Zero-copy asset streaming for game engines. Fast, memory-mapped, and verified.

#include <stdio.h>
#include <stdlib.h>
#include <cyclepack.h>

static void report(const char* what) {
    char msg[256];
    cyclepack_get_last_error_message(msg, sizeof msg);
    printf("%s: error %d: %s\n", what, cyclepack_get_last_error_code(), msg);
}

static void read_entry(const uint8_t key[32], uint64_t min_generation, const char* name) {
    /* The key goes in at open, with the oldest generation this build accepts. */
    CyclePackUnpacker* u = cyclepack_create_unpacker_with_key("sealed.cpk", key, 32, min_generation);
    if (!u) { report("open"); return; }      /* nothing to read: no entry list, no handle */

    uint32_t n = 0;
    if (!cyclepack_get_file_data(u, name, NULL, 0, &n)) {   /* NULL buffer: size query */
        report("size query");
    } else {
        printf("size query: %u bytes\n", n);
        uint8_t* bytes = malloc(n);
        if (bytes && cyclepack_get_file_data(u, name, bytes, n, &n))  /* decrypt + digest check */
            printf("read %u bytes of %s\n", n, name);
        else
            report("read");
        free(bytes);
    }

    /* A sealed archive is never lent out raw: the zero-copy path refuses every entry. */
    uint32_t len = 0;
    if (!cyclepack_get_data_ptr(u, name, &len)) report("get_data_ptr");

    cyclepack_destroy_unpacker(u);
}

int main(void) {
    uint8_t key[32];
    FILE* f = fopen("key.bin", "rb");
    if (!f || fread(key, 1, 32, f) != 32) return 1;
    fclose(f);

    read_entry(key, 7, "meshes/hero.mesh");    /* sealed.cpk is generation 7 */
    read_entry(key, 8, "meshes/hero.mesh");    /* a build that accepts generation 8 and up */

    key[0] ^= 0x01;                            /* one bit off */
    read_entry(key, 0, "meshes/hero.mesh");
    return 0;
}

How checked: compiled with cc -std=c11 -Wall -Wextra against include/cyclepack.h and libcyclepack, no warnings, and run on sealed.cpk, packed as generation 7 with cyclepack-studio pack --compression none --encryption chacha20-poly1305 --key-file key.bin --generation-id 7.

size query: 5000 bytes
read 5000 bytes of meshes/hero.mesh
get_data_ptr: error 1: cyclepack_get_data_ptr: cannot lend 'meshes/hero.mesh': E_ZERO_COPY_ENCRYPTED: 'meshes/hero.mesh' is encrypted; use get_data() to decrypt it
open: error 6: cyclepack_create_unpacker_with_key: cannot open 'sealed.cpk': E_STALE_GENERATION: archive generation 7 is below the minimum generation 8 this reader accepts
open: error 1: cyclepack_create_unpacker_with_key: cannot open 'sealed.cpk': E_WRONG_KEY: this key does not open the archive (the key check in its header does not match)

use cyclepack::lua::runtime::LuaRuntime;
use cyclepack::vfs::{CascStorage, MmapStorage, VfsEngine, VirtualFileSystem};
use std::path::Path;
use std::sync::Arc;

fn main() -> anyhow::Result<()> {
    // One virtual root over every mount, tried by priority; reads take &self
    let mut vfs = VfsEngine::new();

    // A memory-mapped archive, and a content-addressed chunk store
    // (CASC-style, CyclePack's own layout)
    vfs.mount_mmap("assets/", Arc::new(MmapStorage::open("game.cpk")?));
    vfs.mount_casc("textures/", Arc::new(CascStorage::open("stream_cache/")?));

    // Trusted Lua 5.4 runtime for your own scripts; LuaRuntime::sandboxed for mods
    let runtime = LuaRuntime::new()?;
    runtime.load_script(Path::new("scripts/vfs_pipeline.lua"))?;

    // A stored entry is borrowed from the mapping. A compressed or sealed one is
    // refused here (E_ZERO_COPY_*), and vfs.read() decodes it into a copy.
    let slice = vfs.read_slice("assets/meshes/hero.mesh")?;
    println!("resolved {} bytes", slice.as_slice().len());

    Ok(())
}

How checked: built in a scratch crate that depends on this repository by path, and run beside game.cpk, an empty stream_cache/ and a one-line scripts/vfs_pipeline.lua.

[INFO] vfs_pipeline.lua loaded
resolved 5000 bytes

// cyclepack codegen --schema entity.cpack --output generated/ \
//                   --languages cpp,csharp,go,lua,python,rust
// Every target gets a packed struct with the same layout.
struct PlayerState {
    uint64  player_uid;
    float32 position[3];
    float32 orientation[4];
    uint16  current_zone;
    uint32  health_mana_packed;
    bytes   dynamic_payload;   // bytes is a fixed 64-byte field
}

How checked: the command in the first two lines, run on this file. The test suite compiles the generated code and round-trips it through every toolchain on the machine.

⚙️  CyclePack Schema Codegen: "entity.cpack"
✅ Code generation completed successfully! Generated 6 files in "generated/"
   - "generated/entity.hpp"
   - "generated/entity.cs"
   - "generated/entity.go"
   - "generated/entity.lua"
   - "generated/entity.py"
   - "generated/entity.rs"

from cyclepack import SharedMemoryArena

# A 16 MiB slab owned by the engine, in this process. The context manager
# destroys it on the way out; a memoryview taken from it is invalid afterwards.
with SharedMemoryArena(name="game_telemetry", capacity_bytes=16 * 1024 * 1024) as arena:
    # Reserve 256 bytes; the lease is what releases them again
    lease_id, offset, length = arena.allocate(256)

    # A view *into* the engine's slab: Python writes through it,
    # with no copy on the CPython heap
    view = arena.get_memoryview(offset, length)
    view[0:4] = b"SYNC"

    arena.release(lease_id)
    print(f"wrote 4 bytes into a {arena.capacity}-byte arena")

How checked: run with Python 3.12 against the SDK in packages/python and the debug library. Python writes through a memoryview into the engine's slab; nothing is copied onto the CPython heap. The arena lives in this process; it is not shared with other processes.

wrote 4 bytes into a 16777216-byte arena

// Preview renderer: isotropic discs, no depth sort, no culling or LOD, and
// spherical harmonics are ignored. The WASM package is built from the repo.
import { GspxWebGpuRenderer } from "@cyclechain/cyclepack-webgpu";
import init, { GspxWasmReader } from "./pkg/web/cyclepack.js";

await init();
const renderer = new GspxWebGpuRenderer(canvas);
await renderer.init();

// Each chunk's base payload starts at its own 64 KiB-aligned offset, so the
// renderer takes the reader that owns those offsets. Every chunk's 16-byte
// PackedBaseSplat records go into a GPU storage buffer as stored.
const bytes = new Uint8Array(await (await fetch("/assets/scene.gspx")).arrayBuffer());
const reader = GspxWasmReader.from_buffer(bytes);
const scene = renderer.setScene(reader);
reader.free(); // setScene has uploaded every chunk; the reader can go

renderer.render({ view, projection });
console.log(`${scene.splatCount} of ${scene.declaredSplatCount} splats in ${scene.chunkCount} chunks`);

How checked: the API was checked against packages/cyclepack-webgpu/src/renderer.ts. Not run here: it needs a browser with WebGPU.

Not run for this page (needs WebGPU).

-- Lua 5.4: ChaCha20-Poly1305 pipeline writing a CPAK container
--   cyclepack pack --script pipeline.lua --input assets --output game.cpak \
--                  --config password=...
-- Modules are globals the runtime injects; a sandboxed script has no require.

function pack(input_dir, output_file)
    -- A fresh salt per archive, stored in its header for the unpack script
    local salt = crypto.random.bytes(16)
    -- A 32-byte key, stretched from the password with Argon2id
    local key = crypto.kdf.argon2(config.password, salt, 32)

    local builder  = archive.builder()
    builder:set_header("kdf_salt", salt)
    local pipeline = archive.pipeline():add("lz4"):add("chacha20", key)

    for _, path in ipairs(io.walk(input_dir)) do
        if io.is_file(path) then
            -- The name in the archive: the path less the input directory,
            -- whether or not --input ended in a slash
            local name = path:sub(#input_dir + 1):gsub("^/+", "")
            -- Compressed, then encrypted, one entry at a time (each file is read into memory)
            builder:add_stream(name, path, pipeline)
        end
    end

    builder:build(output_file)
    log.info("packed " .. builder:entry_count() .. " entries")
end

How checked: run with the command in its header over the eight demo files. The salt travels in the container header: archive.reader(bytes):get_header("kdf_salt") returned its 16 bytes for the unpack side. For a sealed Format v3 archive that the C, Rust, Node and WASM readers open with its key, use cyclepack-studio pack --encryption chacha20-poly1305 --key-file key.bin --generation-id N instead.

[INFO] packed 8 entries

-- Lua 5.4: .gspx Gaussian splats: frustum cull, then upload
--   cyclepack run splatting.lua

local reader = gspx.open_reader(io.read_file("world/scene.gspx"))
log.info(reader:chunk_count() .. " chunks, " .. reader:total_splats() .. " splats")

-- Six frustum planes, each {a, b, c, d}. Returns the chunk indices inside them.
local visible = reader:cull_frustum({
    {1, 0, 0, 1000}, {-1, 0, 0, 1000}, {0, 1, 0, 1000},
    {0, -1, 0, 1000}, {0, 0, 1, 1000}, {0, 0, -1, 1000},
})

for _, chunk in ipairs(visible) do
    local descriptor = reader:get_chunk_descriptor(chunk)

    -- 16 bytes per PackedBaseSplat, in the layout a GPU storage
    -- buffer takes as-is
    local payload = reader:read_base_payload(chunk)

    log.info(("chunk %d: %d splats, lod %d, %d bytes"):format(
        chunk, descriptor.splat_count, descriptor.lod_level, payload.length))
end

How checked: run with cyclepack run on a .gspx converted from the 64-splat fixture tests/fixtures/studio/gaussian_64_binary_le.ply with --chunk-size 16.

[INFO] 4 chunks, 64 splats
[INFO] chunk 0: 16 splats, lod 0, 256 bytes
[INFO] chunk 1: 16 splats, lod 0, 256 bytes
[INFO] chunk 2: 16 splats, lod 0, 256 bytes
[INFO] chunk 3: 16 splats, lod 0, 256 bytes

-- Lua 5.4: meshlet cluster DAG: open a container, unpack the clusters you need
--   cyclepack-studio convert-mesh --demo --output models/demo.cdag
--   cyclepack run meshlet.lua

local dag = meshlet.load_container(io.read_file("models/demo.cdag"))

local raw_bytes, packed_bytes, ratio = dag:compression_metrics()
log.info(("%d -> %d bytes (%.2fx)"):format(raw_bytes, packed_bytes, ratio))

-- Cluster ids are 0-based. Unpacking returns flat float and index arrays,
-- ready for a vertex buffer without a per-triangle Lua loop.
local mesh = dag:unpack_clusters({0, 1})

log.info(("%d vertices, %d indices"):format(#mesh.positions // 3, #mesh.indices))

How checked: run with cyclepack run on the container that convert-mesh --demo writes (a generated 1,152-triangle mesh, 21 clusters).

[INFO] 46304 -> 20004 bytes (2.31x)
[INFO] 128 vertices, 438 indices

-- Lua 5.4: copy-on-write overlays: patches, snapshots and branches
--   cyclepack run overlay.lua
-- No archive is mounted beneath the overlay in this snippet.

local layers = overlay.create_engine()

-- The patch lands in a new copy-on-write layer
layers:cow_write("assets/hero.mesh", io.read_file("patch/hero.mesh"))
layers:take_snapshot("v2.4")

-- A bad patch is one rollback away
layers:cow_write("assets/hero.mesh", io.read_file("patch/hero_broken.mesh"))
layers:rollback_to_snapshot("v2.4")

-- One branch per build, so QA and live can diverge in the same process
layers:create_branch("live-patch")
layers:switch_branch("live-patch")

for _, path in ipairs(layers:modified_diff()) do
    log.info("overlaid " .. path .. " (" .. layers:read(path).length .. " bytes)")
end

How checked: run with cyclepack run, with a 5,000-byte patch/hero.mesh and a truncated patch/hero_broken.mesh. The rollback restores the 5,000-byte version.

[INFO] overlaid assets/hero.mesh (5000 bytes)

-- Lua 5.4: Merkle root over everything you ship, and a proof per entry
--   cyclepack run security.lua
-- Format v3 already records a BLAKE3 digest per entry; this adds one
-- root you can sign and ship beside the archive.

local paths, leaves = {}, {}
for _, path in ipairs(io.walk("assets")) do
    if io.is_file(path) then
        paths[#paths + 1] = path
        leaves[#leaves + 1] = crypto.hash.sha256(io.read_file(path))
    end
end

-- 32-byte leaves. The root is what you sign.
local root = crypto.merkle.new(leaves)
io.write_file("assets.root", root)

-- A proof for one entry, so a client checks it without the whole set.
-- Leaf indices are 0-based, the way the tree counts them.
local proof = crypto.merkle.proof(leaves, {0})
if not crypto.merkle.verify(root, {0}, {leaves[1]}, #leaves, proof) then
    error("integrity violation: " .. paths[1])
end

log.info("merkle root over " .. #leaves .. " files verified")

How checked: run with cyclepack run over the eight demo files.

[INFO] merkle root over 8 files verified