code wiki / _hdl_build / nx_stream_store.nx
nx_stream_store.nx source
↩ module page · 34 lines · 1854 B
1// nx_stream_store.nx -- LIB (M3): stream a large artifact in fixed-size chunks -> hash incrementally + append to
2// disk, so a multi-GB .safetensors shard needs NO full-size buffer for either hashing OR storing. Composes the
3// shipped streaming SHA-256 (nx_sha256: sha256_init/update/final) -- does not re-implement it. This is the transport
4// half of a verified weights pull: the caller feeds fetched TCP chunks straight through ss_stream_store, then
5// compares the returned digest to the HF lfs.oid (nx_hf_tree_parse) -- fail-closed if they differ. No TLS here ->
6// no M32 collision with the fetch stack (transport + crypto stay in separate compilation units). license_tier: ORIGINAL
7import "nx_sha256.nx"
8import "nx_syscalls.nx"
9
10// one-shot digest convenience (so callers/gates need not import nx_sha256 directly -> avoids double-import).
11func ss_oneshot(src: *u8, n: i64, out32: *u8) -> i64 { sha256_digest(src, n, out32); return 0 }
12
13// Stream src[0..n) in `chunk`-byte pieces: sha256_update each piece + append it to dest_path. Writes the 32-byte
14// streaming digest to out32. Returns bytes written (= n) on success, or -1 if the destination can't be opened.
15// The point: at no moment is more than `chunk` bytes handled -- a 16GB shard streams through a tiny window.
16func ss_stream_store(src: *u8, n: i64, chunk: i64, dest_path: *u8, out32: *u8) -> i64 {
17 let ctx_raw: *u8 = sys_mmap(256)
18 let ctx: *Sha256 = ctx_raw as *Sha256
19 sha256_init(ctx)
20 let fd: i64 = sys_openat_wr(dest_path, 0x1a4)
21 if fd < 0 { return 0 - 1 }
22 var i: i64 = 0
23 while i < n {
24 var c: i64 = chunk
25 if i + c > n { c = n - i }
26 let piece: *u8 = ((src as i64) + i) as *u8
27 sha256_update(ctx, piece, c)
28 sys_write(fd, piece, c)
29 i = i + c
30 }
31 sys_close(fd)
32 sha256_final(ctx, out32)
33 return n
34}