nx_apertus_shard_verify.nx source

1// nx_apertus_shard_verify.nx -- SOVEREIGN fail-closed verify of the streamed Apertus shards: for each .part file on 2// the NAS, stream-hash it with nx_sha256 (8MB window, no full buffer) and compare to the shard's lfs.oid (from the 3// sovereign enumerate nx_apertus_tree_parse). Match -> rename .part -> final (the shard is now a trusted, byte-exact 4// mirror). Mismatch -> leave .part (REJECTED; a corrupt/truncated stream is NEVER promoted). NO TLS import -> no M32 5// clash with the fetch/streaming path. license_tier: ORIGINAL expect_exit: 0 6import "nx_syscalls.nx" 7import "nx_sha256.nx" 8const K_MAGIC_8388608: i64 = 8388608 9const K_MAGIC_1024: i64 = 1024 10 11func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 12func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 } 13func sv_cat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64=off; var i: i64=0; while s[i]!=(0 as u8){dst[o]=s[i];o=o+1;i=i+1} return o } 14func sv_streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while 1==1 { if a[i]!=b[i] {return 0} if a[i]==(0 as u8) {return 1} i=i+1 } return 1 } 15func sv_hexenc(inp: *u8, n: i64, out: *u8) -> i64 { let hx: *u8="0123456789abcdef" as *u8; var i: i64=0; while i<n { out[i*2]=hx[((inp[i] as i64)>>4)&15]; out[i*2+1]=hx[(inp[i] as i64)&15]; i=i+1 } out[n*2]=0 as u8; return n*2 } 16 17// stream-hash a file: sha256 over 8MB windows; out_hex gets the 64-hex digest; returns bytes hashed or -1. 18func sv_hash_file(path: *u8, out_hex: *u8) -> i64 { 19 let ctx_raw: *u8 = sys_mmap(256) 20 let ctx: *Sha256 = ctx_raw as *Sha256 21 sha256_init(ctx) 22 let fd: i64 = sys_openat_rd(path) 23 if fd < 0 { return 0 - 1 } 24 let cap: i64 = K_MAGIC_8388608 25 let buf: *u8 = sys_mmap(cap) 26 var total: i64 = 0 27 var got: i64 = sys_read(fd, buf, cap) 28 while got > 0 { sha256_update(ctx, buf, got); total = total + got; got = sys_read(fd, buf, cap) } 29 sys_close(fd) 30 let dig: *u8 = sys_mmap(32); sha256_final(ctx, dig) 31 sv_hexenc(dig, 32, out_hex) 32 return total 33} 34 35func main() -> i64 { 36 w("=== nx_apertus_shard_verify: stream-hash each .part vs its lfs.oid -> rename on match (fail-closed) ===\n" as *u8) 37 let destbase: *u8 = "/mnt/nas_ai/apertus/" as *u8 38 let names: *i64 = sys_mmap(8 * 8) as *i64 39 let exp: *i64 = sys_mmap(8 * 8) as *i64 40 names[0] = "model-00001-of-00004.safetensors" as *u8 as i64; exp[0] = "3c0d9506c084975849cefc1410dbb0300eafc0f480a5fcbc3db8f54c10500912" as *u8 as i64 41 names[1] = "model-00002-of-00004.safetensors" as *u8 as i64; exp[1] = "a934231fe465a8f4b0028454218d787f02b7cae3f91d886ca73b85ca368f2b21" as *u8 as i64 42 names[2] = "model-00003-of-00004.safetensors" as *u8 as i64; exp[2] = "8c2a98c71209664b1e7c8a2df80782cac1a90cc11feda5cd5c5495c5dd9b584e" as *u8 as i64 43 names[3] = "model-00004-of-00004.safetensors" as *u8 as i64; exp[3] = "6417026ac38ffd73ae5683c568162982c722469683f8040665ddb0a5de6dd442" as *u8 as i64 44 let nn: i64 = 4 45 46 var ok: i64 = 0 47 var i: i64 = 0 48 while i < nn { 49 let name: *u8 = names[i] as *u8 50 let want: *u8 = exp[i] as *u8 51 let part: *u8 = sys_mmap(K_MAGIC_1024); var po: i64 = sv_cat(part, 0, destbase); po = sv_cat(part, po, name); po = sv_cat(part, po, ".part" as *u8); part[po] = 0 as u8 52 let got_hex: *u8 = sys_mmap(80) 53 let n: i64 = sv_hash_file(part, got_hex) 54 w(" " as *u8); w(name); w(" bytes=" as *u8); wn(n); w("\n got =" as *u8); w(got_hex); w("\n want=" as *u8); w(want) 55 if n < 0 { w(" [MISSING .part]\n" as *u8) } else { 56 if sv_streq(got_hex, want) == 1 { 57 let fin: *u8 = sys_mmap(K_MAGIC_1024); var fo: i64 = sv_cat(fin, 0, destbase); fo = sv_cat(fin, fo, name); fin[fo] = 0 as u8 58 sys_renameat(part, fin) 59 w(" [VERIFIED -> renamed to " as *u8); w(name); w("]\n" as *u8) 60 ok = ok + 1 61 } else { w(" [REJECTED: digest mismatch -- .part kept, not promoted]\n" as *u8) } 62 } 63 i = i + 1 64 } 65 w("\n SHARDS VERIFIED: " as *u8); wn(ok); w("/" as *u8); wn(nn); w("\n" as *u8) 66 if ok == nn { w("=== GREEN (all 4 shards byte-exact vs HF lfs.oid, promoted on the NAS) ===\n" as *u8); sys_exit(0); return 0 } 67 w("=== PARTIAL (see per-shard above) ===\n" as *u8); sys_exit(1); return 1 68}