nx_apertus_tree_parse.nx source

1// nx_apertus_tree_parse.nx -- ORGAN B of the sovereign enumerate (SRP: parse only, NO TLS). Reads the raw HF tree 2// response that organ A (nx_apertus_tree_fetch) banked, chunked_decodes it if the dynamic API framed it, and parses 3// the per-shard path + real size + lfs.oid(sha256) via nx_hf_tree_parse -> the shard manifest + total bytes for the 4// verified pull. NO TLS import -> no clash with the fetch stack. license_tier: ORIGINAL expect_exit: 0 5import "nx_syscalls.nx" 6import "nx_http_chunked.nx" 7import "nx_hf_tree_parse.nx" 8const K_MAGIC_4194304: i64 = 4194304 9const K_MAGIC_1000000000: i64 = 1000000000 10 11func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 12func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 } 13func bp_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 14 15func main() -> i64 { 16 w("=== nx_apertus_tree_parse (organ B): parse apertus_tree.raw -> shard manifest ===\n" as *u8) 17 let cap: i64 = K_MAGIC_4194304 18 let raw: *u8 = sys_mmap(cap) 19 let fd: i64 = sys_openat_rd("knowledge/fetched/apertus_tree.raw" as *u8) 20 if fd < 0 { w(" apertus_tree.raw NOT FOUND (run organ A first)\n" as *u8); sys_exit(1); return 1 } 21 var rn: i64 = 0 22 var got: i64 = sys_read(fd, ((raw as i64) + rn) as *u8, cap - rn) 23 while got > 0 { rn = rn + got; got = sys_read(fd, ((raw as i64) + rn) as *u8, cap - rn) } 24 sys_close(fd) 25 w(" raw bytes=" as *u8); wn(rn); w("\n" as *u8) 26 27 // direct parse; if no shards found, the body is chunked-framed -> decode then parse. 28 var body: *u8 = raw 29 var bn: i64 = rn 30 if hft_count(raw, rn, ".safetensors" as *u8) == 0 { 31 let dec: *u8 = sys_mmap(cap) 32 let dn: i64 = chunked_decode(raw, rn, dec) 33 if dn > 0 { if hft_count(dec, dn, ".safetensors" as *u8) > 0 { body = dec; bn = dn; w(" (chunked-decoded " as *u8); wn(dn); w(" bytes)\n" as *u8) } } 34 } 35 36 let shards: i64 = hft_count(body, bn, ".safetensors" as *u8) 37 w(" .safetensors shards=" as *u8); wn(shards); w("\n" as *u8) 38 39 // Real HF entry order is {..."lfs":{"oid":<sha>,"size":N}...,"path":<name>} -- the path FOLLOWS the lfs. So walk 40 // by LFS entry and take the path that follows it; keep only paths ENDING in ".safetensors" (excludes the lfs 41 // tokenizer.json/.pdf and the non-lfs model.safetensors.index.json which merely contains ".safetensors"). 42 var from: i64 = 0 43 var total: i64 = 0 44 var listed: i64 = 0 45 var guard: i64 = 0 46 while guard < 128 { 47 let sha: *u8 = sys_mmap(80) 48 let os: i64 = hft_lfs_oid(body, bn, from, sha) 49 if os < 0 { guard = 128 } else { 50 let sz: i64 = hft_num_field(body, bn, os, "\"size\":" as *u8) 51 let pathbuf: *u8 = sys_mmap(256) 52 let ps: i64 = hft_path(body, bn, os, pathbuf) 53 let pl: i64 = bp_slen(pathbuf) 54 var is_shard: i64 = 0 55 if pl >= 12 { if hft_find(((pathbuf as i64) + (pl - 12)) as *u8, 12, 0, ".safetensors" as *u8) == 0 { is_shard = 1 } } 56 if is_shard == 1 { 57 total = total + sz 58 listed = listed + 1 59 w(" - " as *u8); w(pathbuf); w(" size=" as *u8); wn(sz); w(" sha256=" as *u8); w(sha); w("\n" as *u8) 60 } 61 if ps >= 0 { from = ps + 1 } else { from = os + 1 } 62 } 63 guard = guard + 1 64 } 65 66 w("\n TOTAL shards=" as *u8); wn(listed); w(" total bytes=" as *u8); wn(total); w(" (~" as *u8); wn(total / K_MAGIC_1000000000); w(" GB)\n" as *u8) 67 if listed > 0 { w("=== GREEN (sovereign enumerate: real per-shard path+size+sha256, no WebFetch) ===\n" as *u8); sys_exit(0); return 0 } 68 w("=== RED (no shards parsed -- inspect apertus_tree.raw framing) ===\n" as *u8); sys_exit(1); return 1 69}