nx_safetensors.nx source
↩ module page · 143 lines · 9233 B
1// nx_safetensors.nx -- SOVEREIGN safetensors parser (the neural-weights loader foundation). safetensors is the
2// modern HuggingFace weight format (HiFi-GAN / VITS vocoders, image models, etc. are hosted in it): 8-byte LE
3// header-length, then a JSON header {name:{dtype,shape,data_offsets:[s,e]}, ...}, then the raw tensor bytes. This
4// parses the header (bounded per-tensor brace scan so a tensor's keys don't leak into the next) and returns each
5// tensor's dtype/shape/byte-range -> a pointer into the data section. Paired with nx_https_get_stream (streaming
6// download) this is the sovereign path to ACQUIRE + LOAD a neural model ourselves -- no operator-must-download,
7// the real unblock for human-realistic neural voice (and neural image). Self-test builds a synthetic safetensors
8// in memory + parses it back (metadata + f32 values byte-exact). No JSON lib, no ML dep. license_tier: ORIGINAL expect_exit: 0
9import "nx_syscalls.nx"
10const K_MAGIC_65536: i64 = 65536
11
12func sw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
13func sn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
14func shx(v: i64) -> i64 { let b: *u8=sys_mmap(8); var i: i64=0; while i<8 { let n: i64=(v>>((7-i)*4))&0xF; if n<10 {b[i]=(48+n) as u8} else {b[i]=(87+n) as u8} i=i+1 } sys_write(1,b,8); return 0 }
15func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
16
17// ---- byte readers ----
18func st_u64le(buf: *u8, off: i64) -> i64 { var v: i64=0; var i: i64=0; while i<8 { v = v | ((buf[off+i]&0xff) << (i*8)); i=i+1 } return v }
19func st_u32le(buf: *u8, off: i64) -> i64 { return (buf[off]&0xff) | ((buf[off+1]&0xff)<<8) | ((buf[off+2]&0xff)<<16) | ((buf[off+3]&0xff)<<24) }
20
21// substring search in hdr[start..end) for NUL-terminated pat; return index or -1
22func st_find(hdr: *u8, start: i64, end: i64, pat: *u8) -> i64 {
23 let pl: i64 = slen(pat); if pl==0 {return start}
24 var i: i64 = start
25 while i + pl <= end {
26 var j: i64=0; while j<pl { if hdr[i+j]!=pat[j] {j=pl+1} else {j=j+1} }
27 if j==pl { return i }
28 i = i + 1
29 }
30 return 0-1
31}
32// from '{' at obj_start, return index of the matching '}' (inclusive), or -1
33func st_match_brace(hdr: *u8, obj_start: i64, hlen: i64) -> i64 {
34 var depth: i64 = 0; var i: i64 = obj_start
35 while i < hlen {
36 let c: i64 = hdr[i] as i64
37 if c==123 { depth=depth+1 } // '{'
38 if c==125 { depth=depth-1; if depth==0 { return i } } // '}'
39 i = i + 1
40 }
41 return 0-1
42}
43// parse int array at pos (hdr[pos]=='['), fill arr, return count
44func st_int_array(hdr: *u8, pos: i64, hlen: i64, arr: *i64) -> i64 {
45 if hdr[pos]!=(91 as u8) { return 0 } // '['
46 var i: i64=pos+1; var n: i64=0; var cur: i64=0; var any: i64=0
47 while i<hlen {
48 let c: i64 = hdr[i] as i64
49 if c==93 { if any==1 { arr[n]=cur; n=n+1 } i=hlen+1 } // ']'
50 else { if c==44 { if any==1 { arr[n]=cur; n=n+1; cur=0; any=0 } } // ','
51 else { if c>=48 { if c<=57 { cur=cur*10+(c-48); any=1 } } } }
52 i=i+1
53 }
54 return n
55}
56// find tensor `name` in header; fill dtype_out(str), shape_out[], returns ndim (or -1 if not found). offs_out[2]=byte range.
57func st_tensor(hdr: *u8, hlen: i64, name: *u8, dtype_out: *u8, shape_out: *i64, offs_out: *i64) -> i64 {
58 // build the search key "name":
59 let key: *u8 = sys_mmap(256)
60 key[0]=34 as u8; var kl: i64=1; var a: i64=0 // '"'
61 while name[a]!=(0 as u8){ key[kl]=name[a]; kl=kl+1; a=a+1 }
62 key[kl]=34 as u8; kl=kl+1; key[kl]=58 as u8; kl=kl+1; key[kl]=0 as u8 // '":'
63 let kpos: i64 = st_find(hdr, 0, hlen, key)
64 if kpos<0 { return 0-1 }
65 // find the '{' that starts this tensor's object (right after "name":)
66 var ostart: i64 = kpos + kl
67 var st_found: i64 = 0
68 while st_found==0 { if ostart>=hlen { st_found=2 } else { if hdr[ostart]==(123 as u8) { st_found=1 } else { ostart=ostart+1 } } }
69 if st_found==2 { return 0-1 }
70 let oend: i64 = st_match_brace(hdr, ostart, hlen)
71 if oend<0 { return 0-1 }
72 // within [ostart, oend): extract dtype, shape, data_offsets
73 let dtk: i64 = st_find(hdr, ostart, oend, "\"dtype\":\"" as *u8)
74 var dl: i64=0
75 if dtk>=0 { var p: i64=dtk+9; while p<oend { if hdr[p]==(34 as u8) {p=oend+1} else { dtype_out[dl]=hdr[p]; dl=dl+1; p=p+1 } } }
76 dtype_out[dl]=0 as u8
77 let shk: i64 = st_find(hdr, ostart, oend, "\"shape\":" as *u8)
78 var ndim: i64=0
79 if shk>=0 { ndim = st_int_array(hdr, shk+8, oend, shape_out) }
80 let dok: i64 = st_find(hdr, ostart, oend, "\"data_offsets\":" as *u8)
81 if dok>=0 { st_int_array(hdr, dok+15, oend, offs_out) }
82 return ndim
83}
84
85// ---- synthetic-file writers ----
86func put_u64le(buf: *u8, off: i64, v: i64) -> i64 { var i: i64=0; while i<8 { buf[off+i]=((v>>(i*8))&0xff) as u8; i=i+1 } return 0 }
87func put_u32le(buf: *u8, off: i64, v: i64) -> i64 { var i: i64=0; while i<4 { buf[off+i]=((v>>(i*8))&0xff) as u8; i=i+1 } return 0 }
88func put_str(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){ buf[off+i]=s[i]; i=i+1 } return off+i }
89
90func main() -> i64 {
91 sw("=== nx_safetensors -- sovereign safetensors parser (neural-weights loader foundation) ===\n" as *u8)
92 // Build a synthetic safetensors in memory: tensor "w" F32 [2,3] offsets[0,24]; "b" F32 [3] offsets[24,36].
93 let HDR: *u8 = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2,3],\"data_offsets\":[0,24]},\"b\":{\"dtype\":\"F32\",\"shape\":[3],\"data_offsets\":[24,36]}}" as *u8
94 let hlen: i64 = slen(HDR)
95 let file: *u8 = sys_mmap(K_MAGIC_65536)
96 put_u64le(file, 0, hlen)
97 let dstart: i64 = 8 + hlen
98 var wo: i64 = put_str(file, 8, HDR) // header at [8, 8+hlen)
99 // data section: w = 6 f32 (1.0..6.0), b = 3 f32 (0.5,0.25,0.125)
100 put_u32le(file, dstart+0, 0x3F800000) // 1.0
101 put_u32le(file, dstart+4, 0x40000000) // 2.0
102 put_u32le(file, dstart+8, 0x40400000) // 3.0
103 put_u32le(file, dstart+12, 0x40800000) // 4.0
104 put_u32le(file, dstart+16, 0x40A00000) // 5.0
105 put_u32le(file, dstart+20, 0x40C00000) // 6.0
106 put_u32le(file, dstart+24, 0x3F000000) // 0.5
107 put_u32le(file, dstart+28, 0x3E800000) // 0.25
108 put_u32le(file, dstart+32, 0x3E000000) // 0.125
109
110 // ---- parse it back ----
111 let rd_hlen: i64 = st_u64le(file, 0)
112 let hdr: *u8 = (file as i64 + 8) as *u8 // header bytes
113 sw("header_len read = "); sn(rd_hlen); sw(" (expect "); sn(hlen); sw(")\n" as *u8)
114
115 let dtype: *u8 = sys_mmap(64)
116 let shape: *i64 = sys_mmap(16*8) as *i64
117 let offs: *i64 = sys_mmap(4*8) as *i64
118 let ndw: i64 = st_tensor(hdr, rd_hlen, "w" as *u8, dtype, shape, offs)
119 sw("tensor 'w': dtype="); sw(dtype); sw(" ndim="); sn(ndw); sw(" shape=["); var i: i64=0; while i<ndw { sn(shape[i]); if i<ndw-1 {sw(",")} i=i+1 } sw("] offs=["); sn(offs[0]); sw(","); sn(offs[1]); sw("]\n" as *u8)
120 let w0: i64 = st_u32le(file, dstart + offs[0]) // first f32 of w
121 sw(" w[0] raw f32 = "); shx(w0); sw(" (expect 3f800000 = 1.0)\n" as *u8)
122
123 let dtype2: *u8 = sys_mmap(64)
124 let shape2: *i64 = sys_mmap(16*8) as *i64
125 let offs2: *i64 = sys_mmap(4*8) as *i64
126 let ndb: i64 = st_tensor(hdr, rd_hlen, "b" as *u8, dtype2, shape2, offs2)
127 sw("tensor 'b': dtype="); sw(dtype2); sw(" ndim="); sn(ndb); sw(" shape=["); sn(shape2[0]); sw("] offs=["); sn(offs2[0]); sw(","); sn(offs2[1]); sw("]\n" as *u8)
128 let b0: i64 = st_u32le(file, dstart + offs2[0])
129 sw(" b[0] raw f32 = "); shx(b0); sw(" (expect 3f000000 = 0.5)\n" as *u8)
130
131 // ---- GATE ----
132 var pass: i64=0; var tot: i64=6
133 if rd_hlen==hlen { pass=pass+1; sw("PASS T1 header length round-trips\n" as *u8) } else { sw("FAIL T1\n" as *u8) }
134 if ndw==2 { if shape[0]==2 { if shape[1]==3 { pass=pass+1; sw("PASS T2 tensor 'w' shape [2,3] parsed\n" as *u8) } else {sw("FAIL T2c\n" as *u8)} } else {sw("FAIL T2b\n" as *u8)} } else {sw("FAIL T2a ndim="); sn(ndw); sw("\n" as *u8)}
135 if offs[0]==0 { if offs[1]==24 { pass=pass+1; sw("PASS T3 'w' byte range [0,24] parsed\n" as *u8) } else {sw("FAIL T3b\n" as *u8)} } else {sw("FAIL T3a\n" as *u8)}
136 if (w0&0xFFFFFFFF)==0x3F800000 { pass=pass+1; sw("PASS T4 'w' data readable byte-exact (1.0)\n" as *u8) } else {sw("FAIL T4\n" as *u8)}
137 if ndb==1 { if shape2[0]==3 { pass=pass+1; sw("PASS T5 tensor 'b' shape [3] parsed (per-tensor bounded, no key leak)\n" as *u8) } else {sw("FAIL T5b\n" as *u8)} } else {sw("FAIL T5a\n" as *u8)}
138 if (b0&0xFFFFFFFF)==0x3F000000 { pass=pass+1; sw("PASS T6 'b' data readable byte-exact (0.5)\n" as *u8) } else {sw("FAIL T6\n" as *u8)}
139
140 sw("nx_safetensors pass="); sn(pass); sw("/"); sn(tot)
141 if pass==tot { sw(" GREEN -- can parse safetensors -> with nx_https_get_stream we can ACQUIRE + LOAD a neural model sovereignly (the neural-voice/image unblock).\n" as *u8); sys_exit(0); return 0 }
142 sw(" RED\n" as *u8); sys_exit(1); return 1
143}