nx_apertus_smallfiles.nx source
↩ module page · 63 lines · 3936 B
1// nx_apertus_smallfiles.nx -- mirror the ungated Apertus-8B-Instruct's SMALL files (config + weight-index +
2// tokenizer set -- everything except the 4 multi-GB .safetensors shards) onto the NAS, byte-verified. These fit a
3// single buffer (largest = tokenizer.json ~17MB), so they need no streaming; the shards do (streaming fetch = the
4// last build). TLS-only (no parser/crypto import -> no symbol clash). Additive to /mnt/nas_ai/apertus/.
5// license_tier: ORIGINAL expect_exit: 0
6import "nx_syscalls.nx"
7import "nx_x509_trust_store.nx"
8import "nx_trust_store_load_from_certdata.nx"
9import "nx_https_fetch_follow.nx"
10const K_MAGIC_4194304: i64 = 4194304
11const K_MAGIC_33554432: i64 = 33554432
12
13func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
14func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 }
15func af_cat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64=off; var i: i64=0; while s[i]!=(0 as u8){dst[o]=s[i];o=o+1;i=i+1} return o }
16func af_memeq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if a[i]!=b[i] { return 0 } i=i+1 } return 1 }
17
18func main() -> i64 {
19 w("=== nx_apertus_smallfiles: mirror Apertus-8B-Instruct config/index/tokenizer -> NAS, byte-verified ===\n" as *u8)
20 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
21 if r <= 0 { w("certdata load FAILED\n" as *u8); sys_exit(1); return 1 }
22 let store: *TrustStore = r as *TrustStore
23 let cap: i64 = K_MAGIC_33554432 // 32MB (tokenizer.json ~17MB fits)
24 let out: *u8 = sys_mmap(cap)
25 let rb: *u8 = sys_mmap(cap)
26 let status: *i64 = sys_mmap(8) as *i64
27
28 let base: *u8 = "https://huggingface.co/adamo1139/Apertus-8B-Instruct-2509-ungated/resolve/main/" as *u8
29 let destbase: *u8 = "/mnt/nas_ai/apertus/" as *u8
30
31 let names: *i64 = sys_mmap(8 * 8) as *i64
32 names[0] = "config.json" as *u8 as i64
33 names[1] = "model.safetensors.index.json" as *u8 as i64
34 names[2] = "special_tokens_map.json" as *u8 as i64
35 names[3] = "tokenizer_config.json" as *u8 as i64
36 names[4] = "tokenizer.json" as *u8 as i64
37 let nn: i64 = 5
38
39 var ok: i64 = 0
40 var tot: i64 = 0
41 var i: i64 = 0
42 while i < nn {
43 let name: *u8 = names[i] as *u8
44 let url: *u8 = sys_mmap(512); var uo: i64 = af_cat(url, 0, base); uo = af_cat(url, uo, name); url[uo] = 0 as u8
45 let cn: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
46 w(" " as *u8); w(name); w(" status=" as *u8); wn(status[0]); w(" bytes=" as *u8); wn(cn)
47 if cn <= 0 { w(" FETCH-FAIL\n" as *u8) } else {
48 let dest: *u8 = sys_mmap(512); var doff: i64 = af_cat(dest, 0, destbase); doff = af_cat(dest, doff, name); dest[doff] = 0 as u8
49 let fd: i64 = sys_openat_wr(dest, 0x1a4)
50 if fd < 0 { w(" WRITE-FAIL\n" as *u8) } else {
51 sys_write(fd, out, cn); sys_close(fd)
52 let rfd: i64 = sys_openat_rd(dest); var rn: i64 = 0
53 if rfd >= 0 { rn = sys_read(rfd, rb, cap); sys_close(rfd) }
54 if rn == cn { if af_memeq(out, rb, cn) == 1 { ok = ok + 1; tot = tot + cn; w(" STORED+VERIFIED\n" as *u8) } else { w(" ROUNDTRIP-MISMATCH\n" as *u8) } } else { w(" SHORT-READBACK\n" as *u8) }
55 }
56 }
57 i = i + 1
58 }
59
60 w("\n SMALL FILES MIRRORED: " as *u8); wn(ok); w("/" as *u8); wn(nn); w(" total bytes=" as *u8); wn(tot); w("\n" as *u8)
61 if ok == nn { w("=== GREEN (all Apertus support files on the NAS, byte-verified; only the 4 ~16GB shards remain = streaming fetch) ===\n" as *u8); sys_exit(0); return 0 }
62 w("=== PARTIAL (see per-file above) ===\n" as *u8); sys_exit(1); return 1
63}