nx_apertus_mirror_nas.nx source
↩ module page · 94 lines · 6630 B
1// nx_apertus_mirror_nas.nx -- the NAS-RESIDENT variant of nx_apertus_shards_pull (operator: "i want the NAS doing
2// these things not the laptop where we dev"). IDENTICAL resumable/self-healing streaming pull, but pathed for the
3// NAS's LOCAL disk: reads the CA store from /volume1/ai/apertus/_certdata.txt and streams shards to /volume1/ai/
4// apertus/*.part. Running ON the NAS = no 9p mount, no WSL service, native x86_64 Linux -> fast + stable + the
5// concurrent/parallel path finally viable. Deployed to /volume1/homes/elderwesto/nishihost/ and launched by an
6// allowlisted nx_hostctl `mirror` action (modeled on gendeploy). license_tier: ORIGINAL expect_exit: 0
7import "nx_syscalls.nx"
8import "nx_trust_store_load_from_certdata.nx"
9import "nx_https_fetch_follow.nx"
10import "nx_https_get_stream.nx"
11const K_MAGIC_4096: i64 = 4096
12const K_MAGIC_2048: i64 = 2048
13const K_MAGIC_4194304: i64 = 4194304
14const K_MAGIC_4999776656: i64 = 4999776656
15const K_MAGIC_4882374192: i64 = 4882374192
16const K_MAGIC_4974647808: i64 = 4974647808
17const K_MAGIC_1249928160: i64 = 1249928160
18const K_MAGIC_1024: i64 = 1024
19
20func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
21func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 }
22func sp_cat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64=off; var i: i64=0; while s[i]!=(0 as u8){dst[o]=s[i];o=o+1;i=i+1} return o }
23func file_size(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } let sz: i64=sys_lseek(fd,0,2); sys_close(fd); if sz<0 { return 0 } return sz }
24
25func pull_attempt(url0: *u8, store: *TrustStore, dest_path: *u8, have: i64) -> i64 {
26 let urlbuf: *u8 = sys_mmap(K_MAGIC_4096)
27 var ui: i64=0; while url0[ui]!=0 as u8 { urlbuf[ui]=url0[ui]; ui=ui+1 } urlbuf[ui]=0 as u8
28 var dest_fd: i64 = 0-1
29 if have>0 { dest_fd=sys_openat_append(dest_path,0x1a4) } else { dest_fd=sys_openat_wr(dest_path,0x1a4) }
30 if dest_fd<0 { return 0-100 }
31 let cr: *u8=sys_mmap(32); let priv: *u8=sys_mmap(32); let statusp: *i64=sys_mmap(16) as *i64; let loc: *u8=sys_mmap(K_MAGIC_4096)
32 var hop: i64=0
33 while hop<=8 {
34 let target_raw: *u8=sys_mmap(64); let target: *NxHttpsTarget=target_raw as *NxHttpsTarget
35 target.url=nx_url_new(); target.port=0
36 if nx_https_url_for_fetch(urlbuf,target)!=NX_HTTPS_URL_OK { sys_close(dest_fd); return 0-101 }
37 let fd_p: *i64=(sys_mmap(16)) as *i64
38 if nx_https_url_connect(target,urlbuf,sys_now_realtime_sec(),fd_p)!=NX_HTTPS_CONNECT_OK { sys_close(dest_fd); return 0-102 }
39 let fd: i64=fd_p[0]
40 var i: i64=0; while i<32 { cr[i]=(0xC0+i) as u8; priv[i]=(0xA0+i) as u8; i=i+1 }
41 let vc_raw: *u8=sys_mmap(64); let vc: *TlsValidationContext=vc_raw as *TlsValidationContext
42 vc.store=store; vc.sni_host=urlbuf+target.url.host_off; vc.sni_host_len=target.url.host_len; vc.now_epoch=sys_now_realtime_sec()
43 let sr: i64=nx_tls13_client_session_run(fd,urlbuf+target.url.host_off,target.url.host_len,cr,priv,vc)
44 if sr<=0 { sys_close(fd); sys_close(dest_fd); return 0-103 }
45 let session: *Tls13ClientSession=sr as *Tls13ClientSession
46 let path: *u8=sys_mmap(K_MAGIC_2048); let plen: i64=ff_path(urlbuf,target,path)
47 let rc: i64=nx_https_get_stream(session,fd,path,plen,urlbuf+target.url.host_off,target.url.host_len,have,dest_fd,statusp,loc,K_MAGIC_4096)
48 sys_close(fd)
49 if rc==0 { let resolved: *u8=sys_mmap(K_MAGIC_4096); ff_resolve_location(loc,urlbuf,target,resolved); var k: i64=0; while resolved[k]!=0 as u8 { urlbuf[k]=resolved[k]; k=k+1 } urlbuf[k]=0 as u8; hop=hop+1 }
50 else { sys_close(dest_fd); return rc }
51 }
52 sys_close(dest_fd); return 0-200
53}
54func pull_resumable(url0: *u8, store: *TrustStore, dest_path: *u8, target: i64) -> i64 {
55 var tries: i64=0
56 while tries<20 {
57 let have: i64=file_size(dest_path)
58 if have>=target { return have }
59 w(" attempt from byte " as *u8); wn(have); w(" / " as *u8); wn(target); w("\n" as *u8)
60 let rc: i64=pull_attempt(url0,store,dest_path,have)
61 if rc==(0-9) { let tf: i64=sys_openat_wr(dest_path,0x1a4); if tf>=0 { sys_close(tf) } tries=tries+1 }
62 else { let h2: i64=file_size(dest_path); w(" rc=" as *u8); wn(rc); w(" now " as *u8); wn(h2); w("\n" as *u8); if h2>=target { return h2 } if h2>have { tries=0 } else { tries=tries+1 } }
63 }
64 return 0-1
65}
66
67func main() -> i64 {
68 w("=== nx_apertus_mirror_nas: NAS-resident resumable pull -> /volume1/ai/apertus ===\n" as *u8)
69 let r: i64 = nx_trust_store_load_from_certdata("/volume1/ai/apertus/_certdata.txt" as *u8, 512, K_MAGIC_4194304)
70 if r<=0 { w("certdata load FAILED (stage _certdata.txt to /volume1/ai/apertus/)\n" as *u8); sys_exit(1); return 1 }
71 let store: *TrustStore = r as *TrustStore
72 let base: *u8 = "https://huggingface.co/adamo1139/Apertus-8B-Instruct-2509-ungated/resolve/main/" as *u8
73 let destbase: *u8 = "/volume1/ai/apertus/" as *u8
74 let names: *i64=sys_mmap(8*8) as *i64; let sizes: *i64=sys_mmap(8*8) as *i64
75 names[0]="model-00001-of-00004.safetensors" as *u8 as i64; sizes[0]=K_MAGIC_4999776656
76 names[1]="model-00002-of-00004.safetensors" as *u8 as i64; sizes[1]=K_MAGIC_4882374192
77 names[2]="model-00003-of-00004.safetensors" as *u8 as i64; sizes[2]=K_MAGIC_4974647808
78 names[3]="model-00004-of-00004.safetensors" as *u8 as i64; sizes[3]=K_MAGIC_1249928160
79 let nn: i64 = 4
80 var ok: i64=0
81 var i: i64=0
82 while i<nn {
83 let name: *u8=names[i] as *u8; let target: i64=sizes[i]
84 let url: *u8=sys_mmap(K_MAGIC_1024); var uo: i64=sp_cat(url,0,base); uo=sp_cat(url,uo,name); url[uo]=0 as u8
85 let dest: *u8=sys_mmap(K_MAGIC_1024); var doff: i64=sp_cat(dest,0,destbase); doff=sp_cat(dest,doff,name); doff=sp_cat(dest,doff,".part" as *u8); dest[doff]=0 as u8
86 w(" " as *u8); w(name); w("\n" as *u8)
87 let fin: i64=pull_resumable(url,store,dest,target)
88 if fin>=target { w(" COMPLETE " as *u8); wn(fin); w("\n" as *u8); ok=ok+1 } else { w(" INCOMPLETE " as *u8); wn(fin); w("\n" as *u8) }
89 i=i+1
90 }
91 w("\n NAS SHARDS COMPLETE: " as *u8); wn(ok); w("/" as *u8); wn(nn); w("\n" as *u8)
92 if ok==nn { w("=== GREEN (NAS-resident pull complete) ===\n" as *u8); sys_exit(0); return 0 }
93 w("=== PARTIAL (re-run to resume) ===\n" as *u8); sys_exit(1); return 1
94}