nx_par_pull_nas.nx source
↩ module page · 111 lines · 6766 B
1// nx_par_pull_nas.nx -- PARALLEL concurrent download proven ON THE NAS (the max-speed gap WSL could not run: its
2// service crashed on concurrent fork/TLS; the NAS's real Linux handles it). Forks one worker per file, each its own
3// resumable TLS-1.3 stream, staged to /tmp (NAS-local ext4). Absolute CA path (/volume1/ai/apertus/_certdata.txt)
4// so nx_hostctl can execve it directly (no CWD). Identical logic to nx_par_pull (WSL); only the certdata path
5// differs. license_tier: ORIGINAL expect_exit: 0
6import "nx_syscalls.nx"
7import "nx_trust_store_load_from_certdata.nx"
8import "nx_https_fetch_follow.nx"
9import "nx_https_get_stream.nx"
10const K_MAGIC_4096: i64 = 4096
11const K_MAGIC_2048: i64 = 2048
12const K_MAGIC_4194304: i64 = 4194304
13const K_MAGIC_17078479: i64 = 17078479
14const K_MAGIC_4192195: i64 = 4192195
15const K_MAGIC_7291: i64 = 7291
16const K_MAGIC_1024: i64 = 1024
17
18func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
19func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 }
20func pp_cat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64=off; var i: i64=0; while s[i]!=(0 as u8){dst[o]=s[i];o=o+1;i=i+1} return o }
21func file_size(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } let sz: i64=sys_lseek(fd,0,2); sys_close(fd); if sz<0 { return 0 } return sz }
22
23func pull_attempt(url0: *u8, store: *TrustStore, dest_path: *u8, have: i64) -> i64 {
24 let urlbuf: *u8 = sys_mmap(K_MAGIC_4096)
25 var ui: i64=0; while url0[ui]!=0 as u8 { urlbuf[ui]=url0[ui]; ui=ui+1 } urlbuf[ui]=0 as u8
26 var dest_fd: i64 = 0-1
27 if have>0 { dest_fd=sys_openat_append(dest_path,0x1a4) } else { dest_fd=sys_openat_wr(dest_path,0x1a4) }
28 if dest_fd<0 { return 0-100 }
29 let cr: *u8=sys_mmap(32); let priv: *u8=sys_mmap(32); let statusp: *i64=sys_mmap(16) as *i64; let loc: *u8=sys_mmap(K_MAGIC_4096)
30 var hop: i64=0
31 while hop<=8 {
32 let target_raw: *u8=sys_mmap(64); let target: *NxHttpsTarget=target_raw as *NxHttpsTarget
33 target.url=nx_url_new(); target.port=0
34 if nx_https_url_for_fetch(urlbuf,target)!=NX_HTTPS_URL_OK { sys_close(dest_fd); return 0-101 }
35 let fd_p: *i64=(sys_mmap(16)) as *i64
36 if nx_https_url_connect(target,urlbuf,sys_now_realtime_sec(),fd_p)!=NX_HTTPS_CONNECT_OK { sys_close(dest_fd); return 0-102 }
37 let fd: i64=fd_p[0]
38 var i: i64=0; while i<32 { cr[i]=(0xC0+i) as u8; priv[i]=(0xA0+i) as u8; i=i+1 }
39 let vc_raw: *u8=sys_mmap(64); let vc: *TlsValidationContext=vc_raw as *TlsValidationContext
40 vc.store=store; vc.sni_host=urlbuf+target.url.host_off; vc.sni_host_len=target.url.host_len; vc.now_epoch=sys_now_realtime_sec()
41 let sr: i64=nx_tls13_client_session_run(fd,urlbuf+target.url.host_off,target.url.host_len,cr,priv,vc)
42 if sr<=0 { sys_close(fd); sys_close(dest_fd); return 0-103 }
43 let session: *Tls13ClientSession=sr as *Tls13ClientSession
44 let path: *u8=sys_mmap(K_MAGIC_2048); let plen: i64=ff_path(urlbuf,target,path)
45 let rc: i64=nx_https_get_stream(session,fd,path,plen,urlbuf+target.url.host_off,target.url.host_len,have,dest_fd,statusp,loc,K_MAGIC_4096)
46 sys_close(fd)
47 if rc==0 { let resolved: *u8=sys_mmap(K_MAGIC_4096); ff_resolve_location(loc,urlbuf,target,resolved); var k: i64=0; while resolved[k]!=0 as u8 { urlbuf[k]=resolved[k]; k=k+1 } urlbuf[k]=0 as u8; hop=hop+1 }
48 else { sys_close(dest_fd); return rc }
49 }
50 sys_close(dest_fd); return 0-200
51}
52func pull_one_file(url0: *u8, store: *TrustStore, dest_path: *u8, target: i64) -> i64 {
53 var tries: i64=0
54 while tries<20 {
55 let have: i64=file_size(dest_path)
56 if have>=target { return have }
57 let rc: i64=pull_attempt(url0,store,dest_path,have)
58 if rc==(0-9) { let tf: i64=sys_openat_wr(dest_path,0x1a4); if tf>=0 { sys_close(tf) } tries=tries+1 }
59 else { let h2: i64=file_size(dest_path); if h2>=target { return h2 } if h2>have { tries=0 } else { tries=tries+1 } }
60 }
61 return 0-1
62}
63
64func main() -> i64 {
65 w("=== nx_par_pull_nas: PARALLEL concurrent download ON THE NAS (fork per file, own TLS each) -> /tmp ===\n" as *u8)
66 let r: i64 = nx_trust_store_load_from_certdata("/volume1/ai/apertus/_certdata.txt" as *u8, 512, K_MAGIC_4194304)
67 if r<=0 { w("certdata load FAILED\n" as *u8); sys_exit(1); return 1 }
68 let store: *TrustStore = r as *TrustStore
69 let base: *u8 = "https://huggingface.co/adamo1139/Apertus-8B-Instruct-2509-ungated/resolve/main/" as *u8
70 let names: *i64=sys_mmap(8*8) as *i64; let sizes: *i64=sys_mmap(8*8) as *i64
71 names[0]="tokenizer.json" as *u8 as i64; sizes[0]=K_MAGIC_17078479
72 names[1]="Apertus_Tech_Report.pdf" as *u8 as i64; sizes[1]=K_MAGIC_4192195
73 names[2]="README.md" as *u8 as i64; sizes[2]=K_MAGIC_7291
74 names[3]="config.json" as *u8 as i64; sizes[3]=901
75 let nn: i64 = 4
76 let urls: *i64=sys_mmap(8*8) as *i64; let dests: *i64=sys_mmap(8*8) as *i64
77 var i: i64=0
78 while i<nn {
79 let nm: *u8=names[i] as *u8
80 let u: *u8=sys_mmap(K_MAGIC_1024); var uo: i64=pp_cat(u,0,base); uo=pp_cat(u,uo,nm); u[uo]=0 as u8; urls[i]=u as i64
81 let d: *u8=sys_mmap(K_MAGIC_1024); var doo: i64=pp_cat(d,0,"/tmp/parnas_" as *u8); doo=pp_cat(d,doo,nm); d[doo]=0 as u8; dests[i]=d as i64
82 i=i+1
83 }
84 let pids: *i64=sys_mmap(8*8) as *i64
85 i=0
86 while i<nn {
87 let pid: i64=sys_fork()
88 if pid==0 {
89 let got: i64=pull_one_file(urls[i] as *u8, store, dests[i] as *u8, sizes[i])
90 if got>=sizes[i] { sys_exit(0) }
91 sys_exit(1)
92 }
93 pids[i]=pid
94 w(" forked worker for " as *u8); w(names[i] as *u8); w(" (pid " as *u8); wn(pid); w(")\n" as *u8)
95 i=i+1
96 }
97 var ok: i64=0
98 i=0
99 while i<nn {
100 let st: *i64=sys_mmap(16) as *i64
101 sys_wait4(pids[i], st, 0)
102 let ec: i64=(st[0]>>8)&0xff
103 let sz: i64=file_size(dests[i] as *u8)
104 w(" " as *u8); w(names[i] as *u8); w(" exit=" as *u8); wn(ec); w(" bytes=" as *u8); wn(sz); w("/" as *u8); wn(sizes[i])
105 if sz>=sizes[i] { w(" [OK]\n" as *u8); ok=ok+1 } else { w(" [INCOMPLETE]\n" as *u8) }
106 i=i+1
107 }
108 w("\n PARALLEL WORKERS COMPLETE ON NAS: " as *u8); wn(ok); w("/" as *u8); wn(nn); w("\n" as *u8)
109 if ok==nn { w("=== GREEN (concurrent multi-file download works ON THE NAS -- what crashed WSL) ===\n" as *u8); sys_exit(0); return 0 }
110 w("=== PARTIAL ===\n" as *u8); sys_exit(1); return 1
111}