nx_hifigan_stream_fetch.nx source
↩ module page · 111 lines · 7071 B
1// nx_hifigan_stream_fetch.nx -- SOVEREIGN STREAMING DOWNLOAD of a large LFS model weight file. The high-level
2// fetcher (nx_https_fetch_follow) buffers the whole response in memory and chokes on a 50MB+ body (returned
3// NX_FF_GET=-4 on HF's LFS CDN). This mirrors that fetcher's connect+handshake+redirect loop EXACTLY, but streams
4// the final body straight to a file fd via nx_https_get_stream (8MB window, no giant buffer). HF weights are
5// LFS-hosted -> resolve/main/<file> 302s to cdn-lfs.huggingface.co with a SIGNED url (query string preserved by
6// copying Location verbatim). Chrome-JA3 hello beats CDN anti-bot fingerprinting. This unblocks EVERY sovereign
7// model-weight download (voice/image/LLM), not just this one. expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_csprng.nx"
10import "nx_x509_trust_store.nx"
11import "nx_trust_store_load_from_certdata.nx"
12import "nx_tls13_client_validate_certificate.nx"
13import "nx_tls13_client_session_run.nx"
14import "nx_tls13_chrome_session.nx"
15import "nx_https_url_for_fetch.nx"
16import "nx_https_url_connect.nx"
17import "nx_https_get_stream.nx"
18import "nx_https_redirect.nx"
19const K_MAGIC_8192: i64 = 8192
20const K_MAGIC_4194304: i64 = 4194304
21
22func p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
23func pn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let d: *u8=sys_mmap(24); var k: i64=0; while m>0 {d[k]=(48+(m%10)) as u8; m=m/10; k=k+1} var i: i64=k-1; while i>=0 {let o: *u8=sys_mmap(1); o[0]=d[i]; sys_write(1,o,1); i=i-1} return 0 }
24
25// replicate ff_path: extract the request path (incl. query string) from the URL after host[:port]
26func sf_path(urlbuf: *u8, target: *NxHttpsTarget, path_out: *u8) -> i64 {
27 var pp: i64 = target.url.host_off + target.url.host_len
28 if urlbuf[pp] == 0x3A as u8 {
29 pp = pp + 1
30 while urlbuf[pp] >= 0x30 as u8 { if urlbuf[pp] <= 0x39 as u8 { pp = pp + 1 } else { break } }
31 }
32 if urlbuf[pp] != 0x2F as u8 { path_out[0] = 0x2F as u8; path_out[1] = 0 as u8; return 1 }
33 var k: i64 = 0
34 while urlbuf[pp] != 0 as u8 { path_out[k] = urlbuf[pp]; pp = pp + 1; k = k + 1 }
35 path_out[k] = 0 as u8
36 return k
37}
38
39// streaming redirect-following download to dest_fd. returns body bytes (>=0) or negative error.
40func stream_download(url0: *u8, store: *TrustStore, dest_fd: i64, max_hops: i64, out_status: *i64) -> i64 {
41 let urlbuf: *u8 = sys_mmap(K_MAGIC_8192)
42 var ui: i64 = 0; while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 } urlbuf[ui] = 0 as u8
43 let cr: *u8 = sys_mmap(32); let priv: *u8 = sys_mmap(32)
44 var hop: i64 = 0
45 while hop <= max_hops {
46 let target_raw: *u8 = sys_mmap(64)
47 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
48 target.url = nx_url_new(); target.port = 0
49 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - 1 }
50 let fd_p: *i64 = (sys_mmap(16)) as *i64
51 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - 2 }
52 let fd: i64 = fd_p[0]
53 var i: i64 = 0; nx_csprng_fill(cr, 32); nx_csprng_fill(priv, 32) // CWE-330 (debt 1785970852): were the constants 0xC0../0xA0.. on EVERY session
54 let vc_raw: *u8 = sys_mmap(64)
55 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
56 vc.store = store
57 vc.sni_host = urlbuf + target.url.host_off
58 vc.sni_host_len = target.url.host_len
59 vc.now_epoch = sys_now_realtime_sec()
60 // Chrome-JA3 hello (beats CDN anti-bot fingerprinting + verifies RSA-PSS CertificateVerify)
61 let sr: i64 = nx_tls13_client_session_run_chrome(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc)
62 if sr <= 0 { sys_close(fd); return 0 - 3 }
63 let session: *Tls13ClientSession = sr as *Tls13ClientSession
64 let path: *u8 = sys_mmap(K_MAGIC_8192)
65 let plen: i64 = sf_path(urlbuf, target, path)
66 let loc: *u8 = sys_mmap(K_MAGIC_8192)
67 let r: i64 = nx_https_get_stream(session, fd, path, plen, urlbuf + target.url.host_off, target.url.host_len, 0, dest_fd, out_status, loc, K_MAGIC_8192)
68 sys_close(fd)
69 let status: i64 = out_status[0]
70 if status >= 300 { if status < 400 {
71 // redirect: HF LFS Location is an ABSOLUTE signed CDN url -> copy verbatim (query preserved)
72 var k: i64 = 0; while loc[k] != 0 as u8 { urlbuf[k] = loc[k]; k = k + 1 } urlbuf[k] = 0 as u8
73 p(" -> "); pn(status); p(" redirect to CDN, following...\n" as *u8)
74 hop = hop + 1
75 } else { return r } }
76 else { return r }
77 }
78 return 0 - 99
79}
80
81func main(argc: i64, argv: *i64) -> i64 {
82 let rr: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
83 if rr<=0 { p("certdata load failed\n" as *u8); return 1 }
84 let store: *TrustStore = rr as *TrustStore
85 p("CA roots="); pn(trust_store_count(store)); p("\n" as *u8)
86 // ARG-DRIVEN (reusable): `nx_hifigan_stream_fetch <url> <destpath>` streams ANY HF LFS weight. No args = HiFi-GAN.
87 var url: *u8 = "https://huggingface.co/microsoft/speecht5_hifigan/resolve/main/pytorch_model.bin" as *u8
88 var opath: *u8 = "/home/elderwesto/nx_stage/hifigan_pytorch_model.bin" as *u8
89 if argc >= 3 { url = argv[1] as *u8; opath = argv[2] as *u8 }
90 let dest_fd: i64 = sys_openat_wr(opath, 0x1a4)
91 if dest_fd < 0 { p("open dest failed\n" as *u8); return 1 }
92 p("STREAMING -> "); p(opath); p(" (8MB window, redirect-following)...\n" as *u8)
93 let status_p: *i64 = sys_mmap(8) as *i64
94 let n: i64 = stream_download(url, store, dest_fd, 6, status_p)
95 sys_close(dest_fd)
96 p("final status="); pn(status_p[0]); p(" body_bytes_streamed="); pn(n); p("\n" as *u8)
97 if n < 0 { p("verdict=FAIL (stream error "); pn(n); p(")\n" as *u8); sys_exit(1); return 1 }
98
99 // verify: reopen, peek ZIP magic (PyTorch .bin = ZIP PK\x03\x04)
100 let vfd: i64 = sys_openat_rd(opath)
101 if vfd<0 { p("verdict=FAIL (reopen)\n" as *u8); sys_exit(1); return 1 }
102 let hdr8: *u8 = sys_mmap(64)
103 sys_read(vfd, hdr8, 16); sys_close(vfd)
104 p("first bytes: "); var i: i64=0; while i<8 { let bb: *u8=sys_mmap(4); var nib: i64=(hdr8[i]>>4)&0xF; if nib<10 {bb[0]=(48+nib) as u8} else {bb[0]=(87+nib) as u8} nib=hdr8[i]&0xF; if nib<10 {bb[1]=(48+nib) as u8} else {bb[1]=(87+nib) as u8} bb[2]=32 as u8; sys_write(1,bb,3); i=i+1 } p("\n" as *u8)
105 if hdr8[0]==0x50 as u8 { if hdr8[1]==0x4B as u8 { if hdr8[2]==0x03 as u8 { if hdr8[3]==0x04 as u8 {
106 p("verdict=GREEN (streamed the REAL ~50MB HiFi-GAN weights to disk over our own TLS + it's a valid ZIP/pickle -> sovereign model-weight download WORKS; next = ZIP+pickle reader over this file)\n" as *u8)
107 sys_exit(0); return 0
108 } } } }
109 p("verdict=NOTE (streamed "); pn(n); p(" bytes but first bytes are not the ZIP magic -- inspect above)\n" as *u8)
110 sys_exit(1); return 1
111}