nx_hf_file_fetch.nx source
↩ module page · 135 lines · 7103 B
1// nx_hf_file_fetch.nx -- the STANDARD redirect-following, resumable, self-healing single-file HF downloader (the
2// model-port driver's fetch stage that actually WORKS for HF LFS). nx_forge_model_fetch's Range fetch does NOT
3// follow redirects, but HF `resolve/main/<file>` 302-redirects to the CDN -> it fails. This reuses the PROVEN
4// nx_apertus_shards_pull mechanism (pull_attempt hop-loops connect+TLS+get_stream, following Location on 3xx;
5// pull_resumable resumes from the on-disk size + self-heals) but is FILE-DRIVEN (url/dest/size from
6// data/mp_fetch_{url,dest,size}.txt) so it fetches ANY single model file with no code edit and runs under
7// nx_sov_build_run (no argv). license_tier: ORIGINAL expect_exit: 0
8import "nx_syscalls.nx"
9import "nx_csprng.nx"
10import "nx_trust_store_load_from_certdata.nx"
11import "nx_https_fetch_follow.nx"
12import "nx_https_get_stream.nx"
13const K_MAGIC_4096: i64 = 4096
14const K_MAGIC_2048: i64 = 2048
15const K_MAGIC_4194304: i64 = 4194304
16
17func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func wn(v: i64) -> i64 { var m: i64=v; if m<0{w("-" as *u8);m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1} sys_write(1,o,k); return 0 }
19
20func hff_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
21 let fd: i64 = sys_openat_rd(path)
22 if fd < 0 { return 0 - 1 }
23 var n: i64 = sys_read(fd, buf, cap - 1)
24 sys_close(fd)
25 if n < 0 { return 0 - 1 }
26 var go: i64 = 1
27 while go == 1 {
28 go = 0
29 if n > 0 {
30 let c: i64 = buf[n-1] as i64
31 if c == 10 { n = n - 1; go = 1 }
32 if c == 13 { n = n - 1; go = 1 }
33 if c == 32 { n = n - 1; go = 1 }
34 }
35 }
36 buf[n] = 0 as u8
37 return n
38}
39func hff_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ let c: i64=s[i] as i64; if c>=48{ if c<=57{ v=v*10+(c-48) } } i=i+1 } return v }
40
41func file_size(path: *u8) -> i64 {
42 let fd: i64 = sys_openat_rd(path)
43 if fd < 0 { return 0 }
44 let sz: i64 = sys_lseek(fd, 0, 2)
45 sys_close(fd)
46 if sz < 0 { return 0 }
47 return sz
48}
49
50// ONE attempt: open dest (append if resuming, truncate if fresh), hop-loop connect+handshake, stream (Range=have).
51func pull_attempt(url0: *u8, store: *TrustStore, dest_path: *u8, have: i64) -> i64 {
52 let urlbuf: *u8 = sys_mmap(K_MAGIC_4096)
53 var ui: i64 = 0; while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 } urlbuf[ui] = 0 as u8
54 var dest_fd: i64 = 0 - 1
55 if have > 0 { dest_fd = sys_openat_append(dest_path, 0x1a4) } else { dest_fd = sys_openat_wr(dest_path, 0x1a4) }
56 if dest_fd < 0 { return 0 - 100 }
57 let cr: *u8 = sys_mmap(32)
58 let priv: *u8 = sys_mmap(32)
59 let statusp: *i64 = sys_mmap(16) as *i64
60 let loc: *u8 = sys_mmap(K_MAGIC_4096)
61 var hop: i64 = 0
62 while hop <= 8 {
63 let target_raw: *u8 = sys_mmap(64)
64 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
65 target.url = nx_url_new()
66 target.port = 0
67 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { sys_close(dest_fd); return 0 - 101 }
68 let fd_p: *i64 = (sys_mmap(16)) as *i64
69 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { sys_close(dest_fd); return 0 - 102 }
70 let fd: i64 = fd_p[0]
71 var i: i64 = 0
72 nx_csprng_fill(cr, 32); nx_csprng_fill(priv, 32) // CWE-330 (debt 1785970852): were the constants 0xC0../0xA0.. on EVERY session
73 let vc_raw: *u8 = sys_mmap(64)
74 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
75 vc.store = store
76 vc.sni_host = urlbuf + target.url.host_off
77 vc.sni_host_len = target.url.host_len
78 vc.now_epoch = sys_now_realtime_sec()
79 let sr: i64 = nx_tls13_client_session_run(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc)
80 if sr <= 0 { sys_close(fd); sys_close(dest_fd); return 0 - 103 }
81 let session: *Tls13ClientSession = sr as *Tls13ClientSession
82 let path: *u8 = sys_mmap(K_MAGIC_2048)
83 let plen: i64 = ff_path(urlbuf, target, path)
84 let rc: i64 = nx_https_get_stream(session, fd, path, plen, urlbuf + target.url.host_off, target.url.host_len, have, dest_fd, statusp, loc, K_MAGIC_4096)
85 sys_close(fd)
86 if rc == 0 {
87 let resolved: *u8 = sys_mmap(K_MAGIC_4096)
88 ff_resolve_location(loc, urlbuf, target, resolved)
89 var k: i64 = 0; while resolved[k] != 0 as u8 { urlbuf[k] = resolved[k]; k = k + 1 } urlbuf[k] = 0 as u8
90 hop = hop + 1
91 } else { sys_close(dest_fd); return rc }
92 }
93 sys_close(dest_fd); return 0 - 200
94}
95
96func pull_resumable(url0: *u8, store: *TrustStore, dest_path: *u8, target: i64) -> i64 {
97 var tries: i64 = 0
98 while tries < 40 {
99 let have: i64 = file_size(dest_path)
100 if have >= target { return have }
101 w(" attempt from byte " as *u8); wn(have); w(" / " as *u8); wn(target); w(" ...\n" as *u8)
102 let rc: i64 = pull_attempt(url0, store, dest_path, have)
103 if rc == (0 - 9) {
104 let tf: i64 = sys_openat_wr(dest_path, 0x1a4); if tf >= 0 { sys_close(tf) }
105 tries = tries + 1
106 w(" (server ignored Range -> truncated, restarting)\n" as *u8)
107 } else {
108 let have2: i64 = file_size(dest_path)
109 w(" attempt rc=" as *u8); wn(rc); w(" now at " as *u8); wn(have2); w("\n" as *u8)
110 if have2 >= target { return have2 }
111 if have2 > have { tries = 0 } else { tries = tries + 1 }
112 }
113 }
114 return 0 - 1
115}
116
117func main() -> i64 {
118 w("=== nx_hf_file_fetch: redirect-following resumable single-file HF download ===\n" as *u8)
119 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
120 if r <= 0 { w("certdata load FAILED\n" as *u8); sys_exit(1); return 1 }
121 let store: *TrustStore = r as *TrustStore
122 let url: *u8 = sys_mmap(K_MAGIC_4096)
123 let dest: *u8 = sys_mmap(K_MAGIC_4096)
124 let szs: *u8 = sys_mmap(64)
125 if hff_readfile("data/mp_fetch_url.txt" as *u8, url, K_MAGIC_4096) < 0 { w("no data/mp_fetch_url.txt\n" as *u8); sys_exit(1); return 1 }
126 if hff_readfile("data/mp_fetch_dest.txt" as *u8, dest, K_MAGIC_4096) < 0 { w("no data/mp_fetch_dest.txt\n" as *u8); sys_exit(1); return 1 }
127 if hff_readfile("data/mp_fetch_size.txt" as *u8, szs, 64) < 0 { w("no data/mp_fetch_size.txt\n" as *u8); sys_exit(1); return 1 }
128 let target: i64 = hff_atoi(szs)
129 w(" url: " as *u8); w(url); w("\n dest: " as *u8); w(dest); w("\n size: " as *u8); wn(target); w("\n" as *u8)
130 let fin: i64 = pull_resumable(url, store, dest, target)
131 if fin >= target { w("COMPLETE: " as *u8); wn(fin); w(" bytes\n" as *u8); sys_exit(0); return 0 }
132 w("INCOMPLETE: reached " as *u8); wn(fin); w(" / " as *u8); wn(target); w(" (re-run to resume)\n" as *u8)
133 sys_exit(1)
134 return 1
135}