nx_forge_model_fetch.nx source
↩ module page · 206 lines · 6465 B
1// nx_forge_model_fetch.nx -- L3: sovereign BIG-FILE downloader (Range loop) for model weights.
2// usage: nx_forge_model_fetch <urlfile> <outpath> <expected-bytes>
3// urlfile = file whose content is the (signed CDN) URL -- URLs are shell-hostile, never argv
4// outpath = absolute output path (additive staging: NEVER point at the proven live model)
5// expected = exact byte size (from the origin's X-Linked-Size header)
6// Streams 16MB RFC-9110 Range chunks over sovereign TLS-1.3 (nx_https_fetch_range), appends to
7// outpath, sha256's the stream incrementally, RESUMES from existing size (re-hashing the prefix),
8// retries each chunk 3x. Exits 0 only when size == expected; prints the sha256 hex for comparison
9// against the origin ETag. license_tier: ORIGINAL expect_exit: 0
10import "nx_syscalls.nx"
11import "nx_lib_std.nx"
12import "nx_x509_trust_store.nx"
13import "nx_trust_store_load_from_certdata.nx"
14import "nx_https_fetch_follow.nx"
15const FMF_MAGIC_4096: i64 = 4096
16const FMF_MAGIC_4095: i64 = 4095
17const FMF_MAGIC_4194304: i64 = 4194304
18const FMF_MAGIC_1024: i64 = 1024
19const FMF_MAGIC_1500: i64 = 1500
20
21const FMF_CHUNK: i64 = 16777216
22
23func fmf_wb(fd: i64, p: *u8, n: i64) -> i64 {
24 var off: i64 = 0
25 while off < n {
26 let q: *u8 = p + off
27 let rem: i64 = n - off
28 let w: i64 = sys_write(fd, q, rem)
29 if w <= 0 {
30 let neg: i64 = 0 - 1
31 return neg
32 }
33 off = off + w
34 }
35 return 0
36}
37
38func fmf_hexput(b: *u8, n: i64) -> i64 {
39 let hx: *u8 = "0123456789abcdef" as *u8
40 let o: *u8 = sys_mmap(80) as *u8
41 var i: i64 = 0
42 while i < n {
43 let v: i64 = b[i] as i64
44 let hi: i64 = v / 16
45 let lo: i64 = v % 16
46 let i2: i64 = i * 2
47 o[i2] = hx[hi]
48 let i21: i64 = i2 + 1
49 o[i21] = hx[lo]
50 i = i + 1
51 }
52 let n2: i64 = n * 2
53 sys_write(1, o, n2)
54 return 0
55}
56
57func main(argc: i64, argv: *i64) -> i64 {
58 if argc < 4 {
59 std_putln("usage: nx_forge_model_fetch <urlfile> <outpath> <expected-bytes>" as *u8)
60 sys_exit(2)
61 return 2
62 }
63 let a1: i64 = argv[1]
64 let urlfile: *u8 = a1 as *u8
65 let a2: i64 = argv[2]
66 let outpath: *u8 = a2 as *u8
67 let a3: i64 = argv[3]
68 let expstr: *u8 = a3 as *u8
69 let expected: i64 = std_atoi(expstr)
70 if expected <= 0 {
71 std_putln("FMF bad expected size" as *u8)
72 sys_exit(2)
73 return 2
74 }
75 // url (strip trailing whitespace)
76 let url: *u8 = sys_mmap(FMF_MAGIC_4096) as *u8
77 let ufd: i64 = sys_openat_rd(urlfile)
78 if ufd < 0 {
79 std_putln("FMF urlfile unreadable" as *u8)
80 sys_exit(2)
81 return 2
82 }
83 var un: i64 = sys_read(ufd, url, FMF_MAGIC_4095)
84 sys_close(ufd)
85 var trim: i64 = 1
86 while trim == 1 {
87 trim = 0
88 if un > 0 {
89 let last: i64 = un - 1
90 let c: i64 = url[last] as i64
91 if c == 10 || c == 13 || c == 32 { un = un - 1; trim = 1 }
92 }
93 }
94 url[un] = 0 as u8
95 if un < 12 {
96 std_putln("FMF url too short" as *u8)
97 sys_exit(2)
98 return 2
99 }
100 // trust store
101 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, FMF_MAGIC_4194304)
102 if r <= 0 {
103 std_putln("FMF certdata load failed" as *u8)
104 sys_exit(2)
105 return 2
106 }
107 let store: *TrustStore = r as *TrustStore
108 // sha over the whole stream
109 let ctx_raw: *u8 = sys_mmap(FMF_MAGIC_1024) as *u8
110 let ctx: *Sha256 = ctx_raw as *Sha256
111 sha256_init(ctx)
112 // resume point = existing size; re-hash the prefix
113 var off: i64 = 0
114 let efd: i64 = sys_openat_rd(outpath)
115 let hbuf: *u8 = sys_mmap(FMF_MAGIC_4194304) as *u8
116 if efd >= 0 {
117 var go: i64 = 1
118 while go == 1 {
119 let n: i64 = sys_read(efd, hbuf, FMF_MAGIC_4194304)
120 if n <= 0 { go = 0 }
121 if n > 0 {
122 sha256_update(ctx, hbuf, n)
123 off = off + n
124 }
125 }
126 sys_close(efd)
127 }
128 std_puts("FMF resume-at=" as *u8)
129 std_pdec(off)
130 std_puts(" expected=" as *u8)
131 std_pdec(expected)
132 std_puts("\n" as *u8)
133 if off > expected {
134 std_putln("FMF existing larger than expected -- refusing (wrong file?)" as *u8)
135 sys_exit(3)
136 return 3
137 }
138 let buf: *u8 = sys_mmap(FMF_CHUNK) as *u8
139 let stp: *i64 = sys_mmap(16) as *i64
140 var afd: i64 = 0 - 1
141 if off < expected {
142 afd = sys_openat_append(outpath, 420)
143 if afd < 0 {
144 std_putln("FMF outpath unwritable" as *u8)
145 sys_exit(2)
146 return 2
147 }
148 }
149 var failed: i64 = 0
150 while off < expected && failed == 0 {
151 var rend: i64 = off + FMF_CHUNK - 1
152 let lastb: i64 = expected - 1
153 if rend > lastb { rend = lastb }
154 let want: i64 = rend - off + 1
155 var n: i64 = 0 - 1
156 var tries: i64 = 0
157 while tries < 3 && n != want {
158 n = nx_https_fetch_range(url, store, buf, FMF_CHUNK, off, rend, stp, 0)
159 if n != want {
160 std_puts("FMF retry chunk off=" as *u8)
161 std_pdec(off)
162 std_puts(" got=" as *u8)
163 std_pdec(n)
164 std_puts(" status=" as *u8)
165 let st: i64 = stp[0]
166 std_pdec(st)
167 std_puts("\n" as *u8)
168 sys_sleep_ms(FMF_MAGIC_1500)
169 }
170 tries = tries + 1
171 }
172 if n != want { failed = 1 }
173 if n == want {
174 let w: i64 = fmf_wb(afd, buf, n)
175 if w != 0 { failed = 2 }
176 if w == 0 {
177 sha256_update(ctx, buf, n)
178 off = off + n
179 std_puts("FMF ok off=" as *u8)
180 std_pdec(off)
181 std_puts("/" as *u8)
182 std_pdec(expected)
183 std_puts("\n" as *u8)
184 }
185 }
186 }
187 if afd >= 0 { sys_close(afd) }
188 if failed != 0 {
189 std_puts("FMF FAILED code=" as *u8)
190 std_pdec(failed)
191 std_puts(" at off=" as *u8)
192 std_pdec(off)
193 std_puts("\n" as *u8)
194 sys_exit(1)
195 return 1
196 }
197 let digest: *u8 = sys_mmap(64) as *u8
198 sha256_final(ctx, digest)
199 std_puts("FMF DONE size=" as *u8)
200 std_pdec(off)
201 std_puts(" sha256=" as *u8)
202 fmf_hexput(digest, 32)
203 std_puts("\n" as *u8)
204 sys_exit(0)
205 return 0
206}