nx_model_fetch.nx source
↩ module page · 162 lines · 8052 B
1// nx_model_fetch.nx -- D2: SOVEREIGN model download (HF resolve -> 302 -> CDN TLS -> stream GGUF to disk).
2//
3// The nishi-ecosystem download capability (operator 2026-06-16): no curl/wget/python. Reuses the proven
4// sovereign TLS1.3 stack (D1 nx_hf_probe verified reach + 302 capture). Steps: (1) GET the HF resolve URL
5// -> 302; (2) parse the Location header -> the signed AWS-CDN URL; (3) new TLS session to the CDN host,
6// validate its cert vs the Mozilla CA store, GET the long signed path; (4) find the HTTP body boundary
7// (\r\n\r\n) and write the GGUF body to /tmp/nx_real_model.gguf (the loader's expected path).
8// Diagnostics at every step so one run reveals the whole picture. license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_csprng.nx"
11import "nx_x509_trust_store.nx"
12import "nx_trust_store_load_from_certdata.nx"
13import "nx_tls13_client_validate_certificate.nx"
14import "nx_tls13_client_session_run.nx"
15import "nx_https_url_for_fetch.nx"
16import "nx_https_url_connect.nx"
17import "nx_https_get.nx"
18import "nx_https_get_complete.nx"
19import "nx_http_response_parse.nx"
20const MF_MAGIC_4194304: i64 = 4194304
21const MF_MAGIC_2048: i64 = 2048
22const MF_MAGIC_100000000: i64 = 100000000
23const MF_MAGIC_1048576: i64 = 1048576
24
25const MF_CDN_CAP: i64 = 600000000 // 600 MB buffer for the CDN response (headers + ~400 MB GGUF body)
26
27func mf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
28func mf_putn(v: i64) -> i64 {
29 let bb: *u8 = sys_mmap(28); var m: i64 = v
30 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
31 let t: *u8 = sys_mmap(28); var k: i64 = 0
32 if m == 0 { t[0] = 48 as u8; k = 1 }
33 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
34 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0
35}
36func mf_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
37
38// naive substring search; returns offset of first match in buf[0..n], or -1.
39func mf_find(buf: *u8, n: i64, pat: *u8, pl: i64) -> i64 {
40 if pl == 0 { return -1 }
41 var i: i64 = 0
42 while i + pl <= n {
43 var j: i64 = 0
44 while j < pl { if buf[i + j] != pat[j] { j = pl + 1 } else { j = j + 1 } }
45 if j == pl { return i }
46 i = i + 1
47 }
48 return -1
49}
50
51// index of the next CR or LF at/after start (line end); returns n if none.
52func mf_line_end(buf: *u8, start: i64, n: i64) -> i64 {
53 var i: i64 = start
54 while i < n {
55 if buf[i] == (13 as u8) { return i }
56 if buf[i] == (10 as u8) { return i }
57 i = i + 1
58 }
59 return n
60}
61
62// TLS1.3 GET full_url's `path` on its host into buf (cap); returns total bytes (gc) or negative.
63func mf_fetch(store: *TrustStore, full_url: *u8, path: *u8, path_len: i64, buf: *u8, cap: i64) -> i64 {
64 let cr: *u8 = sys_mmap(32)
65 var i: i64 = 0
66 nx_csprng_fill(cr, 32) // CWE-330 (debt 1785970852): was the constant 0xC0..0xDF
67 let priv: *u8 = sys_mmap(32)
68 i = 0
69 nx_csprng_fill(priv, 32) // CWE-330: the X25519 scalar was the constant 0xA0..0xBF on EVERY session
70 let url_p: *NxUrl = nx_url_new()
71 let target_raw: *u8 = sys_mmap(32)
72 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
73 target.url = url_p
74 target.port = 0
75 if nx_https_url_for_fetch(full_url, target) != NX_HTTPS_URL_OK { return 0 - 41 }
76 let fd_p: *i64 = sys_mmap(16) as *i64
77 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - 42 }
78 let fd: i64 = *fd_p
79 let val_ctx_raw: *u8 = sys_mmap(64)
80 let val_ctx: *TlsValidationContext = val_ctx_raw as *TlsValidationContext
81 val_ctx.store = store
82 val_ctx.sni_host = full_url + target.url.host_off
83 val_ctx.sni_host_len = target.url.host_len
84 val_ctx.now_epoch = sys_now_realtime_sec()
85 let sr: i64 = nx_tls13_client_session_run(fd, full_url + target.url.host_off, target.url.host_len, cr, priv, val_ctx)
86 if sr <= 0 { sys_close(fd); return 0 - (200 + (0 - sr)) }
87 let session: *Tls13ClientSession = sr as *Tls13ClientSession
88 let gc: i64 = nx_https_get_complete(session, fd, path, path_len, full_url + target.url.host_off, target.url.host_len, buf, cap)
89 sys_close(fd)
90 return gc
91}
92
93func main() -> i64 {
94 let cpath: *u8 = "data/mozilla_certdata.txt\x00"
95 let r: i64 = nx_trust_store_load_from_certdata(cpath, 512, MF_MAGIC_4194304)
96 if r <= 0 { mf_puts("MF: certdata load failed\n"); return 1 }
97 let store: *TrustStore = r as *TrustStore
98 mf_puts("CA="); mf_putn(trust_store_count(store)); mf_puts("\n")
99
100 // ---- step 1: GET HF resolve URL -> 302 ----
101 let u1: *u8 = "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf\x00"
102 let p1: *u8 = "/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_k_m.gguf\x00"
103 let buf1: *u8 = sys_mmap(MF_MAGIC_4194304)
104 let gc1: i64 = mf_fetch(store, u1, p1, mf_strlen(p1), buf1, MF_MAGIC_4194304)
105 if gc1 < 0 { mf_puts("MF: resolve fetch failed code="); mf_putn(gc1); mf_puts("\n"); return 5 }
106 let rs1: *i64 = sys_mmap(128) as *i64
107 nx_http_response_parse(buf1, gc1, rs1)
108 mf_puts("step1 resolve ST="); mf_putn(rs1[1]); mf_puts(" bytes="); mf_putn(gc1); mf_puts("\n")
109
110 // ---- step 2: parse Location -> CDN URL ----
111 var off: i64 = mf_find(buf1, gc1, "ocation: " as *u8, 9)
112 if off < 0 { mf_puts("MF: no Location header (status not a redirect?)\n"); return 6 }
113 let lstart: i64 = off + 9
114 let lend: i64 = mf_line_end(buf1, lstart, gc1)
115 let url2: *u8 = sys_mmap(MF_MAGIC_2048)
116 var k: i64 = 0
117 var c2: i64 = lstart
118 while c2 < lend { url2[k] = buf1[c2]; k = k + 1; c2 = c2 + 1 }
119 url2[k] = 0 as u8
120 mf_puts("step2 Location len="); mf_putn(k); mf_puts("\n")
121
122 // ---- step 3: parse CDN host + path from url2, GET it ----
123 let url_p: *NxUrl = nx_url_new()
124 let tgt_raw: *u8 = sys_mmap(32)
125 let tgt: *NxHttpsTarget = tgt_raw as *NxHttpsTarget
126 tgt.url = url_p
127 tgt.port = 0
128 if nx_https_url_for_fetch(url2, tgt) != NX_HTTPS_URL_OK { mf_puts("MF: CDN url parse failed\n"); return 7 }
129 let path2: *u8 = url2 + tgt.url.host_off + tgt.url.host_len
130 let path2_len: i64 = mf_strlen(path2)
131 mf_puts("step3 CDN host_len="); mf_putn(tgt.url.host_len); mf_puts(" path_len="); mf_putn(path2_len); mf_puts("\n")
132
133 let bufc: *u8 = sys_mmap(MF_CDN_CAP)
134 if bufc == (0 as *u8) { mf_puts("MF: CDN buffer mmap failed\n"); return 8 }
135 let gcc: i64 = mf_fetch(store, url2, path2, path2_len, bufc, MF_CDN_CAP)
136 if gcc < 0 { mf_puts("MF: CDN fetch failed code="); mf_putn(gcc); mf_puts("\n"); return 9 }
137 let rsc: *i64 = sys_mmap(128) as *i64
138 nx_http_response_parse(bufc, gcc, rsc)
139 mf_puts("step3 CDN ST="); mf_putn(rsc[1]); mf_puts(" total_bytes="); mf_putn(gcc); mf_puts("\n")
140
141 // ---- step 4: find body (\r\n\r\n) and write to disk ----
142 var boff: i64 = mf_find(bufc, gcc, "\r\n\r\n" as *u8, 4)
143 if boff < 0 { mf_puts("MF: no header/body boundary\n"); return 10 }
144 let bodystart: i64 = boff + 4
145 let bodylen: i64 = gcc - bodystart
146 mf_puts("step4 body_offset="); mf_putn(bodystart); mf_puts(" body_bytes="); mf_putn(bodylen); mf_puts("\n")
147 if bodylen < MF_MAGIC_100000000 { mf_puts("MF: body < 100MB -- another redirect or error, NOT a full GGUF\n"); return 11 }
148
149 let ofd: i64 = sys_openat_wr("/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8, 0x1A4)
150 if ofd <= 0 { mf_puts("MF: open output failed\n"); return 12 }
151 var written: i64 = 0
152 while written < bodylen {
153 var chunk: i64 = bodylen - written
154 if chunk > MF_MAGIC_1048576 { chunk = MF_MAGIC_1048576 }
155 let w: i64 = sys_write(ofd, bufc + bodystart + written, chunk)
156 if w <= 0 { mf_puts("MF: write failed\n"); sys_close(ofd); return 13 }
157 written = written + w
158 }
159 sys_close(ofd)
160 mf_puts("MF-DONE wrote "); mf_putn(written); mf_puts(" bytes -> /tmp/nx_real_model.gguf\n")
161 return 0
162}