nx_fetch_unit.nx source
↩ module page · 177 lines · 6535 B
1// nx_fetch_unit.nx -- the SHARED staged-fetch unit (Rule #15: this logic lived
2// copy-pasted in _dlr_fetch_one + _ed_fetch_html; extracted 2026-06-10 in the
3// standards audit so the next consumer inherits fixes instead of forking them).
4//
5// module: nishi-core.search.fetch_unit
6// depends: nx_https_get.nx stack
7// capability: CORE_COMPUTE
8//
9// nx_fetch_staged(url, bodybuf, cap):
10// validated-HTTPS GET with 301/302/303/307/308 Location follow (https-only,
11// hop-capped FU_HOPS) and Transfer-Encoding chunked DECODE. Returns decoded
12// body length into bodybuf, or 0/negative on failure. Callers decide what
13// the body becomes (html->text, raw html, header probe...).
14
15import "nx_str.nx"
16import "nx_syscalls.nx"
17import "nx_csprng.nx"
18import "nx_x509_trust_store.nx"
19import "nx_pem_loader.nx"
20import "nx_https_get.nx"
21const FU_MAGIC_2048: i64 = 2048
22
23const FU_CAP: i64 = 524288
24const FU_HOPS: i64 = 3 // 1 fetch + up to 2 redirect follows (anti-loop)
25
26func fu_body_off(resp: *u8, n: i64) -> i64 {
27 var i: i64 = 0
28 while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
29 return 0
30}
31func fu_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
32func fu_find_ci(resp: *u8, lim: i64, needle: *u8) -> i64 {
33 let nn: i64 = nx_str_len(needle)
34 if nn == 0 { return 0 - 1 }
35 var i: i64 = 0
36 while i + nn <= lim {
37 var j: i64 = 0
38 var ok: i64 = 1
39 while j < nn {
40 if fu_lc(resp[i + j] as i64) != fu_lc(needle[j] as i64) { ok = 0; j = nn } else { j = j + 1 }
41 }
42 if ok == 1 { return i }
43 i = i + 1
44 }
45 return 0 - 1
46}
47func fu_status(resp: *u8, n: i64) -> i64 {
48 if n < 12 { return 0 }
49 var sp: i64 = 0 - 1
50 var i: i64 = 0
51 while i < 20 { if (resp[i] as i64) == 0x20 { sp = i; i = 20 } else { i = i + 1 } }
52 if sp < 0 { return 0 }
53 if sp + 3 >= n { return 0 }
54 let d0: i64 = (resp[sp+1] as i64) - 0x30
55 if d0 < 0 { return 0 }
56 if d0 > 9 { return 0 }
57 return d0*100 + ((resp[sp+2] as i64) - 0x30)*10 + ((resp[sp+3] as i64) - 0x30)
58}
59func fu_location(resp: *u8, hdr_lim: i64, cururl: *u8, loc: *u8, cap: i64) -> i64 {
60 let at: i64 = fu_find_ci(resp, hdr_lim, "\nlocation:")
61 if at < 0 { return 0 }
62 var p: i64 = at + 10
63 while (resp[p] as i64) == 0x20 { p = p + 1 }
64 var ve: i64 = 0 - 1
65 var s: i64 = p
66 while s < hdr_lim { if (resp[s] as i64) == 0x0D { ve = s; s = hdr_lim } else { s = s + 1 } }
67 if ve <= p { return 0 }
68 var o: i64 = 0
69 if (resp[p] as i64) == 0x2F {
70 let pre: *u8 = "https://"
71 var k: i64 = 0
72 while k < 8 { loc[o] = pre[k]; o = o + 1; k = k + 1 }
73 var h: i64 = 8
74 var go: i64 = 1
75 while go == 1 {
76 let c: i64 = cururl[h] as i64
77 if c == 0 { go = 0 }
78 if c == 0x2F { go = 0 }
79 if go == 1 { if o < cap - 1 { loc[o] = c; o = o + 1 } h = h + 1 }
80 }
81 } else {
82 if fu_find_ci((((resp as i64) + p) as *u8), 8, "https://") != 0 { return 0 }
83 }
84 var q: i64 = p
85 while q < ve { if o < cap - 1 { loc[o] = resp[q]; o = o + 1 } q = q + 1 }
86 loc[o] = 0 as u8
87 return 1
88}
89func fu_dechunk(resp: *u8, bo: i64, n: i64, out: *u8, cap: i64) -> i64 {
90 var p: i64 = bo
91 var w: i64 = 0
92 var go: i64 = 1
93 while go == 1 {
94 var sz: i64 = 0
95 var digits: i64 = 0
96 var rd: i64 = 1
97 while rd == 1 {
98 if p >= n { rd = 0; go = 0 } else {
99 let c: i64 = fu_lc(resp[p] as i64)
100 var dv: i64 = 0 - 1
101 if c >= 0x30 { if c <= 0x39 { dv = c - 0x30 } }
102 if c >= 0x61 { if c <= 0x66 { dv = c - 0x61 + 10 } }
103 if dv >= 0 { sz = sz * 16 + dv; digits = digits + 1; p = p + 1 } else { rd = 0 }
104 }
105 }
106 if go == 1 { if digits == 0 { go = 0 } }
107 if go == 1 {
108 var nl: i64 = 0 - 1
109 var s: i64 = p
110 while s < n { if (resp[s] as i64) == 0x0A { nl = s; s = n } else { s = s + 1 } }
111 if nl < 0 { go = 0 } else {
112 p = nl + 1
113 if sz == 0 { go = 0 } else {
114 var k: i64 = 0
115 while k < sz { if p + k < n { if w < cap { out[w] = resp[p + k]; w = w + 1 } } k = k + 1 }
116 p = p + sz + 2
117 if p >= n { go = 0 }
118 }
119 }
120 }
121 }
122 return w
123}
124
125// staged fetch: validated HTTPS + redirect follow + chunked decode.
126// Decoded body -> bodybuf; returns body length (0/neg = fail).
127func nx_fetch_staged(url: *u8, bodybuf: *u8, cap: i64) -> i64 {
128 let store: *TrustStore = trust_store_alloc(400)
129 if nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt\x00", store) <= 0 { return 0 - 1 }
130 let now: i64 = sys_now_realtime_sec()
131 let out: *u8 = sys_mmap(FU_CAP)
132 let urlbuf: *u8 = sys_mmap(FU_MAGIC_2048)
133 var ul: i64 = 0
134 while url[ul] != (0 as u8) { urlbuf[ul] = url[ul]; ul = ul + 1 }
135 urlbuf[ul] = 0 as u8
136
137 var r: i64 = 0
138 var hop: i64 = 0
139 var fetching: i64 = 1
140 while fetching == 1 {
141 let cr: *u8 = sys_mmap(32); nx_csprng_fill(cr, 32)
142 let pk: *u8 = sys_mmap(32); nx_csprng_fill(pk, 32)
143 r = nx_https_get(urlbuf, cr, pk, store, now, out, FU_CAP)
144 if r <= 0 { return 0 - 2 }
145 let st: i64 = fu_status(out, r)
146 var redir: i64 = 0
147 if st == 301 { redir = 1 }
148 if st == 302 { redir = 1 }
149 if st == 303 { redir = 1 }
150 if st == 307 { redir = 1 }
151 if st == 308 { redir = 1 }
152 fetching = 0
153 if redir == 1 {
154 if hop < FU_HOPS - 1 {
155 let bo0: i64 = fu_body_off(out, r)
156 let loc: *u8 = sys_mmap(FU_MAGIC_2048)
157 if fu_location(out, bo0, urlbuf, loc, FU_MAGIC_2048) == 1 {
158 var c: i64 = 0
159 while loc[c] != (0 as u8) { urlbuf[c] = loc[c]; c = c + 1 }
160 urlbuf[c] = 0 as u8
161 hop = hop + 1
162 fetching = 1
163 }
164 }
165 }
166 }
167
168 let bo: i64 = fu_body_off(out, r)
169 var blen: i64 = r - bo
170 if fu_find_ci(out, bo, "transfer-encoding: chunked") >= 0 {
171 return fu_dechunk(out, bo, r, bodybuf, cap)
172 }
173 if blen > cap { blen = cap }
174 var i: i64 = 0
175 while i < blen { bodybuf[i] = out[bo + i]; i = i + 1 }
176 return blen
177}