nx_url_canon.nx source
↩ module page · 340 lines · 16875 B
1// nx_url_canon.nx -- HTML attribute -> WIRE URL canonicaliser. LIB (no main, no imports).
2//
3// module: nishi-core.search.urlcanon
4// capability: CORE_COMPUTE
5//
6// WHY THIS FILE EXISTS AS A FILE: the same defect was found in two unrelated harvesters on the same
7// day -- the crawler's outlink harvest (debt 1786031857: arxiv + loc.gov urls carrying a literal
8// `&`) and the media ledger (37 of 438 image urls, same shape). Both had hand-rolled attribute
9// copying. Rather than fix it twice and wait for the third, the canonicaliser is ONE definition that
10// every harvester imports. The two RESOLVERS (olh_resolve_root, nx_url_resolve) stay separate on
11// purpose -- merging resolvers to deduplicate is how live capabilities were deleted here before.
12//
13// ⚠DO NOT MOVE THIS INTO nx_url_resolve. That function also resolves HTTP `Location:` headers, which
14// are not HTML: an `&` in a Location header is five literal bytes of the real url, and decoding
15// it would corrupt a correct redirect.
16// ★DECODE BELONGS AT THE LAYER THAT KNOWS THE BYTES CAME FROM HTML, NOT AT THE LAYER THAT ONLY
17// KNOWS THEY ARE A URL.
18// license_tier: ORIGINAL
19
20const UC_SCRATCH: i64 = 4096 // per-call canon scratch; a url longer than this falls back to raw bytes
21
22// ==== URL CANONICALISATION AT HARVEST (debt 1786031857, 2026-08-06) ====
23// A LINK HARVESTED VERBATIM FROM HTML IS NOT A URL UNTIL ITS ENTITIES ARE DECODED. An href is HTML
24// *text*: `?a=1&b=2` is correctly authored `?a=1&b=2`, so copying the attribute byte-for-byte
25// yields a url whose 2nd and later params are named `amp;b`. It 404s, fetch-fails or returns the
26// WRONG page -- and the crawler then blames the HOST and bumps its dead-host streak, so OUR parsing
27// bug becomes PERMANENT coverage loss. MEASURED within minutes of the outcome log shipping, on two
28// independent rows: arxiv.org/search/econ?searchtype=author&query=... (fetchfail) and
29// loc.gov/resource/bdsdcc.18101/?sp=2&st=image (403). It also inflates the R10 param budget,
30// because `amp;st` counts as a param, so a 3-param url is judged as 4.
31//
32// DELIBERATELY NOT nx_html_entity_decode (nx_html_tokenizer.nx), and NOT to be merged with it later:
33// that decoder is correct for HTML *text*, where a numeric reference becomes raw UTF-8. In URL
34// position raw UTF-8 is a malformed request line -- a browser percent-encodes it. Different
35// correctness contract => a separate function, named apart. (It would also drag the whole tokenizer
36// into nx_cc_ingest / nx_cc_warc_ingest / nx_page_ingest, which import this lib and none of the DOM.)
37//
38// Browser parity (WHATWG URL + HTML attribute rules), ONE left-to-right pass, IDEMPOTENT:
39// 1. TAB/LF/CR are REMOVED outright -- pretty-printed HTML splits long hrefs across lines, and a
40// url carrying a literal newline is an invalid request line, not a slow fetch.
41// 2. leading/trailing C0+space are trimmed.
42// 3. character references (named, &#nn;, &#xhh;) are decoded to a codepoint.
43// 4. every emitted byte illegal in a url (C0, space, " < > \ ^ ` { | }, DEL, >=0x80) is
44// percent-encoded; non-ascii codepoints go out as percent-encoded UTF-8.
45// 5. '%' passes through untouched, so an ALREADY-encoded url is never double-encoded -- this is
46// what makes the pass idempotent, which matters because a url can be re-harvested every run.
47// 6. an UNRECOGNISED reference is passed through VERBATIM -- never silently swallow data.
48// KNOWN NON-GOAL: an href that encodes its own scheme (`https://host`) is not detected as
49// absolute, because absolute-vs-relative is judged BEFORE this pass. Not observed in the wild here;
50// recorded so a future reader knows it was considered rather than missed.
51func uc_hexdig(v: i64) -> i64 {
52 if v < 10 { return 48 + v }
53 return 55 + v
54}
55// 1 = this byte must be percent-encoded to ride in a url.
56func uc_url_unsafe(c: i64) -> i64 {
57 if c <= 0x20 { return 1 }
58 if c >= 0x7f { return 1 }
59 if c == 0x22 { return 1 }
60 if c == 0x3c { return 1 }
61 if c == 0x3e { return 1 }
62 if c == 0x5c { return 1 }
63 if c == 0x5e { return 1 }
64 if c == 0x60 { return 1 }
65 if c == 0x7b { return 1 }
66 if c == 0x7c { return 1 }
67 if c == 0x7d { return 1 }
68 return 0
69}
70// emit one byte, percent-encoding when unsafe. returns the new dst offset, or -1 if it would overflow.
71func uc_emit_byte(c: i64, dst: *u8, dp: i64, cap: i64) -> i64 {
72 if uc_url_unsafe(c) == 1 {
73 if dp + 3 > cap { return 0 - 1 }
74 dst[dp] = 37 as u8
75 dst[dp + 1] = uc_hexdig((c >> 4) & 0xf) as u8
76 dst[dp + 2] = uc_hexdig(c & 0xf) as u8
77 return dp + 3
78 }
79 if dp + 1 > cap { return 0 - 1 }
80 dst[dp] = c as u8
81 return dp + 1
82}
83// emit a codepoint as (percent-encoded) UTF-8.
84func uc_emit_cp(cp: i64, dst: *u8, dp: i64, cap: i64) -> i64 {
85 if cp < 0x80 { return uc_emit_byte(cp, dst, dp, cap) }
86 if cp < 0x800 {
87 var o: i64 = uc_emit_byte(0xc0 | (cp >> 6), dst, dp, cap)
88 if o < 0 { return o }
89 return uc_emit_byte(0x80 | (cp & 0x3f), dst, o, cap)
90 }
91 if cp < 0x10000 {
92 var o1: i64 = uc_emit_byte(0xe0 | (cp >> 12), dst, dp, cap)
93 if o1 < 0 { return o1 }
94 o1 = uc_emit_byte(0x80 | ((cp >> 6) & 0x3f), dst, o1, cap)
95 if o1 < 0 { return o1 }
96 return uc_emit_byte(0x80 | (cp & 0x3f), dst, o1, cap)
97 }
98 var o2: i64 = uc_emit_byte(0xf0 | (cp >> 18), dst, dp, cap)
99 if o2 < 0 { return o2 }
100 o2 = uc_emit_byte(0x80 | ((cp >> 12) & 0x3f), dst, o2, cap)
101 if o2 < 0 { return o2 }
102 o2 = uc_emit_byte(0x80 | ((cp >> 6) & 0x3f), dst, o2, cap)
103 if o2 < 0 { return o2 }
104 return uc_emit_byte(0x80 | (cp & 0x3f), dst, o2, cap)
105}
106// src[off..off+n) == lit (NUL-terminated), exact length match.
107func uc_ent_eq(src: *u8, off: i64, n: i64, lit: *u8) -> i64 {
108 var k: i64 = 0
109 while k < n {
110 if lit[k] == (0 as u8) { return 0 }
111 if src[off + k] != lit[k] { return 0 }
112 k = k + 1
113 }
114 if lit[n] != (0 as u8) { return 0 }
115 return 1
116}
117// The named references that actually occur inside href attributes. -1 = unknown (passed through).
118func uc_named_cp(src: *u8, off: i64, n: i64) -> i64 {
119 if uc_ent_eq(src, off, n, "amp" as *u8) == 1 { return 38 }
120 if uc_ent_eq(src, off, n, "AMP" as *u8) == 1 { return 38 }
121 if uc_ent_eq(src, off, n, "quot" as *u8) == 1 { return 34 }
122 if uc_ent_eq(src, off, n, "apos" as *u8) == 1 { return 39 }
123 if uc_ent_eq(src, off, n, "lt" as *u8) == 1 { return 60 }
124 if uc_ent_eq(src, off, n, "gt" as *u8) == 1 { return 62 }
125 if uc_ent_eq(src, off, n, "nbsp" as *u8) == 1 { return 32 }
126 if uc_ent_eq(src, off, n, "sol" as *u8) == 1 { return 47 }
127 if uc_ent_eq(src, off, n, "colon" as *u8) == 1 { return 58 }
128 if uc_ent_eq(src, off, n, "semi" as *u8) == 1 { return 59 }
129 if uc_ent_eq(src, off, n, "equals" as *u8) == 1 { return 61 }
130 if uc_ent_eq(src, off, n, "quest" as *u8) == 1 { return 63 }
131 if uc_ent_eq(src, off, n, "num" as *u8) == 1 { return 35 }
132 if uc_ent_eq(src, off, n, "percnt" as *u8) == 1 { return 37 }
133 if uc_ent_eq(src, off, n, "plus" as *u8) == 1 { return 43 }
134 if uc_ent_eq(src, off, n, "comma" as *u8) == 1 { return 44 }
135 if uc_ent_eq(src, off, n, "period" as *u8) == 1 { return 46 }
136 if uc_ent_eq(src, off, n, "excl" as *u8) == 1 { return 33 }
137 if uc_ent_eq(src, off, n, "ast" as *u8) == 1 { return 42 }
138 if uc_ent_eq(src, off, n, "lpar" as *u8) == 1 { return 40 }
139 if uc_ent_eq(src, off, n, "rpar" as *u8) == 1 { return 41 }
140 if uc_ent_eq(src, off, n, "lsqb" as *u8) == 1 { return 91 }
141 if uc_ent_eq(src, off, n, "rsqb" as *u8) == 1 { return 93 }
142 if uc_ent_eq(src, off, n, "lowbar" as *u8) == 1 { return 95 }
143 if uc_ent_eq(src, off, n, "hyphen" as *u8) == 1 { return 45 }
144 if uc_ent_eq(src, off, n, "tilde" as *u8) == 1 { return 126 }
145 if uc_ent_eq(src, off, n, "dollar" as *u8) == 1 { return 36 }
146 if uc_ent_eq(src, off, n, "commat" as *u8) == 1 { return 64 }
147 if uc_ent_eq(src, off, n, "verbar" as *u8) == 1 { return 124 }
148 return 0 - 1
149}
150// &#nn; / &#xhh; -> codepoint. -1 on any malformed or out-of-range body.
151func uc_numeric_cp(src: *u8, off: i64, n: i64) -> i64 {
152 if n < 2 { return 0 - 1 }
153 let end: i64 = off + n
154 var i: i64 = off + 1
155 var hex: i64 = 0
156 let c0: i64 = src[i] as i64
157 if c0 == 120 { hex = 1; i = i + 1 } else { if c0 == 88 { hex = 1; i = i + 1 } }
158 var v: i64 = 0
159 var got: i64 = 0
160 var bad: i64 = 0
161 while i < end {
162 let d: i64 = src[i] as i64
163 var dv: i64 = 0 - 1
164 if d >= 48 { if d <= 57 { dv = d - 48 } }
165 if hex == 1 {
166 if dv < 0 { if d >= 97 { if d <= 102 { dv = d - 87 } } }
167 if dv < 0 { if d >= 65 { if d <= 70 { dv = d - 55 } } }
168 }
169 if dv < 0 { bad = 1; i = end }
170 else {
171 if hex == 1 { v = v * 16 + dv } else { v = v * 10 + dv }
172 got = 1
173 i = i + 1
174 if v > 0x10ffff { bad = 1; i = end }
175 }
176 }
177 if bad == 1 { return 0 - 1 }
178 if got == 0 { return 0 - 1 }
179 if v == 0 { return 0 - 1 }
180 return v
181}
182// THE CHOKEPOINT. src = raw href bytes (already root-resolved); dst gets the wire url, NUL-terminated.
183// Returns the new length, or 0 on refusal (empty result, or dstcap exhausted) so the caller SKIPS the
184// link rather than fetching a truncated url -- a truncated url is a wrong url, not a shorter one.
185// ---- WIRE NORMALISATION (2026-08-25) --------------------------------------------------------------
186// ADDED HERE, AND NOT AS A FOURTH FILE, DELIBERATELY. A census this session found THREE url
187// canonicalisers in this tree: this one; nx_url_canonical.nx (otpauth:// URIs -- a genuinely different
188// subject, correctly separate); and nx_urlcanon.nx, which declares `wired_status: FULLY_WIRED` and which
189// NOTHING IMPORTS. That dead file ALSO defines a function named nx_url_canon, so the two can never share
190// a build closure -- a symbol collision hiding behind a near-identical filename. Its capability (scheme
191// and host case, default port, fragment) is real and was simply unreachable, so it is folded in HERE,
192// into the copy every harvester actually calls.
193// A SELF-DECLARED wired_status IS A COMMENT, NOT A MEASUREMENT -- the importer census is.
194//
195// THE MEASURED DEFECT THIS CLOSES: our fetcher speaks TLS ONLY, yet http:// links were admitted to the
196// crawl frontier where each is a GUARANTEED fetchfail. Worse, wc_host strips the scheme, so ONE dead
197// http:// row poisons the dead-host entry for every WORKING https:// row of the same host for the rest
198// of that run -- witnessed on www.theguardian.com (four ok 200s, then one http:// fetchfail, then the
199// whole host skipped). On a frontier measured flat at ~1,880 urls those rows are pure loss.
200//
201// RFC 3986 6.2.2.1 (case normalization) and 6.2.3 (scheme-based, default port) are the authority for the
202// first three rules. The https upgrade is POLICY, not RFC, and is a named const rather than a conf row
203// on purpose: this file declares itself "LIB (no main, no imports)" and pulling in a conf reader would
204// break that property for every consumer. If the fetcher ever speaks plaintext, this const is the single
205// place that changes.
206// DELIBERATELY NOT DONE HERE: query-parameter sorting. It is widely published and it is WRONG --
207// parameter order is semantically significant to some servers, so sorting can change the resource.
208const UC_HTTPS_UPGRADE: i64 = 1
209const UC_CH_FRAG: i64 = 35
210
211func uc_lc_ascii(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
212
213// Normalise the WIRE form of an already entity-decoded url, in place. Returns the new length, or 0 to
214// REFUSE -- matching this file's existing contract that a url which cannot be produced correctly is
215// skipped rather than truncated (a truncated url is a WRONG url, and fetching it blames the host for
216// our overflow).
217func uc_wire_norm(b: *u8, n: i64, cap: i64) -> i64 {
218 var len: i64 = n
219 // FRAGMENT: resolved by the client, never sent to the origin, so two urls differing only after '#'
220 // are ONE resource to a crawler.
221 var f: i64 = 0
222 while f < len { if (b[f] as i64) == UC_CH_FRAG { len = f } else { f = f + 1 } }
223 if len <= 0 { return 0 }
224 var se: i64 = 0 - 1
225 var i: i64 = 0
226 while i + 2 < len {
227 if se < 0 { if (b[i] as i64) == 0x3A { if (b[i+1] as i64) == 0x2F { if (b[i+2] as i64) == 0x2F { se = i } } } }
228 i = i + 1
229 }
230 if se < 0 { return len } // schemeless: we do not INVENT a scheme, we leave it exactly as it is
231 var k: i64 = 0
232 while k < se { b[k] = uc_lc_ascii(b[k] as i64) as u8; k = k + 1 }
233 var ishttp: i64 = 0
234 if se == 4 { if (b[0] as i64)==0x68 { if (b[1] as i64)==0x74 { if (b[2] as i64)==0x74 { if (b[3] as i64)==0x70 { ishttp = 1 } } } } }
235 // THE DEFAULT PORT IS A PROPERTY OF THE SCHEME THE AUTHOR WROTE, so it is decided HERE, BEFORE the
236 // upgrade rewrites the scheme. Caught by this file's own gate tooth T7n2: the first version read
237 // b[4]=='s' to pick 443-vs-80 AFTER upgrading, so http://e.org:80/a became https://e.org:80/a --
238 // :80 is not https's default, so the port survived, and an upgrade silently PINNED a port the author
239 // had left implicit. Judging it against the original scheme is the whole fix.
240 var deflt_port: i64 = 443
241 if ishttp == 1 { deflt_port = 80 }
242 if ishttp == 1 { if UC_HTTPS_UPGRADE == 1 {
243 if len + 1 >= cap { return 0 }
244 var m: i64 = len
245 while m > 4 { b[m] = b[m - 1]; m = m - 1 }
246 b[4] = 0x73 as u8
247 len = len + 1
248 se = 5
249 } }
250 var h: i64 = se + 3
251 var g2: i64 = 1
252 while g2 == 1 {
253 if h >= len { g2 = 0 } else {
254 let c: i64 = b[h] as i64
255 if c == 0x2F { g2 = 0 } else { if c == 0x3A { g2 = 0 } else { if c == 0x3F { g2 = 0 } else { b[h] = uc_lc_ascii(c) as u8; h = h + 1 } } }
256 }
257 }
258 if h < len { if (b[h] as i64) == 0x3A {
259 var pe: i64 = h + 1
260 var pv: i64 = 0
261 var digits: i64 = 0
262 var g3: i64 = 1
263 while g3 == 1 {
264 if pe >= len { g3 = 0 } else {
265 let d: i64 = b[pe] as i64
266 if d >= 0x30 { if d <= 0x39 { pv = pv * 10 + (d - 0x30); digits = digits + 1; pe = pe + 1 } else { g3 = 0 } } else { g3 = 0 }
267 }
268 }
269 if digits > 0 {
270 if pv == deflt_port {
271 var w: i64 = h
272 var r: i64 = pe
273 while r < len { b[w] = b[r]; w = w + 1; r = r + 1 }
274 len = w
275 }
276 }
277 } }
278 return len
279}
280
281func nx_url_canon(src: *u8, srclen: i64, dst: *u8, dstcap: i64) -> i64 {
282 var sp: i64 = 0
283 var ep: i64 = srclen
284 var go: i64 = 1
285 while go == 1 { go = 0; if sp < ep { if (src[sp] as i64) <= 0x20 { sp = sp + 1; go = 1 } } }
286 go = 1
287 while go == 1 { go = 0; if ep > sp { if (src[ep - 1] as i64) <= 0x20 { ep = ep - 1; go = 1 } } }
288 var dp: i64 = 0
289 var i: i64 = sp
290 while i < ep {
291 let c: i64 = src[i] as i64
292 var handled: i64 = 0
293 if c == 9 { i = i + 1; handled = 1 }
294 if c == 10 { if handled == 0 { i = i + 1; handled = 1 } }
295 if c == 13 { if handled == 0 { i = i + 1; handled = 1 } }
296 if handled == 0 { if c == 38 {
297 handled = 1
298 var semi: i64 = 0 - 1
299 var scan: i64 = i + 1
300 var lim: i64 = i + 12
301 if lim > ep { lim = ep }
302 while scan < lim {
303 if (src[scan] as i64) == 59 { semi = scan; scan = lim } else { scan = scan + 1 }
304 }
305 var cp: i64 = 0 - 1
306 if semi > 0 {
307 let blen: i64 = semi - (i + 1)
308 if blen > 0 {
309 if (src[i + 1] as i64) == 35 { cp = uc_numeric_cp(src, i + 1, blen) }
310 else { cp = uc_named_cp(src, i + 1, blen) }
311 }
312 }
313 if cp >= 0 {
314 let nd: i64 = uc_emit_cp(cp, dst, dp, dstcap)
315 if nd < 0 { return 0 }
316 dp = nd
317 i = semi + 1
318 } else {
319 let nd2: i64 = uc_emit_byte(38, dst, dp, dstcap)
320 if nd2 < 0 { return 0 }
321 dp = nd2
322 i = i + 1
323 }
324 } }
325 if handled == 0 {
326 let nd3: i64 = uc_emit_byte(c, dst, dp, dstcap)
327 if nd3 < 0 { return 0 }
328 dp = nd3
329 i = i + 1
330 }
331 }
332 if dp <= 0 { return 0 }
333 // WIRE NORMALISATION runs AFTER entity decoding, never before: '&' must become '&' before any
334 // rule reasons about query structure, or the parameter boundaries are wrong.
335 dp = uc_wire_norm(dst, dp, dstcap)
336 if dp <= 0 { return 0 }
337 if dp + 1 > dstcap { return 0 }
338 dst[dp] = 0 as u8
339 return dp
340}