nx_html_tokenizer.nx source
↩ module page · 777 lines · 31157 B
1// nx_html_tokenizer.nx -- HTML tokenizer for the Nishi browser.
2// Phase 1 of docs/NISHI_BROWSER_ROADMAP.md.
3//
4// Recognises:
5// - <!DOCTYPE html> (and any doctype line; we don't parse its body)
6// - <tag> start tag
7// - </tag> end tag
8// - <tag /> self-closing
9// - <tag attr="v"> start tag with attributes
10// - <!-- ... --> comment
11// - text anything outside tags
12//
13// Returns a stream of tokens via a caller-iterated cursor. No
14// allocations beyond the caller's buffer. Sealed token kinds for
15// the next layer (HTML tree builder, Phase 1b).
16//
17// What it does NOT handle yet (named L1b improvements):
18// - HTML5 raw-text / RCDATA tags (script / style / textarea)
19// - foreign content (SVG, MathML namespace switches)
20// - character entity references beyond the basic 5 (amp/lt/gt/quot/apos)
21// - CDATA sections
22// - explicit attribute-value escaping rules around > inside attrs
23//
24// Subset is sufficient for andelinwest.com + ide.html + most modern
25// hand-written HTML. Per cardinal feedback-honest-perf-verdict: we
26// list exactly what's missing.
27//
28// genealogy_id: whatwg_html_tokenizer_2024 + html5_spec_chapter_12 +
29// webwhatwg_html_tokenizer_state_machine
30// lineage_id: nishi_browser_html_tokenizer_q10
31//
32// nx_safety_envelope:
33// intended_use: "HTML5 tokenizer -- substrate browser front-
34// end + a11y-grader input + general HTML parse"
35// sil_target: SIL2 (HTML parsing; XSS vectors gate
36// on tokenizer correctness)
37// asil_target: QM
38// dal_target: DAL C
39// evidence: [WHATWG_HTML5_canonical_basis,
40// sealed_token_kind_enum,
41// bounded_state_machine,
42// attribute_value_unescape_per_spec]
43// hazard_register: [bug-tape-mXSS-script-tag-in-attribute,
44// bug-tape-malformed-comment-CDATA-confusion,
45// bug-tape-charref-overflow-via-large-ncr]
46// residual_risk: "Sanitization (e.g. DOMPurify-class output
47// filtering) is upstream consumer concern;
48// substrate provides token stream only."
49// verdict: NOT_YET_EVALUATED
50
51// (No syscall import needed -- tokenizer is pure byte arithmetic.
52// Removed `import "nx_syscalls_x86_64.nx"` per cross-arch cleanup
53// 2026-05-20 since the tokenizer uses zero syscalls. Importers
54// that need syscalls add `import "nx_syscalls.nx"` themselves --
55// all 5 existing callers already do.)
56
57// Sealed token kind enum.
58const NX_HTML_TOK_UNKNOWN: i64 = 0
59const NX_HTML_TOK_DOCTYPE: i64 = 1
60const NX_HTML_TOK_START_TAG: i64 = 2
61const NX_HTML_TOK_END_TAG: i64 = 3
62const NX_HTML_TOK_SELF_CLOSING: i64 = 4
63const NX_HTML_TOK_TEXT: i64 = 5
64const NX_HTML_TOK_COMMENT: i64 = 6
65const NX_HTML_TOK_EOF: i64 = 7
66const NX_HTML_TOK_N: i64 = 8
67
68// One token's view into the source. Offsets + lengths into the
69// caller-supplied source buffer; no string allocation.
70struct HtmlToken {
71 kind: i64, // NX_HTML_TOK_*
72 src_off: i64, // offset in source where token begins
73 src_len: i64, // byte length of full token in source
74 name_off: i64, // tag-name offset (for START/END/SELF_CLOSING)
75 name_len: i64, // tag-name length
76 body_off: i64, // for COMMENT: comment body; for TEXT: text bytes
77 body_len: i64
78}
79
80// Tokenizer cursor. Caller advances; reuses the same HtmlToken slot.
81struct HtmlCursor {
82 src: *u8,
83 src_len: i64,
84 pos: i64
85}
86
87func nx_html_cursor_init(c: *HtmlCursor, src: *u8, src_len: i64) -> i64 {
88 c.src = src
89 c.src_len = src_len
90 c.pos = 0
91 return 0
92}
93
94// Lowercase ASCII byte (the only case we care about for tag matching).
95func _lc(b: i64) -> i64 {
96 if b >= 65 {
97 if b <= 90 {
98 return b + 32
99 }
100 }
101 return b
102}
103
104// ASCII-whitespace check.
105func _is_ws(b: i64) -> i64 {
106 if b == 32 { return 1 }
107 if b == 9 { return 1 }
108 if b == 10 { return 1 }
109 if b == 13 { return 1 }
110 if b == 12 { return 1 }
111 return 0
112}
113
114// Scan an identifier (tag/attr name): ASCII letter/digit/-/_ etc.
115func _is_name(b: i64) -> i64 {
116 if b >= 97 { if b <= 122 { return 1 } } // a-z
117 if b >= 65 { if b <= 90 { return 1 } } // A-Z
118 if b >= 48 { if b <= 57 { return 1 } } // 0-9
119 if b == 45 { return 1 } // -
120 if b == 95 { return 1 } // _
121 if b == 58 { return 1 } // :
122 return 0
123}
124
125// Walk past whitespace. Returns new pos.
126func _skip_ws(src: *u8, src_len: i64, p: i64) -> i64 {
127 var pos: i64 = p
128 while pos < src_len {
129 if _is_ws(src[pos]) == 1 { pos = pos + 1 }
130 else { return pos }
131 }
132 return pos
133}
134
135// Walk until we see one of the stop bytes. Returns new pos.
136func _scan_name(src: *u8, src_len: i64, p: i64) -> i64 {
137 var pos: i64 = p
138 while pos < src_len {
139 if _is_name(src[pos]) == 1 { pos = pos + 1 }
140 else { return pos }
141 }
142 return pos
143}
144
145// Top-level: produce the next token in the stream. Returns
146// NX_HTML_TOK_EOF when stream is exhausted.
147// HTML5 §13.1.2 void elements: area, base, br, col, embed, hr, img,
148// input, link, meta, param, source, track, wbr. Tokenizer promotes
149// START_TAG -> SELF_CLOSING for any name in this set, regardless of
150// whether the source had an explicit `/>` suffix.
151func _is_void_element(src: *u8, name_off: i64, name_len: i64) -> i64 {
152 if name_len < 2 { return 0 }
153 if name_len > 6 { return 0 }
154 // Lowercase first byte to support both <META> and <meta>.
155 let c0: i64 = _lc(src[name_off])
156 // Two-letter: br, hr
157 if name_len == 2 {
158 let c1: i64 = _lc(src[name_off + 1])
159 if c0 == 0x62 { if c1 == 0x72 { return 1 } } // br
160 if c0 == 0x68 { if c1 == 0x72 { return 1 } } // hr
161 return 0
162 }
163 // Three-letter: col, img, wbr
164 if name_len == 3 {
165 let c1: i64 = _lc(src[name_off + 1])
166 let c2: i64 = _lc(src[name_off + 2])
167 if c0 == 0x63 { if c1 == 0x6f { if c2 == 0x6c { return 1 } } } // col
168 if c0 == 0x69 { if c1 == 0x6d { if c2 == 0x67 { return 1 } } } // img
169 if c0 == 0x77 { if c1 == 0x62 { if c2 == 0x72 { return 1 } } } // wbr
170 return 0
171 }
172 // Four-letter: area, base, link, meta
173 if name_len == 4 {
174 let c1: i64 = _lc(src[name_off + 1])
175 let c2: i64 = _lc(src[name_off + 2])
176 let c3: i64 = _lc(src[name_off + 3])
177 if c0 == 0x61 { if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x61 { return 1 } } } } // area
178 if c0 == 0x62 { if c1 == 0x61 { if c2 == 0x73 { if c3 == 0x65 { return 1 } } } } // base
179 if c0 == 0x6c { if c1 == 0x69 { if c2 == 0x6e { if c3 == 0x6b { return 1 } } } } // link
180 if c0 == 0x6d { if c1 == 0x65 { if c2 == 0x74 { if c3 == 0x61 { return 1 } } } } // meta
181 return 0
182 }
183 // Five-letter: embed, input, param, track
184 if name_len == 5 {
185 let c1: i64 = _lc(src[name_off + 1])
186 let c2: i64 = _lc(src[name_off + 2])
187 let c3: i64 = _lc(src[name_off + 3])
188 let c4: i64 = _lc(src[name_off + 4])
189 if c0 == 0x65 { if c1 == 0x6d { if c2 == 0x62 { if c3 == 0x65 { if c4 == 0x64 { return 1 } } } } } // embed
190 if c0 == 0x69 { if c1 == 0x6e { if c2 == 0x70 { if c3 == 0x75 { if c4 == 0x74 { return 1 } } } } } // input
191 if c0 == 0x70 { if c1 == 0x61 { if c2 == 0x72 { if c3 == 0x61 { if c4 == 0x6d { return 1 } } } } } // param
192 if c0 == 0x74 { if c1 == 0x72 { if c2 == 0x61 { if c3 == 0x63 { if c4 == 0x6b { return 1 } } } } } // track
193 return 0
194 }
195 // Six-letter: source
196 if name_len == 6 {
197 let c1: i64 = _lc(src[name_off + 1])
198 let c2: i64 = _lc(src[name_off + 2])
199 let c3: i64 = _lc(src[name_off + 3])
200 let c4: i64 = _lc(src[name_off + 4])
201 let c5: i64 = _lc(src[name_off + 5])
202 if c0 == 0x73 { if c1 == 0x6f { if c2 == 0x75 { if c3 == 0x72 { if c4 == 0x63 { if c5 == 0x65 { return 1 } } } } } }
203 return 0
204 }
205 return 0
206}
207
208func nx_html_next_token(c: *HtmlCursor, out: *HtmlToken) -> i64 {
209 let src: *u8 = c.src
210 let len: i64 = c.src_len
211 var pos: i64 = c.pos
212
213 out.kind = NX_HTML_TOK_UNKNOWN
214 out.src_off = pos
215 out.src_len = 0
216 out.name_off = 0
217 out.name_len = 0
218 out.body_off = 0
219 out.body_len = 0
220
221 if pos >= len {
222 out.kind = NX_HTML_TOK_EOF
223 c.pos = pos
224 return 0
225 }
226
227 // Outside-tag: text vs tag-start.
228 if src[pos] != 60 { // not '<'
229 let t_start: i64 = pos
230 while pos < len {
231 if src[pos] == 60 { pos = pos + (len + 1) } // exit
232 else { pos = pos + 1 }
233 }
234 if pos > len { pos = pos - (len + 1) }
235 out.kind = NX_HTML_TOK_TEXT
236 out.src_off = t_start
237 out.src_len = pos - t_start
238 out.body_off = t_start
239 out.body_len = pos - t_start
240 c.pos = pos
241 return 0
242 }
243
244 // '<' -- could be tag, end-tag, comment, doctype.
245 pos = pos + 1
246 if pos >= len {
247 out.kind = NX_HTML_TOK_TEXT
248 out.src_off = pos - 1
249 out.src_len = 1
250 c.pos = pos
251 return 0
252 }
253
254 // Comment: <!--
255 if src[pos] == 33 { // '!'
256 if pos + 2 < len {
257 if src[pos+1] == 45 {
258 if src[pos+2] == 45 {
259 // Comment. Skip until "-->".
260 var p: i64 = pos + 3
261 let body_start: i64 = p
262 while p + 2 < len {
263 if src[p] == 45 {
264 if src[p+1] == 45 {
265 if src[p+2] == 62 {
266 out.kind = NX_HTML_TOK_COMMENT
267 out.src_off = c.pos
268 out.body_off = body_start
269 out.body_len = p - body_start
270 c.pos = p + 3
271 out.src_len = c.pos - out.src_off
272 return 0
273 }
274 }
275 }
276 p = p + 1
277 }
278 // Unterminated comment.
279 out.kind = NX_HTML_TOK_COMMENT
280 out.src_off = c.pos
281 out.body_off = body_start
282 out.body_len = len - body_start
283 c.pos = len
284 out.src_len = c.pos - out.src_off
285 return 0
286 }
287 }
288 }
289 // Doctype: <!DOCTYPE ...>
290 // We don't parse contents; just consume to '>'.
291 var pd: i64 = pos + 1
292 while pd < len {
293 if src[pd] == 62 { pd = pd + (len + 1) }
294 else { pd = pd + 1 }
295 }
296 if pd > len { pd = pd - (len + 1) }
297 out.kind = NX_HTML_TOK_DOCTYPE
298 out.src_off = c.pos
299 c.pos = pd + 1
300 if c.pos > len { c.pos = len }
301 out.src_len = c.pos - out.src_off
302 return 0
303 }
304
305 // End tag: </name>
306 if src[pos] == 47 { // '/'
307 pos = pos + 1
308 let name_start: i64 = pos
309 pos = _scan_name(src, len, pos)
310 out.name_off = name_start
311 out.name_len = pos - name_start
312 // skip to '>'
313 while pos < len {
314 if src[pos] == 62 { pos = pos + (len + 1) }
315 else { pos = pos + 1 }
316 }
317 if pos > len { pos = pos - (len + 1) }
318 out.kind = NX_HTML_TOK_END_TAG
319 out.src_off = c.pos
320 c.pos = pos + 1
321 if c.pos > len { c.pos = len }
322 out.src_len = c.pos - out.src_off
323 return 0
324 }
325
326 // Start tag (possibly self-closing): <name ...>
327 let name_start: i64 = pos
328 pos = _scan_name(src, len, pos)
329 let name_end: i64 = pos
330 out.name_off = name_start
331 out.name_len = name_end - name_start
332 var self_closing: i64 = 0
333 // Skip attributes (we just consume them; tree-builder phase parses them).
334 // QUOTE-AWARE (HTML5 §13.2.5 attribute-value states): a '>' or '/' INSIDE a quoted
335 // attribute value is DATA, not markup. Without this, `data-x="a > b"` ended the tag at
336 // the inner '>' and the tag's remainder (`b">`, and any embedded markup) leaked as a
337 // TEXT token that then painted as visible content -- MEASURED on stackoverflow, whose
338 // data-attributes carry '>' and embedded HTML (nx_tok_attr_probe S1/S2/S3, 2026-07-27).
339 // `inq` = 0 outside a value, else the open-quote char (34 '"' or 39 '\''); only the
340 // matching quote closes it. An unterminated value consumes to end-of-input, matching how
341 // a real browser bounds the same malformed markup.
342 // 2026-08-27 -- ALSO UNQUOTED-VALUE-AWARE. Being quote-aware was necessary and NOT sufficient.
343 // HTML5 13.2.5 makes '/' the self-closing marker only in the before-attribute-name and
344 // after-attribute-value states; inside an UNQUOTED attribute value it is an ordinary character.
345 // MEASURED ON danluu.com: it writes hrefs unquoted and its URLs end in a slash, so every
346 // `<a href=https://danluu.com/perf-opt/>` ended in the byte pair `/>` and was tokenized as a
347 // SELF-CLOSING tag. A self-closing <a> takes no children, so every article title fell OUTSIDE
348 // its own anchor and painted as plain black text -- a whole page of links silently becoming
349 // non-links, with nothing failing loudly. The one link that worked was `href=#pt`: the only
350 // href on the page with no trailing slash.
351 // `inuv` = inside an unquoted value; entered after '=' (whitespace skipped) when the value does
352 // not open with a quote, and left ONLY at whitespace or '>'.
353 var inq: i64 = 0
354 var inuv: i64 = 0
355 while pos < len {
356 let bb: i64 = src[pos]
357 if inq != 0 {
358 if bb == inq { inq = 0 }
359 pos = pos + 1
360 } else {
361 if inuv == 1 {
362 if bb == 62 { pos = pos + (len + 1) } // '>' ends the tag
363 else {
364 var isws: i64 = 0
365 if bb == 32 { isws = 1 }
366 if bb == 9 { isws = 1 }
367 if bb == 10 { isws = 1 }
368 if bb == 12 { isws = 1 }
369 if bb == 13 { isws = 1 }
370 if isws == 1 { inuv = 0 }
371 pos = pos + 1
372 }
373 } else {
374 if bb == 34 { inq = 34; pos = pos + 1 } // enter "..."
375 else { if bb == 39 { inq = 39; pos = pos + 1 } // enter '...'
376 else {
377 if bb == 61 {
378 // '=' -- a value follows. Skip inter-token whitespace; a quoted value is picked
379 // up by the branches above on the next pass, anything else is an unquoted value.
380 pos = pos + 1
381 var sk: i64 = 1
382 while sk == 1 {
383 if pos >= len { sk = 0 }
384 else {
385 let vb: i64 = src[pos]
386 var vws: i64 = 0
387 if vb == 32 { vws = 1 }
388 if vb == 9 { vws = 1 }
389 if vb == 10 { vws = 1 }
390 if vb == 12 { vws = 1 }
391 if vb == 13 { vws = 1 }
392 if vws == 1 { pos = pos + 1 } else { sk = 0 }
393 }
394 }
395 if pos < len {
396 let qb: i64 = src[pos]
397 if qb == 34 { inuv = 0 } else { if qb == 39 { inuv = 0 } else { if qb == 62 { inuv = 0 } else { inuv = 1 } } }
398 }
399 }
400 else {
401 if bb == 62 { pos = pos + (len + 1) } // '>'
402 else {
403 if bb == 47 {
404 if pos + 1 < len {
405 if src[pos + 1] == 62 {
406 self_closing = 1
407 pos = pos + 1 // land ON the '>' (NOT past it) -- the shared
408 pos = pos + (len + 1) // `c.pos = pos + 1` below then lands AFTER '>',
409 // matching the plain-'>' branch. `pos + 2` overshot by one, eating the next
410 // char (e.g. the '<' of a following tag -> that tag leaked as TEXT). Bug hit
411 // every XHTML-style `<link/>`/`<br/>`/`<img/>` (Wikipedia ref dedup-style links).
412 } else { pos = pos + 1 }
413 } else { pos = pos + 1 }
414 } else { pos = pos + 1 }
415 }
416 }
417 } }
418 }
419 }
420 }
421 if pos > len { pos = pos - (len + 1) }
422 // HTML5 §13.1.2 void elements: regardless of `/>` syntax, these
423 // tags are always self-closing (no end-tag, no children in the
424 // parse tree). Promote START_TAG -> SELF_CLOSING when the name
425 // matches the void list. Without this, `<meta>` and `<link>`
426 // would push onto the parent stack, swallowing siblings as
427 // descendants (real-world example.com surfaced this 2026-05-20).
428 if self_closing == 0 {
429 if _is_void_element(src, out.name_off, out.name_len) == 1 {
430 self_closing = 1
431 }
432 }
433 if self_closing == 1 {
434 out.kind = NX_HTML_TOK_SELF_CLOSING
435 } else {
436 out.kind = NX_HTML_TOK_START_TAG
437 }
438 out.src_off = c.pos
439 c.pos = pos + 1
440 if c.pos > len { c.pos = len }
441 out.src_len = c.pos - out.src_off
442 return 0
443}
444
445// Compare a tag name (case-insensitive) against a caller-supplied
446// expected name. Convenience for tree-builder.
447func nx_html_name_eq(src: *u8, name_off: i64, name_len: i64,
448 expected: *u8, expected_len: i64) -> i64 {
449 if name_len != expected_len { return 0 }
450 var i: i64 = 0
451 while i < name_len {
452 if _lc(src[name_off + i]) != _lc(expected[i]) { return 0 }
453 i = i + 1
454 }
455 return 1
456}
457
458// Sealed-enum validity gate.
459func nx_html_tok_kind_is_valid(k: i64) -> i64 {
460 if k < 0 { return 0 }
461 if k >= NX_HTML_TOK_N { return 0 }
462 return 1
463}
464
465// -- HTML5 raw-text-mode primitives (Arc B1, NISHI_BROWSER_GOOGLE_IMAGE_SEARCH_ROADMAP)
466//
467// HTML5 §13.2.5.4 / §13.2.5.5 / §13.2.5.6 define four raw-text-class
468// elements where `<` inside content is NOT a tag-start: <script>,
469// <style>, <textarea>, <title>. Inside their bodies, the tokenizer
470// must scan for the matching </tagname> close-sequence
471// (case-insensitive) and treat everything between as a single TEXT
472// token. Real google.com responses contain large <script> blocks
473// that wreck the dumb-tokenizer parse tree without this.
474//
475// API is additive: existing callers (nx_dom_query, nx_layout_from_dom,
476// html_tokenizer_smoke, this file) continue working unchanged.
477// HTML5-aware callers detect a START_TAG for one of the four names
478// via nx_html_is_raw_text_tag and call nx_html_consume_raw_text to
479// advance the cursor past the body and receive a TEXT token for the
480// raw content.
481
482// Returns 1 if name is script/style/textarea/title (case-insensitive).
483func nx_html_is_raw_text_tag(src: *u8, name_off: i64, name_len: i64) -> i64 {
484 if name_len < 5 { return 0 }
485 if name_len > 8 { return 0 }
486 let c0: i64 = _lc(src[name_off])
487 if name_len == 5 {
488 // style, title
489 let c1: i64 = _lc(src[name_off + 1])
490 let c2: i64 = _lc(src[name_off + 2])
491 let c3: i64 = _lc(src[name_off + 3])
492 let c4: i64 = _lc(src[name_off + 4])
493 if c0 == 0x73 { if c1 == 0x74 { if c2 == 0x79 { if c3 == 0x6c { if c4 == 0x65 { return 1 } } } } } // style
494 if c0 == 0x74 { if c1 == 0x69 { if c2 == 0x74 { if c3 == 0x6c { if c4 == 0x65 { return 1 } } } } } // title
495 return 0
496 }
497 if name_len == 6 {
498 // script
499 let c1: i64 = _lc(src[name_off + 1])
500 let c2: i64 = _lc(src[name_off + 2])
501 let c3: i64 = _lc(src[name_off + 3])
502 let c4: i64 = _lc(src[name_off + 4])
503 let c5: i64 = _lc(src[name_off + 5])
504 if c0 == 0x73 { if c1 == 0x63 { if c2 == 0x72 { if c3 == 0x69 { if c4 == 0x70 { if c5 == 0x74 { return 1 } } } } } }
505 return 0
506 }
507 if name_len == 8 {
508 // textarea
509 let c1: i64 = _lc(src[name_off + 1])
510 let c2: i64 = _lc(src[name_off + 2])
511 let c3: i64 = _lc(src[name_off + 3])
512 let c4: i64 = _lc(src[name_off + 4])
513 let c5: i64 = _lc(src[name_off + 5])
514 let c6: i64 = _lc(src[name_off + 6])
515 let c7: i64 = _lc(src[name_off + 7])
516 if c0 == 0x74 {
517 if c1 == 0x65 { if c2 == 0x78 { if c3 == 0x74 { if c4 == 0x61 { if c5 == 0x72 { if c6 == 0x65 { if c7 == 0x61 { return 1 } } } } } } }
518 }
519 return 0
520 }
521 return 0
522}
523
524// After a START_TAG for a raw-text tag has been emitted, the caller
525// invokes this to consume the body up to (but not including) the
526// matching `</tagname>`. Emits a TEXT token spanning the raw bytes;
527// cursor is left positioned at the `<` of the close tag, so the
528// next nx_html_next_token call returns END_TAG normally. If no
529// matching close is found, consumes to EOF (substrate-honest: we
530// surface the truncation rather than fabricating a close).
531//
532// tag_name / tag_name_len describe the tag we are inside, used for
533// case-insensitive close-match. Caller supplies them so we don't
534// have to re-parse the just-emitted START_TAG name.
535func nx_html_consume_raw_text(c: *HtmlCursor, tag_name: *u8, tag_name_len: i64,
536 out: *HtmlToken) -> i64 {
537 let src: *u8 = c.src
538 let len: i64 = c.src_len
539 var pos: i64 = c.pos
540 let body_start: i64 = pos
541
542 out.kind = NX_HTML_TOK_TEXT
543 out.src_off = body_start
544 out.name_off = 0
545 out.name_len = 0
546 out.body_off = body_start
547
548 while pos < len {
549 // Look for `</` then case-insensitive tag_name then '>' or whitespace.
550 if src[pos] != 60 { pos = pos + 1 }
551 else {
552 if pos + 1 >= len { pos = pos + 1 }
553 else {
554 if src[pos + 1] != 47 { pos = pos + 1 } // '/'
555 else {
556 // Match tag name case-insensitively starting at pos+2.
557 var matched: i64 = 1
558 var i: i64 = 0
559 while i < tag_name_len {
560 if pos + 2 + i >= len { matched = 0; i = tag_name_len }
561 else {
562 if _lc(src[pos + 2 + i]) != _lc(tag_name[i]) { matched = 0; i = tag_name_len }
563 else { i = i + 1 }
564 }
565 }
566 if matched == 0 { pos = pos + 1 }
567 else {
568 // Next char must be '>' or whitespace (HTML5 spec).
569 let after: i64 = pos + 2 + tag_name_len
570 if after >= len { pos = pos + 1 }
571 else {
572 let nb: i64 = src[after]
573 if nb == 62 { pos = pos + (len + 1) } // '>'
574 else {
575 if _is_ws(nb) == 1 { pos = pos + (len + 1) }
576 else { pos = pos + 1 }
577 }
578 }
579 }
580 }
581 }
582 }
583 }
584 if pos > len { pos = pos - (len + 1) }
585 out.src_len = pos - body_start
586 out.body_len = pos - body_start
587 c.pos = pos
588 return 0
589}
590
591// -- HTML entity decoder (Arc B1 sibling primitive)
592//
593// Decodes character entity references in `src` into `dst` (cap-bounded).
594// Returns the number of bytes written to dst. Handles:
595// - the basic 5: & < > " '
596// - numeric: &#NN; (decimal) and &#xHH; / &#XHH; (hex)
597// - common named refs: © ® — – …
598// - unrecognised entities pass through unchanged (substrate-honest:
599// never silently swallow data)
600//
601// All output is UTF-8. Code points beyond 0x7F are encoded multi-byte
602// per RFC 3629; surrogate halves and code points beyond 0x10FFFF
603// emit the canonical replacement (U+FFFD 0xEF 0xBF 0xBD).
604func _entity_emit_u32(cp_in: i64, dst: *u8, dst_pos: i64, dst_cap: i64) -> i64 {
605 var p: i64 = dst_pos
606 var cp: i64 = cp_in
607 if cp < 0 { return p }
608 if cp > 0x10FFFF { cp = 0xFFFD }
609 if cp >= 0xD800 { if cp <= 0xDFFF { cp = 0xFFFD } }
610 if cp < 0x80 {
611 if p + 1 > dst_cap { return p }
612 dst[p] = cp as u8
613 return p + 1
614 }
615 if cp < 0x800 {
616 if p + 2 > dst_cap { return p }
617 dst[p] = (0xC0 + (cp >> 6)) as u8
618 dst[p + 1] = (0x80 + (cp & 0x3F)) as u8
619 return p + 2
620 }
621 if cp < 0x10000 {
622 if p + 3 > dst_cap { return p }
623 dst[p] = (0xE0 + (cp >> 12)) as u8
624 dst[p + 1] = (0x80 + ((cp >> 6) & 0x3F)) as u8
625 dst[p + 2] = (0x80 + (cp & 0x3F)) as u8
626 return p + 3
627 }
628 if p + 4 > dst_cap { return p }
629 dst[p] = (0xF0 + (cp >> 18)) as u8
630 dst[p + 1] = (0x80 + ((cp >> 12) & 0x3F)) as u8
631 dst[p + 2] = (0x80 + ((cp >> 6) & 0x3F)) as u8
632 dst[p + 3] = (0x80 + (cp & 0x3F)) as u8
633 return p + 4
634}
635
636// Returns 1 if src[sp..sp+nlen] case-insensitively equals lit[0..nlen].
637func _entity_name_eq(src: *u8, sp: i64, src_len: i64, lit: *u8, nlen: i64) -> i64 {
638 if sp + nlen > src_len { return 0 }
639 var i: i64 = 0
640 while i < nlen {
641 if _lc(src[sp + i]) != _lc(lit[i]) { return 0 }
642 i = i + 1
643 }
644 return 1
645}
646
647// Returns hex digit value (0..15) for ASCII byte, or -1 if not a hex digit.
648func _hex_val(b: i64) -> i64 {
649 if b >= 48 { if b <= 57 { return b - 48 } } // 0-9
650 if b >= 97 { if b <= 102 { return b - 87 } } // a-f
651 if b >= 65 { if b <= 70 { return b - 55 } } // A-F
652 return 0 - 1
653}
654
655// Returns decimal digit value (0..9) for ASCII byte, or -1 if not a digit.
656func _dec_val(b: i64) -> i64 {
657 if b >= 48 { if b <= 57 { return b - 48 } }
658 return 0 - 1
659}
660
661// Parse numeric entity body (bytes between '#' and ';'). Returns
662// code point on success, or -1 if malformed. body_off points to the
663// '#' byte itself.
664func _parse_numeric_entity(src: *u8, body_off: i64, body_len: i64) -> i64 {
665 if body_len < 2 { return 0 - 1 }
666 var p: i64 = body_off + 1
667 let end: i64 = body_off + body_len
668 var base: i64 = 10
669 let c0: i64 = src[p]
670 if c0 == 120 { base = 16; p = p + 1 } // 'x'
671 else { if c0 == 88 { base = 16; p = p + 1 } } // 'X'
672 if p >= end { return 0 - 1 }
673 var cp: i64 = 0
674 while p < end {
675 var dv: i64 = 0 - 1
676 if base == 16 { dv = _hex_val(src[p]) }
677 else { dv = _dec_val(src[p]) }
678 if dv < 0 { return 0 - 1 }
679 cp = cp * base + dv
680 if cp > 0x10FFFF { cp = 0x10FFFF + 1 } // clamp; emitter replaces
681 p = p + 1
682 }
683 return cp
684}
685
686// Look up a named entity body and return its code point, or -1 if unknown.
687// Names are case-sensitive per WHATWG (most are lowercase). Substrate
688// covers the common set; large named-entity table is deferred until
689// real Google response demands it.
690func _lookup_named_entity(src: *u8, body_off: i64, body_len: i64) -> i64 {
691 if body_len == 2 {
692 if _entity_name_eq(src, body_off, body_off + body_len, "lt" as *u8, 2) == 1 { return 0x3C }
693 if _entity_name_eq(src, body_off, body_off + body_len, "gt" as *u8, 2) == 1 { return 0x3E }
694 return 0 - 1
695 }
696 if body_len == 3 {
697 if _entity_name_eq(src, body_off, body_off + body_len, "amp" as *u8, 3) == 1 { return 0x26 }
698 if _entity_name_eq(src, body_off, body_off + body_len, "reg" as *u8, 3) == 1 { return 0xAE }
699 return 0 - 1
700 }
701 if body_len == 4 {
702 if _entity_name_eq(src, body_off, body_off + body_len, "quot" as *u8, 4) == 1 { return 0x22 }
703 if _entity_name_eq(src, body_off, body_off + body_len, "apos" as *u8, 4) == 1 { return 0x27 }
704 if _entity_name_eq(src, body_off, body_off + body_len, "nbsp" as *u8, 4) == 1 { return 0xA0 }
705 if _entity_name_eq(src, body_off, body_off + body_len, "copy" as *u8, 4) == 1 { return 0xA9 }
706 return 0 - 1
707 }
708 if body_len == 5 {
709 if _entity_name_eq(src, body_off, body_off + body_len, "mdash" as *u8, 5) == 1 { return 0x2014 }
710 if _entity_name_eq(src, body_off, body_off + body_len, "ndash" as *u8, 5) == 1 { return 0x2013 }
711 return 0 - 1
712 }
713 if body_len == 6 {
714 if _entity_name_eq(src, body_off, body_off + body_len, "hellip" as *u8, 6) == 1 { return 0x2026 }
715 return 0 - 1
716 }
717 return 0 - 1
718}
719
720func nx_html_entity_decode(src: *u8, src_len: i64, dst: *u8, dst_cap: i64) -> i64 {
721 var sp: i64 = 0
722 var dp: i64 = 0
723 while sp < src_len {
724 if src[sp] != 38 { // not '&'
725 if dp + 1 > dst_cap { return dp }
726 dst[dp] = src[sp]
727 dp = dp + 1
728 sp = sp + 1
729 } else {
730 // Find terminating ';' within 12 bytes (longest common name is 8).
731 var end: i64 = sp + 1
732 var limit: i64 = sp + 12
733 if limit > src_len { limit = src_len }
734 var found: i64 = 0
735 var scan: i64 = sp + 1
736 while scan < limit {
737 if src[scan] == 59 { end = scan; found = 1; scan = limit } // ';' -- exit
738 else { scan = scan + 1 }
739 }
740 if found == 0 {
741 // No terminator -- pass '&' through verbatim.
742 if dp + 1 > dst_cap { return dp }
743 dst[dp] = 38 as u8
744 dp = dp + 1
745 sp = sp + 1
746 } else {
747 // src[sp+1..end] is the entity body. end points at ';'.
748 let body_off: i64 = sp + 1
749 let body_len: i64 = end - body_off
750 var cp: i64 = 0 - 1
751 if body_len > 0 {
752 if src[body_off] == 35 { // '#' numeric
753 cp = _parse_numeric_entity(src, body_off, body_len)
754 } else {
755 cp = _lookup_named_entity(src, body_off, body_len)
756 }
757 }
758 if cp >= 0 {
759 dp = _entity_emit_u32(cp, dst, dp, dst_cap)
760 sp = end + 1
761 } else {
762 // Unrecognised -- pass &name; through verbatim per
763 // substrate-honest: never silently swallow data.
764 var k: i64 = sp
765 while k <= end {
766 if dp + 1 > dst_cap { return dp }
767 dst[dp] = src[k]
768 dp = dp + 1
769 k = k + 1
770 }
771 sp = end + 1
772 }
773 }
774 }
775 }
776 return dp
777}