nx_html_tokenizer.nx source
↩ module page · 726 lines · 28417 B
1// nx_html_tokenizer.nx -- HTML tokenizer for the Nishi browser.
2// Phase 1 of docs/NISHI_BROWSER_ROADMAP.md.
3//
4// Recognises:
5// - <!DOCTYPE html> (and any doctype line; we don't parse its body)
6// - <tag> start tag
7// - </tag> end tag
8// - <tag /> self-closing
9// - <tag attr="v"> start tag with attributes
10// - <!-- ... --> comment
11// - text anything outside tags
12//
13// Returns a stream of tokens via a caller-iterated cursor. No
14// allocations beyond the caller's buffer. Sealed token kinds for
15// the next layer (HTML tree builder, Phase 1b).
16//
17// What it does NOT handle yet (named L1b improvements):
18// - HTML5 raw-text / RCDATA tags (script / style / textarea)
19// - foreign content (SVG, MathML namespace switches)
20// - character entity references beyond the basic 5 (amp/lt/gt/quot/apos)
21// - CDATA sections
22// - explicit attribute-value escaping rules around > inside attrs
23//
24// Subset is sufficient for andelinwest.com + ide.html + most modern
25// hand-written HTML. Per cardinal feedback-honest-perf-verdict: we
26// list exactly what's missing.
27//
28// genealogy_id: whatwg_html_tokenizer_2024 + html5_spec_chapter_12 +
29// webwhatwg_html_tokenizer_state_machine
30// lineage_id: nishi_browser_html_tokenizer_q10
31//
32// nx_safety_envelope:
33// intended_use: "HTML5 tokenizer -- substrate browser front-
34// end + a11y-grader input + general HTML parse"
35// sil_target: SIL2 (HTML parsing; XSS vectors gate
36// on tokenizer correctness)
37// asil_target: QM
38// dal_target: DAL C
39// evidence: [WHATWG_HTML5_canonical_basis,
40// sealed_token_kind_enum,
41// bounded_state_machine,
42// attribute_value_unescape_per_spec]
43// hazard_register: [bug-tape-mXSS-script-tag-in-attribute,
44// bug-tape-malformed-comment-CDATA-confusion,
45// bug-tape-charref-overflow-via-large-ncr]
46// residual_risk: "Sanitization (e.g. DOMPurify-class output
47// filtering) is upstream consumer concern;
48// substrate provides token stream only."
49// verdict: NOT_YET_EVALUATED
50
51// (No syscall import needed -- tokenizer is pure byte arithmetic.
52// Removed `import "nx_syscalls_x86_64.nx"` per cross-arch cleanup
53// 2026-05-20 since the tokenizer uses zero syscalls. Importers
54// that need syscalls add `import "nx_syscalls.nx"` themselves --
55// all 5 existing callers already do.)
56
57// Sealed token kind enum.
58const NX_HTML_TOK_UNKNOWN: i64 = 0
59const NX_HTML_TOK_DOCTYPE: i64 = 1
60const NX_HTML_TOK_START_TAG: i64 = 2
61const NX_HTML_TOK_END_TAG: i64 = 3
62const NX_HTML_TOK_SELF_CLOSING: i64 = 4
63const NX_HTML_TOK_TEXT: i64 = 5
64const NX_HTML_TOK_COMMENT: i64 = 6
65const NX_HTML_TOK_EOF: i64 = 7
66const NX_HTML_TOK_N: i64 = 8
67
68// One token's view into the source. Offsets + lengths into the
69// caller-supplied source buffer; no string allocation.
70struct HtmlToken {
71 kind: i64, // NX_HTML_TOK_*
72 src_off: i64, // offset in source where token begins
73 src_len: i64, // byte length of full token in source
74 name_off: i64, // tag-name offset (for START/END/SELF_CLOSING)
75 name_len: i64, // tag-name length
76 body_off: i64, // for COMMENT: comment body; for TEXT: text bytes
77 body_len: i64
78}
79
80// Tokenizer cursor. Caller advances; reuses the same HtmlToken slot.
81struct HtmlCursor {
82 src: *u8,
83 src_len: i64,
84 pos: i64
85}
86
87func nx_html_cursor_init(c: *HtmlCursor, src: *u8, src_len: i64) -> i64 {
88 c.src = src
89 c.src_len = src_len
90 c.pos = 0
91 return 0
92}
93
94// Lowercase ASCII byte (the only case we care about for tag matching).
95func _lc(b: i64) -> i64 {
96 if b >= 65 {
97 if b <= 90 {
98 return b + 32
99 }
100 }
101 return b
102}
103
104// ASCII-whitespace check.
105func _is_ws(b: i64) -> i64 {
106 if b == 32 { return 1 }
107 if b == 9 { return 1 }
108 if b == 10 { return 1 }
109 if b == 13 { return 1 }
110 if b == 12 { return 1 }
111 return 0
112}
113
114// Scan an identifier (tag/attr name): ASCII letter/digit/-/_ etc.
115func _is_name(b: i64) -> i64 {
116 if b >= 97 { if b <= 122 { return 1 } } // a-z
117 if b >= 65 { if b <= 90 { return 1 } } // A-Z
118 if b >= 48 { if b <= 57 { return 1 } } // 0-9
119 if b == 45 { return 1 } // -
120 if b == 95 { return 1 } // _
121 if b == 58 { return 1 } // :
122 return 0
123}
124
125// Walk past whitespace. Returns new pos.
126func _skip_ws(src: *u8, src_len: i64, p: i64) -> i64 {
127 var pos: i64 = p
128 while pos < src_len {
129 if _is_ws(src[pos]) == 1 { pos = pos + 1 }
130 else { return pos }
131 }
132 return pos
133}
134
135// Walk until we see one of the stop bytes. Returns new pos.
136func _scan_name(src: *u8, src_len: i64, p: i64) -> i64 {
137 var pos: i64 = p
138 while pos < src_len {
139 if _is_name(src[pos]) == 1 { pos = pos + 1 }
140 else { return pos }
141 }
142 return pos
143}
144
145// Top-level: produce the next token in the stream. Returns
146// NX_HTML_TOK_EOF when stream is exhausted.
147// HTML5 §13.1.2 void elements: area, base, br, col, embed, hr, img,
148// input, link, meta, param, source, track, wbr. Tokenizer promotes
149// START_TAG -> SELF_CLOSING for any name in this set, regardless of
150// whether the source had an explicit `/>` suffix.
151func _is_void_element(src: *u8, name_off: i64, name_len: i64) -> i64 {
152 if name_len < 2 { return 0 }
153 if name_len > 6 { return 0 }
154 // Lowercase first byte to support both <META> and <meta>.
155 let c0: i64 = _lc(src[name_off])
156 // Two-letter: br, hr
157 if name_len == 2 {
158 let c1: i64 = _lc(src[name_off + 1])
159 if c0 == 0x62 { if c1 == 0x72 { return 1 } } // br
160 if c0 == 0x68 { if c1 == 0x72 { return 1 } } // hr
161 return 0
162 }
163 // Three-letter: col, img, wbr
164 if name_len == 3 {
165 let c1: i64 = _lc(src[name_off + 1])
166 let c2: i64 = _lc(src[name_off + 2])
167 if c0 == 0x63 { if c1 == 0x6f { if c2 == 0x6c { return 1 } } } // col
168 if c0 == 0x69 { if c1 == 0x6d { if c2 == 0x67 { return 1 } } } // img
169 if c0 == 0x77 { if c1 == 0x62 { if c2 == 0x72 { return 1 } } } // wbr
170 return 0
171 }
172 // Four-letter: area, base, link, meta
173 if name_len == 4 {
174 let c1: i64 = _lc(src[name_off + 1])
175 let c2: i64 = _lc(src[name_off + 2])
176 let c3: i64 = _lc(src[name_off + 3])
177 if c0 == 0x61 { if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x61 { return 1 } } } } // area
178 if c0 == 0x62 { if c1 == 0x61 { if c2 == 0x73 { if c3 == 0x65 { return 1 } } } } // base
179 if c0 == 0x6c { if c1 == 0x69 { if c2 == 0x6e { if c3 == 0x6b { return 1 } } } } // link
180 if c0 == 0x6d { if c1 == 0x65 { if c2 == 0x74 { if c3 == 0x61 { return 1 } } } } // meta
181 return 0
182 }
183 // Five-letter: embed, input, param, track
184 if name_len == 5 {
185 let c1: i64 = _lc(src[name_off + 1])
186 let c2: i64 = _lc(src[name_off + 2])
187 let c3: i64 = _lc(src[name_off + 3])
188 let c4: i64 = _lc(src[name_off + 4])
189 if c0 == 0x65 { if c1 == 0x6d { if c2 == 0x62 { if c3 == 0x65 { if c4 == 0x64 { return 1 } } } } } // embed
190 if c0 == 0x69 { if c1 == 0x6e { if c2 == 0x70 { if c3 == 0x75 { if c4 == 0x74 { return 1 } } } } } // input
191 if c0 == 0x70 { if c1 == 0x61 { if c2 == 0x72 { if c3 == 0x61 { if c4 == 0x6d { return 1 } } } } } // param
192 if c0 == 0x74 { if c1 == 0x72 { if c2 == 0x61 { if c3 == 0x63 { if c4 == 0x6b { return 1 } } } } } // track
193 return 0
194 }
195 // Six-letter: source
196 if name_len == 6 {
197 let c1: i64 = _lc(src[name_off + 1])
198 let c2: i64 = _lc(src[name_off + 2])
199 let c3: i64 = _lc(src[name_off + 3])
200 let c4: i64 = _lc(src[name_off + 4])
201 let c5: i64 = _lc(src[name_off + 5])
202 if c0 == 0x73 { if c1 == 0x6f { if c2 == 0x75 { if c3 == 0x72 { if c4 == 0x63 { if c5 == 0x65 { return 1 } } } } } }
203 return 0
204 }
205 return 0
206}
207
208func nx_html_next_token(c: *HtmlCursor, out: *HtmlToken) -> i64 {
209 let src: *u8 = c.src
210 let len: i64 = c.src_len
211 var pos: i64 = c.pos
212
213 out.kind = NX_HTML_TOK_UNKNOWN
214 out.src_off = pos
215 out.src_len = 0
216 out.name_off = 0
217 out.name_len = 0
218 out.body_off = 0
219 out.body_len = 0
220
221 if pos >= len {
222 out.kind = NX_HTML_TOK_EOF
223 c.pos = pos
224 return 0
225 }
226
227 // Outside-tag: text vs tag-start.
228 if src[pos] != 60 { // not '<'
229 let t_start: i64 = pos
230 while pos < len {
231 if src[pos] == 60 { pos = pos + (len + 1) } // exit
232 else { pos = pos + 1 }
233 }
234 if pos > len { pos = pos - (len + 1) }
235 out.kind = NX_HTML_TOK_TEXT
236 out.src_off = t_start
237 out.src_len = pos - t_start
238 out.body_off = t_start
239 out.body_len = pos - t_start
240 c.pos = pos
241 return 0
242 }
243
244 // '<' -- could be tag, end-tag, comment, doctype.
245 pos = pos + 1
246 if pos >= len {
247 out.kind = NX_HTML_TOK_TEXT
248 out.src_off = pos - 1
249 out.src_len = 1
250 c.pos = pos
251 return 0
252 }
253
254 // Comment: <!--
255 if src[pos] == 33 { // '!'
256 if pos + 2 < len {
257 if src[pos+1] == 45 {
258 if src[pos+2] == 45 {
259 // Comment. Skip until "-->".
260 var p: i64 = pos + 3
261 let body_start: i64 = p
262 while p + 2 < len {
263 if src[p] == 45 {
264 if src[p+1] == 45 {
265 if src[p+2] == 62 {
266 out.kind = NX_HTML_TOK_COMMENT
267 out.src_off = c.pos
268 out.body_off = body_start
269 out.body_len = p - body_start
270 c.pos = p + 3
271 out.src_len = c.pos - out.src_off
272 return 0
273 }
274 }
275 }
276 p = p + 1
277 }
278 // Unterminated comment.
279 out.kind = NX_HTML_TOK_COMMENT
280 out.src_off = c.pos
281 out.body_off = body_start
282 out.body_len = len - body_start
283 c.pos = len
284 out.src_len = c.pos - out.src_off
285 return 0
286 }
287 }
288 }
289 // Doctype: <!DOCTYPE ...>
290 // We don't parse contents; just consume to '>'.
291 var pd: i64 = pos + 1
292 while pd < len {
293 if src[pd] == 62 { pd = pd + (len + 1) }
294 else { pd = pd + 1 }
295 }
296 if pd > len { pd = pd - (len + 1) }
297 out.kind = NX_HTML_TOK_DOCTYPE
298 out.src_off = c.pos
299 c.pos = pd + 1
300 if c.pos > len { c.pos = len }
301 out.src_len = c.pos - out.src_off
302 return 0
303 }
304
305 // End tag: </name>
306 if src[pos] == 47 { // '/'
307 pos = pos + 1
308 let name_start: i64 = pos
309 pos = _scan_name(src, len, pos)
310 out.name_off = name_start
311 out.name_len = pos - name_start
312 // skip to '>'
313 while pos < len {
314 if src[pos] == 62 { pos = pos + (len + 1) }
315 else { pos = pos + 1 }
316 }
317 if pos > len { pos = pos - (len + 1) }
318 out.kind = NX_HTML_TOK_END_TAG
319 out.src_off = c.pos
320 c.pos = pos + 1
321 if c.pos > len { c.pos = len }
322 out.src_len = c.pos - out.src_off
323 return 0
324 }
325
326 // Start tag (possibly self-closing): <name ...>
327 let name_start: i64 = pos
328 pos = _scan_name(src, len, pos)
329 let name_end: i64 = pos
330 out.name_off = name_start
331 out.name_len = name_end - name_start
332 var self_closing: i64 = 0
333 // Skip attributes (we just consume them; tree-builder phase parses them).
334 // QUOTE-AWARE (HTML5 §13.2.5 attribute-value states): a '>' or '/' INSIDE a quoted
335 // attribute value is DATA, not markup. Without this, `data-x="a > b"` ended the tag at
336 // the inner '>' and the tag's remainder (`b">`, and any embedded markup) leaked as a
337 // TEXT token that then painted as visible content -- MEASURED on stackoverflow, whose
338 // data-attributes carry '>' and embedded HTML (nx_tok_attr_probe S1/S2/S3, 2026-07-27).
339 // `inq` = 0 outside a value, else the open-quote char (34 '"' or 39 '\''); only the
340 // matching quote closes it. An unterminated value consumes to end-of-input, matching how
341 // a real browser bounds the same malformed markup.
342 var inq: i64 = 0
343 while pos < len {
344 let bb: i64 = src[pos]
345 if inq != 0 {
346 if bb == inq { inq = 0 }
347 pos = pos + 1
348 } else {
349 if bb == 34 { inq = 34; pos = pos + 1 } // enter "..."
350 else { if bb == 39 { inq = 39; pos = pos + 1 } // enter '...'
351 else {
352 if bb == 62 { pos = pos + (len + 1) } // '>'
353 else {
354 if bb == 47 {
355 if pos + 1 < len {
356 if src[pos + 1] == 62 {
357 self_closing = 1
358 pos = pos + 1 // land ON the '>' (NOT past it) -- the shared
359 pos = pos + (len + 1) // `c.pos = pos + 1` below then lands AFTER '>',
360 // matching the plain-'>' branch. `pos + 2` overshot by one, eating the next
361 // char (e.g. the '<' of a following tag -> that tag leaked as TEXT). Bug hit
362 // every XHTML-style `<link/>`/`<br/>`/`<img/>` (Wikipedia ref dedup-style links).
363 } else { pos = pos + 1 }
364 } else { pos = pos + 1 }
365 } else { pos = pos + 1 }
366 }
367 } }
368 }
369 }
370 if pos > len { pos = pos - (len + 1) }
371 // HTML5 §13.1.2 void elements: regardless of `/>` syntax, these
372 // tags are always self-closing (no end-tag, no children in the
373 // parse tree). Promote START_TAG -> SELF_CLOSING when the name
374 // matches the void list. Without this, `<meta>` and `<link>`
375 // would push onto the parent stack, swallowing siblings as
376 // descendants (real-world example.com surfaced this 2026-05-20).
377 if self_closing == 0 {
378 if _is_void_element(src, out.name_off, out.name_len) == 1 {
379 self_closing = 1
380 }
381 }
382 if self_closing == 1 {
383 out.kind = NX_HTML_TOK_SELF_CLOSING
384 } else {
385 out.kind = NX_HTML_TOK_START_TAG
386 }
387 out.src_off = c.pos
388 c.pos = pos + 1
389 if c.pos > len { c.pos = len }
390 out.src_len = c.pos - out.src_off
391 return 0
392}
393
394// Compare a tag name (case-insensitive) against a caller-supplied
395// expected name. Convenience for tree-builder.
396func nx_html_name_eq(src: *u8, name_off: i64, name_len: i64,
397 expected: *u8, expected_len: i64) -> i64 {
398 if name_len != expected_len { return 0 }
399 var i: i64 = 0
400 while i < name_len {
401 if _lc(src[name_off + i]) != _lc(expected[i]) { return 0 }
402 i = i + 1
403 }
404 return 1
405}
406
407// Sealed-enum validity gate.
408func nx_html_tok_kind_is_valid(k: i64) -> i64 {
409 if k < 0 { return 0 }
410 if k >= NX_HTML_TOK_N { return 0 }
411 return 1
412}
413
414// -- HTML5 raw-text-mode primitives (Arc B1, NISHI_BROWSER_GOOGLE_IMAGE_SEARCH_ROADMAP)
415//
416// HTML5 §13.2.5.4 / §13.2.5.5 / §13.2.5.6 define four raw-text-class
417// elements where `<` inside content is NOT a tag-start: <script>,
418// <style>, <textarea>, <title>. Inside their bodies, the tokenizer
419// must scan for the matching </tagname> close-sequence
420// (case-insensitive) and treat everything between as a single TEXT
421// token. Real google.com responses contain large <script> blocks
422// that wreck the dumb-tokenizer parse tree without this.
423//
424// API is additive: existing callers (nx_dom_query, nx_layout_from_dom,
425// html_tokenizer_smoke, this file) continue working unchanged.
426// HTML5-aware callers detect a START_TAG for one of the four names
427// via nx_html_is_raw_text_tag and call nx_html_consume_raw_text to
428// advance the cursor past the body and receive a TEXT token for the
429// raw content.
430
431// Returns 1 if name is script/style/textarea/title (case-insensitive).
432func nx_html_is_raw_text_tag(src: *u8, name_off: i64, name_len: i64) -> i64 {
433 if name_len < 5 { return 0 }
434 if name_len > 8 { return 0 }
435 let c0: i64 = _lc(src[name_off])
436 if name_len == 5 {
437 // style, title
438 let c1: i64 = _lc(src[name_off + 1])
439 let c2: i64 = _lc(src[name_off + 2])
440 let c3: i64 = _lc(src[name_off + 3])
441 let c4: i64 = _lc(src[name_off + 4])
442 if c0 == 0x73 { if c1 == 0x74 { if c2 == 0x79 { if c3 == 0x6c { if c4 == 0x65 { return 1 } } } } } // style
443 if c0 == 0x74 { if c1 == 0x69 { if c2 == 0x74 { if c3 == 0x6c { if c4 == 0x65 { return 1 } } } } } // title
444 return 0
445 }
446 if name_len == 6 {
447 // script
448 let c1: i64 = _lc(src[name_off + 1])
449 let c2: i64 = _lc(src[name_off + 2])
450 let c3: i64 = _lc(src[name_off + 3])
451 let c4: i64 = _lc(src[name_off + 4])
452 let c5: i64 = _lc(src[name_off + 5])
453 if c0 == 0x73 { if c1 == 0x63 { if c2 == 0x72 { if c3 == 0x69 { if c4 == 0x70 { if c5 == 0x74 { return 1 } } } } } }
454 return 0
455 }
456 if name_len == 8 {
457 // textarea
458 let c1: i64 = _lc(src[name_off + 1])
459 let c2: i64 = _lc(src[name_off + 2])
460 let c3: i64 = _lc(src[name_off + 3])
461 let c4: i64 = _lc(src[name_off + 4])
462 let c5: i64 = _lc(src[name_off + 5])
463 let c6: i64 = _lc(src[name_off + 6])
464 let c7: i64 = _lc(src[name_off + 7])
465 if c0 == 0x74 {
466 if c1 == 0x65 { if c2 == 0x78 { if c3 == 0x74 { if c4 == 0x61 { if c5 == 0x72 { if c6 == 0x65 { if c7 == 0x61 { return 1 } } } } } } }
467 }
468 return 0
469 }
470 return 0
471}
472
473// After a START_TAG for a raw-text tag has been emitted, the caller
474// invokes this to consume the body up to (but not including) the
475// matching `</tagname>`. Emits a TEXT token spanning the raw bytes;
476// cursor is left positioned at the `<` of the close tag, so the
477// next nx_html_next_token call returns END_TAG normally. If no
478// matching close is found, consumes to EOF (substrate-honest: we
479// surface the truncation rather than fabricating a close).
480//
481// tag_name / tag_name_len describe the tag we are inside, used for
482// case-insensitive close-match. Caller supplies them so we don't
483// have to re-parse the just-emitted START_TAG name.
484func nx_html_consume_raw_text(c: *HtmlCursor, tag_name: *u8, tag_name_len: i64,
485 out: *HtmlToken) -> i64 {
486 let src: *u8 = c.src
487 let len: i64 = c.src_len
488 var pos: i64 = c.pos
489 let body_start: i64 = pos
490
491 out.kind = NX_HTML_TOK_TEXT
492 out.src_off = body_start
493 out.name_off = 0
494 out.name_len = 0
495 out.body_off = body_start
496
497 while pos < len {
498 // Look for `</` then case-insensitive tag_name then '>' or whitespace.
499 if src[pos] != 60 { pos = pos + 1 }
500 else {
501 if pos + 1 >= len { pos = pos + 1 }
502 else {
503 if src[pos + 1] != 47 { pos = pos + 1 } // '/'
504 else {
505 // Match tag name case-insensitively starting at pos+2.
506 var matched: i64 = 1
507 var i: i64 = 0
508 while i < tag_name_len {
509 if pos + 2 + i >= len { matched = 0; i = tag_name_len }
510 else {
511 if _lc(src[pos + 2 + i]) != _lc(tag_name[i]) { matched = 0; i = tag_name_len }
512 else { i = i + 1 }
513 }
514 }
515 if matched == 0 { pos = pos + 1 }
516 else {
517 // Next char must be '>' or whitespace (HTML5 spec).
518 let after: i64 = pos + 2 + tag_name_len
519 if after >= len { pos = pos + 1 }
520 else {
521 let nb: i64 = src[after]
522 if nb == 62 { pos = pos + (len + 1) } // '>'
523 else {
524 if _is_ws(nb) == 1 { pos = pos + (len + 1) }
525 else { pos = pos + 1 }
526 }
527 }
528 }
529 }
530 }
531 }
532 }
533 if pos > len { pos = pos - (len + 1) }
534 out.src_len = pos - body_start
535 out.body_len = pos - body_start
536 c.pos = pos
537 return 0
538}
539
540// -- HTML entity decoder (Arc B1 sibling primitive)
541//
542// Decodes character entity references in `src` into `dst` (cap-bounded).
543// Returns the number of bytes written to dst. Handles:
544// - the basic 5: & < > " '
545// - numeric: &#NN; (decimal) and &#xHH; / &#XHH; (hex)
546// - common named refs: © ® — – …
547// - unrecognised entities pass through unchanged (substrate-honest:
548// never silently swallow data)
549//
550// All output is UTF-8. Code points beyond 0x7F are encoded multi-byte
551// per RFC 3629; surrogate halves and code points beyond 0x10FFFF
552// emit the canonical replacement (U+FFFD 0xEF 0xBF 0xBD).
553func _entity_emit_u32(cp_in: i64, dst: *u8, dst_pos: i64, dst_cap: i64) -> i64 {
554 var p: i64 = dst_pos
555 var cp: i64 = cp_in
556 if cp < 0 { return p }
557 if cp > 0x10FFFF { cp = 0xFFFD }
558 if cp >= 0xD800 { if cp <= 0xDFFF { cp = 0xFFFD } }
559 if cp < 0x80 {
560 if p + 1 > dst_cap { return p }
561 dst[p] = cp as u8
562 return p + 1
563 }
564 if cp < 0x800 {
565 if p + 2 > dst_cap { return p }
566 dst[p] = (0xC0 + (cp >> 6)) as u8
567 dst[p + 1] = (0x80 + (cp & 0x3F)) as u8
568 return p + 2
569 }
570 if cp < 0x10000 {
571 if p + 3 > dst_cap { return p }
572 dst[p] = (0xE0 + (cp >> 12)) as u8
573 dst[p + 1] = (0x80 + ((cp >> 6) & 0x3F)) as u8
574 dst[p + 2] = (0x80 + (cp & 0x3F)) as u8
575 return p + 3
576 }
577 if p + 4 > dst_cap { return p }
578 dst[p] = (0xF0 + (cp >> 18)) as u8
579 dst[p + 1] = (0x80 + ((cp >> 12) & 0x3F)) as u8
580 dst[p + 2] = (0x80 + ((cp >> 6) & 0x3F)) as u8
581 dst[p + 3] = (0x80 + (cp & 0x3F)) as u8
582 return p + 4
583}
584
585// Returns 1 if src[sp..sp+nlen] case-insensitively equals lit[0..nlen].
586func _entity_name_eq(src: *u8, sp: i64, src_len: i64, lit: *u8, nlen: i64) -> i64 {
587 if sp + nlen > src_len { return 0 }
588 var i: i64 = 0
589 while i < nlen {
590 if _lc(src[sp + i]) != _lc(lit[i]) { return 0 }
591 i = i + 1
592 }
593 return 1
594}
595
596// Returns hex digit value (0..15) for ASCII byte, or -1 if not a hex digit.
597func _hex_val(b: i64) -> i64 {
598 if b >= 48 { if b <= 57 { return b - 48 } } // 0-9
599 if b >= 97 { if b <= 102 { return b - 87 } } // a-f
600 if b >= 65 { if b <= 70 { return b - 55 } } // A-F
601 return 0 - 1
602}
603
604// Returns decimal digit value (0..9) for ASCII byte, or -1 if not a digit.
605func _dec_val(b: i64) -> i64 {
606 if b >= 48 { if b <= 57 { return b - 48 } }
607 return 0 - 1
608}
609
610// Parse numeric entity body (bytes between '#' and ';'). Returns
611// code point on success, or -1 if malformed. body_off points to the
612// '#' byte itself.
613func _parse_numeric_entity(src: *u8, body_off: i64, body_len: i64) -> i64 {
614 if body_len < 2 { return 0 - 1 }
615 var p: i64 = body_off + 1
616 let end: i64 = body_off + body_len
617 var base: i64 = 10
618 let c0: i64 = src[p]
619 if c0 == 120 { base = 16; p = p + 1 } // 'x'
620 else { if c0 == 88 { base = 16; p = p + 1 } } // 'X'
621 if p >= end { return 0 - 1 }
622 var cp: i64 = 0
623 while p < end {
624 var dv: i64 = 0 - 1
625 if base == 16 { dv = _hex_val(src[p]) }
626 else { dv = _dec_val(src[p]) }
627 if dv < 0 { return 0 - 1 }
628 cp = cp * base + dv
629 if cp > 0x10FFFF { cp = 0x10FFFF + 1 } // clamp; emitter replaces
630 p = p + 1
631 }
632 return cp
633}
634
635// Look up a named entity body and return its code point, or -1 if unknown.
636// Names are case-sensitive per WHATWG (most are lowercase). Substrate
637// covers the common set; large named-entity table is deferred until
638// real Google response demands it.
639func _lookup_named_entity(src: *u8, body_off: i64, body_len: i64) -> i64 {
640 if body_len == 2 {
641 if _entity_name_eq(src, body_off, body_off + body_len, "lt" as *u8, 2) == 1 { return 0x3C }
642 if _entity_name_eq(src, body_off, body_off + body_len, "gt" as *u8, 2) == 1 { return 0x3E }
643 return 0 - 1
644 }
645 if body_len == 3 {
646 if _entity_name_eq(src, body_off, body_off + body_len, "amp" as *u8, 3) == 1 { return 0x26 }
647 if _entity_name_eq(src, body_off, body_off + body_len, "reg" as *u8, 3) == 1 { return 0xAE }
648 return 0 - 1
649 }
650 if body_len == 4 {
651 if _entity_name_eq(src, body_off, body_off + body_len, "quot" as *u8, 4) == 1 { return 0x22 }
652 if _entity_name_eq(src, body_off, body_off + body_len, "apos" as *u8, 4) == 1 { return 0x27 }
653 if _entity_name_eq(src, body_off, body_off + body_len, "nbsp" as *u8, 4) == 1 { return 0xA0 }
654 if _entity_name_eq(src, body_off, body_off + body_len, "copy" as *u8, 4) == 1 { return 0xA9 }
655 return 0 - 1
656 }
657 if body_len == 5 {
658 if _entity_name_eq(src, body_off, body_off + body_len, "mdash" as *u8, 5) == 1 { return 0x2014 }
659 if _entity_name_eq(src, body_off, body_off + body_len, "ndash" as *u8, 5) == 1 { return 0x2013 }
660 return 0 - 1
661 }
662 if body_len == 6 {
663 if _entity_name_eq(src, body_off, body_off + body_len, "hellip" as *u8, 6) == 1 { return 0x2026 }
664 return 0 - 1
665 }
666 return 0 - 1
667}
668
669func nx_html_entity_decode(src: *u8, src_len: i64, dst: *u8, dst_cap: i64) -> i64 {
670 var sp: i64 = 0
671 var dp: i64 = 0
672 while sp < src_len {
673 if src[sp] != 38 { // not '&'
674 if dp + 1 > dst_cap { return dp }
675 dst[dp] = src[sp]
676 dp = dp + 1
677 sp = sp + 1
678 } else {
679 // Find terminating ';' within 12 bytes (longest common name is 8).
680 var end: i64 = sp + 1
681 var limit: i64 = sp + 12
682 if limit > src_len { limit = src_len }
683 var found: i64 = 0
684 var scan: i64 = sp + 1
685 while scan < limit {
686 if src[scan] == 59 { end = scan; found = 1; scan = limit } // ';' -- exit
687 else { scan = scan + 1 }
688 }
689 if found == 0 {
690 // No terminator -- pass '&' through verbatim.
691 if dp + 1 > dst_cap { return dp }
692 dst[dp] = 38 as u8
693 dp = dp + 1
694 sp = sp + 1
695 } else {
696 // src[sp+1..end] is the entity body. end points at ';'.
697 let body_off: i64 = sp + 1
698 let body_len: i64 = end - body_off
699 var cp: i64 = 0 - 1
700 if body_len > 0 {
701 if src[body_off] == 35 { // '#' numeric
702 cp = _parse_numeric_entity(src, body_off, body_len)
703 } else {
704 cp = _lookup_named_entity(src, body_off, body_len)
705 }
706 }
707 if cp >= 0 {
708 dp = _entity_emit_u32(cp, dst, dp, dst_cap)
709 sp = end + 1
710 } else {
711 // Unrecognised -- pass &name; through verbatim per
712 // substrate-honest: never silently swallow data.
713 var k: i64 = sp
714 while k <= end {
715 if dp + 1 > dst_cap { return dp }
716 dst[dp] = src[k]
717 dp = dp + 1
718 k = k + 1
719 }
720 sp = end + 1
721 }
722 }
723 }
724 }
725 return dp
726}