nx_html_to_text.nx source
↩ module page · 476 lines · 19624 B
1// nx_html_to_text.nx -- HTML → plain text rendering for the
2// sovereign nishi browser's text mode.
3//
4// Strips tags, keeps text content, collapses whitespace, decodes 110
5// named entities (data-driven nx_html_entities: quotes/dashes/symbols/
6// accented-latin/arrows -- — ’ © é ...) AND
7// numeric character references (' decimal / ' hex, UTF-8
8// encoded), inserts newlines around block-level elements (p, h1-h6,
9// div, li, br, hr, pre, blockquote, table, tr), suppresses content
10// inside <script> and <style>. nbsp (named or  ) renders as a
11// space in text mode. (X-HTML-ENT-001 + named-table follow-on.)
12//
13// CAPABILITY_COMPLETENESS: PARTIAL. MISSING_CAPABILITIES:
14// - the full HTML5 entity table (2000+ refs; 110 common named
15// + all numeric refs covered) and entities without ';' terminator
16// - CSS-driven inline-vs-block (assumes tag-name defaults)
17// - bidi/RTL text shaping
18//
19// Per [[feedback-no-skip-paths-as-error-codes]] these are named
20// follow-on substrate primitives, not silent skips.
21//
22// Built per F7 post-order DFS.
23// expect_exit: 0
24// license_tier: ORIGINAL
25
26import "nx_syscalls.nx"
27import "nx_html_entities.nx"
28
29// ===== Verdicts ===================================================
30
31const NX_H2T_OK: i64 = 0
32const NX_H2T_ERR_OVERFLOW: i64 = 1
33
34// ===== LEAF helpers ===============================================
35
36func h2t_lc(b: i64) -> i64 {
37 if b >= 0x41 { if b <= 0x5a { return b + 0x20 } }
38 return b
39}
40
41func h2t_is_ws(b: i64) -> i64 {
42 if b == 0x20 { return 1 } // space
43 if b == 0x09 { return 1 } // tab
44 if b == 0x0a { return 1 } // LF
45 if b == 0x0d { return 1 } // CR
46 if b == 0x0c { return 1 } // FF
47 return 0
48}
49
50func h2t_is_name(b: i64) -> i64 {
51 if b >= 0x61 { if b <= 0x7a { return 1 } } // a-z
52 if b >= 0x41 { if b <= 0x5a { return 1 } } // A-Z
53 if b >= 0x30 { if b <= 0x39 { return 1 } } // 0-9
54 return 0
55}
56
57func h2t_emit_byte(out: *u8, out_pos: *i64, cap: i64, b: i64) -> i64 {
58 if out_pos[0] >= cap { return NX_H2T_ERR_OVERFLOW }
59 out[out_pos[0]] = b & 0xff
60 out_pos[0] = out_pos[0] + 1
61 return NX_H2T_OK
62}
63
64// Lowercased ASCII bytes-equal between src[off..off+len] and a literal.
65func h2t_name_eq_lit(src: *u8, off: i64, len: i64, lit: *u8, lit_len: i64) -> i64 {
66 if len != lit_len { return 0 }
67 var i: i64 = 0
68 while i < len {
69 if h2t_lc(src[off + i] & 0xff) != (lit[i] & 0xff) { return 0 }
70 i = i + 1
71 }
72 return 1
73}
74
75// ===== Tag classification (LEAF) ==================================
76
77// Returns 1 if this tag-name should emit a newline before AND after
78// its open/close (block-level). Inline tags (b, i, span, a, em,
79// strong) return 0.
80func h2t_is_block_tag(src: *u8, off: i64, len: i64) -> i64 {
81 // S-CLASS OOM FIX: compare against STRING LITERALS directly -- ZERO per-call mmap. The old version mmap'd ~8
82 // constant tag-name buffers on EVERY call (= every tag in the doc) and never freed them -> ~10 page-leaks per
83 // tag -> OOM-137 on a large multi-doc fold. Behaviour byte-identical; the leak is gone.
84 if h2t_name_eq_lit(src, off, len, "p" as *u8, 1) == 1 { return 1 }
85 if h2t_name_eq_lit(src, off, len, "div" as *u8, 3) == 1 { return 1 }
86 if h2t_name_eq_lit(src, off, len, "li" as *u8, 2) == 1 { return 1 }
87 if h2t_name_eq_lit(src, off, len, "br" as *u8, 2) == 1 { return 1 }
88 if h2t_name_eq_lit(src, off, len, "hr" as *u8, 2) == 1 { return 1 }
89 if h2t_name_eq_lit(src, off, len, "pre" as *u8, 3) == 1 { return 1 }
90 if h2t_name_eq_lit(src, off, len, "tr" as *u8, 2) == 1 { return 1 }
91 if h2t_name_eq_lit(src, off, len, "blockquote" as *u8, 10) == 1 { return 1 }
92 // h1..h6
93 if len == 2 {
94 if h2t_lc(src[off] & 0xff) == 0x68 {
95 let d: i64 = src[off + 1] & 0xff
96 if d >= 0x31 { if d <= 0x36 { return 1 } }
97 }
98 }
99 return 0
100}
101
102// Returns 1 if this is a tag whose contents should be SUPPRESSED
103// (script, style). Browser text mode never shows these bytes.
104func h2t_is_suppress_tag(src: *u8, off: i64, len: i64) -> i64 {
105 // S-CLASS OOM FIX: string literals, zero per-call mmap (was 2 leaked pages per tag).
106 if h2t_name_eq_lit(src, off, len, "script" as *u8, 6) == 1 { return 1 }
107 if h2t_name_eq_lit(src, off, len, "style" as *u8, 5) == 1 { return 1 }
108 return 0
109}
110
111// ===== Entity decode (LEAF) =======================================
112
113// Decode a NUMERIC character reference &#DDD; (decimal) or &#xHH;
114// (hex, x or X) at src[off]. off points at '&', off+1 at '#'.
115// Writes the parsed Unicode codepoint to out_cp. Returns the
116// consumed length (incl. & and ;), or 0 if it is not a well-formed
117// numeric reference (no digits, bad digit, missing ';', or out of the
118// Unicode scalar range). No silent skip -- a malformed ref decodes to
119// 0 so the caller emits '&' literally (per no-skip-paths law).
120func h2t_decode_numeric(src: *u8, src_len: i64, off: i64, out_cp: *i64) -> i64 {
121 var i: i64 = off + 2 // past "&#"
122 if i >= src_len { return 0 }
123 var base: i64 = 10
124 let c0: i64 = src[i] & 0xff
125 if c0 == 0x78 { base = 16; i = i + 1 } // 'x'
126 if c0 == 0x58 { base = 16; i = i + 1 } // 'X'
127 var val: i64 = 0
128 var ndigits: i64 = 0
129 while i < src_len {
130 let c: i64 = src[i] & 0xff
131 if c == 0x3b { // ';' terminator
132 if ndigits == 0 { return 0 }
133 out_cp[0] = val
134 return (i + 1) - off
135 }
136 var d: i64 = 16 // sentinel: >= any base
137 if c >= 0x30 { if c <= 0x39 { d = c - 0x30 } } // 0-9
138 if base == 16 {
139 if c >= 0x61 { if c <= 0x66 { d = (c - 0x61) + 10 } } // a-f
140 if c >= 0x41 { if c <= 0x46 { d = (c - 0x41) + 10 } } // A-F
141 }
142 if d >= base { return 0 } // non-digit before ';'
143 val = (val * base) + d
144 if val > 0x10ffff { return 0 } // outside Unicode range
145 ndigits = ndigits + 1
146 i = i + 1
147 }
148 return 0 // unterminated
149}
150
151// Decode an HTML character reference at src[off]: numeric (&#DDD; /
152// &#xHH;) or a common named entity (& < > " '
153// ). Writes the decoded Unicode codepoint to out_cp. Returns
154// the consumed length (incl. & and ;), or 0 if nothing matched (caller
155// emits '&' as-is and advances 1).
156func h2t_decode_entity(src: *u8, src_len: i64, off: i64, out_cp: *i64) -> i64 {
157 if (src[off] & 0xff) != 0x26 { return 0 } // not '&'
158 if off + 1 >= src_len { return 0 }
159 if (src[off + 1] & 0xff) == 0x23 { // '#' -> numeric ref
160 return h2t_decode_numeric(src, src_len, off, out_cp)
161 }
162 // Named: scan the [A-Za-z0-9] name run, require a ';' terminator, then
163 // look it up in the data-driven table (nx_html_entities, 110 common
164 // entities). Replaces the old 6 hard-coded entities with one organ.
165 var ne: i64 = off + 1
166 var go: i64 = 1
167 while go == 1 {
168 if ne >= src_len { go = 0 }
169 else {
170 if h2t_is_name(src[ne] & 0xff) == 1 { ne = ne + 1 } else { go = 0 }
171 }
172 }
173 let nlen: i64 = ne - (off + 1)
174 if nlen <= 0 { return 0 }
175 if ne >= src_len { return 0 }
176 if (src[ne] & 0xff) != 0x3b { return 0 } // require ';'
177 let name_ptr: *u8 = ((src as i64) + off + 1) as *u8
178 if nx_html_entity_lookup(name_ptr, nlen, out_cp) == 1 {
179 return nlen + 2 // '&' + name + ';'
180 }
181 return 0
182}
183
184// ===== Mid: scan one tag, fill name slot ==========================
185
186// Scan a name (a-z A-Z 0-9) from p, return offset past last name byte.
187func h2t_scan_name(src: *u8, src_len: i64, p: i64) -> i64 {
188 var i: i64 = p
189 while i < src_len {
190 if h2t_is_name(src[i] & 0xff) != 1 { return i }
191 i = i + 1
192 }
193 return i
194}
195
196// Skip from p to the byte after the next '>'. If self-closing
197// (preceding byte is '/'), sets *is_self_close to 1.
198func h2t_skip_to_gt(src: *u8, src_len: i64, p: i64, is_self_close: *i64) -> i64 {
199 var i: i64 = p
200 while i < src_len {
201 let c: i64 = src[i] & 0xff
202 if c == 0x3e {
203 if i > 0 { if (src[i - 1] & 0xff) == 0x2f { is_self_close[0] = 1 } }
204 return i + 1
205 }
206 i = i + 1
207 }
208 return i
209}
210
211// At src[off] == '<'. Returns offset just past the matching '>'.
212// Sets *name_off + *name_len. Comments / doctypes / processing
213// instructions are returned with name_len=0.
214func h2t_scan_tag(
215 src: *u8, src_len: i64, off: i64,
216 name_off: *i64, name_len: *i64,
217 is_close: *i64, is_self_close: *i64
218) -> i64 {
219 var p: i64 = off + 1 // past <
220 is_close[0] = 0
221 is_self_close[0] = 0
222 name_off[0] = p
223 name_len[0] = 0
224 if p >= src_len { return p }
225 if (src[p] & 0xff) == 0x2f { is_close[0] = 1; p = p + 1; name_off[0] = p }
226 if p < src_len {
227 if (src[p] & 0xff) == 0x21 {
228 // <! ... > comment or doctype: skip to '>'
229 return h2t_skip_to_gt(src, src_len, p, is_self_close)
230 }
231 }
232 let name_end: i64 = h2t_scan_name(src, src_len, p)
233 name_len[0] = name_end - p
234 return h2t_skip_to_gt(src, src_len, name_end, is_self_close)
235}
236
237// ===== High: stream render ========================================
238
239// State of the rendering walk:
240// *out_pos -- current write position into out
241// *last_was_ws -- 1 if last emitted byte was ASCII whitespace
242// (used to collapse runs)
243// *suppress_depth -- nonzero when inside script/style (skip all text)
244// *in_pre -- 1 if inside <pre>; preserves whitespace verbatim
245struct H2tState {
246 out_pos: i64,
247 last_was_ws: i64,
248 suppress_depth: i64,
249 in_pre: i64,
250 in_link: i64
251}
252
253func h2t_state_new() -> *H2tState {
254 let s: *H2tState = sys_mmap(64) as *H2tState
255 s.out_pos = 0
256 s.last_was_ws = 1
257 s.suppress_depth = 0
258 s.in_pre = 0
259 s.in_link = 0
260 return s
261}
262
263// Emit a byte verbatim (no whitespace collapse, no state change) -- for inline
264// link markers (STX 0x02 <url> SOH 0x01 <anchor text> ETX 0x03) that the GUI
265// parses into clickable spans. Marker bytes never occur in real text content.
266func h2t_emit_raw(out: *u8, cap: i64, s: *H2tState, b: i64) -> i64 {
267 if s.out_pos >= cap { return NX_H2T_ERR_OVERFLOW }
268 out[s.out_pos] = b & 0xff
269 s.out_pos = s.out_pos + 1
270 return NX_H2T_OK
271}
272
273// Find the href="..." (or '...' or unquoted) attribute value within an <a ...>
274// tag's attribute region [start,end). Requires a word boundary before "href"
275// so data-href etc. don't false-match. Returns 1 + sets out_off/out_len.
276func h2t_find_href(src: *u8, start: i64, end: i64, out_off: *i64, out_len: *i64) -> i64 {
277 var i: i64 = start
278 var st: i64 = 0
279 var hp: i64 = 0 - 1
280 while st == 0 {
281 if (i + 4) > end { st = 2 }
282 else {
283 var m: i64 = 0
284 if h2t_lc(src[i]&0xff)==0x68 { if h2t_lc(src[i+1]&0xff)==0x72 { if h2t_lc(src[i+2]&0xff)==0x65 { if h2t_lc(src[i+3]&0xff)==0x66 { m = 1 } } } }
285 if m == 1 { if i > start { if h2t_is_name(src[i-1]&0xff)==1 { m = 0 } } } // word boundary
286 if m == 1 { hp = i; st = 1 } else { i = i + 1 }
287 }
288 }
289 if st != 1 { return 0 }
290 var j: i64 = hp + 4
291 var sk: i64 = 1
292 while sk == 1 { if j < end { if (src[j]&0xff)==0x20 { j = j + 1 } else { sk = 0 } } else { sk = 0 } }
293 if j >= end { return 0 }
294 if (src[j]&0xff) != 0x3d { return 0 }
295 j = j + 1
296 sk = 1
297 while sk == 1 { if j < end { if (src[j]&0xff)==0x20 { j = j + 1 } else { sk = 0 } } else { sk = 0 } }
298 if j >= end { return 0 }
299 let q: i64 = src[j]&0xff
300 if q == 0x22 {
301 j = j + 1; out_off[0] = j
302 var k: i64 = j; var f: i64 = 0
303 while f == 0 { if k >= end { f = 1 } else { if (src[k]&0xff)==0x22 { f = 1 } else { k = k + 1 } } }
304 out_len[0] = k - j; return 1
305 }
306 if q == 0x27 {
307 j = j + 1; out_off[0] = j
308 var k2: i64 = j; var f2: i64 = 0
309 while f2 == 0 { if k2 >= end { f2 = 1 } else { if (src[k2]&0xff)==0x27 { f2 = 1 } else { k2 = k2 + 1 } } }
310 out_len[0] = k2 - j; return 1
311 }
312 out_off[0] = j
313 var k3: i64 = j; var f3: i64 = 0
314 while f3 == 0 { if k3 >= end { f3 = 1 } else { let cc: i64 = src[k3]&0xff; if cc==0x20 { f3 = 1 } else { if cc==0x3e { f3 = 1 } else { k3 = k3 + 1 } } } }
315 out_len[0] = k3 - j; return 1
316}
317
318// Emit one text byte with whitespace collapsing.
319func h2t_emit_text_byte(out: *u8, cap: i64, s: *H2tState, b: i64) -> i64 {
320 if s.in_pre == 1 {
321 if s.out_pos >= cap { return NX_H2T_ERR_OVERFLOW }
322 out[s.out_pos] = b & 0xff
323 s.out_pos = s.out_pos + 1
324 s.last_was_ws = h2t_is_ws(b & 0xff)
325 return NX_H2T_OK
326 }
327 if h2t_is_ws(b) == 1 {
328 if s.last_was_ws == 1 { return NX_H2T_OK }
329 if s.out_pos >= cap { return NX_H2T_ERR_OVERFLOW }
330 out[s.out_pos] = 0x20
331 s.out_pos = s.out_pos + 1
332 s.last_was_ws = 1
333 return NX_H2T_OK
334 }
335 if s.out_pos >= cap { return NX_H2T_ERR_OVERFLOW }
336 out[s.out_pos] = b & 0xff
337 s.out_pos = s.out_pos + 1
338 s.last_was_ws = 0
339 return NX_H2T_OK
340}
341
342// Force a newline (collapsed if last was already newline-like).
343func h2t_emit_newline(out: *u8, cap: i64, s: *H2tState) -> i64 {
344 if s.out_pos == 0 { return NX_H2T_OK }
345 if s.out_pos > 0 { if out[s.out_pos - 1] == 0x0a { return NX_H2T_OK } }
346 if s.out_pos >= cap { return NX_H2T_ERR_OVERFLOW }
347 out[s.out_pos] = 0x0a
348 s.out_pos = s.out_pos + 1
349 s.last_was_ws = 1
350 return NX_H2T_OK
351}
352
353// Emit a Unicode codepoint as UTF-8 (1-4 bytes) through the text-byte
354// path, so whitespace-collapse + overflow checks still apply. ASCII
355// (cp < 0x80) is a single byte -- byte-identical to the pre-numeric
356// behaviour, so named entities (all < 0x80) are unchanged. Codepoints
357// >= 0x80 (from numeric refs like ’ -> U+2019 ') are encoded
358// faithfully rather than dropped (build intelligence, never strip).
359func h2t_emit_codepoint(out: *u8, cap: i64, s: *H2tState, cp: i64) -> i64 {
360 if cp < 0x80 {
361 return h2t_emit_text_byte(out, cap, s, cp)
362 }
363 if cp < 0x800 {
364 let r1: i64 = h2t_emit_text_byte(out, cap, s, 0xc0 | (cp >> 6))
365 if r1 != NX_H2T_OK { return r1 }
366 return h2t_emit_text_byte(out, cap, s, 0x80 | (cp & 0x3f))
367 }
368 if cp < 0x10000 {
369 let r1: i64 = h2t_emit_text_byte(out, cap, s, 0xe0 | (cp >> 12))
370 if r1 != NX_H2T_OK { return r1 }
371 let r2: i64 = h2t_emit_text_byte(out, cap, s, 0x80 | ((cp >> 6) & 0x3f))
372 if r2 != NX_H2T_OK { return r2 }
373 return h2t_emit_text_byte(out, cap, s, 0x80 | (cp & 0x3f))
374 }
375 let q1: i64 = h2t_emit_text_byte(out, cap, s, 0xf0 | (cp >> 18))
376 if q1 != NX_H2T_OK { return q1 }
377 let q2: i64 = h2t_emit_text_byte(out, cap, s, 0x80 | ((cp >> 12) & 0x3f))
378 if q2 != NX_H2T_OK { return q2 }
379 let q3: i64 = h2t_emit_text_byte(out, cap, s, 0x80 | ((cp >> 6) & 0x3f))
380 if q3 != NX_H2T_OK { return q3 }
381 return h2t_emit_text_byte(out, cap, s, 0x80 | (cp & 0x3f))
382}
383
384// Walk src and render to out. Returns bytes written, or
385// 0-NX_H2T_ERR_OVERFLOW on overflow.
386func nx_html_to_text_x(src: *u8, src_len: i64, out: *u8, out_cap: i64, emit_links: i64) -> i64 {
387 let s: *H2tState = h2t_state_new()
388 let no: *i64 = sys_mmap(8) as *i64
389 let nl: *i64 = sys_mmap(8) as *i64
390 let isc: *i64 = sys_mmap(8) as *i64
391 let sc: *i64 = sys_mmap(8) as *i64
392 let eb: *i64 = sys_mmap(8) as *i64
393 let ho: *i64 = sys_mmap(8) as *i64
394 let hl: *i64 = sys_mmap(8) as *i64
395 let a_lit: *u8 = sys_mmap(8); a_lit[0]=0x61
396 let pre: *u8 = sys_mmap(8); pre[0]=0x70; pre[1]=0x72; pre[2]=0x65
397 var p: i64 = 0
398 while p < src_len {
399 let c: i64 = src[p] & 0xff
400 if c == 0x3c { // '<' tag start
401 let np: i64 = h2t_scan_tag(src, src_len, p, no, nl, isc, sc)
402 if nl[0] > 0 {
403 let is_block: i64 = h2t_is_block_tag(src, no[0], nl[0])
404 let is_supp: i64 = h2t_is_suppress_tag(src, no[0], nl[0])
405 if is_block == 1 { h2t_emit_newline(out, out_cap, s) }
406 if is_supp == 1 {
407 if isc[0] == 1 {
408 if s.suppress_depth > 0 { s.suppress_depth = s.suppress_depth - 1 }
409 } else {
410 if sc[0] == 0 { s.suppress_depth = s.suppress_depth + 1 }
411 }
412 }
413 if h2t_name_eq_lit(src, no[0], nl[0], pre, 3) == 1 {
414 if isc[0] == 1 { s.in_pre = 0 } else { if sc[0] == 0 { s.in_pre = 1 } }
415 }
416 if emit_links == 1 {
417 if h2t_name_eq_lit(src, no[0], nl[0], a_lit, 1) == 1 {
418 if isc[0] == 1 {
419 if s.in_link == 1 { h2t_emit_raw(out, out_cap, s, 0x03); s.in_link = 0 }
420 } else {
421 if sc[0] == 0 {
422 if s.in_link == 0 {
423 let ae: i64 = no[0] + nl[0]
424 if h2t_find_href(src, ae, np - 1, ho, hl) == 1 {
425 if hl[0] > 0 {
426 h2t_emit_raw(out, out_cap, s, 0x02)
427 var hi: i64 = 0
428 while hi < hl[0] { h2t_emit_raw(out, out_cap, s, src[ho[0]+hi]&0xff); hi = hi + 1 }
429 h2t_emit_raw(out, out_cap, s, 0x01)
430 s.in_link = 1
431 }
432 }
433 }
434 }
435 }
436 }
437 }
438 }
439 p = np
440 } else {
441 if s.suppress_depth == 0 {
442 if c == 0x26 { // '&' entity
443 let consumed: i64 = h2t_decode_entity(src, src_len, p, eb)
444 if consumed > 0 {
445 var cp: i64 = eb[0]
446 if cp == 0xa0 { cp = 0x20 } // nbsp -> space (text mode)
447 h2t_emit_codepoint(out, out_cap, s, cp)
448 p = p + consumed
449 } else {
450 h2t_emit_text_byte(out, out_cap, s, c)
451 p = p + 1
452 }
453 } else {
454 h2t_emit_text_byte(out, out_cap, s, c)
455 p = p + 1
456 }
457 } else {
458 p = p + 1
459 }
460 }
461 }
462 // S-class: free this call's per-DOC scratch (the catastrophic per-TAG leak is already killed above by the
463 // string-literal fix). Together = nx_html_to_text now leaks ZERO -> a full multi-doc fold no longer OOMs.
464 let result: i64 = s.out_pos
465 sys_munmap(s as *u8, 64)
466 sys_munmap(no as *u8, 8); sys_munmap(nl as *u8, 8); sys_munmap(isc as *u8, 8); sys_munmap(sc as *u8, 8)
467 sys_munmap(eb as *u8, 8); sys_munmap(ho as *u8, 8); sys_munmap(hl as *u8, 8)
468 sys_munmap(a_lit, 8); sys_munmap(pre, 8)
469 return result
470}
471
472// Stable API (unchanged contract): plain HTML->text, no inline link markers.
473func nx_html_to_text(src: *u8, src_len: i64, out: *u8, out_cap: i64) -> i64 {
474 return nx_html_to_text_x(src, src_len, out, out_cap, 0)
475}
476