nx_html_extract.nx source
↩ module page · 601 lines · 21962 B
1// nx_html_extract.nx -- minimal HTML body-text scraper (state-machine V2).
2//
3// Takes raw HTML bytes (a literotica chapter page, an nhentai gallery
4// info page, a fanfiction.net story, a generic blog post) and emits:
5// - body text (with block tags reduced to newline boundaries)
6// - the first <title>...</title> content
7//
8// PRIORITY: faithful source preservation. Per user spec ("as close
9// to possible to the source") we do NOT normalize whitespace, do NOT
10// collapse newlines into spaces, do NOT re-flow paragraphs. We only:
11// - strip <script>...</script> blocks (executable)
12// - strip <style>...</style> blocks (rendering)
13// - strip <!-- HTML comments -->
14// - strip every other tag's MARKUP, keeping its TEXT CONTENT
15// - decode the common HTML entities
16//
17// Block-level tags (<p>, <br>, <div>, <li>, <h1>..<h6>) cause a
18// newline to be emitted at the open and close, so paragraph
19// structure survives.
20//
21// Per cardinal feedback-build-intelligence-never-strip-features:
22// when we encounter an unfamiliar tag, we strip the tag but KEEP
23// the text content. Silent content stripping is forbidden.
24//
25// nx_safety_envelope:
26// intended_use: "Extract <title> and readable body text
27// from arbitrary HTML, preserving source
28// text bytes verbatim outside of tag/entity
29// markup."
30// sil_target: SIL2
31// asil_target: QM
32// dal_target: DAL C
33// iec_62304_class: NONE
34// evidence: [no_floating_point,
35// bounded_loops_per_jpl_rule_2,
36// adversarial_html_strips_script_style,
37// entity_decode_table_static,
38// explicit_state_machine_no_ad_hoc_breaks,
39// no_re_flow_or_normalization_of_body_text]
40// hazard_register: [bug-tape-malformed-unclosed-script-tag,
41// bug-tape-unknown-entity-passes-through]
42// residual_risk: "Adversarial HTML can construct unclosed
43// <script> spans that swallow the rest of
44// document; bounded EOF guarantees we stop
45// at end of input. Unknown &entities pass
46// through as literal text."
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50
51// ===== HtmlExtract result record ==================================
52
53struct HtmlExtract {
54 title_buf: *u8,
55 title_len: i64,
56 body_buf: *u8,
57 body_len: i64,
58}
59
60const NX_HTML_EXTRACT_BYTES: i64 = 32
61
62// ===== byte helpers ===============================================
63
64func nx_he_load_u8(p: *u8, i: i64) -> i64 {
65 let q: *u8 = ((p as i64) + i) as *u8
66 return *q
67}
68
69func nx_he_store_u8(p: *u8, i: i64, v: i64) {
70 let q: *u8 = ((p as i64) + i) as *u8
71 *q = v as u8
72}
73
74func nx_he_to_lower(c: i64) -> i64 {
75 if c >= 0x41 {
76 if c <= 0x5A { return c | 0x20 }
77 }
78 return c
79}
80
81// ===== case-insensitive literal compare ===========================
82
83func nx_he_match_ci(hay: *u8, off: i64, hay_n: i64,
84 lit: *u8, lit_n: i64) -> i64 {
85 if off + lit_n > hay_n { return 0 }
86 var i: i64 = 0
87 while i < lit_n {
88 let a: i64 = nx_he_to_lower(nx_he_load_u8(hay, off + i))
89 let b: i64 = nx_he_to_lower(nx_he_load_u8(lit, i))
90 if a != b { return 0 }
91 i = i + 1
92 }
93 return 1
94}
95
96// ===== tag-name classifier (block-level open/close emits newline) =
97
98func nx_he_is_block_tag_name(src: *u8, off: i64, end: i64) -> i64 {
99 let n: i64 = end - off
100 if n <= 0 { return 0 }
101
102 let t_p: *u8 = "p" as *u8
103 let t_br: *u8 = "br" as *u8
104 let t_div: *u8 = "div" as *u8
105 let t_h1: *u8 = "h1" as *u8
106 let t_h2: *u8 = "h2" as *u8
107 let t_h3: *u8 = "h3" as *u8
108 let t_h4: *u8 = "h4" as *u8
109 let t_h5: *u8 = "h5" as *u8
110 let t_h6: *u8 = "h6" as *u8
111 let t_li: *u8 = "li" as *u8
112 let t_tr: *u8 = "tr" as *u8
113 let t_td: *u8 = "td" as *u8
114 let t_hr: *u8 = "hr" as *u8
115 let t_pre: *u8 = "pre" as *u8
116 let t_blockquote: *u8 = "blockquote" as *u8
117 let t_article: *u8 = "article" as *u8
118 let t_section: *u8 = "section" as *u8
119
120 if n == 1 {
121 if nx_he_match_ci(src, off, end, t_p, 1) == 1 { return 1 }
122 }
123 if n == 2 {
124 if nx_he_match_ci(src, off, end, t_br, 2) == 1 { return 1 }
125 if nx_he_match_ci(src, off, end, t_h1, 2) == 1 { return 1 }
126 if nx_he_match_ci(src, off, end, t_h2, 2) == 1 { return 1 }
127 if nx_he_match_ci(src, off, end, t_h3, 2) == 1 { return 1 }
128 if nx_he_match_ci(src, off, end, t_h4, 2) == 1 { return 1 }
129 if nx_he_match_ci(src, off, end, t_h5, 2) == 1 { return 1 }
130 if nx_he_match_ci(src, off, end, t_h6, 2) == 1 { return 1 }
131 if nx_he_match_ci(src, off, end, t_li, 2) == 1 { return 1 }
132 if nx_he_match_ci(src, off, end, t_tr, 2) == 1 { return 1 }
133 if nx_he_match_ci(src, off, end, t_td, 2) == 1 { return 1 }
134 if nx_he_match_ci(src, off, end, t_hr, 2) == 1 { return 1 }
135 }
136 if n == 3 {
137 if nx_he_match_ci(src, off, end, t_div, 3) == 1 { return 1 }
138 if nx_he_match_ci(src, off, end, t_pre, 3) == 1 { return 1 }
139 }
140 if n == 7 {
141 if nx_he_match_ci(src, off, end, t_article, 7) == 1 { return 1 }
142 if nx_he_match_ci(src, off, end, t_section, 7) == 1 { return 1 }
143 }
144 if n == 10 {
145 if nx_he_match_ci(src, off, end, t_blockquote, 10) == 1 { return 1 }
146 }
147 return 0
148}
149
150// ===== entity decoder =============================================
151//
152// Decode a single entity starting at '&'. Writes decoded bytes
153// to dst at *dst_pos and returns the position past ';'. Pass-
154// through on unknown.
155
156func nx_he_decode_entity(src: *u8, off: i64, n: i64,
157 dst: *u8, dst_pos: *i64, dst_cap: i64) -> i64 {
158 if off + 2 >= n {
159 if *dst_pos < dst_cap {
160 nx_he_store_u8(dst, *dst_pos, nx_he_load_u8(src, off))
161 *dst_pos = *dst_pos + 1
162 }
163 return off + 1
164 }
165 var end: i64 = off + 1
166 let SCAN: i64 = 12
167 var i: i64 = 0
168 var found: i64 = 0
169 while i < SCAN {
170 if end >= n { i = SCAN }
171 if i < SCAN {
172 if nx_he_load_u8(src, end) == 0x3B { found = 1; i = SCAN }
173 if found == 0 { end = end + 1 }
174 }
175 i = i + 1
176 }
177 if found == 0 {
178 if *dst_pos < dst_cap {
179 nx_he_store_u8(dst, *dst_pos, nx_he_load_u8(src, off))
180 *dst_pos = *dst_pos + 1
181 }
182 return off + 1
183 }
184
185 let body_off: i64 = off + 1
186 let body_len: i64 = end - body_off
187
188 // Named entities (the common ones).
189 let e_amp: *u8 = "amp" as *u8
190 let e_lt: *u8 = "lt" as *u8
191 let e_gt: *u8 = "gt" as *u8
192 let e_quot: *u8 = "quot" as *u8
193 let e_apos: *u8 = "apos" as *u8
194 let e_nbsp: *u8 = "nbsp" as *u8
195
196 if body_len == 3 {
197 if nx_he_match_ci(src, body_off, n, e_amp, 3) == 1 {
198 if *dst_pos < dst_cap {
199 nx_he_store_u8(dst, *dst_pos, 0x26); *dst_pos = *dst_pos + 1
200 }
201 return end + 1
202 }
203 }
204 if body_len == 2 {
205 if nx_he_match_ci(src, body_off, n, e_lt, 2) == 1 {
206 if *dst_pos < dst_cap {
207 nx_he_store_u8(dst, *dst_pos, 0x3C); *dst_pos = *dst_pos + 1
208 }
209 return end + 1
210 }
211 if nx_he_match_ci(src, body_off, n, e_gt, 2) == 1 {
212 if *dst_pos < dst_cap {
213 nx_he_store_u8(dst, *dst_pos, 0x3E); *dst_pos = *dst_pos + 1
214 }
215 return end + 1
216 }
217 }
218 if body_len == 4 {
219 if nx_he_match_ci(src, body_off, n, e_quot, 4) == 1 {
220 if *dst_pos < dst_cap {
221 nx_he_store_u8(dst, *dst_pos, 0x22); *dst_pos = *dst_pos + 1
222 }
223 return end + 1
224 }
225 if nx_he_match_ci(src, body_off, n, e_apos, 4) == 1 {
226 if *dst_pos < dst_cap {
227 nx_he_store_u8(dst, *dst_pos, 0x27); *dst_pos = *dst_pos + 1
228 }
229 return end + 1
230 }
231 if nx_he_match_ci(src, body_off, n, e_nbsp, 4) == 1 {
232 if *dst_pos < dst_cap {
233 nx_he_store_u8(dst, *dst_pos, 0x20); *dst_pos = *dst_pos + 1
234 }
235 return end + 1
236 }
237 }
238
239 // Numeric: &#NNN; or &#xNN;
240 if nx_he_load_u8(src, body_off) == 0x23 {
241 if body_len >= 2 {
242 var hex: i64 = 0
243 var num_off: i64 = body_off + 1
244 var num_len: i64 = body_len - 1
245 let lead: i64 = nx_he_load_u8(src, body_off + 1)
246 if lead == 0x78 { hex = 1; num_off = num_off + 1; num_len = num_len - 1 }
247 if lead == 0x58 { hex = 1; num_off = num_off + 1; num_len = num_len - 1 }
248 var val: i64 = 0
249 var k: i64 = 0
250 var ok: i64 = 1
251 while k < num_len {
252 let cc: i64 = nx_he_load_u8(src, num_off + k)
253 if hex == 1 {
254 if cc >= 0x30 {
255 if cc <= 0x39 { val = val * 16 + (cc - 0x30) }
256 }
257 if cc >= 0x61 {
258 if cc <= 0x66 { val = val * 16 + (cc - 0x57) }
259 }
260 if cc >= 0x41 {
261 if cc <= 0x46 { val = val * 16 + (cc - 0x37) }
262 }
263 }
264 if hex == 0 {
265 if cc >= 0x30 {
266 if cc <= 0x39 { val = val * 10 + (cc - 0x30) }
267 }
268 if cc < 0x30 { ok = 0; k = num_len }
269 if cc > 0x39 { ok = 0; k = num_len }
270 }
271 k = k + 1
272 }
273 if ok == 1 {
274 if val < 0x80 {
275 if *dst_pos < dst_cap {
276 nx_he_store_u8(dst, *dst_pos, val)
277 *dst_pos = *dst_pos + 1
278 }
279 }
280 if val >= 0x80 {
281 if val < 0x800 {
282 if *dst_pos + 1 < dst_cap {
283 nx_he_store_u8(dst, *dst_pos, 0xC0 | (val >> 6))
284 nx_he_store_u8(dst, *dst_pos + 1, 0x80 | (val & 0x3F))
285 *dst_pos = *dst_pos + 2
286 }
287 }
288 if val >= 0x800 {
289 if val < 0x10000 {
290 if *dst_pos + 2 < dst_cap {
291 nx_he_store_u8(dst, *dst_pos, 0xE0 | (val >> 12))
292 nx_he_store_u8(dst, *dst_pos + 1, 0x80 | ((val >> 6) & 0x3F))
293 nx_he_store_u8(dst, *dst_pos + 2, 0x80 | (val & 0x3F))
294 *dst_pos = *dst_pos + 3
295 }
296 }
297 }
298 }
299 return end + 1
300 }
301 }
302 }
303
304 // Pass-through on unknown.
305 var pp: i64 = off
306 while pp <= end {
307 if *dst_pos < dst_cap {
308 nx_he_store_u8(dst, *dst_pos, nx_he_load_u8(src, pp))
309 *dst_pos = *dst_pos + 1
310 }
311 pp = pp + 1
312 }
313 return end + 1
314}
315
316// ===== explicit state-machine extractor ===========================
317//
318// States:
319// 0 NORMAL -- emitting text into body (or skipping)
320// 1 TAG_NAME -- inside <... reading tag name until space or >
321// 2 TAG_ATTRS -- past tag name, scanning for >
322// 3 SCRIPT -- skip until </script>
323// 4 STYLE -- skip until </style>
324// 5 COMMENT -- skip until -->
325// 6 TITLE -- emit content into title_buf until </title>
326
327const NX_HE_STATE_NORMAL: i64 = 0
328const NX_HE_STATE_TAG_NAME: i64 = 1
329const NX_HE_STATE_TAG_ATTRS: i64 = 2
330const NX_HE_STATE_SCRIPT: i64 = 3
331const NX_HE_STATE_STYLE: i64 = 4
332const NX_HE_STATE_COMMENT: i64 = 5
333const NX_HE_STATE_TITLE: i64 = 6
334
335const NX_HE_TITLE_CAP: i64 = 512
336
337func nx_html_extract(src: *u8, n: i64) -> *HtmlExtract {
338 let raw: *u8 = sys_mmap(NX_HTML_EXTRACT_BYTES)
339 let r: *HtmlExtract = raw as *HtmlExtract
340 var body_cap: i64 = n
341 if body_cap < 256 { body_cap = 256 }
342 r.body_buf = sys_mmap(body_cap)
343 r.body_len = 0
344 r.title_buf = sys_mmap(NX_HE_TITLE_CAP)
345 r.title_len = 0
346
347 let body_pos_raw: *u8 = sys_mmap(8)
348 let body_pos: *i64 = body_pos_raw as *i64
349 *body_pos = 0
350 let title_pos_raw: *u8 = sys_mmap(8)
351 let title_pos: *i64 = title_pos_raw as *i64
352 *title_pos = 0
353
354 let t_script: *u8 = "script" as *u8
355 let t_style: *u8 = "style" as *u8
356 let t_title: *u8 = "title" as *u8
357 let t_script_end: *u8 = "</script>" as *u8
358 let t_style_end: *u8 = "</style>" as *u8
359 let t_title_end: *u8 = "</title>" as *u8
360 let t_comment_end: *u8 = "-->" as *u8
361 let t_comment_open:*u8 = "<!--" as *u8
362
363 var state: i64 = NX_HE_STATE_NORMAL
364 var title_done: i64 = 0
365 var tag_name_start: i64 = 0
366 var tag_is_close: i64 = 0
367 var p: i64 = 0
368 let BUDGET: i64 = (n * 2) + 16
369 var iter: i64 = 0
370
371 while p < n {
372 if iter >= BUDGET { p = n }
373 if p < n {
374 let c: i64 = nx_he_load_u8(src, p)
375
376 // ----- state: NORMAL -----
377 if state == NX_HE_STATE_NORMAL {
378 if c == 0x3C {
379 // Check for comment first.
380 if nx_he_match_ci(src, p, n, t_comment_open, 4) == 1 {
381 state = NX_HE_STATE_COMMENT
382 p = p + 4
383 }
384 if state == NX_HE_STATE_NORMAL {
385 tag_name_start = p + 1
386 tag_is_close = 0
387 if p + 1 < n {
388 if nx_he_load_u8(src, p + 1) == 0x2F {
389 tag_is_close = 1
390 tag_name_start = p + 2
391 }
392 }
393 state = NX_HE_STATE_TAG_NAME
394 p = tag_name_start
395 }
396 iter = iter + 1
397 continue
398 }
399 if c == 0x26 {
400 p = nx_he_decode_entity(src, p, n, r.body_buf,
401 body_pos, body_cap)
402 iter = iter + 1
403 continue
404 }
405 if *body_pos < body_cap {
406 nx_he_store_u8(r.body_buf, *body_pos, c)
407 *body_pos = *body_pos + 1
408 }
409 p = p + 1
410 iter = iter + 1
411 continue
412 }
413
414 // ----- state: TAG_NAME -----
415 if state == NX_HE_STATE_TAG_NAME {
416 // Read tag name until space/tab/>
417 var name_end: i64 = p
418 let NB: i64 = 32
419 var ni: i64 = 0
420 var scanning: i64 = 1
421 while scanning == 1 {
422 if ni >= NB { scanning = 0 }
423 if scanning == 1 {
424 if name_end >= n { scanning = 0 }
425 if scanning == 1 {
426 let nc: i64 = nx_he_load_u8(src, name_end)
427 if nc == 0x20 { scanning = 0 }
428 if nc == 0x09 { scanning = 0 }
429 if nc == 0x3E { scanning = 0 }
430 if nc == 0x2F { scanning = 0 }
431 if scanning == 1 { name_end = name_end + 1 }
432 }
433 }
434 ni = ni + 1
435 }
436
437 // Classify tag name.
438 let nlen: i64 = name_end - tag_name_start
439
440 // Special tags.
441 if tag_is_close == 0 {
442 if nlen == 6 {
443 if nx_he_match_ci(src, tag_name_start, n, t_script, 6) == 1 {
444 // Skip attrs then enter SCRIPT.
445 state = NX_HE_STATE_TAG_ATTRS
446 p = name_end
447 iter = iter + 1
448 continue
449 }
450 }
451 if nlen == 5 {
452 if nx_he_match_ci(src, tag_name_start, n, t_style, 5) == 1 {
453 state = NX_HE_STATE_TAG_ATTRS
454 p = name_end
455 iter = iter + 1
456 continue
457 }
458 }
459 if nlen == 5 {
460 if nx_he_match_ci(src, tag_name_start, n, t_title, 5) == 1 {
461 if title_done == 0 {
462 state = NX_HE_STATE_TAG_ATTRS
463 p = name_end
464 iter = iter + 1
465 continue
466 }
467 }
468 }
469 }
470
471 // Emit newline for block-level tags.
472 if nx_he_is_block_tag_name(src, tag_name_start, name_end) == 1 {
473 if *body_pos < body_cap {
474 nx_he_store_u8(r.body_buf, *body_pos, 0x0A)
475 *body_pos = *body_pos + 1
476 }
477 }
478
479 // Skip to '>'
480 state = NX_HE_STATE_TAG_ATTRS
481 p = name_end
482 iter = iter + 1
483 continue
484 }
485
486 // ----- state: TAG_ATTRS -----
487 //
488 // Scan to '>'. If the just-completed tag was script/
489 // style/title, transition to that state. Otherwise
490 // return to NORMAL.
491 if state == NX_HE_STATE_TAG_ATTRS {
492 if c == 0x3E {
493 // Decide which state to enter based on tag name.
494 let was_script: i64 = nx_he_match_ci(src, tag_name_start,
495 n, t_script, 6)
496 let was_style: i64 = nx_he_match_ci(src, tag_name_start,
497 n, t_style, 5)
498 let was_title: i64 = nx_he_match_ci(src, tag_name_start,
499 n, t_title, 5)
500 p = p + 1
501 if tag_is_close == 0 {
502 if was_script == 1 {
503 state = NX_HE_STATE_SCRIPT
504 iter = iter + 1
505 continue
506 }
507 if was_style == 1 {
508 state = NX_HE_STATE_STYLE
509 iter = iter + 1
510 continue
511 }
512 if was_title == 1 {
513 if title_done == 0 {
514 state = NX_HE_STATE_TITLE
515 iter = iter + 1
516 continue
517 }
518 }
519 }
520 state = NX_HE_STATE_NORMAL
521 iter = iter + 1
522 continue
523 }
524 p = p + 1
525 iter = iter + 1
526 continue
527 }
528
529 // ----- state: SCRIPT -----
530 if state == NX_HE_STATE_SCRIPT {
531 if nx_he_match_ci(src, p, n, t_script_end, 9) == 1 {
532 p = p + 9
533 state = NX_HE_STATE_NORMAL
534 iter = iter + 1
535 continue
536 }
537 p = p + 1
538 iter = iter + 1
539 continue
540 }
541
542 // ----- state: STYLE -----
543 if state == NX_HE_STATE_STYLE {
544 if nx_he_match_ci(src, p, n, t_style_end, 8) == 1 {
545 p = p + 8
546 state = NX_HE_STATE_NORMAL
547 iter = iter + 1
548 continue
549 }
550 p = p + 1
551 iter = iter + 1
552 continue
553 }
554
555 // ----- state: COMMENT -----
556 if state == NX_HE_STATE_COMMENT {
557 if nx_he_match_ci(src, p, n, t_comment_end, 3) == 1 {
558 p = p + 3
559 state = NX_HE_STATE_NORMAL
560 iter = iter + 1
561 continue
562 }
563 p = p + 1
564 iter = iter + 1
565 continue
566 }
567
568 // ----- state: TITLE -----
569 if state == NX_HE_STATE_TITLE {
570 if nx_he_match_ci(src, p, n, t_title_end, 8) == 1 {
571 p = p + 8
572 state = NX_HE_STATE_NORMAL
573 title_done = 1
574 iter = iter + 1
575 continue
576 }
577 if c == 0x26 {
578 p = nx_he_decode_entity(src, p, n, r.title_buf,
579 title_pos, NX_HE_TITLE_CAP)
580 iter = iter + 1
581 continue
582 }
583 if *title_pos < NX_HE_TITLE_CAP {
584 nx_he_store_u8(r.title_buf, *title_pos, c)
585 *title_pos = *title_pos + 1
586 }
587 p = p + 1
588 iter = iter + 1
589 continue
590 }
591
592 // Defensive: should never get here.
593 p = p + 1
594 }
595 iter = iter + 1
596 }
597
598 r.body_len = *body_pos
599 r.title_len = *title_pos
600 return r
601}