nx_dr_extract.nx source
↩ module page · 159 lines · 6291 B
1// nx_dr_extract.nx -- sovereign HTML -> main-text extractor (eats the "crude tag-strip" limit
2// named on the published /research-scale scorecard). The old rig's sed 's/<[^>]*>/ /g' removed
3// TAGS but kept <script>/<style> BODIES and HTML comments as judged text, and entity names
4// tokenized as noise words (amp/nbsp/quot). This organ drops script/style/comment content,
5// strips tags to a word separator, kills short &entities;, and squeezes whitespace -- so the
6// judge sees prose, not JavaScript.
7// nx_dr_extract <html_file> -> extracted text on stdout
8// FAIL-LOUD: unreadable file => stderr marker + exit 1 (never a silent empty extraction).
9// Deterministic by construction (pure byte scan). An unterminated <script> eats the remainder
10// (rare, malformed HTML) -- deterministic fail-safe, never a crash.
11// Every func <=6 params (NAS nx_cc >6-arg skew, seq239). No hardware writes (Rule 26).
12// license_tier: ORIGINAL expect_exit: 0
13// module: nishi-core.research.dr_extract depends: nx_syscalls.nx
14import "nx_syscalls.nx"
15const K_MAGIC_262144: i64 = 262144
16
17const K_IN: i64 = 4194304
18
19func ex_alnum(c: i64) -> i64 {
20 if c >= 48 { if c <= 57 { return 1 } }
21 if c >= 65 { if c <= 90 { return 1 } }
22 if c >= 97 { if c <= 122 { return 1 } }
23 return 0
24}
25
26func ex_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
27
28// buf[i..) matches lowercase `name` case-insensitively AND the follower is non-alnum
29// (so <scripty> is NOT <script>). name must be lowercase.
30func ex_matchname(buf: *u8, i: i64, len: i64, name: *u8, nlen: i64) -> i64 {
31 if i + nlen > len { return 0 }
32 var k: i64 = 0
33 while k < nlen {
34 if ex_lc(buf[i+k] as i64) != (name[k] as i64) { return 0 }
35 k = k + 1
36 }
37 if i + nlen >= len { return 1 }
38 if ex_alnum(buf[i+nlen] as i64) == 1 { return 0 }
39 return 1
40}
41
42// i is AT the '<' of an opening <script/<style tag: skip past the matching case-insensitive
43// close tag's '>'. Unterminated block => len (deterministic fail-safe).
44func ex_skipblock(buf: *u8, i: i64, len: i64, name: *u8, nlen: i64) -> i64 {
45 var p: i64 = i
46 var r: i64 = len
47 while p < len { if buf[p] == (62 as u8) { r = p + 1; p = len } else { p = p + 1 } }
48 p = r
49 var out: i64 = len
50 while p < len {
51 var adv: i64 = 1
52 if buf[p] == (60 as u8) {
53 if p + 1 < len {
54 if buf[p+1] == (47 as u8) {
55 if ex_matchname(buf, p + 2, len, name, nlen) == 1 {
56 var q: i64 = p + 2 + nlen
57 var e: i64 = len
58 while q < len { if buf[q] == (62 as u8) { e = q + 1; q = len } else { q = q + 1 } }
59 out = e; p = len; adv = 0
60 }
61 }
62 }
63 }
64 if adv == 1 { p = p + 1 }
65 }
66 return out
67}
68
69// i just past "<!--": return index after "-->" (or len if unterminated).
70func ex_skipcomment(buf: *u8, i: i64, len: i64) -> i64 {
71 var p: i64 = i
72 var r: i64 = len
73 while p + 2 < len {
74 var adv: i64 = 1
75 if buf[p] == (45 as u8) { if buf[p+1] == (45 as u8) { if buf[p+2] == (62 as u8) { r = p + 3; p = len; adv = 0 } } }
76 if adv == 1 { p = p + 1 }
77 }
78 return r
79}
80
81// i at '&': if a short entity (&name; / &#nn; within 10 chars) return index AFTER ';', else -1.
82func ex_entend(buf: *u8, i: i64, len: i64) -> i64 {
83 var j: i64 = i + 1
84 var r: i64 = 0 - 1
85 var stop: i64 = i + 11
86 if stop > len { stop = len }
87 while j < stop {
88 let c: i64 = buf[j] as i64
89 if c == 59 { if j > i + 1 { r = j + 1 } j = stop } else {
90 var ok: i64 = ex_alnum(c)
91 if c == 35 { ok = 1 }
92 if ok == 0 { j = stop } else { j = j + 1 }
93 }
94 }
95 return r
96}
97
98func ex_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
99 let fd: i64 = sys_openat_rd(path)
100 if fd < 0 { return 0 - 1 }
101 var total: i64 = 0
102 var r: i64 = 1
103 while r > 0 {
104 let left: i64 = cap - total
105 if left <= 0 { r = 0 } else {
106 var want: i64 = K_MAGIC_262144
107 if want > left { want = left }
108 r = sys_read(fd, (buf as i64 + total) as *u8, want)
109 if r > 0 { total = total + r }
110 }
111 }
112 sys_close(fd)
113 return total
114}
115
116func ex_err(msg: *u8) -> i64 { var n: i64 = 0; while msg[n] != (0 as u8) { n = n + 1 } sys_write(2, msg, n); return 0 }
117
118func main(argc: i64, argv: *i64) -> i64 {
119 if argc < 2 { ex_err("usage: nx_dr_extract <html_file>\n" as *u8); sys_exit(2); return 2 }
120 let buf: *u8 = sys_mmap(K_IN)
121 let len: i64 = ex_readfile(argv[1] as *u8, buf, K_IN)
122 if len < 0 { ex_err("EXTRACT-ERR: unreadable input\n" as *u8); sys_exit(1); return 1 }
123 let ob: *u8 = sys_mmap(len + 16)
124 var i: i64 = 0
125 var o: i64 = 0
126 var sp: i64 = 1
127 while i < len {
128 let c: i64 = buf[i] as i64
129 var esp: i64 = 0
130 var ech: i64 = 0 - 1
131 if c == 60 {
132 var nx: i64 = 0 - 1
133 if i + 3 < len { if buf[i+1] == (33 as u8) { if buf[i+2] == (45 as u8) { if buf[i+3] == (45 as u8) { nx = ex_skipcomment(buf, i + 4, len) } } } }
134 if nx < 0 { if ex_matchname(buf, i + 1, len, "script" as *u8, 6) == 1 { nx = ex_skipblock(buf, i, len, "script" as *u8, 6) } }
135 if nx < 0 { if ex_matchname(buf, i + 1, len, "style" as *u8, 5) == 1 { nx = ex_skipblock(buf, i, len, "style" as *u8, 5) } }
136 if nx < 0 {
137 var p: i64 = i + 1
138 var r: i64 = len
139 while p < len { if buf[p] == (62 as u8) { r = p + 1; p = len } else { p = p + 1 } }
140 nx = r
141 }
142 i = nx
143 esp = 1
144 } else {
145 if c == 38 {
146 let e: i64 = ex_entend(buf, i, len)
147 if e > 0 { i = e } else { i = i + 1 }
148 esp = 1
149 } else {
150 if c == 9 { esp = 1 } else { if c == 10 { esp = 1 } else { if c == 13 { esp = 1 } else { if c == 32 { esp = 1 } else { ech = c } } } }
151 i = i + 1
152 }
153 }
154 if esp == 1 { if sp == 0 { ob[o] = 32 as u8; o = o + 1; sp = 1 } }
155 if ech >= 0 { ob[o] = ech as u8; o = o + 1; sp = 0 }
156 }
157 sys_write(1, ob, o)
158 sys_exit(0); return 0
159}