nx_block_density.nx source
↩ module page · 586 lines · 29895 B
1// nx_block_density.nx -- BOILERPLATE BLOCK SCORING for HTML: the /compare/webscraping R6 contract
2// (symbol bd_fit_text). Kohlschuetter, Fankhauser, Nejdl (WSDM 2010) shallow text features -- per-block
3// TEXT DENSITY and LINK DENSITY -- computed in integer permil over the flat tag stream, so an indexer
4// stores the article and not the nav, footer and sidebar around it.
5//
6// WHY THIS SHAPE (2026-08-24): nx_html_to_text is a tag-stripping renderer with 51 importers and a frozen
7// byte contract; the /compare row that credited it with a "block-aware boilerplate strip" overclaimed --
8// no density scoring existed anywhere in the tree (link_density / text_density: 0 matches, corpus
9// complete). This lib is the missing ruler. It COMPOSES nx_html_to_text for the final text so the estate
10// still has exactly ONE HTML-to-text renderer: fit = score blocks -> rebuild HTML from the kept spans
11// -> nx_html_to_text. No second entity table, no second whitespace policy, no second tag list.
12//
13// A BLOCK is the span between two block-level tag boundaries (open or close of p, div, li, h1-h6, section,
14// article, aside, nav, header, footer, main, table, tr, td, th, blockquote, pre, ul, ol, dl, dt, dd,
15// figure, figcaption, form, hr, body). script / style / noscript / template / svg bodies are never text and
16// are removed from every block before counting; sup / sub text (citation brackets) is not counted as words.
17// Per block: text bytes (whitespace runs collapsed), markup bytes, words, link words (words that begin
18// inside an <a>). link_density = link_words * 1000 / words, in permil.
19//
20// THE DECISION IS THE PUBLISHED ONE, NOT AN INVENTED SCORE. Calibration on four real pages (2026-08-24,
21// two Wikipedia articles, a Python docs page, an LWN article) refuted a weighted text-density score:
22// Wikipedia prose measures ~7 pct text density and ~60 pct character link density because every link
23// carries ~100 bytes of markup and citation brackets are anchor text -- the score dropped 86 real
24// paragraphs on one page. Kohlschuetter's densitometric classifier uses WORD COUNTS of the previous,
25// current and next block plus word-based link density; the thresholds below are the learned values his
26// reference implementation (boilerpipe NumWordsRulesClassifier) ships, cited on the /compare row:
27// if curr.ld <= ld_curr_max:
28// if prev.ld <= ld_prev_max:
29// if curr.words <= words_curr_short:
30// if next.words <= words_next_short: (prev.words <= words_prev_short ? BOILERPLATE : CONTENT)
31// else CONTENT
32// else CONTENT
33// else: if curr.words <= words_curr_long: (next.words <= words_next_long ? BOILERPLATE : CONTENT)
34// else CONTENT
35// else BOILERPLATE
36// prev/next are the nearest blocks WITH words (empty wrapper and close-tag blocks are structural and kept
37// so a page with no boilerplate rebuilds byte-identical). Blocks inside a semantic element the conf drops
38// (nav / footer / aside by default) are dropped by construction. If nothing survives, or the page has one
39// block, the FULL text is returned and bd_last_fallback_g says so -- a filter that can return an empty
40// page is worse than no filter.
41//
42// EVERY THRESHOLD IS A CONF ROW: knowledge/block_density.conf (key<space>value, nx_lane_conf semantics),
43// read once per process. The defaults are the bootstrap tier of the rule-17 hierarchy and each carries its
44// source; nx_block_density_cli map <page> prints the per-block table a recalibration is read from.
45// license_tier: ORIGINAL module: nishi-core.search.blockdensity No hw writes (Rule 26).
46import "nx_syscalls.nx"
47import "nx_lane_conf.nx"
48import "nx_html_to_text.nx"
49
50const BD_CONF: *u8 = "knowledge/block_density.conf"
51// bootstrap defaults (rule 17: argv > conf > these). Source per row:
52const BD_DEF_LD_CURR_MAX: i64 = 333 // boilerpipe NumWordsRulesClassifier: curr linkDensity <= 0.333333
53const BD_DEF_LD_PREV_MAX: i64 = 556 // prev linkDensity <= 0.555556
54const BD_DEF_WORDS_CURR_SHORT: i64 = 16 // curr numWords <= 16
55const BD_DEF_WORDS_NEXT_SHORT: i64 = 15 // next numWords <= 15
56const BD_DEF_WORDS_PREV_SHORT: i64 = 4 // prev numWords <= 4
57const BD_DEF_WORDS_CURR_LONG: i64 = 40 // (prev linky) curr numWords <= 40
58const BD_DEF_WORDS_NEXT_LONG: i64 = 17 // (prev linky) next numWords <= 17
59const BD_DEF_DROP_NAV: i64 = 1 // HTML5 semantic elements mean what they say; header stays because article headers hold the title
60const BD_DEF_DROP_FOOTER: i64 = 1
61const BD_DEF_DROP_ASIDE: i64 = 1
62const BD_DEF_DROP_HEADER: i64 = 0
63const BD_DEF_DROP_FORM: i64 = 0 // MEASURED 2026-08-24: LWN wraps its whole comment thread in a <form>; dropping forms
64 // erased 12 KB of reader discussion. Short login/search forms are already killed by the
65 // word rule, so the semantic drop bought nothing and cost content.
66const BD_PERMIL: i64 = 1000
67// block record layout (i64 slots): start end text markup linktext words score kept semantic linkwords
68const BD_REC: i64 = 10
69const BD_F_START: i64 = 0
70const BD_F_END: i64 = 1
71const BD_F_TEXT: i64 = 2
72const BD_F_MARKUP: i64 = 3
73const BD_F_LINK: i64 = 4
74const BD_F_WORDS: i64 = 5
75const BD_F_SCORE: i64 = 6
76const BD_F_KEPT: i64 = 7
77const BD_F_SEM: i64 = 8
78const BD_F_LWORDS: i64 = 9
79const BD_KEPT_NO: i64 = 0
80const BD_KEPT_YES: i64 = 1
81const BD_KEPT_RESCUED: i64 = 2
82const BD_KEPT_SEMANTIC: i64 = 3 // dropped because inside nav/footer/aside/form
83const BD_KEPT_STRUCT: i64 = 4 // no words: a wrapper or close tag, kept for structure, contributes no text
84// semantic depth slots
85const BD_SEM_NAV: i64 = 0
86const BD_SEM_FOOTER: i64 = 1
87const BD_SEM_ASIDE: i64 = 2
88const BD_SEM_HEADER: i64 = 3
89const BD_SEM_FORM: i64 = 4
90const BD_SEM_N: i64 = 5
91const BD_LT: i64 = 60
92const BD_GT: i64 = 62
93const BD_SLASH: i64 = 47
94const BD_BANG: i64 = 33
95const BD_DASH: i64 = 45
96const BD_SPACE: i64 = 32
97
98// ---- conf (loaded once per process; every getter below reads the cached copy) ---------------------
99static bd_conf_loaded_g: i64
100static bd_conf_src_g: i64 // 0 = defaults, 1 = conf file supplied at least one row
101static bd_ld_curr_max_g: i64
102static bd_ld_prev_max_g: i64
103static bd_words_curr_short_g: i64
104static bd_words_next_short_g: i64
105static bd_words_prev_short_g: i64
106static bd_words_curr_long_g: i64
107static bd_words_next_long_g: i64
108static bd_drop_nav_g: i64
109static bd_drop_footer_g: i64
110static bd_drop_aside_g: i64
111static bd_drop_header_g: i64
112static bd_drop_form_g: i64
113// ---- last-call stats (announce, never infer) --------------------------------------------------------
114static bd_last_blocks_g: i64
115static bd_last_kept_g: i64
116static bd_last_semantic_g: i64
117static bd_last_structural_g: i64
118static bd_last_dropped_g: i64
119static bd_last_in_g: i64
120static bd_last_fit_html_g: i64
121static bd_last_fallback_g: i64
122
123func bd_conf_one(key: *u8, dflt: i64) -> i64 {
124 let v: i64 = lc_geti(BD_CONF, "" as *u8, key, 0 - 1)
125 if v < 0 { return dflt }
126 bd_conf_src_g = 1
127 return v
128}
129func bd_load_conf() -> i64 {
130 if bd_conf_loaded_g == 1 { return bd_conf_src_g }
131 bd_conf_src_g = 0
132 bd_ld_curr_max_g = bd_conf_one("ld_curr_max" as *u8, BD_DEF_LD_CURR_MAX)
133 bd_ld_prev_max_g = bd_conf_one("ld_prev_max" as *u8, BD_DEF_LD_PREV_MAX)
134 bd_words_curr_short_g = bd_conf_one("words_curr_short" as *u8, BD_DEF_WORDS_CURR_SHORT)
135 bd_words_next_short_g = bd_conf_one("words_next_short" as *u8, BD_DEF_WORDS_NEXT_SHORT)
136 bd_words_prev_short_g = bd_conf_one("words_prev_short" as *u8, BD_DEF_WORDS_PREV_SHORT)
137 bd_words_curr_long_g = bd_conf_one("words_curr_long" as *u8, BD_DEF_WORDS_CURR_LONG)
138 bd_words_next_long_g = bd_conf_one("words_next_long" as *u8, BD_DEF_WORDS_NEXT_LONG)
139 bd_drop_nav_g = bd_conf_one("drop_nav" as *u8, BD_DEF_DROP_NAV)
140 bd_drop_footer_g = bd_conf_one("drop_footer" as *u8, BD_DEF_DROP_FOOTER)
141 bd_drop_aside_g = bd_conf_one("drop_aside" as *u8, BD_DEF_DROP_ASIDE)
142 bd_drop_header_g = bd_conf_one("drop_header" as *u8, BD_DEF_DROP_HEADER)
143 bd_drop_form_g = bd_conf_one("drop_form" as *u8, BD_DEF_DROP_FORM)
144 bd_conf_loaded_g = 1
145 return bd_conf_src_g
146}
147func bd_conf_reset() -> i64 { bd_conf_loaded_g = 0; return 0 }
148
149// ---- PURE CLASSIFIER (gate-tested with explicit thresholds; the conf wrapper below) -----------------
150func bd_text_density(text: i64, markup: i64) -> i64 {
151 if text + markup <= 0 { return 0 }
152 return text * BD_PERMIL / (text + markup)
153}
154// word-based link density in permil: words that BEGIN inside an <a> over all words
155func bd_link_density(words: i64, linkwords: i64) -> i64 {
156 if words <= 0 { return 0 }
157 var l: i64 = linkwords
158 if l > words { l = words }
159 return l * BD_PERMIL / words
160}
161// Kohlschuetter / boilerpipe NumWordsRules decision. 1 = CONTENT, 0 = BOILERPLATE. Explicit thresholds.
162func bd_classify_t(pw: i64, pld: i64, cw: i64, cld: i64, nw: i64,
163 ld_curr_max: i64, ld_prev_max: i64, w_curr_short: i64, w_next_short: i64, w_prev_short: i64,
164 w_curr_long: i64, w_next_long: i64) -> i64 {
165 if cld > ld_curr_max { return 0 }
166 if pld <= ld_prev_max {
167 if cw <= w_curr_short {
168 if nw <= w_next_short {
169 if pw <= w_prev_short { return 0 }
170 return 1
171 }
172 return 1
173 }
174 return 1
175 }
176 if cw <= w_curr_long {
177 if nw <= w_next_long { return 0 }
178 return 1
179 }
180 return 1
181}
182func bd_classify(pw: i64, pld: i64, cw: i64, cld: i64, nw: i64) -> i64 {
183 bd_load_conf()
184 return bd_classify_t(pw, pld, cw, cld, nw, bd_ld_curr_max_g, bd_ld_prev_max_g, bd_words_curr_short_g,
185 bd_words_next_short_g, bd_words_prev_short_g, bd_words_curr_long_g, bd_words_next_long_g)
186}
187
188// ---- tag classification over the raw bytes ------------------------------------------------------------
189func bd_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
190func bd_is_ws(c: i64) -> i64 {
191 if c == 32 { return 1 }
192 if c == 9 { return 1 }
193 if c == 10 { return 1 }
194 if c == 13 { return 1 }
195 if c == 12 { return 1 }
196 return 0
197}
198func bd_is_name(c: i64) -> i64 {
199 if c >= 97 { if c <= 122 { return 1 } }
200 if c >= 65 { if c <= 90 { return 1 } }
201 if c >= 48 { if c <= 57 { return 1 } }
202 return 0
203}
204// lowercase-equal between src[off..off+len) and a lowercase literal
205func bd_name_eq(src: *u8, off: i64, len: i64, lit: *u8) -> i64 {
206 var i: i64 = 0
207 while i < len {
208 let c: i64 = lit[i] as i64
209 if c == 0 { return 0 }
210 if bd_lc(src[off + i] as i64) != c { return 0 }
211 i = i + 1
212 }
213 if lit[len] != (0 as u8) { return 0 }
214 return 1
215}
216// 1 when the tag name is a block boundary
217func bd_is_block(src: *u8, off: i64, len: i64) -> i64 {
218 if len == 1 { if bd_lc(src[off] as i64) == 112 { return 1 } } // p
219 if len == 2 {
220 let c0: i64 = bd_lc(src[off] as i64)
221 let c1: i64 = bd_lc(src[off + 1] as i64)
222 if c0 == 104 { if c1 >= 49 { if c1 <= 54 { return 1 } } } // h1..h6
223 if c0 == 108 { if c1 == 105 { return 1 } } // li
224 if c0 == 116 { if c1 == 114 { return 1 } if c1 == 100 { return 1 } if c1 == 104 { return 1 } } // tr td th
225 if c0 == 117 { if c1 == 108 { return 1 } } // ul
226 if c0 == 111 { if c1 == 108 { return 1 } } // ol
227 if c0 == 100 { if c1 == 108 { return 1 } if c1 == 116 { return 1 } if c1 == 100 { return 1 } } // dl dt dd
228 if c0 == 104 { if c1 == 114 { return 1 } } // hr
229 }
230 if bd_name_eq(src, off, len, "div" as *u8) == 1 { return 1 }
231 if bd_name_eq(src, off, len, "pre" as *u8) == 1 { return 1 }
232 if bd_name_eq(src, off, len, "nav" as *u8) == 1 { return 1 }
233 if bd_name_eq(src, off, len, "body" as *u8) == 1 { return 1 }
234 if bd_name_eq(src, off, len, "main" as *u8) == 1 { return 1 }
235 if bd_name_eq(src, off, len, "form" as *u8) == 1 { return 1 }
236 if bd_name_eq(src, off, len, "table" as *u8) == 1 { return 1 }
237 if bd_name_eq(src, off, len, "aside" as *u8) == 1 { return 1 }
238 if bd_name_eq(src, off, len, "header" as *u8) == 1 { return 1 }
239 if bd_name_eq(src, off, len, "footer" as *u8) == 1 { return 1 }
240 if bd_name_eq(src, off, len, "figure" as *u8) == 1 { return 1 }
241 if bd_name_eq(src, off, len, "section" as *u8) == 1 { return 1 }
242 if bd_name_eq(src, off, len, "article" as *u8) == 1 { return 1 }
243 if bd_name_eq(src, off, len, "blockquote" as *u8) == 1 { return 1 }
244 if bd_name_eq(src, off, len, "figcaption" as *u8) == 1 { return 1 }
245 return 0
246}
247// semantic slot for a tag name, or -1
248func bd_sem_slot(src: *u8, off: i64, len: i64) -> i64 {
249 if bd_name_eq(src, off, len, "nav" as *u8) == 1 { return BD_SEM_NAV }
250 if bd_name_eq(src, off, len, "footer" as *u8) == 1 { return BD_SEM_FOOTER }
251 if bd_name_eq(src, off, len, "aside" as *u8) == 1 { return BD_SEM_ASIDE }
252 if bd_name_eq(src, off, len, "header" as *u8) == 1 { return BD_SEM_HEADER }
253 if bd_name_eq(src, off, len, "form" as *u8) == 1 { return BD_SEM_FORM }
254 return 0 - 1
255}
256func bd_sem_dropped(slot: i64) -> i64 {
257 if slot == BD_SEM_NAV { return bd_drop_nav_g }
258 if slot == BD_SEM_FOOTER { return bd_drop_footer_g }
259 if slot == BD_SEM_ASIDE { return bd_drop_aside_g }
260 if slot == BD_SEM_HEADER { return bd_drop_header_g }
261 if slot == BD_SEM_FORM { return bd_drop_form_g }
262 return 0
263}
264// 1 when the tag opens a body that is never text (skipped to its close tag)
265func bd_is_suppress(src: *u8, off: i64, len: i64) -> i64 {
266 if bd_name_eq(src, off, len, "script" as *u8) == 1 { return 1 }
267 if bd_name_eq(src, off, len, "style" as *u8) == 1 { return 1 }
268 if bd_name_eq(src, off, len, "noscript" as *u8) == 1 { return 1 }
269 if bd_name_eq(src, off, len, "template" as *u8) == 1 { return 1 }
270 if bd_name_eq(src, off, len, "svg" as *u8) == 1 { return 1 }
271 return 0
272}
273// find the end (index just past '>') of the closing tag </name> starting the search at p; n when absent
274func bd_find_close(src: *u8, n: i64, p: i64, nameoff: i64, namelen: i64) -> i64 {
275 var i: i64 = p
276 while i + 2 + namelen < n {
277 if (src[i] as i64) == BD_LT { if (src[i + 1] as i64) == BD_SLASH {
278 var m: i64 = 1
279 var k: i64 = 0
280 while k < namelen { if bd_lc(src[i + 2 + k] as i64) != bd_lc(src[nameoff + k] as i64) { m = 0; k = namelen } else { k = k + 1 } }
281 if m == 1 {
282 var e: i64 = i + 2 + namelen
283 while e < n { if (src[e] as i64) == BD_GT { return e + 1 } e = e + 1 }
284 return n
285 }
286 } }
287 i = i + 1
288 }
289 return n
290}
291func bd_rec(tbl: *i64, i: i64, f: i64) -> i64 { return tbl[i * BD_REC + f] }
292func bd_set(tbl: *i64, i: i64, f: i64, v: i64) -> i64 { tbl[i * BD_REC + f] = v; return 0 }
293
294// ---- THE SCAN: fills the block table; returns the block count ---------------------------------------
295// tbl must hold (count of '<' in src) + 2 records. depth = i64[BD_SEM_N] scratch (zeroed here).
296func bd_scan(src: *u8, n: i64, tbl: *i64, depth: *i64) -> i64 {
297 var d: i64 = 0
298 while d < BD_SEM_N { depth[d] = 0; d = d + 1 }
299 var nb: i64 = 0
300 // open block 0 at byte 0
301 var bi: i64 = 0
302 bd_set(tbl, 0, BD_F_START, 0); bd_set(tbl, 0, BD_F_TEXT, 0); bd_set(tbl, 0, BD_F_MARKUP, 0)
303 bd_set(tbl, 0, BD_F_LINK, 0); bd_set(tbl, 0, BD_F_WORDS, 0); bd_set(tbl, 0, BD_F_SEM, 0); bd_set(tbl, 0, BD_F_LWORDS, 0)
304 nb = 1
305 var in_link: i64 = 0
306 var in_sup: i64 = 0
307 var last_ws: i64 = 1
308 var p: i64 = 0
309 while p < n {
310 let c: i64 = src[p] as i64
311 if c == BD_LT {
312 // comment
313 var handled: i64 = 0
314 if p + 3 < n { if (src[p + 1] as i64) == BD_BANG { if (src[p + 2] as i64) == BD_DASH { if (src[p + 3] as i64) == BD_DASH {
315 var e: i64 = p + 4
316 var f: i64 = 0
317 while f == 0 {
318 if e + 2 >= n { e = n; f = 1 } else {
319 if (src[e] as i64) == BD_DASH { if (src[e + 1] as i64) == BD_DASH { if (src[e + 2] as i64) == BD_GT { e = e + 3; f = 1 } } }
320 if f == 0 { e = e + 1 }
321 }
322 }
323 bd_set(tbl, bi, BD_F_MARKUP, bd_rec(tbl, bi, BD_F_MARKUP) + (e - p))
324 p = e
325 handled = 1
326 } } } }
327 if handled == 0 {
328 // tag: optional '/', name, then to '>'
329 var q: i64 = p + 1
330 var is_close: i64 = 0
331 if q < n { if (src[q] as i64) == BD_SLASH { is_close = 1; q = q + 1 } }
332 let nameoff: i64 = q
333 var nq: i64 = 1
334 while nq == 1 { if q >= n { nq = 0 } else { if bd_is_name(src[q] as i64) == 1 { q = q + 1 } else { nq = 0 } } }
335 let namelen: i64 = q - nameoff
336 var e2: i64 = q
337 var eq: i64 = 1
338 while eq == 1 { if e2 >= n { eq = 0 } else { if (src[e2] as i64) == BD_GT { eq = 0 } else { e2 = e2 + 1 } } }
339 var tagend: i64 = e2 + 1
340 if tagend > n { tagend = n }
341 if namelen == 0 {
342 // a bare '<' in text: count it as one text byte
343 bd_set(tbl, bi, BD_F_TEXT, bd_rec(tbl, bi, BD_F_TEXT) + 1)
344 if in_link == 1 { bd_set(tbl, bi, BD_F_LINK, bd_rec(tbl, bi, BD_F_LINK) + 1) }
345 last_ws = 0
346 p = p + 1
347 } else {
348 var self_close: i64 = 0
349 if tagend >= 2 { if (src[tagend - 2] as i64) == BD_SLASH { self_close = 1 } }
350 if is_close == 0 { if bd_is_suppress(src, nameoff, namelen) == 1 { if self_close == 0 {
351 // skip the whole body to its close tag; all of it is markup
352 let ce: i64 = bd_find_close(src, n, tagend, nameoff, namelen)
353 bd_set(tbl, bi, BD_F_MARKUP, bd_rec(tbl, bi, BD_F_MARKUP) + (ce - p))
354 p = ce
355 tagend = 0 - 1
356 } } }
357 if tagend >= 0 {
358 if bd_name_eq(src, nameoff, namelen, "a" as *u8) == 1 {
359 if is_close == 1 { in_link = 0 } else { if self_close == 0 { in_link = 1 } }
360 }
361 var is_sup: i64 = 0
362 if bd_name_eq(src, nameoff, namelen, "sup" as *u8) == 1 { is_sup = 1 }
363 if bd_name_eq(src, nameoff, namelen, "sub" as *u8) == 1 { is_sup = 1 }
364 if is_sup == 1 {
365 if is_close == 1 { if in_sup > 0 { in_sup = in_sup - 1 } } else { if self_close == 0 { in_sup = in_sup + 1 } }
366 }
367 let ss: i64 = bd_sem_slot(src, nameoff, namelen)
368 if ss >= 0 {
369 if is_close == 1 { if depth[ss] > 0 { depth[ss] = depth[ss] - 1 } } else { if self_close == 0 { depth[ss] = depth[ss] + 1 } }
370 }
371 if bd_is_block(src, nameoff, namelen) == 1 {
372 // close the current block at p, open the next at p (the boundary tag belongs to it)
373 bd_set(tbl, bi, BD_F_END, p)
374 bi = bi + 1
375 nb = nb + 1
376 bd_set(tbl, bi, BD_F_START, p); bd_set(tbl, bi, BD_F_TEXT, 0); bd_set(tbl, bi, BD_F_MARKUP, 0)
377 bd_set(tbl, bi, BD_F_LINK, 0); bd_set(tbl, bi, BD_F_WORDS, 0); bd_set(tbl, bi, BD_F_LWORDS, 0)
378 var sem: i64 = 0
379 var s2: i64 = 0
380 while s2 < BD_SEM_N { if depth[s2] > 0 { if bd_sem_dropped(s2) == 1 { sem = 1 } } s2 = s2 + 1 }
381 bd_set(tbl, bi, BD_F_SEM, sem)
382 last_ws = 1
383 }
384 bd_set(tbl, bi, BD_F_MARKUP, bd_rec(tbl, bi, BD_F_MARKUP) + (tagend - p))
385 p = tagend
386 }
387 }
388 }
389 } else {
390 if in_sup > 0 {
391 // citation brackets and footnote marks: not words, not link words, not text
392 p = p + 1
393 } else {
394 if bd_is_ws(c) == 1 {
395 if last_ws == 0 {
396 bd_set(tbl, bi, BD_F_TEXT, bd_rec(tbl, bi, BD_F_TEXT) + 1)
397 if in_link == 1 { bd_set(tbl, bi, BD_F_LINK, bd_rec(tbl, bi, BD_F_LINK) + 1) }
398 }
399 last_ws = 1
400 } else {
401 if last_ws == 1 {
402 bd_set(tbl, bi, BD_F_WORDS, bd_rec(tbl, bi, BD_F_WORDS) + 1)
403 if in_link == 1 { bd_set(tbl, bi, BD_F_LWORDS, bd_rec(tbl, bi, BD_F_LWORDS) + 1) }
404 }
405 bd_set(tbl, bi, BD_F_TEXT, bd_rec(tbl, bi, BD_F_TEXT) + 1)
406 if in_link == 1 { bd_set(tbl, bi, BD_F_LINK, bd_rec(tbl, bi, BD_F_LINK) + 1) }
407 last_ws = 0
408 }
409 p = p + 1
410 }
411 }
412 }
413 bd_set(tbl, bi, BD_F_END, n)
414 return nb
415}
416
417// ---- DECIDE: semantic drop, structural keep, then the published rule over prev/curr/next ------------
418func bd_prev_content(tbl: *i64, j: i64) -> i64 {
419 var pv: i64 = j - 1
420 while pv >= 0 { if bd_rec(tbl, pv, BD_F_WORDS) > 0 { return pv } pv = pv - 1 }
421 return 0 - 1
422}
423func bd_next_content(tbl: *i64, nb: i64, j: i64) -> i64 {
424 var nx: i64 = j + 1
425 while nx < nb { if bd_rec(tbl, nx, BD_F_WORDS) > 0 { return nx } nx = nx + 1 }
426 return 0 - 1
427}
428func bd_decide(tbl: *i64, nb: i64) -> i64 {
429 var i: i64 = 0
430 var kept: i64 = 0
431 var sem: i64 = 0
432 var structural: i64 = 0
433 while i < nb {
434 let cw: i64 = bd_rec(tbl, i, BD_F_WORDS)
435 let cld: i64 = bd_link_density(cw, bd_rec(tbl, i, BD_F_LWORDS))
436 bd_set(tbl, i, BD_F_SCORE, cld)
437 var k: i64 = BD_KEPT_NO
438 if cw == 0 {
439 // STRUCTURAL: no words (a close tag, a wrapper open). Kept so the rebuilt HTML keeps its block
440 // boundaries and a no-boilerplate page comes back byte-identical; contributes no text.
441 k = BD_KEPT_STRUCT; structural = structural + 1
442 } else {
443 if bd_rec(tbl, i, BD_F_SEM) == 1 { k = BD_KEPT_SEMANTIC; sem = sem + 1 } else {
444 var pw: i64 = 0
445 var pld: i64 = 0
446 let pv: i64 = bd_prev_content(tbl, i)
447 if pv >= 0 { pw = bd_rec(tbl, pv, BD_F_WORDS); pld = bd_link_density(pw, bd_rec(tbl, pv, BD_F_LWORDS)) }
448 var nw: i64 = 0
449 let nx: i64 = bd_next_content(tbl, nb, i)
450 if nx >= 0 { nw = bd_rec(tbl, nx, BD_F_WORDS) }
451 if bd_classify_t(pw, pld, cw, cld, nw, bd_ld_curr_max_g, bd_ld_prev_max_g, bd_words_curr_short_g,
452 bd_words_next_short_g, bd_words_prev_short_g, bd_words_curr_long_g, bd_words_next_long_g) == 1 {
453 k = BD_KEPT_YES; kept = kept + 1
454 }
455 }
456 }
457 bd_set(tbl, i, BD_F_KEPT, k)
458 i = i + 1
459 }
460 bd_last_blocks_g = nb
461 bd_last_kept_g = kept
462 bd_last_semantic_g = sem
463 bd_last_structural_g = structural
464 bd_last_dropped_g = nb - kept - sem - structural
465 return kept
466}
467
468func bd_count_lt(src: *u8, n: i64) -> i64 {
469 var i: i64 = 0
470 var c: i64 = 0
471 while i < n { if (src[i] as i64) == BD_LT { c = c + 1 } i = i + 1 }
472 return c
473}
474
475// ---- THE CONTRACT: fit text. Returns bytes written to out (the nx_html_to_text contract). ------------
476func bd_fit_text(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
477 bd_load_conf()
478 bd_last_in_g = n
479 bd_last_fallback_g = 0
480 if n <= 0 { bd_last_blocks_g = 0; bd_last_kept_g = 0; bd_last_fit_html_g = 0; return 0 }
481 let nlt: i64 = bd_count_lt(src, n)
482 let tbl: *i64 = sys_mmap((nlt + 2) * BD_REC * 8) as *i64
483 let depth: *i64 = sys_mmap(BD_SEM_N * 8) as *i64
484 let nb: i64 = bd_scan(src, n, tbl, depth)
485 let keptn: i64 = bd_decide(tbl, nb)
486 var r: i64 = 0
487 if keptn <= 0 { bd_last_fallback_g = 1 }
488 if nb <= 1 { bd_last_fallback_g = 1 }
489 if bd_last_fallback_g == 1 {
490 bd_last_fit_html_g = n
491 r = nx_html_to_text(src, n, out, cap)
492 } else {
493 let fit: *u8 = sys_mmap(n + 1)
494 var o: i64 = 0
495 var i: i64 = 0
496 while i < nb {
497 let k: i64 = bd_rec(tbl, i, BD_F_KEPT)
498 var take: i64 = 0
499 if k == BD_KEPT_YES { take = 1 }
500 if k == BD_KEPT_RESCUED { take = 1 }
501 if k == BD_KEPT_STRUCT { take = 1 }
502 if take == 1 {
503 // spans are copied VERBATIM and contiguous spans stay contiguous, so a page with no
504 // boilerplate rebuilds to the identical byte string (the gate's negative control)
505 var s: i64 = bd_rec(tbl, i, BD_F_START)
506 let e: i64 = bd_rec(tbl, i, BD_F_END)
507 while s < e { fit[o] = src[s]; o = o + 1; s = s + 1 }
508 }
509 i = i + 1
510 }
511 bd_last_fit_html_g = o
512 r = nx_html_to_text(fit, o, out, cap)
513 sys_munmap(fit, n + 1)
514 }
515 sys_munmap(tbl as *u8, (nlt + 2) * BD_REC * 8)
516 sys_munmap(depth as *u8, BD_SEM_N * 8)
517 return r
518}
519
520// ---- MAP: the per-block table as text, for calibration and for gates (one line per block) ------------
521func bd_map_num(out: *u8, cap: i64, o: i64, v: i64) -> i64 {
522 var oo: i64 = o
523 if v == 0 { if oo < cap { out[oo] = 48 as u8; oo = oo + 1 } return oo }
524 var m: i64 = v
525 if m < 0 { if oo < cap { out[oo] = 45 as u8; oo = oo + 1 } m = 0 - m }
526 let t: *u8 = sys_mmap(32)
527 var k: i64 = 0
528 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
529 while k > 0 { k = k - 1; if oo < cap { out[oo] = t[k]; oo = oo + 1 } }
530 sys_munmap(t, 32)
531 return oo
532}
533func bd_map_str(out: *u8, cap: i64, o: i64, s: *u8) -> i64 {
534 var oo: i64 = o
535 var i: i64 = 0
536 while s[i] != (0 as u8) { if oo < cap { out[oo] = s[i]; oo = oo + 1 } i = i + 1 }
537 return oo
538}
539// writes "blk <i> start=<s> len=<l> text=<t> markup=<m> link=<k> words=<w> td=<> ld=<> score=<> kept=<0|1|2|3>\n" per block
540func bd_map(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
541 bd_load_conf()
542 if n <= 0 { return 0 }
543 let nlt: i64 = bd_count_lt(src, n)
544 let tbl: *i64 = sys_mmap((nlt + 2) * BD_REC * 8) as *i64
545 let depth: *i64 = sys_mmap(BD_SEM_N * 8) as *i64
546 let nb: i64 = bd_scan(src, n, tbl, depth)
547 bd_decide(tbl, nb)
548 var o: i64 = 0
549 o = bd_map_str(out, cap, o, "conf_src=" as *u8)
550 if bd_conf_src_g == 1 { o = bd_map_str(out, cap, o, "file" as *u8) } else { o = bd_map_str(out, cap, o, "defaults" as *u8) }
551 o = bd_map_str(out, cap, o, " ld_curr_max=" as *u8); o = bd_map_num(out, cap, o, bd_ld_curr_max_g)
552 o = bd_map_str(out, cap, o, " ld_prev_max=" as *u8); o = bd_map_num(out, cap, o, bd_ld_prev_max_g)
553 o = bd_map_str(out, cap, o, " words_curr_short=" as *u8); o = bd_map_num(out, cap, o, bd_words_curr_short_g)
554 o = bd_map_str(out, cap, o, " words_next_short=" as *u8); o = bd_map_num(out, cap, o, bd_words_next_short_g)
555 o = bd_map_str(out, cap, o, " words_prev_short=" as *u8); o = bd_map_num(out, cap, o, bd_words_prev_short_g)
556 o = bd_map_str(out, cap, o, " words_curr_long=" as *u8); o = bd_map_num(out, cap, o, bd_words_curr_long_g)
557 o = bd_map_str(out, cap, o, " words_next_long=" as *u8); o = bd_map_num(out, cap, o, bd_words_next_long_g)
558 o = bd_map_str(out, cap, o, " blocks=" as *u8); o = bd_map_num(out, cap, o, nb)
559 o = bd_map_str(out, cap, o, " kept=" as *u8); o = bd_map_num(out, cap, o, bd_last_kept_g)
560 o = bd_map_str(out, cap, o, " semantic=" as *u8); o = bd_map_num(out, cap, o, bd_last_semantic_g)
561 o = bd_map_str(out, cap, o, " structural=" as *u8); o = bd_map_num(out, cap, o, bd_last_structural_g)
562 o = bd_map_str(out, cap, o, " dropped=" as *u8); o = bd_map_num(out, cap, o, bd_last_dropped_g)
563 o = bd_map_str(out, cap, o, "\n" as *u8)
564 var i: i64 = 0
565 while i < nb {
566 let t: i64 = bd_rec(tbl, i, BD_F_TEXT)
567 let m: i64 = bd_rec(tbl, i, BD_F_MARKUP)
568 let l: i64 = bd_rec(tbl, i, BD_F_LINK)
569 o = bd_map_str(out, cap, o, "blk " as *u8); o = bd_map_num(out, cap, o, i)
570 o = bd_map_str(out, cap, o, " start=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_START))
571 o = bd_map_str(out, cap, o, " len=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_END) - bd_rec(tbl, i, BD_F_START))
572 o = bd_map_str(out, cap, o, " text=" as *u8); o = bd_map_num(out, cap, o, t)
573 o = bd_map_str(out, cap, o, " markup=" as *u8); o = bd_map_num(out, cap, o, m)
574 o = bd_map_str(out, cap, o, " link=" as *u8); o = bd_map_num(out, cap, o, l)
575 o = bd_map_str(out, cap, o, " words=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_WORDS))
576 o = bd_map_str(out, cap, o, " lwords=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_LWORDS))
577 o = bd_map_str(out, cap, o, " td=" as *u8); o = bd_map_num(out, cap, o, bd_text_density(t, m))
578 o = bd_map_str(out, cap, o, " ld=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_SCORE))
579 o = bd_map_str(out, cap, o, " kept=" as *u8); o = bd_map_num(out, cap, o, bd_rec(tbl, i, BD_F_KEPT))
580 o = bd_map_str(out, cap, o, "\n" as *u8)
581 i = i + 1
582 }
583 sys_munmap(tbl as *u8, (nlt + 2) * BD_REC * 8)
584 sys_munmap(depth as *u8, BD_SEM_N * 8)
585 return o
586}