nx_docprose.nx source
↩ module page · 329 lines · 15265 B
1// nx_docprose.nx -- LIB: WHERE DOES A STORED DOCUMENT'S REAL CONTENT BEGIN?
2//
3// THE DEFECT THIS EXISTS TO REMOVE, MEASURED LIVE ON nishifamily.com/search 2026-08-25 (site scope,
4// one result page): results were rendered with these as their TITLES --
5// "HTTP/1.1 200 OK Date: Thu, 02 Jul 2026 17:07:07 GMT Content-Type: text/h"
6// "<p align=\"center\">"
7// The renderer was not at fault: it faithfully took "the first non-empty LINE", and the first line of
8// those stored documents really IS an HTTP status line / a raw markup tag, because the evidence-mirror
9// captures in knowledge/fetched are indexed as documents with their wire bytes intact.
10//
11// ROOT vs REACH -- STATED PLAINLY SO NOBODY MISREADS THIS LIB'S SCOPE:
12// The ROOT fix is at INGEST (derive clean text before indexing; nx_block_density's bd_fit_text is the
13// estate's shipping boilerplate remover and is the organ for that job).
14// This lib is the SERVE-SIDE half, and it is NOT a workaround: it repairs the ALREADY-INDEXED corpus
15// with no reindex, and it stays correct afterwards as defence in depth. An ingest fix alone leaves
16// every existing document broken until a full reindex; a serve fix alone leaves the stored text dirty
17// for every other consumer. Both are owed and both are real.
18//
19// POLICY IS CONF, STRUCTURE IS CODE. The protocol facts (an HTTP header block ends at the first blank
20// line, RFC 9112 2.1) are code. The one judgement call -- how many letters make a "word" -- is a conf row.
21// license_tier: ORIGINAL
22import "nx_syscalls.nx"
23import "nx_lane_conf.nx"
24import "nx_textcut.nx"
25
26const DPR_CONF: *u8 = "knowledge/docprose.conf"
27
28// A title line must contain at least one RUN of this many token bytes. This is the definition of
29// "contains a word", not a tuning knob: it exists so a line of pure punctuation ("---", "|", "*")
30// is not served as a document title. Conf-overridable because it is the only judgement here.
31const DPR_DEF_TITLE_MIN_WORDRUN: i64 = 2
32
33// WORK BUDGET, NAMED AND ANNOUNCING (the estate's law: every scan gets a budget derived from its
34// measured corpora, and over-budget must be LOUD, never infinite). This is NOT a fresh guess: it is
35// the SAME 8000-byte head window nx_docportal_search_serve.nx already uses to bound its snippet scan
36// on this exact corpus. Reusing the sibling's calibration beats inventing a second budget that can
37// disagree with it.
38const DPR_DEF_TITLE_SCAN_BYTES: i64 = 8000
39
40const DPR_CH_LF: i64 = 10
41const DPR_CH_CR: i64 = 13
42const DPR_CH_SP: i64 = 32
43const DPR_CH_DASH: i64 = 45
44const DPR_CH_COLON: i64 = 58
45const DPR_CH_LT: i64 = 60
46const DPR_HTTPSIG_N: i64 = 5 // len("HTTP/") -- DERIVED beside the literal it measures, never hand-counted apart from it
47
48static dpr_conf_loaded_g: i64
49static dpr_conf_src_g: i64
50static dpr_title_min_wordrun_g: i64
51static dpr_title_scan_bytes_g: i64
52
53// ANNOUNCE, NEVER INFER -- last-call facts a caller can print instead of assuming.
54static dpr_last_http_skipped_g: i64 // bytes of HTTP header block stepped over
55static dpr_last_lines_skipped_g: i64 // leading header/markup/blank lines stepped over
56static dpr_last_budget_hit_g: i64 // 1 = the scan budget was reached => coverage was PARTIAL
57static dpr_last_degraded_g: i64 // 1 = body located by the DEGRADED html-anchor path, not the exact one
58
59func dpr_conf_one(key: *u8, dflt: i64) -> i64 {
60 let v: i64 = lc_geti(DPR_CONF, "" as *u8, key, 0 - 1)
61 if v < 0 { return dflt }
62 dpr_conf_src_g = 1
63 return v
64}
65func dpr_load_conf() -> i64 {
66 if dpr_conf_loaded_g == 1 { return dpr_conf_src_g }
67 dpr_conf_src_g = 0
68 dpr_title_min_wordrun_g = dpr_conf_one("title_min_wordrun" as *u8, DPR_DEF_TITLE_MIN_WORDRUN)
69 dpr_title_scan_bytes_g = dpr_conf_one("title_scan_bytes" as *u8, DPR_DEF_TITLE_SCAN_BYTES)
70 dpr_conf_loaded_g = 1
71 return dpr_conf_src_g
72}
73func dpr_conf_reset() -> i64 { dpr_conf_loaded_g = 0; return 0 }
74func dpr_last_http_skipped() -> i64 { return dpr_last_http_skipped_g }
75func dpr_last_lines_skipped() -> i64 { return dpr_last_lines_skipped_g }
76func dpr_last_budget_hit() -> i64 { return dpr_last_budget_hit_g }
77// 1 = the body offset came from the DEGRADED html-anchor path (a capture with no blank line), not from
78// the protocol-exact one. A caller that treats those two as the same answer is trusting a weaker rule
79// without knowing it, which is exactly how a degraded reading becomes an unqualified claim.
80func dpr_last_degraded() -> i64 { return dpr_last_degraded_g }
81
82// ---- HTTP capture front matter -------------------------------------------------------------------
83// Returns the offset of the entity body when the buffer GENUINELY BEGINS with a status line, else 0.
84// Deliberately NOT a heuristic scan: a document that merely mentions HTTP headers somewhere in its
85// prose must be left alone. Anchored at byte 0 or it does not fire.
86func dpr_http_body(txt: *u8, n: i64) -> i64 {
87 // RESET THE ANNOUNCE AT ENTRY, NOT AT THE CALLER. It is a LAST-CALL fact, so it must describe THIS
88 // call and no other. Caught by this lib's own neg-control tooth: the flag was reset only in
89 // dpr_content_start, so a direct dpr_http_body call taking the EXACT path returned early and left a
90 // previous degraded call's 1 standing -- every later exact answer then read as degraded.
91 // A STICKY ANNOUNCE IS WORSE THAN NO ANNOUNCE: it reports a weaker rule than the one that answered,
92 // and it does so in the flattering-to-doubt direction, so nobody investigates.
93 dpr_last_degraded_g = 0
94 let sig: *u8 = "HTTP/" as *u8
95 var i: i64 = 0
96 while i < DPR_HTTPSIG_N {
97 if i >= n { return 0 }
98 if txt[i] != sig[i] { return 0 }
99 i = i + 1
100 }
101 // header block ends at the first blank line: LF LF, or CR LF CR LF
102 var p: i64 = 0
103 while p + 1 < n {
104 if (txt[p] as i64) == DPR_CH_LF {
105 if (txt[p + 1] as i64) == DPR_CH_LF { return p + 2 }
106 if p + 2 < n {
107 if (txt[p + 1] as i64) == DPR_CH_CR {
108 if (txt[p + 2] as i64) == DPR_CH_LF { return p + 3 }
109 }
110 }
111 }
112 p = p + 1
113 }
114 // NO BLANK LINE FOUND. This capture LOST ITS HEADER LINE STRUCTURE before it was ever stored --
115 // MEASURED LIVE 2026-08-25 on indexed documents whose headers are SPACE-separated on a single line:
116 // "HTTP/1.1 200 OK Date: Thu, 02 Jul 2026 16:53:48 GMT Content-Type: text/html; charset=utf-8 Vary: ..."
117 // There is no blank line to find, so the protocol-exact rule above correctly declines -- and the SERP
118 // then fell back to showing that header run as the document TITLE.
119 //
120 // DEGRADED PATH, AND IT STILL DOES NOT GUESS. When the capture DECLARED text/html we can anchor on a
121 // structural fact about the DECLARED TYPE rather than on the shape of the bytes: an html entity body
122 // begins at its document start token. We anchor ONLY on <!doctype or <html, never on a bare '<':
123 // a Link header carries "<https://...>; rel=canonical", so a bare-'<' anchor would land INSIDE the
124 // header run and quietly report the wrong body offset -- a confident wrong answer, which is worse
125 // than the honest zero below. A capture of any other declared type gives us nothing sound to anchor
126 // on, so it keeps returning 0.
127 dpr_load_conf()
128 var win: i64 = n
129 if win > dpr_title_scan_bytes_g { win = dpr_title_scan_bytes_g }
130 if dpr_find_ci(txt, 0, win, "content-type" as *u8) < 0 { return 0 }
131 if dpr_find_ci(txt, 0, win, "text/html" as *u8) < 0 { return 0 }
132 var lt: i64 = dpr_find_ci(txt, 0, win, "<!doctype" as *u8)
133 if lt < 0 { lt = dpr_find_ci(txt, 0, win, "<html" as *u8) }
134 if lt <= 0 { return 0 }
135 // ANNOUNCED SEPARATELY from the exact path, so a reader can always tell which rule answered.
136 dpr_last_degraded_g = 1
137 return lt
138}
139
140func dpr_line_end(txt: *u8, n: i64, off: i64) -> i64 {
141 var e: i64 = off
142 var go: i64 = 1
143 while go == 1 {
144 if e >= n { go = 0 } else {
145 if (txt[e] as i64) == DPR_CH_LF { go = 0 } else { e = e + 1 }
146 }
147 }
148 return e
149}
150
151// a line whose first visible byte opens a tag: raw markup that leaked into the stored text
152func dpr_is_markup_line(txt: *u8, n: i64, s: i64, e: i64) -> i64 {
153 let p: i64 = tc_skip_ws(txt, e, s)
154 if p >= e { return 0 }
155 if (txt[p] as i64) == DPR_CH_LT { return 1 }
156 return 0
157}
158
159func dpr_is_namech(c: i64) -> i64 {
160 if tc_is_wordch(c) == 1 { if c < 128 { return 1 } }
161 if c == DPR_CH_DASH { return 1 }
162 return 0
163}
164
165// "Name: value" -- an HTTP field line. The name may not contain spaces, which is what keeps this from
166// firing on ordinary prose ("Chapter 1: Introduction" stops at the space and is correctly NOT a header).
167// Only ever applied to a LEADING RUN of lines, so a "Subject: ..." line inside real prose is safe.
168func dpr_is_header_line(txt: *u8, n: i64, s: i64, e: i64) -> i64 {
169 let sig: *u8 = "HTTP/" as *u8
170 var i: i64 = 0
171 var m: i64 = 1
172 var go: i64 = 1
173 while go == 1 {
174 if i >= DPR_HTTPSIG_N { go = 0 } else {
175 if s + i >= e { m = 0; go = 0 } else {
176 if txt[s + i] != sig[i] { m = 0; go = 0 } else { i = i + 1 }
177 }
178 }
179 }
180 if m == 1 { return 1 }
181 var p: i64 = s
182 var nlen: i64 = 0
183 var g2: i64 = 1
184 while g2 == 1 {
185 if p >= e { g2 = 0 } else {
186 if dpr_is_namech(txt[p] as i64) == 1 { nlen = nlen + 1; p = p + 1 } else { g2 = 0 }
187 }
188 }
189 if nlen == 0 { return 0 }
190 if p >= e { return 0 }
191 if (txt[p] as i64) != DPR_CH_COLON { return 0 }
192 if p + 1 >= e { return 1 }
193 if (txt[p + 1] as i64) == DPR_CH_SP { return 1 }
194 return 0
195}
196
197func dpr_max_wordrun(txt: *u8, s: i64, e: i64) -> i64 {
198 var best: i64 = 0
199 var run: i64 = 0
200 var i: i64 = s
201 while i < e {
202 if tc_is_wordch(txt[i] as i64) == 1 {
203 run = run + 1
204 if run > best { best = run }
205 } else { run = 0 }
206 i = i + 1
207 }
208 return best
209}
210
211// ---- IS THIS DOCUMENT ACTUALLY HTML? --------------------------------------------------------------
212// This gate exists because running a tag stripper over text that is NOT html is DESTRUCTIVE, and the
213// destruction is silent. This estate indexes its own source: nx_html_to_text would eat every `<u8>` type
214// parameter and every `a < b` it could parse as a tag, and the document would still look plausible.
215// So extraction is gated on a STRUCTURAL FACT and never on a tag-density heuristic:
216// (a) the capture's own Content-Type header DECLARED text/html -- the producer said so, authoritative; or
217// (b) a doctype/html/body opening marker appears in the head window.
218// A capture declaring text/plain therefore returns 0 even if its body contains angle brackets, which is
219// exactly the case a density test would get wrong.
220func dpr_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
221
222// case-insensitive substring search over txt[from, to)
223func dpr_find_ci(txt: *u8, from: i64, to: i64, pat: *u8) -> i64 {
224 var pl: i64 = 0
225 while pat[pl] != (0 as u8) { pl = pl + 1 }
226 if pl == 0 { return 0 - 1 }
227 var i: i64 = from
228 while i + pl <= to {
229 var m: i64 = 1
230 var j: i64 = 0
231 while j < pl {
232 if dpr_lc(txt[i + j] as i64) != dpr_lc(pat[j] as i64) { m = 0; j = pl } else { j = j + 1 }
233 }
234 if m == 1 { return i }
235 i = i + 1
236 }
237 return 0 - 1
238}
239
240func dpr_is_html(txt: *u8, n: i64) -> i64 {
241 dpr_load_conf()
242 if n <= 0 { return 0 }
243 let bstart: i64 = dpr_http_body(txt, n)
244 if bstart > 0 {
245 // A real capture: believe its declared type and stop. This is the authoritative branch -- a
246 // capture that says text/plain is NOT html no matter what its body looks like.
247 if dpr_find_ci(txt, 0, bstart, "text/html" as *u8) >= 0 { return 1 }
248 return 0
249 }
250 var win: i64 = dpr_title_scan_bytes_g
251 if win > n { win = n }
252 if dpr_find_ci(txt, 0, win, "<!doctype html" as *u8) >= 0 { return 1 }
253 if dpr_find_ci(txt, 0, win, "<html" as *u8) >= 0 { return 1 }
254 if dpr_find_ci(txt, 0, win, "<body" as *u8) >= 0 { return 1 }
255 return 0
256}
257
258// ---- THE ANSWER: where does real content begin, and what is the title? ----------------------------
259// Steps past the HTTP header block (protocol-exact), then past a LEADING RUN of blank / header-shaped /
260// markup-only lines. Returns that offset. Every consumer of stored document text wants this: the SERP
261// title, the SERP snippet fallback, the /doc view, and any future re-ingest.
262func dpr_content_start(txt: *u8, n: i64) -> i64 {
263 dpr_load_conf()
264 dpr_last_http_skipped_g = 0
265 dpr_last_lines_skipped_g = 0
266 dpr_last_budget_hit_g = 0
267 dpr_last_degraded_g = 0
268 if n <= 0 { return 0 }
269 var off: i64 = dpr_http_body(txt, n)
270 dpr_last_http_skipped_g = off
271 var budget: i64 = off + dpr_title_scan_bytes_g
272 if budget > n { budget = n }
273 var go: i64 = 1
274 while go == 1 {
275 if off >= n { go = 0 } else {
276 if off >= budget { dpr_last_budget_hit_g = 1; go = 0 } else {
277 let e: i64 = dpr_line_end(txt, n, off)
278 let vis: i64 = tc_skip_ws(txt, e, off)
279 var drop: i64 = 0
280 if vis >= e { drop = 1 }
281 if drop == 0 { if dpr_is_header_line(txt, n, vis, e) == 1 { drop = 1 } }
282 if drop == 0 { if dpr_is_markup_line(txt, n, vis, e) == 1 { drop = 1 } }
283 if drop == 1 {
284 dpr_last_lines_skipped_g = dpr_last_lines_skipped_g + 1
285 off = e + 1
286 } else { go = 0 }
287 }
288 }
289 }
290 if off > n { off = n }
291 return off
292}
293
294// THE TITLE. offout/lenout describe a span of txt that is a real first line of content, cut on a token
295// boundary at `cap`. Returns 1 when a titled line was found, 0 when none was (caller falls back and
296// SAYS SO -- a fabricated title is worse than an honest one).
297// exactbox[0] is tc_cut's contract: 0 means the line was one unbroken token longer than cap.
298func dpr_title(txt: *u8, n: i64, cap: i64, offout: *i64, lenout: *i64, exactbox: *i64) -> i64 {
299 dpr_load_conf()
300 offout[0] = 0
301 lenout[0] = 0
302 exactbox[0] = 1
303 if n <= 0 { return 0 }
304 var off: i64 = dpr_content_start(txt, n)
305 var budget: i64 = off + dpr_title_scan_bytes_g
306 if budget > n { budget = n }
307 var go: i64 = 1
308 while go == 1 {
309 if off >= n { go = 0 } else {
310 if off >= budget { dpr_last_budget_hit_g = 1; go = 0 } else {
311 let e: i64 = dpr_line_end(txt, n, off)
312 let vis: i64 = tc_skip_ws(txt, e, off)
313 if vis < e {
314 if dpr_max_wordrun(txt, vis, e) >= dpr_title_min_wordrun_g {
315 let span: i64 = e - vis
316 let l: i64 = tc_cut_trim(((txt as i64) + vis) as *u8, span, cap, exactbox)
317 if l > 0 {
318 offout[0] = vis
319 lenout[0] = l
320 return 1
321 }
322 }
323 }
324 off = e + 1
325 }
326 }
327 }
328 return 0
329}