nx_doc_lexstat.nx source

1// nx_doc_lexstat.nx -- MEASUREMENT PROBE for the keyword-stuffing penalty design (2026-08-04, debt 2// 1785895889): for each <cid> argument, walk the stored doc EXACTLY as serve pass-1 does (same 3// tokenizer, same DSS_TFSCAN cap) and print the statistics the penalty would ride on: token count, 4// per-term tf for the probe terms, and max-term density (tf*1024/tokens). Read-only; no ranking change. 5// usage: nx_doc_lexstat <domain> <term1> <term2> <cid>... 6// license_tier: ORIGINAL 7import "nx_docportal_search_seg.nx" 8// ARM THE CRASH GUARD (2026-08-15). Without this the probe dies on SIGSEGV with ZERO output and the 9// caller sees an empty body -- indistinguishable from a dropped transport response, which is exactly 10// how this fault stayed unattributable for a day. nx_crash.nx turns the signal into a printed fault 11// address, and with a --debug build nx_addr2line turns that address into file:line. 12import "nx_crash.nx" 13 14func lx_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 15func lx_num(v: i64) -> i64 { 16 let bb: *u8 = sys_mmap(28); var m: i64 = v 17 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 18 let t: *u8 = sys_mmap(28); var k: i64 = 0 19 if m == 0 { t[0] = 48 as u8; k = 1 } 20 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 21 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 22 sys_write(1, bb, k); return 0 23} 24func lx_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v } 25 26func main(argc: i64, argv: *i64) -> i64 { 27 nx_crash_guard() // explicit: the IR-level auto-inject only fires when the driver armed the switch 28 if argc < 5 { lx_puts("usage: nx_doc_lexstat <domain> <term1> <term2> <cid>...\n" as *u8); return 1 } 29 let domain: *u8 = argv[1] as *u8 30 let prefix: *u8 = sys_mmap(512) 31 dss_prefix(domain, prefix) 32 // MUST BE ss_open2(prefix, 1), NOT ss_open (2026-08-15). ss_open is ss_open2(prefix, 0) = READ-ALL 33 // into anonymous memory. The live web shard's largest segment alone is 1.58 GB .docs + 490 MB .idx + 34 // 376 MB .pos, and there are 14 segments -- so read-all asks for multiple GB in a short-lived CLI and 35 // dies. MEASURED: this probe faulted on EVERY web cid (existing or absent) while `site` and `trusted` 36 // returned normally and live /search served 200 throughout, because the daemon opens via 37 // ss_open_cached on the mmap-backed path. nx_seg_store's own comment says it: "the big web shard uses 38 // this so the ~1.25M-doc RAM ceiling becomes disk-bound". 39 // ★★★★★★A PROBE THAT OPENS ITS SUBJECT DIFFERENTLY FROM THE SERVE PATH IS NOT MEASURING THE SERVE 40 // PATH -- and this file's own comment claimed it walked "the identical code path" while calling a 41 // different opener. The handle layout is identical either way, so every ss_hget/ss_term reader below 42 // is unchanged; only the backing differs. 43 // OPT-IN OPEN TRACE: this probe is the sanctioned place to measure the shard open, because it runs 44 // the SAME ss_open2 the daemon runs but in a CLI where a 10s cost bothers nobody. Production never 45 // sets this, so the trace costs one integer compare per phase there. 46 ss_open_trace(1) 47 let h: *i64 = ss_open2(prefix, 1) 48 if (h as i64) == 0 { lx_puts("FAIL: shard absent\n" as *u8); return 2 } 49 let tbl: *u8 = sys_mmap(272) 50 ss_tok_table(tbl) 51 let termptrs: *i64 = sys_mmap(2 * 8) as *i64 52 termptrs[0] = argv[2] 53 termptrs[1] = argv[3] 54 let tfout: *i64 = sys_mmap(2 * 8) as *i64 55 let dkey: *u8 = sys_mmap(64) 56 let dp: *i64 = sys_mmap(16) as *i64 57 let dl: *i64 = sys_mmap(16) as *i64 58 var a: i64 = 4 59 while a < argc { 60 let cid: i64 = lx_atoi(argv[a] as *u8) 61 dss_mkkey(cid, dkey) 62 if ss_hget(h, dkey, dp, dl) == 1 { 63 var dn: i64 = dl[0] 64 if dn > DSS_TFSCAN { dn = DSS_TFSCAN } 65 lx_puts("T0" as *u8) 66 let toks: i64 = dss_tf_all(dp[0] as *u8, dn, termptrs, 2, tbl, tfout) 67 lx_puts("T1" as *u8) 68 // PROXIMITY COVERAGE (2026-08-14). dss_prox_all is the OTHER per-candidate walk the serve 69 // path runs (nx_docportal_search_seg.nx:1679, immediately after the tf walk) and it had NO 70 // direct caller outside the daemon -- so it could only ever be exercised by deploying to the 71 // live search surface. That is exactly how it took /search down: a blind hoist of its seven 72 // per-call buffers faulted out-of-bounds on every query and had to be rolled back, and 73 // because it could not be tested in isolation the fault was not attributable. 74 // Called with the SAME arguments the serve path uses -- same dn (DSS_TFSCAN-capped), same 75 // termptrs, same tokenizer table -- so this probe walks the identical code path. 76 // EXTENDS THE EXISTING PROBE rather than adding a second one: nx_doc_lexstat already opens 77 // the shard, resolves cids and loops, and a parallel organ would be a duplicate ruler. 78 // R1i-b: no expansion in this probe, so onterms == nterms == 2 and the call is unchanged. 79 let prox: i64 = dss_prox_all(dp[0] as *u8, dn, termptrs, 2, tbl) 80 var maxtf: i64 = tfout[0] 81 if tfout[1] > maxtf { maxtf = tfout[1] } 82 var dens: i64 = 0 83 if toks > 0 { dens = (maxtf * DSS_MAGIC_1024) / toks } 84 lx_puts("cid=" as *u8); lx_num(cid) 85 lx_puts(" bytes=" as *u8); lx_num(dl[0]) 86 lx_puts(" scanned=" as *u8); lx_num(dn) 87 lx_puts(" tokens=" as *u8); lx_num(toks) 88 lx_puts(" tf1=" as *u8); lx_num(tfout[0]) 89 lx_puts(" tf2=" as *u8); lx_num(tfout[1]) 90 lx_puts(" maxdens_q10=" as *u8); lx_num(dens) 91 lx_puts(" prox=" as *u8); lx_num(prox) 92 // R2d guard trace: the live dcnt/bign the serve path would use + the factor it computes 93 let bign: i64 = ss_doc_count(h) 94 let dc1: i64 = ss_term_dcount(h, termptrs[0] as *u8) 95 let dc2: i64 = ss_term_dcount(h, termptrs[1] as *u8) 96 var maxdc: i64 = dc1 97 var usetf: i64 = tfout[0] 98 if tfout[1] > usetf { usetf = tfout[1]; maxdc = dc2 } 99 lx_puts(" bign=" as *u8); lx_num(bign) 100 lx_puts(" dcnt1=" as *u8); lx_num(dc1) 101 lx_puts(" dcnt2=" as *u8); lx_num(dc2) 102 lx_puts(" factor=" as *u8); lx_num(dss_stuff_factor(usetf, toks, maxdc, bign)) 103 lx_puts("\n" as *u8) 104 } else { 105 lx_puts("cid=" as *u8); lx_num(cid); lx_puts(" ABSENT\n" as *u8) 106 } 107 a = a + 1 108 } 109 return 0 110}