nx_doc_lexstat.nx source

1// nx_doc_lexstat.nx -- MEASUREMENT PROBE for the keyword-stuffing penalty design (2026-08-04, debt 2// 1785895889): for each <cid> argument, walk the stored doc EXACTLY as serve pass-1 does (same 3// tokenizer, same DSS_TFSCAN cap) and print the statistics the penalty would ride on: token count, 4// per-term tf for the probe terms, and max-term density (tf*1024/tokens). Read-only; no ranking change. 5// usage: nx_doc_lexstat <domain> <term1> <term2> <cid>... 6// license_tier: ORIGINAL 7import "nx_docportal_search_seg.nx" 8 9func lx_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 10func lx_num(v: i64) -> i64 { 11 let bb: *u8 = sys_mmap(28); var m: i64 = v 12 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 13 let t: *u8 = sys_mmap(28); var k: i64 = 0 14 if m == 0 { t[0] = 48 as u8; k = 1 } 15 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 16 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 17 sys_write(1, bb, k); return 0 18} 19func lx_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v } 20 21func main(argc: i64, argv: *i64) -> i64 { 22 if argc < 5 { lx_puts("usage: nx_doc_lexstat <domain> <term1> <term2> <cid>...\n" as *u8); return 1 } 23 let domain: *u8 = argv[1] as *u8 24 let prefix: *u8 = sys_mmap(512) 25 dss_prefix(domain, prefix) 26 let h: *i64 = ss_open(prefix) 27 if (h as i64) == 0 { lx_puts("FAIL: shard absent\n" as *u8); return 2 } 28 let tbl: *u8 = sys_mmap(272) 29 ss_tok_table(tbl) 30 let termptrs: *i64 = sys_mmap(2 * 8) as *i64 31 termptrs[0] = argv[2] 32 termptrs[1] = argv[3] 33 let tfout: *i64 = sys_mmap(2 * 8) as *i64 34 let dkey: *u8 = sys_mmap(64) 35 let dp: *i64 = sys_mmap(16) as *i64 36 let dl: *i64 = sys_mmap(16) as *i64 37 var a: i64 = 4 38 while a < argc { 39 let cid: i64 = lx_atoi(argv[a] as *u8) 40 dss_mkkey(cid, dkey) 41 if ss_hget(h, dkey, dp, dl) == 1 { 42 var dn: i64 = dl[0] 43 if dn > DSS_TFSCAN { dn = DSS_TFSCAN } 44 let toks: i64 = dss_tf_all(dp[0] as *u8, dn, termptrs, 2, tbl, tfout) 45 var maxtf: i64 = tfout[0] 46 if tfout[1] > maxtf { maxtf = tfout[1] } 47 var dens: i64 = 0 48 if toks > 0 { dens = (maxtf * DSS_MAGIC_1024) / toks } 49 lx_puts("cid=" as *u8); lx_num(cid) 50 lx_puts(" bytes=" as *u8); lx_num(dl[0]) 51 lx_puts(" scanned=" as *u8); lx_num(dn) 52 lx_puts(" tokens=" as *u8); lx_num(toks) 53 lx_puts(" tf1=" as *u8); lx_num(tfout[0]) 54 lx_puts(" tf2=" as *u8); lx_num(tfout[1]) 55 lx_puts(" maxdens_q10=" as *u8); lx_num(dens) 56 // R2d guard trace: the live dcnt/bign the serve path would use + the factor it computes 57 let bign: i64 = ss_doc_count(h) 58 let dc1: i64 = ss_term_dcount(h, termptrs[0] as *u8) 59 let dc2: i64 = ss_term_dcount(h, termptrs[1] as *u8) 60 var maxdc: i64 = dc1 61 var usetf: i64 = tfout[0] 62 if tfout[1] > usetf { usetf = tfout[1]; maxdc = dc2 } 63 lx_puts(" bign=" as *u8); lx_num(bign) 64 lx_puts(" dcnt1=" as *u8); lx_num(dc1) 65 lx_puts(" dcnt2=" as *u8); lx_num(dc2) 66 lx_puts(" factor=" as *u8); lx_num(dss_stuff_factor(usetf, toks, maxdc, bign)) 67 lx_puts("\n" as *u8) 68 } else { 69 lx_puts("cid=" as *u8); lx_num(cid); lx_puts(" ABSENT\n" as *u8) 70 } 71 a = a + 1 72 } 73 return 0 74}