nx_bm25f_gate.nx source

1// nx_bm25f_gate.nx -- MEASURED proof that BM25F (fielded) beats plain BM25 on the property that matters for 2// onsite search: a query term in the TITLE should win over the same term buried in the BODY. Two docs: 3// doc0 title="estate planning services" body="we help utah families with care and guidance" (estate in TITLE) 4// doc1 title="family law attorney" body="we also handle estate matters for clients today" (estate in BODY) 5// Oracle (all checked from the live scorers, not asserted): 6// R1 BM25F: query 'estate' -> doc0 (TITLE hit ranks #1) 7// R2 plain BM25 body-only: query 'estate' -> doc1 (BODY hit) -- so BM25F FLIPPED the ranking toward the title 8// R3 BM25F: query 'family' -> doc1 (its TITLE) 9// R4 plain BM25 body-only: query 'family' -> -1 (NO body contains it) -- BM25F adds TITLE-FIELD RECALL the 10// body-only ranker misses entirely 11// GREEN iff 4/4. Durable knowledge/status/bm25f_gate.log. license_tier: ORIGINAL 12import "nx_bm25f.nx" // bmf_best + bm_best + bm_token_count (via nx_bm25) 13import "nx_syscalls.nx" 14 15func f_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 16func f_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } 17func f_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 18func f_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 } 19func f_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 20 21func main() -> i64 { 22 f_puts("=== BM25F GATE (fielded: TITLE hit beats BODY hit; vs plain BM25; MEASURED) ===\n" as *u8) 23 let N: i64 = 2 24 let titles: *i64=sys_mmap(8*N) as *i64 25 let bodies: *i64=sys_mmap(8*N) as *i64 26 titles[0]="estate planning services" as *u8 as i64 27 bodies[0]="we help utah families with care and guidance" as *u8 as i64 28 titles[1]="family law attorney" as *u8 as i64 29 bodies[1]="we also handle estate matters for clients today" as *u8 as i64 30 31 let tlens: *i64=sys_mmap(8*N) as *i64 32 let blens: *i64=sys_mmap(8*N) as *i64 33 let dlt: *i64=sys_mmap(8*N) as *i64 34 let dlb: *i64=sys_mmap(8*N) as *i64 35 var tt: i64=0; var tb: i64=0; var i: i64=0 36 while i<N { 37 tlens[i]=f_strlen(titles[i] as *u8); blens[i]=f_strlen(bodies[i] as *u8) 38 dlt[i]=bm_token_count(titles[i] as *u8, tlens[i]); dlb[i]=bm_token_count(bodies[i] as *u8, blens[i]) 39 tt=tt+dlt[i]; tb=tb+dlb[i]; i=i+1 40 } 41 var avgt: i64=tt/N; if avgt<=0 { avgt=1 } 42 var avgb: i64=tb/N; if avgb<=0 { avgb=1 } 43 44 let q_e: *i64=sys_mmap(8) as *i64; q_e[0]="estate" as *u8 as i64 45 let q_f: *i64=sys_mmap(8) as *i64; q_f[0]="family" as *u8 as i64 46 47 let bmf_e: i64=bmf_best(titles, tlens, dlt, bodies, blens, dlb, N, avgt, avgb, q_e, 1) 48 let bm_e: i64=bm_best(bodies, blens, dlb, N, avgb, q_e, 1) 49 let bmf_f: i64=bmf_best(titles, tlens, dlt, bodies, blens, dlb, N, avgt, avgb, q_f, 1) 50 let bm_f: i64=bm_best(bodies, blens, dlb, N, avgb, q_f, 1) 51 52 var pass: i64=0 53 var rows: i64=4 // var, not let: the mutation-derived teeth below add rows at the end 54 f_puts(" R1 BM25F 'estate' -> doc " as *u8); f_num(bmf_e); f_puts(" (expect 0=TITLE)" as *u8) 55 if bmf_e==0 { pass=pass+1; f_puts(" PASS\n" as *u8) } else { f_puts(" FAIL\n" as *u8) } 56 f_puts(" R2 BM25(body) 'estate' -> doc " as *u8); f_num(bm_e); f_puts(" (expect 1=BODY; BM25F flipped it)" as *u8) 57 if bm_e==1 { pass=pass+1; f_puts(" PASS\n" as *u8) } else { f_puts(" FAIL\n" as *u8) } 58 f_puts(" R3 BM25F 'family' -> doc " as *u8); f_num(bmf_f); f_puts(" (expect 1=TITLE)" as *u8) 59 if bmf_f==1 { pass=pass+1; f_puts(" PASS\n" as *u8) } else { f_puts(" FAIL\n" as *u8) } 60 f_puts(" R4 BM25(body) 'family' -> doc " as *u8); f_num(bm_f); f_puts(" (expect -1=body MISSES it; BM25F adds title recall)" as *u8) 61 if bm_f==(0-1) { pass=pass+1; f_puts(" PASS\n" as *u8) } else { f_puts(" FAIL\n" as *u8) } 62 63 f_puts("----\nBM25F rows=" as *u8); f_num(rows); f_puts(" pass=" as *u8); f_num(pass); f_puts("\n" as *u8) 64 let lg: i64=sys_openat_append("knowledge/status/bm25f_gate.log" as *u8, 0x1a4) 65 if lg>=0 { 66 f_w(lg, "BM25F rows=" as *u8); f_wn(lg, rows); f_w(lg, " pass=" as *u8); f_wn(lg, pass) 67 if pass==rows { f_w(lg, " verdict=GREEN\n" as *u8) } else { f_w(lg, " verdict=RED\n" as *u8) } 68 sys_close(lg) 69 } 70 // ---- MUTATION-DERIVED TEETH (2026-08-01, debt 1785604588) ---- 71 // nx_gate_mutation_probe scored this pair 1/4 with THREE survivors, all inside BM25F's DEFINING 72 // mechanism: nx_bm25f.nx:43 the title half of df counting, :61 the TITLE+BODY field combination, 73 // :72 the title-match bonus that is the whole reason BM25F exists rather than plain BM25. 74 // The four existing rows use ONE corpus in which the two documents differ in many ways at once 75 // (different titles AND different bodies AND different lengths), so a ranking stayed correct for 76 // reasons unrelated to the mutated line. 77 // LAW: A TEST WHOSE TWO CASES DIFFER IN MORE THAN ONE WAY CANNOT ATTRIBUTE THE RESULT TO ANY OF 78 // THEM. To measure the title bonus you need two documents identical in every respect EXCEPT which 79 // field carries the term -- a controlled pair, not a realistic one. 80 let M: i64 = 2 81 let ct: *i64=sys_mmap(8*M) as *i64 82 let cb: *i64=sys_mmap(8*M) as *i64 83 // doc0: term in the TITLE. doc1: the SAME two strings with the fields SWAPPED. Identical token 84 // counts, identical field lengths, identical corpus df -- the ONLY difference is placement. 85 ct[0]="widget" as *u8 as i64; cb[0]="alpha beta" as *u8 as i64 86 ct[1]="alpha beta" as *u8 as i64; cb[1]="widget" as *u8 as i64 87 let ctl: *i64=sys_mmap(8*M) as *i64 88 let cbl: *i64=sys_mmap(8*M) as *i64 89 let cdt: *i64=sys_mmap(8*M) as *i64 90 let cdb: *i64=sys_mmap(8*M) as *i64 91 var ci: i64=0; var st: i64=0; var sb: i64=0 92 while ci<M { 93 ctl[ci]=f_strlen(ct[ci] as *u8); cbl[ci]=f_strlen(cb[ci] as *u8) 94 cdt[ci]=bm_token_count(ct[ci] as *u8, ctl[ci]); cdb[ci]=bm_token_count(cb[ci] as *u8, cbl[ci]) 95 st=st+cdt[ci]; sb=sb+cdb[ci]; ci=ci+1 96 } 97 var cavgt: i64=st/M; if cavgt<=0 { cavgt=1 } 98 var cavgb: i64=sb/M; if cavgb<=0 { cavgb=1 } 99 let q_w: *i64=sys_mmap(8) as *i64; q_w[0]="widget" as *u8 as i64 100 let win: i64 = bmf_best(ct, ctl, cdt, cb, cbl, cdb, M, cavgt, cavgb, q_w, 1) 101 rows = rows + 1 102 if win == 0 { pass = pass + 1; f_puts(" T5 PASS TITLE-OVER-BODY: fields swapped, all else identical -> the TITLE doc wins\n" as *u8) } 103 if win != 0 { f_puts(" T5 FAIL TITLE-OVER-BODY: a title hit did NOT outrank an identical body hit\n" as *u8) } 104 // Both fields must CONTRIBUTE: a doc carrying the term in title AND body must outscore the 105 // title-only doc. Kills a combination that drops or subtracts one field's contribution. 106 let bt: *i64=sys_mmap(8*M) as *i64 107 let bb2: *i64=sys_mmap(8*M) as *i64 108 bt[0]="widget" as *u8 as i64; bb2[0]="widget" as *u8 as i64 // both fields 109 bt[1]="widget" as *u8 as i64; bb2[1]="alpha beta" as *u8 as i64 // title only 110 var bi: i64=0; var bt_s: i64=0; var bb_s: i64=0 111 let btl: *i64=sys_mmap(8*M) as *i64 112 let bbl: *i64=sys_mmap(8*M) as *i64 113 let bdt: *i64=sys_mmap(8*M) as *i64 114 let bdb: *i64=sys_mmap(8*M) as *i64 115 while bi<M { 116 btl[bi]=f_strlen(bt[bi] as *u8); bbl[bi]=f_strlen(bb2[bi] as *u8) 117 bdt[bi]=bm_token_count(bt[bi] as *u8, btl[bi]); bdb[bi]=bm_token_count(bb2[bi] as *u8, bbl[bi]) 118 bt_s=bt_s+bdt[bi]; bb_s=bb_s+bdb[bi]; bi=bi+1 119 } 120 var bavgt: i64=bt_s/M; if bavgt<=0 { bavgt=1 } 121 var bavgb: i64=bb_s/M; if bavgb<=0 { bavgb=1 } 122 let win2: i64 = bmf_best(bt, btl, bdt, bb2, bbl, bdb, M, bavgt, bavgb, q_w, 1) 123 rows = rows + 1 124 if win2 == 0 { pass = pass + 1; f_puts(" T6 PASS BOTH-FIELDS: title+body hit outranks title-only -> both fields contribute\n" as *u8) } 125 if win2 != 0 { f_puts(" T6 FAIL BOTH-FIELDS: the body field contributed nothing (or was subtracted)\n" as *u8) } 126 127 if pass==rows { f_puts("BM25F-GATE GREEN (fielded ranking beats plain BM25 on title relevance)\n" as *u8); sys_exit(0); return 0 } 128 f_puts("BM25F-GATE RED\n" as *u8); sys_exit(1); return 1 129}