nx_drbench.nx source

1// nx_drbench.nx -- HotpotQA distractor val[0..99]: the FIRST public-benchmark measurement of the Nishi 2// researcher (census axis [BEN]). Protocol = published distractor eval (10 paras: 2 gold + 8 distractors); 3// retrieve + extract; scored by SQuAD word-F1/EM (nx_qa_score_lib, integer permille) + gold-para recall@2. 4// MODES: mode0 single-hop lexical (IDF-weighted para retrieval + span-heuristic extraction); mode1 mechanical 5// multi-hop (hop-1 best para/sentence -> bridge entities -> expanded second retrieval). TEETH: T1 parse 6// integrity, T2 gold-title resolution, T3 rotated-gold NEG-CONTROL (~0), T4 retrieval floor. 7// MEASURED 2026-07-07 (100-row p0): MODE0 F1=172 EM=13 r@2=640 / MODE1 F1=155 EM=11 r@2=660 -- the honest 8// mechanical ceiling (bridge-not-answer-target failure mode; the climb is census R1/R3). 9// DEBT EATEN 2026-07-07: the ~1100-line engine that lived here moved to nx_qabench_engine.nx (shared with 10// nx_qabench's multi-benchmark ladder) -- this file is now IMPORT + main only, zero duplication, one fix point. 11// ctx slots + engine docs live in nx_qabench_engine.nx. expect_exit: 0 license_tier: ORIGINAL 12import "nx_qabench_engine.nx" 13const K_MAGIC_1024: i64 = 1024 14const K_MAGIC_1572864: i64 = 1572864 15const K_MAGIC_4096: i64 = 4096 16const K_MAGIC_2048: i64 = 2048 17const K_MAGIC_8192: i64 = 8192 18const K_MAGIC_262144: i64 = 262144 19const K_MAGIC_131072: i64 = 131072 20const K_MAGIC_32768: i64 = 32768 21const K_MAGIC_4000: i64 = 4000 22const K_MAGIC_2000: i64 = 2000 23const K_MAGIC_8000: i64 = 8000 24const K_MAGIC_1000000: i64 = 1000000 25 26func main() -> i64 { 27 db_w("=== nx_drbench -- HotpotQA distractor val[0..99]: FIRST public-benchmark measurement of the Nishi researcher ===\n" as *u8) 28 let g: *i64 = sys_mmap(K_MAGIC_1024) as *i64 29 let m0: *u8 = sys_mmap(K_MAGIC_1572864); g[0] = m0 as i64 30 let m4: *u8 = sys_mmap(K_MAGIC_4096); g[4] = m4 as i64 31 let m5: *u8 = sys_mmap(K_MAGIC_2048); g[5] = m5 as i64 32 let m6: *u8 = sys_mmap(64); g[6] = m6 as i64 33 let m7: *u8 = sys_mmap(K_MAGIC_4096); g[7] = m7 as i64 34 let m8: *u8 = sys_mmap(256); g[8] = m8 as i64 35 let m9: *u8 = sys_mmap(256); g[9] = m9 as i64 36 let m11: *u8 = sys_mmap(K_MAGIC_8192); g[11] = m11 as i64 37 let m12: *u8 = sys_mmap(256); g[12] = m12 as i64 38 let m13: *u8 = sys_mmap(256); g[13] = m13 as i64 39 let m15: *u8 = sys_mmap(K_MAGIC_262144); g[15] = m15 as i64 40 let m16: *u8 = sys_mmap(K_MAGIC_8192); g[16] = m16 as i64 41 let m17: *u8 = sys_mmap(K_MAGIC_8192); g[17] = m17 as i64 42 let m18: *u8 = sys_mmap(K_MAGIC_8192); g[18] = m18 as i64 43 let m20: *u8 = sys_mmap(K_MAGIC_262144); g[20] = m20 as i64 44 let m21: *u8 = sys_mmap(K_MAGIC_8192); g[21] = m21 as i64 45 let m22: *u8 = sys_mmap(K_MAGIC_8192); g[22] = m22 as i64 46 let m23: *u8 = sys_mmap(K_MAGIC_131072); g[23] = m23 as i64 47 let m24: *u8 = sys_mmap(K_MAGIC_131072); g[24] = m24 as i64 48 let m25: *u8 = sys_mmap(K_MAGIC_8192); g[25] = m25 as i64 49 let m26: *u8 = sys_mmap(K_MAGIC_8192); g[26] = m26 as i64 50 let m28: *u8 = sys_mmap(K_MAGIC_4096); g[28] = m28 as i64 51 let m29: *u8 = sys_mmap(K_MAGIC_2048); g[29] = m29 as i64 52 let m30: *u8 = sys_mmap(K_MAGIC_2048); g[30] = m30 as i64 53 let m32: *u8 = sys_mmap(K_MAGIC_2048); g[32] = m32 as i64 54 let m33: *u8 = sys_mmap(K_MAGIC_8192); g[33] = m33 as i64 55 let m34: *u8 = sys_mmap(K_MAGIC_8192); g[34] = m34 as i64 56 let m35: *u8 = sys_mmap(K_MAGIC_8192); g[35] = m35 as i64 57 let m36: *u8 = sys_mmap(256); g[36] = m36 as i64 58 let m37: *u8 = sys_mmap(256); g[37] = m37 as i64 59 let m39: *u8 = sys_mmap(K_MAGIC_8192); g[39] = m39 as i64 60 let m40: *u8 = sys_mmap(256); g[40] = m40 as i64 61 let m41: *u8 = sys_mmap(256); g[41] = m41 as i64 62 let m42: *u8 = sys_mmap(256); g[42] = m42 as i64 63 let m43: *u8 = sys_mmap(256); g[43] = m43 as i64 64 let m44: *u8 = sys_mmap(K_MAGIC_1024); g[44] = m44 as i64 65 let m45: *u8 = sys_mmap(128); g[45] = m45 as i64 66 let m46: *u8 = sys_mmap(128); g[46] = m46 as i64 67 let m48: *u8 = sys_mmap(K_MAGIC_2048); g[48] = m48 as i64 68 let m49: *u8 = sys_mmap(256); g[49] = m49 as i64 69 let m50: *u8 = sys_mmap(256); g[50] = m50 as i64 70 let m51: *u8 = sys_mmap(256); g[51] = m51 as i64 71 let m53: *u8 = sys_mmap(K_MAGIC_8192); g[53] = m53 as i64 72 let m54: *u8 = sys_mmap(K_MAGIC_32768); g[54] = m54 as i64 73 let m55: *u8 = sys_mmap(K_MAGIC_32768); g[55] = m55 as i64 74 let m56: *u8 = sys_mmap(K_MAGIC_32768); g[56] = m56 as i64 75 let m57: *u8 = sys_mmap(K_MAGIC_4096); g[57] = m57 as i64 // retrieval presence matrix H[p*128+k] 76 let m58: *u8 = sys_mmap(K_MAGIC_2048); g[58] = m58 as i64 // df[k] per content qterm 77 78 let total: i64 = db_read_raw(g, "knowledge/fetched/drb_hotpot_p0.raw" as *u8, K_MAGIC_1572864) 79 if total == 0-2 { db_w("RED -- drb_hotpot_p0.raw exceeds the 1.5MB raw buffer (raise m0)\n" as *u8); return 1 } 80 if total <= 0 { db_w("RED -- cannot read drb_hotpot_p0.raw (run nx_drbench_fetch first)\n" as *u8); return 1 } 81 let boff: i64 = db_body_start(g, total) 82 g[2] = boff 83 g[1] = total - boff 84 db_w("raw bytes="); db_n(total); db_w(" body="); db_n(g[1]); db_w("\n" as *u8) 85 86 let q: *u8 = g[4] as *u8 87 let a: *u8 = g[5] as *u8 88 let ty: *u8 = g[6] as *u8 89 let gtb: *u8 = g[7] as *u8 90 let gto: *i64 = g[8] as *i64 91 let gtl: *i64 = g[9] as *i64 92 let ptb: *u8 = g[11] as *u8 93 let pto: *i64 = g[12] as *i64 94 let ptl: *i64 = g[13] as *i64 95 let xsc: *i64 = g[43] as *i64 96 let psc: *i64 = g[42] as *i64 97 let shit: *i64 = g[39] as *i64 98 let p0a: *u8 = g[54] as *u8 99 let p1a: *u8 = g[55] as *u8 100 let gaa: *u8 = g[56] as *u8 101 102 var rows: i64 = 0 103 var parse_bad: i64 = 0 104 var gold_bad: i64 = 0 105 var gold_resolved: i64 = 0 106 var gold_total: i64 = 0 107 var sum_f1_0: i64 = 0; var em_0: i64 = 0; var sum_r2_0: i64 = 0 108 var sum_f1_1: i64 = 0; var em_1: i64 = 0; var sum_r2_1: i64 = 0 109 var br_f1_0: i64 = 0; var br_f1_1: i64 = 0; var br_n: i64 = 0 110 var cp_f1_0: i64 = 0; var cp_f1_1: i64 = 0; var cp_n: i64 = 0 111 112 g[3] = 0 113 var live: i64 = 1 114 while live == 1 { 115 if db_find_key(g, "question" as *u8) < 0 { live = 0 } else { 116 let ql: i64 = db_dec_str(g, q, K_MAGIC_4000) 117 db_find_key(g, "answer" as *u8) 118 let al: i64 = db_dec_str(g, a, K_MAGIC_2000) 119 db_find_key(g, "type" as *u8) 120 let tl: i64 = db_dec_str(g, ty, 60) 121 db_find_key(g, "supporting_facts" as *u8) 122 db_find_key(g, "title" as *u8) 123 let rawgt: i64 = db_parse_str_arr(g, gtb, K_MAGIC_4000, gto, gtl, 30) 124 // dedup gold titles in place 125 g[10] = 0 126 var gi: i64 = 0 127 while gi < rawgt { 128 var dup: i64 = 0 129 var gj: i64 = 0 130 while gj < g[10] { 131 if gtl[gi] == gtl[gj] { 132 var eq: i64 = 1 133 var w: i64 = 0 134 while w < gtl[gi] { if gtb[gto[gi]+w] != gtb[gto[gj]+w] { eq = 0 } w = w + 1 } 135 if eq == 1 { dup = 1 } 136 } 137 gj = gj + 1 138 } 139 if dup == 0 { let nd: i64 = g[10]; gto[nd] = gto[gi]; gtl[nd] = gtl[gi]; g[10] = nd + 1 } 140 gi = gi + 1 141 } 142 db_find_key(g, "context" as *u8) 143 db_find_key(g, "title" as *u8) 144 let np0: i64 = db_parse_str_arr(g, ptb, K_MAGIC_8000, pto, ptl, 16) 145 g[14] = np0 146 db_find_key(g, "sentences" as *u8) 147 let npara2: i64 = db_parse_sentences(g) 148 149 var okrow: i64 = 1 150 if ql <= 0 { okrow = 0 } 151 if al <= 0 { okrow = 0 } 152 if g[14] < 2 { okrow = 0 } // HotpotQA distractor is usually 10 paras but some comparison rows carry only the 2 gold paras -- valid 153 if npara2 != g[14] { okrow = 0 } 154 if g[19] <= 0 { okrow = 0 } 155 if g[10] < 1 { okrow = 0 } 156 if okrow == 0 { parse_bad = parse_bad + 1 } else { 157 158 // gold titles must resolve to a context para (byte-exact) 159 var gr: i64 = 0 160 var gsel: i64 = 0 161 while gsel < g[10] { 162 var found: i64 = 0 163 var p: i64 = 0 164 while p < g[14] { 165 if found == 0 { if gtl[gsel] == ptl[p] { 166 var eq2: i64 = 1 167 var w2: i64 = 0 168 while w2 < gtl[gsel] { if gtb[gto[gsel]+w2] != ptb[pto[p]+w2] { eq2 = 0 } w2 = w2 + 1 } 169 if eq2 == 1 { found = 1 } 170 } } 171 p = p + 1 172 } 173 gr = gr + found 174 gsel = gsel + 1 175 } 176 gold_resolved = gold_resolved + gr 177 gold_total = gold_total + g[10] 178 if gr < g[10] { gold_bad = gold_bad + 1 } 179 180 db_norm_row(g) 181 db_score_row(g) 182 183 // ---- mode 0: single-hop ---- 184 let top1: i64 = db_top_para(g, 0-1) 185 let top2: i64 = db_top_para(g, top1) 186 let r2a: i64 = db_recall2(g, top1, top2) 187 let s0: i64 = db_best_sentence(g) 188 let yn: i64 = db_is_yesno(g) 189 let wantnum0: i64 = db_prefer_numeric(g) 190 let pd0: *u8 = (p0a as i64 + rows*256) as *u8 191 if yn == 1 { db_setpred(pd0, "yes" as *u8, 3) } else { 192 var got0: i64 = 0 193 if wantnum0 == 1 { got0 = db_sweep_numeric(g, top1, top2, pd0) } 194 if got0 == 0 { if s0 >= 0 { db_extract(g, s0, pd0) } else { db_setpred(pd0, "none" as *u8, 4) } } 195 } 196 197 // ---- mode 1: multi-hop ---- 198 var p1: i64 = top1 199 var s1: i64 = s0 200 var x2: i64 = 0-1 201 if s1 >= 0 { if p1 >= 0 { 202 db_bridge_from(g, s1, p1) 203 var pp: i64 = 0 204 while pp < g[14] { 205 if pp == p1 { xsc[pp] = 0-1000 } else { 206 let bt: i64 = db_bridge_title_hits(g, pp) 207 let bb: i64 = db_bridge_body_hits(g, pp) 208 xsc[pp] = psc[pp] + 3*bt + 2*bb 209 } 210 pp = pp + 1 211 } 212 var bx2: i64 = 0-1 213 var bxs: i64 = 0-K_MAGIC_1000000 214 pp = 0 215 while pp < g[14] { if xsc[pp] > bxs { bxs = xsc[pp]; bx2 = pp } pp = pp + 1 } 216 x2 = bx2 217 } } 218 let r2b: i64 = db_recall2(g, p1, x2) 219 let pd1: *u8 = (p1a as i64 + rows*256) as *u8 220 if yn == 1 { db_setpred(pd1, "yes" as *u8, 3) } else { 221 var got1: i64 = 0 222 if wantnum0 == 1 { got1 = db_sweep_numeric(g, p1, x2, pd1) } 223 if got1 == 0 { 224 var s2: i64 = 0-1 225 if x2 >= 0 { s2 = db_best_sentence_in(g, x2) } 226 var sfin: i64 = s2 227 if sfin < 0 { sfin = s1 } 228 if s1 >= 0 { if s2 >= 0 { if shit[s1] > shit[s2] + 1 { sfin = s1 } } } 229 if sfin >= 0 { db_extract(g, sfin, pd1) } else { db_setpred(pd1, "none" as *u8, 4) } 230 } 231 } 232 233 // gold copy for scoring + neg-control 234 let gd: *u8 = (gaa as i64 + rows*256) as *u8 235 db_setpred(gd, a, al) 236 237 let f0: i64 = qs_f1(pd0, gd); let e0: i64 = qs_em(pd0, gd) 238 let f1v: i64 = qs_f1(pd1, gd); let e1: i64 = qs_em(pd1, gd) 239 sum_f1_0 = sum_f1_0 + f0; em_0 = em_0 + e0; sum_r2_0 = sum_r2_0 + r2a 240 sum_f1_1 = sum_f1_1 + f1v; em_1 = em_1 + e1; sum_r2_1 = sum_r2_1 + r2b 241 var isbr: i64 = 0 242 if ty[0]==(98 as u8) { isbr = 1 } 243 if isbr == 1 { br_f1_0 = br_f1_0 + f0; br_f1_1 = br_f1_1 + f1v; br_n = br_n + 1 } 244 else { cp_f1_0 = cp_f1_0 + f0; cp_f1_1 = cp_f1_1 + f1v; cp_n = cp_n + 1 } 245 246 // per-row trace (set DBG=1 to inspect extraction failure modes; off in production) 247 let DBG: i64 = 0 248 if DBG == 1 { if rows < 14 { 249 db_w(" [DBG] ty="); db_w(ty); db_w(" f0="); db_n(f0); db_w(" f1="); db_n(f1v) 250 db_w(" | Q="); db_w(q); db_w(" | P0="); db_w(pd0); db_w(" | P1="); db_w(pd1); db_w(" | G="); db_w(gd); db_w("\n" as *u8) 251 } } 252 253 rows = rows + 1 254 } 255 } 256 } 257 258 db_w("rows="); db_n(rows); db_w(" parse_bad="); db_n(parse_bad) 259 db_w(" gold_resolved="); db_n(gold_resolved); db_w("/"); db_n(gold_total); db_w("\n" as *u8) 260 261 // T3 neg-control: mode0 preds vs ROTATED golds 262 var neg_sum: i64 = 0 263 var r: i64 = 0 264 while r < rows { 265 let pd: *u8 = (p0a as i64 + r*256) as *u8 266 var r2x: i64 = r + 1 267 if r2x >= rows { r2x = 0 } 268 let gd2: *u8 = (gaa as i64 + r2x*256) as *u8 269 let nf: i64 = qs_f1(pd, gd2) 270 neg_sum = neg_sum + nf 271 r = r + 1 272 } 273 var neg_avg: i64 = 0 274 if rows > 0 { neg_avg = neg_sum / rows } 275 276 var avg_f1_0: i64 = 0; var avg_r2_0: i64 = 0; var avg_f1_1: i64 = 0; var avg_r2_1: i64 = 0 277 if rows > 0 { 278 avg_f1_0 = sum_f1_0/rows; avg_r2_0 = sum_r2_0/rows 279 avg_f1_1 = sum_f1_1/rows; avg_r2_1 = sum_r2_1/rows 280 } 281 db_w("MODE0 single-hop lexical: F1(permille)="); db_n(avg_f1_0); db_w(" EM="); db_n(em_0); db_w("/"); db_n(rows) 282 db_w(" para-recall@2(permille)="); db_n(avg_r2_0); db_w("\n" as *u8) 283 db_w("MODE1 multi-hop bridge: F1(permille)="); db_n(avg_f1_1); db_w(" EM="); db_n(em_1); db_w("/"); db_n(rows) 284 db_w(" para-recall@2(permille)="); db_n(avg_r2_1); db_w("\n" as *u8) 285 db_w("DELTA (hop - single): F1 "); db_n(avg_f1_1 - avg_f1_0); db_w(" recall@2 "); db_n(avg_r2_1 - avg_r2_0); db_w("\n" as *u8) 286 if br_n > 0 { db_w(" bridge-type ("); db_n(br_n); db_w("): F1 mode0="); db_n(br_f1_0/br_n); db_w(" mode1="); db_n(br_f1_1/br_n); db_w("\n" as *u8) } 287 if cp_n > 0 { db_w(" comparison-type ("); db_n(cp_n); db_w("): F1 mode0="); db_n(cp_f1_0/cp_n); db_w(" mode1="); db_n(cp_f1_1/cp_n); db_w("\n" as *u8) } 288 db_w("NEG-CONTROL rotated-gold avg F1="); db_n(neg_avg); db_w(" (expected near 0)\n" as *u8) 289 290 var pass: i64 = 0 291 if rows == 100 { if parse_bad == 0 { pass = pass + 1 } } 292 if gold_bad == 0 { if gold_total >= 180 { pass = pass + 1 } } 293 if neg_avg < 80 { pass = pass + 1 } 294 if avg_r2_0 >= 300 { pass = pass + 1 } 295 db_w("TEETH T1(parse)+T2(gold)+T3(negctl)+T4(floor) = "); db_n(pass); db_w("/4\n" as *u8) 296 if pass == 4 { 297 db_w("GREEN -- harness valid; numbers above are the HONEST measured baseline (census [BEN] axis: was NEVER-RUN, now MEASURED).\n" as *u8) 298 return 0 299 } 300 db_w("RED -- harness integrity failed\n" as *u8) 301 return 1 302}