nx_qabench_engine.nx source

1// nx_drbench.nx -- R2: the FIRST public-benchmark run of the Nishi researcher (census axis [BEN]: "MEASURED 2// scores on public benchmarks -- NEVER run"). Dataset = HotpotQA distractor validation rows 0..99 (banked by 3// nx_drbench_fetch -> knowledge/fetched/drb_hotpot_p0.raw, sovereign TLS). Protocol = the published distractor 4// eval: each question ships 10 paragraphs (2 gold + 8 distractors); systems retrieve + extract an answer; 5// scored by SQuAD word-F1/EM (nx_qa_score_lib, integer permille) + gold-paragraph recall at 2. 6// TWO MODES measured side by side: 7// mode0 SINGLE-HOP lexical baseline: question-term overlap retrieval -> span heuristic extraction. 8// mode1 MULTI-HOP (the DeepResearcher loop shape, mechanical): hop-1 best para/sentence -> harvest BRIDGE 9// entities (cap-runs + hop-1 title terms not in the question) -> expanded second retrieval -> extract. 10// The LOW numbers ARE the honest baseline (no LLM synth yet -- that is census rung R1). Teeth: 11// T1 parse integrity (100 rows, 10 paras, nonempty q/a) T2 gold titles parsed + resolvable to paragraphs 12// T3 NEG-CONTROL: predictions scored against the WRONG (rotated) golds must collapse to ~0 (metric not rigged) 13// T4 non-degenerate retrieval floor (recall at 2 must beat the ~200 permille random floor) 14// T5 report mode0 vs mode1 delta (measurement, not a forced win -- honest either way) 15// NOTE: all state lives in ONE ctx table g: *i64 passed to every fn (the proven rctx pattern) -- statics hit 16// the known multi-static subscript-read compiler bug (filed; PHASE-A/B bisect proof in the session log). 17// ctx slots: 0 raw 1 blen 2 boff 3 cur 4 q 5 a 6 ty 7 gtb 8 gto 9 gtl 10 ngt 11 ptb 12 pto 13 ptl 14 np 18// 15 sb 16 so 17 sl 18 sp 19 ns 20 nb 21 no 22 nl 23 tko 24 tkl 25 tks 26 tkc 27 ntk 28 qnb 29 qto 30 qtl 19// 31 nqt 32 qk 33 tnb 34 tto 35 ttl 36 tts 37 ttc 38 nttk 39 shit 40 thit 41 bhit 42 psc 43 xsc 20// 44 btb 45 bto 46 btl 47 nbt 48 cnd 49 cno 50 cnl 51 cnt 52 ncn 53 scr 54 p0 55 p1 56 ga 21// expect_exit: 0 license_tier: ORIGINAL 22import "nx_qa_score_lib.nx" 23import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 24const K_MAGIC_262144: i64 = 262144 25const K_MAGIC_260000: i64 = 260000 26const K_MAGIC_5381: i64 = 5381 27const K_MAGIC_77245: i64 = 77245 28const K_MAGIC_1073741789: i64 = 1073741789 29const K_MAGIC_1073741783: i64 = 1073741783 30const K_MAGIC_15000: i64 = 15000 31const K_MAGIC_2000: i64 = 2000 32const K_MAGIC_2048: i64 = 2048 33const K_MAGIC_4096: i64 = 4096 34const K_MAGIC_1000000: i64 = 1000000 35 36// ---------- print helpers ---------- 37func db_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 38// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 39// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 40// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 41// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 42func db_n(v: i64) -> i64 { nxi_out(v); return 0 } 43 44// ---------- file ---------- 45// BOUNDED read (debt eaten 2026-07-07: the old unbounded loop overflowed g[0] when a grown dataset exceeded the 46// buffer -> SIGSEGV; sizing the buffer was a workaround, THIS is the fix). Reads at most cap bytes; if the file 47// has MORE, returns -2 (caller reports "grow the buffer") instead of corrupting memory. Closes fd on all paths. 48func db_read_raw(g: *i64, path: *u8, cap: i64) -> i64 { 49 let raw: *u8 = g[0] as *u8 50 let fd: i64 = sys_openat_rd(path) 51 if fd < 0 { return 0-1 } 52 var total: i64 = 0 53 var r: i64 = 1 54 while r > 0 { 55 var want: i64 = K_MAGIC_262144 56 let left: i64 = cap - total 57 if left <= 0 { 58 // buffer exactly full: probe 1 byte to distinguish exact-fit from oversize 59 var pb: [8]u8 60 let pr: i64 = sys_read(fd, pb, 1) 61 sys_close(fd) 62 if pr > 0 { return 0-2 } 63 return total 64 } 65 if want > left { want = left } 66 r = sys_read(fd, (raw as i64 + total) as *u8, want) 67 if r > 0 { total = total + r } 68 } 69 sys_close(fd) 70 return total 71} 72 73// find body start (after CRLFCRLF) 74func db_body_start(g: *i64, total: i64) -> i64 { 75 let raw: *u8 = g[0] as *u8 76 var i: i64 = 0 77 while i < total-3 { 78 if raw[i]==(13 as u8) { if raw[i+1]==(10 as u8) { if raw[i+2]==(13 as u8) { if raw[i+3]==(10 as u8) { return i+4 } } } } 79 i = i + 1 80 } 81 return 0 82} 83 84func db_b(g: *i64, i: i64) -> i64 { 85 let raw: *u8 = g[0] as *u8 86 return raw[g[2] + i] as i64 87} 88 89// ---------- json scan ---------- 90// does body[pos..] match: QUOTE key QUOTE COLON ? return pos after colon, else -1 91func db_match_key(g: *i64, pos: i64, key: *u8) -> i64 { 92 if db_b(g,pos) != 34 { return 0-1 } 93 var k: i64 = 0 94 var p: i64 = pos + 1 95 while key[k] != (0 as u8) { 96 if p >= g[1] { return 0-1 } 97 if db_b(g,p) != (key[k] as i64) { return 0-1 } 98 p = p + 1; k = k + 1 99 } 100 if db_b(g,p) != 34 { return 0-1 } 101 p = p + 1 102 if db_b(g,p) != 58 { return 0-1 } 103 return p + 1 104} 105 106// scan forward from cur for "key": ; leaves cur after colon. returns 0 ok, -1 not found 107func db_find_key(g: *i64, key: *u8) -> i64 { 108 var i: i64 = g[3] 109 while i < g[1] { 110 if db_b(g,i) == 34 { 111 let m: i64 = db_match_key(g, i, key) 112 if m >= 0 { g[3] = m; return 0 } 113 } 114 i = i + 1 115 } 116 return 0-1 117} 118 119func db_skip_ws(g: *i64) -> i64 { 120 while g[3] < g[1] { 121 var c: i64 = db_b(g, g[3]) 122 var sp: i64 = 0 123 if c==32 { sp=1 } 124 if c==10 { sp=1 } 125 if c==13 { sp=1 } 126 if c==9 { sp=1 } 127 if sp==0 { return 0 } 128 g[3] = g[3] + 1 129 } 130 return 0 131} 132 133// decode a JSON string. cur must be AT the opening quote (after ws). writes out (cap), returns len; cur -> after closing quote. 134func db_dec_str(g: *i64, out: *u8, cap: i64) -> i64 { 135 db_skip_ws(g) 136 if db_b(g, g[3]) != 34 { return 0-1 } 137 g[3] = g[3] + 1 138 var j: i64 = 0 139 while g[3] < g[1] { 140 var c: i64 = db_b(g, g[3]) 141 if c == 34 { g[3] = g[3] + 1; out[j] = (0 as u8); return j } 142 if c == 92 { 143 g[3] = g[3] + 1 144 var e: i64 = db_b(g, g[3]) 145 var oc: i64 = e 146 if e == 110 { oc = 10 } 147 if e == 116 { oc = 9 } 148 if e == 114 { oc = 13 } 149 if e == 98 { oc = 32 } 150 if e == 102 { oc = 32 } 151 if e == 117 { oc = 32; g[3] = g[3] + 4 } 152 if j < cap-2 { out[j] = (oc as u8); j = j + 1 } 153 g[3] = g[3] + 1 154 } else { 155 if j < cap-2 { out[j] = (c as u8); j = j + 1 } 156 g[3] = g[3] + 1 157 } 158 } 159 out[j] = (0 as u8) 160 return j 161} 162 163// parse ["s","s",...] into arena+offs. cur at/before '['. returns count. 164func db_parse_str_arr(g: *i64, arena: *u8, acap: i64, offs: *i64, lens: *i64, maxn: i64) -> i64 { 165 db_skip_ws(g) 166 if db_b(g, g[3]) != 91 { return 0-1 } 167 g[3] = g[3] + 1 168 var n: i64 = 0 169 var bump: i64 = 0 170 while g[3] < g[1] { 171 db_skip_ws(g) 172 var c: i64 = db_b(g, g[3]) 173 if c == 93 { g[3] = g[3] + 1; return n } 174 if c == 44 { g[3] = g[3] + 1 } 175 db_skip_ws(g) 176 if db_b(g, g[3]) == 34 { 177 let dst: *u8 = (arena as i64 + bump) as *u8 178 let ln: i64 = db_dec_str(g, dst, acap - bump) 179 if ln < 0 { return 0-1 } 180 if n < maxn { offs[n] = bump; lens[n] = ln; n = n + 1; bump = bump + ln + 1 } 181 } else { return 0-1 } 182 } 183 return 0-1 184} 185 186// parse [[..],[..]] sentences. fills sb/so/sl/sp + ns (g[19]). returns npara. 187func db_parse_sentences(g: *i64) -> i64 { 188 let sb: *u8 = g[15] as *u8 189 let so: *i64 = g[16] as *i64 190 let sl: *i64 = g[17] as *i64 191 let sp: *i64 = g[18] as *i64 192 db_skip_ws(g) 193 if db_b(g, g[3]) != 91 { return 0-1 } 194 g[3] = g[3] + 1 195 var para: i64 = 0 196 var bump: i64 = 0 197 g[19] = 0 198 while g[3] < g[1] { 199 db_skip_ws(g) 200 var c: i64 = db_b(g, g[3]) 201 if c == 93 { g[3] = g[3] + 1; return para } 202 if c == 44 { g[3] = g[3] + 1; db_skip_ws(g); c = db_b(g, g[3]) } 203 if c == 91 { 204 g[3] = g[3] + 1 205 var live: i64 = 1 206 while live == 1 { 207 db_skip_ws(g) 208 var d: i64 = db_b(g, g[3]) 209 if d == 93 { g[3] = g[3] + 1; live = 0 } 210 if live == 1 { 211 if d == 44 { g[3] = g[3] + 1; db_skip_ws(g) } 212 if db_b(g, g[3]) == 34 { 213 let dst: *u8 = (sb as i64 + bump) as *u8 214 let ln: i64 = db_dec_str(g, dst, K_MAGIC_262144 - bump - 8) 215 if ln < 0 { return 0-1 } 216 if g[19] < 1000 { if bump + ln < K_MAGIC_260000 { 217 let ns0: i64 = g[19] 218 so[ns0] = bump; sl[ns0] = ln; sp[ns0] = para 219 g[19] = ns0 + 1; bump = bump + ln + 1 220 } } 221 } 222 } 223 } 224 para = para + 1 225 } 226 } 227 return 0-1 228} 229 230// ---------- shared semantic-hash (ONE source of truth: nx_semppmi_build writes ids with this, nx_qabench 231// resolves query words with it -- any drift silently zeroes all similarities) ---------- 232func db_semhash(buf: *u8, off: i64, len: i64) -> i64 { 233 var h1: i64 = K_MAGIC_5381 234 var h2: i64 = K_MAGIC_77245 235 var i: i64 = 0 236 while i < len { 237 let c: i64 = buf[off+i] as i64 238 h1 = (h1*33 + c) % K_MAGIC_1073741789 239 h2 = (h2*131 + c) % K_MAGIC_1073741783 240 i = i + 1 241 } 242 return h1 * K_MAGIC_1073741783 + h2 243} 244 245// binary search sorted i64 array; idx or -1 246func db_bsearch_i64(a: *i64, n: i64, v: i64) -> i64 { 247 var lo: i64 = 0 248 var hi: i64 = n - 1 249 while lo <= hi { 250 let mid: i64 = (lo + hi) / 2 251 if a[mid] == v { return mid } 252 if a[mid] < v { lo = mid + 1 } else { hi = mid - 1 } 253 } 254 return 0-1 255} 256 257// ---------- token utilities ---------- 258func db_tok_is(buf: *u8, off: i64, len: i64, lit: *u8) -> i64 { 259 var ll: i64 = 0 260 while lit[ll] != (0 as u8) { ll = ll + 1 } 261 if ll != len { return 0 } 262 var k: i64 = 0 263 while k < len { if buf[off+k] != lit[k] { return 0 } k = k + 1 } 264 return 1 265} 266 267func db_is_stop(buf: *u8, off: i64, len: i64) -> i64 { 268 if db_tok_is(buf,off,len,"is" as *u8)==1 { return 1 } 269 if db_tok_is(buf,off,len,"are" as *u8)==1 { return 1 } 270 if db_tok_is(buf,off,len,"was" as *u8)==1 { return 1 } 271 if db_tok_is(buf,off,len,"were" as *u8)==1 { return 1 } 272 if db_tok_is(buf,off,len,"be" as *u8)==1 { return 1 } 273 if db_tok_is(buf,off,len,"been" as *u8)==1 { return 1 } 274 if db_tok_is(buf,off,len,"do" as *u8)==1 { return 1 } 275 if db_tok_is(buf,off,len,"does" as *u8)==1 { return 1 } 276 if db_tok_is(buf,off,len,"did" as *u8)==1 { return 1 } 277 if db_tok_is(buf,off,len,"what" as *u8)==1 { return 1 } 278 if db_tok_is(buf,off,len,"which" as *u8)==1 { return 1 } 279 if db_tok_is(buf,off,len,"who" as *u8)==1 { return 1 } 280 if db_tok_is(buf,off,len,"whom" as *u8)==1 { return 1 } 281 if db_tok_is(buf,off,len,"whose" as *u8)==1 { return 1 } 282 if db_tok_is(buf,off,len,"when" as *u8)==1 { return 1 } 283 if db_tok_is(buf,off,len,"where" as *u8)==1 { return 1 } 284 if db_tok_is(buf,off,len,"why" as *u8)==1 { return 1 } 285 if db_tok_is(buf,off,len,"how" as *u8)==1 { return 1 } 286 if db_tok_is(buf,off,len,"of" as *u8)==1 { return 1 } 287 if db_tok_is(buf,off,len,"in" as *u8)==1 { return 1 } 288 if db_tok_is(buf,off,len,"on" as *u8)==1 { return 1 } 289 if db_tok_is(buf,off,len,"at" as *u8)==1 { return 1 } 290 if db_tok_is(buf,off,len,"by" as *u8)==1 { return 1 } 291 if db_tok_is(buf,off,len,"for" as *u8)==1 { return 1 } 292 if db_tok_is(buf,off,len,"with" as *u8)==1 { return 1 } 293 if db_tok_is(buf,off,len,"to" as *u8)==1 { return 1 } 294 if db_tok_is(buf,off,len,"from" as *u8)==1 { return 1 } 295 if db_tok_is(buf,off,len,"and" as *u8)==1 { return 1 } 296 if db_tok_is(buf,off,len,"or" as *u8)==1 { return 1 } 297 if db_tok_is(buf,off,len,"as" as *u8)==1 { return 1 } 298 if db_tok_is(buf,off,len,"that" as *u8)==1 { return 1 } 299 if db_tok_is(buf,off,len,"this" as *u8)==1 { return 1 } 300 if db_tok_is(buf,off,len,"it" as *u8)==1 { return 1 } 301 if db_tok_is(buf,off,len,"its" as *u8)==1 { return 1 } 302 if db_tok_is(buf,off,len,"he" as *u8)==1 { return 1 } 303 if db_tok_is(buf,off,len,"she" as *u8)==1 { return 1 } 304 if db_tok_is(buf,off,len,"they" as *u8)==1 { return 1 } 305 if db_tok_is(buf,off,len,"his" as *u8)==1 { return 1 } 306 if db_tok_is(buf,off,len,"her" as *u8)==1 { return 1 } 307 if db_tok_is(buf,off,len,"their" as *u8)==1 { return 1 } 308 if db_tok_is(buf,off,len,"both" as *u8)==1 { return 1 } 309 if db_tok_is(buf,off,len,"same" as *u8)==1 { return 1 } 310 if db_tok_is(buf,off,len,"other" as *u8)==1 { return 1 } 311 if db_tok_is(buf,off,len,"more" as *u8)==1 { return 1 } 312 if db_tok_is(buf,off,len,"most" as *u8)==1 { return 1 } 313 if db_tok_is(buf,off,len,"has" as *u8)==1 { return 1 } 314 if db_tok_is(buf,off,len,"have" as *u8)==1 { return 1 } 315 if db_tok_is(buf,off,len,"had" as *u8)==1 { return 1 } 316 if db_tok_is(buf,off,len,"also" as *u8)==1 { return 1 } 317 if db_tok_is(buf,off,len,"name" as *u8)==1 { return 1 } 318 if db_tok_is(buf,off,len,"named" as *u8)==1 { return 1 } 319 if db_tok_is(buf,off,len,"known" as *u8)==1 { return 1 } 320 if db_tok_is(buf,off,len,"first" as *u8)==1 { return 1 } 321 if db_tok_is(buf,off,len,"born" as *u8)==0 { return 0 } 322 return 1 323} 324 325// is token (buf,off,len) present in question tokens? contentonly 1 = content terms only 326func db_in_question(g: *i64, buf: *u8, off: i64, len: i64, contentonly: i64) -> i64 { 327 let qnb: *u8 = g[28] as *u8 328 let qto: *i64 = g[29] as *i64 329 let qtl: *i64 = g[30] as *i64 330 let qk: *i64 = g[32] as *i64 331 var k: i64 = 0 332 while k < g[31] { 333 var use: i64 = 1 334 if contentonly == 1 { if qk[k] == 0 { use = 0 } } 335 if use == 1 { 336 if qs_tok_eq2(buf, off, len, qnb, qto[k], qtl[k]) == 1 { return 1 } 337 } 338 k = k + 1 339 } 340 return 0 341} 342 343// ---------- per-row normalize + tokenize ---------- 344func db_norm_row(g: *i64) -> i64 { 345 let q: *u8 = g[4] as *u8 346 let qnb: *u8 = g[28] as *u8 347 let qto: *i64 = g[29] as *i64 348 let qtl: *i64 = g[30] as *i64 349 let qk: *i64 = g[32] as *i64 350 let sb: *u8 = g[15] as *u8 351 let so: *i64 = g[16] as *i64 352 let nb: *u8 = g[20] as *u8 353 let no2: *i64 = g[21] as *i64 354 let nl2: *i64 = g[22] as *i64 355 let tko: *i64 = g[23] as *i64 356 let tkl: *i64 = g[24] as *i64 357 let tks: *i64 = g[25] as *i64 358 let tkc: *i64 = g[26] as *i64 359 let ptb: *u8 = g[11] as *u8 360 let pto: *i64 = g[12] as *i64 361 let tnb: *u8 = g[33] as *u8 362 let tto: *i64 = g[34] as *i64 363 let ttl: *i64 = g[35] as *i64 364 let tts: *i64 = g[36] as *i64 365 let ttc: *i64 = g[37] as *i64 366 367 // question 368 let ql: i64 = qs_norm(q, qnb) 369 let nq: i64 = qs_tok(qnb, ql, qto, qtl, 120) 370 g[31] = nq 371 var k: i64 = 0 372 while k < nq { 373 var keep: i64 = 1 374 if db_is_stop(qnb, qto[k], qtl[k]) == 1 { keep = 0 } 375 qk[k] = keep 376 k = k + 1 377 } 378 // sentences 379 g[27] = 0 380 var bump: i64 = 0 381 var s: i64 = 0 382 while s < g[19] { 383 let src: *u8 = (sb as i64 + so[s]) as *u8 384 let dst: *u8 = (nb as i64 + bump) as *u8 385 let nl0: i64 = qs_norm(src, dst) 386 no2[s] = bump; nl2[s] = nl0 387 let t0: i64 = g[27] 388 let cnt: i64 = qs_tok(dst, nl0, (tko as i64 + t0*8) as *i64, (tkl as i64 + t0*8) as *i64, K_MAGIC_15000 - t0) 389 var j: i64 = 0 390 while j < cnt { tko[t0+j] = tko[t0+j] + bump; j = j + 1 } 391 tks[s] = t0; tkc[s] = cnt 392 g[27] = t0 + cnt 393 bump = bump + nl0 + 2 394 s = s + 1 395 } 396 // titles 397 g[38] = 0 398 var tb: i64 = 0 399 var p: i64 = 0 400 while p < g[14] { 401 let tsrc: *u8 = (ptb as i64 + pto[p]) as *u8 402 let tdst: *u8 = (tnb as i64 + tb) as *u8 403 let tl0: i64 = qs_norm(tsrc, tdst) 404 let s0: i64 = g[38] 405 let c2: i64 = qs_tok(tdst, tl0, (tto as i64 + s0*8) as *i64, (ttl as i64 + s0*8) as *i64, 1000 - s0) 406 var j2: i64 = 0 407 while j2 < c2 { tto[s0+j2] = tto[s0+j2] + tb; j2 = j2 + 1 } 408 tts[p] = s0; ttc[p] = c2 409 g[38] = s0 + c2 410 tb = tb + tl0 + 2 411 p = p + 1 412 } 413 return 0 414} 415 416// ---------- scoring ---------- 417func db_sent_hits(g: *i64, s: i64) -> i64 { 418 let nb: *u8 = g[20] as *u8 419 let tko: *i64 = g[23] as *i64 420 let tkl: *i64 = g[24] as *i64 421 let tks: *i64 = g[25] as *i64 422 let tkc: *i64 = g[26] as *i64 423 let qnb: *u8 = g[28] as *u8 424 let qto: *i64 = g[29] as *i64 425 let qtl: *i64 = g[30] as *i64 426 let qk: *i64 = g[32] as *i64 427 var hits: i64 = 0 428 var k: i64 = 0 429 while k < g[31] { 430 if qk[k] == 1 { 431 var found: i64 = 0 432 var j: i64 = 0 433 while j < tkc[s] { 434 if found == 0 { 435 let ti: i64 = tks[s] + j 436 if qs_tok_eq2(nb, tko[ti], tkl[ti], qnb, qto[k], qtl[k]) == 1 { found = 1 } 437 } 438 j = j + 1 439 } 440 hits = hits + found 441 } 442 k = k + 1 443 } 444 return hits 445} 446 447func db_title_hits(g: *i64, p: i64) -> i64 { 448 let tnb: *u8 = g[33] as *u8 449 let tto: *i64 = g[34] as *i64 450 let ttl: *i64 = g[35] as *i64 451 let tts: *i64 = g[36] as *i64 452 let ttc: *i64 = g[37] as *i64 453 let qnb: *u8 = g[28] as *u8 454 let qto: *i64 = g[29] as *i64 455 let qtl: *i64 = g[30] as *i64 456 let qk: *i64 = g[32] as *i64 457 var hits: i64 = 0 458 var k: i64 = 0 459 while k < g[31] { 460 if qk[k] == 1 { 461 var found: i64 = 0 462 var j: i64 = 0 463 while j < ttc[p] { 464 if found == 0 { 465 let ti: i64 = tts[p] + j 466 if qs_tok_eq2(tnb, tto[ti], ttl[ti], qnb, qto[k], qtl[k]) == 1 { found = 1 } 467 } 468 j = j + 1 469 } 470 hits = hits + found 471 } 472 k = k + 1 473 } 474 return hits 475} 476 477func db_body_hits(g: *i64, p: i64) -> i64 { 478 let nb: *u8 = g[20] as *u8 479 let tko: *i64 = g[23] as *i64 480 let tkl: *i64 = g[24] as *i64 481 let tks: *i64 = g[25] as *i64 482 let tkc: *i64 = g[26] as *i64 483 let sp: *i64 = g[18] as *i64 484 let qnb: *u8 = g[28] as *u8 485 let qto: *i64 = g[29] as *i64 486 let qtl: *i64 = g[30] as *i64 487 let qk: *i64 = g[32] as *i64 488 var hits: i64 = 0 489 var k: i64 = 0 490 while k < g[31] { 491 if qk[k] == 1 { 492 var found: i64 = 0 493 var s: i64 = 0 494 while s < g[19] { 495 if found == 0 { if sp[s] == p { 496 var j: i64 = 0 497 while j < tkc[s] { 498 if found == 0 { 499 let ti: i64 = tks[s] + j 500 if qs_tok_eq2(nb, tko[ti], tkl[ti], qnb, qto[k], qtl[k]) == 1 { found = 1 } 501 } 502 j = j + 1 503 } 504 } } 505 s = s + 1 506 } 507 hits = hits + found 508 } 509 k = k + 1 510 } 511 return hits 512} 513 514// single content-qterm k present in para p body? (0/1) 515func db_term_in_body(g: *i64, p: i64, k: i64) -> i64 { 516 let nb: *u8 = g[20] as *u8 517 let tko: *i64 = g[23] as *i64 518 let tkl: *i64 = g[24] as *i64 519 let tks: *i64 = g[25] as *i64 520 let tkc: *i64 = g[26] as *i64 521 let sp: *i64 = g[18] as *i64 522 let qnb: *u8 = g[28] as *u8 523 let qto: *i64 = g[29] as *i64 524 let qtl: *i64 = g[30] as *i64 525 var s: i64 = 0 526 while s < g[19] { 527 if sp[s] == p { 528 var j: i64 = 0 529 while j < tkc[s] { 530 let ti: i64 = tks[s] + j 531 if qs_tok_eq2(nb, tko[ti], tkl[ti], qnb, qto[k], qtl[k]) == 1 { return 1 } 532 j = j + 1 533 } 534 } 535 s = s + 1 536 } 537 return 0 538} 539 540func db_term_in_title(g: *i64, p: i64, k: i64) -> i64 { 541 let tnb: *u8 = g[33] as *u8 542 let tto: *i64 = g[34] as *i64 543 let ttl: *i64 = g[35] as *i64 544 let tts: *i64 = g[36] as *i64 545 let ttc: *i64 = g[37] as *i64 546 let qnb: *u8 = g[28] as *u8 547 let qto: *i64 = g[29] as *i64 548 let qtl: *i64 = g[30] as *i64 549 var j: i64 = 0 550 while j < ttc[p] { 551 let ti: i64 = tts[p] + j 552 if qs_tok_eq2(tnb, tto[ti], ttl[ti], qnb, qto[k], qtl[k]) == 1 { return 1 } 553 j = j + 1 554 } 555 return 0 556} 557 558// IDF-weighted paragraph retrieval (BM25-style discrimination: a term in FEWER paras weighs more). 559// H[p*128+k] = body(bit0)|title(bit1); df[k]=#paras with term; score += (np-df+1) per present term, title 2x. 560func db_score_row(g: *i64) -> i64 { 561 let shit: *i64 = g[39] as *i64 562 let thit: *i64 = g[40] as *i64 563 let psc: *i64 = g[42] as *i64 564 let qk: *i64 = g[32] as *i64 565 let H: *u8 = g[57] as *u8 566 let df: *i64 = g[58] as *i64 567 var s: i64 = 0 568 while s < g[19] { let h: i64 = db_sent_hits(g, s); shit[s] = h; s = s + 1 } 569 // build presence matrix + df over content qterms 570 var k: i64 = 0 571 while k < g[31] { 572 df[k] = 0 573 if qk[k] == 1 { 574 var p: i64 = 0 575 while p < g[14] { 576 let b: i64 = db_term_in_body(g, p, k) 577 let t: i64 = db_term_in_title(g, p, k) 578 var h: i64 = 0 579 if b == 1 { h = h + 1 } 580 if t == 1 { h = h + 2 } 581 H[p*128+k] = h as u8 582 if h != 0 { df[k] = df[k] + 1 } 583 p = p + 1 584 } 585 } 586 k = k + 1 587 } 588 // per-para IDF-weighted score 589 var p2: i64 = 0 590 while p2 < g[14] { 591 var sc: i64 = 0 592 var th: i64 = 0 593 var k2: i64 = 0 594 while k2 < g[31] { 595 if qk[k2] == 1 { 596 let h2: i64 = H[p2*128+k2] as i64 597 if h2 != 0 { 598 let w: i64 = g[14] - df[k2] + 1 // rarer term -> higher weight 599 sc = sc + w 600 if h2 >= 2 { sc = sc + 2*w; th = th + 1 } // title match: extra weight + count 601 } 602 } 603 k2 = k2 + 1 604 } 605 thit[p2] = th 606 psc[p2] = sc 607 p2 = p2 + 1 608 } 609 return 0 610} 611 612func db_bridge_title_hits(g: *i64, p: i64) -> i64 { 613 let tnb: *u8 = g[33] as *u8 614 let tto: *i64 = g[34] as *i64 615 let ttl: *i64 = g[35] as *i64 616 let tts: *i64 = g[36] as *i64 617 let ttc: *i64 = g[37] as *i64 618 let btb: *u8 = g[44] as *u8 619 let bto: *i64 = g[45] as *i64 620 let btl: *i64 = g[46] as *i64 621 var hits: i64 = 0 622 var k: i64 = 0 623 while k < g[47] { 624 var found: i64 = 0 625 var j: i64 = 0 626 while j < ttc[p] { 627 if found == 0 { 628 let ti: i64 = tts[p] + j 629 if qs_tok_eq2(tnb, tto[ti], ttl[ti], btb, bto[k], btl[k]) == 1 { found = 1 } 630 } 631 j = j + 1 632 } 633 hits = hits + found 634 k = k + 1 635 } 636 return hits 637} 638 639func db_bridge_body_hits(g: *i64, p: i64) -> i64 { 640 let nb: *u8 = g[20] as *u8 641 let tko: *i64 = g[23] as *i64 642 let tkl: *i64 = g[24] as *i64 643 let tks: *i64 = g[25] as *i64 644 let tkc: *i64 = g[26] as *i64 645 let sp: *i64 = g[18] as *i64 646 let btb: *u8 = g[44] as *u8 647 let bto: *i64 = g[45] as *i64 648 let btl: *i64 = g[46] as *i64 649 var hits: i64 = 0 650 var k: i64 = 0 651 while k < g[47] { 652 var found: i64 = 0 653 var s: i64 = 0 654 while s < g[19] { 655 if found == 0 { if sp[s] == p { 656 var j: i64 = 0 657 while j < tkc[s] { 658 if found == 0 { 659 let ti: i64 = tks[s] + j 660 if qs_tok_eq2(nb, tko[ti], tkl[ti], btb, bto[k], btl[k]) == 1 { found = 1 } 661 } 662 j = j + 1 663 } 664 } } 665 s = s + 1 666 } 667 hits = hits + found 668 k = k + 1 669 } 670 return hits 671} 672 673// ---------- candidates + extraction ---------- 674// harvest candidates from RAW sentence bytes: capitalized runs + numerics. fills cnd arrays (ncn=g[52]). 675func db_candidates(g: *i64, soff: i64, slen: i64) -> i64 { 676 let sb: *u8 = g[15] as *u8 677 let cnd: *u8 = g[48] as *u8 678 let cno: *i64 = g[49] as *i64 679 let cnl: *i64 = g[50] as *i64 680 let cnt: *i64 = g[51] as *i64 681 g[52] = 0 682 var bump: i64 = 0 683 var i: i64 = 0 684 while i < slen { 685 var c: i64 = sb[soff+i] as i64 686 var alnum: i64 = 0 687 if c>=48 { if c<=57 { alnum=1 } } 688 if c>=65 { if c<=90 { alnum=1 } } 689 if c>=97 { if c<=122 { alnum=1 } } 690 if alnum == 1 { 691 var wend: i64 = i 692 var isnum: i64 = 1 693 var iscap: i64 = 0 694 if c>=65 { if c<=90 { iscap=1 } } 695 while wend < slen { 696 var d: i64 = sb[soff+wend] as i64 697 var a2: i64 = 0 698 if d>=48 { if d<=57 { a2=1 } } 699 if d>=65 { if d<=90 { a2=1; isnum=0 } } 700 if d>=97 { if d<=122 { a2=1; isnum=0 } } 701 if d==44 { if wend+1<slen { var e2: i64 = sb[soff+wend+1] as i64; if e2>=48 { if e2<=57 { a2=1 } } } } // comma between digits: "3,677" stays one number 702 if a2 == 0 { wend = slen + wend + 1 } else { wend = wend + 1 } 703 } 704 if wend > slen { wend = wend - slen - 1 } 705 let wlen: i64 = wend - i 706 if isnum == 1 { 707 if g[52] < 15 { if bump + wlen < K_MAGIC_2000 { 708 var w: i64 = 0 709 while w < wlen { cnd[bump+w] = sb[soff+i+w]; w = w + 1 } 710 cnd[bump+wlen] = (0 as u8) 711 let n0: i64 = g[52] 712 cno[n0] = bump; cnl[n0] = wlen 713 var cty: i64 = 2 714 if wlen == 4 { cty = 3 } 715 cnt[n0] = cty 716 g[52] = n0 + 1; bump = bump + wlen + 1 717 } } 718 i = wend 719 } else { 720 if iscap == 1 { 721 var rend: i64 = wend 722 var words: i64 = 1 723 var go: i64 = 1 724 while go == 1 { 725 if rend < slen { if sb[soff+rend]==(32 as u8) { 726 var nx: i64 = rend + 1 727 if nx < slen { 728 var e: i64 = sb[soff+nx] as i64 729 var ecap: i64 = 0 730 if e>=65 { if e<=90 { ecap=1 } } 731 if e>=48 { if e<=57 { ecap=1 } } 732 if ecap == 1 { if words < 6 { 733 var we2: i64 = nx 734 while we2 < slen { 735 var f: i64 = sb[soff+we2] as i64 736 var a3: i64 = 0 737 if f>=48 { if f<=57 { a3=1 } } 738 if f>=65 { if f<=90 { a3=1 } } 739 if f>=97 { if f<=122 { a3=1 } } 740 if a3 == 0 { we2 = slen + we2 + 1 } else { we2 = we2 + 1 } 741 } 742 if we2 > slen { we2 = we2 - slen - 1 } 743 rend = we2; words = words + 1 744 } else { go = 0 } } else { go = 0 } 745 } else { go = 0 } 746 } else { go = 0 } } else { go = 0 } 747 } 748 let rlen: i64 = rend - i 749 if g[52] < 15 { if bump + rlen < K_MAGIC_2000 { 750 var w2: i64 = 0 751 while w2 < rlen { cnd[bump+w2] = sb[soff+i+w2]; w2 = w2 + 1 } 752 cnd[bump+rlen] = (0 as u8) 753 let n1: i64 = g[52] 754 cno[n1] = bump; cnl[n1] = rlen 755 cnt[n1] = 1 756 g[52] = n1 + 1; bump = bump + rlen + 1 757 } } 758 i = rend 759 } else { 760 i = wend 761 } 762 } 763 } else { 764 i = i + 1 765 } 766 } 767 return g[52] 768} 769 770// candidate echo-check: 1 = candidate fully contained in the question (drop it) 771func db_cand_is_echo(g: *i64, ci: i64) -> i64 { 772 let cnd: *u8 = g[48] as *u8 773 let cno: *i64 = g[49] as *i64 774 let scr: *u8 = g[53] as *u8 775 let src: *u8 = (cnd as i64 + cno[ci]) as *u8 776 let nl0: i64 = qs_norm(src, scr) 777 let toks: *i64 = (scr as i64 + K_MAGIC_2048) as *i64 778 let tlens: *i64 = (scr as i64 + K_MAGIC_4096) as *i64 779 let n: i64 = qs_tok(scr, nl0, toks, tlens, 32) 780 if n == 0 { return 1 } 781 var k: i64 = 0 782 while k < n { 783 if db_in_question(g, scr, toks[k], tlens[k], 0) == 0 { return 0 } 784 k = k + 1 785 } 786 return 1 787} 788 789func db_prefer_numeric(g: *i64) -> i64 { 790 let qnb: *u8 = g[28] as *u8 791 let qto: *i64 = g[29] as *i64 792 let qtl: *i64 = g[30] as *i64 793 var k: i64 = 0 794 while k < g[31] { 795 if db_tok_is(qnb, qto[k], qtl[k], "when" as *u8)==1 { return 1 } 796 if db_tok_is(qnb, qto[k], qtl[k], "year" as *u8)==1 { return 1 } 797 if db_tok_is(qnb, qto[k], qtl[k], "many" as *u8)==1 { return 1 } 798 k = k + 1 799 } 800 return 0 801} 802 803// question wants a YEAR specifically (prefer 4-digit) 804func db_wants_year(g: *i64) -> i64 { 805 let qnb: *u8 = g[28] as *u8 806 let qto: *i64 = g[29] as *i64 807 let qtl: *i64 = g[30] as *i64 808 var k: i64 = 0 809 while k < g[31] { 810 if db_tok_is(qnb, qto[k], qtl[k], "when" as *u8)==1 { return 1 } 811 if db_tok_is(qnb, qto[k], qtl[k], "year" as *u8)==1 { return 1 } 812 k = k + 1 813 } 814 return 0 815} 816 817// candidate ci is a COMMON capitalized word (nationality/pronoun/determiner/number-word) = a likely distractor, not the answer 818func db_cand_is_common(g: *i64, ci: i64) -> i64 { 819 let cnd: *u8 = g[48] as *u8 820 let cno: *i64 = g[49] as *i64 821 let scr: *u8 = g[53] as *u8 822 let src: *u8 = (cnd as i64 + cno[ci]) as *u8 823 let nl0: i64 = qs_norm(src, scr) 824 let toks: *i64 = (scr as i64 + K_MAGIC_2048) as *i64 825 let tlens: *i64 = (scr as i64 + K_MAGIC_4096) as *i64 826 let n: i64 = qs_tok(scr, nl0, toks, tlens, 8) 827 if n != 1 { return 0 } // multi-word runs are proper nouns, not common 828 let o: i64 = toks[0] 829 let l: i64 = tlens[0] 830 if db_tok_is(scr,o,l,"american" as *u8)==1 { return 1 } 831 if db_tok_is(scr,o,l,"british" as *u8)==1 { return 1 } 832 if db_tok_is(scr,o,l,"english" as *u8)==1 { return 1 } 833 if db_tok_is(scr,o,l,"french" as *u8)==1 { return 1 } 834 if db_tok_is(scr,o,l,"german" as *u8)==1 { return 1 } 835 if db_tok_is(scr,o,l,"italian" as *u8)==1 { return 1 } 836 if db_tok_is(scr,o,l,"european" as *u8)==1 { return 1 } 837 if db_tok_is(scr,o,l,"she" as *u8)==1 { return 1 } 838 if db_tok_is(scr,o,l,"he" as *u8)==1 { return 1 } 839 if db_tok_is(scr,o,l,"they" as *u8)==1 { return 1 } 840 if db_tok_is(scr,o,l,"it" as *u8)==1 { return 1 } 841 if db_tok_is(scr,o,l,"the" as *u8)==1 { return 1 } 842 if db_tok_is(scr,o,l,"this" as *u8)==1 { return 1 } 843 if db_tok_is(scr,o,l,"one" as *u8)==1 { return 1 } 844 if db_tok_is(scr,o,l,"two" as *u8)==1 { return 1 } 845 if db_tok_is(scr,o,l,"three" as *u8)==1 { return 1 } 846 return 0 847} 848 849// candidate ci is a MULTI-WORD run (contains a space) = strong proper-noun signal 850func db_cand_multiword(g: *i64, ci: i64) -> i64 { 851 let cnd: *u8 = g[48] as *u8 852 let cno: *i64 = g[49] as *i64 853 let cnl: *i64 = g[50] as *i64 854 var w: i64 = 0 855 while w < cnl[ci] { if cnd[cno[ci]+w]==(32 as u8) { return 1 } w = w + 1 } 856 return 0 857} 858 859// sweep the two retrieved paragraphs for a NUMERIC answer (numeric questions). fills pred, returns 1 if found. 860func db_sweep_numeric(g: *i64, pa: i64, pb: i64, pred: *u8) -> i64 { 861 let so: *i64 = g[16] as *i64 862 let sl: *i64 = g[17] as *i64 863 let sp: *i64 = g[18] as *i64 864 let cnd: *u8 = g[48] as *u8 865 let cno: *i64 = g[49] as *i64 866 let cnl: *i64 = g[50] as *i64 867 let cnt: *i64 = g[51] as *i64 868 let wy: i64 = db_wants_year(g) 869 // (a subject-mention "shit>0" year pass was tried and REGRESSED -- too loose: any sentence naming the 870 // subject qualifies, e.g. "...Earl of Lovelace in 1838" for a birth-year Q. Correct fix = bind the year to 871 // the event cue ("born"), which is dependency-parse-hard; left as a residual plateau item, not patched.) 872 // pass 1: 4-digit (year) when the question wants a year 873 if wy == 1 { 874 var s: i64 = 0 875 while s < g[19] { 876 if sp[s]==pa { if sp[s]<0 { } } // (keep structure explicit) 877 var inpar: i64 = 0 878 if sp[s]==pa { inpar = 1 } 879 if sp[s]==pb { inpar = 1 } 880 if inpar == 1 { 881 db_candidates(g, so[s], sl[s]) 882 var c: i64 = 0 883 while c < g[52] { 884 if cnt[c]==3 { if db_cand_is_echo(g,c)==0 { 885 db_setpred(pred, (cnd as i64 + cno[c]) as *u8, cnl[c]); return 1 886 } } 887 c = c + 1 888 } 889 } 890 s = s + 1 891 } 892 } 893 // pass 2: any number 894 var s2: i64 = 0 895 while s2 < g[19] { 896 var inpar2: i64 = 0 897 if sp[s2]==pa { inpar2 = 1 } 898 if sp[s2]==pb { inpar2 = 1 } 899 if inpar2 == 1 { 900 db_candidates(g, so[s2], sl[s2]) 901 var c2: i64 = 0 902 while c2 < g[52] { 903 if cnt[c2]>=2 { if db_cand_is_echo(g,c2)==0 { 904 db_setpred(pred, (cnd as i64 + cno[c2]) as *u8, cnl[c2]); return 1 905 } } 906 c2 = c2 + 1 907 } 908 } 909 s2 = s2 + 1 910 } 911 return 0 912} 913 914// question is a LOCATION question ("where ...") 915func db_wants_location(g: *i64) -> i64 { 916 let qnb: *u8 = g[28] as *u8 917 let qto: *i64 = g[29] as *i64 918 let qtl: *i64 = g[30] as *i64 919 if g[31] < 1 { return 0 } 920 if db_tok_is(qnb, qto[0], qtl[0], "where" as *u8)==1 { return 1 } 921 return 0 922} 923 924// sentence s carries a LOCATIVE cue (so its proper nouns are likely the place answer, even if it is not the 925// top question-word-match sentence -- the "where" answer usually lives in a follow-up sentence via a pronoun). 926func db_sentence_has_locative(g: *i64, s: i64) -> i64 { 927 let nb: *u8 = g[20] as *u8 928 let tko: *i64 = g[23] as *i64 929 let tkl: *i64 = g[24] as *i64 930 let tks: *i64 = g[25] as *i64 931 let tkc: *i64 = g[26] as *i64 932 var j: i64 = 0 933 while j < tkc[s] { 934 let ti: i64 = tks[s] + j 935 if db_tok_is(nb,tko[ti],tkl[ti],"located" as *u8)==1 { return 1 } 936 if db_tok_is(nb,tko[ti],tkl[ti],"coast" as *u8)==1 { return 1 } 937 if db_tok_is(nb,tko[ti],tkl[ti],"situated" as *u8)==1 { return 1 } 938 if db_tok_is(nb,tko[ti],tkl[ti],"city" as *u8)==1 { return 1 } 939 if db_tok_is(nb,tko[ti],tkl[ti],"country" as *u8)==1 { return 1 } 940 if db_tok_is(nb,tko[ti],tkl[ti],"state" as *u8)==1 { return 1 } 941 if db_tok_is(nb,tko[ti],tkl[ti],"region" as *u8)==1 { return 1 } 942 if db_tok_is(nb,tko[ti],tkl[ti],"capital" as *u8)==1 { return 1 } 943 j = j + 1 944 } 945 return 0 946} 947 948// sentence s is a COMPARISON ("taller than X", "surpassed X") -- its proper noun is a compared entity, NOT the 949// subject's location. These fool a naive location sweep (e.g. "taller than the Washington Monument"). 950func db_sentence_has_comparison(g: *i64, s: i64) -> i64 { 951 let nb: *u8 = g[20] as *u8 952 let tko: *i64 = g[23] as *i64 953 let tkl: *i64 = g[24] as *i64 954 let tks: *i64 = g[25] as *i64 955 let tkc: *i64 = g[26] as *i64 956 var j: i64 = 0 957 while j < tkc[s] { 958 let ti: i64 = tks[s] + j 959 if db_tok_is(nb,tko[ti],tkl[ti],"than" as *u8)==1 { return 1 } 960 if db_tok_is(nb,tko[ti],tkl[ti],"taller" as *u8)==1 { return 1 } 961 if db_tok_is(nb,tko[ti],tkl[ti],"shorter" as *u8)==1 { return 1 } 962 if db_tok_is(nb,tko[ti],tkl[ti],"larger" as *u8)==1 { return 1 } 963 if db_tok_is(nb,tko[ti],tkl[ti],"smaller" as *u8)==1 { return 1 } 964 if db_tok_is(nb,tko[ti],tkl[ti],"surpassed" as *u8)==1 { return 1 } 965 if db_tok_is(nb,tko[ti],tkl[ti],"compared" as *u8)==1 { return 1 } 966 j = j + 1 967 } 968 return 0 969} 970 971// pick the first proper-noun candidate (multiword preferred, else single non-common) in sentence s; non-echo. 972func db_pick_proper_in(g: *i64, s: i64, pred: *u8) -> i64 { 973 let so: *i64 = g[16] as *i64 974 let sl: *i64 = g[17] as *i64 975 let cnd: *u8 = g[48] as *u8 976 let cno: *i64 = g[49] as *i64 977 let cnl: *i64 = g[50] as *i64 978 let cnt: *i64 = g[51] as *i64 979 db_candidates(g, so[s], sl[s]) 980 var c: i64 = 0 981 while c < g[52] { if cnt[c]==1 { if db_cand_multiword(g,c)==1 { if db_cand_is_echo(g,c)==0 { db_setpred(pred,(cnd as i64 + cno[c]) as *u8, cnl[c]); return 1 } } } c = c + 1 } 982 c = 0 983 while c < g[52] { if cnt[c]==1 { if db_cand_is_common(g,c)==0 { if db_cand_is_echo(g,c)==0 { db_setpred(pred,(cnd as i64 + cno[c]) as *u8, cnl[c]); return 1 } } } c = c + 1 } 984 return 0 985} 986 987// sweep the retrieved paragraphs for a LOCATION answer. Distractor-hardened: SKIP comparison clauses (their 988// proper noun is a COMPARED entity, not the subject's place -- "taller than the Washington Monument"). Prefer a 989// locative-cued sentence that mentions the subject (pass 1), else a locative-cued follow-up with a pronoun 990// subject (pass 2). Mirrors db_sweep_numeric. 991func db_sweep_location(g: *i64, pa: i64, pb: i64, pred: *u8) -> i64 { 992 let sp: *i64 = g[18] as *i64 993 let shit: *i64 = g[39] as *i64 994 // pass 1: locative cue + mentions the subject + NOT a comparison 995 var s: i64 = 0 996 while s < g[19] { 997 var inpar: i64 = 0 998 if sp[s]==pa { inpar = 1 } 999 if sp[s]==pb { inpar = 1 } 1000 if inpar == 1 { if db_sentence_has_locative(g, s) == 1 { if shit[s] > 0 { if db_sentence_has_comparison(g, s) == 0 { 1001 if db_pick_proper_in(g, s, pred) == 1 { return 1 } 1002 } } } } 1003 s = s + 1 1004 } 1005 // pass 2: locative cue + NOT a comparison (the "It is located in ..." pronoun-subject follow-up) 1006 var s2: i64 = 0 1007 while s2 < g[19] { 1008 var inpar2: i64 = 0 1009 if sp[s2]==pa { inpar2 = 1 } 1010 if sp[s2]==pb { inpar2 = 1 } 1011 if inpar2 == 1 { if db_sentence_has_locative(g, s2) == 1 { if db_sentence_has_comparison(g, s2) == 0 { 1012 if db_pick_proper_in(g, s2, pred) == 1 { return 1 } 1013 } } } 1014 s2 = s2 + 1 1015 } 1016 return 0 1017} 1018 1019func db_is_yesno(g: *i64) -> i64 { 1020 let qnb: *u8 = g[28] as *u8 1021 let qto: *i64 = g[29] as *i64 1022 let qtl: *i64 = g[30] as *i64 1023 if g[31] < 1 { return 0 } 1024 if db_tok_is(qnb, qto[0], qtl[0], "is" as *u8)==1 { return 1 } 1025 if db_tok_is(qnb, qto[0], qtl[0], "are" as *u8)==1 { return 1 } 1026 if db_tok_is(qnb, qto[0], qtl[0], "was" as *u8)==1 { return 1 } 1027 if db_tok_is(qnb, qto[0], qtl[0], "were" as *u8)==1 { return 1 } 1028 if db_tok_is(qnb, qto[0], qtl[0], "do" as *u8)==1 { return 1 } 1029 if db_tok_is(qnb, qto[0], qtl[0], "does" as *u8)==1 { return 1 } 1030 if db_tok_is(qnb, qto[0], qtl[0], "did" as *u8)==1 { return 1 } 1031 return 0 1032} 1033 1034func db_setpred(dst: *u8, src: *u8, len: i64) -> i64 { 1035 var i: i64 = 0 1036 while i < len { if i < 250 { dst[i] = src[i] } i = i + 1 } 1037 var e: i64 = len 1038 if e > 250 { e = 250 } 1039 dst[e] = (0 as u8) 1040 return 0 1041} 1042 1043// extract answer from sentence s into pred (256B slot). fallback = leading non-question content tokens. 1044func db_extract(g: *i64, s: i64, pred: *u8) -> i64 { 1045 let so: *i64 = g[16] as *i64 1046 let sl: *i64 = g[17] as *i64 1047 let nb: *u8 = g[20] as *u8 1048 let tko: *i64 = g[23] as *i64 1049 let tkl: *i64 = g[24] as *i64 1050 let tks: *i64 = g[25] as *i64 1051 let tkc: *i64 = g[26] as *i64 1052 let cnd: *u8 = g[48] as *u8 1053 let cno: *i64 = g[49] as *i64 1054 let cnl: *i64 = g[50] as *i64 1055 let cnt: *i64 = g[51] as *i64 1056 db_candidates(g, so[s], sl[s]) 1057 let wantnum: i64 = db_prefer_numeric(g) 1058 var pick: i64 = 0-1 1059 if wantnum == 1 { 1060 var c: i64 = 0 1061 while c < g[52] { if pick < 0 { if cnt[c]==3 { if db_cand_is_echo(g,c)==0 { pick = c } } } c = c + 1 } 1062 c = 0 1063 while c < g[52] { if pick < 0 { if cnt[c]==2 { if db_cand_is_echo(g,c)==0 { pick = c } } } c = c + 1 } 1064 } 1065 // pass 1: MULTI-WORD proper-noun run (non-echo) -- beats single common distractors ("American"/"Two") 1066 var c2: i64 = 0 1067 while c2 < g[52] { if pick < 0 { if cnt[c2]==1 { if db_cand_multiword(g,c2)==1 { if db_cand_is_echo(g,c2)==0 { pick = c2 } } } } c2 = c2 + 1 } 1068 // pass 2: single capitalized, non-echo, NOT a common word 1069 c2 = 0 1070 while c2 < g[52] { if pick < 0 { if cnt[c2]==1 { if db_cand_is_common(g,c2)==0 { if db_cand_is_echo(g,c2)==0 { pick = c2 } } } } c2 = c2 + 1 } 1071 // pass 3: any capitalized non-echo (common words as last resort) 1072 c2 = 0 1073 while c2 < g[52] { if pick < 0 { if cnt[c2]==1 { if db_cand_is_echo(g,c2)==0 { pick = c2 } } } c2 = c2 + 1 } 1074 // pass 4: any non-echo candidate 1075 c2 = 0 1076 while c2 < g[52] { if pick < 0 { if db_cand_is_echo(g,c2)==0 { pick = c2 } } c2 = c2 + 1 } 1077 if pick >= 0 { 1078 let src: *u8 = (cnd as i64 + cno[pick]) as *u8 1079 db_setpred(pred, src, cnl[pick]) 1080 return 1 1081 } 1082 var out: i64 = 0 1083 var used: i64 = 0 1084 var j: i64 = 0 1085 while j < tkc[s] { 1086 if used < 8 { 1087 let ti: i64 = tks[s] + j 1088 if db_in_question(g, nb, tko[ti], tkl[ti], 0) == 0 { 1089 var w: i64 = 0 1090 while w < tkl[ti] { if out < 240 { pred[out] = nb[tko[ti]+w]; out = out + 1 } w = w + 1 } 1091 if out < 240 { pred[out] = (32 as u8); out = out + 1 } 1092 used = used + 1 1093 } 1094 } 1095 j = j + 1 1096 } 1097 if out > 0 { out = out - 1 } 1098 pred[out] = (0 as u8) 1099 return 0 1100} 1101 1102// build bridge terms from hop-1 sentence + hop-1 para title: normalized tokens NOT in the question. max 8. (nbt=g[47]) 1103func db_bridge_from(g: *i64, s: i64, p: i64) -> i64 { 1104 let so: *i64 = g[16] as *i64 1105 let sl: *i64 = g[17] as *i64 1106 let tnb: *u8 = g[33] as *u8 1107 let tto: *i64 = g[34] as *i64 1108 let ttl: *i64 = g[35] as *i64 1109 let tts: *i64 = g[36] as *i64 1110 let ttc: *i64 = g[37] as *i64 1111 let btb: *u8 = g[44] as *u8 1112 let bto: *i64 = g[45] as *i64 1113 let btl: *i64 = g[46] as *i64 1114 let cnd: *u8 = g[48] as *u8 1115 let cno: *i64 = g[49] as *i64 1116 let cnt: *i64 = g[51] as *i64 1117 let scr: *u8 = g[53] as *u8 1118 g[47] = 0 1119 var bump: i64 = 0 1120 db_candidates(g, so[s], sl[s]) 1121 var c: i64 = 0 1122 while c < g[52] { 1123 if cnt[c] == 1 { 1124 let src: *u8 = (cnd as i64 + cno[c]) as *u8 1125 let nl0: i64 = qs_norm(src, scr) 1126 let toks: *i64 = (scr as i64 + K_MAGIC_2048) as *i64 1127 let tlens: *i64 = (scr as i64 + K_MAGIC_4096) as *i64 1128 let n: i64 = qs_tok(scr, nl0, toks, tlens, 16) 1129 var k: i64 = 0 1130 while k < n { 1131 if g[47] < 8 { 1132 if db_in_question(g, scr, toks[k], tlens[k], 0) == 0 { 1133 var dup: i64 = 0 1134 var b: i64 = 0 1135 while b < g[47] { if qs_tok_eq2(scr, toks[k], tlens[k], btb, bto[b], btl[b])==1 { dup = 1 } b = b + 1 } 1136 if dup == 0 { 1137 var w: i64 = 0 1138 while w < tlens[k] { btb[bump+w] = scr[toks[k]+w]; w = w + 1 } 1139 let nb0: i64 = g[47] 1140 bto[nb0] = bump; btl[nb0] = tlens[k] 1141 g[47] = nb0 + 1; bump = bump + tlens[k] + 1 1142 } 1143 } 1144 } 1145 k = k + 1 1146 } 1147 } 1148 c = c + 1 1149 } 1150 var j: i64 = 0 1151 while j < ttc[p] { 1152 if g[47] < 8 { 1153 let ti: i64 = tts[p] + j 1154 if db_in_question(g, tnb, tto[ti], ttl[ti], 0) == 0 { 1155 var dup2: i64 = 0 1156 var b2: i64 = 0 1157 while b2 < g[47] { if qs_tok_eq2(tnb, tto[ti], ttl[ti], btb, bto[b2], btl[b2])==1 { dup2 = 1 } b2 = b2 + 1 } 1158 if dup2 == 0 { 1159 var w2: i64 = 0 1160 while w2 < ttl[ti] { btb[bump+w2] = tnb[tto[ti]+w2]; w2 = w2 + 1 } 1161 let nb1: i64 = g[47] 1162 bto[nb1] = bump; btl[nb1] = ttl[ti] 1163 g[47] = nb1 + 1; bump = bump + ttl[ti] + 1 1164 } 1165 } 1166 } 1167 j = j + 1 1168 } 1169 return g[47] 1170} 1171 1172// best sentence index overall (by hits*2 + para title hits, tiebreak earlier); -1 if none 1173func db_best_sentence(g: *i64) -> i64 { 1174 let shit: *i64 = g[39] as *i64 1175 let thit: *i64 = g[40] as *i64 1176 let sp: *i64 = g[18] as *i64 1177 var best: i64 = 0-1 1178 var bs: i64 = 0-1 1179 var s: i64 = 0 1180 while s < g[19] { 1181 let sc: i64 = 2*shit[s] + thit[sp[s]] 1182 if sc > bs { bs = sc; best = s } 1183 s = s + 1 1184 } 1185 return best 1186} 1187 1188// best sentence within para p, scoring question-hits*2 + bridge-hits 1189func db_best_sentence_in(g: *i64, p: i64) -> i64 { 1190 let nb: *u8 = g[20] as *u8 1191 let tko: *i64 = g[23] as *i64 1192 let tkl: *i64 = g[24] as *i64 1193 let tks: *i64 = g[25] as *i64 1194 let tkc: *i64 = g[26] as *i64 1195 let sp: *i64 = g[18] as *i64 1196 let shit: *i64 = g[39] as *i64 1197 let btb: *u8 = g[44] as *u8 1198 let bto: *i64 = g[45] as *i64 1199 let btl: *i64 = g[46] as *i64 1200 var best: i64 = 0-1 1201 var bs: i64 = 0-1 1202 var s: i64 = 0 1203 while s < g[19] { 1204 if sp[s] == p { 1205 var bh: i64 = 0 1206 var k: i64 = 0 1207 while k < g[47] { 1208 var found: i64 = 0 1209 var j: i64 = 0 1210 while j < tkc[s] { 1211 if found == 0 { 1212 let ti: i64 = tks[s] + j 1213 if qs_tok_eq2(nb, tko[ti], tkl[ti], btb, bto[k], btl[k]) == 1 { found = 1 } 1214 } 1215 j = j + 1 1216 } 1217 bh = bh + found 1218 k = k + 1 1219 } 1220 let sc: i64 = 2*shit[s] + bh 1221 if sc > bs { bs = sc; best = s } 1222 } 1223 s = s + 1 1224 } 1225 return best 1226} 1227 1228// gold recall at 2: how many gold titles are in {pa,pb}, permille of ngt 1229func db_recall2(g: *i64, pa: i64, pb: i64) -> i64 { 1230 let gtb: *u8 = g[7] as *u8 1231 let gto: *i64 = g[8] as *i64 1232 let gtl: *i64 = g[9] as *i64 1233 let ptb: *u8 = g[11] as *u8 1234 let pto: *i64 = g[12] as *i64 1235 let ptl: *i64 = g[13] as *i64 1236 if g[10] == 0 { return 0 } 1237 var hit: i64 = 0 1238 var gsel: i64 = 0 1239 while gsel < g[10] { 1240 let gt: *u8 = (gtb as i64 + gto[gsel]) as *u8 1241 var m: i64 = 0 1242 var p: i64 = pa 1243 var round: i64 = 0 1244 while round < 2 { 1245 if p >= 0 { 1246 let pt: *u8 = (ptb as i64 + pto[p]) as *u8 1247 if gtl[gsel] == ptl[p] { 1248 var eq: i64 = 1 1249 var w: i64 = 0 1250 while w < gtl[gsel] { if gt[w] != pt[w] { eq = 0 } w = w + 1 } 1251 if eq == 1 { m = 1 } 1252 } 1253 } 1254 p = pb 1255 round = round + 1 1256 } 1257 hit = hit + m 1258 gsel = gsel + 1 1259 } 1260 return (1000*hit)/g[10] 1261} 1262 1263// top para by psc excluding skip; -1 if none 1264func db_top_para(g: *i64, skip: i64) -> i64 { 1265 let psc: *i64 = g[42] as *i64 1266 var best: i64 = 0-1 1267 var bs: i64 = 0-K_MAGIC_1000000 1268 var p: i64 = 0 1269 while p < g[14] { 1270 if p != skip { 1271 if psc[p] > bs { bs = psc[p]; best = p } 1272 } 1273 p = p + 1 1274 } 1275 return best 1276} 1277 1278// (engine only -- main lives in nx_qabench.nx; shared benchmark engine extracted from nx_drbench)