code wiki / _hdl_build / nx_beir_eval.nx
nx_beir_eval.nx
buildroot/runtime/_hdl_build/nx_beir_eval.nx
about
nx_beir_eval.nx -- SOVEREIGN BEIR harness: real external ground truth (BEIR/nfcorpus) scored by our own
integer BM25 -> mean nDCG@10 (the standard BEIR metric). The HONEST external row the maturity ladder is
gated on (F236): real 3633-doc corpus + 323 test queries + 12335 graded qrels, vs the published BEIR BM25
baseline (nfcorpus nDCG@10 ~0.32, Anserini). BM25 k1=0.9 b=0.4 (BEIR defaults). IDF=log2 (ranking-invariant
vs ln -> identical nDCG). Integer/fixed-point throughout (no float) = bit-reproducible. Forward index
(per-doc sorted term,tf). Tokenizer = lowercase [a-z0-9] runs -> PORTER STEM via nx_stem_lib (2026-08-13), NO stopwords (the stemming rung LANDED -- was the
named gap to the stemmed 0.32 Anserini baseline).
R0d be_bright (2026-09-14): `--set <root>` scores ANY BEIR-shaped set root (<root>/corpus.tsv, queries.tsv,
qrels/test.tsv and, when present, excluded.tsv: BRIGHT lists each query's own source documents there and the benchmark
removes them from that query's ranking, so this harness marks them unrankable for that query and counts them). Every
capacity is now DERIVED from the files (sys_read_file sizes the buffer, newline counts size the tables, the longest line
sizes the token scratch): the old 64 MiB file cap and 4096-document cap would have dropped 99 percent of BRIGHT's
leetcode split in silence, which is the cap class this estate refuses. `--bm25-only` skips the rerank arms (their models
were trained on the web corpus; BRIGHT's published floor is plain BM25, 14.5 average nDCG at 10 on the leaderboard page
read 2026-09-14, search.refs bright). `--bright <root>` is be_bright: this binary runs itself once per split through
tr_run_capture_to (one hour each, the largest split bounds it), prints one JSON row per split, the average over twelve of
twelve (never over a partial set), and writes the estate's row into search.leaderboard beside the published rows with
its rank DERIVED from their scores. The positional grammar is unchanged: [model.safetensors config.json vocab.txt
[max_len [depth [workers]]]] after the flags; workers is the cross-encoder pool width (0 = serial).
expect_exit: 0 license_tier: ORIGINAL
dependencies 12 imports · 0 importers
diagram shows first 10 each side; +2 more imports, +0 more importers in the complete lists below.
imports: nx_qabench_engine.nxnx_ppmi_lib.nxnx_stem.nxnx_ltr_lib.nxnx_dr_densejudge.nxnx_wordpiece_lib.nxnx_bert_ce_lib.nxnx_thread_pool.nxnx_tool_run.nxnx_timefmt.nxnx_intlog.nxnx_beir_arms_lib.nx
imported by: nobody (leaf or entry point)
call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown
structs
| 794 | struct BeCeTask |
consts
| 34 | const BE_ASCII_ZERO: i64 = 48 |
| 35 | const BE_ASCII_NINE: i64 = 57 |
| 36 | const BE_DECIMAL: i64 = 10 |
| 37 | const BE_MAGIC_1024: i64 = 1024 |
| 38 | const BE_MAGIC_1000000: i64 = 1000000 |
| 45 | const BE_KCAND: i64 = 50 |
| 47 | const BE_IDCAP: i64 = 4096 |
| 48 | const K_MAGIC_1024: i64 = 1024 |
| 49 | const K_MAGIC_4096: i64 = 4096 |
| 50 | const K_MAGIC_1900: i64 = 1900 |
| 52 | const BE_FLAG_SET: *u8 = "--set" |
| 53 | const BE_FLAG_BRIGHT: *u8 = "--bright" |
| 54 | const BE_CE_POOL_WORKERS_DEFAULT: i64 = 0 // 0 = every forward on the calling thread; the 6th positional sets the pool width, and since 2026-09-15 the pool's unit of work is ONE WHOLE PAIR (a forward), never a band of one matmul: the forward inside a task is the serial path, so a pooled run scores every pair identically to a serial run and only the wall clock moves (per-matmul banding was measured as NO speedup, search.plan row 1789456036) |
| 55 | const BE_CE_TASK_BYTES: i64 = 96 // BeCeTask: 12 i64 fields; one task record and one scratch set PER CANDIDATE so no two tasks ever share a buffer |
| 59 | const BE_PROGRESS_DIR: *u8 = "knowledge/status/beir_eval." |
| 60 | const BE_PROGRESS_SFX: *u8 = ".progress" |
| 61 | const BE_PROGRESS_SFX_TMP: *u8 = ".progress.tmp" |
| 62 | const BE_SLASH: i64 = 47 |
| 63 | const BE_DOT: i64 = 46 |
| 64 | const BE_USCORE: i64 = 95 |
| 65 | const BE_PROGRESS_BYTES: i64 = 640 |
| 66 | const BE_FLAG_BM25ONLY: *u8 = "--bm25-only" |
| 70 | const BE_FLAG_QUERIES: *u8 = "--queries" |
| 71 | const BE_ROOT_DEFAULT: *u8 = "knowledge/beir/nfcorpus" |
| 72 | const BE_ROOT_DEFAULT_TMP: *u8 = "/tmp/beir/nfcorpus" |
| 73 | const BE_DATASET_DEFAULT: *u8 = "BEIR/nfcorpus" |
| 74 | const BE_F_CORPUS: *u8 = "/corpus.tsv" |
| 75 | const BE_F_QRELS: *u8 = "/qrels/test.tsv" |
| 76 | const BE_F_QUERIES: *u8 = "/queries.tsv" |
| 77 | const BE_F_EXCLUDED: *u8 = "/excluded.tsv" |
| 78 | const BE_SEP: *u8 = "/" |
| 79 | const BE_PATH_CAP: i64 = 1024 |
| 80 | const BE_POS_CAP: i64 = 16 |
| 81 | const BE_NL: i64 = 10 |
| 82 | const BE_TAB: i64 = 9 |
| 83 | const BE_PIPE: i64 = 124 |
| 84 | const BE_DOT: i64 = 46 |
| 85 | const BE_MINUS: i64 = 45 |
| 86 | const BE_I64: i64 = 8 |
| 87 | const BE_G_SLOTS: i64 = 8 |
| 88 | const BE_TOK_BYTES: i64 = 2 // a token needs one byte and a separator, so tokens <= bytes / 2 + lines |
| 89 | const BE_PERMIL_DIGITS: i64 = 10 // the board writes nDCG x 100 with one decimal; permil / 10 . permil % 10 |
| 90 | const BE_BRIGHT_N: i64 = 12 |
| 91 | const BE_PT_TF_MOD: i64 = 1048576 // posting word radix (2^20): doc * radix + tf; a tf at or past it is clamped and COUNTED, never silently wrapped |
| 92 | const BE_BRIGHT_BM25_PUBLISHED_PERMIL: i64 = 145 // the leaderboard page's plain BM25 row, read 2026-09-14 (search.refs bright) |
| 95 | const BE_JSON_BQ_KEY: *u8 = "k_q10\":1228,\"ndcg_at_10_permil\":" |
| 96 | const BE_JSON_FQ_KEY: *u8 = "bm25q-rank, same 50 candidates\",\"ndcg_at_10_permil\":" |
| 97 | const BE_JSON_LONGN_KEY: *u8 = "\"long_n\":" |
| 98 | const BE_JSON_LONGD_KEY: *u8 = "\"long_bm25_permil\":" |
| 99 | const BE_JSON_LONGQ_KEY: *u8 = "\"long_bm25q_permil\":" |
| 100 | const BE_JSON_LONGF_KEY: *u8 = "\"long_fusion_permil\":" |
| 101 | const BE_JSON_NOREP_KEY: *u8 = "\"norepeat_n\":" |
| 102 | const BE_JSON_NOREPSAME_KEY: *u8 = "\"norepeat_top10_identical\":" |
| 103 | const BE_JSON_REP_KEY: *u8 = "\"queries_with_repeated_term\":" |
| 105 | const BE_JSON_CEM_KEY: *u8 = "\"ce_measured\":" |
| 106 | const BE_JSON_CE_KEY: *u8 = "\"ce_ndcg_at_10_permil\":" |
| 107 | const BE_JSON_CEF_KEY: *u8 = "\"cefusion_ndcg_at_10_permil\":" |
| 108 | const BE_JSON_PRF_KEY: *u8 = "\"prf_ndcg_at_10_permil\":" |
| 111 | const BE_PRF_FB_DOCS: i64 = 10 |
| 112 | const BE_PRF_FB_TERMS: i64 = 10 |
| 113 | const BE_PRF_ORIG_PERMIL: i64 = 500 |
| 114 | const BE_PRF_TOP: i64 = 10 |
| 115 | const BE_JSON_CAP: i64 = 65536 // a per-split JSON receipt is a few KB; a capture that fills this is REFUSED, never averaged |
| 116 | const BE_SPLIT_TIMEOUT_MS: i64 = 3600000 // one hour per split: leetcode (413932 documents) bounds it |
| 117 | const BE_LB_A: *u8 = "buildroot/knowledge/compare/search.leaderboard" |
| 118 | const BE_LB_B: *u8 = "knowledge/compare/search.leaderboard" |
| 119 | const BE_LB_TMP: *u8 = ".tmp" |
| 120 | const BE_LB_ROW: *u8 = "lb|bright|" |
| 121 | const BE_LB_SYSTEM: *u8 = "nishi-search" |
| 125 | const BE_LB_URL: *u8 = "https://nishifamily.com/compare/search" |
| 126 | const BE_LB_F_SYSTEM: i64 = 3 |
| 127 | const BE_LB_F_SCORE: i64 = 5 |
| 128 | const BE_LB_SLACK: i64 = 512 |
| 129 | const BE_JSON_NDCG_KEY: *u8 = "\"ndcg_at_10_permil\":" |
| 130 | const BE_JSON_NQ_KEY: *u8 = "\"test_queries_scored\":" |
| 131 | const BE_JSON_DOCS_KEY: *u8 = "\"docs\":" |
| 132 | const BE_SECS_PER_DAY: i64 = 86400 |
| 133 | const BE_CLOCK_WORDS: i64 = 2 |
| 134 | const BE_DATE_LEN: i64 = 10 |
| 135 | const BE_SPLIT_0: *u8 = "biology" |
| 136 | const BE_SPLIT_1: *u8 = "earth_science" |
| 137 | const BE_SPLIT_2: *u8 = "economics" |
| 138 | const BE_SPLIT_3: *u8 = "psychology" |
| 139 | const BE_SPLIT_4: *u8 = "robotics" |
| 140 | const BE_SPLIT_5: *u8 = "stackoverflow" |
| 141 | const BE_SPLIT_6: *u8 = "sustainable_living" |
| 142 | const BE_SPLIT_7: *u8 = "leetcode" |
| 143 | const BE_SPLIT_8: *u8 = "pony" |
| 144 | const BE_SPLIT_9: *u8 = "aops" |
| 145 | const BE_SPLIT_10: *u8 = "theoremqa_questions" |
| 146 | const BE_SPLIT_11: *u8 = "theoremqa_theorems" |
| 426 | const BE_CE_PAIR_CEILING_MS: i64 = 134000 // T11 nx_bert_ce_gate 2026-09-15: one 512-position pair, SERIAL, on the loaded NAS = 133,787,252 us |
functions
| 149 | func be_read(path: *u8, slot: *i64) -> i64 |
| 160 | func be_read_fb(pa: *u8, pb: *u8, slot: *i64) -> i64 |
| 166 | func be_slen(s: *u8) -> i64 |
| 171 | func be_streq(a: *u8, b: *u8) -> i64 |
| 183 | func be_count_byte(buf: *u8, n: i64, ch: i64) -> i64 |
| 190 | func be_maxline(buf: *u8, n: i64) -> i64 |
| 203 | func be_qpath(dst: *u8, root: *u8, qfile: *u8) -> i64 |
| 207 | func be_path3(dst: *u8, a: *u8, b: *u8, c: *u8) -> i64 |
| 222 | func be_path(dst: *u8, a: *u8, b: *u8) -> i64 { return be_path3(dst, a, b, 0 as *u8) } |
| 224 | func be_json_int(buf: *u8, n: i64, key: *u8) -> i64 |
| 252 | func be_split_name(i: i64) -> *u8 |
| 267 | func be_put2(dst: *u8, off: i64, v: i64) -> i64 |
| 273 | func be_today(dst: *u8) -> i64 |
| 291 | func be_field(buf: *u8, s: i64, e: i64, k: i64, lenp: *i64) -> i64 |
| 308 | func be_score_permil(buf: *u8, s: i64, n: i64) -> i64 |
| 322 | func be_starts_at(buf: *u8, s: i64, e: i64, pre: *u8) -> i64 |
| 329 | func be_field_is(buf: *u8, s: i64, e: i64, k: i64, want: *u8) -> i64 |
| 338 | func be_lb_write(permil: i64, today: *u8, org: *u8) -> i64 |
| 427 | func be_split_deadline_ms(setp: *u8, bm25_only: i64, pos: *i64, npos: i64, qfile: *u8) -> i64 |
| 444 | func be_bright(root: *u8, self: *u8, bm25_only: i64, pos: *i64, npos: i64, qfile: *u8) -> i64 |
| 615 | func be_find(buf: *u8, from: i64, limit: i64, ch: i64) -> i64 |
| 628 | func be_tok(buf: *u8, start: i64, end: i64, out: *i64, noff: *i64, scr: *u8) -> i64 |
| 656 | func be_qs1(a: *i64, lo: i64, hi: i64) -> i64 |
| 674 | func be_qs2(key: *i64, pay: *i64, lo: i64, hi: i64) -> i64 |
| 694 | func be_log2_1024(q: i64) -> i64 |
| 706 | func be_disc(r: i64) -> i64 |
| 725 | func be_prox(cbuf: *u8, ts: i64, te: i64, qh: *i64, nqh: i64, scr: *u8, bp: *i64, out: *i64) -> i64 |
| 776 | func be_atoi(s: *u8) -> i64 |
| 808 | func be_ce_task_at(tasks: *u8, i: i64) -> *BeCeTask { return ((tasks as i64) + i * BE_CE_TASK_BYTES) as *BeCeTask } |
| 809 | func _be_ce_task(ctx_i: i64) -> i64 |
| 817 | func be_dec(dst: *u8, off: i64, v: i64) -> i64 |
| 829 | func be_scat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64 = off; var i: i64 = 0; while s[i] != (0 as u8) { dst[o] = s[i]; o = o + 1; i = i + 1 } return o } |
| 835 | func be_progress_path(dst: *u8, qpath: *u8, tmp: i64) -> i64 |
| 852 | func be_progress(qpath: *u8, done: i64, pairs: i64, workers: i64, depth: i64) -> i64 |
| 881 | func main(argc: i64, argv: *i64) -> i64 |