code wiki / _hdl_build / nx_beir_eval.nx

nx_beir_eval.nx

buildroot/runtime/_hdl_build/nx_beir_eval.nx

118813 B2118 linesdepth 10pulls 40 transitivereach 0 importersview sourcekind tooltopic beir
docsdependenciesstructsconstsfunctions

about

nx_beir_eval.nx -- SOVEREIGN BEIR harness: real external ground truth (BEIR/nfcorpus) scored by our own integer BM25 -> mean nDCG@10 (the standard BEIR metric). The HONEST external row the maturity ladder is gated on (F236): real 3633-doc corpus + 323 test queries + 12335 graded qrels, vs the published BEIR BM25 baseline (nfcorpus nDCG@10 ~0.32, Anserini). BM25 k1=0.9 b=0.4 (BEIR defaults). IDF=log2 (ranking-invariant vs ln -> identical nDCG). Integer/fixed-point throughout (no float) = bit-reproducible. Forward index (per-doc sorted term,tf). Tokenizer = lowercase [a-z0-9] runs -> PORTER STEM via nx_stem_lib (2026-08-13), NO stopwords (the stemming rung LANDED -- was the named gap to the stemmed 0.32 Anserini baseline). R0d be_bright (2026-09-14): `--set <root>` scores ANY BEIR-shaped set root (<root>/corpus.tsv, queries.tsv, qrels/test.tsv and, when present, excluded.tsv: BRIGHT lists each query's own source documents there and the benchmark removes them from that query's ranking, so this harness marks them unrankable for that query and counts them). Every capacity is now DERIVED from the files (sys_read_file sizes the buffer, newline counts size the tables, the longest line sizes the token scratch): the old 64 MiB file cap and 4096-document cap would have dropped 99 percent of BRIGHT's leetcode split in silence, which is the cap class this estate refuses. `--bm25-only` skips the rerank arms (their models were trained on the web corpus; BRIGHT's published floor is plain BM25, 14.5 average nDCG at 10 on the leaderboard page read 2026-09-14, search.refs bright). `--bright <root>` is be_bright: this binary runs itself once per split through tr_run_capture_to (one hour each, the largest split bounds it), prints one JSON row per split, the average over twelve of twelve (never over a partial set), and writes the estate's row into search.leaderboard beside the published rows with its rank DERIVED from their scores. The positional grammar is unchanged: [model.safetensors config.json vocab.txt [max_len [depth [workers]]]] after the flags; workers is the cross-encoder pool width (0 = serial). expect_exit: 0 license_tier: ORIGINAL

dependencies 12 imports · 0 importers

nx_qabench_engine.nx nx_ppmi_lib.nx nx_stem.nx nx_ltr_lib.nx nx_dr_densejudge.nx nx_wordpiece_lib.nx nx_bert_ce_lib.nx nx_thread_pool.nx nx_tool_run.nx nx_timefmt.nx nx_beir_eval.nx

diagram shows first 10 each side; +2 more imports, +0 more importers in the complete lists below.

imports: nx_qabench_engine.nxnx_ppmi_lib.nxnx_stem.nxnx_ltr_lib.nxnx_dr_densejudge.nxnx_wordpiece_lib.nxnx_bert_ce_lib.nxnx_thread_pool.nxnx_tool_run.nxnx_timefmt.nxnx_intlog.nxnx_beir_arms_lib.nx

imported by: nobody (leaf or entry point)

call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown

main be_streq be_starts_at be_slen be_slen ↻ be_bright be_today civil_from_days be_put2 db_w db_n nxi_out nxi_fd sys_mmap nxa_die nxa_lock_take nxa_lock_give nxa_report_overrun ccz_cat_num nxi_buf sys_write sys_munmap be_split_name be_path3 be_slen ↻ be_path be_path3 ↻ tr_run_capture_to tr_run_capture_tr tr_run_capture_core tr_run_capture_core_owned tr_run_capture_core_eviden be_split_deadline_ms be_atoi be_qpath be_path ↻ be_path3 ↻ be_count_byte be_json_int be_slen ↻

structs

794struct BeCeTask

consts

34const BE_ASCII_ZERO: i64 = 48
35const BE_ASCII_NINE: i64 = 57
36const BE_DECIMAL: i64 = 10
37const BE_MAGIC_1024: i64 = 1024
38const BE_MAGIC_1000000: i64 = 1000000
45const BE_KCAND: i64 = 50
47const BE_IDCAP: i64 = 4096
48const K_MAGIC_1024: i64 = 1024
49const K_MAGIC_4096: i64 = 4096
50const K_MAGIC_1900: i64 = 1900
52const BE_FLAG_SET: *u8 = "--set"
53const BE_FLAG_BRIGHT: *u8 = "--bright"
54const BE_CE_POOL_WORKERS_DEFAULT: i64 = 0 // 0 = every forward on the calling thread; the 6th positional sets the pool width, and since 2026-09-15 the pool's unit of work is ONE WHOLE PAIR (a forward), never a band of one matmul: the forward inside a task is the serial path, so a pooled run scores every pair identically to a serial run and only the wall clock moves (per-matmul banding was measured as NO speedup, search.plan row 1789456036)
55const BE_CE_TASK_BYTES: i64 = 96 // BeCeTask: 12 i64 fields; one task record and one scratch set PER CANDIDATE so no two tasks ever share a buffer
59const BE_PROGRESS_DIR: *u8 = "knowledge/status/beir_eval."
60const BE_PROGRESS_SFX: *u8 = ".progress"
61const BE_PROGRESS_SFX_TMP: *u8 = ".progress.tmp"
62const BE_SLASH: i64 = 47
63const BE_DOT: i64 = 46
64const BE_USCORE: i64 = 95
65const BE_PROGRESS_BYTES: i64 = 640
66const BE_FLAG_BM25ONLY: *u8 = "--bm25-only"
70const BE_FLAG_QUERIES: *u8 = "--queries"
71const BE_ROOT_DEFAULT: *u8 = "knowledge/beir/nfcorpus"
72const BE_ROOT_DEFAULT_TMP: *u8 = "/tmp/beir/nfcorpus"
73const BE_DATASET_DEFAULT: *u8 = "BEIR/nfcorpus"
74const BE_F_CORPUS: *u8 = "/corpus.tsv"
75const BE_F_QRELS: *u8 = "/qrels/test.tsv"
76const BE_F_QUERIES: *u8 = "/queries.tsv"
77const BE_F_EXCLUDED: *u8 = "/excluded.tsv"
78const BE_SEP: *u8 = "/"
79const BE_PATH_CAP: i64 = 1024
80const BE_POS_CAP: i64 = 16
81const BE_NL: i64 = 10
82const BE_TAB: i64 = 9
83const BE_PIPE: i64 = 124
84const BE_DOT: i64 = 46
85const BE_MINUS: i64 = 45
86const BE_I64: i64 = 8
87const BE_G_SLOTS: i64 = 8
88const BE_TOK_BYTES: i64 = 2 // a token needs one byte and a separator, so tokens <= bytes / 2 + lines
89const BE_PERMIL_DIGITS: i64 = 10 // the board writes nDCG x 100 with one decimal; permil / 10 . permil % 10
90const BE_BRIGHT_N: i64 = 12
91const BE_PT_TF_MOD: i64 = 1048576 // posting word radix (2^20): doc * radix + tf; a tf at or past it is clamped and COUNTED, never silently wrapped
92const BE_BRIGHT_BM25_PUBLISHED_PERMIL: i64 = 145 // the leaderboard page's plain BM25 row, read 2026-09-14 (search.refs bright)
95const BE_JSON_BQ_KEY: *u8 = "k_q10\":1228,\"ndcg_at_10_permil\":"
96const BE_JSON_FQ_KEY: *u8 = "bm25q-rank, same 50 candidates\",\"ndcg_at_10_permil\":"
97const BE_JSON_LONGN_KEY: *u8 = "\"long_n\":"
98const BE_JSON_LONGD_KEY: *u8 = "\"long_bm25_permil\":"
99const BE_JSON_LONGQ_KEY: *u8 = "\"long_bm25q_permil\":"
100const BE_JSON_LONGF_KEY: *u8 = "\"long_fusion_permil\":"
101const BE_JSON_NOREP_KEY: *u8 = "\"norepeat_n\":"
102const BE_JSON_NOREPSAME_KEY: *u8 = "\"norepeat_top10_identical\":"
103const BE_JSON_REP_KEY: *u8 = "\"queries_with_repeated_term\":"
105const BE_JSON_CEM_KEY: *u8 = "\"ce_measured\":"
106const BE_JSON_CE_KEY: *u8 = "\"ce_ndcg_at_10_permil\":"
107const BE_JSON_CEF_KEY: *u8 = "\"cefusion_ndcg_at_10_permil\":"
108const BE_JSON_PRF_KEY: *u8 = "\"prf_ndcg_at_10_permil\":"
111const BE_PRF_FB_DOCS: i64 = 10
112const BE_PRF_FB_TERMS: i64 = 10
113const BE_PRF_ORIG_PERMIL: i64 = 500
114const BE_PRF_TOP: i64 = 10
115const BE_JSON_CAP: i64 = 65536 // a per-split JSON receipt is a few KB; a capture that fills this is REFUSED, never averaged
116const BE_SPLIT_TIMEOUT_MS: i64 = 3600000 // one hour per split: leetcode (413932 documents) bounds it
117const BE_LB_A: *u8 = "buildroot/knowledge/compare/search.leaderboard"
118const BE_LB_B: *u8 = "knowledge/compare/search.leaderboard"
119const BE_LB_TMP: *u8 = ".tmp"
120const BE_LB_ROW: *u8 = "lb|bright|"
121const BE_LB_SYSTEM: *u8 = "nishi-search"
125const BE_LB_URL: *u8 = "https://nishifamily.com/compare/search"
126const BE_LB_F_SYSTEM: i64 = 3
127const BE_LB_F_SCORE: i64 = 5
128const BE_LB_SLACK: i64 = 512
129const BE_JSON_NDCG_KEY: *u8 = "\"ndcg_at_10_permil\":"
130const BE_JSON_NQ_KEY: *u8 = "\"test_queries_scored\":"
131const BE_JSON_DOCS_KEY: *u8 = "\"docs\":"
132const BE_SECS_PER_DAY: i64 = 86400
133const BE_CLOCK_WORDS: i64 = 2
134const BE_DATE_LEN: i64 = 10
135const BE_SPLIT_0: *u8 = "biology"
136const BE_SPLIT_1: *u8 = "earth_science"
137const BE_SPLIT_2: *u8 = "economics"
138const BE_SPLIT_3: *u8 = "psychology"
139const BE_SPLIT_4: *u8 = "robotics"
140const BE_SPLIT_5: *u8 = "stackoverflow"
141const BE_SPLIT_6: *u8 = "sustainable_living"
142const BE_SPLIT_7: *u8 = "leetcode"
143const BE_SPLIT_8: *u8 = "pony"
144const BE_SPLIT_9: *u8 = "aops"
145const BE_SPLIT_10: *u8 = "theoremqa_questions"
146const BE_SPLIT_11: *u8 = "theoremqa_theorems"
426const BE_CE_PAIR_CEILING_MS: i64 = 134000 // T11 nx_bert_ce_gate 2026-09-15: one 512-position pair, SERIAL, on the loaded NAS = 133,787,252 us

functions

149func be_read(path: *u8, slot: *i64) -> i64
160func be_read_fb(pa: *u8, pb: *u8, slot: *i64) -> i64
166func be_slen(s: *u8) -> i64
171func be_streq(a: *u8, b: *u8) -> i64
183func be_count_byte(buf: *u8, n: i64, ch: i64) -> i64
190func be_maxline(buf: *u8, n: i64) -> i64
203func be_qpath(dst: *u8, root: *u8, qfile: *u8) -> i64
called by 3: be_qpathbe_pathbe_bright calls 1: be_slen
207func be_path3(dst: *u8, a: *u8, b: *u8, c: *u8) -> i64
222func be_path(dst: *u8, a: *u8, b: *u8) -> i64 { return be_path3(dst, a, b, 0 as *u8) }
224func be_json_int(buf: *u8, n: i64, key: *u8) -> i64
252func be_split_name(i: i64) -> *u8
267func be_put2(dst: *u8, off: i64, v: i64) -> i64
273func be_today(dst: *u8) -> i64
291func be_field(buf: *u8, s: i64, e: i64, k: i64, lenp: *i64) -> i64
308func be_score_permil(buf: *u8, s: i64, n: i64) -> i64
322func be_starts_at(buf: *u8, s: i64, e: i64, pre: *u8) -> i64
329func be_field_is(buf: *u8, s: i64, e: i64, k: i64, want: *u8) -> i64
338func be_lb_write(permil: i64, today: *u8, org: *u8) -> i64
427func be_split_deadline_ms(setp: *u8, bm25_only: i64, pos: *i64, npos: i64, qfile: *u8) -> i64
444func be_bright(root: *u8, self: *u8, bm25_only: i64, pos: *i64, npos: i64, qfile: *u8) -> i64
615func be_find(buf: *u8, from: i64, limit: i64, ch: i64) -> i64
628func be_tok(buf: *u8, start: i64, end: i64, out: *i64, noff: *i64, scr: *u8) -> i64
656func be_qs1(a: *i64, lo: i64, hi: i64) -> i64
674func be_qs2(key: *i64, pay: *i64, lo: i64, hi: i64) -> i64
694func be_log2_1024(q: i64) -> i64
706func be_disc(r: i64) -> i64
725func be_prox(cbuf: *u8, ts: i64, te: i64, qh: *i64, nqh: i64, scr: *u8, bp: *i64, out: *i64) -> i64
776func be_atoi(s: *u8) -> i64
808func be_ce_task_at(tasks: *u8, i: i64) -> *BeCeTask { return ((tasks as i64) + i * BE_CE_TASK_BYTES) as *BeCeTask }
809func _be_ce_task(ctx_i: i64) -> i64
817func be_dec(dst: *u8, off: i64, v: i64) -> i64
829func be_scat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64 = off; var i: i64 = 0; while s[i] != (0 as u8) { dst[o] = s[i]; o = o + 1; i = i + 1 } return o }
835func be_progress_path(dst: *u8, qpath: *u8, tmp: i64) -> i64
852func be_progress(qpath: *u8, done: i64, pairs: i64, workers: i64, depth: i64) -> i64
881func main(argc: i64, argv: *i64) -> i64