code wiki / _hdl_build / nx_beir_arms_gate.nx
`nx_beir_arms_gate.nx` source

1// nx_beir_arms_gate.nx -- GATE for nx_beir_arms_lib: the arm the BRIGHT sweep writes to the leaderboard is chosen by
2// the declared rules and by nothing else. Every branch is driven IN-PROCESS with the sweep's own inputs (averages,
3// the fusion rule's outcome, how many splits measured the reranker and the feedback arm), so a mutation bite reaches
4// each rule.
5// license_tier: ORIGINAL   No hw writes (Rule 26).
6import "nx_syscalls.nx"
7import "nx_beir_arms_lib.nx"
8import "nx_gate_verdict.nx"
9
10const AG_SPLITS: i64 = 12
11const AG_NONE: i64 = 0 - 1
12
13func ag_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
14
15func main() -> i64 {
16    let ctr: *i64 = gv_ctr()
17    gv_head("nx_beir_arms -- the BRIGHT leaderboard arm is chosen by the pre-declared rules from the sweep's own averages" as *u8)
18
19    // plain only: nothing else eligible
20    gv_check("T1 plain BM25 alone chooses the plain arm" as *u8, (be_lb_choose(150, AG_NONE, 0, 0, AG_NONE, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
21    // the BM25Q fusion arm under its own rule
22    gv_check("T2 fusion whose rule held and whose average is at least plain is chosen" as *u8, (be_lb_choose(150, 155, 1, 0, AG_NONE, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_FUSED) as i64, ctr)
23    gv_check("T3 fusion whose rule FAILED is never chosen, whatever its average" as *u8, (be_lb_choose(150, 190, 0, 0, AG_NONE, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
24    gv_check("T4 fusion equal to plain is chosen (at least), fusion below plain is not" as *u8, (be_lb_choose(150, 150, 1, 0, AG_NONE, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_FUSED) as i64 & (be_lb_choose(150, 149, 1, 0, AG_NONE, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
25    // the cross-encoder arm: twelve of twelve, exceeding plain and the best so far
26    gv_check("T5 crossenc measured on every split and above plain and fusion is chosen" as *u8, (be_lb_choose(150, 155, 1, 0, AG_NONE, AG_SPLITS, 170, 0, AG_SPLITS) == BA_ARM_CE) as i64, ctr)
27    gv_check("T6 crossenc measured on eleven of twelve splits is NEVER chosen -- a partial average is not a placement" as *u8, (be_lb_choose(150, 155, 1, 0, AG_NONE, AG_SPLITS - 1, 300, 300, AG_SPLITS) == BA_ARM_FUSED) as i64, ctr)
28    gv_check("T7 crossenc EQUAL to plain is not chosen (the R0 rule says EXCEED)" as *u8, (be_lb_choose(150, AG_NONE, 0, 0, AG_NONE, AG_SPLITS, 150, 0, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
29    gv_check("T8 crossenc above plain but below the fused arm leaves the fused arm on the row" as *u8, (be_lb_choose(150, 160, 1, 0, AG_NONE, AG_SPLITS, 155, 0, AG_SPLITS) == BA_ARM_FUSED) as i64, ctr)
30    // the cross-encoder fusion arm
31    gv_check("T9 cefusion strictly above crossenc displaces it" as *u8, (be_lb_choose(150, 155, 1, 0, AG_NONE, AG_SPLITS, 170, 171, AG_SPLITS) == BA_ARM_CEF) as i64, ctr)
32    gv_check("T10 cefusion tied with crossenc keeps the simpler pure rerank" as *u8, (be_lb_choose(150, 155, 1, 0, AG_NONE, AG_SPLITS, 170, 170, AG_SPLITS) == BA_ARM_CE) as i64, ctr)
33    gv_check("T11 cefusion above plain while crossenc is below plain is chosen on its own" as *u8, (be_lb_choose(150, AG_NONE, 0, 0, AG_NONE, AG_SPLITS, 140, 160, AG_SPLITS) == BA_ARM_CEF) as i64, ctr)
34    gv_bite("neg-control-unmeasured-reranker a reranker measured on ZERO splits is ignored even with a huge stale value, and chosen once every split measured it" as *u8, (be_lb_choose(150, AG_NONE, 0, 0, AG_NONE, 0, 900, 900, AG_SPLITS) == BA_ARM_PLAIN) as i64, (be_lb_choose(150, AG_NONE, 0, 0, AG_NONE, AG_SPLITS, 900, 0, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
35    // the feedback arm (R0g)
36    gv_check("T14 prf measured on every split and above plain and fusion is chosen" as *u8, (be_lb_choose(150, 155, 1, AG_SPLITS, 160, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PRF) as i64, ctr)
37    gv_check("T15 prf measured on eleven of twelve splits is NEVER chosen" as *u8, (be_lb_choose(150, AG_NONE, 0, AG_SPLITS - 1, 300, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PLAIN) as i64, ctr)
38    gv_check("T16 prf EQUAL to plain is not chosen (EXCEED), and prf below the fused arm leaves fusion on the row" as *u8, (be_lb_choose(150, AG_NONE, 0, AG_SPLITS, 150, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_PLAIN) as i64 & (be_lb_choose(150, 160, 1, AG_SPLITS, 158, 0, AG_NONE, AG_NONE, AG_SPLITS) == BA_ARM_FUSED) as i64, ctr)
39    gv_check("T17 a reranker strictly above prf displaces it; a reranker tied with prf keeps prf" as *u8, (be_lb_choose(150, AG_NONE, 0, AG_SPLITS, 160, AG_SPLITS, 161, 0, AG_SPLITS) == BA_ARM_CE) as i64 & (be_lb_choose(150, AG_NONE, 0, AG_SPLITS, 160, AG_SPLITS, 160, 160, AG_SPLITS) == BA_ARM_PRF) as i64, ctr)
40    // the score travels with the arm
41    gv_check("T12 the row score is the chosen arm's own average" as *u8, (be_lb_arm_score(BA_ARM_PLAIN, 150, 155, 160, 170, 171) == 150) as i64 & (be_lb_arm_score(BA_ARM_FUSED, 150, 155, 160, 170, 171) == 155) as i64 & (be_lb_arm_score(BA_ARM_PRF, 150, 155, 160, 170, 171) == 160) as i64 & (be_lb_arm_score(BA_ARM_CE, 150, 155, 160, 170, 171) == 170) as i64 & (be_lb_arm_score(BA_ARM_CEF, 150, 155, 160, 170, 171) == 171) as i64, ctr)
42    gv_check("T13 every arm has a distinct label and the reranker rows name the licensed weights" as *u8, ag_streq(be_lb_arm_label(BA_ARM_CE), "crossenc-rerank" as *u8) & ag_streq(be_lb_arm_label(BA_ARM_CEF), "crossenc-rrf-fusion" as *u8) & ag_streq(be_lb_arm_label(BA_ARM_PRF), "bm25-rm3-prf" as *u8) & (ag_streq(be_lb_arm_label(BA_ARM_FUSED), be_lb_arm_label(BA_ARM_PLAIN)) == 0) as i64, ctr)
43    gv_kv("arms" as *u8, 5)
44    gv_kv("splits_required" as *u8, AG_SPLITS)
45    return gv_verdict("BEIR-ARMS" as *u8, ctr, "the leaderboard row carries the best arm whose own pre-declared rule held in the same sweep, and never a partial arm average" as *u8)
46}
nx_beir_arms_gate.nx source

`nx_beir_arms_gate.nx` source
