nx_reader_ruler_adversary_gate.nx source
↩ module page · 120 lines · 7952 B
1// nx_reader_ruler_adversary_gate.nx -- ADVERSARY + CRITIC pass on the READER F1 RULER, independently
2// confirming (or discrediting) the 2026-07-13 finding: "the 2-shot surface-form win (+104 on the single-gold
3// standalone bench) was a METRIC ARTIFACT -- on the multi-alias on-protocol metric it did NOT transfer (SQuAD
4// 427 vs 510), so it was reverted."
5//
6// Per the sovereign critic-kernel discipline (see nx_card_adversary_gate): a RULER is trustworthy (LAW) only if
7// it DISCRIMINATES genuine-correct from an adversary built to game it. Here two rulers are compared:
8// single_gold_f1(pred, gold) = qs_f1 against ONE gold string (the standalone bench metric)
9// multi_alias_f1(pred, aliases[]) = max_i qs_f1(pred, alias_i) (the on-protocol qab metric == official SQuAD)
10// The ADVERSARY = a surface-form change ("7" -> "seven") that games single-gold WITHOUT improving true correctness.
11// A trustworthy ruler must give the adversary ZERO gain. We prove: single-gold IS gamed (delta>0), multi-alias is
12// NOT (delta==0). That mechanically CONFIRMS the revert. license_tier: ORIGINAL expect_exit: 0
13import "nx_syscalls.nx"
14import "nx_qa_score_lib.nx"
15
16func g_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
17func g_n(v: i64) -> i64 {
18 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
19 var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
20 let d: *u8 = sys_mmap(24); var k: i64 = 0
21 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
22 var i: i64 = k - 1; while i >= 0 { let o: *u8 = sys_mmap(1); o[0] = d[i]; sys_write(1, o, 1); i = i - 1 }
23 return 0
24}
25func chk(name: *u8, got: i64, want: i64, st: *i64) -> i64 {
26 if got == want { st[0] = st[0] + 1; g_puts(" PASS " as *u8) }
27 else { st[1] = st[1] + 1; g_puts(" FAIL " as *u8) }
28 g_puts(name); g_puts(" got=" as *u8); g_n(got); g_puts(" want=" as *u8); g_n(want); g_puts("\n" as *u8)
29 return 0
30}
31// assert got >= lo (LAW-style threshold check)
32func chk_ge(name: *u8, got: i64, lo: i64, st: *i64) -> i64 {
33 if got >= lo { st[0] = st[0] + 1; g_puts(" PASS " as *u8) }
34 else { st[1] = st[1] + 1; g_puts(" FAIL " as *u8) }
35 g_puts(name); g_puts(" got=" as *u8); g_n(got); g_puts(" need>=" as *u8); g_n(lo); g_puts("\n" as *u8)
36 return 0
37}
38
39// multi-alias F1 = max over gold aliases (the official SQuAD metric: normalize + max-over-golds).
40func malias_f1(pred: *u8, aliases: *i64, na: i64) -> i64 {
41 var best: i64 = 0
42 var i: i64 = 0
43 while i < na {
44 let a: *u8 = aliases[i] as *u8
45 let s: i64 = qs_f1(pred, a)
46 if s > best { best = s }
47 i = i + 1
48 }
49 return best
50}
51
52func main() -> i64 {
53 let st: *i64 = sys_mmap(32) as *i64
54 st[0] = 0; st[1] = 0
55 g_puts("=== nx_reader_ruler_adversary_gate: does the READER F1 ruler survive the surface-form red team? ===\n\n" as *u8)
56
57 // gold "seven" with its surface alias "7" (a real SQuAD answer set; the number-word rows 6/13/36/42).
58 let al_seven: *i64 = sys_mmap(4 * 8) as *i64
59 al_seven[0] = "seven" as i64
60 al_seven[1] = "7" as i64
61 let al_three: *i64 = sys_mmap(4 * 8) as *i64
62 al_three[0] = "three" as i64
63 al_three[1] = "3" as i64
64 let al_seahawks: *i64 = sys_mmap(4 * 8) as *i64
65 al_seahawks[0] = "Seattle Seahawks" as i64
66
67 g_puts("-- PHENOMENON: single-gold penalizes a surface-correct answer that multi-alias credits --\n" as *u8)
68 // T1: single-gold WRONGLY scores the digit "7" as 0 against the word gold "seven".
69 chk("T1 single_gold('7' vs 'seven')==0 (penalizes surface variant)" as *u8, qs_f1("7" as *u8, "seven" as *u8), 0, st)
70 // T2: multi-alias CORRECTLY credits "7" (it is a gold alias) -> the standard SQuAD behavior.
71 chk("T2 multi_alias('7')==1000 (credits the surface variant)" as *u8, malias_f1("7" as *u8, al_seven, 2), 1000, st)
72
73 g_puts("\n-- CRUX: under the trustworthy (multi-alias) ruler, the surface change yields ZERO gain --\n" as *u8)
74 // The 2-shot changed the answer "7" -> "seven". On multi-alias BOTH score 1000 => zero true gain.
75 let ma_digit: i64 = malias_f1("7" as *u8, al_seven, 2)
76 let ma_word: i64 = malias_f1("seven" as *u8, al_seven, 2)
77 chk("T3 multi_alias('seven')==multi_alias('7') (surface change = 0 multi-alias gain)" as *u8, ma_word - ma_digit, 0, st)
78
79 g_puts("\n-- LAW: the surface adversary INFLATES single-gold but NOT multi-alias (mirrors rows 6/13/36/42) --\n" as *u8)
80 // 1-shot answered the digit; 2-shot answered the word. Sum the ruler deltas across the 4 number-word rows.
81 // single-gold delta = (2shot - 1shot) with ONE gold string:
82 var sg_delta: i64 = 0
83 var ma_delta: i64 = 0
84 // rows 6,13 (gold seven): 1shot '7', 2shot 'seven'
85 sg_delta = sg_delta + (qs_f1("seven" as *u8, "seven" as *u8) - qs_f1("7" as *u8, "seven" as *u8))
86 sg_delta = sg_delta + (qs_f1("seven" as *u8, "seven" as *u8) - qs_f1("7" as *u8, "seven" as *u8))
87 ma_delta = ma_delta + (malias_f1("seven" as *u8, al_seven, 2) - malias_f1("7" as *u8, al_seven, 2))
88 ma_delta = ma_delta + (malias_f1("seven" as *u8, al_seven, 2) - malias_f1("7" as *u8, al_seven, 2))
89 // rows 36,42 (gold three): 1shot '3', 2shot 'three'
90 sg_delta = sg_delta + (qs_f1("three" as *u8, "three" as *u8) - qs_f1("3" as *u8, "three" as *u8))
91 sg_delta = sg_delta + (qs_f1("three" as *u8, "three" as *u8) - qs_f1("3" as *u8, "three" as *u8))
92 ma_delta = ma_delta + (malias_f1("three" as *u8, al_three, 2) - malias_f1("3" as *u8, al_three, 2))
93 ma_delta = ma_delta + (malias_f1("three" as *u8, al_three, 2) - malias_f1("3" as *u8, al_three, 2))
94 g_puts(" single_gold surface delta = " as *u8); g_n(sg_delta); g_puts(" (2-shot INFLATES it)\n" as *u8)
95 g_puts(" multi_alias surface delta = " as *u8); g_n(ma_delta); g_puts(" (2-shot gains NOTHING)\n" as *u8)
96 chk_ge("T4 single_gold gamed by surface (delta>=1000)" as *u8, sg_delta, 1000, st)
97 chk("T5 multi_alias NOT gamed (delta==0)" as *u8, ma_delta, 0, st)
98
99 g_puts("\n-- CONTROL: neither ruler is fooled by a genuinely WRONG-CONTENT answer --\n" as *u8)
100 // 2-shot binding regression row 3: answered 'Arizona Cardinals' for gold 'Seattle Seahawks' -> both rulers 0.
101 chk("T6 single_gold(wrong content)==0" as *u8, qs_f1("Arizona Cardinals" as *u8, "Seattle Seahawks" as *u8), 0, st)
102 chk("T7 multi_alias(wrong content)==0" as *u8, malias_f1("Arizona Cardinals" as *u8, al_seahawks, 1), 0, st)
103
104 g_puts("\n-- CRITIC: is multi-alias == the OFFICIAL SQuAD metric? (normalize + max-over-golds) --\n" as *u8)
105 // qs_f1 strips articles (a/an/the) -> 'broken arm' == 'a broken arm' (real row 35). Proves SQuAD normalization.
106 chk("T8 qs_f1('broken arm' vs 'a broken arm')==1000 (article strip = SQuAD norm)" as *u8, qs_f1("broken arm" as *u8, "a broken arm" as *u8), 1000, st)
107 // max-over-aliases is the official multi-answer rule -> multi_alias picks the best-matching gold.
108 chk("T9 multi_alias picks best gold (seven-set, pred 'seven')==1000" as *u8, malias_f1("seven" as *u8, al_seven, 2), 1000, st)
109
110 g_puts("\n=== VERDICT ===\n" as *u8)
111 g_puts("The single-gold ruler is GAMEABLE by surface form (T1+T4): a '7'->'seven' change it scores as +1000/row.\n" as *u8)
112 g_puts("The multi-alias ruler (== official SQuAD: T8+T9) is NOT gamed (T2+T3+T5): that change is worth 0.\n" as *u8)
113 g_puts("=> the 2-shot's +104 standalone win was a SINGLE-GOLD ARTIFACT; the on-protocol -83 is the truth.\n" as *u8)
114 g_puts("=> reverting to 1-shot (measured on the multi-alias/official ruler) was CORRECT. FINDING CONFIRMED.\n\n" as *u8)
115
116 g_puts("PASS=" as *u8); g_n(st[0]); g_puts(" FAIL=" as *u8); g_n(st[1]); g_puts("\n" as *u8)
117 if st[1] > 0 { g_puts("GATE RED\n" as *u8); return 1 }
118 g_puts("GATE GREEN -- ruler-adversary discipline confirms the revert (sovereign, mechanical, deterministic)\n" as *u8)
119 return 0
120}