nx_pool_stale_gate.nx source
↩ module page · 177 lines · 8243 B
1// nx_pool_stale_gate.nx -- STRESS REPRODUCER for the 2026-07-15 run-to-run greedy divergence (proven by
2// exact-repeat 1.5B ruler runs: identical binary+args+weights, different outputs at near-tie forks; 0.5B
3// byte-stable). Hypothesis class: a pooled op's nx_pool_wait(done0+bands) occasionally returns with a band
4// UNFINISHED (early wake / counter anomaly / ctx hijack) -> the next stage reads a stale band -> perturbed
5// logits flip ties. This gate hammers the EXACT usage idiom of nx_nofloat_llm's pooled kernels (delta-wait +
6// shared ctx arena + shared activation scratch, band-parallel disjoint writes) at high op counts and CHECKS
7// three invariants after EVERY wait:
8// I1 STAMP every band wrote its per-op serial stamp (no missing/unfinished band)
9// I2 VALUE every band's dst equals the exact expected sum of THIS op's activations (no stale-input read,
10// no hijacked-ctx compute)
11// I3 COUNTER tasks_completed advanced by exactly +bands across the wait (no over/under-count)
12// Any violation prints op#, band, got/want and the gate goes RED = reproducer captured. A clean 200k-op run
13// is evidence the race is NOT in this idiom (pool+chan+wait) and the hunt moves up/down a layer.
14// T1: 200k ops x n_workers bands, zero violations
15// T2 NEG-CONTROL: sabotage mode (one band deliberately not submitted, wait shortened) MUST be caught by I1
16// Return from main (pool reap). No hw writes (Rule 26). expect_exit: 0 license_tier: ORIGINAL
17import "nx_syscalls.nx"
18import "nx_thread_pool.nx"
19
20const PSG_N: i64 = 4096 // activation length per op
21const PSG_OPS: i64 = 200000 // ops (~70 tokens' worth of 1.5B submit volume)
22
23static g_psg_act: *i64 // shared activation scratch (refilled per op, like xn/hbuf)
24static g_psg_dst: *i64 // band results (disjoint writes)
25static g_psg_stamp: *i64 // per-band op-serial stamps
26static g_psg_ctx: *u8 // shared ctx arena (reused per op, like g_nf_mmarena)
27
28func psg_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
29func psg_n(v: i64) -> i64 {
30 var m: i64 = v
31 if m < 0 { psg_w("-" as *u8); m = 0 - m }
32 let t: *u8 = sys_mmap(24)
33 var k: i64 = 0
34 if m == 0 { t[0] = 48 as u8; k = 1 }
35 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
36 let o: *u8 = sys_mmap(24)
37 var i: i64 = 0
38 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
39 sys_write(1, o, k)
40 return 0
41}
42
43// one band task: dst[band] = sum of act[lo..hi) ; stamp[band] = serial. Mirrors the banded-matmul shape
44// (read shared activations, write a disjoint slot), tiny so 200k ops run in seconds.
45func _psg_task(ctx_i: i64) -> i64 {
46 let c: *i64 = ctx_i as *i64
47 let act: *i64 = c[0] as *i64
48 let lo: i64 = c[1]
49 let hi: i64 = c[2]
50 let band: i64 = c[3]
51 let serial: i64 = c[4]
52 var s: i64 = 0
53 var i: i64 = lo
54 while i < hi { s = s + act[i]; i = i + 1 }
55 g_psg_dst[band] = s
56 g_psg_stamp[band] = serial
57 return 0
58}
59
60func main() -> i64 {
61 psg_w("=== NX-POOL-STALE-GATE -- delta-wait banded-op stress (the 07-15 divergence reproducer) ===\n" as *u8)
62 let p: *NxThreadPool = nx_pool_new(0, 64)
63 var bands: i64 = p.n_workers
64 if bands > 32 { bands = 32 }
65 if bands < 2 { bands = 2 }
66 psg_w("workers=" as *u8); psg_n(p.n_workers); psg_w(" bands=" as *u8); psg_n(bands); psg_w(" ops=" as *u8); psg_n(PSG_OPS); psg_w("\n" as *u8)
67 g_psg_act = sys_mmap(PSG_N * 8) as *i64
68 g_psg_dst = sys_mmap(64 * 8) as *i64
69 g_psg_stamp = sys_mmap(64 * 8) as *i64
70 g_psg_ctx = sys_mmap(64 * 128)
71 let per: i64 = (PSG_N + bands - 1) / bands
72
73 var viol_stamp: i64 = 0
74 var viol_value: i64 = 0
75 var viol_count: i64 = 0
76 var first_op: i64 = 0 - 1
77 let t0: i64 = sys_now_ms()
78 var op: i64 = 1
79 while op <= PSG_OPS {
80 // refill shared activations with op-specific values (like the per-step xn/hbuf refill)
81 var k: i64 = 0
82 while k < PSG_N { g_psg_act[k] = op * 7 + (k % 13); k = k + 1 }
83 // expected band sums, computed serially (the oracle)
84 // submit bands exactly like mm_pool: delta-wait on the shared pool
85 let done0: i64 = nx_pool_n_completed(p)
86 var b: i64 = 0
87 while b < bands {
88 let c: *i64 = ((g_psg_ctx as i64) + b * 128) as *i64
89 c[0] = g_psg_act as i64
90 c[1] = b * per
91 var hi: i64 = (b + 1) * per
92 if hi > PSG_N { hi = PSG_N }
93 c[2] = hi
94 c[3] = b
95 c[4] = op
96 nx_pool_submit(p, _psg_task, c as i64)
97 b = b + 1
98 }
99 nx_pool_wait(p, done0 + bands)
100 let done1: i64 = nx_pool_n_completed(p)
101 // I3 counter delta (== bands unless strays exist)
102 if done1 - done0 != bands { viol_count = viol_count + 1; if first_op < 0 { first_op = op } }
103 // I1 stamps + I2 values
104 b = 0
105 while b < bands {
106 if g_psg_stamp[b] != op {
107 viol_stamp = viol_stamp + 1
108 if first_op < 0 { first_op = op }
109 if viol_stamp < 4 { psg_w(" I1 STALE-BAND op=" as *u8); psg_n(op); psg_w(" band=" as *u8); psg_n(b); psg_w(" stamp=" as *u8); psg_n(g_psg_stamp[b]); psg_w("\n" as *u8) }
110 } else {
111 var want: i64 = 0
112 var i2: i64 = b * per
113 var hi2: i64 = (b + 1) * per
114 if hi2 > PSG_N { hi2 = PSG_N }
115 while i2 < hi2 { want = want + (op * 7 + (i2 % 13)); i2 = i2 + 1 }
116 if g_psg_dst[b] != want {
117 viol_value = viol_value + 1
118 if first_op < 0 { first_op = op }
119 if viol_value < 4 { psg_w(" I2 BAD-VALUE op=" as *u8); psg_n(op); psg_w(" band=" as *u8); psg_n(b); psg_w(" got=" as *u8); psg_n(g_psg_dst[b]); psg_w(" want=" as *u8); psg_n(want); psg_w("\n" as *u8) }
120 }
121 }
122 b = b + 1
123 }
124 op = op + 1
125 }
126 let t1: i64 = sys_now_ms()
127 psg_w("ops=" as *u8); psg_n(PSG_OPS); psg_w(" ms=" as *u8); psg_n(t1 - t0)
128 psg_w(" violations: stamp=" as *u8); psg_n(viol_stamp)
129 psg_w(" value=" as *u8); psg_n(viol_value)
130 psg_w(" counter=" as *u8); psg_n(viol_count)
131 if first_op >= 0 { psg_w(" first_op=" as *u8); psg_n(first_op) }
132 psg_w("\n" as *u8)
133
134 var pass: i64 = 0
135 var ttl: i64 = 0
136 ttl = ttl + 1
137 let ok1a: i64 = (viol_stamp == 0) as i64
138 let ok1b: i64 = (viol_value == 0) as i64
139 let ok1c: i64 = (viol_count == 0) as i64
140 var ok1: i64 = ok1a & ok1b
141 ok1 = ok1 & ok1c
142 psg_w(" T1 200k delta-wait ops, zero stale/value/counter violations: " as *u8)
143 if ok1 == 1 { pass = pass + 1; psg_w("PASS\n" as *u8) } else { psg_w("FAIL (REPRODUCED)\n" as *u8) }
144
145 // T2 NEG-CONTROL: submit bands-1 tasks but wait as if bands, with a 200ms grace -- the missing band's
146 // stamp MUST be stale (proves I1 has teeth; also proves wait cannot invent completions).
147 var k2: i64 = 0
148 while k2 < PSG_N { g_psg_act[k2] = 999983 * 7 + (k2 % 13); k2 = k2 + 1 }
149 let doneN: i64 = nx_pool_n_completed(p)
150 var b2: i64 = 0
151 while b2 < bands - 1 {
152 let c2: *i64 = ((g_psg_ctx as i64) + b2 * 128) as *i64
153 c2[0] = g_psg_act as i64
154 c2[1] = b2 * per
155 var hi3: i64 = (b2 + 1) * per
156 if hi3 > PSG_N { hi3 = PSG_N }
157 c2[2] = hi3
158 c2[3] = b2
159 c2[4] = 999983
160 nx_pool_submit(p, _psg_task, c2 as i64)
161 b2 = b2 + 1
162 }
163 nx_pool_wait(p, doneN + bands - 1)
164 let missing: i64 = bands - 1
165 ttl = ttl + 1
166 let ok2: i64 = (g_psg_stamp[missing] != 999983) as i64
167 psg_w(" T2 NEG-CONTROL missing band detected by the stamp check: " as *u8)
168 if ok2 == 1 { pass = pass + 1; psg_w("PASS\n" as *u8) } else { psg_w("FAIL\n" as *u8) }
169
170 psg_w("NX-POOL-STALE-GATE passed " as *u8); psg_n(pass); psg_w("/" as *u8); psg_n(ttl)
171 if pass == ttl {
172 psg_w(" verdict=GREEN (idiom clean at 200k ops -- the divergence lives elsewhere; move the hunt)\n" as *u8)
173 return 0
174 }
175 psg_w(" verdict=RED (race REPRODUCED in the pool idiom -- fix here, this gate is the ratchet)\n" as *u8)
176 return 1
177}