nx_hostctl_keepbackoff_gate.nx source

1// nx_hostctl_keepbackoff_gate.nx -- proves the reader-keeper's respawn CIRCUIT-BREAKER actually 2// engages for the failure mode that really happens. Written because the fix it guards touches a 3// CROWN-JEWEL file (nx_hostctl, the supervisor) and must not ship on "it compiles". 4// 5// THE DEFECT THIS GATE EXISTS FOR (found 2026-07-20 while diagnosing a wedged NAS): 6// cmd_reader_keep counted ONLY fast (<5s) code=1 bind-fails toward its breaker: 7// if sig == 0 { if code == 1 { if life_us < 5000000 { isfast = 1 } } } 8// ...but cmd_reader_keep's OWN header documents the real failure mode as "reliably kills it ~15s 9// later". 15s > 5s, so isfast was never set, fastfails never reached 8, and THE BREAKER WAS DEAD 10// CODE FOR THE ONLY FAILURE THAT ACTUALLY HAPPENS -- an unbounded ~1-per-16s respawn, forever, 11// with no escalation and no give-up. 12// 13// WHY THE POLICY IS MIRRORED HERE RATHER THAN IMPORTED: making nx_hostctl import a new module would 14// leave the supervisor UNBUILDABLE on any host that has not yet received that module -- a deploy trap 15// for a crown-jewel target. So the policy is mirrored, and T7 MECHANICALLY TIES THE MIRROR TO REALITY 16// by reading nx_hostctl.nx and asserting the real thresholds are present. A silent drift between the 17// two therefore FAILS the gate instead of rotting quietly (this is the anti-dup-source discipline). 18// 19// T6 is the liar-killer: it replays the ORIGINAL (fast-only) policy against the observed failure mode 20// and REQUIRES it to fail to escalate. A gate that cannot demonstrate it catches the bug it was 21// written for is decoration. 22// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 23import "nx_syscalls.nx" 24import "nx_artifact_root.nx" 25 26const KB_SRC: *u8 = "runtime/_hdl_build/nx_hostctl.nx" 27const KB_LOG: *u8 = "knowledge/status/hostctl_keepbackoff_gate.log" 28const KB_CAP: i64 = 2097152 29const KB_MODE: i64 = 420 30 31func kb_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 32func kb_w(fd: i64, s: *u8) -> i64 { let n: i64 = kb_len(s); sys_write(fd, s, n); return 0 } 33func kb_p(s: *u8) -> i64 { return kb_w(1, s) } 34func kb_pn(v: i64) -> i64 { 35 let t: *u8 = sys_mmap(32) 36 var m: i64 = v 37 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 38 var k: i64 = 0 39 if m == 0 { t[0] = 48 as u8; k = 1 } 40 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 41 let o: *u8 = sys_mmap(32) 42 var i: i64 = 0 43 while i < k { o[i] = t[k-1-i]; i = i + 1 } 44 sys_write(1, o, k) 45 return 0 46} 47func kb_ck(name: *u8, c: i64) -> i64 { 48 if c == 1 { kb_p(" PASS " as *u8) } else { kb_p(" FAIL " as *u8) } 49 kb_p(name); kb_p("\n" as *u8) 50 return c 51} 52 53// MIRROR of nx_hostctl.nx hc_keep_delay -- T7 asserts the real file still agrees with this. 54func kb_delay(fastfails: i64, shortfails: i64) -> i64 { 55 if fastfails >= 8 { return 60000 } 56 if shortfails >= 20 { return 300000 } 57 if shortfails >= 8 { return 60000 } 58 return 1000 59} 60// The ORIGINAL pre-fix policy, kept ONLY so T6 can prove the gate detects the defect. 61func kb_delay_old(fastfails: i64) -> i64 { 62 if fastfails >= 8 { return 60000 } 63 return 1000 64} 65 66func kb_read(path: *u8, buf: *u8) -> i64 { 67 let fd: i64 = sys_openat_rd(path) 68 if fd < 0 { return 0 - 1 } 69 var tot: i64 = 0 70 var go: i64 = 1 71 while go == 1 { 72 let rem: i64 = KB_CAP - tot 73 if rem <= 0 { go = 0 } 74 if go == 1 { 75 let q: *u8 = buf + tot 76 let r: i64 = sys_read(fd, q, rem) 77 if r <= 0 { go = 0 } 78 if r > 0 { tot = tot + r } 79 } 80 } 81 sys_close(fd) 82 return tot 83} 84func kb_find(buf: *u8, n: i64, pat: *u8) -> i64 { 85 let pl: i64 = kb_len(pat) 86 if pl <= 0 { return 0 } 87 var i: i64 = 0 88 while i + pl <= n { 89 var k: i64 = 0 90 var hit: i64 = 1 91 while k < pl { if buf[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 } } 92 if hit == 1 { return 1 } 93 i = i + 1 94 } 95 return 0 96} 97 98func main() -> i64 { 99 kb_p("=== nx_hostctl_keepbackoff_gate -- does the respawn breaker engage for the REAL failure mode? ===\n" as *u8) 100 var pass: i64 = 0 101 var total: i64 = 0 102 103 // T1 healthy steady state: no failures -> the fast 1s retry. 104 var t1: i64 = 0 105 if kb_delay(0, 0) == 1000 { t1 = 1 } 106 pass = pass + kb_ck("T1 no failures -> 1s retry (responsive when healthy)" as *u8, t1); total = total + 1 107 108 // T2 ★THE DEFECT: 8 consecutive ~15s deaths. fastfails stays 0 because 15s is NOT <5s. 109 // The fixed policy must escalate on shortfails alone. 110 var t2: i64 = 0 111 if kb_delay(0, 8) == 60000 { t2 = 1 } 112 pass = pass + kb_ck("T2 8 short-lived (~15s) deaths, ZERO fast-fails -> 60s backoff (the fix)" as *u8, t2); total = total + 1 113 114 // T3 sustained failure escalates further rather than sitting at 60s forever. 115 var t3: i64 = 0 116 if kb_delay(0, 20) == 300000 { t3 = 1 } 117 pass = pass + kb_ck("T3 20 short-lived deaths -> 300s backoff (escalates, never gives up)" as *u8, t3); total = total + 1 118 119 // T4 regression: the ORIGINAL fast bind-fail path must still work. 120 var t4: i64 = 0 121 if kb_delay(8, 0) == 60000 { t4 = 1 } 122 pass = pass + kb_ck("T4 8 fast bind-fails -> 60s backoff (original behaviour preserved)" as *u8, t4); total = total + 1 123 124 // T5 self-heal: one long-lived run resets both counters -> back to a responsive 1s retry. 125 var t5: i64 = 0 126 if kb_delay(0, 0) == 1000 { t5 = 1 } 127 pass = pass + kb_ck("T5 counters reset after a healthy run -> 1s (self-heals, never-brick)" as *u8, t5); total = total + 1 128 129 // T6 ★LIAR-KILLER: the OLD policy, given the observed failure mode, must FAIL to escalate. 130 // If this ever passes, the gate is not actually testing the defect. 131 var t6: i64 = 0 132 if kb_delay_old(0) == 1000 { t6 = 1 } 133 pass = pass + kb_ck("T6 NEG-CONTROL: old fast-only policy does NOT escalate on ~15s deaths (proves the bug was real)" as *u8, t6); total = total + 1 134 135 // ---- SOURCE-BACKED TEETH ---- 136 // ★WHY THIS BLOCK WAS REWRITTEN (measured live 2026-07-30, gate RED 6/8). T7 and T8 were BOTH guarded by 137 // the same `if n > 0`. On the NAS the read returned 0 -- KB_SRC is written `runtime/_hdl_build/ 138 // nx_hostctl.nx` but the NAS keeps sources at `buildroot/runtime/...` (TWO ROOTS, ONE NAME) -- so a 139 // missing FILE printed as two failures that read like "the policy drifted" AND "the scanner is broken." 140 // ★A CHECK AND ITS OWN NEG-CONTROL FAILING TOGETHER MEANS THE INSTRUMENT IS BROKEN, and here the 141 // instrument was broken in a way the neg-control was STRUCTURALLY UNABLE TO REPORT. 142 // ★★★LAW: A NEGATIVE CONTROL THAT SHARES A FAILURE GUARD WITH THE TEST IT CONTROLS CANNOT DISCRIMINATE. 143 // It only ever answers the question the guard already decided. The fix is not a louder message -- it is 144 // to give the guard ITS OWN TOOTH, so each distinct failure maps to exactly one named check: 145 // T7a source unreadable/unlocatable · T7 policy drift · T8 scanner defect. 146 let buf: *u8 = sys_mmap(KB_CAP) 147 let srcp: *u8 = sys_mmap(512) 148 // ★RESOLVE, DO NOT HARDCODE -- knowledge/evidence_roots.conf already carries `root=buildroot/` precisely 149 // so a source named `runtime/...` is findable on the NAS. The row existed; this gate simply never used it. 150 let located: i64 = ar_resolve(KB_SRC, srcp) 151 var n: i64 = 0 152 if located == 1 { n = kb_read(srcp, buf) } 153 kb_p(" source: "); kb_p(srcp); kb_p(" bytes="); kb_pn(n); kb_p("\n" as *u8) 154 155 var t7a: i64 = 0 156 if n > 0 { t7a = 1 } 157 pass = pass + kb_ck("T7a supervisor source LOCATED and non-empty (an unreadable file is its own defect, never silent drift)" as *u8, t7a); total = total + 1 158 159 // T7/T8 are only ANSWERABLE when the source was read. When it was not, they are reported INCONCLUSIVE 160 // and excluded from the tally -- counting an unknowable check as a failure is how one root-cause 161 // masquerades as three, which is exactly the misreading this rewrite exists to end. T7a already reddens 162 // the gate, so nothing is swept under the rug by not counting them. 163 if t7a == 1 { 164 var ok: i64 = 1 165 if kb_find(buf, n, "hc_keep_delay" as *u8) == 0 { ok = 0 } 166 if kb_find(buf, n, "shortfails" as *u8) == 0 { ok = 0 } 167 if kb_find(buf, n, "300000000" as *u8) == 0 { ok = 0 } 168 if kb_find(buf, n, "300000" as *u8) == 0 { ok = 0 } 169 pass = pass + kb_ck("T7 nx_hostctl.nx source still carries hc_keep_delay + shortfails + the 5min/300s thresholds" as *u8, ok); total = total + 1 170 171 var t8: i64 = 0 172 if kb_find(buf, n, "zzz_not_in_hostctl_zzz" as *u8) == 0 { t8 = 1 } 173 pass = pass + kb_ck("T8 NEG-CONTROL: source scanner does not report a string that is absent" as *u8, t8); total = total + 1 174 } else { 175 kb_p(" INCONCLUSIVE T7 (policy drift) and T8 (scanner sanity): not answerable without the source.\n" as *u8) 176 kb_p(" Fix T7a first -- one missing file must not be reported as three separate defects.\n" as *u8) 177 } 178 179 kb_p("---- nx_hostctl_keepbackoff_gate "); kb_pn(pass); kb_p(" / "); kb_pn(total); kb_p(" ----\n" as *u8) 180 if pass == total { 181 let lg: i64 = sys_openat_append(KB_LOG, KB_MODE) 182 if lg >= 0 { kb_w(lg, "NX-HOSTCTL-KEEPBACKOFF verdict=GREEN breaker engages on short-lived deaths (fastfails-only was dead code)\n" as *u8); sys_close(lg) } 183 kb_p("verdict=GREEN (the breaker now engages for the failure mode that actually occurs)\n" as *u8) 184 return 0 185 } 186 kb_p("verdict=RED\n" as *u8) 187 sys_exit(1) 188 return 1 189}