nx_ws_crash_gate.nx source

1// nx_ws_crash_gate.nx -- WMS-R10: the END-TO-END CRASH INTEGRATION TEST (the capstone). 2// 3// This is the scenario that STARTED the whole arc -- "claude froze and crashed, find out where we 4// were" -- proven a NON-EVENT. It composes every durability rung at once: 5// R0/R0b framed-append floor : concurrent workers record transitions to the REFLOG with zero tear 6// R2 transition ledger : the append-only reflog = the durable source of truth 7// R8 ws_put_locked : concurrent workers also update the LIVE registry, no lost updates 8// R9 ledger_rebuild : after the crash WIPES the registry, the reflog rebuilds it 9// 10// FLOW: NWORKERS fork; worker w writes NTRANS transitions for workstream w to BOTH the reflog 11// (ledger_append) AND the live registry (ws_put_locked), ending at a deterministic terminal state. 12// Then we simulate a CRASH that loses the derived registry (a fresh, empty registry prefix) while the 13// append-only reflog survives -- the realistic failure mode. Recovery = ledger_rebuild(reflog -> new 14// registry). The end-to-end claim: EVERY workstream's state is recovered, NOTHING is lost. 15// 16// ASSERTIONS (no false green): 17// T1 floor-held : reflog has exactly NWORKERS*NTRANS lines AND flagged==0 (R0 held under 18// concurrency -> the history we recover from is complete and untorn). 19// T2 live-consistent : the concurrently-built LIVE registry has all NWORKERS workstreams at their 20// correct terminal state (R8 lost no update across the shared manifest RMW). 21// T3 recovery-complete : after the crash (wiped registry), ledger_rebuild restores all NWORKERS 22// workstreams with correct terminal states (recovered==N AND correct==N). 23// T4 load-bearing(NEG) : a wiped registry that is NOT rebuilt has ZERO of the N workstreams -- so 24// the recovery is what saves them; if this showed N, the test would be vacuous. 25// GREEN(0) iff T1..T4. Self-log via fa_appendz (dogfoods R0b). Evidence -> ws_crash_gate.log. 26// Sovereign: nx_ws_repair + nx_ws_cas + nx_ws_ledger + nx_workstream_store + nx_framed_append + nx_syscalls. 27// license_tier: ORIGINAL 28import "nx_ws_repair.nx" 29import "nx_ws_cas.nx" 30import "nx_ws_ledger.nx" 31import "nx_workstream_store.nx" 32import "nx_framed_append.nx" 33import "nx_syscalls.nx" 34 35const CW_N: i64 = 12 // concurrent workers (= workstreams 0..11) 36const CW_K: i64 = 8 // transitions per worker (12*8 = 96 < 256 manifest cap) 37const CW_MAXWS: i64 = 64 38const CW_REC: i64 = 256 39const CW_LOG: *u8 = "knowledge/status/ws_crash_gate.log" 40 41// deterministic terminal state for workstream w (spread over the 5 ledger states 0..4) 42func cw_term(w: i64) -> i64 { return (w * 3 + 2) % 5 } 43 44// stdout-only writers 45func so(s: *u8) -> i64 { 46 var n: i64 = 0 47 while s[n] != (0 as u8) { n = n + 1 } 48 sys_write(1, s, n) 49 return 0 50} 51func son(v: i64) -> i64 { 52 let bb: *u8 = sys_mmap(28) 53 var m: i64 = v 54 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 55 let t: *u8 = sys_mmap(28) 56 var k: i64 = 0 57 if m == 0 { t[0] = 48 as u8; k = 1 } 58 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 59 var i: i64 = 0 60 while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 61 sys_write(1, bb, k) 62 return 0 63} 64 65// buffer assembly 66func cw_cat(dst: *u8, off: i64, s: *u8) -> i64 { 67 var i: i64 = 0 68 while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 } 69 return off + i 70} 71func cw_catn(dst: *u8, off: i64, v: i64) -> i64 { 72 var m: i64 = v 73 var o: i64 = off 74 if m == 0 { dst[o] = 48 as u8; return o + 1 } 75 let t: *u8 = sys_mmap(28) 76 var k: i64 = 0 77 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 78 var i: i64 = 0 79 while i < k { dst[o + i] = t[k - 1 - i]; i = i + 1 } 80 return o + k 81} 82func cw_atoi(s: *u8) -> i64 { 83 var v: i64 = 0 84 var i: i64 = 0 85 while s[i] != (0 as u8) { 86 let c: i64 = s[i] as i64 87 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } 88 i = i + 1 89 } 90 return v 91} 92func cw_path(out: *u8, tag: *u8, epoch: i64, suffix: *u8) -> i64 { 93 var o: i64 = 0 94 o = cw_cat(out, o, "/tmp/" as *u8) 95 o = cw_cat(out, o, tag) 96 o = cw_catn(out, o, epoch) 97 o = cw_cat(out, o, suffix) 98 out[o] = 0 as u8 99 return o 100} 101 102// state field (field 2) int of registry record for ws under prefix; -1 if not present. 103func cw_get_state(prefix: *u8, ws: i64) -> i64 { 104 let key: *u8 = sys_mmap(128) 105 var o: i64 = 0 106 o = cw_cat(key, o, "ws:" as *u8) 107 o = cw_catn(key, o, ws) 108 key[o] = 0 as u8 109 let pq: *i64 = sys_mmap(16) as *i64 110 let lq: *i64 = sys_mmap(16) as *i64 111 if ws_get_p(prefix, key, pq, lq) != WS_FOUND { return 0 - 1 } 112 let sb: *u8 = sys_mmap(64) 113 ws_field(pq[0] as *u8, lq[0], 2, sb) 114 return cw_atoi(sb) 115} 116 117// one worker: CW_K transitions for workstream w to BOTH the reflog AND the live registry. 118func cw_worker(reflog: *u8, regp: *u8, w: i64) -> i64 { 119 let pid: i64 = __syscall(39, 0, 0, 0, 0, 0, 0) // getpid (actor) 120 let key: *u8 = sys_mmap(128) 121 var ko: i64 = 0 122 ko = cw_cat(key, ko, "ws:" as *u8) 123 ko = cw_catn(key, ko, w) 124 key[ko] = 0 as u8 125 let val: *u8 = sys_mmap(256) 126 var i: i64 = 0 127 while i < CW_K { 128 var nw: i64 = (i + 1) % 5 129 if i == CW_K - 1 { nw = cw_term(w) } 130 ledger_append(reflog, w, i % 5, nw, pid, i, w * 1000 + i) 131 var vo: i64 = 0 132 vo = cw_catn(val, vo, w) 133 vo = cw_cat(val, vo, "\tE-LIVE\t" as *u8) 134 vo = cw_catn(val, vo, nw) 135 vo = cw_cat(val, vo, "\t0\t-\t-\t-" as *u8) 136 val[vo] = 0 as u8 137 ws_put_locked(regp, key, val) 138 i = i + 1 139 } 140 return 0 141} 142 143// fork CW_N workers over (reflog, regp), wait all. 144func cw_spawn(reflog: *u8, regp: *u8) -> i64 { 145 let pids: *i64 = sys_mmap(8 * (CW_N + 4)) as *i64 146 var w: i64 = 0 147 while w < CW_N { 148 let pid: i64 = sys_fork() 149 if pid == 0 { 150 cw_worker(reflog, regp, w) 151 sys_exit(0) 152 } 153 pids[w] = pid 154 w = w + 1 155 } 156 let st: *i64 = sys_mmap(16) as *i64 157 w = 0 158 while w < CW_N { 159 sys_wait4(pids[w], st, 0) 160 w = w + 1 161 } 162 return 0 163} 164 165// count workstreams 0..CW_N-1 whose state under prefix == cw_term(w) 166func cw_count_correct(prefix: *u8) -> i64 { 167 var ok: i64 = 0 168 var w: i64 = 0 169 while w < CW_N { 170 if cw_get_state(prefix, w) == cw_term(w) { ok = ok + 1 } 171 w = w + 1 172 } 173 return ok 174} 175// count workstreams 0..CW_N-1 PRESENT (state >= 0) under prefix 176func cw_count_present(prefix: *u8) -> i64 { 177 var p: i64 = 0 178 var w: i64 = 0 179 while w < CW_N { 180 if cw_get_state(prefix, w) >= 0 { p = p + 1 } 181 w = w + 1 182 } 183 return p 184} 185 186func main() -> i64 { 187 let epoch: i64 = sys_now_realtime_sec() 188 let reflog: *u8 = sys_mmap(128) 189 let regp_live: *u8 = sys_mmap(128) 190 let regp_rec: *u8 = sys_mmap(128) 191 let regp_norec: *u8 = sys_mmap(128) 192 cw_path(reflog, "cwlog" as *u8, epoch, ".log" as *u8) 193 cw_path(regp_live, "cwlive" as *u8, epoch, "-" as *u8) 194 cw_path(regp_rec, "cwrec" as *u8, epoch, "-" as *u8) 195 cw_path(regp_norec,"cwnr" as *u8, epoch, "-" as *u8) 196 197 // ===== WORK PHASE: concurrent writers -> reflog + live registry ===== 198 cw_spawn(reflog, regp_live) 199 200 // ----- T1 floor-held: reflog complete + untorn ----- 201 let state: *i64 = sys_mmap(8 * CW_MAXWS) as *i64 202 var si: i64 = 0 203 while si < CW_MAXWS { state[si] = 0 - 1; si = si + 1 } 204 let outs: *i64 = sys_mmap(32) as *i64 205 ledger_replay(reflog, state, outs, CW_MAXWS) 206 let lines: i64 = outs[0] 207 let flagged: i64 = outs[2] 208 let want_lines: i64 = CW_N * CW_K 209 var t1: i64 = 0 210 if lines == want_lines { if flagged == 0 { t1 = 1 } } 211 212 // ----- T2 live-registry consistent (R8 lost no update) ----- 213 let live_correct: i64 = cw_count_correct(regp_live) 214 var t2: i64 = 0 215 if live_correct == CW_N { t2 = 1 } 216 217 // ===== CRASH: the derived registry is lost. RECOVER from the reflog. ===== 218 let outs2: *i64 = sys_mmap(32) as *i64 219 ledger_rebuild(reflog, regp_rec, CW_MAXWS, outs2) 220 let recovered: i64 = outs2[3] 221 let rec_correct: i64 = cw_count_correct(regp_rec) 222 var t3: i64 = 0 223 if recovered == CW_N { if rec_correct == CW_N { t3 = 1 } } 224 225 // ----- T4 NEG load-bearing: a wiped registry NOT rebuilt has nothing ----- 226 let norec_present: i64 = cw_count_present(regp_norec) 227 var t4: i64 = 0 228 if norec_present == 0 { t4 = 1 } 229 230 var green: i64 = 1 231 if t1 == 0 { green = 0 } 232 if t2 == 0 { green = 0 } 233 if t3 == 0 { green = 0 } 234 if t4 == 0 { green = 0 } 235 236 so("WMS-R10 end-to-end crash integration gate (R0+R8+R9 capstone)\n" as *u8) 237 so(" T1 floor-held reflog_lines=" as *u8); son(lines); so("/" as *u8); son(want_lines) 238 so(" flagged=" as *u8); son(flagged) 239 if t1 == 1 { so(" PASS\n" as *u8) } else { so(" FAIL\n" as *u8) } 240 so(" T2 live-consistent correct=" as *u8); son(live_correct); so("/" as *u8); son(CW_N) 241 if t2 == 1 { so(" PASS\n" as *u8) } else { so(" FAIL\n" as *u8) } 242 so(" T3 recovery-complete recovered=" as *u8); son(recovered) 243 so(" correct=" as *u8); son(rec_correct); so("/" as *u8); son(CW_N) 244 if t3 == 1 { so(" PASS\n" as *u8) } else { so(" FAIL\n" as *u8) } 245 so(" T4 load-bearing (NEG) norebuild_present=" as *u8); son(norec_present); so("/" as *u8); son(CW_N) 246 if t4 == 1 { so(" PASS(recovery-is-load-bearing)\n" as *u8) } else { so(" FAIL(vacuous)\n" as *u8) } 247 so("verdict=" as *u8) 248 if green == 1 { so("GREEN\n" as *u8) } else { so("RED\n" as *u8) } 249 250 // single assembled evidence record -> fa_appendz (dogfoods R0b) 251 let rec: *u8 = sys_mmap(CW_REC + 16) 252 var o: i64 = 0 253 o = cw_cat(rec, o, "WMS-R10-CRASH epoch=" as *u8); o = cw_catn(rec, o, epoch) 254 o = cw_cat(rec, o, " reflog_lines=" as *u8); o = cw_catn(rec, o, lines) 255 o = cw_cat(rec, o, " flagged=" as *u8); o = cw_catn(rec, o, flagged) 256 o = cw_cat(rec, o, " live=" as *u8); o = cw_catn(rec, o, live_correct) 257 o = cw_cat(rec, o, " recovered=" as *u8); o = cw_catn(rec, o, recovered) 258 o = cw_cat(rec, o, " rec_correct=" as *u8); o = cw_catn(rec, o, rec_correct) 259 o = cw_cat(rec, o, " t1=" as *u8); o = cw_catn(rec, o, t1) 260 o = cw_cat(rec, o, " t2=" as *u8); o = cw_catn(rec, o, t2) 261 o = cw_cat(rec, o, " t3=" as *u8); o = cw_catn(rec, o, t3) 262 o = cw_cat(rec, o, " t4=" as *u8); o = cw_catn(rec, o, t4) 263 o = cw_cat(rec, o, " verdict=" as *u8) 264 if green == 1 { o = cw_cat(rec, o, "GREEN" as *u8) } else { o = cw_cat(rec, o, "RED" as *u8) } 265 rec[o] = 0 as u8 266 fa_appendz(CW_LOG, rec, CW_REC) 267 268 if green == 1 { return 0 } 269 return 1 270}