nx_cachewatch.nx source
↩ module page · 306 lines · 13306 B
1// nx_cachewatch.nx -- IS THE WRITE CACHE ACTUALLY CACHING? The axis nothing in the estate measured.
2//
3// WHY THIS EXISTS (2026-09-03). The box spent a full day refusing every build with load 13-25 while
4// `/api/build` correctly reported an I/O storm. The CAUSE was one condition no organ could see:
5// `md3 : active raid1 nvme1n1p1[1] [2/1] [_U]` -- the NVMe RAID1 behind the write cache lost a device,
6// so the cache was flushed to completion and set `uncacheable`, and from that moment 100% of writes went
7// straight to the RAID5 array. Measured on the live plane: `writes +18,366` == `disk_writes +18,366` ==
8// `uncached_writes +18,366` across two spaced samples, with every SSD-side counter frozen.
9// MEASURED ABSENCE BEFORE BUILDING: `grep -rl flashcache buildroot/runtime/*.nx` returned ZERO organs.
10// The estate could see the SYMPTOM (load, D-state, refused builds) and never the CAUSE.
11//
12// nx_cachewatch check [proc_root] -> one verdict line, exit carries it
13// nx_cachewatch explain [proc_root] -> every field it read, with its LEVEL/COUNTER class named
14//
15// IT READS LEVELS AND REFUSES TO READ COUNTERS AS LEVELS. THIS IS THE WHOLE POINT.
16// `flashcache_stats` carries `dirty_writeback_kb=31543788` and `dirty_write_hits=43493286`. Both are
17// LIFETIME COUNTERS sitting among reads/writes/hits. Read as levels they say "31.5 GB of dirty data is
18// stranded on a degraded unmirrored NVMe" -- an alarming, actionable, WRONG conclusion. The estate's
19// memory recorded exactly that error, and this organ's author was two minutes from publishing it again.
20// The LEVEL lives in `cache_info` as `dirty_blocks`, and it reads 0.
21// A FROZEN COUNTER AND A PINNED LEVEL ARE INDISTINGUISHABLE IN ONE SAMPLE, AND THE ALARMING READING IS
22// THE ONE THAT GETS PUBLISHED. So this organ names the class of every number it prints, and the
23// verdict is computed ONLY from fields it can justify as levels.
24//
25// MODE IS NOT THE DISCRIMINATOR. `cache_info` keeps saying `mode=WRITE_BACK` after the cache has been
26// flushed and disabled -- that field records how the device was CONFIGURED, not whether it is caching
27// now. The discriminator is `flashcache_progress: status=uncacheable...`. A verdict built on `mode`
28// alone would have read GREEN through the entire outage. Proven by its gate's T2: two fixtures with
29// BYTE-IDENTICAL cache_info must return DIFFERENT verdicts.
30//
31// exit: 0 GREEN (caching, mirror intact) | 1 RED (NOT caching -- every write hits the array)
32// | 2 AMBER (caching, mirror degraded -- unmirrored dirty data) | 3 UNOBSERVABLE | 4 usage
33// UNOBSERVABLE IS NOT GREEN. A missing /proc/flashcache means this host has no such cache OR the
34// reader is wrong; both are "I could not look", and neither acquits.
35// PROVEN 8/8 GREEN 2026-09-03, bite-proven killed=1, restores byte-identical.
36// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
37import "nx_syscalls.nx"
38
39const CW_BUF: i64 = 65536
40const CW_NAME: i64 = 512
41const CW_PATH: i64 = 1024
42const CW_DIRBUF: i64 = 65536
43const CW_DENT_RECLEN_OFF: i64 = 16
44const CW_DENT_NAME_OFF: i64 = 19
45const CW_GETDENTS: i64 = 217
46const CW_OPENDIR_FLAGS: i64 = 0x10000
47const CW_AT_FDCWD: i64 = 0 - 100
48const CW_DOT: i64 = 46
49
50const CW_GREEN: i64 = 0
51const CW_RED: i64 = 1
52const CW_AMBER: i64 = 2
53const CW_UNOBS: i64 = 3
54
55func cw_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
56func cw_out(s: *u8) -> i64 { sys_write(1, s, cw_slen(s)); return 0 }
57func cw_num(v: i64) -> i64 {
58 let b: *u8 = sys_mmap(32)
59 var m: i64 = v
60 var i: i64 = 31
61 var neg: i64 = 0
62 if m < 0 { neg = 1; m = 0 - m }
63 if m == 0 { i = i - 1; b[i] = 48 as u8 }
64 while m > 0 { i = i - 1; b[i] = (48 + (m % 10)) as u8; m = m / 10 }
65 if neg == 1 { i = i - 1; b[i] = 45 as u8 }
66 sys_write(1, (b as i64 + i) as *u8, 31 - i)
67 return 0
68}
69func cw_cat(dst: *u8, a: *u8, b: *u8, c: *u8) -> i64 {
70 var o: i64 = 0
71 var i: i64 = 0
72 while a[i] != (0 as u8) { dst[o] = a[i]; o = o + 1; i = i + 1 }
73 i = 0
74 while b[i] != (0 as u8) { dst[o] = b[i]; o = o + 1; i = i + 1 }
75 i = 0
76 while c[i] != (0 as u8) { dst[o] = c[i]; o = o + 1; i = i + 1 }
77 dst[o] = 0 as u8
78 return o
79}
80func cw_read(path: *u8, buf: *u8, cap: i64) -> i64 {
81 let fd: i64 = sys_openat_rd(path)
82 if fd < 0 { return 0 - 1 }
83 var got: i64 = 0
84 var go: i64 = 1
85 while go == 1 {
86 if got >= cap { go = 0 } else {
87 let r: i64 = sys_read(fd, (buf as i64 + got) as *u8, cap - got)
88 if r <= 0 { go = 0 } else { got = got + r }
89 }
90 }
91 sys_close(fd)
92 return got
93}
94// index just past `needle` in buf[0..n), or -1. Exits on a FLAG, never by clobbering the cursor.
95func cw_find(buf: *u8, n: i64, needle: *u8) -> i64 {
96 let m: i64 = cw_slen(needle)
97 if m == 0 { return 0 - 1 }
98 var i: i64 = 0
99 var hit: i64 = 0 - 1
100 while i + m <= n {
101 var j: i64 = 0
102 var same: i64 = 1
103 while j < m {
104 if buf[i + j] != needle[j] { same = 0; j = m } else { j = j + 1 }
105 }
106 if same == 1 { if hit < 0 { hit = i + m } }
107 i = i + 1
108 }
109 return hit
110}
111func cw_int_after(buf: *u8, n: i64, key: *u8) -> i64 {
112 let p: i64 = cw_find(buf, n, key)
113 if p < 0 { return 0 - 1 }
114 var i: i64 = p
115 var v: i64 = 0
116 var got: i64 = 0
117 var go: i64 = 1
118 while go == 1 {
119 if i >= n { go = 0 } else {
120 let c: i64 = buf[i] as i64
121 if c < 48 { go = 0 } else {
122 if c > 57 { go = 0 } else { v = v * 10 + (c - 48); got = 1; i = i + 1 }
123 }
124 }
125 }
126 if got == 0 { return 0 - 1 }
127 return v
128}
129
130// ---- the degraded-mirror axis, read from mdstat ------------------------------------------------
131// A raid1 line prints [N/M]; M < N means a member is missing. Counting them needs no device names,
132// so this cannot go stale when a disk is renamed.
133func cw_degraded_count(buf: *u8, n: i64) -> i64 {
134 var i: i64 = 0
135 var deg: i64 = 0
136 while i + 5 < n {
137 if buf[i] == (91 as u8) {
138 let d1: i64 = buf[i + 1] as i64
139 let sl: i64 = buf[i + 2] as i64
140 let d2: i64 = buf[i + 3] as i64
141 let rb: i64 = buf[i + 4] as i64
142 if sl == 47 {
143 if rb == 93 {
144 if d1 >= 48 { if d1 <= 57 { if d2 >= 48 { if d2 <= 57 {
145 if d2 < d1 { deg = deg + 1 }
146 } } } }
147 }
148 }
149 }
150 i = i + 1
151 }
152 return deg
153}
154
155// ---- find the cache instance without hardcoding its name ---------------------------------------
156// The directory name encodes the volume group and volume (shared_cache_vg1_alloc_cache_1+volume_1),
157// so it is host-specific and MUST NOT be a constant here.
158func cw_first_cache_dir(root: *u8, out: *u8) -> i64 {
159 let dpath: *u8 = sys_mmap(CW_PATH)
160 cw_cat(dpath, root, "/flashcache" as *u8, "" as *u8)
161 let dfd: i64 = __syscall(257, CW_AT_FDCWD, dpath, CW_OPENDIR_FLAGS, 0, 0, 0)
162 if dfd < 0 { return 0 }
163 let buf: *u8 = sys_mmap(CW_DIRBUF)
164 let nm: *u8 = sys_mmap(CW_NAME)
165 var found: i64 = 0
166 var go: i64 = 1
167 // ONE getdents64 CALL IS NOT A DIRECTORY LISTING -- loop until it returns 0.
168 while go == 1 {
169 let nread: i64 = __syscall(CW_GETDENTS, dfd, buf, CW_DIRBUF, 0, 0, 0)
170 if nread <= 0 { go = 0 } else {
171 var pos: i64 = 0
172 while pos < nread {
173 let reclen: i64 = (buf[pos + CW_DENT_RECLEN_OFF] as i64) | ((buf[pos + CW_DENT_RECLEN_OFF + 1] as i64) << 8)
174 var nl: i64 = 0
175 while buf[pos + CW_DENT_NAME_OFF + nl] != (0 as u8) {
176 nm[nl] = buf[pos + CW_DENT_NAME_OFF + nl]
177 nl = nl + 1
178 }
179 nm[nl] = 0 as u8
180 // a cache instance directory is the one carrying a '+' (vg+volume); files at this
181 // level (debug_flags, flashcache_version) never do.
182 if found == 0 {
183 if nl > 0 {
184 if nm[0] != (CW_DOT as u8) {
185 var k: i64 = 0
186 var plus: i64 = 0
187 while k < nl { if nm[k] == (43 as u8) { plus = 1 } k = k + 1 }
188 if plus == 1 {
189 var c: i64 = 0
190 while c <= nl { out[c] = nm[c]; c = c + 1 }
191 found = 1
192 }
193 }
194 }
195 }
196 if reclen <= 0 { pos = nread } else { pos = pos + reclen }
197 }
198 }
199 }
200 sys_close(dfd)
201 return found
202}
203
204func cw_streq(a: *u8, b: *u8) -> i64 {
205 var i: i64 = 0
206 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
207 if b[i] != (0 as u8) { return 0 }
208 return 1
209}
210
211func main(argc: i64, argv: *i64) -> i64 {
212 var verb: *u8 = "check" as *u8
213 if argc > 1 { verb = argv[1] as *u8 }
214 var root: *u8 = "/proc" as *u8
215 if argc > 2 { root = argv[2] as *u8 }
216 var explain: i64 = 0
217 if cw_streq(verb, "explain" as *u8) == 1 { explain = 1 }
218
219 let cname: *u8 = sys_mmap(CW_NAME)
220 let have: i64 = cw_first_cache_dir(root, cname)
221
222 let mdbuf: *u8 = sys_mmap(CW_BUF + 16)
223 let mdpath: *u8 = sys_mmap(CW_PATH)
224 cw_cat(mdpath, root, "/mdstat" as *u8, "" as *u8)
225 let mdn: i64 = cw_read(mdpath, mdbuf, CW_BUF)
226 var degraded: i64 = 0 - 1
227 if mdn > 0 { degraded = cw_degraded_count(mdbuf, mdn) }
228
229 if have == 0 {
230 cw_out("CACHEWATCH verdict=UNOBSERVABLE reason=no-flashcache-instance-under-" as *u8); cw_out(root)
231 cw_out("/flashcache degraded_mirrors=" as *u8); cw_num(degraded)
232 cw_out(" note=this-host-may-have-no-such-cache-OR-this-reader-is-wrong--neither-acquits\n" as *u8)
233 return CW_UNOBS
234 }
235
236 let ci: *u8 = sys_mmap(CW_BUF + 16)
237 let cipath: *u8 = sys_mmap(CW_PATH)
238 let pre: *u8 = sys_mmap(CW_PATH)
239 cw_cat(pre, root, "/flashcache/" as *u8, cname)
240 cw_cat(cipath, pre, "/cache_info" as *u8, "" as *u8)
241 let cin: i64 = cw_read(cipath, ci, CW_BUF)
242 if cin <= 0 {
243 cw_out("CACHEWATCH verdict=UNOBSERVABLE reason=cache_info-unreadable cache=" as *u8); cw_out(cname)
244 cw_out(" degraded_mirrors=" as *u8); cw_num(degraded); cw_out("\n" as *u8)
245 return CW_UNOBS
246 }
247
248 // LEVELS -- the only fields the verdict may use.
249 let dirty: i64 = cw_int_after(ci, cin, "dirty_blocks=" as *u8)
250 let total: i64 = cw_int_after(ci, cin, "total_blocks=" as *u8)
251 let cached: i64 = cw_int_after(ci, cin, "cached_blocks=" as *u8)
252
253 let pg: *u8 = sys_mmap(CW_BUF + 16)
254 let pgpath: *u8 = sys_mmap(CW_PATH)
255 cw_cat(pgpath, pre, "/flashcache_progress" as *u8, "" as *u8)
256 let pgn: i64 = cw_read(pgpath, pg, CW_BUF)
257 var uncacheable: i64 = 0 - 1
258 if pgn > 0 {
259 uncacheable = 0
260 if cw_find(pg, pgn, "uncacheable" as *u8) >= 0 { uncacheable = 1 }
261 }
262
263 if explain == 1 {
264 cw_out("CACHEWATCH-EXPLAIN cache=" as *u8); cw_out(cname); cw_out("\n" as *u8)
265 cw_out(" LEVEL dirty_blocks=" as *u8); cw_num(dirty)
266 cw_out(" (pending dirty data RIGHT NOW -- the field a data-risk claim must rest on)\n" as *u8)
267 cw_out(" LEVEL cached_blocks=" as *u8); cw_num(cached)
268 cw_out(" of total_blocks=" as *u8); cw_num(total); cw_out("\n" as *u8)
269 cw_out(" STATE uncacheable=" as *u8); cw_num(uncacheable)
270 cw_out(" (1 = flushed and DISABLED: every write bypasses the cache)\n" as *u8)
271 cw_out(" LEVEL degraded_mirrors=" as *u8); cw_num(degraded); cw_out("\n" as *u8)
272 cw_out(" REFUSED-AS-LEVEL dirty_writeback_kb, dirty_write_hits -- LIFETIME COUNTERS.\n" as *u8)
273 cw_out(" Reading either as a level reports tens of GB stranded when dirty_blocks is 0.\n" as *u8)
274 }
275
276 if uncacheable < 0 {
277 cw_out("CACHEWATCH verdict=UNOBSERVABLE reason=flashcache_progress-unreadable cache=" as *u8)
278 cw_out(cname); cw_out("\n" as *u8)
279 return CW_UNOBS
280 }
281 if dirty < 0 {
282 cw_out("CACHEWATCH verdict=UNOBSERVABLE reason=no-dirty_blocks-level-in-cache_info cache=" as *u8)
283 cw_out(cname); cw_out("\n" as *u8)
284 return CW_UNOBS
285 }
286
287 if uncacheable == 1 {
288 cw_out("CACHEWATCH verdict=RED cache=" as *u8); cw_out(cname)
289 cw_out(" caching=NO dirty_blocks=" as *u8); cw_num(dirty)
290 cw_out(" degraded_mirrors=" as *u8); cw_num(degraded)
291 cw_out(" impact=EVERY-WRITE-BYPASSES-THE-CACHE-AND-HITS-THE-ARRAY" as *u8)
292 cw_out(" remedy=restore-the-cache-mirror-then-re-enable--this-is-PHYSICAL-not-a-build-defect\n" as *u8)
293 return CW_RED
294 }
295 if degraded > 0 {
296 cw_out("CACHEWATCH verdict=AMBER cache=" as *u8); cw_out(cname)
297 cw_out(" caching=YES dirty_blocks=" as *u8); cw_num(dirty)
298 cw_out(" degraded_mirrors=" as *u8); cw_num(degraded)
299 cw_out(" impact=DIRTY-DATA-IS-CACHED-ON-AN-UNMIRRORED-DEVICE\n" as *u8)
300 return CW_AMBER
301 }
302 cw_out("CACHEWATCH verdict=GREEN cache=" as *u8); cw_out(cname)
303 cw_out(" caching=YES dirty_blocks=" as *u8); cw_num(dirty)
304 cw_out(" degraded_mirrors=0\n" as *u8)
305 return CW_GREEN
306}