nx_ctxtop.nx source

1// nx_ctxtop.nx -- WHO is producing the context switches. The producer-finder for seq1340. 2// 3// nx_procchurn measured the box at 81k-93k ctxsw/sec in every sample (the one CONSTANT in the perf lane) 4// and DISPROVED forks as the driver -- forks swing 15->179->10 while ctxsw stays flat. A box-level rate 5// names no culprit, so this walks /proc/<pid>/status twice and attributes the switches per process. 6// LAW (rising-debt-means-find-the-producer): a rate with no owner is a symptom, not a diagnosis. 7// 8// ★Reports voluntary and nonvoluntary SEPARATELY because they demand OPPOSITE fixes: voluntary = the 9// process blocked itself (poll/read/sleep) -> batch, lengthen polls, go event-driven; nonvoluntary = the 10// scheduler preempted it -> reduce concurrency. A tool printing only the sum sends the fix the wrong way. 11// Exit code = 0 when the sample completed, 3 UNMEASURED (fail-closed, never a confident empty answer). 12// 13// ★2026-08-12 -- PRODUCER-CLASS ATTRIBUTION: the ranked rows name WHO; a final NX-CTXTOP-CLASS line names 14// WHAT KIND (SYSTEM=DSM/Synology, ESTATE=nx_* organs, KERNEL=kthreads, OTHER) so a RED churn axis carries 15// its own verdict instead of a human minting a cap and eyeballing comm strings. Field-measured 2026-08-12: 16// every top-8 producer was DSM (ssrtmpserverd/synorelayd/syno-cloud-clie/redis/kworkers), zero estate. 17// license_tier: ORIGINAL Read-only. No hw writes (Rule 26). expect_exit: 0 18import "nx_ctxtop_lib.nx" 19import "nx_proc_ctl.nx" 20 21const CT_MAXP: i64 = 2048 22const CT_TABLE_BYTES: i64 = 16384 23const CT_DIRBUF: i64 = 65536 24const CT_STATBUF: i64 = 4096 25const CT_TOPK: i64 = 8 26const CT_TOPK_BYTES: i64 = 128 27const CT_WINDOW_MS: i64 = 3000 28const CT_US_PER_MS: i64 = 1000 29const CT_NAMEOFF: i64 = 6 30const CT_EXIT_UNMEASURED: i64 = 3 31const CT_SLASH: i64 = 47 32const CT_EXIT_REFUSED: i64 = 4 33// CT_ADMIT_DEFAULT now lives in nx_ctxtop_lib.nx beside ct_admit_now(), because the driver gates need 34// the SAME calibration and a second copy of 800 is one threshold in two files that can never be tuned 35// together. Moved, not copied. 36const CT_CONFBUF: i64 = 4096 37const CT_LOADBUF: i64 = 128 38 39// producer classes for the attribution verdict (2026-08-12) 40const CTC_SYSTEM: i64 = 0 41const CTC_ESTATE: i64 = 1 42const CTC_KERNEL: i64 = 2 43const CTC_OTHER: i64 = 3 44 45// Walk /proc, recording pid + both ctxsw counters. Returns the number of processes captured. 46// Reports its own capping honestly (L011: partial-as-complete is a bug) via the returned count vs CT_MAXP. 47func ct_scan(pids: *i64, vols: *i64, nvs: *i64, dbuf: *u8, path: *u8, sbuf: *u8) -> i64 { 48 var cnt: i64 = 0 49 // SECOND dirent buffer: the task/ walk is NESTED inside the /proc walk, so it cannot reuse dbuf -- 50 // doing so would corrupt the outer getdents cursor mid-iteration and silently skip processes. 51 let tbuf: *u8 = sys_mmap(CT_DIRBUF) 52 let fd: i64 = sys_openat_rd("/proc" as *u8) 53 if fd < 0 { return 0 } 54 var run: i64 = 1 55 while run == 1 { 56 let n: i64 = sys_getdents64(fd, dbuf, CT_DIRBUF) 57 if n <= 0 { run = 0 } else { 58 var off: i64 = 0 59 while off < n { 60 let rec: *u8 = ((dbuf as i64) + off) as *u8 61 let reclen: i64 = dirent_reclen(rec) 62 if reclen <= 0 { off = n } else { 63 let nm: *u8 = dirent_name(rec) 64 if nm[0] >= (48 as u8) { if nm[0] <= (57 as u8) { if cnt < CT_MAXP { 65 var p: i64 = 0 66 let pre: *u8 = "/proc/" as *u8 67 var a: i64 = 0 68 while pre[a] != (0 as u8) { path[p] = pre[a]; p = p + 1; a = a + 1 } 69 var pidv: i64 = 0 70 a = 0 71 while nm[a] != (0 as u8) { path[p] = nm[a]; pidv = pidv * 10 + ((nm[a] as i64) - 48); p = p + 1; a = a + 1 } 72 let suf: *u8 = "/status" as *u8 73 a = 0 74 while suf[a] != (0 as u8) { path[p] = suf[a]; p = p + 1; a = a + 1 } 75 path[p] = 0 as u8 76 // THREAD-LEVEL WALK (seq1364 gap closed). Reading only /proc/<pid>/status counts 77 // the MAIN THREAD's counters and silently ignores every sibling thread: the box 78 // carries 1809 threads across ~809 processes, so a process-only walk attributed 79 // 40480 of a box-level 116-130k ctxsw/s and left TWO THIRDS unexplained -- while 80 // looking like a complete top-8. Every task has /proc/<pid>/task/<pid>, so walking 81 // task/ ALONE covers the main thread too: no double-count, full coverage. 82 // LAW: A PER-PROCESS COUNTER ON A MULTI-THREADED SYSTEM IS A FLOOR WEARING THE 83 // COSTUME OF A TOTAL. 84 var tp: i64 = p - 7 85 let tsuf: *u8 = "/task" as *u8 86 var ta: i64 = 0 87 while tsuf[ta] != (0 as u8) { path[tp] = tsuf[ta]; tp = tp + 1; ta = ta + 1 } 88 path[tp] = 0 as u8 89 let tfd: i64 = sys_openat_rd(path) 90 if tfd >= 0 { 91 var trun: i64 = 1 92 while trun == 1 { 93 let tn: i64 = sys_getdents64(tfd, tbuf, CT_DIRBUF) 94 if tn <= 0 { trun = 0 } else { 95 var toff: i64 = 0 96 while toff < tn { 97 let trec: *u8 = ((tbuf as i64) + toff) as *u8 98 let treclen: i64 = dirent_reclen(trec) 99 if treclen <= 0 { toff = tn } else { 100 let tnm: *u8 = dirent_name(trec) 101 if tnm[0] >= (48 as u8) { if tnm[0] <= (57 as u8) { if cnt < CT_MAXP { 102 var q: i64 = tp 103 path[q] = CT_SLASH as u8; q = q + 1 104 var tidv: i64 = 0 105 var tb: i64 = 0 106 while tnm[tb] != (0 as u8) { path[q] = tnm[tb]; tidv = tidv * 10 + ((tnm[tb] as i64) - 48); q = q + 1; tb = tb + 1 } 107 let ssuf: *u8 = "/status" as *u8 108 tb = 0 109 while ssuf[tb] != (0 as u8) { path[q] = ssuf[tb]; q = q + 1; tb = tb + 1 } 110 path[q] = 0 as u8 111 let sn2: i64 = rm_read(path, sbuf, CT_STATBUF) 112 if sn2 > 0 { 113 let v2: i64 = rm_field(sbuf, sn2, "voluntary_ctxt_switches:" as *u8) 114 let w2: i64 = rm_field(sbuf, sn2, "nonvoluntary_ctxt_switches:" as *u8) 115 if v2 >= 0 { if w2 >= 0 { 116 pids[cnt] = tidv 117 vols[cnt] = v2 118 nvs[cnt] = w2 119 cnt = cnt + 1 120 } } 121 } 122 } } } 123 toff = toff + treclen 124 } 125 } 126 } 127 } 128 sys_close(tfd) 129 } 130 } } } 131 off = off + reclen 132 } 133 } 134 } 135 } 136 sys_close(fd) 137 return cnt 138} 139 140// print "/proc/<pid>/status" Name: field (comm starts at byte 6, ends at the first newline) 141func ct_putname(pid: i64, path: *u8, sbuf: *u8) -> i64 { 142 var p: i64 = 0 143 let pre: *u8 = "/proc/" as *u8 144 var a: i64 = 0 145 while pre[a] != (0 as u8) { path[p] = pre[a]; p = p + 1; a = a + 1 } 146 p = pc_catn(path, p, pid) 147 let suf: *u8 = "/status" as *u8 148 a = 0 149 while suf[a] != (0 as u8) { path[p] = suf[a]; p = p + 1; a = a + 1 } 150 path[p] = 0 as u8 151 let sn: i64 = rm_read(path, sbuf, CT_STATBUF) 152 if sn <= 0 { rm_puts("<gone>" as *u8); return 0 } 153 var z: i64 = CT_NAMEOFF 154 let out: *u8 = sys_mmap(64) 155 var w: i64 = 0 156 while z < sn { if sbuf[z] == (10 as u8) { z = sn } else { if w < 62 { out[w] = sbuf[z]; w = w + 1 } z = z + 1 } } 157 out[w] = 0 as u8 158 rm_puts(out) 159 sys_munmap(out, 64) 160 return 0 161} 162 163// substring-contains: 1 if needle occurs in the first n bytes of hay, else 0. needle is a *u8 parameter 164// (NOT an inline-cast literal), so hay/needle indexing is safe -- the caller hoists every literal into a 165// `let` before passing it (the ("lit" as *u8)[j] index NEVER matches and compiles clean; §32 hazard). 166// The inner match uses an explicit `ok` flag and only clobbers the INNER cursor j to break, never i -- 167// a loop that exits by clobbering its own search cursor erases the answer (§29). 168func ct_has(hay: *u8, n: i64, needle: *u8) -> i64 { 169 var m: i64 = 0 170 while needle[m] != (0 as u8) { m = m + 1 } 171 if m == 0 { return 0 } 172 if n < m { return 0 } 173 let last: i64 = n - m 174 var i: i64 = 0 175 while i <= last { 176 var j: i64 = 0 177 var ok: i64 = 1 178 while j < m { 179 if hay[(i + j)] != needle[j] { ok = 0; j = m } else { j = j + 1 } 180 } 181 if ok == 1 { return 1 } 182 i = i + 1 183 } 184 return 0 185} 186 187// classify a tid by its OWNING-PROCESS cmdline (all threads share /proc/<tid>/cmdline = the process argv). 188// A kernel thread has an EMPTY cmdline -> KERNEL (a reliable, path-independent detector). DSM/Synology 189// binaries live under /syno, /var/packages, or SurveillanceStation; the estate under nishihost / 190// elderwesto / /volume1/ai. ESTATE is tested FIRST so an estate path is never mistaken for system. 191func ct_class_of(tid: i64, path: *u8, cbuf: *u8) -> i64 { 192 var p: i64 = 0 193 let pre: *u8 = "/proc/" as *u8 194 var a: i64 = 0 195 while pre[a] != (0 as u8) { path[p] = pre[a]; p = p + 1; a = a + 1 } 196 p = pc_catn(path, p, tid) 197 let suf: *u8 = "/cmdline" as *u8 198 a = 0 199 while suf[a] != (0 as u8) { path[p] = suf[a]; p = p + 1; a = a + 1 } 200 path[p] = 0 as u8 201 let n: i64 = rm_read(path, cbuf, CT_STATBUF) 202 if n <= 0 { return CTC_KERNEL } 203 let e1: *u8 = "nishihost" as *u8 204 if ct_has(cbuf, n, e1) == 1 { return CTC_ESTATE } 205 let e2: *u8 = "elderwesto" as *u8 206 if ct_has(cbuf, n, e2) == 1 { return CTC_ESTATE } 207 let e3: *u8 = "/volume1/ai" as *u8 208 if ct_has(cbuf, n, e3) == 1 { return CTC_ESTATE } 209 let s1: *u8 = "syno" as *u8 210 if ct_has(cbuf, n, s1) == 1 { return CTC_SYSTEM } 211 let s2: *u8 = "Surveillance" as *u8 212 if ct_has(cbuf, n, s2) == 1 { return CTC_SYSTEM } 213 let s3: *u8 = "packages/" as *u8 214 if ct_has(cbuf, n, s3) == 1 { return CTC_SYSTEM } 215 let s4: *u8 = "redis" as *u8 216 if ct_has(cbuf, n, s4) == 1 { return CTC_SYSTEM } 217 return CTC_OTHER 218} 219 220func main(argc: i64, argv: *i64) -> i64 { 221 // ADMISSION CONTROL (seq341 / seq1389): this organ walks ALL of /proc and stats every 222 // /proc/<pid>/status -- an O(processes) syscall storm, the same shape seq1318 indicts the supervisor 223 // for. Running it on an already-saturated box is how a diagnostic becomes an outage. Refuse instead. 224 // ★LAW: A DIAGNOSTIC THAT CANNOT REFUSE TO RUN IS A LOAD GENERATOR WITH GOOD INTENTIONS. 225 // 2026-09-02 ARGV CONF (Rule 17: argv > conf file > default), the same shape nx_dstate ships. During 226 // today's storm every producer-naming organ refused to look, so the seat could measure THAT the box was 227 // saturated and could not say BY WHAT -- and a refusal that blinds the operator during the one condition 228 // it exists for protects nothing. Two bounded /proc walks are negligible beside load 30 on an I/O-bound 229 // box. The operator hands a conf path on argv and BOTH the refusal and the report print its provenance, 230 // so an elevated ceiling is announced, never silent; the shipped procchurn.conf is untouched. 231 var confp: *u8 = "knowledge/status/procchurn.conf" as *u8 232 var confsrc: *u8 = "conf-default" as *u8 233 if argc >= 2 { confp = argv[1] as *u8; confsrc = "conf-argv" as *u8 } 234 let cbuf: *u8 = sys_mmap(CT_CONFBUF) 235 let cn: i64 = rm_read(confp, cbuf, CT_CONFBUF) 236 let maxload: i64 = rm_conf(cbuf, cn, "admit-max-load-centi" as *u8, CT_ADMIT_DEFAULT) 237 let lbuf: *u8 = sys_mmap(CT_LOADBUF) 238 let ln: i64 = rm_read("/proc/loadavg" as *u8, lbuf, CT_LOADBUF) 239 let loadc: i64 = ct_load_centi(lbuf, ln) 240 if ct_admit(loadc, maxload) == 0 { 241 rm_puts("NX-CTXTOP verdict=REFUSED load_centi=" as *u8); rm_num(loadc) 242 rm_puts(" max=" as *u8); rm_num(maxload) 243 rm_puts(" src=" as *u8); rm_puts(confsrc); rm_puts(" path=" as *u8); rm_puts(confp) 244 rm_puts(" why=host already saturated; a /proc-walking diagnostic must not add to it (seq341); pass a conf path on argv to raise the ceiling for ONE announced walk\n" as *u8) 245 return CT_EXIT_REFUSED 246 } 247 let p1: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 248 let v1: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 249 let w1: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 250 let p2: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 251 let v2: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 252 let w2: *i64 = sys_mmap(CT_TABLE_BYTES) as *i64 253 let dbuf: *u8 = sys_mmap(CT_DIRBUF) 254 let path: *u8 = sys_mmap(256) 255 let sbuf: *u8 = sys_mmap(CT_STATBUF) 256 257 let t1: i64 = sys_now_us() 258 let n1: i64 = ct_scan(p1, v1, w1, dbuf, path, sbuf) 259 sys_sleep_ms(CT_WINDOW_MS) 260 let t2: i64 = sys_now_us() 261 let n2: i64 = ct_scan(p2, v2, w2, dbuf, path, sbuf) 262 let el: i64 = (t2 - t1) / CT_US_PER_MS 263 264 if n1 <= 0 { rm_puts("NX-CTXTOP verdict=UNMEASURED why=/proc walk returned nothing\n" as *u8); return CT_EXIT_UNMEASURED } 265 if el <= 0 { rm_puts("NX-CTXTOP verdict=UNMEASURED why=zero-length window\n" as *u8); return CT_EXIT_UNMEASURED } 266 267 let tp: *i64 = sys_mmap(CT_TOPK_BYTES) as *i64 268 let tr: *i64 = sys_mmap(CT_TOPK_BYTES) as *i64 269 var used: i64 = 0 270 var tot: i64 = 0 271 var totv: i64 = 0 272 var totn: i64 = 0 273 var i: i64 = 0 274 while i < n2 { 275 let j: i64 = ct_find(p1, n1, p2[i]) 276 if j >= 0 { 277 let dv: i64 = v2[i] - v1[j] 278 let dn: i64 = w2[i] - w1[j] 279 if dv >= 0 { if dn >= 0 { 280 let r: i64 = pc_rate(dv + dn, el) 281 if r > 0 { 282 tot = tot + r 283 totv = totv + pc_rate(dv, el) 284 totn = totn + pc_rate(dn, el) 285 used = ct_topk_insert(tp, tr, CT_TOPK, used, p2[i], r) 286 } 287 } } 288 } 289 i = i + 1 290 } 291 292 rm_puts("NX-CTXTOP window_ms=" as *u8); rm_num(el) 293 rm_puts(" procs=" as *u8); rm_num(n2) 294 rm_puts(" attributed_ctxsw_per_sec=" as *u8); rm_num(tot) 295 rm_puts(" vol=" as *u8); rm_num(totv) 296 rm_puts(" nonvol=" as *u8); rm_num(totn) 297 rm_puts(" admit_max=" as *u8); rm_num(maxload); rm_puts(" admit_src=" as *u8); rm_puts(confsrc); rm_puts(" admit_path=" as *u8); rm_puts(confp); rm_puts("\n" as *u8) 298 var k: i64 = 0 299 while k < used { 300 rm_puts(" #" as *u8); rm_num(k + 1) 301 rm_puts(" pid=" as *u8); rm_num(tp[k]) 302 rm_puts(" ctxsw/s=" as *u8); rm_num(tr[k]) 303 rm_puts(" share=" as *u8); rm_num(ct_share_permil(tr[k], tot)); rm_puts("permil name=" as *u8) 304 ct_putname(tp[k], path, sbuf) 305 rm_puts("\n" as *u8) 306 k = k + 1 307 } 308 if n2 >= CT_MAXP { rm_puts(" WARNING scan CAPPED at " as *u8); rm_num(CT_MAXP); rm_puts(" pids -- coverage INCOMPLETE, totals are a floor\n" as *u8) } 309 310 // ---- PRODUCER-CLASS ATTRIBUTION (2026-08-12) ---- 311 // Name WHAT KIND of producer the ranked churn is, so a RED axis carries its own verdict. Classified 312 // over the ranked top-K (which carried the bulk of attributed churn in field samples) by owning- 313 // process cmdline; scope is printed as top<used> so it is never read as a whole-box population claim 314 // (L: label a bound as a bound). Reuses path/sbuf -- the print loop above has finished with them, so 315 // no extra mmap (the guard organ must not become the load; §admission). 316 if used <= 0 { rm_puts("NX-CTXTOP-CLASS scope=top0 note=no attributed churn to classify\n" as *u8); return 0 } 317 if tot <= 0 { rm_puts("NX-CTXTOP-CLASS scope=top0 note=zero attributed total\n" as *u8); return 0 } 318 var sys_s: i64 = 0 319 var est_s: i64 = 0 320 var ker_s: i64 = 0 321 var oth_s: i64 = 0 322 var kk: i64 = 0 323 while kk < used { 324 let cl: i64 = ct_class_of(tp[kk], path, sbuf) 325 if cl == CTC_SYSTEM { sys_s = sys_s + tr[kk] } 326 if cl == CTC_ESTATE { est_s = est_s + tr[kk] } 327 if cl == CTC_KERNEL { ker_s = ker_s + tr[kk] } 328 if cl == CTC_OTHER { oth_s = oth_s + tr[kk] } 329 kk = kk + 1 330 } 331 // dominant class among the ranked top-K; strict > keeps the incumbent on a tie (conservative, 332 // and label + verdict both derive from the same code so they can never disagree). 333 var domc: i64 = CTC_SYSTEM 334 var domv: i64 = sys_s 335 if est_s > domv { domc = CTC_ESTATE; domv = est_s } 336 if ker_s > domv { domc = CTC_KERNEL; domv = ker_s } 337 if oth_s > domv { domc = CTC_OTHER; domv = oth_s } 338 var dom: *u8 = "SYSTEM" as *u8 339 if domc == CTC_ESTATE { dom = "ESTATE" as *u8 } 340 if domc == CTC_KERNEL { dom = "KERNEL" as *u8 } 341 if domc == CTC_OTHER { dom = "OTHER" as *u8 } 342 rm_puts("NX-CTXTOP-CLASS scope=top" as *u8); rm_num(used) 343 rm_puts(" system_permil=" as *u8); rm_num(ct_share_permil(sys_s, tot)) 344 rm_puts(" estate_permil=" as *u8); rm_num(ct_share_permil(est_s, tot)) 345 rm_puts(" kernel_permil=" as *u8); rm_num(ct_share_permil(ker_s, tot)) 346 rm_puts(" other_permil=" as *u8); rm_num(ct_share_permil(oth_s, tot)) 347 rm_puts(" dominant=" as *u8); rm_puts(dom); rm_puts("\n" as *u8) 348 // The strong "not the estate's to fix" claim is made ONLY against the ACTUAL dominant class -- 349 // never asserted when the dominant is OTHER, and never when ESTATE is on top. 350 if domc == CTC_ESTATE { 351 rm_puts(" ATTRIBUTION: ESTATE-DOMINATED -- nx_* organs are the top churn producer; investigate the named pids\n" as *u8) 352 } else { 353 rm_puts(" ATTRIBUTION: estate is NOT the dominant churn class (dominant=" as *u8); rm_puts(dom) 354 rm_puts(", estate_permil=" as *u8); rm_num(ct_share_permil(est_s, tot)) 355 rm_puts("); the ranked churn is largely not the estate's to fix via the sovereign API\n" as *u8) 356 } 357 return 0 358}