nx_vsz_watchdog.nx source
↩ module page · 179 lines · 7974 B
1// nx_vsz_watchdog.nx -- ONE-SHOT VSZ watchdog CLI (cron/piggyback-invoked): kills a conf-listed daemon whose
2// VSZ crossed its threshold so the nx_hostctl guard respawns it FRESH -- automating the proven manual fix for
3// outage MODE 2 (VSZ-bloat -> fork-fail -> child-exec silently empty; mgmt hit 160GB, restart -> 4.2GB and the
4// exact failed /api/unpack succeeded). DEATH-DECIDER ONLY: respawn is the guard's (no dueling supervisors).
5// usage: nx_vsz_watchdog [conf] [dry] conf default "vsz_watchdog.conf" rows: <cmdline-needle> <max_gb>
6// FAIL-SAFE: no conf -> INERT exit 0; parse failure -> row dropped; unreadable status -> skip; cooldown 600s
7// per row; never kills pid<=300 or itself. Always exits 0 (a watchdog must never fail the cron).
8// license_tier: ORIGINAL expect_exit: 0
9import "nx_syscalls.nx"
10import "nx_vsz_watchdog_core.nx"
11const K_MAGIC_8192: i64 = 8192
12const K_MAGIC_8191: i64 = 8191
13const K_MAGIC_65536: i64 = 65536
14
15func w_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func w_putn(v: i64) -> i64 {
17 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
18 var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
19 let d: *u8 = sys_mmap(24); var k: i64 = 0
20 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
21 let o: *u8 = sys_mmap(24); var wi: i64 = 0
22 while wi < k { o[wi] = d[k-1-wi]; wi = wi + 1 }
23 sys_write(1, o, k)
24 return 0
25}
26func w_streq(a: *u8, b: *u8) -> i64 {
27 var i: i64 = 0
28 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
29 if b[i] != (0 as u8) { return 0 }
30 return 1
31}
32// cooldown statefile per conf row: vszwd_cd_<idx>.ts holding monotonic seconds of the last kill.
33func w_cd_path(idx: i64, out: *u8) -> i64 {
34 var o: i64 = 0
35 let pre: *u8 = "vszwd_cd_" as *u8
36 var a: i64 = 0
37 while pre[a] != (0 as u8) { out[o] = pre[a]; o = o + 1; a = a + 1 }
38 out[o] = (48 + (idx % 10)) as u8; o = o + 1
39 let suf: *u8 = ".ts" as *u8
40 a = 0
41 while suf[a] != (0 as u8) { out[o] = suf[a]; o = o + 1; a = a + 1 }
42 out[o] = 0 as u8
43 return 0
44}
45func w_cd_read(idx: i64) -> i64 {
46 let p: *u8 = sys_mmap(64)
47 w_cd_path(idx, p)
48 let b: *u8 = sys_mmap(64)
49 let n: i64 = vw_read(p, b, 63)
50 if n <= 0 { return 0 }
51 let ep: *i64 = sys_mmap(16) as *i64
52 let v: i64 = vw_num_at(b, n, 0, ep)
53 if v < 0 { return 0 }
54 return v
55}
56func w_cd_write(idx: i64, now: i64) -> i64 {
57 let p: *u8 = sys_mmap(64)
58 w_cd_path(idx, p)
59 let fd: i64 = sys_openat_wr(p, 420)
60 if fd < 0 { return 0 - 1 }
61 let d: *u8 = sys_mmap(32)
62 var m: i64 = now
63 var k: i64 = 0
64 if m == 0 { d[0] = 48 as u8; k = 1 }
65 let t: *u8 = sys_mmap(32)
66 var kk: i64 = 0
67 while m > 0 { t[kk] = (48 + (m % 10)) as u8; m = m / 10; kk = kk + 1 }
68 while kk > 0 { d[k] = t[kk-1]; k = k + 1; kk = kk - 1 }
69 d[k] = 10 as u8
70 sys_write(fd, d, k + 1)
71 sys_close(fd)
72 return 0
73}
74
75func main(argc: i64, argv: *i64) -> i64 {
76 var confp: *u8 = "vsz_watchdog.conf" as *u8
77 if argc >= 2 { confp = argv[1] as *u8 }
78 var dry: i64 = 0
79 if argc >= 3 { if w_streq(argv[2] as *u8, "dry" as *u8) == 1 { dry = 1 } }
80
81 let conf: *u8 = sys_mmap(K_MAGIC_8192)
82 let cn: i64 = vw_read(confp, conf, K_MAGIC_8191)
83 if cn <= 0 { w_puts("[vszwd] INERT (no conf)\n" as *u8); return 0 }
84
85 // parse rows (max 10): needles NUL-terminated in place inside conf
86 let needles: *i64 = sys_mmap(128) as *i64
87 let gbs: *i64 = sys_mmap(128) as *i64
88 let rp: *i64 = sys_mmap(16) as *i64
89 var nrows: i64 = 0
90 var ls: i64 = 0
91 while ls < cn {
92 var le: i64 = ls
93 var go: i64 = 1
94 while go == 1 { go = 0; if le < cn { if conf[le] != (10 as u8) { le = le + 1; go = 1 } } }
95 if nrows < 10 {
96 let gb: i64 = vw_parse_row(conf, ls, le, rp)
97 if gb >= 1 { needles[nrows] = (conf as i64) + rp[0]; gbs[nrows] = gb; nrows = nrows + 1 }
98 }
99 ls = le + 1
100 }
101 if nrows == 0 { w_puts("[vszwd] INERT (no valid rows)\n" as *u8); return 0 }
102
103 let selfpid: i64 = vw_selfpid()
104 let now: i64 = vw_now_s()
105 let fd: i64 = sys_openat_rd("/proc" as *u8)
106 if fd < 0 { w_puts("[vszwd] cannot open /proc\n" as *u8); return 0 }
107 let dbuf: *u8 = sys_mmap(K_MAGIC_65536)
108 let clbuf: *u8 = sys_mmap(K_MAGIC_8192)
109 let path: *u8 = sys_mmap(256)
110 let ep: *i64 = sys_mmap(16) as *i64
111 var scanned: i64 = 0
112 var killed: i64 = 0
113 var over: i64 = 0
114 var run: i64 = 1
115 while run == 1 {
116 let n: i64 = sys_getdents64(fd, dbuf, K_MAGIC_65536)
117 if n <= 0 { run = 0 } else {
118 var off: i64 = 0
119 while off < n {
120 let rec: *u8 = ((dbuf as i64 + off) as *u8)
121 let reclen: i64 = dirent_reclen(rec)
122 if reclen <= 0 { off = n } else {
123 let name: *u8 = dirent_name(rec)
124 if name[0] >= (48 as u8) { if name[0] <= (57 as u8) {
125 let pid: i64 = vw_num_at(name, vw_slen(name), 0, ep)
126 if pid > 300 { if pid != selfpid {
127 // build /proc/<pid>/cmdline
128 var o: i64 = 0
129 let pre: *u8 = "/proc/" as *u8
130 var a: i64 = 0
131 while pre[a] != (0 as u8) { path[o] = pre[a]; o = o + 1; a = a + 1 }
132 a = 0
133 while name[a] != (0 as u8) { path[o] = name[a]; o = o + 1; a = a + 1 }
134 let suf: *u8 = "/cmdline" as *u8
135 a = 0
136 while suf[a] != (0 as u8) { path[o] = suf[a]; o = o + 1; a = a + 1 }
137 path[o] = 0 as u8
138 let cln: i64 = vw_read(path, clbuf, K_MAGIC_8192)
139 if cln > 0 {
140 scanned = scanned + 1
141 var r: i64 = 0
142 while r < nrows {
143 let nd: *u8 = needles[r] as *u8
144 if vw_contains(clbuf, cln, nd, vw_slen(nd)) == 1 {
145 let kb: i64 = vw_vmsize_kb_of(name)
146 let last: i64 = w_cd_read(r)
147 if vw_should_kill(kb, gbs[r], last, now, 600) == 1 {
148 over = over + 1
149 w_puts("[vszwd] OVER pid=" as *u8); w_putn(pid)
150 w_puts(" " as *u8); w_puts(nd)
151 w_puts(" vsz_kb=" as *u8); w_putn(kb)
152 w_puts(" max_gb=" as *u8); w_putn(gbs[r])
153 if dry == 1 { w_puts(" DRY (no kill)\n" as *u8) } else {
154 nx_kill(pid, 15)
155 w_cd_write(r, now)
156 killed = killed + 1
157 w_puts(" KILLED (SIGTERM; guard respawns fresh)\n" as *u8)
158 }
159 }
160 r = nrows
161 } else { r = r + 1 }
162 }
163 }
164 } }
165 } }
166 off = off + reclen
167 }
168 }
169 }
170 }
171 sys_close(fd)
172 w_puts("[vszwd] scanned=" as *u8); w_putn(scanned)
173 w_puts(" over=" as *u8); w_putn(over)
174 w_puts(" killed=" as *u8); w_putn(killed)
175 w_puts(" rows=" as *u8); w_putn(nrows)
176 if dry == 1 { w_puts(" mode=dry" as *u8) }
177 w_puts("\n" as *u8)
178 return 0
179}