code wiki / _hdl_build / nx_store_fold_beat.nx
nx_store_fold_beat.nx source
↩ module page · 223 lines · 9865 B
1// nx_store_fold_beat.nx -- seg-store plane bloat auto-heal (seq1311). The toolreg plane silently
2// grew to 1355 segments -> every ss_get walked all of them -> tools/list 10-22s/page = the 2026-07-30
3// MCP-surface outage. This beat DISCOVERS every knowledge/ plane (any *manifest.txt), and when one
4// exceeds the segment threshold it runs the proven `nx_store_compact <prefix> fold` (store-native
5// ss_compact_cap merge + production-ss_get byte-verify + manifest restore on mismatch).
6//
7// Sweep-row contract: exit 0 = every plane under threshold OR successfully folded (healed);
8// exit 1 = a fold FAILED or a bloated plane is too big to fold safely (surfaced RED, never silent).
9// Thresholds are DATA-DRIVEN (knowledge/store_fold.conf: threshold=N, max_docs_mb=M; defaults 64/256).
10// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
11import "nx_syscalls.nx"
12import "nx_tool_run.nx"
13const FB_MAGIC_4096: i64 = 4096
14const FB_MAGIC_4095: i64 = 4095
15
16const FB_DIRBUF: i64 = 262144
17const FB_MFCAP: i64 = 4194304
18const FB_PATHCAP: i64 = 1024
19const FB_OUTCAP: i64 = 65536
20const FB_NL: i64 = 10
21const FB_DEF_THRESH: i64 = 64
22const FB_DEF_MAXMB: i64 = 256
23const FB_KNOW: *u8 = "knowledge/" as *u8
24const FB_CONF: *u8 = "knowledge/store_fold.conf" as *u8
25const FB_COMPACT: *u8 = "./nx_store_compact.elf" as *u8
26const FB_MB: i64 = 1048576
27
28func fb_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
29func fb_puts(s: *u8) -> i64 { sys_write(1, s, fb_slen(s)); return 0 }
30func fb_putn(v: i64) -> i64 {
31 let t: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
32 var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 }
33 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
34 let b: *u8 = sys_mmap(28); var i: i64 = 0; while i < k { b[i] = t[k-1-i]; i = i + 1 }
35 sys_write(1, b, k); return 0
36}
37func fb_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[off+i] = s[i]; i = i + 1 } return off + i }
38func fb_endswith(name: *u8, nl: i64, sfx: *u8) -> i64 {
39 let sl: i64 = fb_slen(sfx)
40 if nl < sl { return 0 }
41 var i: i64 = 0
42 while i < sl { if name[nl - sl + i] != sfx[i] { return 0 } i = i + 1 }
43 return 1
44}
45// parse "key=<digits>" occurrences in conf; returns value or def
46func fb_conf_val(buf: *u8, n: i64, key: *u8, def: i64) -> i64 {
47 let kl: i64 = fb_slen(key)
48 var i: i64 = 0
49 while i + kl + 1 < n {
50 var m: i64 = 1
51 var c: i64 = 0
52 while c < kl { if buf[i + c] != key[c] { m = 0; c = kl } else { c = c + 1 } }
53 if m == 1 { if buf[i + kl] == (61 as u8) {
54 var v: i64 = 0; var got: i64 = 0
55 var j: i64 = i + kl + 1
56 var go: i64 = 1
57 while go == 1 {
58 if j < n { let ch: i64 = buf[j] as i64
59 if ch >= 48 { if ch <= 57 { v = v * 10 + (ch - 48); got = 1; j = j + 1 } else { go = 0 } } else { go = 0 }
60 } else { go = 0 }
61 }
62 if got == 1 { return v }
63 } }
64 i = i + 1
65 }
66 return def
67}
68func fb_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
69 let fd: i64 = sys_openat_rd(path)
70 if fd < 0 { return 0 - 1 }
71 var n: i64 = 0
72 var go: i64 = 1
73 while go == 1 {
74 let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - n)
75 if r > 0 { n = n + r } else { go = 0 }
76 if n >= cap { go = 0 }
77 }
78 sys_close(fd)
79 return n
80}
81func fb_filesize(path: *u8) -> i64 {
82 let fd: i64 = sys_openat_rd(path)
83 if fd < 0 { return 0 }
84 let sz: i64 = sys_lseek(fd, 0, 2)
85 sys_close(fd)
86 if sz < 0 { return 0 }
87 return sz
88}
89
90// a manifest ROW is a segment only if it ends in "-<digits>" -- the EXACT acceptance rule of
91// nx_store_compact's sc_scan_manifest, so the beat never flags a plane the compactor would refuse.
92// (knowledge/uiqbase-manifest.txt is a UI page TABLE that merely shares the filename shape -- the
93// first beat run flagged it and the compactor fail-closed; count 0 seg rows = not a plane, skip.)
94func fb_is_seg_row(b: *u8, ls: i64, le: i64) -> i64 {
95 var d: i64 = le
96 var g: i64 = 1
97 while g == 1 { g = 0; if d > ls { let c: i64 = b[d-1] as i64; if c >= 48 { if c <= 57 { d = d - 1; g = 1 } } } }
98 if d >= le { return 0 }
99 if d <= ls { return 0 }
100 if b[d-1] != (45 as u8) { return 0 }
101 return 1
102}
103// count seg rows + sum <prefix><row>.docs sizes; segsout[0]=count, bytesout[0]=total
104func fb_plane_stats(prefix: *u8, mfb: *u8, mfn: i64, segsout: *i64, bytesout: *i64) -> i64 {
105 var segs: i64 = 0
106 var total: i64 = 0
107 let path: *u8 = sys_mmap(FB_PATHCAP)
108 var ls: i64 = 0
109 var i: i64 = 0
110 while i <= mfn {
111 var eol: i64 = 0
112 if i == mfn { eol = 1 } else { if mfb[i] == (FB_NL as u8) { eol = 1 } }
113 if eol == 1 {
114 if i > ls { if fb_is_seg_row(mfb, ls, i) == 1 {
115 segs = segs + 1
116 var o: i64 = fb_cat(path, 0, prefix)
117 var c: i64 = 0
118 while c < i - ls { path[o] = mfb[ls + c]; o = o + 1; c = c + 1 }
119 o = fb_cat(path, o, ".docs" as *u8)
120 path[o] = 0 as u8
121 total = total + fb_filesize(path)
122 } }
123 ls = i + 1
124 }
125 i = i + 1
126 }
127 segsout[0] = segs
128 bytesout[0] = total
129 return 0
130}
131
132func main(argc: i64, argv: *i64) -> i64 {
133 // thresholds (data-driven)
134 let cb: *u8 = sys_mmap(FB_MAGIC_4096)
135 let cn: i64 = fb_readfile(FB_CONF, cb, FB_MAGIC_4095)
136 var thresh: i64 = FB_DEF_THRESH
137 var maxmb: i64 = FB_DEF_MAXMB
138 if cn > 0 {
139 thresh = fb_conf_val(cb, cn, "threshold" as *u8, FB_DEF_THRESH)
140 maxmb = fb_conf_val(cb, cn, "max_docs_mb" as *u8, FB_DEF_MAXMB)
141 }
142 // discover planes: every knowledge/*manifest.txt (skip archive/backup/tmp variants)
143 let dfd: i64 = sys_openat_rd(FB_KNOW)
144 if dfd < 0 { fb_puts("FOLD-BEAT REFUSED: cannot open knowledge/\n" as *u8); sys_exit(1); return 1 }
145 let db: *u8 = sys_mmap(FB_DIRBUF)
146 let mfb: *u8 = sys_mmap(FB_MFCAP)
147 let prefix: *u8 = sys_mmap(FB_PATHCAP)
148 let so: *i64 = sys_mmap(16) as *i64
149 let bo: *i64 = sys_mmap(16) as *i64
150 var planes: i64 = 0
151 var folded: i64 = 0
152 var red: i64 = 0
153 var go: i64 = 1
154 while go == 1 {
155 let nd: i64 = sys_getdents64(dfd, db, FB_DIRBUF)
156 if nd <= 0 { go = 0 } else {
157 var p: i64 = 0
158 while p < nd {
159 let reclen: i64 = ((db[p + 16] as i64) | ((db[p + 17] as i64) << 8))
160 let name: *u8 = (db as i64 + p + 19) as *u8
161 let nl: i64 = fb_slen(name)
162 if fb_endswith(name, nl, "manifest.txt" as *u8) == 1 {
163 planes = planes + 1
164 // prefix = knowledge/ + name minus "manifest.txt"
165 var o: i64 = fb_cat(prefix, 0, FB_KNOW)
166 var c: i64 = 0
167 while c < nl - 12 { prefix[o] = name[c]; o = o + 1; c = c + 1 }
168 prefix[o] = 0 as u8
169 let mp: *u8 = sys_mmap(FB_PATHCAP)
170 var mo: i64 = fb_cat(mp, 0, prefix)
171 mo = fb_cat(mp, mo, "manifest.txt" as *u8)
172 mp[mo] = 0 as u8
173 let mfn: i64 = fb_readfile(mp, mfb, FB_MFCAP - 1)
174 if mfn > 0 {
175 fb_plane_stats(prefix, mfb, mfn, so, bo)
176 if so[0] > thresh {
177 if bo[0] > maxmb * FB_MB {
178 fb_puts("FOLD-BEAT RED plane=" as *u8); fb_puts(prefix)
179 fb_puts(" segs=" as *u8); fb_putn(so[0])
180 fb_puts(" docs_bytes=" as *u8); fb_putn(bo[0])
181 fb_puts(" EXCEEDS max_docs_mb=" as *u8); fb_putn(maxmb)
182 fb_puts(" -- too big to fold safely, needs operator attention\n" as *u8)
183 red = red + 1
184 } else {
185 let av: *i64 = sys_mmap(40) as *i64
186 av[0] = FB_COMPACT as i64
187 av[1] = prefix as i64
188 av[2] = "fold" as *u8 as i64
189 av[3] = 0
190 let out: *u8 = sys_mmap(FB_OUTCAP)
191 let ol: *i64 = sys_mmap(16) as *i64
192 let rc: i64 = tr_run_capture(FB_COMPACT, av, out, FB_OUTCAP - 1, ol)
193 if rc == 0 {
194 fb_puts("FOLD-BEAT HEALED plane=" as *u8); fb_puts(prefix)
195 fb_puts(" segs=" as *u8); fb_putn(so[0])
196 fb_puts(" -> folded\n" as *u8)
197 folded = folded + 1
198 } else {
199 fb_puts("FOLD-BEAT RED plane=" as *u8); fb_puts(prefix)
200 fb_puts(" fold rc=" as *u8); fb_putn(rc)
201 fb_puts(" out: " as *u8)
202 sys_write(1, out, ol[0])
203 fb_puts("\n" as *u8)
204 red = red + 1
205 }
206 }
207 }
208 }
209 }
210 p = p + reclen
211 }
212 }
213 }
214 sys_close(dfd)
215 fb_puts("FOLD-BEAT planes=" as *u8); fb_putn(planes)
216 fb_puts(" threshold=" as *u8); fb_putn(thresh)
217 fb_puts(" folded=" as *u8); fb_putn(folded)
218 fb_puts(" red=" as *u8); fb_putn(red)
219 fb_puts("\n" as *u8)
220 if red > 0 { sys_exit(1); return 1 }
221 sys_exit(0)
222 return 0
223}