code wiki / _hdl_build / nx_store_janitor_beat.nx
nx_store_janitor_beat.nx source
↩ module page · 179 lines · 7740 B
1// nx_store_janitor_beat.nx -- standing reclamation of SUPERSEDED seg-store segment files.
2//
3// WHY A BEAT AND NOT A ONE-SHOT: on 2026-07-30 a single manual sweep moved 57,348 MB across 336 planes
4// (14,585 live segment .docs down to 2,973). Within the SAME session debt- went from 1 segment back to 18
5// and down again as siblings wrote and compacted. The seg-store is append-only and every commit writes a NEW
6// segment, so superseded files re-accumulate CONTINUOUSLY. A cleanup that has to be remembered is a
7// workaround; the fix is a standing sweep row that keeps the property true without anyone noticing.
8//
9// SIBLING TO nx_store_fold_beat AND DELIBERATELY ORDERED AFTER IT: fold_beat MERGES a bloated plane (segment
10// COUNT / scan cost), which by construction supersedes the segments it merged. This beat then reclaims those
11// files (disk / directory hygiene). Fold makes garbage; janitor takes it out. Neither replaces the other.
12//
13// Forks the proven ./nx_store_janitor.elf per plane exactly as fold_beat forks nx_store_compact -- the worker
14// owns the safety argument (manifest IS the definition of live, REFUSES on an unreadable manifest, moves aside
15// into retired/ and never unlinks, and re-reads the manifest afterwards to restore anything a concurrent
16// writer made live mid-walk). This beat adds discovery and aggregation ONLY; it must not re-implement policy.
17//
18// Sweep-row contract: exit 0 = every plane reclaimed or already clean; exit 1 = a plane REFUSED (unknown live
19// set) or a live segment was moved and restored -- surfaced RED, never silent.
20// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
21import "nx_syscalls.nx"
22import "nx_tool_run.nx"
23
24const JB_DIRBUF: i64 = 262144
25const JB_PATHCAP: i64 = 1024
26const JB_OUTCAP: i64 = 65536
27const JB_STORE: *u8 = "knowledge/store" as *u8
28const JB_JANITOR: *u8 = "./nx_store_janitor.elf" as *u8
29const JB_MANI: *u8 = "manifest.txt" as *u8
30const JB_MANILEN: i64 = 12
31const JB_STDOUT: i64 = 1
32const JB_NL: i64 = 10
33const JB_AVCAP: i64 = 40
34
35func jb_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
36func jb_puts(s: *u8) -> i64 { sys_write(JB_STDOUT, s, jb_len(s)); return 0 }
37func jb_putn(v: i64) -> i64 {
38 let t: *u8 = sys_mmap(32)
39 var m: i64 = v
40 if m < 0 { m = 0 - m }
41 var k: i64 = 0
42 if m == 0 { t[0] = 48 as u8; k = 1 }
43 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
44 let b: *u8 = sys_mmap(32)
45 var i: i64 = 0
46 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
47 sys_write(JB_STDOUT, b, k)
48 sys_munmap(t, 32)
49 sys_munmap(b, 32)
50 return 0
51}
52// suffix test against a LOCAL string, never a const *u8 index -- CONST[i] is a banked landmine that can
53// compile clean and then read garbage (nishilang gotchas).
54func jb_ends_manifest(nm: *u8) -> i64 {
55 let n: i64 = jb_len(nm)
56 if n <= JB_MANILEN {
57 return 0
58 }
59 let want: *u8 = "manifest.txt" as *u8
60 var i: i64 = 0
61 var ok: i64 = 1
62 while i < JB_MANILEN {
63 if nm[n - JB_MANILEN + i] != want[i] {
64 ok = 0
65 i = JB_MANILEN
66 } else {
67 i = i + 1
68 }
69 }
70 return ok
71}
72// pull "moved=" style integer out of the worker's line; -1 if absent
73func jb_field(buf: *u8, n: i64, key: *u8) -> i64 {
74 let kl: i64 = jb_len(key)
75 var i: i64 = 0
76 while i + kl < n {
77 var ok: i64 = 1
78 var k: i64 = 0
79 while k < kl { if buf[i + k] != key[k] { ok = 0; k = kl } else { k = k + 1 } }
80 if ok == 1 {
81 var v: i64 = 0
82 var j: i64 = i + kl
83 var any: i64 = 0
84 while j < n {
85 let c: i64 = buf[j]
86 var digit: i64 = 0
87 if c >= 48 {
88 if c <= 57 { digit = 1 }
89 }
90 if digit == 1 {
91 v = v * 10 + (c - 48)
92 any = 1
93 j = j + 1
94 } else {
95 j = n
96 }
97 }
98 if any == 1 { return v }
99 return 0 - 1
100 }
101 i = i + 1
102 }
103 return 0 - 1
104}
105
106func main(argc: i64, argv: *i64) -> i64 {
107 let dbuf: *u8 = sys_mmap(JB_DIRBUF)
108 let prefix: *u8 = sys_mmap(JB_PATHCAP)
109 let out: *u8 = sys_mmap(JB_OUTCAP)
110 let ol: *i64 = sys_mmap(16) as *i64
111 var planes: i64 = 0
112 var swept: i64 = 0
113 var files: i64 = 0
114 var mb: i64 = 0
115 var refused: i64 = 0
116 var restored: i64 = 0
117
118 let fd: i64 = sys_openat_rd(JB_STORE)
119 if fd < 0 { jb_puts("JANITOR-BEAT RED cannot open knowledge/store\n" as *u8); return 1 }
120 var go: i64 = 1
121 while go == 1 {
122 let nb: i64 = sys_getdents64(fd, dbuf, JB_DIRBUF)
123 if nb <= 0 { go = 0 } else {
124 var off: i64 = 0
125 while off < nb {
126 let rec: *u8 = (dbuf as i64 + off) as *u8
127 let rl: i64 = dirent_reclen(rec)
128 if rl <= 0 { off = nb } else {
129 let nm: *u8 = dirent_name(rec)
130 if jb_ends_manifest(nm) == 1 {
131 let nl: i64 = jb_len(nm)
132 var po: i64 = 0
133 var z: i64 = 0
134 let sdir: *u8 = "knowledge/store" as *u8
135 while sdir[z] != (0 as u8) {
136 prefix[po] = sdir[z]
137 po = po + 1
138 z = z + 1
139 }
140 prefix[po] = 47 as u8; po = po + 1
141 var q: i64 = 0
142 while q < nl - JB_MANILEN { prefix[po] = nm[q]; po = po + 1; q = q + 1 }
143 prefix[po] = 0 as u8
144 planes = planes + 1
145 let av: *i64 = sys_mmap(JB_AVCAP) as *i64
146 av[0] = JB_JANITOR as i64
147 av[1] = prefix as i64
148 av[2] = "apply" as *u8 as i64
149 av[3] = 0
150 let rc: i64 = tr_run_capture(JB_JANITOR, av, out, JB_OUTCAP - 1, ol)
151 let mv: i64 = jb_field(out, ol[0], "MOVED=" as *u8)
152 let rm: i64 = jb_field(out, ol[0], "reclaimable_MB=" as *u8)
153 let rs: i64 = jb_field(out, ol[0], "RACE-RESTORED=" as *u8)
154 if rs > 0 { restored = restored + rs
155 jb_puts("JANITOR-BEAT RACE-RESTORED plane=" as *u8); jb_puts(prefix); jb_puts("\n" as *u8) }
156 // worker exit 3 = REFUSED (unknown live set) or a live segment could not be restored
157 if rc == 3 { refused = refused + 1
158 jb_puts("JANITOR-BEAT REFUSED plane=" as *u8); jb_puts(prefix); jb_puts("\n" as *u8) }
159 if mv > 0 { swept = swept + 1; files = files + mv; if rm > 0 { mb = mb + rm } }
160 sys_munmap(av as *u8, JB_AVCAP)
161 }
162 off = off + rl
163 }
164 }
165 }
166 }
167 sys_close(fd)
168
169 jb_puts("JANITOR-BEAT planes=" as *u8); jb_putn(planes)
170 jb_puts(" swept=" as *u8); jb_putn(swept)
171 jb_puts(" files_moved=" as *u8); jb_putn(files)
172 jb_puts(" MB=" as *u8); jb_putn(mb)
173 jb_puts(" refused=" as *u8); jb_putn(refused)
174 jb_puts(" race_restored=" as *u8); jb_putn(restored)
175 if refused > 0 { jb_puts(" verdict=RED-a-plane-had-an-UNKNOWN-LIVE-SET-nothing-swept-there\n" as *u8); return 1 }
176 if restored > 0 { jb_puts(" verdict=RED-a-LIVE-segment-was-moved-and-restored-investigate-the-writer\n" as *u8); return 1 }
177 jb_puts(" verdict=GREEN-superseded-files-reclaimed-into-retired\n" as *u8)
178 return 0
179}