nx_tsvdrain.nx source
↩ module page · 350 lines · 18492 B
1// nx_tsvdrain.nx -- DRAIN THE TSV LANE: migrate every UNCONSUMED .tsv into a sovereign plane, then retire
2// the third-party file, reversibly. Composes nx_tsvcensus's partition with the two proven organs.
3//
4// WHY: nx_tsvcensus named 44 unconsumed .tsv files across the two knowledge trees. Doing them by hand is
5// two tool calls plus a verification each -- the same tooling gap that made the census itself look "too
6// slow", and a tooling gap is never a licence to sample a subset and call the lane drained.
7//
8// MIGRATE-THEN-RETIRE IS THE ORDER, AND IT IS NOT COSMETIC. Retiring alone would discard measured rows;
9// migrating alone would leave the third-party format in the tree. Evidence files are NOT a special case for
10// the ACTION -- every row survives as sovereign data -- they are only a reason never to retire WITHOUT
11// migrating first.
12//
13// FAIL-SAFE BY CONSTRUCTION (rule 26 / the destructive-heuristic law):
14// - DRY IS THE DEFAULT. Passing nothing prints the plan and changes nothing. Only the literal verb
15// `apply` acts, so a mistyped argument does nothing rather than something.
16// - A file is retired ONLY IF its migrate printed TSV-MIGRATE GREEN *and* verified=1. Any other outcome
17// leaves the source exactly where it is and is counted as SKIPPED, never as done.
18// - Retirement goes through nx_retire_path, which renames and never deletes.
19// - If the census cannot prove coverage, NOTHING is planned and NOTHING is applied.
20//
21// usage: nx_tsvdrain <tsv-root> <src-root> <ext> [apply]
22// exit 0 OK | 1 SOME-SKIPPED | 3 UNPROVEN (partial coverage -- no plan issued) | 2 usage
23// license_tier: ORIGINAL Writes only via nx_tsv_migrate (additive) and nx_retire_path (rename). Rule 26.
24import "nx_syscalls.nx"
25import "nx_tool_run.nx"
26import "nx_lib_std.nx"
27
28const TD_SHELLTOOL: *u8 = "/volume1/homes/elderwesto/nishihost/nx_shelltool.elf"
29const TD_MIGRATE: *u8 = "/volume1/homes/elderwesto/nishihost/nx_tsv_migrate.elf"
30const TD_RETIRE: *u8 = "/volume1/homes/elderwesto/nishihost/nx_retire_path.elf"
31const TD_PLANE_ROOT: *u8 = "knowledge/store/"
32// Subprocess captures: size unknowable before the run, so NAMED for that one purpose, and truncation
33// ANNOUNCES and REFUSES rather than silently yielding a partial answer.
34const TD_CAPTURE_CAP: i64 = 8388608
35const TD_SMALL_CAP: i64 = 65536
36const TD_MAXTSV: i64 = 8192
37const TD_RC_SKIPPED: i64 = 1
38const TD_RC_USAGE: i64 = 2
39const TD_RC_UNPROVEN: i64 = 3
40const TD_SLASH: i64 = 47
41const TD_LF: i64 = 10
42const TD_DASH: i64 = 45
43const TD_DOT: i64 = 46
44
45func td_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
46func td_putn(v: i64) -> i64 {
47 let b: *u8 = sys_mmap(32)
48 let t: *u8 = sys_mmap(32)
49 var m: i64 = v
50 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
51 var k: i64 = 0
52 if m == 0 { t[0] = 48 as u8; k = 1 }
53 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
54 var i: i64 = 0
55 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
56 sys_write(1, b, k)
57 return 0
58}
59func td_at(buf: *u8, i: i64, n: i64, needle: *u8) -> i64 {
60 var k: i64 = 0
61 while needle[k] != (0 as u8) {
62 if i + k >= n { return 0 }
63 if buf[i + k] != needle[k] { return 0 }
64 k = k + 1
65 }
66 return 1
67}
68func td_has(buf: *u8, n: i64, needle: *u8) -> i64 {
69 var i: i64 = 0
70 while i < n { if td_at(buf, i, n, needle) == 1 { return 1 } i = i + 1 }
71 return 0
72}
73func td_num_after(buf: *u8, n: i64, needle: *u8) -> i64 {
74 var found: i64 = 0 - 1
75 var i: i64 = 0
76 while i < n {
77 if td_at(buf, i, n, needle) == 1 {
78 var j: i64 = i
79 while needle[j - i] != (0 as u8) { j = j + 1 }
80 var v: i64 = 0
81 var got: i64 = 0
82 var go: i64 = 1
83 while go == 1 {
84 if j >= n { go = 0 } else {
85 let c: i64 = buf[j] as i64
86 if c < 48 { go = 0 } else {
87 if c > 57 { go = 0 } else { v = v * 10 + (c - 48); got = 1; j = j + 1 }
88 }
89 }
90 }
91 if got == 1 { found = v }
92 }
93 i = i + 1
94 }
95 return found
96}
97func td_has_span(hay: *u8, hl: i64, nee: *u8, ns: i64, nl: i64) -> i64 {
98 if nl <= 0 { return 0 }
99 var i: i64 = 0
100 while i + nl <= hl {
101 var k: i64 = 0
102 var ok: i64 = 1
103 while k < nl { if hay[i + k] != nee[ns + k] { ok = 0; k = nl } else { k = k + 1 } }
104 if ok == 1 { return 1 }
105 i = i + 1
106 }
107 return 0
108}
109func td_partial(buf: *u8, n: i64) -> i64 {
110 if td_has(buf, n, "NX-TRUNCATED" as *u8) == 1 { return 1 }
111 if td_has(buf, n, "OUTPUT-IS-PARTIAL" as *u8) == 1 { return 1 }
112 if td_has(buf, n, "BUDGET-EXCEEDED" as *u8) == 1 { return 1 }
113 return 0
114}
115// copy [s,s+l) of src into a fresh NUL-terminated buffer
116func td_dup(src: *u8, s: i64, l: i64) -> *u8 {
117 let d: *u8 = sys_mmap(l + 2)
118 var i: i64 = 0
119 while i < l { d[i] = src[s + i]; i = i + 1 }
120 d[l] = 0 as u8
121 return d
122}
123// plane prefix = TD_PLANE_ROOT + the PATH (not the basename) with '/' -> '_', minus .tsv, + "-"
124// ⚠DERIVING THIS FROM THE BASENAME WAS A REAL DATA-INTEGRITY DEFECT, caught by READING THE DRY OUTPUT on
125// 2026-08-14 before any apply ever ran. Several research subdirectories each carry a manifest file and a
126// counts file under the SAME leaf name, so eight unrelated sources derived just two plane prefixes and
127// would have been silently merged -- and a merge is not something a later reader could untangle.
128// ⚠⚠THE FILENAMES ARE DELIBERATELY NOT SPELLED OUT ABOVE. Writing them here made this organ's own comment
129// a CONSUMER of those files: the very next census read consumed 163->171 and planned 43->35, because the
130// reference scan found the names in this explanation of them. ★★★PROSE IS SOURCE BYTES, AND A DETECTOR THAT
131// NAMES ITS SUBJECTS IN A COMMENT CONTAMINATES ITS OWN MEASUREMENT -- second occurrence in one day.
132// ★A NAME DERIVED FROM PART OF A UNIQUE KEY IS NOT UNIQUE. The PATH is what distinguishes these files, so
133// the path is what the plane must be named from. The collision check at the call site exists because I got
134// this wrong once: the fix is in the path, not in remembering.
135func td_plane(path: *u8, pl: i64) -> *u8 {
136 let d: *u8 = sys_mmap(pl + 64)
137 var i: i64 = 0
138 while TD_PLANE_ROOT[i] != (0 as u8) { d[i] = TD_PLANE_ROOT[i]; i = i + 1 }
139 var s: i64 = 0
140 var lead: i64 = 1
141 while lead == 1 {
142 lead = 0
143 if s + 2 < pl {
144 if (path[s] as i64) == TD_DOT {
145 if (path[s+1] as i64) == TD_DOT {
146 if (path[s+2] as i64) == TD_SLASH { s = s + 3; lead = 1 }
147 }
148 }
149 }
150 }
151 var stem: i64 = pl
152 var j: i64 = s
153 while j < pl { if (path[j] as i64) == TD_DOT { stem = j } j = j + 1 }
154 var k: i64 = s
155 while k < stem {
156 if (path[k] as i64) == TD_SLASH { d[i] = 95 as u8 } else { d[i] = path[k] }
157 i = i + 1; k = k + 1
158 }
159 d[i] = TD_DASH as u8; i = i + 1
160 d[i] = 0 as u8
161 return d
162}
163
164func main(argc: i64, argv: *i64) -> i64 {
165 if argc < 5 {
166 td_puts("usage: nx_tsvdrain <workdir> <tsv-root> <src-root> <ext> [apply]\n" as *u8)
167 td_puts(" DRY BY DEFAULT -- only the literal verb apply changes anything.\n" as *u8)
168 td_puts(" workdir is chdir'd into FIRST, so every path below is relative to it and none needs a\n" as *u8)
169 td_puts(" leading '..'. That is load-bearing: nx_retire_path REFUSES TRAVERSAL, so the first apply\n" as *u8)
170 td_puts(" run migrated 42 files and retired ZERO purely because it was launched from the build\n" as *u8)
171 td_puts(" tree. A TOOL'S WORKING DIRECTORY IS PART OF ITS INTERFACE.\n" as *u8)
172 td_puts(" exit 0 OK | 1 SOME-SKIPPED | 3 UNPROVEN | 2 usage\n" as *u8)
173 return TD_RC_USAGE
174 }
175 let wdir: *u8 = argv[1] as *u8
176 if sys_chdir(wdir) < 0 {
177 td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=chdir-failed workdir=" as *u8); td_puts(wdir)
178 td_puts(" -- refusing to run against paths that are not what the caller meant.\n" as *u8)
179 return TD_RC_UNPROVEN
180 }
181 let troot: *u8 = argv[2] as *u8
182 let sroot: *u8 = argv[3] as *u8
183 let ext: *u8 = argv[4] as *u8
184 var apply: i64 = 0
185 if argc >= 6 {
186 let a: *u8 = argv[5] as *u8
187 if td_at(a, 0, 5, "apply" as *u8) == 1 { apply = 1 }
188 }
189
190 let av: *i64 = sys_mmap(64) as *i64
191 av[0] = TD_SHELLTOOL as i64
192 av[1] = "glob" as *u8 as i64
193 av[2] = "*.tsv" as *u8 as i64
194 av[3] = troot as i64
195 av[4] = 0
196 let gbuf: *u8 = sys_mmap(TD_CAPTURE_CAP + 1)
197 let glen: *i64 = sys_mmap(16) as *i64
198 let grc: i64 = tr_run_capture(TD_SHELLTOOL, av, gbuf, TD_CAPTURE_CAP, glen)
199 if grc < 0 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=harness-failure pass=glob\n" as *u8); return TD_RC_UNPROVEN }
200 let gn: i64 = glen[0]
201 if td_partial(gbuf, gn) == 1 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=glob-truncated\n" as *u8); return TD_RC_UNPROVEN }
202 if td_num_after(gbuf, gn, "corpus_complete=" as *u8) != 1 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=glob-coverage\n" as *u8); return TD_RC_UNPROVEN }
203
204 let av2: *i64 = sys_mmap(64) as *i64
205 av2[0] = TD_SHELLTOOL as i64
206 av2[1] = "grep" as *u8 as i64
207 av2[2] = ".tsv" as *u8 as i64
208 av2[3] = sroot as i64
209 av2[4] = ext as i64
210 av2[5] = 0
211 let rbuf: *u8 = sys_mmap(TD_CAPTURE_CAP + 1)
212 let rlen: *i64 = sys_mmap(16) as *i64
213 let rrc: i64 = tr_run_capture(TD_SHELLTOOL, av2, rbuf, TD_CAPTURE_CAP, rlen)
214 if rrc < 0 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=harness-failure pass=grep\n" as *u8); return TD_RC_UNPROVEN }
215 let rn: i64 = rlen[0]
216 if td_partial(rbuf, rn) == 1 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=grep-truncated\n" as *u8); return TD_RC_UNPROVEN }
217 if td_num_after(rbuf, rn, "corpus_complete=" as *u8) != 1 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=grep-coverage\n" as *u8); return TD_RC_UNPROVEN }
218
219 // ★★★SAME GUARD AS THE CENSUS, AND IT MATTERS MORE HERE BECAUSE THIS ORGAN RETIRES FILES. A partition
220 // over the wrong tree still reconciles, so `RECONCILES` alone never proves the subject. nx_shelltool
221 // reports files=N for the tree it actually walked; both counts are printed, and when they are EQUAL
222 // the caller is scanning one tree twice -- the signature of a wrong root argument.
223 // Refused only at ZERO: that is not a tuned bar but the point where the searcher saw nothing, and a
224 // zero source scan would make EVERY tsv read UNCONSUMED for want of a reader -- i.e. it would hand a
225 // destructive sweep the entire corpus as its worklist.
226 let gfiles: i64 = td_num_after(gbuf, gn, "files=" as *u8)
227 let rfiles: i64 = td_num_after(rbuf, rn, "files=" as *u8)
228 if gfiles == 0 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=empty-tsv-tree files=0 -- check tsv-root\n" as *u8); return TD_RC_UNPROVEN }
229 if rfiles == 0 { td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=empty-src-tree files=0 -- every tsv would read UNCONSUMED for want of a reader; REFUSING to plan a sweep on that\n" as *u8); return TD_RC_UNPROVEN }
230 var total: i64 = 0
231 var consumed: i64 = 0
232 var planned: i64 = 0
233 var migrated: i64 = 0
234 var retired: i64 = 0
235 var skipped: i64 = 0
236 // A TOMBSTONE IS A CORRECT OUTCOME, NOT A FAILURE: counted apart from skipped, and it does NOT make
237 // the run non-zero. A deprecated registry may be left deliberately comment-only so that any parser
238 // still pointed at it reads zero rows and fails closed, with its comments carrying the redirect to the
239 // plane that replaced it. Retiring one deletes a signpost.
240 // ★★★UNCONSUMED DOES NOT MEAN RETIRE-ME, AND A WORKLIST THAT KEEPS RE-PROPOSING SOMETHING YOU MUST
241 // NEVER DO IS ONE EVERYBODY LEARNS TO IGNORE.
242 // ⚠⚠NO FILENAME APPEARS IN THIS COMMENT, AND THAT IS STRUCTURAL, NOT STYLE. This organ's reference scan
243 // reads organ source, so naming a measured file HERE makes this file its consumer. It happened three
244 // times in one day; the third time did not merely skew a count, it moved the subject into CONSUMED so
245 // it left the worklist and the tombstone branch below never executed -- the documentation silently
246 // disabled the feature it documented. ★★★AN INSTRUMENT'S SOURCE MUST NOT CONTAIN THE DATA IT MEASURES;
247 // worked examples belong in the memory record, and the source carries the RULE.
248 var tomb: i64 = 0
249 td_puts("=== NX-TSVDRAIN mode=" as *u8)
250 if apply == 1 { td_puts("APPLY" as *u8) } else { td_puts("DRY (nothing is changed)" as *u8) }
251 td_puts(" ===\n" as *u8)
252
253 // every planned plane, so a collision REFUSES instead of merging two sources into one plane
254 let seen: *i64 = sys_mmap(TD_MAXTSV*8) as *i64
255 var i: i64 = 0
256 var lstart: i64 = 0
257 while i <= gn {
258 var eol: i64 = 0
259 if i == gn { eol = 1 } else { if (gbuf[i] as i64) == TD_LF { eol = 1 } }
260 if eol == 1 {
261 let e: i64 = i
262 if e > lstart {
263 if (gbuf[lstart] as i64) != TD_DASH {
264 if td_at(gbuf, e - 4, gn, ".tsv" as *u8) == 1 {
265 total = total + 1
266 var b: i64 = lstart
267 var j: i64 = lstart
268 while j < e { if (gbuf[j] as i64) == TD_SLASH { b = j + 1 } j = j + 1 }
269 if td_has_span(rbuf, rn, gbuf, b, e - b) == 1 { consumed = consumed + 1 }
270 else {
271 planned = planned + 1
272 let path: *u8 = td_dup(gbuf, lstart, e - lstart)
273 let plane: *u8 = td_plane(path, e - lstart)
274 var dup: i64 = 0
275 var q: i64 = 0
276 while q + 1 < planned { if std_streq(plane, seen[q] as *u8) == 1 { dup = 1 } q = q + 1 }
277 if dup == 1 {
278 td_puts("NX-TSVDRAIN verdict=UNPROVEN reason=plane-collision path=" as *u8)
279 td_puts(path); td_puts(" plane=" as *u8); td_puts(plane)
280 td_puts(" -- two sources derive one plane; REFUSING so nothing is merged.\n" as *u8)
281 return TD_RC_UNPROVEN
282 }
283 seen[planned - 1] = plane as i64
284 td_puts(" " as *u8); td_puts(path); td_puts(" -> " as *u8); td_puts(plane)
285 if apply == 1 {
286 let mv: *i64 = sys_mmap(64) as *i64
287 mv[0] = TD_MIGRATE as i64
288 mv[1] = path as i64
289 mv[2] = plane as i64
290 mv[3] = 0
291 let mbuf: *u8 = sys_mmap(TD_SMALL_CAP + 1)
292 let mlen: *i64 = sys_mmap(16) as *i64
293 let mrc: i64 = tr_run_capture(TD_MIGRATE, mv, mbuf, TD_SMALL_CAP, mlen)
294 var green: i64 = 0
295 if mrc >= 0 {
296 if td_has(mbuf, mlen[0], "TSV-MIGRATE GREEN" as *u8) == 1 {
297 if td_num_after(mbuf, mlen[0], "\"verified\":" as *u8) == 1 { green = 1 }
298 }
299 }
300 if green == 1 {
301 migrated = migrated + 1
302 let rv: *i64 = sys_mmap(64) as *i64
303 rv[0] = TD_RETIRE as i64
304 rv[1] = "retire" as *u8 as i64
305 rv[2] = path as i64
306 rv[3] = 0
307 let rb: *u8 = sys_mmap(TD_SMALL_CAP + 1)
308 let rl: *i64 = sys_mmap(16) as *i64
309 let rr: i64 = tr_run_capture(TD_RETIRE, rv, rb, TD_SMALL_CAP, rl)
310 if rr >= 0 {
311 if td_num_after(rb, rl[0], "\"reversible\":" as *u8) == 1 {
312 retired = retired + 1
313 td_puts(" MIGRATED+RETIRED\n" as *u8)
314 } else { skipped = skipped + 1; td_puts(" MIGRATED, RETIRE-REFUSED (source left in place)\n" as *u8) }
315 } else { skipped = skipped + 1; td_puts(" MIGRATED, RETIRE-HARNESS-FAIL (source left in place)\n" as *u8) }
316 } else {
317 if td_has(mbuf, mlen[0], "no data lines" as *u8) == 1 {
318 tomb = tomb + 1
319 td_puts(" TOMBSTONE: comment-only by design -- correctly LEFT IN PLACE, not a failure\n" as *u8)
320 } else {
321 skipped = skipped + 1
322 td_puts(" SKIPPED: migrate did not report GREEN+verified -- source untouched\n" as *u8)
323 }
324 }
325 } else { td_puts(" [dry]\n" as *u8) }
326 }
327 }
328 }
329 }
330 lstart = i + 1
331 }
332 i = i + 1
333 }
334
335 td_puts("tsv_files_walked=" as *u8); td_putn(gfiles)
336 td_puts(" src_files_walked=" as *u8); td_putn(rfiles)
337 td_puts("\ntsv_total=" as *u8); td_putn(total)
338 td_puts(" consumed=" as *u8); td_putn(consumed)
339 td_puts(" planned=" as *u8); td_putn(planned)
340 td_puts(" migrated=" as *u8); td_putn(migrated)
341 td_puts(" retired=" as *u8); td_putn(retired)
342 td_puts(" skipped=" as *u8); td_putn(skipped)
343 td_puts(" tombstones=" as *u8); td_putn(tomb)
344 td_puts(" sum=" as *u8); td_putn(consumed + planned)
345 if consumed + planned == total { td_puts(" partition=RECONCILES" as *u8) } else { td_puts(" partition=LEAK" as *u8) }
346 td_puts("\n" as *u8)
347 if skipped > 0 { td_puts("verdict=SOME-SKIPPED\n" as *u8); return TD_RC_SKIPPED }
348 td_puts("verdict=OK\n" as *u8)
349 return 0
350}