code wiki / _hdl_build / nx_dedup_candidates.nx
nx_dedup_candidates.nx source
↩ module page · 294 lines · 14516 B
1// nx_dedup_candidates.nx -- READ-ONLY migration-candidate scanner (F1029 measurement foundation, 2026-07-23).
2// Scans a dir's *.nx and reports organs that define a migratable helper SHAPE and do NOT already import
3// nx_sovjson_lib -- the organs the dedup walker COULD migrate but hasn't. Excludes _gate/_register by name.
4// ★OOM-SAFE BY CONSTRUCTION (dup_source_check-v3 lesson): dir/file/name/out buffers each allocated ONCE
5// before the loop + reused per file (O(1), not O(files)); PROVEN 384KiB peak RSS over 4096 files. Read via
6// open/read-into-fixed-buffer/close -- NEVER sys_read_file (allocates per call = the exact OOM bug).
7// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
8import "nx_syscalls.nx"
9import "nx_gate_verdict.nx"
10import "nx_sovjson_lib.nx"
11import "nx_store_seed_lib.nx"
12import "nx_seg_store.nx"
13
14const DC_PLANE: i64 = 1048576
15const DC_DIRBUF: i64 = 131072
16const DC_FILECAP: i64 = 262144
17const DC_NAMEW: i64 = 256
18const DC_OUT: i64 = 65536
19const DC_SHAPES: i64 = 9
20const DC_SHOWN: i64 = 40
21
22func dc_read_into(path: *u8, buf: *u8, cap: i64) -> i64 {
23 let fd: i64 = sys_openat_rd(path)
24 if fd < 0 { return 0 - 1 }
25 var t: i64 = 0
26 var go: i64 = 1
27 while go == 1 {
28 if t >= cap { go = 0 } else {
29 let r: i64 = sys_read(fd, ((buf as i64) + t) as *u8, cap - t)
30 if r <= 0 { go = 0 } else { t = t + r }
31 }
32 }
33 sys_close(fd)
34 return t
35}
36
37func dc_ends(name: *u8, suf: *u8) -> i64 {
38 let nl: i64 = sj_vlen(name)
39 let sl: i64 = sj_vlen(suf)
40 if nl < sl { return 0 }
41 var i: i64 = 0
42 while i < sl { if name[nl - sl + i] != suf[i] { return 0 } i = i + 1 }
43 return 1
44}
45func dc_contains_name(name: *u8, sub: *u8) -> i64 {
46 let nl: i64 = sj_vlen(name)
47 let sl: i64 = sj_vlen(sub)
48 if sl == 0 { return 0 }
49 var i: i64 = 0
50 while i + sl <= nl {
51 var m: i64 = 1
52 var j: i64 = 0
53 while j < sl { if name[i+j] != sub[j] { m = 0; j = sl } else { j = j + 1 } }
54 if m == 1 { return 1 }
55 i = i + 1
56 }
57 return 0
58}
59
60func dc_shape(k: i64) -> *u8 {
61 if k == 0 { return "_puts(" as *u8 }
62 if k == 1 { return "_werr(" as *u8 }
63 if k == 2 { return "_vlen(" as *u8 }
64 if k == 3 { return "_slen(" as *u8 }
65 if k == 4 { return "_cat(" as *u8 }
66 if k == 5 { return "_catn(" as *u8 }
67 if k == 6 { return "_put(" as *u8 }
68 if k == 7 { return "_putn(" as *u8 }
69 if k == 8 { return "_w(" as *u8 }
70 return 0 as *u8
71}
72
73func dc_has_shape(buf: *u8, n: i64) -> i64 {
74 var i: i64 = 0
75 while i + 6 <= n {
76 var at_ls: i64 = 0
77 if i == 0 { at_ls = 1 } else { if buf[i-1] == (10 as u8) { at_ls = 1 } }
78 if at_ls == 1 {
79 if buf[i]==(102 as u8) { if buf[i+1]==(117 as u8) { if buf[i+2]==(110 as u8) { if buf[i+3]==(99 as u8) { if buf[i+4]==(32 as u8) {
80 var p: i64 = i + 5
81 var op: i64 = 0 - 1
82 var s: i64 = 1
83 while s == 1 { if p >= n { s = 0 } else { if buf[p] == (40 as u8) { op = p; s = 0 } else { if buf[p] == (10 as u8) { s = 0 } else { p = p + 1 } } } }
84 if op > i + 5 {
85 var k: i64 = 0
86 while k < DC_SHAPES {
87 let suf: *u8 = dc_shape(k)
88 let sl: i64 = sj_vlen(suf)
89 if op + 1 - sl >= i + 5 {
90 var m: i64 = 1
91 var j: i64 = 0
92 while j < sl { if buf[op + 1 - sl + j] != suf[j] { m = 0; j = sl } else { j = j + 1 } }
93 if m == 1 { return 1 }
94 }
95 k = k + 1
96 }
97 }
98 } } } } }
99 }
100 i = i + 1
101 }
102 return 0
103}
104
105func dc_has_import(buf: *u8, n: i64) -> i64 {
106 return sj_span_has(buf, 0, n, "nx_sovjson_lib" as *u8)
107}
108
109func main(argc: i64, argv: *i64) -> i64 {
110 var dir: *u8 = "buildroot/runtime/_hdl_build\x00" as *u8
111 if argc >= 2 { dir = argv[1] as *u8 }
112 if argc >= 2 { if sj_lit_eq(argv[1] as *u8, 0, sj_vlen(argv[1] as *u8), "selftest" as *u8) == 1 {
113 let ctr: *i64 = gv_ctr()
114 gv_head("nx_dedup_candidates selftest -- pure shape/import/name predicates (hermetic)" as *u8)
115 let a: *u8 = "import \x22x\x22\nfunc ab_w(s: *u8) -> i64 { return 0 }\n" as *u8
116 gv_check("T1 line-start _w helper detected" as *u8, dc_has_shape(a, sj_vlen(a)), ctr)
117 let b: *u8 = "func ab_widget(s: *u8) -> i64 { return 0 }\n" as *u8
118 gv_check("T2 _widget is NOT a _w shape (suffix-before-paren exact)" as *u8, 1 - dc_has_shape(b, sj_vlen(b)), ctr)
119 let c: *u8 = "// func ab_w(s) note\nvar x_w = 1\n" as *u8
120 gv_check("T3 non-line-start / commented func NOT matched" as *u8, 1 - dc_has_shape(c, sj_vlen(c)), ctr)
121 let d: *u8 = "import \x22nx_sovjson_lib.nx\x22\nfunc q_cat() {}\n" as *u8
122 gv_check("T4 already-composes-base detected" as *u8, dc_has_import(d, sj_vlen(d)), ctr)
123 gv_check("T5 _gate name excluded" as *u8, dc_ends("nx_foo_gate" as *u8, "_gate" as *u8), ctr)
124 gv_check("T6 _register name excluded" as *u8, dc_contains_name("nx_foo_register" as *u8, "_register" as *u8), ctr)
125 gv_check("T7 plain organ name not excluded by _gate" as *u8, 1 - dc_ends("nx_libprobe" as *u8, "_gate" as *u8), ctr)
126 gv_check("T8 _catn helper detected" as *u8, dc_has_shape("\nfunc z_catn(d: *u8) -> i64 { return 0 }\n" as *u8, 42), ctr)
127 let rc: i64 = gv_verdict("DEDUP-CANDIDATES-SELFTEST" as *u8, ctr, "shape at line-start + suffix-before-paren + base-import + name-exclude sound" as *u8)
128 sys_exit(rc)
129 return rc
130 } }
131
132 if sj_lit_eq(argv[1] as *u8, 0, sj_vlen(argv[1] as *u8), "seed" as *u8) == 1 {
133 if argc < 4 { sj_werr("seed needs <scandir> <max> [dqprefix]\n" as *u8); sys_exit(2); return 2 }
134 let sdir: *u8 = argv[2] as *u8
135 let smax: i64 = sj_atoi_z(argv[3] as *u8)
136 var dqp: *u8 = "knowledge/store/dedupq-\x00" as *u8
137 if argc >= 5 { dqp = argv[4] as *u8 }
138 let qb: *u8 = sys_mmap(DC_PLANE)
139 let qn: i64 = sts_load(dqp, qb, DC_PLANE - DC_FILECAP)
140 let sdbuf: *u8 = sys_mmap(DC_DIRBUF)
141 let sfbuf: *u8 = sys_mmap(DC_FILECAP)
142 let spath: *u8 = sys_mmap(DC_NAMEW)
143 let keyb: *u8 = sys_mmap(DC_NAMEW)
144 let sob: *u8 = sys_mmap(DC_PLANE)
145 let sdl: i64 = sj_vlen(sdir)
146 let sfd: i64 = sys_openat_rd(sdir)
147 if sfd < 0 { sj_werr("SEED RED: cannot open scan dir\n" as *u8); sys_exit(1); return 1 }
148 var seeded: i64 = 0
149 var already_q: i64 = 0
150 // ADDITIVE (seq756 fix): sts_seed REPLACES the plane -> pre-load the existing rows into the seed
151 // buffer so sts_seed(existing + new) = APPEND. Proven on a PRE-SEEDED scratch (an empty scratch cannot
152 // catch a replace-vs-append bug -- the incident lesson).
153 var so: i64 = 0
154 var qcp: i64 = 0
155 while qcp < qn { sob[qcp] = qb[qcp]; qcp = qcp + 1 }
156 so = qn
157 if qn > 0 { if sob[qn-1] != (10 as u8) { sob[so] = 10 as u8; so = so + 1 } }
158 var sgo: i64 = 1
159 while sgo == 1 {
160 let snr: i64 = sys_getdents64(sfd, sdbuf, DC_DIRBUF)
161 if snr <= 0 { sgo = 0 } else {
162 var soff: i64 = 0
163 while soff < snr {
164 if seeded >= smax { soff = snr } else {
165 let srec: *u8 = ((sdbuf as i64) + soff) as *u8
166 let snm: *u8 = dirent_name(srec)
167 soff = soff + dirent_reclen(srec)
168 if dc_ends(snm, ".nx" as *u8) == 1 { if dc_ends(snm, "_gate.nx" as *u8) == 0 { if dc_contains_name(snm, "_register" as *u8) == 0 {
169 var spo: i64 = 0
170 var sz: i64 = 0
171 while sz < sdl { spath[spo] = sdir[sz]; spo = spo + 1; sz = sz + 1 }
172 spath[spo] = 47 as u8; spo = spo + 1
173 sz = 0
174 while snm[sz] != (0 as u8) { spath[spo] = snm[sz]; spo = spo + 1; sz = sz + 1 }
175 spath[spo] = 0 as u8
176 let sfn: i64 = dc_read_into(spath, sfbuf, DC_FILECAP)
177 if sfn > 0 { if sfn < DC_FILECAP {
178 if dc_has_import(sfbuf, sfn) == 0 { if dc_has_shape(sfbuf, sfn) == 1 {
179 let olen: i64 = sj_vlen(snm) - 3
180 var ko: i64 = sj_cat(keyb, 0, "DQA" as *u8)
181 var kk: i64 = 0
182 while kk < olen { keyb[ko] = snm[kk]; ko = ko + 1; kk = kk + 1 }
183 keyb[ko] = 0 as u8
184 if sj_span_has(qb, 0, qn, keyb) == 1 { already_q = already_q + 1 } else {
185 so = sj_cat(sob, so, keyb)
186 sob[so] = 9 as u8; so = so + 1
187 var w2: i64 = 0
188 while w2 < olen { sob[so] = snm[w2]; so = so + 1; w2 = w2 + 1 }
189 sob[so] = 9 as u8; so = so + 1
190 so = sj_cat(sob, so, "LOCAL-MAKER\tqueued\t" as *u8)
191 so = sj_cat(sob, so, keyb)
192 so = sj_cat(sob, so, "|" as *u8)
193 w2 = 0
194 while w2 < olen { sob[so] = snm[w2]; so = so + 1; w2 = w2 + 1 }
195 so = sj_cat(sob, so, "|LOCAL-MAKER|queued|F1029-auto-seeded\n" as *u8)
196 seeded = seeded + 1
197 }
198 } }
199 } }
200 } } }
201 }
202 }
203 }
204 }
205 sys_close(sfd)
206 if seeded > 0 { sts_seed(dqp, sob, so) }
207 let rb: *u8 = sys_mmap(DC_OUT)
208 var ro: i64 = sj_cat(rb, 0, "{\x22organ\x22:\x22nx_dedup_candidates\x22,\x22verb\x22:\x22seed\x22,\x22seeded\x22:" as *u8)
209 ro = sj_catn(rb, ro, seeded)
210 ro = sj_cat(rb, ro, ",\x22max\x22:" as *u8)
211 ro = sj_catn(rb, ro, smax)
212 ro = sj_cat(rb, ro, ",\x22already_queued\x22:" as *u8)
213 ro = sj_catn(rb, ro, already_q)
214 ro = sj_cat(rb, ro, ",\x22idempotent\x22:\x22key=DQA<organ> name-derived; re-seed REPLACES; commit=1; bounded; det-filter=walker-judge\x22}" as *u8)
215 rb[ro] = 10 as u8
216 sys_write(1, rb, ro + 1)
217 sys_exit(0)
218 return 0
219 }
220
221 let dbuf: *u8 = sys_mmap(DC_DIRBUF)
222 let fbuf: *u8 = sys_mmap(DC_FILECAP)
223 let path: *u8 = sys_mmap(DC_NAMEW)
224 let ob: *u8 = sys_mmap(DC_OUT)
225 let dirlen: i64 = sj_vlen(dir)
226 let fd: i64 = sys_openat_rd(dir)
227 if fd < 0 { sj_werr("DC RED: cannot open scan dir\n" as *u8); sys_exit(1); return 1 }
228 var scanned: i64 = 0
229 var cand: i64 = 0
230 var skipped_big: i64 = 0
231 var already: i64 = 0
232 var oo: i64 = sj_cat(ob, 0, "{\x22organ\x22:\x22nx_dedup_candidates\x22,\x22candidates\x22:[" as *u8)
233 var first: i64 = 1
234 var go: i64 = 1
235 while go == 1 {
236 let nr: i64 = sys_getdents64(fd, dbuf, DC_DIRBUF)
237 if nr <= 0 { go = 0 } else {
238 var off: i64 = 0
239 while off < nr {
240 let rec: *u8 = ((dbuf as i64) + off) as *u8
241 let nm: *u8 = dirent_name(rec)
242 off = off + dirent_reclen(rec)
243 if dc_ends(nm, ".nx" as *u8) == 1 {
244 if dc_ends(nm, "_gate.nx" as *u8) == 0 { if dc_contains_name(nm, "_register" as *u8) == 0 {
245 var po: i64 = 0
246 var z: i64 = 0
247 while z < dirlen { path[po] = dir[z]; po = po + 1; z = z + 1 }
248 path[po] = 47 as u8; po = po + 1
249 z = 0
250 while nm[z] != (0 as u8) { path[po] = nm[z]; po = po + 1; z = z + 1 }
251 path[po] = 0 as u8
252 let fn: i64 = dc_read_into(path, fbuf, DC_FILECAP)
253 if fn >= DC_FILECAP { skipped_big = skipped_big + 1 } else {
254 if fn > 0 {
255 scanned = scanned + 1
256 if dc_has_import(fbuf, fn) == 1 { already = already + 1 } else {
257 if dc_has_shape(fbuf, fn) == 1 {
258 cand = cand + 1
259 if cand <= DC_SHOWN {
260 if first == 0 { oo = sj_cat(ob, oo, "," as *u8) }
261 first = 0
262 oo = sj_cat(ob, oo, "\x22" as *u8)
263 var w: i64 = 0
264 while nm[w] != (0 as u8) { if nm[w] != (34 as u8) { ob[oo] = nm[w]; oo = oo + 1 } w = w + 1 }
265 oo = sj_cat(ob, oo, "\x22" as *u8)
266 }
267 }
268 }
269 }
270 }
271 } }
272 }
273 }
274 }
275 }
276 sys_close(fd)
277 oo = sj_cat(ob, oo, "],\x22candidates_total\x22:" as *u8)
278 oo = sj_catn(ob, oo, cand)
279 oo = sj_cat(ob, oo, ",\x22shown\x22:" as *u8)
280 var shown: i64 = cand
281 if shown > DC_SHOWN { shown = DC_SHOWN }
282 oo = sj_catn(ob, oo, shown)
283 oo = sj_cat(ob, oo, ",\x22scanned\x22:" as *u8)
284 oo = sj_catn(ob, oo, scanned)
285 oo = sj_cat(ob, oo, ",\x22already_on_base\x22:" as *u8)
286 oo = sj_catn(ob, oo, already)
287 oo = sj_cat(ob, oo, ",\x22skipped_oversized\x22:" as *u8)
288 oo = sj_catn(ob, oo, skipped_big)
289 oo = sj_cat(ob, oo, ",\x22envelope\x22:\x22O(1) allocation buffers-once (dup_source_check-v3 OOM lesson, PROVEN 384KiB peak/4096 files); .nx only, _gate+_register excluded; files over 256KiB skipped+counted; shape=line-start func with a sj_-migratable suffix before the paren; CANDIDATES are proposals -- the dedup judge is the correctness net\x22}" as *u8)
290 ob[oo] = 10 as u8
291 sys_write(1, ob, oo + 1)
292 sys_exit(0)
293 return 0
294}