nx_bundle_gen.nx source
↩ module page · 473 lines · 17890 B
1// nx_bundle_gen.nx -- pure-NishiLang shard-bundle generator.
2//
3// Walks specs/algo_registry/ to collect shape-driven primitives, then
4// emits runtime/_bundle_shard_<i>.nx files where each shard inlines
5// renamed copies of multiple primitives' test.nx, plus a master main()
6// that invokes every test_<nx>() and emits PASS/FAIL via sys_write.
7//
8// Replaces sed-per-primitive in bash (which cost ~139s wall for 1192
9// primitives via 1192 subshells). Pure NishiLang means substrate self-
10// generates its own bundles -- aligns with the "no host tools for own
11// bookkeeping" directive.
12//
13// Rewriting rules per inlined test.nx:
14// 1. Strip lines starting with `import ` (bundle re-emits the shared
15// imports at the top once)
16// 2. Rename `func main() -> nx_int` -> `func test_<nx>() -> nx_int`
17// and `func main() -> i64` -> `func test_<nx>() -> nx_int`
18// 3. Replace `__syscall(93, N, 0, 0, 0, 0, 0)` (process exit syscall)
19// with plain `N` so failure propagates to the bundle's master main
20
21// nx_safety_envelope:
22// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
23// sil_target: SIL1
24// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
25// verdict: NOT_YET_EVALUATED
26
27import "nx_syscalls.nx"
28import "nx_runtime.nx"
29import "nx_dirent.nx"
30import "nx_fcntl.nx"
31import "nx_tier.nx"
32
33const NX_BG_PATH_BUF: nx_size = 512
34const NX_BG_FIELD_CAP: nx_size = 4096
35const NX_BG_SHARD_SIZE: nx_int = 200
36const NX_BG_MAX_PRIMS: nx_size = 1048576 // 1M slots; mmap is lazy
37const NX_BG_NAME_LEN: nx_size = 64 // bytes per name slot
38const NX_BG_SCRATCH: nx_size = 65536 // per-prim scratch for test.nx
39
40// ===== string + IO primitives =========================================
41
42func bg_str_cat(dst: *u8, off: nx_idx, src: *u8) -> nx_idx {
43 var i: nx_idx = 0
44 while src[i] != 0 {
45 dst[off + i] = src[i]
46 i = i + 1
47 }
48 dst[off + i] = 0
49 return off + i
50}
51
52func bg_write_all(fd: nx_fd, buf: *u8, n: nx_size) -> nx_size {
53 var written: nx_size = 0
54 while written < n {
55 let p: *u8 = ((buf as nx_size) + written) as *u8
56 let m: nx_errno = sys_write(fd, p, n - written)
57 if m <= 0 { return written }
58 written = written + m
59 }
60 return written
61}
62
63func bg_wrs(fd: nx_fd, s: *u8) -> nx_int {
64 bg_write_all(fd, s, strlen(s))
65 return 0
66}
67
68// Substring search in buf[start..end). Returns offset or -1.
69func bg_find(buf: *u8, start: nx_idx, end: nx_idx, needle: *u8, nlen: nx_size) -> nx_idx {
70 if nlen == 0 { return start }
71 if (end as nx_size) < (nlen as nx_size) { return -1 }
72 var p: nx_idx = start
73 let last: nx_idx = end - nlen
74 while p <= last {
75 var j: nx_idx = 0
76 var hit: nx_int = 1
77 while j < nlen {
78 if buf[p + j] != needle[j] { hit = 0; j = nlen }
79 j = j + 1
80 }
81 if hit == 1 { return p }
82 p = p + 1
83 }
84 return -1
85}
86
87// Read "key":"<value>" field from JSON-ish buf into out. Returns value len.
88func bg_read_field(buf: *u8, n: nx_size, key: *u8, out: *u8) -> nx_size {
89 let pat: *u8 = sys_mmap(NX_BUF_SMALL)
90 var po: nx_idx = 0
91 pat[po] = 34; po = po + 1
92 var ki: nx_idx = 0
93 while key[ki] != 0 {
94 pat[po] = key[ki]; po = po + 1
95 ki = ki + 1
96 }
97 pat[po] = 34; po = po + 1
98 pat[po] = 58; po = po + 1
99 pat[po] = 0
100 let p: nx_idx = bg_find(buf, 0, n, pat, po)
101 if p < 0 { out[0] = 0; return 0 }
102 var q: nx_idx = p + po
103 while q < n {
104 let c: nx_int = buf[q] as nx_int
105 if c == 34 { q = q + 1; var l: nx_size = 0
106 while q < n {
107 if buf[q] as nx_int == 34 { out[l] = 0; return l }
108 if l < NX_BG_FIELD_CAP - 1 { out[l] = buf[q]; l = l + 1 }
109 q = q + 1
110 }
111 out[l] = 0
112 return l
113 }
114 if c != 32 { if c != 9 { if c != 10 { if c != 13 { out[0] = 0; return 0 } } } }
115 q = q + 1
116 }
117 out[0] = 0
118 return 0
119}
120
121func bg_file_exists(path: *u8) -> nx_int {
122 let fd: nx_fd = nx_openat(NX_AT_FDCWD, path, NX_O_RDONLY, 0)
123 if fd < 0 { return 0 }
124 sys_close(fd)
125 return 1
126}
127
128// ===== collect prims ====================================================
129
130// Walks specs/algo_registry/ via nx_dirent. For each subdir with a
131// meta.json that contains "shape" and a corresponding runtime/<nx>_test.nx,
132// stores nx_module at prims_buf[i * NX_BG_NAME_LEN]. Returns count.
133func bg_collect_prims(prims_buf: *u8) -> nx_size {
134 let fd: nx_fd = nx_openat(NX_AT_FDCWD,
135 "nxc2/specs/algo_registry" as *u8,
136 NX_O_RDONLY | NX_O_DIRECTORY, 0)
137 if fd < 0 { return 0 }
138 let dirent_buf: *u8 = sys_mmap(NX_BUF_LARGE)
139 let dr_raw: *u8 = sys_mmap(NX_DIRENT_BYTES)
140 let dr: *NxDirent = dr_raw as *NxDirent
141 var count: nx_size = 0
142 let meta_path: *u8 = sys_mmap(NX_BG_PATH_BUF)
143 let test_path: *u8 = sys_mmap(NX_BG_PATH_BUF)
144 let field_buf: *u8 = sys_mmap(NX_BG_FIELD_CAP)
145 var batch: nx_size = nx_dirent_read(fd, dirent_buf, NX_BUF_LARGE)
146 while batch > 0 {
147 var off: nx_idx = 0
148 while off < batch {
149 let next_off: nx_idx = nx_dirent_iter(dirent_buf, off, batch, dr)
150 if next_off <= 0 { off = batch + 1 }
151 if off <= batch {
152 let nm_len: nx_size = nx_dirent_name_len(dr)
153 var is_dot: nx_int = 0
154 if nm_len == 1 { if dr.name[0] == 46 { is_dot = 1 } }
155 if nm_len == 2 { if dr.name[0] == 46 { if dr.name[1] == 46 { is_dot = 1 } } }
156 if is_dot == 0 {
157 var o: nx_idx = 0
158 o = bg_str_cat(meta_path, o, "nxc2/specs/algo_registry/" as *u8)
159 o = bg_str_cat(meta_path, o, dr.name)
160 o = bg_str_cat(meta_path, o, "/meta.json" as *u8)
161 let len_p: *nx_size = (sys_mmap(NX_SIZEOF_NX_SIZE)) as *nx_size
162 len_p[0] = 0
163 let meta_buf: *u8 = sys_read_file(meta_path, len_p)
164 if (meta_buf as nx_size) != 0 {
165 let meta_n: nx_size = len_p[0]
166 // shape-driven?
167 let shape_pos: nx_idx = bg_find(meta_buf, 0, meta_n,
168 "\"shape\":" as *u8, 8)
169 if shape_pos >= 0 {
170 let nx_len: nx_size = bg_read_field(meta_buf, meta_n,
171 "nx_module" as *u8, field_buf)
172 if nx_len > 0 {
173 // Does runtime/<nx>_test.nx exist?
174 var to: nx_idx = 0
175 to = bg_str_cat(test_path, to, "nxc2/runtime/" as *u8)
176 to = bg_str_cat(test_path, to, field_buf)
177 to = bg_str_cat(test_path, to, "_test.nx" as *u8)
178 if bg_file_exists(test_path) == 1 {
179 if count < NX_BG_MAX_PRIMS {
180 let slot: *u8 = ((prims_buf as nx_size) + count * NX_BG_NAME_LEN) as *u8
181 var si: nx_idx = 0
182 while field_buf[si] != 0 {
183 if si < NX_BG_NAME_LEN - 1 {
184 slot[si] = field_buf[si]
185 }
186 si = si + 1
187 }
188 slot[si] = 0
189 count = count + 1
190 }
191 }
192 }
193 }
194 }
195 }
196 off = next_off
197 }
198 }
199 batch = nx_dirent_read(fd, dirent_buf, NX_BUF_LARGE)
200 }
201 sys_close(fd)
202 return count
203}
204
205// ===== test.nx rewriter =================================================
206
207// Find length of line starting at pos in buf[..n). Returns position of
208// first newline (or n if line runs to EOF, in which case it is unterminated).
209func bg_line_end(buf: *u8, pos: nx_idx, n: nx_size) -> nx_idx {
210 var p: nx_idx = pos
211 while p < n {
212 if buf[p] == 10 { return p }
213 p = p + 1
214 }
215 return n
216}
217
218// Does the line buf[pos..le) start (after optional leading spaces/tabs)
219// with the literal "import "?
220func bg_is_import_line(buf: *u8, pos: nx_idx, le: nx_idx) -> nx_int {
221 var p: nx_idx = pos
222 var done: nx_int = 0
223 while done == 0 {
224 if p >= le { done = 1 }
225 if done == 0 {
226 let c: nx_int = buf[p] as nx_int
227 if c == 32 { p = p + 1 }
228 if c == 9 { p = p + 1 }
229 if c != 32 { if c != 9 { done = 1 } }
230 }
231 }
232 if p + 7 > le { return 0 }
233 if buf[p+0] != 105 { return 0 } // i
234 if buf[p+1] != 109 { return 0 } // m
235 if buf[p+2] != 112 { return 0 } // p
236 if buf[p+3] != 111 { return 0 } // o
237 if buf[p+4] != 114 { return 0 } // r
238 if buf[p+5] != 116 { return 0 } // t
239 if buf[p+6] != 32 { return 0 } // space
240 return 1
241}
242
243// Write one rewritten line to fd. Applies rules 2+3 to the line content.
244func bg_emit_line(fd: nx_fd, src: *u8, pos: nx_idx, le: nx_idx, nx_mod: *u8) -> nx_int {
245 // Rule 2: rename "func main() -> nx_int" or "func main() -> i64".
246 let main_at: nx_idx = bg_find(src, pos, le, "func main()" as *u8, 11)
247 if main_at >= 0 {
248 // Emit prefix [pos, main_at)
249 bg_write_all(fd, ((src as nx_size) + pos) as *u8, main_at - pos)
250 // Emit replacement function header.
251 bg_wrs(fd, "func test_" as *u8)
252 bg_wrs(fd, nx_mod)
253 bg_wrs(fd, "()" as *u8)
254 // Skip "func main()" (11 chars). Emit the remainder; if return type
255 // is i64 rewrite to nx_int for consistency with bundle aggregator.
256 var rest: nx_idx = main_at + 11
257 let arrow_i64: nx_idx = bg_find(src, rest, le, "-> i64" as *u8, 6)
258 if arrow_i64 >= 0 {
259 bg_write_all(fd, ((src as nx_size) + rest) as *u8, arrow_i64 - rest)
260 bg_wrs(fd, "-> nx_int" as *u8)
261 rest = arrow_i64 + 6
262 }
263 bg_write_all(fd, ((src as nx_size) + rest) as *u8, le - rest)
264 sys_write(fd, "\n" as *u8, 1)
265 return 0
266 }
267
268 // Rule 3: substitute every __syscall(93, N, 0, 0, 0, 0, 0) -> N.
269 // (Guard flags rather than mutating ip-as-sentinel; the latter
270 // combined with the clip-to-le below ate the digit + suffix.)
271 var p: nx_idx = pos
272 var outer_done: nx_int = 0
273 while outer_done == 0 {
274 if p >= le { outer_done = 1 }
275 if outer_done == 0 {
276 let sc_at: nx_idx = bg_find(src, p, le, "__syscall(93," as *u8, 13)
277 if sc_at < 0 {
278 bg_write_all(fd, ((src as nx_size) + p) as *u8, le - p)
279 outer_done = 1
280 }
281 if sc_at >= 0 {
282 bg_write_all(fd, ((src as nx_size) + p) as *u8, sc_at - p)
283 var ip: nx_idx = sc_at + 13
284 // skip leading whitespace before N
285 var ws_done: nx_int = 0
286 while ws_done == 0 {
287 if ip >= le { ws_done = 1 }
288 if ws_done == 0 {
289 let cc: nx_int = src[ip] as nx_int
290 if cc == 32 { ip = ip + 1 }
291 if cc == 9 { ip = ip + 1 }
292 if cc != 32 { if cc != 9 { ws_done = 1 } }
293 }
294 }
295 // optional sign
296 if ip < le { if src[ip] as nx_int == 45 {
297 sys_write(fd, "-" as *u8, 1)
298 ip = ip + 1
299 } }
300 // digits
301 var dig_done: nx_int = 0
302 while dig_done == 0 {
303 if ip >= le { dig_done = 1 }
304 if dig_done == 0 {
305 let d: nx_int = src[ip] as nx_int
306 var is_digit: nx_int = 0
307 if d >= 48 { if d <= 57 { is_digit = 1 } }
308 if is_digit == 1 {
309 let single: *u8 = ((src as nx_size) + ip) as *u8
310 sys_write(fd, single, 1)
311 ip = ip + 1
312 }
313 if is_digit == 0 { dig_done = 1 }
314 }
315 }
316 // skip past closing ')'
317 var rp_done: nx_int = 0
318 while rp_done == 0 {
319 if ip >= le { rp_done = 1 }
320 if rp_done == 0 {
321 if src[ip] as nx_int == 41 { ip = ip + 1; rp_done = 1 }
322 if rp_done == 0 { ip = ip + 1 }
323 }
324 }
325 p = ip
326 }
327 }
328 }
329 sys_write(fd, "\n" as *u8, 1)
330 return 0
331}
332
333func bg_emit_rewritten_test(fd: nx_fd, src: *u8, src_n: nx_size, nx_mod: *u8) -> nx_int {
334 var pos: nx_idx = 0
335 while pos < src_n {
336 let le: nx_idx = bg_line_end(src, pos, src_n)
337 if bg_is_import_line(src, pos, le) == 0 {
338 bg_emit_line(fd, src, pos, le, nx_mod)
339 }
340 pos = le + 1
341 }
342 return 0
343}
344
345// ===== shard emitter ====================================================
346
347func bg_emit_shard(shard_idx: nx_int, prims_buf: *u8, lo: nx_size, hi: nx_size) -> nx_int {
348 // path = nxc2/runtime/_bundle_shard_<i>.nx
349 let path: *u8 = sys_mmap(NX_BG_PATH_BUF)
350 var o: nx_idx = 0
351 o = bg_str_cat(path, o, "nxc2/runtime/_bundle_shard_" as *u8)
352 // append decimal shard index
353 let idx_buf: *u8 = sys_mmap(32)
354 var idx_n: nx_idx = 0
355 var v: nx_int = shard_idx
356 if v == 0 { idx_buf[0] = 48; idx_n = 1 }
357 if v > 0 {
358 let tmp: *u8 = sys_mmap(32)
359 var tn: nx_idx = 0
360 while v > 0 {
361 tmp[tn] = (48 + (v - (v / 10) * 10)) as u8
362 tn = tn + 1
363 v = v / 10
364 }
365 // reverse
366 var k: nx_idx = 0
367 while k < tn {
368 idx_buf[k] = tmp[tn - 1 - k]
369 k = k + 1
370 }
371 idx_n = tn
372 }
373 var ki: nx_idx = 0
374 while ki < idx_n {
375 path[o] = idx_buf[ki]
376 o = o + 1
377 ki = ki + 1
378 }
379 o = bg_str_cat(path, o, ".nx" as *u8)
380
381 let fd: nx_fd = nx_open_wr(path)
382 if fd < 0 { return 0 }
383
384 bg_wrs(fd, "import \"nx_syscalls.nx\"\n" as *u8)
385 bg_wrs(fd, "import \"nx_runtime.nx\"\n" as *u8)
386 bg_wrs(fd, "import \"nx_tier.nx\"\n" as *u8)
387 bg_wrs(fd, "\n" as *u8)
388
389 // Impl imports for primitives in shard.
390 var i: nx_size = lo
391 while i < hi {
392 let nx_slot: *u8 = ((prims_buf as nx_size) + i * NX_BG_NAME_LEN) as *u8
393 bg_wrs(fd, "import \"" as *u8)
394 bg_wrs(fd, nx_slot)
395 bg_wrs(fd, ".nx\"\n" as *u8)
396 i = i + 1
397 }
398 bg_wrs(fd, "\n" as *u8)
399
400 // emit_line helper.
401 bg_wrs(fd, "func emit_line(tag: *u8, name: *u8) -> nx_int {\n" as *u8)
402 bg_wrs(fd, " sys_write(1, tag, strlen(tag))\n" as *u8)
403 bg_wrs(fd, " sys_write(1, name, strlen(name))\n" as *u8)
404 bg_wrs(fd, " sys_write(1, \"\\n\" as *u8, 1)\n" as *u8)
405 bg_wrs(fd, " return 0\n}\n\n" as *u8)
406
407 // Inline rewritten tests.
408 i = lo
409 let scratch_len: *nx_size = (sys_mmap(NX_SIZEOF_NX_SIZE)) as *nx_size
410 while i < hi {
411 let nx_slot2: *u8 = ((prims_buf as nx_size) + i * NX_BG_NAME_LEN) as *u8
412 let tp: *u8 = sys_mmap(NX_BG_PATH_BUF)
413 var to: nx_idx = 0
414 to = bg_str_cat(tp, to, "nxc2/runtime/" as *u8)
415 to = bg_str_cat(tp, to, nx_slot2)
416 to = bg_str_cat(tp, to, "_test.nx" as *u8)
417 scratch_len[0] = 0
418 let test_buf: *u8 = sys_read_file(tp, scratch_len)
419 if (test_buf as nx_size) != 0 {
420 bg_emit_rewritten_test(fd, test_buf, scratch_len[0], nx_slot2)
421 bg_wrs(fd, "\n" as *u8)
422 }
423 i = i + 1
424 }
425
426 // Master main.
427 bg_wrs(fd, "func main() -> nx_exit {\n" as *u8)
428 bg_wrs(fd, " var r: nx_int = 0\n" as *u8)
429 i = lo
430 while i < hi {
431 let nx_slot3: *u8 = ((prims_buf as nx_size) + i * NX_BG_NAME_LEN) as *u8
432 bg_wrs(fd, " r = test_" as *u8)
433 bg_wrs(fd, nx_slot3)
434 bg_wrs(fd, "()\n" as *u8)
435 bg_wrs(fd, " if r != 0 { emit_line(\"FAIL \" as *u8, \"" as *u8)
436 bg_wrs(fd, nx_slot3)
437 bg_wrs(fd, "\" as *u8) }\n" as *u8)
438 bg_wrs(fd, " if r == 0 { emit_line(\"PASS \" as *u8, \"" as *u8)
439 bg_wrs(fd, nx_slot3)
440 bg_wrs(fd, "\" as *u8) }\n" as *u8)
441 i = i + 1
442 }
443 bg_wrs(fd, " return 0\n}\n" as *u8)
444 sys_close(fd)
445 return 1
446}
447
448func main() -> nx_exit {
449 let prims_buf: *u8 = sys_mmap(NX_BG_MAX_PRIMS * NX_BG_NAME_LEN)
450 let n: nx_size = bg_collect_prims(prims_buf)
451 if n == 0 {
452 let err: *u8 = "nx_bundle_gen: no shape-driven primitives found\n" as *u8
453 sys_write(NX_FD_STDOUT, err, strlen(err))
454 return 1
455 }
456 let n_shards: nx_int = ((n + NX_BG_SHARD_SIZE - 1) / NX_BG_SHARD_SIZE) as nx_int
457 var s: nx_int = 0
458 while s < n_shards {
459 let lo: nx_size = s * NX_BG_SHARD_SIZE
460 var hi: nx_size = lo + NX_BG_SHARD_SIZE
461 if hi > n { hi = n }
462 bg_emit_shard(s, prims_buf, lo, hi)
463 s = s + 1
464 }
465 let msg1: *u8 = "nx_bundle_gen: primitives=" as *u8
466 sys_write(NX_FD_STDOUT, msg1, strlen(msg1))
467 print_i64(n as nx_int)
468 let msg2: *u8 = " shards=" as *u8
469 sys_write(NX_FD_STDOUT, msg2, strlen(msg2))
470 print_i64(n_shards)
471 sys_write(NX_FD_STDOUT, "\n" as *u8, 1)
472 return 0
473}