nx_primitive_synth.nx source
↩ module page · 435 lines · 15643 B
1// nx_synth.nx -- substrate-native per-primitive synthesizer.
2//
3// Per user directive 2026-05-14: "i want you building nishilang to
4// the point it can [ingest algorithms and create primitives]" -- the
5// substrate must be the actor, not bash.
6//
7// Reads nxc2/specs/algo_corpus.txt (TSV: name, module, shape, summary,
8// KEY=VAL ...) and emits substituted .nx impl + test files under
9// nxc2/runtime/ using shape templates in nxc2/specs/algo_shapes/<shape>/.
10//
11// Design rules:
12// - Every mmap happens at startup; the per-row loop allocates ZERO.
13// - One open output file at a time, stream sys_write per chunk.
14// No 16 MiB write buffer to accumulate state across primitives.
15// - Hard cap NX_SYNTH_MAX_PRIMITIVES per run.
16// - Wired through nx_disk_budget for graceful halt on disk pressure.
17// - Function signatures kept <= 8 params (until nxc2 supports stack-passed
18// args 9+). Cross-function state packed via NxSynthCtx.
19//
20// genealogy_id: substrate_self_hosting_synth_2026_05_14
21// lineage_id: per_primitive_emit_no_batch
22
23// nx_safety_envelope:
24// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
25// sil_target: SIL1
26// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
27// verdict: NOT_YET_EVALUATED
28
29import "nx_syscalls.nx"
30import "nx_runtime.nx"
31import "nx_tier.nx"
32import "nx_fcntl.nx"
33import "nx_disk_budget.nx"
34
35// ===== caps + sizing =================================================
36
37// Curation knob: the corpus has a curated section (rows 1..N) followed
38// by a bulk auto-generated section (sub_999, add_500, ... -- archived
39// separately). Default cap targets the curated section. Raise this
40// to include the auto-generated tail, or filter the corpus upstream.
41const NX_SYNTH_MAX_PRIMITIVES: nx_int = 200
42const NX_SYNTH_DISK_BUDGET: nx_size = 16777216
43
44const NX_SYNTH_PATH_BUF: nx_size = 256
45const NX_SYNTH_NAME_LEN: nx_size = 64
46const NX_SYNTH_TPL_CAP: nx_size = 32768
47const NX_SYNTH_KV_CAP: nx_int = 32
48const NX_SYNTH_KV_KEY: nx_size = 64
49const NX_SYNTH_KV_VAL: nx_size = 512
50
51const NX_SYNTH_TAB: nx_int = 9
52const NX_SYNTH_NL: nx_int = 10
53const NX_SYNTH_EQ: nx_int = 61
54const NX_SYNTH_LBRACE: nx_int = 123
55const NX_SYNTH_RBRACE: nx_int = 125
56
57// ===== string helpers ================================================
58
59func sy_strlen(s: *u8) -> nx_size {
60 var n: nx_size = 0
61 while s[n] != 0 { n = n + 1 }
62 return n
63}
64
65func sy_str_eq(a: *u8, b: *u8) -> nx_int {
66 var i: nx_idx = 0
67 while a[i] != 0 {
68 if a[i] != b[i] { return 0 }
69 i = i + 1
70 }
71 if b[i] != 0 { return 0 }
72 return 1
73}
74
75func sy_str_cpy(dst: *u8, src: *u8) -> nx_size {
76 var i: nx_size = 0
77 while src[i] != 0 {
78 dst[i] = src[i]
79 i = i + 1
80 }
81 dst[i] = 0
82 return i
83}
84
85func sy_str_cat(dst: *u8, off: nx_idx, src: *u8) -> nx_idx {
86 var i: nx_idx = 0
87 while src[i] != 0 {
88 dst[off + i] = src[i]
89 i = i + 1
90 }
91 dst[off + i] = 0
92 return off + i
93}
94
95// ===== scanning helpers (substrate stop-flag pattern) ================
96
97func sy_find_byte(corpus: *u8, start: nx_idx, end: nx_idx, c: nx_int) -> nx_idx {
98 var p: nx_idx = start
99 var stop: nx_int = 0
100 while stop == 0 {
101 if p >= end { stop = 1 }
102 if stop == 0 {
103 if corpus[p] == c { stop = 1 }
104 }
105 if stop == 0 { p = p + 1 }
106 }
107 return p
108}
109
110func sy_find_byte2(corpus: *u8, start: nx_idx, end: nx_idx, c1: nx_int, c2: nx_int) -> nx_idx {
111 var p: nx_idx = start
112 var stop: nx_int = 0
113 while stop == 0 {
114 if p >= end { stop = 1 }
115 if stop == 0 {
116 if corpus[p] == c1 { stop = 1 }
117 if stop == 0 { if corpus[p] == c2 { stop = 1 } }
118 }
119 if stop == 0 { p = p + 1 }
120 }
121 return p
122}
123
124func sy_copy_range(corpus: *u8, lo: nx_idx, hi: nx_idx, dst: *u8, cap: nx_size) -> nx_size {
125 let n: nx_size = (hi - lo) as nx_size
126 var w: nx_size = n
127 if w > cap - 1 { w = cap - 1 }
128 var i: nx_idx = 0
129 while i < (w as nx_idx) {
130 dst[i] = corpus[lo + i]
131 i = i + 1
132 }
133 dst[w] = 0
134 return w
135}
136
137// ===== row parser ====================================================
138//
139// Parse corpus[lo..hi) into nx_mod_out/shape_out + KV table (keys_buf,
140// vals_buf, n_kv_out). Returns 1 on success, 0 on malformed row.
141
142func sy_parse_row(
143 corpus: *u8, lo: nx_idx, hi: nx_idx,
144 nx_mod_out: *u8, shape_out: *u8,
145 keys_buf: *u8, vals_buf: *u8,
146 n_kv_out: *nx_size
147) -> nx_int {
148 nx_mod_out[0] = 0
149 shape_out[0] = 0
150 n_kv_out[0] = 0
151
152 var p: nx_idx = lo
153 var col: nx_int = 0
154
155 while p < hi {
156 let q: nx_idx = sy_find_byte2(corpus, p, hi, NX_SYNTH_TAB, NX_SYNTH_NL)
157
158 if col == 1 {
159 sy_copy_range(corpus, p, q, nx_mod_out, NX_SYNTH_NAME_LEN)
160 }
161 if col == 2 {
162 sy_copy_range(corpus, p, q, shape_out, NX_SYNTH_NAME_LEN)
163 }
164 if col >= 4 {
165 let eqp: nx_idx = sy_find_byte(corpus, p, q, NX_SYNTH_EQ)
166 if eqp < q {
167 let n_kv: nx_size = n_kv_out[0]
168 if (n_kv as nx_int) < NX_SYNTH_KV_CAP {
169 let key_slot: *u8 = ((keys_buf as nx_size) + n_kv * NX_SYNTH_KV_KEY) as *u8
170 let val_slot: *u8 = ((vals_buf as nx_size) + n_kv * NX_SYNTH_KV_VAL) as *u8
171 sy_copy_range(corpus, p, eqp, key_slot, NX_SYNTH_KV_KEY)
172 sy_copy_range(corpus, eqp + 1, q, val_slot, NX_SYNTH_KV_VAL)
173 n_kv_out[0] = n_kv + 1
174 }
175 }
176 }
177
178 col = col + 1
179 p = q + 1
180 if q >= hi { p = hi }
181 }
182
183 if nx_mod_out[0] == 0 { return 0 }
184 if shape_out[0] == 0 { return 0 }
185 return 1
186}
187
188// ===== KV lookup =====================================================
189
190func sy_lookup(keys_buf: *u8, vals_buf: *u8, n_kv: nx_size, key: *u8, out: *u8) -> nx_int {
191 var i: nx_size = 0
192 while i < n_kv {
193 let k_slot: *u8 = ((keys_buf as nx_size) + i * NX_SYNTH_KV_KEY) as *u8
194 if sy_str_eq(k_slot, key) == 1 {
195 let v_slot: *u8 = ((vals_buf as nx_size) + i * NX_SYNTH_KV_VAL) as *u8
196 sy_str_cpy(out, v_slot)
197 return 1
198 }
199 i = i + 1
200 }
201 out[0] = 0
202 return 0
203}
204
205// ===== context (packs >8 args for sy_emit_substituted) ==============
206
207struct NxSynthCtx {
208 keys_buf: *u8,
209 vals_buf: *u8,
210 n_kv: nx_size,
211 nx_mod: *u8,
212 key_scratch: *u8,
213 val_scratch: *u8,
214}
215
216const NX_SYNTH_CTX_BYTES: nx_size = 48
217
218// ===== template streaming substitution =================================
219
220func sy_emit_substituted(tpl: *u8, tpl_n: nx_size, ctx: *NxSynthCtx, fd: nx_fd) -> nx_int {
221 let keys_buf: *u8 = ctx.keys_buf
222 let vals_buf: *u8 = ctx.vals_buf
223 let n_kv: nx_size = ctx.n_kv
224 let nx_mod: *u8 = ctx.nx_mod
225 let key_scratch: *u8 = ctx.key_scratch
226 let val_scratch: *u8 = ctx.val_scratch
227
228 var pos: nx_idx = 0
229 while pos < (tpl_n as nx_idx) {
230 var placeholder: nx_int = 0
231 if pos + 3 < (tpl_n as nx_idx) {
232 if tpl[pos] == NX_SYNTH_LBRACE {
233 if tpl[pos + 1] == NX_SYNTH_LBRACE { placeholder = 1 }
234 }
235 }
236
237 if placeholder == 1 {
238 var end: nx_idx = pos + 2
239 var found: nx_int = 0
240 while found == 0 {
241 if end + 1 >= (tpl_n as nx_idx) { found = 2 }
242 if found == 0 {
243 if tpl[end] == NX_SYNTH_RBRACE {
244 if tpl[end + 1] == NX_SYNTH_RBRACE { found = 1 }
245 }
246 if found == 0 { end = end + 1 }
247 }
248 }
249 if found == 1 {
250 sy_copy_range(tpl, pos + 2, end, key_scratch, NX_SYNTH_KV_KEY)
251 let nx_module_lit: *u8 = "NX_MODULE" as *u8
252 var emitted: nx_int = 0
253 if sy_str_eq(key_scratch, nx_module_lit) == 1 {
254 let mlen: nx_size = sy_strlen(nx_mod)
255 sys_write(fd, nx_mod, mlen)
256 emitted = 1
257 }
258 if emitted == 0 {
259 sy_lookup(keys_buf, vals_buf, n_kv, key_scratch, val_scratch)
260 let vlen: nx_size = sy_strlen(val_scratch)
261 if vlen > 0 {
262 sys_write(fd, val_scratch, vlen)
263 }
264 }
265 pos = end + 2
266 }
267 if found == 2 {
268 let one: *u8 = ((tpl as nx_size) + pos) as *u8
269 sys_write(fd, one, 1)
270 pos = pos + 1
271 }
272 }
273
274 if placeholder == 0 {
275 // Search starts at pos+1 so even when tpl[pos] is a lone '{'
276 // (e.g., a function-body opener) we still advance one byte
277 // per iteration instead of hanging.
278 var search_from: nx_idx = pos + 1
279 if search_from > (tpl_n as nx_idx) { search_from = tpl_n as nx_idx }
280 let run_end: nx_idx = sy_find_byte(tpl, search_from, tpl_n as nx_idx, NX_SYNTH_LBRACE)
281 let run_buf: *u8 = ((tpl as nx_size) + pos) as *u8
282 let run_len: nx_size = (run_end - pos) as nx_size
283 sys_write(fd, run_buf, run_len)
284 pos = run_end
285 }
286 }
287 return 0
288}
289
290// ===== shape file loader =============================================
291
292func sy_load_shape_file(
293 shape: *u8, leaf: *u8,
294 out_buf: *u8, path_buf: *u8, len_p: *nx_size
295) -> nx_size {
296 var o: nx_idx = 0
297 o = sy_str_cat(path_buf, o, "nxc2/specs/algo_shapes/" as *u8)
298 o = sy_str_cat(path_buf, o, shape)
299 o = sy_str_cat(path_buf, o, "/" as *u8)
300 o = sy_str_cat(path_buf, o, leaf)
301 len_p[0] = 0
302 let data: *u8 = sys_read_file(path_buf, len_p)
303 if (data as nx_size) == 0 { return 0 }
304 let n: nx_size = len_p[0]
305 var cap: nx_size = n
306 if cap > NX_SYNTH_TPL_CAP - 1 { cap = NX_SYNTH_TPL_CAP - 1 }
307 var i: nx_idx = 0
308 while i < (cap as nx_idx) {
309 out_buf[i] = data[i]
310 i = i + 1
311 }
312 out_buf[cap] = 0
313 return cap
314}
315
316// ===== main ==========================================================
317
318func main() -> nx_exit {
319 let corpus_len_p: *nx_size = (sys_mmap(NX_SIZEOF_NX_SIZE)) as *nx_size
320 corpus_len_p[0] = 0
321 let corpus: *u8 = sys_read_file("nxc2/specs/algo_corpus.txt" as *u8, corpus_len_p)
322 if (corpus as nx_size) == 0 {
323 sys_write(NX_FD_STDOUT, "nx_synth: cannot read corpus\n" as *u8, 29)
324 return 1
325 }
326 let corpus_n: nx_size = corpus_len_p[0]
327
328 let nx_mod_buf: *u8 = sys_mmap(NX_SYNTH_NAME_LEN)
329 let shape_buf: *u8 = sys_mmap(NX_SYNTH_NAME_LEN)
330 let last_shape: *u8 = sys_mmap(NX_SYNTH_NAME_LEN)
331 let keys_buf: *u8 = sys_mmap((NX_SYNTH_KV_CAP as nx_size) * NX_SYNTH_KV_KEY)
332 let vals_buf: *u8 = sys_mmap((NX_SYNTH_KV_CAP as nx_size) * NX_SYNTH_KV_VAL)
333 let n_kv_p: *nx_size = (sys_mmap(NX_SIZEOF_NX_SIZE)) as *nx_size
334 let path_buf: *u8 = sys_mmap(NX_SYNTH_PATH_BUF)
335 let path2_buf: *u8 = sys_mmap(NX_SYNTH_PATH_BUF)
336 let shape_path: *u8 = sys_mmap(NX_SYNTH_PATH_BUF)
337 let impl_tpl_buf: *u8 = sys_mmap(NX_SYNTH_TPL_CAP)
338 let test_tpl_buf: *u8 = sys_mmap(NX_SYNTH_TPL_CAP)
339 let key_scratch: *u8 = sys_mmap(NX_SYNTH_KV_KEY)
340 let val_scratch: *u8 = sys_mmap(NX_SYNTH_KV_VAL)
341 let len_scratch: *nx_size = (sys_mmap(NX_SIZEOF_NX_SIZE)) as *nx_size
342 let ctx_raw: *u8 = sys_mmap(NX_SYNTH_CTX_BYTES)
343 let ctx: *NxSynthCtx = ctx_raw as *NxSynthCtx
344 ctx.keys_buf = keys_buf
345 ctx.vals_buf = vals_buf
346 ctx.nx_mod = nx_mod_buf
347 ctx.key_scratch = key_scratch
348 ctx.val_scratch = val_scratch
349
350 last_shape[0] = 0
351 var impl_n: nx_size = 0
352 var test_n: nx_size = 0
353
354 let budget: *NxDiskBudget = nx_disk_budget_new(NX_SYNTH_DISK_BUDGET)
355
356 var n_emitted: nx_int = 0
357 var n_skipped: nx_int = 0
358 var pos: nx_idx = 0
359
360 while pos < (corpus_n as nx_idx) {
361 if n_emitted >= NX_SYNTH_MAX_PRIMITIVES { pos = corpus_n as nx_idx }
362
363 if pos < (corpus_n as nx_idx) {
364 let eol: nx_idx = sy_find_byte(corpus, pos, corpus_n as nx_idx, NX_SYNTH_NL)
365
366 let parsed: nx_int = sy_parse_row(corpus, pos, eol,
367 nx_mod_buf, shape_buf,
368 keys_buf, vals_buf, n_kv_p)
369 var did_emit: nx_int = 0
370 if parsed == 1 {
371 var o: nx_idx = 0
372 o = sy_str_cat(path_buf, o, "nxc2/runtime/" as *u8)
373 o = sy_str_cat(path_buf, o, nx_mod_buf)
374 o = sy_str_cat(path_buf, o, ".nx" as *u8)
375
376 let probe_fd: nx_fd = nx_openat(NX_AT_FDCWD, path_buf, NX_O_RDONLY, 0)
377 var already_shipped: nx_int = 0
378 if probe_fd >= 0 {
379 sys_close(probe_fd)
380 already_shipped = 1
381 }
382
383 if already_shipped == 0 {
384 if sy_str_eq(shape_buf, last_shape) == 0 {
385 impl_n = sy_load_shape_file(shape_buf, "template.nx" as *u8,
386 impl_tpl_buf, shape_path, len_scratch)
387 test_n = sy_load_shape_file(shape_buf, "test_template.nx" as *u8,
388 test_tpl_buf, shape_path, len_scratch)
389 sy_str_cpy(last_shape, shape_buf)
390 }
391
392 if impl_n > 0 {
393 ctx.n_kv = n_kv_p[0]
394 let fd: nx_fd = __syscall(SYS_OPENAT, AT_FDCWD, path_buf as i64,
395 0x241, 0x1A4, 0, 0)
396 if fd >= 0 {
397 sy_emit_substituted(impl_tpl_buf, impl_n, ctx, fd)
398 sys_close(fd)
399
400 if test_n > 0 {
401 var o2: nx_idx = 0
402 o2 = sy_str_cat(path2_buf, o2, "nxc2/runtime/" as *u8)
403 o2 = sy_str_cat(path2_buf, o2, nx_mod_buf)
404 o2 = sy_str_cat(path2_buf, o2, "_test.nx" as *u8)
405 let tfd: nx_fd = __syscall(SYS_OPENAT, AT_FDCWD, path2_buf as i64,
406 0x241, 0x1A4, 0, 0)
407 if tfd >= 0 {
408 sy_emit_substituted(test_tpl_buf, test_n, ctx, tfd)
409 sys_close(tfd)
410 }
411 }
412
413 n_emitted = n_emitted + 1
414 did_emit = 1
415 sys_write(NX_FD_STDOUT, " emitted: " as *u8, 11)
416 sys_write(NX_FD_STDOUT, nx_mod_buf, sy_strlen(nx_mod_buf))
417 sys_write(NX_FD_STDOUT, "\n" as *u8, 1)
418 }
419 }
420 }
421 }
422 if did_emit == 0 { n_skipped = n_skipped + 1 }
423 pos = eol + 1
424 }
425 }
426
427 sys_write(NX_FD_STDOUT, "nx_synth: emitted=" as *u8, 18)
428 print_i64(n_emitted as i64)
429 sys_write(NX_FD_STDOUT, " skipped=" as *u8, 9)
430 print_i64(n_skipped as i64)
431 sys_write(NX_FD_STDOUT, " disk_pct=" as *u8, 10)
432 print_i64(nx_disk_budget_pct_used(budget) as i64)
433 sys_write(NX_FD_STDOUT, "\n" as *u8, 1)
434 return 0
435}