nx_galx_fill.nx source

1// nx_galx_fill.nx -- SOVEREIGN Nishi corpus-fill batch-runner. Replaces the bash split/cp/awk loop: 2// counts the lines in knowledge/status/galx_corpus_full.txt, then for s = start; s < total; s += BATCH: 3// fork() + execve(_galx_prod_ingest.elf, [s, BATCH]) + wait4(). Each ingest slice runs in a FRESH 4// process, so the kernel reclaims all of its mmaps on exit -> RSS stays bounded (~3GB/slice) with NO 5// sys_munmap (which the toolchain lacks). All logic is NishiLang; wsl is only the last-mile launcher. 6// Usage: nx_galx_fill <start_line> (resume offset; 0 = from the beginning). license_tier: ORIGINAL 7import "nx_syscalls.nx" 8 9const DRIVER: *u8 = "/mnt/c/Users/elder/nishi-core/nxc2/runtime/_hdl_build/_galx_prod_ingest.elf" as *u8 10// BATCH = images per fresh ingest process. Each image's idempotency scan (ss_get) mmaps the growing 11// store unbounded (no munmap in the toolchain), so per-process RSS ~= BATCH * per-image-scan-cost, and 12// that cost grows with store size. BATCH is argv[2] (default below) so the slice size can be tuned to 13// keep RSS bounded as the corpus grows -- a large store needs a smaller BATCH. license: data-driven (rule 11). 14const BATCH_DEFAULT: i64 = 500 15 16func fp(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 17func fn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1}; sys_write(1,b,k); return 0 } 18func f_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ if s[i]>=(48 as u8){ if s[i]<=(57 as u8){ v=v*10+((s[i]-(48 as u8)) as i64) } } i=i+1 } return v } 19func f_itoa(v: i64, buf: *u8) -> i64 { var m: i64=v; if m<0{m=0}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{buf[i]=t[k-1-i];i=i+1}; buf[k]=0 as u8; return 0 } 20 21func main(argc: i64, argv: *i64) -> i64 { 22 var start: i64 = 0 23 if argc >= 2 { start = f_atoi(argv[1] as *u8) } 24 var batch: i64 = BATCH_DEFAULT 25 if argc >= 3 { batch = f_atoi(argv[2] as *u8) } 26 if batch < 1 { batch = BATCH_DEFAULT } 27 // argv[3] overrides the per-slice ingest driver (default = the per-image prod ingest; the bulk 28 // ingest _galx_bulk_ingest.elf is passed here for the 230k full-corpus load). 29 var driver: *u8 = DRIVER 30 if argc >= 4 { driver = argv[3] as *u8 } 31 32 let lszp: *i64 = sys_mmap(16) as *i64 33 let lst: *u8 = sys_read_file("knowledge/status/galx_corpus_full.txt" as *u8, lszp) 34 let lsz: i64 = lszp[0] 35 if (lst as i64) == 0 { fp("FILL FAIL: no galx_corpus_full.txt\n" as *u8); sys_exit(1); return 1 } 36 var total: i64 = 0 37 var i: i64 = 0 38 while i < lsz { if lst[i] == (10 as u8) { total = total + 1 } i = i + 1 } 39 fp("FILL start=" as *u8); fn(start); fp(" total=" as *u8); fn(total); fp(" batch=" as *u8); fn(batch); fp("\n" as *u8) 40 41 let sbuf: *u8 = sys_mmap(32) 42 let cbuf: *u8 = sys_mmap(32) 43 let cargv: *i64 = sys_mmap(32) as *i64 44 let cenv: *i64 = sys_mmap(8) as *i64 45 cenv[0] = 0 46 let st: *i64 = sys_mmap(8) as *i64 47 var s: i64 = start 48 while s < total { 49 f_itoa(s, sbuf) 50 f_itoa(batch, cbuf) 51 cargv[0] = driver as i64 52 cargv[1] = sbuf as i64 53 cargv[2] = cbuf as i64 54 cargv[3] = 0 55 let pid: i64 = sys_fork() 56 if pid == 0 { 57 sys_execve(driver, cargv, cenv) 58 sys_exit(127) 59 } 60 if pid > 0 { sys_wait4(pid, st, 0) } 61 fp("FILL slice @" as *u8); fn(s); fp(" done\n" as *u8) 62 s = s + batch 63 } 64 fp("FILL ALL DONE total=" as *u8); fn(total); fp("\n" as *u8) 65 sys_exit(0) 66 return 0 67}