nx_library_consume.nx source

1// nx_library_consume.nx -- the CONSUME engine: ingest a queue of fetched science documents into the sovereign 2// no-link-rot library (nx_library). The queue is an INLINED sovereign seed (extensible; grows as the 3// researcher fetches more during downtime -- the eventual data-driven source list lives in the seg_store, NOT 4// a TSV). Each doc -> content-addressed CID + dedup + running index + metadata. Idempotent (re-run = all DUP). 5// This is "downtime consuming": fetch (nx_research_fetch) lands files in knowledge/fetched/, this sweeps them 6// into the corpus so the knowledge is owned + reproducibly citable forever. license_tier: ORIGINAL 7import "nx_syscalls.nx" 8import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 9import "nx_library.nx" 10 11func cw_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 12// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 13// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 14// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 15// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 16func cw_n(v: i64) -> i64 { nxi_out(v); return 0 } 17func cw_pfx(cid: *u8, k: i64) -> i64 { var i: i64=0; while i<k { if cid[i]==(0 as u8){break} let o: *u8=sys_mmap(1); o[0]=cid[i]; sys_write(1,o,1); i=i+1 } return 0 } 18 19func cons_one(path: *u8, src: *u8, title: *u8, nw: *i64, dup: *i64) -> i64 { 20 let cid: *u8 = sys_mmap(128) 21 let st: i64 = lib_ingest(path, src, title, "2026-06-20" as *u8, cid) 22 cw_w(" + "); cw_w(title); cw_w(" -> "); cw_pfx(cid, 18) 23 if st == 1 { cw_w("... [NEW]\n"); nw[0]=nw[0]+1 } else { if st == 0 { cw_w("... [DUP]\n"); dup[0]=dup[0]+1 } else { cw_w("... [ERR/missing]\n") } } 24 return st 25} 26 27func main() -> i64 { 28 cw_w("=== NISHI LIBRARY CONSUME (sweep fetched science docs into the no-link-rot corpus) ===\n") 29 let nw: *i64 = sys_mmap(8) as *i64; let dup: *i64 = sys_mmap(8) as *i64; nw[0]=0; dup[0]=0 30 31 cons_one("knowledge/fetched/nist/codata_allascii.txt" as *u8, "https://physics.nist.gov/cuu/Constants/Table/allascii.txt" as *u8, "NIST CODATA 2022 Fundamental Physical Constants" as *u8, nw, dup) 32 cons_one("knowledge/fetched/nist/atomic_weights.raw" as *u8, "https://physics.nist.gov/cgi-bin/Compositions" as *u8, "NIST Atomic Weights and Isotopic Compositions (AME2020)" as *u8, nw, dup) 33 cons_one("knowledge/fetched/ciaaw/abridged_atomic_weights.html" as *u8, "https://ciaaw.org/abridged-atomic-weights.htm" as *u8, "IUPAC CIAAW Abridged Standard Atomic Weights" as *u8, nw, dup) 34 cons_one("knowledge/fetched/ciaaw/atomic_weights.html" as *u8, "https://ciaaw.org/atomic-weights.htm" as *u8, "IUPAC CIAAW Standard Atomic Weights (full, with intervals)" as *u8, nw, dup) 35 cons_one("knowledge/fetched/wsl_config.html" as *u8, "https://learn.microsoft.com/en-us/windows/wsl/wsl-config" as *u8, "Microsoft WSL .wslconfig advanced settings reference (orchestration stability)" as *u8, nw, dup) 36 37 cw_w("consumed: NEW="); cw_n(nw[0]); cw_w(" DUP="); cw_n(dup[0]); cw_w("\n") 38 cw_w("--- running index (the no-rot corpus) ---\n") 39 cw_w("documents="); cw_n(lib_count()); cw_w("\n") 40 let cids: *i64 = sys_mmap(8 * 256) as *i64 41 let nc: i64 = lib_list(cids, 256) 42 let title: *u8 = sys_mmap(256); let by: *u8 = sys_mmap(32) 43 var i: i64 = 0 44 while i < nc { 45 let cid: *u8 = cids[i] as *u8 46 cw_w(" ["); cw_pfx(cid, 16); cw_w("...] ") 47 if lib_meta(cid, "title" as *u8, title, 256) == 1 { cw_w(title) } 48 if lib_meta(cid, "bytes" as *u8, by, 32) == 1 { cw_w(" ("); cw_w(by); cw_w("B)") } 49 cw_w("\n") 50 i = i + 1 51 } 52 sys_exit(0); return 0 53}