nx_library_foundation.nx source

1// nx_library_foundation.nx -- close the foundationing gap: every harvested doc on disk must be FOUNDATIONED 2// (CID + ed25519 provenance + lineage traced to god) before it counts as library knowledge. Operator law: 3// "we dont want to ingest non foundationed data -- if we consume we trace + build its lineage properly back to 4// god." This auto-discovers every knowledge/library/*.txt (getdents64, so the running index grows itself), 5// and for each: ingest_foundation (cid_of + prov_make[DOWNLOADED] + ed25519 sign + verify with 3 liar-kill 6// negatives + walk genesis_lineage UNI-WIRE->ORIGIN) AND lib_ingest (store by CID in the no-link-rot corpus). 7// Idempotent: lib_ingest returns DUP for docs already stored, so re-running only foundations, never double-adds. 8// Honest provenance: source is the VERIFIABLE origin class from the filename (wiki-article / source-readme / 9// nishi-internal-research / external-doc) -- never a fabricated upstream URL. license_tier: ORIGINAL 10import "nx_syscalls.nx" 11import "nx_library.nx" 12import "nx_ingest_foundation.nx" 13const LF_MAGIC_65536: i64 = 65536 14 15const LF_DIR: *u8 = "knowledge/library" 16const LF_LOG: *u8 = "knowledge/status/library_foundation.log" 17const LF_CREDDIR: *u8 = "knowledge/foundation/cred" 18const LF_DATE: *u8 = "2026-06-20" 19const LF_AGENT: *u8 = "nishi-librarian" 20 21func lf_getdents(fd: i64, buf: *u8, count: i64) -> i64 { return __syscall(217, fd, buf, count, 0, 0, 0) } 22func lf_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 23func lf_cat(dst: *u8, o: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[o]=s[i];o=o+1;i=i+1} dst[o]=0 as u8; return o } 24 25func lf_ends(name: *u8, suf: *u8) -> i64 { 26 let nl: i64=lf_slen(name); let sl: i64=lf_slen(suf); if sl>nl {return 0} 27 var i: i64=0; while i<sl { if name[nl-sl+i]!=suf[i]{return 0} i=i+1 } return 1 28} 29func lf_has(name: *u8, lit: *u8) -> i64 { 30 let nl: i64=lf_slen(name); let ll: i64=lf_slen(lit); if ll==0 {return 0} 31 var i: i64=0 32 while i+ll<=nl { var j: i64=0; var ok: i64=1; while j<ll { if name[i+j]!=lit[j]{ok=0;j=ll}else{j=j+1} } if ok==1 {return 1} i=i+1 } 33 return 0 34} 35func lf_is_txt(name: *u8) -> i64 { return lf_ends(name, ".txt" as *u8) } 36 37// honest origin class from the filename -- what the file VERIFIABLY is, not an invented URL. 38func lf_source(name: *u8, out: *u8) -> i64 { 39 if lf_ends(name, "_wiki.txt" as *u8)==1 { return lf_cat(out, 0, "harvest-mirror:wiki-article" as *u8) } 40 if lf_ends(name, "_readme.txt" as *u8)==1 { return lf_cat(out, 0, "harvest-mirror:source-readme" as *u8) } 41 if lf_has(name, "deep-research" as *u8)==1 { return lf_cat(out, 0, "nishi-internal-research-artifact" as *u8) } 42 if lf_has(name, "grounded" as *u8)==1 { return lf_cat(out, 0, "nishi-internal-research-artifact" as *u8) } 43 if lf_has(name, "best-practices" as *u8)==1 { return lf_cat(out, 0, "nishi-internal-research-artifact" as *u8) } 44 if lf_has(name, "backlog" as *u8)==1 { return lf_cat(out, 0, "nishi-internal-research-artifact" as *u8) } 45 return lf_cat(out, 0, "harvest-mirror:external-doc" as *u8) 46} 47 48// foundation + ingest ONE doc. counts[0]=scanned [1]=new+GREEN [2]=dup(foundationed, already stored) [3]=RED 49// [4]=resume-skipped (credential already on disk -- OOM-safe resume across fresh-process passes). 50// returns 3 RESUME-SKIP / 2 NEW+GREEN / 1 DUP / 0 RED. 51func lf_one(name: *u8, counts: *i64) -> i64 { 52 counts[0] = counts[0] + 1 53 let path: *u8 = sys_mmap(512); var o: i64 = lf_cat(path, 0, LF_DIR); path[o]=47 as u8; o=o+1; lf_cat(path, o, name) 54 let source: *u8 = sys_mmap(128); lf_source(name, source) 55 let cred: *u8 = sys_mmap(512); o = lf_cat(cred, 0, LF_CREDDIR); cred[o]=47 as u8; o=o+1; o=lf_cat(cred, o, name); lf_cat(cred, o, ".nxpc1" as *u8) 56 // RESUME: if this doc's credential already exists on disk, skip (already foundationed a prior pass). 57 // This makes the full-library run OOM-safe: re-invoke as fresh processes until caught up. 58 let exfd: i64 = sys_openat_rd(cred) 59 if exfd >= 0 { sys_close(exfd); counts[4] = counts[4] + 1; return 3 } 60 let logtag: *u8 = sys_mmap(160); o = lf_cat(logtag, 0, "LIBFOUND " as *u8); lf_cat(logtag, o, name) 61 let cid: *u8 = sys_mmap(160) 62 var found: i64 = ingest_foundation(path, source, LF_DATE, LF_AGENT, cred, LF_LOG, logtag, cid) 63 // READ-BACK. ingest_foundation returns GREEN from an IN-MEMORY sign+verify; it never confirms the 64 // credential reached disk. Proven necessary 2026-08-01: a full pass printed "ALL scanned docs 65 // foundationed GREEN" for 146 docs while the NEXT pass reported resume-skip=0 -- the resume path 66 // reads exactly this credential, so 0 skips means every write had vanished. The organ already 67 // contained its own read-back and the summary line simply never consulted it. 68 // A WRITER IS ONLY TESTED BY A READ-BACK: a doc without a credential on disk is NOT foundationed, 69 // whatever the signature said, because every downstream consumer keys off the FILE. 70 let vfd: i64 = sys_openat_rd(cred) 71 if vfd >= 0 { sys_close(vfd) } else { found = 0 } 72 let cid2: *u8 = sys_mmap(160) 73 let ing: i64 = lib_ingest(path, source, name, LF_DATE, cid2) 74 if found == 1 { 75 if ing == 1 { counts[1] = counts[1] + 1; return 2 } 76 counts[2] = counts[2] + 1; return 1 77 } 78 counts[3] = counts[3] + 1; return 0 79} 80 81// walk LF_DIR, foundation+ingest every *.txt. fills counts[0..4] (caller zeroes). returns scanned count. 82func lf_run(counts: *i64) -> i64 { 83 // PORTABLE MKDIR IDIOM: x86_64 mkdirat=258 with AT_FDCWD, the form used across nx_mkdirp / 84 // nx_evidence_gather / nx_media_gather / nx_epub_read. The previous raw __syscall(83,...) was 85 // legacy x86 mkdir(2) -- a number that is in NEITHER the rv64 table (mkdirat=34) NOR the 86 // translated x86 form (258), so it only ever had a chance of working on one lane. We build for 87 // more than rv64, so a bare arch-specific number is a silent per-lane failure, and this one 88 // failed silently: the credentials never landed while every doc still printed verdict=GREEN. 89 // EEXIST is harmless -- this is idempotent by construction (rule 10). 90 // RUNTIME-COMPUTED syscall number, not a literal. A LITERAL goes through the rv64->x86 91 // constant-translate path (nx_x86_64_ctx maps 34->258), so a bare `258` can be rewritten out 92 // from under you; storing it in memory first is the documented escape hatch used by 93 // sys_symlinkat ("forced RUNTIME") and nx_cms_gate (`nb[0]=258`). The original raw 83 was 94 // legacy x86 mkdir(2) -- in NEITHER table -- and my first fix used a literal 258, which is the 95 // same trap one layer shallower. We build for more than rv64: a bare arch-specific number is a 96 // silent per-lane failure. EEXIST is harmless (idempotent, rule 10). 97 let nbox: *i64 = sys_mmap(16) as *i64 98 nbox[0] = 258 99 __syscall(nbox[0], AT_FDCWD, "knowledge/foundation" as *u8 as i64, 448, 0, 0, 0) 100 __syscall(nbox[0], AT_FDCWD, LF_CREDDIR as i64, 448, 0, 0, 0) 101 sys_munmap(nbox as *u8, 16) 102 let fd: i64 = sys_openat_rd(LF_DIR) 103 if fd < 0 { return 0 } 104 let gbuf: *u8 = sys_mmap(LF_MAGIC_65536) 105 var nread: i64 = lf_getdents(fd, gbuf, LF_MAGIC_65536) 106 while nread > 0 { 107 var off: i64 = 0 108 while off < nread { 109 let reclen: i64 = (gbuf[off+16] as i64) | ((gbuf[off+17] as i64) << 8) 110 if reclen <= 0 { off = nread } else { 111 let name: *u8 = gbuf + off + 19 112 if lf_is_txt(name) == 1 { lf_one(name, counts) } 113 off = off + reclen 114 } 115 } 116 nread = lf_getdents(fd, gbuf, LF_MAGIC_65536) 117 } 118 sys_close(fd) 119 return counts[0] 120} 121 122func lf_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 123func lf_n(v: i64) -> i64 { var m: i64=v; if m<0{lf_w("-");m=0-m} let t:*u8=sys_mmap(24); var k:i64=0; if m==0{t[0]=48 as u8;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i:i64=0; let o:*u8=sys_mmap(24); while i<k{o[i]=t[k-1-i];i=i+1}; sys_write(1,o,k); return 0 } 124 125func main() -> i64 { 126 lf_w("=== NX-LIBRARY-FOUNDATION: foundation+ingest every harvested doc (CID + ed25519 provenance + lineage-to-god) ===\n") 127 let counts: *i64 = sys_mmap(64) as *i64 128 var z: i64=0; while z<8 { counts[z]=0; z=z+1 } 129 let before: i64 = lib_count() 130 lf_run(counts) 131 let after: i64 = lib_count() 132 lf_w(" scanned="); lf_n(counts[0]); lf_w(" new+GREEN="); lf_n(counts[1]); lf_w(" resume-skip="); lf_n(counts[4]); lf_w(" dup(already stored)="); lf_n(counts[2]); lf_w(" RED(non-foundationed)="); lf_n(counts[3]); lf_w("\n") 133 lf_w(" library running index: "); lf_n(before); lf_w(" -> "); lf_n(after); lf_w(" docs\n") 134 if counts[3] == 0 { lf_w(" ALL scanned docs foundationed GREEN -- zero non-foundationed data in the library.\n") } 135 // CAUGHT-UP signal for the fresh-process driver: no new docs foundationed this pass. 136 // CAUGHT-UP means "nothing left to do", and counts[1]==0 alone CANNOT distinguish that from 137 // "every single doc failed" -- both produce zero NEW foundationed docs. Caught live 2026-08-01: 138 // with the read-back in place a pass of 146 RED printed FOUNDATION-CAUGHT-UP, the exact inversion 139 // the read-back had just fixed one layer down. 140 // A COMPLETION SIGNAL DERIVED FROM "NO PROGRESS" IS ALSO A TOTAL-FAILURE SIGNAL -- require the 141 // absence of failures, not the absence of work. 142 if counts[3] > 0 { 143 lf_w(" FOUNDATION-BLOCKED: "); lf_n(counts[3]) 144 lf_w(" doc(s) produced NO credential on disk. NOT caught up -- the writes are failing.\n") 145 sys_exit(1); return 1 146 } 147 if counts[1] == 0 { lf_w(" FOUNDATION-CAUGHT-UP: every library doc has a credential.\n"); sys_exit(0); return 0 } 148 lf_w(" FOUNDATION-MORE: re-invoke (fresh process) to continue past the OOM leak wall.\n") 149 sys_exit(0); return 0 150}