nx_lib_uxf_ingest.nx source

1// nx_lib_uxf_ingest.nx -- Library arc R4a: content-address the CALIBRE CATALOG into the UXF envelope 2// ("everything in our superior format"). Reads the book index (idx_book.wsl), and for each book builds a 3// canonical UXF DOC record {bk=path, ti=basename, fmt=ext} -> uxf_cid_profiled(UXF_DOC) -> append to a 4// content-addressed seg_store (batched across segments because ss_begin's writer is 1 MiB). Then a gate 5// proves it: count, retrieve-by-CID byte-faithful, no-fabrication (absent CID -> not found), tamper-evident 6// (one flipped byte -> different CID), distinct books -> distinct CIDs. 7// 8// This makes each book a tamper-evident, dedupable, retrievable UXF record (the LIB02/LIB04 moat applied to 9// the real 5761-book catalog). HONEST: metadata records here; per-book CONTENT conversion (epub/pdf text -> 10// UXF doc) is R4b. Reuses nx_canon_cid + nx_uxf_cid + nx_seg_store. No hardware writes (Rule 26). 11// license_tier: ORIGINAL 12import "nx_syscalls.nx" 13import "nx_canon_cid.nx" 14import "nx_uxf_cid.nx" 15import "nx_seg_store.nx" 16const IU_MAGIC_2097152: i64 = 2097152 17const IU_MAGIC_8192: i64 = 8192 18const IU_MAGIC_8000: i64 = 8000 19 20const IU_IDX: *u8 = "knowledge/staging/media/idx_book.wsl" 21const IU_PREFIX: *u8 = "knowledge/staging/media/uxf_lib/" 22 23func iu_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 24func iu_n(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 } 25 26func iu_read(path: *u8, buf: *u8, cap: i64) -> i64 { 27 let fd: i64 = sys_openat_rd(path) 28 if fd < 0 { return 0 - 1 } 29 var n: i64 = 0; var go: i64 = 1 30 while go == 1 { let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - 1 - n); if r <= 0 { go = 0 } else { n = n + r } if n >= cap - 1 { go = 0 } } 31 sys_close(fd) 32 return n 33} 34 35// last index of byte b in s[0..len), else -1 36func iu_last(s: *u8, len: i64, b: i64) -> i64 { var i: i64=0; var r: i64=0-1; while i<len { if s[i]==(b as u8) { r=i } i=i+1 } return r } 37 38func iu_eq(a: *u8, an: i64, bp: *u8, bn: i64) -> i64 { if an!=bn { return 0 } var i: i64=0; while i<an { if a[i]!=bp[i] { return 0 } i=i+1 } return 1 } 39 40// build a UXF DOC record for a NUL-terminated book path into `canon`; write its profiled CID into `cid`. 41// returns canon byte length. 42func iu_record(path: *u8, plen: i64, canon: *u8, cid: *u8) -> i64 { 43 let keys: *i64 = sys_mmap(64) as *i64 44 let vals: *i64 = sys_mmap(64) as *i64 45 let slash: i64 = iu_last(path, plen, 47) 46 var base: *u8 = path 47 if slash >= 0 { base = (path as i64 + slash + 1) as *u8 } 48 let dot: i64 = iu_last(path, plen, 46) 49 var ext: *u8 = "" as *u8 50 if dot >= 0 { ext = (path as i64 + dot + 1) as *u8 } 51 keys[0] = ("bk\x00") as i64; vals[0] = (path as i64) 52 keys[1] = ("ti\x00") as i64; vals[1] = (base as i64) 53 keys[2] = ("fmt\x00") as i64; vals[2] = (ext as i64) 54 let clen: i64 = canon_encode(keys, vals, 3, canon) 55 uxf_cid_profiled(UXF_DOC, canon, clen, cid) 56 return clen 57} 58 59func main() -> i64 { 60 iu_p("=== nx_lib_uxf_ingest: content-address the Calibre catalog into UXF ===\n" as *u8) 61 let buf: *u8 = sys_mmap(IU_MAGIC_2097152) 62 let n: i64 = iu_read(IU_IDX, buf, IU_MAGIC_2097152) 63 if n <= 0 { iu_p("UXF-INGEST verdict=RED reason=idx-missing\n" as *u8); sys_exit(1); return 1 } 64 65 let pathb: *u8 = sys_mmap(IU_MAGIC_8192) 66 let canon: *u8 = sys_mmap(IU_MAGIC_8192) 67 let cid: *u8 = sys_mmap(128) 68 // remembered records for the gate 69 let cid0: *u8 = sys_mmap(128); let canon0: *u8 = sys_mmap(IU_MAGIC_8192); var clen0: i64 = 0 70 let cid1: *u8 = sys_mmap(128) 71 72 var w: *i64 = ss_begin() 73 var segid: i64 = 0 74 var count: i64 = 0 75 var s: i64 = 0 76 while s < n { 77 var e: i64 = s 78 var fnd: i64 = 0 79 while fnd == 0 { if e >= n { fnd = 1 } else { if buf[e]==(10 as u8) { fnd = 1 } else { e = e + 1 } } } 80 let llen: i64 = e - s 81 if llen > 0 { if llen < IU_MAGIC_8000 { 82 var c: i64 = 0 83 while c < llen { pathb[c] = buf[s+c]; c = c + 1 } 84 pathb[llen] = 0 as u8 85 let clen: i64 = iu_record(pathb, llen, canon, cid) 86 // remember the first two records for the gate 87 if count == 0 { var z: i64=0; while z<72 { cid0[z]=cid[z]; z=z+1 } z=0; while z<clen { canon0[z]=canon[z]; z=z+1 } clen0=clen } 88 if count == 1 { var z: i64=0; while z<72 { cid1[z]=cid[z]; z=z+1 } } 89 // append (key=cid, val=canon); on writer-full, commit this segment + start a new one 90 let rc: i64 = ss_add(w, 1, cid, canon, clen) 91 if rc != 0 { 92 ss_commit(IU_PREFIX, w, segid) 93 segid = segid + 1 94 w = ss_begin() 95 ss_add(w, 1, cid, canon, clen) 96 } 97 count = count + 1 98 } } 99 s = e + 1 100 } 101 ss_commit(IU_PREFIX, w, segid) 102 iu_p(" ingested books="); iu_n(count); iu_p(" segments="); iu_n(segid+1); iu_p(" -> "); iu_p(IU_PREFIX); iu_p("\n" as *u8) 103 104 // ---------------- GATE ---------------- 105 var pass: i64 = 0; var tot: i64 = 0 106 let h: *i64 = ss_open(IU_PREFIX) 107 let pp: *i64 = sys_mmap(8) as *i64 108 let pl: *i64 = sys_mmap(8) as *i64 109 110 // A1: retrieve book #0 by its CID -> found + byte-faithful + CID recomputes 111 tot=tot+1 112 let r0: i64 = ss_hget(h, cid0, pp, pl) 113 var a1: i64 = 0 114 if r0 == 1 { 115 let got: *u8 = pp[0] as *u8 116 if iu_eq(got, pl[0], canon0, clen0) == 1 { 117 let rc2: *u8 = sys_mmap(128) 118 uxf_cid_profiled(UXF_DOC, got, pl[0], rc2) 119 if iu_eq(rc2, 72, cid0, 72) == 1 { a1 = 1 } 120 } 121 } 122 iu_p(" A1 retrieve-by-CID byte-faithful + content-addressed: "); if a1==1 { pass=pass+1; iu_p("PASS\n" as *u8) } else { iu_p("FAIL\n" as *u8) } 123 124 // A2 (no-fabrication): a fabricated CID must NOT be found 125 tot=tot+1 126 let fake: *u8 = "nxc1-02-0000000000000000000000000000000000000000000000000000000000000000\x00" as *u8 127 let rf: i64 = ss_hget(h, fake, pp, pl) 128 iu_p(" A2 absent CID -> not found (rf="); iu_n(rf); iu_p("): "); if rf <= 0 { pass=pass+1; iu_p("PASS\n" as *u8) } else { iu_p("FAIL\n" as *u8) } 129 130 // A3 (tamper-evident): flip one byte of book #0's canon -> different CID 131 tot=tot+1 132 let tcan: *u8 = sys_mmap(IU_MAGIC_8192) 133 var z: i64 = 0; while z < clen0 { tcan[z]=canon0[z]; z=z+1 } 134 let lb: i64 = tcan[clen0-1] 135 tcan[clen0-1] = ((lb + 1) & 255) as u8 136 let tcid: *u8 = sys_mmap(128) 137 uxf_cid_profiled(UXF_DOC, tcan, clen0, tcid) 138 iu_p(" A3 tamper -> different CID: "); if iu_eq(tcid, 72, cid0, 72) == 0 { pass=pass+1; iu_p("PASS\n" as *u8) } else { iu_p("FAIL\n" as *u8) } 139 140 // A4 (discriminating): book #0 and book #1 have distinct CIDs 141 tot=tot+1 142 iu_p(" A4 distinct books -> distinct CIDs: "); if iu_eq(cid0, 72, cid1, 72) == 0 { pass=pass+1; iu_p("PASS\n" as *u8) } else { iu_p("FAIL\n" as *u8) } 143 144 // A5 count sanity 145 tot=tot+1 146 iu_p(" A5 ingested > 0: "); if count > 0 { pass=pass+1; iu_p("PASS\n" as *u8) } else { iu_p("FAIL\n" as *u8) } 147 148 iu_p("\nUXF-INGEST-GATE "); iu_n(pass); iu_p("/"); iu_n(tot) 149 if pass == tot { iu_p(" verdict=GREEN\n" as *u8); sys_exit(0); return 0 } 150 iu_p(" verdict=RED\n" as *u8); sys_exit(1); return 1 151}