nx_library_harvest.nx source

1// nx_library_harvest.nx -- ASSIGNMENT D (tutored): turn the team's DARK knowledge into a persistent, searchable 2// index. The Researcher's deep-research output (knowledge/research/*.md) was never indexed -- invisible to the 3// team's own search. This harvest reads those docs, combines them into a line-oriented corpus, builds the 4// inverted index (nx_inv_build_from_jsonl tokenizes each line), and PERSISTS it (nx_inv_save) so the team builds 5// once and queries forever. Proof: a term that exists ONLY in the dark research (not the old 52-line Library) 6// becomes searchable from the on-disk index. Composes the team's own primitives -- Claude tutored the doc-reader 7// composition (the novel rung), the team owns the index + persistence + syscalls. license_tier: ORIGINAL 8import "nx_search_inverted_persist.nx" 9 10func hv_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 11func hv_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } 12 13// concatenate each corpus file (paths[0..n), each a *u8 cast to i64) into out_path, newline-separated, so the 14// index treats lines as rows. Returns bytes written (skips files that don't read). 15func harvest_combine(paths: *i64, n: i64, out_path: *u8) -> i64 { 16 let fd: i64 = sys_openat_wr(out_path, 0x1a4) 17 if fd < 0 { return 0 - 1 } 18 var total: i64 = 0; var i: i64 = 0 19 while i < n { 20 let lenbox: *i64 = sys_mmap(8) as *i64 21 let buf: *u8 = sys_read_file(paths[i] as *u8, lenbox) 22 if buf != 0 as *u8 { 23 var off: i64 = 0 24 while off < lenbox[0] { 25 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, lenbox[0] - off) 26 if w <= 0 { off = lenbox[0] } else { off = off + w; total = total + w } 27 } 28 sys_write(fd, "\n" as *u8, 1); total = total + 1 29 } 30 i = i + 1 31 } 32 sys_close(fd) 33 return total 34} 35 36func hv_query(idx: *NxInvIndex, term: *u8, tlen: i64) -> i64 { 37 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult 38 let rowids: *i64 = sys_mmap(8*256) as *i64 39 nx_inv_query_term(idx, term, tlen, rowids, 256, res) 40 return res.postings_count 41} 42 43func main() -> i64 { 44 hv_puts("=== ASSIGNMENT D: harvest the DARK research into a persistent searchable index ===\n" as *u8) 45 // the dark corpus: ALL deep-research + specs the team produced (overnight harvest -> grow the Library). 46 let paths: *i64 = sys_mmap(8*32) as *i64 47 paths[0] = "knowledge/research/2026-06-04-historic-resource-constrained.md" as *u8 as i64 48 paths[1] = "knowledge/research/2026-06-04-image-precision-lossless-language.md" as *u8 as i64 49 paths[2] = "knowledge/research/2026-06-04-model-switching-timesync.md" as *u8 as i64 50 paths[3] = "knowledge/research/2026-06-04-objective-testing-judging.md" as *u8 as i64 51 paths[4] = "knowledge/research/2026-06-04-semantic-retrieval-and-llm-backup.md" as *u8 as i64 52 paths[5] = "knowledge/research/2026-06-04-shared-llm-backbone.md" as *u8 as i64 53 paths[6] = "knowledge/research/2026-06-04-vram-bottleneck.md" as *u8 as i64 54 paths[7] = "knowledge/research/2026-06-04-website-ux-andelinwest.md" as *u8 as i64 55 paths[8] = "knowledge/research/2026-06-05-automated-prioritization.md" as *u8 as i64 56 paths[9] = "knowledge/research/2026-06-05-autonomous-git-orchestration.md" as *u8 as i64 57 paths[10] = "knowledge/research/2026-06-05-compile-troubleshooting.md" as *u8 as i64 58 paths[11] = "knowledge/research/2026-06-05-polite-autoscaling.md" as *u8 as i64 59 paths[12] = "knowledge/research/2026-06-05-sovereign-ereader.md" as *u8 as i64 60 paths[13] = "knowledge/research/2026-06-05-sovereign-information-management.md" as *u8 as i64 61 paths[14] = "knowledge/research/2026-06-05-sovereign-web-hosting.md" as *u8 as i64 62 paths[15] = "knowledge/specs/2026-06-05-librarian-structure-the-library-assignment.md" as *u8 as i64 63 paths[16] = "knowledge/specs/2026-06-05-nishi-ereader-assignment.md" as *u8 as i64 64 paths[17] = "knowledge/specs/2026-06-05-nishi-reader-format-spec.md" as *u8 as i64 65 paths[18] = "knowledge/specs/2026-06-05-sovereign-information-management-spec.md" as *u8 as i64 66 paths[19] = "knowledge/specs/2026-06-05-spore-trust-bootstrap-assignment.md" as *u8 as i64 67 let np: i64 = 20 68 69 let bytes: i64 = harvest_combine(paths, np, "/tmp/nishi_corpus.txt" as *u8) 70 hv_puts(" combined " as *u8); hv_num(np); hv_puts(" dark docs -> corpus bytes=" as *u8); hv_num(bytes); hv_puts("\n" as *u8) 71 if bytes <= 0 { hv_puts(" harvest read nothing (paths missing?)\n" as *u8); sys_exit(1); return 1 } 72 73 let idx: *NxInvIndex = nx_inv_build_from_jsonl("/tmp/nishi_corpus.txt" as *u8) 74 if idx == 0 as *NxInvIndex { hv_puts(" build failed\n" as *u8); sys_exit(1); return 1 } 75 let sv: i64 = nx_inv_save(idx, "/tmp/nishi_library.idx" as *u8) 76 hv_puts(" indexed rows=" as *u8); hv_num(idx.n_rows); hv_puts(" persisted=" as *u8); hv_num(sv); hv_puts(" (1=OK)\n" as *u8) 77 78 // query terms that exist ONLY in the dark research, NOT the old 52-line Library. 79 let live_paged: i64 = hv_query(idx, "pagedattention" as *u8, 14) 80 let idx2: *NxInvIndex = nx_inv_load("/tmp/nishi_library.idx" as *u8) 81 if idx2 == 0 as *NxInvIndex { hv_puts(" reload failed\n" as *u8); sys_exit(1); return 1 } 82 let disk_paged: i64 = hv_query(idx2, "pagedattention" as *u8, 14) 83 let disk_ppmi: i64 = hv_query(idx2, "ppmi" as *u8, 4) 84 hv_puts(" query 'pagedattention': in-mem=" as *u8); hv_num(live_paged); hv_puts(" on-disk=" as *u8); hv_num(disk_paged) 85 hv_puts(" query 'ppmi' on-disk=" as *u8); hv_num(disk_ppmi); hv_puts("\n----\n" as *u8) 86 87 var ok: i64 = 1 88 if sv != NX_INV_OK { ok = 0 } 89 if live_paged <= 0 { ok = 0 } // the dark term IS now indexed 90 if disk_paged != live_paged { ok = 0 } // and survives the disk round-trip 91 if disk_ppmi <= 0 { ok = 0 } // a second dark term, from the on-disk index 92 if ok == 1 { 93 hv_puts(" PROVEN: dark deep-research is now a PERSISTENT searchable index -- terms invisible to the old\n" as *u8) 94 hv_puts(" 52-line Library ('pagedattention','ppmi') are queryable on-disk. This is the harvest -> S-class index path.\n" as *u8) 95 sys_exit(0); return 0 96 } 97 hv_puts(" FAIL\n" as *u8); sys_exit(1); return 1 98}