code wiki / _hdl_build / nx_library_harvest_v2.nx
nx_library_harvest_v2.nx source
↩ module page · 175 lines · 9030 B
1// nx_library_harvest_v2.nx -- ASSIGNMENT D SCALE-UP (tutored): the v1 harvest proved dark->indexed but had two
2// named v1 limits: (1) 20 HARDCODED paths -- new research lands dark again; (2) per-LINE rows -- a query returns
3// a line number, not a document. v2 closes both: a getdents dir-walk (the team's own nx_dirent, same primitive
4// as nx_wall_scan_native) discovers every .md/.txt under the knowledge dirs, and each document becomes ONE row
5// (newlines flattened to spaces) so rowid == docid; a DOCMAP file (line i = path of docid i) makes every query
6// result name its document. Index + docmap persist to knowledge/index/ -- DURABLE, not /tmp (the Archivist
7// reboot lesson: /tmp artifacts died; knowledge/ survives). Gate: row count must EQUAL docmap count (the
8// alignment invariant), and a dark term must be queryable from the RELOADED on-disk index with its rowids in
9// docmap range. Subdirectory recursion (knowledge/library/mirror/) = the next rung, skipped via d_type.
10// license_tier: ORIGINAL
11import "nx_search_inverted_persist.nx"
12import "nx_dirent.nx"
13
14const LH2_DIR_BUF: i64 = 65536
15const LH2_PATH_CAP: i64 = 4096
16
17func lh2_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func lh2_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
19func lh2_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
20func lh2_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
21
22// suffix match: name ends in ".md" or ".txt"
23func lh2_indexable(name: *u8, n: i64) -> i64 {
24 if n > 3 { if name[n-3]==(46 as u8) { if name[n-2]==(109 as u8) { if name[n-1]==(100 as u8) { return 1 } } } }
25 if n > 4 { if name[n-4]==(46 as u8) { if name[n-3]==(116 as u8) { if name[n-2]==(120 as u8) { if name[n-1]==(116 as u8) { return 1 } } } } }
26 return 0
27}
28
29// write one document into the corpus as ONE row: every \n \r \t -> space, then a single terminating \n.
30// Returns 1 if a non-empty row was written (caller then appends the docmap line), else 0.
31func lh2_emit_doc(corpus_fd: i64, buf: *u8, len: i64) -> i64 {
32 if len <= 0 { return 0 }
33 let row: *u8 = sys_mmap(len + 1)
34 var nonspace: i64 = 0
35 var i: i64 = 0
36 while i < len {
37 var c: i64 = buf[i] as i64
38 if c == 10 { c = 32 }
39 if c == 13 { c = 32 }
40 if c == 9 { c = 32 }
41 if c != 32 { nonspace = 1 }
42 row[i] = c as u8
43 i = i + 1
44 }
45 if nonspace == 0 { return 0 } // all-whitespace doc would become an EMPTY row the builder skips -> docmap drift
46 row[len] = 10 as u8
47 var off: i64 = 0
48 while off < len + 1 {
49 let w: i64 = sys_write(corpus_fd, ((row as i64) + off) as *u8, len + 1 - off)
50 if w <= 0 { return 0 - 1 }
51 off = off + w
52 }
53 return 1
54}
55
56// walk ONE directory (no recursion -- d_type DIR skipped), harvesting every indexable regular file.
57// Returns docs harvested from this dir, or -1 on open failure.
58func lh2_harvest_dir(dirpath: *u8, corpus_fd: i64, docmap_fd: i64) -> i64 {
59 let fd: i64 = nx_open_rd(dirpath)
60 if fd < 0 { return 0 - 1 }
61 let dirbuf: *u8 = sys_mmap(LH2_DIR_BUF + 64)
62 let pathbuf: *u8 = sys_mmap(LH2_PATH_CAP)
63 let lenbox: *i64 = sys_mmap(16) as *i64
64 var docs: i64 = 0
65 let d: *NxDirent = sys_mmap(NX_DIRENT_BYTES) as *NxDirent // heap struct, the proven x86-lane idiom (no stack structs)
66 var done: i64 = 0
67 while done == 0 {
68 let n: i64 = nx_dirent_read(fd, dirbuf, LH2_DIR_BUF)
69 if n <= 0 { done = 1 }
70 else {
71 var off: i64 = 0
72 while off < n {
73 off = nx_dirent_iter(dirbuf, off, n, d)
74 if off < 0 { off = n }
75 else {
76 if d.dtype != NX_DT_DIR {
77 let nlen: i64 = nx_dirent_name_len(d)
78 if lh2_indexable(d.name, nlen) == 1 {
79 var po: i64 = lh2_cat(pathbuf, 0, dirpath)
80 po = lh2_cat(pathbuf, po, "/" as *u8)
81 po = lh2_cat(pathbuf, po, d.name)
82 pathbuf[po] = 0 as u8
83 let buf: *u8 = sys_read_file(pathbuf, lenbox)
84 if buf != 0 as *u8 {
85 let wrote: i64 = lh2_emit_doc(corpus_fd, buf, lenbox[0])
86 if wrote == 1 {
87 // docmap line i = path of docid i: the row and the line append in LOCKSTEP
88 sys_write(docmap_fd, pathbuf, po)
89 sys_write(docmap_fd, "\n" as *u8, 1)
90 docs = docs + 1
91 }
92 }
93 }
94 }
95 }
96 }
97 }
98 }
99 sys_close(fd)
100 return docs
101}
102
103func lh2_query(idx: *NxInvIndex, term: *u8, rowids: *i64, cap: i64, res: *NxInvQueryResult) -> i64 {
104 nx_inv_query_term(idx, term, lh2_strlen(term), rowids, cap, res)
105 return res.postings_count
106}
107
108func main() -> i64 {
109 lh2_puts("=== ASSIGNMENT D v2: dir-walk harvest -> per-DOC rows -> durable on-disk index + docmap ===\n" as *u8)
110 sys_mkdir("knowledge/index" as *u8, 0x1ed) // 0755; idempotent (EEXIST ignored)
111
112 let corpus_path: *u8 = "knowledge/index/library_corpus.txt" as *u8
113 let docmap_path: *u8 = "knowledge/index/library.docmap" as *u8
114 let index_path: *u8 = "knowledge/index/library.idx" as *u8
115
116 let corpus_fd: i64 = sys_openat_wr(corpus_path, 0x1a4)
117 let docmap_fd: i64 = sys_openat_wr(docmap_path, 0x1a4)
118 if corpus_fd < 0 { lh2_puts(" FAIL open corpus\n" as *u8); sys_exit(1); return 1 }
119 if docmap_fd < 0 { lh2_puts(" FAIL open docmap\n" as *u8); sys_exit(1); return 1 }
120
121 // the knowledge dirs that go dark today; ADD A ROW to extend coverage (data-driven, not a code change)
122 var total: i64 = 0
123 let d1: i64 = lh2_harvest_dir("knowledge/research" as *u8, corpus_fd, docmap_fd)
124 let d2: i64 = lh2_harvest_dir("knowledge/specs" as *u8, corpus_fd, docmap_fd)
125 let d3: i64 = lh2_harvest_dir("knowledge/library" as *u8, corpus_fd, docmap_fd)
126 let d4: i64 = lh2_harvest_dir("knowledge/status" as *u8, corpus_fd, docmap_fd)
127 if d1 > 0 { total = total + d1 }
128 if d2 > 0 { total = total + d2 }
129 if d3 > 0 { total = total + d3 }
130 if d4 > 0 { total = total + d4 }
131 sys_close(corpus_fd)
132 sys_close(docmap_fd)
133 lh2_puts(" walked research=" as *u8); lh2_num(d1)
134 lh2_puts(" specs=" as *u8); lh2_num(d2)
135 lh2_puts(" library=" as *u8); lh2_num(d3)
136 lh2_puts(" status=" as *u8); lh2_num(d4)
137 lh2_puts(" -> docs=" as *u8); lh2_num(total); lh2_puts("\n" as *u8)
138 if total <= 0 { lh2_puts(" FAIL harvest found nothing\n" as *u8); sys_exit(1); return 1 }
139
140 let idx: *NxInvIndex = nx_inv_build_from_jsonl(corpus_path)
141 if idx == 0 as *NxInvIndex { lh2_puts(" FAIL index build\n" as *u8); sys_exit(1); return 1 }
142 let sv: i64 = nx_inv_save(idx, index_path)
143 lh2_puts(" indexed rows=" as *u8); lh2_num(idx.n_rows)
144 lh2_puts(" persisted=" as *u8); lh2_num(sv); lh2_puts(" (1=OK)\n" as *u8)
145
146 // GATE 1: the alignment invariant -- every doc is exactly one row, so docmap line i NAMES rowid i
147 var ok: i64 = 1
148 if idx.n_rows != total { ok = 0; lh2_puts(" FAIL rows != docs (docmap drift)\n" as *u8) }
149 if sv != NX_INV_OK { ok = 0; lh2_puts(" FAIL persist\n" as *u8) }
150
151 // GATE 2: dark terms queryable from the RELOADED on-disk index, rowids inside docmap range
152 let idx2: *NxInvIndex = nx_inv_load(index_path)
153 if idx2 == 0 as *NxInvIndex { lh2_puts(" FAIL reload\n" as *u8); sys_exit(1); return 1 }
154 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult
155 let rowids: *i64 = sys_mmap(8*256) as *i64
156 let p1: i64 = lh2_query(idx2, "pagedattention" as *u8, rowids, 256, res)
157 var rid_ok: i64 = 1
158 var ri: i64 = 0
159 while ri < res.n_rowids_filled { if rowids[ri] < 0 { rid_ok = 0 } if rowids[ri] >= total { rid_ok = 0 } ri = ri + 1 }
160 let p2: i64 = lh2_query(idx2, "tantivy" as *u8, rowids, 256, res)
161 lh2_puts(" on-disk query 'pagedattention'=" as *u8); lh2_num(p1)
162 lh2_puts(" 'tantivy'=" as *u8); lh2_num(p2)
163 lh2_puts(" rowids-in-range=" as *u8); lh2_num(rid_ok); lh2_puts("\n----\n" as *u8)
164 if p1 <= 0 { ok = 0 }
165 if p2 <= 0 { ok = 0 }
166 if rid_ok != 1 { ok = 0 }
167
168 if ok == 1 {
169 lh2_puts(" HARVEST-V2-PROVEN: dir-walked docs=" as *u8); lh2_num(total)
170 lh2_puts(" rows==docs, dark terms queryable on-disk, rowid->docmap aligned. No hardcoded paths.\n" as *u8)
171 sys_exit(0); return 0
172 }
173 lh2_puts(" HARVEST-V2-FAIL\n" as *u8)
174 sys_exit(1); return 1
175}