code wiki / _hdl_build / nx_library_harvest_v2.nx

nx_library_harvest_v2.nx

buildroot/runtime/_hdl_build/nx_library_harvest_v2.nx

9030 B175 linesdepth 5pulls 7 transitivereach 0 importersview sourcekind tooltopic library
docsdependenciesstructsconstsfunctions

about

nx_library_harvest_v2.nx -- ASSIGNMENT D SCALE-UP (tutored): the v1 harvest proved dark->indexed but had two named v1 limits: (1) 20 HARDCODED paths -- new research lands dark again; (2) per-LINE rows -- a query returns a line number, not a document. v2 closes both: a getdents dir-walk (the team's own nx_dirent, same primitive as nx_wall_scan_native) discovers every .md/.txt under the knowledge dirs, and each document becomes ONE row (newlines flattened to spaces) so rowid == docid; a DOCMAP file (line i = path of docid i) makes every query result name its document. Index + docmap persist to knowledge/index/ -- DURABLE, not /tmp (the Archivist reboot lesson: /tmp artifacts died; knowledge/ survives). Gate: row count must EQUAL docmap count (the alignment invariant), and a dark term must be queryable from the RELOADED on-disk index with its rowids in docmap range. Subdirectory recursion (knowledge/library/mirror/) = the next rung, skipped via d_type. license_tier: ORIGINAL

dependencies 2 imports · 0 importers

nx_search_inverted_persist.nx nx_dirent.nx nx_library_harvest_v2.nx

imports: nx_search_inverted_persist.nxnx_dirent.nx

imported by: nobody (leaf or entry point)

call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown

main lh2_puts sys_mkdir sys_openat_wr lh2_harvest_dir nx_open_rd nx_openat nx_dirent_read nx_dirent_iter nx_dirent_name_len lh2_indexable lh2_cat sys_read_file sys_openat_rd sys_lseek sys_mmap sys_read sys_close lh2_emit_doc lh2_num nx_inv_build_from_jsonl sys_read_file ↻ nx_inv_new_sized nx_inv_index_row nx_inv_is_token_char nx_inv_hash_bytes_lower nx_inv_add_posting nx_inv_count_token nx_inv_slot_at nx_inv_slot_hash nx_inv_slot_set_hash nx_inv_slot_set_postings_c nx_inv_slot_postings_count nx_inv_finalize_offsets nx_inv_slot_at ↻ nx_inv_slot_hash ↻ nx_inv_slot_postings_count ↻ nx_inv_slot_set_postings_o nx_inv_slot_set_write_curs nx_inv_emit_row

structs

none

consts

14const LH2_DIR_BUF: i64 = 65536
15const LH2_PATH_CAP: i64 = 4096

functions

17func lh2_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
called by 1: main
18func lh2_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
called by 1: main
19func lh2_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
called by 1: lh2_query
20func lh2_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
called by 1: lh2_harvest_dir
23func lh2_indexable(name: *u8, n: i64) -> i64
called by 1: lh2_harvest_dir
31func lh2_emit_doc(corpus_fd: i64, buf: *u8, len: i64) -> i64
called by 1: lh2_harvest_dir
58func lh2_harvest_dir(dirpath: *u8, corpus_fd: i64, docmap_fd: i64) -> i64
103func lh2_query(idx: *NxInvIndex, term: *u8, rowids: *i64, cap: i64, res: *NxInvQueryResult) -> i64
called by 1: main calls 2: nx_inv_query_termlh2_strlen
108func main() -> i64