code wiki / _hdl_build / nx_vocab_extract.nx

nx_vocab_extract.nx

buildroot/runtime/_hdl_build/nx_vocab_extract.nx

9552 B152 linesdepth 3pulls 3 transitivereach 0 importersview sourcekind tool

docs dependencies structs consts functions

about

nx_vocab_extract.nx -- extract the distinct, SORTED vocab STRINGS from a manifest/source so the gallery's hash-based search can power a LIVE /api/suggest (autocomplete) + fuzzy. Same tokenization as the index (alnum runs len>=2, lowercased, pure-numeric scrubbed). LINE-AWARE: tokenizes only the TEXT after the first TAB per line, so the leading id/cid field (e.g. galx_search.tsv = cid<TAB>prompt) does NOT pollute the vocab. SCALABLE: FNV open-addressing hash-set dedup + iterative bottom-up mergesort (handles the gallery's ~10^5 vocab). usage: nx_vocab_extract <manifest> <vocab_out> (CLI: writes the sorted vocab file) nx_vocab_extract (gate: andelinwest regression + /suggest assertions) license_tier: ORIGINAL

dependencies 2 imports · 0 importers

imports: nx_syscalls.nxnx_prefix.nx

imported by: nobody (leaf or entry point)

call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown

structs

none

consts

12	const VX_MAGIC_2166136261: i64 = 2166136261
13	const VX_MAGIC_16777619: i64 = 16777619
15	const VX_HT: i64 = 524288 // 2^19 hash slots (load < 0.4 at ~180k terms)
16	const VX_STORE: i64 = 33554432 // 32MB term store
17	const VX_MAXV: i64 = 262144 // max distinct terms

functions

19	func vx_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } called by 1: main calls 1: sys_write
20	func vx_pn(v: i64) -> i64 { let bb: u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } called by 1: main calls 2: sys_mmap sys_write
21	func vx_putt(t: u8) -> i64 { var n: i64=0; while t[n]!=(0 as u8){n=n+1} sys_write(1,t,n); sys_write(1," " as u8,1); return 0 } called by 1: main calls 1: sys_write
22	func vx_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } called by 2: vx_extract_file main calls 1: sys_write
23	func vx_wn(fd: i64, v: i64) -> i64 { let bb: u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 } called by 1: main calls 2: sys_mmap sys_write
25	func vx_is_tok(c: i64) -> i64 { if c>=48 { if c<=57 { return 1 } } if c>=65 { if c<=90 { return 1 } } if c>=97 { if c<=122 { return 1 } } return 0 } called by 1: vx_extract_file
26	func vx_lower(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c } called by 3: vx_fnv vx_eq_lower vx_extract_file
27	func vx_fnv(buf: u8, off: i64, len: i64) -> i64 { var h: i64=VX_MAGIC_2166136261; var i: i64=0; while i<len { h = h ^ vx_lower(buf[off+i] as i64); h = h VX_MAGIC_16777619; i=i+1 } return h } called by 1: vx_extract_file calls 1: vx_lower
29	func vx_eq_lower(term: u8, buf: u8, off: i64, len: i64) -> i64 called by 1: vx_extract_file calls 1: vx_lower
36	func vx_strcmp(a: u8, b: u8) -> i64 called by 1: vx_msort
43	func vx_msort(ptrs: i64, n: i64, tmp: i64) -> i64 called by 1: vx_extract_file calls 1: vx_strcmp
65	func vx_extract_file(manifest: u8, out: u8, store: u8, ptrs: i64, cnt: i64, ht: i64, tmp: *i64, kmin: i64) -> i64 called by 1: main calls 10: sys_mmap sys_read_file vx_is_tok vx_fnv vx_lower vx_eq_lower+4
116	func main(argc: i64, argv: *i64) -> i64 calls 11: sys_mmap vx_extract_file vx_puts sys_exit vx_pn vr_prefix_collect+5