code wiki / _hdl_build / nx_vocab_extract.nx
nx_vocab_extract.nx source
↩ module page · 152 lines · 9552 B
1// nx_vocab_extract.nx -- extract the distinct, SORTED vocab STRINGS from a manifest/source so the gallery's
2// hash-based search can power a LIVE /api/suggest (autocomplete) + fuzzy. Same tokenization as the index
3// (alnum runs len>=2, lowercased, pure-numeric scrubbed). LINE-AWARE: tokenizes only the TEXT after the first TAB
4// per line, so the leading id/cid field (e.g. galx_search.tsv = cid<TAB>prompt) does NOT pollute the vocab.
5// SCALABLE: FNV open-addressing hash-set dedup + iterative bottom-up mergesort (handles the gallery's ~10^5 vocab).
6//
7// usage: nx_vocab_extract <manifest> <vocab_out> (CLI: writes the sorted vocab file)
8// nx_vocab_extract (gate: andelinwest regression + /suggest assertions)
9// license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_prefix.nx"
12const VX_MAGIC_2166136261: i64 = 2166136261
13const VX_MAGIC_16777619: i64 = 16777619
14
15const VX_HT: i64 = 524288 // 2^19 hash slots (load < 0.4 at ~180k terms)
16const VX_STORE: i64 = 33554432 // 32MB term store
17const VX_MAXV: i64 = 262144 // max distinct terms
18
19func vx_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
20func vx_pn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
21func vx_putt(t: *u8) -> i64 { var n: i64=0; while t[n]!=(0 as u8){n=n+1} sys_write(1,t,n); sys_write(1," " as *u8,1); return 0 }
22func vx_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
23func vx_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
24
25func vx_is_tok(c: i64) -> i64 { if c>=48 { if c<=57 { return 1 } } if c>=65 { if c<=90 { return 1 } } if c>=97 { if c<=122 { return 1 } } return 0 }
26func vx_lower(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c }
27func vx_fnv(buf: *u8, off: i64, len: i64) -> i64 { var h: i64=VX_MAGIC_2166136261; var i: i64=0; while i<len { h = h ^ vx_lower(buf[off+i] as i64); h = h * VX_MAGIC_16777619; i=i+1 } return h }
28// compare stored NUL-term lowercased `term` vs raw buf[off..off+len) (lowercased on the fly)
29func vx_eq_lower(term: *u8, buf: *u8, off: i64, len: i64) -> i64 {
30 var i: i64=0
31 while i<len { if term[i]==(0 as u8) { return 0 } if (term[i] as i64)!=vx_lower(buf[off+i] as i64) { return 0 } i=i+1 }
32 if term[i]!=(0 as u8) { return 0 }
33 return 1
34}
35// lexicographic strcmp of NUL-terminated strings
36func vx_strcmp(a: *u8, b: *u8) -> i64 {
37 var i: i64=0
38 while a[i]!=(0 as u8) { if b[i]==(0 as u8) { return 1 } if a[i]!=b[i] { if (a[i] as i64)<(b[i] as i64) { return 0-1 } return 1 } i=i+1 }
39 if b[i]!=(0 as u8) { return 0-1 }
40 return 0
41}
42// iterative bottom-up mergesort of ptrs[0..n) (string order), tmp = scratch of size n
43func vx_msort(ptrs: *i64, n: i64, tmp: *i64) -> i64 {
44 var width: i64=1
45 while width<n {
46 var lo: i64=0
47 while lo<n {
48 var mid: i64=lo+width; if mid>n { mid=n }
49 var hi: i64=lo+width+width; if hi>n { hi=n }
50 var a: i64=lo; var b: i64=mid; var k: i64=lo
51 while k<hi {
52 if a<mid { if b<hi { if vx_strcmp(ptrs[a] as *u8, ptrs[b] as *u8)<=0 { tmp[k]=ptrs[a]; a=a+1 } else { tmp[k]=ptrs[b]; b=b+1 } } else { tmp[k]=ptrs[a]; a=a+1 } } else { tmp[k]=ptrs[b]; b=b+1 }
53 k=k+1
54 }
55 lo=lo+width+width
56 }
57 var c: i64=lo; c=0; while c<n { ptrs[c]=tmp[c]; c=c+1 }
58 width=width+width
59 }
60 return 0
61}
62
63// extract -> store/ptrs (sorted, frequency-filtered), write vocab file. returns kept count (or -1 read fail).
64// kmin = minimum corpus occurrences to keep a term (drops user-prompt typos/fragments; kmin=1 keeps all).
65func vx_extract_file(manifest: *u8, out: *u8, store: *u8, ptrs: *i64, cnt: *i64, ht: *i64, tmp: *i64, kmin: i64) -> i64 {
66 let box: *i64=sys_mmap(16) as *i64
67 let buf: *u8=sys_read_file(manifest, box)
68 if buf==0 as *u8 { return 0-1 }
69 let n: i64=box[0]
70 var z: i64=0; while z<VX_HT { ht[z]=0; z=z+1 }
71 var nv: i64=0; var so: i64=0
72 var i: i64=0; var past_tab: i64=0
73 while i<n {
74 let ch: i64=buf[i] as i64
75 if ch==10 { past_tab=0; i=i+1 } else {
76 if ch==9 { past_tab=1; i=i+1 } else {
77 if past_tab==0 { i=i+1 } else {
78 if vx_is_tok(ch)==0 { i=i+1 } else {
79 let s: i64=i; var go: i64=1
80 while go==1 { if i>=n { go=0 } else { let c2: i64=buf[i] as i64; if vx_is_tok(c2)==1 { i=i+1 } else { go=0 } } }
81 let len: i64=i-s
82 if len>=2 {
83 var num: i64=1; var t: i64=s; while t<i { let c: i64=buf[t] as i64; if c<48 { num=0 } else { if c>57 { num=0 } } t=t+1 }
84 if num==0 {
85 let h: i64=vx_fnv(buf, s, len)
86 var slot: i64=h & (VX_HT-1)
87 var guard: i64=0
88 while guard<VX_HT {
89 let cur: i64=ht[slot]
90 if cur==0 {
91 if nv<VX_MAXV { if so+len+1 < VX_STORE {
92 let base: i64=(store as i64)+so
93 var bb2: i64=0; while bb2<len { store[so+bb2]=vx_lower(buf[s+bb2] as i64) as u8; bb2=bb2+1 }
94 store[so+len]=0 as u8
95 ht[slot]=nv+1; ptrs[nv]=base; cnt[nv]=1; nv=nv+1; so=so+len+1
96 } }
97 guard=VX_HT
98 } else {
99 if vx_eq_lower(ptrs[cur-1] as *u8, buf, s, len)==1 { cnt[cur-1]=cnt[cur-1]+1; guard=VX_HT } else { slot=(slot+1) & (VX_HT-1); guard=guard+1 }
100 }
101 }
102 }
103 }
104 } } } }
105 }
106 // frequency filter: keep terms occurring >= kmin times (drops 1-2x typos/fragments at scale)
107 var w: i64=0; var r: i64=0
108 while r<nv { if cnt[r]>=kmin { ptrs[w]=ptrs[r]; w=w+1 } r=r+1 }
109 let kept: i64=w
110 vx_msort(ptrs, kept, tmp)
111 let fd: i64=sys_openat_wr(out, 0x1a4)
112 if fd>=0 { var k: i64=0; while k<kept { vx_w(fd, ptrs[k] as *u8); vx_w(fd, "\n" as *u8); k=k+1 } sys_close(fd) }
113 return kept
114}
115
116func main(argc: i64, argv: *i64) -> i64 {
117 let store: *u8=sys_mmap(VX_STORE)
118 let ptrs: *i64=sys_mmap(8*VX_MAXV) as *i64
119 let cnt: *i64=sys_mmap(8*VX_MAXV) as *i64
120 let ht: *i64=sys_mmap(8*VX_HT) as *i64
121 let tmp: *i64=sys_mmap(8*VX_MAXV) as *i64
122
123 if argc>=3 {
124 let mf: *u8=argv[1] as *u8; let ot: *u8=argv[2] as *u8
125 var kmin: i64=1
126 if argc>=4 { let ka: *u8=argv[3] as *u8; var kv: i64=0; var ki: i64=0; while ka[ki]!=(0 as u8) { let c: i64=ka[ki] as i64; if c>=48 { if c<=57 { kv=kv*10+(c-48) } } ki=ki+1 } if kv>0 { kmin=kv } }
127 let nv: i64=vx_extract_file(mf, ot, store, ptrs, cnt, ht, tmp, kmin)
128 if nv<0 { vx_puts("VOCAB-EXTRACT FAIL: cannot read manifest\n" as *u8); sys_exit(1); return 1 }
129 vx_puts("VOCAB-EXTRACT ok terms="); vx_pn(nv); vx_puts(" (kmin="); vx_pn(kmin); vx_puts(") -> "); vx_puts(ot); vx_puts("\n" as *u8)
130 sys_exit(0); return 0
131 }
132
133 vx_puts("=== VOCAB EXTRACT gate (andelinwest regression + /suggest) ===\n" as *u8)
134 let nv: i64=vx_extract_file("knowledge/index/andelinwest.manifest" as *u8, "knowledge/index/andelinwest.vocab" as *u8, store, ptrs, cnt, ht, tmp, 1)
135 if nv<0 { vx_puts("FAIL: andelinwest.manifest not found\n" as *u8); sys_exit(1); return 1 }
136 vx_puts(" distinct terms: " as *u8); vx_pn(nv); vx_puts("\n" as *u8)
137 let out: *i64=sys_mmap(8*64) as *i64
138 var pass: i64=0; var total: i64=0
139 let m1: i64=vr_prefix_collect(ptrs, nv, "div\x00" as *u8, 3, out)
140 vx_puts(" 'div' -> " as *u8); var i: i64=0; while i<m1 { vx_putt(ptrs[out[i]] as *u8); i=i+1 } vx_puts("\n" as *u8)
141 total=total+1; if m1>=1 { pass=pass+1; vx_puts(" T1 'div'->divorce PASS\n" as *u8) } else { vx_puts(" T1 FAIL\n" as *u8) }
142 let m2: i64=vr_prefix_collect(ptrs, nv, "cust\x00" as *u8, 4, out)
143 total=total+1; if m2>=1 { pass=pass+1; vx_puts(" T2 'cust'->custody PASS\n" as *u8) } else { vx_puts(" T2 FAIL\n" as *u8) }
144 let m3: i64=vr_prefix_collect(ptrs, nv, "ins\x00" as *u8, 3, out)
145 total=total+1; if m3>=1 { pass=pass+1; vx_puts(" T3 'ins'->insurance PASS\n" as *u8) } else { vx_puts(" T3 FAIL\n" as *u8) }
146 total=total+1; if nv>=15 { pass=pass+1; vx_puts(" T4 real vocab (>=15) PASS\n" as *u8) } else { vx_puts(" T4 FAIL\n" as *u8) }
147 vx_puts("----\nVOCAB-EXTRACT(scalable) gate " as *u8); vx_pn(pass); vx_puts("/" as *u8); vx_pn(total); vx_puts(" passed\n" as *u8)
148 let lg: i64=sys_openat_append("knowledge/status/vocab_extract_gate.log" as *u8, 0x1a4)
149 if lg>=0 { vx_w(lg, "VOCAB-EXTRACT(scalable) gate pass=" as *u8); vx_wn(lg, pass); vx_w(lg, "/" as *u8); vx_wn(lg, total); vx_w(lg, " terms=" as *u8); vx_wn(lg, nv); if pass==total { vx_w(lg, " GREEN\n" as *u8) } else { vx_w(lg, " RED\n" as *u8) } sys_close(lg) }
150 if pass==total { vx_puts("VOCAB-EXTRACT GREEN\n" as *u8); sys_exit(0); return 0 }
151 vx_puts("VOCAB-EXTRACT RED\n" as *u8); sys_exit(1); return 1
152}