code wiki / _hdl_build / nx_vec_hybrid_gate.nx
nx_vec_hybrid_gate.nx source
↩ module page · 79 lines · 7217 B
1// nx_vec_hybrid_gate.nx -- INTEGRATION proof that R-VEC-0 (kernel) + R-VEC-1 (embeddings) + R-VEC-5 (RRF fusion)
2// COMPOSE into working sovereign hybrid semantic search -- the felt S-class moment, measured end-to-end.
3//
4// Scenario (the classic lexical-recall failure): user query = "marriage". Only doc0 literally contains the token
5// "marriage"; doc1 is the OTHER family-law doc (custody/children/parents/divorce) and never says "marriage", so
6// pure keyword/BM25 search CANNOT find it. Sovereign distributional embeddings know marriage ~ divorce/custody
7// (same context), so the vector ranker surfaces doc1; RRF fusion then RECOVERS doc1 into the results -- without
8// losing the exact match (doc0 stays #1) and without false positives (the estate/injury docs stay out).
9//
10// This is the whole point of hybrid search: semantic recall on top of lexical precision. All sovereign, no-float,
11// no external weights, no third party. expect_exit: 0 license_tier: ORIGINAL
12import "nx_syscalls.nx"
13import "nx_vec_kernel.nx"
14import "nx_vec_embed.nx"
15import "nx_vec_fuse.nx"
16
17func hy_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func hy_pn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
19func hy_lst(list: *i64, n: i64) -> i64 { var i: i64=0; while i<n { hy_puts("doc" as *u8); hy_pn(list[i]); hy_puts(" " as *u8); i=i+1 } if n==0 { hy_puts("(none)" as *u8) } return 0 }
20func hy_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
21func hy_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
22
23func rank_of(list: *i64, n: i64, id: i64) -> i64 { var i: i64=0; while i<n { if list[i]==id { return i+1 } i=i+1 } return 0 }
24
25func main() -> i64 {
26 hy_puts("=== HYBRID SEARCH INTEGRATION -- kernel + embeddings + RRF, end-to-end ===\n" as *u8)
27 let M: *i64=sys_mmap(8*VE_V*VE_V) as *i64
28 var z: i64=0; while z<VE_V*VE_V { M[z]=0; z=z+1 }
29 let s1: *i64=sys_mmap(8*8) as *i64; s1[0]=0;s1[1]=1;s1[2]=2;s1[3]=3;s1[4]=4; ve_build_co(M,s1,5) // doc0 family (+marriage)
30 let s2: *i64=sys_mmap(8*8) as *i64; s2[0]=1;s2[1]=2;s2[2]=5;s2[3]=0; ve_build_co(M,s2,4) // doc1 family (NO marriage)
31 let s3: *i64=sys_mmap(8*8) as *i64; s3[0]=6;s3[1]=7;s3[2]=8;s3[3]=9;s3[4]=10; ve_build_co(M,s3,5) // doc2 estate
32 let s4: *i64=sys_mmap(8*8) as *i64; s4[0]=7;s4[1]=8;s4[2]=6;s4[3]=11; ve_build_co(M,s4,4) // doc3 estate
33 let s5: *i64=sys_mmap(8*8) as *i64; s5[0]=12;s5[1]=13;s5[2]=14;s5[3]=15;s5[4]=16; ve_build_co(M,s5,5) // doc4 injury
34
35 let dptr: *i64=sys_mmap(8*8) as *i64; let dlen: *i64=sys_mmap(8*8) as *i64
36 dptr[0]=s1 as i64; dlen[0]=5; dptr[1]=s2 as i64; dlen[1]=4; dptr[2]=s3 as i64; dlen[2]=5; dptr[3]=s4 as i64; dlen[3]=4; dptr[4]=s5 as i64; dlen[4]=5
37 let ND: i64=5
38 let QID: i64=4 // query term = "marriage"
39 let qrow: *i64=vr_embed(M, QID)
40
41 // (1) LEXICAL list: docs whose token set literally contains the query term
42 let lex: *i64=sys_mmap(8*8) as *i64; var nL: i64=0
43 var i: i64=0
44 while i<ND { let terms: *i64=dptr[i] as *i64; var has: i64=0; var j: i64=0; while j<dlen[i] { if terms[j]==QID { has=1 } j=j+1 } if has==1 { lex[nL]=i; nL=nL+1 } i=i+1 }
45
46 // (2) VECTOR list: cos(query, doc) for each doc, ranked desc, cos>0
47 let vcos: *i64=sys_mmap(8*8) as *i64; let tmp: *i64=sys_mmap(8*VE_V) as *i64
48 i=0; while i<ND { ve_docvec(M, dptr[i] as *i64, dlen[i], tmp); vcos[i]=vr_cos_milli(qrow, tmp, VE_V); i=i+1 }
49 let vlist: *i64=sys_mmap(8*8) as *i64; var nV: i64=0
50 i=0; while i<ND { if vcos[i]>0 { vlist[nV]=i; nV=nV+1 } i=i+1 }
51 var a: i64=0
52 while a<nV { var best: i64=a; var b: i64=a+1; while b<nV { if vcos[vlist[b]]>vcos[vlist[best]] { best=b } b=b+1 } let t: i64=vlist[a]; vlist[a]=vlist[best]; vlist[best]=t; a=a+1 }
53
54 // (3) FUSE
55 let score: *i64=sys_mmap(8*8) as *i64; var z2: i64=0; while z2<ND { score[z2]=0; z2=z2+1 }
56 vr_fuse_add(score, lex, nL); vr_fuse_add(score, vlist, nV)
57 let fused: *i64=sys_mmap(8*8) as *i64; let nF: i64=vr_fuse_rank(score, ND, fused)
58
59 hy_puts(" query=\"marriage\" (doc0 family+marriage, doc1 family-NO-marriage, doc2/3 estate, doc4 injury)\n" as *u8)
60 hy_puts(" LEXICAL (keyword): " as *u8); hy_lst(lex, nL); hy_puts("\n" as *u8)
61 hy_puts(" VECTOR (semantic): " as *u8); hy_lst(vlist, nV); hy_puts(" [cos doc0=" as *u8); hy_pn(vcos[0]); hy_puts(" doc1=" as *u8); hy_pn(vcos[1]); hy_puts(" doc4=" as *u8); hy_pn(vcos[4]); hy_puts("]\n" as *u8)
62 hy_puts(" HYBRID (RRF): " as *u8); hy_lst(fused, nF); hy_puts("\n" as *u8)
63
64 var pass: i64=0; var total: i64=0
65 total=total+1; hy_puts(" T1 keyword search MISSES doc1 (no 'marriage' token) " as *u8); if rank_of(lex,nL,1)==0 { pass=pass+1; hy_puts("PASS\n" as *u8) } else { hy_puts("FAIL\n" as *u8) }
66 total=total+1; hy_puts(" T2 semantic FINDS doc1 in top-2 (meaning, not tokens) " as *u8); let vr1: i64=rank_of(vlist,nV,1); if vr1>=1 { if vr1<=2 { pass=pass+1; hy_puts("PASS\n" as *u8) } else { hy_puts("FAIL\n" as *u8) } } else { hy_puts("FAIL\n" as *u8) }
67 total=total+1; hy_puts(" T3 HYBRID RECOVERS doc1 (recall win over keyword) " as *u8); if rank_of(fused,nF,1)>=1 { pass=pass+1; hy_puts("PASS\n" as *u8) } else { hy_puts("FAIL\n" as *u8) }
68 total=total+1; hy_puts(" T4 hybrid keeps the exact match #1 (doc0) " as *u8); if fused[0]==0 { pass=pass+1; hy_puts("PASS\n" as *u8) } else { hy_puts("FAIL\n" as *u8) }
69 total=total+1; hy_puts(" T5 no false positives (injury doc4 stays out) " as *u8); if rank_of(fused,nF,4)==0 { pass=pass+1; hy_puts("PASS\n" as *u8) } else { hy_puts("FAIL\n" as *u8) }
70
71 hy_puts("----\nHYBRID-SEARCH integration " as *u8); hy_pn(pass); hy_puts("/" as *u8); hy_pn(total); hy_puts(" passed\n" as *u8)
72 hy_puts("HONEST: the three vector rungs COMPOSE -- semantic recall recovers a doc keyword search cannot reach,\n" as *u8)
73 hy_puts(" keeping lexical precision. Proven on the controlled corpus + ALL sovereign; wiring this pipeline\n" as *u8)
74 hy_puts(" into the LIVE nx_onsite_search over the real manifest is the next integration step.\n" as *u8)
75 let lg: i64=sys_openat_append("knowledge/status/vec_hybrid_gate.log" as *u8, 0x1a4)
76 if lg>=0 { hy_w(lg, "HYBRID integration pass=" as *u8); hy_wn(lg, pass); hy_w(lg, "/" as *u8); hy_wn(lg, total); hy_w(lg, " lex_misses_doc1+hybrid_recovers" as *u8); if pass==total { hy_w(lg, " GREEN\n" as *u8) } else { hy_w(lg, " RED\n" as *u8) } sys_close(lg) }
77 if pass==total { hy_puts("HYBRID GREEN (sovereign hybrid semantic search proven end-to-end)\n" as *u8); sys_exit(0); return 0 }
78 hy_puts("HYBRID RED\n" as *u8); sys_exit(1); return 1
79}