nx_distrib_embed.nx source
↩ module page · 110 lines · 5967 B
1// nx_distrib_embed.nx -- R3 rung 2 (R3b): a SOVEREIGN DISTRIBUTIONAL embedder = the Jina-Embeddings replacement,
2// count-based (NOT a trained neural net, NO float, NO training loop): "you shall know a word by the company it
3// keeps" (Firth 1957 / Turney-Pantel 2010). Word vector = its co-occurrence row over a corpus; doc vector = sum
4// of its words' rows; similarity = integer cosine (isqrt). This is the SEMANTIC lever nx_vec_index needed. THE
5// PROOF (what BM25 canNOT do): query 'cat' retrieves the DOG doc over the CAR doc even though NEITHER contains
6// the word 'cat' -- pure semantic (shared context), zero lexical overlap. main() = KAT gate + neg-control.
7// HONEST SCOPE: proves the METHOD on a controlled mini-corpus (docs pre-tokenized to word-IDs to isolate the
8// embedder from string-tokenization). Scaling to the real 519-doc library (tokenizer + big matrix) = R3b-scale.
9// license_tier: ORIGINAL expect_exit: 0
10import "nx_syscalls.nx"
11const K_MAGIC_4096: i64 = 4096
12
13func dw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
14func dn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m; sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
15
16func isqrt(n: i64) -> i64 {
17 if n<0 { return 0 }
18 if n<2 { return n }
19 var x: i64 = n
20 var y: i64 = (x+1)/2
21 while y<x { x=y; y=(x + n/x)/2 }
22 return x
23}
24
25func vdot(a: *i64, ao: i64, b: *i64, bo: i64, d: i64) -> i64 {
26 var s: i64=0; var i: i64=0
27 while i<d { s=s+a[ao+i]*b[bo+i]; i=i+1 }
28 return s
29}
30
31// cosine in permille between raw vectors a[ao..] and b[bo..] (normalizes internally via isqrt).
32func cos2(a: *i64, ao: i64, b: *i64, bo: i64, d: i64) -> i64 {
33 let da: i64 = vdot(a,ao,a,ao,d)
34 let db: i64 = vdot(b,bo,b,bo,d)
35 if da==0 { return 0 }
36 if db==0 { return 0 }
37 let dp: i64 = vdot(a,ao,b,bo,d)
38 let denom: i64 = isqrt(da*db)
39 if denom==0 { return 0 }
40 return (dp*1000)/denom
41}
42
43// accumulate whole-doc co-occurrence for one doc into M[V x V].
44func add_doc(M: *i64, V: i64, doc: *i64, len: i64) -> i64 {
45 var i: i64=0
46 while i<len {
47 var j: i64=0
48 while j<len { if i!=j { M[doc[i]*V + doc[j]] = M[doc[i]*V + doc[j]] + 1 } j=j+1 }
49 i=i+1
50 }
51 return 0
52}
53
54// doc vector = sum of the co-occurrence rows of the doc's words.
55func doc_vec(M: *i64, V: i64, doc: *i64, len: i64, out: *i64) -> i64 {
56 var c: i64=0; while c<V { out[c]=0; c=c+1 }
57 var i: i64=0
58 while i<len { var cc: i64=0; while cc<V { out[cc]=out[cc]+M[doc[i]*V+cc]; cc=cc+1 } i=i+1 }
59 return 0
60}
61
62func chk(label: *u8, got: i64, exp: i64, tot: *i64) -> i64 {
63 dw(" "); dw(label); dw(" = "); dn(got)
64 if got==exp { dw(" PASS"); tot[0]=tot[0]+1 } else { dw(" FAIL(exp "); dn(exp); dw(")") }
65 tot[1]=tot[1]+1; dw("\n"); return 0
66}
67
68func main() -> i64 {
69 // vocab ids: 0=cat 1=dog 2=furry 3=pet 4=animal 5=car 6=truck 7=road 8=drive 9=wheels 10=the 11=is 12=a 13=on
70 let V: i64 = 14
71 let tot: *i64 = sys_mmap(64) as *i64
72 tot[0]=0; tot[1]=0
73 dw("=== nx_distrib_embed -- sovereign distributional (co-occurrence) embedder: semantic retrieval, no-float, no-train ===\n")
74
75 let M: *i64 = sys_mmap(K_MAGIC_4096) as *i64
76 var z: i64=0; while z<V*V { M[z]=0; z=z+1 }
77
78 let d0: *i64 = sys_mmap(64) as *i64; d0[0]=0; d0[1]=11; d0[2]=12; d0[3]=2; d0[4]=3; d0[5]=4 // cat is a furry pet animal
79 let d1: *i64 = sys_mmap(64) as *i64; d1[0]=1; d1[1]=11; d1[2]=12; d1[3]=2; d1[4]=3; d1[5]=4 // dog is a furry pet animal
80 let d2: *i64 = sys_mmap(64) as *i64; d2[0]=5; d2[1]=8; d2[2]=13; d2[3]=7; d2[4]=9 // car drive on road wheels
81 let d3: *i64 = sys_mmap(64) as *i64; d3[0]=6; d3[1]=8; d3[2]=13; d3[3]=7; d3[4]=9 // truck drive on road wheels
82 add_doc(M,V,d0,6); add_doc(M,V,d1,6); add_doc(M,V,d2,5); add_doc(M,V,d3,5)
83
84 dw("word-level semantics (cosine of co-occurrence rows):\n")
85 chk("cos(cat, dog) [both animals, shared context]" as *u8, cos2(M,0*V,M,1*V,V), 1000, tot)
86 chk("cos(car, truck) [both vehicles, shared context]" as *u8, cos2(M,5*V,M,6*V,V), 1000, tot)
87 chk("cos(cat, car) [animal vs vehicle, disjoint] " as *u8, cos2(M,0*V,M,5*V,V), 0, tot)
88
89 // SEMANTIC RETRIEVAL PROOF: query 'cat'. Candidate docs = DOG-sentence and CAR-sentence. NEITHER contains 'cat'.
90 let dv1: *i64 = sys_mmap(256) as *i64; doc_vec(M,V,d1,6,dv1) // dog sentence
91 let dv2: *i64 = sys_mmap(256) as *i64; doc_vec(M,V,d2,5,dv2) // car sentence
92 let c1: i64 = cos2(M,0*V, dv1,0, V) // cat vs dog-doc
93 let c2: i64 = cos2(M,0*V, dv2,0, V) // cat vs car-doc
94 dw("SEMANTIC RETRIEVAL (query='cat'; neither candidate contains the word 'cat'):\n")
95 dw(" cat -> DOG-doc = "); dn(c1); dw(" cat -> CAR-doc = "); dn(c2); dw("\n")
96 var okret: i64=0; if c1>c2 { okret=1 }
97 if okret==1 { dw(" DOG-doc outranks CAR-doc by MEANING (BM25 would tie both at 0) PASS\n"); tot[0]=tot[0]+1 } else { dw(" FAIL: no semantic lift\n") }
98 tot[1]=tot[1]+1
99 chk(" car-doc similarity to 'cat' (must be 0) " as *u8, c2, 0, tot)
100
101 // NEG-CONTROL: a semantically-unrelated doc must NOT outrank a related one.
102 dw(" NEG-CONTROL: unrelated(car-doc) must NOT outrank related(dog-doc) -> ")
103 var neg: i64=0; if c2<c1 { dw("car("); dn(c2); dw(") < dog("); dn(c1); dw(") CAUGHT\n"); neg=1 } else { dw("BROKEN\n") }
104 if neg==1 { tot[0]=tot[0]+1 } tot[1]=tot[1]+1
105
106 dw("DISTRIB-EMBED KAT "); dn(tot[0]); dw("/"); dn(tot[1]); dw("\n")
107 if tot[0]==tot[1] { dw("GREEN -- sovereign distributional embedder: retrieves by MEANING (zero lexical overlap), no float, no training. Feeds nx_vec_index. R3b-scale = run over the real 519-doc library.\n"); return 0 }
108 dw("RED -- distributional embedder KAT failed\n")
109 return 1
110}