code wiki / (root) / nx_lib_semantic.nx
`nx_lib_semantic.nx` source

↩ module page · 222 lines · 10380 B
1// nx_lib_semantic.nx -- R3b-SCALE: run the sovereign distributional embedder over the REAL banked library
2// (the 6 deep-research dr_*.txt) so semantic retrieval goes LIVE on actual documents. Pipeline: read_file ->
3// tokenize (lowercase, len>=4 stopword floor) -> bounded vocab (V<=600) -> windowed word-word co-occurrence ->
4// doc vector = sum of member words' co-occ rows -> per-doc max-scale (cosine-invariant, bounds magnitude) ->
5// integer cosine. No float, no training, no dep. GATE asserts only STRUCTURAL invariants (self-sim=1000,
6// self-retrieval top-1, zero-query=0) so it CANNOT be gamed; the semantic CLUSTERING is MEASURED + REPORTED,
7// never pre-asserted (that would be the rigged-gate sin). HONEST CAP: first 3000 content-words/doc (logged).
8// license_tier: ORIGINAL  expect_exit: 0
9import "nx_syscalls.nx"
10const K_MAGIC_3000: i64 = 3000
11const K_MAGIC_8388608: i64 = 8388608
12const K_MAGIC_65536: i64 = 65536
13const K_MAGIC_8192: i64 = 8192
14const K_MAGIC_200000: i64 = 200000
15const K_MAGIC_3200000: i64 = 3200000
16
17func lw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func ln(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m; sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
19
20func read_file(path: *u8, buf: *u8, cap: i64, lenbox: *i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd<0 { return 0 }
  let n: i64 = sys_read(fd, buf, cap)
  sys_close(fd)
  if n<=0 { return 0 }
  lenbox[0]=n
  return n
28}
29
30func isqrt(n: i64) -> i64 {
  if n<0 { return 0 }
  if n<2 { return n }
  var x: i64 = n
  var y: i64 = (x+1)/2
  while y<x { x=y; y=(x + n/x)/2 }
  return x
37}
38
39func vdot(a: *i64, ao: i64, b: *i64, bo: i64, d: i64) -> i64 {
  var s: i64=0; var i: i64=0
  while i<d { s=s+a[ao+i]*b[bo+i]; i=i+1 }
  return s
43}
44
45// cosine permille; magnitudes bounded by per-doc max-scaling so da*db and dp*1000 fit i64. self-sim exact 1000.
46func cos2(a: *i64, ao: i64, b: *i64, bo: i64, d: i64) -> i64 {
  let da: i64 = vdot(a,ao,a,ao,d)
  let db: i64 = vdot(b,bo,b,bo,d)
  if da==0 { return 0 }
  if db==0 { return 0 }
  let dp: i64 = vdot(a,ao,b,bo,d)
  let denom: i64 = isqrt(da*db)
  if denom==0 { return 0 }
  return (dp*1000)/denom
55}
56
57// vocab: vst[0]=count vst[1]=bytes_used. linear-scan lookup; add if room. returns id or -1 if full.
58func vocab_id(vbuf: *u8, voff: *i64, vlen: *i64, vst: *i64, w: *u8, wl: i64, vmax: i64) -> i64 {
  var i: i64=0; let vc: i64=vst[0]
  while i<vc {
      if vlen[i]==wl {
          var k: i64=0; var eq: i64=1
          while k<wl { if vbuf[voff[i]+k]!=w[k] { eq=0; k=wl } else { k=k+1 } }
          if eq==1 { return i }
      }
      i=i+1
  }
  if vc>=vmax { return 0-1 }
  let used: i64=vst[1]
  var k2: i64=0
  while k2<wl { vbuf[used+k2]=w[k2]; k2=k2+1 }
  voff[vc]=used; vlen[vc]=wl; vst[1]=used+wl; vst[0]=vc+1
  return vc
74}
75
76// tokenize file buffer -> doc d's token-id row (tok[d*MAXTOK..]); records ntok[d]. len>=4 stopword floor.
77func tokenize_doc(fb: *u8, flen: i64, d: i64, tok: *i64, MAXTOK: i64, ntok: *i64, vbuf: *u8, voff: *i64, vlen: *i64, vst: *i64, vmax: i64) -> i64 {
  let wtmp: *u8 = sys_mmap(128)
  var wl: i64=0; var nt: i64=0; var i: i64=0
  while i<=flen {
      var c: i64=32
      if i<flen { c = fb[i] as i64 }
      if c>=65 { if c<=90 { c=c+32 } }
      var isw: i64=0
      if c>=97 { if c<=122 { isw=1 } }
      if c>=48 { if c<=57 { isw=1 } }
      if isw==1 { if wl<127 { wtmp[wl]=(c as u8); wl=wl+1 } }
      else {
          if wl>=4 { if nt<MAXTOK {
              let id: i64 = vocab_id(vbuf,voff,vlen,vst,wtmp,wl,vmax)
              if id>=0 { tok[d*MAXTOK+nt]=id; nt=nt+1 }
          } }
          wl=0
      }
      i=i+1
  }
  ntok[d]=nt
  return 0
99}
100
101// windowed word-word co-occurrence for doc d into M[V x V] (symmetric).
102func build_cooc(tok: *i64, d: i64, MAXTOK: i64, nt: i64, M: *i64, V: i64, W: i64) -> i64 {
  var i: i64=0
  while i<nt {
      let a: i64=tok[d*MAXTOK+i]
      var j: i64=i+1
      var jend: i64=i+W; if jend>nt { jend=nt }
      while j<jend {
          let b: i64=tok[d*MAXTOK+j]
          if a!=b { M[a*V+b]=M[a*V+b]+1; M[b*V+a]=M[b*V+a]+1 }
          j=j+1
      }
      i=i+1
  }
  return 0
116}
117
118// doc vector = sum of UNIQUE member words' co-occ rows, then max-scaled to 1000 (cosine-invariant, bounds magnitude).
119func build_dvec(tok: *i64, d: i64, MAXTOK: i64, nt: i64, M: *i64, V: i64, dvec: *i64, seen: *i64) -> i64 {
  var c: i64=0; while c<V { dvec[d*V+c]=0; seen[c]=0; c=c+1 }
  var i: i64=0
  while i<nt {
      let m: i64=tok[d*MAXTOK+i]
      if seen[m]==0 { seen[m]=1; var cc: i64=0; while cc<V { dvec[d*V+cc]=dvec[d*V+cc]+M[m*V+cc]; cc=cc+1 } }
      i=i+1
  }
  var maxc: i64=0; c=0
  while c<V { if dvec[d*V+c]>maxc { maxc=dvec[d*V+c] } c=c+1 }
  if maxc>1000 { c=0; while c<V { dvec[d*V+c]=(dvec[d*V+c]*1000)/maxc; c=c+1 } }
  return 0
131}
132
133func main() -> i64 {
  let V: i64=600; let MAXTOK: i64=K_MAGIC_3000; let W: i64=6; let ND: i64=6
  let FB: i64=K_MAGIC_8388608
  lw("=== nx_lib_semantic -- R3b-SCALE: distributional embedder over the REAL banked corpus (6 dr_*.txt) ===\n")
137
  let filebuf: *u8 = sys_mmap(FB)
  let vbuf: *u8 = sys_mmap(K_MAGIC_65536)
  let voff: *i64 = sys_mmap(K_MAGIC_8192) as *i64
  let vlen: *i64 = sys_mmap(K_MAGIC_8192) as *i64
  let vst: *i64 = sys_mmap(64) as *i64; vst[0]=0; vst[1]=0
  let tok: *i64 = sys_mmap(K_MAGIC_200000) as *i64
  let ntok: *i64 = sys_mmap(64) as *i64
  let M: *i64 = sys_mmap(K_MAGIC_3200000) as *i64
  var z: i64=0; while z<V*V { M[z]=0; z=z+1 }
  let dvec: *i64 = sys_mmap(K_MAGIC_65536) as *i64
  let seen: *i64 = sys_mmap(K_MAGIC_8192) as *i64
  let lenbox: *i64 = sys_mmap(64) as *i64
150
  // read + tokenize + co-occurrence, doc by doc (M accumulates globally)
  read_file("knowledge/library/dr_deepresearcher_paper.txt" as *u8, filebuf, FB, lenbox); tokenize_doc(filebuf, lenbox[0], 0, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 0, MAXTOK, ntok[0], M, V, W)
  read_file("knowledge/library/dr_deepresearcher_repo.txt" as *u8, filebuf, FB, lenbox);  tokenize_doc(filebuf, lenbox[0], 1, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 1, MAXTOK, ntok[1], M, V, W)
  read_file("knowledge/library/dr_tongyi_deepresearch.txt" as *u8, filebuf, FB, lenbox);  tokenize_doc(filebuf, lenbox[0], 2, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 2, MAXTOK, ntok[2], M, V, W)
  read_file("knowledge/library/dr_deepsearcher_rag.txt" as *u8, filebuf, FB, lenbox);     tokenize_doc(filebuf, lenbox[0], 3, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 3, MAXTOK, ntok[3], M, V, W)
  read_file("knowledge/library/dr_dzhng_deepresearch.txt" as *u8, filebuf, FB, lenbox);   tokenize_doc(filebuf, lenbox[0], 4, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 4, MAXTOK, ntok[4], M, V, W)
  read_file("knowledge/library/dr_multimodal.txt" as *u8, filebuf, FB, lenbox);           tokenize_doc(filebuf, lenbox[0], 5, tok, MAXTOK, ntok, vbuf, voff, vlen, vst, V); build_cooc(tok, 5, MAXTOK, ntok[5], M, V, W)
158
  var d: i64=0; while d<ND { build_dvec(tok, d, MAXTOK, ntok[d], M, V, dvec, seen); d=d+1 }
160
  lw("corpus: vocab="); ln(vst[0]); lw(" content-words (cap "); ln(V); lw("); tokens/doc(cap "); ln(MAXTOK); lw("): ")
  d=0; while d<ND { ln(ntok[d]); lw(" "); d=d+1 } lw("\n")
163
  // labels
  let l0: *u8="deepresearcher_paper" as *u8; let l1: *u8="deepresearcher_repo" as *u8; let l2: *u8="tongyi" as *u8
  let l3: *u8="deepsearcher" as *u8; let l4: *u8="dzhng" as *u8; let l5: *u8="multimodal" as *u8
167
  let tot: *i64 = sys_mmap(64) as *i64; tot[0]=0; tot[1]=0
169
  // STRUCTURAL GATE 1: self-similarity == 1000 for every doc
  lw("\n[gate] self-similarity (must be 1000):\n")
  d=0
  while d<ND {
      let s: i64=cos2(dvec, d*V, dvec, d*V, V)
      lw("  doc "); ln(d); lw(" self="); ln(s)
      if s==1000 { lw("  PASS\n"); tot[0]=tot[0]+1 } else { lw("  FAIL\n") }
      tot[1]=tot[1]+1
      d=d+1
  }
180
  // STRUCTURAL GATE 2: self-retrieval -- query=doc d retrieves d as top-1
  lw("[gate] self-retrieval top-1:\n")
  d=0
  while d<ND {
      var best: i64=0-1; var bestc: i64=0-1
      var e: i64=0
      while e<ND { let c: i64=cos2(dvec,d*V,dvec,e*V,V); if c>bestc { bestc=c; best=e } e=e+1 }
      if best==d { tot[0]=tot[0]+1 } else { lw("  doc "); ln(d); lw(" FAIL top1="); ln(best); lw("\n") }
      tot[1]=tot[1]+1
      d=d+1
  }
  lw("  self-retrieval done\n")
193
  // STRUCTURAL GATE 3 (NEG-CONTROL): a zero query must score 0 against a real doc
  let zq: *i64 = sys_mmap(K_MAGIC_8192) as *i64; var zc: i64=0; while zc<V { zq[zc]=0; zc=zc+1 }
  lw("[gate] NEG-CONTROL zero-query vs doc0 -> ")
  let znc: i64=cos2(zq,0,dvec,0*V,V)
  if znc==0 { lw("0  PASS (empty query matches nothing)\n"); tot[0]=tot[0]+1 } else { lw("BROKEN\n") }
  tot[1]=tot[1]+1
200
  // MEASURED SEMANTIC FINDING (reported, NOT pre-asserted): nearest neighbour of each doc
  lw("\n[MEASURED] nearest semantic neighbour per doc (cosine permille, self excluded):\n")
  d=0
  while d<ND {
      var best: i64=0-1; var bestc: i64=0-1
      var e: i64=0
      while e<ND { if e!=d { let c: i64=cos2(dvec,d*V,dvec,e*V,V); if c>bestc { bestc=c; best=e } } e=e+1 }
      lw("  "); if d==0{lw(l0)} if d==1{lw(l1)} if d==2{lw(l2)} if d==3{lw(l3)} if d==4{lw(l4)} if d==5{lw(l5)}
      lw("  ->  ")
      if best==0{lw(l0)} if best==1{lw(l1)} if best==2{lw(l2)} if best==3{lw(l3)} if best==4{lw(l4)} if best==5{lw(l5)}
      lw("  (cos="); ln(bestc); lw(")\n")
      d=d+1
  }
214
  lw("\nLIB-SEMANTIC KAT "); ln(tot[0]); lw("/"); ln(tot[1]); lw("\n")
  if tot[0]==tot[1] {
      lw("GREEN -- distributional embedder LIVE on the real corpus: structural invariants hold; nearest-neighbour = the MEASURED semantic map above (RL/paper docs should cluster). This is the R3 dense retriever running on actual banked documents.\n")
      return 0
  }
  lw("RED -- structural invariant failed\n")
  return 1
222}
nx_lib_semantic.nx source

`nx_lib_semantic.nx` source
