code wiki / _hdl_build / nx_research_miner.nx
`nx_research_miner.nx` source

↩ module page · 278 lines · 13179 B
1// nx_research_miner.nx -- THE NISHI MINER: flags RICH VEINS in the researched corpus = the high-value seams
2// worth mining DEEPER to push capability to S-class exceed. Scores every unique content (vein) in the content
3// registry by TWO grounded signals: (1) CROSS-TOPIC multiplicity -- content that many research topics need is a
4// rich shared seam (reuses the registry nx_research_grow builds); (2) INNOVATION/OPTIMIZATION signal density --
5// occurrences of capability-pushing stems (optim/efficien/benchmark/novel/outperform/neural/lossless/...) in the
6// doc head, i.e. material about optimizing + innovating. richness = xtopic*XT_W + value. Ranks desc, writes
7// knowledge/index/rich_veins.log, and flags the top seams S-CLASS-EXCEED (where to mine next). 100% offline over
8// what we already fetched -- no network, deterministic, gateable. expect_exit: 0  license_tier: ORIGINAL
9import "nx_search_inverted_persist.nx"
10import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
11
12const RM_CAP:        i64 = 65536      // max unique veins
13const RM_SCAN:       i64 = 16384      // value-signal scan window (doc head)
14const RM_XT_WEIGHT:  i64 = 100        // cross-topic weight (a shared seam is a strong vein signal)
15const RM_TOPK:       i64 = 15         // how many top veins to flag S-CLASS-EXCEED
16
17func rm_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
19// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
20// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
21// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
22func rm_num(v: i64) -> i64 { nxi_out(v); return 0 }
23func rm_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
24func rm_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
25func rm_catbuf(dst: *u8, off: i64, src: *u8, n: i64) -> i64 { var i: i64=0; while i<n { dst[off+i]=src[i]; i=i+1 } return off+n }
26func rm_catnum(dst: *u8, off: i64, v: i64) -> i64 {
  if v == 0 { dst[off]=48 as u8; return off+1 }
  let tmp: *u8=sys_mmap(28); var m: i64=v; var k: i64=0
  while m>0 { tmp[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
  var i: i64=0; while i<k { dst[off+i]=tmp[k-1-i]; i=i+1 }
  return off+k
32}
33
34func rm_field_end(buf: *u8, i: i64, end: i64) -> i64 {
  var p: i64=i; var stop: i64=0
  while stop==0 {
      if p >= end { stop=1 }
      else { if buf[p]==(32 as u8) { stop=1 } else { if buf[p]==(10 as u8) { stop=1 } else { p=p+1 } } }
  }
  return p
41}
42func rm_parse_dec(buf: *u8, start: i64, fend: i64) -> i64 {
  var v: i64=0; var p: i64=start
  while p < fend { let c: i64=buf[p] as i64; if c>=48 { if c<=57 { v=v*10+(c-48) } } p=p+1 }
  return v
46}
47func rm_streq_rng(buf: *u8, a_off: i64, a_len: i64, b_off: i64, b_len: i64) -> i64 {
  if a_len != b_len { return 0 }
  var i: i64=0
  while i < a_len { if buf[a_off+i] != buf[b_off+i] { return 0 } i=i+1 }
  return 1
52}
53
54// case-insensitive substring occurrences of `needle` in first `hlen` bytes of `hay`
55func rm_count(hay: *u8, hlen: i64, needle: *u8) -> i64 {
  let nlen: i64 = rm_strlen(needle)
  if nlen == 0 { return 0 }
  var cnt: i64=0; var i: i64=0
  while i + nlen <= hlen {
      var j: i64=0; var ok: i64=1
      while j < nlen {
          var a: i64 = hay[i+j] as i64
          var b: i64 = needle[j] as i64
          if a >= 65 { if a <= 90 { a = a + 32 } }
          if b >= 65 { if b <= 90 { b = b + 32 } }
          if a != b { ok=0; j=nlen } else { j=j+1 }
      }
      if ok==1 { cnt=cnt+1 }
      i=i+1
  }
  return cnt
72}
73
74// innovation/optimization signal density over the doc head (capability-pushing stems)
75func rm_value(buf: *u8, len: i64) -> i64 {
  var n: i64 = len
  if n > RM_SCAN { n = RM_SCAN }
  var v: i64 = 0
  v = v + rm_count(buf, n, "optim" as *u8)        // optimize/optimization/optimal
  v = v + rm_count(buf, n, "efficien" as *u8)     // efficient/efficiency
  v = v + rm_count(buf, n, "benchmark" as *u8)
  v = v + rm_count(buf, n, "novel" as *u8)
  v = v + rm_count(buf, n, "outperform" as *u8)
  v = v + rm_count(buf, n, "breakthrough" as *u8)
  v = v + rm_count(buf, n, "accelerat" as *u8)
  v = v + rm_count(buf, n, "scalab" as *u8)
  v = v + rm_count(buf, n, "lossless" as *u8)
  v = v + rm_count(buf, n, "compress" as *u8)
  v = v + rm_count(buf, n, "quantiz" as *u8)
  v = v + rm_count(buf, n, "neural" as *u8)
  v = v + rm_count(buf, n, "embedding" as *u8)
  v = v + rm_count(buf, n, "gradient" as *u8)
  v = v + rm_count(buf, n, "throughput" as *u8)
  v = v + rm_count(buf, n, "latency" as *u8)
  v = v + rm_count(buf, n, "parallel" as *u8)
  v = v + rm_count(buf, n, "innovat" as *u8)       // innovate/innovation
  v = v + rm_count(buf, n, "state-of-the-art" as *u8)
  v = v + rm_count(buf, n, "performance" as *u8)
  return v
100}
101
102func main() -> i64 {
  rm_puts("=== nx_research_miner: flag RICH VEINS (high-value seams) for S-class-exceed mining ===\n" as *u8)
  let reg_path: *u8 = "knowledge/index/content_registry.txt" as *u8
  let rbox: *i64 = sys_mmap(16) as *i64
  let reg: *u8 = sys_read_file(reg_path, rbox)
  if reg == 0 as *u8 { rm_puts(" NO registry yet (run nx_research_grow first)\n" as *u8); sys_exit(1); return 1 }
  let rlen: i64 = rbox[0]
109
  // parse registry lines: <fp> <topic> <path>
  let lfp:  *i64 = sys_mmap(8*RM_CAP) as *i64
  let ltoff: *i64 = sys_mmap(8*RM_CAP) as *i64
  let ltlen: *i64 = sys_mmap(8*RM_CAP) as *i64
  let lpoff: *i64 = sys_mmap(8*RM_CAP) as *i64
  let lplen: *i64 = sys_mmap(8*RM_CAP) as *i64
  var nlines: i64 = 0
  var i: i64 = 0
  while i < rlen {
      let f1: i64 = rm_field_end(reg, i, rlen)
      let fp: i64 = rm_parse_dec(reg, i, f1)
      var j: i64 = f1
      if j < rlen { if reg[j]==(32 as u8) { j=j+1 } }
      let f2: i64 = rm_field_end(reg, j, rlen)
      let toff: i64 = j; let tl: i64 = f2 - j
      var p: i64 = f2
      if p < rlen { if reg[p]==(32 as u8) { p=p+1 } }
      let f3: i64 = rm_field_end(reg, p, rlen)
      let poff: i64 = p; let pl: i64 = f3 - p
      var e: i64 = f3
      var st: i64 = 0
      while st==0 { if e>=rlen { st=1 } else { if reg[e]==(10 as u8) { st=1 } else { e=e+1 } } }
      if e < rlen { e=e+1 }
      if nlines < RM_CAP { lfp[nlines]=fp; ltoff[nlines]=toff; ltlen[nlines]=tl; lpoff[nlines]=poff; lplen[nlines]=pl; nlines=nlines+1 }
      i = e
  }
136
  // unique veins: first occurrence of each fp = canonical; xtopic = distinct topics across its lines
  let ufp:   *i64 = sys_mmap(8*RM_CAP) as *i64
  let ucpoff: *i64 = sys_mmap(8*RM_CAP) as *i64   // canonical path off/len
  let ucplen: *i64 = sys_mmap(8*RM_CAP) as *i64
  let uxt:   *i64 = sys_mmap(8*RM_CAP) as *i64    // cross-topic count
  let uval:  *i64 = sys_mmap(8*RM_CAP) as *i64    // value-signal density
  let usc:   *i64 = sys_mmap(8*RM_CAP) as *i64    // richness score
  var nveins: i64 = 0
  var li: i64 = 0
  while li < nlines {
      // already counted?
      var seen: i64 = 0
      var u: i64 = 0
      while u < nveins { if ufp[u]==lfp[li] { seen=1 } u=u+1 }
      if seen == 0 {
          // distinct topics for this fp
          var xt: i64 = 0
          var lj: i64 = 0
          while lj < nlines {
              if lfp[lj]==lfp[li] {
                  // distinct vs earlier same-fp lines
                  var dup: i64 = 0
                  var lk: i64 = 0
                  while lk < lj {
                      if lfp[lk]==lfp[li] { if rm_streq_rng(reg, ltoff[lk], ltlen[lk], ltoff[lj], ltlen[lj])==1 { dup=1 } }
                      lk=lk+1
                  }
                  if dup==0 { xt=xt+1 }
              }
              lj=lj+1
          }
          if nveins < RM_CAP {
              ufp[nveins]=lfp[li]; ucpoff[nveins]=lpoff[li]; ucplen[nveins]=lplen[li]; uxt[nveins]=xt
              nveins=nveins+1
          }
      }
      li=li+1
  }
175
  // value-signal density per vein, INCREMENTAL: cache value by content fingerprint (vein_value_cache.txt) so
  // only NEW docs are read+scanned each run. The per-run cost used to be reading ALL docs (the cycle's miner
  // bottleneck at full scale); now a re-run with no new content does ZERO doc reads. Cache key = content fp
  // (same content => same value), so it stays correct as the corpus grows.
  let vc_fp:  *i64 = sys_mmap(8*RM_CAP) as *i64
  let vc_val: *i64 = sys_mmap(8*RM_CAP) as *i64
  var nvc: i64 = 0
  let vcbox: *i64 = sys_mmap(16) as *i64
  let vcbuf: *u8 = sys_read_file("knowledge/index/vein_value_cache.txt" as *u8, vcbox)
  if vcbuf != 0 as *u8 {
      let vclen: i64 = vcbox[0]
      var ci2: i64 = 0
      while ci2 < vclen {
          let f1: i64 = rm_field_end(vcbuf, ci2, vclen)
          let fpv: i64 = rm_parse_dec(vcbuf, ci2, f1)
          var j2: i64 = f1; if j2<vclen { if vcbuf[j2]==(32 as u8) { j2=j2+1 } }
          let f2: i64 = rm_field_end(vcbuf, j2, vclen)
          let vv: i64 = rm_parse_dec(vcbuf, j2, f2)
          var e2: i64 = f2; var st2: i64=0
          while st2==0 { if e2>=vclen { st2=1 } else { if vcbuf[e2]==(10 as u8) { st2=1 } else { e2=e2+1 } } }
          if e2<vclen { e2=e2+1 }
          if nvc < RM_CAP { vc_fp[nvc]=fpv; vc_val[nvc]=vv; nvc=nvc+1 }
          ci2 = e2
      }
  }
  let vc_fd: i64 = sys_openat_append("knowledge/index/vein_value_cache.txt" as *u8, 0x1a4)
  let pbuf: *u8 = sys_mmap(RM_CAP)
  let dbox: *i64 = sys_mmap(16) as *i64
  var vi: i64 = 0
  while vi < nveins {
      var val: i64 = 0 - 1
      var c: i64 = 0
      while c < nvc { if vc_fp[c]==ufp[vi] { val = vc_val[c] } c=c+1 }   // cached by content fp?
      if val < 0 {
          // miss: read canonical doc head, compute value, cache it (append + in-memory)
          var po: i64 = rm_catbuf(pbuf, 0, ((reg as i64)+ucpoff[vi]) as *u8, ucplen[vi]); pbuf[po]=0 as u8
          let buf: *u8 = sys_read_file(pbuf, dbox)
          val = 0
          if buf != 0 as *u8 { val = rm_value(buf, dbox[0]) }
          if vc_fd >= 0 {
              let cl: *u8 = sys_mmap(64); var cp2: i64 = 0
              cp2 = rm_catnum(cl, cp2, ufp[vi]); cl[cp2]=32 as u8; cp2=cp2+1
              cp2 = rm_catnum(cl, cp2, val); cl[cp2]=10 as u8; cp2=cp2+1
              sys_write(vc_fd, cl, cp2)
          }
          if nvc < RM_CAP { vc_fp[nvc]=ufp[vi]; vc_val[nvc]=val; nvc=nvc+1 }
      }
      uval[vi] = val
      usc[vi] = uxt[vi]*RM_XT_WEIGHT + val
      vi=vi+1
  }
  if vc_fd >= 0 { sys_close(vc_fd) }
228
  // selection-sort veins by score desc (carry fp/path/xt/val/score in lockstep)
  var a: i64 = 0
  while a < nveins {
      var best: i64 = a
      var b: i64 = a+1
      while b < nveins { if usc[b] > usc[best] { best=b } b=b+1 }
      if best != a {
          let ts: i64=usc[a]; usc[a]=usc[best]; usc[best]=ts
          let tx: i64=uxt[a]; uxt[a]=uxt[best]; uxt[best]=tx
          let tv: i64=uval[a]; uval[a]=uval[best]; uval[best]=tv
          let tf: i64=ufp[a]; ufp[a]=ufp[best]; ufp[best]=tf
          let to: i64=ucpoff[a]; ucpoff[a]=ucpoff[best]; ucpoff[best]=to
          let tl: i64=ucplen[a]; ucplen[a]=ucplen[best]; ucplen[best]=tl
      }
      a=a+1
  }
245
  // write rich_veins.log (all veins, desc), flag top-K S-CLASS-EXCEED
  let vfd: i64 = sys_openat_wr("knowledge/index/rich_veins.log" as *u8, 0x1a4)
  var flagged: i64 = 0
  var r: i64 = 0
  while r < nveins {
      let line: *u8 = sys_mmap(RM_CAP+256); var lp: i64 = 0
      if r < RM_TOPK { lp=rm_cat(line,lp,"S-CLASS-EXCEED-VEIN " as *u8); flagged=flagged+1 } else { lp=rm_cat(line,lp,"vein " as *u8) }
      lp=rm_cat(line,lp,"score=" as *u8); lp=rm_catnum(line,lp,usc[r])
      lp=rm_cat(line,lp," xtopic=" as *u8); lp=rm_catnum(line,lp,uxt[r])
      lp=rm_cat(line,lp," value=" as *u8); lp=rm_catnum(line,lp,uval[r])
      lp=rm_cat(line,lp," path=" as *u8); lp=rm_catbuf(line,lp,((reg as i64)+ucpoff[r]) as *u8,ucplen[r])
      line[lp]=10 as u8; lp=lp+1
      if vfd >= 0 { sys_write(vfd, line, lp) }
      r=r+1
  }
  if vfd >= 0 { sys_close(vfd) }
262
  rm_puts(" veins="); rm_num(nveins); rm_puts(" flagged_S-class="); rm_num(flagged)
  rm_puts(" -> knowledge/index/rich_veins.log\n" as *u8)
  rm_puts(" TOP VEINS (mine these deeper):\n" as *u8)
  var t: i64 = 0
  while t < 8 {
      if t < nveins {
          rm_puts("  #"); rm_num(t+1); rm_puts(" score="); rm_num(usc[t])
          rm_puts(" xtopic="); rm_num(uxt[t]); rm_puts(" value="); rm_num(uval[t])
          rm_puts(" "); sys_write(1, ((reg as i64)+ucpoff[t]) as *u8, ucplen[t]); rm_puts("\n" as *u8)
      }
      t=t+1
  }
  if nveins <= 0 { rm_puts(" NO veins\n" as *u8); sys_exit(1); return 1 }
  rm_puts(" MINER-OK\n" as *u8)
  sys_exit(0); return 0
278}
nx_research_miner.nx source

`nx_research_miner.nx` source
