code wiki / _hdl_build / nx_search_race.nx
`nx_search_race.nx` source

↩ module page · 255 lines · 10966 B
1// nx_search_race.nx -- IM6b: THE BEAT-FTS5 SEARCH RACE (the full-text-search
2// EXCEED measurement on the REAL unified team-log corpus).
3//
4// Two lanes, SAME input (pm_plan_durable.log + cap_registry_durable.log,
5// unique non-empty lines), SAME 9-query set (6 terms + 3 ANDs) x 200
6// iterations, each lane self-timing ONLY its engine work:
7//   SOV lane: /tmp/nx_race_search_worker.sov.elf (chunked fsync'd commits
8//             w/ .keys+.terms/.post; handle + postings queries)
9//   FTS5 lane: python3 sqlite3 FTS5 (the incumbent full-text engine; run as
10//             an EXTERNAL COMPETITOR PROCESS, harness .py written to /tmp at
11//             runtime -- the repo stays pure NishiLang)
12//
13// Verdict law: numbers are MEASURED and logged win or lose; every per-query
14// COUNT must match between lanes or the race is INVALID (correctness before
15// speed).  Exit 0 only on SOV-WINS-BOTH.  Durable: infomgmt_race.log.
16// license_tier: ORIGINAL
17
18import "nx_syscalls.nx"
19import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
20const K_MAGIC_8192: i64 = 8192
21const K_MAGIC_8000: i64 = 8000
22
23func sx_p(s: *u8) -> i64 {
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  sys_write(1, s, n)
  return 0
28}
29
30func sx_fp(fd: i64, s: *u8) -> i64 {
  var n: i64 = 0
  while s[n] != (0 as u8) { n = n + 1 }
  sys_write(fd, s, n)
  return 0
35}
36
37// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
38// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
39// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
40// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
41func sx_fn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
42
43func sx_cat(dst: *u8, off: i64, s: *u8) -> i64 {
  var i: i64 = 0
  while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 }
  return off + i
47}
48
49func sx_catn(dst: *u8, off: i64, v: i64) -> i64 {
  var m: i64 = v
  var o: i64 = off
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { dst[o + i] = t[k - 1 - i]; i = i + 1 }
  return o + k
59}
60
61func sx_run(path: *u8, a1: *u8, a2: *u8, a3: *u8, outpath: *u8) -> i64 {
  let pid: i64 = sys_fork()
  if pid == 0 {
      let ofd: i64 = sys_openat_wr(outpath, 0x1a4)
      if ofd >= 0 { sys_dup3(ofd, 1, 0) }
      let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
      if dn >= 0 { sys_dup3(dn, 2, 0) }
      let argv: *i64 = sys_mmap(8 * 6) as *i64
      argv[0] = path as i64
      var na: i64 = 1
      if (a1 as i64) != 0 { argv[na] = a1 as i64; na = na + 1 }
      if (a2 as i64) != 0 { argv[na] = a2 as i64; na = na + 1 }
      if (a3 as i64) != 0 { argv[na] = a3 as i64; na = na + 1 }
      argv[na] = 0
      let envp: *i64 = sys_mmap(16) as *i64
      envp[0] = 0
      sys_execve(path, argv, envp)
      sys_exit(127)
  }
  let st: *i64 = sys_mmap(16) as *i64
  sys_wait4(pid, st, 0)
  return st[0]
83}
84
85func sx_num_after(path: *u8, pat: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let b: *u8 = sys_mmap(K_MAGIC_8192)
  let n: i64 = sys_read(fd, b, K_MAGIC_8000)
  sys_close(fd)
  if n <= 0 { return 0 - 1 }
  var pl: i64 = 0
  while pat[pl] != (0 as u8) { pl = pl + 1 }
  var i: i64 = 0
  while i + pl <= n {
      var hit: i64 = 1
      var k: i64 = 0
      while k < pl {
          if hit == 1 { if b[i + k] != pat[k] { hit = 0 } }
          k = k + 1
      }
      if hit == 1 {
          var v: i64 = 0
          var p: i64 = i + pl
          var any: i64 = 0
          var go: i64 = 1
          while go == 1 {
              if p >= n { go = 0 }
              if go == 1 {
                  let c: i64 = b[p]
                  if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; p = p + 1 } else { go = 0 } } else { go = 0 }
              }
          }
          if any == 1 { return v }
          return 0 - 1
      }
      i = i + 1
  }
  return 0 - 1
120}
121
122// the FTS5 competitor harness (.py) -- /tmp only, never the repo
123func sx_write_py(path: *u8) -> i64 {
  let fd: i64 = sys_openat_wr(path, 0x1a4)
  if fd < 0 { return 0 - 1 }
  sx_fp(fd, "import sqlite3, time, sys\n" as *u8)
  sx_fp(fd, "mode=sys.argv[1]; db=sys.argv[2]\n" as *u8)
  sx_fp(fd, "docs=[]; seen=set()\n" as *u8)
  sx_fp(fd, "for f in ('knowledge/status/pm_plan_durable.log','knowledge/status/cap_registry_durable.log'):\n" as *u8)
  sx_fp(fd, "    for L in open(f, encoding='utf-8', errors='replace'):\n" as *u8)
  sx_fp(fd, "        L=L.rstrip('\\n')\n" as *u8)
  sx_fp(fd, "        if L=='' or L in seen: continue\n" as *u8)
  sx_fp(fd, "        seen.add(L); docs.append(L)\n" as *u8)
  sx_fp(fd, "QT=['infomgmt','compaction','postings','sovereign','capreg','gate']\n" as *u8)
  sx_fp(fd, "QA=[('infomgmt','postings'),('sovereign','gate'),('compaction','archive')]\n" as *u8)
  sx_fp(fd, "if mode=='write':\n" as *u8)
  sx_fp(fd, "    t0=time.monotonic_ns()\n" as *u8)
  sx_fp(fd, "    c=sqlite3.connect(db)\n" as *u8)
  sx_fp(fd, "    c.execute('CREATE VIRTUAL TABLE docs USING fts5(line)')\n" as *u8)
  sx_fp(fd, "    cur=c.cursor()\n" as *u8)
  sx_fp(fd, "    for d in docs: cur.execute('INSERT INTO docs VALUES(?)',(d,))\n" as *u8)
  sx_fp(fd, "    c.commit(); c.close()\n" as *u8)
  sx_fp(fd, "    t1=time.monotonic_ns()\n" as *u8)
  sx_fp(fd, "    print('SQLW us=%d count=%d' % ((t1-t0)//1000, len(docs)))\n" as *u8)
  sx_fp(fd, "else:\n" as *u8)
  sx_fp(fd, "    t0=time.monotonic_ns()\n" as *u8)
  sx_fp(fd, "    c=sqlite3.connect(db)\n" as *u8)
  sx_fp(fd, "    qc=[0]*9\n" as *u8)
  sx_fp(fd, "    for it in range(200):\n" as *u8)
  sx_fp(fd, "        for i,t in enumerate(QT):\n" as *u8)
  sx_fp(fd, "            qc[i]=c.execute('SELECT count(*) FROM docs WHERE docs MATCH ?',(t,)).fetchone()[0]\n" as *u8)
  sx_fp(fd, "        for i,(a,b) in enumerate(QA):\n" as *u8)
  sx_fp(fd, "            qc[6+i]=c.execute('SELECT count(*) FROM docs WHERE docs MATCH ?',(a+' AND '+b,)).fetchone()[0]\n" as *u8)
  sx_fp(fd, "    c.close()\n" as *u8)
  sx_fp(fd, "    t1=time.monotonic_ns()\n" as *u8)
  sx_fp(fd, "    print('SQLQ us=%d %s count=%d' % ((t1-t0)//1000, ' '.join('q%d=%d'%(i,v) for i,v in enumerate(qc)), len(docs)))\n" as *u8)
  sys_close(fd)
  return 0
159}
160
161func main() -> i64 {
  sx_p("=== IM6b SEARCH RACE: sovereign postings vs sqlite FTS5 (team-log corpus, 9 queries x 200) ===\n" as *u8)
  let wpath: *u8 = "/tmp/nx_race_search_worker.sov.elf" as *u8
  let probe: i64 = sys_openat_rd(wpath)
  if probe >= 0 { sys_close(probe) } else {
      sx_run("_offc/nx_sov_build_run.elf" as *u8, "nx_race_search_worker" as *u8, 0 as *u8, 0 as *u8, "/tmp/srace_build.out" as *u8)
  }
168
  let id: i64 = sys_now_us()
  let sovpfx: *u8 = sys_mmap(128)
  var o: i64 = 0
  o = sx_cat(sovpfx, o, "/tmp/srsov" as *u8)
  o = sx_catn(sovpfx, o, id)
  o = sx_cat(sovpfx, o, "-" as *u8)
  sovpfx[o] = 0 as u8
  let dbp: *u8 = sys_mmap(128)
  o = 0
  o = sx_cat(dbp, o, "/tmp/srsql" as *u8)
  o = sx_catn(dbp, o, id)
  o = sx_cat(dbp, o, ".db" as *u8)
  dbp[o] = 0 as u8
182
  if sx_write_py("/tmp/srace_fts5.py" as *u8) != 0 { sx_p(" py harness write FAILED\n" as *u8); return 1 }
184
  sx_run(wpath, "write" as *u8, sovpfx, 0 as *u8, "/tmp/srace_sw.out" as *u8)
  sx_run(wpath, "query" as *u8, sovpfx, 0 as *u8, "/tmp/srace_sq.out" as *u8)
  let py: *u8 = "/usr/bin/python3" as *u8
  sx_run(py, "/tmp/srace_fts5.py" as *u8, "write" as *u8, dbp, "/tmp/srace_qw.out" as *u8)
  sx_run(py, "/tmp/srace_fts5.py" as *u8, "query" as *u8, dbp, "/tmp/srace_qq.out" as *u8)
190
  let sw: i64 = sx_num_after("/tmp/srace_sw.out" as *u8, "SOVW us=" as *u8)
  let swc: i64 = sx_num_after("/tmp/srace_sw.out" as *u8, "count=" as *u8)
  let sq: i64 = sx_num_after("/tmp/srace_sq.out" as *u8, "SOVQ us=" as *u8)
  let sqc: i64 = sx_num_after("/tmp/srace_sq.out" as *u8, "count=" as *u8)
  let qw: i64 = sx_num_after("/tmp/srace_qw.out" as *u8, "SQLW us=" as *u8)
  let qwc: i64 = sx_num_after("/tmp/srace_qw.out" as *u8, "count=" as *u8)
  let qq: i64 = sx_num_after("/tmp/srace_qq.out" as *u8, "SQLQ us=" as *u8)
  let qqc: i64 = sx_num_after("/tmp/srace_qq.out" as *u8, "count=" as *u8)
199
  // per-query correctness cross-check (q0..q8 must match pairwise)
  var counts_ok: i64 = 0
  if swc > 0 { if swc == sqc { if swc == qwc { if swc == qqc { counts_ok = 1 } } } }
  let pat: *u8 = sys_mmap(16)
  var qi: i64 = 0
  while qi < 9 {
      var po: i64 = 0
      po = sx_cat(pat, po, "q" as *u8)
      po = sx_catn(pat, po, qi)
      po = sx_cat(pat, po, "=" as *u8)
      pat[po] = 0 as u8
      let a: i64 = sx_num_after("/tmp/srace_sq.out" as *u8, pat)
      let b: i64 = sx_num_after("/tmp/srace_qq.out" as *u8, pat)
      if a < 0 { counts_ok = 0 }
      if a != b { counts_ok = 0 }
      qi = qi + 1
  }
  var win_w: i64 = 0
  if sw >= 0 { if qw >= 0 { if sw <= qw { win_w = 1 } } }
  var win_q: i64 = 0
  if sq >= 0 { if qq >= 0 { if sq <= qq { win_q = 1 } } }
221
  let lfd: i64 = sys_openat_append("knowledge/status/infomgmt_race.log" as *u8, 0x1a4)
  var fdi: i64 = 0
  while fdi < 2 {
      var fd: i64 = 1
      if fdi == 1 { fd = lfd }
      if fd > 0 {
          sx_fp(fd, "RACE epoch=" as *u8)
          sx_fn(fd, sys_now_realtime_sec())
          sx_fp(fd, " workload=logsearch docs=" as *u8)
          sx_fn(fd, swc)
          sx_fp(fd, " sov_index_us=" as *u8)
          sx_fn(fd, sw)
          sx_fp(fd, " fts5_index_us=" as *u8)
          sx_fn(fd, qw)
          sx_fp(fd, " sov_query_us=" as *u8)
          sx_fn(fd, sq)
          sx_fp(fd, " fts5_query_us=" as *u8)
          sx_fn(fd, qq)
          sx_fp(fd, " counts_ok=" as *u8)
          sx_fn(fd, counts_ok)
          if counts_ok == 1 {
              if win_w == 1 { if win_q == 1 { sx_fp(fd, " verdict=SOV-WINS-BOTH\n" as *u8) } }
              if win_w == 1 { if win_q == 0 { sx_fp(fd, " verdict=MIXED-QUERY-LOSS (optimization rung: bitpack/FST dict + segment merge)\n" as *u8) } }
              if win_w == 0 { if win_q == 1 { sx_fp(fd, " verdict=MIXED-INDEX-LOSS (optimization rung: single-segment bulk build + batch fsync)\n" as *u8) } }
              if win_w == 0 { if win_q == 0 { sx_fp(fd, " verdict=SOV-LOSES-BOTH (file defect: slow is a bug)\n" as *u8) } }
          }
          if counts_ok != 1 { sx_fp(fd, " verdict=INVALID (counts disagree -- correctness before speed)\n" as *u8) }
      }
      fdi = fdi + 1
  }
  if lfd > 0 { sys_close(lfd) }
  if counts_ok == 1 { if win_w == 1 { if win_q == 1 { return 0 } } }
  return 1
255}
nx_search_race.nx source

`nx_search_race.nx` source
