code wiki / _hdl_build / nx_capsearch.nx
`nx_capsearch.nx` source

↩ module page · 431 lines · 19480 B
1// nx_capsearch.nx -- CAPABILITY SEARCH (2026-08-06). "What tool does X?" answered from the estate's OWN
2// registry, server-side, so an AGENT or a script can ask it. This is the organ the ecosystem was missing:
3// grounded capability retrieval EXISTED at /code/ask but was browser-only client-side JS plus a local
4// llama-server on the operator's laptop, so no agent and no headless caller could ever reach it, while
5// /api/search has scopes site|web|trusted and NO capability scope -- querying it for `nx_page_verify`,
6// the literal name of one of our own 869 tools, returns arxiv.org and kernel.org.
7//
8// THE RANKER IS NOT INVENTED. It is /code/ask's own `tooltop` (stem-hit on the name + word-boundary hit
9// on the title) plus three GENERIC improvements, each measured against a HELD-OUT gold set before being
10// kept: nx_common_tasks' 45 team-authored task->canonical-organ rows, which predate this organ. The task
11// labels were deliberately NOT folded in as aliases -- that would score ~1000 and mean nothing.
12//   known good (/code/ask tooltop) : hit@1 400  hit@5 640  hit@10 720  MRR 482  (permil)
13//   + call-grammar field + idf     : hit@1 400  hit@5 640  hit@10 720  MRR 507
14//   + morphological prefix match   : hit@1 520  hit@5 760  hit@10 800  MRR 620   <- shipped
15// +120 permil hit@1 (+30% rel), +138 MRR (+29% rel) over the known good.
16//
17// CORPUS = knowledge/tool_schemas.conf (name + title) joined to knowledge/tool_grammar.conf (the derived
18// argv contract from nx_toolgrammar) PLUS, since 2026-08-24, knowledge/status/libindex.tsv -- THE LIBRARY
19// SURFACE. This organ ranked REGISTERED TOOLS ONLY, and a library has no registration row by design, so
20// every library in the estate was invisible to the one instrument CLAUDE.md orders run before building
21// anything. Measured cost the day it was fixed: two FALSE ABSENT rows on one compare board (a LIVE
22// constraint solver and a complete Mei-2007 erosion pipeline that a sibling board had called
23// "catalogue-proven absent" for a week), six private isqrt copies written beside each other, and a
24// capsearch zero read as corroboration of absence. The lib corpus is emitted by nx_libindex (PG1) from
25// cl_is_lib, the same classifier the adoption ladder uses, so this organ and the board cannot disagree
26// about what a library is. Every result now carries its SURFACE ("tool" is MCP-callable, "lib" is
27// import-only) so a lib hit is never mistaken for something a caller can invoke.
28// Both corpora are read fresh per call, so a re-harvest is LIVE.
29// INTEGER-ONLY scoring (no floats anywhere in this estate): weights are scaled by CS_SCALE.
30// HONEST: a query whose tokens match nothing returns an EMPTY result list -- it never falls back to
31// "here are some tools anyway". No evidence, no answer. And the corpus cap is ANNOUNCED: rows beyond it
32// are COUNTED as dropped and corpus_complete flips to 0, never silently ranked as if they did not exist.
33//   nx_capsearch <query words...>  -> JSON {query, considered, tools, libs, dropped, corpus_complete,
34//                                           results:[{tool,surface,score,what,call}]}
35// license_tier: ORIGINAL   No hw writes (Rule 26).   expect_exit: 0
36import "nx_syscalls.nx"
37import "nx_store_seed_lib.nx"
38const CS_MAGIC_1024: i64 = 1024
39
40const CS_SCHEMAS: *u8 = "knowledge/tool_schemas.conf" as *u8
41const CS_GRAMMAR: *u8 = "knowledge/tool_grammar.conf" as *u8
42// The library surface: a SOVEREIGN ROW PLANE seeded by nx_libindex (never a .tsv file in the plane -- that
43// shape was retired 2026-08-13 and the first cut of this join repeated it for a day). Read through
44// sts_load_fit, which returns the whole plane or REFUSES; it never hands back a partial. ABSENT is legal
45// (the plane is seeded by nx_libindex on its own beat) and is reported as libs=0 corpus_complete=0 rather
46// than refused, so the tool surface still answers.
47const CS_LIBIDX: *u8 = "knowledge/store/libindex-" as *u8
48// sized: the schema conf is ~125KB today; refuse-on-full rather than rank a truncated corpus
49const CS_FCAP: i64 = 1 << 21
50// The row ceiling. Measured 2026-08-24: registry 1,309 tools; the library surface adds every lib in both
51// runtime roots. Raised from 1,600 (which the lib corpus alone would have bound) and made ANNOUNCING:
52// a bind is counted in `dropped` and forces corpus_complete=0. A silent cap was the defect class this
53// whole change exists to close.
54const CS_MAXTOOLS: i64 = 8192
55// declared: query tokens beyond this are ignored (declared, never silent)
56const CS_MAXTOK: i64 = 12
57// sized: a query token longer than this cannot match anything useful
58const CS_TOKCAP: i64 = 64
59// results returned
60const CS_TOPN: i64 = 8
61// idf-ish damping numerator, integer-scaled: w = CS_SCALE / (3 + df)
62const CS_SCALE: i64 = 1000000
63// morphological match: two words sharing this many leading chars are the same term (mine~miner)
64const CS_MORPH: i64 = 4
65// a token shorter than this is noise
66const CS_MINTOK: i64 = 3
67// surface ids per row: which buffer the row's offsets index
68const CS_SURF_TOOL: i64 = 0
69const CS_SURF_LIB: i64 = 1
70
71func csw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
72func csn(v: i64) -> i64 {
  let b: *u8 = sys_mmap(32)
  var x: i64 = v
  var i: i64 = 31
  if x == 0 { b[i] = 48 as u8; i = i - 1 }
  while x > 0 { b[i] = ((48 + (x - ((x / 10) * 10))) as u8); x = x / 10; i = i - 1 }
  sys_write(1, ((b as i64) + i + 1) as *u8, 31 - i)
  return 0
80}
81// JSON-escape n bytes of buf to stdout (quote, backslash, controls)
82func csj(buf: *u8, off: i64, n: i64) -> i64 {
  let o: *u8 = sys_mmap(8)
  var i: i64 = 0
  while i < n {
      let c: i64 = buf[off + i] as i64
      if c == 34 { csw("\\\"" as *u8) } else {
          if c == 92 { csw("\\\\" as *u8) } else {
              if c < 32 { csw(" " as *u8) } else { o[0] = c as u8; sys_write(1, o, 1) }
          }
      }
      i = i + 1
  }
  return 0
95}
96func cs_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
97// [a-z0-9_] after lowering
98func cs_isw(c: i64) -> i64 {
  let l: i64 = cs_lower(c)
  if l >= 97 { if l <= 122 { return 1 } }
  if l >= 48 { if l <= 57 { return 1 } }
  if l == 95 { return 1 }
  return 0
104}
105// idx-th TAB field of buf[ls,le) -> out2=(off,len); 1 present / 0 absent
106func cs_field(buf: *u8, ls: i64, le: i64, idx: i64, out2: *i64) -> i64 {
  var fi: i64 = 0
  var s: i64 = ls
  var i: i64 = ls
  while i <= le {
      var sep: i64 = 0
      if i == le { sep = 1 } else { if buf[i] == (9 as u8) { sep = 1 } }
      if sep == 1 {
          if fi == idx { out2[0] = s; out2[1] = i - s; return 1 }
          fi = fi + 1
          s = i + 1
      }
      i = i + 1
  }
  return 0
121}
122// do buf[a,alen) and tok share >= CS_MORPH leading chars (or are equal)? case-insensitive on buf side.
123func cs_morph(buf: *u8, a: i64, alen: i64, tok: *u8, tlen: i64) -> i64 {
  var n: i64 = alen
  if tlen < n { n = tlen }
  if alen == tlen {
      var m: i64 = 1
      var i: i64 = 0
      while i < alen { if cs_lower(buf[a+i] as i64) != (tok[i] as i64) { m = 0; i = alen } else { i = i + 1 } }
      if m == 1 { return 2 }
  }
  if n < CS_MORPH { return 0 }
  var j: i64 = 0
  while j < n { if cs_lower(buf[a+j] as i64) != (tok[j] as i64) { return 0 } j = j + 1 }
  return 1
136}
137// name hit: whole name == tok -> 3 ; an '_'-separated part == tok -> 2 ; a part shares a prefix -> 1
138func cs_namehit(buf: *u8, off: i64, len: i64, tok: *u8, tlen: i64) -> i64 {
  if cs_morph(buf, off, len, tok, tlen) == 2 { return 3 }
  var best: i64 = 0
  var s: i64 = off
  var i: i64 = off
  while i <= off + len {
      var sep: i64 = 0
      if i == off + len { sep = 1 } else { if buf[i] == (95 as u8) { sep = 1 } }
      if sep == 1 {
          if i > s {
              let m: i64 = cs_morph(buf, s, i - s, tok, tlen)
              if m == 2 { if best < 2 { best = 2 } }
              if m == 1 { if best < 1 { best = 1 } }
          }
          s = i + 1
      }
      i = i + 1
  }
  return best
157}
158// text hit: a whole word == tok -> 2 ; a word shares a prefix -> 1 ; else 0
159func cs_texthit(buf: *u8, off: i64, len: i64, tok: *u8, tlen: i64) -> i64 {
  var best: i64 = 0
  var s: i64 = 0 - 1
  var i: i64 = off
  while i <= off + len {
      var isw: i64 = 0
      if i < off + len { isw = cs_isw(buf[i] as i64) }
      if isw == 1 {
          if s < 0 { s = i }
      } else {
          if s >= 0 {
              let m: i64 = cs_morph(buf, s, i - s, tok, tlen)
              if m == 2 { if best < 2 { best = 2 } }
              if m == 1 { if best < 1 { best = 1 } }
              s = 0 - 1
          }
      }
      i = i + 1
  }
  return best
179}
180
181// The buffer a row's offsets index, by surface. One place, so no consumer picks the wrong one.
182func cs_buf(surf: i64, sb: *u8, lb: *u8) -> *u8 { if surf == CS_SURF_LIB { return lb } return sb }
183
184func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 {
      csw("{\"organ\":\"nx_capsearch\",\"refused\":\"no query -- usage: nx_capsearch <query words...>\"}\n" as *u8)
      sys_exit(2)
      return 2
  }
  // ---- corpus: the tool surface ----
  let szp: *i64 = sys_mmap(16) as *i64
  let sb: *u8 = sys_read_file(CS_SCHEMAS, szp)
  if (sb as i64) == 0 {
      csw("{\"organ\":\"nx_capsearch\",\"refused\":\"knowledge/tool_schemas.conf unreadable\"}\n" as *u8)
      sys_exit(2)
      return 2
  }
  let sn: i64 = szp[0]
  let gzp: *i64 = sys_mmap(16) as *i64
  var gb: *u8 = sys_read_file(CS_GRAMMAR, gzp)
  var gn: i64 = 0
  if (gb as i64) != 0 { gn = gzp[0] }
  // ---- corpus: the library surface, a row plane read whole-or-refused (absent is reported, never refused) ----
  let lzp: *i64 = sys_mmap(16) as *i64
  var lb: *u8 = sts_load_fit(CS_LIBIDX, lzp)
  var ln: i64 = 0
  var lib_present: i64 = 0
  if (lb as i64) != 0 { ln = lzp[0]; if ln > 0 { lib_present = 1 } }
  if lib_present == 0 { lb = sb }        // a valid buffer to index; no lib row will ever point into it
210
  let nm_o: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let nm_l: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let ti_o: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let ti_l: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let gr_o: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let gr_l: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  let surf: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  var nt: i64 = 0
  var ntools: i64 = 0
  var nlibs: i64 = 0
  var dropped: i64 = 0
  let f2: *i64 = sys_mmap(16) as *i64
  var ls: i64 = 0
  var i: i64 = 0
  while i <= sn {
      var eol: i64 = 0
      if i == sn { eol = 1 } else { if sb[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          if i > ls { if sb[ls] != (35 as u8) {
              if cs_field(sb, ls, i, 0, f2) == 1 {
                  let no: i64 = f2[0]
                  let nl: i64 = f2[1]
                  if nl > 0 {
                      if nt < CS_MAXTOOLS {
                          nm_o[nt] = no
                          nm_l[nt] = nl
                          ti_o[nt] = 0
                          ti_l[nt] = 0
                          if cs_field(sb, ls, i, 1, f2) == 1 { ti_o[nt] = f2[0]; ti_l[nt] = f2[1] }
                          gr_o[nt] = 0
                          gr_l[nt] = 0
                          surf[nt] = CS_SURF_TOOL
                          nt = nt + 1
                          ntools = ntools + 1
                      } else { dropped = dropped + 1 }
                  }
              }
          } }
          ls = i + 1
      }
      i = i + 1
  }
  // join the derived grammar by name (tool rows only -- a lib has no argv contract)
  if gn > 0 {
      var gls: i64 = 0
      var gi: i64 = 0
      while gi <= gn {
          var geol: i64 = 0
          if gi == gn { geol = 1 } else { if gb[gi] == (10 as u8) { geol = 1 } }
          if geol == 1 {
              if gi > gls { if gb[gls] != (35 as u8) {
                  if cs_field(gb, gls, gi, 0, f2) == 1 {
                      let gno: i64 = f2[0]
                      let gnl: i64 = f2[1]
                      if cs_field(gb, gls, gi, 1, f2) == 1 {
                          let gvo: i64 = f2[0]
                          let gvl: i64 = f2[1]
                          var k: i64 = 0
                          while k < nt {
                              if nm_l[k] == gnl {
                                  var m: i64 = 1
                                  var c: i64 = 0
                                  while c < gnl { if sb[nm_o[k]+c] != gb[gno+c] { m = 0; c = gnl } else { c = c + 1 } }
                                  if m == 1 { gr_o[k] = gvo; gr_l[k] = gvl; k = nt } else { k = k + 1 }
                              } else { k = k + 1 }
                          }
                      }
                  }
              } }
              gls = gi + 1
          }
          gi = gi + 1
      }
  }
  // library rows: name<TAB>path<TAB>title<TAB> sym1 sym2 ... -- the searchable text is everything from
  // the title onward, so a query for a SYMBOL finds the lib that declares it. The path field is skipped:
  // a directory name would match every query containing "runtime".
  if lib_present == 1 {
      var lls: i64 = 0
      var li: i64 = 0
      while li <= ln {
          var leol: i64 = 0
          if li == ln { leol = 1 } else { if lb[li] == (10 as u8) { leol = 1 } }
          if leol == 1 {
              if li > lls { if lb[lls] != (35 as u8) {
                  if cs_field(lb, lls, li, 0, f2) == 1 {
                      let lno: i64 = f2[0]
                      let lnl: i64 = f2[1]
                      if lnl > 0 {
                          if nt < CS_MAXTOOLS {
                              nm_o[nt] = lno
                              nm_l[nt] = lnl
                              ti_o[nt] = 0
                              ti_l[nt] = 0
                              if cs_field(lb, lls, li, 2, f2) == 1 { ti_o[nt] = f2[0]; ti_l[nt] = li - f2[0] }
                              gr_o[nt] = 0
                              gr_l[nt] = 0
                              surf[nt] = CS_SURF_LIB
                              nt = nt + 1
                              nlibs = nlibs + 1
                          } else { dropped = dropped + 1 }
                      }
                  }
              } }
              lls = li + 1
          }
          li = li + 1
      }
  }
  var corpus_complete: i64 = 1
  if dropped > 0 { corpus_complete = 0 }
  if lib_present == 0 { corpus_complete = 0 }
323
  // ---- tokenize the query from argv[1..] ----
  let tokbuf: *u8 = sys_mmap(CS_MAXTOK * CS_TOKCAP)
  let toklen: *i64 = sys_mmap(CS_MAXTOK * 8) as *i64
  var ntok: i64 = 0
  let qecho: *u8 = sys_mmap(CS_MAGIC_1024)
  var qn: i64 = 0
  var a: i64 = 1
  while a < argc {
      let s: *u8 = argv[a] as *u8
      var p: i64 = 0
      var cur: i64 = 0
      while s[p] != (0 as u8) {
          let c: i64 = s[p] as i64
          if qn < 1000 { qecho[qn] = c as u8; qn = qn + 1 }
          if cs_isw(c) == 1 {
              if ntok < CS_MAXTOK { if cur < CS_TOKCAP - 1 { tokbuf[ntok*CS_TOKCAP + cur] = cs_lower(c) as u8; cur = cur + 1 } }
          } else {
              if cur >= CS_MINTOK { if ntok < CS_MAXTOK { toklen[ntok] = cur; ntok = ntok + 1 } }
              cur = 0
          }
          p = p + 1
      }
      if cur >= CS_MINTOK { if ntok < CS_MAXTOK { toklen[ntok] = cur; ntok = ntok + 1 } }
      if a + 1 < argc { if qn < 1000 { qecho[qn] = 32 as u8; qn = qn + 1 } }
      a = a + 1
  }
  if ntok == 0 {
      csw("{\"organ\":\"nx_capsearch\",\"query\":\"" as *u8); csj(qecho, 0, qn)
      csw("\",\"considered\":" as *u8); csn(nt)
      csw(",\"tools\":" as *u8); csn(ntools); csw(",\"libs\":" as *u8); csn(nlibs)
      csw(",\"dropped\":" as *u8); csn(dropped); csw(",\"corpus_complete\":" as *u8); csn(corpus_complete)
      csw(",\"results\":[],\"note\":\"no usable query token (>=3 chars) -- not guessing\"}\n" as *u8)
      sys_exit(0)
      return 0
  }
359
  // ---- df per token, then score ----
  let score: *i64 = sys_mmap(CS_MAXTOOLS * 8) as *i64
  var z: i64 = 0
  while z < nt { score[z] = 0; z = z + 1 }
  var t: i64 = 0
  while t < ntok {
      let tk: *u8 = ((tokbuf as i64) + t * CS_TOKCAP) as *u8
      let tl: i64 = toklen[t]
      var df: i64 = 0
      var d: i64 = 0
      while d < nt {
          let rb: *u8 = cs_buf(surf[d], sb, lb)
          var hit: i64 = 0
          if cs_namehit(rb, nm_o[d], nm_l[d], tk, tl) > 0 { hit = 1 }
          if hit == 0 { if ti_l[d] > 0 { if cs_texthit(rb, ti_o[d], ti_l[d], tk, tl) > 0 { hit = 1 } } }
          if hit == 0 { if gr_l[d] > 0 { if cs_texthit(gb, gr_o[d], gr_l[d], tk, tl) > 0 { hit = 1 } } }
          if hit == 1 { df = df + 1 }
          d = d + 1
      }
      let w: i64 = CS_SCALE / (3 + df)
      var e: i64 = 0
      while e < nt {
          let eb: *u8 = cs_buf(surf[e], sb, lb)
          var s2: i64 = 0
          let nh: i64 = cs_namehit(eb, nm_o[e], nm_l[e], tk, tl)
          if nh > 0 { s2 = s2 + nh * 3 * w }
          if ti_l[e] > 0 { let th: i64 = cs_texthit(eb, ti_o[e], ti_l[e], tk, tl); if th > 0 { s2 = s2 + th * w } }
          if gr_l[e] > 0 { let gh: i64 = cs_texthit(gb, gr_o[e], gr_l[e], tk, tl); if gh > 0 { s2 = s2 + (gh * w) / 2 } }
          if s2 > 0 { score[e] = score[e] + s2 }
          e = e + 1
      }
      t = t + 1
  }
393
  // ---- top-N selection (no sort: N passes of argmax, N is 8) ----
  csw("{\"organ\":\"nx_capsearch\",\"query\":\"" as *u8); csj(qecho, 0, qn)
  csw("\",\"considered\":" as *u8); csn(nt)
  csw(",\"tools\":" as *u8); csn(ntools); csw(",\"libs\":" as *u8); csn(nlibs)
  csw(",\"dropped\":" as *u8); csn(dropped); csw(",\"corpus_complete\":" as *u8); csn(corpus_complete)
  csw(",\"ranker\":\"code-ask tooltop + call-grammar field + idf damping + morphological prefix (held-out gold: hit@1 520 permil vs 400 known-good); library surface joined 2026-08-24\",\"results\":[" as *u8)
  var emitted: i64 = 0
  var r: i64 = 0
  while r < CS_TOPN {
      var bi: i64 = 0 - 1
      var bs: i64 = 0
      var q: i64 = 0
      while q < nt {
          if score[q] > bs { bs = score[q]; bi = q }
          q = q + 1
      }
      if bi < 0 { r = CS_TOPN } else {
          let ob: *u8 = cs_buf(surf[bi], sb, lb)
          if emitted > 0 { csw("," as *u8) }
          csw("{\"tool\":\"" as *u8); csj(ob, nm_o[bi], nm_l[bi])
          csw("\",\"surface\":\"" as *u8)
          if surf[bi] == CS_SURF_LIB { csw("lib" as *u8) } else { csw("tool" as *u8) }
          csw("\",\"score\":" as *u8); csn(bs)
          csw(",\"what\":\"" as *u8)
          if ti_l[bi] > 0 { csj(ob, ti_o[bi], ti_l[bi]) }
          csw("\",\"call\":\"" as *u8)
          if gr_l[bi] > 0 { csj(gb, gr_o[bi], gr_l[bi]) }
          csw("\"}" as *u8)
          emitted = emitted + 1
          score[bi] = 0
          r = r + 1
      }
  }
  csw("],\"returned\":" as *u8); csn(emitted)
  csw("}\n" as *u8)
  sys_exit(0)
  return 0
431}
nx_capsearch.nx source

`nx_capsearch.nx` source
