`nx_learn_rank.nx` source

↩ module page · 424 lines · 19972 B
1// nx_learn_rank.nx -- the learnings-rail ranker, COMPOSED not rebuilt.
2//
3// module: nishi-core.learn.rank
4// depends: fx.nx, syscalls.nx, nx_bm25.nx, nx_rrf.nx, nx_rank_fused.nx
5// capability: CORE_COMPUTE
6//
7// WHY THIS EXISTS AND WHY IT IS SMALL: nishi-ops/learn/nx_rank.ps1 implements BM25 + RRF + a severity
8// blend in PowerShell -- judgement in shell, which the standing order forbids. The obvious remedy is
9// "write a ranking organ", and that is WRONG: measured 2026-08-16 (corpus_complete=1) the estate
10// already ships nx_bm25 (+_lib,+_gate), nx_rrf, nx_rank_fused (+test), nx_intlog (+gate) and
11// nx_bm25_naive as a brute-force oracle. A third BM25 would be the duplicate-ruler defect -- the very
12// one nx_rank.ps1's OWN header cites (nx_ppmi_lib's two silently-diverged copies of one routine) as
13// its reason for existing. So this file contains NO ranking mathematics. It is a DRIVER: read a
14// corpus, call the incumbents, apply the declared blend, print an order.
15//
16// CONTRACT (pre-declared BEFORE the first run, so the result cannot rationalise the design):
17//   the top-N ORDER emitted here must equal nx_rank.ps1's top-N order on the same fixture.
18//   SCORES may differ -- Q16.16 fixed point here vs IEEE doubles there -- but ORDER is the contract.
19//
20// INPUT  corpus file, one doc per line:  <rank>\t<slug>\t<text>
21//        rank is the severity 1..star_max; a malformed line is SKIPPED AND COUNTED, never guessed.
22// OUTPUT one line per hit:  <rank>\t<slug>\t<q16.16 score>   then a DECLARED envelope line.
23//
24//   nx_learn_rank <corpus> <topN> <conf> <query words...>
25// exit: 0 ranked (possibly zero hits) | 2 usage | 4 corpus unreadable
26// license_tier: ORIGINAL   No hw writes (Rule 26).
27
28// ONE import, deliberately. nx_rank_fused already carries fx, syscalls, nx_source_tier, nx_bm25 and
29// nx_rrf, so naming any of them again here is not belt-and-braces -- it risks a second copy of every
30// symbol they define. ★AN IMPORT LIST THAT RESTATES A DEPENDENCY'S OWN DEPENDENCIES IS A DUPLICATE,
31// NOT A SAFEGUARD -- import the capstone and let it carry its graph.
32// ⚠PROVENANCE OF THAT RULE, stated because the evidence is NOT reproducible on this tree: it was
33// measured 2026-08-16 on a LAPTOP-LOCAL FORK of nx_bm25.nx (a Q16.16 variant that also defined
34// nx_bm25_tf, which nx_search_inverted.nx already defines) where the duplicate import stopped the
35// build outright. THIS tree's nx_bm25.nx is a different implementation -- milli/micro integer units,
36// no nx_bm25_tf -- so that exact error cannot occur here. The rule stands; the anecdote is local.
37// ★A COMMENT THAT REPORTS A CONDITION FROM ANOTHER TREE AS IF IT WERE THIS ONE'S SENDS THE NEXT
38// READER HUNTING A DEFECT THAT IS NOT THERE.
39import "nx_rank_fused.nx"
40
41// Rule 11: every weight is DATA. These are FALLBACKS used only when the conf omits a key, and they
42// mirror learn_markers.conf so the two cannot disagree silently. star_max is the severity ceiling and
43// has two consumers (the miner's rank scale and this blend) -- see that file's star_max row.
44const LR_DEF_BLEND_REL_NUM: i64 = 100    // blend_relevance 1.00, held as hundredths so the conf's
45const LR_DEF_BLEND_SEV_NUM: i64 = 35     // blend_severity  0.35, decimal text parses without floats
46const LR_DEF_STAR_MAX: i64 = 6
47// Prefix-stem length for the SECOND retrieval view, mirroring learn_markers.conf's stem_prefix_len.
48// This is what lets a query for "verification" reach a law that says "verify": exact BM25 structurally
49// cannot, and nx_rank.ps1's own suite proves it (T8 asserts the miss, T9 asserts fusion recovers it).
50// A single-view organ would silently REGRESS that recall, which is why order-equality in exact mode was
51// necessary but not sufficient to call this a replacement.
52const LR_DEF_STEM_PREFIX: i64 = 6
53const LR_HUNDRED: i64 = 100
54const LR_TAB: i64 = 9
55const LR_NL: i64 = 10
56const LR_MAXDOCS: i64 = 20000
57const LR_MAXTERMS: i64 = 64
58
59func lr_p(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
60func lr_e(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 }
61func lr_n(fd: i64, v: i64) -> i64 {
  let t: *u8 = sys_mmap(32)
  var m: i64 = v
  if m < 0 { sys_write(fd, "-" as *u8, 1); m = 0 - m }
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let b: *u8 = sys_mmap(32)
  var i: i64 = 0
  while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(fd, b, k)
  return 0
73}
74func lr_byte(fd: i64, c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c as u8; sys_write(fd, b, 1); return 0 }
75func lr_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
76
77// decimal text -> integer hundredths, so "0.35" becomes 35 with no float anywhere.
78// Accepts "N", "N.D", "N.DD". Trailing digits beyond two are TRUNCATED, not rounded, and that choice
79// is stated rather than left for a reader to infer.
80func lr_hundredths(s: *u8, n: i64) -> i64 {
  var whole: i64 = 0
  var frac: i64 = 0
  var digits: i64 = 0
  var seen_dot: i64 = 0
  var i: i64 = 0
  while i < n {
      let c: i64 = s[i] as i64
      if c == 46 { seen_dot = 1 } else {
          if c >= 48 { if c <= 57 {
              if seen_dot == 0 { whole = whole * 10 + (c - 48) } else {
                  if digits < 2 { frac = frac * 10 + (c - 48); digits = digits + 1 }
              }
          } }
      }
      i = i + 1
  }
  while digits < 2 { frac = frac * 10; digits = digits + 1 }
  return whole * LR_HUNDRED + frac
99}
100
101// one `key=value` lookup over a conf buffer. Comments and blanks skipped; FIRST '=' splits, matching
102// learn_markers.conf's stated format exactly rather than inventing a second grammar.
103func lr_conf_get(buf: *u8, n: i64, key: *u8, out: *u8, outcap: i64) -> i64 {
  let kl: i64 = lr_slen(key)
  var i: i64 = 0
  while i < n {
      // same flag discipline as the corpus scanner below -- no sentinel in the cursor
      var le: i64 = i
      var eol: i64 = 0
      while eol == 0 {
          if le >= n { eol = 1 } else {
              if buf[le] == (LR_NL as u8) { eol = 1 } else { le = le + 1 }
          }
      }
      var s: i64 = i
      var lead: i64 = 0
      while lead == 0 {
          if s >= le { lead = 1 } else {
              if buf[s] == (32 as u8) { s = s + 1 } else { lead = 1 }
          }
      }
      // s == le means a blank/whitespace-only line: there is no byte to classify, and reading
      // buf[s] there would inspect the newline (or one past the buffer on the final line).
      if s < le { if buf[s] != (35 as u8) {
          var eq: i64 = 0 - 1
          var j: i64 = s
          while j < le { if buf[j] == (61 as u8) { if eq < 0 { eq = j } } j = j + 1 }
          if eq > s {
              var m: i64 = 1
              if eq - s != kl { m = 0 }
              if m == 1 {
                  var t: i64 = 0
                  while t < kl { if buf[s + t] != key[t] { m = 0 } t = t + 1 }
              }
              if m == 1 {
                  var vs: i64 = eq + 1
                  var vn: i64 = le - vs
                  if vn >= outcap { vn = outcap - 1 }
                  var w: i64 = 0
                  while w < vn { out[w] = buf[vs + w]; w = w + 1 }
                  out[vn] = 0 as u8
                  return vn
              }
          }
      } }
      i = le + 1
  }
  return 0 - 1
149}
150
151// Is c a token byte? Must agree with what nx_bm25 counts as a token, or the stemmed view would be
152// tokenised on one boundary and scored on another. Alphanumeric only, matching the estate's tokenizers.
153func lr_is_tok(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return 1 } }
  if c >= 65 { if c <= 90 { return 1 } }
  if c >= 97 { if c <= 122 { return 1 } }
  return 0
158}
159
160// Write a PREFIX-STEMMED copy of src[0..n) into dst: every token truncated to `plen` bytes, separators
161// collapsed to a single space. Returns bytes written.
162// The output is never longer than the input (tokens only shrink), so the caller sizes dst from the
163// SOURCE and there is no capacity to guess. Truncation is the whole point here, not a limitation: it
164// is the morphological view.
165func lr_stem_into(dst: *u8, src: *u8, n: i64, plen: i64) -> i64 {
  var o: i64 = 0
  var i: i64 = 0
  while i < n {
      if lr_is_tok(src[i] as i64) == 0 {
          if o > 0 { if dst[o - 1] != (32 as u8) { dst[o] = 32 as u8; o = o + 1 } }
          i = i + 1
      } else {
          // SEPARATE FLAG, NOT A SENTINEL IN THE CURSOR. The first cut of this loop ended a token by
          // assigning `i = n + 1`, which does not end the token -- it ends the WHOLE SCAN, so only the
          // first word of every document was ever stemmed and the second view matched nothing. That
          // is the identical cursor-clobber defect this same file already hit in its corpus scanner
          // (docs=1 on a 5-line fixture), written a second time within the hour.
          // A LAW YOU JUST APPLIED IN ONE LOOP DOES NOT TRANSFER ITSELF TO THE NEXT LOOP YOU WRITE.
          // Caught only because the T9 fixture was re-run with the exact-matching term REMOVED: with
          // "reversibility firmware" it passed on the strength of "firmware" alone.
          var k: i64 = 0
          var tokend: i64 = 0
          while tokend == 0 {
              if i >= n { tokend = 1 } else {
                  if lr_is_tok(src[i] as i64) == 1 {
                      if k < plen { dst[o] = src[i]; o = o + 1; k = k + 1 }
                      i = i + 1
                  } else { tokend = 1 }
              }
          }
          if o > 0 { dst[o] = 32 as u8; o = o + 1 }
      }
  }
  return o
195}
196
197func main(argc: i64, argv: *i64) -> i64 {
  if argc < 5 {
      lr_e("usage: nx_learn_rank <corpus> <topN> <conf> <query words...>\n" as *u8)
      sys_exit(2)
      return 2
  }
  let corpus_path: *u8 = argv[1] as *u8
  let topn: i64 = lr_hundredths(argv[2] as *u8, lr_slen(argv[2] as *u8)) / LR_HUNDRED
  let conf_path: *u8 = argv[3] as *u8
206
  // --- conf: every weight is data (rule 11). Missing keys fall back to the constants above.
  var blend_rel: i64 = LR_DEF_BLEND_REL_NUM
  var blend_sev: i64 = LR_DEF_BLEND_SEV_NUM
  var star_max: i64 = LR_DEF_STAR_MAX
  var stem_prefix: i64 = LR_DEF_STEM_PREFIX
  let cszp: *i64 = sys_mmap(16) as *i64
  let cbuf: *u8 = sys_read_file(conf_path, cszp)
  if (cbuf as i64) != 0 {
      let cn: i64 = cszp[0]
      let vb: *u8 = sys_mmap(64)
      if lr_conf_get(cbuf, cn, "blend_relevance" as *u8, vb, 64) >= 0 { blend_rel = lr_hundredths(vb, lr_slen(vb)) }
      if lr_conf_get(cbuf, cn, "blend_severity" as *u8, vb, 64) >= 0 { blend_sev = lr_hundredths(vb, lr_slen(vb)) }
      if lr_conf_get(cbuf, cn, "stem_prefix_len" as *u8, vb, 64) >= 0 {
          let sp: i64 = lr_hundredths(vb, lr_slen(vb)) / LR_HUNDRED
          if sp > 0 { stem_prefix = sp }
      }
      if lr_conf_get(cbuf, cn, "star_max" as *u8, vb, 64) >= 0 {
          let sm: i64 = lr_hundredths(vb, lr_slen(vb)) / LR_HUNDRED
          // a zero or negative scale is a broken conf, not a tuning choice: refuse it and keep the
          // fallback rather than propagate a divide-by-zero into every score.
          if sm > 0 { star_max = sm }
      }
  }
230
  // --- corpus: <rank>\t<slug>\t<text> per line. Buffer sized from the FILE by sys_read_file, so
  // there is no read ceiling to guess and no short read to detect.
  let szp: *i64 = sys_mmap(16) as *i64
  let buf: *u8 = sys_read_file(corpus_path, szp)
  if (buf as i64) == 0 {
      lr_e("CORPUS UNREADABLE: " as *u8); lr_e(corpus_path); lr_e("\n" as *u8)
      sys_exit(4)
      return 4
  }
  let n: i64 = szp[0]
241
  let texts: **u8 = sys_mmap(LR_MAXDOCS * 8) as **u8
  let tlens: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  let ranks: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  let slugs: **u8 = sys_mmap(LR_MAXDOCS * 8) as **u8
  let sllen: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  var ndocs: i64 = 0
  var malformed: i64 = 0
  var capped: i64 = 0
250
  var i: i64 = 0
  while i < n {
      // SEPARATE FLAG, never a sentinel written into the cursor. Writing `le = n + 1` to break out
      // erases the very position being searched for -- the estate has banked that defect four times
      // in one day, and this organ reproduced it on its first run: a 5-line fixture parsed as
      // docs=1. The declared envelope is what exposed it; a bare "0 results" would have read as
      // "no matches" rather than "the scanner is broken".
      var le: i64 = i
      var eol: i64 = 0
      while eol == 0 {
          if le >= n { eol = 1 } else {
              if buf[le] == (LR_NL as u8) { eol = 1 } else { le = le + 1 }
          }
      }
      let end: i64 = le
      if end > i {
          var t1: i64 = 0 - 1
          var t2: i64 = 0 - 1
          var j: i64 = i
          while j < end {
              if buf[j] == (LR_TAB as u8) {
                  if t1 < 0 { t1 = j } else { if t2 < 0 { t2 = j } }
              }
              j = j + 1
          }
          if t1 < 0 { malformed = malformed + 1 } else {
              if t2 < 0 { malformed = malformed + 1 } else {
                  if ndocs >= LR_MAXDOCS { capped = 1 } else {
                      var r: i64 = 0
                      var k: i64 = i
                      while k < t1 {
                          let c: i64 = buf[k] as i64
                          if c >= 48 { if c <= 57 { r = r * 10 + (c - 48) } }
                          k = k + 1
                      }
                      ranks[ndocs] = r
                      slugs[ndocs] = (buf as i64 + t1 + 1) as *u8
                      sllen[ndocs] = t2 - t1 - 1
                      texts[ndocs] = (buf as i64 + t2 + 1) as *u8
                      tlens[ndocs] = end - t2 - 1
                      ndocs = ndocs + 1
                  }
              }
          }
      }
      i = le + 1
  }
298
  // NO SILENT CAP: if the corpus outgrew the table say so on stderr, because a truncated ranking that
  // prints normally is a measurement nobody knows is partial.
  if capped == 1 {
      lr_e("REFUSED: corpus exceeds LR_MAXDOCS -- the printed order would be a PREFIX, not a ranking\n" as *u8)
      sys_exit(4)
      return 4
  }
306
  // --- query terms straight from argv (already tokenized by the caller's shell split)
  let qterms: **u8 = sys_mmap(LR_MAXTERMS * 8) as **u8
  let qlens: *i64 = sys_mmap(LR_MAXTERMS * 8) as *i64
  var nterms: i64 = 0
  var a: i64 = 4
  while a < argc {
      if nterms < LR_MAXTERMS {
          qterms[nterms] = argv[a] as *u8
          qlens[nterms] = lr_slen(argv[a] as *u8)
          nterms = nterms + 1
      }
      a = a + 1
  }
320
  // --- THE INCUMBENTS DO THE MATHEMATICS. This file only sequences them.
  let scores: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  nx_bm25_score(texts, tlens, ndocs, qterms, qlens, nterms, scores)
324
  let order: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  nx_rank_order_desc(scores, ndocs, order)
327
  let rrf: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  var z: i64 = 0
  while z < ndocs { rrf[z] = 0; z = z + 1 }
  nx_rrf_add(order, ndocs, rrf)
332
  // ---- SECOND VIEW: prefix-stemmed BM25, fused by RRF ------------------------------------------
  // WHY THIS IS NOT OPTIONAL. Exact BM25 cannot match "reversibility" to a law that says
  // "reversible" -- nx_rank.ps1's suite asserts that miss (T8) and then asserts fusion recovers it
  // (T9). An exact-only organ would pass an order-equality check in `exact` mode and silently lose
  // that recall in the `fused` mode production actually runs. RRF fuses by RANK, so adding a view is
  // one more nx_rrf_add over one more order -- which is why the incumbent's header says swapping in a
  // dense view later is "a one-view change, not a rewrite". This is that same seam, used once.
  // The stem arena is sized from the CORPUS (stemming only shrinks tokens), so no capacity is guessed.
  let sarena: *u8 = sys_mmap(n + ndocs + 1)
  let stexts: **u8 = sys_mmap(LR_MAXDOCS * 8) as **u8
  let slens: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  var soff: i64 = 0
  var sd: i64 = 0
  while sd < ndocs {
      let dstp: *u8 = (sarena as i64 + soff) as *u8
      let wrote: i64 = lr_stem_into(dstp, texts[sd], tlens[sd], stem_prefix)
      stexts[sd] = dstp
      slens[sd] = wrote
      soff = soff + wrote + 1
      sd = sd + 1
  }
  // the query is stemmed with the SAME transform, or the two sides would not meet
  let qarena: *u8 = sys_mmap(LR_MAXTERMS * 256)
  let sqterms: **u8 = sys_mmap(LR_MAXTERMS * 8) as **u8
  let sqlens: *i64 = sys_mmap(LR_MAXTERMS * 8) as *i64
  var qoff: i64 = 0
  var qi: i64 = 0
  while qi < nterms {
      let qd: *u8 = (qarena as i64 + qoff) as *u8
      var w: i64 = 0
      while w < qlens[qi] { if w < stem_prefix { qd[w] = qterms[qi][w] } w = w + 1 }
      var kl: i64 = qlens[qi]
      if kl > stem_prefix { kl = stem_prefix }
      qd[kl] = 0 as u8
      sqterms[qi] = qd
      sqlens[qi] = kl
      qoff = qoff + kl + 1
      qi = qi + 1
  }
  let sscores: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  nx_bm25_score(stexts, slens, ndocs, sqterms, sqlens, nterms, sscores)
  let sorder: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  nx_rank_order_desc(sscores, ndocs, sorder)
  nx_rrf_add(sorder, ndocs, rrf)
377
  // severity SCALES relevance, never adds to it (learn_markers.conf states the same rule):
  //   final = rrf * (blend_relevance + blend_severity * rank / star_max)
  // Multiplicative keeps zero relevance at zero, which is what lets the ranker honestly return
  // NOTHING rather than surfacing a severe-but-irrelevant law on any query.
  let fin: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  var d: i64 = 0
  while d < ndocs {
      let sevfrac: i64 = fx_from_frac(ranks[d], star_max)
      let factor: i64 = fx_from_frac(blend_rel, LR_HUNDRED) + fx_mul(fx_from_frac(blend_sev, LR_HUNDRED), sevfrac)
      fin[d] = fx_mul(rrf[d], factor)
      // A doc must stay out of the order only if it matched NOTHING in EITHER view. Testing the
      // exact view alone would zero precisely the docs the stemmed view exists to recover -- the
      // T9 case ("reversibility" reaching "reversible") would be found and then discarded one line
      // later. AN EXCLUSION RULE WRITTEN FOR ONE VIEW SILENTLY CANCELS EVERY VIEW ADDED AFTER IT.
      if scores[d] <= 0 { if sscores[d] <= 0 { fin[d] = 0 } }
      d = d + 1
  }
395
  let ford: *i64 = sys_mmap(LR_MAXDOCS * 8) as *i64
  nx_rank_order_desc(fin, ndocs, ford)
398
  var shown: i64 = 0
  var p: i64 = 0
  while p < ndocs {
      let id: i64 = ford[p]
      if fin[id] > 0 { if shown < topn {
          lr_n(1, ranks[id]); lr_byte(1, LR_TAB)
          sys_write(1, slugs[id], sllen[id]); lr_byte(1, LR_TAB)
          lr_n(1, fin[id]); lr_byte(1, LR_NL)
          shown = shown + 1
      } }
      p = p + 1
  }
411
  // DECLARED ENVELOPE: the counts a reader needs to know whether this answer is complete, including
  // the lines this organ refused to guess at.
  lr_e("LEARN-RANK docs=" as *u8); lr_n(2, ndocs)
  lr_e(" terms=" as *u8); lr_n(2, nterms)
  lr_e(" shown=" as *u8); lr_n(2, shown)
  lr_e(" malformed_skipped=" as *u8); lr_n(2, malformed)
  lr_e(" blend_rel_hundredths=" as *u8); lr_n(2, blend_rel)
  lr_e(" blend_sev_hundredths=" as *u8); lr_n(2, blend_sev)
  lr_e(" star_max=" as *u8); lr_n(2, star_max)
  lr_e("\n" as *u8)
  sys_exit(0)
  return 0
424}
nx_learn_rank.nx source

`nx_learn_rank.nx` source
