code wiki / _hdl_build / nx_search_census.nx
`nx_search_census.nx` source

↩ module page · 534 lines · 30185 B
1// nx_search_census.nx -- G1: THE SEARCH CORPUS, THROUGHPUT AND DIVERSITY CENSUS.
2// /compare/search rung G1, contracts idx_corpus_census and wc_throughput_census.
3//
4// WHY IT EXISTS. On 2026-09-03 the operator could see that the index was not growing and the BOARD COULD
5// NOT. /compare/search carried four latency rows and a relevance row and NOTHING for corpus size, crawl
6// throughput or host diversity; the one row that touched scale had already withheld its Nishi figure as
7// "prose, never a measurement" and nobody had measured it since. Nothing else in the estate measured it
8// either -- knowledge/status/growth_series.log tracks the PM board, debts and maturity and carries no
9// corpus field at all -- so the one axis the operator was complaining about was invisible to every
10// consumer and therefore priced at zero by the ranker.
11//
12// WHAT IT REFUSES TO DO. It prints LEVELS and never a rate. A rate needs two samples in time and the
13// estate ALREADY has a time spine, nx_growth_series, whose report verb computes d/w/m deltas per axis. A
14// second spine here would be the duplicate-ruler defect, so this organ is a PRODUCER for that one: it
15// TRUNCATE-writes a canonical status file and lets the spine carry the trajectory. The throughput axis
16// is the DIFFERENCE between two spine rows, never a number this organ invents.
17//
18// EVERY COUNT CARRIES ITS COVERAGE, and that is not a slogan here. The key census is COMPLETE over the
19// manifest and says so, with its partition summed and the sum printed. The outcome partition is a
20// DECLARED TAIL WINDOW and says so. This organ was written in the same session a sibling validator was
21// caught reading the OLDEST 3.2 percent of a 129 MB log while printing "sampling newest-first" -- a
22// window published without its denominator reads as a population, and that is exactly how a three-week
23// old fossil became evidence about today.
24//
25// ONE FAIRNESS RULER. The fair verdict is wc_frontier_fair, imported from the ENFORCER itself, and the
26// host parser is the crawler's own wc_host. Re-deriving either here would be a second ruler that agrees
27// today and diverges silently on the next edit to one side.
28//
29// DECLARED GAPS, NAMED RATHER THAN HIDDEN. (1) doc= counts seg-store ENTRIES, which are append-only, so
30// it is an UPPER BOUND on distinct documents and the output says so on its own line; distinct-key dedup
31// is the named next rung. (2) The fr: pending-versus-done split is NOT reported: splitting it needs an
32// ss_hget per frontier key and the live shard holds millions, so the honest choices were a silent sample
33// or a named absence, and this takes the absence.
34//   usage: nx_search_census [shard-prefix] [outcomes-log]
35// license_tier: ORIGINAL  expect_exit: 0
36import "nx_web_crawl_step.nx"   // wc_frontier_fair (shared fairness ruler) + wc_host (shared host parser)
37import "nx_intlog.nx"          // cs_crossover (S1: the corpus-scale side of the published BM25 crossover)
38
39const SC_SHARD:   *u8 = "knowledge/store/dp-web-pub-"
40const SC_LOG:     *u8 = "knowledge/status/crawl_outcomes.log"
41const SC_STATUS:  *u8 = "knowledge/status/search_census.status"
42// DECLARED window over the outcome log TAIL: how much recent crawl history one census reads. Printed
43// beside every figure derived from it, because a window without its denominator reads as a population.
44const SC_LOGCAP:    i64 = 4194304
45// Open-addressed distinct-host set for the outcome window, sized well above any plausible host count in
46// one window so the load factor stays low. A genuine hash collision UNDERCOUNTS hosts, which is wrong in
47// the direction of reporting LESS diversity than there is -- the safe direction for a rung whose entire
48// claim is that diversity is too low.
49const SC_HOSTSLOTS: i64 = 65536
50const SC_HOSTBUF:   i64 = 64
51const SC_URLBUF:    i64 = 4096
52const SC_STATBUF:   i64 = 4096
53const SC_HASH_MUL:  i64 = 1000003
54const SC_TAB:       i64 = 9
55const SC_NL:        i64 = 10
56const SC_PERMIL:    i64 = 1000
57
58func sc_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
59func sc_n(v: i64) -> i64 {
  let b: *u8 = sys_mmap(32); var x: i64 = v; var i: i64 = 31
  if x < 0 { x = 0 - x }
  if x == 0 { b[i] = 48 as u8; i = i - 1 }
  while x > 0 { b[i] = (48 + x % 10) as u8; x = x / 10; i = i - 1 }
  if v < 0 { b[i] = 45 as u8; i = i - 1 }
  sys_write(1, ((b as i64) + i + 1) as *u8, 31 - i); return 0
66}
67func sc_cat(out: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 } return o }
68func sc_catn(out: *u8, o: i64, v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { out[o] = 45 as u8; o = o + 1; m = 0 - m }
  if m == 0 { out[o] = 48 as u8; return o + 1 }
  var pw: i64 = 1
  while m / pw >= 10 { pw = pw * 10 }
  while pw > 0 { out[o] = (48 + (m / pw) % 10) as u8; o = o + 1; pw = pw / 10 }
  return o
76}
77// Prefix test by LITERAL, never by character codes: a string spelled as char codes is the same defect as
78// a hand-counted length -- no reader recognises it and no grep finds it.
79func sc_lit_at(p: *u8, len: i64, lit: *u8) -> i64 {
  var i: i64 = 0
  while lit[i] != (0 as u8) {
      if i >= len { return 0 }
      if p[i] != lit[i] { return 0 }
      i = i + 1
  }
  return 1
87}
88// Same test against a seg-store key entry, whose bytes start at eo+5.
89func sc_key_at(kb: *u8, eo: i64, kl: i64, lit: *u8) -> i64 {
  let kp: *u8 = ((kb as i64) + eo + 5) as *u8
  return sc_lit_at(kp, kl, lit)
92}
93// Multiply-accumulate hash. Deliberately no xor: this dialect's bitwise surface is not something a
94// census should be the first organ to depend on, and a collision here only undercounts hosts.
95// Parse the epoch field in place. Needed because a byte window is not a TIME window and only the
96// timestamps say which.
97func sc_atoi_at(p: *u8, s: i64, e: i64) -> i64 {
  var v: i64 = 0
  var i: i64 = s
  while i < e { let c: i64 = p[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 }
  return v
102}
103func sc_hash(p: *u8, n: i64) -> i64 {
  var h: i64 = 1
  var i: i64 = 0
  while i < n { h = h * SC_HASH_MUL + (p[i] as i64); if h < 0 { h = 0 - h } i = i + 1 }
  if h <= 0 { h = 1 }
  return h
109}
110// 1 when this host is NEWLY distinct in the set, 0 when already present or unusable.
111func sc_host_new(tbl: *i64, slots: i64, hp: *u8, hl: i64) -> i64 {
  if hl <= 0 { return 0 }
  let h: i64 = sc_hash(hp, hl)
  var idx: i64 = h % slots
  var guard: i64 = 0
  while guard < slots {
      if tbl[idx] == 0 { tbl[idx] = h; return 1 }
      if tbl[idx] == h { return 0 }
      idx = idx + 1
      if idx >= slots { idx = 0 }
      guard = guard + 1
  }
  return 0
124}
125
126// ---- THE TWO CONTRACT SYMBOLS. These are the names /compare/search watch rows measure, so they are the
127// REAL work and not wrappers shaped like it: a completion signal that keys on a name rewards writing the
128// name, so each of these must be the function that actually produces the number its row publishes. Both
129// are also the reason this organ is gateable -- a gate can drive them in-process with a fixture shard and
130// a fixture buffer, without forking anything or touching the live index.
131//
132// idx_corpus_census: out[0]=doc out[1]=url out[2]=frontier out[3]=priors out[4]=other out[5]=tombstones.
133// Returns TOTAL entries walked, so the caller can check the partition SUMS instead of trusting it.
134func idx_corpus_census(h: *i64, out: *i64) -> i64 {
  var i: i64 = 0
  while i < 6 { out[i] = 0; i = i + 1 }
  if (h as i64) == 0 { return 0 - 1 }   // UNOBSERVABLE, and -1 says so: an absent shard is not an empty one
  let ns: i64 = h[0]
  var all: i64 = 0
  var s: i64 = 0
  while s < ns {
      let kb: *u8 = h[1 + 8 * s] as *u8
      if h[2 + 8 * s] >= 8 {
          let m: i64 = ss_r32(kb, 4)
          var e: i64 = 0
          while e < m {
              let eo: i64 = 8 + 4 * m + ss_r32(kb, 8 + 4 * e)
              let kl: i64 = ss_r32(kb, eo + 1)
              all = all + 1
              if (kb[eo] as i64) == 2 { out[5] = out[5] + 1 } else {
                  if sc_key_at(kb, eo, kl, "doc:" as *u8) == 1 { out[0] = out[0] + 1 } else {
                  if sc_key_at(kb, eo, kl, "url:" as *u8) == 1 { out[1] = out[1] + 1 } else {
                  if sc_key_at(kb, eo, kl, "fr:" as *u8) == 1 { out[2] = out[2] + 1 } else {
                  if sc_key_at(kb, eo, kl, "pr:" as *u8) == 1 { out[3] = out[3] + 1 } else {
                      out[4] = out[4] + 1
                  } } } }
              }
              e = e + 1
          }
      }
      s = s + 1
  }
  return all
164}
165// wc_throughput_census: out[0]=hostcap out[1]=ok out[2]=thin out[3]=dup out[4]=robots out[5]=http
166// out[6]=other out[7]=distinct_hosts. Returns ROWS parsed, so the caller can check the class partition
167// SUMS to it. Buffers are passed IN, never allocated here: this runs once per row of a 4 MB window and
168// an allocation in a hot loop is how a census becomes the load it was measuring.
169// SLOTS IS A PARAMETER, NOT AN ASSUMPTION (2026-09-03, found by this organ's own gate on its FIRST run).
170// It previously took hosttbl from the caller and then indexed it with the hardcoded SC_HOSTSLOTS, so the
171// ALLOCATION and the INDEX BOUND lived in two places and any caller that sized its table differently got
172// a buffer overrun. The gate allocated 1024 slots, the callee indexed 65536, and it SIGSEGV'd -- which is
173// the good outcome, because the ship refused to promote a crashing gate. A callee that assumes the size
174// of a buffer it did not allocate is the same defect class as a hand-counted length beside a literal:
175// two copies of one number, drifting silently. Now there is one.
176func wc_throughput_census(buf: *u8, n: i64, out: *i64, hosttbl: *i64, slots: i64, hbuf: *u8, ubuf: *u8) -> i64 {
  var i0: i64 = 0
  while i0 < 10 { out[i0] = 0; i0 = i0 + 1 }
  var rows: i64 = 0
  var p: i64 = 0
  while p < n {
      // row extent is [p, re) -- a SEPARATE cursor, never the loop variable, so the scan can still say
      // where it stopped instead of clobbering its own position in order to exit
      var re: i64 = p
      var f: i64 = 0
      while f == 0 { if re >= n { f = 1 } else { if (buf[re] as i64) == SC_NL { f = 1 } else { re = re + 1 } } }
      if re > p {
          // row shape: <epoch> TAB <class> TAB <status> TAB <textlen> TAB <url>
          var t1: i64 = 0 - 1
          var t1e: i64 = 0 - 1
          var t4: i64 = 0 - 1
          var tabs: i64 = 0
          var i: i64 = p
          while i < re {
              if (buf[i] as i64) == SC_TAB {
                  tabs = tabs + 1
                  if tabs == 1 { t1 = i + 1 }
                  if tabs == 2 { t1e = i }
                  if tabs == 4 { t4 = i + 1 }
              }
              i = i + 1
          }
          if t1 >= 0 { if t1e > t1 {
              rows = rows + 1
              // THE WINDOW'S TIME SPAN, not just its byte size. A byte window is NOT a time window,
              // and only the timestamps say which hours it covers. Measured the day this was written:
              // a 4 MB tail of the outcome log spans about 27 HOURS, so a permil computed over it
              // averages a change made 2 hours ago into invisibility -- hostcap_defer_permil read 927
              // while hand-sampled post-fix windows read ZERO, and both were correct about their own
              // span. Declaring partial=1 was not enough; the SPAN is the envelope that matters.
              let ep: i64 = sc_atoi_at(buf, p, t1 - 1)
              if out[8] == 0 { out[8] = ep }
              out[9] = ep
              let cp: *u8 = ((buf as i64) + t1) as *u8
              let clen: i64 = t1e - t1
              // Prefix match on purpose: robots-cd belongs WITH robots, one class and not two.
              if sc_lit_at(cp, clen, "hostcap" as *u8) == 1 { out[0] = out[0] + 1 } else {
              if sc_lit_at(cp, clen, "robots" as *u8) == 1 { out[4] = out[4] + 1 } else {
              if sc_lit_at(cp, clen, "thin" as *u8) == 1 { out[2] = out[2] + 1 } else {
              if sc_lit_at(cp, clen, "dup" as *u8) == 1 { out[3] = out[3] + 1 } else {
              if sc_lit_at(cp, clen, "http" as *u8) == 1 { out[5] = out[5] + 1 } else {
              if sc_lit_at(cp, clen, "ok" as *u8) == 1 { out[1] = out[1] + 1 } else {
                  out[6] = out[6] + 1
              } } } } } }
              if t4 >= 0 {
                  let ul: i64 = re - t4
                  if ul > 0 { if ul < SC_URLBUF - 1 {
                      var y: i64 = 0
                      while y < ul { ubuf[y] = buf[t4 + y]; y = y + 1 }
                      ubuf[ul] = 0 as u8
                      let hl: i64 = wc_host(ubuf, hbuf)
                      if hl > 0 { out[7] = out[7] + sc_host_new(hosttbl, slots, hbuf, hl) }
                  } }
              }
          } }
      }
      p = re + 1
  }
  return rows
240}
241
242// ---- S1: CORPUS-SCALE SIDE OF THE PUBLISHED BM25 CROSSOVER. Indexed token occurrences are summed from every
243// live segment's NXQ1 positions sidecar: one varint run header per (term, doc) pair carries that pair's tf, so
244// the sum over every run is the exact count of token occurrences the tokenizer emitted into the index. A segment
245// without a readable sidecar contributes nothing and is COUNTED, so the figure is a LOWER BOUND whenever
246// pos_missing or pos_bad is above 0 -- and a lower bound already past the crossover still proves ABOVE, while a
247// lower bound under it proves nothing and says so. The crossover, its citation key and the study's own scope
248// caveat are DATA in knowledge/search_crossover.conf; the decision is nx_intlog's cs_crossover (the S1 symbol).
249const SC_CROSSOVER_CONF: *u8 = "knowledge/search_crossover.conf"
250const SC_POS_EXT: *u8 = ".pos"
251const SC_POS_MAGIC_N: i64 = 78
252const SC_POS_MAGIC_X: i64 = 88
253const SC_POS_MAGIC_Q: i64 = 81
254const SC_POS_MAGIC_1: i64 = 49
255const SC_POS_HDR: i64 = 8
256const SC_CONF_VALCAP: i64 = 2048
257const SC_INTBUF: i64 = 64
258const SC_SPACE: i64 = 32
259const SC_CR: i64 = 13
260// Sum every run's npos in one NXQ1 blob. outs[0]=tokens outs[1]=runs outs[2]=terms. 0 ok, -1 not an NXQ1 blob,
261// -2 a run cursor left the blob (truncated or corrupt): the partial sum is NOT trusted by the caller.
262func cs_pos_tokens(qb: *u8, qsz: i64, outs: *i64) -> i64 {
  outs[0] = 0; outs[1] = 0; outs[2] = 0
  if qsz < SC_POS_HDR { return 0 - 1 }
  if (qb[0] as i64) != SC_POS_MAGIC_N { return 0 - 1 }
  if (qb[1] as i64) != SC_POS_MAGIC_X { return 0 - 1 }
  if (qb[2] as i64) != SC_POS_MAGIC_Q { return 0 - 1 }
  if (qb[3] as i64) != SC_POS_MAGIC_1 { return 0 - 1 }
  let nterms: i64 = ss_r32(qb, 4)
  let qbase: i64 = SC_POS_HDR + 4 * nterms
  if qbase > qsz { return 0 - 2 }
  let cur: *i64 = sys_mmap(16) as *i64
  var t: i64 = 0
  while t < nterms {
      var e: i64 = qsz
      if t + 1 < nterms { e = qbase + ss_r32(qb, SC_POS_HDR + 4 * (t + 1)) }
      if e > qsz { e = qsz }
      cur[0] = qbase + ss_r32(qb, SC_POS_HDR + 4 * t)
      while cur[0] < e {
          let np: i64 = ss_vr(qb, cur)
          var sk: i64 = 0
          while sk < np { if cur[0] >= e { return 0 - 2 } ss_vr(qb, cur); sk = sk + 1 }
          outs[0] = outs[0] + np
          outs[1] = outs[1] + 1
      }
      if cur[0] > e { return 0 - 2 }
      t = t + 1
  }
  outs[2] = nterms
  return 0
291}
292// One key-value line of a conf: copies the value (to end of line, CR stripped) into out. Returns bytes copied,
293// 0 when the key has no line. A key must start its line and be followed by one space.
294func sc_conf_val(cf: *u8, n: i64, key: *u8, out: *u8, cap: i64) -> i64 {
  var kl: i64 = 0
  while key[kl] != (0 as u8) { kl = kl + 1 }
  var i: i64 = 0
  while i < n {
      var e: i64 = i
      var f: i64 = 0
      while f == 0 { if e >= n { f = 1 } else { if (cf[e] as i64) == SC_NL { f = 1 } else { e = e + 1 } } }
      if e - i > kl { if (cf[i + kl] as i64) == SC_SPACE { if sc_lit_at(((cf as i64) + i) as *u8, e - i, key) == 1 {
          var o: i64 = 0
          var j: i64 = i + kl + 1
          while j < e { if (cf[j] as i64) != SC_CR { if o < cap - 1 { out[o] = cf[j]; o = o + 1 } } j = j + 1 }
          out[o] = 0 as u8
          return o
      } } }
      i = e + 1
  }
  out[0] = 0 as u8
  return 0
313}
314func sc_conf_int(cf: *u8, n: i64, key: *u8) -> i64 {
  let vb: *u8 = sys_mmap(SC_INTBUF)
  let vn: i64 = sc_conf_val(cf, n, key, vb, SC_INTBUF)
  if vn <= 0 { return 0 }
  return sc_atoi_at(vb, 0, vn)
319}
320
321func main(argc: i64, argv: *i64) -> i64 {
  var prefix: *u8 = SC_SHARD
  if argc >= 2 { prefix = argv[1] as *u8 }
  var logp: *u8 = SC_LOG
  if argc >= 3 { logp = argv[2] as *u8 }
326
  sc_w("=== NX-SEARCH-CENSUS -- LEVELS ONLY; the RATE is the difference between two nx_growth_series rows, never a number this organ invents ===\n" as *u8)
328
  // ---- 1. THE KEY CENSUS, COMPLETE over the manifest: every segment, every entry.
  // ss_open2(...,1) is the MMAP open. The read-all ss_open pulls the WHOLE multi-GB shard into anon
  // RAM -- the OOM class the compactor was rewritten to kill. A census must never be why the box falls
  // over, and a measurement that costs an outage is not a measurement anyone will keep running.
  let h: *i64 = ss_open2(prefix, 1)
  if (h as i64) == 0 {
      sc_w("SHARD UNREADABLE prefix=" as *u8); sc_w(prefix)
      sc_w("\n  UNOBSERVABLE, not zero -- an absent shard and an empty shard are different facts and only one of them is a defect. An axis that cannot see must abstain, never acquit.\nverdict=SKIP\n" as *u8)
      return 3
  }
  let ns: i64 = h[0]
  // COMPOSE the contract function; main FORMATS and never re-implements. Two copies of this walk would
  // be a duplicate ruler: they agree the day they are written and drift on the next edit to either one.
  let kc: *i64 = sys_mmap(64) as *i64
  let e_all: i64 = idx_corpus_census(h, kc)
  let e_doc: i64 = kc[0]
  let e_url: i64 = kc[1]
  let e_fr: i64 = kc[2]
  let e_pr: i64 = kc[3]
  let e_other: i64 = kc[4]
  let e_tomb: i64 = kc[5]
  sc_w("SHARD prefix=" as *u8); sc_w(prefix); sc_w(" segments=" as *u8); sc_n(ns); sc_w("\n" as *u8)
  sc_w("KEYS doc=" as *u8); sc_n(e_doc)
  sc_w(" url=" as *u8); sc_n(e_url)
  sc_w(" frontier=" as *u8); sc_n(e_fr)
  sc_w(" priors=" as *u8); sc_n(e_pr)
  sc_w(" other=" as *u8); sc_n(e_other)
  sc_w(" tombstones=" as *u8); sc_n(e_tomb)
  sc_w(" total=" as *u8); sc_n(e_all); sc_w("\n" as *u8)
  // A PARTITION IS A CLAIM: CHECK THE PARTS SUM, AND PRINT THE CHECK. An unexplained residual is a leak;
  // an explained one is a decision. Silence about it is neither.
  let e_sum: i64 = e_doc + e_url + e_fr + e_pr + e_other + e_tomb
  sc_w("PARTITION parts=" as *u8); sc_n(e_sum); sc_w(" total=" as *u8); sc_n(e_all)
  if e_sum == e_all { sc_w(" SUMS=1 coverage_complete=1 corpus_complete=1 (every segment in the manifest, every entry in each)\n" as *u8) } else { sc_w(" SUMS=0 RESIDUAL -- do not trust these buckets\n" as *u8) }
  sc_w("ENVELOPE doc= counts append-only ENTRIES, so it is an UPPER BOUND on distinct documents (a rewritten doc counts once per segment it lands in). NO consumer may publish doc= as a document count until the distinct-key dedup rung lands.\n" as *u8)
  sc_w("ENVELOPE frontier= is likewise an upper bound on PENDING urls: the pending-versus-done split needs one value read per key and the shard holds millions, so it is a NAMED ABSENCE here rather than a silent sample.\n" as *u8)
365
  // ---- 1b. S1: THE CORPUS-SCALE SIDE OF THE PUBLISHED BM25 CROSSOVER (composes cs_pos_tokens + cs_crossover).
  let cxb: *i64 = sys_mmap(16) as *i64
  cxb[0] = 0
  let cxcf: *u8 = sys_read_file(SC_CROSSOVER_CONF, cxb)
  var cx_tokens: i64 = 0
  let cx_ref: *u8 = sys_mmap(SC_INTBUF)
  let cx_cav: *u8 = sys_mmap(SC_CONF_VALCAP)
  cx_ref[0] = 0 as u8
  cx_cav[0] = 0 as u8
  if (cxcf as i64) != 0 {
      cx_tokens = sc_conf_int(cxcf, cxb[0], "crossover_tokens" as *u8)
      sc_conf_val(cxcf, cxb[0], "ref" as *u8, cx_ref, SC_INTBUF)
      sc_conf_val(cxcf, cxb[0], "caveat" as *u8, cx_cav, SC_CONF_VALCAP)
  }
  let pxs: *i64 = sys_mmap(16) as *i64
  let pxo: *i64 = sys_mmap(32) as *i64
  var tok_total: i64 = 0
  var pos_runs: i64 = 0
  var pos_ok: i64 = 0
  var pos_missing: i64 = 0
  var pos_bad: i64 = 0
  let qp: *u8 = sys_mmap(SC_URLBUF)
  var s2: i64 = 0
  while s2 < ns {
      let nm: *u8 = ss_open_segname(h, s2)
      if (nm as i64) == 0 { pos_missing = pos_missing + 1 } else {
          var qo: i64 = sc_cat(qp, 0, prefix)
          qo = sc_cat(qp, qo, nm)
          qo = sc_cat(qp, qo, SC_POS_EXT)
          qp[qo] = 0 as u8
          pxs[0] = 0
          let qb: *u8 = ss_readall(qp, pxs)
          if (qb as i64) == 0 { pos_missing = pos_missing + 1 } else {
              let prc: i64 = cs_pos_tokens(qb, pxs[0], pxo)
              if prc == 0 { tok_total = tok_total + pxo[0]; pos_runs = pos_runs + pxo[1]; pos_ok = pos_ok + 1 } else { pos_bad = pos_bad + 1 }
              ss_freeall(qb, pxs[0])   // the paired free for ss_readall, released per segment: a census must never become the load it measures
          }
      }
      s2 = s2 + 1
  }
  var cx_side: i64 = 0 - 1
  var sidew: *u8 = "NO-CONF" as *u8
  if cx_tokens > 0 {
      let cxr: i64 = cs_crossover(tok_total, cx_tokens)
      if cxr == CS_SIDE_ABOVE { cx_side = CS_SIDE_ABOVE; sidew = "ABOVE" as *u8 }
      else { if cxr == CS_SIDE_UNMEASURED { sidew = "UNMEASURED" as *u8 } else { if pos_missing + pos_bad == 0 { cx_side = CS_SIDE_BELOW; sidew = "BELOW" as *u8 } else { sidew = "BELOW-OR-UNCOVERED" as *u8 } } }   // zero indexed tokens is an UNMEASURED corpus, never a BELOW (the three-state contract)
  }
  sc_w("CROSSOVER indexed_tokens=" as *u8); sc_n(tok_total)
  sc_w(" runs=" as *u8); sc_n(pos_runs)
  sc_w(" segments_with_pos=" as *u8); sc_n(pos_ok)
  sc_w(" pos_missing=" as *u8); sc_n(pos_missing)
  sc_w(" pos_bad=" as *u8); sc_n(pos_bad)
  sc_w(" crossover_tokens=" as *u8); sc_n(cx_tokens)
  sc_w(" side=" as *u8); sc_w(sidew)
  sc_w(" ref=" as *u8); sc_w(cx_ref)
  sc_w(" scope=" as *u8); sc_w(cx_cav); sc_w("\n" as *u8)
  sc_w("ENVELOPE indexed_tokens= counts the token OCCURRENCES the tokenizer emitted into the positions sidecars (unindexed bytes are not tokens here), summed over every segment whose sidecar was read whole; with pos_missing or pos_bad above 0 it is a LOWER BOUND, and ABOVE is still asserted from a lower bound because a bound already past the crossover proves the side, while BELOW is asserted only with every segment covered.\n" as *u8)
423
  // ---- 2. THE OUTCOME PARTITION over a DECLARED TAIL WINDOW.
  let buf: *u8 = sys_mmap(SC_LOGCAP)
  let fd: i64 = sys_openat_rd(logp)
  var flen: i64 = 0
  var n: i64 = 0
  var startoff: i64 = 0
  if fd >= 0 {
      flen = sys_lseek(fd, 0, 2)
      if flen > SC_LOGCAP { startoff = flen - SC_LOGCAP }
      sys_lseek(fd, startoff, 0)
      var go: i64 = 1
      while go == 1 {
          let r: i64 = sys_read(fd, ((buf as i64) + n) as *u8, SC_LOGCAP - n)
          if r <= 0 { go = 0 } else { n = n + r }
          if n >= SC_LOGCAP { go = 0 }
      }
      sys_close(fd)
  }
  // A seeked window almost never lands on a row boundary: DROP the partial leading row rather than
  // count half a record into a class.
  if startoff > 0 {
      var lead: i64 = 0
      var q: i64 = 0
      var fnd: i64 = 0
      while fnd == 0 { if q >= n { fnd = 1 } else { if (buf[q] as i64) == SC_NL { lead = q + 1; fnd = 1 } else { q = q + 1 } } }
      if lead > 0 {
          var mv: i64 = 0
          while mv < n - lead { buf[mv] = buf[mv + lead]; mv = mv + 1 }
          n = n - lead
      }
  }
  let hosttbl: *i64 = sys_mmap(SC_HOSTSLOTS * 8) as *i64
  let hbuf: *u8 = sys_mmap(SC_HOSTBUF)
  let ubuf: *u8 = sys_mmap(SC_URLBUF)   // allocated ONCE: never allocate in a hot loop
  // COMPOSE the contract function; main FORMATS and never re-implements. The buffers are passed IN so
  // nothing allocates per row -- a census that allocates in its hot loop becomes the load it measures.
  let tc: *i64 = sys_mmap(96) as *i64
  let rows: i64 = wc_throughput_census(buf, n, tc, hosttbl, SC_HOSTSLOTS, hbuf, ubuf)
  let c_hostcap: i64 = tc[0]
  let c_ok: i64 = tc[1]
  let c_thin: i64 = tc[2]
  let c_dup: i64 = tc[3]
  let c_robots: i64 = tc[4]
  let c_http: i64 = tc[5]
  let c_other: i64 = tc[6]
  let hosts: i64 = tc[7]
  var partial: i64 = 0
  if startoff > 0 { partial = 1 }
  sc_w("OUTCOMES log_bytes=" as *u8); sc_n(flen)
  sc_w(" window_bytes=" as *u8); sc_n(n)
  sc_w(" rows=" as *u8); sc_n(rows)
  if partial == 1 { sc_w(" scope=TAIL partial=1 (older rows NOT scanned -- a DECLARED window, never the population)\n" as *u8) } else { sc_w(" scope=FULL partial=0\n" as *u8) }
  let w_first: i64 = tc[8]
  let w_last: i64 = tc[9]
  sc_w("WINDOW_SPAN first_epoch=" as *u8); sc_n(w_first)
  sc_w(" last_epoch=" as *u8); sc_n(w_last)
  sc_w(" span_s=" as *u8); sc_n(w_last - w_first)
  sc_w(" -- EVERY CLASS COUNT AND PERMIL BELOW IS AN AVERAGE OVER THIS SPAN. A change made more recently than the span is averaged toward invisibility: measured 2026-09-03, this window spanned 51763s (14.4h, and note the author had ESTIMATED 27h by extrapolating a growth rate -- the measurement beat the estimate by 2x, which is why this field exists at all) and reported hostcap_defer_permil=925 while hand-sampled post-fix windows read ZERO. Both readings were correct about their own span. AND A SECOND TRAP LIVES HERE: A PERMIL OVER ROWS IS NOT A PERMIL OVER TIME. Rows are not uniformly distributed in time -- a deferral costs no fetch and is emitted in bursts, while a successful fetch is slow -- so a period that DEFERS heavily contributes far more rows per second than one that FETCHES, and a row-weighted average over-weights it. With about 27 percent of this window post-fix BY TIME, a naive reading would predict ~730 permil; it reads 925 because the pre-fix rows are denser, not because the fix is failing. To judge a RECENT change, read the log tail directly or shrink SC_LOGCAP; do not reason from this permil alone.\n" as *u8)
  sc_w("CLASSES hostcap=" as *u8); sc_n(c_hostcap)
  sc_w(" ok=" as *u8); sc_n(c_ok)
  sc_w(" thin=" as *u8); sc_n(c_thin)
  sc_w(" dup=" as *u8); sc_n(c_dup)
  sc_w(" robots=" as *u8); sc_n(c_robots)
  sc_w(" http=" as *u8); sc_n(c_http)
  sc_w(" other=" as *u8); sc_n(c_other); sc_w("\n" as *u8)
  let c_sum: i64 = c_hostcap + c_ok + c_thin + c_dup + c_robots + c_http + c_other
  sc_w("PARTITION classes=" as *u8); sc_n(c_sum); sc_w(" rows=" as *u8); sc_n(rows)
  if c_sum == rows { sc_w(" SUMS=1\n" as *u8) } else { sc_w(" SUMS=0 RESIDUAL -- do not trust these class counts\n" as *u8) }
  // THE TWO NUMBERS RUNG G0 IS GRADED ON. Splitting the classes matters and is not cosmetic: a thin or
  // robots loss is the JS-render rung R1, not the host-diversity rung G0, and a single throughput
  // figure would attribute one to the other and send the next reader at the wrong rung.
  var defer_permil: i64 = 0
  if rows > 0 { defer_permil = c_hostcap * SC_PERMIL / rows }
  sc_w("DIVERSITY distinct_hosts=" as *u8); sc_n(hosts)
  sc_w(" hostcap_defer_permil=" as *u8); sc_n(defer_permil)
  sc_w(" window_quota=" as *u8); sc_n(WC_WINQ)
  let fair: i64 = wc_frontier_fair(rows, hosts)
  if fair == 1 { sc_w(" fair=1\n" as *u8) } else { sc_w(" fair=0 -- this many rows could not have been held to the quota across this few hosts\n" as *u8) }
502
  // ---- 3. THE CANONICAL STATUS FILE: TRUNCATE-write, one state, verdict LAST so a positional reader
  // (gv_last_line) is safe. Deliberately NOT append-only -- the trajectory belongs to nx_growth_series,
  // and a second append-only spine here is precisely the defect this organ refuses to commit.
  let sb: *u8 = sys_mmap(SC_STATBUF)
  var o: i64 = 0
  o = sc_cat(sb, o, "corpus_doc_entries " as *u8); o = sc_catn(sb, o, e_doc)
  o = sc_cat(sb, o, "\ncorpus_url_entries " as *u8); o = sc_catn(sb, o, e_url)
  o = sc_cat(sb, o, "\nfrontier_entries " as *u8); o = sc_catn(sb, o, e_fr)
  o = sc_cat(sb, o, "\nshard_segments " as *u8); o = sc_catn(sb, o, ns)
  o = sc_cat(sb, o, "\nshard_entries_total " as *u8); o = sc_catn(sb, o, e_all)
  o = sc_cat(sb, o, "\noutcome_rows " as *u8); o = sc_catn(sb, o, rows)
  o = sc_cat(sb, o, "\noutcome_window_partial " as *u8); o = sc_catn(sb, o, partial)
  o = sc_cat(sb, o, "\noutcome_window_span_s " as *u8); o = sc_catn(sb, o, w_last - w_first)
  o = sc_cat(sb, o, "\noutcome_window_first_epoch " as *u8); o = sc_catn(sb, o, w_first)
  o = sc_cat(sb, o, "\noutcome_window_last_epoch " as *u8); o = sc_catn(sb, o, w_last)
  o = sc_cat(sb, o, "\nhostcap_defer_permil " as *u8); o = sc_catn(sb, o, defer_permil)
  o = sc_cat(sb, o, "\ndistinct_hosts " as *u8); o = sc_catn(sb, o, hosts)
  o = sc_cat(sb, o, "\nfrontier_fair " as *u8); o = sc_catn(sb, o, fair)
  o = sc_cat(sb, o, "\ncorpus_indexed_tokens " as *u8); o = sc_catn(sb, o, tok_total)
  o = sc_cat(sb, o, "\ncorpus_pos_uncovered " as *u8); o = sc_catn(sb, o, pos_missing + pos_bad)
  o = sc_cat(sb, o, "\ncrossover_tokens " as *u8); o = sc_catn(sb, o, cx_tokens)
  o = sc_cat(sb, o, "\ncrossover_side " as *u8); o = sc_catn(sb, o, cx_side)
  o = sc_cat(sb, o, "\nverdict=GREEN\n" as *u8)
  if ss_writefile(SC_STATUS, sb, o) == 0 {
      sc_w("STATUS wrote " as *u8); sc_n(o); sc_w(" bytes -> " as *u8); sc_w(SC_STATUS); sc_w("\n" as *u8)
  } else {
      sc_w("STATUS WRITE FAILED -- the census ran and published nothing, which from outside is indistinguishable from never running\n" as *u8)
  }
531
  sc_w("verdict=GREEN\n" as *u8)
  return 0
534}
nx_search_census.nx source

`nx_search_census.nx` source
