code wiki / _hdl_build / nx_corpus_ingest.nx
`nx_corpus_ingest.nx` source

↩ module page · 681 lines · 36150 B
1// nx_corpus_ingest.nx -- SOVEREIGN corpus ingestion: a DIRECTORY of documents -> a domain's PUBLIC seg_store
2// shard, searchable at once (ss_write_seg builds .terms per segment; the store IS the index -- no tsv, no
3// derived artifact). This is the front door for ALL THREE nishi-search fronts (operator 2026-07-03):
4//   library:    nx_corpus_ingest knowledge/library nishifamily.com          (.txt/.md -> /doc text views)
5//   site pages: nx_corpus_ingest <pagesdir> andelinwest.com /               (.html -> text + url:<cid> row,
6//               so the SERP result links to the REAL page instead of /doc)
7//   web:        the crawler drops fetched pages in a dir -> same ingest with absolute-URL prefix rows
8//   estate:     nx_corpus_ingest tree <root> estate estate://<root>/         (E1, 2026-09-14: every source file
9//               under a tree, recursively, one document per file, its relative path as the url, its identifiers
10//               anchored, and a changed file REPLACING its previous document through a path row + tombstones)
11// Mechanics: getdents64 walk (flat dir, or the recursive tree walk) -> .txt/.md raw, .html/.htm via
12// nx_html_to_text (script/style suppressed, entities decoded) -> cid = the same polynomial hash
13// nx_dp_tsv_migrate uses (content-addressed, idempotent) -> skip-if-already-present (re-runs are clean) ->
14// RAW ss_add doc:<cid> (+ url:<cid>) batched into 1MB writer segments, each committed under a FRESH segid
15// (the seg store's ss_next_segid -- never clobbers existing segments). ONE ingest body (ci_ingest_file) and ONE
16// writer put (ci_put) serve both walks: two copies would agree the day they were written and drift on the next
17// edit. license_tier: ORIGINAL
18import "nx_docportal_search_seg.nx"
19import "nx_html_to_text.nx"
20import "nx_funchead_lib.nx"
21const CI_MAGIC_1125899906842597: i64 = 1125899906842597
22const CI_MAGIC_131072: i64 = 131072
23const CI_MAGIC_1024: i64 = 1024
24const CI_MAGIC_65536: i64 = 65536
25
26const CI_DOCCAP:   i64 = 900000   // per-doc byte cap (writer segments are 1MB; a doc must fit with headroom)
27const CI_MINDOC:   i64 = 20       // skip empty-ish extractions
28const CI_RAWCAP:   i64 = 8388608  // largest raw file we read
29const CI_DT_DIR:   i64 = 4        // dirent d_type of a directory
30const CI_WORD:     i64 = 8
31const CI_BOX:      i64 = 16
32const CI_KEYBUF:   i64 = 64
33const CI_PREFIXBUF: i64 = 512
34const CI_DIGIT0:   i64 = 48
35const CI_DIGIT9:   i64 = 57
36const CI_DECBUF:   i64 = 24
37const CI_LF:       i64 = 10
38const CI_KIND_LIVE: i64 = 1       // seg-store entry kinds: a live row, and a tombstone that retires a key
39const CI_KIND_TOMB: i64 = 2
40const CI_DOC_JOURNAL: i64 = 3     // ci_kind_source: an append-only journal (.jrnl .log), ingested TAIL-anchored (E3)
41const CI_DOC_BOARD: i64 = 4       // ci_kind_source: a compare board (.plan .matrix), ingested whole AND row by row (E3b)
42const CI_ROW_SEP: *u8 = "#"       // a row document's url is the board url, this separator, the row index
43const CI_ROW_RUNG: *u8 = "rung|"  // a rung row carries the definition token of its id, the way a function head does
44const CI_CONF_PIPE: i64 = 124     // estate_corpora.conf field separator
45const CI_CONF_HASH: i64 = 35      // estate_corpora.conf comment lead byte
46const CI_CR: i64 = 13             // a CRLF conf leaves this as the last byte of a row; stripped, never parsed as data
47const CI_STATE_WORDS: i64 = 12
48const CI_ANCHOR_PAD: i64 = 64     // the anchor tail is never longer than the text it came from; this is the newline and slack
49const CI_TREE_MAXDEPTH: i64 = 12  // a source tree deeper than this is not a source tree; announced, never silent
50
51func ci_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
52func ci_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
53func ci_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
54// decimal digits of a non-negative value into out (no terminator); returns the digit count. THE one digit emitter
55// here: the keys, the path row value and the report all spell numbers through it.
56func ci_dec(v: i64, out: *u8) -> i64 {
  if v == 0 { out[0] = CI_DIGIT0 as u8; return 1 }
  let t: *u8 = sys_mmap(CI_DECBUF)
  var k: i64 = 0
  var m: i64 = v
  while m > 0 { t[k] = (CI_DIGIT0 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var j: i64 = 0
  while j < k { out[j] = t[k - 1 - j]; j = j + 1 }
  return k
65}
66func ci_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(CI_DECBUF + 4)
  var m: i64 = v
  var o: i64 = 0
  if m < 0 { bb[0] = 45 as u8; o = 1; m = 0 - m }
  o = o + ci_dec(m, (bb as i64 + o) as *u8)
  sys_write(1, bb, o); return 0
73}
74// digits in p[0..n) as a value; a non-digit byte is skipped (the path row value is digits only)
75func ci_atoi(p: *u8, n: i64) -> i64 {
  var v: i64 = 0
  var i: i64 = 0
  while i < n { let c: i64 = p[i] as i64; if c >= CI_DIGIT0 { if c <= CI_DIGIT9 { v = v * 10 + (c - CI_DIGIT0) } } i = i + 1 }
  return v
80}
81
82// stable positive content id -- SAME polynomial as nx_dp_tsv_migrate.mg_hash (idempotent across organs)
83func ci_hash(s: *u8, n: i64) -> i64 {
  var h: i64 = CI_MAGIC_1125899906842597
  var i: i64 = 0
  while i < n { h = (h * 131) + (s[i] as i64); i = i + 1 }
  if h < 0 { h = 0 - h }
  return h & 0x7fffffffffffffff
89}
90// build "<pfx><id>" NUL-terminated into out; returns its length. ONE key builder for url:/fp:/pth: (a literal each,
91// never a string spelled as character codes -- the earlier url:/fp: builders were, and no grep could find them).
92func ci_mkkey_pfx(pfx: *u8, id: i64, out: *u8) -> i64 {
  var o: i64 = ci_cat(out, 0, pfx)
  o = o + ci_dec(id, (out as i64 + o) as *u8)
  out[o] = 0 as u8
  return o
97}
98// build "url:<cid>" (MUST match the serve side's dsv_mkurlkey)
99func ci_mkurlkey(cid: i64, out: *u8) -> i64 { return ci_mkkey_pfx("url:" as *u8, cid, out) }
100// build "fp:<cid>" -- the near-duplicate FINGERPRINT row, sibling of doc:/url: (2026-08-16).
101// WHY IT EXISTS: every ingest organ ALREADY computes nx_simhash_fingerprint and then THROWS IT AWAY --
102// it dedupes within one run against keptfp[] and vanishes at exit. So intra-host near-duplication, which
103// is the real doorway signature (many urls, one template, one swapped word), could only be asked of the
104// corpus by re-fetching and re-computing ALL of it. Persisting the value that was already computed turns
105// that into a query. Capture once, ask forever.
106// MEASURED 2026-08-16, the reason this is the right signal and redirect depth is not: bestbeeg.top/key/dudh
107// returns 200 with ZERO redirects and a 9,399-byte body. A hop-count detector fires on nothing there; the
108// duplication is visible, the redirection is not.
109// ADDITIVE BY CONSTRUCTION: a new key prefix. Readers scanning doc:/url:/fr:/hd:/pol:/pr:/out: are
110// untouched, and an absent fp: row simply means "ingested before this shipped" -- never a wrong answer.
111func ci_mkfpkey(cid: i64, out: *u8) -> i64 { return ci_mkkey_pfx("fp:" as *u8, cid, out) }
112// build "fe:<cid>" -- the FETCH EPOCH row, sibling of doc:/url:/fp: (S12, 2026-09-17). Every crawler knows the moment it
113// fetched a kept page and then threw it away; the only ledger was crawl_outcomes.log, which no result can be joined to at
114// query time. Stored beside the doc, a result can say how old its bytes are. ADDITIVE BY CONSTRUCTION: readers of the
115// other prefixes are untouched, and an absent fe: row means "ingested before this shipped" -- unobserved, never an age of zero.
116func ci_mkfekey(cid: i64, out: *u8) -> i64 { return ci_mkkey_pfx("fe:" as *u8, cid, out) }
117// build "pth:<hash of the url tail>" -- the PATH row (E1, 2026-09-14): value = the decimal cid of the document that
118// currently stands for that path, so a re-ingest of a CHANGED file finds its predecessor and tombstones it instead
119// of leaving two live documents for one file. Additive: readers of the older prefixes never see it.
120func ci_mkpthkey(pathhash: i64, out: *u8) -> i64 { return ci_mkkey_pfx("pth:" as *u8, pathhash, out) }
121// does name end with the null-terminated suffix? (case-sensitive; corpus files are ours)
122func ci_ends(name: *u8, suf: *u8) -> i64 {
  let nl: i64 = ci_len(name)
  let sl: i64 = ci_len(suf)
  if nl < sl { return 0 }
  var i: i64 = 0
  while i < sl { if name[nl - sl + i] != suf[i] { return 0 } i = i + 1 }
  return 1
129}
130func ci_contains(name: *u8, ndl: *u8) -> i64 {
  let nl: i64 = ci_len(name)
  let dl: i64 = ci_len(ndl)
  if dl == 0 { return 0 }
  var i: i64 = 0
  while i + dl <= nl {
      var j: i64 = 0
      var hit: i64 = 1
      while j < dl { if name[i + j] != ndl[j] { hit = 0; j = dl } else { j = j + 1 } }
      if hit == 1 { return 1 }
      i = i + 1
  }
  return 0
143}
144// classify: 1 = raw text (.txt/.md), 2 = html (.html/.htm), 0 = skip (.bak anywhere, everything else)
145func ci_kind(name: *u8) -> i64 {
  if ci_contains(name, ".bak" as *u8) == 1 { return 0 }
  if ci_ends(name, ".txt" as *u8) == 1 { return 1 }
  if ci_ends(name, ".md" as *u8) == 1 { return 1 }
  if ci_ends(name, ".rst" as *u8) == 1 { return 1 }
  if ci_ends(name, ".html" as *u8) == 1 { return 2 }
  if ci_ends(name, ".htm" as *u8) == 1 { return 2 }
  return 0
153}
154// E1 source kinds: the estate's own text -- NishiLang, markdown, plain text, confs and the compare boards.
155// Everything else (binaries, fossils, media) is skipped and COUNTED as skipped_other.
156func ci_kind_source(name: *u8) -> i64 {
  if ci_contains(name, ".bak" as *u8) == 1 { return 0 }
  if ci_ends(name, ".nx" as *u8) == 1 { return 1 }
  if ci_ends(name, ".md" as *u8) == 1 { return 1 }
  if ci_ends(name, ".txt" as *u8) == 1 { return 1 }
  if ci_ends(name, ".conf" as *u8) == 1 { return 1 }
  if ci_ends(name, ".plan" as *u8) == 1 { return CI_DOC_BOARD }
  if ci_ends(name, ".matrix" as *u8) == 1 { return CI_DOC_BOARD }
  if ci_ends(name, ".sota" as *u8) == 1 { return 1 }
  if ci_ends(name, ".gates" as *u8) == 1 { return 1 }
  if ci_ends(name, ".axes" as *u8) == 1 { return 1 }
  if ci_ends(name, ".bench" as *u8) == 1 { return 1 }
  if ci_ends(name, ".list" as *u8) == 1 { return 1 }
  if ci_ends(name, ".status" as *u8) == 1 { return 1 }
  if ci_ends(name, ".stamp" as *u8) == 1 { return 1 }
  if ci_ends(name, ".jrnl" as *u8) == 1 { return CI_DOC_JOURNAL }
  if ci_ends(name, ".log" as *u8) == 1 { return CI_DOC_JOURNAL }
  if ci_ends(name, ".refs" as *u8) == 1 { return 1 }
  return 0
175}
176// directories the tree walk never enters: hidden entries (which also covers . and ..), build fossils, job
177// artifacts, retired copies, the stores and the fetched mirrors -- none of them is the estate's own text.
178// exact name equality. A SUFFIX test here skipped _hdl_build -- the tree that holds most current organs -- on the
179// first estate ingest, because _hdl_build ends with _build; MEASURED 2026-09-14 by a query for a lib that was not
180// there. A skip list matches whole names, never tails.
181func ci_name_eq(name: *u8, lit: *u8) -> i64 {
  if ci_len(name) != ci_len(lit) { return 0 }
  return ci_contains(name, lit)
184}
185func ci_tree_skipdir(name: *u8) -> i64 {
  let dot: *u8 = "." as *u8
  if name[0] == dot[0] { return 1 }
  if ci_name_eq(name, "_build" as *u8) == 1 { return 1 }
  if ci_name_eq(name, "_jobs" as *u8) == 1 { return 1 }
  if ci_name_eq(name, "retired" as *u8) == 1 { return 1 }
  if ci_name_eq(name, "store" as *u8) == 1 { return 1 }
  if ci_name_eq(name, "fetched" as *u8) == 1 { return 1 }
  return 0
194}
195
196// Delegates to the seg store's ONE derivation (ss_next_segid, dynamic manifest read): a shard past the old
197// 256-name buffer -- the estate source tree commits hundreds of segments -- can no longer overrun anything here.
198func ci_next_segid(prefix: *u8) -> i64 {
  return ss_next_segid(prefix)
200}
201// THE INGEST STATE, one per run, shared by both walks: [0]=writer [1]=segid [2]=prefix [3]=store handle
202// (0 = fresh shard, fine) [4]=pbox [5]=lbox [6]=counts [7]=key [8]=ukey [9]=urlval [10]=hout [11]=anchors
203// (1 = the tree walk: identifier anchors appended to every text document and a path row kept per file).
204// counts[0]=ingested counts[1]=skipped-present counts[2]=skipped-other counts[3]=segments-committed
205// counts[4]=replaced (a changed file whose previous document was tombstoned)
206func ci_state_new(prefix: *u8, counts: *i64) -> *i64 {
  counts[0] = 0; counts[1] = 0; counts[2] = 0; counts[3] = 0; counts[4] = 0
  let st: *i64 = sys_mmap(CI_STATE_WORDS * CI_WORD) as *i64
  st[0] = ss_begin() as i64
  st[1] = ci_next_segid(prefix)
  st[2] = prefix as i64
  st[3] = ss_open(prefix) as i64   // open the CURRENT store once for skip-if-present and the path rows
  st[4] = sys_mmap(CI_BOX) as i64
  st[5] = sys_mmap(CI_BOX) as i64
  st[6] = counts as i64
  st[7] = sys_mmap(CI_KEYBUF) as i64
  st[8] = sys_mmap(CI_KEYBUF) as i64
  st[9] = sys_mmap(CI_MAGIC_1024) as i64
  st[10] = sys_mmap(CI_RAWCAP + CI_MAGIC_65536) as i64
  st[11] = 0
  return st
222}
223// commit whatever the writer still holds
224func ci_state_finish(st: *i64) -> i64 {
  let w: *i64 = st[0] as *i64
  let counts: *i64 = st[6] as *i64
  if w[1] > 0 {
      if ss_commit(st[2] as *u8, w, st[1]) == 0 { counts[3] = counts[3] + 1 } else { ci_puts("  FINAL-COMMIT-FAIL\n" as *u8) }
  }
  return 0
231}
232// ONE writer put: a row into the current writer, and when the writer is full, commit the segment, open the next
233// and put again. -1 only when a row cannot fit an EMPTY writer (a document past the segment size).
234func ci_put(st: *i64, kind: i64, key: *u8, val: *u8, vlen: i64) -> i64 {
  var w: *i64 = st[0] as *i64
  if ss_add(w, kind, key, val, vlen) >= 0 { return 0 }
  let counts: *i64 = st[6] as *i64
  if ss_commit(st[2] as *u8, w, st[1]) != 0 { ci_puts("  COMMIT-FAIL seg=" as *u8); ci_num(st[1]); ci_puts("\n" as *u8) } else { counts[3] = counts[3] + 1 }
  st[1] = st[1] + 1
  w = ss_begin()
  st[0] = w as i64
  if ss_add(w, kind, key, val, vlen) < 0 { return 0 - 1 }
  return 0
244}
245// E1b DEFINITION ANCHORS: one token per function head, the def prefix glued to the collapsed name (ss_def_token, the
246// same derivation the query side uses), appended after the identifier anchors so the file that DECLARES a symbol
247// carries a term no caller carries. The heads come from nx_funchead_lib (the scanner nx_funcmine reads), never a
248// second scanner. The head table is sized from the text itself: a head is at least CI_DEF_MINHEAD bytes, so
249// n / CI_DEF_MINHEAD + 1 entries can never truncate -- a derived bound, not a guessed cap.
250const CI_DEF_MINHEAD: i64 = 7      // the shortest head: func, a space, a one-byte name, the paren
251func ci_def_anchors(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
  let maxh: i64 = n / CI_DEF_MINHEAD + 1
  let starts: *i64 = sys_mmap(maxh * CI_WORD) as *i64
  let noffs: *i64 = sys_mmap(maxh * CI_WORD) as *i64
  let nlens: *i64 = sys_mmap(maxh * CI_WORD) as *i64
  let nf: i64 = fh_scan(src, n, starts, noffs, nlens, maxh)
  // the token buffer is sized from the longest head this file declares plus the prefix: derived, never a guessed cap
  var maxnl: i64 = 0
  var m: i64 = 0
  while m < nf { if nlens[m] > maxnl { maxnl = nlens[m] } m = m + 1 }
  let tokcap: i64 = maxnl + ci_len(SS_DEF_PREFIX) + 1
  let tok: *u8 = sys_mmap(tokcap)
  var o: i64 = 0
  var k: i64 = 0
  while k < nf {
      let tl: i64 = ss_def_token((src as i64 + noffs[k]) as *u8, nlens[k], tok, tokcap)
      if o + tl + 1 < cap {
          var j: i64 = 0
          while j < tl { out[o] = tok[j]; o = o + 1; j = j + 1 }
          out[o] = SS_SP as u8; o = o + 1
      }
      k = k + 1
  }
  sys_munmap(starts, maxh * CI_WORD)
  sys_munmap(noffs, maxh * CI_WORD)
  sys_munmap(nlens, maxh * CI_WORD)
  sys_munmap(tok, tokcap)
  return o
279}
280// ONE FILE INTO THE WRITER -- the single ingest body. kind 1 = raw text, 2 = html. urlname is the url tail
281// appended to urlprefix (the bare file name for the flat walk, the relative path for the tree walk).
282func ci_ingest_file(st: *i64, path: *u8, kind: i64, urlprefix: *u8, urlname: *u8) -> i64 {
  let counts: *i64 = st[6] as *i64
  let h: *i64 = st[3] as *i64
  let pbox: *i64 = st[4] as *i64
  let lbox: *i64 = st[5] as *i64
  let key: *u8 = st[7] as *u8
  let ukey: *u8 = st[8] as *u8
  let urlval: *u8 = st[9] as *u8
  let hout: *u8 = st[10] as *u8
  let szp: *i64 = sys_mmap(CI_BOX) as *i64
  let raw: *u8 = ss_readall(path, szp)
  var rn: i64 = szp[0]
  if rn > CI_RAWCAP { rn = 0 }   // absurd file -> skip, never OOM
  if rn <= 0 { counts[2] = counts[2] + 1; return 0 }
  var txt: *u8 = raw
  var tn: i64 = rn
  if kind == 2 {
      let hr: i64 = nx_html_to_text(raw, rn, hout, CI_RAWCAP + CI_MAGIC_65536)
      if hr <= 0 { tn = 0 } else { txt = hout; tn = hr }
  }
  // E3: an append-only journal is anchored at its TAIL -- the newest CI_DOCCAP bytes -- because the head of an
  // append-only file is its past; the tail is the document that answers a question about now.
  if kind == CI_DOC_JOURNAL { if tn > CI_DOCCAP { txt = (raw as i64 + (tn - CI_DOCCAP)) as *u8; tn = CI_DOCCAP } }
  if tn > CI_DOCCAP { tn = CI_DOCCAP }
  if tn < CI_MINDOC { counts[2] = counts[2] + 1; ss_freeall(raw, szp[0]); return 0 }
  // IDENTIFIER ANCHORS (E1): the tree walk appends every underscored identifier's collapsed form after a newline,
  // so the tokenizer indexes the whole identifier and the file that defines it ranks first for it.
  var doc: *u8 = txt
  var dn: i64 = tn
  var ab: *u8 = 0 as *u8
  var abn: i64 = 0
  if st[11] == 1 {
      // three lengths: the text, its identifier anchors (never longer than the text) and its definition anchors
      // (one token per head, each shorter than the head it came from), plus the newline and slack
      abn = tn + tn + tn + CI_ANCHOR_PAD
      ab = sys_mmap(abn)
      var ao: i64 = 0
      var ci: i64 = 0
      while ci < tn { ab[ao] = txt[ci]; ao = ao + 1; ci = ci + 1 }
      ab[ao] = CI_LF as u8; ao = ao + 1
      ao = ao + ss_ident_anchors(txt, tn, (ab as i64 + ao) as *u8, abn - ao)
      ao = ao + ci_def_anchors(txt, tn, (ab as i64 + ao) as *u8, abn - ao)
      doc = ab
      dn = ao
      if dn > CI_DOCCAP { dn = CI_DOCCAP }
  }
  let r: i64 = ci_ingest_doc(st, doc, dn, urlprefix, urlname)
  // E3b: a board is also ingested ROW BY ROW; the row pass runs whether or not the whole-file document was present,
  // because the rows are content-addressed themselves (a present row costs one key lookup, never a second copy)
  if r >= 0 { if kind == CI_DOC_BOARD { ci_ingest_board_rows(st, txt, tn, urlprefix, urlname) } }
  // the writer copied the bytes it needs; the raw file and the anchored copy are released per document so a
  // tree of thousands of sources never becomes the load it indexes
  ss_freeall(raw, szp[0])
  if abn > 0 { sys_munmap(ab, abn) }
  return 0
337}
338// ONE DOCUMENT INTO THE STORE: the content-addressed skip, the path row's tombstones for a replaced document, the doc
339// row, the url row and the path row. Shared by the file ingest and the board-row ingest (E3b): two copies would agree
340// the day they were written and drift on the next edit. Returns 1 ingested, 0 present, -1 when the put failed.
341func ci_ingest_doc(st: *i64, doc: *u8, dn: i64, urlprefix: *u8, urlname: *u8) -> i64 {
  let counts: *i64 = st[6] as *i64
  let h: *i64 = st[3] as *i64
  let pbox: *i64 = st[4] as *i64
  let lbox: *i64 = st[5] as *i64
  let key: *u8 = st[7] as *u8
  let ukey: *u8 = st[8] as *u8
  let urlval: *u8 = st[9] as *u8
  let cid: i64 = ci_hash(doc, dn)
  dss_mkkey(cid, key)
  var present: i64 = 0
  if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { present = 1 } }
  if present == 1 { counts[1] = counts[1] + 1; return 0 }
  // THE PATH ROW (E1): when this path already stands for an older document, that document and its url row are
  // tombstoned before the replacement lands -- one live document per file, and a changed file is a replacement,
  // never a second copy beside the first.
  let pkey: *u8 = sys_mmap(CI_KEYBUF)
  var pathhash: i64 = 0
  if st[11] == 1 {
      pathhash = ci_hash(urlname, ci_len(urlname))
      ci_mkpthkey(pathhash, pkey)
      if (h as i64) != 0 { if ss_hget(h, pkey, pbox, lbox) == 1 {
          let oldcid: i64 = ci_atoi(pbox[0] as *u8, lbox[0])
          if oldcid != 0 { if oldcid != cid {
              let okey: *u8 = sys_mmap(CI_KEYBUF)
              dss_mkkey(oldcid, okey)
              ci_put(st, CI_KIND_TOMB, okey, 0 as *u8, 0)
              ci_mkurlkey(oldcid, okey)
              ci_put(st, CI_KIND_TOMB, okey, 0 as *u8, 0)
              counts[4] = counts[4] + 1
              sys_munmap(okey, CI_KEYBUF)
          } }
      } }
  }
  if ci_put(st, CI_KIND_LIVE, key, doc, dn) < 0 { counts[2] = counts[2] + 1; sys_munmap(pkey, CI_KEYBUF); return 0 - 1 }
  counts[0] = counts[0] + 1
  // NULL POINTER = no url rows. NEVER an empty-string literal here: `"" as *u8`
  // miscompiles to an unterminated pool pointer (the 2026-07-03 library-url bug --
  // 596 docs got "corpus-ingest: dir=..." hrefs; same codegen-gotcha family as the
  // inline cast-index one). Gate T8 pins the no-urlprefix path clean.
  if (urlprefix as i64) != 0 { if ci_len(urlprefix) > 0 {
      var uo: i64 = ci_cat(urlval, 0, urlprefix)
      uo = ci_cat(urlval, uo, urlname)
      ci_mkurlkey(cid, ukey)
      ci_put(st, CI_KIND_LIVE, ukey, urlval, uo)
  } }
  if st[11] == 1 {
      let pv: *u8 = sys_mmap(CI_DECBUF)
      let pvn: i64 = ci_dec(cid, pv)
      ci_put(st, CI_KIND_LIVE, pkey, pv, pvn)
      sys_munmap(pv, CI_DECBUF)
  }
  sys_munmap(pkey, CI_KEYBUF)
  return 1
395}
396// E3b BOARD ROWS AS DOCUMENTS: every non-empty, non-comment row of a .plan or .matrix is its own document, url =
397// the board url + CI_ROW_SEP + the row index, with its own path row (a changed row replaces itself), its identifier
398// anchors, and for a `rung|<id>|` row the definition token of the id -- so a rung id ranks its rung row first the way
399// a function name ranks its head. The whole-file document beside it still answers prose. Returns rows ingested.
400func ci_ingest_board_rows(st: *i64, txt: *u8, tn: i64, urlprefix: *u8, urlname: *u8) -> i64 {
  let counts: *i64 = st[6] as *i64
  let un: i64 = ci_len(urlname)
  let rncap: i64 = un + ci_len(CI_ROW_SEP) + CI_DECBUF + 1
  let rowname: *u8 = sys_mmap(rncap)
  let rungw: *u8 = CI_ROW_RUNG
  let rungl: i64 = ci_len(rungw)
  var rows: i64 = 0
  var idx: i64 = 0
  var i: i64 = 0
  while i < tn {
      var e: i64 = i
      var found: i64 = 0
      while found == 0 { if e >= tn { found = 1 } else { if (txt[e] as i64) == CI_LF { found = 1 } else { e = e + 1 } } }
      var le: i64 = e
      if le > i { if (txt[le - 1] as i64) == CI_CR { le = le - 1 } }
      let rl: i64 = le - i
      if rl >= CI_MINDOC { if (txt[i] as i64) != CI_CONF_HASH {
          let row: *u8 = (txt as i64 + i) as *u8
          // the row, a newline, its identifier anchors and (rung rows) the definition token of the id: three row
          // lengths plus slack, the same derived bound the file ingest uses
          let cap: i64 = rl + rl + rl + CI_ANCHOR_PAD
          let rb: *u8 = sys_mmap(cap)
          var o: i64 = 0
          var c: i64 = 0
          while c < rl { rb[o] = row[c]; o = o + 1; c = c + 1 }
          rb[o] = CI_LF as u8; o = o + 1
          o = o + ss_ident_anchors(row, rl, (rb as i64 + o) as *u8, cap - o)
          var isrung: i64 = 1
          if rl <= rungl { isrung = 0 }
          var q: i64 = 0
          while q < rungl { if isrung == 1 { if row[q] != rungw[q] { isrung = 0 } } q = q + 1 }
          if isrung == 1 {
              var idn: i64 = 0
              var stop: i64 = 0
              while stop == 0 { if rungl + idn >= rl { stop = 1 } else { if (row[rungl + idn] as i64) == CI_CONF_PIPE { stop = 1 } else { idn = idn + 1 } } }
              if idn > 0 { if o + idn + rungl + 1 < cap {
                  let dl: i64 = ss_def_token((row as i64 + rungl) as *u8, idn, (rb as i64 + o) as *u8, cap - o)
                  o = o + dl
                  rb[o] = SS_SP as u8; o = o + 1
              } }
          }
          var ro: i64 = ci_cat(rowname, 0, urlname)
          ro = ci_cat(rowname, ro, CI_ROW_SEP)
          ro = ro + ci_dec(idx, (rowname as i64 + ro) as *u8)
          rowname[ro] = 0 as u8
          if ci_ingest_doc(st, rb, o, urlprefix, rowname) == 1 { rows = rows + 1 }
          sys_munmap(rb, cap)
      } }
      idx = idx + 1
      i = e + 1
  }
  counts[5] = counts[5] + rows
  sys_munmap(rowname, rncap)
  return rows
455}
456
457// THE FLAT INGEST. Returns docs ingested (>=0), or -1 open-fail. counts as in ci_state_new.
458func ci_run(dir: *u8, domain: *u8, urlprefix: *u8, counts: *i64) -> i64 {
  let prefix: *u8 = sys_mmap(CI_PREFIXBUF)
  dss_prefix(domain, prefix)
  let st: *i64 = ci_state_new(prefix, counts)
  let fd: i64 = sys_openat_rd(dir)
  if fd < 0 { ci_puts("corpus-ingest: cannot open dir " as *u8); ci_puts(dir); ci_puts("\n" as *u8); return 0 - 1 }
  let dbuf: *u8 = sys_mmap(CI_MAGIC_131072)
  let path: *u8 = sys_mmap(CI_MAGIC_1024)
  var go: i64 = 1
  while go == 1 {
      let nr: i64 = sys_getdents64(fd, dbuf, CI_MAGIC_131072)
      if nr <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nr {
              let rec: *u8 = (dbuf as i64 + off) as *u8
              let ty: i64 = dirent_type(rec)
              let nm: *u8 = dirent_name(rec)
              off = off + dirent_reclen(rec)
              if ty != CI_DT_DIR {
                  let kind: i64 = ci_kind(nm)
                  if kind == 0 { counts[2] = counts[2] + 1 } else {
                      var po: i64 = ci_cat(path, 0, dir)
                      po = ci_cat(path, po, "/" as *u8)
                      po = ci_cat(path, po, nm)
                      path[po] = 0 as u8
                      ci_ingest_file(st, path, kind, urlprefix, nm)
                  }
              }
          }
      }
  }
  sys_close(fd)
  ci_state_finish(st)
  return counts[0]
492}
493
494// E1 -- THE TREE WALK: every source file under dir, recursively, urlname = the path relative to the root.
495// Returns files ingested or skipped-present (walked), so a caller can tell an empty tree from a refusal.
496func ci_walk_tree(st: *i64, dir: *u8, rel: *u8, depth: i64, urlprefix: *u8) -> i64 {
  if depth > CI_TREE_MAXDEPTH { ci_puts("corpus-ingest: tree deeper than CI_TREE_MAXDEPTH at " as *u8); ci_puts(dir); ci_puts(" -- not entered\n" as *u8); return 0 }
  let fd: i64 = sys_openat_rd(dir)
  if fd < 0 { return 0 }
  let counts: *i64 = st[6] as *i64
  let dbuf: *u8 = sys_mmap(CI_MAGIC_131072)
  let path: *u8 = sys_mmap(CI_MAGIC_1024)
  let sub: *u8 = sys_mmap(CI_MAGIC_1024)
  var walked: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let nr: i64 = sys_getdents64(fd, dbuf, CI_MAGIC_131072)
      if nr <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nr {
              let rec: *u8 = (dbuf as i64 + off) as *u8
              let ty: i64 = dirent_type(rec)
              let nm: *u8 = dirent_name(rec)
              off = off + dirent_reclen(rec)
              var po: i64 = ci_cat(path, 0, dir)
              po = ci_cat(path, po, "/" as *u8)
              po = ci_cat(path, po, nm)
              path[po] = 0 as u8
              var so: i64 = ci_cat(sub, 0, rel)
              if so > 0 { so = ci_cat(sub, so, "/" as *u8) }
              so = ci_cat(sub, so, nm)
              sub[so] = 0 as u8
              if ty == CI_DT_DIR {
                  if ci_tree_skipdir(nm) == 0 { walked = walked + ci_walk_tree(st, path, sub, depth + 1, urlprefix) }
              } else {
                  let kind: i64 = ci_kind_source(nm)
                  if kind == 0 { counts[2] = counts[2] + 1 } else { ci_ingest_file(st, path, kind, urlprefix, sub); walked = walked + 1 }
              }
          }
      }
  }
  sys_close(fd)
  sys_munmap(dbuf, CI_MAGIC_131072)
  sys_munmap(path, CI_MAGIC_1024)
  sys_munmap(sub, CI_MAGIC_1024)
  return walked
537}
538// E1 -- THE CODE BASE AS A CORPUS (search rung E1, 2026-09-14): every text file under a source tree becomes one
539// document in the domain's shard, its path relative to the root as the url under <urlprefix>, its underscored
540// identifiers anchored so a query for one finds the file that defines it, and a path row so a changed file
541// REPLACES its previous document. The shard is dp-<domain>-pub- like every other ingest; no public scope resolves
542// to the estate domain, so the corpus is reachable only through nx_estate_search until an auth-gated scope lands
543// (E4). Returns files walked (>= 0) or -1 when the root cannot be opened.
544func ci_ingest_source_tree(root: *u8, domain: *u8, urlprefix: *u8, counts: *i64) -> i64 {
  let prefix: *u8 = sys_mmap(CI_PREFIXBUF)
  dss_prefix(domain, prefix)
  let st: *i64 = ci_state_new(prefix, counts)
  st[11] = 1
  let fd0: i64 = sys_openat_rd(root)
  if fd0 < 0 { ci_puts("corpus-ingest: cannot open root " as *u8); ci_puts(root); ci_puts("\n" as *u8); return 0 - 1 }
  sys_close(fd0)
  let walked: i64 = ci_walk_tree(st, root, "" as *u8, 0, urlprefix)
  ci_state_finish(st)
  return walked
555}
556
557// E3 THE ESTATE'S OWN RECORD AS CORPORA: one conf row per tree (root|domain|urlprefix, knowledge/estate_corpora.conf;
558// a row whose first byte is the hash is a comment; a trailing CR is stripped so a CRLF conf parses clean), each row
559// walked through the ONE tree ingest above with the counts summed across rows. Returns the row count (0 = nothing
560// walked), -1 when the conf cannot be read. Adding a corpus is a data edit here, never a second organ or clock row.
561func ci_ingest_boards(confpath: *u8, counts: *i64) -> i64 {
  let szp: *i64 = sys_mmap(CI_BOX) as *i64
  let buf: *u8 = ss_readall(confpath, szp)
  let n: i64 = szp[0]
  if n <= 0 { return 0 - 1 }
  let root: *u8 = sys_mmap(CI_MAGIC_1024)
  let dom: *u8 = sys_mmap(CI_KEYBUF)
  let pfx: *u8 = sys_mmap(CI_MAGIC_1024)
  let rc: *i64 = sys_mmap(CI_KEYBUF) as *i64
  let words: i64 = CI_KEYBUF / CI_WORD
  var rows: i64 = 0
  var i: i64 = 0
  while i < n {
      var e: i64 = i
      var found: i64 = 0
      while found == 0 { if e >= n { found = 1 } else { if (buf[e] as i64) == CI_LF { found = 1 } else { e = e + 1 } } }
      var le: i64 = e
      if le > i { if (buf[le - 1] as i64) == CI_CR { le = le - 1 } }
      if le > i { if (buf[i] as i64) != CI_CONF_HASH {
          var f: i64 = 0
          var o: i64 = 0
          var k: i64 = i
          pfx[0] = 0 as u8
          while k < le {
              let c: i64 = buf[k] as i64
              if c == CI_CONF_PIPE {
                  if f == 0 { root[o] = 0 as u8 }
                  if f == 1 { dom[o] = 0 as u8 }
                  f = f + 1
                  o = 0
              } else {
                  if f == 0 { if o < CI_MAGIC_1024 - 1 { root[o] = c as u8; o = o + 1 } }
                  if f == 1 { if o < CI_KEYBUF - 1 { dom[o] = c as u8; o = o + 1 } }
                  if f == 2 { if o < CI_MAGIC_1024 - 1 { pfx[o] = c as u8; o = o + 1 } }
              }
              k = k + 1
          }
          if f == 0 { root[o] = 0 as u8 }
          if f == 1 { dom[o] = 0 as u8 }
          if f == 2 { pfx[o] = 0 as u8 }
          if f >= 1 {
              var z: i64 = 0
              while z < words { rc[z] = 0; z = z + 1 }
              var up: *u8 = pfx
              if pfx[0] == (0 as u8) { up = 0 as *u8 }
              ci_puts("  boards row root=" as *u8); ci_puts(root); ci_puts(" domain=" as *u8); ci_puts(dom)
              let walked: i64 = ci_ingest_source_tree(root, dom, up, rc)
              ci_puts(" walked=" as *u8); ci_num(walked); ci_puts(" ingested=" as *u8); ci_num(rc[0]); ci_puts(" replaced=" as *u8); ci_num(rc[4]); ci_puts(" rows=" as *u8); ci_num(rc[5]); ci_puts(" skipped_other=" as *u8); ci_num(rc[2]); ci_puts("\n" as *u8)
              var q: i64 = 0
              while q < words { counts[q] = counts[q] + rc[q]; q = q + 1 }
              rows = rows + 1
          }
      } }
      i = e + 1
  }
  ss_freeall(buf, szp[0])
  return rows
618}
619
620func main(argc: i64, argv: *i64) -> i64 {
  if argc < 3 { ci_puts("usage: nx_corpus_ingest <dir> <domain> [urlprefix] | nx_corpus_ingest tree <root> <domain> [urlprefix]\n" as *u8); return 1 }
  let a1: *u8 = argv[1] as *u8
  let treew: *u8 = "tree" as *u8
  var is_tree: i64 = 0
  if ci_len(a1) == ci_len(treew) { if ci_contains(a1, treew) == 1 { is_tree = 1 } }
  if is_tree == 1 {
      if argc < 4 { ci_puts("usage: nx_corpus_ingest tree <root> <domain> [urlprefix]\n" as *u8); return 1 }
      let root: *u8 = argv[2] as *u8
      let tdomain: *u8 = argv[3] as *u8
      var tprefix: *u8 = 0 as *u8
      if argc >= 5 { tprefix = argv[4] as *u8 }
      let tcounts: *i64 = sys_mmap(CI_KEYBUF) as *i64
      let walked: i64 = ci_ingest_source_tree(root, tdomain, tprefix, tcounts)
      if walked < 0 { return 2 }
      ci_puts("corpus-ingest: tree=" as *u8); ci_puts(root)
      ci_puts(" -> shard dp-" as *u8); ci_puts(tdomain); ci_puts("-pub-  walked=" as *u8); ci_num(walked)
      ci_puts(" ingested=" as *u8); ci_num(tcounts[0])
      ci_puts(" replaced=" as *u8); ci_num(tcounts[4])
      ci_puts(" skipped_present=" as *u8); ci_num(tcounts[1])
      ci_puts(" skipped_other=" as *u8); ci_num(tcounts[2])
      ci_puts(" segments=" as *u8); ci_num(tcounts[3])
      ci_puts(" rows=" as *u8); ci_num(tcounts[5])
      ci_puts("\n" as *u8)
      if tcounts[0] + tcounts[1] == 0 { ci_puts("corpus-ingest: NOTHING ingested (empty tree or all skipped)\n" as *u8); return 3 }
      return 0
  }
  let boardsw: *u8 = "boards" as *u8
  var is_boards: i64 = 0
  if ci_len(a1) == ci_len(boardsw) { if ci_contains(a1, boardsw) == 1 { is_boards = 1 } }
  if is_boards == 1 {
      let bcounts: *i64 = sys_mmap(CI_KEYBUF) as *i64
      let rows: i64 = ci_ingest_boards(argv[2] as *u8, bcounts)
      if rows < 0 { ci_puts("corpus-ingest: boards conf unreadable\n" as *u8); return 2 }
      ci_puts("corpus-ingest: boards conf=" as *u8); ci_puts(argv[2] as *u8)
      ci_puts(" rows=" as *u8); ci_num(rows)
      ci_puts(" ingested=" as *u8); ci_num(bcounts[0])
      ci_puts(" replaced=" as *u8); ci_num(bcounts[4])
      ci_puts(" skipped_present=" as *u8); ci_num(bcounts[1])
      ci_puts(" skipped_other=" as *u8); ci_num(bcounts[2])
      ci_puts(" segments=" as *u8); ci_num(bcounts[3])
      ci_puts(" rows=" as *u8); ci_num(bcounts[5])
      ci_puts("\n" as *u8)
      if rows == 0 { ci_puts("corpus-ingest: NOTHING ingested (no rows in the conf)\n" as *u8); return 3 }
      return 0
  }
  let dir: *u8 = a1
  let domain: *u8 = argv[2] as *u8
  var urlprefix: *u8 = 0 as *u8   // null = no url rows ("" literal miscompiles -- see ci_ingest_file)
  if argc >= 4 { urlprefix = argv[3] as *u8 }
  let counts: *i64 = sys_mmap(CI_KEYBUF) as *i64
  let rc: i64 = ci_run(dir, domain, urlprefix, counts)
  if rc < 0 { return 2 }
  ci_puts("corpus-ingest: dir=" as *u8); ci_puts(dir)
  ci_puts(" -> shard dp-" as *u8); ci_puts(domain); ci_puts("-pub-  ingested=" as *u8); ci_num(counts[0])
  ci_puts(" skipped_present=" as *u8); ci_num(counts[1])
  ci_puts(" skipped_other=" as *u8); ci_num(counts[2])
  ci_puts(" segments=" as *u8); ci_num(counts[3])
  ci_puts("\n" as *u8)
  if counts[0] + counts[1] == 0 { ci_puts("corpus-ingest: NOTHING ingested (empty dir or all skipped)\n" as *u8); return 3 }
  return 0
681}
nx_corpus_ingest.nx source

`nx_corpus_ingest.nx` source
