code wiki / _hdl_build / nx_corpus_ingest_gate.nx
`nx_corpus_ingest_gate.nx` source

↩ module page · 341 lines · 22657 B
1// nx_corpus_ingest_gate.nx -- GATE for the corpus-ingest front door (dir -> searchable sovereign shard).
2// Builds a real test dir (/tmp) with: a raw .txt, an .html (tags/script/entities exercise nx_html_to_text),
3// a .bak (must be skipped), and TWO ~600KB files (forces the 1MB writer to batch across >=2 segments).
4// Assertions are RUN-INDEPENDENT (content-addressed skip-if-present makes re-runs clean): search truths +
5// the persisted manifest segment count, never per-run counters alone. On the gate base class (gv_check /
6// gv_verdict) since 2026-09-14 so /api/promote and /api/gate_run read the verdict from the exit code.
7// license_tier: ORIGINAL  expect_exit: 0
8import "nx_corpus_ingest.nx"
9import "nx_g_puts_lib.nx"
10import "nx_gate_verdict.nx"
11
12func g_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
13func g_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
21}
22func g_contains(hay: *u8, hn: i64, needle: *u8) -> i64 {
  let nl: i64 = g_len(needle)
  if nl == 0 { return 1 }
  var i: i64 = 0
  while i + nl <= hn {
      var m: i64 = 1
      var j: i64 = 0
      while j < nl { if hay[i + j] != needle[j] { m = 0; j = nl } else { j = j + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
34}
35func g_write(path: *u8, data: *u8, n: i64) -> i64 {
  let fd: i64 = sys_openat_wr(path, 0x1a4)
  if fd < 0 { return 0 - 1 }
  sys_write(fd, data, n); sys_close(fd); return 0
39}
40
41// T9 fixture values, NAMED because the build ratchet counts inline literals
42const G_MODE_DIR: i64 = 0x1ed
43const G_COUNTS: i64 = 64
44const G_PREFIXBUF: i64 = 512
45const G_KEYBUF: i64 = 64
46const G_BOX: i64 = 16
47const G_TOP: i64 = 10
48const G_TREE_PRESENT: i64 = 7   // the tree fixture on a re-run: 4 files present (x.nx y.md w.nx b.plan) + b.plan's 3 rows (E3b), rows are content-addressed documents too
49const G_JOURNAL_BYTES: i64 = 1000000   // T12: a journal fixture LARGER than CI_DOCCAP, so only its tail window can be the document
50const G_JOURNAL_TAILROOM: i64 = 32     // bytes held back for the tail marker
51
52func main() -> i64 {
  g_puts("=== nx_corpus_ingest gate (dir -> sovereign searchable shard; html->text; url rows; batching; tree walk; anchors) ===\n" as *u8)
  let ctr: *i64 = gv_ctr()
  let dir: *u8 = "/tmp/ci_gate_dir" as *u8
  sys_mkdir(dir, 0x1ed)   // exists-already is fine; files are rewritten each run
  let fa: *u8 = "Alpha zenithterm ingestion check for the sovereign corpus front door. Tokenizer needs words." as *u8
  g_write("/tmp/ci_gate_dir/a.txt" as *u8, fa, g_len(fa))
  let fb: *u8 = "<html><head><style>x{color:red}</style></head><body><h1>Beta estateterm page</h1><p>Planning &amp; probate content here.</p><script>evilmarker()</script></body></html>" as *u8
  g_write("/tmp/ci_gate_dir/b.html" as *u8, fb, g_len(fb))
  let fc: *u8 = "bakmarkerterm should never be indexed from a .bak file" as *u8
  g_write("/tmp/ci_gate_dir/c.bak" as *u8, fc, g_len(fc))
  // two ~600KB files -> the 1MB writer MUST split them across segments
  let big: *u8 = sys_mmap(700000)
  var bo: i64 = 0
  var r: i64 = 0
  while r < 30000 { bo = bo + 0; let w: *u8 = "filler corpus words " as *u8; var j: i64 = 0; while w[j] != (0 as u8) { big[bo] = w[j]; bo = bo + 1; j = j + 1 } r = r + 1 }
  let m1: *u8 = "bigsegterm1 " as *u8
  var j1: i64 = 0; while m1[j1] != (0 as u8) { big[bo] = m1[j1]; bo = bo + 1; j1 = j1 + 1 }
  g_write("/tmp/ci_gate_dir/big1.txt" as *u8, big, bo)
  // swap the unique marker for the second file (same filler, different tail = different cid)
  var bo2: i64 = bo - 12
  let m2: *u8 = "bigsegterm2 " as *u8
  var j2: i64 = 0; while m2[j2] != (0 as u8) { big[bo2] = m2[j2]; bo2 = bo2 + 1; j2 = j2 + 1 }
  g_write("/tmp/ci_gate_dir/big2.txt" as *u8, big, bo2)
76
  let dom: *u8 = "cigatetest" as *u8
  let counts: *i64 = sys_mmap(64) as *i64
  let rc: i64 = ci_run(dir, dom, "/" as *u8, counts)
  g_puts("  ci_run rc=" as *u8); g_num(rc)
  g_puts(" ingested=" as *u8); g_num(counts[0]); g_puts(" present=" as *u8); g_num(counts[1])
  g_puts(" other=" as *u8); g_num(counts[2]); g_puts(" segs=" as *u8); g_num(counts[3]); g_puts("\n" as *u8)
83
  let cids: *i64 = sys_mmap(64 * 8) as *i64
  let scores: *i64 = sys_mmap(64 * 8) as *i64
86
  // T1: the raw .txt doc is searchable
  let n1: i64 = dss_search(dom, "zenithterm" as *u8, 10, cids, scores, 10)
  gv_check("T1 raw .txt searchable" as *u8, (n1 == 1) as i64, ctr)
90
  // T2: the .html doc was TEXT-EXTRACTED (searchable term; no tags; script suppressed; entity decoded)
  let n2: i64 = dss_search(dom, "estateterm" as *u8, 10, cids, scores, 10)
  var t2: i64 = 0
  if n2 == 1 {
      let prefix: *u8 = sys_mmap(512); dss_prefix(dom, prefix)
      let h: *i64 = ss_open(prefix)
      let key: *u8 = sys_mmap(64); dss_mkkey(cids[0], key)
      let dp: *i64 = sys_mmap(16) as *i64; let dl: *i64 = sys_mmap(16) as *i64
      if (h as i64) != 0 { if ss_hget(h, key, dp, dl) == 1 {
          let txt: *u8 = dp[0] as *u8
          if g_contains(txt, dl[0], "Beta estateterm page" as *u8) == 1 {
              if g_contains(txt, dl[0], "<h1>" as *u8) == 0 {
                  if g_contains(txt, dl[0], "evilmarker" as *u8) == 0 {
                      if g_contains(txt, dl[0], "Planning & probate" as *u8) == 1 { t2 = 1 }
                  }
              }
          }
      } }
  }
  gv_check("T2 html->text (tags gone, script suppressed, entity decoded)" as *u8, t2, ctr)
111
  // T3: the html doc carries its url:<cid> row = "/b.html" (SERP links to the real page)
  var t3: i64 = 0
  if n2 == 1 {
      let prefix3: *u8 = sys_mmap(512); dss_prefix(dom, prefix3)
      let h3: *i64 = ss_open(prefix3)
      let uk: *u8 = sys_mmap(64); ci_mkurlkey(cids[0], uk)
      let up: *i64 = sys_mmap(16) as *i64; let ul: *i64 = sys_mmap(16) as *i64
      if (h3 as i64) != 0 { if ss_hget(h3, uk, up, ul) == 1 {
          if ul[0] == 7 { if g_contains(up[0] as *u8, ul[0], "/b.html" as *u8) == 1 { t3 = 1 } }
      } }
  }
  gv_check("T3 url:<cid> row = /b.html" as *u8, t3, ctr)
124
  // T4 NEG: the .bak file was never indexed
  let n4: i64 = dss_search(dom, "bakmarkerterm" as *u8, 13, cids, scores, 10)
  gv_check("T4 neg-control-.bak skipped" as *u8, (n4 == 0) as i64, ctr)
128
  // T5: BOTH big docs searchable (the writer batched across segments without losing either)
  let n5a: i64 = dss_search(dom, "bigsegterm1" as *u8, 11, cids, scores, 10)
  let n5b: i64 = dss_search(dom, "bigsegterm2" as *u8, 11, cids, scores, 10)
  gv_check("T5 both >0.5MB docs searchable across segments" as *u8, ((n5a == 1) as i64) * ((n5b == 1) as i64), ctr)
133
  // T6: the shard REALLY spans >=2 segments (persisted manifest truth, run-independent)
  let prefix6: *u8 = sys_mmap(512); dss_prefix(dom, prefix6)
  let segs: *i64 = sys_mmap(256 * 8) as *i64
  let nseg: i64 = ss_manifest_file(prefix6, "manifest.txt" as *u8, segs)
  gv_check("T6 manifest >=2 segments" as *u8, (nseg >= 2) as i64, ctr)
139
  // T7: idempotent re-run -- nothing double-ingested (zenithterm still exactly 1 hit)
  let counts2: *i64 = sys_mmap(64) as *i64
  ci_run(dir, dom, "/" as *u8, counts2)
  let n7: i64 = dss_search(dom, "zenithterm" as *u8, 10, cids, scores, 10)
  var t7: i64 = 0
  if n7 == 1 { if counts2[0] == 0 { if counts2[1] >= 4 { t7 = 1 } } }
  gv_check("T7 idempotent re-run (0 new, all present, still 1 hit)" as *u8, t7, ctr)
147
  // T8 NEG (the 2026-07-03 library-url regression): NO urlprefix (null) -> NO url row may exist.
  // The `"" as *u8` empty literal miscompiled to an unterminated pool pointer and silently stamped
  // every doc with a junk href -- this row pins the no-urlprefix path clean forever.
  let dom2: *u8 = "cigatetest2" as *u8
  let counts3: *i64 = sys_mmap(64) as *i64
  ci_run(dir, dom2, 0 as *u8, counts3)
  let n8b: i64 = dss_search(dom2, "estateterm" as *u8, 10, cids, scores, 10)
  var t8: i64 = 0
  if n8b == 1 {
      let prefix8: *u8 = sys_mmap(512); dss_prefix(dom2, prefix8)
      let h8: *i64 = ss_open(prefix8)
      let uk8: *u8 = sys_mmap(64); ci_mkurlkey(cids[0], uk8)
      let up8: *i64 = sys_mmap(16) as *i64; let ul8: *i64 = sys_mmap(16) as *i64
      if (h8 as i64) != 0 { if ss_hget(h8, uk8, up8, ul8) != 1 { t8 = 1 } }
  }
  gv_check("T8 neg-control-null urlprefix -> NO url row stamped" as *u8, t8, ctr)
164
  // T9 (E1): the recursive source-tree walk -- nested dirs entered, the relative path as the url, fossil and
  // hidden dirs never entered, and the whole thing idempotent on a re-run. Fixture files are rewritten each run.
  // A FRESH FIXTURE SHARD EVERY RUN: the previous manifest is RETIRED (renamed, never deleted) before the tree
  // ingest, or documents left by an earlier binary -- without anchors or path rows -- would sit beside this run's
  // and no count below could hold. A gate that is not idempotent reports on its first run and lies on every run after.
  let dom9: *u8 = "cigatetree" as *u8
  let prefix9r: *u8 = sys_mmap(G_PREFIXBUF); dss_prefix(dom9, prefix9r)
  let mf9: *u8 = sys_mmap(G_PREFIXBUF)
  var mo9: i64 = ci_cat(mf9, 0, prefix9r); mo9 = ci_cat(mf9, mo9, "manifest.txt" as *u8); mf9[mo9] = 0 as u8
  let mr9: *u8 = sys_mmap(G_PREFIXBUF)
  var ro9: i64 = ci_cat(mr9, 0, prefix9r); ro9 = ci_cat(mr9, ro9, "manifest.retired" as *u8); mr9[ro9] = 0 as u8
  sys_renameat(mf9, mr9)
  let tr: *u8 = "/tmp/ci_gate_tree" as *u8
  sys_mkdir(tr, G_MODE_DIR)
  // the T12 journal lives in ITS OWN root; a first cut wrote it into this tree, where it survived to the next run and
  // moved T9f's present count from three to four -- a fixture that leaks across runs lies on every run after the first.
  // Any stray copy from that cut is moved out (a rename of an absent file is a harmless refusal).
  sys_mkdir("/tmp/ci_gate_jrnl" as *u8, G_MODE_DIR)
  sys_renameat("/tmp/ci_gate_tree/beat.jrnl" as *u8, "/tmp/ci_gate_jrnl/beat.jrnl.stray" as *u8)
  sys_mkdir("/tmp/ci_gate_tree/sub" as *u8, G_MODE_DIR)
  sys_mkdir("/tmp/ci_gate_tree/sub/deep" as *u8, G_MODE_DIR)
  sys_mkdir("/tmp/ci_gate_tree/_build" as *u8, G_MODE_DIR)
  sys_mkdir("/tmp/ci_gate_tree/.hidden" as *u8, G_MODE_DIR)
  let tx: *u8 = "// a source file for the tree walk\nfunc treeterm_alpha() -> i64 { return 1 }   // treewalkterm marks this file\n" as *u8
  g_write("/tmp/ci_gate_tree/sub/deep/x.nx" as *u8, tx, g_len(tx))
  let ty: *u8 = "treemdterm is a markdown note under the root of the tree walk fixture" as *u8
  g_write("/tmp/ci_gate_tree/y.md" as *u8, ty, g_len(ty))
  let tz: *u8 = "buildfossilterm must never be indexed from a _build directory of the tree" as *u8
  g_write("/tmp/ci_gate_tree/_build/z.nx" as *u8, tz, g_len(tz))
  let th: *u8 = "hiddenterm must never be indexed from a hidden directory of the tree" as *u8
  g_write("/tmp/ci_gate_tree/.hidden/h.nx" as *u8, th, g_len(th))
  // a directory whose name merely ENDS with _build must be entered: the skip list matches whole names
  sys_mkdir("/tmp/ci_gate_tree/_hdl_build" as *u8, G_MODE_DIR)
  let tw: *u8 = "hdlbuildterm lives under a directory whose name only ends with build and must be indexed" as *u8
  g_write("/tmp/ci_gate_tree/_hdl_build/w.nx" as *u8, tw, g_len(tw))
  // E3b fixture board: a comment row, a rung row, a rungrole row and a log row; the rung row earns the definition
  // token of its id and every row is its own document beside the whole file
  let tb: *u8 = "# fixture board\nrung|ZZ9|A fixture rung with a title|zz_symbol|note text for the fixture rung row\nrungrole|ZZ9|substrate|T-FX|why the fixture rung stands where it stands\nlog|1|ZZ9|measure|ZZ9 ZZ9 ZZ9 mentioned thrice in a measure row about the fixture rung\n" as *u8
  g_write("/tmp/ci_gate_tree/b.plan" as *u8, tb, g_len(tb))
  let counts9: *i64 = sys_mmap(G_COUNTS) as *i64
  let walked9: i64 = ci_ingest_source_tree(tr, dom9, "estate://tree/" as *u8, counts9)
  g_puts("  ci_ingest_source_tree walked=" as *u8); g_num(walked9); g_puts(" ingested=" as *u8); g_num(counts9[0]); g_puts(" present=" as *u8); g_num(counts9[1]); g_puts(" other=" as *u8); g_num(counts9[2]); g_puts(" segs=" as *u8); g_num(counts9[3]); g_puts("\n" as *u8)
  let q9: *u8 = "treewalkterm" as *u8
  let n9: i64 = dss_search(dom9, q9, g_len(q9), cids, scores, G_TOP)
  gv_check("T9 a source file two directories down is searchable" as *u8, (n9 == 1) as i64, ctr)
  var t9b: i64 = 0
  if n9 == 1 {
      let prefix9: *u8 = sys_mmap(G_PREFIXBUF); dss_prefix(dom9, prefix9)
      let h9: *i64 = ss_open(prefix9)
      let uk9: *u8 = sys_mmap(G_KEYBUF); ci_mkurlkey(cids[0], uk9)
      let up9: *i64 = sys_mmap(G_BOX) as *i64; let ul9: *i64 = sys_mmap(G_BOX) as *i64
      let want9: *u8 = "estate://tree/sub/deep/x.nx" as *u8
      if (h9 as i64) != 0 { if ss_hget(h9, uk9, up9, ul9) == 1 {
          if ul9[0] == g_len(want9) { if g_contains(up9[0] as *u8, ul9[0], want9) == 1 { t9b = 1 } }
      } }
  }
  gv_check("T9b its url row is the path relative to the root under the prefix" as *u8, t9b, ctr)
  let q9c: *u8 = "treemdterm" as *u8
  let n9c: i64 = dss_search(dom9, q9c, g_len(q9c), cids, scores, G_TOP)
  gv_check("T9c a markdown file at the root is searchable" as *u8, (n9c == 1) as i64, ctr)
  let q9d: *u8 = "buildfossilterm" as *u8
  let n9d: i64 = dss_search(dom9, q9d, g_len(q9d), cids, scores, G_TOP)
  gv_check("T9d neg-control-a _build directory is never entered" as *u8, (n9d == 0) as i64, ctr)
  let q9e: *u8 = "hiddenterm" as *u8
  let n9e: i64 = dss_search(dom9, q9e, g_len(q9e), cids, scores, G_TOP)
  gv_check("T9e neg-control-a hidden directory is never entered" as *u8, (n9e == 0) as i64, ctr)
  let q9g: *u8 = "hdlbuildterm" as *u8
  let n9g: i64 = dss_search(dom9, q9g, g_len(q9g), cids, scores, G_TOP)
  gv_check("T9g a directory whose name only ends with build IS entered (the skip list matches whole names)" as *u8, (n9g == 1) as i64, ctr)
  let counts9b: *i64 = sys_mmap(G_COUNTS) as *i64
  ci_ingest_source_tree(tr, dom9, "estate://tree/" as *u8, counts9b)
  let n9f: i64 = dss_search(dom9, q9, g_len(q9), cids, scores, G_TOP)
  var t9f: i64 = 0
  if n9f == 1 { if counts9b[0] == 0 { if counts9b[1] == G_TREE_PRESENT { t9f = 1 } } }
  gv_check("T9f idempotent re-run (0 new, both present, still 1 hit)" as *u8, t9f, ctr)
  // T10 (E1): identifier anchors and the path row -- a changed file REPLACES its document, and an underscored
  // identifier is searchable whole through its collapsed anchor
  let q10: *u8 = "treetermalpha" as *u8
  let n10: i64 = dss_search(dom9, q10, g_len(q10), cids, scores, G_TOP)
  gv_check("T10 the collapsed identifier of func tree_term_alpha is an anchor the walk indexed" as *u8, (n10 == 1) as i64, ctr)
  let tx2: *u8 = "// the same file, rewritten\nfunc tree_term_alpha() -> i64 { return 2 }   // treeretermwalk marks the second version\n" as *u8
  g_write("/tmp/ci_gate_tree/sub/deep/x.nx" as *u8, tx2, g_len(tx2))
  let counts10: *i64 = sys_mmap(G_COUNTS) as *i64
  ci_ingest_source_tree(tr, dom9, "estate://tree/" as *u8, counts10)
  g_puts("  re-ingest after a change: ingested=" as *u8); g_num(counts10[0]); g_puts(" replaced=" as *u8); g_num(counts10[4]); g_puts(" present=" as *u8); g_num(counts10[1]); g_puts("\n" as *u8)
  gv_check("T10b the changed file counts as ONE replacement" as *u8, (counts10[4] == 1) as i64, ctr)
  let q10c: *u8 = "treeretermwalk" as *u8
  let n10c: i64 = dss_search(dom9, q10c, g_len(q10c), cids, scores, G_TOP)
  gv_check("T10c the new version is searchable" as *u8, (n10c == 1) as i64, ctr)
  let n10d: i64 = dss_search(dom9, q9, g_len(q9), cids, scores, G_TOP)
  gv_check("T10d neg-control-the old version is tombstoned: its own marker answers zero" as *u8, (n10d == 0) as i64, ctr)
  let n10e: i64 = dss_search(dom9, q10, g_len(q10), cids, scores, G_TOP)
  gv_check("T10e the identifier anchor still answers exactly one document, the replacement" as *u8, (n10e == 1) as i64, ctr)
  // T13 (E3b): board rows as documents -- the fixture board's rung row is its own document with the row index in
  // its url and carries the definition token of its id; the whole file is still a document of its own.
  let q13: *u8 = "defzz9" as *u8
  let n13: i64 = dss_search(dom9, q13, g_len(q13), cids, scores, G_TOP)
  gv_check("T13 the rung row's definition token is indexed exactly once (the rung row document)" as *u8, (n13 == 1) as i64, ctr)
  var t13b: i64 = 0
  if n13 == 1 {
      let prefix13: *u8 = sys_mmap(G_PREFIXBUF); dss_prefix(dom9, prefix13)
      let h13: *i64 = ss_open(prefix13)
      let uk13: *u8 = sys_mmap(G_KEYBUF); ci_mkurlkey(cids[0], uk13)
      let up13: *i64 = sys_mmap(G_BOX) as *i64; let ul13: *i64 = sys_mmap(G_BOX) as *i64
      let want13: *u8 = "estate://tree/b.plan#1" as *u8
      if (h13 as i64) != 0 { if ss_hget(h13, uk13, up13, ul13) == 1 {
          if ul13[0] == g_len(want13) { if g_contains(up13[0] as *u8, ul13[0], want13) == 1 { t13b = 1 } }
      } }
  }
  gv_check("T13b the rung row's url is the board url with its row index (the comment row is row 0)" as *u8, t13b, ctr)
  let q13c: *u8 = "zz9" as *u8
  let n13c: i64 = dss_search(dom9, q13c, g_len(q13c), cids, scores, G_TOP)
  gv_check("T13c the id is found in the whole-file document AND its rows (at least four documents)" as *u8, (n13c >= 4) as i64, ctr)
  // T11 (E1b): a definition token per function head -- the def prefix glued to the collapsed name -- so the file
  // that DECLARES a symbol carries a term no caller carries. The rewritten x.nx declares tree_term_alpha; the word
  // treeretermwalk in the same file is not a head and must earn no token.
  let q11: *u8 = "deftreetermalpha" as *u8
  let n11: i64 = dss_search(dom9, q11, g_len(q11), cids, scores, G_TOP)
  gv_check("T11 the definition token of func tree_term_alpha is an anchor the walk indexed" as *u8, (n11 == 1) as i64, ctr)
  let q11b: *u8 = "deftreeretermwalk" as *u8
  let n11b: i64 = dss_search(dom9, q11b, g_len(q11b), cids, scores, G_TOP)
  gv_check("T11b neg-control-a-word-that-is-not-a-function-head earns no definition token" as *u8, (n11b == 0) as i64, ctr)
  // T12 (E3): the boards verb -- one conf row per tree (root|domain|urlprefix, a comment row ignored, a CRLF row
  // tolerated), every row walked through the ONE tree ingest with the counts summed; and an append-only journal is
  // anchored at its TAIL (the newest CI_DOCCAP bytes), so a marker in the head of an oversized journal is NOT indexed
  // while one in its tail is. Fresh shard every run, the same retire-the-manifest discipline as T9.
  let domb: *u8 = "cigateboards" as *u8
  let prefixbr: *u8 = sys_mmap(G_PREFIXBUF); dss_prefix(domb, prefixbr)
  let mfb: *u8 = sys_mmap(G_PREFIXBUF)
  var mob: i64 = ci_cat(mfb, 0, prefixbr); mob = ci_cat(mfb, mob, "manifest.txt" as *u8); mfb[mob] = 0 as u8
  let mrb: *u8 = sys_mmap(G_PREFIXBUF)
  var rob: i64 = ci_cat(mrb, 0, prefixbr); rob = ci_cat(mrb, rob, "manifest.retired" as *u8); mrb[rob] = 0 as u8
  sys_renameat(mfb, mrb)
  let jb: *u8 = sys_mmap(G_JOURNAL_BYTES)
  let jh: *u8 = "jrnlheadterm " as *u8
  var jo: i64 = 0
  var jhi: i64 = 0
  while jh[jhi] != (0 as u8) { jb[jo] = jh[jhi]; jo = jo + 1; jhi = jhi + 1 }
  while jo < G_JOURNAL_BYTES - G_JOURNAL_TAILROOM { let w2: *u8 = "journal filler row " as *u8; var wj: i64 = 0; while w2[wj] != (0 as u8) { jb[jo] = w2[wj]; jo = jo + 1; wj = wj + 1 } }
  let jt: *u8 = " jrnltailterm\n" as *u8
  var jti: i64 = 0
  while jt[jti] != (0 as u8) { jb[jo] = jt[jti]; jo = jo + 1; jti = jti + 1 }
  g_write("/tmp/ci_gate_jrnl/beat.jrnl" as *u8, jb, jo)
  let cf0: *u8 = "# boards fixture: three roots, one comment row\n/tmp/ci_gate_tree|cigateboards|estate://b1/" as *u8
  let cf1: *u8 = "\n/tmp/ci_gate_dir|cigateboards|estate://b2/\n/tmp/ci_gate_jrnl|cigateboards|estate://b3/\n" as *u8
  let cfb: *u8 = sys_mmap(G_PREFIXBUF)
  var co: i64 = 0
  var c0: i64 = 0
  while cf0[c0] != (0 as u8) { cfb[co] = cf0[c0]; co = co + 1; c0 = c0 + 1 }
  cfb[co] = CI_CR as u8; co = co + 1
  var c1: i64 = 0
  while cf1[c1] != (0 as u8) { cfb[co] = cf1[c1]; co = co + 1; c1 = c1 + 1 }
  g_write("/tmp/ci_gate_boards.conf" as *u8, cfb, co)
  let counts12: *i64 = sys_mmap(G_COUNTS) as *i64
  let rows12: i64 = ci_ingest_boards("/tmp/ci_gate_boards.conf" as *u8, counts12)
  g_puts("  ci_ingest_boards rows=" as *u8); g_num(rows12); g_puts(" ingested=" as *u8); g_num(counts12[0]); g_puts(" other=" as *u8); g_num(counts12[2]); g_puts("\n" as *u8)
  gv_check("T12 the boards verb walks every conf row (three roots) and ignores the comment row" as *u8, (rows12 == 3) as i64, ctr)
  let n12b: i64 = dss_search(domb, "zenithterm" as *u8, 10, cids, scores, G_TOP)
  gv_check("T12b the second root's raw text document is searchable under the boards domain" as *u8, (n12b == 1) as i64, ctr)
  let n12c: i64 = dss_search(domb, q10c, g_len(q10c), cids, scores, G_TOP)
  gv_check("T12c the first root's source file is searchable under the boards domain (its CRLF row parsed clean)" as *u8, (n12c == 1) as i64, ctr)
  let q12d: *u8 = "jrnltailterm" as *u8
  let n12d: i64 = dss_search(domb, q12d, g_len(q12d), cids, scores, G_TOP)
  gv_check("T12d a journal's TAIL marker is indexed" as *u8, (n12d == 1) as i64, ctr)
  let q12e: *u8 = "jrnlheadterm" as *u8
  let n12e: i64 = dss_search(domb, q12e, g_len(q12e), cids, scores, G_TOP)
  gv_check("T12e neg-control-a-journal-head-marker-past-the-tail-window is not indexed" as *u8, (n12e == 0) as i64, ctr)
332
  gv_values_head()
  gv_kv("dir_ingested" as *u8, counts[0])
  gv_kv("boards_rows" as *u8, rows12)
  gv_kv("dir_segments" as *u8, nseg)
  gv_kv("tree_walked" as *u8, walked9)
  gv_kv("tree_ingested" as *u8, counts9[0])
  gv_kv("tree_replaced" as *u8, counts10[4])
  return gv_verdict("NX-CORPUS-INGEST-GATE" as *u8, ctr, "the corpus front door: a flat dir and a source tree into a searchable sovereign shard with url rows, identifier and definition anchors, replacement tombstones and idempotent re-runs" as *u8)
341}
nx_corpus_ingest_gate.nx source

`nx_corpus_ingest_gate.nx` source
