code wiki / _hdl_build / nx_estate_search_gate.nx
`nx_estate_search_gate.nx` source

↩ module page · 231 lines · 18074 B
1// nx_estate_search_gate.nx -- teeth for the estate query (nx_estate_search_lib.es_query) on a fixture shard it
2// writes itself through the seg store: two documents under a fixture domain, one with a url row and one without,
3// one carrying control bytes; then (E1b) a definer and a caller of one identifier, where BM25 alone ranks the
4// caller first and the definition tier ranks the definer first. No fork, no live shard. Declared in
5// knowledge/organ_gate.conf because the one-segment-strip convention resolved nx_estate_search to nx_estate_gate --
6// the WILLS gate -- on the first ship, a name classifier keying on a prefix. license_tier: ORIGINAL  expect_exit: 0
7import "nx_estate_search_lib.nx"
8import "nx_gate_verdict.nx"
9import "nx_uxf_decode.nx"   // E8: the round-trip witness reads the envelope back
10
11const GE_DOMAIN: *u8 = "esgatetest"
12const GE_NONE: *u8 = "esgatenone-never-ingested"
13const GE_OUTCAP: i64 = 16384
14const GE_TOP: i64 = 5
15const GE_PREFIXBUF: i64 = 512
16const GE_KEYBUF: i64 = 64
17const GE_BOX: i64 = 16
18const GE_ONE: i64 = 1
19
20func ge_write_doc(w: *i64, prefix: *u8, txt: *u8, url: *u8) -> i64 {
  let n: i64 = es_slen(txt)
  let cid: i64 = ci_hash_like(txt, n)
  let key: *u8 = sys_mmap(GE_KEYBUF)
  dss_mkkey(cid, key)
  ss_add(w, GE_ONE, key, txt, n)
  if (url as i64) != 0 {
      let ukey: *u8 = sys_mmap(GE_KEYBUF)
      dsv_mkurlkey(cid, ukey)
      ss_add(w, GE_ONE, ukey, url, es_slen(url))
  }
  return cid
32}
33// the ingest's content id is a polynomial hash the seg store does not export; the gate needs ANY stable id per
34// document, and the url/doc keys are looked up by the id the gate itself wrote, so the polynomial here need not
35// match the ingest's -- it only has to be deterministic. Positive by construction.
36func ci_hash_like(s: *u8, n: i64) -> i64 {
  var h: i64 = 1125899906842597
  var i: i64 = 0
  while i < n { h = (h * 131) + (s[i] as i64); i = i + 1 }
  if h < 0 { h = 0 - h }
  return h & 0x7fffffffffffffff
42}
43// does the FIRST rendered line (up to its newline) carry the needle? The rank-1 witness for the tier teeth.
44func ge_first_line_has(out: *u8, n: i64, needle: *u8) -> i64 {
  // the newline's position is the answer, so the loop exits on a FLAG and never writes a sentinel into its cursor
  // (a sentinel past the end erased the position and made this witness search the whole output -- caught 2026-09-14)
  var e: i64 = 0
  var found: i64 = 0
  while found == 0 { if e >= n { found = 1 } else { if out[e] == (ES_LF as u8) { found = 1 } else { e = e + 1 } } }
  if es_find(out, e, needle) >= 0 { return 1 }
  return 0
52}
53
54// E8 witnesses: byte equality over a binary buffer (canon bytes carry NULs inside their length fields, so a
55// NUL-terminated find cannot judge them), and a decoded field lookup on the canon lib's own byte order (cc_cmp).
56func ge_bytes_eq(a: *u8, b: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n { if a[i] != b[i] { return 0 } i = i + 1 }
  return 1
60}
61func ge_field_is(keys: *i64, vals: *i64, n: i64, key: *u8, val: *u8) -> i64 {
  var i: i64 = 0
  while i < n {
      if cc_cmp(keys[i] as *u8, key) == 0 { if cc_cmp(vals[i] as *u8, val) == 0 { return 1 } }
      i = i + 1
  }
  return 0
68}
69
70func main() -> i64 {
  gv_puts("=== nx_estate_search gate (es_query on a fixture shard: url row, doc fallback, snippet folding, unreadable shard, definition precedence) ===\n" as *u8)
  let ctr: *i64 = gv_ctr()
  let prefix: *u8 = sys_mmap(GE_PREFIXBUF)
  dss_prefix(GE_DOMAIN, prefix)
  let segid: i64 = ss_next_segid(prefix)
  let w: *i64 = ss_begin()
  let ta: *u8 = "alphaestateterm lives in file one of the estate search fixture and this is its text" as *u8
  let ua: *u8 = "estate://fx/one.nx" as *u8
  ge_write_doc(w, prefix, ta, ua)
  let tb: *u8 = "betaestateterm lives in file two of the fixture with no url row of its own" as *u8
  ge_write_doc(w, prefix, tb, 0 as *u8)
  let tc: *u8 = "gammaestateterm has a newline\nand a tab\tinside its first bytes for the snippet fold" as *u8
  ge_write_doc(w, prefix, tc, 0 as *u8)
  let crc: i64 = ss_commit(prefix, w, segid)
  gv_check("T1 fixture-reached: the fixture segment committed" as *u8, (crc == 0) as i64, ctr)
86
  let out: *u8 = sys_mmap(GE_OUTCAP)
  let olen: *i64 = sys_mmap(GE_BOX) as *i64
  let segs: *i64 = sys_mmap(GE_BOX) as *i64
  let qa: *u8 = "alphaestateterm" as *u8
  let na: i64 = es_query(GE_DOMAIN, qa, es_slen(qa), GE_TOP, out, GE_OUTCAP, olen, segs)
  let ra: i64 = olen[0]   // captured now: later queries rewrite olen and a value read at the end would be theirs
  gv_check("T2 a term in one document answers exactly one hit" as *u8, (na == 1) as i64, ctr)
  gv_check("T2b the hit line carries the url row" as *u8, (es_find(out, olen[0], ua) >= 0) as i64, ctr)
  gv_check("T2c the hit line carries the snippet text" as *u8, (es_find(out, olen[0], "lives in file one" as *u8) >= 0) as i64, ctr)
  gv_check("T2d the shard's segment count is reported" as *u8, (segs[0] >= 1) as i64, ctr)
  let qb: *u8 = "betaestateterm" as *u8
  let nb: i64 = es_query(GE_DOMAIN, qb, es_slen(qb), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T3 a document without a url row still answers" as *u8, (nb == 1) as i64, ctr)
  gv_check("T3b and its line falls back to doc:<cid>" as *u8, (es_find(out, olen[0], "doc:" as *u8) >= 0) as i64, ctr)
  let qc: *u8 = "gammaestateterm" as *u8
  let nc: i64 = es_query(GE_DOMAIN, qc, es_slen(qc), GE_TOP, out, GE_OUTCAP, olen, segs)
  var t4: i64 = 0
  if nc == 1 {
      // the ONLY newline in the rendered line must be its terminator, and no tab may survive past the third field
      var lf: i64 = 0
      var k: i64 = 0
      while k < olen[0] { if out[k] == (ES_LF as u8) { lf = lf + 1 } k = k + 1 }
      if lf == 1 { if es_find(out, olen[0], "has a newline and a tab inside" as *u8) >= 0 { t4 = 1 } }
  }
  gv_check("T4 control bytes inside the document are folded to spaces in the snippet" as *u8, t4, ctr)
  let qz: *u8 = "zetaestatetermabsent" as *u8
  let nz: i64 = es_query(GE_DOMAIN, qz, es_slen(qz), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T5 neg-control-an-absent-term answers zero hits, not an error" as *u8, (nz == 0) as i64, ctr)
  let nn: i64 = es_query(GE_NONE, qa, es_slen(qa), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T6 neg-control-a-domain-never-ingested is UNREADABLE, never zero hits" as *u8, (nn == ES_UNREADABLE) as i64, ctr)
  // T8 (E1 anchors): a query term with underscores is also asked in its collapsed form, the anchor the tree ingest
  // writes; the fixture carries the collapsed token the way an anchored document would
  let td: *u8 = "deltaestateterm file text with the anchor tail alphabetagamma after a newline" as *u8
  let w8: *i64 = ss_begin()
  ge_write_doc(w8, prefix, td, 0 as *u8)
  let crc8: i64 = ss_commit(prefix, w8, ss_next_segid(prefix))
  gv_check("T8 fixture-reached: the anchored document committed" as *u8, (crc8 == 0) as i64, ctr)
  let q8: *u8 = "alpha_beta_gamma" as *u8
  let n8: i64 = es_query(GE_DOMAIN, q8, es_slen(q8), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T8b an underscored query term finds the document through its collapsed anchor" as *u8, (n8 == 1) as i64, ctr)
  let q8c: *u8 = "alpha_beta" as *u8
  let n8c: i64 = es_query(GE_DOMAIN, q8c, es_slen(q8c), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T8c neg-control-a-different-identifier collapses to a token nobody wrote and answers zero" as *u8, (n8c == 0) as i64, ctr)
  // T9 (E1b definition precedence): the CALLER mentions epsilon_estate_term eight times and carries eight collapsed
  // anchors; the DEFINER mentions it once, is padded so length normalisation cannot lift it, and carries the
  // definition token the tree ingest writes per function head. BM25 alone ranks the caller first on every term;
  // the tier ranks the definer first. Both documents carry the theta identifier for the neg-control below.
  let tdef: *u8 = "func epsilon_estate_term() -> i64 { return 1 }   declared once here, in a longer file with filler words so that length normalisation cannot lift it: one two three four five six seven eight nine ten eleven twelve, and theta_estate_term once\nepsilonestateterm thetaestateterm defepsilonestateterm" as *u8
  let udef: *u8 = "estate://fx/def.nx" as *u8
  let tcall: *u8 = "epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() epsilon_estate_term() called eight times, theta_estate_term theta_estate_term theta_estate_term theta_estate_term theta_estate_term\nepsilonestateterm epsilonestateterm epsilonestateterm epsilonestateterm epsilonestateterm epsilonestateterm epsilonestateterm epsilonestateterm thetaestateterm thetaestateterm thetaestateterm thetaestateterm thetaestateterm" as *u8
  let ucall: *u8 = "estate://fx/call.nx" as *u8
  let w9: *i64 = ss_begin()
  ge_write_doc(w9, prefix, tdef, udef)
  ge_write_doc(w9, prefix, tcall, ucall)
  let crc9: i64 = ss_commit(prefix, w9, ss_next_segid(prefix))
  gv_check("T9 fixture-reached: the definer and the caller committed" as *u8, (crc9 == 0) as i64, ctr)
  let q9: *u8 = "epsilon_estate_term" as *u8
  let n9: i64 = es_query(GE_DOMAIN, q9, es_slen(q9), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T9a the identifier query finds both the definer and the caller (its typed words also match the alpha fixture, so at least two hits)" as *u8, ((n9 >= 2) as i64) * ((es_find(out, olen[0], udef) >= 0) as i64) * ((es_find(out, olen[0], ucall) >= 0) as i64), ctr)
  gv_check("T9b the DEFINER ranks first: its url opens the first rendered line" as *u8, ge_first_line_has(out, olen[0], udef), ctr)
  let q9c: *u8 = "theta_estate_term" as *u8
  let n9c: i64 = es_query(GE_DOMAIN, q9c, es_slen(q9c), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T9c neg-control-an-identifier-nobody-defines keeps the score order: the caller with five mentions ranks first" as *u8, ((n9c >= 2) as i64) * ge_first_line_has(out, olen[0], ucall), ctr)
  gv_check("T9d es_def_precedence is a tier before the score" as *u8, ((es_def_precedence(1, 5, 0, 500) == 1) as i64) * ((es_def_precedence(0, 500, 1, 5) == 0) as i64) * ((es_def_precedence(0, 7, 0, 5) == 1) as i64) * ((es_def_precedence(0, 5, 0, 7) == 0) as i64), ctr)
  // T10 (E3b definition precedence for a LOOKUP): a one-term query asks for its definition token too, so a rung id
  // ranks the rung row that carries defzz9 above a log row that mentions ZZ9 seven times; a two-term prose query
  // asks for no definition token and the log row keeps its BM25 place.
  let trung: *u8 = "rung|ZZ9|A fixture rung with a title|zz_symbol|note text\nzzsymbol defzz9" as *u8
  let urung: *u8 = "estate://fx/b.plan#1" as *u8
  let tlog: *u8 = "log|1|ZZ9|measure|ZZ9 ZZ9 ZZ9 ZZ9 ZZ9 ZZ9 mentioned six more times in a measure row about the fixture rung" as *u8
  let ulog: *u8 = "estate://fx/b.plan#3" as *u8
  let w10: *i64 = ss_begin()
  ge_write_doc(w10, prefix, trung, urung)
  ge_write_doc(w10, prefix, tlog, ulog)
  let crc10: i64 = ss_commit(prefix, w10, ss_next_segid(prefix))
  gv_check("T10 fixture-reached: the rung row and the log row committed" as *u8, (crc10 == 0) as i64, ctr)
  let q10: *u8 = "ZZ9" as *u8
  let n10: i64 = es_query(GE_DOMAIN, q10, es_slen(q10), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T10a a one-term lookup finds both rows" as *u8, ((n10 >= 2) as i64) * ((es_find(out, olen[0], urung) >= 0) as i64) * ((es_find(out, olen[0], ulog) >= 0) as i64), ctr)
  gv_check("T10b the RUNG ROW ranks first for its id through the definition token" as *u8, ge_first_line_has(out, olen[0], urung), ctr)
  let q10c: *u8 = "ZZ9 measure" as *u8
  let n10c: i64 = es_query(GE_DOMAIN, q10c, es_slen(q10c), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T10c neg-control-a-two-term-prose-query asks for no definition token: the log row keeps its score place" as *u8, ((n10c >= 2) as i64) * ge_first_line_has(out, olen[0], ulog), ctr)
  // T11 (E4 corpus scope): the scorer's scope prefix keeps only hits whose url row starts with it -- the definer
  // under estate://fx/def, nothing under estate://fx/b. for the same query, and the full set again once cleared.
  let pfx11: *u8 = "estate://fx/def" as *u8
  dss_set_scope_prefix(pfx11, es_slen(pfx11))
  let n11: i64 = es_query(GE_DOMAIN, q9, es_slen(q9), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T11 a scope prefix keeps only the hit under it (the definer, one hit)" as *u8, ((n11 == 1) as i64) * ((es_find(out, olen[0], udef) >= 0) as i64), ctr)
  let pfx11b: *u8 = "estate://fx/b." as *u8
  dss_set_scope_prefix(pfx11b, es_slen(pfx11b))
  let n11b: i64 = es_query(GE_DOMAIN, q9, es_slen(q9), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T11b neg-control-a-scope-that-holds-no-hit-for-the-query answers zero, never a leak from another prefix" as *u8, (n11b == 0) as i64, ctr)
  dss_set_scope_prefix(0 as *u8, 0)
  let n11c: i64 = es_query(GE_DOMAIN, q9, es_slen(q9), GE_TOP, out, GE_OUTCAP, olen, segs)
  gv_check("T11c the cleared scope answers the whole set again" as *u8, (n11c == n9) as i64, ctr)
  gv_check("T11d dss_scope_filter: empty prefix passes, a matching prefix passes, a short url and a different prefix fail" as *u8, ((dss_scope_filter(udef, es_slen(udef), pfx11, 0) == 1) as i64) * ((dss_scope_filter(udef, es_slen(udef), pfx11, es_slen(pfx11)) == 1) as i64) * ((dss_scope_filter(pfx11, es_slen(pfx11) - 1, pfx11, es_slen(pfx11)) == 0) as i64) * ((dss_scope_filter(ucall, es_slen(ucall), pfx11, es_slen(pfx11)) == 0) as i64), ctr)
  // T12 (E8 envelope): the rendered answer canon-encodes to exactly the size it declares, with the head fields and
  // four per hit, under a CID that carries profile UXF_SEARCH; the same answer twice is byte-identical; canon_decode
  // returns every field and the fixture url and query read back; a different query gives a different CID; and the
  // envelope REFUSES text it cannot parse or a hit count the text disagrees with.
  let na12: i64 = es_query(GE_DOMAIN, qa, es_slen(qa), GE_TOP, out, GE_OUTCAP, olen, segs)
  let need12: i64 = es_uxf_envelope_size(out, olen[0], qa, na12, GE_TOP, prefix, segs[0])
  gv_check("T12 fixture-reached: the envelope sizes a one-hit answer" as *u8, ((na12 == 1) as i64) * ((need12 > 0) as i64), ctr)
  let canon12: *u8 = sys_mmap(need12)
  let cid12: *u8 = sys_mmap(UXF_CID_LEN + 1)
  let f12: *i64 = sys_mmap(GE_BOX) as *i64
  let got12: i64 = es_uxf_envelope(out, olen[0], qa, na12, GE_TOP, prefix, segs[0], canon12, cid12, f12)
  gv_check_eq("T12a the encoder wrote exactly the size it declared" as *u8, got12, need12, ctr)
  gv_check_eq("T12b fields = head fields + hit fields per hit" as *u8, f12[0], ES_UXF_HEAD_FIELDS + na12 * ES_UXF_HIT_FIELDS, ctr)
  gv_check_eq("T12c the CID carries profile UXF_SEARCH and reads back through uxf_codec_of_cid" as *u8, uxf_codec_of_cid(cid12), UXF_SEARCH, ctr)
  let canon12b: *u8 = sys_mmap(need12)
  let cid12b: *u8 = sys_mmap(UXF_CID_LEN + 1)
  let got12b: i64 = es_uxf_envelope(out, olen[0], qa, na12, GE_TOP, prefix, segs[0], canon12b, cid12b, f12)
  gv_check("T12d determinism: the same answer twice is byte-identical canon under the same CID" as *u8, ((got12b == got12) as i64) * ge_bytes_eq(canon12, canon12b, got12) * ge_bytes_eq(cid12, cid12b, UXF_CID_LEN), ctr)
  let dk: *i64 = sys_mmap(f12[0] * ES_WORD) as *i64
  let dv: *i64 = sys_mmap(f12[0] * ES_WORD) as *i64
  let nd: i64 = canon_decode(canon12, got12, dk, dv, f12[0])
  gv_check_eq("T12e canon_decode round-trips the field count" as *u8, nd, f12[0], ctr)
  gv_check("T12f the decoded q is the query and hit.1.url is the fixture url" as *u8, ge_field_is(dk, dv, nd, "q" as *u8, qa) * ge_field_is(dk, dv, nd, "hit.1.url" as *u8, ua), ctr)
  gv_check("T12g neg-control-a-hit-count-the-text-disagrees-with-is-refused (hits+1 over a one-line answer returns -1)" as *u8, (es_uxf_envelope_size(out, olen[0], qa, na12 + 1, GE_TOP, prefix, segs[0]) == 0 - 1) as i64, ctr)
  let bad12: *u8 = "1 a line with no tab columns\n" as *u8
  gv_check("T12h neg-control-unparseable-rendered-text-is-refused (a line without its tabs returns -1)" as *u8, (es_uxf_envelope_size(bad12, es_slen(bad12), qa, 1, GE_TOP, prefix, segs[0]) == 0 - 1) as i64, ctr)
  let nb12: i64 = es_query(GE_DOMAIN, qb, es_slen(qb), GE_TOP, out, GE_OUTCAP, olen, segs)
  let needb12: i64 = es_uxf_envelope_size(out, olen[0], qb, nb12, GE_TOP, prefix, segs[0])
  let canonb12: *u8 = sys_mmap(needb12)
  let cidb12: *u8 = sys_mmap(UXF_CID_LEN + 1)
  es_uxf_envelope(out, olen[0], qb, nb12, GE_TOP, prefix, segs[0], canonb12, cidb12, f12)
  gv_check("T12i neg-control-a-different-query-yields-a-different-cid" as *u8, ((needb12 > 0) as i64) * ((ge_bytes_eq(cid12, cidb12, UXF_CID_LEN) == 0) as i64), ctr)
  let sm: *u8 = sys_mmap(GE_OUTCAP)
  let sn: i64 = es_summary(sm, 0, qa, na, GE_TOP, prefix, segs[0])
  gv_check("T7 the summary line names the query, hits and shard" as *u8, ((es_find(sm, sn, "ESTATE-SEARCH q=alphaestateterm hits=1" as *u8) >= 0) as i64) * ((es_find(sm, sn, prefix) >= 0) as i64), ctr)
219
  gv_values_head()
  gv_kv("fixture_segid" as *u8, segid)
  gv_kv("hits_alpha" as *u8, na)
  gv_kv("rendered_bytes_alpha" as *u8, ra)
  gv_kv("hits_epsilon" as *u8, n9)
  gv_kv("hits_theta" as *u8, n9c)
  gv_kv("hits_zz9" as *u8, n10)
  gv_kv("uxf_bytes_alpha" as *u8, got12)
  gv_kv("uxf_fields_alpha" as *u8, f12[0])
  gv_kv("uxf_profile_alpha" as *u8, uxf_codec_of_cid(cid12))
  return gv_verdict("NX-ESTATE-SEARCH-GATE" as *u8, ctr, "the estate query renders a url-or-doc line with a folded snippet per hit on a fixture shard, answers zero for an absent term and UNREADABLE for an absent shard, and ranks the file that defines an identifier above the files that only mention it" as *u8)
231}
nx_estate_search_gate.nx source

`nx_estate_search_gate.nx` source
