nx_estate_search_lib.nx source

1// nx_estate_search_lib.nx -- THE ESTATE QUERY, as a library so a gate drives it in-process on a fixture shard 2// (search rungs E1/E1b/E4, 2026-09-14). es_query ranks with dss_search (the SAME scorer the public search runs: one 3// ruler, two corpora) and renders one line per hit into a caller buffer: rank TAB score TAB url (the url row, 4// else doc:<cid>) TAB snippet (the first ES_SNIPPET bytes of the document with control bytes folded to spaces). 5// It returns the hit count, or ES_UNREADABLE when the shard cannot be opened -- an absent shard and an empty one 6// are different facts and only one of them is a defect. 7// E1b DEFINITION PRECEDENCE: an underscored term is also asked as its definition token (ss_def_token, the token 8// the tree ingest writes once per function head), and the hits are re-ordered by a TIER before the score -- a 9// document carrying a typed term's definition token ranks above one that only mentions the term, whatever the 10// BM25 magnitude (es_def_precedence, the shape of dss_exact_precedence: a lexicographic key needs no coefficient, 11// and a discount constant here would be a magic number nobody could justify). A one-word function name has no 12// definition tier -- a NAMED residual, the same condition ss_ident_collapse already draws. license_tier: ORIGINAL 13import "nx_docportal_search_serve.nx" 14import "nx_uxf_cid.nx" // E8: the search answer's machine envelope is a UXF profile document (canonical bytes + profiled CID) 15 16const ES_DOMAIN: *u8 = "estate" 17const ES_TOP_DEFAULT: i64 = 10 18const ES_TOP_MAX: i64 = 50 19const ES_QBUF: i64 = 1024 20const ES_SNIPPET: i64 = 160 21const ES_PREFIXBUF: i64 = 512 22const ES_KEYBUF: i64 = 64 23const ES_BOX: i64 = 16 24const ES_WORD: i64 = 8 25const ES_NUMBUF: i64 = 28 26const ES_TAB: i64 = 9 27const ES_LF: i64 = 10 28const ES_SPACE: i64 = 32 29const ES_DIGIT0: i64 = 48 30const ES_DIGIT9: i64 = 57 31const ES_MMAP_OPEN: i64 = 1 // ss_open2 mode: mmap the segments, never read a multi-GB shard into anon RAM 32const ES_UNREADABLE: i64 = 0 - 3 // es_query: the shard could not be opened (the program exits 3 on it) 33const ES_LINE_RESERVE: i64 = 256 // per hit: rank, score, tabs, doc:<cid> and the newline beside the url and snippet 34const ES_DEFOFF: i64 = DSS_MAXTERMS // at most one definition token per query term the scorer will read 35const ES_LOOKUP_TERMS: i64 = 1 // a one-term query is a LOOKUP: it also asks for that term's definition token (a rung id, a bare name); prose has more 36 37func es_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 38func es_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i } 39func es_catn(d: *u8, o: i64, v: i64) -> i64 { 40 var m: i64 = v 41 var oo: i64 = o 42 if m < 0 { d[oo] = 45 as u8; oo = oo + 1; m = 0 - m } 43 if m == 0 { d[oo] = ES_DIGIT0 as u8; return oo + 1 } 44 var pw: i64 = 1 45 while m / pw >= 10 { pw = pw * 10 } 46 while pw > 0 { d[oo] = (ES_DIGIT0 + (m / pw) % 10) as u8; oo = oo + 1; pw = pw / 10 } 47 return oo 48} 49func es_starts(a: *u8, p: *u8) -> i64 { var i: i64 = 0; while p[i] != (0 as u8) { if a[i] != p[i] { return 0 } i = i + 1 } return 1 } 50func es_atoi(s: *u8) -> i64 { 51 var v: i64 = 0 52 var i: i64 = 0 53 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= ES_DIGIT0 { if c <= ES_DIGIT9 { v = v * 10 + (c - ES_DIGIT0) } } i = i + 1 } 54 return v 55} 56// first occurrence of a NUL-terminated needle in hay[0..n), -1 when absent (the gate's own witness) 57func es_find(hay: *u8, n: i64, needle: *u8) -> i64 { 58 let nl: i64 = es_slen(needle) 59 if nl == 0 { return 0 } 60 var i: i64 = 0 61 while i + nl <= n { 62 var j: i64 = 0 63 var hit: i64 = 1 64 while j < nl { if hay[i + j] != needle[j] { hit = 0; j = nl } else { j = j + 1 } } 65 if hit == 1 { return i } 66 i = i + 1 67 } 68 return 0 - 1 69} 70// E1b: does tok (NUL-terminated) occur in doc[0..dn) as a whole word, case-folded the way the tokenizer folds? The 71// definition anchors sit in the document's anchor tail as space-separated tokens, so the boundary test is the 72// identifier alphabet (ss_is_ident_byte). A linear scan over the mapped document, bounded by ES_TOP_MAX hits. 73func es_has_word(doc: *u8, dn: i64, tok: *u8) -> i64 { 74 let tl: i64 = es_slen(tok) 75 if tl == 0 { return 0 } 76 var i: i64 = 0 77 while i + tl <= dn { 78 var j: i64 = 0 79 var hit: i64 = 1 80 while j < tl { if dss_lower(doc[i + j] as i64) != dss_lower(tok[j] as i64) { hit = 0; j = tl } else { j = j + 1 } } 81 if hit == 1 { 82 var lb: i64 = 1 83 if i > 0 { if ss_is_ident_byte(doc[i - 1] as i64) == 1 { lb = 0 } } 84 var rb: i64 = 1 85 if i + tl < dn { if ss_is_ident_byte(doc[i + tl] as i64) == 1 { rb = 0 } } 86 if lb == 1 { if rb == 1 { return 1 } } 87 } 88 i = i + 1 89 } 90 return 0 91} 92// E1b CONTRACT (es_def_precedence): a hit carrying more of the query's definition tokens ranks first; equal tiers 93// fall through to the score. A TIER, not a coefficient (the shape of dss_exact_precedence). Returns 1 when a wins. 94func es_def_precedence(defa: i64, sca: i64, defb: i64, scb: i64) -> i64 { 95 if defa > defb { return 1 } 96 if defa < defb { return 0 } 97 if sca > scb { return 1 } 98 return 0 99} 100 101// THE QUERY. Renders up to `top` hits into out[0..cap) and stores the byte count in olen[0]; returns the hit count 102// or ES_UNREADABLE. segs_out[0] receives the shard's segment count for the caller's summary line. 103func es_query(domain: *u8, q: *u8, qn: i64, top: i64, out: *u8, cap: i64, olen: *i64, segs_out: *i64) -> i64 { 104 olen[0] = 0 105 segs_out[0] = 0 106 var t: i64 = top 107 if t < 1 { t = 1 } 108 if t > ES_TOP_MAX { t = ES_TOP_MAX } 109 let prefix: *u8 = sys_mmap(ES_PREFIXBUF) 110 dss_prefix(domain, prefix) 111 // ABSENT and EMPTY are different facts: a shard nobody ingested has no manifest and is UNREADABLE (the caller 112 // names the ingest); a shard with a manifest and no segments is empty and answers zero hits honestly. 113 let mfp: *u8 = sys_mmap(ES_PREFIXBUF) 114 var mo: i64 = es_cat(mfp, 0, prefix) 115 mo = es_cat(mfp, mo, "manifest.txt" as *u8) 116 mfp[mo] = 0 as u8 117 let mfd: i64 = sys_openat_rd(mfp) 118 if mfd < 0 { return ES_UNREADABLE } 119 sys_close(mfd) 120 let h: *i64 = ss_open2(prefix, ES_MMAP_OPEN) 121 if (h as i64) == 0 { return ES_UNREADABLE } 122 segs_out[0] = h[0] 123 let cids: *i64 = sys_mmap(ES_TOP_MAX * ES_WORD) as *i64 124 let scores: *i64 = sys_mmap(ES_TOP_MAX * ES_WORD) as *i64 125 // IDENTIFIER EXPANSION (E1) + DEFINITION TOKENS (E1b): a query term carrying an underscore is also asked in its 126 // collapsed form (the anchor every source file carries for every identifier it mentions) and as its definition 127 // token (the anchor a file carries only for the functions it declares). Same derivation both sides: 128 // ss_ident_collapse and ss_def_token. The expansion stops at the scorer's own term cap (DSS_MAXTERMS) so nothing 129 // past it is dropped silently by the scorer; the typed terms always come first. 130 // the expanded query is at most the typed bytes three times over (typed, collapsed, definition) plus a prefix and a 131 // separator per term the scorer will read -- derived from the input, never a guessed ceiling 132 let pfxlen: i64 = es_slen(SS_DEF_PREFIX) 133 let q2cap: i64 = qn + qn + qn + DSS_MAXTERMS * (pfxlen + 1 + 1) + 1 134 let q2: *u8 = sys_mmap(q2cap) 135 var qn2: i64 = 0 136 var qi: i64 = 0 137 var nterms: i64 = 0 138 if qn > 0 { nterms = 1 } 139 while qi < qn { 140 q2[qn2] = q[qi] 141 qn2 = qn2 + 1 142 if (q[qi] as i64) == ES_SPACE { nterms = nterms + 1 } 143 qi = qi + 1 144 } 145 let termcap: i64 = qn + pfxlen + 1 // one typed term, its collapsed twin or its definition token: bounded by the query itself 146 let term: *u8 = sys_mmap(termcap) 147 let coll: *u8 = sys_mmap(termcap) 148 let defcap: i64 = qn + DSS_MAXTERMS * (pfxlen + 1) + 1 // every definition token NUL-separated: derived from the query 149 let defs: *u8 = sys_mmap(defcap) 150 let defoff: *i64 = sys_mmap(ES_DEFOFF * ES_WORD) as *i64 151 var ndef: i64 = 0 152 var dfo: i64 = 0 153 let nterms0: i64 = nterms // the TYPED term count, before any expansion: one term is a lookup 154 var ts: i64 = 0 155 var tp: i64 = 0 156 while tp <= qn { 157 var atend: i64 = 0 158 if tp == qn { atend = 1 } else { if (q[tp] as i64) == ES_SPACE { atend = 1 } } 159 if atend == 1 { 160 let tl: i64 = tp - ts 161 if tl > 0 { if tl < termcap { 162 var k: i64 = 0 163 while k < tl { term[k] = q[ts + k]; k = k + 1 } 164 term[tl] = 0 as u8 165 let cl: i64 = ss_ident_collapse(term, coll, termcap) 166 if cl > 0 { if nterms < DSS_MAXTERMS { if qn2 + 1 + cl < q2cap { 167 q2[qn2] = ES_SPACE as u8; qn2 = qn2 + 1 168 var m: i64 = 0 169 while m < cl { q2[qn2] = coll[m]; qn2 = qn2 + 1; m = m + 1 } 170 nterms = nterms + 1 171 } } } 172 // the definition token is asked for an underscored identifier, and for the one term of a LOOKUP query 173 // (a rung id, a bare name): a multi-term prose query is left exactly as typed 174 var wantdef: i64 = 0 175 if cl > 0 { wantdef = 1 } 176 if nterms0 == ES_LOOKUP_TERMS { wantdef = 1 } 177 if wantdef == 1 { 178 let dfl: i64 = ss_def_token(term, tl, coll, termcap) 179 if nterms < DSS_MAXTERMS { if ndef < ES_DEFOFF { if dfo + dfl + 1 < defcap { if qn2 + 1 + dfl < q2cap { 180 q2[qn2] = ES_SPACE as u8; qn2 = qn2 + 1 181 defoff[ndef] = dfo 182 var m2: i64 = 0 183 while m2 < dfl { q2[qn2] = coll[m2]; qn2 = qn2 + 1; defs[dfo] = coll[m2]; dfo = dfo + 1; m2 = m2 + 1 } 184 defs[dfo] = 0 as u8; dfo = dfo + 1 185 ndef = ndef + 1 186 nterms = nterms + 1 187 } } } } 188 } 189 } } 190 ts = tp + 1 191 } 192 tp = tp + 1 193 } 194 q2[qn2] = 0 as u8 195 // E1b: with definition tokens in play the scorer is asked for its full window (ES_TOP_MAX) so the tier has room 196 // to lift a definer the caller's `top` would have cut; without them the fetch is exactly `top`, byte-identical to E1. 197 var fetch: i64 = t 198 if ndef > 0 { fetch = ES_TOP_MAX } 199 var n: i64 = dss_search(domain, q2, qn2, cids, scores, fetch) 200 let key: *u8 = sys_mmap(ES_KEYBUF) 201 let ukey: *u8 = sys_mmap(ES_KEYBUF) 202 let dp: *i64 = sys_mmap(ES_BOX) as *i64 203 let dl: *i64 = sys_mmap(ES_BOX) as *i64 204 if ndef > 0 { 205 let dtier: *i64 = sys_mmap(ES_TOP_MAX * ES_WORD) as *i64 206 var ti: i64 = 0 207 while ti < n { 208 dtier[ti] = 0 209 dss_mkkey(cids[ti], key) 210 if ss_hget(h, key, dp, dl) == 1 { 211 var di: i64 = 0 212 while di < ndef { if es_has_word(dp[0] as *u8, dl[0], (defs as i64 + defoff[di]) as *u8) == 1 { dtier[ti] = dtier[ti] + 1 } di = di + 1 } 213 } 214 ti = ti + 1 215 } 216 // selection order by the tier then the score: at most ES_TOP_MAX hits, so the square is small 217 var s: i64 = 0 218 while s < n { 219 var best: i64 = s 220 var j: i64 = s + 1 221 while j < n { if es_def_precedence(dtier[j], scores[j], dtier[best], scores[best]) == 1 { best = j } j = j + 1 } 222 if best != s { 223 let tc: i64 = cids[s]; cids[s] = cids[best]; cids[best] = tc 224 let tsc: i64 = scores[s]; scores[s] = scores[best]; scores[best] = tsc 225 let tt: i64 = dtier[s]; dtier[s] = dtier[best]; dtier[best] = tt 226 } 227 s = s + 1 228 } 229 if n > t { n = t } 230 } 231 var o: i64 = 0 232 var i: i64 = 0 233 while i < n { 234 if o + ES_LINE_RESERVE + ES_SNIPPET + ES_QBUF >= cap { i = n } else { 235 o = es_catn(out, o, i + 1); out[o] = ES_TAB as u8; o = o + 1 236 o = es_catn(out, o, scores[i]); out[o] = ES_TAB as u8; o = o + 1 237 dsv_mkurlkey(cids[i], ukey) 238 var haveurl: i64 = 0 239 if ss_hget(h, ukey, dp, dl) == 1 { 240 let up: *u8 = dp[0] as *u8 241 var ul: i64 = dl[0] 242 if ul > ES_QBUF { ul = ES_QBUF } 243 var k: i64 = 0 244 while k < ul { out[o] = up[k]; o = o + 1; k = k + 1 } 245 haveurl = 1 246 } 247 if haveurl == 0 { o = es_cat(out, o, "doc:" as *u8); o = es_catn(out, o, cids[i]) } 248 out[o] = ES_TAB as u8; o = o + 1 249 dss_mkkey(cids[i], key) 250 if ss_hget(h, key, dp, dl) == 1 { 251 let dt: *u8 = dp[0] as *u8 252 var m: i64 = dl[0] 253 if m > ES_SNIPPET { m = ES_SNIPPET } 254 var j: i64 = 0 255 while j < m { var c: i64 = dt[j] as i64; if c < ES_SPACE { c = ES_SPACE } out[o] = c as u8; o = o + 1; j = j + 1 } 256 } 257 out[o] = ES_LF as u8; o = o + 1 258 i = i + 1 259 } 260 } 261 olen[0] = o 262 return n 263} 264// the positional summary line, written LAST by the program: ESTATE-SEARCH q=... hits=... top=... shard=... segments=... 265func es_summary(out: *u8, o0: i64, q: *u8, hits: i64, top: i64, prefix: *u8, segs: i64) -> i64 { 266 var o: i64 = es_cat(out, o0, "ESTATE-SEARCH q=" as *u8) 267 o = es_cat(out, o, q) 268 o = es_cat(out, o, " hits=" as *u8); o = es_catn(out, o, hits) 269 o = es_cat(out, o, " top=" as *u8); o = es_catn(out, o, top) 270 o = es_cat(out, o, " shard=" as *u8); o = es_cat(out, o, prefix) 271 o = es_cat(out, o, " segments=" as *u8); o = es_catn(out, o, segs) 272 out[o] = ES_LF as u8 273 return o + 1 274} 275 276// ---- E8: THE MACHINE ENVELOPE (search rung E8, 2026-09-14) ------------------------------------------------------- 277// The answer above is text for a reader. The envelope below is the SAME answer as a UXF profile document: the fields 278// of the summary line plus one field per hit column, canon-encoded (NXR1, key-sorted, length-prefixed, no escaping) 279// and content-addressed under profile UXF_SEARCH. Two queries over the same shard state that render the same text 280// carry the same CID; one changed byte anywhere changes it. JSON and the TAB text are projections of these bytes, 281// never the other way round. Field keys: q hits top shard segments and hit.<i>.rank/score/url/snippet (i from 1). 282const ES_UXF_HEAD_FIELDS: i64 = 5 // q hits top shard segments 283const ES_UXF_HIT_FIELDS: i64 = 4 // rank score url snippet 284const ES_UXF_HIT_COLS: i64 = ES_UXF_HIT_FIELDS - 1 // TAB separators per rendered hit line 285const ES_UXF_KEYLEN: i64 = 24 // "hit." + up to two digits of ES_TOP_MAX + ".snippet" + NUL, the longest key 286 287// one NUL-terminated copy of src[0..n) in fresh memory (canon_encode reads NUL-terminated strings; the rendered text has none) 288func es_uxf_dup(src: *u8, n: i64) -> *u8 { 289 let d: *u8 = sys_mmap(n + 1) 290 var i: i64 = 0 291 while i < n { d[i] = src[i]; i = i + 1 } 292 d[n] = 0 as u8 293 return d 294} 295 296// key "hit.<i>.<col>" into a fresh buffer; returns its pointer 297func es_uxf_hitkey(i: i64, col: *u8) -> *u8 { 298 let k: *u8 = sys_mmap(ES_UXF_KEYLEN) 299 var o: i64 = es_cat(k, 0, "hit." as *u8) 300 o = es_catn(k, o, i) 301 k[o] = 46 as u8; o = o + 1 // '.' 302 o = es_cat(k, o, col) 303 k[o] = 0 as u8 304 return k 305} 306 307// Build the envelope from the rendered text (rendered[0..rn), one hit per line, columns TAB-separated) and the summary 308// values. Writes the canonical bytes to canon_out (which MUST be sized by es_uxf_envelope_size on the same inputs, or 309// pass 0 and read only the size), the profiled CID to cid (UXF_CID_LEN + 1 bytes). Returns the canonical byte length, 310// fields_out[0] = the field count. A rendered line with fewer than ES_UXF_HIT_COLS tabs is malformed and returns -1: 311// the envelope refuses to describe an answer it cannot parse, it never encodes a partial one. 312func es_uxf_envelope(rendered: *u8, rn: i64, q: *u8, hits: i64, top: i64, prefix: *u8, segs: i64, canon_out: *u8, cid: *u8, fields_out: *i64) -> i64 { 313 let maxf: i64 = ES_UXF_HEAD_FIELDS + hits * ES_UXF_HIT_FIELDS 314 let keys: *i64 = sys_mmap(maxf * ES_WORD) as *i64 315 let vals: *i64 = sys_mmap(maxf * ES_WORD) as *i64 316 var nf: i64 = 0 317 let nb: *u8 = sys_mmap(ES_NUMBUF) 318 keys[nf] = "q" as *u8 as i64; vals[nf] = q as i64; nf = nf + 1 319 var l: i64 = es_catn(nb, 0, hits); keys[nf] = "hits" as *u8 as i64; vals[nf] = es_uxf_dup(nb, l) as i64; nf = nf + 1 320 l = es_catn(nb, 0, top); keys[nf] = "top" as *u8 as i64; vals[nf] = es_uxf_dup(nb, l) as i64; nf = nf + 1 321 keys[nf] = "shard" as *u8 as i64; vals[nf] = prefix as i64; nf = nf + 1 322 l = es_catn(nb, 0, segs); keys[nf] = "segments" as *u8 as i64; vals[nf] = es_uxf_dup(nb, l) as i64; nf = nf + 1 323 var p: i64 = 0 324 var hi: i64 = 1 325 while p < rn { 326 if hi > hits { return 0 - 1 } // more lines than hits: the text and the count disagree, refuse 327 // the four columns of this line: rank score url snippet; the snippet runs to the newline and may hold no tab 328 let cs: *i64 = sys_mmap((ES_UXF_HIT_FIELDS + 1) * ES_WORD) as *i64 // column starts, cs[4] = line end 329 cs[0] = p 330 var col: i64 = 1 331 var eol: i64 = 0 // the line's newline was seen (an explicit flag: the cursor p is never written as a sentinel) 332 var go: i64 = 1 333 while go == 1 { 334 if p >= rn { go = 0 } 335 else { 336 let c: u8 = rendered[p] 337 p = p + 1 338 if c == (ES_LF as u8) { cs[ES_UXF_HIT_FIELDS] = p - 1; eol = 1; go = 0 } 339 else { if col < ES_UXF_HIT_FIELDS { if c == (ES_TAB as u8) { cs[col] = p; col = col + 1 } } } 340 } 341 } 342 if eol == 0 { cs[ES_UXF_HIT_FIELDS] = rn } // the last line may end at the buffer instead of a newline 343 if col < ES_UXF_HIT_FIELDS { return 0 - 1 } // fewer than three tabs before the line ended 344 keys[nf] = es_uxf_hitkey(hi, "rank" as *u8) as i64; vals[nf] = es_uxf_dup((rendered as i64 + cs[0]) as *u8, cs[1] - 1 - cs[0]) as i64; nf = nf + 1 345 keys[nf] = es_uxf_hitkey(hi, "score" as *u8) as i64; vals[nf] = es_uxf_dup((rendered as i64 + cs[1]) as *u8, cs[2] - 1 - cs[1]) as i64; nf = nf + 1 346 keys[nf] = es_uxf_hitkey(hi, "url" as *u8) as i64; vals[nf] = es_uxf_dup((rendered as i64 + cs[2]) as *u8, cs[3] - 1 - cs[2]) as i64; nf = nf + 1 347 keys[nf] = es_uxf_hitkey(hi, "snippet" as *u8) as i64; vals[nf] = es_uxf_dup((rendered as i64 + cs[3]) as *u8, cs[ES_UXF_HIT_FIELDS] - cs[3]) as i64; nf = nf + 1 348 hi = hi + 1 349 } 350 if hi - 1 != hits { return 0 - 1 } // fewer lines than hits: same disagreement, same refusal 351 fields_out[0] = nf 352 let need: i64 = canon_encode_size(keys, vals, nf) 353 if (canon_out as i64) == 0 { return need } 354 let got: i64 = canon_encode(keys, vals, nf, canon_out) 355 uxf_cid_profiled(UXF_SEARCH, canon_out, got, cid) 356 return got 357} 358 359// the exact canonical size for these inputs (the caller sizes its buffer from this, never from a constant) 360func es_uxf_envelope_size(rendered: *u8, rn: i64, q: *u8, hits: i64, top: i64, prefix: *u8, segs: i64) -> i64 { 361 let fb: *i64 = sys_mmap(ES_BOX) as *i64 362 return es_uxf_envelope(rendered, rn, q, hits, top, prefix, segs, 0 as *u8, 0 as *u8, fb) 363}