nx_incumbent_lib.nx source
↩ module page · 1427 lines · 61815 B
1// nx_incumbent_lib.nx -- THE ONE RULER FOR A RUNG CONTRACT'S INCUMBENTS AND THE RECORD'S REJECTIONS (engineshift ES28).
2//
3// Operator 2026-09-14, on the search R0 review: "this is the stuff we need the search and nishi estate to do
4// proactively not rely on you treasure hunting." A prior seat rebuilt the R0 rerank arm as a learned fusion over
5// PPMI while the estate already held the trained dense embedding (knowledge/index/embed_v1.bin), the lib that
6// scores with it (nx_dr_densejudge) and, in that lib's own header, the MEASURED REJECTION of exactly that fusion
7// (DR-13). The reviewing seat found all three by glob and header reads: a retrieval failure, not a discovery.
8// nx_capsearch ranks registered tools and libs by name, title and grammar; it cannot see a header's prose, a data
9// asset, or a rejection. This lib answers a rung's contract from the record itself, and nx_compare_rank prints the
10// answer on every open rung so no seat starts a build from its own memory.
11//
12// THE INDEX (a derived artefact, stamped, refused when stale): one row per organ source in the two runtime roots
13// org|<name>|<root>|<header words, lowercase, unique, first 12 bytes each>|<knowledge/ paths the header names>|<rejection lines>
14// and one per data asset under knowledge/index: asset|<name>|<name words>
15// headed by # NX-DERIVED incumbent index generated_at=<epoch> files=<n> head_truncated=<n> assets=<n> corpus_complete=<0|1>
16// The HEADER of an organ is its leading comment block (the // lines before the first code line); its words, its
17// basename parts and its column-0 func names are one bag. A header longer than the read window is COUNTED
18// (head_truncated), never silently cut. A bag word is kept to 12 bytes for display; no key or match reads past 8.
19//
20// THE MATCH: the contract terms are the rung's title, symbol and note tokenised to lowercase runs of letters and
21// digits at least min_tok long, deduplicated by their MATCH PREFIX (rerank and reranker are one term, beat and
22// beats are one term); an all-digit term is DROPPED and counted (a number is a measurement, never a capability word:
23// measured 2026-09-14, the 321 in "321 permille" matched three throwaway test organs at a rare-token weight). A term
24// matches a bag word when their leading bytes agree -- the first morph_long bytes for a term at least that long,
25// else the WHOLE term (mine~miner is the prefix rule nx_capsearch measured on its gold set at 4 bytes; here the
26// 4-byte key only narrows the posting chain, because measured on 2026-09-14 the 4-byte cut made dense match density
27// and wired match wire). Each matched term weighs idf_scale / (idf_base + df), df
28// being the number of organs carrying it (the weighting nx_capsearch ships), and a term carried by more than
29// df_max_permil of the organs is DROPPED as non-discriminating and COUNTED, so no stop-word list is hand-written:
30// the corpus decides which words say nothing. An organ's score is the sum over its distinct matched terms, and a
31// term that also hits the organ's own NAME parts adds its idf a second time: the name is the organ's own
32// declaration of what it is, a second piece of evidence weighed like the first (the rule nx_capsearch's known-good
33// tooltop applies, a stem hit on the name counted beside a hit in the title). MEASURED 2026-09-14 on the live tree
34// (20,820 organs): ranking by a name TIER first put an organ named for two process words (wired, exceed) second
35// for R0 at a tiny weight, so the name hit is weighed, never sorted on. The rung's own organ is excluded.
36// THE LIST IS THE ANSWER, NEVER A THRESHOLD: the top topn organs print with their name hits, score and matched
37// terms, then the data assets those headers name plus assets whose own name matches a term, then the rejection
38// lines those headers carry. PRIOR-REJECTIONS is its own axis: beyond the top organs, every organ whose NAME carries
39// a contract term and whose header records a rejection is listed (capped by rej_orgs, overflow counted), because a
40// rejection on an organ named for the contract's own word is the record answering the contract -- that is how R0
41// reaches nx_dr_densejudge (named dense) and its DR-13 line, and through that header the embed_v1.bin asset, while
42// the BEIR lane's own organs hold the top of the incumbents list. NONE when nothing matches, with the terms tried.
43// COVERAGE travels with every answer: files scanned, headers truncated, corpus_complete, index age and state.
44// COMPOSES sys_read_file, sys_getdents64 (the ps_has loop shape of nx_presubmit_lib), sys_openat_wr + sys_renameat
45// (atomic publish), sys_clock_gettime_real. Every number lives in knowledge/incumbent.conf (conf > defaults).
46// Scratch buffers are allocated once per build or per match, never per file or per word (no allocation in a hot loop).
47// license_tier: ORIGINAL Writes only the index it is asked to write. No hw writes (Rule 26).
48import "nx_syscalls.nx"
49
50const INC_I64: i64 = 8
51const INC_NL: i64 = 10
52const INC_CR: i64 = 13
53const INC_TAB: i64 = 9
54const INC_PIPE: i64 = 124
55const INC_SPACE: i64 = 32
56const INC_SLASH: i64 = 47
57const INC_SEMI: i64 = 59
58const INC_EQ: i64 = 61
59const INC_HASHCH: i64 = 35
60const INC_DOT: i64 = 46
61const INC_COLON: i64 = 58
62const INC_LPAREN: i64 = 40
63const INC_RPAREN: i64 = 41
64const INC_COMMA: i64 = 44
65const INC_DQUOTE: i64 = 34
66const INC_SQUOTE: i64 = 39
67const INC_RBRACK: i64 = 93
68const INC_GT: i64 = 62
69const INC_MINUS: i64 = 45
70const INC_A: i64 = 97
71const INC_Z: i64 = 122
72const INC_UA: i64 = 65
73const INC_UZ: i64 = 90
74const INC_D0: i64 = 48
75const INC_D9: i64 = 57
76const INC_CH_N: i64 = 110
77const INC_CH_X: i64 = 120
78const INC_CASE_DELTA: i64 = 32
79const INC_KEY_BYTES: i64 = 8 // a key packs up to 8 leading bytes into one i64; no match reads past 8
80const INC_WORD_KEEP: i64 = 12 // a bag word or a term is kept to 12 bytes for display
81const INC_BYTE_BITS: i64 = 8
82const INC_HASH_MUL: i64 = 1099511628211 // an odd multiplier for the packed key (the FNV-1a prime)
83const INC_HASH_SHIFT: i64 = 29
84const INC_NAME_CAP: i64 = 256
85const INC_PATH_CAP: i64 = 1024
86const INC_NUM_CAP: i64 = 32
87const INC_DIRBUF: i64 = 65536 // the getdents window nx_presubmit_lib uses (PS_DIRBUF)
88const INC_SETCAP: i64 = 4096 // per-file word set slots (a header carries hundreds of words, never thousands)
89const INC_HASH_MIN: i64 = 1024
90const INC_CONF_A: *u8 = "knowledge/incumbent.conf"
91const INC_CONF_B: *u8 = "../knowledge/incumbent.conf"
92const INC_REJ_A: *u8 = "knowledge/incumbent_reject.conf"
93const INC_REJ_B: *u8 = "../knowledge/incumbent_reject.conf"
94const INC_INDEX_A: *u8 = "knowledge/status/incumbent_index.txt"
95const INC_INDEX_B: *u8 = "../knowledge/status/incumbent_index.txt"
96const INC_TMP_SUFFIX: *u8 = ".tmp"
97const INC_ROOT_RT_A: *u8 = "buildroot/runtime"
98const INC_ROOT_HDL_A: *u8 = "buildroot/runtime/_hdl_build"
99const INC_ASSET_A: *u8 = "knowledge/index"
100const INC_ROOT_RT_B: *u8 = "runtime"
101const INC_ROOT_HDL_B: *u8 = "runtime/_hdl_build"
102const INC_ASSET_B: *u8 = "../knowledge/index"
103const INC_HEAD_MARK: *u8 = "# NX-DERIVED incumbent index generated_at="
104const INC_ORG_TAG: *u8 = "org|"
105const INC_ASSET_TAG: *u8 = "asset|"
106const INC_KNOW: *u8 = "knowledge/"
107const INC_FUNC: *u8 = "func "
108const INC_FUNC_LEN: i64 = 5
109const INC_REJ_SEP: *u8 = ";; "
110const INC_ORG_FIELDS: i64 = 6
111const INC_ASSET_FIELDS: i64 = 3
112const INC_ROW_SLACK: i64 = 1024 // per row beyond the header window: name, root, separators, asset paths
113// defaults: bootstrap values, overridden by knowledge/incumbent.conf (the configuration hierarchy)
114const INC_D_HEAD_BYTES: i64 = 8192
115const INC_D_MORPH_LONG: i64 = 6
116const INC_D_MORPH_SHORT: i64 = 4
117const INC_D_MIN_TOK: i64 = 3
118const INC_D_TOPN: i64 = 8
119const INC_D_DF_MAX_PERMIL: i64 = 250
120const INC_D_IDF_SCALE: i64 = 1000000
121const INC_D_IDF_BASE: i64 = 3
122const INC_D_REJ_PER_ORG: i64 = 2
123const INC_D_REJ_LINE_CAP: i64 = 160
124const INC_D_INDEX_MAX_AGE_S: i64 = 172800
125const INC_D_MAX_TERMS: i64 = 62
126const INC_D_REJ_ORGS: i64 = 8
127const INC_PERMIL: i64 = 1000
128// conf slots
129const INC_C_HEAD_BYTES: i64 = 0
130const INC_C_MORPH_LONG: i64 = 1
131const INC_C_MORPH_SHORT: i64 = 2
132const INC_C_MIN_TOK: i64 = 3
133const INC_C_TOPN: i64 = 4
134const INC_C_DF_MAX_PERMIL: i64 = 5
135const INC_C_IDF_SCALE: i64 = 6
136const INC_C_IDF_BASE: i64 = 7
137const INC_C_REJ_PER_ORG: i64 = 8
138const INC_C_REJ_LINE_CAP: i64 = 9
139const INC_C_INDEX_MAX_AGE_S: i64 = 10
140const INC_C_MAX_TERMS: i64 = 11
141const INC_C_SRC: i64 = 12 // 1 conf file read, 0 defaults
142const INC_C_REJ_ORGS: i64 = 13 // organs listed under PRIOR-REJECTIONS beyond the top organs
143const INC_C_N: i64 = 14
144// index g-box slots
145const INC_G_BUF: i64 = 0
146const INC_G_N: i64 = 1
147const INC_G_ROWS: i64 = 2
148const INC_G_R_NAME: i64 = 3
149const INC_G_R_ROOT: i64 = 4
150const INC_G_R_WORDS: i64 = 5
151const INC_G_R_ASSETS: i64 = 6
152const INC_G_R_REJ: i64 = 7
153const INC_G_ASSETS: i64 = 8
154const INC_G_A_NAME: i64 = 9
155const INC_G_A_WORDS: i64 = 10
156const INC_G_HCAP: i64 = 11
157const INC_G_HKEY: i64 = 12
158const INC_G_HHEAD: i64 = 13
159const INC_G_PWORD: i64 = 14
160const INC_G_PROW: i64 = 15
161const INC_G_PNEXT: i64 = 16
162const INC_G_NPOST: i64 = 17
163const INC_G_GEN: i64 = 18
164const INC_G_FILES: i64 = 19
165const INC_G_TRUNC: i64 = 20
166const INC_G_COMPLETE: i64 = 21
167const INC_G_STATE: i64 = 22
168const INC_G_AGE: i64 = 23
169const INC_G_PATH: i64 = 24
170const INC_G_SEEN: i64 = 25 // per-row scratch stamp for df counting
171const INC_G_SCORE: i64 = 26
172const INC_G_MASK: i64 = 27
173const INC_G_TIER: i64 = 28 // per-row scratch: contract terms hitting the organ's name parts
174const INC_G_NIDF: i64 = 29 // per-row scratch: the idf of the terms hitting the organ's name parts
175const INC_G_N_SLOTS: i64 = 30
176// index states
177const INC_S_FRESH: i64 = 0
178const INC_S_STALE: i64 = 1
179const INC_S_ABSENT: i64 = 2
180const INC_S_MALFORMED: i64 = 3
181// result g-box slots
182const INC_R_N: i64 = 0
183const INC_R_ROW: i64 = 1
184const INC_R_SCORE: i64 = 2
185const INC_R_MASK: i64 = 3
186const INC_R_NTERMS: i64 = 4
187const INC_R_TERMS: i64 = 5
188const INC_R_TOFF: i64 = 6
189const INC_R_DROPPED: i64 = 7
190const INC_R_DROPMASK: i64 = 8
191const INC_R_OVER: i64 = 9
192const INC_R_MATCHED: i64 = 10
193const INC_R_TIER: i64 = 11
194const INC_R_NUMERIC: i64 = 12 // all-digit terms dropped
195const INC_R_TIDF: i64 = 13 // per-term idf weight (0 when dropped or unmatched)
196const INC_R_N_SLOTS: i64 = 14
197// build envelope slots
198const INC_E_FILES: i64 = 0
199const INC_E_TRUNC: i64 = 1
200const INC_E_ASSETS: i64 = 2
201const INC_E_COMPLETE: i64 = 3
202const INC_E_GEN: i64 = 4
203const INC_E_BYTES: i64 = 5
204const INC_E_N: i64 = 6
205// build context slots (one box, allocated once per build)
206const INC_B_CONF: i64 = 0
207const INC_B_ENV: i64 = 1
208const INC_B_REJ: i64 = 2
209const INC_B_NREJ: i64 = 3
210const INC_B_SET: i64 = 4
211const INC_B_ROOT: i64 = 5
212const INC_B_NAME: i64 = 6
213const INC_B_WORD: i64 = 7
214const INC_B_POS: i64 = 8
215const INC_B_CAND: i64 = 9
216const INC_B_HEAD: i64 = 10
217const INC_B_N: i64 = 11
218// render scratch slots
219const INC_V_ASTART: i64 = 0
220const INC_V_NA: i64 = 1
221const INC_V_N: i64 = 2
222const INC_NONE: i64 = 0 - 1
223const INC_CLOCK_WORDS: i64 = 2
224
225static inc_numbuf: i64
226
227// ---- tiny string helpers (bounded where they write) ----
228func inc_slen(s: *u8) -> i64 {
229 var n: i64 = 0
230 while s[n] != (0 as u8) { n = n + 1 }
231 return n
232}
233func inc_cat(d: *u8, o: i64, s: *u8) -> i64 {
234 var i: i64 = 0
235 var oo: i64 = o
236 while s[i] != (0 as u8) { d[oo] = s[i]; oo = oo + 1; i = i + 1 }
237 d[oo] = 0 as u8
238 return oo
239}
240// bounded cat: never writes past cap-1; the caller announces a cut
241func inc_catb(d: *u8, o: i64, cap: i64, s: *u8) -> i64 {
242 var i: i64 = 0
243 var oo: i64 = o
244 while s[i] != (0 as u8) { if oo < cap - 1 { d[oo] = s[i]; oo = oo + 1 } i = i + 1 }
245 d[oo] = 0 as u8
246 return oo
247}
248func inc_catn(d: *u8, o: i64, cap: i64, v: i64) -> i64 {
249 if inc_numbuf == 0 { inc_numbuf = sys_mmap(INC_NUM_CAP) as i64 }
250 let t: *u8 = inc_numbuf as *u8
251 var m: i64 = v
252 var k: i64 = 0
253 var neg: i64 = 0
254 if m < 0 { neg = 1; m = 0 - m }
255 if m == 0 { t[k] = INC_D0 as u8; k = k + 1 }
256 while m > 0 { t[k] = (INC_D0 + m - (m / 10) * 10) as u8; m = m / 10; k = k + 1 }
257 var oo: i64 = o
258 if neg == 1 { if oo < cap - 1 { d[oo] = INC_MINUS as u8; oo = oo + 1 } }
259 while k > 0 { k = k - 1; if oo < cap - 1 { d[oo] = t[k]; oo = oo + 1 } }
260 d[oo] = 0 as u8
261 return oo
262}
263func inc_streq(a: *u8, b: *u8) -> i64 {
264 var i: i64 = 0
265 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
266 if b[i] != (0 as u8) { return 0 }
267 return 1
268}
269func inc_starts(s: *u8, pre: *u8) -> i64 {
270 var i: i64 = 0
271 while pre[i] != (0 as u8) { if s[i] != pre[i] { return 0 } i = i + 1 }
272 return 1
273}
274// the index of the first NL at or after p, or n
275func inc_lend(buf: *u8, n: i64, p: i64) -> i64 {
276 var e: i64 = p
277 var go: i64 = 1
278 while go == 1 { if e >= n { go = 0 } else { if buf[e] == (INC_NL as u8) { go = 0 } else { e = e + 1 } } }
279 return e
280}
281// the index of the first space or NUL at or after p, or n
282func inc_wend(buf: *u8, n: i64, p: i64) -> i64 {
283 var e: i64 = p
284 var go: i64 = 1
285 while go == 1 { if e >= n { go = 0 } else { if buf[e] == (INC_SPACE as u8) { go = 0 } else { if buf[e] == (0 as u8) { go = 0 } else { e = e + 1 } } } }
286 return e
287}
288// does s[0..n) contain needle (plain substring)
289func inc_contains(s: *u8, n: i64, needle: *u8) -> i64 {
290 let m: i64 = inc_slen(needle)
291 if m == 0 { return 0 }
292 var i: i64 = 0
293 while i + m <= n {
294 var j: i64 = 0
295 var ok: i64 = 1
296 while j < m { if s[i + j] != needle[j] { ok = 0; j = m } else { j = j + 1 } }
297 if ok == 1 { return 1 }
298 i = i + 1
299 }
300 return 0
301}
302func inc_is_alnum(b: i64) -> i64 {
303 if b >= INC_A { if b <= INC_Z { return 1 } }
304 if b >= INC_UA { if b <= INC_UZ { return 1 } }
305 if b >= INC_D0 { if b <= INC_D9 { return 1 } }
306 return 0
307}
308func inc_is_digit(b: i64) -> i64 {
309 if b < INC_D0 { return 0 }
310 if b > INC_D9 { return 0 }
311 return 1
312}
313func inc_lower(b: i64) -> i64 {
314 if b >= INC_UA { if b <= INC_UZ { return b + INC_CASE_DELTA } }
315 return b
316}
317// decimal digits at s[off..] -> value; stops at the first non-digit
318func inc_atoi(s: *u8, off: i64) -> i64 {
319 var v: i64 = 0
320 var i: i64 = off
321 var go: i64 = 1
322 while go == 1 {
323 let b: i64 = s[i] as i64
324 if inc_is_digit(b) == 0 { go = 0 } else { v = v * 10 + (b - INC_D0); i = i + 1 }
325 }
326 return v
327}
328// the value after "key=" anywhere in the NUL-terminated line s; INC_NONE when the key is absent
329func inc_kv_int(s: *u8, key: *u8) -> i64 {
330 let n: i64 = inc_slen(s)
331 let m: i64 = inc_slen(key)
332 var i: i64 = 0
333 while i + m + 1 <= n {
334 var j: i64 = 0
335 var ok: i64 = 1
336 while j < m { if s[i + j] != key[j] { ok = 0; j = m } else { j = j + 1 } }
337 if ok == 1 { if s[i + m] == (INC_EQ as u8) { return inc_atoi(s, i + m + 1) } }
338 i = i + 1
339 }
340 return INC_NONE
341}
342func inc_now() -> i64 {
343 let ts: *i64 = sys_mmap(INC_CLOCK_WORDS * INC_I64) as *i64
344 ts[0] = 0
345 sys_clock_gettime_real(ts)
346 return ts[0]
347}
348func inc_ends_nx(nm: *u8, l: i64) -> i64 {
349 if l <= 3 { return 0 }
350 if nm[l - 3] != (INC_DOT as u8) { return 0 }
351 if nm[l - 2] != (INC_CH_N as u8) { return 0 }
352 if nm[l - 1] != (INC_CH_X as u8) { return 0 }
353 return 1
354}
355// the organ name of a source path: the bytes after the last / with a trailing .nx removed
356func inc_organ_name(path: *u8, out: *u8) -> i64 {
357 let n: i64 = inc_slen(path)
358 var s: i64 = 0
359 var i: i64 = 0
360 while i < n { if path[i] == (INC_SLASH as u8) { s = i + 1 } i = i + 1 }
361 var e: i64 = n
362 if inc_ends_nx(path, n) == 1 { e = n - 3 }
363 var o: i64 = 0
364 while s < e { if o < INC_NAME_CAP - 1 { out[o] = path[s]; o = o + 1 } s = s + 1 }
365 out[o] = 0 as u8
366 return o
367}
368// is name one of the space-separated tokens of buf[from..o)
369func inc_listed(buf: *u8, from: i64, o: i64, name: *u8) -> i64 {
370 var s: i64 = from
371 let nl: i64 = inc_slen(name)
372 while s < o {
373 let e: i64 = inc_wend(buf, o, s)
374 if e - s == nl {
375 var j: i64 = 0
376 var eq: i64 = 1
377 while j < nl { if buf[s + j] != name[j] { eq = 0; j = nl } else { j = j + 1 } }
378 if eq == 1 { return 1 }
379 }
380 s = e + 1
381 }
382 return 0
383}
384// where am I: the serving root (buildroot/runtime opens) or buildroot itself (runtime opens)
385func inc_cwd_is_nishihost() -> i64 {
386 let fd: i64 = sys_openat_rd(INC_ROOT_RT_A)
387 if fd < 0 { return 0 }
388 sys_close(fd)
389 return 1
390}
391
392// ---- conf ----
393func inc_conf_take(c: *i64, ln: *u8, key: *u8, slot: i64) -> i64 {
394 let v: i64 = inc_kv_int(ln, key)
395 if v != INC_NONE { c[slot] = v }
396 return 0
397}
398// reads knowledge/incumbent.conf (then ../knowledge/), key=value lines; every slot falls back to its default
399func inc_conf_load() -> *i64 {
400 let c: *i64 = sys_mmap(INC_C_N * INC_I64) as *i64
401 c[INC_C_HEAD_BYTES] = INC_D_HEAD_BYTES
402 c[INC_C_MORPH_LONG] = INC_D_MORPH_LONG
403 c[INC_C_MORPH_SHORT] = INC_D_MORPH_SHORT
404 c[INC_C_MIN_TOK] = INC_D_MIN_TOK
405 c[INC_C_TOPN] = INC_D_TOPN
406 c[INC_C_DF_MAX_PERMIL] = INC_D_DF_MAX_PERMIL
407 c[INC_C_IDF_SCALE] = INC_D_IDF_SCALE
408 c[INC_C_IDF_BASE] = INC_D_IDF_BASE
409 c[INC_C_REJ_PER_ORG] = INC_D_REJ_PER_ORG
410 c[INC_C_REJ_LINE_CAP] = INC_D_REJ_LINE_CAP
411 c[INC_C_INDEX_MAX_AGE_S] = INC_D_INDEX_MAX_AGE_S
412 c[INC_C_MAX_TERMS] = INC_D_MAX_TERMS
413 c[INC_C_REJ_ORGS] = INC_D_REJ_ORGS
414 c[INC_C_SRC] = 0
415 let lenp: *i64 = sys_mmap(INC_I64) as *i64
416 lenp[0] = 0
417 var buf: *u8 = sys_read_file(INC_CONF_A, lenp)
418 if lenp[0] <= 0 { buf = sys_read_file(INC_CONF_B, lenp) }
419 if lenp[0] <= 0 { return c }
420 c[INC_C_SRC] = 1
421 let n: i64 = lenp[0]
422 var p: i64 = 0
423 while p < n {
424 let e: i64 = inc_lend(buf, n, p)
425 buf[e] = 0 as u8
426 let ln: *u8 = (buf as i64 + p) as *u8
427 if ln[0] != (INC_HASHCH as u8) {
428 inc_conf_take(c, ln, "head_bytes" as *u8, INC_C_HEAD_BYTES)
429 inc_conf_take(c, ln, "morph_long" as *u8, INC_C_MORPH_LONG)
430 inc_conf_take(c, ln, "morph_short" as *u8, INC_C_MORPH_SHORT)
431 inc_conf_take(c, ln, "min_tok" as *u8, INC_C_MIN_TOK)
432 inc_conf_take(c, ln, "topn" as *u8, INC_C_TOPN)
433 inc_conf_take(c, ln, "df_max_permil" as *u8, INC_C_DF_MAX_PERMIL)
434 inc_conf_take(c, ln, "idf_scale" as *u8, INC_C_IDF_SCALE)
435 inc_conf_take(c, ln, "idf_base" as *u8, INC_C_IDF_BASE)
436 inc_conf_take(c, ln, "rej_per_org" as *u8, INC_C_REJ_PER_ORG)
437 inc_conf_take(c, ln, "rej_line_cap" as *u8, INC_C_REJ_LINE_CAP)
438 inc_conf_take(c, ln, "index_max_age_s" as *u8, INC_C_INDEX_MAX_AGE_S)
439 inc_conf_take(c, ln, "max_terms" as *u8, INC_C_MAX_TERMS)
440 inc_conf_take(c, ln, "rej_orgs" as *u8, INC_C_REJ_ORGS)
441 }
442 p = e + 1
443 }
444 if c[INC_C_MORPH_LONG] > INC_KEY_BYTES { c[INC_C_MORPH_LONG] = INC_KEY_BYTES }
445 if c[INC_C_MORPH_SHORT] > c[INC_C_MORPH_LONG] { c[INC_C_MORPH_SHORT] = c[INC_C_MORPH_LONG] }
446 if c[INC_C_MAX_TERMS] > INC_D_MAX_TERMS { c[INC_C_MAX_TERMS] = INC_D_MAX_TERMS }
447 return c
448}
449// the rejection phrases: knowledge/incumbent_reject.conf (then ../knowledge/), one per line, # comments; a
450// NUL-separated buffer with count[0] phrases (0 when the conf is absent: then no line can be a rejection, announced)
451func inc_reject_load(count: *i64) -> *u8 {
452 let lenp: *i64 = sys_mmap(INC_I64) as *i64
453 lenp[0] = 0
454 var buf: *u8 = sys_read_file(INC_REJ_A, lenp)
455 if lenp[0] <= 0 { buf = sys_read_file(INC_REJ_B, lenp) }
456 count[0] = 0
457 if lenp[0] <= 0 { return sys_mmap(INC_I64) }
458 let n: i64 = lenp[0]
459 let out: *u8 = sys_mmap(n + 2)
460 var o: i64 = 0
461 var p: i64 = 0
462 while p < n {
463 let e: i64 = inc_lend(buf, n, p)
464 if e > p { if buf[p] != (INC_HASHCH as u8) {
465 var i: i64 = p
466 while i < e { if buf[i] != (INC_CR as u8) { out[o] = buf[i]; o = o + 1 } i = i + 1 }
467 out[o] = 0 as u8
468 o = o + 1
469 count[0] = count[0] + 1
470 } }
471 p = e + 1
472 }
473 out[o] = 0 as u8
474 return out
475}
476func inc_reject_nth(rej: *u8, k: i64) -> *u8 {
477 var p: i64 = 0
478 var i: i64 = 0
479 while i < k { while rej[p] != (0 as u8) { p = p + 1 } p = p + 1; i = i + 1 }
480 return (rej as i64 + p) as *u8
481}
482
483// ---- the word bag ----
484// the next alnum run in s[pos..n): lowercased into out (kept to INC_WORD_KEEP, the whole run consumed); returns the
485// run's full length, 0 when none is left; pos advances past the run
486func inc_next_word(s: *u8, n: i64, pos: *i64, out: *u8) -> i64 {
487 var p: i64 = pos[0]
488 var found: i64 = 0
489 var go: i64 = 1
490 while go == 1 { if p >= n { go = 0 } else { if inc_is_alnum(s[p] as i64) == 1 { found = 1; go = 0 } else { p = p + 1 } } }
491 if found == 0 { pos[0] = n; out[0] = 0 as u8; return 0 }
492 var l: i64 = 0
493 go = 1
494 while go == 1 {
495 if p >= n { go = 0 } else {
496 if inc_is_alnum(s[p] as i64) == 1 { if l < INC_WORD_KEEP { out[l] = inc_lower(s[p] as i64) as u8 } l = l + 1; p = p + 1 }
497 else { go = 0 }
498 }
499 }
500 pos[0] = p
501 var kept: i64 = l
502 if kept > INC_WORD_KEEP { kept = INC_WORD_KEEP }
503 out[kept] = 0 as u8
504 return l
505}
506// pack the first take bytes of a word (fewer when shorter) into an i64 key; never 0 for a non-empty word
507func inc_key(w: *u8, take: i64) -> i64 {
508 var k: i64 = 0
509 var i: i64 = 0
510 var go: i64 = 1
511 while go == 1 { if i >= take { go = 0 } else { if w[i] == (0 as u8) { go = 0 } else { k = k | ((w[i] as i64) << (i * INC_BYTE_BITS)); i = i + 1 } } }
512 return k
513}
514func inc_hash(k: i64, cap: i64) -> i64 {
515 var h: i64 = k * INC_HASH_MUL
516 h = h ^ (h >> INC_HASH_SHIFT)
517 return h & (cap - 1)
518}
519
520// ---- the index build ----
521// list the .nx entries of dir into names (INC_NAME_CAP bytes each); want == 0 counts only; INC_NONE = unreadable dir
522func inc_list_dir(dir: *u8, names: *u8, want: i64, srconly: i64) -> i64 {
523 let fd: i64 = sys_openat_rd(dir)
524 if fd < 0 { return INC_NONE }
525 let dbuf: *u8 = sys_mmap(INC_DIRBUF)
526 var count: i64 = 0
527 var go: i64 = 1
528 while go == 1 {
529 let nr: i64 = sys_getdents64(fd, dbuf, INC_DIRBUF)
530 if nr <= 0 { go = 0 } else {
531 var off: i64 = 0
532 while off < nr {
533 let rec: *u8 = (dbuf as i64 + off) as *u8
534 let nm: *u8 = dirent_name(rec)
535 let l: i64 = inc_slen(nm)
536 var ok: i64 = 0
537 if srconly == 1 { ok = inc_ends_nx(nm, l) }
538 else { if nm[0] != (INC_DOT as u8) { if dirent_type(rec) == DT_REG { ok = 1 } } }
539 if ok == 1 {
540 if want > 0 { if count < want {
541 let dst: *u8 = (names as i64 + count * INC_NAME_CAP) as *u8
542 var i: i64 = 0
543 while i < l { if i < INC_NAME_CAP - 1 { dst[i] = nm[i] } i = i + 1 }
544 if l < INC_NAME_CAP { dst[l] = 0 as u8 } else { dst[INC_NAME_CAP - 1] = 0 as u8 }
545 } }
546 count = count + 1
547 }
548 off = off + dirent_reclen(rec)
549 }
550 }
551 }
552 sys_close(fd)
553 return count
554}
555// read up to cap bytes of path; returns bytes read (INC_NONE when unreadable)
556func inc_read_head(path: *u8, buf: *u8, cap: i64) -> i64 {
557 let fd: i64 = sys_openat_rd(path)
558 if fd < 0 { return INC_NONE }
559 var got: i64 = 0
560 var go: i64 = 1
561 while go == 1 {
562 if got >= cap { go = 0 } else {
563 let r: i64 = sys_read(fd, (buf as i64 + got) as *u8, cap - got)
564 if r <= 0 { go = 0 } else { got = got + r }
565 }
566 }
567 sys_close(fd)
568 return got
569}
570// add a word (lowercase) to the per-file set keyed on its first 8 bytes; returns 1 when new
571func inc_set_add(set: *i64, cap: i64, w: *u8) -> i64 {
572 let k: i64 = inc_key(w, INC_KEY_BYTES)
573 if k == 0 { return 0 }
574 var h: i64 = inc_hash(k, cap)
575 var probes: i64 = 0
576 while probes < cap {
577 if set[h] == 0 { set[h] = k; return 1 }
578 if set[h] == k { return 0 }
579 h = (h + 1) & (cap - 1)
580 probes = probes + 1
581 }
582 return 0
583}
584// emit every word of s[0..n) (>= min_tok, new to the set) into out at o, space-separated after the first
585func inc_bag_emit(b: *i64, s: *u8, n: i64, out: *u8, o: i64, firstp: *i64) -> i64 {
586 let c: *i64 = b[INC_B_CONF] as *i64
587 let set: *i64 = b[INC_B_SET] as *i64
588 let w: *u8 = b[INC_B_WORD] as *u8
589 let pos: *i64 = b[INC_B_POS] as *i64
590 var oo: i64 = o
591 pos[0] = 0
592 var l: i64 = inc_next_word(s, n, pos, w)
593 while l > 0 {
594 if l >= c[INC_C_MIN_TOK] { if inc_set_add(set, INC_SETCAP, w) == 1 {
595 if firstp[0] == 0 { oo = inc_cat(out, oo, " " as *u8) }
596 oo = inc_cat(out, oo, w)
597 firstp[0] = 0
598 } }
599 l = inc_next_word(s, n, pos, w)
600 }
601 return oo
602}
603// the end of the leading comment block of hb[0..got): // lines with blank lines allowed; trunc[0]=1 when the block
604// runs to the end of a full window
605func inc_head_end(hb: *u8, got: i64, cap: i64, trunc: *i64) -> i64 {
606 var hend: i64 = 0
607 var p: i64 = 0
608 var go: i64 = 1
609 trunc[0] = 0
610 while go == 1 {
611 if p >= got { go = 0; if got >= cap { trunc[0] = 1 } }
612 else {
613 let eol: i64 = inc_lend(hb, got, p)
614 var blank: i64 = 1
615 var q: i64 = p
616 while q < eol { let bq: i64 = hb[q] as i64; if bq != INC_SPACE { if bq != INC_CR { if bq != INC_TAB { blank = 0 } } } q = q + 1 }
617 var iscomment: i64 = 0
618 if eol - p >= 2 { if hb[p] == (INC_SLASH as u8) { if hb[p + 1] == (INC_SLASH as u8) { iscomment = 1 } } }
619 if iscomment == 1 { hend = eol; p = eol + 1 }
620 else { if blank == 1 { p = eol + 1 } else { go = 0 } }
621 }
622 }
623 return hend
624}
625// is b a byte that ends a knowledge/ path token
626func inc_path_stop(b: i64) -> i64 {
627 if b == INC_SPACE { return 1 }
628 if b == INC_RPAREN { return 1 }
629 if b == INC_COMMA { return 1 }
630 if b == INC_SEMI { return 1 }
631 if b == INC_DQUOTE { return 1 }
632 if b == INC_SQUOTE { return 1 }
633 if b == INC_NL { return 1 }
634 if b == INC_CR { return 1 }
635 if b == INC_PIPE { return 1 }
636 if b == INC_RBRACK { return 1 }
637 if b == INC_GT { return 1 }
638 return 0
639}
640// one organ source -> one index row appended at out[o..]; returns the new offset
641func inc_row_for(b: *i64, path: *u8, out: *u8, o: i64) -> i64 {
642 let c: *i64 = b[INC_B_CONF] as *i64
643 let env: *i64 = b[INC_B_ENV] as *i64
644 let rej: *u8 = b[INC_B_REJ] as *u8
645 let nrej: i64 = b[INC_B_NREJ]
646 let set: *i64 = b[INC_B_SET] as *i64
647 let hb: *u8 = b[INC_B_HEAD] as *u8
648 let nm: *u8 = b[INC_B_NAME] as *u8
649 let cand: *u8 = b[INC_B_CAND] as *u8
650 let cap: i64 = c[INC_C_HEAD_BYTES]
651 let got: i64 = inc_read_head(path, hb, cap)
652 if got < 0 { return o }
653 inc_organ_name(path, nm)
654 var i: i64 = 0
655 while i < INC_SETCAP { set[i] = 0; i = i + 1 }
656 var oo: i64 = inc_cat(out, o, INC_ORG_TAG)
657 oo = inc_cat(out, oo, nm)
658 oo = inc_cat(out, oo, "|" as *u8)
659 oo = inc_cat(out, oo, b[INC_B_ROOT] as *u8)
660 oo = inc_cat(out, oo, "|" as *u8)
661 let trunc: *i64 = sys_mmap(INC_I64) as *i64
662 let hend: i64 = inc_head_end(hb, got, cap, trunc)
663 if trunc[0] == 1 { env[INC_E_TRUNC] = env[INC_E_TRUNC] + 1 }
664 // the bag: basename parts, header words, column-0 func names anywhere in the window
665 let firstp: *i64 = sys_mmap(INC_I64) as *i64
666 firstp[0] = 1
667 oo = inc_bag_emit(b, nm, inc_slen(nm), out, oo, firstp)
668 oo = inc_bag_emit(b, hb, hend, out, oo, firstp)
669 var q: i64 = 0
670 while q < got {
671 var atline: i64 = 0
672 if q == 0 { atline = 1 } else { if hb[q - 1] == (INC_NL as u8) { atline = 1 } }
673 if atline == 1 { if q + INC_FUNC_LEN < got { if inc_starts((hb as i64 + q) as *u8, INC_FUNC) == 1 {
674 let eol: i64 = inc_lend(hb, got, q)
675 var e2: i64 = q + INC_FUNC_LEN
676 var go: i64 = 1
677 while go == 1 { if e2 >= eol { go = 0 } else { if hb[e2] == (INC_LPAREN as u8) { go = 0 } else { e2 = e2 + 1 } } }
678 oo = inc_bag_emit(b, (hb as i64 + q + INC_FUNC_LEN) as *u8, e2 - q - INC_FUNC_LEN, out, oo, firstp)
679 } } }
680 q = q + 1
681 }
682 // assets: knowledge/ paths named in the header, unique within the row
683 oo = inc_cat(out, oo, "|" as *u8)
684 let astart: i64 = oo
685 var afirst: i64 = 1
686 var k: i64 = 0
687 let kl: i64 = inc_slen(INC_KNOW)
688 while k + kl <= hend {
689 if inc_starts((hb as i64 + k) as *u8, INC_KNOW) == 1 {
690 var e3: i64 = k
691 var stop: i64 = 0
692 while stop == 0 { if e3 >= hend { stop = 1 } else { if inc_path_stop(hb[e3] as i64) == 1 { stop = 1 } else { e3 = e3 + 1 } } }
693 var e4: i64 = e3
694 if e4 > k { if hb[e4 - 1] == (INC_DOT as u8) { e4 = e4 - 1 } else { if hb[e4 - 1] == (INC_COLON as u8) { e4 = e4 - 1 } } }
695 var ci: i64 = 0
696 while k + ci < e4 { if ci < INC_PATH_CAP - 1 { cand[ci] = hb[k + ci] } ci = ci + 1 }
697 if ci >= INC_PATH_CAP { ci = INC_PATH_CAP - 1 }
698 cand[ci] = 0 as u8
699 if ci > kl { if inc_listed(out, astart, oo, cand) == 0 {
700 if afirst == 0 { oo = inc_cat(out, oo, " " as *u8) }
701 oo = inc_cat(out, oo, cand)
702 afirst = 0
703 } }
704 k = e3
705 } else { k = k + 1 }
706 }
707 // rejection lines: header lines carrying a phrase, up to rej_per_org, each cut at rej_line_cap; pipes and
708 // semicolons become slashes so the row grammar holds
709 oo = inc_cat(out, oo, "|" as *u8)
710 var kept: i64 = 0
711 var p: i64 = 0
712 while p < hend {
713 let e5: i64 = inc_lend(hb, hend, p)
714 if kept < c[INC_C_REJ_PER_ORG] {
715 var hit: i64 = 0
716 var ri: i64 = 0
717 while ri < nrej { if inc_contains((hb as i64 + p) as *u8, e5 - p, inc_reject_nth(rej, ri)) == 1 { hit = 1; ri = nrej } else { ri = ri + 1 } }
718 if hit == 1 {
719 if kept > 0 { oo = inc_cat(out, oo, INC_REJ_SEP) }
720 var s3: i64 = p
721 var lead: i64 = 1
722 while lead == 1 { if s3 >= e5 { lead = 0 } else { let bl: i64 = hb[s3] as i64; if bl == INC_SLASH { s3 = s3 + 1 } else { if bl == INC_SPACE { s3 = s3 + 1 } else { lead = 0 } } } }
723 var wrote: i64 = 0
724 while s3 < e5 {
725 if wrote < c[INC_C_REJ_LINE_CAP] {
726 var b2: i64 = hb[s3] as i64
727 if b2 == INC_PIPE { b2 = INC_SLASH }
728 if b2 == INC_SEMI { b2 = INC_SLASH }
729 if b2 == INC_CR { b2 = INC_SPACE }
730 out[oo] = b2 as u8
731 oo = oo + 1
732 wrote = wrote + 1
733 }
734 s3 = s3 + 1
735 }
736 kept = kept + 1
737 }
738 }
739 p = e5 + 1
740 }
741 out[oo] = INC_NL as u8
742 oo = oo + 1
743 out[oo] = 0 as u8
744 env[INC_E_FILES] = env[INC_E_FILES] + 1
745 return oo
746}
747// one asset entry -> asset|name|words
748func inc_asset_row(b: *i64, name: *u8, out: *u8, o: i64) -> i64 {
749 let set: *i64 = b[INC_B_SET] as *i64
750 var i: i64 = 0
751 while i < INC_SETCAP { set[i] = 0; i = i + 1 }
752 var oo: i64 = inc_cat(out, o, INC_ASSET_TAG)
753 oo = inc_cat(out, oo, name)
754 oo = inc_cat(out, oo, "|" as *u8)
755 let firstp: *i64 = sys_mmap(INC_I64) as *i64
756 firstp[0] = 1
757 oo = inc_bag_emit(b, name, inc_slen(name), out, oo, firstp)
758 out[oo] = INC_NL as u8
759 oo = oo + 1
760 out[oo] = 0 as u8
761 return oo
762}
763// write the whole index atomically: <out_path>.tmp then rename; returns bytes written or INC_NONE
764func inc_publish(out_path: *u8, head: *u8, rows: *u8, rows_len: i64) -> i64 {
765 let tmp: *u8 = sys_mmap(INC_PATH_CAP)
766 var t: i64 = inc_cat(tmp, 0, out_path)
767 t = inc_cat(tmp, t, INC_TMP_SUFFIX)
768 let fd: i64 = sys_openat_wr(tmp, MODE_0644)
769 if fd < 0 { return INC_NONE }
770 let hl: i64 = inc_slen(head)
771 let w1: i64 = sys_write(fd, head, hl)
772 var w2: i64 = 0
773 var off: i64 = 0
774 var go: i64 = 1
775 while go == 1 {
776 if off >= rows_len { go = 0 } else {
777 let r: i64 = sys_write(fd, (rows as i64 + off) as *u8, rows_len - off)
778 if r <= 0 { go = 0 } else { off = off + r; w2 = w2 + r }
779 }
780 }
781 sys_close(fd)
782 if w1 != hl { return INC_NONE }
783 if w2 != rows_len { return INC_NONE }
784 if sys_renameat(tmp, out_path) != 0 { return INC_NONE }
785 return hl + rows_len
786}
787func inc_fill_names(dir: *u8, cnt: i64, srconly: i64, env: *i64) -> *u8 {
788 let names: *u8 = sys_mmap((cnt + 1) * INC_NAME_CAP)
789 if cnt > 0 { let r: i64 = inc_list_dir(dir, names, cnt, srconly); if r != cnt { env[INC_E_COMPLETE] = 0 } }
790 return names
791}
792// BUILD: the two source roots and the asset dir -> the index at out_path. Returns the envelope box.
793func inc_index_build(c: *i64, root_a: *u8, root_b: *u8, asset_dir: *u8, out_path: *u8) -> *i64 {
794 let env: *i64 = sys_mmap(INC_E_N * INC_I64) as *i64
795 var k: i64 = 0
796 while k < INC_E_N { env[k] = 0; k = k + 1 }
797 env[INC_E_COMPLETE] = 1
798 env[INC_E_GEN] = inc_now()
799 let scratch: *u8 = sys_mmap(INC_I64)
800 var ca: i64 = inc_list_dir(root_a, scratch, 0, 1)
801 var cb: i64 = inc_list_dir(root_b, scratch, 0, 1)
802 var cs: i64 = inc_list_dir(asset_dir, scratch, 0, 0)
803 if ca < 0 { ca = 0; env[INC_E_COMPLETE] = 0 }
804 if cb < 0 { cb = 0; env[INC_E_COMPLETE] = 0 }
805 if cs < 0 { cs = 0; env[INC_E_COMPLETE] = 0 }
806 let names_a: *u8 = inc_fill_names(root_a, ca, 1, env)
807 let names_b: *u8 = inc_fill_names(root_b, cb, 1, env)
808 let names_s: *u8 = inc_fill_names(asset_dir, cs, 0, env)
809 // the rows buffer, sized from the population (mmap is lazy: headroom costs address space, not memory)
810 let per: i64 = c[INC_C_HEAD_BYTES] + INC_ROW_SLACK
811 let rows: *u8 = sys_mmap((ca + cb + cs + 2) * per)
812 let nrejp: *i64 = sys_mmap(INC_I64) as *i64
813 let rej: *u8 = inc_reject_load(nrejp)
814 let b: *i64 = sys_mmap(INC_B_N * INC_I64) as *i64
815 b[INC_B_CONF] = c as i64
816 b[INC_B_ENV] = env as i64
817 b[INC_B_REJ] = rej as i64
818 b[INC_B_NREJ] = nrejp[0]
819 b[INC_B_SET] = sys_mmap(INC_SETCAP * INC_I64) as i64
820 b[INC_B_NAME] = sys_mmap(INC_NAME_CAP) as i64
821 b[INC_B_WORD] = sys_mmap(INC_WORD_KEEP + 1) as i64
822 b[INC_B_POS] = sys_mmap(INC_I64) as i64
823 b[INC_B_CAND] = sys_mmap(INC_PATH_CAP) as i64
824 b[INC_B_HEAD] = sys_mmap(c[INC_C_HEAD_BYTES] + 1) as i64
825 let path: *u8 = sys_mmap(INC_PATH_CAP)
826 var o: i64 = 0
827 var i: i64 = 0
828 b[INC_B_ROOT] = root_a as i64
829 while i < ca {
830 var t: i64 = inc_cat(path, 0, root_a)
831 t = inc_cat(path, t, "/" as *u8)
832 t = inc_cat(path, t, (names_a as i64 + i * INC_NAME_CAP) as *u8)
833 o = inc_row_for(b, path, rows, o)
834 i = i + 1
835 }
836 b[INC_B_ROOT] = root_b as i64
837 i = 0
838 while i < cb {
839 var t2: i64 = inc_cat(path, 0, root_b)
840 t2 = inc_cat(path, t2, "/" as *u8)
841 t2 = inc_cat(path, t2, (names_b as i64 + i * INC_NAME_CAP) as *u8)
842 o = inc_row_for(b, path, rows, o)
843 i = i + 1
844 }
845 i = 0
846 while i < cs {
847 o = inc_asset_row(b, (names_s as i64 + i * INC_NAME_CAP) as *u8, rows, o)
848 env[INC_E_ASSETS] = env[INC_E_ASSETS] + 1
849 i = i + 1
850 }
851 // the head, written last so its counts are the counts
852 let head: *u8 = sys_mmap(INC_PATH_CAP)
853 var h: i64 = inc_cat(head, 0, INC_HEAD_MARK)
854 h = inc_catn(head, h, INC_PATH_CAP, env[INC_E_GEN])
855 h = inc_cat(head, h, " files=" as *u8); h = inc_catn(head, h, INC_PATH_CAP, env[INC_E_FILES])
856 h = inc_cat(head, h, " head_truncated=" as *u8); h = inc_catn(head, h, INC_PATH_CAP, env[INC_E_TRUNC])
857 h = inc_cat(head, h, " assets=" as *u8); h = inc_catn(head, h, INC_PATH_CAP, env[INC_E_ASSETS])
858 h = inc_cat(head, h, " corpus_complete=" as *u8); h = inc_catn(head, h, INC_PATH_CAP, env[INC_E_COMPLETE])
859 h = inc_cat(head, h, " reject_phrases=" as *u8); h = inc_catn(head, h, INC_PATH_CAP, nrejp[0])
860 h = inc_cat(head, h, "\n" as *u8)
861 env[INC_E_BYTES] = inc_publish(out_path, head, rows, o)
862 return env
863}
864
865// ---- the index load ----
866// split a NUL-terminated row into fields at |; fld[i] = start offsets; returns the field count
867func inc_split(ln: *u8, fld: *i64, maxf: i64) -> i64 {
868 var f: i64 = 1
869 var i: i64 = 0
870 fld[0] = 0
871 while ln[i] != (0 as u8) {
872 if ln[i] == (INC_PIPE as u8) { ln[i] = 0 as u8; if f < maxf { fld[f] = i + 1; f = f + 1 } }
873 i = i + 1
874 }
875 return f
876}
877func inc_hput(g: *i64, key: i64, word: i64, row: i64) -> i64 {
878 let cap: i64 = g[INC_G_HCAP]
879 let hkey: *i64 = g[INC_G_HKEY] as *i64
880 let hhead: *i64 = g[INC_G_HHEAD] as *i64
881 let pword: *i64 = g[INC_G_PWORD] as *i64
882 let prow: *i64 = g[INC_G_PROW] as *i64
883 let pnext: *i64 = g[INC_G_PNEXT] as *i64
884 var h: i64 = inc_hash(key, cap)
885 var probes: i64 = 0
886 var slot: i64 = INC_NONE
887 while probes < cap {
888 if hkey[h] == 0 { hkey[h] = key; hhead[h] = INC_NONE; slot = h; probes = cap }
889 else { if hkey[h] == key { slot = h; probes = cap } else { h = (h + 1) & (cap - 1); probes = probes + 1 } }
890 }
891 if slot == INC_NONE { return INC_NONE }
892 let np: i64 = g[INC_G_NPOST]
893 pword[np] = word
894 prow[np] = row
895 pnext[np] = hhead[slot]
896 hhead[slot] = np
897 g[INC_G_NPOST] = np + 1
898 return slot
899}
900func inc_hfind(g: *i64, key: i64) -> i64 {
901 let cap: i64 = g[INC_G_HCAP]
902 let hkey: *i64 = g[INC_G_HKEY] as *i64
903 var h: i64 = inc_hash(key, cap)
904 var probes: i64 = 0
905 while probes < cap {
906 if hkey[h] == 0 { return INC_NONE }
907 if hkey[h] == key { return h }
908 h = (h + 1) & (cap - 1)
909 probes = probes + 1
910 }
911 return INC_NONE
912}
913func inc_state_name(s: i64) -> *u8 {
914 if s == INC_S_FRESH { return "FRESH" as *u8 }
915 if s == INC_S_STALE { return "STALE" as *u8 }
916 if s == INC_S_ABSENT { return "ABSENT" as *u8 }
917 return "MALFORMED" as *u8
918}
919// LOAD the index at path (now = the clock, or a value a gate hands in). Always returns a box; read INC_G_STATE.
920func inc_load_path(c: *i64, path: *u8, now: i64) -> *i64 {
921 let g: *i64 = sys_mmap(INC_G_N_SLOTS * INC_I64) as *i64
922 var k: i64 = 0
923 while k < INC_G_N_SLOTS { g[k] = 0; k = k + 1 }
924 g[INC_G_PATH] = path as i64
925 g[INC_G_STATE] = INC_S_ABSENT
926 let lenp: *i64 = sys_mmap(INC_I64) as *i64
927 lenp[0] = 0
928 let buf: *u8 = sys_read_file(path, lenp)
929 if lenp[0] <= 0 { return g }
930 let n: i64 = lenp[0]
931 g[INC_G_BUF] = buf as i64
932 g[INC_G_N] = n
933 if inc_starts(buf, INC_HEAD_MARK) == 0 { g[INC_G_STATE] = INC_S_MALFORMED; return g }
934 // pass 1: count rows and an upper bound on words (spaces in org rows)
935 var rows: i64 = 0
936 var assets: i64 = 0
937 var words: i64 = 0
938 var p: i64 = 0
939 while p < n {
940 let e: i64 = inc_lend(buf, n, p)
941 if inc_starts((buf as i64 + p) as *u8, INC_ORG_TAG) == 1 {
942 rows = rows + 1
943 var q: i64 = p
944 while q < e { if buf[q] == (INC_SPACE as u8) { words = words + 1 } q = q + 1 }
945 words = words + 1
946 }
947 if inc_starts((buf as i64 + p) as *u8, INC_ASSET_TAG) == 1 { assets = assets + 1 }
948 p = e + 1
949 }
950 g[INC_G_R_NAME] = sys_mmap((rows + 1) * INC_I64) as i64
951 g[INC_G_R_ROOT] = sys_mmap((rows + 1) * INC_I64) as i64
952 g[INC_G_R_WORDS] = sys_mmap((rows + 1) * INC_I64) as i64
953 g[INC_G_R_ASSETS] = sys_mmap((rows + 1) * INC_I64) as i64
954 g[INC_G_R_REJ] = sys_mmap((rows + 1) * INC_I64) as i64
955 g[INC_G_A_NAME] = sys_mmap((assets + 1) * INC_I64) as i64
956 g[INC_G_A_WORDS] = sys_mmap((assets + 1) * INC_I64) as i64
957 g[INC_G_SEEN] = sys_mmap((rows + 1) * INC_I64) as i64
958 g[INC_G_SCORE] = sys_mmap((rows + 1) * INC_I64) as i64
959 g[INC_G_MASK] = sys_mmap((rows + 1) * INC_I64) as i64
960 g[INC_G_TIER] = sys_mmap((rows + 1) * INC_I64) as i64
961 g[INC_G_NIDF] = sys_mmap((rows + 1) * INC_I64) as i64
962 var cap: i64 = INC_HASH_MIN
963 while cap < words * 2 { cap = cap * 2 }
964 g[INC_G_HCAP] = cap
965 g[INC_G_HKEY] = sys_mmap(cap * INC_I64) as i64
966 g[INC_G_HHEAD] = sys_mmap(cap * INC_I64) as i64
967 g[INC_G_PWORD] = sys_mmap((words + 1) * INC_I64) as i64
968 g[INC_G_PROW] = sys_mmap((words + 1) * INC_I64) as i64
969 g[INC_G_PNEXT] = sys_mmap((words + 1) * INC_I64) as i64
970 g[INC_G_NPOST] = 0
971 let rname: *i64 = g[INC_G_R_NAME] as *i64
972 let rroot: *i64 = g[INC_G_R_ROOT] as *i64
973 let rwords: *i64 = g[INC_G_R_WORDS] as *i64
974 let rassets: *i64 = g[INC_G_R_ASSETS] as *i64
975 let rrej: *i64 = g[INC_G_R_REJ] as *i64
976 let aname: *i64 = g[INC_G_A_NAME] as *i64
977 let awords: *i64 = g[INC_G_A_WORDS] as *i64
978 let fld: *i64 = sys_mmap(INC_ORG_FIELDS * INC_I64) as *i64
979 // pass 2: the head, then the rows (NUL-terminated in place) and their postings
980 var ri: i64 = 0
981 var ai: i64 = 0
982 var first: i64 = 1
983 p = 0
984 while p < n {
985 let e2: i64 = inc_lend(buf, n, p)
986 buf[e2] = 0 as u8
987 let ln: *u8 = (buf as i64 + p) as *u8
988 if first == 1 {
989 first = 0
990 g[INC_G_GEN] = inc_kv_int(ln, "generated_at" as *u8)
991 g[INC_G_FILES] = inc_kv_int(ln, "files" as *u8)
992 g[INC_G_TRUNC] = inc_kv_int(ln, "head_truncated" as *u8)
993 g[INC_G_COMPLETE] = inc_kv_int(ln, "corpus_complete" as *u8)
994 } else {
995 if inc_starts(ln, INC_ORG_TAG) == 1 {
996 let nf: i64 = inc_split(ln, fld, INC_ORG_FIELDS)
997 if nf >= INC_ORG_FIELDS { if ri < rows {
998 rname[ri] = ln as i64 + fld[1]
999 rroot[ri] = ln as i64 + fld[2]
1000 rwords[ri] = ln as i64 + fld[3]
1001 rassets[ri] = ln as i64 + fld[4]
1002 rrej[ri] = ln as i64 + fld[5]
1003 // postings: one per bag word, keyed on its first morph_short bytes; the words field becomes NUL-separated
1004 let ws: *u8 = rwords[ri] as *u8
1005 let wl: i64 = inc_slen(ws)
1006 var s: i64 = 0
1007 while s < wl {
1008 let e3: i64 = inc_wend(ws, wl, s)
1009 if e3 > s {
1010 ws[e3] = 0 as u8
1011 let kk: i64 = inc_key((ws as i64 + s) as *u8, c[INC_C_MORPH_SHORT])
1012 if kk != 0 { inc_hput(g, kk, ws as i64 + s, ri) }
1013 }
1014 s = e3 + 1
1015 }
1016 ri = ri + 1
1017 } }
1018 }
1019 if inc_starts(ln, INC_ASSET_TAG) == 1 {
1020 let nf2: i64 = inc_split(ln, fld, INC_ASSET_FIELDS)
1021 if nf2 >= INC_ASSET_FIELDS { if ai < assets {
1022 aname[ai] = ln as i64 + fld[1]
1023 awords[ai] = ln as i64 + fld[2]
1024 ai = ai + 1
1025 } }
1026 }
1027 }
1028 p = e2 + 1
1029 }
1030 g[INC_G_ROWS] = ri
1031 g[INC_G_ASSETS] = ai
1032 if g[INC_G_GEN] <= 0 { g[INC_G_STATE] = INC_S_MALFORMED; return g }
1033 g[INC_G_AGE] = now - g[INC_G_GEN]
1034 if g[INC_G_AGE] < 0 { g[INC_G_STATE] = INC_S_STALE; return g }
1035 if g[INC_G_AGE] > c[INC_C_INDEX_MAX_AGE_S] { g[INC_G_STATE] = INC_S_STALE; return g }
1036 g[INC_G_STATE] = INC_S_FRESH
1037 return g
1038}
1039// the index at its standing path (knowledge/status, then ../knowledge/status), judged against the clock
1040func inc_load(c: *i64) -> *i64 {
1041 let now: i64 = inc_now()
1042 let g: *i64 = inc_load_path(c, INC_INDEX_A, now)
1043 if g[INC_G_STATE] == INC_S_ABSENT { return inc_load_path(c, INC_INDEX_B, now) }
1044 return g
1045}
1046
1047// ---- the match ----
1048// does the query term (full length tlen) match the bag word: the first morph_long bytes when the term is that long,
1049// else the term's own first morph_short bytes (or all of it when shorter)
1050func inc_term_hits(c: *i64, term: *u8, tlen: i64, word: *u8) -> i64 {
1051 // a term shorter than morph_long is matched WHOLE (measured 2026-09-14: cutting a 5-letter term to the 4-byte key
1052 // made dense match density and wired match wire, and both filled the rejection axis); the morph_short key only
1053 // narrows the posting chain the walk reads
1054 var take: i64 = c[INC_C_MORPH_LONG]
1055 if tlen < c[INC_C_MORPH_LONG] { take = tlen }
1056 var i: i64 = 0
1057 while i < take { if word[i] != term[i] { return 0 } i = i + 1 }
1058 return 1
1059}
1060// the name hits of an organ: out2[0] = how many matched terms hit its name parts, out2[1] = the sum of their idf
1061func inc_name_hits(c: *i64, r: *i64, name: *u8, w: *u8, pos: *i64, out2: *i64) -> i64 {
1062 let terms: *u8 = r[INC_R_TERMS] as *u8
1063 let toff: *i64 = r[INC_R_TOFF] as *i64
1064 let tidf: *i64 = r[INC_R_TIDF] as *i64
1065 let nt: i64 = r[INC_R_NTERMS]
1066 let nl: i64 = inc_slen(name)
1067 out2[0] = 0
1068 out2[1] = 0
1069 var t: i64 = 0
1070 while t < nt {
1071 if tidf[t] > 0 {
1072 let term: *u8 = (terms as i64 + toff[t]) as *u8
1073 var tl: i64 = inc_slen(term)
1074 if tl > INC_KEY_BYTES { tl = INC_KEY_BYTES }
1075 pos[0] = 0
1076 var hit: i64 = 0
1077 var l: i64 = inc_next_word(name, nl, pos, w)
1078 while l > 0 { if hit == 0 { if inc_term_hits(c, term, tl, w) == 1 { hit = 1 } } l = inc_next_word(name, nl, pos, w) }
1079 if hit == 1 { out2[0] = out2[0] + 1; out2[1] = out2[1] + tidf[t] }
1080 }
1081 t = t + 1
1082 }
1083 return out2[0]
1084}
1085// MATCH one contract (title, symbol, note) against the loaded index; excl is the rung's own organ name (or "")
1086func inc_match(g: *i64, c: *i64, title: *u8, sym: *u8, note: *u8, excl: *u8) -> *i64 {
1087 let r: *i64 = sys_mmap(INC_R_N_SLOTS * INC_I64) as *i64
1088 var k: i64 = 0
1089 while k < INC_R_N_SLOTS { r[k] = 0; k = k + 1 }
1090 let rows: i64 = g[INC_G_ROWS]
1091 let maxt: i64 = c[INC_C_MAX_TERMS]
1092 // terms: unique lowercase words of title + symbol + note, in that order, deduplicated by match prefix; all-digit
1093 // runs dropped and counted
1094 let tl: i64 = inc_slen(title) + inc_slen(sym) + inc_slen(note) + 3
1095 let text: *u8 = sys_mmap(tl + 1)
1096 var o: i64 = inc_cat(text, 0, title)
1097 o = inc_cat(text, o, " " as *u8)
1098 o = inc_cat(text, o, sym)
1099 o = inc_cat(text, o, " " as *u8)
1100 o = inc_cat(text, o, note)
1101 let terms: *u8 = sys_mmap((maxt + 1) * (INC_WORD_KEEP + 1))
1102 let toff: *i64 = sys_mmap((maxt + 1) * INC_I64) as *i64
1103 let tlen: *i64 = sys_mmap((maxt + 1) * INC_I64) as *i64
1104 let tidf: *i64 = sys_mmap((maxt + 1) * INC_I64) as *i64
1105 let w: *u8 = sys_mmap(INC_WORD_KEEP + 1)
1106 let pos: *i64 = sys_mmap(INC_I64) as *i64
1107 pos[0] = 0
1108 var nt: i64 = 0
1109 var to: i64 = 0
1110 var l: i64 = inc_next_word(text, o, pos, w)
1111 while l > 0 {
1112 if l >= c[INC_C_MIN_TOK] {
1113 var alldig: i64 = 1
1114 var di: i64 = 0
1115 while w[di] != (0 as u8) { if inc_is_digit(w[di] as i64) == 0 { alldig = 0 } di = di + 1 }
1116 var dup: i64 = alldig
1117 if alldig == 1 { r[INC_R_NUMERIC] = r[INC_R_NUMERIC] + 1 }
1118 var t: i64 = 0
1119 while t < nt {
1120 let tt: *u8 = (terms as i64 + toff[t]) as *u8
1121 if inc_term_hits(c, tt, tlen[t], w) == 1 { dup = 1 } else { if inc_term_hits(c, w, l, tt) == 1 { dup = 1 } }
1122 if dup == 1 { t = nt } else { t = t + 1 }
1123 }
1124 if dup == 0 {
1125 if nt < maxt {
1126 toff[nt] = to
1127 tlen[nt] = l
1128 if tlen[nt] > INC_KEY_BYTES { tlen[nt] = INC_KEY_BYTES }
1129 to = inc_cat(terms, to, w) + 1
1130 nt = nt + 1
1131 } else { r[INC_R_OVER] = r[INC_R_OVER] + 1 }
1132 }
1133 }
1134 l = inc_next_word(text, o, pos, w)
1135 }
1136 r[INC_R_NTERMS] = nt
1137 r[INC_R_TERMS] = terms as i64
1138 r[INC_R_TOFF] = toff as i64
1139 r[INC_R_TIDF] = tidf as i64
1140 let seen: *i64 = g[INC_G_SEEN] as *i64
1141 let score: *i64 = g[INC_G_SCORE] as *i64
1142 let mask: *i64 = g[INC_G_MASK] as *i64
1143 let tier: *i64 = g[INC_G_TIER] as *i64
1144 let nidf: *i64 = g[INC_G_NIDF] as *i64
1145 var i: i64 = 0
1146 while i < rows { seen[i] = 0; score[i] = 0; mask[i] = 0; tier[i] = 0; nidf[i] = 0; i = i + 1 }
1147 let hhead: *i64 = g[INC_G_HHEAD] as *i64
1148 let pword: *i64 = g[INC_G_PWORD] as *i64
1149 let prow: *i64 = g[INC_G_PROW] as *i64
1150 let pnext: *i64 = g[INC_G_PNEXT] as *i64
1151 let rname: *i64 = g[INC_G_R_NAME] as *i64
1152 let dfmax: i64 = rows * c[INC_C_DF_MAX_PERMIL] / INC_PERMIL
1153 var t2: i64 = 0
1154 while t2 < nt {
1155 let term: *u8 = (terms as i64 + toff[t2]) as *u8
1156 let key: i64 = inc_key(term, c[INC_C_MORPH_SHORT])
1157 let h: i64 = inc_hfind(g, key)
1158 tidf[t2] = 0
1159 if h != INC_NONE {
1160 // walk 1: df = distinct rows whose word agrees on the term's full match length
1161 var df: i64 = 0
1162 var pi: i64 = hhead[h]
1163 let stamp: i64 = t2 + 1
1164 while pi != INC_NONE {
1165 if inc_term_hits(c, term, tlen[t2], pword[pi] as *u8) == 1 { let rw: i64 = prow[pi]; if seen[rw] != stamp { seen[rw] = stamp; df = df + 1 } }
1166 pi = pnext[pi]
1167 }
1168 if df > dfmax { r[INC_R_DROPPED] = r[INC_R_DROPPED] + 1; r[INC_R_DROPMASK] = r[INC_R_DROPMASK] | (1 << t2) }
1169 else { if df > 0 {
1170 let wgt: i64 = c[INC_C_IDF_SCALE] / (c[INC_C_IDF_BASE] + df)
1171 tidf[t2] = wgt
1172 let bit: i64 = 1 << t2
1173 pi = hhead[h]
1174 while pi != INC_NONE {
1175 if inc_term_hits(c, term, tlen[t2], pword[pi] as *u8) == 1 {
1176 let rw2: i64 = prow[pi]
1177 if (mask[rw2] & bit) == 0 { mask[rw2] = mask[rw2] | bit; score[rw2] = score[rw2] + wgt }
1178 }
1179 pi = pnext[pi]
1180 }
1181 } }
1182 }
1183 t2 = t2 + 1
1184 }
1185 // the rung's own organ never answers its own contract
1186 if excl[0] != (0 as u8) { i = 0; while i < rows { if inc_streq(rname[i] as *u8, excl) == 1 { score[i] = 0; mask[i] = 0 } i = i + 1 } }
1187 // the name hits of every matching row: how many matched terms hit its name parts (tier) and their idf, added to
1188 // the score as a second piece of evidence weighed like the first
1189 let nh: *i64 = sys_mmap(2 * INC_I64) as *i64
1190 var matched: i64 = 0
1191 i = 0
1192 while i < rows { if score[i] > 0 { matched = matched + 1; inc_name_hits(c, r, rname[i] as *u8, w, pos, nh); tier[i] = nh[0]; nidf[i] = nh[1]; score[i] = score[i] + nh[1] } i = i + 1 }
1193 r[INC_R_MATCHED] = matched
1194 // top-N by score (ties by row order): selection, N is small. The scores of the selected rows are zeroed here;
1195 // tier[] survives for the rejection axis of the renderer.
1196 let topn: i64 = c[INC_C_TOPN]
1197 let rrow: *i64 = sys_mmap((topn + 1) * INC_I64) as *i64
1198 let rsc: *i64 = sys_mmap((topn + 1) * INC_I64) as *i64
1199 let rmk: *i64 = sys_mmap((topn + 1) * INC_I64) as *i64
1200 let rtr: *i64 = sys_mmap((topn + 1) * INC_I64) as *i64
1201 var nres: i64 = 0
1202 var round: i64 = 0
1203 while round < topn {
1204 var best: i64 = INC_NONE
1205 var bs: i64 = 0
1206 i = 0
1207 while i < rows { if score[i] > bs { bs = score[i]; best = i } i = i + 1 }
1208 if best == INC_NONE { round = topn }
1209 else { rrow[nres] = best; rsc[nres] = bs; rmk[nres] = mask[best]; rtr[nres] = tier[best]; nres = nres + 1; score[best] = 0; round = round + 1 }
1210 }
1211 r[INC_R_N] = nres
1212 r[INC_R_ROW] = rrow as i64
1213 r[INC_R_SCORE] = rsc as i64
1214 r[INC_R_MASK] = rmk as i64
1215 r[INC_R_TIER] = rtr as i64
1216 return r
1217}
1218// does any word of the asset's name words match any kept term
1219func inc_asset_hits(c: *i64, r: *i64, ws: *u8) -> i64 {
1220 let terms: *u8 = r[INC_R_TERMS] as *u8
1221 let toff: *i64 = r[INC_R_TOFF] as *i64
1222 let tidf: *i64 = r[INC_R_TIDF] as *i64
1223 let nt: i64 = r[INC_R_NTERMS]
1224 let wl: i64 = inc_slen(ws)
1225 var t: i64 = 0
1226 while t < nt {
1227 if (r[INC_R_DROPMASK] & (1 << t)) == 0 {
1228 let term: *u8 = (terms as i64 + toff[t]) as *u8
1229 var tl: i64 = inc_slen(term)
1230 if tl > INC_KEY_BYTES { tl = INC_KEY_BYTES }
1231 var s: i64 = 0
1232 while s < wl {
1233 let e: i64 = inc_wend(ws, wl, s)
1234 if e > s { if inc_term_hits(c, term, tl, (ws as i64 + s) as *u8) == 1 { return 1 } }
1235 s = e + 1
1236 }
1237 }
1238 t = t + 1
1239 }
1240 return 0
1241}
1242// append the assets a row's header names (unique in the list so far) to out; v = [astart, na]
1243func inc_render_assets(g: *i64, row: i64, out: *u8, o: i64, cap: i64, v: *i64) -> i64 {
1244 let rassets: *i64 = g[INC_G_R_ASSETS] as *i64
1245 let alist: *u8 = rassets[row] as *u8
1246 let al: i64 = inc_slen(alist)
1247 let one: *u8 = sys_mmap(INC_PATH_CAP)
1248 var oo: i64 = o
1249 var s: i64 = 0
1250 while s < al {
1251 let e: i64 = inc_wend(alist, al, s)
1252 if e > s {
1253 var j: i64 = 0
1254 while j < e - s { if j < INC_PATH_CAP - 1 { one[j] = alist[s + j] } j = j + 1 }
1255 if j >= INC_PATH_CAP { j = INC_PATH_CAP - 1 }
1256 one[j] = 0 as u8
1257 if inc_listed(out, v[INC_V_ASTART], oo, one) == 0 { if v[INC_V_NA] > 0 { oo = inc_catb(out, oo, cap, " " as *u8) } oo = inc_catb(out, oo, cap, one); v[INC_V_NA] = v[INC_V_NA] + 1 }
1258 }
1259 s = e + 1
1260 }
1261 return oo
1262}
1263// append one row's rejection lines to out; v[INC_V_N] counts rows written
1264func inc_render_rej(g: *i64, row: i64, out: *u8, o: i64, cap: i64, v: *i64) -> i64 {
1265 let rname: *i64 = g[INC_G_R_NAME] as *i64
1266 let rrej: *i64 = g[INC_G_R_REJ] as *i64
1267 let rj: *u8 = rrej[row] as *u8
1268 if rj[0] == (0 as u8) { return o }
1269 var oo: i64 = o
1270 if v[INC_V_N] > 0 { oo = inc_catb(out, oo, cap, " " as *u8) }
1271 oo = inc_catb(out, oo, cap, rname[row] as *u8)
1272 oo = inc_catb(out, oo, cap, ": " as *u8)
1273 oo = inc_catb(out, oo, cap, rj)
1274 v[INC_V_N] = v[INC_V_N] + 1
1275 return oo
1276}
1277// is row a rejection-axis candidate: a name hit, a rejection line on record, not among the top results
1278func inc_rej_cand(g: *i64, r: *i64, row: i64) -> i64 {
1279 let tier: *i64 = g[INC_G_TIER] as *i64
1280 if tier[row] <= 0 { return 0 }
1281 let rrej: *i64 = g[INC_G_R_REJ] as *i64
1282 let rj: *u8 = rrej[row] as *u8
1283 if rj[0] == (0 as u8) { return 0 }
1284 let rrow: *i64 = r[INC_R_ROW] as *i64
1285 var q: i64 = 0
1286 while q < r[INC_R_N] { if rrow[q] == row { return 0 } q = q + 1 }
1287 return 1
1288}
1289// RENDER one answer into out (cap bytes, a cut announced with [CUT]): INCUMBENTS(rid): ... | ASSETS: ... | PRIOR-REJECTIONS: ...
1290func inc_render(g: *i64, c: *i64, r: *i64, rid: *u8, out: *u8, cap: i64) -> i64 {
1291 var o: i64 = inc_catb(out, 0, cap, "INCUMBENTS(" as *u8)
1292 o = inc_catb(out, o, cap, rid)
1293 o = inc_catb(out, o, cap, "): " as *u8)
1294 let n: i64 = r[INC_R_N]
1295 let terms: *u8 = r[INC_R_TERMS] as *u8
1296 let toff: *i64 = r[INC_R_TOFF] as *i64
1297 let nt: i64 = r[INC_R_NTERMS]
1298 let rname: *i64 = g[INC_G_R_NAME] as *i64
1299 let rrej: *i64 = g[INC_G_R_REJ] as *i64
1300 let rrow: *i64 = r[INC_R_ROW] as *i64
1301 let rsc: *i64 = r[INC_R_SCORE] as *i64
1302 let rmk: *i64 = r[INC_R_MASK] as *i64
1303 let rtr: *i64 = r[INC_R_TIER] as *i64
1304 if n == 0 {
1305 o = inc_catb(out, o, cap, "NONE (0 of " as *u8)
1306 o = inc_catn(out, o, cap, g[INC_G_ROWS])
1307 o = inc_catb(out, o, cap, " organs match; terms tried=" as *u8)
1308 var t0: i64 = 0
1309 while t0 < nt { if t0 > 0 { o = inc_catb(out, o, cap, "," as *u8) } o = inc_catb(out, o, cap, (terms as i64 + toff[t0]) as *u8); t0 = t0 + 1 }
1310 o = inc_catb(out, o, cap, " dropped_common=" as *u8)
1311 o = inc_catn(out, o, cap, r[INC_R_DROPPED])
1312 o = inc_catb(out, o, cap, " dropped_numeric=" as *u8)
1313 o = inc_catn(out, o, cap, r[INC_R_NUMERIC])
1314 o = inc_catb(out, o, cap, ")" as *u8)
1315 if o >= cap - 1 { o = inc_cat(out, cap - 6, "[CUT]" as *u8) }
1316 return o
1317 }
1318 var i: i64 = 0
1319 while i < n {
1320 if i > 0 { o = inc_catb(out, o, cap, " " as *u8) }
1321 o = inc_catb(out, o, cap, rname[rrow[i]] as *u8)
1322 o = inc_catb(out, o, cap, "(name" as *u8)
1323 o = inc_catn(out, o, cap, rtr[i])
1324 o = inc_catb(out, o, cap, " idf" as *u8)
1325 o = inc_catn(out, o, cap, rsc[i])
1326 o = inc_catb(out, o, cap, ":" as *u8)
1327 var t: i64 = 0
1328 var firstt: i64 = 1
1329 while t < nt { if (rmk[i] & (1 << t)) != 0 { if firstt == 0 { o = inc_catb(out, o, cap, "," as *u8) } o = inc_catb(out, o, cap, (terms as i64 + toff[t]) as *u8); firstt = 0 } t = t + 1 }
1330 o = inc_catb(out, o, cap, ")" as *u8)
1331 i = i + 1
1332 }
1333 o = inc_catb(out, o, cap, " of " as *u8)
1334 o = inc_catn(out, o, cap, r[INC_R_MATCHED])
1335 o = inc_catb(out, o, cap, " matching, dropped_common=" as *u8)
1336 o = inc_catn(out, o, cap, r[INC_R_DROPPED])
1337 o = inc_catb(out, o, cap, " dropped_numeric=" as *u8)
1338 o = inc_catn(out, o, cap, r[INC_R_NUMERIC])
1339 // the rejection axis beyond the top organs: every organ whose NAME carries a contract term and whose header
1340 // records a rejection (a rejection on an organ named for the contract's own word is the record answering the
1341 // contract); capped by rej_orgs, the overflow counted
1342 // the candidates ranked by the idf of their name hits: the rarer the contract word in an organ's name, the more
1343 // specifically that organ is named for the contract (a name hit on a common word is how a generic organ intrudes)
1344 let nidf: *i64 = g[INC_G_NIDF] as *i64
1345 let xcap: i64 = c[INC_C_REJ_ORGS]
1346 let xrows: *i64 = sys_mmap((xcap + 1) * INC_I64) as *i64
1347 var nx: i64 = 0
1348 var cands: i64 = 0
1349 var ri: i64 = 0
1350 while ri < g[INC_G_ROWS] { if inc_rej_cand(g, r, ri) == 1 { cands = cands + 1 } ri = ri + 1 }
1351 var round: i64 = 0
1352 while round < xcap {
1353 var best: i64 = INC_NONE
1354 var bv: i64 = INC_NONE
1355 ri = 0
1356 while ri < g[INC_G_ROWS] {
1357 if inc_rej_cand(g, r, ri) == 1 {
1358 var taken: i64 = 0
1359 var q: i64 = 0
1360 while q < nx { if xrows[q] == ri { taken = 1; q = nx } else { q = q + 1 } }
1361 if taken == 0 { if nidf[ri] > bv { bv = nidf[ri]; best = ri } }
1362 }
1363 ri = ri + 1
1364 }
1365 if best == INC_NONE { round = xcap } else { xrows[nx] = best; nx = nx + 1; round = round + 1 }
1366 }
1367 let xover: i64 = cands - nx
1368 // assets: named in the matched headers and the rejection organs' headers (unique), then assets whose own name
1369 // matches a term
1370 o = inc_catb(out, o, cap, " | ASSETS: " as *u8)
1371 let v: *i64 = sys_mmap(INC_V_N * INC_I64 + INC_I64) as *i64
1372 v[INC_V_ASTART] = o
1373 v[INC_V_NA] = 0
1374 v[INC_V_N] = 0
1375 i = 0
1376 while i < n { o = inc_render_assets(g, rrow[i], out, o, cap, v); i = i + 1 }
1377 i = 0
1378 while i < nx { o = inc_render_assets(g, xrows[i], out, o, cap, v); i = i + 1 }
1379 let aname: *i64 = g[INC_G_A_NAME] as *i64
1380 let awords: *i64 = g[INC_G_A_WORDS] as *i64
1381 var ai: i64 = 0
1382 while ai < g[INC_G_ASSETS] {
1383 if inc_asset_hits(c, r, awords[ai] as *u8) == 1 { if inc_listed(out, v[INC_V_ASTART], o, aname[ai] as *u8) == 0 {
1384 if v[INC_V_NA] > 0 { o = inc_catb(out, o, cap, " " as *u8) }
1385 o = inc_catb(out, o, cap, aname[ai] as *u8)
1386 o = inc_catb(out, o, cap, "(by-name)" as *u8)
1387 v[INC_V_NA] = v[INC_V_NA] + 1
1388 } }
1389 ai = ai + 1
1390 }
1391 if v[INC_V_NA] == 0 { o = inc_catb(out, o, cap, "none" as *u8) }
1392 // rejections: the top organs' lines, then the rejection axis
1393 o = inc_catb(out, o, cap, " | PRIOR-REJECTIONS: " as *u8)
1394 v[INC_V_N] = 0
1395 i = 0
1396 while i < n { o = inc_render_rej(g, rrow[i], out, o, cap, v); i = i + 1 }
1397 i = 0
1398 while i < nx { o = inc_render_rej(g, xrows[i], out, o, cap, v); i = i + 1 }
1399 if v[INC_V_N] == 0 { o = inc_catb(out, o, cap, "none recorded in the matched headers or on any name-tier organ" as *u8) }
1400 if xover > 0 { o = inc_catb(out, o, cap, " (+" as *u8); o = inc_catn(out, o, cap, xover); o = inc_catb(out, o, cap, " more name-tier organs record rejections; raise rej_orgs to list them)" as *u8) }
1401 if o >= cap - 1 { o = inc_cat(out, cap - 6, "[CUT]" as *u8) }
1402 return o
1403}
1404// the coverage envelope line for a loaded index
1405func inc_envelope(g: *i64, out: *u8, cap: i64) -> i64 {
1406 var o: i64 = inc_catb(out, 0, cap, "INCUMBENTS index=" as *u8)
1407 o = inc_catb(out, o, cap, g[INC_G_PATH] as *u8)
1408 o = inc_catb(out, o, cap, " state=" as *u8)
1409 o = inc_catb(out, o, cap, inc_state_name(g[INC_G_STATE]))
1410 o = inc_catb(out, o, cap, " generated_at=" as *u8)
1411 o = inc_catn(out, o, cap, g[INC_G_GEN])
1412 o = inc_catb(out, o, cap, " age_s=" as *u8)
1413 o = inc_catn(out, o, cap, g[INC_G_AGE])
1414 o = inc_catb(out, o, cap, " organs=" as *u8)
1415 o = inc_catn(out, o, cap, g[INC_G_ROWS])
1416 o = inc_catb(out, o, cap, " files=" as *u8)
1417 o = inc_catn(out, o, cap, g[INC_G_FILES])
1418 o = inc_catb(out, o, cap, " head_truncated=" as *u8)
1419 o = inc_catn(out, o, cap, g[INC_G_TRUNC])
1420 o = inc_catb(out, o, cap, " assets=" as *u8)
1421 o = inc_catn(out, o, cap, g[INC_G_ASSETS])
1422 o = inc_catb(out, o, cap, " corpus_complete=" as *u8)
1423 o = inc_catn(out, o, cap, g[INC_G_COMPLETE])
1424 o = inc_catb(out, o, cap, " postings=" as *u8)
1425 o = inc_catn(out, o, cap, g[INC_G_NPOST])
1426 return o
1427}