code wiki / _hdl_build / nx_docportal_search_seg.nx
nx_docportal_search_seg.nx
buildroot/runtime/_hdl_build/nx_docportal_search_seg.nx
about
nx_docportal_search_seg.nx -- SOVEREIGN seg_store-native onsite search.
Operator 2026-07-02: "we dont want to use tsv we want to use nishi ecosystem from the hardware rung up each rung."
This RETIRES the derived flat-file bridge (knowledge/index/<domain>_src.tsv -> nx_onsite_index -> separate .idx)
for onsite search. It queries the domain's PUBLIC seg_store shard's OWN term postings DIRECTLY:
ss_open(shard) -> per query-term ss_term (current-state postings, tombstone-shadowed) -> ss_hget doc text ->
rank by (# distinct query terms matched, then total term frequency).
The store IS the index by construction: ss_write_seg builds .terms for EVERY segment (nx_seg_store:690), so no
derived artifact and no flat file appear anywhere in the path. Cross-shard isolation is inherited (a public
query ss_open's ONLY the -pub- shard; the -prv- shard's files are never opened). license_tier: ORIGINAL
Shard/key format MUST match nx_docportal_lib (dp_prefix/dp_key): prefix "knowledge/store/dp-<domain>-pub-",
key "doc:<cid>". Kept inlined (not imported) so this stays a leaf on nx_seg_store alone -> gate-able in isolation.
dependencies 6 imports · 22 importers
diagram shows first 10 each side; +0 more imports, +12 more importers in the complete lists below.
imports: nx_seg_store.nxnx_intlog.nxnx_quicksort.nxnx_editdist.nxnx_stem.nxnx_didyoumean.nx
imported by: nx_corpus_ingest.nxnx_didyoumean_gate.nxnx_doc_lexstat.nxnx_docportal_ingest_gate.nxnx_docportal_policy_gate.nxnx_docportal_search_seg_gate.nxnx_docportal_search_serve.nxnx_docportal_search_serve_snip_20260912.nxnx_docportal_search_sticky_gate.nxnx_dp_leak_gate.nxnx_dp_reclassify.nxnx_dp_reclassify_gate.nxnx_dp_tsv_migrate.nxnx_dss_query.nxnx_nist_grader.nxnx_qrels_bench.nxnx_seg_imp_build.nxnx_shard_compact.nxnx_tpl_ingest.nxnx_url_scrub.nxnx_web_search_probe.nxnx_web_shard_compact.nx
structs
| none |
consts
| 19 | const DSC_MAGIC_262144: i64 = 262144 |
| 20 | const DSC_MAGIC_2097152: i64 = 2097152 |
| 21 | const DSC_MAGIC_4096: i64 = 4096 |
| 25 | const DSC_PREFIXBUF: i64 = 512 |
| 26 | const DSC_TOKBUF: i64 = 64 |
| 27 | const DSC_POSBUF: i64 = 16 |
| 28 | const DSC_ASCII_0: i64 = 48 |
| 29 | const DSC_DEC: i64 = 10 |
| 31 | const DSS_MAGIC_1125899906842597: i64 = 1125899906842597 |
| 32 | const DSS_MAGIC_1024: i64 = 1024 |
| 34 | const DSS_MAXTERMS: i64 = 16 |
| 36 | const DSS_BQ_IDXBITS: i64 = 4096 // > DSS_MAXCAND: a candidate index rides in the low bits of one sort key |
| 37 | const DSS_BQ_SMAX: i64 = 1099511627776 // 2^40, above any score this scorer can produce (16 terms x a 25-bit idf x a 12-bit tfnorm x a 2x proximity factor < 2^31), so (SMAX - score) keeps keys positive and the ascending canonical sort orders scores DESCENDING |
| 38 | const DSS_BQ_RRF_SCALE: i64 = 16777216 // 2^24: the reciprocal-rank numerator, so 1/(k+r) stays order-preserving as an integer over every rank the candidate cap allows (adjacent ranks at 2048 still differ by 3) |
| 39 | const DSS_BQ_FMAX: i64 = 16777216 // above any fused sum (two reciprocals of at most SCALE/(k+1) each), so the fused key is positive too |
| 40 | const DSS_MAXHITS: i64 = 2048 // per-term candidacy cap. FIVE-CELL MEASUREMENT 2026-07-25 -- |
| 50 | const DSS_MAXCAND: i64 = 2048 // unique candidate docs cap (same five-cell measurement) |
| 51 | const DSS_PHRASEHITS: i64 = 512 // phrase candidacy cap -- ss_phrase's internal rank buffers are |
| 53 | const DSS_POL_SEARCH: i64 = 1 // owner-consent search bit -- MUST equal nx_docportal_lib DP_USE_PUB_SEARCH |
| 54 | const DSS_TFSCAN: i64 = 8192 // per-doc tf-scan cap (perf, 2026-07-24 32768->8192): the TWO per-candidate walks (tf + proximity) dominate WEB latency on large HTML->text docs; tf saturates ~5 so 8KB of leading content keeps ranking (|d| now comes from byte-length, cap-independent), and the rare-term floor + postings-truth keep deep matches findable. 4x less scanning = the p95 win until precomputed-tf-in-postings lands. |
| 55 | const DSS_SHORT_FLOOR: i64 = 2048 // seq877: min candidates handed to full BM25 (was a bare 128). |
| 59 | const DSS_S1_TFK: i64 = 2 // seq871 stage-1 tf saturation constant: weight = idf*tf/(tf+K). |
| 64 | const DSS_RARE_DCOUNT: i64 = 4 // a POSTED term this rare (<=4 docs corpus-wide) gets a tf=1 floor when the scan cap hid it: near-unique terms (names, ids, gate markers) stay findable even at a huge doc's tail, while COMMON terms hidden by the cap stay dropped (deep-boilerplate noise -- the julia-kyoka complaint). Postings are full-doc truth; the floor only restores what they assert. |
| 65 | const DSS_HOSTCAP: i64 = 3 // web scope ONLY: max results per host per page (host-crowding cap). Over-cap hits still appear via phase-B overflow -- down-rank, never delete. Site/trusted shards are never capped. |
| 79 | const DSS_ENTITYCAP: i64 = 1 |
| 80 | const DSS_PR_SCALE: i64 = 1000000000 // == nx_pagerank PR_SCALE; pr: values are ppb ranks in [0, DSS_PR_SCALE] |
| 81 | const DSS_PR_BOOST: i64 = 3072 // web authority: score *= (1024 + rank*BOOST/SCALE)/1024 -- a top page (~5e8) ~2.5x |
| 82 | const DSS_PR_MAXFAC: i64 = 4096 // cap the authority factor at 4x (no single super-authority page dominates) |
| 83 | const DSS_TRUST_BOOST: i64 = 2048 // R2c TRUST PRIOR (2026-08-04, debt 1785895889): bounded 2x for a host |
| 88 | const DSS_TRUST_MAXH: i64 = 96 // seed hosts held per query (the file is ~60 rows) |
| 96 | const DSS_STOP_DFPCT: i64 = 12 // term in >12% of docs = stopword (Zipf: function words cluster here) |
| 97 | const DSS_STOP_DISCOUNT: i64 = 12 // stopword idf /= this (weak tiebreaker, not deleted) |
| 98 | const DSS_STOP_MINCORPUS: i64 = 1000 // stopword detection only above this doc count (protects tiny corpora) |
| 99 | const DSS_IDF_Q10_MIN: i64 = 1 // one quantum of the Q10 fixed point (1/1024): the SMALLEST positive weight a term the |
| 181 | const DSC_PATHBUF: i64 = 560 |
| 182 | const DSC_STATBUF: i64 = 160 |
| 183 | const DSC_SIGBUF: i64 = 32 |
| 302 | const DSQ_MAXR: i64 = 30 // max results memoised per entry; >= the SERP page size served |
| 303 | const DSQ_SLOTS: i64 = 2048 // capacity BOUND (entries). Fixed table => fixed memory. |
| 304 | const DSQ_ENT_W: i64 = 2 // S11-c: the memoised pin (pinned, cid) so a memo hit announces what it serves |
| 305 | const DSQ_SLOTW: i64 = 6 + 2 * DSQ_MAXR + DSQ_ENT_W // words/slot: key_head,gen,nres,total,phrase + cids + scores + entity + key_tail |
| 306 | const DSQ_HDRW: i64 = 1 // word 0 = current generation |
| 307 | const DSQ_KEYBUF: i64 = 512 |
| 323 | const DCC_SLOTS: i64 = 512 // capacity BOUND (entries). Fixed table => fixed memory. |
| 324 | const DCC_TEXTB: i64 = 64 // bytes of correction text kept per slot |
| 325 | const DCC_TEXTW: i64 = DCC_TEXTB / 8 |
| 326 | const DCC_SLOTW: i64 = 4 + DCC_TEXTW // head_key, gen, len, <text words>, tail_key |
| 361 | const DSQ_UPPER_LO: i64 = 65 // 'A' |
| 362 | const DSQ_UPPER_HI: i64 = 90 // 'Z' |
| 363 | const DSQ_CASE_DELTA: i64 = 32 // 'a' - 'A' |
| 457 | const DSS_FE_KEY_BYTES: i64 = 32 // "fe:" + 20 decimal digits + NUL |
| 458 | const DSS_FRESH_SLOTS: i64 = 3 // out[0] observed, out[1] newest epoch, out[2] median age (seconds) |
| 459 | const DSS_FRESH_MAX: i64 = 64 // ages scratch: a result PAGE, tens of rows; callers cap n here and say so |
| 855 | const DSS_PROX_SCALE: i64 = 1024 |
| 856 | const DSS_PROX_WREF: i64 = 8 // reference window in TOKENS; span << WREF -> near-full strength |
| 857 | const DSS_PROX_BOOST: i64 = 1024 // max boost: prox=SCALE -> factor 2048/1024 = 2x (bounded, capped) |
| 858 | const DSS_PROX_OCAP: i64 = 512 // max query-term occurrences tracked (bounded serve cost) |
| 859 | const DSS_TITLE_TOKENS: i64 = 6 // R1c: leading tokens treated as the TITLE / lead field |
| 860 | const DSS_TITLE_BONUS: i64 = 384 // R1c: per-distinct-term title hit (BM25F-lite), summed then capped at SCALE |
| 867 | const DSS_COV_FLOOR: i64 = 256 // coord factor for a doc matching only the FIRST of >=2 distinct query terms |
| 883 | const DSS_SEARCHPAGE_PEN: i64 = 512 // 512/1024 = 0.5x on a detected search/query-echo URL (web scope only) |
| 884 | const DSS_S9_PAGE_KINDS: i64 = 2 // S9: page kinds per coverage level in the precedence key -- content, results page |
| 885 | const DSS_I64_BYTES: i64 = 8 // bytes per i64 slot (the composite key array) |
| 886 | const DSS_S9_ENTITY_TERMS: i64 = 2 // S9b: a name lookup is one or two typed words (the slug prior's shape gate) |
| 890 | const DSS_AUTH_IDF_MIN: i64 = 512 // 512/1024 = 50% of the query content idf-mass required to earn the PR boost |
| 918 | const DSS_SLUG_BOOST: i64 = 6144 // Q10 6.0x: must clear the mention-wall (measured: ~140 short torrent stubs |
| 920 | const DSS_SLUG_MINLEN: i64 = 4 // ignore short segments/terms (/en/, /id/) -- too weak to mark an entity |
| 925 | const DSS_SLUG_RAREK: i64 = 512 |
| 934 | const DSS_STUFF_DENS: i64 = 16 |
| 935 | const DSS_STUFF_PEN: i64 = 512 |
| 936 | const DSS_STUFF_MINTOKS: i64 = 64 |
| 991 | const DSS_ENT_PATH_DEFAULT: *u8 = "knowledge/status/entitypin.tsv" |
| 992 | const DSS_ENT_MAXROWS: i64 = 4096 |
| 993 | const DSS_ENT_KEYCAP: i64 = 128 |
| 994 | const DSS_ENT_STAT_BYTES: i64 = 256 |
| 995 | const DSS_ENT_STAT_SIZE_OFF: i64 = 48 // st_size in the x86-64 stat buffer |
| 996 | const DSS_ENT_STAT_MTIME_OFF: i64 = 88 // st_mtim.tv_sec |
| 997 | const DSS_ENT_READ_CHUNK: i64 = 65536 |
| 998 | const DSS_ENT_TAB: i64 = 9 |
| 999 | const DSS_ENT_NL: i64 = 10 |
| 1000 | const DSS_ENT_SPACE: i64 = 32 |
| 1001 | const DSS_ENT_DIGIT0: i64 = 48 |
| 1002 | const DSS_ENT_DIGIT9: i64 = 57 |
| 1003 | const DSS_ENT_TEN: i64 = 10 |
| 1004 | const DSS_ENT_BOX_SLOTS: i64 = 4 |
| 1184 | const DSS_ENT_FIELD_JP: i64 = 0 |
| 1185 | const DSS_ENT_FIELD_URL: i64 = 1 |
| 1306 | const DSS_HEADTOK: i64 = 1280 |
| 1307 | const DSS_L0_OCC: i64 = 512 // per-candidate occurrence cap == DSS_PROX_OCAP (the same window the walk kept) |
| 1308 | const DSS_L0_TSHIFT: i64 = 32 // packed occurrence key = position * DSS_L0_TSHIFT + term (DSS_MAXTERMS < 32) |
| 1447 | const DSS_PH_SLOTS: i64 = 16 |
| 2106 | const DSS_SPELL_MARK: i64 = 0 - 2 |
functions
| 113 | func dss_last_estimated() -> i64 { return dss_est_g } |
| 116 | func dss_prefix(domain: *u8, out: *u8) -> i64 |
| 131 | func dss_mkkey(cid: i64, out: *u8) -> i64 |
| 172 | func dsc_web_prefix_is(prefix: *u8) -> i64 |
| 184 | func dsc_manifest_sig(prefix: *u8, sig: *i64) -> i64 |
| 204 | func dss_open_maybe_cached(prefix: *u8) -> *i64 |
| 255 | func dss_web_cache_refresh() -> i64 |
| 274 | func dss_web_index_segments() -> i64 |
| 280 | func dss_web_cache_sig(sig: *i64) -> i64 called by 1: main |
| 330 | func dsq_init() -> i64 |
| 343 | func dsq_bump() -> i64 called by 1: dss_refresh_and_invalidate |
| 348 | func dsq_putn(out: *u8, off: i64, v: i64) -> i64 called by 1: dsq_key |
| 383 | func dsq_key(domain: *u8, q: *u8, qn: i64, max: i64, offset: i64, div: i64) -> i64 |
| 411 | func dss_refresh_and_invalidate() -> i64 |
| 423 | func dss_mkpolkey(cid: i64, out: *u8) -> i64 |
| 439 | func dss_mkurlkey(cid: i64, out: *u8) -> i64 |
| 461 | func dss_mkfekey(cid: i64, out: *u8) -> i64 called by 1: dss_doc_fetch_epoch |
| 478 | func dss_doc_fetch_epoch(h: *i64, cid: i64) -> i64 |
| 495 | func dss_fresh_stats(fe: *i64, n0: i64, now: i64, out: *i64) -> i64 |
| 522 | func dss_url_host(u: *u8, ul: i64, out: *u8) -> i64 |
| 548 | func dss_is_web(domain: *u8) -> i64 |
| 555 | func dss_hosthash(u: *u8, ul: i64, scratch: *u8) -> i64 |
| 579 | func dss_entityhash(u: *u8, ul: i64, scratch: *u8) -> i64 |
| 630 | func dss_urlcid(s: *u8, n: i64) -> i64 called by 1: dss_search_off_div |
| 638 | func dss_prkey(cid: i64, out: *u8) -> i64 called by 1: dss_search_off_div |
| 654 | func dss_url_host_match(u: *u8, ul: i64, host: *u8) -> i64 |
| 692 | func dss_url_path_has(u: *u8, ul: i64, pat: *u8, seganchor: i64) -> i64 |
| 718 | func dss_key_cid(kp: *u8, kl: i64) -> i64 called by 1: dss_search_off_div |
| 734 | func dss_streq(tok: *u8, term: *u8) -> i64 |
| 745 | func dss_tf(doc: *u8, dn: i64, term: *u8, tbl: *u8) -> i64 |
| 784 | func dss_set_expand_off(v: i64) -> i64 { dss_expand_off = v; return dss_expand_off } |
| 791 | func dss_set_scope_prefix(p: *u8, n: i64) -> i64 { dss_scope_pfx = p; dss_scope_pfx_n = n; return n } |
| 793 | func dss_scope_stats() -> i64 { return dss_scope_us } |
| 796 | func dss_scope_filter(u: *u8, ul: i64, pfx: *u8, pn: i64) -> i64 called by 1: dss_search_off_div |
| 810 | func dss_tf_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8, tf_out: *i64) -> i64 |
| 878 | func dss_popcount(m: i64) -> i64 { var c: i64 = 0; var x: i64 = m; while x != 0 { c = c + (x & 1); x = x >> 1 } return c } called by 1: dss_search_off_div |
| 891 | func dss_sub_at(u: *u8, n: i64, i: i64, pat: *u8, plen: i64) -> i64 called by 1: dss_is_search_url |
| 897 | func dss_is_search_url(u: *u8, n: i64) -> i64 |
| 937 | func dss_stuff_factor(tf: i64, toks: i64, dcnt: i64, bign: i64) -> i64 called by 1: main |
| 951 | func dss_host_label_match(u: *u8, ul: i64, term: *u8, scratch: *u8) -> i64 |
| 980 | func dss_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } |
| 1021 | func dss_ent_init() -> i64 |
| 1042 | func dss_ent_set_path(p: *u8) -> i64 { dss_ent_init(); dss_ent_path = p; dss_ent_size = 0 - 1; dss_ent_mtime = 0 - 1; return 0 } |
| 1043 | func dss_ent_set(pinned: i64, cid: i64) -> i64 { dss_ent_init(); dss_ent_box[0] = pinned; dss_ent_box[1] = cid; return 0 } |
| 1046 | func dss_entity_stats(out: *i64) -> i64 { dss_ent_init(); out[0] = dss_ent_box[0]; out[1] = dss_ent_box[1]; out[2] = dss_ent_n; return 3 } |
| 1048 | func dss_ent_reload() -> i64 |
| 1134 | func dss_ent_lookup(termptrs: *i64, onterms: i64) -> i64 |
| 1168 | func dss_ent_hit_by_cid(cid: i64) -> i64 |
| 1176 | func dss_ent_memo_save(w: *i64) -> i64 { dss_ent_init(); w[0] = dss_ent_box[0]; w[1] = dss_ent_box[1]; return 0 } |
| 1177 | func dss_ent_memo_load(w: *i64) -> i64 |
| 1186 | func dss_entity_field(which: i64, dst: *u8, cap: i64) -> i64 |
| 1201 | func dss_entity_titles() -> i64 |
| 1209 | func dss_slug_seg_eq(u: *u8, n: i64, i: i64, term: *u8) -> i64 |
| 1227 | func dss_slug_match_term(u: *u8, n: i64, term: *u8) -> i64 |
| 1332 | func dss_bq_set(repeats: i64, moved: i64) -> i64 |
| 1338 | func dss_bm25q_stats(out: *i64) -> i64 |
| 1346 | func dss_qtf_map(termptrs: *i64, nterms: i64, qfirst: *i64, qcnt: *i64) -> i64 |
| 1373 | func dss_bm25q_fuse(plain: *i64, q: *i64, n: i64) -> i64 |
| 1433 | func dss_l0_stats(out: *i64) -> i64 called by 1: dss_api_search |
| 1455 | func dss_adj_typed(opos: *i64, oidx: *i64, nocc: i64, ntyped: i64, endtok: i64) -> i64 called by 1: dss_prox_score |
| 1497 | func dss_ph_init() -> i64 |
| 1503 | func dss_ph_fin(tsel: i64, t0: i64) -> i64 called by 1: dss_search_off_div |
| 1510 | func dss_phase_stats(out: *i64) -> i64 called by 1: dss_api_search |
| 1519 | func dss_prox_score(opos: *i64, oidx: *i64, nocc: i64, present: i64, tseen: *i64, nterms: i64, endtok: i64) -> i64 |
| 1567 | func dss_l0_locate(h: *i64, key: *u8, outs: *i64, vo: *i64, vl: *i64) -> i64 |
| 1607 | func dss_tf_postings(prefix: *u8, h: *i64, cand_cid: *i64, ncand: i64, termptrs: *i64, nterms: i64, tfmat: *i64, cand_prox: *i64, cand_capped: *i64, cand_dl: *i64, cand_l0: *i64, cand_phrase: *i64) -> i64 called by 1: dss_search_off_div calls 9: dss_mkkeydss_l0_locatess_manifest_dynss_catss_loadfiless_r32+3 |
| 1844 | func dss_prox_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8) -> i64 |
| 1911 | func dss_correct(domain: *u8, q: *u8, qn: i64, out: *u8, outcap: i64) -> i64 called by 6: maindsv_serp_restdss_api_searchdss_servedss_api_searchmain calls 2: dss_correct_rawdsq_key |
| 1952 | func dss_correct_raw(domain: *u8, q: *u8, qn: i64, out: *u8, outcap: i64) -> i64 called by 1: dss_correct calls 10: dss_prefixdss_open_maybe_cachedss_tok_tabless_tok_next2ss_term_dcountss_term_count+4 |
| 2092 | func dss_spell_stats(out: *i64) -> i64 |
| 2107 | func dss_spell_suggest(domain: *u8, q: *u8, qn: i64, out: *u8, outcap: i64) -> i64 |
| 2166 | func dss_suggest(domain: *u8, pfx: *u8, pn: i64, out: *u8, maxn: i64) -> i64 called by 4: maindss_api_suggestdss_api_suggestmain calls 4: dss_prefixdss_open_maybe_cachedss_term_countss_term_at |
| 2249 | func dss_tlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } |
| 2267 | func dss_stem_expand(h: *i64, termstore: *u8, termptrs: *i64, nterms: i64, orig_nterms: i64, owner: *i64) -> i64 |
| 2435 | func dss_owner_map(owner: *i64, n: i64) -> i64 called by 1: dss_search_off_div |
| 2449 | func dss_posted_floor(tf: i64, capped: i64, posted: i64, dcount: i64) -> i64 called by 1: dss_search_off_div |
| 2465 | func dss_exact_precedence(exa: i64, sca: i64, exb: i64, scb: i64) -> i64 called by 1: dss_search_off_div |
| 2471 | func dss_search_off_div(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64, webdiv: i64) -> i64 called by 4: dss_search_offmainng_rankmain calls 44: dss_ph_initdss_bq_setdss_ent_setdss_prefixdss_open_maybe_cachedss_tok_table+38 |
| 3725 | func dss_search_off(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64) -> i64 |
| 3773 | func dss_search(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64) -> i64 |