code wiki / _hdl_build / nx_docportal_search_seg.nx
nx_docportal_search_seg.nx
buildroot/runtime/_hdl_build/nx_docportal_search_seg.nx
about
nx_docportal_search_seg.nx -- SOVEREIGN seg_store-native onsite search.
Operator 2026-07-02: "we dont want to use tsv we want to use nishi ecosystem from the hardware rung up each rung."
This RETIRES the derived flat-file bridge (knowledge/index/<domain>_src.tsv -> nx_onsite_index -> separate .idx)
for onsite search. It queries the domain's PUBLIC seg_store shard's OWN term postings DIRECTLY:
ss_open(shard) -> per query-term ss_term (current-state postings, tombstone-shadowed) -> ss_hget doc text ->
rank by (# distinct query terms matched, then total term frequency).
The store IS the index by construction: ss_write_seg builds .terms for EVERY segment (nx_seg_store:690), so no
derived artifact and no flat file appear anywhere in the path. Cross-shard isolation is inherited (a public
query ss_open's ONLY the -pub- shard; the -prv- shard's files are never opened). license_tier: ORIGINAL
Shard/key format MUST match nx_docportal_lib (dp_prefix/dp_key): prefix "knowledge/store/dp-<domain>-pub-",
key "doc:<cid>". Kept inlined (not imported) so this stays a leaf on nx_seg_store alone -> gate-able in isolation.
dependencies 4 imports · 18 importers
diagram shows first 10 each side; +0 more imports, +8 more importers in the complete lists below.
imports: nx_seg_store.nxnx_intlog.nxnx_editdist.nxnx_stem.nx
imported by: nx_corpus_ingest.nxnx_doc_lexstat.nxnx_docportal_ingest_gate.nxnx_docportal_policy_gate.nxnx_docportal_search_seg_gate.nxnx_docportal_search_serve.nxnx_dp_leak_gate.nxnx_dp_reclassify.nxnx_dp_reclassify_gate.nxnx_dp_tsv_migrate.nxnx_dss_query.nxnx_nist_grader.nxnx_seg_imp_build.nxnx_shard_compact.nxnx_tpl_ingest.nxnx_url_scrub.nxnx_web_search_probe.nxnx_web_shard_compact.nx
structs
| none |
consts
| 20 | const DSC_PREFIXBUF: i64 = 512 |
| 21 | const DSC_TOKBUF: i64 = 64 |
| 22 | const DSC_POSBUF: i64 = 16 |
| 23 | const DSC_ASCII_0: i64 = 48 |
| 24 | const DSC_DEC: i64 = 10 |
| 26 | const DSS_MAGIC_1125899906842597: i64 = 1125899906842597 |
| 27 | const DSS_MAGIC_1024: i64 = 1024 |
| 29 | const DSS_MAXTERMS: i64 = 16 |
| 30 | const DSS_MAXHITS: i64 = 2048 // per-term candidacy cap. FIVE-CELL MEASUREMENT 2026-07-25 -- |
| 40 | const DSS_MAXCAND: i64 = 2048 // unique candidate docs cap (same five-cell measurement) |
| 41 | const DSS_PHRASEHITS: i64 = 512 // phrase candidacy cap -- ss_phrase's internal rank buffers are |
| 43 | const DSS_POL_SEARCH: i64 = 1 // owner-consent search bit -- MUST equal nx_docportal_lib DP_USE_PUB_SEARCH |
| 44 | const DSS_TFSCAN: i64 = 8192 // per-doc tf-scan cap (perf, 2026-07-24 32768->8192): the TWO per-candidate walks (tf + proximity) dominate WEB latency on large HTML->text docs; tf saturates ~5 so 8KB of leading content keeps ranking (|d| now comes from byte-length, cap-independent), and the rare-term floor + postings-truth keep deep matches findable. 4x less scanning = the p95 win until precomputed-tf-in-postings lands. |
| 45 | const DSS_SHORT_FLOOR: i64 = 2048 // seq877: min candidates handed to full BM25 (was a bare 128). |
| 49 | const DSS_S1_TFK: i64 = 2 // seq871 stage-1 tf saturation constant: weight = idf*tf/(tf+K). |
| 54 | const DSS_RARE_DCOUNT: i64 = 4 // a POSTED term this rare (<=4 docs corpus-wide) gets a tf=1 floor when the scan cap hid it: near-unique terms (names, ids, gate markers) stay findable even at a huge doc's tail, while COMMON terms hidden by the cap stay dropped (deep-boilerplate noise -- the julia-kyoka complaint). Postings are full-doc truth; the floor only restores what they assert. |
| 55 | const DSS_HOSTCAP: i64 = 3 // web scope ONLY: max results per host per page (host-crowding cap). Over-cap hits still appear via phase-B overflow -- down-rank, never delete. Site/trusted shards are never capped. |
| 56 | const DSS_PR_SCALE: i64 = 1000000000 // == nx_pagerank PR_SCALE; pr: values are ppb ranks in [0, DSS_PR_SCALE] |
| 57 | const DSS_PR_BOOST: i64 = 3072 // web authority: score *= (1024 + rank*BOOST/SCALE)/1024 -- a top page (~5e8) ~2.5x |
| 58 | const DSS_PR_MAXFAC: i64 = 4096 // cap the authority factor at 4x (no single super-authority page dominates) |
| 59 | const DSS_TRUST_BOOST: i64 = 2048 // R2c TRUST PRIOR (2026-08-04, debt 1785895889): bounded 2x for a host |
| 64 | const DSS_TRUST_MAXH: i64 = 96 // seed hosts held per query (the file is ~60 rows) |
| 72 | const DSS_STOP_DFPCT: i64 = 12 // term in >12% of docs = stopword (Zipf: function words cluster here) |
| 73 | const DSS_STOP_DISCOUNT: i64 = 12 // stopword idf /= this (weak tiebreaker, not deleted) |
| 74 | const DSS_STOP_MINCORPUS: i64 = 1000 // stopword detection only above this doc count (protects tiny corpora) |
| 127 | const DSC_PATHBUF: i64 = 560 |
| 128 | const DSC_STATBUF: i64 = 160 |
| 129 | const DSC_SIGBUF: i64 = 32 |
| 434 | const DSS_PROX_SCALE: i64 = 1024 |
| 435 | const DSS_PROX_WREF: i64 = 8 // reference window in TOKENS; span << WREF -> near-full strength |
| 436 | const DSS_PROX_BOOST: i64 = 1024 // max boost: prox=SCALE -> factor 2048/1024 = 2x (bounded, capped) |
| 437 | const DSS_PROX_OCAP: i64 = 512 // max query-term occurrences tracked (bounded serve cost) |
| 438 | const DSS_TITLE_TOKENS: i64 = 6 // R1c: leading tokens treated as the TITLE / lead field |
| 439 | const DSS_TITLE_BONUS: i64 = 384 // R1c: per-distinct-term title hit (BM25F-lite), summed then capped at SCALE |
| 446 | const DSS_COV_FLOOR: i64 = 256 // coord factor for a doc matching only the FIRST of >=2 distinct query terms |
| 452 | const DSS_SEARCHPAGE_PEN: i64 = 512 // 512/1024 = 0.5x on a detected search/query-echo URL (web scope only) |
| 456 | const DSS_AUTH_IDF_MIN: i64 = 512 // 512/1024 = 50% of the query content idf-mass required to earn the PR boost |
| 484 | const DSS_SLUG_BOOST: i64 = 6144 // Q10 6.0x: must clear the mention-wall (measured: ~140 short torrent stubs |
| 486 | const DSS_SLUG_MINLEN: i64 = 4 // ignore short segments/terms (/en/, /id/) -- too weak to mark an entity |
| 491 | const DSS_SLUG_RAREK: i64 = 512 |
| 500 | const DSS_STUFF_DENS: i64 = 16 |
| 501 | const DSS_STUFF_PEN: i64 = 512 |
| 502 | const DSS_STUFF_MINTOKS: i64 = 64 |
functions
| 77 | func dss_prefix(domain: *u8, out: *u8) -> i64 |
| 92 | func dss_mkkey(cid: i64, out: *u8) -> i64 |
| 118 | func dsc_web_prefix_is(prefix: *u8) -> i64 called by 1: dss_open_maybe_cached |
| 130 | func dsc_manifest_sig(prefix: *u8, sig: *i64) -> i64 |
| 150 | func dss_open_maybe_cached(prefix: *u8) -> *i64 |
| 173 | func dss_web_cache_refresh() -> i64 |
| 184 | func dss_mkpolkey(cid: i64, out: *u8) -> i64 |
| 200 | func dss_mkurlkey(cid: i64, out: *u8) -> i64 |
| 215 | func dss_url_host(u: *u8, ul: i64, out: *u8) -> i64 |
| 241 | func dss_is_web(domain: *u8) -> i64 |
| 248 | func dss_hosthash(u: *u8, ul: i64, scratch: *u8) -> i64 |
| 259 | func dss_urlcid(s: *u8, n: i64) -> i64 called by 1: dss_search_off_div |
| 267 | func dss_prkey(cid: i64, out: *u8) -> i64 called by 1: dss_search_off_div |
| 283 | func dss_url_host_match(u: *u8, ul: i64, host: *u8) -> i64 |
| 321 | func dss_url_path_has(u: *u8, ul: i64, pat: *u8, seganchor: i64) -> i64 |
| 347 | func dss_key_cid(kp: *u8, kl: i64) -> i64 called by 1: dss_search_off_div |
| 363 | func dss_streq(tok: *u8, term: *u8) -> i64 |
| 374 | func dss_tf(doc: *u8, dn: i64, term: *u8, tbl: *u8) -> i64 |
| 392 | func dss_tf_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8, tf_out: *i64) -> i64 |
| 447 | func dss_popcount(m: i64) -> i64 { var c: i64 = 0; var x: i64 = m; while x != 0 { c = c + (x & 1); x = x >> 1 } return c } |
| 457 | func dss_sub_at(u: *u8, n: i64, i: i64, pat: *u8, plen: i64) -> i64 called by 1: dss_is_search_url |
| 463 | func dss_is_search_url(u: *u8, n: i64) -> i64 |
| 503 | func dss_stuff_factor(tf: i64, toks: i64, dcnt: i64, bign: i64) -> i64 called by 1: main |
| 517 | func dss_host_label_match(u: *u8, ul: i64, term: *u8, scratch: *u8) -> i64 |
| 546 | func dss_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } called by 1: dss_slug_seg_eq |
| 549 | func dss_slug_seg_eq(u: *u8, n: i64, i: i64, term: *u8) -> i64 |
| 567 | func dss_slug_match_term(u: *u8, n: i64, term: *u8) -> i64 |
| 579 | func dss_prox_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8) -> i64 |
| 673 | func dss_correct(domain: *u8, q: *u8, qn: i64, out: *u8, outcap: i64) -> i64 |
| 771 | func dss_suggest(domain: *u8, pfx: *u8, pn: i64, out: *u8, maxn: i64) -> i64 called by 3: maindss_api_suggestmain calls 5: dss_prefixdss_open_maybe_cachedss_term_countss_term_atsys_munmap |
| 854 | func dss_tlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } |
| 860 | func dss_stem_expand(h: *i64, termstore: *u8, termptrs: *i64, nterms: i64, orig_nterms: i64) -> i64 |
| 946 | func dss_search_off_div(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64, webdiv: i64) -> i64 called by 4: dss_search_offmainng_rankmain calls 29: dss_prefixdss_open_maybe_cachedss_tok_tabless_tok_next2dss_streqdss_stem_expand+23 |
| 1851 | func dss_search_off(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64) -> i64 |
| 1855 | func dss_search(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64) -> i64 |