code wiki / (root) / nx_search_inverted.nx

nx_search_inverted.nx

buildroot/runtime/nx_search_inverted.nx

26152 B691 linesdepth 3pulls 3 transitivereach 201 importersview sourcekind librarytopic search

docs dependencies structs consts functions

about

nx_search_inverted.nx -- token-postings inverted index. module: nishi-core.search.inverted depends: nishi-core.io.syscalls disk_kb: 6 capability: CORE_IO wired_status: FULLY_WIRED license_tier: PUBLIC_NISHI_SUBSTRATE genealogy_id: salton_1971_inverted_index + fnv_1a_64bit_hash + nishi_build_the_system_cardinal_2026 Search-engine Brick #2. Replaces O(N*M) linear scan with O(M_query * log(V) + sum(postings_per_qtoken)) lookup. At 2.6M rows + ~50K vocab: sub-millisecond query latency. In-memory index built from JSONL once; held in process memory. File-backed persistence queued (nx_search_inverted_persist). Tokenization: split on whitespace + ASCII punctuation, lowercase ASCII letters, keep alphanumerics. Minimum token length = 2. Maximum token length = 64. Hash: FNV-1a 64-bit (per nx_canary_value substrate convention).

dependencies 1 imports · 37 importers

diagram shows first 10 each side; +0 more imports, +27 more importers in the complete lists below.

imports: syscalls.nx

imported by: nx_artifact_store.nxnx_corpus_measure.nxnx_corpus_search.nxnx_crawl_doc.nxnx_diora_index_demo.nxnx_diora_live_reach.nxnx_diora_live_reach_guarded.nxnx_diora_live_serp.nxnx_dms_search.nxnx_docstore_test.nxnx_entity_discover.nxnx_html_to_nishi_page.nxnx_index_migrate_gate.nxnx_inv_bounded_gate.nxnx_live_crawl.nxnx_live_https.nxnx_nishi_page_emit.nxnx_pipeline_walker.nxnx_research_digest.nxnx_search_handler_flow.nxnx_search_inverted_persist.nxnx_search_inverted_test.nxnx_search_neutrality_gate.nxnx_search_onsite_engine.nxnx_search_query_parser.nxnx_search_server.nxnx_search_snippet_extract.nxnx_simhash.nxnx_warc_index.nxnx_wiki_archive_router.nxnx_wiki_https_daemon.nxnx_wiki_https_daemon_mv.nxnx_wiki_index_builder.nxnx_wiki_main.nxnx_wiki_routes.nxnx_wiki_search_wiring.nxnx_wiki_vhost_table.nx

structs

161	struct NxInvIndex
623	struct NxInvQueryResult

consts

28	const NX_MAGIC_4294967296: i64 = 4294967296
29	const NX_MAGIC_4194304: i64 = 4194304
33	const NX_INV_OK: i64 = 1
34	const NX_INV_FILE_NOT_FOUND: i64 = 2
35	const NX_INV_BAD_ARGS: i64 = 3
36	const NX_INV_NO_MATCHES: i64 = 4
37	const NX_INV_OUT_OF_VOCAB: i64 = 5
38	const NX_INV_BUILD_FAIL: i64 = 6
52	const NX_INV_FNV1A_OFFSET_BASIS: i64 = -3750763034362895579
53	const NX_INV_FNV1A_PRIME: i64 = 1099511628211
102	const NX_INV_VOCAB_CAPACITY: i64 = 131072 // 128 K slots (~50% load at 50K vocab)
103	const NX_INV_MAX_POSTINGS_PER: i64 = 16384 // cap per token (most tokens have <100)
104	const NX_INV_MAX_TOTAL_ROWS: i64 = 8388608 // 8 M rows ceiling
105	const NX_INV_MIN_TOKEN_LEN: i64 = 2
106	const NX_INV_MAX_TOKEN_LEN: i64 = 64
159	const NX_INV_SLOT_BYTES: i64 = 32
171	const NX_INV_INDEX_BYTES: i64 = 56 // 7 fields * 8 bytes
632	const NX_INV_QUERY_RESULT_BYTES: i64 = 48

functions

40	func nx_inv_verdict_name(v: i64) -> *u8
55	func nx_inv_hash_bytes_lower(bytes: *u8, n: i64) -> i64 called by 32: nx_artifact_store_add nx_artifact_store_find_by_path cm_has cs_has cs_ndocs nx_crawl_search+26
83	func nx_inv_is_token_char(c: i64) -> i64 called by 12: nx_dms_search_query gs_bm_doclen gs_bm_tf gs_bm25_emit oi_scrub_numeric nx_bm25_tf+6
131	func nx_bm25_tf(text: *u8, n: i64, term_hash: i64) -> i64 called by 13: nx_crawl_search nx_crawl_search main main main main+7 calls 2: nx_inv_is_token_char nx_inv_hash_bytes_lower
175	func nx_inv_new(expected_rows: i64) -> *NxInvIndex called by 19: main main main main main main+13
202	func nx_inv_new_sized(cap: i64) -> *NxInvIndex called by 4: main main nx_inv_build_from_jsonl main
221	func nx_inv_new_bounded(cap: i64, post_cap: i64) -> *NxInvIndex called by 2: main main
241	func nx_inv_slot_at(idx: NxInvIndex, slot_i: i64) -> u8 called by 4: nx_inv_count_token nx_inv_finalize_offsets nx_inv_emit_posting nx_inv_lookup_slot
245	func nx_inv_slot_hash(slot: *u8) -> i64 called by 4: nx_inv_count_token nx_inv_finalize_offsets nx_inv_emit_posting nx_inv_lookup_slot
250	func nx_inv_slot_set_hash(slot: *u8, h: i64) called by 1: nx_inv_count_token
255	func nx_inv_slot_postings_offset(slot: *u8) -> i64 called by 7: cm_has cs_has mig_query bg_has nx_inv_emit_posting nx_inv_query_term+1
260	func nx_inv_slot_set_postings_offset(slot: *u8, off: i64) called by 1: nx_inv_finalize_offsets
265	func nx_inv_slot_postings_count(slot: *u8) -> i64 called by 6: mig_query nx_inv_count_token nx_inv_finalize_offsets nx_inv_emit_posting nx_inv_query_term wig_query
270	func nx_inv_slot_set_postings_count(slot: *u8, c: i64) called by 1: nx_inv_count_token
275	func nx_inv_slot_write_cursor(slot: *u8) -> i64 called by 6: cm_has cs_has cs_ndocs bg_has nx_inv_emit_posting nx_inv_query_term
280	func nx_inv_slot_set_write_cursor(slot: *u8, c: i64) called by 2: nx_inv_finalize_offsets nx_inv_emit_posting
286	func nx_inv_count_token(idx: *NxInvIndex, token_hash: i64) -> i64 called by 1: nx_inv_add_posting calls 5: nx_inv_slot_at nx_inv_slot_hash nx_inv_slot_set_hash nx_inv_slot_set_postings_count nx_inv_slot_postings_count
322	func nx_inv_finalize_offsets(idx: *NxInvIndex) -> i64 called by 24: main main main main main main+18 calls 5: nx_inv_slot_at nx_inv_slot_hash nx_inv_slot_postings_count nx_inv_slot_set_postings_offset nx_inv_slot_set_write_cursor
348	func nx_inv_emit_posting(idx: *NxInvIndex, token_hash: i64, row_id: i64) -> i64 called by 1: nx_inv_emit_row calls 6: nx_inv_slot_at nx_inv_slot_hash nx_inv_slot_postings_count nx_inv_slot_postings_offset nx_inv_slot_write_cursor nx_inv_slot_set_write_cursor
392	func nx_inv_add_posting(idx: *NxInvIndex, token_hash: i64, row_id: i64) -> i64 called by 2: nx_inv_index_row main calls 1: nx_inv_count_token
400	func nx_inv_lookup_slot(idx: NxInvIndex, token_hash: i64) -> u8 called by 7: cm_has cs_has cs_ndocs mig_query bg_has nx_inv_query_term+1 calls 2: nx_inv_slot_at nx_inv_slot_hash
424	func nx_inv_index_row( called by 25: main main main main main main+19 calls 3: nx_inv_is_token_char nx_inv_hash_bytes_lower nx_inv_add_posting
474	func nx_inv_emit_row( called by 23: main main main main main main+17 calls 3: nx_inv_is_token_char nx_inv_hash_bytes_lower nx_inv_emit_posting
523	func nx_inv_build_from_jsonl(jsonl_path: u8) -> NxInvIndex called by 6: main main main main main main_disabled calls 5: sys_read_file nx_inv_new_sized nx_inv_index_row nx_inv_finalize_offsets nx_inv_emit_row
634	func nx_inv_query_term( called by 22: main main main nx_idx_term_count s_qcount gs_bm25_emit+16 calls 5: nx_inv_hash_bytes_lower nx_inv_lookup_slot nx_inv_slot_postings_count nx_inv_slot_postings_offset nx_inv_slot_write_cursor