nx_diora_index_demo.nx source
↩ module page · 82 lines · 3466 B
1// nx_diora_index_demo.nx -- proof that retrieval runs on Nishi's OWN
2// sovereign inverted index, not a borrowed one.
3//
4// module: nishi-core.search.bench.diora_index_demo
5// depends: fx.nx, nx_search_inverted.nx (FNV-1a postings index), nx_diora_data.nx
6// capability: APP_RUNNABLE
7//
8// Builds the verified Diora corpus into nx_search_inverted (the sovereign
9// FNV-1a token-postings index, Salton-1971 lineage, zero external libs) via
10// the two-pass build, then RETRIEVES by term. The postings counts prove the
11// index ingested our corpus and answers term queries -- the index, tokenizer,
12// hashing and postings are all ours, bits-up. (Open scope: the corpus TEXT is
13// research-grounded, not yet filled by our own crawler over the bits-up
14// DNS/TLS/HTTP stack -- that crawl is the next milestone.)
15
16import "fx.nx"
17import "nx_str.nx"
18import "nx_search_inverted.nx"
19import "nx_diora_data.nx"
20
21func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
22func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
23func nx_put_i64(n: i64) -> i64 {
24 if n == 0 { nx_putc(0x30); return 0 }
25 var v: i64 = n
26 if v < 0 { nx_putc(0x2D); v = 0 - v }
27 let buf: *u8 = sys_mmap(32)
28 var k: i64 = 0
29 while v > 0 { buf[k] = 0x30 + (v - (v / 10) * 10); v = v / 10; k = k + 1 }
30 while k > 0 { k = k - 1; sys_write(1, (((buf as i64) + k) as *u8), 1) }
31 return 0
32}
33
34func nx_idx_term_count(idx: *NxInvIndex, term: *u8) -> i64 {
35 let rowids: *i64 = sys_mmap(128 * 8) as *i64
36 let rr: *NxInvQueryResult = sys_mmap(NX_INV_QUERY_RESULT_BYTES) as *NxInvQueryResult
37 nx_inv_query_term(idx, term, nx_str_len(term), rowids, 128, rr)
38 return rr.postings_count
39}
40
41func nx_idx_report(idx: *NxInvIndex, term: *u8) -> i64 {
42 nx_puts(" ");
43 nx_puts(term)
44 var n: i64 = nx_str_len(term)
45 while n < 12 { nx_putc(0x20); n = n + 1 }
46 nx_puts("-> "); nx_put_i64(nx_idx_term_count(idx, term)); nx_puts(" docs\n")
47 return 0
48}
49
50func main() -> i64 {
51 let qn: i64 = nx_diora_qn()
52 let qurl: **u8 = sys_mmap(qn * 8) as **u8
53 let qtit: **u8 = sys_mmap(qn * 8) as **u8
54 let qfac: *i64 = sys_mmap(qn * 8) as *i64
55 let qgrd: *i64 = sys_mmap(qn * 8) as *i64
56 let qpri: *i64 = sys_mmap(qn * 8) as *i64
57 nx_diora_fill_qrels(qurl, qtit, qfac, qgrd, qpri)
58
59 // --- build OUR index: two-pass (count, finalize, emit) over doc titles ---
60 let idx: *NxInvIndex = nx_inv_new(qn + 16)
61 var i: i64 = 0
62 while i < qn { nx_inv_index_row(idx, qtit[i], nx_str_len(qtit[i]), i); i = i + 1 }
63 nx_inv_finalize_offsets(idx)
64 i = 0
65 while i < qn { nx_inv_emit_row(idx, qtit[i], nx_str_len(qtit[i]), i); i = i + 1 }
66
67 nx_puts("=== Nishi SOVEREIGN inverted index (FNV-1a, Salton-1971 lineage, zero ext libs) ===\n")
68 nx_puts("indexed "); nx_put_i64(qn); nx_puts(" verified Diora docs into OUR OWN index\n")
69 nx_puts("vocab slots occupied: "); nx_put_i64(idx.vocab_occupied); nx_putc(0x0A)
70 nx_puts("retrieval by term (sovereign postings lookup):\n")
71 nx_idx_report(idx, "diora")
72 nx_idx_report(idx, "baird")
73 nx_idx_report(idx, "wikipedia")
74 nx_idx_report(idx, "imdb")
75 nx_idx_report(idx, "getty")
76 nx_idx_report(idx, "interview")
77 nx_idx_report(idx, "instagram")
78 nx_idx_report(idx, "wedding")
79 nx_idx_report(idx, "worth")
80 nx_puts("=> retrieval is OURS bits-up; next: own crawler fills the corpus over our DNS/TLS/HTTP.\n")
81 return 0
82}