_wiki_search_gate.nx source

1// _wiki_search_gate.nx -- THE RANKED-SEARCH GATE (wiki R3). 2// 3// Re-proves, from a REAL run (no fabricated GREEN), that the wiki's search is 4// (1) genuinely BM25-RANKED -- ordered by term density, not mere presence -- and 5// (2) UNIFIED with the no-link-rot archive -- content that lives only in the 6// content-addressed archive (never in the live doc store) is still found. 7// 8// Rows (judged by the WIKISEARCH marker; FAIL any -> RED, never silenced): 9// rank_ok docA mentions "sovereign" 5x, docB 1x (both same length-class). 10// Real BM25 must rank docA STRICTLY ABOVE docB -- assert on the 11// ranked rowid ORDER (ground truth), not a bare rc==const. A 12// presence/AND ranker would tie them; BM25 separates them by tf. 13// archive_ok A page whose body contains the UNIQUE term "zarquontoken" is 14// archived into the wikiarchive store but is NOT in the live doc 15// store. Searching "zarquontoken" must FIND it via the seg_store 16// term index folded in by nx_wbs_gather_archive (kind=ARCHIVE). 17// neg_ok Bogus "zzqnosuchterm" -> 0 hits across doc store AND archive. 18// 19// Determinism: a FRESH archive prefix keyed by the wall-clock epoch (empty store 20// each run => idempotent re-run, Rule 10). The PRODUCTION archive prefix is 21// knowledge/store/wikiarchive- (nx_wiki_archive). nx_sites_daemon UNTOUCHED. 22// 23// Verdict line (stdout + knowledge/status/wiki_search_gate.log): 24// WIKISEARCH results=<n> rank_ok=<0|1> archive_ok=<0|1> neg_ok=<0|1> verdict=GREEN|RED 25// 26// Pure NishiLang, NO SQL, NO .sh/.py/.js, no new .tsv/.conf. license_tier: ORIGINAL 27import "nx_syscalls.nx" 28import "nx_seg_store.nx" 29import "wiki/nx_wiki_archive.nx" 30import "wiki/nx_wiki_index_builder.nx" 31import "wiki/nx_wiki_bm25_search.nx" 32 33const WSG_LOG: *u8 = "knowledge/status/wiki_search_gate.log" 34 35// ---- io helpers ---- 36func wsg_w(fd: i64, s: *u8) -> i64 { 37 var n: i64 = 0 38 while s[n] != (0 as u8) { n = n + 1 } 39 sys_write(fd, s, n) 40 return 0 41} 42func wsg_n(fd: i64, v: i64) -> i64 { 43 let bb: *u8 = sys_mmap(28) 44 var m: i64 = v 45 if m < 0 { m = 0 - m; sys_write(fd, "-" as *u8, 1) } 46 let t: *u8 = sys_mmap(28) 47 var k: i64 = 0 48 if m == 0 { t[0] = 48 as u8; k = 1 } 49 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 50 var i: i64 = 0 51 while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 52 sys_write(fd, bb, k) 53 return 0 54} 55// write to BOTH stdout (gate evidence) and the durable log 56func wsg_w2(lfd: i64, s: *u8) -> i64 { wsg_w(1, s); if lfd >= 0 { wsg_w(lfd, s) } return 0 } 57func wsg_n2(lfd: i64, v: i64) -> i64 { wsg_n(1, v); if lfd >= 0 { wsg_n(lfd, v) } return 0 } 58func wsg_p(s: *u8) -> i64 { wsg_w(1, s); return 0 } 59func wsg_pn(v: i64) -> i64 { wsg_n(1, v); return 0 } 60func wsg_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 61 62// concat NUL-terminated s into dst at off; returns new off 63func wsg_cat(dst: *u8, off: i64, s: *u8) -> i64 { 64 var i: i64 = 0 65 while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 } 66 return off + i 67} 68func wsg_catn(dst: *u8, off: i64, v: i64) -> i64 { 69 var m: i64 = v 70 var o: i64 = off 71 if m < 0 { dst[o] = 45 as u8; o = o + 1; m = 0 - m } 72 let t: *u8 = sys_mmap(28) 73 var k: i64 = 0 74 if m == 0 { t[0] = 48 as u8; k = 1 } 75 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 76 var i: i64 = 0 77 while i < k { dst[o + i] = t[k - 1 - i]; i = i + 1 } 78 return o + k 79} 80 81func main() -> i64 { 82 wsg_p("WIKISEARCH-GATE: start (BM25 ranked search + no-rot archive unification)\n" as *u8) 83 84 // ---- FRESH per-run archive prefix: knowledge/store/wikiarchive-search-<epoch>- ---- 85 let prefix: *u8 = sys_mmap(256) 86 var po: i64 = 0 87 po = wsg_cat(prefix, po, "knowledge/store/wikiarchive-search-" as *u8) 88 po = wsg_catn(prefix, po, sys_now_realtime_sec()) 89 po = wsg_cat(prefix, po, "-" as *u8) 90 prefix[po] = 0 as u8 91 wsg_p(" archive prefix = " as *u8); wsg_p(prefix); wsg_p("\n" as *u8) 92 93 // ===== BUILD a LIVE doc store with DISCRIMINATING term frequencies ===== 94 // docA: "sovereign" x5 (substantive) docB: "sovereign" x1 (thin mention), 95 // same length-class so the SEPARATION is driven by tf, not by length. Four 96 // filler docs (docC..docF) do NOT contain "sovereign", so across the 6-doc 97 // corpus "sovereign" is a genuinely DISCRIMINATING term (df=2 of N=6 => the 98 // BM25 IDF stays positive, the regime where tf-density ranking is meaningful 99 // -- not a term so ubiquitous that IDF goes negative). Real BM25, honest 100 // corpus: docA must outrank docB on term density. 101 let store: *NxWikiDocStore = sys_mmap(256) as *NxWikiDocStore 102 let rc_si: i64 = nx_wiki_doc_store_init(store, 16, 65536, 65536, 1048576) 103 if rc_si != 0 { wsg_p(" FATAL doc_store_init rc="); wsg_pn(rc_si); wsg_p("\n" as *u8) } 104 105 let bodyA: *u8 = "sovereign sovereign sovereign sovereign sovereign substrate runtime engine kernel" as *u8 106 let bodyB: *u8 = "sovereign wikipedia aggregator portal directory listing index summary overview page" as *u8 107 let bodyC: *u8 = "weather climate rainfall temperature humidity forecast barometer seasonal pattern chart" as *u8 108 let bodyD: *u8 = "recipe cooking baking flour sugar butter oven timer kitchen utensil mixing whisk bowl" as *u8 109 let bodyE: *u8 = "garden planting soil seed water sunlight harvest compost mulch trowel spade fence trellis" as *u8 110 let bodyF: *u8 = "music melody rhythm harmony tempo chord scale octave instrument string drum flute brass" as *u8 111 let rA: i64 = nx_wiki_doc_store_add(store, "Sovereign Substrate" as *u8, 20, "/wiki/sovereign-substrate" as *u8, 25, bodyA, wsg_slen(bodyA)) 112 let rB: i64 = nx_wiki_doc_store_add(store, "Aggregator Portal" as *u8, 17, "/wiki/aggregator-portal" as *u8, 23, bodyB, wsg_slen(bodyB)) 113 let rC: i64 = nx_wiki_doc_store_add(store, "Weather" as *u8, 7, "/wiki/weather" as *u8, 13, bodyC, wsg_slen(bodyC)) 114 let rD: i64 = nx_wiki_doc_store_add(store, "Recipe" as *u8, 6, "/wiki/recipe" as *u8, 12, bodyD, wsg_slen(bodyD)) 115 let rE: i64 = nx_wiki_doc_store_add(store, "Garden" as *u8, 6, "/wiki/garden" as *u8, 12, bodyE, wsg_slen(bodyE)) 116 let rF: i64 = nx_wiki_doc_store_add(store, "Music" as *u8, 5, "/wiki/music" as *u8, 11, bodyF, wsg_slen(bodyF)) 117 wsg_p(" doc store: docA(rowid="); wsg_pn(rA); wsg_p(",sovereign x5) docB(rowid="); wsg_pn(rB); wsg_p(",sovereign x1) + 4 filler (no 'sovereign'), N="); wsg_pn(nx_wiki_doc_store_count(store)); wsg_p("\n" as *u8) 118 119 // ===== ARCHIVE a page whose body has a UNIQUE term, NOT in the doc store ===== 120 // "zarquontoken" appears ONLY in this archived blob. If search finds it, the 121 // archive term index (ss_term) was genuinely folded into ranking. 122 let s_uniq: *u8 = "zarquon-archive-page" as *u8 123 let bodyZ: *u8 = "# Zarquon Archive\nThis archived page contains the unique token zarquontoken nowhere in the live doc store.\n" as *u8 124 let w: *i64 = ss_begin() 125 let cidZ: *u8 = sys_mmap(80) 126 let raz: i64 = war_archive_page(w, s_uniq, bodyZ, wsg_slen(bodyZ), cidZ) 127 let cz: i64 = ss_commit(prefix, w, 0) 128 wsg_p(" archived zarquon page (war rc="); wsg_pn(raz); wsg_p(" commit rc="); wsg_pn(cz); wsg_p(" cid="); wsg_p(cidZ); wsg_p(")\n" as *u8) 129 130 // ===== ranked-result scratch ===== 131 let out: *NxWikiRanked = sys_mmap(128) as *NxWikiRanked 132 nx_wiki_ranked_init(out, 64) 133 134 // ----- query "sovereign": expect ranked DOC hits, docA above docB ----- 135 let qt1: *i64 = sys_mmap(8 * 2) as *i64 136 let ql1: *i64 = sys_mmap(8 * 2) as *i64 137 qt1[0] = "sovereign" as *u8 as i64 138 ql1[0] = 9 139 nx_wiki_bm25_rank(store, prefix, qt1 as **u8, ql1, 1, out, 20) 140 let n1: i64 = nx_wiki_ranked_count(out) 141 // find the rank POSITION of docA(rowid rA) and docB(rowid rB) among DOC hits 142 var posA: i64 = 0 - 1 143 var posB: i64 = 0 - 1 144 var scoreA: i64 = 0 145 var scoreB: i64 = 0 146 var i: i64 = 0 147 while i < n1 { 148 if nx_wiki_ranked_kind_at(out, i) == NX_WBS_SRC_DOC { 149 let rid: i64 = nx_wiki_ranked_rowid_at(out, i) 150 if rid == rA { posA = i; scoreA = nx_wiki_ranked_score_at(out, i) } 151 if rid == rB { posB = i; scoreB = nx_wiki_ranked_score_at(out, i) } 152 } 153 i = i + 1 154 } 155 // rank_ok: both present, docA strictly ABOVE docB (lower index = higher rank) 156 // AND docA's BM25 score strictly exceeds docB's (density, not presence). 157 var rank_ok: i64 = 0 158 if posA >= 0 { if posB >= 0 { if posA < posB { if scoreA > scoreB { rank_ok = 1 } } } } 159 wsg_p(" query 'sovereign': results="); wsg_pn(n1); 160 wsg_p(" posA="); wsg_pn(posA); wsg_p(" posB="); wsg_pn(posB); 161 wsg_p(" scoreA="); wsg_pn(scoreA); wsg_p(" scoreB="); wsg_pn(scoreB); 162 wsg_p(" rank_ok="); wsg_pn(rank_ok); wsg_p("\n" as *u8) 163 164 // ----- query "zarquontoken": must be FOUND, and via the ARCHIVE ----- 165 let qt2: *i64 = sys_mmap(8 * 2) as *i64 166 let ql2: *i64 = sys_mmap(8 * 2) as *i64 167 qt2[0] = "zarquontoken" as *u8 as i64 168 ql2[0] = 12 169 nx_wiki_bm25_rank(store, prefix, qt2 as **u8, ql2, 1, out, 20) 170 let n2: i64 = nx_wiki_ranked_count(out) 171 // archive_ok: at least one hit, and it is an ARCHIVE-kind row (proves the 172 // seg_store term index supplied content the live doc store never had). 173 var archive_ok: i64 = 0 174 var arch_hits: i64 = 0 175 i = 0 176 while i < n2 { 177 if nx_wiki_ranked_kind_at(out, i) == NX_WBS_SRC_ARCHIVE { arch_hits = arch_hits + 1 } 178 i = i + 1 179 } 180 if arch_hits >= 1 { archive_ok = 1 } 181 wsg_p(" query 'zarquontoken': results="); wsg_pn(n2); wsg_p(" archive_hits="); wsg_pn(arch_hits); wsg_p(" archive_ok="); wsg_pn(archive_ok); wsg_p("\n" as *u8) 182 183 // ----- query "zzqnosuchterm": 0 hits anywhere ----- 184 let qt3: *i64 = sys_mmap(8 * 2) as *i64 185 let ql3: *i64 = sys_mmap(8 * 2) as *i64 186 qt3[0] = "zzqnosuchterm" as *u8 as i64 187 ql3[0] = 13 188 nx_wiki_bm25_rank(store, prefix, qt3 as **u8, ql3, 1, out, 20) 189 let n3: i64 = nx_wiki_ranked_count(out) 190 var neg_ok: i64 = 0 191 if n3 == 0 { neg_ok = 1 } 192 wsg_p(" query 'zzqnosuchterm': results="); wsg_pn(n3); wsg_p(" neg_ok="); wsg_pn(neg_ok); wsg_p("\n" as *u8) 193 194 // ===== VERDICT (judged by the marker; results=<n> reports the ranked query) ===== 195 var green: i64 = 1 196 if rank_ok != 1 { green = 0 } 197 if archive_ok != 1 { green = 0 } 198 if neg_ok != 1 { green = 0 } 199 200 let lfd: i64 = sys_openat_append(WSG_LOG, 420) 201 wsg_w2(lfd, "WIKISEARCH results=" as *u8); wsg_n2(lfd, n1) 202 wsg_w2(lfd, " rank_ok=" as *u8); wsg_n2(lfd, rank_ok) 203 wsg_w2(lfd, " archive_ok=" as *u8); wsg_n2(lfd, archive_ok) 204 wsg_w2(lfd, " neg_ok=" as *u8); wsg_n2(lfd, neg_ok) 205 if green == 1 { wsg_w2(lfd, " verdict=GREEN\n" as *u8) } else { wsg_w2(lfd, " verdict=RED\n" as *u8) } 206 if lfd >= 0 { sys_close(lfd) } 207 208 if green == 1 { 209 sys_exit(0) 210 return 0 211 } 212 sys_exit(1) 213 return 1 214}