nx_recall_prox.nx source
↩ module page · 116 lines · 5031 B
1// nx_recall_prox.nx -- serve-time TERM-PROXIMITY signal for the recall reranker.
2//
3// module: nishi-core.search.recall_prox
4// depends: nx_syscalls.nx
5// capability: CORE_COMPUTE
6// wired_status: LIBRARY (SERP wiring into dss_search_off_div PASS-1/PASS-2 = R1b, claims-coordinated)
7// genealogy_id: buttcher_clarke_cormack_2006_proximity (term-proximity relevance)
8//
9// WHY: the live SERP ranks by BM25 (+PageRank) only. dss PASS-1 ALREADY loads each candidate's doc
10// TEXT to count tf -- so a proximity signal (query terms that CLUSTER = more relevant) is computable
11// at serve time with NO dense index. Byte-substring based (tokenizer-free) = low blast-radius; it
12// fuses in exactly like the existing authority multiply. Honest: this is a proximity signal, NOT a
13// neural cross-encoder; it moves the ranking on the "terms-together" cases (the JPEG-vs-IR problem).
14//
15// prox_score(text, tn, terms, termlens, nterms) -> strength in [0, PROX_SCALE]:
16// the smallest byte-window containing ALL present query terms; tighter + more-terms = higher.
17// single-term query (or <2 terms present) -> 0 (no proximity signal, ranking unchanged).
18//
19// ⚠⚠SERVE-PATH ALIGNMENT (do NOT wire this raw into dss_search_off_div): this fn is byte-SUBSTRING
20// based -- correct for general text, but the engine's dss_tf_all matches by TOKENIZING via
21// ss_tok_next2 + dss_streq (token-EXACT, normalized). Wiring substring proximity into the SERP
22// would MIS-FIRE (case/stem/substring-in-word) and REGRESS live ranking. The serve version = a
23// SEPARATE dss_prox_all inside the search organ that clones dss_tf_all's tokenize loop + records
24// token positions -> the SAME min-window logic. This fn stays the gated ALGORITHM proof + a general
25// (non-tokenized) proximity primitive. See [[project-nishi-information-finding-sota-2026-07-17]] R1b.
26
27import "nx_syscalls.nx"
28
29const PROX_SCALE: i64 = 1024
30const PROX_WREF: i64 = 200 // reference window (bytes): span << WREF -> near-full strength
31const PROX_OCAP: i64 = 256 // max occurrences tracked (bounded serve cost)
32
33func px_eq_at(text: *u8, pos: i64, tn: i64, pat: *u8, pl: i64) -> i64 {
34 if pos + pl > tn { return 0 }
35 var i: i64 = 0
36 while i < pl { if text[pos+i] != pat[i] { return 0 } i = i + 1 }
37 return 1
38}
39func px_find(text: *u8, tn: i64, from: i64, pat: *u8, pl: i64) -> i64 {
40 if pl <= 0 { return 0 - 1 }
41 var p: i64 = from
42 while p + pl <= tn {
43 if px_eq_at(text, p, tn, pat, pl) == 1 { return p }
44 p = p + 1
45 }
46 return 0 - 1
47}
48
49// terms[i] = a *u8 term string cast to i64; termlens[i] = its length; nterms <= 64.
50func prox_score(text: *u8, tn: i64, terms: *i64, termlens: *i64, nterms: i64) -> i64 {
51 if nterms <= 1 { return 0 }
52 if tn <= 0 { return 0 }
53 let opos: *i64 = sys_mmap(PROX_OCAP * 8) as *i64
54 let oidx: *i64 = sys_mmap(PROX_OCAP * 8) as *i64
55 let seen: *i64 = sys_mmap(64 * 8) as *i64
56 var z: i64 = 0
57 while z < nterms { seen[z] = 0; z = z + 1 }
58 var nocc: i64 = 0
59 var present: i64 = 0
60 var ti: i64 = 0
61 while ti < nterms {
62 let pat: *u8 = terms[ti] as *u8
63 let pl: i64 = termlens[ti]
64 var from: i64 = 0
65 var more: i64 = 1
66 while more == 1 {
67 let pos: i64 = px_find(text, tn, from, pat, pl)
68 if pos < 0 { more = 0 }
69 else {
70 if nocc < PROX_OCAP { opos[nocc] = pos; oidx[nocc] = ti; nocc = nocc + 1 }
71 if seen[ti] == 0 { seen[ti] = 1; present = present + 1 }
72 from = pos + 1
73 if nocc >= PROX_OCAP { more = 0 }
74 }
75 }
76 ti = ti + 1
77 }
78 if present <= 1 { return 0 }
79 // selection-sort occurrences by position
80 var a: i64 = 0
81 while a < nocc {
82 var best: i64 = a
83 var b: i64 = a + 1
84 while b < nocc { if opos[b] < opos[best] { best = b } b = b + 1 }
85 if best != a {
86 let tp: i64 = opos[a]; opos[a] = opos[best]; opos[best] = tp
87 let tx: i64 = oidx[a]; oidx[a] = oidx[best]; oidx[best] = tx
88 }
89 a = a + 1
90 }
91 // two-pointer: smallest window [L,R] containing all `present` distinct terms
92 let cnt: *i64 = sys_mmap(64 * 8) as *i64
93 var zz: i64 = 0
94 while zz < nterms { cnt[zz] = 0; zz = zz + 1 }
95 var distinct: i64 = 0
96 var lft: i64 = 0
97 var minspan: i64 = tn + 1
98 var r: i64 = 0
99 while r < nocc {
100 let tr: i64 = oidx[r]
101 if cnt[tr] == 0 { distinct = distinct + 1 }
102 cnt[tr] = cnt[tr] + 1
103 while distinct == present {
104 let span: i64 = opos[r] - opos[lft]
105 if span < minspan { minspan = span }
106 let tl: i64 = oidx[lft]
107 cnt[tl] = cnt[tl] - 1
108 if cnt[tl] == 0 { distinct = distinct - 1 }
109 lft = lft + 1
110 }
111 r = r + 1
112 }
113 // strength = coverage(present/nterms) scaled by tightness(WREF/(WREF+minspan))
114 let base: i64 = (present * PROX_SCALE) / nterms
115 return (base * PROX_WREF) / (PROX_WREF + minspan)
116}