nx_semantic_retrieval.nx source

1// nx_semantic_retrieval.nx -- SPEC A built bits-up: match a query BEYOND exact terms via co-occurrence. 2// Terms that appear together across the corpus are related; so a document can be relevant to a query 3// even when it never contains the query word, as long as it contains terms that CO-OCCUR with it. This 4// is the rung where retrieval can exceed BM25 (which scores 0 when the exact term is absent). 5// Built on Layer-8 integer ALU ops only (the spec's primitives): COUNT (term presence), IMUL+IADD 6// (vector dot product), ISQRT (cosine magnitude) -- no floats, no learned embeddings. 7// Honest boundary: co-occurrence over a small corpus is a weak proxy for true semantics; learned 8// embeddings (the EMBED_TRAIN primitive the Researcher flagged) are the LLM rung, not faked here. 9// license_tier: ORIGINAL Pairs with nx_research_extract (re_has/re_count) + nx_bm25 (exact baseline). 10 11import "nx_research_extract.nx" // re_has / re_count 12import "nx_syscalls.nx" 13 14// L8 integer sqrt (digit-by-digit, no multiply in the loop) -- the cosine magnitude primitive, bits-up. 15func sr_isqrt(x: i64) -> i64 { 16 if x < 2 { return x } 17 var num: i64 = x; var r: i64 = 0; var b: i64 = 1 18 while b * 4 <= num { b = b * 4 } 19 while b > 0 { 20 if num >= r + b { num = num - r - b; r = (r / 2) + b } else { r = r / 2 } 21 b = b / 4 22 } 23 return r 24} 25 26func sr_doc(ptrs: *i64, k: i64) -> *u8 { return ptrs[k] as *u8 } 27 28// co-occurrence: number of docs containing BOTH terms a and b -- the semantic link strength. 29func sr_cooc(ptrs: *i64, lens: *i64, N: i64, a: *u8, b: *u8) -> i64 { 30 var c: i64 = 0; var i: i64 = 0 31 while i < N { if re_has(sr_doc(ptrs, i), lens[i], a) == 1 { if re_has(sr_doc(ptrs, i), lens[i], b) == 1 { c = c + 1 } } i = i + 1 } 32 return c 33} 34 35// the query's co-occurrence VECTOR over the vocabulary: qvec[t] = cooc(query_term, vocab[t]). 36func sr_build_qvec(ptrs: *i64, lens: *i64, N: i64, qterm: *u8, vocab: *i64, V: i64, qvec: *i64) -> i64 { 37 var t: i64 = 0 38 while t < V { qvec[t] = sr_cooc(ptrs, lens, N, qterm, vocab[t] as *u8); t = t + 1 } 39 return 0 40} 41 42// semantic score of doc k = dot(qvec, doc-presence-vector) = sum of qvec[t] for vocab terms present. 43func sr_score(ptrs: *i64, lens: *i64, k: i64, qvec: *i64, vocab: *i64, V: i64) -> i64 { 44 var s: i64 = 0; var t: i64 = 0 45 while t < V { if re_has(sr_doc(ptrs, k), lens[k], vocab[t] as *u8) == 1 { s = s + qvec[t] } t = t + 1 } 46 return s 47} 48 49// the exact-match baseline (what BM25/substring sees): occurrences of the literal query term. 50func sr_exact(ptrs: *i64, lens: *i64, k: i64, qterm: *u8) -> i64 { return re_count(sr_doc(ptrs, k), lens[k], qterm) } 51 52func sr_best(ptrs: *i64, lens: *i64, N: i64, qvec: *i64, vocab: *i64, V: i64) -> i64 { 53 var best: i64 = 0 - 1; var bestscore: i64 = 0; var i: i64 = 0 54 while i < N { let sc: i64 = sr_score(ptrs, lens, i, qvec, vocab, V); if sc > bestscore { bestscore = sc; best = i } i = i + 1 } 55 return best 56} 57 58// integer cosine similarity x1000 between the query vector and doc k's presence vector (uses ISQRT). 59func sr_cosine_milli(ptrs: *i64, lens: *i64, k: i64, qvec: *i64, vocab: *i64, V: i64) -> i64 { 60 var dot: i64 = 0; var qmag2: i64 = 0; var dmag2: i64 = 0; var t: i64 = 0 61 while t < V { 62 let dv: i64 = re_has(sr_doc(ptrs, k), lens[k], vocab[t] as *u8) // 0/1 63 dot = dot + qvec[t] * dv 64 qmag2 = qmag2 + qvec[t] * qvec[t] 65 dmag2 = dmag2 + dv * dv 66 t = t + 1 67 } 68 let denom: i64 = sr_isqrt(qmag2) * sr_isqrt(dmag2) 69 if denom <= 0 { return 0 } 70 return (dot * 1000) / denom 71}