nx_research_digest.nx source
↩ module page · 70 lines · 2862 B
1// nx_research_digest.nx -- end-to-end EXTRACTIVE research digest (sovereign,
2// no LLM). Given a query + a set of sources (id, class, text) + candidate
3// claim terms, it returns:
4// - sources RANKED by BM25 relevance to the query (out_src_order)
5// - each claim's CONFIDENCE by independent corroboration (out_claim_conf)
6// - the corroboration ledger (caller reads citations via nx_corrob_cite)
7//
8// A source "asserts" a claim iff its text contains the claim term (nx_bm25_tf).
9// Corroboration uses the independence rule (nx_research_corroborate): echoes of
10// one source-class don't add confidence. This is the rigorous, cited digest
11// the Nishi researcher can return TODAY -- the generative prose synthesis on
12// top is the LLM-gated step (ai/accel lock). Composes nx_bm25 + nx_corroborate
13// (DRY #15). The live FETCH that supplies the sources = nx_crawl_https (the
14// delegated step), wired next.
15// license_tier: ORIGINAL
16
17import "nx_syscalls.nx"
18import "nx_bm25.nx"
19import "nx_search_inverted.nx"
20import "nx_research_corroborate.nx"
21
22// item indices sorted by score desc (ties -> lower index first).
23func _rd_order_desc(scores: *i64, n: i64, order_out: *i64) -> i64 {
24 let used: *i64 = sys_mmap(n * 8) as *i64
25 var i: i64 = 0
26 while i < n { used[i] = 0; i = i + 1 }
27 var k: i64 = 0
28 while k < n {
29 var best: i64 = 0 - 1
30 var bestsc: i64 = 0
31 var j: i64 = 0
32 while j < n {
33 if used[j] == 0 {
34 if best < 0 { best = j; bestsc = scores[j] }
35 else { if scores[j] > bestsc { best = j; bestsc = scores[j] } }
36 }
37 j = j + 1
38 }
39 used[best] = 1
40 order_out[k] = best
41 k = k + 1
42 }
43 return 0
44}
45
46func nx_research_digest(
47 src_ids: *i64, src_classes: *i64, src_texts: **u8, src_lens: *i64, nsrc: i64,
48 claim_ids: *i64, claim_terms: **u8, claim_term_lens: *i64, nclaims: i64,
49 q_terms: **u8, q_term_lens: *i64, nterms: i64,
50 out_src_order: *i64, out_claim_conf: *i64, corrob: *NxCorrob) -> i64 {
51 // 1. rank sources by BM25 relevance to the query
52 let bm: *i64 = sys_mmap(nsrc * 8) as *i64
53 nx_bm25_score(src_texts, src_lens, nsrc, q_terms, q_term_lens, nterms, bm)
54 _rd_order_desc(bm, nsrc, out_src_order)
55 // 2. corroborate each claim: a source asserts it iff its text contains the term
56 var ci: i64 = 0
57 while ci < nclaims {
58 let chash: i64 = nx_inv_hash_bytes_lower(claim_terms[ci], claim_term_lens[ci])
59 var si: i64 = 0
60 while si < nsrc {
61 if nx_bm25_tf(src_texts[si], src_lens[si], chash) > 0 {
62 nx_corrob_assert(corrob, claim_ids[ci], src_ids[si], src_classes[si])
63 }
64 si = si + 1
65 }
66 out_claim_conf[ci] = nx_corrob_confidence(corrob, claim_ids[ci])
67 ci = ci + 1
68 }
69 return 0
70}