nx_dms_search.nx source
↩ module page · 161 lines · 5537 B
1// nx_dms_search.nx -- consent-gated search over the licensed library (DMS rung 4).
2//
3// Ranks media by relevance to a text query using the existing nx_bm25 scorer,
4// but the candidate set is FIRST filtered through nx_dms_is_visible -- so a
5// search returns ONLY currently-licensed, active assets, ranked. An unlicensed
6// asset that is a STRONGER textual match is still never surfaced; revoke or
7// expire a license and it drops out of results the same instant.
8//
9// Composes: nx_dms_catalog (visibility gate) + nx_rights_ledger (consent) +
10// nx_bm25 (ranking) + nx_search_inverted (tokenizer). DRY #15 -- no new
11// ranker, no new tokenizer.
12//
13// RUNG 1 (this file): in-memory, BM25 over the visible candidate set. Scaling
14// to a persistent inverted index over the whole catalog (nx_search_inverted +
15// nx_seg_store) = later rung (flagged, NOT a silent cap).
16// license_tier: ORIGINAL
17
18import "nx_syscalls.nx"
19import "nx_rights_ledger.nx"
20import "nx_dms_catalog.nx"
21import "nx_bm25.nx"
22import "nx_search_inverted.nx"
23
24struct NxDmsSearchDoc {
25 asset_id: nx_int,
26 text_ptr: nx_size, // the metadata text pointer, held as an integer
27 text_len: nx_int,
28}
29
30struct NxDmsSearchIndex {
31 docs: *NxDmsSearchDoc,
32 capacity: nx_size,
33 count: nx_size,
34}
35
36const NX_DMS_SDOC_BYTES: nx_size = 24 // 3 fields * 8 bytes
37
38func _sx_strlen(s: *u8) -> i64 {
39 var n: i64 = 0
40 while s[n] != (0 as u8) { n = n + 1 }
41 return n
42}
43
44func nx_dms_search_new(capacity: nx_size) -> *NxDmsSearchIndex {
45 let sx: *NxDmsSearchIndex = (sys_mmap(24)) as *NxDmsSearchIndex
46 sx.docs = (sys_mmap(capacity * NX_DMS_SDOC_BYTES)) as *NxDmsSearchDoc
47 sx.capacity = capacity
48 sx.count = 0
49 return sx
50}
51
52func _sx_at(sx: *NxDmsSearchIndex, idx: nx_size) -> *NxDmsSearchDoc {
53 return (sx.docs as i64 + (idx as i64) * NX_DMS_SDOC_BYTES) as *NxDmsSearchDoc
54}
55
56// register searchable metadata text for a catalogued asset (null-terminated).
57func nx_dms_search_add(sx: *NxDmsSearchIndex, asset_id: nx_int, text: *u8) -> nx_int {
58 if sx.count >= sx.capacity { return -1 }
59 let d: *NxDmsSearchDoc = _sx_at(sx, sx.count)
60 d.asset_id = asset_id
61 d.text_ptr = text as i64
62 d.text_len = _sx_strlen(text)
63 sx.count = sx.count + 1
64 return 0
65}
66
67// Query: returns up to out_max licensed+active asset ids ranked by BM25 (most-
68// relevant first); out_scores gets the matching Q16.16 scores. Returns the
69// number of results written. Only positively-scored AND currently-visible
70// assets are returned.
71func nx_dms_search_query(sx: *NxDmsSearchIndex, cat: *NxDmsCatalog, rl: *NxRightsLedger,
72 query: *u8, now_us: nx_size,
73 out_ids: *i64, out_scores: *i64, out_max: i64) -> i64 {
74 let ndoc: i64 = sx.count as i64
75 if ndoc <= 0 { return 0 }
76
77 // 1. candidate set = visible (licensed + active) docs ONLY
78 let cand_texts: **u8 = sys_mmap((ndoc + 1) * 8) as **u8
79 let cand_lens: *i64 = sys_mmap((ndoc + 1) * 8) as *i64
80 let cand_ids: *i64 = sys_mmap((ndoc + 1) * 8) as *i64
81 var ncand: i64 = 0
82 var i: nx_size = 0
83 while i < sx.count {
84 let d: *NxDmsSearchDoc = _sx_at(sx, i)
85 if nx_dms_is_visible(cat, rl, d.asset_id, now_us) == 1 {
86 cand_texts[ncand] = (d.text_ptr) as *u8
87 cand_lens[ncand] = d.text_len
88 cand_ids[ncand] = d.asset_id
89 ncand = ncand + 1
90 }
91 i = i + 1
92 }
93 if ncand <= 0 { return 0 }
94
95 // 2. tokenize the query (alphanumeric runs, length >= 2)
96 let qn: i64 = _sx_strlen(query)
97 let q_terms: **u8 = sys_mmap((qn + 1) * 8) as **u8
98 let q_lens: *i64 = sys_mmap((qn + 1) * 8) as *i64
99 var nterms: i64 = 0
100 var p: i64 = 0
101 while p < qn {
102 if nx_inv_is_token_char(query[p] as i64) == 0 {
103 p = p + 1
104 } else {
105 let start: i64 = p
106 var run: i64 = 1
107 while run == 1 {
108 run = 0
109 if p < qn { if nx_inv_is_token_char(query[p] as i64) == 1 { p = p + 1; run = 1 } }
110 }
111 if (p - start) >= 2 {
112 q_terms[nterms] = ((query as i64) + start) as *u8
113 q_lens[nterms] = p - start
114 nterms = nterms + 1
115 }
116 }
117 }
118 if nterms <= 0 { return 0 }
119
120 // 3. BM25 score over the visible candidate set
121 let scores: *i64 = sys_mmap((ncand + 1) * 8) as *i64
122 nx_bm25_score(cand_texts, cand_lens, ncand, q_terms, q_lens, nterms, scores)
123
124 // 4. rank candidate indices by score desc (selection sort; ncand is small)
125 let order: *i64 = sys_mmap((ncand + 1) * 8) as *i64
126 var k: i64 = 0
127 while k < ncand { order[k] = k; k = k + 1 }
128 var a: i64 = 0
129 while a < ncand {
130 var best: i64 = a
131 var b: i64 = a + 1
132 while b < ncand {
133 if scores[order[b]] > scores[order[best]] { best = b }
134 b = b + 1
135 }
136 let tmp: i64 = order[a]
137 order[a] = order[best]
138 order[best] = tmp
139 a = a + 1
140 }
141
142 // 5. emit positively-scored results, most-relevant first, up to out_max
143 var nout: i64 = 0
144 var r: i64 = 0
145 while r < ncand {
146 let ci: i64 = order[r]
147 if scores[ci] > 0 {
148 if nout < out_max {
149 out_ids[nout] = cand_ids[ci]
150 out_scores[nout] = scores[ci]
151 nout = nout + 1
152 }
153 }
154 r = r + 1
155 }
156 return nout
157}
158
159func nx_dms_search_count(sx: *NxDmsSearchIndex) -> nx_size {
160 return sx.count
161}