nx_intelmine_lib.nx source
↩ module page · 326 lines · 15430 B
1// nx_intelmine_lib.nx -- THE PROPOSER: mined signals become candidate rows on the /compare board that owns them
2// (/compare/intelmine IM3 im_propose_rows, IM24 ic_propose). LIB. Inputs already exist as data: the review rubric over
3// a title's journal (nx_reviewmine_lib: defect and feature tables per tier) and the installed-mode census row (what the
4// title ships). knowledge/reviewmine/capability_map.conf routes each signal (kind|term) to a domain and a rung title.
5// Output: <outdir>/<domain>.proposed rows, APPENDED and deduped by (appid, kind, term); the proposer never writes a
6// board -- a seat adjudicates a proposal with the goal map, and only then does a rung land.
7//
8// SIGNALS (the operator's split): DEFECT (a defect term in the does-not-meet or mixed tiers), WE-DO-BETTER (a feature
9// complained about at least as often as praised), DEMANDED (a feature praised but asked for more: a demand cue beside
10// it), THEY-DO-WELL (praised more than complained -- take it), SHIPS (an installed fingerprint: the title ships it).
11// Row: prop|<epoch>|<appid>|<name>|<signal>|<kind>|<term>|<domain>|<rung title>|<evidence>
12// license_tier: ORIGINAL
13import "nx_syscalls.nx"
14import "nx_reviewmine_lib.nx"
15import "nx_installed_census_lib.nx"
16
17const IP_MAP_MAX: i64 = 1024
18const IP_ARENA: i64 = 262144
19const IP_NAME_CAP: i64 = 512
20const IP_ROW_CAP: i64 = 4096
21const IP_PATH_CAP: i64 = 1024
22// class count and names come from the census lib (IC_CLASS_N, ic_class_name): ONE table, never a second copy
23const IP_KIND_W: i64 = 32 // one capability_map row: the kind cell
24const IP_TERM_W: i64 = 64 // the term cell
25const IP_DOMAIN_W: i64 = 64 // the domain cell
26const IP_CLASS_W: i64 = 256 // one census class-label cell (labels joined by semicolons)
27const IP_IDKEY_CAP: i64 = 64 // the |<appid>| dedupe key
28const IP_EVIDENCE_CAP: i64 = 256 // the evidence field of one proposal row
29const IP_F_KIND: i64 = 0 // capability_map.conf fields
30const IP_F_TERM: i64 = 1
31const IP_F_DOMAIN: i64 = 2
32const IP_F_TITLE: i64 = 3
33const IP_CENSUS_F_APPID: i64 = 1 // i| census row fields
34const IP_CENSUS_F_NAME: i64 = 2
35const IP_CENSUS_FIRST_CLASS_FIELD: i64 = 11 // engine sits in field 11 of an i| census row (0 = the i marker)
36const IP_CH_SEMI: i64 = 59
37const IP_CH_LBRACE: i64 = 123
38const IP_CH_RBRACE: i64 = 125
39const IP_DUP_KEY_CAP: i64 = 512
40
41static ip_kind: *u8 // IP_MAP_MAX * IP_KIND_W
42static ip_term: *u8 // IP_MAP_MAX * IP_TERM_W
43static ip_domain: *u8 // IP_MAP_MAX * IP_DOMAIN_W
44static ip_title_off: *i64
45static ip_arena: *u8
46static ip_used: i64
47static ip_n: i64
48static ip_rows_written: i64
49static ip_rows_dup: i64
50static ip_rows_unmapped: i64
51static ip_row: *u8
52static ip_key: *u8
53static ip_census_name: *u8
54static ip_census_class: *u8 // IC_CLASS_N * IP_CLASS_W
55
56func ip_init() -> i64 {
57 if (ip_kind as i64) != 0 { return 0 }
58 ip_kind = sys_mmap(IP_MAP_MAX * IP_KIND_W); ip_term = sys_mmap(IP_MAP_MAX * IP_TERM_W); ip_domain = sys_mmap(IP_MAP_MAX * IP_DOMAIN_W)
59 ip_title_off = sys_mmap(IP_MAP_MAX * RM_I64_BYTES) as *i64; ip_arena = sys_mmap(IP_ARENA)
60 ip_row = sys_mmap(IP_ROW_CAP); ip_key = sys_mmap(IP_DUP_KEY_CAP)
61 ip_census_name = sys_mmap(IP_NAME_CAP); ip_census_class = sys_mmap(IC_CLASS_N * IP_CLASS_W)
62 ip_used = 0; ip_n = 0; ip_rows_written = 0; ip_rows_dup = 0; ip_rows_unmapped = 0
63 return 0
64}
65func ip_lc(c: i64) -> i64 { if c >= RM_UPPER_A { if c <= RM_UPPER_Z { return c + RM_CASE_DELTA } } return c }
66func ip_streq_ci(a: *u8, b: *u8) -> i64 {
67 var i: i64 = 0
68 while a[i] != (0 as u8) { if ip_lc(a[i] as i64) != ip_lc(b[i] as i64) { return 0 } i = i + 1 }
69 if b[i] != (0 as u8) { return 0 }
70 return 1
71}
72// field f (0-based, pipe-separated) of a line [s,e) into out; returns len or -1
73func ip_field(b: *u8, s: i64, e: i64, f: i64, out: *u8, cap: i64) -> i64 {
74 var k: i64 = 0
75 var st: i64 = s
76 var i: i64 = s
77 while i <= e {
78 var c: i64 = RM_PIPE
79 if i < e { c = b[i] as i64 }
80 if c == RM_PIPE {
81 if k == f { var l: i64 = i - st; if l > cap - 1 { l = cap - 1 } rm_catn(out, 0, (b as i64 + st) as *u8, l); out[l] = 0 as u8; return l }
82 k = k + 1; st = i + 1
83 }
84 i = i + 1
85 }
86 out[0] = 0 as u8
87 return 0 - 1
88}
89// ---- the map: kind|term|domain|title template ----
90func ip_map_load(path: *u8) -> i64 {
91 ip_init()
92 ip_n = 0; ip_used = 0
93 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
94 lp[0] = 0
95 let b: *u8 = sys_read_file(path, lp)
96 if (b as i64) == 0 { return 0 }
97 let n: i64 = lp[0]
98 let tmp: *u8 = sys_mmap(IP_ROW_CAP)
99 var i: i64 = 0
100 while i < n {
101 let e0: i64 = rm_line_end(b, n, i)
102 var e: i64 = e0
103 if e > i { if (b[e - 1] as i64) == RM_CR { e = e - 1 } }
104 if e > i { if (b[i] as i64) != IP_CH_SEMI { if ip_n < IP_MAP_MAX {
105 if ip_field(b, i, e, IP_F_KIND, (ip_kind as i64 + ip_n * IP_KIND_W) as *u8, IP_KIND_W) > 0 {
106 if ip_field(b, i, e, IP_F_TERM, (ip_term as i64 + ip_n * IP_TERM_W) as *u8, IP_TERM_W) > 0 {
107 if ip_field(b, i, e, IP_F_DOMAIN, (ip_domain as i64 + ip_n * IP_DOMAIN_W) as *u8, IP_DOMAIN_W) > 0 {
108 let tl: i64 = ip_field(b, i, e, IP_F_TITLE, tmp, IP_ROW_CAP)
109 if tl > 0 { if ip_used + tl + 1 < IP_ARENA {
110 ip_title_off[ip_n] = ip_used
111 rm_catn(ip_arena, ip_used, tmp, tl)
112 ip_used = ip_used + tl
113 ip_arena[ip_used] = 0 as u8; ip_used = ip_used + 1
114 ip_n = ip_n + 1
115 } }
116 }
117 }
118 }
119 } } }
120 i = e0 + 1
121 }
122 return ip_n
123}
124func ip_map_count() -> i64 { return ip_n }
125func ip_map_find(kind: *u8, term: *u8) -> i64 {
126 var i: i64 = 0
127 while i < ip_n {
128 if rm_streq((ip_kind as i64 + i * IP_KIND_W) as *u8, kind) == 1 { if ip_streq_ci((ip_term as i64 + i * IP_TERM_W) as *u8, term) == 1 { return i } }
129 i = i + 1
130 }
131 return 0 - 1
132}
133func ip_map_domain(i: i64) -> *u8 { return (ip_domain as i64 + i * IP_DOMAIN_W) as *u8 }
134// the template with {title} replaced
135func ip_map_title(i: i64, title: *u8, out: *u8, cap: i64) -> i64 {
136 let t: *u8 = (ip_arena as i64 + ip_title_off[i]) as *u8
137 var o: i64 = 0
138 var k: i64 = 0
139 while t[k] != (0 as u8) {
140 if (t[k] as i64) == IP_CH_LBRACE {
141 // {title}
142 let tt: *u8 = "{title}" as *u8
143 if rm_find(t, rm_slen(t), tt, k) == k { if o + rm_slen(title) + 1 < cap { o = rm_cat(out, o, title) } k = k + rm_slen(tt) }
144 else { if o + 1 < cap { out[o] = t[k]; o = o + 1 } k = k + 1 }
145 } else { if o + 1 < cap { out[o] = t[k]; o = o + 1 } k = k + 1 }
146 }
147 out[o] = 0 as u8
148 return o
149}
150// ---- the census row for an appid: name + class labels ----
151func ip_census_load(census_path: *u8, appid: i64) -> i64 {
152 ip_init()
153 ip_census_name[0] = 0 as u8
154 var c: i64 = 0
155 while c < IC_CLASS_N { let d: *u8 = (ip_census_class as i64 + c * IP_CLASS_W) as *u8; d[0] = RM_MINUS as u8; d[1] = 0 as u8; c = c + 1 }
156 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
157 lp[0] = 0
158 let b: *u8 = sys_read_file(census_path, lp)
159 if (b as i64) == 0 { return 0 }
160 let n: i64 = lp[0]
161 let tmp: *u8 = sys_mmap(IP_NAME_CAP)
162 var i: i64 = 0
163 while i < n {
164 let e: i64 = rm_line_end(b, n, i)
165 if e > i + RM_ROW_PREFIX_LEN { if (b[i] as i64) == IC_ROW_MARK { if (b[i + 1] as i64) == RM_PIPE {
166 if ip_field(b, i, e, IP_CENSUS_F_APPID, tmp, IP_NAME_CAP) > 0 { if rm_atoi(tmp) == appid {
167 ip_field(b, i, e, IP_CENSUS_F_NAME, ip_census_name, IP_NAME_CAP)
168 var k: i64 = 0
169 while k < IC_CLASS_N { ip_field(b, i, e, IP_CENSUS_FIRST_CLASS_FIELD + k, (ip_census_class as i64 + k * IP_CLASS_W) as *u8, IP_CLASS_W); k = k + 1 }
170 return 1
171 } }
172 } } }
173 i = e + 1
174 }
175 return 0
176}
177func ip_census_name_get() -> *u8 { return ip_census_name }
178func ip_census_class_get(c: i64) -> *u8 { return (ip_census_class as i64 + c * IP_CLASS_W) as *u8 }
179// ---- emit one proposal row, deduped by (appid, kind, term) inside <outdir>/<domain>.proposed ----
180func ip_emit(outdir: *u8, epoch: i64, appid: i64, name: *u8, signal: *u8, kind: *u8, term: *u8, mapi: i64, evidence: *u8) -> i64 {
181 let path: *u8 = sys_mmap(IP_PATH_CAP)
182 var o: i64 = rm_cat(path, 0, outdir)
183 if o > 0 { if (path[o - 1] as i64) != RM_CH_SLASH { path[o] = RM_CH_SLASH as u8; o = o + 1 } }
184 o = rm_cat(path, o, ip_map_domain(mapi))
185 o = rm_cat(path, o, ".proposed" as *u8)
186 path[o] = 0 as u8
187 // dedupe key: |<appid>|... we scan for "|<kind>|<term>|" on a line that also carries "|<appid>|"
188 var ko: i64 = 0
189 ip_key[ko] = RM_PIPE as u8; ko = ko + 1
190 ko = rm_cat(ip_key, ko, kind)
191 ip_key[ko] = RM_PIPE as u8; ko = ko + 1
192 ko = rm_cat(ip_key, ko, term)
193 ip_key[ko] = RM_PIPE as u8; ko = ko + 1
194 ip_key[ko] = 0 as u8
195 let idk: *u8 = sys_mmap(IP_IDKEY_CAP)
196 var io: i64 = 0
197 idk[io] = RM_PIPE as u8; io = io + 1
198 io = rm_cati(idk, io, appid)
199 idk[io] = RM_PIPE as u8; io = io + 1
200 idk[io] = 0 as u8
201 let lp: *i64 = sys_mmap(RM_I64_PAIR) as *i64
202 lp[0] = 0
203 let b: *u8 = sys_read_file(path, lp)
204 if (b as i64) != 0 {
205 let n: i64 = lp[0]
206 var i: i64 = 0
207 while i < n {
208 let e: i64 = rm_line_end(b, n, i)
209 if e > i {
210 let ln: i64 = e - i
211 let lb: *u8 = (b as i64 + i) as *u8
212 if rm_find(lb, ln, idk, 0) >= 0 { if rm_find(lb, ln, ip_key, 0) >= 0 { ip_rows_dup = ip_rows_dup + 1; return 0 } }
213 }
214 i = e + 1
215 }
216 }
217 var r: i64 = rm_cat(ip_row, 0, "prop|" as *u8)
218 r = rm_cati(ip_row, r, epoch)
219 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cati(ip_row, r, appid)
220 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, name)
221 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, signal)
222 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, kind)
223 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, term)
224 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, ip_map_domain(mapi))
225 ip_row[r] = RM_PIPE as u8; r = r + 1
226 let tb: *u8 = sys_mmap(IP_ROW_CAP)
227 ip_map_title(mapi, name, tb, IP_ROW_CAP - 1)
228 r = rm_cat(ip_row, r, tb)
229 ip_row[r] = RM_PIPE as u8; r = r + 1; r = rm_cat(ip_row, r, evidence)
230 ip_row[r] = RM_LF as u8; r = r + 1
231 if rm_file_append(path, ip_row, r) < 0 { return 0 - 1 }
232 ip_rows_written = ip_rows_written + 1
233 rm_w("PROPOSED " as *u8); rm_wb(ip_row, r - 1); rm_w("\n" as *u8)
234 return 1
235}
236func ip_rows_written_get() -> i64 { return ip_rows_written }
237func ip_rows_dup_get() -> i64 { return ip_rows_dup }
238func ip_rows_unmapped_get() -> i64 { return ip_rows_unmapped }
239func ip_counters_reset() -> i64 { ip_rows_written = 0; ip_rows_dup = 0; ip_rows_unmapped = 0; return 0 }
240// ---- proposals from the rubric already run (rm_rubric) ----
241func ip_propose_defects(outdir: *u8, epoch: i64, appid: i64, name: *u8, top: i64, min_support: i64) -> i64 {
242 let out: *i64 = sys_mmap((top + 1) * RM_I64_BYTES) as *i64
243 let nd: i64 = rm_rubric_rank(RM_RANK_DEFECT, top, min_support, out)
244 let ev: *u8 = sys_mmap(IP_EVIDENCE_CAP)
245 var r: i64 = 0
246 var made: i64 = 0
247 while r < nd {
248 let sl: i64 = out[r]
249 let term: *u8 = rm_vocab_tok(sl)
250 let mi: i64 = ip_map_find("defect" as *u8, term)
251 if mi < 0 { ip_rows_unmapped = ip_rows_unmapped + 1 }
252 else {
253 var o: i64 = rm_cat(ev, 0, "does_not_meet=" as *u8); o = rm_cati(ev, o, rm_vocab_tier(sl, RM_TIER_DNM))
254 o = rm_cat(ev, o, " mixed=" as *u8); o = rm_cati(ev, o, rm_vocab_tier(sl, RM_TIER_MIXED))
255 o = rm_cat(ev, o, " meets=" as *u8); o = rm_cati(ev, o, rm_vocab_tier(sl, RM_TIER_MEETS))
256 o = rm_cat(ev, o, " exceeds=" as *u8); o = rm_cati(ev, o, rm_vocab_tier(sl, RM_TIER_EXCEEDS))
257 o = rm_cat(ev, o, " dnm_lift_permil=" as *u8); o = rm_cati(ev, o, rm_vocab_dnm_lift(sl))
258 ev[o] = 0 as u8
259 if ip_emit(outdir, epoch, appid, name, "DEFECT" as *u8, "defect" as *u8, term, mi, ev) == 1 { made = made + 1 }
260 }
261 r = r + 1
262 }
263 return made
264}
265func ip_propose_features(outdir: *u8, epoch: i64, appid: i64, name: *u8, top: i64, min_support: i64) -> i64 {
266 let out: *i64 = sys_mmap((top + 1) * RM_I64_BYTES) as *i64
267 let nf: i64 = rm_rubric_rank(RM_RANK_FEATURE, top, min_support, out)
268 let ev: *u8 = sys_mmap(IP_EVIDENCE_CAP)
269 var r: i64 = 0
270 var made: i64 = 0
271 while r < nf {
272 let sl: i64 = out[r]
273 let term: *u8 = rm_vocab_tok(sl)
274 let mi: i64 = ip_map_find("feature" as *u8, term)
275 if mi < 0 { ip_rows_unmapped = ip_rows_unmapped + 1 }
276 else {
277 let praised: i64 = rm_vocab_tier(sl, RM_TIER_EXCEEDS) + rm_vocab_tier(sl, RM_TIER_MEETS)
278 let complained: i64 = rm_vocab_tier(sl, RM_TIER_DNM)
279 let demanded: i64 = rm_vocab_dem(sl)
280 var signal: *u8 = "THEY-DO-WELL" as *u8
281 if complained >= praised { signal = "WE-DO-BETTER" as *u8 }
282 else { if demanded >= min_support { signal = "DEMANDED" as *u8 } }
283 var o: i64 = rm_cat(ev, 0, "praised=" as *u8); o = rm_cati(ev, o, praised)
284 o = rm_cat(ev, o, " complained=" as *u8); o = rm_cati(ev, o, complained)
285 o = rm_cat(ev, o, " mixed=" as *u8); o = rm_cati(ev, o, rm_vocab_tier(sl, RM_TIER_MIXED))
286 o = rm_cat(ev, o, " demanded=" as *u8); o = rm_cati(ev, o, demanded)
287 ev[o] = 0 as u8
288 if ip_emit(outdir, epoch, appid, name, signal, "feature" as *u8, term, mi, ev) == 1 { made = made + 1 }
289 }
290 r = r + 1
291 }
292 return made
293}
294// ---- proposals from the installed census row (ip_census_load first) ----
295func ip_propose_installed(outdir: *u8, epoch: i64, appid: i64, name: *u8) -> i64 {
296 var made: i64 = 0
297 let lab: *u8 = sys_mmap(IP_CLASS_W)
298 var c: i64 = 0
299 while c < IC_CLASS_N {
300 let labels: *u8 = ip_census_class_get(c)
301 // split on ';'
302 var s: i64 = 0
303 var i: i64 = 0
304 var go: i64 = 1
305 while go == 1 {
306 var ch: i64 = 0
307 if labels[i] != (0 as u8) { ch = labels[i] as i64 }
308 if ch == IP_CH_SEMI { ch = 0 }
309 if ch == 0 {
310 let l: i64 = i - s
311 if l > 0 { if l < IP_CLASS_W - 1 {
312 rm_catn(lab, 0, (labels as i64 + s) as *u8, l); lab[l] = 0 as u8
313 if rm_streq(lab, "-" as *u8) == 0 {
314 let mi: i64 = ip_map_find(ic_class_name(c), lab)
315 if mi < 0 { ip_rows_unmapped = ip_rows_unmapped + 1 }
316 else { if ip_emit(outdir, epoch, appid, name, "SHIPS" as *u8, ic_class_name(c), lab, mi, "fingerprint=installed-tree" as *u8) == 1 { made = made + 1 } }
317 }
318 } }
319 if labels[i] == (0 as u8) { go = 0 } else { s = i + 1 }
320 }
321 i = i + 1
322 }
323 c = c + 1
324 }
325 return made
326}