nx_analyst_insight.nx source
↩ module page · 460 lines · 19804 B
1// nx_analyst_insight.nx -- AUTOMATED INSIGHT MINING (F1003): point at a dataset with NO target column and
2// get back RANKED findings. Every other analyst entry point we have answers a question you already framed
3// (ar_analyze needs a target); this one finds the question.
4//
5// THE HONESTY PROBLEM THIS IS BUILT AROUND. Scanning all column pairs means testing C(k,2) hypotheses and
6// reporting the best -- which is p-hacking with extra steps unless two things are true:
7// (1) the significance bar is FAMILY-WISE over every hypothesis actually tested (Bonferroni over the pair
8// count, via nx_analyst_infer), not per-test .05; and
9// (2) the report DISCLOSES how many hypotheses were tested and what level survived, so a reader can see
10// the multiplicity instead of being handed only the winners.
11// A miner that quietly reports "the top 5 correlations" out of 190 tested pairs is manufacturing findings.
12// Both are enforced below and gated.
13//
14// FINDING CLASSES: RELATIONSHIP (significant |r|) · NONLINEAR-MONOTONE (Spearman clears the bar and beats
15// Pearson by a configured margin -- the pair moves together but not in a straight line, which a
16// correlation-only scan reports as "moderate" and a human then misreads) · CONSTANT · ID-LIKE ·
17// SKEWED · OUTLIERS. Thresholds and score weights are DATA (analyst_insight.conf), not literals (rule 11).
18// license_tier: ORIGINAL No hardware writes (Rule 26).
19import "nx_syscalls.nx"
20import "_hdl_build/nx_analyst_data.nx"
21import "_hdl_build/nx_analyst_multi.nx"
22import "nx_analyst_infer.nx"
23const IN_MAGIC_1400: i64 = 1400
24const IN_MAGIC_1200: i64 = 1200
25const IN_MAGIC_1000000: i64 = 1000000
26const IN_MAGIC_4096: i64 = 4096
27
28const IN_MAXF: i64 = 2048 // finding slots
29const IN_CONF_CAP: i64 = 4096
30const IN_PROF: i64 = 16
31const IN_TAB: i64 = 9
32const IN_HASH: i64 = 35
33const IN_NL: i64 = 10
34const IN_CR: i64 = 13
35const IN_ZERO: i64 = 48
36// finding kinds
37const IN_K_REL: i64 = 1
38const IN_K_NONLIN: i64 = 2
39const IN_K_CONST: i64 = 3
40const IN_K_IDLIKE: i64 = 4
41const IN_K_SKEW: i64 = 5
42const IN_K_OUTLIER: i64 = 6
43
44func ins_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
45func ins_catn(d: *u8, o: i64, v: i64) -> i64 {
46 var oo: i64 = o
47 var m: i64 = v
48 if m < 0 { d[oo] = 45 as u8; oo = oo + 1; m = 0 - m }
49 let t: *u8 = sys_mmap(24)
50 var k: i64 = 0
51 if m == 0 { t[0] = IN_ZERO as u8; k = 1 }
52 while m > 0 { t[k] = (IN_ZERO + (m % 10)) as u8; m = m / 10; k = k + 1 }
53 var i: i64 = k - 1
54 while i >= 0 { d[oo] = t[i]; oo = oo + 1; i = i - 1 }
55 return oo
56}
57func ins_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
58// EXACT distinct count. ad_profile's AD_DISTINCT_PCT comes from the HLL sketch, which over-estimates at
59// small n and printed "137pct unique" for 40 rows -- fine as a scale estimate, wrong as the basis of a
60// threshold decision. O(n^2) is affordable at analyst scale and it cannot exceed n. (debt filed for the
61// >100pct percentage itself.)
62func ins_distinct(col: *i64, n: i64) -> i64 {
63 var d: i64 = 0
64 var i: i64 = 0
65 while i < n {
66 var seen: i64 = 0
67 var j: i64 = 0
68 while j < i { if col[j] == col[i] { seen = 1 } j = j + 1 }
69 if seen == 0 { d = d + 1 }
70 i = i + 1
71 }
72 return d
73}
74func ins_atoi_rng(buf: *u8, a: i64, b: i64) -> i64 {
75 var v: i64 = 0
76 var neg: i64 = 0
77 var i: i64 = a
78 if i < b { if buf[i] == (45 as u8) { neg = 1; i = i + 1 } }
79 while i < b {
80 let c: i64 = buf[i] as i64
81 if c >= IN_ZERO { if c <= (IN_ZERO + 9) { v = v * 10 + (c - IN_ZERO) } }
82 i = i + 1
83 }
84 if neg == 1 { return 0 - v }
85 return v
86}
87// one tunable from knowledge/registry/analyst_insight.conf ("key <TAB> value"); `dflt` when absent.
88// rule 11/17: weights and thresholds are configuration, not literals compiled into the scan.
89func ins_conf(key: *u8, dflt: i64) -> i64 {
90 let fd: i64 = sys_openat_rd("knowledge/registry/analyst_insight.conf" as *u8)
91 if fd < 0 { return dflt }
92 let cb: *u8 = sys_mmap(IN_CONF_CAP)
93 var n: i64 = 0
94 var r: i64 = 1
95 while r > 0 { if n >= IN_CONF_CAP { r = 0 } else { r = sys_read(fd, (((cb as i64) + n) as *u8), IN_CONF_CAP - n); if r > 0 { n = n + r } } }
96 sys_close(fd)
97 var out: i64 = dflt
98 var found: i64 = 0
99 var ls: i64 = 0
100 while ls < n {
101 var le: i64 = ls
102 var e: i64 = 0
103 while e == 0 { if le >= n { e = 1 } else { if cb[le] == (IN_NL as u8) { e = 1 } else { le = le + 1 } } }
104 var te: i64 = le
105 if te > ls { if cb[te-1] == (IN_CR as u8) { te = te - 1 } }
106 if te > ls { if cb[ls] != (IN_HASH as u8) { if found == 0 {
107 var tb: i64 = ls
108 var f: i64 = 0
109 while f == 0 { if tb >= te { f = 1 } else { if cb[tb] == (IN_TAB as u8) { f = 1 } else { tb = tb + 1 } } }
110 if tb < te {
111 var same: i64 = 1
112 var j: i64 = 0
113 while key[j] != (0 as u8) { if ls + j >= tb { same = 0 } else { if cb[ls + j] != key[j] { same = 0 } } j = j + 1 }
114 if ls + j != tb { same = 0 }
115 if same == 1 { out = ins_atoi_rng(cb, tb + 1, te); found = 1 }
116 }
117 } } }
118 ls = le + 1
119 }
120 return out
121}
122
123// record one finding
124func ins_add(kind: *i64, ca: *i64, cb2: *i64, sc: *i64, st: *i64, cnt: i64, k: i64, a: i64, b: i64, score: i64, stat: i64) -> i64 {
125 if cnt >= IN_MAXF { return cnt }
126 kind[cnt] = k
127 ca[cnt] = a
128 cb2[cnt] = b
129 sc[cnt] = score
130 st[cnt] = stat
131 return cnt + 1
132}
133
134// ai_mine_scan -- the SCAN half of ai_mine, factored out so a JSON caller (nx_analyze `mine`) and the
135// text emitter can share ONE multiple-comparison scan instead of two that drift. Fills the parallel
136// finding arrays and writes meta[0]=npair meta[1]=surviving-pairs meta[2]=per_test_alpha; returns nf.
137// ⚠ai_mine still inlines its own copy of this logic (behaviour-preserving refactor deferred to avoid a
138// fragile 90-line exact-match edit over a flaky link) -- debt filed to converge them next touch.
139func ai_mine_scan(cols: *i64, names: *i64, ncol: i64, n: i64, kind: *i64, fa: *i64, fb: *i64, fs: *i64, ft: *i64, meta: *i64) -> i64 {
140 let nl_gap: i64 = ins_conf("nonlinear_gap_permil" as *u8, 120)
141 let skew_t: i64 = ins_conf("skew_ratio_permil" as *u8, IN_MAGIC_1400)
142 let idlike: i64 = ins_conf("id_like_pct" as *u8, 90)
143 let w_rel: i64 = ins_conf("w_relationship" as *u8, 1000)
144 let w_nl: i64 = ins_conf("w_nonlinear" as *u8, IN_MAGIC_1200)
145 let w_c: i64 = ins_conf("w_constant" as *u8, 800)
146 let w_id: i64 = ins_conf("w_id_like" as *u8, 400)
147 let w_sk: i64 = ins_conf("w_skew" as *u8, 700)
148 let w_ol: i64 = ins_conf("w_outlier" as *u8, 600)
149 let npair: i64 = ncol * (ncol - 1) / 2
150 var mfam: i64 = npair
151 if mfam < 1 { mfam = 1 }
152 let per_alpha: i64 = ai_alpha_for_m(mfam)
153 var nf: i64 = 0
154 var surv: i64 = 0
155 var i: i64 = 0
156 while i < ncol {
157 var j: i64 = i + 1
158 while j < ncol {
159 let ci: *i64 = cols[i] as *i64
160 let cj: *i64 = cols[j] as *i64
161 let rp: i64 = am_pearson_milli(ci, cj, n)
162 let sp: i64 = ai_spearman_milli(ci, cj, n)
163 var ar: i64 = ins_abs(rp)
164 var as2: i64 = ins_abs(sp)
165 if ar > 1000 { ar = 0 - 1 }
166 if as2 > 1000 { as2 = 0 - 1 }
167 var rel_ok: i64 = 0
168 var sp_ok: i64 = 0
169 if ar >= 0 { rel_ok = ai_r_sig_bonferroni(rp, n, mfam) }
170 if as2 >= 0 { sp_ok = ai_r_sig_bonferroni(sp, n, mfam) }
171 if rel_ok == 1 {
172 surv = surv + 1
173 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_REL, i, j, w_rel * ar / 1000, rp)
174 }
175 if sp_ok == 1 { if as2 - ar >= nl_gap {
176 if rel_ok != 1 { surv = surv + 1 }
177 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_NONLIN, i, j, w_nl * as2 / 1000, sp)
178 } }
179 j = j + 1
180 }
181 i = i + 1
182 }
183 let prof: *i64 = sys_mmap(8 * IN_PROF) as *i64
184 i = 0
185 while i < ncol {
186 let ci: *i64 = cols[i] as *i64
187 ad_profile(ci, n, 10, prof)
188 if prof[AD_MIN] == prof[AD_MAX] {
189 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_CONST, i, 0 - 1, w_c, prof[AD_MIN])
190 } else {
191 let dpct: i64 = ins_distinct(ci, n) * 100 / n
192 if dpct >= idlike {
193 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_IDLIKE, i, 0 - 1, w_id, dpct)
194 }
195 let md: i64 = prof[AD_MEDIAN]
196 if md != 0 {
197 let ratio: i64 = ins_abs(prof[AD_MEAN]) * 1000 / ins_abs(md)
198 var sk: i64 = 0
199 if ratio > skew_t { sk = 1 }
200 if ratio * skew_t < IN_MAGIC_1000000 { sk = 1 }
201 if sk == 1 { nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_SKEW, i, 0 - 1, w_sk, ratio) }
202 }
203 let ol: i64 = prof[AD_OUT_HI] + prof[AD_OUT_LO]
204 if ol > 0 {
205 var cl: i64 = ol
206 if cl > 10 { cl = 10 }
207 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_OUTLIER, i, 0 - 1, w_ol * cl / 10, ol)
208 }
209 }
210 i = i + 1
211 }
212 var a: i64 = 0
213 while a < nf {
214 var best: i64 = a
215 var b: i64 = a + 1
216 while b < nf { if fs[b] > fs[best] { best = b } b = b + 1 }
217 if best != a {
218 let tk: i64 = kind[a]
219 let ta: i64 = fa[a]
220 let tb2: i64 = fb[a]
221 let ts: i64 = fs[a]
222 let tt: i64 = ft[a]
223 kind[a] = kind[best]
224 fa[a] = fa[best]
225 fb[a] = fb[best]
226 fs[a] = fs[best]
227 ft[a] = ft[best]
228 kind[best] = tk
229 fa[best] = ta
230 fb[best] = tb2
231 fs[best] = ts
232 ft[best] = tt
233 }
234 a = a + 1
235 }
236 meta[0] = npair
237 meta[1] = surv
238 meta[2] = per_alpha
239 return nf
240}
241
242// THE FRONT DOOR: mine `cols` (ncol columns of n rows, names parallel) for findings, ranked. No target.
243func ai_mine(cols: *i64, names: *i64, ncol: i64, n: i64, out: *u8, cap: i64) -> i64 {
244 if ncol <= 0 { return 0 }
245 if n <= 0 { return 0 }
246 if cap < IN_MAGIC_4096 { return 0 }
247 let maxf: i64 = ins_conf("max_findings" as *u8, 12)
248 let nl_gap: i64 = ins_conf("nonlinear_gap_permil" as *u8, 120)
249 let skew_t: i64 = ins_conf("skew_ratio_permil" as *u8, IN_MAGIC_1400)
250 let idlike: i64 = ins_conf("id_like_pct" as *u8, 90)
251 let w_rel: i64 = ins_conf("w_relationship" as *u8, 1000)
252 let w_nl: i64 = ins_conf("w_nonlinear" as *u8, IN_MAGIC_1200)
253 let w_c: i64 = ins_conf("w_constant" as *u8, 800)
254 let w_id: i64 = ins_conf("w_id_like" as *u8, 400)
255 let w_sk: i64 = ins_conf("w_skew" as *u8, 700)
256 let w_ol: i64 = ins_conf("w_outlier" as *u8, 600)
257
258 // hypotheses tested = every unordered column pair. This count IS the multiple-comparison family.
259 let npair: i64 = ncol * (ncol - 1) / 2
260 var mfam: i64 = npair
261 if mfam < 1 { mfam = 1 }
262 let per_alpha: i64 = ai_alpha_for_m(mfam)
263
264 let kind: *i64 = sys_mmap(8 * IN_MAXF) as *i64
265 let fa: *i64 = sys_mmap(8 * IN_MAXF) as *i64
266 let fb: *i64 = sys_mmap(8 * IN_MAXF) as *i64
267 let fs: *i64 = sys_mmap(8 * IN_MAXF) as *i64
268 let ft: *i64 = sys_mmap(8 * IN_MAXF) as *i64
269 var nf: i64 = 0
270 var surv: i64 = 0
271
272 // ---- pairwise scan --------------------------------------------------------------------------
273 var i: i64 = 0
274 while i < ncol {
275 var j: i64 = i + 1
276 while j < ncol {
277 let ci: *i64 = cols[i] as *i64
278 let cj: *i64 = cols[j] as *i64
279 let rp: i64 = am_pearson_milli(ci, cj, n)
280 let sp: i64 = ai_spearman_milli(ci, cj, n)
281 var ar: i64 = ins_abs(rp)
282 var as2: i64 = ins_abs(sp)
283 // a correlation against a degenerate column is UNDEFINED, not strong: drop the pair rather
284 // than let a sentinel magnitude outrank real structure
285 if ar > 1000 { ar = 0 - 1 }
286 if as2 > 1000 { as2 = 0 - 1 }
287 // family-wise bar over ALL pairs tested, not per-test .05
288 var rel_ok: i64 = 0
289 var sp_ok: i64 = 0
290 if ar >= 0 { rel_ok = ai_r_sig_bonferroni(rp, n, mfam) }
291 if as2 >= 0 { sp_ok = ai_r_sig_bonferroni(sp, n, mfam) }
292 if rel_ok == 1 {
293 surv = surv + 1
294 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_REL, i, j, w_rel * ar / 1000, rp)
295 }
296 // monotone but NOT linear: ranks agree far better than raw values do
297 if sp_ok == 1 { if as2 - ar >= nl_gap {
298 if rel_ok != 1 { surv = surv + 1 }
299 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_NONLIN, i, j, w_nl * as2 / 1000, sp)
300 } }
301 j = j + 1
302 }
303 i = i + 1
304 }
305
306 // ---- per-column shape findings ----------------------------------------------------------------
307 let prof: *i64 = sys_mmap(8 * IN_PROF) as *i64
308 i = 0
309 while i < ncol {
310 let ci: *i64 = cols[i] as *i64
311 ad_profile(ci, n, 10, prof)
312 if prof[AD_MIN] == prof[AD_MAX] {
313 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_CONST, i, 0 - 1, w_c, prof[AD_MIN])
314 } else {
315 let dpct: i64 = ins_distinct(ci, n) * 100 / n
316 if dpct >= idlike {
317 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_IDLIKE, i, 0 - 1, w_id, dpct)
318 }
319 let md: i64 = prof[AD_MEDIAN]
320 if md != 0 {
321 let ratio: i64 = ins_abs(prof[AD_MEAN]) * 1000 / ins_abs(md)
322 var sk: i64 = 0
323 if ratio > skew_t { sk = 1 }
324 if ratio * skew_t < IN_MAGIC_1000000 { sk = 1 }
325 if sk == 1 { nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_SKEW, i, 0 - 1, w_sk, ratio) }
326 }
327 let ol: i64 = prof[AD_OUT_HI] + prof[AD_OUT_LO]
328 if ol > 0 {
329 var cl: i64 = ol
330 if cl > 10 { cl = 10 }
331 nf = ins_add(kind, fa, fb, fs, ft, nf, IN_K_OUTLIER, i, 0 - 1, w_ol * cl / 10, ol)
332 }
333 }
334 i = i + 1
335 }
336
337 // ---- rank by score (selection sort; nf is small by construction) ------------------------------
338 var a: i64 = 0
339 while a < nf {
340 var best: i64 = a
341 var b: i64 = a + 1
342 while b < nf { if fs[b] > fs[best] { best = b } b = b + 1 }
343 if best != a {
344 let tk: i64 = kind[a]
345 let ta: i64 = fa[a]
346 let tb2: i64 = fb[a]
347 let ts: i64 = fs[a]
348 let tt: i64 = ft[a]
349 kind[a] = kind[best]
350 fa[a] = fa[best]
351 fb[a] = fb[best]
352 fs[a] = fs[best]
353 ft[a] = ft[best]
354 kind[best] = tk
355 fa[best] = ta
356 fb[best] = tb2
357 fs[best] = ts
358 ft[best] = tt
359 }
360 a = a + 1
361 }
362
363 // ---- emit ------------------------------------------------------------------------------------
364 var o: i64 = 0
365 o = ins_cat(out, o, "=== INSIGHT SCAN (no target column) ===\nrows=" as *u8)
366 o = ins_catn(out, o, n)
367 o = ins_cat(out, o, " columns=" as *u8)
368 o = ins_catn(out, o, ncol)
369 o = ins_cat(out, o, "\nHYPOTHESES TESTED=" as *u8)
370 o = ins_catn(out, o, npair)
371 o = ins_cat(out, o, " (every column pair). Scanning all pairs and reporting the best is p-hacking unless the bar accounts for it, so significance here is FAMILY-WISE .05 via Bonferroni" as *u8)
372 if per_alpha > 0 {
373 o = ins_cat(out, o, " -- per-test alpha " as *u8)
374 o = ins_catn(out, o, per_alpha)
375 o = ins_cat(out, o, "/1000.\n" as *u8)
376 } else {
377 o = ins_cat(out, o, " -- but " as *u8)
378 o = ins_catn(out, o, npair)
379 o = ins_cat(out, o, " pairs need a level STRICTER THAN THE CRITICAL TABLE HOLDS, so NO pairwise finding can be certified at this width. Reduce the column count or raise n.\n" as *u8)
380 }
381 o = ins_cat(out, o, "PAIRS SURVIVING CORRECTION=" as *u8)
382 o = ins_catn(out, o, surv)
383 o = ins_cat(out, o, " of " as *u8)
384 o = ins_catn(out, o, npair)
385 o = ins_cat(out, o, "\n\n-- FINDINGS (ranked) --\n" as *u8)
386
387 var shown: i64 = 0
388 var f: i64 = 0
389 let cls: *u8 = sys_mmap(128)
390 while f < nf {
391 if shown < maxf { if o + 512 < cap {
392 shown = shown + 1
393 o = ins_cat(out, o, " " as *u8)
394 o = ins_catn(out, o, shown)
395 o = ins_cat(out, o, ". " as *u8)
396 let na: *u8 = names[fa[f]] as *u8
397 if kind[f] == IN_K_REL {
398 am_class(0 as *u8, ft[f], cls)
399 o = ins_cat(out, o, "[RELATIONSHIP] " as *u8)
400 o = ins_cat(out, o, na)
401 o = ins_cat(out, o, " <-> " as *u8)
402 o = ins_cat(out, o, names[fb[f]] as *u8)
403 o = ins_cat(out, o, ": r=" as *u8)
404 o = ins_catn(out, o, ft[f])
405 o = ins_cat(out, o, "/1000 (" as *u8)
406 o = ins_cat(out, o, cls)
407 o = ins_cat(out, o, "), clears the family-wise bar" as *u8)
408 }
409 if kind[f] == IN_K_NONLIN {
410 o = ins_cat(out, o, "[NONLINEAR-MONOTONE] " as *u8)
411 o = ins_cat(out, o, na)
412 o = ins_cat(out, o, " <-> " as *u8)
413 o = ins_cat(out, o, names[fb[f]] as *u8)
414 o = ins_cat(out, o, ": Spearman=" as *u8)
415 o = ins_catn(out, o, ft[f])
416 o = ins_cat(out, o, "/1000 -- they move together consistently but NOT in a straight line, which a correlation-only scan under-reports" as *u8)
417 }
418 if kind[f] == IN_K_CONST {
419 o = ins_cat(out, o, "[CONSTANT] " as *u8)
420 o = ins_cat(out, o, na)
421 o = ins_cat(out, o, " never varies (value " as *u8)
422 o = ins_catn(out, o, ft[f])
423 o = ins_cat(out, o, ") -- it can explain nothing" as *u8)
424 }
425 if kind[f] == IN_K_IDLIKE {
426 o = ins_cat(out, o, "[ID-LIKE] " as *u8)
427 o = ins_cat(out, o, na)
428 o = ins_cat(out, o, " is " as *u8)
429 o = ins_catn(out, o, ft[f])
430 o = ins_cat(out, o, "pct unique -- likely a key, not a measurement; correlations with it are usually artefacts" as *u8)
431 }
432 if kind[f] == IN_K_SKEW {
433 o = ins_cat(out, o, "[SKEWED] " as *u8)
434 o = ins_cat(out, o, na)
435 o = ins_cat(out, o, ": mean/median=" as *u8)
436 o = ins_catn(out, o, ft[f])
437 o = ins_cat(out, o, "/1000 -- the average is not a typical value here; prefer the median" as *u8)
438 }
439 if kind[f] == IN_K_OUTLIER {
440 o = ins_cat(out, o, "[OUTLIERS] " as *u8)
441 o = ins_cat(out, o, na)
442 o = ins_cat(out, o, " has " as *u8)
443 o = ins_catn(out, o, ft[f])
444 o = ins_cat(out, o, " value(s) beyond 3 sigma -- check them before trusting any mean" as *u8)
445 }
446 o = ins_cat(out, o, "\n" as *u8)
447 } }
448 f = f + 1
449 }
450 if shown == 0 { o = ins_cat(out, o, " (none -- nothing in this dataset clears the bar once multiplicity is accounted for)\n" as *u8) }
451 if nf > shown {
452 o = ins_cat(out, o, "\n-- " as *u8)
453 o = ins_catn(out, o, nf - shown)
454 o = ins_cat(out, o, " further finding(s) not shown (max_findings=" as *u8)
455 o = ins_catn(out, o, maxf)
456 o = ins_cat(out, o, "); the cap is declared rather than silently truncating.\n" as *u8)
457 }
458 out[o] = 0 as u8
459 return o
460}