nx_analyst_multi.nx source

1// nx_analyst_multi.nx -- LIB: MULTI-COLUMN relationship analysis -- the "find patterns" half of the general 2// data analyst (rung 3). Integer Pearson correlation between two columns + a data-driven strength class + 3// a KEY-DRIVER finder (which of N candidate columns most correlates with a target). Composes nx_dataframe. 4// OVERFLOW-AWARE: uses CENTERED sums (subtract the integer mean first) so intermediates stay ~n*V^2 rather 5// than n^2*V^2 -- safe for moderate n*V (documented bound); i128/normalized scale = the follow-on for very 6// large streams. license_tier: ORIGINAL 7import "nx_syscalls.nx" 8import "_hdl_build/nx_dataframe.nx" 9 10const AM_R_DEGENERATE: i64 = 0 - 2000 // sentinel: a column has zero variance -> correlation undefined 11 12// Pearson correlation * 1000 (range -1000..1000). Centered on integer means. Returns AM_R_DEGENERATE if 13// either column is constant (zero variance). n must be > 1. 14func am_pearson_milli(x: *i64, y: *i64, n: i64) -> i64 { 15 if n < 2 { return AM_R_DEGENERATE } 16 let rem: *i64 = sys_mmap(8) as *i64 17 let mx: i64 = df_mean(x, n, rem) 18 let my: i64 = df_mean(y, n, rem) 19 var sxy: i64 = 0 20 var sxx: i64 = 0 21 var syy: i64 = 0 22 var i: i64 = 0 23 while i < n { 24 let dx: i64 = x[i] - mx 25 let dy: i64 = y[i] - my 26 sxy = sxy + dx * dy 27 sxx = sxx + dx * dx 28 syy = syy + dy * dy 29 i = i + 1 30 } 31 if sxx <= 0 { return AM_R_DEGENERATE } 32 if syy <= 0 { return AM_R_DEGENERATE } 33 // r = sxy / sqrt(sxx*syy). r_milli = sxy*1000 / isqrt(sxx*syy). 34 let denom: i64 = df_isqrt(sxx * syy) 35 if denom <= 0 { return AM_R_DEGENERATE } 36 var r: i64 = (sxy * 1000) / denom 37 if r > 1000 { r = 1000 } 38 if r < 0 - 1000 { r = 0 - 1000 } 39 return r 40} 41 42// covariance (centered, /n) -- the unnormalized relationship, for completeness. 43func am_covariance(x: *i64, y: *i64, n: i64) -> i64 { 44 if n < 1 { return 0 } 45 let rem: *i64 = sys_mmap(8) as *i64 46 let mx: i64 = df_mean(x, n, rem) 47 let my: i64 = df_mean(y, n, rem) 48 var s: i64 = 0 49 var i: i64 = 0 50 while i < n { s = s + (x[i] - mx) * (y[i] - my); i = i + 1 } 51 return s / n 52} 53 54// classify a correlation coefficient (r_milli) into words. |r| thresholds are the conventional social-science 55// bands (data-driven, named): >=800 very strong, >=500 strong, >=300 moderate, >=100 weak, else negligible. 56func am_class(r_milli: *u8, r: i64, out: *u8) -> i64 { 57 if r == AM_R_DEGENERATE { 58 var k: i64 = 0 59 let s: *u8 = "undefined (constant column)" as *u8 60 while s[k] != (0 as u8) { out[k] = s[k]; k = k + 1 } 61 out[k] = 0 as u8 62 return k 63 } 64 var a: i64 = r 65 if a < 0 { a = 0 - a } 66 var o: i64 = 0 67 if r < 0 { let neg: *u8 = "negative " as *u8; var j: i64 = 0; while neg[j] != (0 as u8) { out[o] = neg[j]; o = o + 1; j = j + 1 } } 68 if r > 0 { let pos: *u8 = "positive " as *u8; var j2: i64 = 0; while pos[j2] != (0 as u8) { out[o] = pos[j2]; o = o + 1; j2 = j2 + 1 } } 69 var band: *u8 = "negligible" as *u8 70 if a >= 100 { band = "weak" as *u8 } 71 if a >= 300 { band = "moderate" as *u8 } 72 if a >= 500 { band = "strong" as *u8 } 73 if a >= 800 { band = "very strong" as *u8 } 74 var b: i64 = 0 75 while band[b] != (0 as u8) { out[o] = band[b]; o = o + 1; b = b + 1 } 76 out[o] = 0 as u8 77 return o 78} 79 80// KEY-DRIVER: of `ncol` candidate columns (cols = array of column pointers), which most correlates (by |r|) 81// with `target`? Writes the winning column index to out_idx and its r_milli to out_r. Returns 1 if found. 82// NOTE the inline-cast-index gotcha: bind cols[c] to a local BEFORE indexing. 83func am_key_driver(target: *i64, cols: *i64, ncol: i64, n: i64, out_idx: *i64, out_r: *i64) -> i64 { 84 var best: i64 = 0 - 1 85 var best_abs: i64 = 0 - 1 86 var best_r: i64 = 0 87 var c: i64 = 0 88 while c < ncol { 89 let colc: *i64 = cols[c] as *i64 90 let r: i64 = am_pearson_milli(target, colc, n) 91 if r != AM_R_DEGENERATE { 92 var a: i64 = r 93 if a < 0 { a = 0 - a } 94 if a > best_abs { best_abs = a; best = c; best_r = r } 95 } 96 c = c + 1 97 } 98 if best < 0 { return 0 } 99 out_idx[0] = best 100 out_r[0] = best_r 101 return 1 102}