nx_analyst_data.nx source

1// nx_analyst_data.nx -- LIB: the GENERAL DATA ANALYST the operator said "isn't there" (2026-07-10). The 2// existing nx_analyst is a forensic COST tracer for one vertical; this one points at an ARBITRARY i64 column 3// and PROFILES it (count / distinct / min / max / mean / median / stddev / p95 / p99 / outliers) then EXPLAINS 4// it in words (cardinality class, skew, outliers) -- the "profile + find patterns + explain" front door. 5// Composes the MEASURED primitives: nx_dataframe (exact aggregation) + nx_dataframe_approx (HLL distinct at 6// scale). Data-driven anomaly flags, no hardcoded thresholds beyond the named statistical constants. 7// license_tier: ORIGINAL 8import "nx_syscalls.nx" 9import "_hdl_build/nx_dataframe.nx" 10import "_hdl_build/nx_dataframe_approx.nx" 11 12// profile slots (out_prof must hold >=12 i64): 13const AD_COUNT: i64 = 0 14const AD_DISTINCT: i64 = 1 15const AD_MIN: i64 = 2 16const AD_MAX: i64 = 3 17const AD_MEAN: i64 = 4 18const AD_MEDIAN: i64 = 5 19const AD_STDDEV: i64 = 6 20const AD_P95: i64 = 7 21const AD_P99: i64 = 8 22const AD_OUT_HI: i64 = 9 23const AD_OUT_LO: i64 = 10 24const AD_DISTINCT_PCT: i64 = 11 25 26// compute the full profile of col[0..n) into out_prof. lg_k for the HLL distinct estimate (10 = impl max). 27func ad_profile(col: *i64, n: i64, lg_k: i64, out_prof: *i64) -> i64 { 28 if n <= 0 { var z: i64 = 0; while z < 12 { out_prof[z] = 0; z = z + 1 } return 0 } 29 out_prof[AD_COUNT] = n 30 out_prof[AD_DISTINCT] = dfa_distinct(col, n, lg_k) 31 out_prof[AD_MIN] = df_min(col, n) 32 out_prof[AD_MAX] = df_max(col, n) 33 let rem: *i64 = sys_mmap(8) as *i64 34 out_prof[AD_MEAN] = df_mean(col, n, rem) 35 out_prof[AD_MEDIAN] = df_quantile(col, n, 500) 36 let vm: i64 = df_var_milli(col, n) 37 out_prof[AD_STDDEV] = df_isqrt(vm / 1000) 38 out_prof[AD_P95] = df_quantile(col, n, 950) 39 out_prof[AD_P99] = df_quantile(col, n, 990) 40 // outliers beyond 3 sigma (data-driven: mean +/- 3*stddev) 41 let mean: i64 = out_prof[AD_MEAN] 42 let sd: i64 = out_prof[AD_STDDEV] 43 let hi_thr: i64 = mean + 3 * sd 44 let lo_thr: i64 = mean - 3 * sd 45 out_prof[AD_OUT_HI] = df_filter_count(col, n, 3, hi_thr) // > hi 46 out_prof[AD_OUT_LO] = df_filter_count(col, n, 1, lo_thr) // < lo 47 var dp: i64 = 0 48 if out_prof[AD_DISTINCT] > 0 { dp = (out_prof[AD_DISTINCT] * 100) / n } 49 out_prof[AD_DISTINCT_PCT] = dp 50 return 0 51} 52 53func adx_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i } 54 55func ad_catn(dst: *u8, off: i64, v: i64) -> i64 { 56 var o: i64 = off 57 var m: i64 = v 58 if m < 0 { dst[o] = 45 as u8; o = o + 1; m = 0 - m } 59 let t: *u8 = sys_mmap(24) 60 var k: i64 = 0 61 if m == 0 { t[0] = 48 as u8; k = 1 } 62 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 63 var i: i64 = k - 1 64 while i >= 0 { dst[o] = t[i]; o = o + 1; i = i - 1 } 65 return o 66} 67 68// render a human-readable profile + explanation from a computed profile. Returns bytes written. 69func ad_report(name: *u8, out_prof: *i64, out: *u8, cap: i64) -> i64 { 70 if cap < 1024 { return 0 } 71 var o: i64 = 0 72 o = adx_cat(out, o, "DATA PROFILE: " as *u8) 73 o = adx_cat(out, o, name) 74 o = adx_cat(out, o, "\n rows=" as *u8) 75 o = ad_catn(out, o, out_prof[AD_COUNT]) 76 o = adx_cat(out, o, " distinct~" as *u8) 77 o = ad_catn(out, o, out_prof[AD_DISTINCT]) 78 o = adx_cat(out, o, " (" as *u8) 79 o = ad_catn(out, o, out_prof[AD_DISTINCT_PCT]) 80 o = adx_cat(out, o, "% unique)\n min=" as *u8) 81 o = ad_catn(out, o, out_prof[AD_MIN]) 82 o = adx_cat(out, o, " max=" as *u8) 83 o = ad_catn(out, o, out_prof[AD_MAX]) 84 o = adx_cat(out, o, " mean=" as *u8) 85 o = ad_catn(out, o, out_prof[AD_MEAN]) 86 o = adx_cat(out, o, " median=" as *u8) 87 o = ad_catn(out, o, out_prof[AD_MEDIAN]) 88 o = adx_cat(out, o, " stddev=" as *u8) 89 o = ad_catn(out, o, out_prof[AD_STDDEV]) 90 o = adx_cat(out, o, "\n p95=" as *u8) 91 o = ad_catn(out, o, out_prof[AD_P95]) 92 o = adx_cat(out, o, " p99=" as *u8) 93 o = ad_catn(out, o, out_prof[AD_P99]) 94 o = adx_cat(out, o, " outliers(>3sd)=" as *u8) 95 o = ad_catn(out, o, out_prof[AD_OUT_HI] + out_prof[AD_OUT_LO]) 96 o = adx_cat(out, o, "\n READ: " as *u8) 97 98 // ---- data-driven explanation ---- 99 let d: i64 = out_prof[AD_DISTINCT] 100 let dp: i64 = out_prof[AD_DISTINCT_PCT] 101 let mean: i64 = out_prof[AD_MEAN] 102 let median: i64 = out_prof[AD_MEDIAN] 103 var wrote: i64 = 0 104 if d <= 1 { o = adx_cat(out, o,"CONSTANT column (single value). " as *u8); wrote = 1 } 105 if wrote == 0 { if d < 20 { o = adx_cat(out, o,"low-cardinality / categorical (" as *u8); o = ad_catn(out, o, d); o = adx_cat(out, o," classes). " as *u8); wrote = 1 } } 106 if wrote == 0 { if dp >= 90 { o = adx_cat(out, o,"near-unique -- likely an id/key column. " as *u8); wrote = 1 } } 107 if wrote == 0 { o = adx_cat(out, o,"continuous numeric. " as *u8) } 108 // skew: compare mean vs median (>20% gap) 109 if median > 0 { 110 if mean * 10 > median * 12 { o = adx_cat(out, o,"RIGHT-SKEWED (mean >> median -- a high tail pulls the average up). " as *u8) } 111 if median * 10 > mean * 12 { o = adx_cat(out, o,"LEFT-SKEWED (median >> mean). " as *u8) } 112 } 113 let outl: i64 = out_prof[AD_OUT_HI] + out_prof[AD_OUT_LO] 114 if outl > 0 { o = adx_cat(out, o,"Has " as *u8); o = ad_catn(out, o, outl); o = adx_cat(out, o," outliers beyond 3 sigma -- inspect. " as *u8) } 115 if outl == 0 { o = adx_cat(out, o,"No 3-sigma outliers. " as *u8) } 116 o = adx_cat(out, o, "\n" as *u8) 117 if o + 2 >= cap { return o } 118 out[o] = 0 as u8 119 return o 120}