code wiki / _hdl_build / nx_dataframe.nx
nx_dataframe.nx source
↩ module page · 180 lines · 6847 B
1// nx_dataframe.nx -- LIB: the AGGREGATION FRONT DOOR the analytics stack was missing (operator 2026-07-10:
2// "our data engineer and analytics is toy level"). The audit found real PRIMITIVES (sketches, columnar,
3// seg_store) but NO numeric aggregation, NO GROUP BY, NO value predicate scan -- you hand-rolled everything.
4// This is the first rung: a general INTEGER-EXACT columnar aggregation engine over an i64 column (or two
5// parallel key/value columns). 100%-integer -> BIT-REPRODUCIBLE aggregates (the sovereign determinism exceed;
6// pandas/DuckDB float sums are order-dependent, these are not). Seeds: aggregation-frontend, descriptive-
7// stats-general, value-range-scan, and GROUP BY (dataframe-ops). Approximate distinct/quantile at scale =
8// the next rung (wire nx_sketch_hll / nx_sketch_tdigest behind this same front door). license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_vecmath.nx"
11
12// ---- reductions over a column col[0..n) ----
13func df_count(col: *i64, n: i64) -> i64 { return n }
14
15func df_sum(col: *i64, n: i64) -> i64 {
16 var s: i64 = 0
17 var i: i64 = 0
18 while i < n { s = s + col[i]; i = i + 1 }
19 return s
20}
21
22func df_min(col: *i64, n: i64) -> i64 {
23 if n <= 0 { return 0 }
24 var m: i64 = col[0]
25 var i: i64 = 1
26 while i < n { if col[i] < m { m = col[i] } i = i + 1 }
27 return m
28}
29
30func df_max(col: *i64, n: i64) -> i64 {
31 if n <= 0 { return 0 }
32 var m: i64 = col[0]
33 var i: i64 = 1
34 while i < n { if col[i] > m { m = col[i] } i = i + 1 }
35 return m
36}
37
38// integer mean (floor toward zero for the quotient); remainder via out_rem if wanted (pass 0 to skip).
39func df_mean(col: *i64, n: i64, out_rem: *i64) -> i64 {
40 if n <= 0 { if (out_rem as i64) != 0 { out_rem[0] = 0 } return 0 }
41 let s: i64 = df_sum(col, n)
42 if (out_rem as i64) != 0 { out_rem[0] = s - (s / n) * n }
43 return s / n
44}
45
46// mean scaled by 1000 (milli-units) -> fractional mean without float. mean_milli/1000 = mean.
47func df_mean_milli(col: *i64, n: i64) -> i64 {
48 if n <= 0 { return 0 }
49 let s: i64 = df_sum(col, n)
50 return (s * 1000) / n
51}
52
53// population variance in MILLI-units (var*1000, integer): E[x^2] - E[x]^2, scaled. Guards overflow only up to
54// the caller's data range (i64). var_milli/1000 = variance.
55func df_var_milli(col: *i64, n: i64) -> i64 {
56 if n <= 0 { return 0 }
57 var sq: i64 = 0
58 var i: i64 = 0
59 while i < n { sq = sq + col[i] * col[i]; i = i + 1 }
60 let s: i64 = df_sum(col, n)
61 let ex2: i64 = (sq * 1000) / n // E[x^2] * 1000
62 let ex: i64 = s / n // integer E[x]
63 let exsq: i64 = ex * ex * 1000 // (floor E[x])^2 * 1000
64 let v: i64 = ex2 - exsq
65 if v < 0 { return 0 }
66 return v
67}
68
69// integer sqrt (Newton) -- for stddev = isqrt(var). Operates on plain integer input.
70func df_isqrt(v: i64) -> i64 { return vm_isqrt(v) }
71
72// ---- predicate scan (value-range-scan seed): count values matching op vs threshold ----
73// op: 0 == , 1 < , 2 <= , 3 > , 4 >=
74func df_filter_count(col: *i64, n: i64, op: i64, thr: i64) -> i64 {
75 var c: i64 = 0
76 var i: i64 = 0
77 while i < n {
78 let v: i64 = col[i]
79 var hit: i64 = 0
80 if op == 0 { if v == thr { hit = 1 } }
81 if op == 1 { if v < thr { hit = 1 } }
82 if op == 2 { if v <= thr { hit = 1 } }
83 if op == 3 { if v > thr { hit = 1 } }
84 if op == 4 { if v >= thr { hit = 1 } }
85 if hit == 1 { c = c + 1 }
86 i = i + 1
87 }
88 return c
89}
90
91// sum of values matching a predicate (filter -> sum, the WHERE ... SUM seed).
92func df_filter_sum(col: *i64, n: i64, op: i64, thr: i64) -> i64 {
93 var s: i64 = 0
94 var i: i64 = 0
95 while i < n {
96 let v: i64 = col[i]
97 var hit: i64 = 0
98 if op == 0 { if v == thr { hit = 1 } }
99 if op == 1 { if v < thr { hit = 1 } }
100 if op == 2 { if v <= thr { hit = 1 } }
101 if op == 3 { if v > thr { hit = 1 } }
102 if op == 4 { if v >= thr { hit = 1 } }
103 if hit == 1 { s = s + v }
104 i = i + 1
105 }
106 return s
107}
108
109// ---- exact median / quantile via in-place-ish sort of a COPY (small-N exact; sketches for scale) ----
110// q_permil in [0,1000]: 500 = median. Uses a copy so the caller's column is untouched. Nearest-rank.
111func df_quantile(col: *i64, n: i64, q_permil: i64) -> i64 {
112 if n <= 0 { return 0 }
113 let cp: *i64 = sys_mmap(n * 8) as *i64
114 var i: i64 = 0
115 while i < n { cp[i] = col[i]; i = i + 1 }
116 // insertion sort (exact, stable enough for quantiles; N expected modest at this rung)
117 var a: i64 = 1
118 while a < n {
119 let key: i64 = cp[a]
120 var b: i64 = a - 1
121 var placing: i64 = 1
122 while placing == 1 {
123 if b >= 0 { if cp[b] > key { cp[b + 1] = cp[b]; b = b - 1 } else { placing = 0 } } else { placing = 0 }
124 }
125 cp[b + 1] = key
126 a = a + 1
127 }
128 var rank: i64 = (q_permil * (n - 1)) / 1000
129 if rank < 0 { rank = 0 }
130 if rank >= n { rank = n - 1 }
131 return cp[rank]
132}
133
134// ---- GROUP BY (dataframe-ops seed): parallel key[]/val[] columns -> per-distinct-key aggregate ----
135// Writes distinct keys into out_keys and their aggregate into out_agg; returns the group count.
136// agg_kind: 0 = count, 1 = sum. O(n * groups) -- fine at this rung; hash groupby is the scale follow-on.
137func df_groupby(keys: *i64, vals: *i64, n: i64, agg_kind: i64, out_keys: *i64, out_agg: *i64, max_groups: i64) -> i64 {
138 var g: i64 = 0
139 var i: i64 = 0
140 while i < n {
141 let k: i64 = keys[i]
142 // find existing group
143 var found: i64 = 0 - 1
144 var j: i64 = 0
145 while j < g { if out_keys[j] == k { found = j; j = g } else { j = j + 1 } }
146 if found < 0 {
147 if g < max_groups {
148 out_keys[g] = k
149 if agg_kind == 0 { out_agg[g] = 1 } else { out_agg[g] = vals[i] }
150 g = g + 1
151 }
152 } else {
153 if agg_kind == 0 { out_agg[found] = out_agg[found] + 1 } else { out_agg[found] = out_agg[found] + vals[i] }
154 }
155 i = i + 1
156 }
157 return g
158}
159
160// ---- INNER JOIN (relational-join seed): match left keys to right keys, emit paired value rows ----
161// For each left row, find the FIRST matching right key; emit (lval,rval) into out. Returns matched-row count.
162func df_inner_join(lkeys: *i64, lvals: *i64, ln: i64, rkeys: *i64, rvals: *i64, rn: i64, out_l: *i64, out_r: *i64, max_out: i64) -> i64 {
163 var o: i64 = 0
164 var i: i64 = 0
165 while i < ln {
166 var j: i64 = 0
167 var done: i64 = 0
168 while j < rn {
169 if done == 0 {
170 if rkeys[j] == lkeys[i] {
171 if o < max_out { out_l[o] = lvals[i]; out_r[o] = rvals[j]; o = o + 1 }
172 done = 1
173 }
174 }
175 j = j + 1
176 }
177 i = i + 1
178 }
179 return o
180}