code wiki / _hdl_build / nx_dataframe.nx
nx_dataframe.nx source
↩ module page · 185 lines · 6940 B
1// nx_dataframe.nx -- LIB: the AGGREGATION FRONT DOOR the analytics stack was missing (operator 2026-07-10:
2// "our data engineer and analytics is toy level"). The audit found real PRIMITIVES (sketches, columnar,
3// seg_store) but NO numeric aggregation, NO GROUP BY, NO value predicate scan -- you hand-rolled everything.
4// This is the first rung: a general INTEGER-EXACT columnar aggregation engine over an i64 column (or two
5// parallel key/value columns). 100%-integer -> BIT-REPRODUCIBLE aggregates (the sovereign determinism exceed;
6// pandas/DuckDB float sums are order-dependent, these are not). Seeds: aggregation-frontend, descriptive-
7// stats-general, value-range-scan, and GROUP BY (dataframe-ops). Approximate distinct/quantile at scale =
8// the next rung (wire nx_sketch_hll / nx_sketch_tdigest behind this same front door). license_tier: ORIGINAL
9import "nx_syscalls.nx"
10
11// ---- reductions over a column col[0..n) ----
12func df_count(col: *i64, n: i64) -> i64 { return n }
13
14func df_sum(col: *i64, n: i64) -> i64 {
15 var s: i64 = 0
16 var i: i64 = 0
17 while i < n { s = s + col[i]; i = i + 1 }
18 return s
19}
20
21func df_min(col: *i64, n: i64) -> i64 {
22 if n <= 0 { return 0 }
23 var m: i64 = col[0]
24 var i: i64 = 1
25 while i < n { if col[i] < m { m = col[i] } i = i + 1 }
26 return m
27}
28
29func df_max(col: *i64, n: i64) -> i64 {
30 if n <= 0 { return 0 }
31 var m: i64 = col[0]
32 var i: i64 = 1
33 while i < n { if col[i] > m { m = col[i] } i = i + 1 }
34 return m
35}
36
37// integer mean (floor toward zero for the quotient); remainder via out_rem if wanted (pass 0 to skip).
38func df_mean(col: *i64, n: i64, out_rem: *i64) -> i64 {
39 if n <= 0 { if (out_rem as i64) != 0 { out_rem[0] = 0 } return 0 }
40 let s: i64 = df_sum(col, n)
41 if (out_rem as i64) != 0 { out_rem[0] = s - (s / n) * n }
42 return s / n
43}
44
45// mean scaled by 1000 (milli-units) -> fractional mean without float. mean_milli/1000 = mean.
46func df_mean_milli(col: *i64, n: i64) -> i64 {
47 if n <= 0 { return 0 }
48 let s: i64 = df_sum(col, n)
49 return (s * 1000) / n
50}
51
52// population variance in MILLI-units (var*1000, integer): E[x^2] - E[x]^2, scaled. Guards overflow only up to
53// the caller's data range (i64). var_milli/1000 = variance.
54func df_var_milli(col: *i64, n: i64) -> i64 {
55 if n <= 0 { return 0 }
56 var sq: i64 = 0
57 var i: i64 = 0
58 while i < n { sq = sq + col[i] * col[i]; i = i + 1 }
59 let s: i64 = df_sum(col, n)
60 let ex2: i64 = (sq * 1000) / n // E[x^2] * 1000
61 let ex: i64 = s / n // integer E[x]
62 let exsq: i64 = ex * ex * 1000 // (floor E[x])^2 * 1000
63 let v: i64 = ex2 - exsq
64 if v < 0 { return 0 }
65 return v
66}
67
68// integer sqrt (Newton) -- for stddev = isqrt(var). Operates on plain integer input.
69func df_isqrt(v: i64) -> i64 {
70 if v <= 0 { return 0 }
71 var x: i64 = v
72 var y: i64 = (x + 1) / 2
73 while y < x { x = y; y = (x + v / x) / 2 }
74 return x
75}
76
77// ---- predicate scan (value-range-scan seed): count values matching op vs threshold ----
78// op: 0 == , 1 < , 2 <= , 3 > , 4 >=
79func df_filter_count(col: *i64, n: i64, op: i64, thr: i64) -> i64 {
80 var c: i64 = 0
81 var i: i64 = 0
82 while i < n {
83 let v: i64 = col[i]
84 var hit: i64 = 0
85 if op == 0 { if v == thr { hit = 1 } }
86 if op == 1 { if v < thr { hit = 1 } }
87 if op == 2 { if v <= thr { hit = 1 } }
88 if op == 3 { if v > thr { hit = 1 } }
89 if op == 4 { if v >= thr { hit = 1 } }
90 if hit == 1 { c = c + 1 }
91 i = i + 1
92 }
93 return c
94}
95
96// sum of values matching a predicate (filter -> sum, the WHERE ... SUM seed).
97func df_filter_sum(col: *i64, n: i64, op: i64, thr: i64) -> i64 {
98 var s: i64 = 0
99 var i: i64 = 0
100 while i < n {
101 let v: i64 = col[i]
102 var hit: i64 = 0
103 if op == 0 { if v == thr { hit = 1 } }
104 if op == 1 { if v < thr { hit = 1 } }
105 if op == 2 { if v <= thr { hit = 1 } }
106 if op == 3 { if v > thr { hit = 1 } }
107 if op == 4 { if v >= thr { hit = 1 } }
108 if hit == 1 { s = s + v }
109 i = i + 1
110 }
111 return s
112}
113
114// ---- exact median / quantile via in-place-ish sort of a COPY (small-N exact; sketches for scale) ----
115// q_permil in [0,1000]: 500 = median. Uses a copy so the caller's column is untouched. Nearest-rank.
116func df_quantile(col: *i64, n: i64, q_permil: i64) -> i64 {
117 if n <= 0 { return 0 }
118 let cp: *i64 = sys_mmap(n * 8) as *i64
119 var i: i64 = 0
120 while i < n { cp[i] = col[i]; i = i + 1 }
121 // insertion sort (exact, stable enough for quantiles; N expected modest at this rung)
122 var a: i64 = 1
123 while a < n {
124 let key: i64 = cp[a]
125 var b: i64 = a - 1
126 var placing: i64 = 1
127 while placing == 1 {
128 if b >= 0 { if cp[b] > key { cp[b + 1] = cp[b]; b = b - 1 } else { placing = 0 } } else { placing = 0 }
129 }
130 cp[b + 1] = key
131 a = a + 1
132 }
133 var rank: i64 = (q_permil * (n - 1)) / 1000
134 if rank < 0 { rank = 0 }
135 if rank >= n { rank = n - 1 }
136 return cp[rank]
137}
138
139// ---- GROUP BY (dataframe-ops seed): parallel key[]/val[] columns -> per-distinct-key aggregate ----
140// Writes distinct keys into out_keys and their aggregate into out_agg; returns the group count.
141// agg_kind: 0 = count, 1 = sum. O(n * groups) -- fine at this rung; hash groupby is the scale follow-on.
142func df_groupby(keys: *i64, vals: *i64, n: i64, agg_kind: i64, out_keys: *i64, out_agg: *i64, max_groups: i64) -> i64 {
143 var g: i64 = 0
144 var i: i64 = 0
145 while i < n {
146 let k: i64 = keys[i]
147 // find existing group
148 var found: i64 = 0 - 1
149 var j: i64 = 0
150 while j < g { if out_keys[j] == k { found = j; j = g } else { j = j + 1 } }
151 if found < 0 {
152 if g < max_groups {
153 out_keys[g] = k
154 if agg_kind == 0 { out_agg[g] = 1 } else { out_agg[g] = vals[i] }
155 g = g + 1
156 }
157 } else {
158 if agg_kind == 0 { out_agg[found] = out_agg[found] + 1 } else { out_agg[found] = out_agg[found] + vals[i] }
159 }
160 i = i + 1
161 }
162 return g
163}
164
165// ---- INNER JOIN (relational-join seed): match left keys to right keys, emit paired value rows ----
166// For each left row, find the FIRST matching right key; emit (lval,rval) into out. Returns matched-row count.
167func df_inner_join(lkeys: *i64, lvals: *i64, ln: i64, rkeys: *i64, rvals: *i64, rn: i64, out_l: *i64, out_r: *i64, max_out: i64) -> i64 {
168 var o: i64 = 0
169 var i: i64 = 0
170 while i < ln {
171 var j: i64 = 0
172 var done: i64 = 0
173 while j < rn {
174 if done == 0 {
175 if rkeys[j] == lkeys[i] {
176 if o < max_out { out_l[o] = lvals[i]; out_r[o] = rvals[j]; o = o + 1 }
177 done = 1
178 }
179 }
180 j = j + 1
181 }
182 i = i + 1
183 }
184 return o
185}