nx_survey_stats.nx source
↩ module page · 425 lines · 16686 B
1// nx_survey_stats.nx -- the INSIGHTS engine over survey responses (operator 2026-07-10: "we have the survey
2// front end but we dont have any of the statistics or all the capabilities that power meaningful insights").
3// CONSUMES nx_survey_engine's effective ballots (se_ballots -> vout[i]/lout[i] = ballot record bytes) and
4// question records; turns raw responses into the analytics a real survey product delivers:
5// - descriptive: n/min/max/mean/median/mode/stddev (fixed-point permille, integer isqrt)
6// - CX metrics: NPS (promoters-detractors), top/bottom-box, 95% confidence margin on a proportion
7// - cross-tabulation: r x c contingency of one question by another
8// - significance: chi-square test of independence vs a DATA-DRIVEN critical table (alpha .05), FAIL-CLOSED
9// past df 10 (never a false "significant") -- the adversary's own guard
10// - key drivers: Pearson correlation of every scale question against a chosen outcome, ranked by |r|
11// - segment analysis: mean of a target question within each level of a segment question
12// - trend: before/after split on real ballot timestamps
13// - welfare triage rollup: composes nx_survey's sv_triage/sv_band to count crisis/strained/stable
14// - open text: sovereign tokenized theme frequency (honest: frequency-based, NOT neural topic/sentiment)
15// 100% integer / fixed-point, no floats, no hardware writes. Every stat is hand-KAT'd in the gate and carries
16// an adversarial neg-control (a fabricated significance / correlation must be rejected). license_tier: ORIGINAL
17import "nx_survey_engine.nx"
18import "nx_vecmath.nx"
19const ST_MAGIC_3841: i64 = 3841
20const ST_MAGIC_5991: i64 = 5991
21const ST_MAGIC_7815: i64 = 7815
22const ST_MAGIC_9488: i64 = 9488
23const ST_MAGIC_11070: i64 = 11070
24const ST_MAGIC_12592: i64 = 12592
25const ST_MAGIC_14067: i64 = 14067
26const ST_MAGIC_15507: i64 = 15507
27const ST_MAGIC_16919: i64 = 16919
28const ST_MAGIC_18307: i64 = 18307
29const ST_MAGIC_9223372036854775807: i64 = 9223372036854775807
30const ST_MAGIC_4096: i64 = 4096
31
32const ST_SCALE: i64 = 1000 // permille fixed-point
33const ST_Z95SQ: i64 = 3841600 // (1.96 * 1000)^2 = 3841600, the 95% two-tailed z^2 in permille^2
34
35// floor(sqrt(n)) via Newton's method on i64 (the one primitive every stat leans on)
36func st_isqrt(n: i64) -> i64 { return vm_isqrt(n) }
37
38// integer answer to qid in one ballot, or -1 when absent / non-numeric
39func st_answer_int(vp: *u8, vl: i64, qid: *u8) -> i64 {
40 let ab: *u8 = sys_mmap(64)
41 let n: i64 = se_answer_of(vp, vl, qid, ab, 64)
42 if n <= 0 { return 0 - 1 }
43 if se_digits(ab) == 0 { return 0 - 1 }
44 return sv_atoi(ab)
45}
46// the ballot's commit timestamp (usec field), or -1
47func st_ballot_usec(vp: *u8, vl: i64) -> i64 {
48 let ks: *i64 = sys_mmap(8 * 40) as *i64
49 let vs: *i64 = sys_mmap(8 * 40) as *i64
50 let nf: i64 = canon_decode(vp, vl, ks, vs, 40)
51 if nf < 0 { return 0 - 1 }
52 let v: *u8 = sv_get(ks, vs, nf, "usec" as *u8)
53 if (v as i64) == 0 { return 0 - 1 }
54 return sv_atoi(v)
55}
56// collect the numeric answers to qid across ballots into xs[]; returns count
57func st_collect(vout: *i64, lout: *i64, nb: i64, qid: *u8, xs: *i64) -> i64 {
58 var m: i64 = 0
59 var i: i64 = 0
60 while i < nb {
61 let v: i64 = st_answer_int(vout[i] as *u8, lout[i], qid)
62 if v >= 0 { xs[m] = v; m = m + 1 }
63 i = i + 1
64 }
65 return m
66}
67// collect ballots answering BOTH qidX and qidY numerically -> paired xs[]/ys[]; returns pair count
68func st_collect_pairs(vout: *i64, lout: *i64, nb: i64, qx: *u8, qy: *u8, xs: *i64, ys: *i64) -> i64 {
69 var m: i64 = 0
70 var i: i64 = 0
71 while i < nb {
72 let a: i64 = st_answer_int(vout[i] as *u8, lout[i], qx)
73 let b: i64 = st_answer_int(vout[i] as *u8, lout[i], qy)
74 if a >= 0 { if b >= 0 { xs[m] = a; ys[m] = b; m = m + 1 } }
75 i = i + 1
76 }
77 return m
78}
79// ascending insertion sort of a fresh copy (median/mode); writes into dst[]
80func st_sortcopy(xs: *i64, m: i64, dst: *i64) -> i64 {
81 var i: i64 = 0
82 while i < m { dst[i] = xs[i]; i = i + 1 }
83 i = 1
84 while i < m {
85 let cur: i64 = dst[i]
86 var j: i64 = i - 1
87 var go: i64 = 1
88 while go == 1 {
89 if j < 0 { go = 0 } else {
90 if dst[j] > cur { dst[j + 1] = dst[j]; j = j - 1 } else { go = 0 }
91 }
92 }
93 dst[j + 1] = cur
94 i = i + 1
95 }
96 return 0
97}
98// descriptive stats into out[7]: [0]=n [1]=min [2]=max [3]=mean_pm [4]=median_pm [5]=stddev_pm [6]=mode
99func st_desc(xs: *i64, m: i64, out: *i64) -> i64 {
100 if m <= 0 { var z: i64 = 0; while z < 7 { out[z] = 0; z = z + 1 } return 0 }
101 var sum: i64 = 0
102 var sumsq: i64 = 0
103 var mn: i64 = xs[0]
104 var mx: i64 = xs[0]
105 var i: i64 = 0
106 while i < m {
107 let v: i64 = xs[i]
108 sum = sum + v
109 sumsq = sumsq + v * v
110 if v < mn { mn = v }
111 if v > mx { mx = v }
112 i = i + 1
113 }
114 out[0] = m
115 out[1] = mn
116 out[2] = mx
117 out[3] = sum * ST_SCALE / m
118 // median from sorted copy
119 let srt: *i64 = sys_mmap(8 * (m + 2)) as *i64
120 st_sortcopy(xs, m, srt)
121 if m % 2 == 1 { out[4] = srt[m / 2] * ST_SCALE } else { out[4] = (srt[m / 2 - 1] + srt[m / 2]) * ST_SCALE / 2 }
122 // stddev permille = isqrt( (n*sumsq - sum^2) * SCALE^2 ) / n (population)
123 let numer: i64 = m * sumsq - sum * sum
124 if numer <= 0 { out[5] = 0 } else { out[5] = st_isqrt(numer * ST_SCALE * ST_SCALE) / m }
125 // mode = value with the longest run in the sorted copy
126 var best: i64 = srt[0]
127 var bestc: i64 = 1
128 var curc: i64 = 1
129 i = 1
130 while i < m {
131 if srt[i] == srt[i - 1] { curc = curc + 1 } else { curc = 1 }
132 if curc > bestc { bestc = curc; best = srt[i] }
133 i = i + 1
134 }
135 out[6] = best
136 return 0
137}
138// NPS into out[4]: [0]=nps(-100..100) [1]=promoters [2]=passives [3]=detractors. prom_min/det_max = cutoffs (DATA).
139func st_nps(xs: *i64, m: i64, prom_min: i64, det_max: i64, out: *i64) -> i64 {
140 var pr: i64 = 0
141 var de: i64 = 0
142 var i: i64 = 0
143 while i < m {
144 if xs[i] >= prom_min { pr = pr + 1 } else { if xs[i] <= det_max { de = de + 1 } }
145 i = i + 1
146 }
147 out[1] = pr
148 out[2] = m - pr - de
149 out[3] = de
150 if m == 0 { out[0] = 0 } else { out[0] = (pr - de) * 100 / m }
151 return 0
152}
153// top-box permille: share of answers with value >= maxval - boxsize + 1
154func st_topbox(xs: *i64, m: i64, maxval: i64, boxsize: i64) -> i64 {
155 if m == 0 { return 0 }
156 let thr: i64 = maxval - boxsize + 1
157 var c: i64 = 0
158 var i: i64 = 0
159 while i < m { if xs[i] >= thr { c = c + 1 } i = i + 1 }
160 return c * ST_SCALE / m
161}
162// 95% two-tailed confidence margin (permille) on proportion k/n: 1.96*sqrt(p(1-p)/n)
163func st_ci_margin(k: i64, n: i64) -> i64 {
164 if n <= 0 { return 0 }
165 if k < 0 { return 0 }
166 if k > n { return 0 }
167 // margin_pm = isqrt( Z95SQ * k * (n-k) / n^3 )
168 return st_isqrt(ST_Z95SQ * k * (n - k) / (n * n * n))
169}
170
171// ---- cross-tabulation + chi-square ----
172// fill an ra x cb contingency table (row-major, table[i*cb+j]) of ballots answering BOTH qa and qb;
173// returns the grand total counted. Out-of-range choice indices are skipped (defensive).
174func st_xtab(vout: *i64, lout: *i64, nb: i64, qa: *u8, qb: *u8, ra: i64, cb: i64, table: *i64) -> i64 {
175 var z: i64 = 0
176 while z < ra * cb { table[z] = 0; z = z + 1 }
177 var grand: i64 = 0
178 var i: i64 = 0
179 while i < nb {
180 let a: i64 = st_answer_int(vout[i] as *u8, lout[i], qa)
181 let b: i64 = st_answer_int(vout[i] as *u8, lout[i], qb)
182 if a >= 0 { if a < ra { if b >= 0 { if b < cb {
183 table[a * cb + b] = table[a * cb + b] + 1
184 grand = grand + 1
185 } } } }
186 i = i + 1
187 }
188 return grand
189}
190// chi-square critical value at alpha=0.05 (permille), df 1..10; FAIL-CLOSED (huge) past the table so we can
191// never falsely declare significance for a df we do not have a bound for (the adversary's guard).
192func st_crit05_pm(df: i64) -> i64 {
193 if df == 1 { return ST_MAGIC_3841 }
194 if df == 2 { return ST_MAGIC_5991 }
195 if df == 3 { return ST_MAGIC_7815 }
196 if df == 4 { return ST_MAGIC_9488 }
197 if df == 5 { return ST_MAGIC_11070 }
198 if df == 6 { return ST_MAGIC_12592 }
199 if df == 7 { return ST_MAGIC_14067 }
200 if df == 8 { return ST_MAGIC_15507 }
201 if df == 9 { return ST_MAGIC_16919 }
202 if df == 10 { return ST_MAGIC_18307 }
203 return ST_MAGIC_9223372036854775807
204}
205// chi-square of independence on table[r*c]; out[0]=chisq_pm out[1]=df. Returns 1 significant (>=crit .05), else 0.
206func st_chisq(table: *i64, r: i64, c: i64, out: *i64) -> i64 {
207 let rowt: *i64 = sys_mmap(8 * (r + 1)) as *i64
208 let colt: *i64 = sys_mmap(8 * (c + 1)) as *i64
209 var i: i64 = 0
210 while i < r { rowt[i] = 0; i = i + 1 }
211 var j: i64 = 0
212 while j < c { colt[j] = 0; j = j + 1 }
213 var nn: i64 = 0
214 i = 0
215 while i < r {
216 j = 0
217 while j < c {
218 let o: i64 = table[i * c + j]
219 rowt[i] = rowt[i] + o
220 colt[j] = colt[j] + o
221 nn = nn + o
222 j = j + 1
223 }
224 i = i + 1
225 }
226 out[1] = (r - 1) * (c - 1)
227 if nn == 0 { out[0] = 0; return 0 }
228 var chi: i64 = 0
229 i = 0
230 while i < r {
231 j = 0
232 while j < c {
233 if rowt[i] > 0 { if colt[j] > 0 {
234 let o: i64 = table[i * c + j]
235 let d: i64 = o * nn - rowt[i] * colt[j]
236 chi = chi + (d * d) * ST_SCALE / (nn * rowt[i] * colt[j])
237 } }
238 j = j + 1
239 }
240 i = i + 1
241 }
242 out[0] = chi
243 let crit: i64 = st_crit05_pm(out[1])
244 if chi >= crit { return 1 }
245 return 0
246}
247
248// Pearson correlation (permille, -1000..1000) of m paired values; 0 when undefined (flat variable)
249func st_corr(xs: *i64, ys: *i64, m: i64) -> i64 {
250 if m < 2 { return 0 }
251 var sx: i64 = 0
252 var sy: i64 = 0
253 var sxy: i64 = 0
254 var sxx: i64 = 0
255 var syy: i64 = 0
256 var i: i64 = 0
257 while i < m {
258 let x: i64 = xs[i]
259 let y: i64 = ys[i]
260 sx = sx + x
261 sy = sy + y
262 sxy = sxy + x * y
263 sxx = sxx + x * x
264 syy = syy + y * y
265 i = i + 1
266 }
267 let num: i64 = m * sxy - sx * sy
268 let dxx: i64 = m * sxx - sx * sx
269 let dyy: i64 = m * syy - sy * sy
270 if dxx <= 0 { return 0 }
271 if dyy <= 0 { return 0 }
272 let den: i64 = st_isqrt(dxx * dyy)
273 if den == 0 { return 0 }
274 return num * ST_SCALE / den
275}
276
277// segment means: for each level 0..seg_n-1 of seg_qid, count + mean_pm of tgt_qid; into out_n[]/out_meanpm[]
278func st_segment_means(vout: *i64, lout: *i64, nb: i64, seg_qid: *u8, tgt_qid: *u8, seg_n: i64, out_n: *i64, out_meanpm: *i64) -> i64 {
279 let sum: *i64 = sys_mmap(8 * (seg_n + 1)) as *i64
280 var s: i64 = 0
281 while s < seg_n { sum[s] = 0; out_n[s] = 0; out_meanpm[s] = 0; s = s + 1 }
282 var i: i64 = 0
283 while i < nb {
284 let g: i64 = st_answer_int(vout[i] as *u8, lout[i], seg_qid)
285 let t: i64 = st_answer_int(vout[i] as *u8, lout[i], tgt_qid)
286 if g >= 0 { if g < seg_n { if t >= 0 {
287 sum[g] = sum[g] + t
288 out_n[g] = out_n[g] + 1
289 } } }
290 i = i + 1
291 }
292 s = 0
293 while s < seg_n {
294 if out_n[s] > 0 { out_meanpm[s] = sum[s] * ST_SCALE / out_n[s] }
295 s = s + 1
296 }
297 return 0
298}
299// before/after trend on real ballot timestamps: out[5] = [n_before, mean_before_pm, n_after, mean_after_pm, delta_pm]
300func st_trend_split(vout: *i64, lout: *i64, nb: i64, qid: *u8, split_usec: i64, out: *i64) -> i64 {
301 var nb0: i64 = 0
302 var sb0: i64 = 0
303 var na0: i64 = 0
304 var sa0: i64 = 0
305 var i: i64 = 0
306 while i < nb {
307 let v: i64 = st_answer_int(vout[i] as *u8, lout[i], qid)
308 if v >= 0 {
309 let ts: i64 = st_ballot_usec(vout[i] as *u8, lout[i])
310 if ts >= 0 {
311 if ts < split_usec { nb0 = nb0 + 1; sb0 = sb0 + v } else { na0 = na0 + 1; sa0 = sa0 + v }
312 }
313 }
314 i = i + 1
315 }
316 out[0] = nb0
317 out[2] = na0
318 var mb: i64 = 0
319 var ma: i64 = 0
320 if nb0 > 0 { mb = sb0 * ST_SCALE / nb0 }
321 if na0 > 0 { ma = sa0 * ST_SCALE / na0 }
322 out[1] = mb
323 out[3] = ma
324 out[4] = ma - mb
325 return 0
326}
327// welfare triage rollup: compose sv_triage/sv_band per ballot; out[3]=[stable,strained,crisis]. wqids are the
328// ballot field keys ("a_<qid>"), weights + band thresholds are DATA (rule 11).
329func st_triage_rollup(vout: *i64, lout: *i64, nb: i64, wqids: *i64, weights: *i64, nw: i64, t_str: i64, t_cri: i64, out: *i64) -> i64 {
330 out[0] = 0
331 out[1] = 0
332 out[2] = 0
333 var i: i64 = 0
334 while i < nb {
335 let ks: *i64 = sys_mmap(8 * 48) as *i64
336 let vs: *i64 = sys_mmap(8 * 48) as *i64
337 let nf: i64 = canon_decode(vout[i] as *u8, lout[i], ks, vs, 48)
338 if nf > 0 {
339 let sc: i64 = sv_triage(ks, vs, nf, wqids, weights, nw)
340 let band: i64 = sv_band(sc, t_str, t_cri)
341 out[band] = out[band] + 1
342 }
343 i = i + 1
344 }
345 return 0
346}
347
348// ---- open-text theme frequency (sovereign; frequency-based, honest NOT neural) ----
349// common English stopwords so themes surface content words, not glue
350func st_stopword(t: *u8) -> i64 {
351 if se_seq(t, "the" as *u8) == 1 { return 1 }
352 if se_seq(t, "and" as *u8) == 1 { return 1 }
353 if se_seq(t, "for" as *u8) == 1 { return 1 }
354 if se_seq(t, "are" as *u8) == 1 { return 1 }
355 if se_seq(t, "our" as *u8) == 1 { return 1 }
356 if se_seq(t, "was" as *u8) == 1 { return 1 }
357 if se_seq(t, "with" as *u8) == 1 { return 1 }
358 if se_seq(t, "that" as *u8) == 1 { return 1 }
359 if se_seq(t, "this" as *u8) == 1 { return 1 }
360 if se_seq(t, "have" as *u8) == 1 { return 1 }
361 if se_seq(t, "would" as *u8) == 1 { return 1 }
362 if se_seq(t, "could" as *u8) == 1 { return 1 }
363 if se_seq(t, "you" as *u8) == 1 { return 1 }
364 if se_seq(t, "your" as *u8) == 1 { return 1 }
365 if se_seq(t, "not" as *u8) == 1 { return 1 }
366 if se_seq(t, "but" as *u8) == 1 { return 1 }
367 if se_seq(t, "get" as *u8) == 1 { return 1 }
368 if se_seq(t, "from" as *u8) == 1 { return 1 }
369 if se_seq(t, "they" as *u8) == 1 { return 1 }
370 if se_seq(t, "can" as *u8) == 1 { return 1 }
371 return 0
372}
373// tally content-word frequency across all text answers to qid. Parallel arrays terms_out[]/freq_out[] hold up
374// to cap distinct terms (pool-backed term strings). Returns distinct-term count (bounded by cap).
375func st_text_themes(vout: *i64, lout: *i64, nb: i64, qid: *u8, terms_out: *i64, freq_out: *i64, cap: i64) -> i64 {
376 let pool: *u8 = sys_mmap(cap * 40 + ST_MAGIC_4096)
377 var pooloff: i64 = 0
378 var nt: i64 = 0
379 let tbl: *u8 = sys_mmap(272)
380 ss_tok_table(tbl)
381 let ab: *u8 = sys_mmap(ST_MAGIC_4096)
382 let tok: *u8 = sys_mmap(40)
383 let pos: *i64 = sys_mmap(16) as *i64
384 var i: i64 = 0
385 while i < nb {
386 let al: i64 = se_answer_of(vout[i] as *u8, lout[i], qid, ab, ST_MAGIC_4096)
387 if al > 0 {
388 pos[0] = 0
389 var go: i64 = 1
390 while go == 1 {
391 let tl: i64 = ss_tok_next2(ab, al, pos, tok, tbl)
392 if tl < 0 { go = 0 } else {
393 if st_stopword(tok) == 0 {
394 // linear find/insert (term counts are small; cap bounds it)
395 var f: i64 = 0 - 1
396 var k: i64 = 0
397 while k < nt { if se_seq(terms_out[k] as *u8, tok) == 1 { f = k; k = nt } else { k = k + 1 } }
398 if f >= 0 { freq_out[f] = freq_out[f] + 1 } else {
399 if nt < cap {
400 let dst: *u8 = (pool as i64 + pooloff) as *u8
401 var x: i64 = 0
402 while x <= tl { dst[x] = tok[x]; x = x + 1 }
403 pooloff = pooloff + tl + 1
404 terms_out[nt] = dst as i64
405 freq_out[nt] = 1
406 nt = nt + 1
407 }
408 }
409 }
410 }
411 }
412 }
413 i = i + 1
414 }
415 return nt
416}
417// frequency of one exact term across text answers (the gate's deterministic oracle)
418func st_term_freq(vout: *i64, lout: *i64, nb: i64, qid: *u8, term: *u8) -> i64 {
419 let terms: *i64 = sys_mmap(8 * 256) as *i64
420 let freqs: *i64 = sys_mmap(8 * 256) as *i64
421 let nt: i64 = st_text_themes(vout, lout, nb, qid, terms, freqs, 256)
422 var i: i64 = 0
423 while i < nt { if se_seq(terms[i] as *u8, term) == 1 { return freqs[i] } i = i + 1 }
424 return 0
425}