code wiki / (root) / nx_survey_stats.nx
`nx_survey_stats.nx` source

↩ module page · 431 lines · 16805 B
1// nx_survey_stats.nx -- the INSIGHTS engine over survey responses (operator 2026-07-10: "we have the survey
2// front end but we dont have any of the statistics or all the capabilities that power meaningful insights").
3// CONSUMES nx_survey_engine's effective ballots (se_ballots -> vout[i]/lout[i] = ballot record bytes) and
4// question records; turns raw responses into the analytics a real survey product delivers:
5//   - descriptive: n/min/max/mean/median/mode/stddev (fixed-point permille, integer isqrt)
6//   - CX metrics: NPS (promoters-detractors), top/bottom-box, 95% confidence margin on a proportion
7//   - cross-tabulation: r x c contingency of one question by another
8//   - significance: chi-square test of independence vs a DATA-DRIVEN critical table (alpha .05), FAIL-CLOSED
9//     past df 10 (never a false "significant") -- the adversary's own guard
10//   - key drivers: Pearson correlation of every scale question against a chosen outcome, ranked by |r|
11//   - segment analysis: mean of a target question within each level of a segment question
12//   - trend: before/after split on real ballot timestamps
13//   - welfare triage rollup: composes nx_survey's sv_triage/sv_band to count crisis/strained/stable
14//   - open text: sovereign tokenized theme frequency (honest: frequency-based, NOT neural topic/sentiment)
15// 100% integer / fixed-point, no floats, no hardware writes. Every stat is hand-KAT'd in the gate and carries
16// an adversarial neg-control (a fabricated significance / correlation must be rejected). license_tier: ORIGINAL
17import "nx_survey_engine.nx"
18const ST_MAGIC_3841: i64 = 3841
19const ST_MAGIC_5991: i64 = 5991
20const ST_MAGIC_7815: i64 = 7815
21const ST_MAGIC_9488: i64 = 9488
22const ST_MAGIC_11070: i64 = 11070
23const ST_MAGIC_12592: i64 = 12592
24const ST_MAGIC_14067: i64 = 14067
25const ST_MAGIC_15507: i64 = 15507
26const ST_MAGIC_16919: i64 = 16919
27const ST_MAGIC_18307: i64 = 18307
28const ST_MAGIC_9223372036854775807: i64 = 9223372036854775807
29const ST_MAGIC_4096: i64 = 4096
30
31const ST_SCALE: i64 = 1000        // permille fixed-point
32const ST_Z95SQ: i64 = 3841600     // (1.96 * 1000)^2 = 3841600, the 95% two-tailed z^2 in permille^2
33
34// floor(sqrt(n)) via Newton's method on i64 (the one primitive every stat leans on)
35func st_isqrt(n: i64) -> i64 {
  if n <= 0 { return 0 }
  if n < 4 { return 1 }
  var x: i64 = n
  var y: i64 = (x + 1) / 2
  while y < x { x = y; y = (x + n / x) / 2 }
  return x
42}
43
44// integer answer to qid in one ballot, or -1 when absent / non-numeric
45func st_answer_int(vp: *u8, vl: i64, qid: *u8) -> i64 {
  let ab: *u8 = sys_mmap(64)
  let n: i64 = se_answer_of(vp, vl, qid, ab, 64)
  if n <= 0 { return 0 - 1 }
  if se_digits(ab) == 0 { return 0 - 1 }
  return sv_atoi(ab)
51}
52// the ballot's commit timestamp (usec field), or -1
53func st_ballot_usec(vp: *u8, vl: i64) -> i64 {
  let ks: *i64 = sys_mmap(8 * 40) as *i64
  let vs: *i64 = sys_mmap(8 * 40) as *i64
  let nf: i64 = canon_decode(vp, vl, ks, vs, 40)
  if nf < 0 { return 0 - 1 }
  let v: *u8 = sv_get(ks, vs, nf, "usec" as *u8)
  if (v as i64) == 0 { return 0 - 1 }
  return sv_atoi(v)
61}
62// collect the numeric answers to qid across ballots into xs[]; returns count
63func st_collect(vout: *i64, lout: *i64, nb: i64, qid: *u8, xs: *i64) -> i64 {
  var m: i64 = 0
  var i: i64 = 0
  while i < nb {
      let v: i64 = st_answer_int(vout[i] as *u8, lout[i], qid)
      if v >= 0 { xs[m] = v; m = m + 1 }
      i = i + 1
  }
  return m
72}
73// collect ballots answering BOTH qidX and qidY numerically -> paired xs[]/ys[]; returns pair count
74func st_collect_pairs(vout: *i64, lout: *i64, nb: i64, qx: *u8, qy: *u8, xs: *i64, ys: *i64) -> i64 {
  var m: i64 = 0
  var i: i64 = 0
  while i < nb {
      let a: i64 = st_answer_int(vout[i] as *u8, lout[i], qx)
      let b: i64 = st_answer_int(vout[i] as *u8, lout[i], qy)
      if a >= 0 { if b >= 0 { xs[m] = a; ys[m] = b; m = m + 1 } }
      i = i + 1
  }
  return m
84}
85// ascending insertion sort of a fresh copy (median/mode); writes into dst[]
86func st_sortcopy(xs: *i64, m: i64, dst: *i64) -> i64 {
  var i: i64 = 0
  while i < m { dst[i] = xs[i]; i = i + 1 }
  i = 1
  while i < m {
      let cur: i64 = dst[i]
      var j: i64 = i - 1
      var go: i64 = 1
      while go == 1 {
          if j < 0 { go = 0 } else {
              if dst[j] > cur { dst[j + 1] = dst[j]; j = j - 1 } else { go = 0 }
          }
      }
      dst[j + 1] = cur
      i = i + 1
  }
  return 0
103}
104// descriptive stats into out[7]: [0]=n [1]=min [2]=max [3]=mean_pm [4]=median_pm [5]=stddev_pm [6]=mode
105func st_desc(xs: *i64, m: i64, out: *i64) -> i64 {
  if m <= 0 { var z: i64 = 0; while z < 7 { out[z] = 0; z = z + 1 } return 0 }
  var sum: i64 = 0
  var sumsq: i64 = 0
  var mn: i64 = xs[0]
  var mx: i64 = xs[0]
  var i: i64 = 0
  while i < m {
      let v: i64 = xs[i]
      sum = sum + v
      sumsq = sumsq + v * v
      if v < mn { mn = v }
      if v > mx { mx = v }
      i = i + 1
  }
  out[0] = m
  out[1] = mn
  out[2] = mx
  out[3] = sum * ST_SCALE / m
  // median from sorted copy
  let srt: *i64 = sys_mmap(8 * (m + 2)) as *i64
  st_sortcopy(xs, m, srt)
  if m % 2 == 1 { out[4] = srt[m / 2] * ST_SCALE } else { out[4] = (srt[m / 2 - 1] + srt[m / 2]) * ST_SCALE / 2 }
  // stddev permille = isqrt( (n*sumsq - sum^2) * SCALE^2 ) / n   (population)
  let numer: i64 = m * sumsq - sum * sum
  if numer <= 0 { out[5] = 0 } else { out[5] = st_isqrt(numer * ST_SCALE * ST_SCALE) / m }
  // mode = value with the longest run in the sorted copy
  var best: i64 = srt[0]
  var bestc: i64 = 1
  var curc: i64 = 1
  i = 1
  while i < m {
      if srt[i] == srt[i - 1] { curc = curc + 1 } else { curc = 1 }
      if curc > bestc { bestc = curc; best = srt[i] }
      i = i + 1
  }
  out[6] = best
  return 0
143}
144// NPS into out[4]: [0]=nps(-100..100) [1]=promoters [2]=passives [3]=detractors. prom_min/det_max = cutoffs (DATA).
145func st_nps(xs: *i64, m: i64, prom_min: i64, det_max: i64, out: *i64) -> i64 {
  var pr: i64 = 0
  var de: i64 = 0
  var i: i64 = 0
  while i < m {
      if xs[i] >= prom_min { pr = pr + 1 } else { if xs[i] <= det_max { de = de + 1 } }
      i = i + 1
  }
  out[1] = pr
  out[2] = m - pr - de
  out[3] = de
  if m == 0 { out[0] = 0 } else { out[0] = (pr - de) * 100 / m }
  return 0
158}
159// top-box permille: share of answers with value >= maxval - boxsize + 1
160func st_topbox(xs: *i64, m: i64, maxval: i64, boxsize: i64) -> i64 {
  if m == 0 { return 0 }
  let thr: i64 = maxval - boxsize + 1
  var c: i64 = 0
  var i: i64 = 0
  while i < m { if xs[i] >= thr { c = c + 1 } i = i + 1 }
  return c * ST_SCALE / m
167}
168// 95% two-tailed confidence margin (permille) on proportion k/n: 1.96*sqrt(p(1-p)/n)
169func st_ci_margin(k: i64, n: i64) -> i64 {
  if n <= 0 { return 0 }
  if k < 0 { return 0 }
  if k > n { return 0 }
  // margin_pm = isqrt( Z95SQ * k * (n-k) / n^3 )
  return st_isqrt(ST_Z95SQ * k * (n - k) / (n * n * n))
175}
176
177// ---- cross-tabulation + chi-square ----
178// fill an ra x cb contingency table (row-major, table[i*cb+j]) of ballots answering BOTH qa and qb;
179// returns the grand total counted. Out-of-range choice indices are skipped (defensive).
180func st_xtab(vout: *i64, lout: *i64, nb: i64, qa: *u8, qb: *u8, ra: i64, cb: i64, table: *i64) -> i64 {
  var z: i64 = 0
  while z < ra * cb { table[z] = 0; z = z + 1 }
  var grand: i64 = 0
  var i: i64 = 0
  while i < nb {
      let a: i64 = st_answer_int(vout[i] as *u8, lout[i], qa)
      let b: i64 = st_answer_int(vout[i] as *u8, lout[i], qb)
      if a >= 0 { if a < ra { if b >= 0 { if b < cb {
          table[a * cb + b] = table[a * cb + b] + 1
          grand = grand + 1
      } } } }
      i = i + 1
  }
  return grand
195}
196// chi-square critical value at alpha=0.05 (permille), df 1..10; FAIL-CLOSED (huge) past the table so we can
197// never falsely declare significance for a df we do not have a bound for (the adversary's guard).
198func st_crit05_pm(df: i64) -> i64 {
  if df == 1 { return ST_MAGIC_3841 }
  if df == 2 { return ST_MAGIC_5991 }
  if df == 3 { return ST_MAGIC_7815 }
  if df == 4 { return ST_MAGIC_9488 }
  if df == 5 { return ST_MAGIC_11070 }
  if df == 6 { return ST_MAGIC_12592 }
  if df == 7 { return ST_MAGIC_14067 }
  if df == 8 { return ST_MAGIC_15507 }
  if df == 9 { return ST_MAGIC_16919 }
  if df == 10 { return ST_MAGIC_18307 }
  return ST_MAGIC_9223372036854775807
210}
211// chi-square of independence on table[r*c]; out[0]=chisq_pm out[1]=df. Returns 1 significant (>=crit .05), else 0.
212func st_chisq(table: *i64, r: i64, c: i64, out: *i64) -> i64 {
  let rowt: *i64 = sys_mmap(8 * (r + 1)) as *i64
  let colt: *i64 = sys_mmap(8 * (c + 1)) as *i64
  var i: i64 = 0
  while i < r { rowt[i] = 0; i = i + 1 }
  var j: i64 = 0
  while j < c { colt[j] = 0; j = j + 1 }
  var nn: i64 = 0
  i = 0
  while i < r {
      j = 0
      while j < c {
          let o: i64 = table[i * c + j]
          rowt[i] = rowt[i] + o
          colt[j] = colt[j] + o
          nn = nn + o
          j = j + 1
      }
      i = i + 1
  }
  out[1] = (r - 1) * (c - 1)
  if nn == 0 { out[0] = 0; return 0 }
  var chi: i64 = 0
  i = 0
  while i < r {
      j = 0
      while j < c {
          if rowt[i] > 0 { if colt[j] > 0 {
              let o: i64 = table[i * c + j]
              let d: i64 = o * nn - rowt[i] * colt[j]
              chi = chi + (d * d) * ST_SCALE / (nn * rowt[i] * colt[j])
          } }
          j = j + 1
      }
      i = i + 1
  }
  out[0] = chi
  let crit: i64 = st_crit05_pm(out[1])
  if chi >= crit { return 1 }
  return 0
252}
253
254// Pearson correlation (permille, -1000..1000) of m paired values; 0 when undefined (flat variable)
255func st_corr(xs: *i64, ys: *i64, m: i64) -> i64 {
  if m < 2 { return 0 }
  var sx: i64 = 0
  var sy: i64 = 0
  var sxy: i64 = 0
  var sxx: i64 = 0
  var syy: i64 = 0
  var i: i64 = 0
  while i < m {
      let x: i64 = xs[i]
      let y: i64 = ys[i]
      sx = sx + x
      sy = sy + y
      sxy = sxy + x * y
      sxx = sxx + x * x
      syy = syy + y * y
      i = i + 1
  }
  let num: i64 = m * sxy - sx * sy
  let dxx: i64 = m * sxx - sx * sx
  let dyy: i64 = m * syy - sy * sy
  if dxx <= 0 { return 0 }
  if dyy <= 0 { return 0 }
  let den: i64 = st_isqrt(dxx * dyy)
  if den == 0 { return 0 }
  return num * ST_SCALE / den
281}
282
283// segment means: for each level 0..seg_n-1 of seg_qid, count + mean_pm of tgt_qid; into out_n[]/out_meanpm[]
284func st_segment_means(vout: *i64, lout: *i64, nb: i64, seg_qid: *u8, tgt_qid: *u8, seg_n: i64, out_n: *i64, out_meanpm: *i64) -> i64 {
  let sum: *i64 = sys_mmap(8 * (seg_n + 1)) as *i64
  var s: i64 = 0
  while s < seg_n { sum[s] = 0; out_n[s] = 0; out_meanpm[s] = 0; s = s + 1 }
  var i: i64 = 0
  while i < nb {
      let g: i64 = st_answer_int(vout[i] as *u8, lout[i], seg_qid)
      let t: i64 = st_answer_int(vout[i] as *u8, lout[i], tgt_qid)
      if g >= 0 { if g < seg_n { if t >= 0 {
          sum[g] = sum[g] + t
          out_n[g] = out_n[g] + 1
      } } }
      i = i + 1
  }
  s = 0
  while s < seg_n {
      if out_n[s] > 0 { out_meanpm[s] = sum[s] * ST_SCALE / out_n[s] }
      s = s + 1
  }
  return 0
304}
305// before/after trend on real ballot timestamps: out[5] = [n_before, mean_before_pm, n_after, mean_after_pm, delta_pm]
306func st_trend_split(vout: *i64, lout: *i64, nb: i64, qid: *u8, split_usec: i64, out: *i64) -> i64 {
  var nb0: i64 = 0
  var sb0: i64 = 0
  var na0: i64 = 0
  var sa0: i64 = 0
  var i: i64 = 0
  while i < nb {
      let v: i64 = st_answer_int(vout[i] as *u8, lout[i], qid)
      if v >= 0 {
          let ts: i64 = st_ballot_usec(vout[i] as *u8, lout[i])
          if ts >= 0 {
              if ts < split_usec { nb0 = nb0 + 1; sb0 = sb0 + v } else { na0 = na0 + 1; sa0 = sa0 + v }
          }
      }
      i = i + 1
  }
  out[0] = nb0
  out[2] = na0
  var mb: i64 = 0
  var ma: i64 = 0
  if nb0 > 0 { mb = sb0 * ST_SCALE / nb0 }
  if na0 > 0 { ma = sa0 * ST_SCALE / na0 }
  out[1] = mb
  out[3] = ma
  out[4] = ma - mb
  return 0
332}
333// welfare triage rollup: compose sv_triage/sv_band per ballot; out[3]=[stable,strained,crisis]. wqids are the
334// ballot field keys ("a_<qid>"), weights + band thresholds are DATA (rule 11).
335func st_triage_rollup(vout: *i64, lout: *i64, nb: i64, wqids: *i64, weights: *i64, nw: i64, t_str: i64, t_cri: i64, out: *i64) -> i64 {
  out[0] = 0
  out[1] = 0
  out[2] = 0
  var i: i64 = 0
  while i < nb {
      let ks: *i64 = sys_mmap(8 * 48) as *i64
      let vs: *i64 = sys_mmap(8 * 48) as *i64
      let nf: i64 = canon_decode(vout[i] as *u8, lout[i], ks, vs, 48)
      if nf > 0 {
          let sc: i64 = sv_triage(ks, vs, nf, wqids, weights, nw)
          let band: i64 = sv_band(sc, t_str, t_cri)
          out[band] = out[band] + 1
      }
      i = i + 1
  }
  return 0
352}
353
354// ---- open-text theme frequency (sovereign; frequency-based, honest NOT neural) ----
355// common English stopwords so themes surface content words, not glue
356func st_stopword(t: *u8) -> i64 {
  if se_seq(t, "the" as *u8) == 1 { return 1 }
  if se_seq(t, "and" as *u8) == 1 { return 1 }
  if se_seq(t, "for" as *u8) == 1 { return 1 }
  if se_seq(t, "are" as *u8) == 1 { return 1 }
  if se_seq(t, "our" as *u8) == 1 { return 1 }
  if se_seq(t, "was" as *u8) == 1 { return 1 }
  if se_seq(t, "with" as *u8) == 1 { return 1 }
  if se_seq(t, "that" as *u8) == 1 { return 1 }
  if se_seq(t, "this" as *u8) == 1 { return 1 }
  if se_seq(t, "have" as *u8) == 1 { return 1 }
  if se_seq(t, "would" as *u8) == 1 { return 1 }
  if se_seq(t, "could" as *u8) == 1 { return 1 }
  if se_seq(t, "you" as *u8) == 1 { return 1 }
  if se_seq(t, "your" as *u8) == 1 { return 1 }
  if se_seq(t, "not" as *u8) == 1 { return 1 }
  if se_seq(t, "but" as *u8) == 1 { return 1 }
  if se_seq(t, "get" as *u8) == 1 { return 1 }
  if se_seq(t, "from" as *u8) == 1 { return 1 }
  if se_seq(t, "they" as *u8) == 1 { return 1 }
  if se_seq(t, "can" as *u8) == 1 { return 1 }
  return 0
378}
379// tally content-word frequency across all text answers to qid. Parallel arrays terms_out[]/freq_out[] hold up
380// to cap distinct terms (pool-backed term strings). Returns distinct-term count (bounded by cap).
381func st_text_themes(vout: *i64, lout: *i64, nb: i64, qid: *u8, terms_out: *i64, freq_out: *i64, cap: i64) -> i64 {
  let pool: *u8 = sys_mmap(cap * 40 + ST_MAGIC_4096)
  var pooloff: i64 = 0
  var nt: i64 = 0
  let tbl: *u8 = sys_mmap(272)
  ss_tok_table(tbl)
  let ab: *u8 = sys_mmap(ST_MAGIC_4096)
  let tok: *u8 = sys_mmap(40)
  let pos: *i64 = sys_mmap(16) as *i64
  var i: i64 = 0
  while i < nb {
      let al: i64 = se_answer_of(vout[i] as *u8, lout[i], qid, ab, ST_MAGIC_4096)
      if al > 0 {
          pos[0] = 0
          var go: i64 = 1
          while go == 1 {
              let tl: i64 = ss_tok_next2(ab, al, pos, tok, tbl)
              if tl < 0 { go = 0 } else {
                  if st_stopword(tok) == 0 {
                      // linear find/insert (term counts are small; cap bounds it)
                      var f: i64 = 0 - 1
                      var k: i64 = 0
                      while k < nt { if se_seq(terms_out[k] as *u8, tok) == 1 { f = k; k = nt } else { k = k + 1 } }
                      if f >= 0 { freq_out[f] = freq_out[f] + 1 } else {
                          if nt < cap {
                              let dst: *u8 = (pool as i64 + pooloff) as *u8
                              var x: i64 = 0
                              while x <= tl { dst[x] = tok[x]; x = x + 1 }
                              pooloff = pooloff + tl + 1
                              terms_out[nt] = dst as i64
                              freq_out[nt] = 1
                              nt = nt + 1
                          }
                      }
                  }
              }
          }
      }
      i = i + 1
  }
  return nt
422}
423// frequency of one exact term across text answers (the gate's deterministic oracle)
424func st_term_freq(vout: *i64, lout: *i64, nb: i64, qid: *u8, term: *u8) -> i64 {
  let terms: *i64 = sys_mmap(8 * 256) as *i64
  let freqs: *i64 = sys_mmap(8 * 256) as *i64
  let nt: i64 = st_text_themes(vout, lout, nb, qid, terms, freqs, 256)
  var i: i64 = 0
  while i < nt { if se_seq(terms[i] as *u8, term) == 1 { return freqs[i] } i = i + 1 }
  return 0
431}
nx_survey_stats.nx source

`nx_survey_stats.nx` source
