code wiki / (root) / nx_abstat_gate.nx
`nx_abstat_gate.nx` source

↩ module page · 151 lines · 8987 B
1// nx_abstat_gate.nx -- THE REFEREE FOR THE A/B CONTRAST CORE (nx_abstat_lib).
2//
3// A statistics organ is the one kind of organ whose output nobody can eyeball, so every claim it makes has
4// to be bitten here. The teeth below are ordered by what they would cost if they were wrong: a detector that
5// fires on noise is worse than no detector, so the false-positive control comes before the sensitivity one.
6//
7// Every tooth prints the numbers it tested (gv_check_eq / gv_kv), because a PASS-only gate is unfalsifiable
8// from the outside and makes an independent second method structurally impossible.
9// license_tier: ORIGINAL   No hw writes (Rule 26).
10import "nx_syscalls.nx"
11import "nx_gate_verdict.nx"
12import "nx_abstat_lib.nx"
13
14const ABG_N: i64 = 40                 // comfortably over NX_MC_MIN_N so the sample floor is not what is under test
15const ABG_TIGHT: i64 = 100            // a run of 100 samples for the below-MDE tooth
16
17// deterministic, no PRNG: a fixed sawtooth around a centre, so every run of this gate sees identical inputs
18func abg_saw(i: i64, centre: i64, spread: i64) -> i64 {
  let phase: i64 = i % 5
  if phase == 0 { return centre - spread }
  if phase == 1 { return centre + spread }
  if phase == 2 { return centre }
  if phase == 3 { return centre - spread }
  return centre + spread
25}
26
27func abg_fill(r: *i64, axis: i64, arm: i64, n: i64, centre: i64, spread: i64) -> i64 {
  var i: i64 = 0
  while i < n { ab_push(r, axis, arm, abg_saw(i, centre, spread)); i = i + 1 }
  return 0
31}
32
33func abg_region() -> *i64 {
  let r: *i64 = sys_mmap(AB_WORDS * 8)
  ab_init(r)
  return r
37}
38
39func main() -> i64 {
  let ctr: *i64 = gv_ctr()
  gv_head("nx_abstat_gate -- the A/B contrast core" as *u8)
42
  // ---- the composed rulers are the incumbents, not fresh numbers ------------------------------------
  gv_check_eq("welford-equivalent-to-incumbent" as *u8, ab_welford_matches_incumbent(4096, 5120), 1, ctr)
  gv_check_eq("bonferroni-table-is-the-incumbent-5axes" as *u8, ab_crit_t_q10(5), 2641, ctr)
  gv_check_eq("bonferroni-table-is-the-incumbent-1axis" as *u8, ab_crit_t_q10(1), 2007, ctr)
  gv_check("bonferroni-actually-tightens-with-more-axes" as *u8,
           (ab_crit_t_q10(5) > ab_crit_t_q10(1)) as i64, ctr)
  gv_check_eq("sample-floor-is-the-incumbent-min-n" as *u8, ab_min_n(), 30, ctr)
50
  // ---- an unmeasured thing must never read as a measured zero --------------------------------------
  let e: *i64 = abg_region()
  gv_check_eq("empty-arm-mean-is-UNOBSERVED-not-zero" as *u8, ab_mean_q10(e, 0, AB_ARM_B), 0 - 1, ctr)
  ab_push(e, 0, AB_ARM_B, 7)
  gv_check_eq("one-sample-variance-is-UNOBSERVED-not-zero" as *u8, ab_var_q10(e, 0, AB_ARM_B), 0 - 1, ctr)
56
  // ---- NEG-CONTROL: a fixture the test CANNOT fail must be named, never scored ----------------------
  // Both arms constant. Delta is zero and variance is zero: reporting NO-CHANGE here would publish a
  // non-experiment as a measurement. The required answer is INSUFFICIENT with the reason named.
  let z: *i64 = abg_region()
  abg_fill(z, 0, AB_ARM_B, ABG_N, 100, 0)
  abg_fill(z, 0, AB_ARM_A, ABG_N, 100, 0)
  gv_check_eq("neg-control-zero-variance-reason-is-ZEROVAR" as *u8, ab_axis_reason(z, 0), AB_R_ZEROVAR, ctr)
  gv_check_eq("neg-control-zero-variance-axis-is-INSUFF" as *u8,
              ab_axis_verdict(z, 0, 1, 1, 0), AB_AX_INSUFF, ctr)
66
  // ---- NEG-CONTROL: under the sample floor abstains, and says WHICH arm was short -------------------
  let f: *i64 = abg_region()
  abg_fill(f, 0, AB_ARM_B, 29, 100, 3)
  abg_fill(f, 0, AB_ARM_A, ABG_N, 120, 3)
  gv_check_eq("neg-control-short-baseline-names-arm-B" as *u8, ab_axis_reason(f, 0), AB_R_FEW_B, ctr)
  gv_check_eq("neg-control-short-baseline-axis-is-INSUFF" as *u8,
              ab_axis_verdict(f, 0, 1, 1, 0), AB_AX_INSUFF, ctr)
74
  // ---- THE FALSE-POSITIVE CONTROL: two arms from the SAME distribution must NOT read as improved ----
  // Same centre, same spread, different phase. A detector that calls this UP fires on noise, and a
  // detector that fires on noise is worse than none because it teaches everyone to ignore it.
  let noise: *i64 = abg_region()
  var i: i64 = 0
  while i < ABG_N { ab_push(noise, 0, AB_ARM_B, abg_saw(i, 100, 4)); i = i + 1 }
  i = 0
  while i < ABG_N { ab_push(noise, 0, AB_ARM_A, abg_saw(i + 2, 100, 4)); i = i + 1 }
  let nv: i64 = ab_axis_verdict(noise, 0, 1, 1, 0)
  gv_kv("noise_control_delta_q10" as *u8, ab_delta_q10(noise, 0))
  gv_check("false-positive-control-same-distribution-not-UP" as *u8, (nv != AB_AX_UP) as i64, ctr)
  gv_check("false-positive-control-same-distribution-not-DOWN" as *u8, (nv != AB_AX_DOWN) as i64, ctr)
87
  // ---- SENSITIVITY: a real separation must be caught ------------------------------------------------
  let up: *i64 = abg_region()
  abg_fill(up, 0, AB_ARM_B, ABG_N, 100, 3)
  abg_fill(up, 0, AB_ARM_A, ABG_N, 130, 3)
  gv_kv("improve_control_delta_q10" as *u8, ab_delta_q10(up, 0))
  gv_check_eq("detects-a-real-improvement" as *u8, ab_axis_verdict(up, 0, 1, 1, 0), AB_AX_UP, ctr)
94
  // ---- THE DIRECTION FLAG IS LOAD-BEARING: identical data, opposite verdict -------------------------
  // Same region, same samples, higher_is_better flipped. If this tooth passed trivially the flag would be
  // decorative, and a frame-time axis (smaller is better) would be graded backwards in production.
  gv_check_eq("direction-flag-inverts-the-same-data" as *u8, ab_axis_verdict(up, 0, 1, 0, 0), AB_AX_DOWN, ctr)
99
  // ---- a real regression, measured in its own right --------------------------------------------------
  let dn: *i64 = abg_region()
  abg_fill(dn, 0, AB_ARM_B, ABG_N, 130, 3)
  abg_fill(dn, 0, AB_ARM_A, ABG_N, 100, 3)
  gv_check_eq("detects-a-real-regression" as *u8, ab_axis_verdict(dn, 0, 1, 1, 0), AB_AX_DOWN, ctr)
105
  // ---- BELOW-MDE IS ITS OWN STATE: significant is not the same as worth shipping --------------------
  // A large n makes a two-unit shift statistically significant. With a declared MDE of 10 units it must
  // report BELOW_MDE, not UP -- otherwise the organ launders statistical significance into progress.
  let sm: *i64 = abg_region()
  abg_fill(sm, 0, AB_ARM_B, ABG_TIGHT, 100, 1)
  abg_fill(sm, 0, AB_ARM_A, ABG_TIGHT, 102, 1)
  let smv: i64 = ab_axis_verdict(sm, 0, 1, 1, 10 * 1024)
  gv_kv("below_mde_delta_q10" as *u8, ab_delta_q10(sm, 0))
  gv_check_eq("small-but-significant-is-BELOW-MDE-not-UP" as *u8, smv, AB_AX_BELOW_MDE, ctr)
  gv_check_eq("same-effect-with-a-low-MDE-is-UP" as *u8, ab_axis_verdict(sm, 0, 1, 1, 1024), AB_AX_UP, ctr)
116
  // ---- family aggregation ---------------------------------------------------------------------------
  let axv: *i64 = sys_mmap(8 * 8)
  axv[0] = AB_AX_UP; axv[1] = AB_AX_UP; axv[2] = AB_AX_UP; axv[3] = AB_AX_DOWN
  gv_check_eq("regressed-dominates-improved" as *u8, ab_family_verdict(axv, 4), AB_V_REGRESSED, ctr)
  axv[0] = AB_AX_UP; axv[1] = AB_AX_FLAT; axv[2] = AB_AX_FLAT; axv[3] = AB_AX_FLAT
  gv_check_eq("one-clean-rise-is-IMPROVED" as *u8, ab_family_verdict(axv, 4), AB_V_IMPROVED, ctr)
  axv[0] = AB_AX_FLAT; axv[1] = AB_AX_FLAT; axv[2] = AB_AX_FLAT; axv[3] = AB_AX_FLAT
  gv_check_eq("all-flat-is-NO-CHANGE" as *u8, ab_family_verdict(axv, 4), AB_V_NO_CHANGE, ctr)
125
  // ---- THE PASS-ON-THE-EMPTY-SET TOOTH ---------------------------------------------------------------
  axv[0] = AB_AX_INSUFF; axv[1] = AB_AX_INSUFF; axv[2] = AB_AX_INSUFF; axv[3] = AB_AX_INSUFF
  gv_check_eq("zero-testable-axes-is-INSUFFICIENT-not-NO-CHANGE" as *u8,
              ab_family_verdict(axv, 4), AB_V_INSUFFICIENT, ctr)
  gv_check_eq("tested-axis-denominator-is-published" as *u8, ab_tested_axes(axv, 4), 0, ctr)
  gv_check_eq("no-axes-at-all-is-INSUFFICIENT" as *u8, ab_family_verdict(axv, 0), AB_V_INSUFFICIENT, ctr)
132
  // ---- the values a second method can recompute from the declared inputs ----------------------------
  gv_values_head()
  gv_kv("crit_t_q10_1axis" as *u8, ab_crit_t_q10(1))
  gv_kv("crit_t_q10_5axes" as *u8, ab_crit_t_q10(5))
  gv_kv("crit_t_q10_16axes" as *u8, ab_crit_t_q10(16))
  gv_kv("min_samples_per_arm" as *u8, ab_min_n())
  gv_kv("region_words" as *u8, ab_words())
  gv_kv("max_axes" as *u8, ab_max_axes())
  gv_kv("improve_mean_B_q10" as *u8, ab_mean_q10(up, 0, AB_ARM_B))
  gv_kv("improve_mean_A_q10" as *u8, ab_mean_q10(up, 0, AB_ARM_A))
  gv_kv("improve_var_B_q10" as *u8, ab_var_q10(up, 0, AB_ARM_B))
  gv_kv("improve_var_A_q10" as *u8, ab_var_q10(up, 0, AB_ARM_A))
  gv_kv("improve_se2_q10" as *u8, ab_se2_q10(up, 0))
  gv_kv("noise_var_B_q10" as *u8, ab_var_q10(noise, 0, AB_ARM_B))
  gv_kv("noise_se2_q10" as *u8, ab_se2_q10(noise, 0))
148
  return gv_verdict("nx_abstat_gate" as *u8, ctr,
      "two-arm contrast with Bonferroni-corrected per-axis tests; abstains rather than acquits" as *u8)
151}
nx_abstat_gate.nx source

`nx_abstat_gate.nx` source
