nx_abstat_aa_gate.nx source
↩ module page · 376 lines · 26668 B
1// nx_abstat_aa_gate.nx -- A/A CALIBRATION, A-PRIORI SAMPLE SIZE, ALWAYS-VALID MONITORING and CUPED for the
2// contrast core (gameengine GE51): the four published requirements for an honest improvement claim, each read
3// from a mirrored primary source and each proven here by a seeded fixture with a KNOWN effect.
4//
5// (1) the A/A bar -- "the null hypothesis should be rejected about 5 percent of the time when a 95 percent
6// confidence level is used" (Kohavi et al., KDD 2007, ref ge-expguide). nx_abstat_gate carries ONE
7// same-distribution trial; one trial cannot measure a RATE. This gate runs thousands of synthetic
8// experiments from a SEEDED generator and asserts the rejection rate sits inside a binomial envelope
9// DERIVED from the trial count -- a rate that is too HIGH is a lying instrument, a rate that is too LOW is
10// a blunt one, and both fail here.
11// (2) a-priori sample size -- n = (4 r sigma / Delta)^2 at 95 percent confidence and 90 percent power, gated
12// against the paper's own worked examples (over 1.6 million; 25,600; under 500,000) so the number the
13// acceptance page prints before reading the arms is the number the field would compute.
14// (3) ALWAYS-VALID INFERENCE (Johari, Pekelis, Walsh, ref ge-alwaysvalid) -- the same A/A experiments are now
15// MONITORED CONTINUOUSLY, one look per pushed pair from the sample floor to the end, and two rules are
16// consulted at every look: the fixed-horizon verdict (the practice the paper measures at "fivefold"
17// Type I inflation) and ab_always_valid (the mSPRT statistic, rejecting when its running maximum reaches
18// ln(1/alpha)). The bar: the always-valid rate stays inside the honest envelope while the peeked
19// fixed-horizon rate leaves it -- a positive and a negative control on the SAME sample stream, so the
20// discrimination is measured, never assumed. The mixing variance is derived from the planted effect the
21// way the estate derives it from the pre-declared MDE; the closed form is KAT-checked against values
22// derived by hand from the formula before any rate is trusted.
23// (4) CUPED (Deng, Xu, Kohavi, Walker, ref ge-cuped) -- paired fixtures (a pre-period covariate X and a
24// metric Y = X + independent noise, so the correlation is KNOWN: rho^2 = 1/2 and theta = 1) prove the
25// estimator's identities from the paper (theta, rho^2, var(Ycv) = var(Y)(1 - rho^2)) on 400,000 pairs,
26// then the A/A rate of the adjusted contrast stays in the envelope, and the POWER of the adjusted
27// contrast at a planted effect exceeds the plain contrast's power on the SAME samples -- the paper's
28// claim ("the same power from half the users") measured as a rate, not asserted.
29// plus a POWER control for the plain test: the same generator with a planted effect must reject almost
30// always, or the A/A rate is cheap for the wrong reason (a test that never rejects has a perfect
31// false-positive rate).
32// DETERMINISTIC BY CONSTRUCTION: the estate's nofloat gates' LCG (seeded), so every rate is reproducible and a
33// RED is an arithmetic change, never a dice roll. Samples are Irwin-Hall-4 (the sum of four uniforms), whose
34// variance is derived from the uniform's width rather than typed. license_tier: ORIGINAL No hw writes.
35import "nx_syscalls.nx"
36import "nx_gate_verdict.nx"
37import "nx_abstat_lib.nx"
38
39// the seeded generator the nofloat gates already use (Knuth MMIX multiplier and increment), masked to non-negative
40const AA_LCG_MUL: i64 = 6364136223846793005
41const AA_LCG_INC: i64 = 1442695040888963407
42const AA_LCG_MASK: i64 = 0x7FFFFFFFFFFFFFFF
43const AA_SEED: i64 = 20260904 // the date this gate was written; any fixed seed is a fixture fact
44const AA_UNIFORM_W: i64 = 1000 // one uniform draw in [0, AA_UNIFORM_W)
45const AA_IH_K: i64 = 4 // Irwin-Hall order: sum of four uniforms, near-normal, integer
46// A/A: TRIALS synthetic experiments of N_PER_ARM samples per arm from ONE distribution
47const AA_TRIALS: i64 = 2000
48const AA_N_PER_ARM: i64 = 100
49const AA_AXES: i64 = 1 // one axis: alpha_each = 0.05, critical t 1.96 (the incumbent's table)
50const AA_ALPHA_PERMIL: i64 = 50 // the bar: about 5 percent of A/A runs reject
51const AA_ENVELOPE_SIGMAS: i64 = 3 // the envelope half-width in binomial standard deviations
52const AA_MDE_Q10: i64 = 1 // an MDE small enough never to turn a rejection into BELOW_MDE
53// POWER: the same generator with a planted mean shift must reject almost always
54const AA_EFFECT: i64 = 405 // five standard errors: se_diff at N_PER_ARM is isqrt(2 x 333333 / 100) = 81 (derived below, printed)
55const AA_POWER_FLOOR_PERMIL: i64 = 900 // the paper's 90 percent power bar
56// SAMPLE-SIZE KATS, in a unit of one ten-thousandth of a dollar so every figure in the paper is integral
57const KS_R: i64 = 2
58const KS_REV_SIGMA: i64 = 300000 // 30 dollars
59const KS_REV_MDE: i64 = 1875 // 5 percent of a 3.75 dollar mean
60const KS_REV_N: i64 = 1638400 // (4 x 2 x 30 / 0.1875)^2 = 1280^2, "over 1.6 million"
61const KS_CHK_SIGMA: i64 = 5000 // 0.5, the Bernoulli std-dev at p = 0.5
62const KS_CHK_MDE: i64 = 250 // 5 percent of 0.5
63const KS_CHK_N: i64 = 25600 // the paper's own figure
64const KS_CONV_SIGMA: i64 = 2179 // sqrt(0.05 x 0.95) = 0.2179
65const KS_CONV_MDE: i64 = 25 // 5 percent of 0.05
66const KS_CONV_BAR: i64 = 500000 // "less than 500,000 users"
67const KS_RELAX: i64 = 4 // relaxing the MDE by 4 drops n by 16: the square law the paper states
68const KS_SQUARE_LAW: i64 = 16
69const AA_I64_BYTES: i64 = 8
70const AA_Q10: i64 = 1024
71// ALWAYS-VALID KATS -- every expected value derived BY HAND from the closed form in nx_abstat_lib and written
72// beside its derivation, so a reader can recompute them without running anything; CPython re-derived the same
73// four numbers from the formula on 2026-09-04 (a second method class):
74// ln Lambda = (1/2) ln( V / (V + tau2) ) + tau2 d^2 / ( 2 V (V + tau2) ), all Q10 (1024 = 1.0)
75const KV_LN_TWO: i64 = 710 // ab_ln_q10(2048): ln 2 = 0.693147 x 1024 = 709.8, the lib's own rounding
76const KV_BAR_20: i64 = 3068 // ab_av_bar_q10(50) = ln 20 = 2.995732 x 1024 = 3067.6
77const KV_BAR_TOL: i64 = 4 // one integer-log rounding step either side
78const KV_A_NULL: i64 = 0 - 355 // d=0, V=1, tau2=1: (1/2) ln(1/2) = -0.346574 x 1024 = -354.9
79const KV_A_TWO: i64 = 669 // d=2, V=1, tau2=1: -0.346574 + 1 x 4 / (2 x 1 x 2) = 0.653426 x 1024 = 669.1
80const KV_A_WIDE: i64 = 0 - 114 // d=0, V=4, tau2=1: (1/2) ln(4/5) = -0.111572 x 1024 = -114.2
81const KV_TOL: i64 = 3
82// CUPED fixture: X = one Irwin-Hall draw, Y = X + an INDEPENDENT Irwin-Hall draw (+ the planted effect on arm A)
83// => var(X) = v, var(Y) = 2v, cov = v => theta = 1 (1024 Q10), rho^2 = 1/2 (512 Q10), var(Ycv)/var(Y) = 1/2
84const CV_THETA_EXPECT: i64 = 1024
85const CV_RHO2_EXPECT: i64 = 512
86const CV_IDENT_TOL_PERMIL: i64 = 30 // 3 percent: the identities hold on 400,000 pairs to well inside this
87const CV_EFFECT: i64 = 230 // two standard errors of the PLAIN contrast on Y (se_y = isqrt(4 x 333333 / 100) = 115):
88 // the plain test lands near 50 percent power, where a halved variance shows most
89const CV_POWER_GAIN_FLOOR_PERMIL: i64 = 150 // the adjusted contrast must win by at least this much on the same samples
90const CV_XY_WORDS: i64 = 2
91const CV_OUT_WORDS: i64 = 2
92const CV_CNT_WORDS: i64 = 3
93const CV_CONST_X: i64 = 7 // the constant covariate of the neg-control
94const CV_CONST_N: i64 = 4
95
96func aa_lcg(st: *i64) -> i64 { st[0] = (st[0] * AA_LCG_MUL + AA_LCG_INC) & AA_LCG_MASK; return st[0] }
97func aa_uniform(st: *i64) -> i64 { return aa_lcg(st) % AA_UNIFORM_W }
98func aa_sample(st: *i64) -> i64 {
99 var s: i64 = 0
100 var k: i64 = 0
101 while k < AA_IH_K { s = s + aa_uniform(st); k = k + 1 }
102 return s
103}
104func aa_isqrt(v: i64) -> i64 {
105 if v <= 0 { return 0 }
106 var x: i64 = v
107 var y: i64 = (x + 1) / 2
108 while y < x { x = y; y = (x + v / x) / 2 }
109 return x
110}
111func aa_is_reject(v: i64) -> i64 {
112 if v == AB_AX_UP { return 1 }
113 if v == AB_AX_DOWN { return 1 }
114 if v == AB_AX_BELOW_MDE { return 1 }
115 return 0
116}
117// one synthetic experiment: both arms from the generator, arm A shifted by `effect`; returns the axis verdict
118func aa_trial(st: *i64, reg: *i64, effect: i64) -> i64 {
119 ab_init(reg)
120 var i: i64 = 0
121 while i < AA_N_PER_ARM {
122 ab_push(reg, 0, AB_ARM_B, aa_sample(st))
123 ab_push(reg, 0, AB_ARM_A, aa_sample(st) + effect)
124 i = i + 1
125 }
126 return ab_axis_verdict(reg, 0, AA_AXES, 1, AA_MDE_Q10)
127}
128// count rejections of the null (UP or DOWN) over TRIALS experiments; BELOW_MDE is a rejection too at this MDE
129func aa_reject_count(st: *i64, reg: *i64, effect: i64, insuff: *i64) -> i64 {
130 var rej: i64 = 0
131 var t: i64 = 0
132 while t < AA_TRIALS {
133 let v: i64 = aa_trial(st, reg, effect)
134 rej = rej + aa_is_reject(v)
135 if v == AB_AX_INSUFF { insuff[0] = insuff[0] + 1 }
136 t = t + 1
137 }
138 return rej
139}
140
141// ---- (3) one experiment monitored CONTINUOUSLY --------------------------------------------------------
142// After every paired push from the sample floor onward, two rules look at the SAME running contrast:
143// out[0] = 1 if the always-valid statistic ever reached the bar (its running max, Theorem 1),
144// out[1] = 1 if the fixed-horizon verdict rejected at ANY look (the peeking practice the paper measures).
145func aa_seq_trial(st: *i64, reg: *i64, effect: i64, tau2: i64, bar: i64, out: *i64) -> i64 {
146 ab_init(reg)
147 out[0] = 0
148 out[1] = 0
149 let floor_n: i64 = ab_min_n()
150 var i: i64 = 0
151 while i < AA_N_PER_ARM {
152 ab_push(reg, 0, AB_ARM_B, aa_sample(st))
153 ab_push(reg, 0, AB_ARM_A, aa_sample(st) + effect)
154 i = i + 1
155 if i >= floor_n {
156 let d: i64 = ab_delta_q10(reg, 0)
157 let se2: i64 = ab_se2_q10(reg, 0)
158 let lnl: i64 = ab_always_valid(d, se2, tau2)
159 if lnl != AB_UNOBSERVED { if lnl >= bar { out[0] = 1 } }
160 out[1] = out[1] + aa_is_reject(ab_axis_verdict(reg, 0, AA_AXES, 1, AA_MDE_Q10))
161 }
162 }
163 if out[1] > 0 { out[1] = 1 }
164 return 0
165}
166// rates over TRIALS monitored experiments: cnt[0] always-valid rejections, cnt[1] peeked fixed-horizon rejections
167func aa_seq_counts(st: *i64, reg: *i64, effect: i64, tau2: i64, bar: i64, out: *i64, cnt: *i64) -> i64 {
168 cnt[0] = 0
169 cnt[1] = 0
170 var t: i64 = 0
171 while t < AA_TRIALS {
172 aa_seq_trial(st, reg, effect, tau2, bar, out)
173 cnt[0] = cnt[0] + out[0]
174 cnt[1] = cnt[1] + out[1]
175 t = t + 1
176 }
177 return 0
178}
179
180// ---- (4) CUPED fixture: paired (X, Y) with a KNOWN correlation ----------------------------------------
181func aa_cv_pair(st: *i64, xy: *i64, effect: i64) -> i64 {
182 let x: i64 = aa_sample(st)
183 xy[0] = x
184 xy[1] = x + aa_sample(st) + effect
185 return 0
186}
187// one paired experiment: the CUPED accumulator and the plain accumulator see the SAME y samples, so their
188// verdicts are a paired comparison. use_cal=1 also feeds the whole-run calibration accumulator.
189// out[0] = the CUPED verdict, out[1] = the plain verdict. Returns the trial's pooled theta.
190func aa_cv_trial(st: *i64, cv: *i64, reg: *i64, cal: *i64, use_cal: i64, xy: *i64, effect: i64, out: *i64) -> i64 {
191 ab_cuped_init(cv)
192 ab_init(reg)
193 var i: i64 = 0
194 while i < AA_N_PER_ARM {
195 aa_cv_pair(st, xy, 0)
196 ab_cuped(cv, AB_ARM_B, xy[1], xy[0])
197 ab_push(reg, 0, AB_ARM_B, xy[1])
198 if use_cal == 1 { ab_cuped(cal, AB_ARM_B, xy[1], xy[0]) }
199 aa_cv_pair(st, xy, effect)
200 ab_cuped(cv, AB_ARM_A, xy[1], xy[0])
201 ab_push(reg, 0, AB_ARM_A, xy[1])
202 if use_cal == 1 { ab_cuped(cal, AB_ARM_A, xy[1], xy[0]) }
203 i = i + 1
204 }
205 let theta: i64 = ab_cuped_theta_q10(cv)
206 out[0] = ab_cuped_verdict(cv, theta, AA_AXES, 1, AA_MDE_Q10)
207 out[1] = ab_axis_verdict(reg, 0, AA_AXES, 1, AA_MDE_Q10)
208 return theta
209}
210// cnt[0] CUPED rejections, cnt[1] plain rejections, cnt[2] CUPED INSUFF, over TRIALS paired experiments
211func aa_cv_counts(st: *i64, cv: *i64, reg: *i64, cal: *i64, use_cal: i64, xy: *i64, effect: i64, out: *i64, cnt: *i64) -> i64 {
212 cnt[0] = 0
213 cnt[1] = 0
214 cnt[2] = 0
215 var t: i64 = 0
216 while t < AA_TRIALS {
217 aa_cv_trial(st, cv, reg, cal, use_cal, xy, effect, out)
218 cnt[0] = cnt[0] + aa_is_reject(out[0])
219 cnt[1] = cnt[1] + aa_is_reject(out[1])
220 if out[0] == AB_AX_INSUFF { cnt[2] = cnt[2] + 1 }
221 t = t + 1
222 }
223 return 0
224}
225// |a - expected| <= |expected| x tol_permil / 1000
226func aa_within_permil(a: i64, expected: i64, tol_permil: i64) -> i64 {
227 var d: i64 = a - expected
228 if d < 0 { d = 0 - d }
229 var e: i64 = expected
230 if e < 0 { e = 0 - e }
231 if d * 1000 <= e * tol_permil { return 1 }
232 return 0
233}
234
235func main() -> i64 {
236 let ctr: *i64 = gv_ctr()
237 gv_puts("nx_abstat_aa_gate -- A/A rate, a-priori sample size, always-valid monitoring and CUPED behind an honest improvement claim\n\n" as *u8)
238 let st: *i64 = sys_mmap(AA_I64_BYTES) as *i64
239 st[0] = AA_SEED
240 let reg: *i64 = sys_mmap(ab_words() * AA_I64_BYTES) as *i64
241 let insuff: *i64 = sys_mmap(AA_I64_BYTES) as *i64
242 insuff[0] = 0
243
244 // ---- the generator is what it claims: variance derived, not typed ------------------------------
245 // Irwin-Hall-4 over uniform(0..W-1): var = K x (W^2 - 1) / 12; the std error of a two-arm mean
246 // difference at n per arm is sqrt(2 var / n). Printed so the planted effect reads in standard errors.
247 let var_u: i64 = (AA_UNIFORM_W * AA_UNIFORM_W - 1) / 12
248 let var_s: i64 = AA_IH_K * var_u
249 let se_diff: i64 = aa_isqrt(2 * var_s / AA_N_PER_ARM)
250 gv_check("fixture-derivation-planted-effect-is-at-least-four-standard-errors (power is not luck)" as *u8, AA_EFFECT >= 4 * se_diff, ctr)
251
252 // ---- (1) A/A: the null is rejected about 5 percent of the time ---------------------------------
253 let rej: i64 = aa_reject_count(st, reg, 0, insuff)
254 let expected: i64 = AA_TRIALS * AA_ALPHA_PERMIL / 1000
255 // binomial sd = sqrt(trials x p x (1-p)); computed in permil arithmetic then rooted
256 let sd: i64 = aa_isqrt(AA_TRIALS * AA_ALPHA_PERMIL * (1000 - AA_ALPHA_PERMIL) / (1000 * 1000))
257 let lo: i64 = expected - AA_ENVELOPE_SIGMAS * sd
258 let hi: i64 = expected + AA_ENVELOPE_SIGMAS * sd
259 gv_puts(" [A/A] trials=" as *u8); gv_num(AA_TRIALS); gv_puts(" rejections=" as *u8); gv_num(rej); gv_puts(" expected=" as *u8); gv_num(expected); gv_puts(" envelope=[" as *u8); gv_num(lo); gv_puts("," as *u8); gv_num(hi); gv_puts("] insufficient=" as *u8); gv_num(insuff[0]); gv_puts("\n" as *u8)
260 gv_check_eq("fixture-reached-every-A/A-trial-was-testable (INSUFF count)" as *u8, insuff[0], 0, ctr)
261 gv_check("A/A-rejection-rate-is-not-too-HIGH (a lying instrument rejects the null it was handed)" as *u8, rej <= hi, ctr)
262 gv_check("A/A-rejection-rate-is-not-too-LOW (a blunt instrument never rejects; that is not a calibration)" as *u8, rej >= lo, ctr)
263 gv_check("anti-vacuity-the-envelope-excludes-zero-rejections" as *u8, lo > 0, ctr)
264
265 // ---- POWER control: a planted effect is found almost always ------------------------------------
266 insuff[0] = 0
267 let rej_eff: i64 = aa_reject_count(st, reg, AA_EFFECT, insuff)
268 let power_permil: i64 = rej_eff * 1000 / AA_TRIALS
269 gv_puts(" [power] effect=" as *u8); gv_num(AA_EFFECT); gv_puts(" se_diff~" as *u8); gv_num(se_diff); gv_puts(" rejections=" as *u8); gv_num(rej_eff); gv_puts(" power_permil=" as *u8); gv_num(power_permil); gv_puts("\n" as *u8)
270 gv_check("positive-control-planted-effect-rejects-at-or-above-the-90-percent-power-bar" as *u8, power_permil >= AA_POWER_FLOOR_PERMIL, ctr)
271 gv_check("neg-control-power-run-is-not-the-A/A-run (it must reject FAR more often)" as *u8, rej_eff > hi, ctr)
272
273 // ---- (2) a-priori sample size against the paper's worked examples ------------------------------
274 gv_check_eq("required-n-revenue-example-is-1638400 (4x2x30/0.1875 squared, over 1.6 million)" as *u8, ab_sample_size_aa(KS_R, KS_REV_SIGMA, KS_REV_MDE), KS_REV_N, ctr)
275 gv_check_eq("required-n-checkout-example-is-25600 (the paper's own figure)" as *u8, ab_sample_size_aa(KS_R, KS_CHK_SIGMA, KS_CHK_MDE), KS_CHK_N, ctr)
276 let conv: i64 = ab_sample_size_aa(KS_R, KS_CONV_SIGMA, KS_CONV_MDE)
277 gv_check("required-n-conversion-example-is-under-500000 (the paper's bound)" as *u8, conv < KS_CONV_BAR, ctr)
278 gv_check("required-n-conversion-example-is-not-trivially-small (above the revenue example / 10)" as *u8, conv > KS_REV_N / 10, ctr)
279 gv_check_eq("square-law-relaxing-the-MDE-4x-drops-n-16x (the paper's own arithmetic)" as *u8, ab_sample_size_aa(KS_R, KS_CHK_SIGMA, KS_CHK_MDE) / ab_sample_size_aa(KS_R, KS_CHK_SIGMA, KS_CHK_MDE * KS_RELAX), KS_SQUARE_LAW, ctr)
280 gv_check_eq("neg-control-zero-MDE-is-UNOBSERVED-not-a-huge-number" as *u8, ab_sample_size_aa(KS_R, KS_CHK_SIGMA, 0), AB_UNOBSERVED, ctr)
281 gv_check_eq("neg-control-zero-variants-is-UNOBSERVED" as *u8, ab_sample_size_aa(0, KS_CHK_SIGMA, KS_CHK_MDE), AB_UNOBSERVED, ctr)
282
283 // ---- (3) always-valid inference: the closed form, then the monitored rates ---------------------
284 gv_check_eq("kat-ln-of-one-is-zero (ab_ln_q10 on a Q10 one)" as *u8, ab_ln_q10(AA_Q10), 0, ctr)
285 gv_check_eq("kat-ln-of-two-is-the-lib's-ln2 (710 Q10)" as *u8, ab_ln_q10(2 * AA_Q10), KV_LN_TWO, ctr)
286 gv_check_eq("kat-ln-of-a-half-is-minus-ln2 (the sign survives the integer log)" as *u8, ab_ln_q10(AA_Q10 / 2), 0 - KV_LN_TWO, ctr)
287 gv_check_near("kat-rejection-bar-at-alpha-50-permil-is-ln20 (2.9957 Q10 3068)" as *u8, ab_av_bar_q10(AA_ALPHA_PERMIL), KV_BAR_20, KV_BAR_TOL, ctr)
288 gv_check_near("kat-lnLambda-at-zero-contrast-V1-tau1-is-half-ln-half (-355)" as *u8, ab_always_valid(0, AA_Q10, AA_Q10), KV_A_NULL, KV_TOL, ctr)
289 gv_check_near("kat-lnLambda-at-contrast-2-V1-tau1-is-669 (the exponent term equals one)" as *u8, ab_always_valid(2 * AA_Q10, AA_Q10, AA_Q10), KV_A_TWO, KV_TOL, ctr)
290 gv_check_near("kat-lnLambda-at-zero-contrast-V4-tau1-is-half-ln-four-fifths (-114)" as *u8, ab_always_valid(0, 4 * AA_Q10, AA_Q10), KV_A_WIDE, KV_TOL, ctr)
291 gv_check_near("kat-lnLambda-is-even-in-the-contrast (a negative contrast scores the same evidence)" as *u8, ab_always_valid(0 - 2 * AA_Q10, AA_Q10, AA_Q10), KV_A_TWO, KV_TOL, ctr)
292 gv_check_eq("neg-control-zero-variance-is-UNOBSERVED-not-infinite-evidence" as *u8, ab_always_valid(2 * AA_Q10, 0, AA_Q10), AB_UNOBSERVED, ctr)
293 gv_check_eq("neg-control-zero-mixing-variance-is-UNOBSERVED" as *u8, ab_always_valid(2 * AA_Q10, AA_Q10, 0), AB_UNOBSERVED, ctr)
294 // the mixing variance is derived from the effect the experiment is declared to care about, as the lib derives
295 // it from the MDE; here that is the planted effect, in Q10
296 let tau2: i64 = ab_av_tau2_from_mde_q10(AA_EFFECT * AA_Q10)
297 let bar: i64 = ab_av_bar_q10(AA_ALPHA_PERMIL)
298 let out: *i64 = sys_mmap(CV_OUT_WORDS * AA_I64_BYTES) as *i64
299 let cnt: *i64 = sys_mmap(CV_CNT_WORDS * AA_I64_BYTES) as *i64
300 aa_seq_counts(st, reg, 0, tau2, bar, out, cnt)
301 let av_rej: i64 = cnt[0]
302 let peek_rej: i64 = cnt[1]
303 gv_puts(" [monitored A/A] looks-per-trial=" as *u8); gv_num(AA_N_PER_ARM - ab_min_n() + 1); gv_puts(" always_valid_rejections=" as *u8); gv_num(av_rej); gv_puts(" peeked_fixed_horizon_rejections=" as *u8); gv_num(peek_rej); gv_puts(" envelope_hi=" as *u8); gv_num(hi); gv_puts(" tau2_q10=" as *u8); gv_num(tau2); gv_puts(" bar_q10=" as *u8); gv_num(bar); gv_puts("\n" as *u8)
304 gv_check("always-valid-A/A-rate-under-continuous-monitoring-stays-inside-the-honest-envelope (Type I controlled at every look)" as *u8, av_rej <= hi, ctr)
305 gv_check("neg-control-peeked-fixed-horizon-A/A-rate-LEAVES-the-envelope (the inflation the paper measures)" as *u8, peek_rej > hi, ctr)
306 gv_check("discrimination-always-valid-rejects-strictly-less-often-than-peeking-on-the-same-stream" as *u8, av_rej < peek_rej, ctr)
307 aa_seq_counts(st, reg, AA_EFFECT, tau2, bar, out, cnt)
308 let av_power_permil: i64 = cnt[0] * 1000 / AA_TRIALS
309 gv_puts(" [monitored power] always_valid_rejections=" as *u8); gv_num(cnt[0]); gv_puts(" power_permil=" as *u8); gv_num(av_power_permil); gv_puts("\n" as *u8)
310 gv_check("positive-control-always-valid-finds-the-planted-effect-at-or-above-the-90-percent-power-bar" as *u8, av_power_permil >= AA_POWER_FLOOR_PERMIL, ctr)
311
312 // ---- (4) CUPED: the paper's identities on a known-correlation fixture, then rates -------------
313 let cv: *i64 = sys_mmap(ab_cuped_words() * AA_I64_BYTES) as *i64
314 let cal: *i64 = sys_mmap(ab_cuped_words() * AA_I64_BYTES) as *i64
315 let xy: *i64 = sys_mmap(CV_XY_WORDS * AA_I64_BYTES) as *i64
316 ab_cuped_init(cal)
317 aa_cv_counts(st, cv, reg, cal, 1, xy, 0, out, cnt)
318 let cv_rej: i64 = cnt[0]
319 let plain_rej_paired: i64 = cnt[1]
320 let cv_insuff: i64 = cnt[2]
321 let theta_cal: i64 = ab_cuped_theta_q10(cal)
322 let rho2_cal: i64 = ab_cuped_rho2_q10(cal)
323 let n_cal: i64 = ab_cuped_n(cal, AB_CV_POOL)
324 let var_y_b: i64 = cal[ab_cv_base(AB_ARM_B) + AB_CV_M2Y] / ((ab_cuped_n(cal, AB_ARM_B) - 1) * AA_Q10)
325 let var_cv_b: i64 = ab_cuped_var_q10(cal, AB_ARM_B, theta_cal)
326 let ratio_q10: i64 = ab_ratio_q10(var_cv_b, var_y_b)
327 gv_puts(" [CUPED identities] pairs=" as *u8); gv_num(n_cal); gv_puts(" theta_q10=" as *u8); gv_num(theta_cal); gv_puts(" rho2_q10=" as *u8); gv_num(rho2_cal); gv_puts(" var_y=" as *u8); gv_num(var_y_b); gv_puts(" var_cv=" as *u8); gv_num(var_cv_b); gv_puts(" ratio_q10=" as *u8); gv_num(ratio_q10); gv_puts("\n" as *u8)
328 gv_check("fixture-reached-the-calibration-pool-holds-every-pair (2 x trials x n)" as *u8, n_cal == 2 * AA_TRIALS * AA_N_PER_ARM, ctr)
329 gv_check("cuped-theta-is-cov-over-var-and-equals-one-on-this-fixture (eq 4, within 3 percent)" as *u8, aa_within_permil(theta_cal, CV_THETA_EXPECT, CV_IDENT_TOL_PERMIL), ctr)
330 gv_check("cuped-rho-squared-is-one-half-on-this-fixture (within 3 percent)" as *u8, aa_within_permil(rho2_cal, CV_RHO2_EXPECT, CV_IDENT_TOL_PERMIL), ctr)
331 gv_check("cuped-variance-ratio-var(Ycv)/var(Y)-equals-one-minus-rho-squared (eq 5, within 3 percent)" as *u8, aa_within_permil(ratio_q10, AA_Q10 - rho2_cal, CV_IDENT_TOL_PERMIL), ctr)
332 gv_check("cuped-adjusted-variance-never-exceeds-the-plain-variance" as *u8, var_cv_b <= var_y_b, ctr)
333 gv_check_eq("fixture-reached-every-CUPED-A/A-trial-was-testable (INSUFF count)" as *u8, cv_insuff, 0, ctr)
334 gv_puts(" [CUPED A/A] cuped_rejections=" as *u8); gv_num(cv_rej); gv_puts(" plain_rejections_same_samples=" as *u8); gv_num(plain_rej_paired); gv_puts(" envelope=[" as *u8); gv_num(lo); gv_puts("," as *u8); gv_num(hi); gv_puts("]\n" as *u8)
335 gv_check("cuped-A/A-rejection-rate-is-not-too-HIGH (variance reduction must not manufacture findings)" as *u8, cv_rej <= hi, ctr)
336 gv_check("cuped-A/A-rejection-rate-is-not-too-LOW" as *u8, cv_rej >= lo, ctr)
337 // neg-control: with theta forced to zero the adjusted contrast IS the plain contrast, exactly
338 let theta_last: i64 = aa_cv_trial(st, cv, reg, cal, 0, xy, 0, out)
339 gv_check_eq("neg-control-theta-zero-reproduces-the-plain-delta-EXACTLY (the adjustment is the only difference)" as *u8, ab_cuped_delta_q10(cv, 0), ab_delta_q10(reg, 0), ctr)
340 gv_check("the-per-trial-theta-is-read-from-the-pool-and-is-near-one-too" as *u8, aa_within_permil(theta_last, CV_THETA_EXPECT, 5 * CV_IDENT_TOL_PERMIL), ctr)
341 // POWER on the same samples: the adjusted contrast must find a planted effect more often than the plain one
342 aa_cv_counts(st, cv, reg, cal, 0, xy, CV_EFFECT, out, cnt)
343 let cv_power: i64 = cnt[0] * 1000 / AA_TRIALS
344 let plain_power: i64 = cnt[1] * 1000 / AA_TRIALS
345 let se_y: i64 = aa_isqrt(2 * 2 * var_s / AA_N_PER_ARM)
346 gv_puts(" [CUPED power] effect=" as *u8); gv_num(CV_EFFECT); gv_puts(" se_y~" as *u8); gv_num(se_y); gv_puts(" cuped_power_permil=" as *u8); gv_num(cv_power); gv_puts(" plain_power_permil=" as *u8); gv_num(plain_power); gv_puts("\n" as *u8)
347 gv_check("fixture-derivation-cuped-effect-is-about-two-plain-standard-errors (where a halved variance shows)" as *u8, CV_EFFECT >= 2 * se_y - se_y / 10, ctr)
348 gv_check("cuped-power-exceeds-plain-power-on-the-same-samples-by-the-declared-floor (the paper's sensitivity claim, measured)" as *u8, cv_power - plain_power >= CV_POWER_GAIN_FLOOR_PERMIL, ctr)
349 gv_check("neg-control-plain-power-at-two-standard-errors-is-not-already-near-one (the comparison had room to move)" as *u8, plain_power < AA_POWER_FLOOR_PERMIL, ctr)
350 // neg-control: a constant covariate carries no information and the estimator must refuse, never divide by zero
351 ab_cuped_init(cv)
352 var k: i64 = 0
353 while k < CV_CONST_N { ab_cuped(cv, AB_ARM_B, aa_sample(st), CV_CONST_X); ab_cuped(cv, AB_ARM_A, aa_sample(st), CV_CONST_X); k = k + 1 }
354 gv_check_eq("neg-control-constant-covariate-theta-is-UNOBSERVED" as *u8, ab_cuped_theta_q10(cv), AB_UNOBSERVED, ctr)
355
356 gv_values_head()
357 gv_kv("aa_trials" as *u8, AA_TRIALS)
358 gv_kv("aa_rejections" as *u8, rej)
359 gv_kv("aa_expected" as *u8, expected)
360 gv_kv("aa_envelope_lo" as *u8, lo)
361 gv_kv("aa_envelope_hi" as *u8, hi)
362 gv_kv("power_permil" as *u8, power_permil)
363 gv_kv("required_n_revenue" as *u8, ab_sample_size_aa(KS_R, KS_REV_SIGMA, KS_REV_MDE))
364 gv_kv("required_n_conversion" as *u8, conv)
365 gv_kv("av_rejections" as *u8, av_rej)
366 gv_kv("peeked_rejections" as *u8, peek_rej)
367 gv_kv("av_power_permil" as *u8, av_power_permil)
368 gv_kv("av_bar_q10" as *u8, bar)
369 gv_kv("cuped_theta_q10" as *u8, theta_cal)
370 gv_kv("cuped_rho2_q10" as *u8, rho2_cal)
371 gv_kv("cuped_var_ratio_q10" as *u8, ratio_q10)
372 gv_kv("cuped_aa_rejections" as *u8, cv_rej)
373 gv_kv("cuped_power_permil" as *u8, cv_power)
374 gv_kv("plain_power_permil_same_samples" as *u8, plain_power)
375 return gv_verdict("nx_abstat_aa_gate" as *u8, ctr, "seeded A/A rates inside a binomial envelope derived from the trial count for the plain, the continuously-monitored always-valid and the CUPED contrasts, the peeked fixed-horizon rate as the neg-control that leaves it, planted-effect power floors and a paired power gain, the paper's sample-size examples and the closed form's hand-derived values as known answers" as *u8)
376}