nx_frameab.nx source
↩ module page · 372 lines · 19295 B
1// nx_frameab.nx -- THE FIRST REAL A/B ON THE ESTATE: two frametrace windows, three axes, one corrected verdict.
2//
3// WHY THIS ORGAN AND WHY NOW. nx_abstat_lib shipped the two-arm contrast arithmetic, and the acceptance panel
4// supplies exactly ONE voting lens (a single percept scalar over a whole frame), so K=1 and the family-wise
5// correction was a no-op with nothing to correct across. This organ is the wiring that fixes that for the
6// PERFORMANCE subject, and it needs no new capture, no lens and no calibration corpus, because the arms
7// ALREADY EXIST: knowledge/store/frametrace- is append-only and its last column carries EVERY INDIVIDUAL
8// FRAME TIME of the window -- 256 replicated samples per row against a floor of 30.
9//
10// THE CONFOUND GUARD IS THE POINT, NOT A COURTESY. The live plane holds beach rows at budget 94, 84 and 6 ms.
11// Budget is derived from the display refresh, so ROWS AT DIFFERENT BUDGETS CAME FROM DIFFERENT MACHINES, and
12// contrasting them would measure the hardware while reporting a verdict about the build. This organ REFUSES
13// such a pair by name (CONFOUNDED-BUDGET) instead of returning a confident number.
14// * A comparison whose arms differ in a dimension you did not intend to vary is not an experiment, and it
15// fails in the flattering direction: a faster machine reads as a better build.
16//
17// THREE AXES, ALL PER-FRAME, ALL FROM THE SAME WINDOW, SO ALL WITH REAL WITHIN-ARM VARIANCE:
18// 0 frame time ms -- LOWER is better
19// 1 over-budget indicator -- LOWER is better (1 when the frame missed its budget)
20// 2 jank indicator -- LOWER is better (1 when the frame exceeded 2x its own window median,
21// the shipped-industry definition nx_perf_lib already uses)
22// K=3 puts the Bonferroni critical t at 2447 rather than 2007: three correlated views of one window would
23// otherwise give three chances to find an improvement. Bonferroni is conservative under ANY correlation,
24// which is exactly why it is the right correction for axes drawn from the same frames.
25//
26// THE MINIMUM DETECTABLE EFFECT IS DERIVED, NOT PICKED. Frame samples are INTEGER MILLISECONDS, so an effect
27// below 1 ms is beneath the instrument's own resolution and must not be called an improvement however
28// significant it is. That is the MDE for axis 0 and its provenance is printed. The indicator axes are already
29// rates, so their MDE is 0 and BELOW-MDE can never fire on them -- stated here rather than left to be found.
30//
31// KNOWN LIMIT, NAMED RATHER THAN HIDDEN: a rate axis whose arms are BOTH constant (every frame over budget in
32// one window, none in the other) has an undefined t statistic, so this organ reports INSUFFICIENT with the
33// reason ZERO-VARIANCE even though the underlying shift is real and large. That is the safe direction --
34// abstain, never acquit -- but the right instrument for a rate is a two-proportion test, which this organ
35// does not have. Measured live on its own fixture: a 100-percent-to-0 over-budget shift read INSUFF.
36//
37// COMPOSES, NEVER RE-IMPLEMENTS: fpc_row_frames for the CSV (the incumbent parser, with its own bad-input
38// handling), pf_sample/pf_n/pf_budget/pf_pct for the ring, ab_push/ab_axis_verdict/ab_family_verdict for the
39// statistics. This organ owns only the SELECTION of two rows and the refusal.
40//
41// nx_frameab <world> [mde_ms] [plane=<prefix>] -- contrast the two most recent windows for <world>
42// EXIT: 0 IMPROVED or NO-CHANGE | 1 REGRESSED | 2 usage | 3 INSUFFICIENT (abstained, reason named)
43// | 4 REFUSED (confound named) | 5 plane unreadable or fewer than two rows
44// license_tier: ORIGINAL No hw writes (Rule 26).
45import "nx_syscalls.nx"
46import "nx_frame_pacing_lib.nx"
47import "nx_abstat_lib.nx"
48
49const FAB_ROWCAP: i64 = 65536
50// THE UNIT OF REPLICATION IS THE WINDOW, NOT THE FRAME, AND THE FIRST CUT OF THIS ORGAN GOT IT WRONG.
51// Pushing 256 frames from ONE window as 256 observations is PSEUDOREPLICATION: frames inside a single run
52// share a thermal state, a GPU clock, a scene and a page load, so they are not independent draws. Treating
53// them as independent inflates n by the window length and shrinks the standard error by its square root --
54// about 16x here -- which makes a confident verdict out of one run. The honest sample size is the number of
55// WINDOWS, so each window contributes exactly ONE observation per axis and NX_MC_MIN_N means 30 RUNS.
56// * A LARGE n ASSEMBLED FROM ONE RUN IS NOT REPLICATION, IT IS THE SAME MEASUREMENT WRITTEN DOWN N TIMES,
57// AND IT FAILS IN THE FLATTERING DIRECTION BECAUSE IT MAKES EVERYTHING LOOK SIGNIFICANT.
58const FAB_AXES: i64 = 3
59const FAB_AX_FRAME: i64 = 0
60const FAB_AX_OVER: i64 = 1
61const FAB_AX_JANK: i64 = 2
62const FAB_MDE_MS_DEFAULT: i64 = 1 // DERIVED: samples are integer ms, so 1 ms is the instrument resolution
63const FAB_JANK_MULT: i64 = 2 // nx_perf_lib PF_SPIKE_MULT, the shipped jank definition
64const FAB_MEDIAN_PERMIL: i64 = 500
65const FAB_US_PER_MS: i64 = 1000 // a window MEAN is carried in microseconds so it survives a 1 ms frame timer
66const FAB_PERMIL: i64 = 1000
67const FAB_NUMCAP: i64 = 32
68
69const FAB_EXIT_OK: i64 = 0
70const FAB_EXIT_REGRESSED: i64 = 1
71const FAB_EXIT_USAGE: i64 = 2
72const FAB_EXIT_INSUFF: i64 = 3
73const FAB_EXIT_REFUSED: i64 = 4
74const FAB_EXIT_IO: i64 = 5
75
76func fab_p(s: *u8) -> i64 { sys_write(1, s, fpc_slen(s)); return 0 }
77func fab_pn(v: i64) -> i64 {
78 let t: *u8 = sys_mmap(FAB_NUMCAP)
79 var n: i64 = 0
80 var x: i64 = v
81 if x < 0 { t[0] = 45; n = 1; x = 0 - x }
82 if x == 0 { t[n] = 48; n = n + 1 } else {
83 let d: *u8 = sys_mmap(FAB_NUMCAP)
84 var k: i64 = 0
85 while x > 0 { d[k] = (48 + (x % 10)) as u8; x = x / 10; k = k + 1 }
86 while k > 0 { k = k - 1; t[n] = d[k]; n = n + 1 }
87 }
88 sys_write(1, t, n)
89 return 0
90}
91func fab_kv(k: *u8, v: i64) -> i64 { fab_p(k); fab_p("=" as *u8); fab_pn(v); fab_p("\n" as *u8); return 0 }
92func fab_has_prefix(s: *u8, n: i64, p: *u8) -> i64 {
93 let pl: i64 = fpc_slen(p)
94 if n < pl { return 0 }
95 var i: i64 = 0
96 while i < pl { if s[i] != p[i] { return 0 } i = i + 1 }
97 return 1
98}
99
100// the LAST TWO rows for a world, oldest of the pair into out_b (baseline), newest into out_a (candidate).
101// Returns the number of matching rows FOUND (0, 1 or 2+), -1 when the plane is unreadable. Selection only --
102// the row grammar itself stays fpc's.
103func fab_last_two(plane: *u8, world: *u8, out_b: *u8, out_a: *u8, rows_seen: *i64) -> i64 {
104 let lp: *i64 = sys_mmap(16) as *i64
105 let b: *u8 = sts_load_fit(plane, lp)
106 rows_seen[0] = 0
107 if (b as i64) == 0 { return 0 - 1 }
108 let n: i64 = lp[0]
109 var s1: i64 = 0 - 1
110 var e1: i64 = 0
111 var s2: i64 = 0 - 1
112 var e2: i64 = 0
113 var found: i64 = 0
114 var i: i64 = 0
115 while i < n {
116 let e: i64 = fpc_eol(b, n, i)
117 if e > i { if b[i] != (FPC_CH_HASH as u8) {
118 rows_seen[0] = rows_seen[0] + 1
119 let ws: i64 = fpc_col_start(b, i, e, FPC_COL_WORLD)
120 if ws >= 0 { if fpc_slice_eq(b, ws, fpc_col_end(b, ws, e), world) == 1 {
121 s1 = s2; e1 = e2 // previous newest slides down to baseline
122 s2 = i; e2 = e
123 found = found + 1
124 } }
125 } }
126 i = e + 1
127 }
128 if found < 2 { return found }
129 var o: i64 = 0
130 var j: i64 = s1
131 while j < e1 { if o < FAB_ROWCAP - 1 { if b[j] != (FPC_CH_CR as u8) { out_b[o] = b[j]; o = o + 1 } } j = j + 1 }
132 out_b[o] = 0 as u8
133 o = 0
134 j = s2
135 while j < e2 { if o < FAB_ROWCAP - 1 { if b[j] != (FPC_CH_CR as u8) { out_a[o] = b[j]; o = o + 1 } } j = j + 1 }
136 out_a[o] = 0 as u8
137 return found
138}
139
140// THE CONFOUND DECISION, as a callable so a gate can bite it in-process rather than by driving main().
141// Budget follows the display refresh, so two windows at different budgets came from different machines.
142// Returns 1 when the pair is CONFOUNDED and must be refused, 0 when it may be contrasted.
143func fab_confounded(bud_b: i64, bud_a: i64) -> i64 {
144 if bud_b != bud_a { return 1 }
145 return 0
146}
147// the two per-frame indicators, named so the gate tests the DERIVATION and not just the plumbing
148func fab_over_of(ms: i64, budget: i64) -> i64 { if ms > budget { return 1 } return 0 }
149func fab_jank_of(ms: i64, median: i64) -> i64 {
150 if median <= 0 { return 0 }
151 if ms > (median * FAB_JANK_MULT) { return 1 }
152 return 0
153}
154
155// push one parsed window into an arm: frame ms, over-budget indicator, jank indicator.
156// The median is the ARM'S OWN window median, because jank is defined relative to the window a frame sat in --
157// using a shared median would let a slower arm launder its spikes against the faster arm's baseline.
158func fab_push_arm(reg: *i64, arm: i64, r: *i64, budget: i64) -> i64 {
159 let cnt: i64 = pf_n(r)
160 let med: i64 = pf_pct(r, FAB_MEDIAN_PERMIL)
161 var i: i64 = 0
162 while i < cnt {
163 let ms: i64 = pf_sample(r, i)
164 ab_push(reg, FAB_AX_FRAME, arm, ms)
165 ab_push(reg, FAB_AX_OVER, arm, fab_over_of(ms, budget))
166 ab_push(reg, FAB_AX_JANK, arm, fab_jank_of(ms, med))
167 i = i + 1
168 }
169 return cnt
170}
171
172// ONE OBSERVATION PER WINDOW -- the production path, and the fix for the pseudoreplication above.
173// A window contributes its MEAN frame time (in MICROSECONDS, so a window mean still varies when every
174// individual frame is quantised to the same integer millisecond), its over-budget RATE and its jank RATE,
175// each as a single number in permil. The run is the sample; its frames are not.
176// fab_push_arm above is KEPT and is the frame-level helper the gate uses to build synthetic arms with a
177// known distribution -- it is how the STATISTICS are tested, while this is how PRODUCTION is measured.
178// Naming them separately is deliberate: one is a test fixture builder, the other is the real estimator,
179// and collapsing them is how the wrong unit got shipped in the first place.
180func fab_push_window(reg: *i64, arm: i64, r: *i64, budget: i64) -> i64 {
181 let cnt: i64 = pf_n(r)
182 if cnt <= 0 { return 0 }
183 let med: i64 = pf_pct(r, FAB_MEDIAN_PERMIL)
184 var sum: i64 = 0
185 var over: i64 = 0
186 var jank: i64 = 0
187 var i: i64 = 0
188 while i < cnt {
189 let ms: i64 = pf_sample(r, i)
190 sum = sum + ms
191 over = over + fab_over_of(ms, budget)
192 jank = jank + fab_jank_of(ms, med)
193 i = i + 1
194 }
195 ab_push(reg, FAB_AX_FRAME, arm, (sum * FAB_US_PER_MS) / cnt)
196 ab_push(reg, FAB_AX_OVER, arm, (over * FAB_PERMIL) / cnt)
197 ab_push(reg, FAB_AX_JANK, arm, (jank * FAB_PERMIL) / cnt)
198 return 1
199}
200
201func fab_axis_name(a: i64) -> *u8 {
202 if a == FAB_AX_FRAME { return "frame_ms" as *u8 }
203 if a == FAB_AX_OVER { return "over_budget_rate" as *u8 }
204 return "jank_rate" as *u8
205}
206func fab_ax_word(v: i64) -> *u8 {
207 if v == AB_AX_UP { return "UP" as *u8 }
208 if v == AB_AX_DOWN { return "DOWN" as *u8 }
209 if v == AB_AX_FLAT { return "FLAT" as *u8 }
210 if v == AB_AX_BELOW_MDE { return "BELOW-MDE" as *u8 }
211 return "INSUFF" as *u8
212}
213func fab_reason_word(v: i64) -> *u8 {
214 if v == AB_R_OK { return "OK" as *u8 }
215 if v == AB_R_FEW_B { return "TOO-FEW-SAMPLES-BASELINE" as *u8 }
216 if v == AB_R_FEW_A { return "TOO-FEW-SAMPLES-CANDIDATE" as *u8 }
217 if v == AB_R_ZEROVAR { return "ZERO-VARIANCE-BOTH-ARMS-NOT-A-TEST" as *u8 }
218 if v == AB_R_NOTBINARY { return "NOT-A-PROPORTION-AXIS" as *u8 }
219 return "OVERFLOW-REFUSED" as *u8
220}
221
222func main(argc: i64, argv: *i64) -> i64 {
223 if argc < 2 {
224 fab_p("usage: nx_frameab <world> [mde_ms] [plane=<prefix>]\n" as *u8)
225 fab_p(" contrasts the two most recent frametrace- windows for <world>: three per-frame axes,\n" as *u8)
226 fab_p(" Bonferroni-corrected across them, REFUSING a pair whose budgets differ (different hosts).\n" as *u8)
227 return FAB_EXIT_USAGE
228 }
229 let world: *u8 = argv[1] as *u8
230 if fpc_world_safe(world, fpc_slen(world)) == 0 {
231 fab_p("REFUSED bad-world-name: a world is [a-z0-9_] and names a plane row\n" as *u8)
232 return FAB_EXIT_REFUSED
233 }
234 var mde_ms: i64 = FAB_MDE_MS_DEFAULT
235 var mde_src: *u8 = "DERIVED-sample-quantum-integer-ms" as *u8
236 // plane= is SCANNED, not a positional slot: a gate drives this organ over a /tmp fixture plane, and an
237 // override that had to sit at a fixed index would collide with the optional mde argument. (The estate has
238 // already paid for that lesson once, in a mutation harness whose 4th positional slot did not exist.)
239 var plane: *u8 = FPC_PLANE
240 var plane_src: *u8 = "PRODUCTION" as *u8
241 var ai: i64 = 2
242 while ai < argc {
243 let av: *u8 = argv[ai] as *u8
244 let al: i64 = fpc_slen(av)
245 if fab_has_prefix(av, al, "plane=" as *u8) == 1 {
246 plane = av + 6
247 plane_src = "ARGV-OVERRIDE" as *u8
248 } else {
249 mde_ms = fpc_slice_int(av, 0, al)
250 mde_src = "ARGV" as *u8
251 }
252 ai = ai + 1
253 }
254 if mde_ms < 0 { mde_ms = FAB_MDE_MS_DEFAULT; mde_src = "DERIVED-argv-was-negative" as *u8 }
255
256 let rb: *u8 = sys_mmap(FAB_ROWCAP)
257 let ra: *u8 = sys_mmap(FAB_ROWCAP)
258 let seen: *i64 = sys_mmap(16) as *i64
259 let found: i64 = fab_last_two(plane, world, rb, ra, seen)
260 fab_p("NX-FRAMEAB world=" as *u8); fab_p(world)
261 fab_p(" plane=" as *u8); fab_p(plane)
262 fab_p(" plane_src=" as *u8); fab_p(plane_src); fab_p("\n" as *u8)
263 fab_kv("plane_rows_scanned" as *u8, seen[0])
264 fab_kv("rows_for_this_world" as *u8, found)
265 if found < 0 {
266 fab_p("REFUSED plane-unreadable: the frametrace- plane could not be loaded\n" as *u8)
267 return FAB_EXIT_IO
268 }
269 if found < 2 {
270 fab_p("ABSTAIN NEED-TWO-WINDOWS: a contrast needs a baseline and a candidate, and this world has fewer\n" as *u8)
271 return FAB_EXIT_IO
272 }
273
274 let lb: i64 = fpc_slen(rb)
275 let la: i64 = fpc_slen(ra)
276 let bud_b: i64 = fpc_col_int(rb, 0, lb, FPC_COL_BUDGET)
277 let bud_a: i64 = fpc_col_int(ra, 0, la, FPC_COL_BUDGET)
278 let ts_b: i64 = fpc_col_int(rb, 0, lb, FPC_COL_TS)
279 let ts_a: i64 = fpc_col_int(ra, 0, la, FPC_COL_TS)
280 fab_kv("baseline_ts" as *u8, ts_b)
281 fab_kv("candidate_ts" as *u8, ts_a)
282 fab_kv("baseline_budget_ms" as *u8, bud_b)
283 fab_kv("candidate_budget_ms" as *u8, bud_a)
284
285 // THE CONFOUND GUARD. Budget follows the display refresh, so differing budgets mean differing hosts and
286 // the contrast would measure the machine while reporting on the build. Refuse, and say so.
287 if fab_confounded(bud_b, bud_a) == 1 {
288 fab_p("REFUSED CONFOUNDED-BUDGET: the two windows ran under different frame budgets, which follow the\n" as *u8)
289 fab_p(" display refresh, so they are different HOSTS. A verdict here would grade the machine, not the\n" as *u8)
290 fab_p(" build. Capture both arms on one host, or add a host column to the plane and select on it.\n" as *u8)
291 return FAB_EXIT_REFUSED
292 }
293 if bud_b <= 0 {
294 fab_p("ABSTAIN NO-BUDGET: the rows carry no usable budget, so over-budget cannot be decided\n" as *u8)
295 return FAB_EXIT_INSUFF
296 }
297
298 let pb: *i64 = sys_mmap(pf_words() * 8)
299 let pa: *i64 = sys_mmap(pf_words() * 8)
300 pf_init(pb)
301 pf_init(pa)
302 let nb: i64 = fpc_row_frames(rb, lb, pb, bud_b)
303 let na: i64 = fpc_row_frames(ra, la, pa, bud_a)
304 fab_kv("baseline_frames_parsed" as *u8, nb)
305 fab_kv("candidate_frames_parsed" as *u8, na)
306 if nb < 0 { fab_p("ABSTAIN FRAMES-UNPARSABLE-BASELINE\n" as *u8); return FAB_EXIT_INSUFF }
307 if na < 0 { fab_p("ABSTAIN FRAMES-UNPARSABLE-CANDIDATE\n" as *u8); return FAB_EXIT_INSUFF }
308
309 let reg: *i64 = sys_mmap(ab_words() * 8)
310 ab_init(reg)
311 // ONE observation per window, not one per frame -- see the pseudoreplication note at the top. With two
312 // windows this gives n=1 per arm, which is far under the sample floor, so the organ ABSTAINS. That is
313 // the honest state of this measurement today and not a defect: it takes 30 RUNS per arm, not 30 frames.
314 fab_push_window(reg, AB_ARM_B, pb, bud_b)
315 fab_push_window(reg, AB_ARM_A, pa, bud_a)
316 fab_p("unit_of_replication=WINDOW windows_per_arm=1 needed=" as *u8); fab_pn(ab_min_n()); fab_p("\n" as *u8)
317
318 fab_kv("axes" as *u8, FAB_AXES)
319 fab_kv("bonferroni_crit_t_q10" as *u8, ab_crit_t_q10(FAB_AXES))
320 fab_kv("min_samples_per_arm" as *u8, ab_min_n())
321 fab_p("mde_ms=" as *u8); fab_pn(mde_ms); fab_p(" provenance=" as *u8); fab_p(mde_src); fab_p("\n" as *u8)
322
323 let axv: *i64 = sys_mmap(FAB_AXES * 8)
324 var a: i64 = 0
325 while a < FAB_AXES {
326 // every axis here is LOWER-IS-BETTER, and that is passed explicitly rather than defaulted.
327 // AXIS 0 IS CONTINUOUS (milliseconds) and AXES 1-2 ARE PROPORTIONS (0/1 per frame), so each is
328 // graded by its OWN instrument: a rate put through the continuous test abstains on its strongest
329 // findings, because with both arms constant the two-sample t has no denominator.
330 var mde: i64 = 0
331 var v: i64 = 0
332 if a == FAB_AX_FRAME {
333 mde = mde_ms * 1024
334 v = ab_axis_verdict(reg, a, FAB_AXES, 0, mde)
335 } else {
336 v = ab_prop_verdict(reg, a, FAB_AXES, 0, mde)
337 }
338 axv[a] = v
339 fab_p("AXIS " as *u8); fab_p(fab_axis_name(a))
340 fab_p(" verdict=" as *u8); fab_p(fab_ax_word(v))
341 // the reason comes from the SAME test that produced the verdict, or the line contradicts itself
342 var rsn: i64 = AB_R_OK
343 if a == FAB_AX_FRAME { rsn = ab_axis_reason(reg, a) } else { rsn = ab_prop_reason(reg, a) }
344 fab_p(" reason=" as *u8); fab_p(fab_reason_word(rsn))
345 fab_p(" n_b=" as *u8); fab_pn(ab_n(reg, a, AB_ARM_B))
346 fab_p(" n_a=" as *u8); fab_pn(ab_n(reg, a, AB_ARM_A))
347 fab_p(" mean_b_q10=" as *u8); fab_pn(ab_mean_q10(reg, a, AB_ARM_B))
348 fab_p(" mean_a_q10=" as *u8); fab_pn(ab_mean_q10(reg, a, AB_ARM_A))
349 fab_p(" var_b_q10=" as *u8); fab_pn(ab_var_q10(reg, a, AB_ARM_B))
350 fab_p(" var_a_q10=" as *u8); fab_pn(ab_var_q10(reg, a, AB_ARM_A))
351 fab_p(" delta_q10=" as *u8); fab_pn(ab_delta_q10(reg, a))
352 if a == FAB_AX_FRAME {
353 fab_p(" test=welch-t se2_q10=" as *u8); fab_pn(ab_se2_q10(reg, a))
354 } else {
355 fab_p(" test=two-proportion se2_q10=" as *u8); fab_pn(ab_prop_se2_q10(reg, a))
356 fab_p(" pooled_p_q10=" as *u8); fab_pn(ab_pooled_p_q10(reg, a))
357 }
358 fab_p("\n" as *u8)
359 a = a + 1
360 }
361
362 let fam: i64 = ab_family_verdict(axv, FAB_AXES)
363 let tested: i64 = ab_tested_axes(axv, FAB_AXES)
364 // THE DENOMINATOR TRAVELS WITH THE VERDICT: a family computed over 1 testable axis is not the same claim
365 // as one computed over 3, and without this number an abstaining family reads as a confident NO-CHANGE.
366 fab_p("tested_axes=" as *u8); fab_pn(tested); fab_p(" of " as *u8); fab_pn(FAB_AXES); fab_p("\n" as *u8)
367 if fam == AB_V_IMPROVED { fab_p("verdict=IMPROVED\n" as *u8); return FAB_EXIT_OK }
368 if fam == AB_V_REGRESSED { fab_p("verdict=REGRESSED\n" as *u8); return FAB_EXIT_REGRESSED }
369 if fam == AB_V_NO_CHANGE { fab_p("verdict=NO-CHANGE\n" as *u8); return FAB_EXIT_OK }
370 fab_p("verdict=INSUFFICIENT\n" as *u8)
371 return FAB_EXIT_INSUFF
372}