code wiki / _hdl_build / nx_cms_abtest_gate.nx
nx_cms_abtest_gate.nx source
↩ module page · 151 lines · 7523 B
1// nx_cms_abtest_gate.nx -- CMS A/B-TESTING + PERSONALIZATION gate: land the missing Nelio/Google-Optimize
2// class HONESTLY by proving the sovereign deterministic assignment engine over 600 synthetic visitors --
3// stickiness, valid range, experiment independence, a non-degenerate even split, a weighted 90/10 rollout,
4// degenerate-safety, and bounded reproducible positions. Positive AND negative controls (a broken even
5// split or a weighting that ignores weights would FAIL the two-sided tolerances). Appends "CMSGATE row=
6// nx_cms_abtest ab-testing-personalize ... verdict=PASS" to knowledge/status/cms_gate.log ONLY if every
7// assertion holds (no fake-green). Exit 0 iff all pass. license_tier: ORIGINAL
8import "nx_cms_abtest.nx"
9import "nx_syscalls.nx"
10
11const ABG_N: i64 = 600 // synthetic visitor population
12
13func ag_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
14func ag_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
15func ag_cat(dst: *u8, off: i64, s: *u8) -> i64 { var o: i64=off; var k: i64=0; while s[k]!=(0 as u8){dst[o]=s[k];o=o+1;k=k+1} return o }
16func ag_catnum(dst: *u8, off: i64, v: i64) -> i64 {
17 var o: i64=off; let t: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; var k: i64=0
18 if m==0 {t[0]=48 as u8;k=1}; while m>0 {t[k]=(48+(m%10)) as u8; m=m/10; k=k+1}
19 var i: i64=0; while i<k {dst[o]=t[k-1-i]; o=o+1; i=i+1} return o
20}
21
22// write "u"<idx> (NUL-terminated) into buf; returns length
23func ag_vid(buf: *u8, idx: i64) -> i64 {
24 buf[0] = 117 as u8 // 'u'
25 let t: *u8 = sys_mmap(28); var m: i64=idx; var k: i64=0
26 if m==0 {t[0]=48 as u8;k=1}; while m>0 {t[k]=(48+(m%10)) as u8; m=m/10; k=k+1}
27 var o: i64 = 1; var i: i64 = 0
28 while i<k {buf[o]=t[k-1-i]; o=o+1; i=i+1}
29 buf[o] = 0 as u8
30 return o
31}
32
33func ag_row(id: i64, ok: i64, what: *u8) -> i64 {
34 ag_w("ABROW " as *u8); ag_num(id); ag_w(" " as *u8)
35 if ok==1 { ag_w("PASS " as *u8) } else { ag_w("FAIL " as *u8) }
36 ag_w(what); ag_w("\n" as *u8)
37 return ok
38}
39
40func main() -> i64 {
41 var pass: i64 = 0
42 var rows: i64 = 0
43 var ok: i64 = 0
44 let vid: *u8 = sys_mmap(64)
45
46 // R0: sticky/deterministic -- same (experiment, visitor) -> same arm across repeated calls
47 let s1: i64 = ab_assign("homepage-hero" as *u8, "visitor-42" as *u8, 2)
48 let s2: i64 = ab_assign("homepage-hero" as *u8, "visitor-42" as *u8, 2)
49 ok = 0; if s1 == s2 { ok = 1 }
50 rows=rows+1; pass=pass+ag_row(0, ok, "sticky: same (experiment,visitor) -> same arm, no stored state" as *u8)
51
52 // R1: valid range -- for k=2,3,4 every assignment is in [0,k)
53 ok = 1
54 var i: i64 = 0
55 while i < ABG_N {
56 ag_vid(vid, i)
57 let a2: i64 = ab_assign("exp-range" as *u8, vid, 2); if a2 < 0 { ok = 0 } if a2 >= 2 { ok = 0 }
58 let a3: i64 = ab_assign("exp-range" as *u8, vid, 3); if a3 < 0 { ok = 0 } if a3 >= 3 { ok = 0 }
59 let a4: i64 = ab_assign("exp-range" as *u8, vid, 4); if a4 < 0 { ok = 0 } if a4 >= 4 { ok = 0 }
60 i = i + 1
61 }
62 rows=rows+1; pass=pass+ag_row(1, ok, "range: every assignment in [0,num_variants) for k=2,3,4" as *u8)
63
64 // R2: experiment independence -- some visitor lands in a different arm under a different salt
65 var diff: i64 = 0
66 i = 0
67 while i < ABG_N {
68 ag_vid(vid, i)
69 let xa: i64 = ab_assign("exp-A" as *u8, vid, 2)
70 let xb: i64 = ab_assign("exp-B" as *u8, vid, 2)
71 if xa != xb { diff = diff + 1 }
72 i = i + 1
73 }
74 ok = 0; if diff > 0 { ok = 1 }
75 rows=rows+1; pass=pass+ag_row(2, ok, "independence: differently-salted experiments assign independently" as *u8)
76
77 // R3: non-degenerate EVEN split (k=2) -- both arms get 33%..67% (catches an all-one-bucket bug)
78 var c0: i64 = 0; var c1: i64 = 0
79 i = 0
80 while i < ABG_N {
81 ag_vid(vid, i)
82 let a: i64 = ab_assign("exp-even" as *u8, vid, 2)
83 if a == 0 { c0 = c0 + 1 } else { c1 = c1 + 1 }
84 i = i + 1
85 }
86 let lo: i64 = (ABG_N * 33) / 100
87 let hi: i64 = (ABG_N * 67) / 100
88 ok = 0; if c0 >= lo { if c0 <= hi { if c1 >= lo { if c1 <= hi { ok = 1 } } } }
89 rows=rows+1; pass=pass+ag_row(3, ok, "even split: both arms within 33%..67% over 600 visitors (no degenerate arm)" as *u8)
90
91 // R4: weighted 90/10 rollout -- variant 0 gets 80%..99% (two-sided: not 50/50, not all-v0)
92 let wts: *i64 = sys_mmap(32) as *i64
93 wts[0] = 9000; wts[1] = 1000
94 var w0: i64 = 0
95 i = 0
96 while i < ABG_N {
97 ag_vid(vid, i)
98 let a: i64 = ab_assign_weighted("exp-rollout" as *u8, vid, wts, 2)
99 if a == 0 { w0 = w0 + 1 }
100 i = i + 1
101 }
102 let wlo: i64 = (ABG_N * 80) / 100
103 let whi: i64 = (ABG_N * 99) / 100
104 ok = 0; if w0 >= wlo { if w0 <= whi { ok = 1 } }
105 rows=rows+1; pass=pass+ag_row(4, ok, "weighted 90/10: variant0 within 80%..99% (weights honored, not ignored)" as *u8)
106
107 // R5: weighted assignment is always in range AND a 50/50 weight is ~even (sanity on the band math)
108 let wts2: *i64 = sys_mmap(32) as *i64
109 wts2[0] = 5000; wts2[1] = 5000
110 var e0: i64 = 0; var rangeok: i64 = 1
111 i = 0
112 while i < ABG_N {
113 ag_vid(vid, i)
114 let a: i64 = ab_assign_weighted("exp-5050" as *u8, vid, wts2, 2)
115 if a < 0 { rangeok = 0 } if a >= 2 { rangeok = 0 }
116 if a == 0 { e0 = e0 + 1 }
117 i = i + 1
118 }
119 ok = 0; if rangeok == 1 { if e0 >= lo { if e0 <= hi { ok = 1 } } }
120 rows=rows+1; pass=pass+ag_row(5, ok, "weighted in-range; 50/50 weight ~even (band math correct)" as *u8)
121
122 // R6: degenerate-safe -- single-variant / zero-variant collapse to control arm 0 (no crash, no OOB)
123 ok = 0
124 if ab_assign("e" as *u8, "v" as *u8, 1) == 0 { if ab_assign("e" as *u8, "v" as *u8, 0) == 0 { ok = 1 } }
125 rows=rows+1; pass=pass+ag_row(6, ok, "degenerate-safe: num_variants<=1 -> control arm 0" as *u8)
126
127 // R7: positions are reproducible and bounded [0,RESOLUTION)
128 let p1: i64 = ab_position("exp-pos" as *u8, "visitor-7" as *u8)
129 let p2: i64 = ab_position("exp-pos" as *u8, "visitor-7" as *u8)
130 ok = 0; if p1 == p2 { if p1 >= 0 { if p1 < NX_AB_RESOLUTION { ok = 1 } } }
131 rows=rows+1; pass=pass+ag_row(7, ok, "position reproducible & bounded [0,10000)" as *u8)
132
133 ag_w("CMS-ABTEST-GATE rows=" as *u8); ag_num(rows); ag_w(" pass=" as *u8); ag_num(pass)
134 ag_w(" even=" as *u8); ag_num(c0); ag_w("/" as *u8); ag_num(c1)
135 ag_w(" w90=" as *u8); ag_num(w0); ag_w("/" as *u8); ag_num(ABG_N); ag_w("\n" as *u8)
136
137 if pass == rows {
138 let line: *u8 = sys_mmap(256)
139 var off: i64 = ag_cat(line, 0, "CMSGATE row=nx_cms_abtest ab-testing-personalize visitors=" as *u8)
140 off = ag_catnum(line, off, ABG_N)
141 off = ag_cat(line, off, " rows=" as *u8); off = ag_catnum(line, off, rows)
142 off = ag_cat(line, off, " pass=" as *u8); off = ag_catnum(line, off, pass)
143 off = ag_cat(line, off, " verdict=PASS\n" as *u8)
144 let gf: i64 = sys_openat_append("knowledge/status/cms_gate.log" as *u8, 0x1a4)
145 if gf >= 0 { sys_write(gf, line, off); sys_close(gf) }
146 ag_w("CMS-ABTEST-GATE verdict=PASS -- ab-testing-personalize recorded in cms_gate.log\n" as *u8)
147 sys_exit(0); return 0
148 }
149 ag_w("CMS-ABTEST-GATE verdict=FAIL -- NOT recorded (no fake-green)\n" as *u8)
150 sys_exit(1); return 1
151}