nx_codegen_bench_gate.nx source
↩ module page · 152 lines · 8685 B
1// nx_codegen_bench_gate.nx -- OBJECTIVE code-gen grader (eats the anti-navel-gazing debt, 2026-07-15).
2// nx_capability_triage graded code-gen = MEASURE(bench=none): we had NO objective way to grade generated
3// code, so any "our coding is improving" claim was navel-gazing. This is the missing instrument: coding
4// tasks with OBJECTIVE (input->expected) test cases; a candidate solution (a function = "generated code")
5// is run against every case; PASS iff ALL cases match. The grade is not our opinion -- it is whether the
6// code COMPUTES THE RIGHT ANSWER. The load-bearing ANTI-NAVEL-GAZING property (T2): the grader must FAIL
7// wrong code -- a grader that passes broken solutions is exactly the navel-gazing the operator warned about.
8// T1 CORRECT solutions PASS (the tasks are gradable; reference code scores 100%)
9// T2 SOUND / anti-navel-gazing: BROKEN solutions FAIL (the grader rejects wrong code -- the whole point)
10// T3 SCORE computed = pass-rate over candidates (the metric), persisted to a durable bench ledger (living)
11// T4 DETERMINISM: re-grading is byte-stable
12// The candidate here is a compiled-in function (proves the GRADER sound); the real GENERATOR (forge e1 coder
13// -> candidate SOURCE -> nx_cc compile -> this grader) + a standard suite (HumanEval-style) = the next rungs
14// that turn this into the external number proving "we reduce Claude coding". expect_exit: 0 license_tier: ORIGINAL
15import "nx_syscalls.nx"
16import "nx_gate_verdict.nx"
17import "nx_stage_path.nx"
18
19func cb_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
20func cb_n(v: i64) -> i64 {
21 var m: i64 = v
22 if m < 0 { cb_w("-" as *u8); m = 0 - m }
23 let t: *u8 = sys_mmap(24)
24 var k: i64 = 0
25 if m == 0 { t[0] = 48 as u8; k = 1 }
26 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
27 let o: *u8 = sys_mmap(24)
28 var i: i64 = 0
29 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
30 sys_write(1, o, k)
31 return 0
32}
33
34// ===== TASK 1: max2(a,b) -> larger =====
35func t1_correct(a: i64, b: i64) -> i64 { if a > b { return a } return b }
36func t1_broken(a: i64, b: i64) -> i64 { return a } // always first arg -- WRONG
37// ===== TASK 2: sum_n(n) -> 1+2+...+n =====
38func t2_correct(n: i64, unused: i64) -> i64 { var s: i64=0; var i: i64=1; while i<=n { s=s+i; i=i+1 } return s }
39func t2_broken(n: i64, unused: i64) -> i64 { return n } // just returns n -- WRONG
40// ===== TASK 3: is_prime(n) -> 1/0 =====
41func t3_correct(n: i64, unused: i64) -> i64 { if n < 2 { return 0 } var d: i64=2; while d*d<=n { if (n/d)*d==n { return 0 } d=d+1 } return 1 }
42func t3_broken(n: i64, unused: i64) -> i64 { return 1 } // everything prime -- WRONG
43
44// grade a candidate func(a,b)->i64 against `ncase` test cases (ins2 unused for 1-arg tasks). PASS(1)/FAIL(0).
45func cb_grade(cand: func(i64,i64) -> i64, ins1: *i64, ins2: *i64, exp: *i64, ncase: i64) -> i64 {
46 var i: i64 = 0
47 while i < ncase {
48 let got: i64 = cand(ins1[i], ins2[i])
49 if got != exp[i] { return 0 }
50 i = i + 1
51 }
52 return 1
53}
54
55func main() -> i64 {
56 cb_w("=== NX-CODEGEN-BENCH -- objective code-gen grader (compute-the-right-answer, anti-navel-gazing) ===\n" as *u8)
57 // task test vectors
58 let a1: *i64 = sys_mmap(8*8) as *i64
59 let b1: *i64 = sys_mmap(8*8) as *i64
60 let e1: *i64 = sys_mmap(8*8) as *i64
61 a1[0]=3; b1[0]=5; e1[0]=5
62 a1[1]=7; b1[1]=2; e1[1]=7
63 a1[2]=4; b1[2]=4; e1[2]=4
64 a1[3]=0; b1[3]=9; e1[3]=9
65 let n1: i64 = 4
66 let a2: *i64 = sys_mmap(8*8) as *i64
67 let z2: *i64 = sys_mmap(8*8) as *i64
68 let e2: *i64 = sys_mmap(8*8) as *i64
69 a2[0]=5; e2[0]=15
70 a2[1]=1; e2[1]=1
71 a2[2]=10; e2[2]=55
72 let n2: i64 = 3
73 let a3: *i64 = sys_mmap(8*8) as *i64
74 let z3: *i64 = sys_mmap(8*8) as *i64
75 let e3: *i64 = sys_mmap(8*8) as *i64
76 a3[0]=7; e3[0]=1
77 a3[1]=8; e3[1]=0
78 a3[2]=2; e3[2]=1
79 a3[3]=1; e3[3]=0
80 a3[4]=9; e3[4]=0
81 let n3: i64 = 5
82
83 // grade the CORRECT candidates
84 let c1: i64 = cb_grade(t1_correct, a1, b1, e1, n1)
85 let c2: i64 = cb_grade(t2_correct, a2, z2, e2, n2)
86 let c3: i64 = cb_grade(t3_correct, a3, z3, e3, n3)
87 let corr_pass: i64 = c1 + c2 + c3
88 // grade the BROKEN candidates
89 let b1r: i64 = cb_grade(t1_broken, a1, b1, e1, n1)
90 let b2r: i64 = cb_grade(t2_broken, a2, z2, e2, n2)
91 let b3r: i64 = cb_grade(t3_broken, a3, z3, e3, n3)
92 let brok_pass: i64 = b1r + b2r + b3r
93
94 cb_w(" TASK CORRECT-cand BROKEN-cand\n" as *u8)
95 cb_w(" max2(a,b) " as *u8); if c1==1 { cb_w("PASS " as *u8) } else { cb_w("FAIL " as *u8) } if b1r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
96 cb_w(" sum_n(n) " as *u8); if c2==1 { cb_w("PASS " as *u8) } else { cb_w("FAIL " as *u8) } if b2r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
97 cb_w(" is_prime(n) " as *u8); if c3==1 { cb_w("PASS " as *u8) } else { cb_w("FAIL " as *u8) } if b3r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
98 cb_w("\n SCORE: correct candidates " as *u8); cb_n(corr_pass); cb_w("/3 pass; broken candidates " as *u8); cb_n(brok_pass); cb_w("/3 pass (must be 0 -- grader rejects wrong code)\n" as *u8)
99 let ratepm: i64 = (corr_pass * 1000) / 3
100 cb_w(" code-gen pass-rate on the reference set: " as *u8); cb_n(ratepm/10); cb_w("%\n\n" as *u8)
101
102 // persist to a durable bench ledger (living trend, like the zero-claude ledger)
103 let ts: i64 = sys_now_realtime_sec()
104 let rec: *u8 = sys_mmap(256)
105 var ro: i64 = 0
106 let hdr: *u8 = "CGB ts=" as *u8
107 var hi: i64=0; while hdr[hi]!=(0 as u8){ rec[ro]=hdr[hi]; ro=ro+1; hi=hi+1 }
108 var m: i64=ts; let tb: *u8=sys_mmap(24); var tk: i64=0; while m>0 { tb[tk]=(48+(m%10)) as u8; m=m/10; tk=tk+1 } var ti: i64=tk-1; while ti>=0 { rec[ro]=tb[ti]; ro=ro+1; ti=ti-1 }
109 let mid: *u8 = " generator=reference tasks=3 pass=" as *u8
110 var mi: i64=0; while mid[mi]!=(0 as u8){ rec[ro]=mid[mi]; ro=ro+1; mi=mi+1 }
111 var pm: i64=corr_pass; if pm==0 { rec[ro]=48 as u8; ro=ro+1 } else { let pb: *u8=sys_mmap(8); var pk: i64=0; while pm>0 { pb[pk]=(48+(pm%10)) as u8; pm=pm/10; pk=pk+1 } var pj: i64=pk-1; while pj>=0 { rec[ro]=pb[pj]; ro=ro+1; pj=pj-1 } }
112 rec[ro]=10 as u8; ro=ro+1
113 sp_skip_unless("CODEGEN-BENCH-GATE" as *u8, sp_path("" as *u8, sys_mmap(SP_PATH_MAX)))
114 let path: *u8 = sp_path("codegen_bench_ledger.log" as *u8, sys_mmap(SP_PATH_MAX))
115 let lenb: *i64 = sys_mmap(8) as *i64
116 let old: *u8 = sys_read_file(path, lenb)
117 let full: *u8 = sys_mmap(65536)
118 var wo: i64 = 0
119 if (old as i64) != 0 { var z4: i64=0; while z4 < lenb[0] { full[wo]=old[z4]; wo=wo+1; z4=z4+1 } }
120 var z5: i64=0; while z5 < ro { full[wo]=rec[z5]; wo=wo+1; z5=z5+1 }
121 let fd: i64 = sys_openat_wr(path, 0x1a4)
122 var wrote: i64 = 0
123 if fd >= 0 { wrote = sys_write(fd, full, wo); sys_close(fd) }
124
125 // ---- teeth ----
126 var pass: i64 = 0
127 var ttl: i64 = 0
128 ttl = ttl + 1
129 cb_w(" T1 CORRECT solutions all PASS (tasks gradable, reference=100%): " as *u8)
130 if corr_pass == 3 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
131 ttl = ttl + 1
132 cb_w(" T2 SOUND/anti-navel-gazing: BROKEN solutions all FAIL (grader rejects wrong code): " as *u8)
133 if brok_pass == 0 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL (a broken candidate slipped through = navel-gazing grader)\n" as *u8) }
134 ttl = ttl + 1
135 let c1b: i64 = cb_grade(t1_correct, a1, b1, e1, n1)
136 cb_w(" T3 DETERMINISM (re-grade byte-stable): " as *u8)
137 if c1b == c1 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
138 ttl = ttl + 1
139 cb_w(" T4 PERSISTED to durable bench ledger (living trend): " as *u8)
140 if wrote > 0 { pass = pass + 1; cb_w("PASS (" as *u8); cb_n(wrote); cb_w("B)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
141
142 cb_w("NX-CODEGEN-BENCH-GATE passed " as *u8); cb_n(pass); cb_w("/" as *u8); cb_n(ttl)
143 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
144 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
145 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
146 let ctr__dry: *i64 = gv_ctr()
147 ctr__dry[0] = pass
148 ctr__dry[1] = ttl
149 let rc__dry: i64 = gv_verdict("CODEGEN-BENCH-GATE" as *u8, ctr__dry, "objective code-gen grader: rejects wrong code -- the anti-navel-gazing instrument for 'is our coding improving')" as *u8)
150 sys_exit(rc__dry)
151 return rc__dry
152}