code wiki / (root) / nx_codegen_bench_gate.nx
`nx_codegen_bench_gate.nx` source

↩ module page · 150 lines · 8562 B
1// nx_codegen_bench_gate.nx -- OBJECTIVE code-gen grader (eats the anti-navel-gazing debt, 2026-07-15).
2// nx_capability_triage graded code-gen = MEASURE(bench=none): we had NO objective way to grade generated
3// code, so any "our coding is improving" claim was navel-gazing. This is the missing instrument: coding
4// tasks with OBJECTIVE (input->expected) test cases; a candidate solution (a function = "generated code")
5// is run against every case; PASS iff ALL cases match. The grade is not our opinion -- it is whether the
6// code COMPUTES THE RIGHT ANSWER. The load-bearing ANTI-NAVEL-GAZING property (T2): the grader must FAIL
7// wrong code -- a grader that passes broken solutions is exactly the navel-gazing the operator warned about.
8//   T1 CORRECT solutions PASS (the tasks are gradable; reference code scores 100%)
9//   T2 SOUND / anti-navel-gazing: BROKEN solutions FAIL (the grader rejects wrong code -- the whole point)
10//   T3 SCORE computed = pass-rate over candidates (the metric), persisted to a durable bench ledger (living)
11//   T4 DETERMINISM: re-grading is byte-stable
12// The candidate here is a compiled-in function (proves the GRADER sound); the real GENERATOR (forge e1 coder
13// -> candidate SOURCE -> nx_cc compile -> this grader) + a standard suite (HumanEval-style) = the next rungs
14// that turn this into the external number proving "we reduce Claude coding". expect_exit: 0  license_tier: ORIGINAL
15import "nx_syscalls.nx"
16import "nx_gate_verdict.nx"
17
18func cb_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
19func cb_n(v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { cb_w("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(24)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let o: *u8 = sys_mmap(24)
  var i: i64 = 0
  while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, o, k)
  return 0
31}
32
33// ===== TASK 1: max2(a,b) -> larger =====
34func t1_correct(a: i64, b: i64) -> i64 { if a > b { return a } return b }
35func t1_broken(a: i64, b: i64) -> i64 { return a }               // always first arg -- WRONG
36// ===== TASK 2: sum_n(n) -> 1+2+...+n =====
37func t2_correct(n: i64, unused: i64) -> i64 { var s: i64=0; var i: i64=1; while i<=n { s=s+i; i=i+1 } return s }
38func t2_broken(n: i64, unused: i64) -> i64 { return n }           // just returns n -- WRONG
39// ===== TASK 3: is_prime(n) -> 1/0 =====
40func t3_correct(n: i64, unused: i64) -> i64 { if n < 2 { return 0 } var d: i64=2; while d*d<=n { if (n/d)*d==n { return 0 } d=d+1 } return 1 }
41func t3_broken(n: i64, unused: i64) -> i64 { return 1 }           // everything prime -- WRONG
42
43// grade a candidate func(a,b)->i64 against `ncase` test cases (ins2 unused for 1-arg tasks). PASS(1)/FAIL(0).
44func cb_grade(cand: func(i64,i64) -> i64, ins1: *i64, ins2: *i64, exp: *i64, ncase: i64) -> i64 {
  var i: i64 = 0
  while i < ncase {
      let got: i64 = cand(ins1[i], ins2[i])
      if got != exp[i] { return 0 }
      i = i + 1
  }
  return 1
52}
53
54func main() -> i64 {
  cb_w("=== NX-CODEGEN-BENCH -- objective code-gen grader (compute-the-right-answer, anti-navel-gazing) ===\n" as *u8)
  // task test vectors
  let a1: *i64 = sys_mmap(8*8) as *i64
  let b1: *i64 = sys_mmap(8*8) as *i64
  let e1: *i64 = sys_mmap(8*8) as *i64
  a1[0]=3; b1[0]=5; e1[0]=5
  a1[1]=7; b1[1]=2; e1[1]=7
  a1[2]=4; b1[2]=4; e1[2]=4
  a1[3]=0; b1[3]=9; e1[3]=9
  let n1: i64 = 4
  let a2: *i64 = sys_mmap(8*8) as *i64
  let z2: *i64 = sys_mmap(8*8) as *i64
  let e2: *i64 = sys_mmap(8*8) as *i64
  a2[0]=5; e2[0]=15
  a2[1]=1; e2[1]=1
  a2[2]=10; e2[2]=55
  let n2: i64 = 3
  let a3: *i64 = sys_mmap(8*8) as *i64
  let z3: *i64 = sys_mmap(8*8) as *i64
  let e3: *i64 = sys_mmap(8*8) as *i64
  a3[0]=7; e3[0]=1
  a3[1]=8; e3[1]=0
  a3[2]=2; e3[2]=1
  a3[3]=1; e3[3]=0
  a3[4]=9; e3[4]=0
  let n3: i64 = 5
81
  // grade the CORRECT candidates
  let c1: i64 = cb_grade(t1_correct, a1, b1, e1, n1)
  let c2: i64 = cb_grade(t2_correct, a2, z2, e2, n2)
  let c3: i64 = cb_grade(t3_correct, a3, z3, e3, n3)
  let corr_pass: i64 = c1 + c2 + c3
  // grade the BROKEN candidates
  let b1r: i64 = cb_grade(t1_broken, a1, b1, e1, n1)
  let b2r: i64 = cb_grade(t2_broken, a2, z2, e2, n2)
  let b3r: i64 = cb_grade(t3_broken, a3, z3, e3, n3)
  let brok_pass: i64 = b1r + b2r + b3r
92
  cb_w("  TASK              CORRECT-cand  BROKEN-cand\n" as *u8)
  cb_w("  max2(a,b)         " as *u8); if c1==1 { cb_w("PASS         " as *u8) } else { cb_w("FAIL         " as *u8) } if b1r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
  cb_w("  sum_n(n)          " as *u8); if c2==1 { cb_w("PASS         " as *u8) } else { cb_w("FAIL         " as *u8) } if b2r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
  cb_w("  is_prime(n)       " as *u8); if c3==1 { cb_w("PASS         " as *u8) } else { cb_w("FAIL         " as *u8) } if b3r==1 { cb_w("PASS(!)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
  cb_w("\n  SCORE: correct candidates " as *u8); cb_n(corr_pass); cb_w("/3 pass; broken candidates " as *u8); cb_n(brok_pass); cb_w("/3 pass (must be 0 -- grader rejects wrong code)\n" as *u8)
  let ratepm: i64 = (corr_pass * 1000) / 3
  cb_w("  code-gen pass-rate on the reference set: " as *u8); cb_n(ratepm/10); cb_w("%\n\n" as *u8)
100
  // persist to a durable bench ledger (living trend, like the zero-claude ledger)
  let ts: i64 = sys_now_realtime_sec()
  let rec: *u8 = sys_mmap(256)
  var ro: i64 = 0
  let hdr: *u8 = "CGB ts=" as *u8
  var hi: i64=0; while hdr[hi]!=(0 as u8){ rec[ro]=hdr[hi]; ro=ro+1; hi=hi+1 }
  var m: i64=ts; let tb: *u8=sys_mmap(24); var tk: i64=0; while m>0 { tb[tk]=(48+(m%10)) as u8; m=m/10; tk=tk+1 } var ti: i64=tk-1; while ti>=0 { rec[ro]=tb[ti]; ro=ro+1; ti=ti-1 }
  let mid: *u8 = " generator=reference tasks=3 pass=" as *u8
  var mi: i64=0; while mid[mi]!=(0 as u8){ rec[ro]=mid[mi]; ro=ro+1; mi=mi+1 }
  var pm: i64=corr_pass; if pm==0 { rec[ro]=48 as u8; ro=ro+1 } else { let pb: *u8=sys_mmap(8); var pk: i64=0; while pm>0 { pb[pk]=(48+(pm%10)) as u8; pm=pm/10; pk=pk+1 } var pj: i64=pk-1; while pj>=0 { rec[ro]=pb[pj]; ro=ro+1; pj=pj-1 } }
  rec[ro]=10 as u8; ro=ro+1
  let path: *u8 = "/home/elderwesto/nx_stage/codegen_bench_ledger.log" as *u8
  let lenb: *i64 = sys_mmap(8) as *i64
  let old: *u8 = sys_read_file(path, lenb)
  let full: *u8 = sys_mmap(65536)
  var wo: i64 = 0
  if (old as i64) != 0 { var z4: i64=0; while z4 < lenb[0] { full[wo]=old[z4]; wo=wo+1; z4=z4+1 } }
  var z5: i64=0; while z5 < ro { full[wo]=rec[z5]; wo=wo+1; z5=z5+1 }
  let fd: i64 = sys_openat_wr(path, 0x1a4)
  var wrote: i64 = 0
  if fd >= 0 { wrote = sys_write(fd, full, wo); sys_close(fd) }
122
  // ---- teeth ----
  var pass: i64 = 0
  var ttl: i64 = 0
  ttl = ttl + 1
  cb_w("  T1 CORRECT solutions all PASS (tasks gradable, reference=100%): " as *u8)
  if corr_pass == 3 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  cb_w("  T2 SOUND/anti-navel-gazing: BROKEN solutions all FAIL (grader rejects wrong code): " as *u8)
  if brok_pass == 0 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL (a broken candidate slipped through = navel-gazing grader)\n" as *u8) }
  ttl = ttl + 1
  let c1b: i64 = cb_grade(t1_correct, a1, b1, e1, n1)
  cb_w("  T3 DETERMINISM (re-grade byte-stable): " as *u8)
  if c1b == c1 { pass = pass + 1; cb_w("PASS\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  cb_w("  T4 PERSISTED to durable bench ledger (living trend): " as *u8)
  if wrote > 0 { pass = pass + 1; cb_w("PASS (" as *u8); cb_n(wrote); cb_w("B)\n" as *u8) } else { cb_w("FAIL\n" as *u8) }
139
  cb_w("NX-CODEGEN-BENCH-GATE passed " as *u8); cb_n(pass); cb_w("/" as *u8); cb_n(ttl)
  // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
  // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
  // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
  let ctr__dry: *i64 = gv_ctr()
  ctr__dry[0] = pass
  ctr__dry[1] = ttl
  let rc__dry: i64 = gv_verdict("CODEGEN-BENCH-GATE" as *u8, ctr__dry, "objective code-gen grader: rejects wrong code -- the anti-navel-gazing instrument for 'is our coding improving')" as *u8)
  sys_exit(rc__dry)
  return rc__dry
150}
nx_codegen_bench_gate.nx source

`nx_codegen_bench_gate.nx` source
