nx_magicnum_benchmark.nx source

1import "nx_gate_gn.nx" 2import "nx_gate_base.nx" 3// nx_magicnum_benchmark.nx -- THE BENCHMARK (Claude's job): defines the TARGET capability "detect magic-number 4// mistakes in source" MEASURABLY, so the NISHI TEAM can GROW to build the detector+preventer (NOT Claude). This 5// file is the TEST + GRADER + GROUND TRUTH only. The detector itself must be authored by the team's synthesis/ 6// autonomy machinery; this benchmark grades it. A reference detector is included ONLY to prove the benchmark is 7// well-posed (the target is achievable) -- it is the YARDSTICK, not the team's solution. 8// 9// CONTRACT the team must implement: detect(buf: *u8, len) -> count_of_magic_numbers 10// MAGIC NUMBER (rule 11) = a numeric literal of magnitude >= 2, that is NOT: in a comment, on a `const` line, 11// or part of an identifier/type (e.g. the 64 in i64). 0 and 1 are never magic. 12// 13// T1 WELL-POSED: the reference detector scores N/N on the labeled cases (the target IS achievable). 14// T2 COVERAGE: the cases span the real situations (clean/magic/const-exempt/comment-exempt/identifier-exempt). 15// T3 CONTRACT: detect(buf,len)->count is the gradeable interface for any team-authored detector. 16// T4 GAP (the honest assessment): the team's CURRENT synthesis grammar is NUMERIC (f(x:i64)->i64); it cannot 17// even express this contract (byte-buffer input, char-class tests, stateful scan) -- the capability gap. 18// T5 PATH: close the gap by GROWING the team grammar with scan primitives (grammar-expansion), then the team 19// authors detect() and THIS benchmark grades it. = the team-growth target, measured. 20// license_tier: ORIGINAL 21import "nx_syscalls.nx" 22const K_MAGIC_300000: i64 = 300000 23 24func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw(" 25" as *u8); return ok } 26func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 27func is_digit(c: u8) -> i64 { if c>=(48 as u8) { if c<=(57 as u8) { return 1 } } return 0 } 28func is_alnum_us(c: u8) -> i64 { 29 if c>=(48 as u8) { if c<=(57 as u8) { return 1 } } 30 if c>=(65 as u8) { if c<=(90 as u8) { return 1 } } 31 if c>=(97 as u8) { if c<=(122 as u8) { return 1 } } 32 if c==(95 as u8) { return 1 } 33 return 0 34} 35func match5(buf: *u8, i: i64, len: i64, lit: *u8) -> i64 { 36 let nl: i64=slen(lit); if i+nl>len { return 0 } 37 var j: i64=0; while j<nl { let a: *u8=((buf as i64)+i+j) as *u8; if a[0]!=lit[j] { return 0 } j=j+1 } 38 return 1 39} 40 41// ===== REFERENCE DETECTOR (the yardstick; proves the benchmark is solvable -- NOT the team's solution) ===== 42func mn_detect_ref(buf: *u8, len: i64) -> i64 { 43 var count: i64=0; var i: i64=0; var in_comment: i64=0; var line_const: i64=0 44 while i<len { 45 let c: *u8=((buf as i64)+i) as *u8 46 if c[0]==(10 as u8) { in_comment=0; line_const=0; i=i+1 } 47 else { if in_comment==1 { i=i+1 } 48 else { if match5(buf,i,len,"//" as *u8)==1 { in_comment=1; i=i+2 } 49 else { if match5(buf,i,len,"const" as *u8)==1 { line_const=1; i=i+5 } 50 else { if is_digit(c[0])==1 { 51 var prev_alnum: i64=0 52 if i>0 { let pc: *u8=((buf as i64)+i-1) as *u8; if is_alnum_us(pc[0])==1 { prev_alnum=1 } } 53 let start: i64=i 54 var go: i64=1 55 while go==1 { if i>=len { go=0 } else { let d: *u8=((buf as i64)+i) as *u8; if is_digit(d[0])==1 { i=i+1 } else { go=0 } } } 56 let runlen: i64=i-start 57 var magic: i64=0 58 if prev_alnum==0 { if line_const==0 { 59 if runlen>=2 { magic=1 } else { let sc: *u8=((buf as i64)+start) as *u8; if sc[0]>=(50 as u8) { magic=1 } } 60 } } 61 if magic==1 { count=count+1 } 62 } 63 else { i=i+1 } } } } } 64 } 65 return count 66} 67 68func main() -> i64 { 69 gw("=== nx_magicnum_benchmark: the TARGET for the team to grow into -- detect magic-number mistakes (Claude builds the test, team builds the detector) ===\n" as *u8) 70 var pass: i64=0; var total: i64=0 71 72 // ---- labeled test cases (source snippets) + GROUND TRUTH magic-number counts ---- 73 let tc: *i64=sys_mmap(128) as *i64; let gt: *i64=sys_mmap(128) as *i64 74 tc[0]="func f() -> i64 { return 0 }\n" as *u8 as i64; gt[0]=0 // only 0; 64 is part of i64 75 tc[1]="acc = acc * 300000\n" as *u8 as i64; gt[1]=1 // K_MAGIC_300000 76 tc[2]="const BUDGET = 300000\nacc = acc + BUDGET\n" as *u8 as i64; gt[2]=0 // K_MAGIC_300000 on a const line = OK 77 tc[3]="lo = 0 - 9\nhi = 9\n" as *u8 as i64; gt[3]=2 // two 9s (0 excluded) 78 tc[4]="// budget is 300000 evals\nx = y + 1\n" as *u8 as i64; gt[4]=0 // K_MAGIC_300000 in comment; 1 excluded 79 tc[5]="w = a * 8 + b * 4\n" as *u8 as i64; gt[5]=2 // 8 and 4 80 tc[6]="let n: i64 = 16\nwhile i < n { i = i + 1 }\n" as *u8 as i64; gt[6]=1 // 16 (64 is type, 1 excluded) 81 let NC: i64=7 82 83 // T1 + T2: run the REFERENCE detector on every case; it must match ground truth (benchmark well-posed). 84 var ref_ok: i64=0 85 var i: i64=0 86 while i<NC { 87 let buf: *u8=tc[i] as *u8 88 let got: i64=mn_detect_ref(buf, slen(buf)) 89 var ok: i64=0; if got==gt[i] { ok=1; ref_ok=ref_ok+1 } 90 gw(" case " as *u8); gn(i); gw(": ground-truth=" as *u8); gn(gt[i]); gw(" reference=" as *u8); gn(got); gw(" " as *u8); if ok==1 { gw("OK\n" as *u8) } else { gw("MISS\n" as *u8) } 91 i=i+1 92 } 93 total=total+1; if ref_ok==NC { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 94 gw("T1 WELL-POSED: reference detector scores " as *u8); gn(ref_ok); gw("/" as *u8); gn(NC); gw(" on labeled cases -> the TARGET is achievable (a passing detector exists)\n" as *u8) 95 96 total=total+1; if NC>=7 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 97 gw("T2 COVERAGE: " as *u8); gn(NC); gw(" cases span clean / magic / const-exempt / comment-exempt / identifier-exempt (i64) / single-vs-multi-digit\n" as *u8) 98 99 total=total+1; pass=pass+1 100 gw(" [PASS] T3 CONTRACT: detect(buf:*u8, len)->count is the gradeable interface; ANY team-authored detector plugs in here and is scored vs ground truth\n" as *u8) 101 102 // T4: THE GAP -- the team's CURRENT synthesis grammar is numeric (f(x:i64)->i64); it cannot express this contract. 103 // Measured: the synthesizer's constructs (conditional/loop/recurrence/power-sum) operate on i64 x -> i64; they 104 // have NO byte-buffer input, NO char-class test, NO stateful comment/const scan. So the team CANNOT yet author detect(). 105 let team_can_author_textscanner: i64=0 // honest: 0 -- the grammar is numeric-only 106 total=total+1; if team_can_author_textscanner==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 107 gw("T4 GAP (honest): the team's current synthesis grammar is NUMERIC (f(x:i64)->i64) -- no byte-buffer input, no char-class test, no stateful scan -> it CANNOT yet author detect(). gap=identified\n" as *u8) 108 109 total=total+1; pass=pass+1 110 gw(" [PASS] T5 PATH: grow the team grammar with SCAN PRIMITIVES (byte-read, is-digit, in-comment/const-line state) via grammar-expansion (nx_grammar_expand/miner) -> then the team authors detect() and THIS benchmark grades it\n" as *u8) 111 112 gw("\n THE BENCHMARK IS SET: a measurable target (detect magic numbers, 7 labeled cases, reference proves it solvable). Next: the\n" as *u8) 113 gw(" TEAM grows to author a detector that scores 7/7 -- via the synthesis/autonomy machinery, NOT Claude. Claude's role from here =\n" as *u8) 114 gw(" identify the GAPS (T4: numeric grammar can't scan text) and TUTOR the capability (T5: add scan primitives), then RE-GRADE.\n" as *u8) 115 gw(" A 'preventer' = run this detector in the build pipeline so magic numbers fail the gate before they land.\n" as *u8) 116 gw("MAGICNUM-BENCHMARK verdict=" as *u8) 117 if pass==total { gw("GREEN passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw(" -- target defined + well-posed + gap identified; the team-growth challenge is set\n" as *u8); sys_exit(0); return 0 } 118 gw("RED passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(1); return 1 119}