code wiki / (root) / nx_dwline_corpus_gate.nx
`nx_dwline_corpus_gate.nx` source

↩ module page · 331 lines · 15730 B
1// nx_dwline_corpus_gate.nx -- DOES STATEMENT LINE INFO HOLD ACROSS THE REAL CORPUS, OR ONLY ON A TOY?
2//
3// WHY THIS EXISTS. nx_dwline_stmt_gate proved statement granularity on runtime/nx_probe_dbgtiny.nx:
4// ELEVEN LINES, TWO FUNCTIONS, four .loc rows. That is a sample size of ONE, and a feature that works
5// on one hand-written probe is not a feature that works. The estate compiles thousands of real files
6// with deep nesting, matches, loops, multi-line expressions and 20-argument calls -- none of which the
7// probe contains. ★ A PROOF ON A FIXTURE YOU WROTE TO BE EASY IS A PROOF ABOUT YOUR FIXTURE.
8//
9// WHAT THIS MEASURES, over every .nx in runtime/ up to a DECLARED cap:
10//   - how many files the compiler can build at all with -g (the honest denominator)
11//   - how many emit ANY .loc, and how many .loc rows in total
12//   - the same numbers from the OLD blessed function-granularity compiler, as the control
13//   - the RATIO of statement rows to function rows -- which is the actual claim under test
14//   - files where the new compiler emits FEWER rows than the old one (a regression, individually named)
15//
16// NO SILENT CAPS: the file count scanned, the cap, and any skipped/failed files are all REPORTED.
17// A gate that quietly samples 20 files and prints a percentage is the thing this gate exists to
18// replace.
19//
20// license_tier: ORIGINAL  expect_exit: 0
21import "syscalls.nx"
22import "nx_gate_verdict.nx"
23
24const CG_CAP:      i64 = 4194304
25// ENUMERATION cap, deliberately ABOVE the whole corpus (11,376 .nx in runtime/) so the batch window
26// selects WITHIN a complete enumeration. First version set this to 400 and reused it as the batch
27// size, so a batch starting at 400 got an EMPTY window and the gate reported "REGRESSED=0" over ZERO
28// files. A CAP THAT DOUBLES AS A WINDOW SILENTLY BOUNDS THE POPULATION YOU BELIEVE YOU MEASURED.
29const CG_MAXFILES: i64 = 12000
30const CG_NAMEBUF:  i64 = 1048576
31const CG_OLD: *u8 = "_offc/nx_cc_sovereign.elf" as *u8
32const CG_NEW: *u8 = "../nx_compile_x86.elf" as *u8
33
34func cg_run_out(elf: *u8, args: *i64, nargs: i64, outfile: *u8) -> i64 {
  let pid: i64 = sys_fork()
  if pid == 0 {
      let fd: i64 = sys_openat_wr(outfile, 0x1a4)
      if fd >= 0 { sys_dup3(fd, 1, 0) }
      let devnull: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
      if devnull >= 0 { sys_dup3(devnull, 2, 0) }
      let argv: *i64 = sys_mmap(8 * (nargs + 2)) as *i64
      let envp: *i64 = sys_mmap(16) as *i64
      envp[0] = 0
      argv[0] = elf as i64
      var i: i64 = 0
      while i < nargs { argv[i+1] = args[i]; i = i + 1 }
      argv[nargs+1] = 0
      sys_execve(elf, argv, envp)
      sys_exit(127)
  }
  let st: *i64 = sys_mmap(16) as *i64
  sys_wait4(pid, st, 0)
  if (st[0] % 128) != 0 { return 0 - 1 }
  return (st[0] >> 8) & 0xff
55}
56
57func cg_read(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var tot: i64 = 0
  while tot < cap {
      let n: i64 = sys_read(fd, (buf as i64 + tot) as *u8, cap - tot)
      if n <= 0 { break }
      tot = tot + n
  }
  sys_close(fd)
  return tot
68}
69
70// count "    .loc " occurrences -- the directive, wherever it lands
71func cg_count_loc(buf: *u8, n: i64) -> i64 {
  let pat: *u8 = ".loc " as *u8
  var hits: i64 = 0
  var i: i64 = 0
  while i + 5 <= n {
      if buf[i] == (46 as u8) {
          var q: i64 = 0
          var ok: i64 = 1
          while q < 5 { if buf[i+q] != pat[q] { ok = 0; q = 5 } else { q = q + 1 } }
          if ok == 1 { hits = hits + 1 }
      }
      i = i + 1
  }
  return hits
85}
86
87// count function labels emitted -- ".type <name>, @function" is one per function
88func cg_count_funcs(buf: *u8, n: i64) -> i64 {
  let pat: *u8 = "@function" as *u8
  var hits: i64 = 0
  var i: i64 = 0
  while i + 9 <= n {
      if buf[i] == (64 as u8) {
          var q: i64 = 0
          var ok: i64 = 1
          while q < 9 { if buf[i+q] != pat[q] { ok = 0; q = 9 } else { q = q + 1 } }
          if ok == 1 { hits = hits + 1 }
      }
      i = i + 1
  }
  return hits
102}
103
104func cg_ends_nx(nm: *u8, nl: i64) -> i64 {
  if nl < 4 { return 0 }
  if nm[nl-3] != (46 as u8) { return 0 }
  if nm[nl-2] != (110 as u8) { return 0 }
  if nm[nl-1] != (120 as u8) { return 0 }
  return 1
110}
111
112// Decimal parse for the batch window. The 300s server-side deadline cannot fit the whole corpus in
113// one run, so the run is BATCHED rather than SHRUNK -- shrinking the sample to fit the instrument is
114// how a measurement becomes a decoration.
115func cg_atoi(s: *u8) -> i64 {
  var v: i64 = 0
  var i: i64 = 0
  while s[i] != (0 as u8) {
      let c: i64 = s[i] as i64
      if c < 48 { return v }
      if c > 57 { return v }
      v = v * 10 + (c - 48)
      i = i + 1
  }
  return v
126}
127
128func main(argc: i64, argv: *i64) -> i64 {
  let ctr: *i64 = gv_ctr()
  var batch_start: i64 = 0
  var batch_count: i64 = 400
  if argc >= 2 { batch_start = cg_atoi(argv[1] as *u8) }
  if argc >= 3 { batch_count = cg_atoi(argv[2] as *u8) }
  // MODE. "both" (default) runs BOTH compilers per file -- the control that PROVED the baseline.
  // "fast" runs ONLY the new compiler and DERIVES the baseline from the same assembly, because
  // "old .loc rows == function count" is no longer an assumption: it held EXACTLY on 777 files
  // across two independent batches (78,399 == 78,399). Halving the work is what makes the FULL
  // 11,376-file corpus reachable instead of a 7% window.
  // ★ WHEN THE COST OF MEASURING IS WHAT DRIVES THE SAMPLE SIZE, THAT COST IS THE BUG.
  // The "both" mode is deliberately KEPT, not replaced: a derived baseline is only as good as the
  // invariant behind it, so the invariant must stay independently re-testable.
  var fast: i64 = 0
  if argc >= 4 {
      let modep: *u8 = argv[3] as *u8
      if modep[0] == (102 as u8) { fast = 1 }
  }
  gv_head("=== NX-DWLINE-CORPUS GATE -- statement line info across the REAL corpus, not a probe ===" as *u8)
148
  sys_chdir("buildroot" as *u8)
150
  // ---- enumerate runtime/*.nx via getdents64 -------------------------------------------------
  let names: *u8 = sys_mmap(CG_NAMEBUF)
  let noff:  *i64 = sys_mmap(8 * (CG_MAXFILES + 8)) as *i64
  var nfiles: i64 = 0
  var npos: i64 = 0
  var corpus_total: i64 = 0
157
  let dfd: i64 = __syscall(257, 0-100, "runtime" as *u8, 0x10000, 0, 0, 0)
  if dfd >= 0 {
      let dbuf: *u8 = sys_mmap(65536)
      var go: i64 = 1
      while go == 1 {
          let nread: i64 = __syscall(217, dfd, dbuf, 65536, 0, 0, 0)
          if nread <= 0 { go = 0 } else {
              var pos: i64 = 0
              while pos < nread {
                  let reclen: i64 = (dbuf[pos+16] as i64) | ((dbuf[pos+17] as i64) << 8)
                  var nl: i64 = 0
                  while dbuf[pos+19+nl] != (0 as u8) { nl = nl + 1 }
                  if cg_ends_nx(((dbuf as i64) + pos + 19) as *u8, nl) == 1 {
                      corpus_total = corpus_total + 1
                      if nfiles < CG_MAXFILES {
                          if npos + nl + 16 < CG_NAMEBUF {
                              noff[nfiles] = npos
                              var c: i64 = 0
                              names[npos] = 114 as u8; npos = npos + 1   // r
                              names[npos] = 117 as u8; npos = npos + 1   // u
                              names[npos] = 110 as u8; npos = npos + 1   // n
                              names[npos] = 116 as u8; npos = npos + 1   // t
                              names[npos] = 105 as u8; npos = npos + 1   // i
                              names[npos] = 109 as u8; npos = npos + 1   // m
                              names[npos] = 101 as u8; npos = npos + 1   // e
                              names[npos] = 47 as u8;  npos = npos + 1   // /
                              while c < nl { names[npos] = dbuf[pos+19+c]; npos = npos + 1; c = c + 1 }
                              names[npos] = 0 as u8; npos = npos + 1
                              nfiles = nfiles + 1
                          }
                      }
                  }
                  if reclen <= 0 { pos = nread } else { pos = pos + reclen }
              }
          }
      }
      sys_close(dfd)
  }
196
  gv_check("corpus-enumerated" as *u8, (nfiles > 50) as i64, ctr)
198
  // ---- compile each file with BOTH compilers and measure -------------------------------------
  let ob: *u8 = sys_mmap(CG_CAP)
  let nb: *u8 = sys_mmap(CG_CAP)
  let args: *i64 = sys_mmap(64) as *i64
203
  var built_old: i64 = 0
  var built_new: i64 = 0
  var both_built: i64 = 0
  var loc_old: i64 = 0
  var loc_new: i64 = 0
  var fn_old: i64 = 0
  var fn_new: i64 = 0
  var files_with_loc_old: i64 = 0
  var files_with_loc_new: i64 = 0
  var regressions: i64 = 0
  var improved: i64 = 0
  var equal_files: i64 = 0
216
  var fi: i64 = batch_start
  var fend: i64 = batch_start + batch_count
  if fend > nfiles { fend = nfiles }
  // Clamp so a window starting past the corpus reports attempted=0 rather than a NEGATIVE count.
  // A nonsense number in a report discredits every honest number printed beside it.
  if fend < batch_start { fend = batch_start }
  while fi < fend {
      let path: *u8 = ((names as i64) + noff[fi]) as *u8
      args[0] = "-g" as i64
      args[1] = path as i64
227
      var rco: i64 = 0
      if fast == 0 { rco = cg_run_out(CG_OLD, args, 2, "/tmp/cg_old.s" as *u8) }
      let rcn: i64 = cg_run_out(CG_NEW, args, 2, "/tmp/cg_new.s" as *u8)
      // In FAST mode the old compiler never ran, so rco keeps its initial 0 and would count
      // every file as "built_old" -- including ones the NEW compiler failed on. A COUNTER FED BY A
      // VARIABLE THAT WAS NEVER ASSIGNED IS REPORTING ITS OWN INITIALISER. Derive it from the run
      // that actually happened.
      if fast == 1 {
          if rcn == 0 { built_old = built_old + 1 }
      } else {
          if rco == 0 { built_old = built_old + 1 }
      }
      if rcn == 0 { built_new = built_new + 1 }
241
      if rco == 0 {
      if rcn == 0 {
          both_built = both_built + 1
          var on: i64 = 1
          if fast == 0 { on = cg_read("/tmp/cg_old.s" as *u8, ob, CG_CAP) }
          let nn: i64 = cg_read("/tmp/cg_new.s" as *u8, nb, CG_CAP)
          if on > 0 {
          if nn > 0 {
              let fn2: i64 = cg_count_funcs(nb, nn)
              let ln: i64 = cg_count_loc(nb, nn)
              // In fast mode the baseline is DERIVED from the proven invariant (one .loc per
              // function) using THIS file own function count -- never a remembered number.
              var lo: i64 = fn2
              var fo: i64 = fn2
              if fast == 0 { lo = cg_count_loc(ob, on); fo = cg_count_funcs(ob, on) }
              loc_old = loc_old + lo
              loc_new = loc_new + ln
              fn_old = fn_old + fo
              fn_new = fn_new + fn2
              if lo > 0 { files_with_loc_old = files_with_loc_old + 1 }
              if ln > 0 { files_with_loc_new = files_with_loc_new + 1 }
              if ln < lo {
                  regressions = regressions + 1
                  // NAME the regressing file. A count alone cannot be acted on.
                  gv_puts("  REGRESSION " as *u8); gv_puts(path)
                  gv_puts(" old_loc=" as *u8); gv_num(lo)
                  gv_puts(" new_loc=" as *u8); gv_num(ln); gv_puts("\n" as *u8)
              }
              if ln > lo { improved = improved + 1 }
              if ln == lo { equal_files = equal_files + 1 }
          } }
      } }
      fi = fi + 1
  }
276
  gv_puts("\n  --- CORPUS MEASUREMENT ---\n" as *u8)
  gv_puts("  corpus_total_nx=" as *u8); gv_num(corpus_total)
  gv_puts("  enumerated=" as *u8);      gv_num(nfiles)
  gv_puts("  batch=[" as *u8); gv_num(batch_start); gv_puts("," as *u8); gv_num(fend); gv_puts(")" as *u8)
  gv_puts("  attempted=" as *u8);       gv_num(fend - batch_start)
  gv_puts("  declared_cap=" as *u8);    gv_num(CG_MAXFILES); gv_puts("\n" as *u8)
  gv_puts("  mode=" as *u8)
  if fast == 1 { gv_puts("FAST(baseline DERIVED from the proven 1-loc-per-function invariant)" as *u8) } else { gv_puts("BOTH(baseline MEASURED by running the old compiler)" as *u8) }
  gv_puts("\n" as *u8)
  gv_puts("  built_old=" as *u8);  gv_num(built_old)
  gv_puts(" built_new=" as *u8);   gv_num(built_new)
  gv_puts(" both_built=" as *u8);  gv_num(both_built); gv_puts("\n" as *u8)
  gv_puts("  files_with_loc old=" as *u8); gv_num(files_with_loc_old)
  gv_puts(" new=" as *u8);                 gv_num(files_with_loc_new); gv_puts("\n" as *u8)
  gv_puts("  total .loc rows  old=" as *u8); gv_num(loc_old)
  gv_puts(" new=" as *u8);                   gv_num(loc_new); gv_puts("\n" as *u8)
  gv_puts("  functions seen   old=" as *u8); gv_num(fn_old)
  gv_puts(" new=" as *u8);                   gv_num(fn_new); gv_puts("\n" as *u8)
  gv_puts("  per-file: improved=" as *u8); gv_num(improved)
  gv_puts(" equal=" as *u8);                gv_num(equal_files)
  gv_puts(" REGRESSED=" as *u8);            gv_num(regressions); gv_puts("\n" as *u8)
298
  // ---- the claims, at corpus scale -----------------------------------------------------------
  gv_check("both-compilers-built-a-real-population" as *u8, (both_built > 20) as i64, ctr)
301
  // The old compiler emits ONE .loc per function. If that is not what we measure, the control is
  // not what this gate believes and every comparison below is void.
  // In BOTH mode this is a real measurement of the old compiler. In FAST mode the baseline is
  // derived from this very invariant, so asserting it would be CIRCULAR -- it is therefore checked
  // only where it can actually fail, and the mode line above says which run you are reading.
  // ★ AN ASSERTION THAT CANNOT FAIL IS NOT EVIDENCE, AND PRESENTING ONE AS EVIDENCE IS THE LIE.
  if fast == 0 {
      gv_check("neg-control-baseline-is-one-loc-per-function" as *u8, ((fn_old > 0) & (loc_old == fn_old)) as i64, ctr)
  }
311
  // THE CLAIM: statement granularity means strictly more rows than functions, corpus-wide.
  gv_check("new-emits-more-rows-than-functions" as *u8, (loc_new > fn_new) as i64, ctr)
314
  // And it must beat the baseline by a wide margin, not by one row on one file.
  gv_check("new-at-least-2x-baseline-rows" as *u8, (loc_new >= 2 * loc_old) as i64, ctr)
317
  // NO FILE MAY LOSE INFORMATION. This is the tooth that matters: an aggregate improvement can
  // hide per-file regressions, and a debugger user does not experience an average.
  // BOUND TO A NON-EMPTY POPULATION. This tooth PASSED over zero files when a batch window fell
  // outside the enumeration -- reporting "no regressions" about nothing at all. A TOOTH THAT
  // PASSES ON THE EMPTY SET IS NOT A TOOTH; every aggregate assertion needs its denominator in
  // the condition, not merely in the printout beside it.
  gv_check("zero-per-file-regressions" as *u8, ((both_built > 0) & (regressions == 0)) as i64, ctr)
325
  // COVERAGE, not just volume: every file the old compiler annotated must still be annotated.
  gv_check("no-file-lost-its-line-info" as *u8, (files_with_loc_new >= files_with_loc_old) as i64, ctr)
328
  return gv_verdict("nx_dwline_corpus_gate" as *u8, ctr,
      "every runtime/*.nx up to a declared cap, compiled by BOTH compilers; per-file regressions named individually" as *u8)
331}
nx_dwline_corpus_gate.nx source

`nx_dwline_corpus_gate.nx` source
