nx_bench_census_lib.nx source
↩ module page · 126 lines · 6184 B
1// nx_bench_census_lib.nx -- ecosystem BENCHMARK-COVERAGE census (the anti-navel-gazing engine scaled to
2// EVERYTHING). Question: of the LIVE tool surface, how many are graded against an EXTERNAL SOTA oracle
3// (measured, not self-asserted)? An ungraded tool is honest DEBT, NEVER silently GREEN. Data-driven:
4// registry rows <tool> <TAB> <oracle> <TAB> <verdict> verdict in {PARITY|EXCEED|UNIQUE|GAP|UNMEASURED}
5// live toolset the tool_allowlist.conf (field 0 of each non-comment line) = the DENOMINATOR
6// A tool with a PARITY/EXCEED/UNIQUE/GAP row is GRADED (GAP counts -- it's MEASURED + named, not hidden).
7// Coverage = graded / live. The uncovered set is PRINTED = the exact debt to eat lap by lap.
8// license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
11
12const BC_TAB: i64 = 9
13const BC_NL: i64 = 10
14const BC_HASH: i64 = 35 // '#' comment line
15const BC_NAME_CAP: i64 = 256
16const BC_VERD_CAP: i64 = 64
17const BC_PERMILLE: i64 = 1000
18const BC_ASCII_0: i64 = 48 // '0' (decimal print)
19const BC_C_LIVE: i64 = 0 // counts[] slot indices (named so array indexing is rule-11 clean)
20const BC_C_GRADED: i64 = 1
21const BC_C_EXCEED: i64 = 2
22const BC_C_GAP: i64 = 3
23const BC_C_UNMEAS: i64 = 4
24const BC_F_VERDICT: i64 = 2 // registry field index: <tool>=0 <oracle>=1 <verdict>=2
25
26func bc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
27// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
28// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
29// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
30// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
31func bc_putn(v: i64) -> i64 { nxi_out(v); return 0 }
32func bc_seq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
33
34// extract the fidx-th (0-based) TAB-separated field of line [ls,le) into out (NUL-terminated); return len
35func bc_field(buf: *u8, ls: i64, le: i64, fidx: i64, out: *u8) -> i64 {
36 var cur: i64 = 0
37 var i: i64 = ls
38 while cur < fidx {
39 if i >= le { out[0] = 0 as u8; return 0 }
40 if buf[i] == (BC_TAB as u8) { cur = cur + 1 }
41 i = i + 1
42 }
43 var o: i64 = 0
44 var go: i64 = 1
45 while go == 1 {
46 if i >= le { go = 0 } else {
47 if buf[i] == (BC_TAB as u8) { go = 0 } else { out[o] = buf[i]; o = o + 1; i = i + 1 }
48 }
49 }
50 out[o] = 0 as u8
51 return o
52}
53// a verdict string is "graded" iff it is one of the measured labels (GAP included -- named, not hidden)
54func bc_is_graded(v: *u8) -> i64 {
55 if bc_seq(v, "PARITY" as *u8) == 1 { return 1 }
56 if bc_seq(v, "EXCEED" as *u8) == 1 { return 1 }
57 if bc_seq(v, "UNIQUE" as *u8) == 1 { return 1 }
58 if bc_seq(v, "GAP" as *u8) == 1 { return 1 }
59 return 0
60}
61// find `tool` in registry reg[0..rn); write its verdict (field 2) into vout; 1 found / 0 not-in-registry
62func bc_lookup(reg: *u8, rn: i64, tool: *u8, vout: *u8) -> i64 {
63 let nm: *u8 = sys_mmap(BC_NAME_CAP)
64 var ls: i64 = 0
65 var i: i64 = 0
66 while i <= rn {
67 var eol: i64 = 0
68 if i == rn { eol = 1 } else { if reg[i] == (BC_NL as u8) { eol = 1 } }
69 if eol == 1 {
70 if i > ls { if reg[ls] != (BC_HASH as u8) {
71 bc_field(reg, ls, i, 0, nm)
72 if bc_seq(nm, tool) == 1 { bc_field(reg, ls, i, BC_F_VERDICT, vout); return 1 }
73 } }
74 ls = i + 1
75 }
76 i = i + 1
77 }
78 vout[0] = 0 as u8
79 return 0
80}
81// census: for each live tool (field0 of allowlist), join the registry; print the UNCOVERED debt.
82// counts[0]=live total, [1]=graded, [2]=exceed, [3]=gap, [4]=unmeasured. Returns the uncovered count.
83func bc_census(allow: *u8, an: i64, reg: *u8, rn: i64, counts: *i64) -> i64 {
84 var live: i64 = 0
85 var graded: i64 = 0
86 var exceed: i64 = 0
87 var gap: i64 = 0
88 var unmeasured: i64 = 0
89 let nm: *u8 = sys_mmap(BC_NAME_CAP)
90 let vd: *u8 = sys_mmap(BC_VERD_CAP)
91 bc_puts("--- UNMEASURED (external-SOTA benchmark = DEBT to eat) ---\n" as *u8)
92 var ls: i64 = 0
93 var i: i64 = 0
94 while i <= an {
95 var eol: i64 = 0
96 if i == an { eol = 1 } else { if allow[i] == (BC_NL as u8) { eol = 1 } }
97 if eol == 1 {
98 if i > ls { if allow[ls] != (BC_HASH as u8) {
99 bc_field(allow, ls, i, 0, nm)
100 if nm[0] != (0 as u8) {
101 live = live + 1
102 if bc_lookup(reg, rn, nm, vd) == 1 {
103 if bc_is_graded(vd) == 1 {
104 graded = graded + 1
105 if bc_seq(vd, "EXCEED" as *u8) == 1 { exceed = exceed + 1 }
106 if bc_seq(vd, "GAP" as *u8) == 1 { gap = gap + 1 }
107 } else { unmeasured = unmeasured + 1; bc_puts(" " as *u8); bc_puts(nm); bc_puts(" (registry=UNMEASURED)\n" as *u8) }
108 } else { unmeasured = unmeasured + 1; bc_puts(" " as *u8); bc_puts(nm); bc_puts(" (no registry row)\n" as *u8) }
109 }
110 } }
111 ls = i + 1
112 }
113 i = i + 1
114 }
115 counts[BC_C_LIVE] = live; counts[BC_C_GRADED] = graded; counts[BC_C_EXCEED] = exceed; counts[BC_C_GAP] = gap; counts[BC_C_UNMEAS] = unmeasured
116 return unmeasured
117}
118// print the report + coverage permille; returns coverage permille
119func bc_report(counts: *i64) -> i64 {
120 var cov: i64 = 0
121 if counts[BC_C_LIVE] > 0 { cov = counts[BC_C_GRADED] * BC_PERMILLE / counts[BC_C_LIVE] }
122 bc_puts("=== BENCH-COVERAGE: " as *u8); bc_putn(counts[BC_C_GRADED]); bc_puts("/" as *u8); bc_putn(counts[BC_C_LIVE])
123 bc_puts(" live tools externally SOTA-graded (" as *u8); bc_putn(cov); bc_puts(" permille); exceed=" as *u8); bc_putn(counts[BC_C_EXCEED])
124 bc_puts(" named-gap=" as *u8); bc_putn(counts[BC_C_GAP]); bc_puts(" UNMEASURED=" as *u8); bc_putn(counts[BC_C_UNMEAS]); bc_puts("\n" as *u8)
125 return cov
126}