nx_autograde_page.nx source
↩ module page · 300 lines · 18611 B
1// nx_autograde_page.nx -- the AUTONOMOUS GRADING dashboard for /compare/autograde, written for HUMANS
2// (operator 2026-07-16: "this is garbage output... what human can understand your weird output" -- v1 rendered
3// raw ledger keys/permille/ge4 jargon. v2 rule: every goal is a PLAIN QUESTION a person would ask, every answer
4// is YES/NOT YET/NOT MEASURED + a percentage bar + ONE sentence of what it means. No permille, no key=value).
5// Presentation ONLY: every number comes from the persisted ledgers written by MEASURING ORGANS (nx_autograde,
6// nx_zero_claude_ledger, nx_codegen_bench). This organ TRANSLATES them to English; it never grades.
7// argv[1]=html (default) | json. Emits to STDOUT. license_tier: ORIGINAL expect_exit: 0
8import "nx_syscalls.nx"
9const K_MAGIC_999999: i64 = 999999
10
11func w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
12func wn(v: i64) -> i64 {
13 var m: i64 = v
14 if m < 0 { w("-" as *u8); m = 0 - m }
15 let t: *u8 = sys_mmap(24)
16 var k: i64 = 0
17 if m == 0 { t[0] = 48 as u8; k = 1 }
18 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
19 let o: *u8 = sys_mmap(24)
20 var i: i64 = 0
21 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
22 sys_write(1, o, k)
23 return 0
24}
25func slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
26func streq(a: *u8, b: *u8) -> i64 {
27 var i: i64 = 0
28 var go: i64 = 1
29 var res: i64 = 1
30 while go == 1 {
31 let ca: i64 = a[i] as i64
32 let cb: i64 = b[i] as i64
33 if ca != cb { res = 0; go = 0 }
34 else { if ca == 0 { go = 0 } else { i = i + 1 } }
35 }
36 return res
37}
38func findf(buf: *u8, from: i64, n: i64, needle: *u8) -> i64 {
39 let m: i64 = slen(needle)
40 if m == 0 { return 0 - 1 }
41 var i: i64 = from
42 while i + m <= n {
43 var j: i64 = 0
44 var ok: i64 = 1
45 while j < m { if buf[i+j] != needle[j] { ok = 0; j = m } else { j = j + 1 } }
46 if ok == 1 { return i }
47 i = i + 1
48 }
49 return 0 - 1
50}
51func pint(buf: *u8, n: i64, p: i64) -> i64 {
52 var i: i64 = p
53 if i >= n { return 0 - K_MAGIC_999999 }
54 var neg: i64 = 0
55 if (buf[i] as i64) == 45 { neg = 1; i = i + 1 }
56 var v: i64 = 0
57 var any: i64 = 0
58 var go: i64 = 1
59 while go == 1 {
60 if i >= n { go = 0 }
61 else {
62 let d: i64 = buf[i] as i64
63 if d >= 48 { if d <= 57 { v = v*10 + (d-48); any = 1; i = i + 1 } else { go = 0 } }
64 else { go = 0 }
65 }
66 }
67 if any == 0 { return 0 - K_MAGIC_999999 }
68 if neg == 1 { return 0 - v }
69 return v
70}
71func tok_after(buf: *u8, ls: i64, le: i64, key: *u8, dst: *u8, dcap: i64) -> i64 {
72 let ki: i64 = findf(buf, ls, le, key)
73 if ki < 0 { dst[0] = 0 as u8; return 0 }
74 var i: i64 = ki + slen(key)
75 var j: i64 = 0
76 var go: i64 = 1
77 while go == 1 {
78 if i >= le { go = 0 }
79 else { let c: i64 = buf[i] as i64
80 if c == 32 { go = 0 } else { if c == 10 { go = 0 } else { if j >= dcap-1 { go = 0 } else { dst[j] = buf[i]; j = j + 1; i = i + 1 } } } }
81 }
82 dst[j] = 0 as u8
83 return j
84}
85func int_after(buf: *u8, ls: i64, le: i64, key: *u8) -> i64 {
86 let ki: i64 = findf(buf, ls, le, key)
87 if ki < 0 { return 0 - K_MAGIC_999999 }
88 return pint(buf, le, ki + slen(key))
89}
90func read_ledger(path: *u8, lenout: *i64) -> *u8 {
91 let b: *u8 = sys_read_file(path, lenout)
92 if (b as i64) == 0 { lenout[0] = 0 }
93 return b
94}
95// verdict of target `name` in the LAST autograde run: 1=MET 2=NOTMET 0=UNMEAS -1=absent. value via valout.
96func target_verdict(ag: *u8, n: i64, last_ts: i64, name: *u8, valout: *i64) -> i64 {
97 let nb: *u8 = sys_mmap(64)
98 let vb: *u8 = sys_mmap(32)
99 var p: i64 = 0
100 var res: i64 = 0 - 1
101 while p < n {
102 var e: i64 = p
103 var f: i64 = 0
104 while f == 0 { if e >= n { f = 1 } else { if ag[e] == (10 as u8) { f = 1 } else { e = e + 1 } } }
105 let t: i64 = int_after(ag, p, e, "ts=" as *u8)
106 if t == last_ts {
107 tok_after(ag, p, e, "name=" as *u8, nb, 64)
108 if streq(nb, name) == 1 {
109 valout[0] = int_after(ag, p, e, "value=" as *u8)
110 tok_after(ag, p, e, "verdict=" as *u8, vb, 32)
111 if streq(vb, "MET" as *u8) == 1 { res = 1 }
112 else { if streq(vb, "NOTMET" as *u8) == 1 { res = 2 } else { res = 0 } }
113 }
114 }
115 p = e + 1
116 }
117 return res
118}
119// emit one scoreboard row: a question, a YES/NOT-YET/NOT-MEASURED chip, and a plain sentence
120func row(question: *u8, verdict: i64, sentence: *u8) -> i64 {
121 w("<div class='q'><div class='qq'>" as *u8)
122 w(question)
123 w("</div><div>" as *u8)
124 if verdict == 1 { w("<span class='chip yes'>✓ YES</span>" as *u8) }
125 if verdict == 2 { w("<span class='chip notyet'>NOT YET</span>" as *u8) }
126 if verdict == 0 { w("<span class='chip nomeas'>NOT MEASURED YET</span>" as *u8) }
127 if verdict < 0 { w("<span class='chip nomeas'>NO DATA</span>" as *u8) }
128 w(" <span class='ans'>" as *u8)
129 w(sentence)
130 w("</span></div></div>\n" as *u8)
131 return 0
132}
133
134func main(argc: i64, argv: *i64) -> i64 {
135 var jsonmode: i64 = 0
136 if argc >= 2 {
137 let a1: i64 = argv[1]
138 if streq(a1 as *u8, "json" as *u8) == 1 { jsonmode = 1 }
139 }
140 let ts: i64 = sys_now_realtime_sec()
141
142 // ---- load ledgers ----
143 let agn: *i64 = sys_mmap(8) as *i64
144 let ag: *u8 = read_ledger("/home/elderwesto/nx_stage/autograde_ledger.log" as *u8, agn)
145 let zcn: *i64 = sys_mmap(8) as *i64
146 let zc: *u8 = read_ledger("/home/elderwesto/nx_stage/zero_claude_ledger.log" as *u8, zcn)
147
148 // ---- last autograde run ts ----
149 var last_ts: i64 = 0
150 var p: i64 = 0
151 while p < agn[0] {
152 var e: i64 = p
153 var f: i64 = 0
154 while f == 0 { if e >= agn[0] { f = 1 } else { if ag[e] == (10 as u8) { f = 1 } else { e = e + 1 } } }
155 let t: i64 = int_after(ag, p, e, "ts=" as *u8)
156 if t > last_ts { last_ts = t }
157 p = e + 1
158 }
159
160 // ---- pull each target's verdict ----
161 let vbox: *i64 = sys_mmap(8) as *i64
162 vbox[0] = 0
163 let v_checker: i64 = target_verdict(ag, agn[0], last_ts, "codegen-grader-sound" as *u8, vbox)
164 let v_fixloop: i64 = target_verdict(ag, agn[0], last_ts, "autofix-loop-sound" as *u8, vbox)
165 let v_suite: i64 = target_verdict(ag, agn[0], last_ts, "codegen-ref-passrate" as *u8, vbox)
166 let v_gcc: i64 = target_verdict(ag, agn[0], last_ts, "exceed-gcc-scalar" as *u8, vbox)
167 let v_local: i64 = target_verdict(ag, agn[0], last_ts, "local-codegen-green" as *u8, vbox)
168 let swbbox: *i64 = sys_mmap(8) as *i64
169 swbbox[0] = 0
170 let v_swb: i64 = target_verdict(ag, agn[0], last_ts, "local-fix-resolve-rate" as *u8, swbbox)
171 var swbpct: i64 = 0
172 if swbbox[0] != (0 - K_MAGIC_999999) { swbpct = swbbox[0] }
173 let liftbox: *i64 = sys_mmap(8) as *i64
174 liftbox[0] = 0
175 let v_lift: i64 = target_verdict(ag, agn[0], last_ts, "zero-claude-lift" as *u8, liftbox)
176 var liftpct: i64 = 0
177 if liftbox[0] != (0 - K_MAGIC_999999) { liftpct = (liftbox[0] + 5) / 10 }
178
179 // ---- zero-claude trend runs (permille -> percent) ----
180 let modbuf: *u8 = sys_mmap(32)
181
182 if jsonmode == 1 {
183 w("{\"v\":2,\"api\":\"nishi-compare\",\"generated_unix\":" as *u8); wn(ts)
184 w(",\"domain\":\"autograde\",\"kind\":\"dashboard\",\"title\":\"Is our system standing on its own? -- live scoreboard\"" as *u8)
185 w(",\"plain_english\":true,\"measured_by\":\"the system itself (nx_autograde runs the tests; this page only translates the results)\"" as *u8)
186 w(",\"scoreboard\":[" as *u8)
187 w("{\"question\":\"Can we automatically check whether generated code is correct?\",\"answer\":\"" as *u8)
188 if v_checker == 1 { w("yes" as *u8) } else { w("not yet" as *u8) }
189 w("\"},{\"question\":\"Can the system find a bug, ask permission, fix it, and undo the fix?\",\"answer\":\"" as *u8)
190 if v_fixloop == 1 { w("yes" as *u8) } else { w("not yet" as *u8) }
191 w("\"},{\"question\":\"How much of the work runs with zero Claude cost?\",\"answer\":\"" as *u8)
192 wn(liftpct)
193 w(" percent, goal 100\"},{\"question\":\"Does the code test suite itself work?\",\"answer\":\"" as *u8)
194 if v_suite == 1 { w("yes" as *u8) } else { w("not yet" as *u8) }
195 w("\"},{\"question\":\"Is our own compiler as fast as the industry standard (gcc)?\",\"answer\":\"" as *u8)
196 if v_gcc == 0 { w("not measured yet -- we refuse to guess" as *u8) } else { if v_gcc == 1 { w("yes" as *u8) } else { w("not yet" as *u8) } }
197 w("\"},{\"question\":\"Can the local AI produce working code by itself?\",\"answer\":\"" as *u8)
198 if v_local == 1 { w("yes for small fixes: it finds the failing function itself, writes the fix, and the machine verifies it -- zero Claude" as *u8) } else { w("not yet" as *u8) }
199 w("\"},{\"question\":\"Graded by the industry bug-fixing benchmark's own rules (SWE-bench contract), what's our resolve rate?\",\"answer\":\"" as *u8)
200 if v_swb == 1 { wn(swbpct); w(" percent of our foreign-bug instances resolved (fix passes, nothing else breaks); sovereign analog, not the Python set" as *u8) } else { w("not measured yet" as *u8) }
201 w("\"}],\"claude_free_trend_percent\":[" as *u8)
202 var first: i64 = 1
203 p = 0
204 while p < zcn[0] {
205 var e3: i64 = p
206 var f3: i64 = 0
207 while f3 == 0 { if e3 >= zcn[0] { f3 = 1 } else { if zc[e3] == (10 as u8) { f3 = 1 } else { e3 = e3 + 1 } } }
208 let lp: i64 = int_after(zc, p, e3, "liftpm=" as *u8)
209 if lp != (0 - K_MAGIC_999999) {
210 tok_after(zc, p, e3, "model=" as *u8, modbuf, 32)
211 if first == 0 { w("," as *u8) }
212 first = 0
213 w("{\"local_model\":\"" as *u8); w(modbuf)
214 w("\",\"percent\":" as *u8); wn((lp + 5) / 10)
215 w("}" as *u8)
216 }
217 p = e3 + 1
218 }
219 w("]}" as *u8)
220 return 0
221 }
222
223 // ---------- HTML ----------
224 w("<!DOCTYPE html>\n<html lang=\"en\"><head><meta charset='utf-8'><meta name='viewport' content='width=device-width, initial-scale=1'>\n" as *u8)
225 w("<title>Is our system standing on its own? -- live scoreboard</title>\n" as *u8)
226 w("<style>body{font-family:-apple-system,Segoe UI,Roboto,sans-serif;max-width:760px;margin:0 auto;padding:0 20px 6vh;line-height:1.65;color:rgb(22,22,34)}h1{color:rgb(42,77,143);font-size:1.7rem}h2{font-size:1.15rem;margin-top:30px}.q{border:1px solid rgb(227,227,234);border-radius:12px;padding:14px 18px;margin:12px 0;box-shadow:0 1px 3px rgba(16,20,28,0.06)}.qq{font-weight:700;margin-bottom:6px}.chip{display:inline-block;font-size:.8rem;font-weight:800;padding:3px 12px;border-radius:20px;color:rgb(255,255,255);vertical-align:middle}.yes{background:rgb(20,130,70)}.notyet{background:rgb(200,120,30)}.nomeas{background:rgb(130,130,140)}.ans{font-size:.95rem;margin-left:8px}.trend{margin:10px 0}.tl{font-size:.9rem;margin-bottom:2px}.tb{background:rgb(236,238,245);border-radius:9px;height:26px;position:relative;overflow:hidden}.tf{background:linear-gradient(90deg,rgb(42,77,143),rgb(90,60,190));height:100%;border-radius:9px 0 0 9px}.tp{position:absolute;left:10px;top:2px;color:rgb(255,255,255);font-weight:800;font-size:.85rem}.mut{color:rgb(102,102,102);font-size:.85rem}@media(prefers-color-scheme:dark){body{background:rgb(15,15,20);color:rgb(230,230,238)}.q{border-color:rgb(38,38,47)}.tb{background:rgb(30,30,40)}}</style></head><body>\n" as *u8)
227 w("<p class='mut'><a href='/compare'>Nishi Compare</a> › Standing on our own</p>\n" as *u8)
228 w("<h1>Is our system standing on its own?</h1>\n" as *u8)
229 w("<p>We are building toward one goal: our own local system does the work, and Claude (the paid AI) is needed less and less. This scoreboard is measured <b>by the system itself</b> — it runs its own tests and reports what actually happened. It cannot be sweet-talked: anything not truly measured says <b>NOT MEASURED YET</b> instead of pretending.</p>\n" as *u8)
230
231 w("<h2>The scoreboard</h2>\n" as *u8)
232 row("Can we automatically check whether generated code is correct?" as *u8, v_checker,
233 "Our checker compiles the code, runs it, and compares the answers. It passed all its self-tests — including the important one: it correctly REJECTS wrong code instead of approving everything." as *u8)
234 row("Can the system find a bug, ask permission, fix it, and undo the fix?" as *u8, v_fixloop,
235 "Proven end to end: it spotted a planted bug on its own, wrote the finding to an approval queue (nothing changes without a person saying yes), applied the fix, re-tested it, and restored the original perfectly." as *u8)
236 // the lift row gets its own bar treatment below the chip
237 w("<div class='q'><div class='qq'>How much of the work already runs with zero Claude cost?</div><div>" as *u8)
238 if v_lift == 1 { w("<span class='chip yes'>✓ DONE</span>" as *u8) } else { w("<span class='chip notyet'>" as *u8); wn(liftpct); w("% SO FAR</span>" as *u8) }
239 w(" <span class='ans'>Goal: 100%. On our test set, " as *u8)
240 wn(liftpct)
241 w("% of tasks were answered by our own local model and verified correct by our own checker — no Claude involved, no tokens spent. The rest still get sent to Claude.</span></div>" as *u8)
242 w("<div class='trend' style='margin-top:10px'>" as *u8)
243 p = 0
244 var run: i64 = 0
245 while p < zcn[0] {
246 var e6: i64 = p
247 var f6: i64 = 0
248 while f6 == 0 { if e6 >= zcn[0] { f6 = 1 } else { if zc[e6] == (10 as u8) { f6 = 1 } else { e6 = e6 + 1 } } }
249 let lp6: i64 = int_after(zc, p, e6, "liftpm=" as *u8)
250 if lp6 != (0 - K_MAGIC_999999) {
251 run = run + 1
252 tok_after(zc, p, e6, "model=" as *u8, modbuf, 32)
253 let pct: i64 = (lp6 + 5) / 10
254 w("<div class='tl'>measurement " as *u8); wn(run)
255 w(" — local model size " as *u8); w(modbuf)
256 w("</div><div class='tb'><div class='tf' style='width:" as *u8)
257 wn(pct)
258 w("%'></div><div class='tp'>" as *u8)
259 wn(pct)
260 w("%</div></div>" as *u8)
261 }
262 p = e6 + 1
263 }
264 w("</div><div class='mut'>The jump came from upgrading the local model. Next jumps: teach it more skills, and train it on exactly the tasks it still hands to Claude.</div></div>\n" as *u8)
265
266 row("Does the code test suite itself work?" as *u8, v_suite,
267 "Health check: known-good solutions pass all tests (so a failure really means the code is wrong, not that the tests are broken)." as *u8)
268 // the gcc row renders the compiler lane's own lap ledger dynamically (value = ratio x1000; 1000 = parity)
269 let gccval: *i64 = sys_mmap(8) as *i64
270 gccval[0] = 0
271 let v_gcc2: i64 = target_verdict(ag, agn[0], last_ts, "exceed-gcc-scalar" as *u8, gccval)
272 w("<div class='q'><div class='qq'>Is our own compiler as fast as the industry standard (gcc)?</div><div>" as *u8)
273 if v_gcc2 == 1 { w("<span class='chip yes'>✓ YES</span> <span class='ans'>At or faster than gcc on the official measured lap — parity is the bar, and we cleared it.</span>" as *u8) }
274 else { if v_gcc2 == 2 {
275 let behind: i64 = (gccval[0] - 1000 + 5) / 10
276 w("<span class='chip notyet'>ALMOST — " as *u8)
277 wn(behind)
278 w("% SLOWER</span> <span class='ans'>Official lap from the compiler team's own ledger: " as *u8)
279 wn(behind)
280 w("% slower than gcc on the toughest scalar loop (and already faster than gcc on one benchmark). The register-allocator work now in progress is expected to close it; every new lap updates this row automatically.</span>" as *u8)
281 } else { w("<span class='chip nomeas'>NOT MEASURED YET</span> <span class='ans'>We refuse to guess: until an official lap is recorded, this row stays unmeasured.</span>" as *u8) } }
282 w("</div></div>\n" as *u8)
283
284 if v_swb == 1 {
285 w("<div class='q'><div class='qq'>Graded by the industry bug-fixing benchmark's own rules, how does our fixer score?</div><div><span class='chip yes'>✓ " as *u8)
286 wn(swbpct)
287 w("% RESOLVED</span> <span class='ans'>SWE-bench is the field's standard bug-fixing test: patch a real issue, and it only counts if the broken tests pass AND nothing that worked before breaks. We graded our own fix loop by that exact cheat-proof rule — it resolves " as *u8)
288 wn(swbpct)
289 w("% of our bug set. Honest scope: this is our own sovereign set under SWE-bench's rules, not its Python problems (a no-Python system can't run those); we claim the same standard, not parity with agents that run the real set.</span></div></div>\n" as *u8)
290 }
291 w("<h2>Can our local AI write working code by itself yet?</h2>\n" as *u8)
292 if v_local == 1 {
293 w("<p><b>YES for small fixes — with honest limits.</b> The local model now runs the whole repair loop on its own: it reads the test results to <b>find which function is broken</b>, pulls that function's source, writes the corrected code, and the independent checker confirms the repair — compiled, ran, exact right answers, zero Claude anywhere in the content. Proven on different bugs in different functions, including a repair shape it was never shown. Writing whole programs from scratch is still beyond it, so every remaining failure is saved automatically as training material. The checker is what keeps this honest: code only counts when it runs and produces exactly the right answer.</p>\n" as *u8)
294 } else {
295 w("<p><b>Not yet — and here is the honest picture.</b> The small model just parrots text back. The bigger model now writes real programs, but like a beginner: 2 of its last 3 programs compiled and ran, yet gave wrong answers (it invented variables it never defined, and guessed instead of reading the input). Every one of these failures is saved automatically and becomes training material. The checker above is what keeps this honest: a program only counts when it runs and produces exactly the right answer.</p>\n" as *u8)
296 }
297
298 w("<p class='mut'>Updated automatically each time the system re-measures itself. This page only translates the measurement records into plain English — the numbers come from the machine, not from anyone's opinion. <a href='/compare/autograde/api.json'>machine-readable version</a></p>\n</body></html>\n" as *u8)
299 return 0
300}