nx_benchmark_suite_registry.nx source

1// nx_benchmark_suite_registry.nx -- the RED-TEAM META-HONESTY organ: censuses our own CENSUSES. For each 2// workstream it names the EXTERNAL benchmark SUITE (like MSU for video, Unsloth for LLM-training) and marks 3// whether we are WIRED to it (ran the real tool h2h, both numbers) or UNWIRED (self-graded only = our score 4// is SUSPECT). Operator 2026-07-10: "get suites of tools up for benchmarking so we stop giving ourselves super 5// positive ratings when the nishi red team and critic ... should be calling out our shit." This organ deflates 6// inflated self-ratings BY CONSTRUCTION: an UNWIRED workstream's high self-score is printed as UNVERIFIED, and 7// the "self-congratulation risk" = the count of high self-scores with no external measurement. NOT a self-gate 8// that says we're great -- a self-gate that says WHERE WE HAVE NO RIGHT TO CLAIM ANYTHING YET. license_tier: ORIGINAL 9import "nx_syscalls.nx" 10 11func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 12func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 13func wpad(s: *u8, width: i64) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); var i: i64=n; while i<width { sys_write(1," " as *u8,1); i=i+1 } return 0 } 14// wired status: 2=WIRED(h2h vs external tool, both #s) 1=PARTIAL(oracle referenced, not full h2h) 0=UNWIRED(self-graded=SUSPECT) 15func wstr(c: i64) -> i64 { if c==2 { w("WIRED " as *u8); return 0 } if c==1 { w("PARTIAL " as *u8); return 0 } w("UNWIRED! " as *u8); return 0 } 16 17// one row: workstream, external suite, wired-status, our self-score (permille or -1 n/a), note. returns wired pts. 18func row(ws: *u8, suite: *u8, wired: i64, selfscore: i64, note: *u8) -> i64 { 19 w(" "); wpad(ws, 16); w("suite="); wpad(suite, 30); wstr(wired) 20 w(" self="); if selfscore>=0 { wn(selfscore); w("permil" as *u8) } else { w(" n/a " as *u8) } 21 if wired<2 { if selfscore>=700 { w(" ⚠UNVERIFIED-HIGH" as *u8) } } 22 w(" "); w(note); w("\n" as *u8) 23 return wired 24} 25 26func main() -> i64 { 27 w("=== nx_benchmark_suite_registry: do we grade ourselves, or does an EXTERNAL suite grade us? (red-team honesty) ===\n\n" as *u8) 28 w(" RULE: a self-gate GREEN is NECESSARY-not-sufficient. A quality/SOTA claim needs the EXTERNAL benchmark SUITE\n" as *u8) 29 w(" (real tool run h2h on the same task). UNWIRED + high self-score = we have NO RIGHT to the claim yet.\n\n" as *u8) 30 w(" WORKSTREAM EXTERNAL BENCHMARK SUITE STATUS SELF-SCORE HONEST NOTE\n" as *u8) 31 w(" ------------------------------------------------------------------------------------------------------------\n" as *u8) 32 var wired: i64=0; var total: i64=0; var risk: i64=0 33 34 // WIRED (external tool actually run h2h) -- the ones we can stand on 35 wired=wired+row("video-codec" as *u8, "MSU (ffmpeg x264/x265/AV1+xiph)" as *u8, 2, -1, "GOLD: caught qp*188 21x-gap invisible to self-gates; BD-rate measured" as *u8); total=total+2 36 wired=wired+row("js-vm" as *u8, "V8 (Octane, h2h)" as *u8, 2, -1, "4.09x vs tree-walker MEASURED; Octane run vs V8" as *u8); total=total+2 37 wired=wired+row("crypto" as *u8, "RFC vectors / NIST KAT" as *u8, 2, -1, "KAT-verified vs the standard's own vectors (external truth)" as *u8); total=total+2 38 wired=wired+row("llm-determinism" as *u8, "PyTorch 2.11 float32 (h2h RUN)" as *u8, 2, -1, "★WON+MEASURED 07-10: PyTorch f32 sum order-dependent 16777216/78/80 (none exact); ours integer 16777220 exact+identical" as *u8); total=total+2 39 wired=wired+row("compute-speed" as *u8, "PyTorch/cuBLAS matmul (h2h RUN)" as *u8, 2, -1, "★MEASURED-BEHIND 07-10: ours int-SIMD 8.4Gop/s vs PyTorch CPU 129/cuBLAS 16383/tensor-core 48993 GFLOPs = 15x-5800x BEHIND. NO speed claim" as *u8) 40 if 1==1 { total=total+2 } // compute-speed's tally (WIRED-measured above) 41 // PARTIAL (oracle exists / half-run) 42 wired=wired+row("llm-quality" as *u8, "PyTorch/transformers ppl (h2h RUN)" as *u8, 2, -1, "★MATCH+MEASURED 07-10: our integer ppl 7.37 vs float ref 7.02 = 5pct apart (quant noise) -> inference FAITHFUL + ruler CORRECT" as *u8); total=total+2 43 wired=wired+row("llm-inference-e2e" as *u8, "llama.cpp tok/s (NOT run)" as *u8, 1, -1, "end-to-end serve tok/s vs llama.cpp NOT h2h yet (compute-speed proxy says ~behind); lm-eval MMLU also pending" as *u8); total=total+2 44 wired=wired+row("llm-training" as *u8, "PyTorch CPU (real-scale LM h2h RUN)" as *u8, 2, -1, "★MEASURED 07-11: REAL d=64 ffn=256 LM, BOTH learn 24/24. SIMD-wired (fwd+bwd __i16x16_madd) ours 6.46ms/step vs PyTorch 3.72ms = 1.73x SLOWER (was 4.09x scalar; SIMD grind closed it). Remaining=scalar tail (LUT exp, causal-attn/rmsnorm SIMD). NO speed claim till WON" as *u8); total=total+2 45 // OS LANE -- added 2026-07-20 (osbench, rungs F852-F861). The OS was ABSENT from this ledger 46 // entirely while being the CRITICAL-PATH lane. Publishing our own absence is the first honest act. 47 wired=wired+row("os-kernel" as *u8, "LMbench/sysbench h2h same metal" as *u8, 0, -1, "nx_os_compete is a NARRATIVE scorecard: 1 measured axis (source bytes) then hardcoded EXCEED/BEHIND strings. ZERO kernel-perf numbers. F853 nx_osbench_micro builds the real one" as *u8); total=total+2 48 wired=wired+row("os-ipc" as *u8, "pipe/unix-socket RTT h2h" as *u8, 0, -1, "no IPC throughput or context-switch measurement of ANY kind exists in the tree; F853 axis" as *u8); total=total+2 49 wired=wired+row("os-boot" as *u8, "systemd-analyze / bootchart" as *u8, 0, -1, "boot-to-userland never timed vs Linux on the same silicon; gated on the F856 bare-metal node" as *u8); total=total+2 50 51 // UNWIRED (self-graded only -- the inflation zone) 52 if row("office" as *u8, "Google Docs / LibreOffice" as *u8, 0, 608, "census = OUR judgment of the product, not a measured run vs it" as *u8)<2 { if 608>=700 { risk=risk+1 } } total=total+2 53 if row("crm" as *u8, "Salesforce / HubSpot" as *u8, 0, 1000, "1000permil is a SELF-gate; never run vs the real CRM" as *u8)<2 { risk=risk+1 } total=total+2 54 if row("authz" as *u8, "SpiceDB / OpenFGA / Cedar" as *u8, 0, 764, "28/28 self-gate; SpiceDB not stood up + run h2h" as *u8)<2 { if 764>=700 { risk=risk+1 } } total=total+2 55 if row("containers" as *u8, "Docker / containerd" as *u8, 0, 950, "950permil self; no Docker h2h (startup/throughput/OCI-conformance)" as *u8)<2 { risk=risk+1 } total=total+2 56 if row("deploy-host" as *u8, "Vercel / Fly.io" as *u8, 0, 720, "self-census; no measured deploy-time/cold-start vs them" as *u8)<2 { if 720>=700 { risk=risk+1 } } total=total+2 57 if row("world-gen" as *u8, "Infinigen (banked)" as *u8, 1, 600, "adversarial vs BANKED Infinigen outputs (good) but not run live h2h" as *u8)<2 { total=total+0 } total=total+2 58 if row("data-eng" as *u8, "pandas / DuckDB / Spark" as *u8, 0, 586, "self-gates; no h2h vs pandas/DuckDB on the same dataset+query" as *u8)<2 { risk=risk+1 } total=total+2 59 60 let pm: i64=(wired*1000)/total 61 w("\n BENCHMARK-WIRED COVERAGE: " as *u8); wn(wired); w("/" as *u8); wn(total); w(" points = " as *u8); wn(pm); w(" permille of our scoring is EXTERNALLY MEASURED\n" as *u8) 62 w(" SELF-CONGRATULATION RISK: " as *u8); wn(risk); w(" workstreams carry a HIGH self-score with NO external tool run (UNVERIFIED-HIGH above)\n" as *u8) 63 w("\n ★07-10 FIRST LLM h2h RUN: llm-determinism WIRED -- PyTorch 2.11 (cu128) float32 sum is order-dependent (three\n" as *u8) 64 w(" answers 16777216/78/80, NONE exact) while ours (integer) is 16777220 exact+identical -> we BEAT PyTorch on\n" as *u8) 65 w(" reproducibility, MEASURED, both numbers in hand. HONEST SCOPE: ONE axis; on SPEED PyTorch/CUDA far faster (we lose).\n" as *u8) 66 w(" HONEST VERDICT: video(MSU)/js(V8)/crypto(KAT)/llm-determinism(PyTorch) are benchmark-WIRED -- MSU is the proof it matters\n" as *u8) 67 w(" (it turned 'codec sound' into a measured 21x gap we then FIXED). MOST high self-scores (crm 1000, containers 950,\n" as *u8) 68 w(" authz 764, deploy 720, office 608, data-eng 586, llm) are UNWIRED = self-graded = we have NOT earned the claim.\n" as *u8) 69 w(" ★THE WORK: stand up each UNWIRED suite (external tool + standard tasks + run scripts) -> the red-team/critic then\n" as *u8) 70 w(" grades against the SUITE, not our own GREEN. Priority = highest self-score + zero external # first (most inflation).\n" as *u8) 71 w(" ★07-11 FOUR LLM axes now WIRED vs PyTorch: determinism(WON) · quality/ppl(MATCH 5pct) · compute-speed(LOSE 15-5800x) ·\n" as *u8) 72 w(" training-speed(LOSE ~8x, same-convergence; root=software-float tape). Honest LLM verdict: FAITHFUL+DETERMINISTIC but SLOW.\n" as *u8) 73 w(" ★07-11 PATH-TO-CLOSE MEASURED (not asserted): integer training = 7x faster than software-float on identical work\n" as *u8) 74 w(" (nx_train_int_vs_softfloat), recovering ~7 of the 8x PyTorch gap -> scalar-int ~= PyTorch CPU, SIMD the margin to PASS.\n" as *u8) 75 w(" ★07-11 COMPLETE SOVEREIGN INTEGER TRANSFORMER LM now EXISTS + LEARNS end-to-end (nx_intfp_lm_train 6/6 next-token):\n" as *u8) 76 w(" embed+attn+SwiGLU+2 RMSNorm+2 residual+head+cross-entropy, ALL Q20 integer no-float, 12 gates GREEN (every op +\n" as *u8) 77 w(" both compositions + full layer 84/84 all gradchecked). Sovereign HALF of the h2h READY. Still UNBEATEN externally\n" as *u8) 78 w(" until it's SCALED (real dims/corpus) + SIMD-gemm-wired + the h2h RUN and WON -- NO training SOTA claim before that.\n" as *u8) 79 w(" Suites still to stand up (build queue): llm-inference-e2e(llama.cpp tok/s + lm-eval MMLU) · containers(Docker) ·\n" as *u8) 80 w(" authz(SpiceDB) · data-eng(pandas/DuckDB) · office(LibreOffice) · deploy(Fly/Vercel) · crm(SuiteCRM open).\n" as *u8) 81 w("NX-BENCHMARK-SUITE-REGISTRY verdict=GREEN (honest accounting emitted; " as *u8); wn(risk); w(" inflated self-scores flagged, " as *u8); wn(pm); w("permil externally-wired)\n" as *u8) 82 return 0 83}