nx_effbench_research_fetch.nx source
↩ module page · 79 lines · 5506 B
1// nx_effbench_research_fetch.nx -- SOVEREIGN researcher: GROUND the "are we S-class EXCEED vs Unsloth & friends"
2// benchmark in REAL fetched facts (Rule 4: benchmarks need VERIFIED competitor claims, never assumptions). The
3// incumbents = efficient LLM training/inference frameworks (Unsloth, llama.cpp, QLoRA, GPTQ, AWQ, bitsandbytes/
4// LLM.int8, LoRA). We measure our no-float Q16 stack against THEIR real claims (speedup, memory, bit-width) so
5// the EXCEED verdict is measured + honest (no-wave), not a floor-claim.
6// Mirrors the proven fetcher: sovereign TLS-1.3 (nx_https_fetch_follow) + Mozilla CA, idempotent skip-if-have,
7// saves each source to knowledge/fetched/eff_*.raw so the corpus is greppable to CITE. expect_exit: 0
8import "nx_syscalls.nx"
9import "nx_x509_trust_store.nx"
10import "nx_trust_store_load_from_certdata.nx"
11import "nx_https_fetch_follow.nx"
12const K_MAGIC_4194304: i64 = 4194304
13const K_MAGIC_8388608: i64 = 8388608
14
15func df_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func df_putn(v: i64) -> i64 {
17 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
18 var m: i64 = v
19 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
20 let d: *u8 = sys_mmap(24); var k: i64 = 0
21 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
22 var j: i64 = k - 1
23 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
24 return 0
25}
26func have_file(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
27func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
28 if have_file(opath) == 1 { df_puts(opath); df_puts(" [have-skip]\n"); return 1 }
29 let status: *i64 = sys_mmap(8) as *i64
30 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
31 df_puts(url); df_puts(" status="); df_putn(status[0]); df_puts(" bytes="); df_putn(n)
32 if n <= 0 { df_puts(" FETCH-FAIL\n"); return 0 }
33 var gz: i64 = 0
34 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
35 if gz == 1 { df_puts(" [GZIP-skip]\n"); return 0 }
36 let fd: i64 = sys_openat_wr(opath, 0x1a4)
37 if fd < 0 { df_puts(" SAVE-FAIL\n"); return 0 }
38 sys_write(fd, out, n); sys_close(fd)
39 df_puts(" SAVED\n")
40 return 1
41}
42
43func main() -> i64 {
44 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
45 if r <= 0 { df_puts("EFFBENCH: certdata load failed\n"); return 1 }
46 let store: *TrustStore = r as *TrustStore
47 df_puts("CA roots="); df_putn(trust_store_count(store)); df_puts("\n")
48 let cap: i64 = K_MAGIC_8388608
49 let out: *u8 = sys_mmap(cap)
50 var ok: i64 = 0
51
52 df_puts("== EFFICIENT FINE-TUNING FRAMEWORKS (the incumbents we benchmark against) ==\n")
53 ok = ok + fetch_save("https://github.com/unslothai/unsloth" as *u8, "knowledge/fetched/eff_unsloth.raw" as *u8, store, out, cap)
54 ok = ok + fetch_save("https://arxiv.org/abs/2305.14314" as *u8, "knowledge/fetched/eff_qlora.raw" as *u8, store, out, cap)
55 ok = ok + fetch_save("https://arxiv.org/abs/2106.09685" as *u8, "knowledge/fetched/eff_lora.raw" as *u8, store, out, cap)
56
57 df_puts("== QUANTIZATION (post-training + int8 + activation-aware) ==\n")
58 ok = ok + fetch_save("https://arxiv.org/abs/2210.17323" as *u8, "knowledge/fetched/eff_gptq.raw" as *u8, store, out, cap)
59 ok = ok + fetch_save("https://arxiv.org/abs/2306.00978" as *u8, "knowledge/fetched/eff_awq.raw" as *u8, store, out, cap)
60 ok = ok + fetch_save("https://arxiv.org/abs/2208.07339" as *u8, "knowledge/fetched/eff_llmint8.raw" as *u8, store, out, cap)
61 ok = ok + fetch_save("https://arxiv.org/abs/2402.17764" as *u8, "knowledge/fetched/eff_bitnet158.raw" as *u8, store, out, cap)
62 ok = ok + fetch_save("https://arxiv.org/abs/1712.05877" as *u8, "knowledge/fetched/eff_intonly.raw" as *u8, store, out, cap)
63
64 df_puts("== CPU / SOVEREIGN-ADJACENT INFERENCE + our DETERMINISM axis ==\n")
65 ok = ok + fetch_save("https://github.com/ggerganov/llama.cpp" as *u8, "knowledge/fetched/eff_llamacpp.raw" as *u8, store, out, cap)
66 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Llama.cpp" as *u8, "knowledge/fetched/eff_llamacpp_wiki.raw" as *u8, store, out, cap)
67 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Floating-point_arithmetic" as *u8, "knowledge/fetched/eff_float.raw" as *u8, store, out, cap)
68 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Reproducibility" as *u8, "knowledge/fetched/eff_reproducibility.raw" as *u8, store, out, cap)
69
70 df_puts("== DEEPER: KV/throughput/quant SOTA + determinism foundation + model-format portability ==\n")
71 ok = ok + fetch_save("https://arxiv.org/abs/2309.06180" as *u8, "knowledge/fetched/eff_vllm.raw" as *u8, store, out, cap)
72 ok = ok + fetch_save("https://arxiv.org/abs/2211.10438" as *u8, "knowledge/fetched/eff_smoothquant.raw" as *u8, store, out, cap)
73 ok = ok + fetch_save("https://github.com/NVIDIA/TensorRT-LLM" as *u8, "knowledge/fetched/eff_tensorrtllm.raw" as *u8, store, out, cap)
74 ok = ok + fetch_save("https://en.wikipedia.org/wiki/IEEE_754" as *u8, "knowledge/fetched/eff_ieee754.raw" as *u8, store, out, cap)
75 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Open_Neural_Network_Exchange" as *u8, "knowledge/fetched/eff_onnx.raw" as *u8, store, out, cap)
76
77 df_puts("EFFBENCH READABLE SOURCES SAVED: "); df_putn(ok); df_puts(" / 17\n")
78 return 0
79}