nx_forge_rag_ab.nx source
↩ module page · 136 lines · 5719 B
1// nx_forge_rag_ab.nx -- THE RAG VALUE A/B (2026-07-17): base-pack vs rag-augmented-pack on the
2// held-out fbench tasks, same engine, same judge, greedy pass@1 (mirrors nx_forge_curves cv_task:
3// fe_run maxc=1 maxnew=400 nsamples=0). Because the 0.5B floors at 0 on fbench pass@1 (A1 measured),
4// each leg ALSO classifies the failure from the banked loop ctx (/tmp/forge_loop_ctx.txt) --
5// 1=GREEN 2=COMPILE-FAIL 3=RUN-FAIL 4=OUTPUT-MISS -9=unknown -- the softer signal with real
6// discrimination on a weak model. Rows -> sovereign store knowledge/store/rag_ab keyed
7// "<leg>:<task>" = [rc, class, us] (latest-wins; re-runnable suite command per growth-loop law).
8// Pre-step (documented): regenerate rag packs via _offc/nx_forge_rag.elf augment <taskf> <mini> ... 2 <ragpack>.
9// usage: nx_forge_rag_ab <port> <chat>
10// license_tier: ORIGINAL expect_exit: 0
11import "nx_forge_engine.nx"
12import "nx_sov_ledger.nx"
13
14const AB_CTXCAP: i64 = 65536
15
16func ab_read(path: *u8, buf: *u8, cap: i64) -> i64 {
17 let fd: i64 = sys_openat_rd(path)
18 if fd < 0 { return 0 - 1 }
19 var total: i64 = 0
20 var going: i64 = 1
21 while going == 1 {
22 let room: i64 = cap - 1 - total
23 if room <= 0 { going = 0 }
24 if going == 1 {
25 let p: *u8 = buf + total
26 let n: i64 = sys_read(fd, p, room)
27 if n <= 0 { going = 0 }
28 if n > 0 { total = total + n }
29 }
30 }
31 sys_close(fd)
32 buf[total] = 0 as u8
33 return total
34}
35
36func ab_find(hay: *u8, hn: i64, needle: *u8) -> i64 {
37 var nl: i64 = 0
38 while needle[nl] != (0 as u8) { nl = nl + 1 }
39 if nl == 0 { return 0 }
40 var i: i64 = 0
41 while i + nl <= hn {
42 var k: i64 = 0
43 var ok: i64 = 1
44 while k < nl { if hay[i+k] != needle[k] { ok = 0; k = nl } else { k = k + 1 } }
45 if ok == 1 { return i }
46 i = i + 1
47 }
48 return 0 - 1
49}
50
51// classify a finished leg: rc>=1 GREEN=1; else read the banked last-failure ctx and discriminate.
52func ab_classify(rc: i64) -> i64 {
53 if rc >= 1 { return 1 }
54 let buf: *u8 = sys_mmap(AB_CTXCAP) as *u8
55 let n: i64 = ab_read("/tmp/forge_loop_ctx.txt" as *u8, buf, AB_CTXCAP)
56 if n <= 0 { return 0 - 9 }
57 let c: i64 = ab_find(buf, n, "COMPILE-FAIL" as *u8)
58 if c >= 0 { return 2 }
59 let jr: i64 = ab_find(buf, n, "output-mismatch" as *u8)
60 if jr >= 0 { return 4 }
61 let r: i64 = ab_find(buf, n, "run-exit=" as *u8)
62 if r >= 0 {
63 let z: i64 = ab_find(buf, n, "run-exit=0" as *u8)
64 if z < 0 { return 3 }
65 return 4
66 }
67 let m: i64 = ab_find(buf, n, "MISMATCH" as *u8)
68 if m >= 0 { return 4 }
69 return 0 - 9
70}
71
72func ab_row(key: *u8, rc: i64, cls: i64) -> i64 {
73 let vals: *i64 = sys_mmap(3 * 8) as *i64
74 vals[0] = rc
75 vals[1] = cls
76 vals[2] = sys_now_us()
77 sov_put_ints("knowledge/store/rag_ab" as *u8, key, vals, 3)
78 return 0
79}
80
81// one leg: engine greedy pass@1 on (organ, task, expect, pack); returns class.
82func ab_leg(key: *u8, organ: *u8, taskf: *u8, expectf: *u8, packf: *u8, port: i64, chat: i64) -> i64 {
83 std_puts("AB-LEG " as *u8)
84 std_puts(key)
85 std_puts(" ..." as *u8)
86 std_puts("\n" as *u8)
87 let rc: i64 = fe_run(organ, taskf, expectf, packf, port, 1, 400, chat, 0, 0)
88 let cls: i64 = ab_classify(rc)
89 ab_row(key, rc, cls)
90 std_puts("AB-ROW " as *u8)
91 std_puts(key)
92 std_puts(" rc=" as *u8)
93 std_pdec(rc)
94 std_puts(" class=" as *u8)
95 std_pdec(cls)
96 std_puts(" (1=GREEN 2=COMPILE 3=RUN 4=OUTPUT)" as *u8)
97 std_puts("\n" as *u8)
98 return cls
99}
100
101func main(argc: i64, argv: *i64) -> i64 {
102 if argc < 3 {
103 std_putln("usage: nx_forge_rag_ab <port> <chat>" as *u8)
104 sys_exit(2)
105 return 2
106 }
107 let a1: i64 = argv[1]
108 let p1: *u8 = a1 as *u8
109 let port: i64 = std_atoi(p1)
110 let a2: i64 = argv[2]
111 let p2: *u8 = a2 as *u8
112 let chat: i64 = std_atoi(p2)
113 std_putln("FORGE-RAG-AB: base-pack vs rag-pack, greedy pass@1 + failure-class, held-out fbench" as *u8)
114 let c1b: i64 = ab_leg("base:T1" as *u8, "abb_t1" as *u8, "knowledge/forge/fix/flp_task_t1.txt" as *u8, "knowledge/forge/fix/expect_lT1.bin" as *u8, "knowledge/forge/nishilang_pack_mini_v1.txt" as *u8, port, chat)
115 let c1r: i64 = ab_leg("rag:T1" as *u8, "abr_t1" as *u8, "knowledge/forge/fix/flp_task_t1.txt" as *u8, "knowledge/forge/fix/expect_lT1.bin" as *u8, "knowledge/forge/ragpack_t1.txt" as *u8, port, chat)
116 let c2b: i64 = ab_leg("base:T2" as *u8, "abb_t2" as *u8, "knowledge/forge/fix/flp_task_t2.txt" as *u8, "knowledge/forge/fix/expect_lT2.bin" as *u8, "knowledge/forge/nishilang_pack_mini_v1.txt" as *u8, port, chat)
117 let c2r: i64 = ab_leg("rag:T2" as *u8, "abr_t2" as *u8, "knowledge/forge/fix/flp_task_t2.txt" as *u8, "knowledge/forge/fix/expect_lT2.bin" as *u8, "knowledge/forge/ragpack_t2.txt" as *u8, port, chat)
118 let c3b: i64 = ab_leg("base:T3" as *u8, "abb_t3" as *u8, "knowledge/forge/fix/flp_task_t3.txt" as *u8, "knowledge/forge/fix/expect_lT3.bin" as *u8, "knowledge/forge/nishilang_pack_mini_v1.txt" as *u8, port, chat)
119 let c3r: i64 = ab_leg("rag:T3" as *u8, "abr_t3" as *u8, "knowledge/forge/fix/flp_task_t3.txt" as *u8, "knowledge/forge/fix/expect_lT3.bin" as *u8, "knowledge/forge/ragpack_t3.txt" as *u8, port, chat)
120 std_puts("FORGE-RAG-AB SUMMARY classes base=[" as *u8)
121 std_pdec(c1b)
122 std_puts("," as *u8)
123 std_pdec(c2b)
124 std_puts("," as *u8)
125 std_pdec(c3b)
126 std_puts("] rag=[" as *u8)
127 std_pdec(c1r)
128 std_puts("," as *u8)
129 std_pdec(c2r)
130 std_puts("," as *u8)
131 std_pdec(c3r)
132 std_puts("] (1=GREEN best, 4=OUTPUT-MISS closest-miss, 2=COMPILE worst)" as *u8)
133 std_puts("\n" as *u8)
134 sys_exit(0)
135 return 0
136}