code wiki / _hdl_build / nx_ale_suite.nx
nx_ale_suite.nx source
↩ module page · 207 lines · 9803 B
1// nx_ale_suite.nx -- ALE-R4: the sovereign ALE SEED-TASK SUITE runner. Reads a manifest (one task
2// spec path per line, '#' comments skipped) and MEASURES each task end-to-end via _offc/
3// nx_ale_harness.elf (the ALE-R1 measure: run agent -> INDEPENDENT deterministic grade -> [0,1]),
4// collecting the per-task milli-scores and aggregating (n_tasks / n_scored / min / mean). The
5// benchmark layer over the measure -- "author N code-graded tasks + measure them" -> one suite
6// verdict + a per-task score breakdown. Reuses the harness (DRY): no duplicated agent/grade logic.
7// args mode: nx_ale_suite <manifest> -> measure that manifest, print SUITE + per-task scores.
8// no-arg: SELF-GATE on the real seed manifest (knowledge/specs/ale_suite.manifest) -> POS all
9// tasks scored + bounded + >=4 tasks -> mean; NEG baked control (a manifest with a BROKEN task
10// -> harness fails -> that task UNSCORED -> all_scored=0) proves the suite DETECTS unsolved (no
11// vacuous green). SUITEGATE verdict=GREEN/RED to knowledge/status/ale_suite.log + stdout.
12// Landmines: nested ifs (no &&/||), flat exprs, <=6 args/func, no empty-string literal, strings via
13// Write. license_tier: ORIGINAL
14//
15// module: nishi-core.ale.suite
16// depends: nishi-core.sys.syscalls
17// capability: ALE_SEED_TASK_SUITE
18import "nx_syscalls.nx"
19const K_MAGIC_65536: i64 = 65536
20const K_MAGIC_1000001: i64 = 1000001
21const K_MAGIC_2048: i64 = 2048
22const K_MAGIC_2047: i64 = 2047
23
24func su_p(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
25func su_fp(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 }
26func su_fn(fd: i64, v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(fd, "-" as *u8, 1) }; let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = 48; k = 1 }; while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }; var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }; sys_write(fd, bb, k); return 0 }
27func su_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
28func su_unlink(path: *u8) -> i64 { return __syscall(263, AT_FDCWD, path, 0, 0, 0, 0) }
29
30func su_read(path: *u8, buf: *u8, cap: i64) -> i64 {
31 let fd: i64 = sys_openat_rd(path)
32 if fd < 0 { return 0 }
33 var n: i64 = 0
34 var go: i64 = 1
35 while go == 1 {
36 let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - 1 - n)
37 if r <= 0 { go = 0 } else { n = n + r }
38 if n >= cap - 1 { go = 0 }
39 }
40 sys_close(fd)
41 return n
42}
43
44func su_score_of(path: *u8) -> i64 {
45 let sb: *u8 = sys_mmap(64)
46 let n: i64 = su_read(path, sb, 64)
47 var v: i64 = 0
48 var i: i64 = 0
49 while i < n { if sb[i] >= (48 as u8) { if sb[i] <= (57 as u8) { v = v * 10 + (sb[i] - 48) } } i = i + 1 }
50 return v
51}
52
53func su_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
54 su_unlink(path)
55 let fd: i64 = sys_openat_wr(path, 0x1a4)
56 if fd < 0 { return 0 }
57 if n > 0 { sys_write(fd, buf, n) }
58 sys_close(fd)
59 return 1
60}
61
62// dst = prefix + decimal(idx) (NUL-terminated)
63func su_idx_path(dst: *u8, prefix: *u8, idx: i64) -> i64 {
64 var o: i64 = 0
65 var i: i64 = 0
66 while prefix[i] != (0 as u8) { dst[o] = prefix[i]; o = o + 1; i = i + 1 }
67 var m: i64 = idx
68 let t: *u8 = sys_mmap(28)
69 var k: i64 = 0
70 if m == 0 { t[0] = 48 as u8; k = 1 }
71 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
72 var p: i64 = 0
73 while p < k { dst[o] = t[k - 1 - p]; o = o + 1; p = p + 1 }
74 dst[o] = 0 as u8
75 return o
76}
77
78// fork/execve _offc/nx_ale_harness.elf <task> <sb> <out> <score>; child exit code (0 ok, !=0 fail).
79func su_run_harness(task: *u8, sb: *u8, out: *u8, score: *u8) -> i64 {
80 let pid: i64 = sys_fork()
81 if pid == 0 {
82 let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
83 if dn >= 0 { sys_dup3(dn, 1, 0); sys_dup3(dn, 2, 0) }
84 let argv: *i64 = sys_mmap(48) as *i64
85 argv[0] = "_offc/nx_ale_harness.elf" as *u8 as i64
86 argv[1] = task as i64
87 argv[2] = sb as i64
88 argv[3] = out as i64
89 argv[4] = score as i64
90 argv[5] = 0
91 let envp: *i64 = sys_mmap(16) as *i64
92 envp[0] = 0
93 sys_execve("_offc/nx_ale_harness.elf" as *u8, argv, envp)
94 sys_exit(127)
95 }
96 let st: *i64 = sys_mmap(16) as *i64
97 sys_wait4(pid, st, 0)
98 return (st[0] >> 8) & 0xff
99}
100
101// measure every task in the manifest. res[0]=n_tasks res[1]=n_scored res[2]=sum res[3]=min.
102func su_run_suite(manifestpath: *u8, res: *i64, verbose: i64) -> i64 {
103 let buf: *u8 = sys_mmap(K_MAGIC_65536)
104 let n: i64 = su_read(manifestpath, buf, K_MAGIC_65536)
105 var n_tasks: i64 = 0
106 var n_scored: i64 = 0
107 var sum: i64 = 0
108 var minv: i64 = K_MAGIC_1000001
109 var idx: i64 = 0
110 var ls: i64 = 0
111 var i: i64 = 0
112 while i <= n {
113 var eol: i64 = 0
114 if i == n { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
115 if eol == 1 {
116 if i > ls { if buf[ls] != (35 as u8) {
117 let tpath: *u8 = sys_mmap(K_MAGIC_2048)
118 var k: i64 = 0
119 var p: i64 = ls
120 while p < i { if k < K_MAGIC_2047 { tpath[k] = buf[p]; k = k + 1 } p = p + 1 }
121 tpath[k] = 0 as u8
122 if k > 0 {
123 n_tasks = n_tasks + 1
124 let sb: *u8 = sys_mmap(256); su_idx_path(sb, "/tmp/_suite_sb_" as *u8, idx)
125 let out: *u8 = sys_mmap(256); su_idx_path(out, "/tmp/_suite_out_" as *u8, idx)
126 let sc: *u8 = sys_mmap(256); su_idx_path(sc, "/tmp/_suite_sc_" as *u8, idx)
127 sys_mkdir(sb, 0x1ed); sys_mkdir(out, 0x1ed)
128 su_unlink(sc)
129 let rc: i64 = su_run_harness(tpath, sb, out, sc)
130 var score: i64 = 0 - 1
131 if rc == 0 { score = su_score_of(sc) }
132 var scored: i64 = 0
133 if rc == 0 { if score >= 0 { if score <= 1000 { scored = 1 } } }
134 if scored == 1 { n_scored = n_scored + 1; sum = sum + score; if score < minv { minv = score } }
135 if verbose == 1 { su_p(" task=" as *u8); su_p(tpath); su_p(" score=" as *u8); su_fn(1, score); su_p(" scored=" as *u8); su_fn(1, scored); su_p("\n" as *u8) }
136 idx = idx + 1
137 }
138 } }
139 ls = i + 1
140 }
141 i = i + 1
142 }
143 if n_scored == 0 { minv = 0 }
144 res[0] = n_tasks; res[1] = n_scored; res[2] = sum; res[3] = minv
145 return 0
146}
147
148func main(argc: i64, argv: *i64) -> i64 {
149 if argc >= 2 {
150 let res: *i64 = sys_mmap(64) as *i64
151 su_p("=== ALE suite (run manifest) ===\n" as *u8)
152 su_run_suite(argv[1] as *u8, res, 1)
153 var mean1: i64 = 0
154 if res[1] > 0 { mean1 = res[2] / res[1] }
155 su_p("SUITE n_tasks=" as *u8); su_fn(1, res[0]); su_p(" scored=" as *u8); su_fn(1, res[1]); su_p(" mean=" as *u8); su_fn(1, mean1); su_p(" min=" as *u8); su_fn(1, res[3]); su_p(" mode=RUN\n" as *u8)
156 sys_exit(0); return 0
157 }
158
159 su_p("=== ALE-suite gate (ALE-R4: seed-task SUITE measured via the ALE-R1 harness) ===\n" as *u8)
160 let res: *i64 = sys_mmap(64) as *i64
161 su_run_suite("knowledge/specs/ale_suite.manifest" as *u8, res, 1)
162 let nt: i64 = res[0]
163 let ns: i64 = res[1]
164 var mean: i64 = 0
165 if ns > 0 { mean = res[2] / ns }
166 let mn: i64 = res[3]
167
168 var c_count: i64 = 0
169 if nt >= 4 { c_count = 1 }
170 var c_allscored: i64 = 0
171 if nt > 0 { if ns == nt { c_allscored = 1 } }
172 var c_bounded: i64 = 0
173 if mn >= 0 { if mean <= 1000 { c_bounded = 1 } }
174
175 // NEG baked control: a manifest naming a BROKEN (nonexistent) task -> harness fails -> the task
176 // is present but UNSCORED -> all_scored would be false -> the suite DETECTS it (no vacuous green).
177 su_write_file("/tmp/_suite_neg.manifest" as *u8, "knowledge/specs/_NO_SUCH_TASK_suite_neg.txt\n" as *u8, 44)
178 let resn: *i64 = sys_mmap(64) as *i64
179 su_run_suite("/tmp/_suite_neg.manifest" as *u8, resn, 0)
180 var c_neg: i64 = 0
181 if resn[0] >= 1 { if resn[1] < resn[0] { c_neg = 1 } }
182
183 su_p(" n_tasks=" as *u8); su_fn(1, nt); su_p(" scored=" as *u8); su_fn(1, ns); su_p(" mean=" as *u8); su_fn(1, mean); su_p(" min=" as *u8); su_fn(1, mn)
184 su_p("\n c_count4=" as *u8); su_fn(1, c_count); su_p(" c_all_scored=" as *u8); su_fn(1, c_allscored); su_p(" c_bounded=" as *u8); su_fn(1, c_bounded); su_p(" c_neg_detects=" as *u8); su_fn(1, c_neg); su_p("\n" as *u8)
185
186 var allok: i64 = 1
187 if c_count == 0 { allok = 0 }
188 if c_allscored == 0 { allok = 0 }
189 if c_bounded == 0 { allok = 0 }
190 if c_neg == 0 { allok = 0 }
191
192 let lfd: i64 = sys_openat_append("knowledge/status/ale_suite.log" as *u8, 0x1a4)
193 if allok == 1 {
194 su_p("SUITEGATE verdict=GREEN n_tasks=" as *u8); su_fn(1, nt); su_p(" all_scored=1 mean=" as *u8); su_fn(1, mean); su_p(" neg_detects=1 rung=ALE-R4\n" as *u8)
195 if lfd >= 0 {
196 su_fp(lfd, "SUITEGATE verdict=GREEN n_tasks=" as *u8); su_fn(lfd, nt)
197 su_fp(lfd, " all_scored=1 mean=" as *u8); su_fn(lfd, mean)
198 su_fp(lfd, " min=" as *u8); su_fn(lfd, mn)
199 su_fp(lfd, " neg_detects=1 rung=ALE-R4 epoch=" as *u8); su_fn(lfd, sys_now_realtime_sec())
200 su_fp(lfd, "\n" as *u8); sys_close(lfd)
201 }
202 sys_exit(0); return 0
203 }
204 su_p("SUITEGATE verdict=RED\n" as *u8)
205 if lfd >= 0 { su_fp(lfd, "SUITEGATE verdict=RED nt=" as *u8); su_fn(lfd, nt); su_fp(lfd, " ns=" as *u8); su_fn(lfd, ns); su_fp(lfd, " c_neg=" as *u8); su_fn(lfd, c_neg); su_fp(lfd, "\n" as *u8); sys_close(lfd) }
206 sys_exit(1); return 1
207}