code wiki / _hdl_build / nx_ale_harness.nx
nx_ale_harness.nx source
↩ module page · 281 lines · 12402 B
1// nx_ale_harness.nx -- ALE-R1: THE sovereign ALE-style MEASURE harness organ. Composes the
2// existing green phase organs into the end-to-end deployer loop on a task spec:
3// ingest task -> run agent (produce artifact) -> resolve the DECLARED reference (post-run, from
4// the task's grader| field, NEVER staged into the sandbox = no leakage) -> INDEPENDENT
5// deterministic grade -> [0,1] (milli 0..1000) score + log.
6// The harness's reported score is the INDEPENDENT nx_ale_grade of (task's grader|reference|, agent
7// artifact) -- NOT the agent's self_score: the measure is owned by the rubric, not the agent.
8// Reuses _offc/nx_ale_agent.elf (ALE-R2d, itself plan->exec->verify) + _offc/nx_ale_grade.elf
9// (ALE-R1a) via fork/execve (DRY -- no duplicated phase logic). Deterministic (no clock/rand in the
10// loop); no-leakage (the reference is a grader| field resolved AFTER the agent finished).
11// args mode: nx_ale_harness <task> <sandbox> <outdir> <scoreout>
12// -> runs the measure, writes the milli-score to <scoreout>, prints HARNESS score=<n> mode=RUN.
13// no-arg mode: SELF-GATE on the golden fixture (ale_exec_examples/task_exec.txt) with baked
14// controls -> C1 SCORES-0to1 (good->1000, bounded into [0,1000]), C2 NO-LEAKAGE (a decoy
15// reference.txt planted in the sandbox is ignored -> score unchanged), C3 DETERMINISTIC (re-run
16// identical), C4 DISCRIMINATES (a WRONG artifact grades <1000 -> the measure bites, not
17// vacuous) -> HARNESSGATE verdict=GREEN/RED to knowledge/status/ale_harness.log + stdout.
18// Landmines: nested ifs (no &&/||), flat exprs, <=6 args/func, no empty-string literal, strings via
19// Write, openat_wr no O_TRUNC (unlink the scoreout before each grade so a shorter score can't leave
20// trailing digits). license_tier: ORIGINAL
21//
22// module: nishi-core.ale.harness
23// depends: nishi-core.sys.syscalls
24// capability: ALE_MEASURE_HARNESS
25import "nx_syscalls.nx"
26const K_MAGIC_262144: i64 = 262144
27const K_MAGIC_2048: i64 = 2048
28const K_MAGIC_4096: i64 = 4096
29
30func h_p(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
31func h_fp(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 }
32func h_fn(fd: i64, v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m }; let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = 48; k = 1 }; while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }; var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }; sys_write(fd, bb, k); return 0 }
33func h_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
34
35func h_unlink(path: *u8) -> i64 { return __syscall(263, AT_FDCWD, path, 0, 0, 0, 0) }
36
37// read whole file into buf (cap), return byte count (0 on open-fail / empty)
38func h_read(path: *u8, buf: *u8, cap: i64) -> i64 {
39 let fd: i64 = sys_openat_rd(path)
40 if fd < 0 { return 0 }
41 var n: i64 = 0
42 var go: i64 = 1
43 while go == 1 {
44 let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - 1 - n)
45 if r <= 0 { go = 0 } else { n = n + r }
46 if n >= cap - 1 { go = 0 }
47 }
48 sys_close(fd)
49 return n
50}
51
52// parse leading non-negative decimal score from a file (0 if none/empty)
53func h_score_of(path: *u8) -> i64 {
54 let sb: *u8 = sys_mmap(64)
55 let n: i64 = h_read(path, sb, 64)
56 var v: i64 = 0
57 var i: i64 = 0
58 while i < n {
59 if sb[i] >= (48 as u8) { if sb[i] <= (57 as u8) { v = v * 10 + (sb[i] - 48) } }
60 i = i + 1
61 }
62 return v
63}
64
65// write n bytes from buf to a fresh file at path. 1/0.
66func h_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
67 h_unlink(path)
68 let fd: i64 = sys_openat_wr(path, 0x1a4)
69 if fd < 0 { return 0 }
70 if n > 0 { sys_write(fd, buf, n) }
71 sys_close(fd)
72 return 1
73}
74
75// copy the VALUE of the first line that BEGINS with key ("contract|field|") into out (NUL-term);
76// value = bytes after key up to '\n' or EOF. comment lines ("# ...") never begin with the key.
77// returns value length (0 if not found).
78func h_field_value(buf: *u8, n: i64, key: *u8, out: *u8, cap: i64) -> i64 {
79 out[0] = 0 as u8
80 let kl: i64 = h_len(key)
81 var i: i64 = 0
82 var ls: i64 = 0
83 var done: i64 = 0
84 var olen: i64 = 0
85 while i <= n {
86 var eol: i64 = 0
87 if i == n { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
88 if eol == 1 {
89 if done == 0 {
90 if ls + kl <= i {
91 var k: i64 = 0
92 var eq: i64 = 1
93 while k < kl { if buf[ls + k] != key[k] { eq = 0; k = kl } else { k = k + 1 } }
94 if eq == 1 {
95 var p: i64 = ls + kl
96 var o: i64 = 0
97 while p < i { if o < cap - 1 { out[o] = buf[p]; o = o + 1 } p = p + 1 }
98 out[o] = 0 as u8
99 olen = o
100 done = 1
101 }
102 }
103 }
104 ls = i + 1
105 }
106 i = i + 1
107 }
108 return olen
109}
110
111// dst = a + "/" + b (NUL-terminated)
112func h_join(dst: *u8, a: *u8, b: *u8) -> i64 {
113 var o: i64 = 0
114 var i: i64 = 0
115 while a[i] != (0 as u8) { dst[o] = a[i]; o = o + 1; i = i + 1 }
116 dst[o] = 47 as u8; o = o + 1
117 i = 0
118 while b[i] != (0 as u8) { dst[o] = b[i]; o = o + 1; i = i + 1 }
119 dst[o] = 0 as u8
120 return o
121}
122
123// fork/execve _offc/nx_ale_agent.elf <task> <sandbox> <outdir>; child exit code (silenced IO).
124func h_run_agent(task: *u8, sandbox: *u8, outdir: *u8) -> i64 {
125 let pid: i64 = sys_fork()
126 if pid == 0 {
127 let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
128 if dn >= 0 { sys_dup3(dn, 1, 0); sys_dup3(dn, 2, 0) }
129 let argv: *i64 = sys_mmap(48) as *i64
130 argv[0] = "_offc/nx_ale_agent.elf" as *u8 as i64
131 argv[1] = task as i64
132 argv[2] = sandbox as i64
133 argv[3] = outdir as i64
134 argv[4] = 0
135 let envp: *i64 = sys_mmap(16) as *i64
136 envp[0] = 0
137 sys_execve("_offc/nx_ale_agent.elf" as *u8, argv, envp)
138 sys_exit(127)
139 }
140 let st: *i64 = sys_mmap(16) as *i64
141 sys_wait4(pid, st, 0)
142 return (st[0] >> 8) & 0xff
143}
144
145// fork/execve _offc/nx_ale_grade.elf <ref> <artifact> <scoreout>; child exit code (independent grade).
146func h_run_grade(ref: *u8, art: *u8, scoreout: *u8) -> i64 {
147 let pid: i64 = sys_fork()
148 if pid == 0 {
149 let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
150 if dn >= 0 { sys_dup3(dn, 1, 0); sys_dup3(dn, 2, 0) }
151 let argv: *i64 = sys_mmap(48) as *i64
152 argv[0] = "_offc/nx_ale_grade.elf" as *u8 as i64
153 argv[1] = ref as i64
154 argv[2] = art as i64
155 argv[3] = scoreout as i64
156 argv[4] = 0
157 let envp: *i64 = sys_mmap(16) as *i64
158 envp[0] = 0
159 sys_execve("_offc/nx_ale_grade.elf" as *u8, argv, envp)
160 sys_exit(127)
161 }
162 let st: *i64 = sys_mmap(16) as *i64
163 sys_wait4(pid, st, 0)
164 return (st[0] >> 8) & 0xff
165}
166
167// THE MEASURE on (task, sandbox, outdir, scoreout): run agent -> INDEPENDENT grade of the agent
168// artifact against the task's DECLARED grader|reference|. returns milli-score 0..1000; -1 on phase fail.
169func h_measure(task: *u8, sandbox: *u8, outdir: *u8, scoreout: *u8) -> i64 {
170 sys_mkdir(sandbox, 0x1ed)
171 sys_mkdir(outdir, 0x1ed)
172 let tbuf: *u8 = sys_mmap(K_MAGIC_262144)
173 let tn: i64 = h_read(task, tbuf, K_MAGIC_262144)
174 if tn <= 0 { return 0 - 1 }
175 let ref: *u8 = sys_mmap(K_MAGIC_2048)
176 let apath: *u8 = sys_mmap(K_MAGIC_2048)
177 h_field_value(tbuf, tn, "grader|reference|" as *u8, ref, K_MAGIC_2048)
178 h_field_value(tbuf, tn, "task|artifact_path|" as *u8, apath, K_MAGIC_2048)
179 if h_len(ref) == 0 { return 0 - 1 }
180 if h_len(apath) == 0 { return 0 - 1 }
181 let rca: i64 = h_run_agent(task, sandbox, outdir)
182 if rca != 0 { return 0 - 1 }
183 let art: *u8 = sys_mmap(K_MAGIC_4096)
184 h_join(art, sandbox, apath)
185 h_unlink(scoreout)
186 let rcg: i64 = h_run_grade(ref, art, scoreout)
187 if rcg != 0 { return 0 - 1 }
188 return h_score_of(scoreout)
189}
190
191func main(argc: i64, argv: *i64) -> i64 {
192 if argc >= 5 {
193 let sc: i64 = h_measure(argv[1] as *u8, argv[2] as *u8, argv[3] as *u8, argv[4] as *u8)
194 h_p("HARNESS score=" as *u8); h_fn(1, sc); h_p(" mode=RUN\n" as *u8)
195 if sc < 0 { sys_exit(2); return 2 }
196 sys_exit(0); return 0
197 }
198
199 // ---- SELF-GATE on the golden fixture ----
200 let task: *u8 = "knowledge/specs/ale_exec_examples/task_exec.txt" as *u8
201 h_p("=== ALE-harness gate (ALE-R1: THE measure -- run agent -> INDEPENDENT grade -> [0,1]) ===\n" as *u8)
202
203 // C1: good fixture -> score 1000, bounded into [0,1000]
204 let s1: i64 = h_measure(task, "/tmp/_aleh_sb1" as *u8, "/tmp/_aleh_out1" as *u8, "/tmp/_aleh_score1" as *u8)
205 var c1: i64 = 0
206 if s1 == 1000 { c1 = 1 }
207 var bounded: i64 = 0
208 if s1 >= 0 { if s1 <= 1000 { bounded = 1 } }
209
210 // C2 no-leakage: plant a DECOY reference.txt inside the sandbox; the measure resolves the
211 // reference from the task's grader| field, never the sandbox -> score unchanged.
212 sys_mkdir("/tmp/_aleh_sb2" as *u8, 0x1ed)
213 let decoy: *u8 = "unit|row_count|999\nunit|header|XXX\nunit|sorted_by|nope\nunit|checksum|dead\n" as *u8
214 h_write_file("/tmp/_aleh_sb2/reference.txt" as *u8, decoy, h_len(decoy))
215 let s2: i64 = h_measure(task, "/tmp/_aleh_sb2" as *u8, "/tmp/_aleh_out2" as *u8, "/tmp/_aleh_score2" as *u8)
216 var c2: i64 = 0
217 if s2 == s1 { c2 = 1 }
218
219 // C3 deterministic: re-run on a fresh sandbox -> identical score
220 let s3: i64 = h_measure(task, "/tmp/_aleh_sb3" as *u8, "/tmp/_aleh_out3" as *u8, "/tmp/_aleh_score3" as *u8)
221 var c3: i64 = 0
222 if s3 == s1 { c3 = 1 }
223
224 // C4 discriminates: a WRONG artifact (no unit matches the reference) grades <1000 -> the
225 // measure bites; it is not vacuously 1000. Grade independently against the real reference.
226 let tbuf: *u8 = sys_mmap(K_MAGIC_262144)
227 let tn: i64 = h_read(task, tbuf, K_MAGIC_262144)
228 let ref: *u8 = sys_mmap(K_MAGIC_2048)
229 h_field_value(tbuf, tn, "grader|reference|" as *u8, ref, K_MAGIC_2048)
230 let wrong: *u8 = "unit|row_count|ZZZ\nunit|nope|0\n" as *u8
231 h_write_file("/tmp/_aleh_wrong.units" as *u8, wrong, h_len(wrong))
232 h_unlink("/tmp/_aleh_scorew" as *u8)
233 let rcw: i64 = h_run_grade(ref, "/tmp/_aleh_wrong.units" as *u8, "/tmp/_aleh_scorew" as *u8)
234 let sw: i64 = h_score_of("/tmp/_aleh_scorew" as *u8)
235 var c4: i64 = 0
236 if rcw == 0 { if sw >= 0 { if sw < 1000 { c4 = 1 } } }
237
238 h_p(" score_good=" as *u8); h_fn(1, s1)
239 h_p(" score_decoy=" as *u8); h_fn(1, s2)
240 h_p(" score_rerun=" as *u8); h_fn(1, s3)
241 h_p(" score_wrong=" as *u8); h_fn(1, sw)
242 h_p("\n c1_scores_0to1=" as *u8); h_fn(1, c1)
243 h_p(" bounded=" as *u8); h_fn(1, bounded)
244 h_p(" c2_no_leakage=" as *u8); h_fn(1, c2)
245 h_p(" c3_deterministic=" as *u8); h_fn(1, c3)
246 h_p(" c4_discriminates=" as *u8); h_fn(1, c4)
247 h_p("\n" as *u8)
248
249 var allok: i64 = 1
250 if c1 == 0 { allok = 0 }
251 if bounded == 0 { allok = 0 }
252 if c2 == 0 { allok = 0 }
253 if c3 == 0 { allok = 0 }
254 if c4 == 0 { allok = 0 }
255
256 let lfd: i64 = sys_openat_append("knowledge/status/ale_harness.log" as *u8, 0x1a4)
257 if allok == 1 {
258 h_p("HARNESSGATE verdict=GREEN score=1000 scores_0to1=1 no_leakage=1 deterministic=1 discriminates=1 rung=ALE-R1\n" as *u8)
259 if lfd >= 0 {
260 h_fp(lfd, "HARNESSGATE verdict=GREEN score=" as *u8); h_fn(lfd, s1)
261 h_fp(lfd, " scores_0to1=1 no_leakage=1 deterministic=1 discriminates=1 score_wrong=" as *u8); h_fn(lfd, sw)
262 h_fp(lfd, " rung=ALE-R1 epoch=" as *u8); h_fn(lfd, sys_now_realtime_sec())
263 h_fp(lfd, "\n" as *u8)
264 sys_close(lfd)
265 }
266 sys_exit(0)
267 return 0
268 }
269 h_p("HARNESSGATE verdict=RED (a check did not fire)\n" as *u8)
270 if lfd >= 0 {
271 h_fp(lfd, "HARNESSGATE verdict=RED c1=" as *u8); h_fn(lfd, c1)
272 h_fp(lfd, " bounded=" as *u8); h_fn(lfd, bounded)
273 h_fp(lfd, " c2=" as *u8); h_fn(lfd, c2)
274 h_fp(lfd, " c3=" as *u8); h_fn(lfd, c3)
275 h_fp(lfd, " c4=" as *u8); h_fn(lfd, c4)
276 h_fp(lfd, "\n" as *u8)
277 sys_close(lfd)
278 }
279 sys_exit(1)
280 return 1
281}