nx_research_verify_synth_gate.nx source
↩ module page · 72 lines · 5152 B
1// nx_research_verify_synth_gate.nx -- liar-kill gate for the S1<->S2 SELF-VERIFYING RESEARCHER.
2// TIER-1 (deterministic, $0, no model): fixture "synthesized" answers over numeric sources. A correct answer's
3// numbers are CONFIRMED against their cited sources; an answer with FABRICATED numbers (50.0 not in a source
4// that says 51.0/86.8; 999999 not in the source) is DISCREDITED -- the researcher catches its OWN numeric
5// hallucination. An uncited numeric claim is UNVERIFIABLE. TIER-2 (marker knowledge/index/rvs_model.on):
6// run the REAL composed rvs_answer (Qwen synth -> self-verify). license_tier: ORIGINAL
7import "nx_research_verify_synth.nx"
8import "nx_syscalls.nx"
9
10func st_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func st_putn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
12func st_len(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
13
14func main() -> i64 {
15 var pass: i64 = 0
16 var total: i64 = 0
17 let out: *i64 = sys_mmap(64) as *i64
18 let out2: *i64 = sys_mmap(64) as *i64
19
20 let srcv: *i64 = sys_mmap(8*8) as *i64
21 srcv[0] = ("The logistic regression baseline reaches an F1 of 51.0 and human performance is 86.8 percent." as *u8) as i64
22 srcv[1] = ("The dataset contains 107785 question answer pairs across 536 articles." as *u8) as i64
23
24 let a_good: *u8 = "Human performance is 86.8 [1]. The dataset has 107785 pairs [2]." as *u8
25 let a_hall: *u8 = "Human performance is 50.0 [1]. The dataset has 999999 pairs [2]." as *u8
26 let a_unci: *u8 = "The score is 86.8." as *u8
27
28 // T1: a CORRECT answer -- both numbers present in their cited sources -> CONFIRMED, none discredited
29 total = total + 1
30 rvs_verify_answer(a_good, st_len(a_good), srcv, 2, 5, out)
31 st_puts("T1 correct: numeric=" as *u8); st_putn(out[0]); st_puts(" confirmed=" as *u8); st_putn(out[1]); st_puts(" discredited=" as *u8); st_putn(out[2]); st_puts(" unverifiable=" as *u8); st_putn(out[3]); st_puts("\n" as *u8)
32 if out[0] == 2 { if out[1] == 2 { if out[2] == 0 { pass = pass + 1 } else { st_puts("T1 FAIL false-discredit\n" as *u8) } } else { st_puts("T1 FAIL not-all-confirmed\n" as *u8) } } else { st_puts("T1 FAIL numeric-count\n" as *u8) }
33
34 // T2 LIAR-KILL: FABRICATED numbers (50.0, 999999 not in the cited sources) MUST be DISCREDITED
35 total = total + 1
36 rvs_verify_answer(a_hall, st_len(a_hall), srcv, 2, 5, out)
37 st_puts("T2 hallucinated: numeric=" as *u8); st_putn(out[0]); st_puts(" confirmed=" as *u8); st_putn(out[1]); st_puts(" discredited=" as *u8); st_putn(out[2]); st_puts("\n" as *u8)
38 if out[0] == 2 { if out[2] == 2 { if out[1] == 0 { pass = pass + 1 } else { st_puts("T2 FAIL some-confirmed\n" as *u8) } } else { st_puts("T2 FAIL fabrication-not-caught\n" as *u8) } } else { st_puts("T2 FAIL numeric-count\n" as *u8) }
39
40 // T3: an UNCITED numeric claim can't be verified -> UNVERIFIABLE
41 total = total + 1
42 rvs_verify_answer(a_unci, st_len(a_unci), srcv, 2, 5, out)
43 st_puts("T3 uncited: numeric=" as *u8); st_putn(out[0]); st_puts(" unverifiable=" as *u8); st_putn(out[3]); st_puts("\n" as *u8)
44 if out[0] == 1 { if out[3] == 1 { pass = pass + 1 } else { st_puts("T3 FAIL uncited-not-unverifiable\n" as *u8) } } else { st_puts("T3 FAIL numeric-count\n" as *u8) }
45
46 // T4: determinism -- same answer+sources -> identical grade
47 total = total + 1
48 rvs_verify_answer(a_good, st_len(a_good), srcv, 2, 5, out)
49 rvs_verify_answer(a_good, st_len(a_good), srcv, 2, 5, out2)
50 var t4: i64 = 1
51 if out[0] != out2[0] { t4 = 0 }
52 if out[1] != out2[1] { t4 = 0 }
53 if out[2] != out2[2] { t4 = 0 }
54 if out[3] != out2[3] { t4 = 0 }
55 if t4 == 1 { pass = pass + 1 } else { st_puts("T4 FAIL not-deterministic\n" as *u8) }
56
57 // T5 (marker-gated): the REAL composed self-verifying researcher (Qwen synth -> self-verify). Skipped (GREEN) w/o model.
58 let mk: i64 = sys_openat_rd("knowledge/index/rvs_model.on" as *u8)
59 if mk >= 0 {
60 sys_close(mk)
61 total = total + 1
62 let ans: *u8 = sys_mmap(2048)
63 let al: i64 = rvs_answer("What is human performance on this dataset?" as *u8, srcv, 2, 5, ans, out)
64 st_puts("T5 MODEL ans=[" as *u8); if al > 0 { sys_write(1, ans, al) } st_puts("] numeric=" as *u8); st_putn(out[0]); st_puts(" confirmed=" as *u8); st_putn(out[1]); st_puts(" discredited=" as *u8); st_putn(out[2]); st_puts("\n" as *u8)
65 if al > 0 { pass = pass + 1 } else { st_puts("T5 FAIL model-empty-or-unready\n" as *u8) }
66 } else { st_puts("T5 SKIP (no rvs_model.on; Tier-1 self-verification proven; composed model path = S1 Tier-2 + this logic)\n" as *u8) }
67
68 st_puts("VERIFY-SYNTH " as *u8); st_putn(pass); st_puts("/" as *u8); st_putn(total); st_puts("\n" as *u8)
69 if pass == total { st_puts("VERIFY-SYNTH ALL-PASS\n" as *u8); sys_exit(0) }
70 sys_exit(1)
71 return 1
72}