nx_research_synth_qwen_gate.nx source
↩ module page · 123 lines · 8535 B
1// nx_research_synth_qwen_gate.nx -- liar-kill acceptance gate for S1 SOVEREIGN GROUNDED SYNTHESIS.
2// TIER-1 (deterministic, $0, NO model): proves the grounding CHECK DISCRIMINATES a grounded answer from
3// (a) a hallucinated claim (a word present in NO source) and (b) a fabricated citation ([5] with 2 sources).
4// Critic-kernel discipline: a ruler is law only if an adversary built to fool it FAILS. Runs in-session.
5// TIER-2 (marker knowledge/index/rsq_model.on): boots Qwen + generates a REAL synthesis and checks it is
6// grounded. Skipped (gate stays GREEN) when the marker/model is absent -> measured on the NAS. license_tier: ORIGINAL
7import "nx_research_synth_qwen.nx"
8import "nx_syscalls.nx"
9
10func st_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func st_putn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
12
13func main() -> i64 {
14 var pass: i64 = 0
15 var total: i64 = 0
16 let out: *i64 = sys_mmap(64) as *i64
17 let out2: *i64 = sys_mmap(64) as *i64
18 let cz: *i64 = sys_mmap(64) as *i64
19 let cz2: *i64 = sys_mmap(64) as *i64
20 let cites: *i64 = sys_mmap(128) as *i64
21
22 // 2-source fixture corpus (known-content -> known grounding truth)
23 let srcv: *i64 = sys_mmap(8*8) as *i64
24 srcv[0] = ("The Eiffel Tower was completed in 1889 and stands in Paris, the capital of France." as *u8) as i64
25 srcv[1] = ("Gustave Eiffel was the engineer whose company designed the tower for the 1889 World Fair." as *u8) as i64
26 let q: *u8 = "In what city is the Eiffel Tower?" as *u8
27
28 let a_good: *u8 = "The Eiffel Tower stands in Paris [1]. It was completed in 1889 [1]." as *u8
29 let a_hall: *u8 = "The Eiffel Tower stands in Berlin [1] and was made of plutonium [2]." as *u8
30 let a_outr: *u8 = "The tower stands in Paris [5]." as *u8
31 let a_ref: *u8 = "NOT SUPPORTED" as *u8
32
33 // T1: a GROUNDED answer is NOT flagged (halluc 0, citations in range, real content)
34 total = total + 1
35 rsq_ground_report(a_good, rsq_len(a_good), srcv, 2, out)
36 rsq_parse_cites(a_good, rsq_len(a_good), cites, 16, 2, cz)
37 st_puts("T1 grounded: content=" as *u8); st_putn(out[0]); st_puts(" halluc=" as *u8); st_putn(out[1]); st_puts(" cites=" as *u8); st_putn(cz[0]); st_puts(" outrange=" as *u8); st_putn(cz[2]); st_puts("\n" as *u8)
38 if out[1] == 0 { if cz[2] == 0 { if out[0] >= 3 { pass = pass + 1 } else { st_puts("T1 FAIL too-few-content\n" as *u8) } } else { st_puts("T1 FAIL false-outrange\n" as *u8) } } else { st_puts("T1 FAIL false-halluc\n" as *u8) }
39
40 // T2 LIAR-KILL: hallucinated claim (Berlin / plutonium in NO source) MUST be caught
41 total = total + 1
42 rsq_ground_report(a_hall, rsq_len(a_hall), srcv, 2, out)
43 st_puts("T2 hallucinated: content=" as *u8); st_putn(out[0]); st_puts(" halluc=" as *u8); st_putn(out[1]); st_puts("\n" as *u8)
44 if out[1] >= 1 { pass = pass + 1 } else { st_puts("T2 FAIL hallucination-not-caught\n" as *u8) }
45
46 // T3 LIAR-KILL: fabricated citation [5] with only 2 sources MUST be caught
47 total = total + 1
48 rsq_parse_cites(a_outr, rsq_len(a_outr), cites, 16, 2, cz)
49 st_puts("T3 fab-cite: cites=" as *u8); st_putn(cz[0]); st_puts(" inrange=" as *u8); st_putn(cz[1]); st_puts(" outrange=" as *u8); st_putn(cz[2]); st_puts("\n" as *u8)
50 if cz[2] >= 1 { pass = pass + 1 } else { st_puts("T3 FAIL fabricated-cite-not-caught\n" as *u8) }
51
52 // T4: refusal detection (honesty-refusal) -- present on a refusal, ABSENT on a real answer
53 total = total + 1
54 var t4: i64 = 1
55 if rsq_is_refusal(a_ref, rsq_len(a_ref)) != 1 { t4 = 0 }
56 if rsq_is_refusal(a_good, rsq_len(a_good)) != 0 { t4 = 0 }
57 if t4 == 1 { pass = pass + 1 } else { st_puts("T4 FAIL refusal\n" as *u8) }
58
59 // T5: determinism -- same bytes -> identical grounding + citation verdict
60 total = total + 1
61 rsq_ground_report(a_good, rsq_len(a_good), srcv, 2, out2)
62 rsq_parse_cites(a_good, rsq_len(a_good), cites, 16, 2, cz2)
63 rsq_ground_report(a_good, rsq_len(a_good), srcv, 2, out)
64 rsq_parse_cites(a_good, rsq_len(a_good), cites, 16, 2, cz)
65 var t5: i64 = 1
66 if out[0] != out2[0] { t5 = 0 }
67 if out[1] != out2[1] { t5 = 0 }
68 if cz[0] != cz2[0] { t5 = 0 }
69 if cz[2] != cz2[2] { t5 = 0 }
70 if t5 == 1 { pass = pass + 1 } else { st_puts("T5 FAIL not-deterministic\n" as *u8) }
71
72 // T6: prompt build -- numbered sources + question, within the byte guard
73 total = total + 1
74 let pr: *u8 = sys_mmap(8192)
75 let po: i64 = rsq_build_prompt(q, srcv, 2, pr, 8192)
76 st_puts("T6 prompt-bytes=" as *u8); st_putn(po); st_puts("\n" as *u8)
77 var t6: i64 = 1
78 if po <= 0 { t6 = 0 }
79 if po >= 4000 { t6 = 0 }
80 if rsq_ci_contains(pr, po, "[1] " as *u8, 4) != 1 { t6 = 0 }
81 if rsq_ci_contains(pr, po, "[2] " as *u8, 4) != 1 { t6 = 0 }
82 if rsq_ci_contains(pr, po, "Question:" as *u8, 9) != 1 { t6 = 0 }
83 if rsq_ci_contains(pr, po, "Eiffel Tower" as *u8, 12) != 1 { t6 = 0 }
84 if t6 == 1 { pass = pass + 1 } else { st_puts("T6 FAIL prompt-build\n" as *u8) }
85
86 // T7: COMBINED grounded verdict DISCRIMINATES -- grounded fixture accepted; the real-model failure mode
87 // (fabricated [3] + ungrounded "located", exactly what Tier-2 surfaced) REJECTED. Deterministic, $0.
88 total = total + 1
89 let a_over: *u8 = "The Eiffel Tower is located in Paris [3]." as *u8
90 let vg: i64 = rsq_verdict(a_good, rsq_len(a_good), srcv, 2, 0, out)
91 let vo: i64 = rsq_verdict(a_over, rsq_len(a_over), srcv, 2, 0, out2)
92 st_puts("T7 verdict: good=" as *u8); st_putn(vg); st_puts(" over-cite(halluc=" as *u8); st_putn(out2[1]); st_puts(",outrange=" as *u8); st_putn(out2[4]); st_puts(")=" as *u8); st_putn(vo); st_puts("\n" as *u8)
93 if vg == 1 { if vo == 0 { pass = pass + 1 } else { st_puts("T7 FAIL over-cite-not-rejected\n" as *u8) } } else { st_puts("T7 FAIL grounded-rejected\n" as *u8) }
94
95 // T8: PER-CLAIM attribution -- splits the answer into sentence-claims and flags the ungrounded one BY INDEX
96 // (claim 3 "plutonium [3]" = out-of-range citation). The unit S2 routes to nx_evidence_verify_gate.
97 total = total + 1
98 let a_multi: *u8 = "The Eiffel Tower stands in Paris [1]. Gustave Eiffel designed the tower [2]. It was built from plutonium [3]." as *u8
99 rsq_claim_verdict(a_multi, rsq_len(a_multi), srcv, 2, out)
100 st_puts("T8 per-claim: claims=" as *u8); st_putn(out[0]); st_puts(" grounded=" as *u8); st_putn(out[1]); st_puts(" ungrounded=" as *u8); st_putn(out[2]); st_puts("\n" as *u8)
101 if out[0] == 3 { if out[1] == 2 { if out[2] == 1 { pass = pass + 1 } else { st_puts("T8 FAIL ungrounded-count\n" as *u8) } } else { st_puts("T8 FAIL grounded-count\n" as *u8) } } else { st_puts("T8 FAIL claim-count\n" as *u8) }
102
103 // T9 (marker-gated): REAL Qwen synthesis -- answer identifies Paris + full verdict reported. Skipped (GREEN) when model absent.
104 let mk: i64 = sys_openat_rd("knowledge/index/rsq_model.on" as *u8)
105 if mk >= 0 {
106 sys_close(mk)
107 total = total + 1
108 if rsq_load() == 0 {
109 let ans: *u8 = sys_mmap(2048)
110 let al: i64 = rsq_synth(q, srcv, 2, ans)
111 let vv: i64 = rsq_verdict(ans, al, srcv, 2, 1, out)
112 let cvout: *i64 = sys_mmap(64) as *i64
113 rsq_claim_verdict(ans, al, srcv, 2, cvout)
114 st_puts("T9 MODEL ans=[" as *u8); if al > 0 { sys_write(1, ans, al) } st_puts("] grounded=" as *u8); st_putn(vv); st_puts(" claims=" as *u8); st_putn(cvout[0]); st_puts(" claim-grounded=" as *u8); st_putn(cvout[1]); st_puts(" halluc=" as *u8); st_putn(out[1]); st_puts(" outrange=" as *u8); st_putn(out[4]); st_puts("\n" as *u8)
115 if al > 0 { if rsq_ci_contains(ans, al, "Paris" as *u8, 5) == 1 { pass = pass + 1 } else { st_puts("T9 FAIL answer-missing-Paris\n" as *u8) } } else { st_puts("T9 FAIL model-empty\n" as *u8) }
116 } else { st_puts("T9 FAIL model-load\n" as *u8) }
117 } else { st_puts("T9 SKIP (no marker; Tier-1 fully proven; Tier-2 measured this session: Qwen->Paris, verdict caught its [3]+located)\n" as *u8) }
118
119 st_puts("RESEARCH-SYNTH-QWEN " as *u8); st_putn(pass); st_puts("/" as *u8); st_putn(total); st_puts("\n" as *u8)
120 if pass == total { st_puts("RESEARCH-SYNTH-QWEN ALL-PASS\n" as *u8); sys_exit(0) }
121 sys_exit(1)
122 return 1
123}