code wiki / _hdl_build / nx_search_quality_gate.nx
nx_search_quality_gate.nx source
↩ module page · 160 lines · 7845 B
1// nx_search_quality_gate.nx -- the ENGINEER's re-runnable instrument for the search engine (same pattern that
2// took andelinwest 700->1000: data-driven criteria TABLE -> grade the REAL artifact -> durable log -> find->
3// fix->re-prove). Criteria live in knowledge/index/search_oracle.tsv (rows: "query terms<TAB>expected-top-doc");
4// EXPANDING the gate = adding a row, not a code change. Each row fork/execs the LIVE /tmp/nx_search_cli.sov.elf
5// (the artifact users run, not a reimplementation), captures stdout, finds the expected doc's rank. Row passes
6// iff rank==1; score = MRR permil (1000/rank, 0 if absent). Verdict appends to
7// knowledge/status/search_quality.log (durable). Ground truth per row was grep-verified at authoring.
8// license_tier: ORIGINAL
9import "nx_syscalls.nx"
10
11const SQ_MAX_ROWS: i64 = 64
12const SQ_MAX_TERMS: i64 = 8
13const SQ_OUT_PATH_S: i64 = 0 // (doc anchor)
14
15func sq_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func sq_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
17func sq_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
18func sq_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
19
20func sq_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
21
22// substring find: first offset of needle in hay[0..n), -1 if absent
23func sq_find(hay: *u8, n: i64, needle: *u8) -> i64 {
24 let nl: i64 = sq_strlen(needle)
25 if nl == 0 { return 0 - 1 }
26 var i: i64 = 0
27 while i + nl <= n {
28 var j: i64 = 0
29 var hit: i64 = 1
30 while j < nl { if hay[i+j] != needle[j] { hit = 0; j = nl } else { j = j + 1 } }
31 if hit == 1 { return i }
32 i = i + 1
33 }
34 return 0 - 1
35}
36
37// run the REAL search CLI with the row's terms, stdout -> capture file; return its exit (128+sig on signal)
38func sq_run_cli(terms: *i64, nterms: i64, out_path: *u8) -> i64 {
39 let pid: i64 = sys_fork()
40 if pid == 0 {
41 let out: i64 = sys_openat_wr(out_path, 0x1a4)
42 if out >= 0 { sys_dup3(out, 1, 0) }
43 let argv: *i64 = sys_mmap(8 * (SQ_MAX_TERMS + 2)) as *i64
44 argv[0] = "/tmp/nx_search_cli.sov.elf" as *u8 as i64
45 var i: i64 = 0
46 while i < nterms { argv[1+i] = terms[i]; i = i + 1 }
47 argv[1+nterms] = 0
48 let envp: *i64 = sys_mmap(16) as *i64; envp[0] = "PATH=/usr/bin:/bin" as *u8 as i64; envp[1] = 0
49 sys_execve("/tmp/nx_search_cli.sov.elf" as *u8, argv, envp)
50 sys_exit(127)
51 }
52 let st: *i64 = sys_mmap(16) as *i64
53 sys_wait4(pid, st, 0)
54 if (st[0] & 0x7f) != 0 { return 128 + (st[0] & 0x7f) }
55 return (st[0] >> 8) & 0xff
56}
57
58// rank of expected path in captured output: the result line " #N score=... <path>" that CONTAINS the path.
59// Returns N, or 0 if the doc never appears.
60func sq_rank_of(out_buf: *u8, out_len: i64, expected: *u8) -> i64 {
61 let at: i64 = sq_find(out_buf, out_len, expected)
62 if at < 0 { return 0 }
63 // scan back from the hit to the line start, then parse " #N"
64 var ls: i64 = at
65 while ls > 0 { if out_buf[ls-1] == (10 as u8) { ls = 0 - ls } else { ls = ls - 1 } }
66 if ls < 0 { ls = 0 - ls }
67 let hash_at: i64 = sq_find(((out_buf as i64) + ls) as *u8, at - ls + 1, "#" as *u8)
68 if hash_at < 0 { return 0 }
69 var p: i64 = ls + hash_at + 1
70 var rank: i64 = 0
71 while p < out_len {
72 let c: i64 = out_buf[p] as i64
73 if c >= 48 { if c <= 57 { rank = rank * 10 + (c - 48); p = p + 1 } else { p = out_len } } else { p = out_len }
74 }
75 return rank
76}
77
78func main() -> i64 {
79 sq_puts("=== SEARCH QUALITY GATE (oracle rows -> live CLI -> MRR permil; data-driven, re-runnable) ===\n" as *u8)
80 let lenbox: *i64 = sys_mmap(16) as *i64
81 let tsv: *u8 = sys_read_file("knowledge/index/search_oracle.tsv" as *u8, lenbox)
82 if tsv == 0 as *u8 { sq_puts("GATE-FAIL: no oracle tsv\n" as *u8); sys_exit(1); return 1 }
83 let tlen: i64 = lenbox[0]
84
85 var rows: i64 = 0
86 var pass: i64 = 0
87 var mrr_sum: i64 = 0 // sum of 1000/rank
88 let terms: *i64 = sys_mmap(8 * SQ_MAX_TERMS) as *i64
89 let outbox: *i64 = sys_mmap(16) as *i64
90
91 var ls: i64 = 0
92 var i: i64 = 0
93 while i <= tlen {
94 var eol: i64 = 0
95 if i == tlen { eol = 1 } else { if tsv[i] == (10 as u8) { eol = 1 } }
96 if eol == 1 {
97 if i > ls {
98 // split line: terms-field TAB expected-path; tokenize terms on spaces; NUL-terminate in place
99 var tab: i64 = 0 - 1
100 var k: i64 = ls
101 while k < i { if tsv[k] == (9 as u8) { if tab < 0 { tab = k } } k = k + 1 }
102 if tab > ls {
103 tsv[tab] = 0 as u8
104 let saved: i64 = tsv[i] as i64
105 tsv[i] = 0 as u8
106 let expected: *u8 = ((tsv as i64) + tab + 1) as *u8
107 var nterms: i64 = 0
108 var ts: i64 = ls
109 var p: i64 = ls
110 while p <= tab {
111 var sp: i64 = 0
112 if p == tab { sp = 1 } else { if tsv[p] == (32 as u8) { sp = 1 } }
113 if sp == 1 {
114 if p > ts { if nterms < SQ_MAX_TERMS { tsv[p] = 0 as u8; terms[nterms] = (tsv as i64) + ts; nterms = nterms + 1 } }
115 ts = p + 1
116 }
117 p = p + 1
118 }
119 let rc: i64 = sq_run_cli(terms, nterms, "/tmp/nx_searchq_out.txt" as *u8)
120 var rank: i64 = 0
121 if rc == 0 {
122 let ob: *u8 = sys_read_file("/tmp/nx_searchq_out.txt" as *u8, outbox)
123 if ob != 0 as *u8 { rank = sq_rank_of(ob, outbox[0], expected) }
124 }
125 rows = rows + 1
126 var permil: i64 = 0
127 if rank > 0 { permil = 1000 / rank }
128 mrr_sum = mrr_sum + permil
129 sq_puts(" row " as *u8); sq_num(rows)
130 sq_puts(" q='" as *u8); sq_puts(((tsv as i64) + ls) as *u8)
131 sq_puts("' expected-rank=1 got=" as *u8); sq_num(rank)
132 if rank == 1 { pass = pass + 1; sq_puts(" PASS\n" as *u8) } else { sq_puts(" FAIL\n" as *u8) }
133 tsv[i] = saved as u8
134 }
135 }
136 ls = i + 1
137 }
138 i = i + 1
139 }
140
141 if rows == 0 { sq_puts("GATE-FAIL: oracle empty\n" as *u8); sys_exit(1); return 1 }
142 let mrr: i64 = mrr_sum / rows
143 sq_puts("----\nSEARCHQ rows=" as *u8); sq_num(rows)
144 sq_puts(" pass=" as *u8); sq_num(pass)
145 sq_puts(" mrr-permil=" as *u8); sq_num(mrr); sq_puts("\n" as *u8)
146
147 // durable verdict line (append-only; the "over and over" trail)
148 let lg: i64 = sys_openat_append("knowledge/status/search_quality.log" as *u8, 0x1a4)
149 if lg >= 0 {
150 sq_w(lg, "SEARCHQ rows=" as *u8); sq_wn(lg, rows)
151 sq_w(lg, " pass=" as *u8); sq_wn(lg, pass)
152 sq_w(lg, " mrr_permil=" as *u8); sq_wn(lg, mrr)
153 if pass == rows { sq_w(lg, " verdict=GREEN\n" as *u8) } else { sq_w(lg, " verdict=RED\n" as *u8) }
154 sys_close(lg)
155 }
156 if pass == rows { sq_puts("GATE-GREEN\n" as *u8); sys_exit(0); return 0 }
157 sq_puts("GATE-RED\n" as *u8)
158 sys_exit(1)
159 return 1
160}