code wiki / _hdl_build / nx_swebv_ingest.nx
nx_swebv_ingest.nx source
↩ module page · 193 lines · 8087 B
1// nx_swebv_ingest.nx -- F787 rung-2: parse the five fetched SWE-bench Verified /rows pages
2// (knowledge/fetched/swebv_rows_{000..400}.raw, ingested by nx_swebv_research_fetch) into the
3// per-instance INDEX plane knowledge/store/swebvinst- : one 6-col row per instance
4// instance_id<TAB>repo<TAB>base_commit<TAB>version<TAB>difficulty<TAB>page-tag
5// The plane is the queryable index; the big fields (patch/test_patch/problem_statement/
6// FAIL_TO_PASS/PASS_TO_PASS) STAY in the .raw corpus, addressed by page-tag + instance_id.
7// PARSER SOUNDNESS: anchors start with a BARE quote byte; inside any JSON string value every
8// quote is escaped (backslash-quote), so a bare-quote needle can never match inside a value --
9// plus a prev-byte backslash check. Field order per row (features 0..12) means sequential
10// forward scanning; difficulty is the last field before the next row anchor.
11// TOOTH (D001 verdict contract): exactly 500 rows parsed + seeded -> verdict=GREEN, else RED.
12// ONE sts_seed commit = idempotent re-run. Envelope: page cap 4MiB/file, out cap 256KiB.
13// expect_exit: 0 license_tier: ORIGINAL No hw writes (Rule 26).
14import "nx_store_seed_lib.nx"
15import "nx_seg_store.nx"
16import "nx_syscalls.nx"
17
18static g_out: *u8
19static g_o: i64
20static g_rows: i64
21
22const SVI_PREFIX: *u8 = "knowledge/store/swebvinst-" as *u8
23const SVI_FCAP: i64 = 4194304
24const SVI_OCAP: i64 = 262144
25const SVI_VCAP: i64 = 256
26const SVI_TAB: i64 = 9
27const SVI_NL: i64 = 10
28const SVI_Q: i64 = 34
29const SVI_BSL: i64 = 92
30const SVI_STDERR: i64 = 2
31const SVI_WANT: i64 = 500
32const SVI_EXIT_IO: i64 = 1
33const SVI_EXIT_RED: i64 = 3
34const SVI_D0: i64 = 48
35const SVI_B10: i64 = 10
36const SVI_NUMB: i64 = 24
37
38func sv_werr(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(SVI_STDERR, s, n); return 0 }
39func sv_wn(v: i64) -> i64 {
40 if v == 0 { sys_write(SVI_STDERR, "0" as *u8, 1); return 0 }
41 var m: i64 = v
42 let d: *u8 = sys_mmap(SVI_NUMB)
43 var k: i64 = 0
44 while m > 0 { d[k] = (SVI_D0 + (m % SVI_B10)) as u8; m = m / SVI_B10; k = k + 1 }
45 let o: *u8 = sys_mmap(SVI_NUMB)
46 var i: i64 = 0
47 while i < k { o[i] = d[k - 1 - i]; i = i + 1 }
48 sys_write(SVI_STDERR, o, k)
49 return 0
50}
51func sv_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
52// bounded file read (NOT sys_read_file -- its 4GB mmap per call is the known leak class)
53func sv_read(path: *u8, buf: *u8, cap: i64) -> i64 {
54 let fd: i64 = sys_openat_rd(path)
55 if fd < 0 { return 0 - 1 }
56 var n: i64 = 0
57 var go: i64 = 1
58 while go == 1 {
59 let base: i64 = buf as i64
60 let r: i64 = sys_read(fd, (base + n) as *u8, cap - n)
61 if r <= 0 { go = 0 } else { n = n + r }
62 if n >= cap { go = 0 }
63 }
64 sys_close(fd)
65 return n
66}
67// find needle (first byte = bare quote) at/after `from`; prev byte must not be backslash; -1 if absent
68func sv_find(buf: *u8, n: i64, needle: *u8, from: i64) -> i64 {
69 let m: i64 = sv_slen(needle)
70 if m == 0 { return 0 - 1 }
71 var i: i64 = from
72 while i + m <= n {
73 var j: i64 = 0
74 var ok: i64 = 1
75 while j < m { if buf[i+j] != needle[j] { ok = 0; j = m } else { j = j + 1 } }
76 if ok == 1 { if i > 0 { if buf[i-1] == (SVI_BSL as u8) { ok = 0 } } }
77 if ok == 1 { return i }
78 i = i + 1
79 }
80 return 0 - 1
81}
82// value of the field whose `needle` ("key":") begins at/after `from`: copy chars until the next
83// bare quote into val (cap SVI_VCAP-1), return position AFTER the closing quote; -1 if absent
84func sv_field(buf: *u8, n: i64, needle: *u8, from: i64, val: *u8) -> i64 {
85 let p: i64 = sv_find(buf, n, needle, from)
86 if p < 0 { return 0 - 1 }
87 let vs: i64 = p + sv_slen(needle)
88 var i: i64 = vs
89 var k: i64 = 0
90 var go: i64 = 1
91 while go == 1 {
92 if i >= n { go = 0 }
93 else {
94 if buf[i] == (SVI_Q as u8) { go = 0 }
95 else {
96 if k < SVI_VCAP - 1 { val[k] = buf[i]; k = k + 1 }
97 i = i + 1
98 }
99 }
100 }
101 val[k] = 0 as u8
102 return i + 1
103}
104func sv_emit(s: *u8) -> i64 {
105 var j: i64 = 0
106 while s[j] != (0 as u8) { if g_o < SVI_OCAP - 2 { g_out[g_o] = s[j]; g_o = g_o + 1 } j = j + 1 }
107 return 0
108}
109func sv_emitc(c: i64) -> i64 {
110 if g_o < SVI_OCAP - 2 { g_out[g_o] = c as u8; g_o = g_o + 1 }
111 return 0
112}
113// scan one page buffer: rows anchored at "row":{"repo":" ; fields in schema order
114func sv_scan(buf: *u8, n: i64, tag: *u8) -> i64 {
115 let repo: *u8 = sys_mmap(SVI_VCAP)
116 let iid: *u8 = sys_mmap(SVI_VCAP)
117 let bc: *u8 = sys_mmap(SVI_VCAP)
118 let ver: *u8 = sys_mmap(SVI_VCAP)
119 let dif: *u8 = sys_mmap(SVI_VCAP)
120 var p: i64 = 0
121 var go: i64 = 1
122 while go == 1 {
123 let a: i64 = sv_find(buf, n, "\"row\":{\"repo\":\"" as *u8, p)
124 if a < 0 { go = 0 }
125 else {
126 var q: i64 = sv_field(buf, n, "\"repo\":\"" as *u8, a, repo)
127 if q < 0 { go = 0 }
128 else {
129 q = sv_field(buf, n, "\"instance_id\":\"" as *u8, q, iid)
130 if q < 0 { go = 0 }
131 else {
132 q = sv_field(buf, n, "\"base_commit\":\"" as *u8, q, bc)
133 if q < 0 { go = 0 }
134 else {
135 q = sv_field(buf, n, "\"version\":\"" as *u8, q, ver)
136 if q < 0 { go = 0 }
137 else {
138 q = sv_field(buf, n, "\"difficulty\":\"" as *u8, q, dif)
139 if q < 0 { go = 0 }
140 else {
141 sv_emit(iid)
142 sv_emitc(SVI_TAB)
143 sv_emit(repo)
144 sv_emitc(SVI_TAB)
145 sv_emit(bc)
146 sv_emitc(SVI_TAB)
147 sv_emit(ver)
148 sv_emitc(SVI_TAB)
149 sv_emit(dif)
150 sv_emitc(SVI_TAB)
151 sv_emit(tag)
152 sv_emitc(SVI_NL)
153 g_rows = g_rows + 1
154 p = q
155 }
156 }
157 }
158 }
159 }
160 }
161 }
162 return 0
163}
164func sv_page(path: *u8, tag: *u8, fbuf: *u8) -> i64 {
165 let n: i64 = sv_read(path, fbuf, SVI_FCAP)
166 if n <= 0 { sv_werr("SWEBV-INGEST page ABSENT: " as *u8); sv_werr(path); sv_werr("\n" as *u8); return 0 }
167 sv_scan(fbuf, n, tag)
168 return 1
169}
170
171func main() -> i64 {
172 g_out = sys_mmap(SVI_OCAP)
173 g_o = 0
174 g_rows = 0
175 let fbuf: *u8 = sys_mmap(SVI_FCAP)
176 var pages: i64 = 0
177 pages = pages + sv_page("knowledge/fetched/swebv_rows_000.raw" as *u8, "p000" as *u8, fbuf)
178 pages = pages + sv_page("knowledge/fetched/swebv_rows_100.raw" as *u8, "p100" as *u8, fbuf)
179 pages = pages + sv_page("knowledge/fetched/swebv_rows_200.raw" as *u8, "p200" as *u8, fbuf)
180 pages = pages + sv_page("knowledge/fetched/swebv_rows_300.raw" as *u8, "p300" as *u8, fbuf)
181 pages = pages + sv_page("knowledge/fetched/swebv_rows_400.raw" as *u8, "p400" as *u8, fbuf)
182 if pages < 5 { sv_werr("SWEBV-INGEST FAIL verdict=RED pages=" as *u8); sv_wn(pages); sv_werr("/5\n" as *u8); sys_exit(SVI_EXIT_IO); return SVI_EXIT_IO }
183 if g_rows != SVI_WANT { sv_werr("SWEBV-INGEST FAIL verdict=RED rows=" as *u8); sv_wn(g_rows); sv_werr(" want=500 (store NOT touched, fail-closed)\n" as *u8); sys_exit(SVI_EXIT_RED); return SVI_EXIT_RED }
184 let rc: i64 = sts_seed(SVI_PREFIX, g_out, g_o)
185 if rc < 0 { sv_werr("SWEBV-INGEST FAIL verdict=RED store-commit\n" as *u8); sys_exit(SVI_EXIT_IO); return SVI_EXIT_IO }
186 sv_werr("SWEBV-INGEST OK verdict=GREEN rows=" as *u8)
187 sv_wn(g_rows)
188 sv_werr(" bytes=" as *u8)
189 sv_wn(g_o)
190 sv_werr(" plane=swebvinst- (500 public Verified instances indexed; corpus stays in knowledge/fetched)\n" as *u8)
191 sys_exit(0)
192 return 0
193}