nx_parquet_gate.nx source
↩ module page · 448 lines · 24227 B
1// nx_parquet_gate.nx -- GATE for the sovereign parquet reader (nx_parquet_lib), driven IN-PROCESS.
2//
3// Every wire rule the reader relies on is proven on PLANTED bytes with a known answer and a neg-control that must
4// refuse: thrift varints and zigzag (a 150, a -1, an eleven-byte overrun, a truncated one), field headers (short and
5// long form, delta accumulation, STOP), list headers (short and long form), the container skipper (a nested struct,
6// a map, and a nesting deeper than the bound), the RLE and bit-packed hybrid (an RLE run, a bit-packed group, a mixed
7// stream, width zero, a truncated run, a count over capacity), snappy (literal plus copies of all three offset widths,
8// an overlapping copy, an offset before the buffer, a declared length the stream does not produce, an output that
9// does not fit), and pq_open (not a parquet file, a footer length past the file, a truncated tail, a footer length
10// bumped by five bytes). Then the REAL bytes: the BRIGHT pony examples and documents files (search.refs brightds26,
11// pinned in knowledge/fetched) must reproduce the counts the dataset card declares (112 examples, 7894 documents),
12// resolve a list column by its column name and not by its element name, reconstruct 2219 gold_ids over 112 rows with
13// repetition levels, decode the first query and the first ids to their known bytes, refuse an out-of-range leaf or row
14// group, and write the documents TSV deterministically (two writes byte-identical, 7894 rows, no null rows, the first
15// row starting with the first document id). The real-file teeth SKIP, never acquit, when the fixtures are absent.
16// license_tier: ORIGINAL No hw writes (Rule 26).
17import "nx_syscalls.nx"
18import "nx_gate_verdict.nx"
19import "nx_parquet_lib.nx"
20
21const G_ROOT: *u8 = "/tmp/nx_parquet_gate"
22const G_NOTPQ: *u8 = "/tmp/nx_parquet_gate/not_parquet.bin"
23const G_BADLEN: *u8 = "/tmp/nx_parquet_gate/footer_past_eof.parquet"
24const G_TRUNC: *u8 = "/tmp/nx_parquet_gate/truncated.parquet"
25const G_BUMPED: *u8 = "/tmp/nx_parquet_gate/footer_bumped.parquet"
26const G_TSV_A: *u8 = "/tmp/nx_parquet_gate/docs_a.tsv"
27const G_TSV_B: *u8 = "/tmp/nx_parquet_gate/docs_b.tsv"
28const G_EX_A: *u8 = "knowledge/fetched/cmp_search_bright_examples_pony.parquet"
29const G_EX_B: *u8 = "../knowledge/fetched/cmp_search_bright_examples_pony.parquet"
30const G_DOC_A: *u8 = "knowledge/fetched/cmp_search_bright_documents_pony.parquet"
31const G_DOC_B: *u8 = "../knowledge/fetched/cmp_search_bright_documents_pony.parquet"
32const G_NOTPQ_BYTES: *u8 = "hello world, not a parquet file"
33const G_MODE_DIR: i64 = 0x1ed
34const G_I64: i64 = 8
35const G_FIX_CAP: i64 = 128
36const G_OUT_CAP: i64 = 64
37const G_OUT_SMALL: i64 = 4
38const G_TRUNC_KEEP: i64 = 20000
39const G_BUMP: i64 = 5
40const G_DEEP: i64 = 70
41const G_CH_A: i64 = 97
42const G_CH_B: i64 = 98
43const G_CH_C: i64 = 99
44const G_CH_D: i64 = 100
45const G_CH_E: i64 = 101
46const G_CH_F: i64 = 102
47const G_CH_X: i64 = 120
48const G_CH_Z: i64 = 122
49const G_BAD_LEAF: i64 = 99
50const G_BAD_RG: i64 = 5
51// what the BRIGHT card declares for pony (brightds26, read 2026-09-14) and what the pinned files carry (measured)
52const G_EX_ROWS: i64 = 112
53const G_EX_LEAVES: i64 = 7
54const G_EX_RGS: i64 = 1
55const G_EX_LEAF_QUERY: i64 = 0
56const G_EX_LEAF_ID: i64 = 2
57const G_EX_LEAF_GOLD: i64 = 5
58const G_EX_GOLD_ENTRIES: i64 = 2219
59const G_EX_GOLD_DICT: i64 = 43
60const G_EX_PAGES: i64 = 2
61const G_EX_ENCMASK_RLE_DICT: i64 = 256
62const G_EX_QUERY0_LEN: i64 = 237
63const G_QUERY_PREFIX: *u8 = "I will use the programming language pony"
64const G_DOC_ROWS: i64 = 7894
65const G_DOC_RGS: i64 = 8
66const G_DOC_LEAVES: i64 = 2
67const G_DOC_TSV_BYTES: i64 = 2317793
68const G_DOC_FIRST_ID: *u8 = "Pony/src-builtin-runtime_options-_1.txt"
69const G_SNAPPY_OUT: *u8 = "abcdefabcdz"
70const G_SNAPPY_OUT_LEN: i64 = 11
71const G_SNAPPY2_OUT: *u8 = "abcdabcd"
72const G_SNAPPY2_OUT_LEN: i64 = 8
73const G_SNAPPY3_OUT: *u8 = "aaaaa"
74const G_SNAPPY3_OUT_LEN: i64 = 5
75const G_HYBRID_N: i64 = 8
76const G_HYBRID_MIXED_N: i64 = 10
77const G_RLE_N: i64 = 5
78const G_RLE_VALUE: i64 = 3
79const G_VARINT_150: i64 = 150
80const G_LIST_LONG: i64 = 20
81
82func g_slen(s: *u8) -> i64 {
83 var n: i64 = 0
84 while (s[n] & PQ_BYTE) as i64 != 0 { n = n + 1 }
85 return n
86}
87func g_write_bytes(path: *u8, buf: *u8, n: i64) -> i64 {
88 let fd: i64 = sys_openat_wr(path, MODE_0644)
89 if fd < 0 { return 0 }
90 var off: i64 = 0
91 while off < n {
92 let r: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off)
93 if r <= 0 { sys_close(fd); return 0 }
94 off = off + r
95 }
96 sys_close(fd)
97 return 1
98}
99// buf[off..off+len) starts with the NUL-terminated s
100func g_starts(buf: *u8, off: i64, len: i64, s: *u8) -> i64 {
101 let n: i64 = g_slen(s)
102 if len < n { return 0 }
103 var i: i64 = 0
104 while i < n { if (buf[off + i] & PQ_BYTE) as i64 != (s[i] & PQ_BYTE) as i64 { return 0 } i = i + 1 }
105 return 1
106}
107func g_bytes_are(buf: *u8, off: i64, len: i64, s: *u8) -> i64 {
108 if len != g_slen(s) { return 0 }
109 return g_starts(buf, off, len, s)
110}
111func g_files_equal(a: *u8, b: *u8) -> i64 {
112 let la: *i64 = sys_mmap(G_I64) as *i64
113 let lb: *i64 = sys_mmap(G_I64) as *i64
114 la[0] = 0
115 lb[0] = 0
116 let ba: *u8 = sys_read_file(a, la)
117 let bb: *u8 = sys_read_file(b, lb)
118 if (ba as i64) == 0 || (bb as i64) == 0 { return 0 }
119 if la[0] != lb[0] { return 0 }
120 if la[0] <= 0 { return 0 }
121 var i: i64 = 0
122 while i < la[0] { if ba[i] != bb[i] { return 0 } i = i + 1 }
123 return 1
124}
125// open a fixture from the serving root or from buildroot (the two CWDs a gate runs under)
126func g_open2(pq: *i64, a: *u8, b: *u8) -> i64 {
127 let rc: i64 = pq_open(pq, a)
128 if rc != PQ_E_OPEN { return rc }
129 return pq_open(pq, b)
130}
131// the whole file behind a fixture path pair; len lands in lenp[0]
132func g_read2(a: *u8, b: *u8, lenp: *i64) -> *u8 {
133 lenp[0] = 0
134 let x: *u8 = sys_read_file(a, lenp)
135 if (x as i64) != 0 && lenp[0] > 0 { return x }
136 lenp[0] = 0
137 return sys_read_file(b, lenp)
138}
139
140func main() -> i64 {
141 gv_head("=== nx_parquet_gate -- the sovereign parquet reader proven on planted bytes and on the BRIGHT pony files ===" as *u8)
142 let c: *i64 = gv_ctr()
143 sys_mkdir(G_ROOT, G_MODE_DIR)
144 let cur: *i64 = sys_mmap(TC_SLOTS * G_I64) as *i64
145 let st: *i64 = sys_mmap(TS_DEPTH * TS_SLOTS * G_I64) as *i64
146 let vb: *u8 = sys_mmap(G_FIX_CAP)
147 let ob: *u8 = sys_mmap(G_OUT_CAP)
148 let lv: *i64 = sys_mmap(G_FIX_CAP * G_I64) as *i64
149
150 // ---- varint and zigzag
151 vb[0] = 0x96 as u8
152 vb[1] = 0x01 as u8
153 tc_init(cur, vb, 0, 2)
154 gv_check_eq("varint-0x96-0x01-decodes-to-150" as *u8, tc_varint(cur), G_VARINT_150, c)
155 gv_check_eq("varint-consumed-exactly-two-bytes" as *u8, cur[TC_POS], 2, c)
156 vb[0] = 0x01 as u8
157 tc_init(cur, vb, 0, 1)
158 gv_check_eq("zigzag-1-is-minus-one" as *u8, tc_zz(cur), 0 - 1, c)
159 vb[0] = 0x02 as u8
160 tc_init(cur, vb, 0, 1)
161 gv_check_eq("zigzag-2-is-one" as *u8, tc_zz(cur), 1, c)
162 vb[0] = 0x00 as u8
163 tc_init(cur, vb, 0, 1)
164 gv_check_eq("zigzag-0-is-zero" as *u8, tc_zz(cur), 0, c)
165 vb[0] = 0x03 as u8
166 tc_init(cur, vb, 0, 1)
167 gv_check_eq("zigzag-3-is-minus-two" as *u8, tc_zz(cur), 0 - 2, c)
168 var i: i64 = 0
169 while i < 11 { vb[i] = 0xFF as u8; i = i + 1 }
170 tc_init(cur, vb, 0, 11)
171 tc_varint(cur)
172 gv_check_eq("neg-control-an-eleven-byte-varint-is-refused-THRIFT" as *u8, cur[TC_ERR], PQ_E_THRIFT, c)
173 vb[0] = 0x80 as u8
174 tc_init(cur, vb, 0, 1)
175 tc_varint(cur)
176 gv_check_eq("neg-control-a-varint-cut-after-its-continuation-bit-is-refused-TRUNCATED" as *u8, cur[TC_ERR], PQ_E_TRUNC, c)
177
178 // ---- field headers and list headers
179 vb[0] = 0x15 as u8
180 tc_init(cur, vb, 0, 1)
181 gv_check_eq("field-header-0x15-is-type-I32" as *u8, tc_field(cur), TC_I32, c)
182 gv_check_eq("field-header-0x15-is-field-1" as *u8, cur[TC_FID], 1, c)
183 vb[0] = 0x05 as u8
184 vb[1] = 0x0E as u8
185 tc_init(cur, vb, 0, 2)
186 gv_check_eq("long-form-field-header-reads-type-I32" as *u8, tc_field(cur), TC_I32, c)
187 gv_check_eq("long-form-field-id-is-the-zigzag-that-follows (7)" as *u8, cur[TC_FID], 7, c)
188 vb[0] = 0x15 as u8
189 vb[1] = 0x02 as u8
190 vb[2] = 0x26 as u8
191 tc_init(cur, vb, 0, 3)
192 tc_field(cur)
193 gv_check_eq("field-1-value-zigzag-2-is-one" as *u8, tc_zz(cur), 1, c)
194 gv_check_eq("delta-2-after-field-1-reads-type-I64" as *u8, tc_field(cur), TC_I64, c)
195 gv_check_eq("delta-2-after-field-1-is-field-3" as *u8, cur[TC_FID], 3, c)
196 vb[0] = 0x00 as u8
197 tc_init(cur, vb, 0, 1)
198 gv_check_eq("a-zero-byte-is-STOP" as *u8, tc_field(cur), TC_STOP, c)
199 vb[0] = 0x2C as u8
200 tc_init(cur, vb, 0, 1)
201 gv_check_eq("list-header-0x2C-has-size-2" as *u8, tc_list_head(cur), 2, c)
202 gv_check_eq("list-header-0x2C-elements-are-STRUCT" as *u8, cur[TC_ET], TC_STRUCT, c)
203 vb[0] = 0xF5 as u8
204 vb[1] = 0x14 as u8
205 tc_init(cur, vb, 0, 2)
206 gv_check_eq("long-form-list-header-size-is-the-varint-that-follows (20)" as *u8, tc_list_head(cur), G_LIST_LONG, c)
207 gv_check_eq("long-form-list-header-elements-are-I32" as *u8, cur[TC_ET], TC_I32, c)
208
209 // ---- the container skipper: struct{1: i32 1, 2: struct{1: binary ab}, 3: list<i32>[1, 2]}
210 vb[0] = 0x15 as u8
211 vb[1] = 0x02 as u8
212 vb[2] = 0x1C as u8
213 vb[3] = 0x18 as u8
214 vb[4] = 0x02 as u8
215 vb[5] = G_CH_A as u8
216 vb[6] = G_CH_B as u8
217 vb[7] = 0x00 as u8
218 vb[8] = 0x19 as u8
219 vb[9] = 0x25 as u8
220 vb[10] = 0x02 as u8
221 vb[11] = 0x04 as u8
222 vb[12] = 0x00 as u8
223 tc_init(cur, vb, 0, 13)
224 tc_skip(cur, TC_STRUCT, st)
225 gv_check_eq("skip-of-a-nested-struct-ends-exactly-at-its-STOP (13 bytes)" as *u8, cur[TC_POS], 13, c)
226 gv_check_eq("skip-of-a-nested-struct-raises-no-error" as *u8, cur[TC_ERR], 0, c)
227 gv_check_eq("skip-restores-the-outer-last-field-id" as *u8, cur[TC_LAST], 0, c)
228 // map{1 pair: key binary x, value i32 5}
229 vb[0] = 0x01 as u8
230 vb[1] = 0x85 as u8
231 vb[2] = 0x01 as u8
232 vb[3] = G_CH_X as u8
233 vb[4] = 0x0A as u8
234 tc_init(cur, vb, 0, 5)
235 tc_skip(cur, TC_MAP, st)
236 gv_check_eq("skip-of-a-map-consumes-its-header-key-and-value (5 bytes)" as *u8, cur[TC_POS], 5, c)
237 gv_check_eq("skip-of-a-map-raises-no-error" as *u8, cur[TC_ERR], 0, c)
238 i = 0
239 while i < G_DEEP { vb[i] = 0x19 as u8; i = i + 1 }
240 vb[G_DEEP] = 0x09 as u8
241 tc_init(cur, vb, 0, G_DEEP + 1)
242 tc_skip(cur, TC_LIST, st)
243 gv_check_eq("neg-control-a-nesting-deeper-than-the-bound-is-refused-THRIFT" as *u8, cur[TC_ERR], PQ_E_THRIFT, c)
244
245 // ---- the hybrid
246 vb[0] = 0x0A as u8
247 vb[1] = 0x03 as u8
248 gv_check_eq("rle-run-of-five-consumes-two-bytes" as *u8, pq_hybrid(vb, 0, 2, 2, G_RLE_N, lv, G_FIX_CAP), 2, c)
249 gv_check_eq("rle-run-first-value-is-3" as *u8, lv[0], G_RLE_VALUE, c)
250 gv_check_eq("rle-run-last-value-is-3" as *u8, lv[G_RLE_N - 1], G_RLE_VALUE, c)
251 vb[0] = 0x03 as u8
252 vb[1] = 0xAA as u8
253 gv_check_eq("bit-packed-group-of-eight-consumes-two-bytes" as *u8, pq_hybrid(vb, 0, 2, 1, G_HYBRID_N, lv, G_FIX_CAP), 2, c)
254 gv_check_eq("bit-packed-values-are-LSB-first: value 0 is 0" as *u8, lv[0], 0, c)
255 gv_check_eq("bit-packed-values-are-LSB-first: value 1 is 1" as *u8, lv[1], 1, c)
256 gv_check_eq("bit-packed-values-are-LSB-first: value 7 is 1" as *u8, lv[7], 1, c)
257 vb[0] = 0x03 as u8
258 vb[1] = 0xAA as u8
259 vb[2] = 0x04 as u8
260 vb[3] = 0x01 as u8
261 gv_check_eq("a-mixed-stream-consumes-four-bytes" as *u8, pq_hybrid(vb, 0, 4, 1, G_HYBRID_MIXED_N, lv, G_FIX_CAP), 4, c)
262 gv_check_eq("a-mixed-stream-last-value-is-the-rle-value (1)" as *u8, lv[G_HYBRID_MIXED_N - 1], 1, c)
263 gv_check_eq("width-zero-consumes-nothing" as *u8, pq_hybrid(vb, 0, 4, 0, 4, lv, G_FIX_CAP), 0, c)
264 gv_check_eq("width-zero-values-are-zero" as *u8, lv[3], 0, c)
265 vb[0] = 0x0A as u8
266 gv_check_eq("neg-control-an-rle-run-without-its-value-is-refused-TRUNCATED" as *u8, pq_hybrid(vb, 0, 1, 2, G_RLE_N, lv, G_FIX_CAP), PQ_E_TRUNC, c)
267 gv_check_eq("neg-control-a-count-over-capacity-is-refused-CAPACITY" as *u8, pq_hybrid(vb, 0, 1, 2, G_RLE_N, lv, 2), PQ_E_CAPACITY, c)
268
269 // ---- snappy: literal abcdef, copy 1-byte offset (len 4, offset 6), literal z
270 vb[0] = 0x0B as u8
271 vb[1] = 0x14 as u8
272 vb[2] = G_CH_A as u8
273 vb[3] = G_CH_B as u8
274 vb[4] = G_CH_C as u8
275 vb[5] = G_CH_D as u8
276 vb[6] = G_CH_E as u8
277 vb[7] = G_CH_F as u8
278 vb[8] = 0x01 as u8
279 vb[9] = 0x06 as u8
280 vb[10] = 0x00 as u8
281 vb[11] = G_CH_Z as u8
282 gv_check_eq("snappy-literal-copy1-literal-produces-11-bytes" as *u8, pq_snappy(vb, 0, 12, ob, 0, G_OUT_CAP), G_SNAPPY_OUT_LEN, c)
283 gv_check("snappy-output-is-abcdefabcdz" as *u8, g_bytes_are(ob, 0, G_SNAPPY_OUT_LEN, G_SNAPPY_OUT), c)
284 vb[9] = 0x14 as u8
285 gv_check_eq("neg-control-a-copy-offset-before-the-buffer-is-refused-SNAPPY" as *u8, pq_snappy(vb, 0, 12, ob, 0, G_OUT_CAP), PQ_E_SNAPPY, c)
286 vb[9] = 0x06 as u8
287 vb[0] = 0x0C as u8
288 gv_check_eq("neg-control-a-declared-length-the-stream-does-not-produce-is-refused-SNAPPY" as *u8, pq_snappy(vb, 0, 12, ob, 0, G_OUT_CAP), PQ_E_SNAPPY, c)
289 vb[0] = 0x0B as u8
290 gv_check_eq("neg-control-an-output-that-does-not-fit-is-refused-CAPACITY" as *u8, pq_snappy(vb, 0, 12, ob, 0, G_OUT_SMALL), PQ_E_CAPACITY, c)
291 // copy with a 2-byte offset: literal abcd then copy len 4 offset 4
292 vb[0] = 0x08 as u8
293 vb[1] = 0x0C as u8
294 vb[2] = G_CH_A as u8
295 vb[3] = G_CH_B as u8
296 vb[4] = G_CH_C as u8
297 vb[5] = G_CH_D as u8
298 vb[6] = 0x0E as u8
299 vb[7] = 0x04 as u8
300 vb[8] = 0x00 as u8
301 gv_check_eq("snappy-copy2-produces-8-bytes" as *u8, pq_snappy(vb, 0, 9, ob, 0, G_OUT_CAP), G_SNAPPY2_OUT_LEN, c)
302 gv_check("snappy-copy2-output-is-abcdabcd" as *u8, g_bytes_are(ob, 0, G_SNAPPY2_OUT_LEN, G_SNAPPY2_OUT), c)
303 // copy with a 4-byte offset: the same, offset as four bytes
304 vb[6] = 0x0F as u8
305 vb[7] = 0x04 as u8
306 vb[8] = 0x00 as u8
307 vb[9] = 0x00 as u8
308 vb[10] = 0x00 as u8
309 gv_check_eq("snappy-copy4-produces-8-bytes" as *u8, pq_snappy(vb, 0, 11, ob, 0, G_OUT_CAP), G_SNAPPY2_OUT_LEN, c)
310 gv_check("snappy-copy4-output-is-abcdabcd" as *u8, g_bytes_are(ob, 0, G_SNAPPY2_OUT_LEN, G_SNAPPY2_OUT), c)
311 // an overlapping copy: literal a then copy len 4 offset 1 -> aaaaa
312 vb[0] = 0x05 as u8
313 vb[1] = 0x00 as u8
314 vb[2] = G_CH_A as u8
315 vb[3] = 0x01 as u8
316 vb[4] = 0x01 as u8
317 gv_check_eq("snappy-overlapping-copy-produces-5-bytes" as *u8, pq_snappy(vb, 0, 5, ob, 0, G_OUT_CAP), G_SNAPPY3_OUT_LEN, c)
318 gv_check("snappy-overlapping-copy-output-is-aaaaa" as *u8, g_bytes_are(ob, 0, G_SNAPPY3_OUT_LEN, G_SNAPPY3_OUT), c)
319
320 // ---- pq_open refusals on planted files
321 let pq: *i64 = sys_mmap(PQ_SLOTS * G_I64) as *i64
322 gv_check("fixture-written: a file that is not parquet" as *u8, g_write_bytes(G_NOTPQ, G_NOTPQ_BYTES, g_slen(G_NOTPQ_BYTES)), c)
323 gv_check_eq("neg-control-a-file-without-the-magic-is-refused-NOT-PARQUET" as *u8, pq_open(pq, G_NOTPQ), PQ_E_MAGIC, c)
324 vb[0] = PQ_MAGIC_P as u8
325 vb[1] = PQ_MAGIC_A as u8
326 vb[2] = PQ_MAGIC_R as u8
327 vb[3] = PQ_MAGIC_1 as u8
328 vb[4] = 0xE8 as u8
329 vb[5] = 0x03 as u8
330 vb[6] = 0x00 as u8
331 vb[7] = 0x00 as u8
332 vb[8] = PQ_MAGIC_P as u8
333 vb[9] = PQ_MAGIC_A as u8
334 vb[10] = PQ_MAGIC_R as u8
335 vb[11] = PQ_MAGIC_1 as u8
336 gv_check("fixture-written: both magics and a footer length of 1000 in a 12-byte file" as *u8, g_write_bytes(G_BADLEN, vb, 12), c)
337 gv_check_eq("neg-control-a-footer-length-past-the-file-is-refused-TRUNCATED" as *u8, pq_open(pq, G_BADLEN), PQ_E_TRUNC, c)
338
339 // ---- the real bytes: BRIGHT pony examples
340 let lenp: *i64 = sys_mmap(G_I64) as *i64
341 let exb: *u8 = g_read2(G_EX_A, G_EX_B, lenp)
342 let exlen: i64 = lenp[0]
343 if gv_need("the pinned BRIGHT pony examples parquet (knowledge/fetched, search.refs brightds26)" as *u8, (exlen > 0) as i64, c) == 1 {
344 gv_check("fixture-written: the examples file cut at 20000 bytes" as *u8, g_write_bytes(G_TRUNC, exb, G_TRUNC_KEEP), c)
345 gv_check_eq("neg-control-a-file-cut-before-its-footer-is-refused-NOT-PARQUET (no tail magic)" as *u8, pq_open(pq, G_TRUNC), PQ_E_MAGIC, c)
346 let flen: i64 = pq_le32(exb, exlen - PQ_TAIL_LEN)
347 let bumped: i64 = flen + G_BUMP
348 exb[exlen - PQ_TAIL_LEN] = (bumped & PQ_BYTE) as u8
349 exb[exlen - PQ_TAIL_LEN + 1] = ((bumped >> 8) & PQ_BYTE) as u8
350 gv_check("fixture-written: the examples file with its footer length bumped by five" as *u8, g_write_bytes(G_BUMPED, exb, exlen), c)
351 let rcb: i64 = pq_open(pq, G_BUMPED)
352 gv_check("neg-control-a-bumped-footer-length-is-refused (any refusal, never OK)" as *u8, (rcb < 0) as i64, c)
353 gv_kv("bumped_footer_refusal_code" as *u8, rcb)
354 let rce: i64 = g_open2(pq, G_EX_A, G_EX_B)
355 gv_check_eq("examples-opens-OK" as *u8, rce, 0, c)
356 gv_check_eq("examples-declares-112-rows (the card's num_examples for pony)" as *u8, pq[PQ_NROWS], G_EX_ROWS, c)
357 gv_check_eq("examples-has-7-leaves" as *u8, pq[PQ_NLEAF], G_EX_LEAVES, c)
358 gv_check_eq("examples-has-1-row-group" as *u8, pq[PQ_NRG], G_EX_RGS, c)
359 gv_check_eq("a-list-column-resolves-by-its-column-name (gold_ids is leaf 5)" as *u8, pq_leaf_named(pq, "gold_ids" as *u8), G_EX_LEAF_GOLD, c)
360 gv_check_eq("a-flat-column-resolves-by-its-name (query is leaf 0)" as *u8, pq_leaf_named(pq, "query" as *u8), G_EX_LEAF_QUERY, c)
361 gv_check_eq("neg-control-the-writer's-element-name-is-not-a-column" as *u8, pq_leaf_named(pq, "element" as *u8), PQ_NONE, c)
362 let pg: *i64 = pr_alloc(pq, 0, G_EX_LEAF_GOLD)
363 let rcg: i64 = pq_read_column(pq, 0, G_EX_LEAF_GOLD, pg)
364 gv_check_eq("gold_ids-decodes-2219-entries" as *u8, rcg, G_EX_GOLD_ENTRIES, c)
365 gv_check_eq("gold_ids-entries-reconstruct-112-rows-from-repetition-levels" as *u8, pg[PR_ROWS], G_EX_ROWS, c)
366 gv_check_eq("gold_ids-every-entry-is-present (no nulls)" as *u8, pg[PR_NONNULL], G_EX_GOLD_ENTRIES, c)
367 gv_check_eq("gold_ids-dictionary-has-43-values" as *u8, pg[PR_DICTN], G_EX_GOLD_DICT, c)
368 gv_check_eq("gold_ids-is-two-pages (dictionary then data)" as *u8, pg[PR_PAGES], G_EX_PAGES, c)
369 gv_check_eq("gold_ids-values-are-RLE_DICTIONARY-encoded (mask bit 8)" as *u8, pg[PR_ENCMASK], G_EX_ENCMASK_RLE_DICT, c)
370 let grep: *i64 = pg[PR_REP] as *i64
371 let gdef: *i64 = pg[PR_DEF] as *i64
372 let grow: *i64 = pg[PR_ROW] as *i64
373 var cont: i64 = 0
374 var k: i64 = 0
375 while k < pg[PR_N] { if grep[k] == 1 && gdef[k] == 3 { cont = cont + 1 } k = k + 1 }
376 gv_check("gold_ids-carries-list-continuations (rep 1, def 3)" as *u8, (cont > 0) as i64, c)
377 gv_check_eq("gold_ids-first-entry-starts-row-0" as *u8, grow[0], 0, c)
378 gv_check_eq("gold_ids-first-entry-has-rep-0" as *u8, grep[0], 0, c)
379 gv_check_eq("gold_ids-last-entry-belongs-to-row-111" as *u8, grow[pg[PR_N] - 1], G_EX_ROWS - 1, c)
380 pr_free(pg)
381 let pqy: *i64 = pr_alloc(pq, 0, G_EX_LEAF_QUERY)
382 let rcq: i64 = pq_read_column(pq, 0, G_EX_LEAF_QUERY, pqy)
383 gv_check_eq("query-decodes-112-entries" as *u8, rcq, G_EX_ROWS, c)
384 let qoff: *i64 = pqy[PR_OFF] as *i64
385 let qlen: *i64 = pqy[PR_LEN] as *i64
386 gv_check_eq("query-0-is-237-bytes" as *u8, qlen[0], G_EX_QUERY0_LEN, c)
387 gv_check("query-0-starts-with-the-known-text" as *u8, g_starts(pqy[PR_ARENA] as *u8, qoff[0], qlen[0], G_QUERY_PREFIX), c)
388 pr_free(pqy)
389 let pid: *i64 = pr_alloc(pq, 0, G_EX_LEAF_ID)
390 let rci: i64 = pq_read_column(pq, 0, G_EX_LEAF_ID, pid)
391 gv_check_eq("id-decodes-112-entries" as *u8, rci, G_EX_ROWS, c)
392 let ioff: *i64 = pid[PR_OFF] as *i64
393 let ilen: *i64 = pid[PR_LEN] as *i64
394 gv_check("id-0-is-the-string-0" as *u8, g_bytes_are(pid[PR_ARENA] as *u8, ioff[0], ilen[0], "0" as *u8), c)
395 gv_check("id-2-is-the-string-2" as *u8, g_bytes_are(pid[PR_ARENA] as *u8, ioff[2], ilen[2], "2" as *u8), c)
396 pr_free(pid)
397 gv_check_eq("neg-control-a-leaf-out-of-range-is-refused-OUT-OF-RANGE" as *u8, pq_read_column(pq, 0, G_BAD_LEAF, pid), PQ_E_ARG, c)
398 gv_check_eq("neg-control-a-row-group-out-of-range-is-refused-OUT-OF-RANGE" as *u8, pq_read_column(pq, G_BAD_RG, 0, pid), PQ_E_ARG, c)
399 let stats0: *i64 = sys_mmap(PT_SLOTS * G_I64) as *i64
400 gv_check_eq("neg-control-tsv-of-a-list-leaf-is-refused-OUT-OF-RANGE" as *u8, pq_tsv(pq, G_EX_LEAF_ID, G_EX_LEAF_GOLD, G_TSV_A, stats0), PQ_E_ARG, c)
401 }
402
403 // ---- the real bytes: BRIGHT pony documents
404 let pqd: *i64 = sys_mmap(PQ_SLOTS * G_I64) as *i64
405 let rcd: i64 = g_open2(pqd, G_DOC_A, G_DOC_B)
406 if gv_need("the pinned BRIGHT pony documents parquet (knowledge/fetched, search.refs brightds26)" as *u8, (rcd != PQ_E_OPEN) as i64, c) == 1 {
407 gv_check_eq("documents-opens-OK" as *u8, rcd, 0, c)
408 gv_check_eq("documents-declares-7894-rows (the card's num_examples for pony documents)" as *u8, pqd[PQ_NROWS], G_DOC_ROWS, c)
409 gv_check_eq("documents-has-8-row-groups" as *u8, pqd[PQ_NRG], G_DOC_RGS, c)
410 gv_check_eq("documents-has-2-leaves" as *u8, pqd[PQ_NLEAF], G_DOC_LEAVES, c)
411 var rows_id: i64 = 0
412 var rows_content: i64 = 0
413 var worst: i64 = 0
414 var r: i64 = 0
415 while r < pqd[PQ_NRG] {
416 let pa: *i64 = pr_alloc(pqd, r, 0)
417 let ra: i64 = pq_read_column(pqd, r, 0, pa)
418 if ra < 0 { worst = ra } else { rows_id = rows_id + pa[PR_ROWS] }
419 pr_free(pa)
420 let pb: *i64 = pr_alloc(pqd, r, 1)
421 let rb: i64 = pq_read_column(pqd, r, 1, pb)
422 if rb < 0 { worst = rb } else { rows_content = rows_content + pb[PR_ROWS] }
423 pr_free(pb)
424 r = r + 1
425 }
426 gv_check_eq("every-documents-row-group-decodes (no refusal)" as *u8, worst, 0, c)
427 gv_check_eq("documents-id-rows-across-8-row-groups-sum-to-7894" as *u8, rows_id, G_DOC_ROWS, c)
428 gv_check_eq("documents-content-rows-across-8-row-groups-sum-to-7894" as *u8, rows_content, G_DOC_ROWS, c)
429 let stats: *i64 = sys_mmap(PT_SLOTS * G_I64) as *i64
430 gv_check_eq("tsv-of-id-and-content-writes-OK" as *u8, pq_tsv(pqd, 0, 1, G_TSV_A, stats), 0, c)
431 gv_check_eq("tsv-writes-7894-rows" as *u8, stats[PT_WRITTEN], G_DOC_ROWS, c)
432 gv_check_eq("tsv-skips-no-null-rows" as *u8, stats[PT_NULLROWS], 0, c)
433 gv_check_eq("tsv-is-2317793-bytes (the measured size, a determinism KAT)" as *u8, stats[PT_BYTES], G_DOC_TSV_BYTES, c)
434 let stats2: *i64 = sys_mmap(PT_SLOTS * G_I64) as *i64
435 gv_check_eq("tsv-written-a-second-time-writes-OK" as *u8, pq_tsv(pqd, 0, 1, G_TSV_B, stats2), 0, c)
436 gv_check("two-tsv-writes-are-byte-identical" as *u8, g_files_equal(G_TSV_A, G_TSV_B), c)
437 let tl: *i64 = sys_mmap(G_I64) as *i64
438 tl[0] = 0
439 let tb: *u8 = sys_read_file(G_TSV_A, tl)
440 gv_check("tsv-first-row-starts-with-the-first-document-id" as *u8, g_starts(tb, 0, tl[0], G_DOC_FIRST_ID), c)
441 gv_check_eq("tsv-first-row-has-a-tab-after-the-id" as *u8, (tb[g_slen(G_DOC_FIRST_ID)] & PQ_BYTE) as i64, PQ_TAB, c)
442 gv_values_head()
443 gv_kv("documents_rows" as *u8, rows_id)
444 gv_kv("documents_tsv_bytes" as *u8, stats[PT_BYTES])
445 gv_kv("examples_bytes" as *u8, exlen)
446 }
447 return gv_verdict("nx_parquet_gate" as *u8, c, "the sovereign parquet reader proven on planted bytes with known answers and neg-controls for every wire rule it relies on (thrift varints, zigzag, field and list headers, the bounded container skipper, the RLE and bit-packed hybrid, snappy with all three copy widths, pq_open on files that are not parquet, cut, or carry a footer length past the file or bumped by five), and on the pinned BRIGHT pony files: the card's declared counts reproduce (112 examples, 7894 documents over 8 row groups), a list column resolves by its column name, 2219 gold_ids reconstruct 112 rows from repetition levels, the first query and ids decode to their known bytes, out-of-range reads and a list-leaf tsv refuse by name, and the documents TSV is byte-identical across two writes; the real-file teeth SKIP when the fixtures are absent" as *u8)
448}