nx_npy_gate.nx source
↩ module page · 150 lines · 7672 B
1// nx_npy_gate.nx -- proves the .npy parser reads what is there and REFUSES the three things that
2// corrupt silently. procgen PG38's opening move, gateable on its own before any model byte is fetched.
3//
4// WHY THE NEG-CONTROLS ARE THE POINT. A forward-only suite here would pass for a parser that ignores
5// fortran_order, guesses an itemsize and trusts the declared shape -- and every one of those returns
6// PLAUSIBLE NUMBERS rather than an error. A transposed basis, a factor-of-two offset walk, or a read
7// past the end of a member all look like data. So T7-T10 are the load-bearing teeth.
8//
9// THE FIXTURE IS ASSEMBLED AT RUNTIME, never read from disk: the parser's own subject is a byte layout,
10// so a byte layout built in memory is the honest fixture and there is no file to go stale. The teeth are
11// ARITHMETIC over that layout (count = product of dims; data_off + count*itemsize = total), which makes
12// them their own oracle in the mathcore sense -- no second implementation to disagree with.
13//
14// license_tier: ORIGINAL expect_exit: 0
15import "nx_syscalls.nx"
16import "nx_gate_verdict.nx"
17import "nx_npy_lib.nx"
18
19// The f8 case alone needs 5023*3*8 = 120,552 payload bytes. The FIRST run of this gate sized this at
20// 65536 and ng_mk wrote past the end, corrupting a NEIGHBOURING fixture so the GOOD buffer reported
21// bad magic -- a harness bug masquerading as a parser bug. Sized from the largest fixture, with room.
22const NG_CAP: i64 = 262144
23const NG_Q: i64 = 39 // '\'' by name, the header's own quoting
24
25func ng_puts(b: *u8, o0: i64, s: *u8) -> i64 {
26 var o: i64 = o0
27 var i: i64 = 0
28 while s[i] != (0 as u8) { b[o] = s[i]; o = o + 1; i = i + 1 }
29 return o
30}
31
32func ng_putb(b: *u8, o: i64, v: i64) -> i64 { b[o] = v as u8; return o + 1 }
33
34// Assemble a v1 .npy: magic, version, u16 header length, header text, then `nbytes` of zero payload.
35// Returns the total length. descr and shape_txt are supplied so a tooth can vary ONE field at a time.
36func ng_mk(b: *u8, descr: *u8, fortran: *u8, shape_txt: *u8, nbytes: i64) -> i64 {
37 var o: i64 = 0
38 o = ng_putb(b, o, 147)
39 o = ng_puts(b, o, "NUMPY" as *u8)
40 o = ng_putb(b, o, 1)
41 o = ng_putb(b, o, 0)
42 let hlen_at: i64 = o
43 o = ng_putb(b, o, 0)
44 o = ng_putb(b, o, 0)
45 let hstart: i64 = o
46 o = ng_puts(b, o, "{" as *u8)
47 o = ng_putb(b, o, NG_Q); o = ng_puts(b, o, "descr" as *u8); o = ng_putb(b, o, NG_Q)
48 o = ng_puts(b, o, ": " as *u8)
49 o = ng_putb(b, o, NG_Q); o = ng_puts(b, o, descr); o = ng_putb(b, o, NG_Q)
50 o = ng_puts(b, o, ", " as *u8)
51 o = ng_putb(b, o, NG_Q); o = ng_puts(b, o, "fortran_order" as *u8); o = ng_putb(b, o, NG_Q)
52 o = ng_puts(b, o, ": " as *u8)
53 o = ng_puts(b, o, fortran)
54 o = ng_puts(b, o, ", " as *u8)
55 o = ng_putb(b, o, NG_Q); o = ng_puts(b, o, "shape" as *u8); o = ng_putb(b, o, NG_Q)
56 o = ng_puts(b, o, ": " as *u8)
57 o = ng_puts(b, o, shape_txt)
58 o = ng_puts(b, o, ", }" as *u8)
59 let hlen: i64 = o - hstart
60 b[hlen_at] = (hlen % 256) as u8
61 b[hlen_at + 1] = ((hlen / 256) % 256) as u8
62 var k: i64 = 0
63 while k < nbytes { b[o + k] = 0 as u8; k = k + 1 }
64 return o + nbytes
65}
66
67func main() -> i64 {
68 let ctr: *i64 = gv_ctr()
69 gv_head("=== nx_npy_gate -- the .npy parser reads what is there and refuses what corrupts silently ===" as *u8)
70
71 let dims: *i64 = sys_mmap(64) as *i64
72 let out: *i64 = sys_mmap(64) as *i64
73
74 // ---- the good case: 5023 x 3 float32, the shape a vertex basis actually has ----
75 let b: *u8 = sys_mmap(NG_CAP)
76 let need: i64 = 5023 * 3 * 4
77 let total: i64 = ng_mk(b, "<f4" as *u8, "False" as *u8, "(5023, 3)" as *u8, need)
78
79 gv_check_eq("recognises-npy-magic", npy_is_npy(b, total), 1, ctr)
80 gv_check_eq("reads-major-version", npy_major(b, total), 1, ctr)
81
82 let rc: i64 = npy_open(b, total, dims, out)
83 gv_check_eq("opens-a-well-formed-array", rc, NPY_OK, ctr)
84 gv_check_eq("derives-itemsize-from-descr-not-assumed", out[1], 4, ctr)
85 gv_check_eq("reads-dtype-kind-float", out[2], NPY_KIND_F, ctr)
86 gv_check_eq("reads-ndim", out[3], 2, ctr)
87 gv_check_eq("reads-dim0", dims[0], 5023, ctr)
88 gv_check_eq("reads-dim1", dims[1], 3, ctr)
89 // ALGEBRAIC, its own oracle: the count is the product of the dims
90 gv_check_eq("elem-count-is-product-of-dims", out[4], 5023 * 3, ctr)
91 // PARTITION SUMS: header + payload accounts for every byte of the member
92 gv_check_eq("partition-sums-header-plus-payload", out[0] + out[4] * out[1], total, ctr)
93
94 // ---- itemsize must be DERIVED: same shape, f8, must double the payload accounting ----
95 let b8: *u8 = sys_mmap(NG_CAP)
96 let need8: i64 = 5023 * 3 * 8
97 let tot8: i64 = ng_mk(b8, "<f8" as *u8, "False" as *u8, "(5023, 3)" as *u8, need8)
98 let rc8: i64 = npy_open(b8, tot8, dims, out)
99 gv_check_eq("f8-opens", rc8, NPY_OK, ctr)
100 gv_check_eq("f8-itemsize-is-eight-not-four", out[1], 8, ctr)
101 gv_check_eq("f8-partition-sums", out[0] + out[4] * out[1], tot8, ctr)
102
103 // ---- NEG-CONTROL 1: fortran_order True is the TRANSPOSED-BYTES trap. Every length check still
104 // passes; only an explicit refusal stops a scrambled basis being read as a good one. ----
105 let bf: *u8 = sys_mmap(NG_CAP)
106 let totf: i64 = ng_mk(bf, "<f4" as *u8, "True" as *u8, "(5023, 3)" as *u8, need)
107 gv_check_eq("neg-control-fortran-order-refused", npy_open(bf, totf, dims, out), NPY_ERR_FORTRAN, ctr)
108 gv_check_eq("neg-control-fortran-order-detected", npy_fortran_order(bf, totf), 1, ctr)
109 gv_check_eq("c-order-reads-false", npy_fortran_order(b, total), 0, ctr)
110
111 // ---- NEG-CONTROL 2: an unknown dtype must REFUSE, never guess an itemsize ----
112 let bz: *u8 = sys_mmap(NG_CAP)
113 let totz: i64 = ng_mk(bz, "<z4" as *u8, "False" as *u8, "(4, 4)" as *u8, 64)
114 gv_check_eq("neg-control-unknown-dtype-refused", npy_open(bz, totz, dims, out), NPY_ERR_DTYPE, ctr)
115
116 // ---- NEG-CONTROL 3: a shape that claims more elements than bytes present must REFUSE.
117 // Without this the parser walks off the end of the ZIP member into the next one. ----
118 let bs: *u8 = sys_mmap(NG_CAP)
119 let tots: i64 = ng_mk(bs, "<f4" as *u8, "False" as *u8, "(5023, 3)" as *u8, 16)
120 gv_check_eq("neg-control-declared-size-exceeds-bytes-refused", npy_open(bs, tots, dims, out), NPY_ERR_SIZE, ctr)
121
122 // ---- NEG-CONTROL 4: a buffer that is not npy at all ----
123 let bb: *u8 = sys_mmap(64)
124 var i: i64 = 0
125 while i < 32 { bb[i] = 65 as u8; i = i + 1 }
126 gv_check_eq("neg-control-bad-magic-refused", npy_open(bb, 32, dims, out), NPY_ERR_MAGIC, ctr)
127 gv_check_eq("neg-control-bad-magic-not-recognised", npy_is_npy(bb, 32), 0, ctr)
128
129 // ---- a 1-D array, because a basis file carries both shapes ----
130 let b1: *u8 = sys_mmap(NG_CAP)
131 let tot1: i64 = ng_mk(b1, "<i4" as *u8, "False" as *u8, "(300,)" as *u8, 1200)
132 gv_check_eq("1d-opens", npy_open(b1, tot1, dims, out), NPY_OK, ctr)
133 gv_check_eq("1d-ndim-is-one", out[3], 1, ctr)
134 gv_check_eq("1d-count", out[4], 300, ctr)
135 gv_check_eq("1d-kind-int", out[2], NPY_KIND_I, ctr)
136
137 gv_values_head()
138 let rc2: i64 = npy_open(b, total, dims, out)
139 gv_kv("open_rc" as *u8, rc2)
140 gv_kv("data_off" as *u8, out[0])
141 gv_kv("itemsize" as *u8, out[1])
142 gv_kv("dtype_kind" as *u8, out[2])
143 gv_kv("ndim" as *u8, out[3])
144 gv_kv("elem_count" as *u8, out[4])
145 gv_kv("payload_bytes" as *u8, out[5])
146 gv_kv("total_member_bytes" as *u8, total)
147 gv_kv("header_len" as *u8, npy_hdr_len(b, total))
148
149 return gv_verdict("nx_npy_gate" as *u8, ctr, "npy header parsed; fortran-order, unknown dtype, oversized shape and bad magic all refused by name" as *u8)
150}