nx_genfix.nx source
↩ module page · 210 lines · 8035 B
1// nx_genfix.nx -- shared loader for image-generation oracle fixtures. MODEL-AGNOSTIC.
2//
3// WHY THIS EXISTS
4// The sovereign gen migration verifies each re-implemented op against a tensor dumped from
5// the engine that ships images today. Those dumps used to be produced by throwaway
6// instrumentation into a per-session temp directory; both were reaped, and the 13 organs that
7// depended on them each carried their own private copy of the dead path, so all 13 died at
8// once and each needed its own edit.
9//
10// WHY IT IS NOT NAMED FOR A MODEL
11// Z-Image is one model, not the lane. Flux, SD3/MMDiT, SDXL-UNet and Qwen-Image all need the
12// same treatment, and they differ in dimensions, block naming and product capability (negative
13// prompting, CFG, LoRA stacking). So nothing here may hardcode a model: the model id is an
14// argument, and every SHAPE is read from the manifest.tsv the oracle tap writes beside the
15// dumps. An organ that hardcodes 3840 is an organ that silently verifies the wrong model the
16// first time it is pointed at another one.
17//
18// LAYOUT
19// <root>/<model_id>/manifest.tsv name \t ne0 \t ne1 \t ne2 \t ne3 \t type \t nelem \t bytes
20// <root>/<model_id>/<name>.f32 raw little-endian f32
21// license_tier: ORIGINAL
22
23import "nx_syscalls.nx"
24const K_MAGIC_1024: i64 = 1024
25const K_MAGIC_1048576: i64 = 1048576
26
27// Fixture root. Durable -- NOT a temp directory; that is the defect this file exists to fix.
28func nx_genfix_root() -> *u8 {
29 let r: *u8 = "/mnt/c/Users/elder/nishi-fixtures/" as *u8
30 return r
31}
32
33func _gf_cpy(dst: *u8, at: i64, src: *u8, n: i64) -> i64 {
34 var o: i64 = at
35 var i: i64 = 0
36 while i < n { dst[o] = src[i]; o = o + 1; i = i + 1 }
37 return o
38}
39
40func _gf_cpyz(dst: *u8, at: i64, src: *u8) -> i64 {
41 var o: i64 = at
42 var i: i64 = 0
43 while src[i] != (0 as u8) { dst[o] = src[i]; o = o + 1; i = i + 1 }
44 return o
45}
46
47// Join a tap scope and an op into "<scope>.<op>" -- the naming the oracle tap emits.
48// Returns the length written. Lets an organ be pointed at any block without a new build.
49func nx_genfix_name(out: *u8, scope: *u8, op: *u8, ol: i64) -> i64 {
50 var o: i64 = _gf_cpyz(out, 0, scope)
51 out[o] = 0x2E; o = o + 1 // '.'
52 o = _gf_cpy(out, o, op, ol)
53 out[o] = 0
54 return o
55}
56
57// Build "<root><model>/<name>.f32"
58func _gf_path(model: *u8, name: *u8, nl: i64) -> *u8 {
59 let path: *u8 = sys_mmap(K_MAGIC_1024)
60 var o: i64 = _gf_cpyz(path, 0, nx_genfix_root())
61 o = _gf_cpyz(path, o, model)
62 path[o] = 0x2F; o = o + 1 // '/'
63 o = _gf_cpy(path, o, name, nl)
64 path[o] = 0x2E; o = o + 1 // '.'
65 path[o] = 0x66; o = o + 1 // 'f'
66 path[o] = 0x33; o = o + 1 // '3'
67 path[o] = 0x32; o = o + 1 // '2'
68 path[o] = 0
69 return path
70}
71
72// Load <root>/<model>/<name>.f32.
73//
74// Returns 0 on open failure OR short read. The previous per-organ loaders returned their
75// buffer unconditionally, so a missing or truncated fixture arrived as a page of zeros and
76// was reported as a numeric mismatch -- a harness fault wearing the clothes of a real result.
77func nx_genfix_load(model: *u8, name: *u8, nl: i64, n_floats: i64) -> *u8 {
78 let path: *u8 = _gf_path(model, name, nl)
79 let fd: i64 = sys_openat_rd(path)
80 if fd < 0 { return 0 as *u8 }
81
82 let bytes: i64 = n_floats * 4
83 let buf: *u8 = sys_mmap(bytes + 64)
84 var tot: i64 = 0
85 var go: i64 = 1
86 while go == 1 {
87 let r: i64 = sys_read(fd, ((buf as i64) + tot) as *u8, bytes - tot)
88 if r <= 0 { go = 0 } else { tot = tot + r; if tot >= bytes { go = 0 } }
89 }
90 sys_close(fd)
91 if tot < bytes { return 0 as *u8 }
92 return buf
93}
94
95// Build "<root><model>/<name>.raw"
96func _gf_path_raw(model: *u8, name: *u8, nl: i64) -> *u8 {
97 let path: *u8 = sys_mmap(K_MAGIC_1024)
98 var o: i64 = _gf_cpyz(path, 0, nx_genfix_root())
99 o = _gf_cpyz(path, o, model)
100 path[o] = 0x2F; o = o + 1 // '/'
101 o = _gf_cpy(path, o, name, nl)
102 path[o] = 0x2E; o = o + 1 // '.'
103 path[o] = 0x72; o = o + 1 // 'r'
104 path[o] = 0x61; o = o + 1 // 'a'
105 path[o] = 0x77; o = o + 1 // 'w'
106 path[o] = 0
107 return path
108}
109
110// Load <root>/<model>/<name>.raw -- the tensor's NATIVE bytes.
111//
112// The .f32 sibling is ggml's own dequantization, which is the right reference for an f32 kernel
113// and the WRONG input for a quantized one: a sovereign Q8_0 dequant-dot must read the same int8
114// blocks the engine reads, or it is not the same computation and its speed is not comparable.
115func nx_genfix_load_raw(model: *u8, name: *u8, nl: i64, n_bytes: i64) -> *u8 {
116 let path: *u8 = _gf_path_raw(model, name, nl)
117 let fd: i64 = sys_openat_rd(path)
118 if fd < 0 { return 0 as *u8 }
119 let buf: *u8 = sys_mmap(n_bytes + 64)
120 var tot: i64 = 0
121 var go: i64 = 1
122 while go == 1 {
123 let r: i64 = sys_read(fd, ((buf as i64) + tot) as *u8, n_bytes - tot)
124 if r <= 0 { go = 0 } else { tot = tot + r; if tot >= n_bytes { go = 0 } }
125 }
126 sys_close(fd)
127 if tot < n_bytes { return 0 as *u8 }
128 return buf
129}
130
131// ===== manifest =====================================================
132//
133// Shapes are READ, never assumed. The old fixture set shipped bare .f32 blobs whose shape
134// lived only in the reader, so once the producer was gone the bytes were unreadable.
135
136func _gf_read_manifest(model: *u8, out_len: *i64) -> *u8 {
137 let path: *u8 = sys_mmap(K_MAGIC_1024)
138 var o: i64 = _gf_cpyz(path, 0, nx_genfix_root())
139 o = _gf_cpyz(path, o, model)
140 path[o] = 0x2F; o = o + 1
141 o = _gf_cpyz(path, o, "manifest.tsv" as *u8)
142 path[o] = 0
143
144 let fd: i64 = sys_openat_rd(path)
145 if fd < 0 { return 0 as *u8 }
146 let CAP: i64 = K_MAGIC_1048576
147 let buf: *u8 = sys_mmap(CAP)
148 var tot: i64 = 0
149 var go: i64 = 1
150 while go == 1 {
151 let r: i64 = sys_read(fd, ((buf as i64) + tot) as *u8, CAP - tot)
152 if r <= 0 { go = 0 } else { tot = tot + r; if tot >= CAP { go = 0 } }
153 }
154 sys_close(fd)
155 out_len[0] = tot
156 return buf
157}
158
159// Look up one tensor. Fills ne[0..3] and returns its element count, or -1 if absent.
160// An absent name is a HARD error for the caller: it means the tap never emitted this tensor,
161// which is a different fault from a numeric mismatch and must not be confused with one.
162func nx_genfix_dims(model: *u8, name: *u8, nl: i64, ne: *i64) -> i64 {
163 let lenp: *i64 = sys_mmap(16) as *i64
164 lenp[0] = 0
165 let m: *u8 = _gf_read_manifest(model, lenp)
166 if (m as i64) == 0 { return 0 - 1 }
167 let mlen: i64 = lenp[0]
168
169 var pos: i64 = 0
170 while pos < mlen {
171 // does the row at pos start with name followed by TAB?
172 var k: i64 = 0
173 var hit: i64 = 1
174 while k < nl {
175 if (pos + k) >= mlen { hit = 0; k = nl }
176 else {
177 if m[pos + k] != name[k] { hit = 0; k = nl }
178 else { k = k + 1 }
179 }
180 }
181 if hit == 1 {
182 if (pos + nl) >= mlen { hit = 0 }
183 else { if m[pos + nl] != (0x09 as u8) { hit = 0 } }
184 }
185
186 if hit == 1 {
187 // parse 4 tab-separated integers after the name
188 var p: i64 = pos + nl
189 var f: i64 = 0
190 var nelem: i64 = 1
191 while f < 4 {
192 p = p + 1 // skip the TAB
193 var v: i64 = 0
194 while p < mlen && m[p] >= (0x30 as u8) && m[p] <= (0x39 as u8) {
195 v = v * 10 + ((m[p] as i64) - 48)
196 p = p + 1
197 }
198 ne[f] = v
199 nelem = nelem * v
200 f = f + 1
201 }
202 return nelem
203 }
204
205 // advance to the next line
206 while pos < mlen && m[pos] != (0x0A as u8) { pos = pos + 1 }
207 pos = pos + 1
208 }
209 return 0 - 1
210}