nx_bundle_ingest_lib.nx source
↩ module page · 337 lines · 15343 B
1// nx_bundle_ingest_lib.nx -- THE COMMUNITY BUNDLE WALKER (/compare/modding MD1 bundle_ingest, 2026-09-06): a ZIP from a rip
2// page or a mod host is walked by CONTENT TYPE, never by filename. Composes the incumbents and adds no second ruler:
3// nx_zip.nx the central-directory reader (EOCD -> CD -> local header), the reflected CRC-32
4// nx_deflate.nx RFC 1951 INFLATE for method-8 members, sized from the member's DECLARED usize
5// nx_assetprobe_lib.nx the ONE identify / classify / claim ruler (the CLI probe and this walker share it)
6// Every member gets a row: method, sizes, the magic the bytes carry, the category that magic implies, the extension's
7// claim, the probe verdict, the inflate outcome, and -- for a texture -- which model or material member REFERENCES it by
8// basename inside its own bytes (the mesh's material slots, read from the mesh, not guessed from a folder). A member the
9// ruler cannot type is REPORTED as unknown, never dropped; a method the walker cannot open is REPORTED as unsupported.
10// The partition (model material texture motion audio video bundle unknown) is asserted to SUM to the member count.
11// license_tier: ORIGINAL No hw writes (Rule 26).
12import "nx_syscalls.nx"
13import "nx_zip.nx"
14import "nx_deflate.nx"
15import "nx_assetprobe_lib.nx"
16
17const BI_METHOD_STORE: i64 = 0
18const BI_METHOD_DEFLATE: i64 = 8
19// inflate outcomes
20const BI_INF_STORED: i64 = 0 // method 0: bytes read in place, CRC verified
21const BI_INF_OK: i64 = 1 // method 8: inflated to exactly usize, CRC verified
22const BI_INF_FAILED: i64 = 2 // the stream or the CRC refused
23const BI_INF_UNSUPPORTED: i64 = 3 // a method this walker does not open (reported by number)
24const BI_INF_NOT_READ: i64 = 5
25const BI_INF_SIZE_MISMATCH: i64 = 6
26const BI_INF_CRC_MISMATCH: i64 = 7
27const BI_INF_LIMIT: i64 = 8
28const BI_INF_BADLOCAL: i64 = 4 // local header missing or outside the file
29// probe verdicts (the CLI's own words)
30const BI_V_UNIDENTIFIED: i64 = 0
31const BI_V_UNCLAIMED: i64 = 1
32const BI_V_AGREE: i64 = 2
33const BI_V_DISAGREE: i64 = 3
34const BI_V_UNREAD: i64 = 4 // no bytes to probe (inflate failed or unsupported)
35// texture binding
36const BI_BIND_NA: i64 = 0
37const BI_BIND_BOUND: i64 = 1
38const BI_BIND_UNBOUND: i64 = 2
39// per-member record slots
40const BI_F_METHOD: i64 = 0
41const BI_F_CSIZE: i64 = 1
42const BI_F_USIZE: i64 = 2
43const BI_F_MAGIC: i64 = 3
44const BI_F_CAT: i64 = 4
45const BI_F_CLAIM: i64 = 5
46const BI_F_VERDICT: i64 = 6
47const BI_F_INFLATE: i64 = 7
48const BI_F_BIND: i64 = 8
49const BI_F_BOUND_TO: i64 = 9
50const BI_F_NAMEOFF: i64 = 10
51const BI_F_NAMELEN: i64 = 11
52const BI_F_DATA: i64 = 12
53const BI_F_DLEN: i64 = 13
54const BI_F_CRC: i64 = 14
55const BI_F_CRC_ACTUAL: i64 = 15
56const BI_F_N: i64 = 16
57// partition slots
58const BI_P_MODEL: i64 = 0
59const BI_P_MATERIAL: i64 = 1
60const BI_P_TEXTURE: i64 = 2
61const BI_P_MOTION: i64 = 3
62const BI_P_AUDIO: i64 = 4
63const BI_P_VIDEO: i64 = 5
64const BI_P_BUNDLE: i64 = 6
65const BI_P_UNKNOWN: i64 = 7
66const BI_P_STORED: i64 = 8
67const BI_P_INFLATED: i64 = 9
68const BI_P_FAILED: i64 = 10
69const BI_P_UNSUPPORTED: i64 = 11
70const BI_P_BOUND: i64 = 12
71const BI_P_UNBOUND: i64 = 13
72const BI_P_N: i64 = 16
73const BI_ZIP_FLD_N: i64 = 8
74const BI_NAME_CAP: i64 = 1024
75
76func bi_rec(tbl: *i64, i: i64) -> *i64 { return ((tbl as i64) + i * BI_F_N * 8) as *i64 }
77func bi_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
78
79// member count from the EOCD, or -1 when the container is not a zip / truncated (REFUSE, never zero members)
80func bi_count(b: *u8, n: i64) -> i64 { return nxzip_count(b, n) }
81
82// the data offset of a member's LOCAL entry for ANY method (nxzip_entry_data refuses non-STORED by contract); -1 if the
83// local header is missing, mislabelled or outside the file
84func bi_local_data(b: *u8, n: i64, fld: *i64) -> i64 {
85 let lo: i64 = fld[NX_ZIP_FLD_LOCALOFF]
86 let cd: i64 = fld[NX_ZIP_FLD_NAMEOFF] - NX_ZIP_CD_LEN
87 if lo < 0 { return 0 - 1 }
88 if lo > n - NX_ZIP_LOCAL_LEN { return 0 - 1 }
89 if cd < 0 { return 0 - 1 }
90 if cd > n - NX_ZIP_CD_LEN { return 0 - 1 }
91 if nxzip_r32(b, lo) != NX_ZIP_SIG_LOCAL { return 0 - 1 }
92 let flags: i64 = nxzip_r16(b, cd + 8)
93 if (flags & 1) != 0 { return 0 - 1 }
94 if nxzip_r16(b, lo + 6) != flags { return 0 - 1 }
95 if nxzip_r16(b, lo + 8) != fld[NX_ZIP_FLD_METHOD] { return 0 - 1 }
96 let nlen: i64 = nxzip_r16(b, lo + 26)
97 let elen: i64 = nxzip_r16(b, lo + 28)
98 if nlen != fld[NX_ZIP_FLD_NAMELEN] { return 0 - 1 }
99 if nlen > n - lo - NX_ZIP_LOCAL_LEN { return 0 - 1 }
100 if nlen > n - fld[NX_ZIP_FLD_NAMEOFF] { return 0 - 1 }
101 var i: i64 = 0
102 while i < nlen {
103 if b[lo + NX_ZIP_LOCAL_LEN + i] != b[fld[NX_ZIP_FLD_NAMEOFF] + i] { return 0 - 1 }
104 i = i + 1
105 }
106 let doff: i64 = lo + NX_ZIP_LOCAL_LEN + nlen + elen
107 if doff > n { return 0 - 1 }
108 let csize: i64 = fld[NX_ZIP_FLD_CSIZE]
109 let usize: i64 = fld[NX_ZIP_FLD_USIZE]
110 if csize < 0 { return 0 - 1 }
111 if usize < 0 { return 0 - 1 }
112 if csize == NX_ZIP_U32 { return 0 - 1 }
113 if usize == NX_ZIP_U32 { return 0 - 1 }
114 if csize > n - doff { return 0 - 1 }
115 if doff + csize > nxzip_cd_start(b,n) { return 0 - 1 }
116 if (flags & 8) == 0 {
117 if nxzip_r32(b,lo+14) != fld[NX_ZIP_FLD_CRC] { return 0 - 1 }
118 if nxzip_r32(b,lo+18) != csize { return 0 - 1 }
119 if nxzip_r32(b,lo+22) != usize { return 0 - 1 }
120 }
121 return doff
122}
123
124// Decode one already validated directory record. Limit is supplied by the caller; unselected entries are never inflated.
125func bi_decode_member(b: *u8, n: i64, fld: *i64, r: *i64, limit: i64) -> i64 {
126 r[BI_F_DATA] = 0
127 r[BI_F_DLEN] = 0
128 r[BI_F_CRC_ACTUAL] = 0 - 1
129 let doff: i64 = bi_local_data(b,n,fld)
130 if doff < 0 { return BI_INF_BADLOCAL }
131 if r[BI_F_USIZE] > limit { return BI_INF_LIMIT }
132 var data: *u8 = b + doff
133 if r[BI_F_METHOD] == BI_METHOD_STORE {
134 if r[BI_F_CSIZE] != r[BI_F_USIZE] { return BI_INF_SIZE_MISMATCH }
135 } else {
136 if r[BI_F_METHOD] != BI_METHOD_DEFLATE { return BI_INF_UNSUPPORTED }
137 var cap: i64 = r[BI_F_USIZE]
138 if cap == 0 { cap = 1 }
139 let dr: *NxDeflateResult = nx_deflate_inflate(data,r[BI_F_CSIZE] as nx_int,cap as nx_int)
140 if (dr as i64) == 0 { return BI_INF_FAILED }
141 if dr.error_code != NX_DEF_OK { return BI_INF_FAILED }
142 if (dr.output_size as i64) != r[BI_F_USIZE] { return BI_INF_SIZE_MISMATCH }
143 if (dr.bytes_consumed as i64) != r[BI_F_CSIZE] { return BI_INF_SIZE_MISMATCH }
144 data = dr.output_data
145 }
146 r[BI_F_CRC_ACTUAL] = nxzip_crc32(data,r[BI_F_USIZE])
147 if r[BI_F_CRC_ACTUAL] != r[BI_F_CRC] { return BI_INF_CRC_MISMATCH }
148 r[BI_F_DATA] = data as i64
149 r[BI_F_DLEN] = r[BI_F_USIZE]
150 if r[BI_F_METHOD] == BI_METHOD_STORE { return BI_INF_STORED }
151 return BI_INF_OK
152}
153
154// the basename of names[off..off+len) (after the last /), written NUL-terminated and lowercased into out
155func bi_basename(names: *u8, off: i64, len: i64, out: *u8, cap: i64) -> i64 {
156 var s: i64 = 0
157 var i: i64 = 0
158 while i < len { if names[off + i] == (47 as u8) { s = i + 1 } i = i + 1 }
159 var w: i64 = 0
160 i = s
161 while i < len { if w < cap - 1 { out[w] = bi_lc((names[off + i] & 0xff) as i64) as u8; w = w + 1 } i = i + 1 }
162 out[w] = 0 as u8
163 return w
164}
165
166// does hay[0..hn) contain needle (case-insensitive, needle already lowercased)? bounded to the member's own bytes
167func bi_contains_ci(hay: *u8, hn: i64, needle: *u8, nn: i64) -> i64 {
168 if nn <= 0 { return 0 }
169 if nn > hn { return 0 }
170 var i: i64 = 0
171 while i + nn <= hn {
172 var k: i64 = 0
173 var ok: i64 = 1
174 while k < nn {
175 if bi_lc((hay[i + k] & 0xff) as i64) != (needle[k] & 0xff) as i64 { ok = 0; k = nn } else { k = k + 1 }
176 }
177 if ok == 1 { return 1 }
178 i = i + 1
179 }
180 return 0
181}
182
183// Walk every central-directory member of the zip at b[0..n). Fills tbl (BI_F_N slots per member, at most maxm rows) and
184// copies names into `names` (a shared arena, cap bytes). Returns the number of rows written, or -1 (no EOCD), -2 (a
185// central-directory record refused), -3 (maxm too small for the declared count -- the caller sized it from bi_count).
186func bi_walk_selected(b: *u8, n: i64, tbl: *i64, names: *u8, ncap: i64, options: *i64) -> i64 {
187 let maxm: i64 = options[0]
188 let selected: i64 = options[1]
189 let limit: i64 = options[2]
190 let total: i64 = nxzip_count(b, n)
191 if total < 0 { return 0 - 1 }
192 if total > maxm { return 0 - 3 }
193 var cdp: i64 = nxzip_cd_start(b, n)
194 let fld: *i64 = sys_mmap(8 * BI_ZIP_FLD_N) as *i64
195 if (fld as i64) <= 0 { return 0 - 5 }
196 if selected >= total { return 0 - 6 }
197 if limit < 0 { return 0 - 7 }
198 var noff: i64 = 0
199 var k: i64 = 0
200 while k < total {
201 if nxzip_read_cd(b, n, cdp, fld) != 1 { return 0 - 2 }
202 let r: *i64 = bi_rec(tbl, k)
203 r[BI_F_METHOD] = fld[NX_ZIP_FLD_METHOD]
204 r[BI_F_CSIZE] = fld[NX_ZIP_FLD_CSIZE]
205 r[BI_F_USIZE] = fld[NX_ZIP_FLD_USIZE]
206 r[BI_F_CRC] = fld[NX_ZIP_FLD_CRC]
207 // name into the arena, NUL-terminated so ap_claim can read it
208 let nl: i64 = fld[NX_ZIP_FLD_NAMELEN]
209 if nl >= ncap - noff { sys_munmap(fld as *u8,8 * BI_ZIP_FLD_N); return 0 - 4 }
210 r[BI_F_NAMEOFF] = noff
211 r[BI_F_NAMELEN] = nl
212 var q: i64 = 0
213 while q < nl { if noff + q < ncap - 1 { names[noff + q] = b[fld[NX_ZIP_FLD_NAMEOFF] + q] } q = q + 1 }
214 if noff + nl < ncap { names[noff + nl] = 0 as u8 }
215 noff = noff + nl + 1
216 r[BI_F_DATA] = 0
217 r[BI_F_DLEN] = 0
218 r[BI_F_CRC_ACTUAL] = 0 - 1
219 r[BI_F_INFLATE] = BI_INF_NOT_READ
220 if selected == (0 - 1) { r[BI_F_INFLATE] = bi_decode_member(b,n,fld,r,limit) }
221 else { if selected == k { r[BI_F_INFLATE] = bi_decode_member(b,n,fld,r,limit) } }
222 // the ruler: content first, the name only as the claim it is checked against
223 r[BI_F_CLAIM] = ap_claim(names + r[BI_F_NAMEOFF])
224 if r[BI_F_DLEN] > 0 {
225 r[BI_F_MAGIC] = ap_identify(r[BI_F_DATA] as *u8, r[BI_F_DLEN])
226 r[BI_F_CAT] = ap_cat(r[BI_F_MAGIC])
227 if r[BI_F_MAGIC] == AP_UNKNOWN { r[BI_F_VERDICT] = BI_V_UNIDENTIFIED }
228 else { if r[BI_F_CLAIM] == AP_UNKNOWN { r[BI_F_VERDICT] = BI_V_UNCLAIMED }
229 else { if r[BI_F_CLAIM] == r[BI_F_MAGIC] { r[BI_F_VERDICT] = BI_V_AGREE } else { r[BI_F_VERDICT] = BI_V_DISAGREE } } }
230 } else {
231 r[BI_F_MAGIC] = AP_UNKNOWN
232 r[BI_F_CAT] = AP_C_UNKNOWN
233 r[BI_F_VERDICT] = BI_V_UNREAD
234 }
235 r[BI_F_BIND] = BI_BIND_NA
236 r[BI_F_BOUND_TO] = 0 - 1
237 cdp = fld[NX_ZIP_FLD_NEXTCD]
238 k = k + 1
239 }
240 sys_munmap(fld as *u8,8 * BI_ZIP_FLD_N)
241 if cdp != nxzip_find_eocd(b,n) { return 0 - 2 }
242 return total
243}
244
245// Legacy full-content walk preserves its six-argument contract. Selected=-2 inventories metadata only.
246func bi_walk(b: *u8, n: i64, tbl: *i64, maxm: i64, names: *u8, ncap: i64) -> i64 {
247 let options: *i64 = sys_mmap(24) as *i64
248 if (options as i64) <= 0 { return 0 - 5 }
249 options[0] = maxm
250 options[1] = 0 - 1
251 options[2] = NX_ZIP_U32 - 1
252 let result: i64 = bi_walk_selected(b,n,tbl,names,ncap,options)
253 sys_munmap(options as *u8,24)
254 return result
255}
256
257// Bind every texture member to the model or material member whose OWN BYTES name it (basename, case-insensitive). A texture
258// nobody references is UNBOUND -- reported, never guessed onto a mesh. Returns the number bound.
259func bi_bind_textures(tbl: *i64, m: i64, names: *u8) -> i64 {
260 let base: *u8 = sys_mmap(BI_NAME_CAP)
261 var bound: i64 = 0
262 var i: i64 = 0
263 while i < m {
264 let t: *i64 = bi_rec(tbl, i)
265 if t[BI_F_CAT] == AP_C_TEXTURE {
266 t[BI_F_BIND] = BI_BIND_UNBOUND
267 let bl: i64 = bi_basename(names, t[BI_F_NAMEOFF], t[BI_F_NAMELEN], base, BI_NAME_CAP)
268 var j: i64 = 0
269 while j < m {
270 let s: *i64 = bi_rec(tbl, j)
271 var owner: i64 = 0
272 if s[BI_F_CAT] == AP_C_MODEL { owner = 1 }
273 if s[BI_F_CAT] == AP_C_MATERIAL { owner = 1 }
274 if owner == 1 { if s[BI_F_DLEN] > 0 { if t[BI_F_BIND] == BI_BIND_UNBOUND {
275 if bi_contains_ci(s[BI_F_DATA] as *u8, s[BI_F_DLEN], base, bl) == 1 { t[BI_F_BIND] = BI_BIND_BOUND; t[BI_F_BOUND_TO] = j; bound = bound + 1 }
276 } } }
277 j = j + 1
278 }
279 }
280 i = i + 1
281 }
282 return bound
283}
284
285// the partition over the rows: categories, inflate outcomes, bindings. Returns the category sum (must equal m).
286func bi_partition(tbl: *i64, m: i64, p: *i64) -> i64 {
287 var i: i64 = 0
288 while i < BI_P_N { p[i] = 0; i = i + 1 }
289 i = 0
290 while i < m {
291 let r: *i64 = bi_rec(tbl, i)
292 let c: i64 = r[BI_F_CAT]
293 if c == AP_C_MODEL { p[BI_P_MODEL] = p[BI_P_MODEL] + 1 }
294 else { if c == AP_C_MATERIAL { p[BI_P_MATERIAL] = p[BI_P_MATERIAL] + 1 }
295 else { if c == AP_C_TEXTURE { p[BI_P_TEXTURE] = p[BI_P_TEXTURE] + 1 }
296 else { if c == AP_C_MOTION { p[BI_P_MOTION] = p[BI_P_MOTION] + 1 }
297 else { if c == AP_C_AUDIO { p[BI_P_AUDIO] = p[BI_P_AUDIO] + 1 }
298 else { if c == AP_C_VIDEO { p[BI_P_VIDEO] = p[BI_P_VIDEO] + 1 }
299 else { if c == AP_C_BUNDLE { p[BI_P_BUNDLE] = p[BI_P_BUNDLE] + 1 }
300 else { p[BI_P_UNKNOWN] = p[BI_P_UNKNOWN] + 1 } } } } } } }
301 let f: i64 = r[BI_F_INFLATE]
302 if f == BI_INF_STORED { p[BI_P_STORED] = p[BI_P_STORED] + 1 }
303 if f == BI_INF_OK { p[BI_P_INFLATED] = p[BI_P_INFLATED] + 1 }
304 if f == BI_INF_FAILED { p[BI_P_FAILED] = p[BI_P_FAILED] + 1 }
305 if f == BI_INF_BADLOCAL { p[BI_P_FAILED] = p[BI_P_FAILED] + 1 }
306 if f >= BI_INF_SIZE_MISMATCH { p[BI_P_FAILED] = p[BI_P_FAILED] + 1 }
307 if f == BI_INF_UNSUPPORTED { p[BI_P_UNSUPPORTED] = p[BI_P_UNSUPPORTED] + 1 }
308 if r[BI_F_BIND] == BI_BIND_BOUND { p[BI_P_BOUND] = p[BI_P_BOUND] + 1 }
309 if r[BI_F_BIND] == BI_BIND_UNBOUND { p[BI_P_UNBOUND] = p[BI_P_UNBOUND] + 1 }
310 i = i + 1
311 }
312 return p[BI_P_MODEL] + p[BI_P_MATERIAL] + p[BI_P_TEXTURE] + p[BI_P_MOTION] + p[BI_P_AUDIO] + p[BI_P_VIDEO] + p[BI_P_BUNDLE] + p[BI_P_UNKNOWN]
313}
314
315func bi_verdict_name(v: i64) -> *u8 {
316 if v == BI_V_AGREE { return "AGREE" as *u8 }
317 if v == BI_V_DISAGREE { return "DISAGREE" as *u8 }
318 if v == BI_V_UNCLAIMED { return "UNCLAIMED" as *u8 }
319 if v == BI_V_UNREAD { return "UNREAD" as *u8 }
320 return "UNIDENTIFIED" as *u8
321}
322func bi_inflate_name(f: i64) -> *u8 {
323 if f == BI_INF_NOT_READ { return "NOT-READ" as *u8 }
324 if f == BI_INF_SIZE_MISMATCH { return "SIZE-MISMATCH" as *u8 }
325 if f == BI_INF_CRC_MISMATCH { return "CRC-MISMATCH" as *u8 }
326 if f == BI_INF_LIMIT { return "OUTPUT-LIMIT" as *u8 }
327 if f == BI_INF_STORED { return "STORED" as *u8 }
328 if f == BI_INF_OK { return "OK" as *u8 }
329 if f == BI_INF_FAILED { return "FAILED" as *u8 }
330 if f == BI_INF_BADLOCAL { return "BAD-LOCAL-HEADER" as *u8 }
331 return "UNSUPPORTED-METHOD" as *u8
332}
333func bi_bind_name(v: i64) -> *u8 {
334 if v == BI_BIND_BOUND { return "BOUND" as *u8 }
335 if v == BI_BIND_UNBOUND { return "UNBOUND" as *u8 }
336 return "-" as *u8
337}