nx_zip_extract1.nx source
↩ module page · 164 lines · 6629 B
1// nx_zip_extract1.nx -- sovereign single-entry ZIP extractor (STORED or DEFLATE).
2// Reads a .zip whose FIRST local file entry is the payload (the shape HF ships for
3// small single-file dataset archives, e.g. whyen-wang/coco_keypoints/data/
4// keypoints_validation.zip -> keypoints_validation.json), inflates it with the
5// shipped sovereign nx_deflate_inflate, and writes the decompressed bytes out.
6//
7// This is the "eat the debt" companion to the main-API dataset fetch: the HF
8// datasets-server (datasets-server.huggingface.co) 503s our client for EVERY route
9// (its own /healthcheck 503s -> HF-side/edge, not a fetcher bug), so dataset CONTENT
10// is reached via the WORKING main API (tree -> LFS resolve -> file) and decompressed
11// here. 100% sovereign: own TLS fetch (upstream) + own DEFLATE (here); the only
12// non-Nishi input is the fetched archive bytes.
13//
14// Config (relative to nxc2 cwd; the build/run launcher does not forward argv):
15// data/mp_zip_in.txt -- path to the .zip on disk
16// data/mp_zip_out.txt -- path to write the decompressed entry
17//
18// nx_safety_envelope:
19// intended_use: decompress a fetched dataset archive (offline, local file)
20// sil_target: SIL1
21// verdict: NOT_YET_EVALUATED
22// genealogy_id: pkware/appnote + ietf/rfc_1951 lineage_id: nx_zip_extract1_v1
23import "nx_syscalls.nx"
24import "nx_deflate.nx"
25const K_MAGIC_4096: i64 = 4096
26const K_MAGIC_1024: i64 = 1024
27const K_MAGIC_65536: i64 = 65536
28
29func ze_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return n }
30func ze_putn(v: i64) -> i64 {
31 let b: *u8 = sys_mmap(28)
32 var m: i64 = v
33 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
34 let t: *u8 = sys_mmap(28)
35 var k: i64 = 0
36 if m == 0 { t[0] = 48 as u8; k = 1 }
37 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
38 var i: i64 = 0
39 while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
40 sys_write(1, b, k)
41 return k
42}
43
44// little-endian readers over a byte buffer (u8 loads zero-extend -> correct for bytes)
45func ze_u16(b: *u8, o: i64) -> i64 { return (b[o] as i64) + ((b[o+1] as i64) << 8) }
46func ze_u32(b: *u8, o: i64) -> i64 {
47 return (b[o] as i64) + ((b[o+1] as i64) << 8) + ((b[o+2] as i64) << 16) + ((b[o+3] as i64) << 24)
48}
49
50// read up to cap bytes of `path` into buf (looping); returns byte count or -1.
51func ze_read_all(path: *u8, buf: *u8, cap: i64) -> i64 {
52 let fd: i64 = sys_openat_rd(path)
53 if fd < 0 { return 0 - 1 }
54 var off: i64 = 0
55 var go: i64 = 1
56 while go == 1 {
57 let n: i64 = sys_read(fd, ((buf as i64) + off) as *u8, cap - off)
58 if n <= 0 { go = 0 } else { off = off + n; if off >= cap { go = 0 } }
59 }
60 sys_close(fd)
61 return off
62}
63
64// write exactly n bytes of buf to `path` (create/trunc, 0644); returns n or negative.
65func ze_write_all(path: *u8, buf: *u8, n: i64) -> i64 {
66 let fd: i64 = sys_openat_wr(path, 0x1a4)
67 if fd < 0 { return 0 - 1 }
68 var off: i64 = 0
69 while off < n {
70 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off)
71 if w <= 0 { sys_close(fd); return 0 - 2 }
72 off = off + w
73 }
74 sys_close(fd)
75 return off
76}
77
78// load first line of a config file (trim at CR/LF/space) into out (NUL-term); returns len.
79func ze_cfg(path: *u8, out: *u8, cap: i64) -> i64 {
80 let fd: i64 = sys_openat_rd(path)
81 if fd < 0 { out[0] = 0 as u8; return 0 - 1 }
82 let tmp: *u8 = sys_mmap(K_MAGIC_4096)
83 let n: i64 = sys_read(fd, tmp, K_MAGIC_4096)
84 sys_close(fd)
85 var k: i64 = 0
86 var go: i64 = 1
87 while go == 1 {
88 if k >= n { go = 0 } else {
89 let c: i64 = tmp[k] as i64
90 if c == 10 { go = 0 } else {
91 if c == 13 { go = 0 } else {
92 if c == 32 { go = 0 } else {
93 out[k] = tmp[k]
94 k = k + 1
95 if k >= cap - 1 { go = 0 }
96 }
97 }
98 }
99 }
100 }
101 out[k] = 0 as u8
102 return k
103}
104
105func main() -> i64 {
106 let inpath: *u8 = sys_mmap(K_MAGIC_1024)
107 let outpath: *u8 = sys_mmap(K_MAGIC_1024)
108 ze_cfg("data/mp_zip_in.txt" as *u8, inpath, K_MAGIC_1024)
109 ze_cfg("data/mp_zip_out.txt" as *u8, outpath, K_MAGIC_1024)
110 ze_puts("in=" as *u8); ze_puts(inpath); ze_puts("\n" as *u8)
111 ze_puts("out=" as *u8); ze_puts(outpath); ze_puts("\n" as *u8)
112
113 let cap: i64 = 8 * K_MAGIC_1024 * K_MAGIC_1024
114 let zip: *u8 = sys_mmap(cap)
115 let zn: i64 = ze_read_all(inpath, zip, cap)
116 ze_puts("zip_bytes=" as *u8); ze_putn(zn); ze_puts("\n" as *u8)
117 if zn < 30 { ze_puts("ERR short/no-file\n" as *u8); sys_exit(1) }
118
119 // local file header: sig 0x04034b50 (PK\x03\x04)
120 if zip[0] != (0x50 as u8) { ze_puts("ERR not-zip\n" as *u8); sys_exit(1) }
121 if zip[1] != (0x4b as u8) { ze_puts("ERR not-zip\n" as *u8); sys_exit(1) }
122 let method: i64 = ze_u16(zip, 8)
123 let csize: i64 = ze_u32(zip, 18)
124 let usize: i64 = ze_u32(zip, 22)
125 let namelen: i64 = ze_u16(zip, 26)
126 let extralen:i64 = ze_u16(zip, 28)
127 let dataoff: i64 = 30 + namelen + extralen
128 ze_puts("method=" as *u8); ze_putn(method)
129 ze_puts(" csize=" as *u8); ze_putn(csize)
130 ze_puts(" usize=" as *u8); ze_putn(usize)
131 ze_puts(" name=" as *u8)
132 var i: i64 = 0
133 while i < namelen { sys_write(1, ((zip as i64) + 30 + i) as *u8, 1); i = i + 1 }
134 ze_puts("\n" as *u8)
135
136 var outn: i64 = 0
137 var obuf: *u8 = 0 as *u8
138 if method == 0 {
139 outn = csize
140 obuf = ((zip as i64) + dataoff) as *u8
141 ze_puts("STORED (no compression)\n" as *u8)
142 } else {
143 let res: *NxDeflateResult = nx_deflate_inflate(((zip as i64) + dataoff) as *u8, csize, usize + K_MAGIC_65536)
144 if (res as i64) == 0 { ze_puts("ERR inflate-null\n" as *u8); sys_exit(1) }
145 if res.error_code != NX_DEF_OK { ze_puts("ERR inflate-code=" as *u8); ze_putn(res.error_code); ze_puts("\n" as *u8); sys_exit(1) }
146 outn = res.output_size
147 obuf = res.output_data
148 }
149 ze_puts("inflated_bytes=" as *u8); ze_putn(outn); ze_puts("\n" as *u8)
150
151 let w: i64 = ze_write_all(outpath, obuf, outn)
152 ze_puts("wrote=" as *u8); ze_putn(w); ze_puts(" -> " as *u8); ze_puts(outpath); ze_puts("\n" as *u8)
153
154 ze_puts("--- head(240) ---\n" as *u8)
155 var j: i64 = 0
156 var stop: i64 = 240
157 if outn < stop { stop = outn }
158 while j < stop { sys_write(1, ((obuf as i64) + j) as *u8, 1); j = j + 1 }
159 ze_puts("\n--- end head ---\n" as *u8)
160
161 if w == outn { ze_puts("ZIP-EXTRACT OK\n" as *u8); sys_exit(0) }
162 sys_exit(1)
163 return 0
164}