nx_packmirror_lib.nx source
↩ module page · 259 lines · 12690 B
1// nx_packmirror_lib.nx -- PACK-MANIFEST PARSE + VERIFY CORE (debt 1787064637; operator 2026-08-18:
2// media archiving by hand-driven 2-step fetches cannot scale and leaves per-file verification
3// unproven). Dialect v1: the pCloud showpublink JSON (the mining pattern's banked manifest).
4// STRUCTURAL FACTS THIS PARSER RESTS ON (read from the real manifest, stated so a format drift is
5// diagnosable): a FILE entry carries "fileid" (folders carry "folderid", which does not contain the
6// quoted needle); within an entry the observed field order is "name" ... "fileid" ... "size", and the
7// next entry's fields follow AFTER this entry's size -- so name = the LAST "name" before the fileid,
8// size = the FIRST "size" after it, both bounded by the neighbouring fileid positions.
9// FAIL-CLOSED: an entry missing either field counts UNPARSED, never a guessed row. The dest-name
10// sanitizer maps every byte outside [A-Za-z0-9._-] to '_' -- a manifest name can therefore never
11// escape the destination directory (path traversal is structurally impossible, not filtered).
12// license_tier: ORIGINAL No hw writes (Rule 26).
13import "nx_syscalls.nx"
14
15const PM_MAX_FILES: i64 = 4096 // a pack beyond this REFUSES loudly (partial-manifest announce), never truncates silently
16const PM_NAME_CAP: i64 = 512 // one sanitized dest name
17const PM_UNPARSED_SLOT: i64 = 0 // counts[] slot: entries missing a required field
18
19func pm_b(buf: *u8, o: i64) -> i64 { return buf[o] as i64 & 0xff }
20func pm_findn(buf: *u8, start: i64, end: i64, needle: *u8) -> i64 {
21 var m: i64 = 0
22 while needle[m] != (0 as u8) { m = m + 1 }
23 var i: i64 = start
24 while i + m <= end {
25 var k: i64 = 0
26 while k < m { if (buf[i+k] as i64) != (needle[k] as i64) { k = m + 9 } else { k = k + 1 } }
27 if k == m { return i }
28 i = i + 1
29 }
30 return 0 - 1
31}
32// last occurrence of needle in [start,limit) -- the backward half of the entry-order fact.
33func pm_rfindn(buf: *u8, start: i64, limit: i64, needle: *u8) -> i64 {
34 var best: i64 = 0 - 1
35 var at: i64 = pm_findn(buf, start, limit, needle)
36 while at >= 0 {
37 best = at
38 at = pm_findn(buf, at + 1, limit, needle)
39 }
40 return best
41}
42// integer after a key position (skips spaces); -1 when no digits follow.
43func pm_int_after(buf: *u8, at: i64, end: i64) -> i64 {
44 var p: i64 = at
45 while p < end { if pm_b(buf,p) == 32 { p = p + 1 } else { break } }
46 var v: i64 = 0 - 1
47 while p < end {
48 let c: i64 = pm_b(buf, p)
49 if c >= 48 { if c <= 57 { if v < 0 { v = 0 } v = v * 10 + (c - 48); p = p + 1 } else { break } } else { break }
50 }
51 return v
52}
53// dest-safe name copy: bytes outside [A-Za-z0-9._-] become '_'; returns length (0 = unusable).
54func pm_sanitize(src: *u8, soff: i64, send: i64, dst: *u8, dcap: i64) -> i64 {
55 var o: i64 = 0
56 var p: i64 = soff
57 while p < send {
58 if o >= dcap - 1 { break }
59 var c: i64 = pm_b(src, p)
60 var ok: i64 = 0
61 if c >= 48 { if c <= 57 { ok = 1 } }
62 if c >= 65 { if c <= 90 { ok = 1 } }
63 if c >= 97 { if c <= 122 { ok = 1 } }
64 if c == 46 { ok = 1 }
65 if c == 95 { ok = 1 }
66 if c == 45 { ok = 1 }
67 if ok == 0 { c = 95 }
68 dst[o] = c as u8
69 o = o + 1
70 p = p + 1
71 }
72 dst[o] = 0 as u8
73 return o
74}
75// parse the manifest: per FILE entry emit fileid, declared size, sanitized name (PM_NAME_CAP slot each).
76// returns file count; counts[PM_UNPARSED_SLOT] = entries with a fileid but a missing name/size.
77// A count that hits maxn is a REFUSAL CONDITION for the caller (announce, never a silent floor).
78func pm_parse_manifest(buf: *u8, n: i64, ids: *i64, sizes: *i64, names: *u8, maxn: i64, counts: *i64) -> i64 {
79 counts[PM_UNPARSED_SLOT] = 0
80 var count: i64 = 0
81 var fid_at: i64 = pm_findn(buf, 0, n, "\"fileid\":" as *u8)
82 while fid_at >= 0 {
83 if count >= maxn { return count }
84 let next_at: i64 = pm_findn(buf, fid_at + 1, n, "\"fileid\":" as *u8)
85 var bound: i64 = n
86 if next_at >= 0 { bound = next_at }
87 let fid: i64 = pm_int_after(buf, fid_at + 9, n)
88 var nm_ok: i64 = 0
89 let nm_at: i64 = pm_rfindn(buf, 0, fid_at, "\"name\":" as *u8)
90 var nm_s: i64 = 0 - 1
91 var nm_e: i64 = 0 - 1
92 if nm_at >= 0 {
93 var q: i64 = nm_at + 7
94 while q < fid_at { if pm_b(buf,q) == 32 { q = q + 1 } else { break } }
95 if q < fid_at { if pm_b(buf,q) == 34 {
96 nm_s = q + 1
97 var e: i64 = nm_s
98 while e < fid_at { if pm_b(buf,e) == 34 { break } e = e + 1 }
99 if e < fid_at { nm_e = e; nm_ok = 1 }
100 } }
101 }
102 var sz: i64 = 0 - 1
103 let sz_at: i64 = pm_findn(buf, fid_at, bound, "\"size\":" as *u8)
104 if sz_at >= 0 { sz = pm_int_after(buf, sz_at + 7, bound) }
105 if fid >= 0 { if nm_ok == 1 { if sz >= 0 {
106 ids[count] = fid
107 sizes[count] = sz
108 pm_sanitize(buf, nm_s, nm_e, ((names as i64) + count * PM_NAME_CAP) as *u8, PM_NAME_CAP)
109 count = count + 1
110 } else { counts[PM_UNPARSED_SLOT] = counts[PM_UNPARSED_SLOT] + 1 }
111 } else { counts[PM_UNPARSED_SLOT] = counts[PM_UNPARSED_SLOT] + 1 } }
112 fid_at = next_at
113 }
114 return count
115}
116// ---- DIALECT 2: a 4chan thread JSON (a.4cdn.org/<board>/thread/<no>.json), the /ldg/ mining shape.
117// STRUCTURAL FACTS (read from the banked thread bytes, stated so drift is diagnosable): a post that
118// carries media has, in this order, "ext":".webm" ... "tim":<i64> ... "fsize":<i64>; posts without
119// media have none of the three. The nx_4chan adapter names the archived file <tim><ext>. So one entry
120// = one "tim": occurrence; ext = the LAST "ext": before it (bounded by the previous tim), fsize = the
121// FIRST "fsize": after it (bounded by the next tim). Missing ext or fsize -> UNPARSED, never guessed.
122// Emits: sizes[i] = fsize, names[i] = "<tim><ext>", ids[i] = tim (a stable id for receipts).
123func pm_parse_thread(buf: *u8, n: i64, ids: *i64, sizes: *i64, names: *u8, maxn: i64, counts: *i64) -> i64 {
124 counts[PM_UNPARSED_SLOT] = 0
125 var count: i64 = 0
126 var prev_tim: i64 = 0
127 var tim_at: i64 = pm_findn(buf, 0, n, "\"tim\":" as *u8)
128 while tim_at >= 0 {
129 if count >= maxn { return count }
130 let next_at: i64 = pm_findn(buf, tim_at + 1, n, "\"tim\":" as *u8)
131 var bound: i64 = n
132 if next_at >= 0 { bound = next_at }
133 let tim: i64 = pm_int_after(buf, tim_at + 6, n)
134 // ext: last "ext":"..." between the previous tim and this one
135 var ext_ok: i64 = 0
136 var ext_s: i64 = 0 - 1
137 var ext_e: i64 = 0 - 1
138 let ext_at: i64 = pm_rfindn(buf, prev_tim, tim_at, "\"ext\":" as *u8)
139 if ext_at >= 0 {
140 var q: i64 = ext_at + 6
141 if q < tim_at { if pm_b(buf,q) == 34 {
142 ext_s = q + 1
143 var e: i64 = ext_s
144 while e < tim_at { if pm_b(buf,e) == 34 { break } e = e + 1 }
145 if e < tim_at { ext_e = e; ext_ok = 1 }
146 } }
147 }
148 var fs: i64 = 0 - 1
149 let fs_at: i64 = pm_findn(buf, tim_at, bound, "\"fsize\":" as *u8)
150 if fs_at >= 0 { fs = pm_int_after(buf, fs_at + 8, bound) }
151 if tim >= 0 { if ext_ok == 1 { if fs >= 0 {
152 ids[count] = tim
153 sizes[count] = fs
154 let dst: *u8 = ((names as i64) + count * PM_NAME_CAP) as *u8
155 // name = decimal tim + ext, both already dest-safe by construction (digits, dot, letters)
156 var o: i64 = 0
157 let t: *u8 = sys_mmap(32)
158 var x: i64 = tim
159 var k: i64 = 0
160 if x == 0 { t[0] = 48 as u8; k = 1 }
161 while x > 0 { t[k] = (48 + x % 10) as u8; x = x / 10; k = k + 1 }
162 var i: i64 = 0
163 while i < k { dst[o] = t[k-1-i]; o = o + 1; i = i + 1 }
164 var p: i64 = ext_s
165 while p < ext_e { if o < PM_NAME_CAP - 1 { dst[o] = buf[p]; o = o + 1 } p = p + 1 }
166 dst[o] = 0 as u8
167 count = count + 1
168 } else { counts[PM_UNPARSED_SLOT] = counts[PM_UNPARSED_SLOT] + 1 }
169 } else { counts[PM_UNPARSED_SLOT] = counts[PM_UNPARSED_SLOT] + 1 } }
170 prev_tim = tim_at
171 tim_at = next_at
172 }
173 return count
174}
175// existing dest at the exact declared size? 1 = yes (skip), 0 = no.
176func pm_dest_ok(path: *u8, declared: i64) -> i64 {
177 let fd: i64 = sys_openat_rd(path)
178 if fd < 0 { return 0 }
179 let sz: i64 = sys_lseek(fd, 0, 2)
180 sys_close(fd)
181 if sz == declared { return 1 }
182 return 0
183}
184
185// ---- BEPIS DIALECT (2026-08-18, watch contract pm_bepis on /compare/koikatsu; SSOT recipe
186// knowledge/library/bepis/INGEST_RECIPE.md; grounding fixture AI_248895.meta.json, pin-verified).
187// One BepisDB metadata response: {"type":"success","data":{"cardType":"AI","id":248895,...,
188// "fileSize":1403340,"sha256CardHash":"<base64 of the raw 32-byte digest>",...}}
189// STRUCTURAL FACTS (read from the real response, stated so a format drift is a loud parse failure):
190// "cardType" occurs ONCE per response at the data level, and the card's own "id" is the FIRST
191// "id": after it (the uploader's id comes later in the object); fileSize and sha256CardHash sit in
192// the same data object. A manifest file = one or more responses concatenated (the miner appends one
193// per card). Dest name mirrors the MEASURED file-URL shape: <cardType>_<id zero-padded to 6>.png.
194// hashes: PM_HASH_CAP stride per entry, the base64 EXACTLY as served -- decoding is the verifier's
195// job, so the parser stays a parser.
196const PM_HASH_CAP: i64 = 48 // base64 of a 32-byte digest = 44 chars + NUL, with margin
197const PM_ID_PAD: i64 = 6 // BepisDB zero-pads card ids to 6 digits in file URLs (measured)
198const PM_CT_MAX: i64 = 16 // longest cardType code observed is KKCLOTHING (10); 16 refuses garbage
199func pm_parse_bepis(buf: *u8, n: i64, ids: *i64, sizes: *i64, names: *u8, maxn: i64, counts: *i64, hashes: *u8) -> i64 {
200 counts[PM_UNPARSED_SLOT] = 0
201 var count: i64 = 0
202 var ct_at: i64 = pm_findn(buf, 0, n, "\"cardType\":\"" as *u8)
203 while ct_at >= 0 {
204 if count >= maxn { return count }
205 let next_at: i64 = pm_findn(buf, ct_at + 1, n, "\"cardType\":\"" as *u8)
206 var bound: i64 = n
207 if next_at >= 0 { bound = next_at }
208 let cs: i64 = ct_at + 12
209 var ce: i64 = cs
210 while ce < bound { if pm_b(buf,ce) == 34 { break } ce = ce + 1 }
211 var idv: i64 = 0 - 1
212 let id_at: i64 = pm_findn(buf, ce, bound, "\"id\":" as *u8)
213 if id_at >= 0 { idv = pm_int_after(buf, id_at + 5, bound) }
214 var fs: i64 = 0 - 1
215 let fs_at: i64 = pm_findn(buf, ce, bound, "\"fileSize\":" as *u8)
216 if fs_at >= 0 { fs = pm_int_after(buf, fs_at + 11, bound) }
217 var hs: i64 = 0 - 1
218 var he: i64 = 0 - 1
219 let h_at: i64 = pm_findn(buf, ce, bound, "\"sha256CardHash\":\"" as *u8)
220 if h_at >= 0 {
221 hs = h_at + 18
222 he = hs
223 while he < bound { if pm_b(buf,he) == 34 { break } he = he + 1 }
224 if he >= bound { hs = 0 - 1 }
225 }
226 var okrow: i64 = 0
227 if idv >= 0 { if fs >= 0 { if hs >= 0 { if ce > cs { if ce - cs < PM_CT_MAX { okrow = 1 } } } } }
228 if okrow == 1 {
229 ids[count] = idv
230 sizes[count] = fs
231 let dst: *u8 = ((names as i64) + count * PM_NAME_CAP) as *u8
232 var o: i64 = 0
233 var q: i64 = cs
234 while q < ce { dst[o] = buf[q]; o = o + 1; q = q + 1 }
235 dst[o] = 95 as u8; o = o + 1
236 let t: *u8 = sys_mmap(32)
237 var x: i64 = idv
238 var k: i64 = 0
239 if x == 0 { t[0] = 48 as u8; k = 1 }
240 while x > 0 { t[k] = (48 + x % 10) as u8; x = x / 10; k = k + 1 }
241 var padk: i64 = k
242 while padk < PM_ID_PAD { dst[o] = 48 as u8; o = o + 1; padk = padk + 1 }
243 var i2: i64 = 0
244 while i2 < k { dst[o] = t[k-1-i2]; o = o + 1; i2 = i2 + 1 }
245 let ext: *u8 = ".png" as *u8
246 var e2: i64 = 0
247 while ext[e2] != (0 as u8) { dst[o] = ext[e2]; o = o + 1; e2 = e2 + 1 }
248 dst[o] = 0 as u8
249 let hp: *u8 = ((hashes as i64) + count * PM_HASH_CAP) as *u8
250 var ho: i64 = 0
251 q = hs
252 while q < he { if ho < PM_HASH_CAP - 1 { hp[ho] = buf[q]; ho = ho + 1 } q = q + 1 }
253 hp[ho] = 0 as u8
254 count = count + 1
255 } else { counts[PM_UNPARSED_SLOT] = counts[PM_UNPARSED_SLOT] + 1 }
256 ct_at = next_at
257 }
258 return count
259}