nx_entity_media.nx source
↩ module page · 481 lines · 20030 B
1// nx_entity_media.nx -- MULTI-IMAGE entity media gather.
2//
3// Supersedes the 1-image demo in nx_media_gather.nx, which fetched exactly ONE image and only if its URL
4// contained a hardcoded "upload.wikimedia.org". Here: fetch an entity page over sovereign validated HTTPS,
5// extract ALL image URLs, fetch each (polite pacing), sniff + sha256 + content-address into
6// knowledge/media/<entity>/, dedupe by digest within the run, and append a provenance row per stored asset.
7//
8// No host allowlist: any http(s) image URL is eligible. Bounds are PARAMETERS, not literals -- the intended
9// limits are hardware and politeness, nothing else. A hit cap is REPORTED (capped=1), never silent.
10//
11// usage: nx_entity_media <page-url> <entity-slug> [max_imgs] [pace_ms]
12
13import "nx_str.nx"
14import "nx_syscalls.nx"
15import "nx_csprng.nx"
16import "nx_x509_trust_store.nx"
17import "nx_pem_loader.nx"
18import "nx_https_get.nx"
19import "nx_html_extract_imgs.nx"
20import "nx_media_sniff.nx"
21import "nx_sha256.nx"
22import "nx_inflate.nx"
23import "nx_http_dechunk.nx"
24const EM_MAGIC_2048: i64 = 2048
25
26const EM_PAGE_CAP: i64 = 4194304
27const EM_IMG_CAP: i64 = 8388608
28// Bounds sized so HARDWARE is the limit, not a literal. 4096 urls costs 32KB+32KB of offset/length
29// arrays, 128KB of dedupe digests and a 2MB url arena -- irrelevant against the 8MB image buffer.
30const EM_URLBUF: i64 = 2097152
31const EM_MAX_IMGS: i64 = 4096
32const EM_PATHBUF: i64 = 4096
33const EM_DEF_PACE_MS: i64 = 800
34
35func em_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
36func em_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 }
37
38func em_num(v: i64, o: *u8) -> i64 {
39 let t: *u8 = sys_mmap(32)
40 var m: i64 = v
41 var k: i64 = 0
42 if m < 0 { m = 0 - m }
43 if m == 0 { t[0] = 48; k = 1 }
44 while m > 0 { t[k] = 48 + (m % 10); m = m / 10; k = k + 1 }
45 var i: i64 = 0
46 while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
47 o[k] = 0 as u8
48 return k
49}
50
51func em_pi(v: i64) -> i64 { let o: *u8 = sys_mmap(32); let n: i64 = em_num(v, o); sys_write(1, o, n); return 0 }
52func em_wn(fd: i64, v: i64) -> i64 { let o: *u8 = sys_mmap(32); let n: i64 = em_num(v, o); sys_write(fd, o, n); return 0 }
53
54func em_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[off+i] = s[i]; i = i + 1 } return off + i }
55
56func em_mkdir(path: *u8) -> i64 { return __syscall(258, -100, path, 0x1ed, 0, 0, 0) }
57
58func em_atoi(s: *u8) -> i64 {
59 var i: i64 = 0
60 var v: i64 = 0
61 while s[i] != (0 as u8) {
62 let c: i64 = s[i] as i64
63 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
64 i = i + 1
65 }
66 return v
67}
68
69func em_hex(d: *u8, out: *u8) -> i64 {
70 let hx: *u8 = "0123456789abcdef" as *u8
71 var i: i64 = 0
72 while i < 32 {
73 let b: i64 = d[i] as i64
74 out[i*2] = hx[(b >> 4) & 15]
75 out[i*2+1] = hx[b & 15]
76 i = i + 1
77 }
78 out[64] = 0 as u8
79 return 64
80}
81
82func em_body_off(resp: *u8, n: i64) -> i64 {
83 var i: i64 = 0
84 while i + 3 < n {
85 if resp[i] == (13 as u8) { if resp[i+1] == (10 as u8) { if resp[i+2] == (13 as u8) { if resp[i+3] == (10 as u8) { return i + 4 } } } }
86 i = i + 1
87 }
88 return 0
89}
90
91func em_write_file(path: *u8, buf: *u8, n: i64) -> i64 {
92 let fd: i64 = sys_openat_wr(path, 0x1a4)
93 if fd < 0 { return 0 - 1 }
94 var off: i64 = 0
95 while off < n {
96 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off)
97 if w <= 0 { sys_close(fd); return 0 - 1 }
98 off = off + w
99 }
100 sys_close(fd)
101 return n
102}
103
104// http(s) scheme only -- rejects data: URIs, protocol-relative and relative leftovers
105func em_is_http(u: *u8, n: i64) -> i64 {
106 if n < 8 { return 0 }
107 if u[0] != (104 as u8) { return 0 }
108 if u[1] != (116 as u8) { return 0 }
109 if u[2] != (116 as u8) { return 0 }
110 if u[3] != (112 as u8) { return 0 }
111 return 1
112}
113
114// has this exact content already been stored in THIS run?
115func em_seen(seen: *u8, nseen: i64, dg: *u8) -> i64 {
116 var i: i64 = 0
117 while i < nseen {
118 var same: i64 = 1
119 var b: i64 = 0
120 while b < 32 { if seen[i*32+b] != dg[b] { same = 0 } b = b + 1 }
121 if same == 1 { return 1 }
122 i = i + 1
123 }
124 return 0
125}
126
127func em_ledger(lg: i64, entity: *u8, page: *u8, iurl: *u8, n: i64, hex: *u8, kind: *u8) -> i64 {
128 if lg < 0 { return 0 }
129 em_w(lg, entity); em_w(lg, "\t" as *u8)
130 em_w(lg, page); em_w(lg, "\t" as *u8)
131 em_w(lg, iurl); em_w(lg, "\t" as *u8)
132 em_wn(lg, n); em_w(lg, "\t" as *u8)
133 em_w(lg, hex); em_w(lg, "\t" as *u8)
134 em_w(lg, kind); em_w(lg, "\tVERIFIED\n" as *u8)
135 return 0
136}
137
138func em_doss(df: i64, entity: *u8, iurl: *u8, hex: *u8, kind: *u8, n: i64) -> i64 {
139 if df < 0 { return 0 }
140 em_w(df, "media asset entity=" as *u8); em_w(df, entity)
141 em_w(df, " image=" as *u8); em_w(df, iurl)
142 em_w(df, " sha256=" as *u8); em_w(df, hex)
143 em_w(df, " kind=" as *u8); em_w(df, kind)
144 em_w(df, " bytes=" as *u8); em_wn(df, n)
145 em_w(df, " transport=sovereign-validated-tls13 status=verified\n" as *u8)
146 return 0
147}
148
149// FULL-SIZE RESOLUTION. We gather <img src> AS RENDERED, and rendered means the THUMBNAIL the page chose
150// to display. Measured on a real corpus: 19 of 19 assets were under 400px, average longest side 79px --
151// the index was being built on postage stamps. Both major sources expose the original by a deterministic
152// URL transform:
153// wikimedia /commons/thumb/5/5d/X.jpg/250px-X.jpg -> /commons/5/5d/X.jpg
154// (drop the "/thumb" segment, then drop the trailing rendered-size component)
155// javdatabase /idolimages/thumb/x.webp -> /idolimages/full/x.webp
156// Returns 1 and fills `out` when a transform applied, else 0. The caller TRIES THE ORIGINAL FIRST and
157// FALLS BACK to the thumbnail, so a source that does not honour the transform still yields an asset.
158// does url[0..n) end in a raster extension we actually decode?
159func em_ends(u: *u8, n: i64, s: *u8) -> i64 {
160 let sl: i64 = nx_str_len(s)
161 if sl > n { return 0 }
162 var i: i64 = 0
163 while i < sl { if u[n - sl + i] != s[i] { return 0 } i = i + 1 }
164 return 1
165}
166
167func em_raster_ext(u: *u8, n: i64) -> i64 {
168 if em_ends(u, n, ".jpg" as *u8) == 1 { return 1 }
169 if em_ends(u, n, ".jpeg" as *u8) == 1 { return 1 }
170 if em_ends(u, n, ".png" as *u8) == 1 { return 1 }
171 if em_ends(u, n, ".webp" as *u8) == 1 { return 1 }
172 if em_ends(u, n, ".gif" as *u8) == 1 { return 1 }
173 if em_ends(u, n, ".JPG" as *u8) == 1 { return 1 }
174 if em_ends(u, n, ".JPEG" as *u8) == 1 { return 1 }
175 if em_ends(u, n, ".PNG" as *u8) == 1 { return 1 }
176 return 0
177}
178
179func em_upsize(u: *u8, n: i64, out: *u8) -> i64 {
180 // idolimages/thumb/ -> idolimages/full/
181 let a: *u8 = "/idolimages/thumb/" as *u8
182 let al: i64 = nx_str_len(a)
183 var i: i64 = 0
184 while i + al <= n {
185 var hit: i64 = 1
186 var j: i64 = 0
187 while j < al { if u[i+j] != a[j] { hit = 0; j = al } else { j = j + 1 } }
188 if hit == 1 {
189 var o: i64 = 0
190 while o < i { out[o] = u[o]; o = o + 1 }
191 o = em_cat(out, o, "/idolimages/full/" as *u8)
192 var k: i64 = i + al
193 while k < n { out[o] = u[k]; o = o + 1; k = k + 1 }
194 out[o] = 0 as u8
195 return 1
196 }
197 i = i + 1
198 }
199
200 // /thumb/ -> drop the segment, then drop the trailing rendered-size component
201 let b: *u8 = "/thumb/" as *u8
202 let bl: i64 = nx_str_len(b)
203 var t: i64 = 0 - 1
204 var p: i64 = 0
205 while p + bl <= n {
206 if t < 0 {
207 var hit2: i64 = 1
208 var j2: i64 = 0
209 while j2 < bl { if u[p+j2] != b[j2] { hit2 = 0; j2 = bl } else { j2 = j2 + 1 } }
210 if hit2 == 1 { t = p }
211 }
212 p = p + 1
213 }
214 if t < 0 { return 0 }
215
216 var o2: i64 = 0
217 var c2: i64 = 0
218 while c2 < t { out[o2] = u[c2]; o2 = o2 + 1; c2 = c2 + 1 }
219 var k2: i64 = t + 6 // skip "/thumb", keep the following "/"
220 while k2 < n { out[o2] = u[k2]; o2 = o2 + 1; k2 = k2 + 1 }
221 // truncate at the LAST '/' -- that removes the "250px-Name.jpg" rendered component
222 var last: i64 = 0 - 1
223 var q: i64 = 0
224 while q < o2 { if out[q] == (47 as u8) { last = q } q = q + 1 }
225 if last <= 0 { return 0 }
226 out[last] = 0 as u8
227 // ★A SUCCESSFUL FETCH OF THE WRONG THING DEFEATS A FALLBACK KEYED ON FETCH FAILURE.
228 // A Commons thumbnail is often a PNG *rendering* of a non-raster original (.svg, .pdf, .ogv, .tif).
229 // Upsizing those returns the SOURCE -- a 200 OK we cannot decode -- so the fetch-failure fallback never
230 // fires and we silently lose the one usable form. Only upsize when the ORIGINAL is itself a raster we
231 // decode; otherwise keep the rendered thumbnail, which is the decodable artifact.
232 if em_raster_ext(out, last) == 0 { return 0 }
233 return 1
234}
235
236// Content-Encoding: gzip. The sovereign HTTPS client requests gzip and returns the body still
237// COMPRESSED; handing those bytes to an HTML extractor yields a SILENT zero-images result rather
238// than an error. inf_gunzip has existed in nx_inflate.nx all along -- this is the missing compose.
239// Returns the inflated length written to dst, or 0 when src is not a gzip stream.
240func em_inflate_if_gzip(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 {
241 if n < 3 { return 0 }
242 if src[0] != (31 as u8) { return 0 }
243 if src[1] != (139 as u8) { return 0 }
244 let dn: i64 = inf_gunzip(src, n, dst, cap)
245 if dn <= 0 { return 0 }
246 return dn
247}
248
249func main(argc: i64, argv: *i64) -> i64 {
250 var url: *u8 = "https://en.wikipedia.org/wiki/Diora_Baird" as *u8
251 var entity: *u8 = "diora_baird" as *u8
252 var maxn: i64 = EM_MAX_IMGS
253 var pace: i64 = EM_DEF_PACE_MS
254 if argc >= 2 { url = argv[1] as *u8 }
255 if argc >= 3 { entity = argv[2] as *u8 }
256 if argc >= 4 { maxn = em_atoi(argv[3] as *u8) }
257 if argc >= 5 { pace = em_atoi(argv[4] as *u8) }
258 if maxn <= 0 { maxn = EM_MAX_IMGS }
259 if maxn > EM_MAX_IMGS { maxn = EM_MAX_IMGS }
260 if pace < 0 { pace = 0 }
261
262 em_puts("=== ENTITY MEDIA GATHER (multi-image, sovereign TLS) ===\n entity=" as *u8)
263 em_puts(entity); em_puts(" page=" as *u8); em_puts(url)
264 em_puts(" max=" as *u8); em_pi(maxn); em_puts(" pace_ms=" as *u8); em_pi(pace); em_puts("\n" as *u8)
265
266 let store: *TrustStore = trust_store_alloc(400)
267 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store)
268 if nroots <= 0 { em_puts("ENTITY-MEDIA-FAIL no-roots\n" as *u8); sys_exit(1); return 1 }
269
270 let cr: *u8 = sys_mmap(32)
271 let pk: *u8 = sys_mmap(32)
272 nx_csprng_fill(cr, 32)
273 nx_csprng_fill(pk, 32)
274 let page: *u8 = sys_mmap(EM_PAGE_CAP)
275 let pr: i64 = nx_https_get(url, cr, pk, store, sys_now_realtime_sec(), page, EM_PAGE_CAP)
276 em_puts(" page fetch bytes=" as *u8); em_pi(pr); em_puts("\n" as *u8)
277 if pr <= 0 { em_puts("ENTITY-MEDIA-FAIL page-fetch\n" as *u8); sys_exit(1); return 1 }
278
279 let bo: i64 = em_body_off(page, pr)
280 let body: *u8 = ((page as i64) + bo) as *u8
281 let blen: i64 = pr - bo
282
283 // WIRE ORDER MATTERS: Transfer-Encoding framing is the OUTER layer, Content-Encoding the inner.
284 // A chunked+gzip response begins with a hex chunk-size line, so the gzip magic is NOT at body[0]
285 // and a naive gzip probe reports 'identity' and hands chunk-framed binary to the extractor.
286 // nx_http_dechunk returns negative on non-chunked input, so attempt-then-fall-back is safe.
287 var raw: *u8 = body
288 var rawlen: i64 = blen
289 let dechunk: *u8 = sys_mmap(EM_PAGE_CAP)
290 let cn: i64 = nx_http_dechunk(body, blen, dechunk, EM_PAGE_CAP)
291 if cn > 0 { raw = dechunk; rawlen = cn }
292
293 var html: *u8 = raw
294 var hlen: i64 = rawlen
295 let infbuf: *u8 = sys_mmap(EM_PAGE_CAP)
296 let pdn: i64 = em_inflate_if_gzip(raw, rawlen, infbuf, EM_PAGE_CAP)
297 if pdn > 0 { html = infbuf; hlen = pdn }
298
299 em_puts(" transfer=" as *u8)
300 if cn > 0 { em_puts("chunked dechunked=" as *u8); em_pi(cn) } else { em_puts("identity" as *u8) }
301 em_puts(" encoding=" as *u8)
302 if pdn > 0 { em_puts("gzip inflated=" as *u8); em_pi(pdn) } else { em_puts("identity" as *u8) }
303 em_puts(" html_bytes=" as *u8); em_pi(hlen); em_puts("\n" as *u8)
304
305 let url_buf: *u8 = sys_mmap(EM_URLBUF)
306 let offs: *i64 = sys_mmap(8 * EM_MAX_IMGS) as *i64
307 let lens: *i64 = sys_mmap(8 * EM_MAX_IMGS) as *i64
308 let nimgs: i64 = nx_html_extract_imgs(html, hlen, url, nx_str_len(url), url_buf, EM_URLBUF, offs, lens, EM_MAX_IMGS)
309 em_puts(" image urls extracted=" as *u8); em_pi(nimgs); em_puts("\n" as *u8)
310 if nimgs <= 0 { em_puts("ENTITY-MEDIA-FAIL no-images\n" as *u8); sys_exit(1); return 1 }
311
312 em_mkdir("knowledge/media" as *u8)
313 em_mkdir("knowledge/library" as *u8)
314 let dir: *u8 = sys_mmap(EM_PATHBUF)
315 var dof: i64 = em_cat(dir, 0, "knowledge/media/" as *u8)
316 dof = em_cat(dir, dof, entity)
317 dir[dof] = 0 as u8
318 em_mkdir(dir)
319
320 let dpath: *u8 = sys_mmap(EM_PATHBUF)
321 var dpo: i64 = em_cat(dpath, 0, "knowledge/library/media-" as *u8)
322 dpo = em_cat(dpath, dpo, entity)
323 dpo = em_cat(dpath, dpo, ".txt" as *u8)
324 dpath[dpo] = 0 as u8
325
326 let lg: i64 = sys_openat_append("knowledge/media/media_ledger.tsv" as *u8, 0x1a4)
327 let df: i64 = sys_openat_append(dpath, 0x1a4)
328
329 let img: *u8 = sys_mmap(EM_IMG_CAP)
330 let iurl: *u8 = sys_mmap(EM_PATHBUF)
331 let fpath: *u8 = sys_mmap(EM_PATHBUF)
332 let dg: *u8 = sys_mmap(32)
333 let hex: *u8 = sys_mmap(72)
334 let seen: *u8 = sys_mmap(32 * EM_MAX_IMGS)
335 let imginf: *u8 = sys_mmap(EM_IMG_CAP)
336 let imgdech: *u8 = sys_mmap(EM_IMG_CAP)
337
338 var nseen: i64 = 0
339 var stored: i64 = 0
340 var dupes: i64 = 0
341 var skipped: i64 = 0
342 var failed: i64 = 0
343 var nonimg: i64 = 0
344 var capped: i64 = 0
345 var bytes_total: i64 = 0
346 // FORMAT CENSUS: 'non_image' previously conflated 'not an image' with 'an image format we do not
347 // sniff'. Gather is now format-AGNOSTIC (store any real image); decode coverage is measured
348 // separately by the indexer. That split is what makes the next build target decidable.
349 var n_jpeg: i64 = 0
350 var n_png: i64 = 0
351 var n_webp: i64 = 0
352 var n_gif: i64 = 0
353 var n_bmp: i64 = 0
354 var n_tiff: i64 = 0
355 var n_svg: i64 = 0
356 var n_upsized: i64 = 0
357 var n_tinyskip: i64 = 0
358
359 // FLAT loop: one guard flag, max nesting depth 2. A deep else-chain hides its own mis-nesting.
360 var idx: i64 = 0
361 while idx < nimgs {
362 var st: i64 = 0
363 let up: *u8 = ((url_buf as i64) + offs[idx]) as *u8
364 let ul: i64 = lens[idx]
365
366 if stored >= maxn { st = 9; capped = 1 }
367 if st == 0 { if em_is_http(up, ul) == 0 { st = 1; skipped = skipped + 1 } }
368 if st == 0 { if ul + 1 >= EM_PATHBUF { st = 1; skipped = skipped + 1 } }
369
370 if st == 0 {
371 var o: i64 = 0
372 while o < ul { iurl[o] = up[o]; o = o + 1 }
373 iurl[ul] = 0 as u8
374 if pace > 0 { sys_sleep_ms(pace) }
375 nx_csprng_fill(cr, 32)
376 nx_csprng_fill(pk, 32)
377 }
378
379 // Try the FULL-SIZE original first; fall back to the rendered thumbnail if it does not resolve.
380 var ir: i64 = 0
381 if st == 0 {
382 let full: *u8 = sys_mmap(EM_PATHBUF)
383 if em_upsize(iurl, ul, full) == 1 {
384 ir = nx_https_get(full, cr, pk, store, sys_now_realtime_sec(), img, EM_IMG_CAP)
385 if ir > 0 { n_upsized = n_upsized + 1 }
386 if ir <= 0 { if pace > 0 { sys_sleep_ms(pace) } }
387 }
388 if ir <= 0 {
389 nx_csprng_fill(cr, 32)
390 nx_csprng_fill(pk, 32)
391 ir = nx_https_get(iurl, cr, pk, store, sys_now_realtime_sec(), img, EM_IMG_CAP)
392 }
393 if ir <= 0 { st = 1; failed = failed + 1 }
394 }
395
396 var iblen: i64 = 0
397 var ibody: *u8 = img
398 if st == 0 {
399 let ibo: i64 = em_body_off(img, ir)
400 ibody = ((img as i64) + ibo) as *u8
401 iblen = ir - ibo
402 if iblen <= 0 { st = 1; failed = failed + 1 }
403 let icn: i64 = nx_http_dechunk(ibody, iblen, imgdech, EM_IMG_CAP)
404 if icn > 0 { ibody = imgdech; iblen = icn }
405 let idn: i64 = em_inflate_if_gzip(ibody, iblen, imginf, EM_IMG_CAP)
406 if idn > 0 { ibody = imginf; iblen = idn }
407 }
408
409 var kind: *u8 = "unknown" as *u8
410 var ext: *u8 = ".bin" as *u8
411 if st == 0 {
412 var isimg: i64 = 0
413 if nx_sniff_is_jpeg(ibody, iblen) == 1 { kind = "jpeg" as *u8; ext = ".jpg" as *u8; isimg = 1; n_jpeg = n_jpeg + 1 }
414 if nx_sniff_is_png(ibody, iblen) == 1 { kind = "png" as *u8; ext = ".png" as *u8; isimg = 1; n_png = n_png + 1 }
415 if nx_sniff_is_webp(ibody, iblen) == 1 { kind = "webp" as *u8; ext = ".webp" as *u8; isimg = 1; n_webp = n_webp + 1 }
416 if nx_sniff_is_gif(ibody, iblen) == 1 { kind = "gif" as *u8; ext = ".gif" as *u8; isimg = 1; n_gif = n_gif + 1 }
417 if nx_sniff_is_bmp(ibody, iblen) == 1 { kind = "bmp" as *u8; ext = ".bmp" as *u8; isimg = 1; n_bmp = n_bmp + 1 }
418 if nx_sniff_is_tiff(ibody, iblen) == 1 { kind = "tiff" as *u8; ext = ".tif" as *u8; isimg = 1; n_tiff = n_tiff + 1 }
419 if isimg == 0 { st = 1; nonimg = nonimg + 1 }
420 if isimg == 0 { if nx_sniff_is_svg_token(ibody, iblen) == 1 { n_svg = n_svg + 1 } }
421 }
422
423 // CHROME FLOOR: page furniture (flag icons, edit pencils, spacers) is tiny. A real photograph is
424 // not 900 bytes. Rejecting by size here is crude but it is measured against the actual corpus, and
425 // it is REPORTED so the number never hides.
426 if st == 0 { if iblen < EM_MAGIC_2048 { st = 1; n_tinyskip = n_tinyskip + 1 } }
427
428 if st == 0 {
429 sha256_digest(ibody, iblen, dg)
430 if em_seen(seen, nseen, dg) == 1 { st = 1; dupes = dupes + 1 }
431 }
432
433 if st == 0 {
434 var b: i64 = 0
435 while b < 32 { seen[nseen*32 + b] = dg[b]; b = b + 1 }
436 nseen = nseen + 1
437 em_hex(dg, hex)
438 var fo: i64 = em_cat(fpath, 0, dir)
439 fo = em_cat(fpath, fo, "/" as *u8)
440 fo = em_cat(fpath, fo, hex)
441 fo = em_cat(fpath, fo, ext)
442 fpath[fo] = 0 as u8
443 let wr: i64 = em_write_file(fpath, ibody, iblen)
444 if wr != iblen { st = 1; failed = failed + 1 }
445 if st == 0 { stored = stored + 1; bytes_total = bytes_total + iblen }
446 if st == 0 { em_ledger(lg, entity, url, iurl, iblen, hex, kind) }
447 if st == 0 { em_doss(df, entity, iurl, hex, kind, iblen) }
448 }
449
450 idx = idx + 1
451 }
452
453 if lg >= 0 { sys_close(lg) }
454 if df >= 0 { sys_close(df) }
455
456 em_puts("{\"tool\":\"nx_entity_media\",\"entity\":\"" as *u8); em_puts(entity)
457 em_puts("\",\"page\":\"" as *u8); em_puts(url)
458 em_puts("\",\"urls_extracted\":" as *u8); em_pi(nimgs)
459 em_puts(",\"stored\":" as *u8); em_pi(stored)
460 em_puts(",\"dupes\":" as *u8); em_pi(dupes)
461 em_puts(",\"non_image\":" as *u8); em_pi(nonimg)
462 em_puts(",\"skipped_non_http\":" as *u8); em_pi(skipped)
463 em_puts(",\"failed\":" as *u8); em_pi(failed)
464 em_puts(",\"bytes\":" as *u8); em_pi(bytes_total)
465 em_puts(",\"fmt\":{\"jpeg\":" as *u8); em_pi(n_jpeg)
466 em_puts(",\"png\":" as *u8); em_pi(n_png)
467 em_puts(",\"webp\":" as *u8); em_pi(n_webp)
468 em_puts(",\"gif\":" as *u8); em_pi(n_gif)
469 em_puts(",\"bmp\":" as *u8); em_pi(n_bmp)
470 em_puts(",\"tiff\":" as *u8); em_pi(n_tiff)
471 em_puts(",\"svg_rejected\":" as *u8); em_pi(n_svg)
472 em_puts(",\"upsized_to_original\":" as *u8); em_pi(n_upsized)
473 em_puts(",\"chrome_under_2kb_skipped\":" as *u8); em_pi(n_tinyskip)
474 em_puts("}" as *u8)
475 em_puts(",\"capped_at_max\":" as *u8); em_pi(capped)
476 em_puts(",\"dir\":\"" as *u8); em_puts(dir)
477 em_puts("\"}\n" as *u8)
478 em_puts("ENTITY-MEDIA-OK\n" as *u8)
479 sys_exit(0)
480 return 0
481}