code wiki / (root) / nx_entity_media.nx

nx_entity_media.nx source

↩ module page · 481 lines · 20030 B

1// nx_entity_media.nx -- MULTI-IMAGE entity media gather. 2// 3// Supersedes the 1-image demo in nx_media_gather.nx, which fetched exactly ONE image and only if its URL 4// contained a hardcoded "upload.wikimedia.org". Here: fetch an entity page over sovereign validated HTTPS, 5// extract ALL image URLs, fetch each (polite pacing), sniff + sha256 + content-address into 6// knowledge/media/<entity>/, dedupe by digest within the run, and append a provenance row per stored asset. 7// 8// No host allowlist: any http(s) image URL is eligible. Bounds are PARAMETERS, not literals -- the intended 9// limits are hardware and politeness, nothing else. A hit cap is REPORTED (capped=1), never silent. 10// 11// usage: nx_entity_media <page-url> <entity-slug> [max_imgs] [pace_ms] 12 13import "nx_str.nx" 14import "nx_syscalls.nx" 15import "nx_csprng.nx" 16import "nx_x509_trust_store.nx" 17import "nx_pem_loader.nx" 18import "nx_https_get.nx" 19import "nx_html_extract_imgs.nx" 20import "nx_media_sniff.nx" 21import "nx_sha256.nx" 22import "nx_inflate.nx" 23import "nx_http_dechunk.nx" 24const EM_MAGIC_2048: i64 = 2048 25 26const EM_PAGE_CAP: i64 = 4194304 27const EM_IMG_CAP: i64 = 8388608 28// Bounds sized so HARDWARE is the limit, not a literal. 4096 urls costs 32KB+32KB of offset/length 29// arrays, 128KB of dedupe digests and a 2MB url arena -- irrelevant against the 8MB image buffer. 30const EM_URLBUF: i64 = 2097152 31const EM_MAX_IMGS: i64 = 4096 32const EM_PATHBUF: i64 = 4096 33const EM_DEF_PACE_MS: i64 = 800 34 35func em_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 } 36func em_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, nx_str_len(s)); return 0 } 37 38func em_num(v: i64, o: *u8) -> i64 { 39 let t: *u8 = sys_mmap(32) 40 var m: i64 = v 41 var k: i64 = 0 42 if m < 0 { m = 0 - m } 43 if m == 0 { t[0] = 48; k = 1 } 44 while m > 0 { t[k] = 48 + (m % 10); m = m / 10; k = k + 1 } 45 var i: i64 = 0 46 while i < k { o[i] = t[k - 1 - i]; i = i + 1 } 47 o[k] = 0 as u8 48 return k 49} 50 51func em_pi(v: i64) -> i64 { let o: *u8 = sys_mmap(32); let n: i64 = em_num(v, o); sys_write(1, o, n); return 0 } 52func em_wn(fd: i64, v: i64) -> i64 { let o: *u8 = sys_mmap(32); let n: i64 = em_num(v, o); sys_write(fd, o, n); return 0 } 53 54func em_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[off+i] = s[i]; i = i + 1 } return off + i } 55 56func em_mkdir(path: *u8) -> i64 { return __syscall(258, -100, path, 0x1ed, 0, 0, 0) } 57 58func em_atoi(s: *u8) -> i64 { 59 var i: i64 = 0 60 var v: i64 = 0 61 while s[i] != (0 as u8) { 62 let c: i64 = s[i] as i64 63 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } 64 i = i + 1 65 } 66 return v 67} 68 69func em_hex(d: *u8, out: *u8) -> i64 { 70 let hx: *u8 = "0123456789abcdef" as *u8 71 var i: i64 = 0 72 while i < 32 { 73 let b: i64 = d[i] as i64 74 out[i*2] = hx[(b >> 4) & 15] 75 out[i*2+1] = hx[b & 15] 76 i = i + 1 77 } 78 out[64] = 0 as u8 79 return 64 80} 81 82func em_body_off(resp: *u8, n: i64) -> i64 { 83 var i: i64 = 0 84 while i + 3 < n { 85 if resp[i] == (13 as u8) { if resp[i+1] == (10 as u8) { if resp[i+2] == (13 as u8) { if resp[i+3] == (10 as u8) { return i + 4 } } } } 86 i = i + 1 87 } 88 return 0 89} 90 91func em_write_file(path: *u8, buf: *u8, n: i64) -> i64 { 92 let fd: i64 = sys_openat_wr(path, 0x1a4) 93 if fd < 0 { return 0 - 1 } 94 var off: i64 = 0 95 while off < n { 96 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off) 97 if w <= 0 { sys_close(fd); return 0 - 1 } 98 off = off + w 99 } 100 sys_close(fd) 101 return n 102} 103 104// http(s) scheme only -- rejects data: URIs, protocol-relative and relative leftovers 105func em_is_http(u: *u8, n: i64) -> i64 { 106 if n < 8 { return 0 } 107 if u[0] != (104 as u8) { return 0 } 108 if u[1] != (116 as u8) { return 0 } 109 if u[2] != (116 as u8) { return 0 } 110 if u[3] != (112 as u8) { return 0 } 111 return 1 112} 113 114// has this exact content already been stored in THIS run? 115func em_seen(seen: *u8, nseen: i64, dg: *u8) -> i64 { 116 var i: i64 = 0 117 while i < nseen { 118 var same: i64 = 1 119 var b: i64 = 0 120 while b < 32 { if seen[i*32+b] != dg[b] { same = 0 } b = b + 1 } 121 if same == 1 { return 1 } 122 i = i + 1 123 } 124 return 0 125} 126 127func em_ledger(lg: i64, entity: *u8, page: *u8, iurl: *u8, n: i64, hex: *u8, kind: *u8) -> i64 { 128 if lg < 0 { return 0 } 129 em_w(lg, entity); em_w(lg, "\t" as *u8) 130 em_w(lg, page); em_w(lg, "\t" as *u8) 131 em_w(lg, iurl); em_w(lg, "\t" as *u8) 132 em_wn(lg, n); em_w(lg, "\t" as *u8) 133 em_w(lg, hex); em_w(lg, "\t" as *u8) 134 em_w(lg, kind); em_w(lg, "\tVERIFIED\n" as *u8) 135 return 0 136} 137 138func em_doss(df: i64, entity: *u8, iurl: *u8, hex: *u8, kind: *u8, n: i64) -> i64 { 139 if df < 0 { return 0 } 140 em_w(df, "media asset entity=" as *u8); em_w(df, entity) 141 em_w(df, " image=" as *u8); em_w(df, iurl) 142 em_w(df, " sha256=" as *u8); em_w(df, hex) 143 em_w(df, " kind=" as *u8); em_w(df, kind) 144 em_w(df, " bytes=" as *u8); em_wn(df, n) 145 em_w(df, " transport=sovereign-validated-tls13 status=verified\n" as *u8) 146 return 0 147} 148 149// FULL-SIZE RESOLUTION. We gather <img src> AS RENDERED, and rendered means the THUMBNAIL the page chose 150// to display. Measured on a real corpus: 19 of 19 assets were under 400px, average longest side 79px -- 151// the index was being built on postage stamps. Both major sources expose the original by a deterministic 152// URL transform: 153// wikimedia /commons/thumb/5/5d/X.jpg/250px-X.jpg -> /commons/5/5d/X.jpg 154// (drop the "/thumb" segment, then drop the trailing rendered-size component) 155// javdatabase /idolimages/thumb/x.webp -> /idolimages/full/x.webp 156// Returns 1 and fills `out` when a transform applied, else 0. The caller TRIES THE ORIGINAL FIRST and 157// FALLS BACK to the thumbnail, so a source that does not honour the transform still yields an asset. 158// does url[0..n) end in a raster extension we actually decode? 159func em_ends(u: *u8, n: i64, s: *u8) -> i64 { 160 let sl: i64 = nx_str_len(s) 161 if sl > n { return 0 } 162 var i: i64 = 0 163 while i < sl { if u[n - sl + i] != s[i] { return 0 } i = i + 1 } 164 return 1 165} 166 167func em_raster_ext(u: *u8, n: i64) -> i64 { 168 if em_ends(u, n, ".jpg" as *u8) == 1 { return 1 } 169 if em_ends(u, n, ".jpeg" as *u8) == 1 { return 1 } 170 if em_ends(u, n, ".png" as *u8) == 1 { return 1 } 171 if em_ends(u, n, ".webp" as *u8) == 1 { return 1 } 172 if em_ends(u, n, ".gif" as *u8) == 1 { return 1 } 173 if em_ends(u, n, ".JPG" as *u8) == 1 { return 1 } 174 if em_ends(u, n, ".JPEG" as *u8) == 1 { return 1 } 175 if em_ends(u, n, ".PNG" as *u8) == 1 { return 1 } 176 return 0 177} 178 179func em_upsize(u: *u8, n: i64, out: *u8) -> i64 { 180 // idolimages/thumb/ -> idolimages/full/ 181 let a: *u8 = "/idolimages/thumb/" as *u8 182 let al: i64 = nx_str_len(a) 183 var i: i64 = 0 184 while i + al <= n { 185 var hit: i64 = 1 186 var j: i64 = 0 187 while j < al { if u[i+j] != a[j] { hit = 0; j = al } else { j = j + 1 } } 188 if hit == 1 { 189 var o: i64 = 0 190 while o < i { out[o] = u[o]; o = o + 1 } 191 o = em_cat(out, o, "/idolimages/full/" as *u8) 192 var k: i64 = i + al 193 while k < n { out[o] = u[k]; o = o + 1; k = k + 1 } 194 out[o] = 0 as u8 195 return 1 196 } 197 i = i + 1 198 } 199 200 // /thumb/ -> drop the segment, then drop the trailing rendered-size component 201 let b: *u8 = "/thumb/" as *u8 202 let bl: i64 = nx_str_len(b) 203 var t: i64 = 0 - 1 204 var p: i64 = 0 205 while p + bl <= n { 206 if t < 0 { 207 var hit2: i64 = 1 208 var j2: i64 = 0 209 while j2 < bl { if u[p+j2] != b[j2] { hit2 = 0; j2 = bl } else { j2 = j2 + 1 } } 210 if hit2 == 1 { t = p } 211 } 212 p = p + 1 213 } 214 if t < 0 { return 0 } 215 216 var o2: i64 = 0 217 var c2: i64 = 0 218 while c2 < t { out[o2] = u[c2]; o2 = o2 + 1; c2 = c2 + 1 } 219 var k2: i64 = t + 6 // skip "/thumb", keep the following "/" 220 while k2 < n { out[o2] = u[k2]; o2 = o2 + 1; k2 = k2 + 1 } 221 // truncate at the LAST '/' -- that removes the "250px-Name.jpg" rendered component 222 var last: i64 = 0 - 1 223 var q: i64 = 0 224 while q < o2 { if out[q] == (47 as u8) { last = q } q = q + 1 } 225 if last <= 0 { return 0 } 226 out[last] = 0 as u8 227 // ★A SUCCESSFUL FETCH OF THE WRONG THING DEFEATS A FALLBACK KEYED ON FETCH FAILURE. 228 // A Commons thumbnail is often a PNG *rendering* of a non-raster original (.svg, .pdf, .ogv, .tif). 229 // Upsizing those returns the SOURCE -- a 200 OK we cannot decode -- so the fetch-failure fallback never 230 // fires and we silently lose the one usable form. Only upsize when the ORIGINAL is itself a raster we 231 // decode; otherwise keep the rendered thumbnail, which is the decodable artifact. 232 if em_raster_ext(out, last) == 0 { return 0 } 233 return 1 234} 235 236// Content-Encoding: gzip. The sovereign HTTPS client requests gzip and returns the body still 237// COMPRESSED; handing those bytes to an HTML extractor yields a SILENT zero-images result rather 238// than an error. inf_gunzip has existed in nx_inflate.nx all along -- this is the missing compose. 239// Returns the inflated length written to dst, or 0 when src is not a gzip stream. 240func em_inflate_if_gzip(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 { 241 if n < 3 { return 0 } 242 if src[0] != (31 as u8) { return 0 } 243 if src[1] != (139 as u8) { return 0 } 244 let dn: i64 = inf_gunzip(src, n, dst, cap) 245 if dn <= 0 { return 0 } 246 return dn 247} 248 249func main(argc: i64, argv: *i64) -> i64 { 250 var url: *u8 = "https://en.wikipedia.org/wiki/Diora_Baird" as *u8 251 var entity: *u8 = "diora_baird" as *u8 252 var maxn: i64 = EM_MAX_IMGS 253 var pace: i64 = EM_DEF_PACE_MS 254 if argc >= 2 { url = argv[1] as *u8 } 255 if argc >= 3 { entity = argv[2] as *u8 } 256 if argc >= 4 { maxn = em_atoi(argv[3] as *u8) } 257 if argc >= 5 { pace = em_atoi(argv[4] as *u8) } 258 if maxn <= 0 { maxn = EM_MAX_IMGS } 259 if maxn > EM_MAX_IMGS { maxn = EM_MAX_IMGS } 260 if pace < 0 { pace = 0 } 261 262 em_puts("=== ENTITY MEDIA GATHER (multi-image, sovereign TLS) ===\n entity=" as *u8) 263 em_puts(entity); em_puts(" page=" as *u8); em_puts(url) 264 em_puts(" max=" as *u8); em_pi(maxn); em_puts(" pace_ms=" as *u8); em_pi(pace); em_puts("\n" as *u8) 265 266 let store: *TrustStore = trust_store_alloc(400) 267 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt" as *u8, store) 268 if nroots <= 0 { em_puts("ENTITY-MEDIA-FAIL no-roots\n" as *u8); sys_exit(1); return 1 } 269 270 let cr: *u8 = sys_mmap(32) 271 let pk: *u8 = sys_mmap(32) 272 nx_csprng_fill(cr, 32) 273 nx_csprng_fill(pk, 32) 274 let page: *u8 = sys_mmap(EM_PAGE_CAP) 275 let pr: i64 = nx_https_get(url, cr, pk, store, sys_now_realtime_sec(), page, EM_PAGE_CAP) 276 em_puts(" page fetch bytes=" as *u8); em_pi(pr); em_puts("\n" as *u8) 277 if pr <= 0 { em_puts("ENTITY-MEDIA-FAIL page-fetch\n" as *u8); sys_exit(1); return 1 } 278 279 let bo: i64 = em_body_off(page, pr) 280 let body: *u8 = ((page as i64) + bo) as *u8 281 let blen: i64 = pr - bo 282 283 // WIRE ORDER MATTERS: Transfer-Encoding framing is the OUTER layer, Content-Encoding the inner. 284 // A chunked+gzip response begins with a hex chunk-size line, so the gzip magic is NOT at body[0] 285 // and a naive gzip probe reports 'identity' and hands chunk-framed binary to the extractor. 286 // nx_http_dechunk returns negative on non-chunked input, so attempt-then-fall-back is safe. 287 var raw: *u8 = body 288 var rawlen: i64 = blen 289 let dechunk: *u8 = sys_mmap(EM_PAGE_CAP) 290 let cn: i64 = nx_http_dechunk(body, blen, dechunk, EM_PAGE_CAP) 291 if cn > 0 { raw = dechunk; rawlen = cn } 292 293 var html: *u8 = raw 294 var hlen: i64 = rawlen 295 let infbuf: *u8 = sys_mmap(EM_PAGE_CAP) 296 let pdn: i64 = em_inflate_if_gzip(raw, rawlen, infbuf, EM_PAGE_CAP) 297 if pdn > 0 { html = infbuf; hlen = pdn } 298 299 em_puts(" transfer=" as *u8) 300 if cn > 0 { em_puts("chunked dechunked=" as *u8); em_pi(cn) } else { em_puts("identity" as *u8) } 301 em_puts(" encoding=" as *u8) 302 if pdn > 0 { em_puts("gzip inflated=" as *u8); em_pi(pdn) } else { em_puts("identity" as *u8) } 303 em_puts(" html_bytes=" as *u8); em_pi(hlen); em_puts("\n" as *u8) 304 305 let url_buf: *u8 = sys_mmap(EM_URLBUF) 306 let offs: *i64 = sys_mmap(8 * EM_MAX_IMGS) as *i64 307 let lens: *i64 = sys_mmap(8 * EM_MAX_IMGS) as *i64 308 let nimgs: i64 = nx_html_extract_imgs(html, hlen, url, nx_str_len(url), url_buf, EM_URLBUF, offs, lens, EM_MAX_IMGS) 309 em_puts(" image urls extracted=" as *u8); em_pi(nimgs); em_puts("\n" as *u8) 310 if nimgs <= 0 { em_puts("ENTITY-MEDIA-FAIL no-images\n" as *u8); sys_exit(1); return 1 } 311 312 em_mkdir("knowledge/media" as *u8) 313 em_mkdir("knowledge/library" as *u8) 314 let dir: *u8 = sys_mmap(EM_PATHBUF) 315 var dof: i64 = em_cat(dir, 0, "knowledge/media/" as *u8) 316 dof = em_cat(dir, dof, entity) 317 dir[dof] = 0 as u8 318 em_mkdir(dir) 319 320 let dpath: *u8 = sys_mmap(EM_PATHBUF) 321 var dpo: i64 = em_cat(dpath, 0, "knowledge/library/media-" as *u8) 322 dpo = em_cat(dpath, dpo, entity) 323 dpo = em_cat(dpath, dpo, ".txt" as *u8) 324 dpath[dpo] = 0 as u8 325 326 let lg: i64 = sys_openat_append("knowledge/media/media_ledger.tsv" as *u8, 0x1a4) 327 let df: i64 = sys_openat_append(dpath, 0x1a4) 328 329 let img: *u8 = sys_mmap(EM_IMG_CAP) 330 let iurl: *u8 = sys_mmap(EM_PATHBUF) 331 let fpath: *u8 = sys_mmap(EM_PATHBUF) 332 let dg: *u8 = sys_mmap(32) 333 let hex: *u8 = sys_mmap(72) 334 let seen: *u8 = sys_mmap(32 * EM_MAX_IMGS) 335 let imginf: *u8 = sys_mmap(EM_IMG_CAP) 336 let imgdech: *u8 = sys_mmap(EM_IMG_CAP) 337 338 var nseen: i64 = 0 339 var stored: i64 = 0 340 var dupes: i64 = 0 341 var skipped: i64 = 0 342 var failed: i64 = 0 343 var nonimg: i64 = 0 344 var capped: i64 = 0 345 var bytes_total: i64 = 0 346 // FORMAT CENSUS: 'non_image' previously conflated 'not an image' with 'an image format we do not 347 // sniff'. Gather is now format-AGNOSTIC (store any real image); decode coverage is measured 348 // separately by the indexer. That split is what makes the next build target decidable. 349 var n_jpeg: i64 = 0 350 var n_png: i64 = 0 351 var n_webp: i64 = 0 352 var n_gif: i64 = 0 353 var n_bmp: i64 = 0 354 var n_tiff: i64 = 0 355 var n_svg: i64 = 0 356 var n_upsized: i64 = 0 357 var n_tinyskip: i64 = 0 358 359 // FLAT loop: one guard flag, max nesting depth 2. A deep else-chain hides its own mis-nesting. 360 var idx: i64 = 0 361 while idx < nimgs { 362 var st: i64 = 0 363 let up: *u8 = ((url_buf as i64) + offs[idx]) as *u8 364 let ul: i64 = lens[idx] 365 366 if stored >= maxn { st = 9; capped = 1 } 367 if st == 0 { if em_is_http(up, ul) == 0 { st = 1; skipped = skipped + 1 } } 368 if st == 0 { if ul + 1 >= EM_PATHBUF { st = 1; skipped = skipped + 1 } } 369 370 if st == 0 { 371 var o: i64 = 0 372 while o < ul { iurl[o] = up[o]; o = o + 1 } 373 iurl[ul] = 0 as u8 374 if pace > 0 { sys_sleep_ms(pace) } 375 nx_csprng_fill(cr, 32) 376 nx_csprng_fill(pk, 32) 377 } 378 379 // Try the FULL-SIZE original first; fall back to the rendered thumbnail if it does not resolve. 380 var ir: i64 = 0 381 if st == 0 { 382 let full: *u8 = sys_mmap(EM_PATHBUF) 383 if em_upsize(iurl, ul, full) == 1 { 384 ir = nx_https_get(full, cr, pk, store, sys_now_realtime_sec(), img, EM_IMG_CAP) 385 if ir > 0 { n_upsized = n_upsized + 1 } 386 if ir <= 0 { if pace > 0 { sys_sleep_ms(pace) } } 387 } 388 if ir <= 0 { 389 nx_csprng_fill(cr, 32) 390 nx_csprng_fill(pk, 32) 391 ir = nx_https_get(iurl, cr, pk, store, sys_now_realtime_sec(), img, EM_IMG_CAP) 392 } 393 if ir <= 0 { st = 1; failed = failed + 1 } 394 } 395 396 var iblen: i64 = 0 397 var ibody: *u8 = img 398 if st == 0 { 399 let ibo: i64 = em_body_off(img, ir) 400 ibody = ((img as i64) + ibo) as *u8 401 iblen = ir - ibo 402 if iblen <= 0 { st = 1; failed = failed + 1 } 403 let icn: i64 = nx_http_dechunk(ibody, iblen, imgdech, EM_IMG_CAP) 404 if icn > 0 { ibody = imgdech; iblen = icn } 405 let idn: i64 = em_inflate_if_gzip(ibody, iblen, imginf, EM_IMG_CAP) 406 if idn > 0 { ibody = imginf; iblen = idn } 407 } 408 409 var kind: *u8 = "unknown" as *u8 410 var ext: *u8 = ".bin" as *u8 411 if st == 0 { 412 var isimg: i64 = 0 413 if nx_sniff_is_jpeg(ibody, iblen) == 1 { kind = "jpeg" as *u8; ext = ".jpg" as *u8; isimg = 1; n_jpeg = n_jpeg + 1 } 414 if nx_sniff_is_png(ibody, iblen) == 1 { kind = "png" as *u8; ext = ".png" as *u8; isimg = 1; n_png = n_png + 1 } 415 if nx_sniff_is_webp(ibody, iblen) == 1 { kind = "webp" as *u8; ext = ".webp" as *u8; isimg = 1; n_webp = n_webp + 1 } 416 if nx_sniff_is_gif(ibody, iblen) == 1 { kind = "gif" as *u8; ext = ".gif" as *u8; isimg = 1; n_gif = n_gif + 1 } 417 if nx_sniff_is_bmp(ibody, iblen) == 1 { kind = "bmp" as *u8; ext = ".bmp" as *u8; isimg = 1; n_bmp = n_bmp + 1 } 418 if nx_sniff_is_tiff(ibody, iblen) == 1 { kind = "tiff" as *u8; ext = ".tif" as *u8; isimg = 1; n_tiff = n_tiff + 1 } 419 if isimg == 0 { st = 1; nonimg = nonimg + 1 } 420 if isimg == 0 { if nx_sniff_is_svg_token(ibody, iblen) == 1 { n_svg = n_svg + 1 } } 421 } 422 423 // CHROME FLOOR: page furniture (flag icons, edit pencils, spacers) is tiny. A real photograph is 424 // not 900 bytes. Rejecting by size here is crude but it is measured against the actual corpus, and 425 // it is REPORTED so the number never hides. 426 if st == 0 { if iblen < EM_MAGIC_2048 { st = 1; n_tinyskip = n_tinyskip + 1 } } 427 428 if st == 0 { 429 sha256_digest(ibody, iblen, dg) 430 if em_seen(seen, nseen, dg) == 1 { st = 1; dupes = dupes + 1 } 431 } 432 433 if st == 0 { 434 var b: i64 = 0 435 while b < 32 { seen[nseen*32 + b] = dg[b]; b = b + 1 } 436 nseen = nseen + 1 437 em_hex(dg, hex) 438 var fo: i64 = em_cat(fpath, 0, dir) 439 fo = em_cat(fpath, fo, "/" as *u8) 440 fo = em_cat(fpath, fo, hex) 441 fo = em_cat(fpath, fo, ext) 442 fpath[fo] = 0 as u8 443 let wr: i64 = em_write_file(fpath, ibody, iblen) 444 if wr != iblen { st = 1; failed = failed + 1 } 445 if st == 0 { stored = stored + 1; bytes_total = bytes_total + iblen } 446 if st == 0 { em_ledger(lg, entity, url, iurl, iblen, hex, kind) } 447 if st == 0 { em_doss(df, entity, iurl, hex, kind, iblen) } 448 } 449 450 idx = idx + 1 451 } 452 453 if lg >= 0 { sys_close(lg) } 454 if df >= 0 { sys_close(df) } 455 456 em_puts("{\"tool\":\"nx_entity_media\",\"entity\":\"" as *u8); em_puts(entity) 457 em_puts("\",\"page\":\"" as *u8); em_puts(url) 458 em_puts("\",\"urls_extracted\":" as *u8); em_pi(nimgs) 459 em_puts(",\"stored\":" as *u8); em_pi(stored) 460 em_puts(",\"dupes\":" as *u8); em_pi(dupes) 461 em_puts(",\"non_image\":" as *u8); em_pi(nonimg) 462 em_puts(",\"skipped_non_http\":" as *u8); em_pi(skipped) 463 em_puts(",\"failed\":" as *u8); em_pi(failed) 464 em_puts(",\"bytes\":" as *u8); em_pi(bytes_total) 465 em_puts(",\"fmt\":{\"jpeg\":" as *u8); em_pi(n_jpeg) 466 em_puts(",\"png\":" as *u8); em_pi(n_png) 467 em_puts(",\"webp\":" as *u8); em_pi(n_webp) 468 em_puts(",\"gif\":" as *u8); em_pi(n_gif) 469 em_puts(",\"bmp\":" as *u8); em_pi(n_bmp) 470 em_puts(",\"tiff\":" as *u8); em_pi(n_tiff) 471 em_puts(",\"svg_rejected\":" as *u8); em_pi(n_svg) 472 em_puts(",\"upsized_to_original\":" as *u8); em_pi(n_upsized) 473 em_puts(",\"chrome_under_2kb_skipped\":" as *u8); em_pi(n_tinyskip) 474 em_puts("}" as *u8) 475 em_puts(",\"capped_at_max\":" as *u8); em_pi(capped) 476 em_puts(",\"dir\":\"" as *u8); em_puts(dir) 477 em_puts("\"}\n" as *u8) 478 em_puts("ENTITY-MEDIA-OK\n" as *u8) 479 sys_exit(0) 480 return 0 481}