code wiki / _hdl_build / nx_cbx_pages.nx
`nx_cbx_pages.nx` source

↩ module page · 800 lines · 37892 B
1// nx_cbx_pages.nx -- SOVEREIGN CBX page-image extraction (Reader-arc RUNG 3, FORMAT-FIX rev "cfix":2).
2//
3// The operator headline ask: "convert pdf into cbx type zoom" so books read easily on mobile; then
4// "the formats are broken like for the comics" -- this rev EATS that debt. CBX = comic-archive style:
5// ordered page IMAGES the browser renders + pinch/pan/zooms natively. KEY INSIGHT: the browser is the
6// last-mile image renderer/zoomer (allowed). Nishi only EXTRACTS page image BYTES sovereignly, serves them.
7//
8// CONTAINERS (routed by MAGIC BYTES, never extension -- wild comics LIE: zip renamed .cbr, rar renamed .cbz):
9//   (A) ZIP  (cbz)          -- central-directory walker (proven, from nx_epub_read.nx), store+deflate.
10//   (B) RAR4 (cbr)          -- sovereign block walker, STORED (method 0x30) entries extracted. Comics keep
11//       JPEG pages STORED (compressing JPEG gains ~0%), so store-only covers the real corpus; entries that
12//       ARE compressed are COUNTED and reported CBX-RAR-COMPRESSED-UNSUPPORTED -- honest, never fake pages.
13//   (C) RAR5 (cbr)          -- vint block walker, method bits 7..9 == 0 (store) extracted; same honesty.
14//   (D) 7z   (cb7)          -- CBX-7Z-UNSUPPORTED honest skip (LZMA decode not built yet).
15//   (E) Scanned/image PDF   -- /DCTDecode image XObjects = raw JPEG between stream/endstream. Anchor on
16//       /DCTDecode (NEVER bare "stream"), confirm SOI ff d8, scan to endstream. Vector/text PDF -> honest skip.
17//
18// FORMAT-FIX rev (the broken-comics debt this rev eats):
19//   1. NATURAL page sort  -- ascii sort put "page10" before "page2" in any non-zero-padded comic = pages
20//      OUT OF ORDER. name_cmp_nat compares digit runs numerically (case-insensitive elsewhere).
21//   2. PER-PAGE magic gate -- __MACOSX/._foo.jpg AppleDouble junk passed the extension filter and was
22//      WRITTEN as a broken page. Now every page's BYTES must pass img_sig_ok or the entry is skipped+logged.
23//   3. CBR support (B)(C) + magic sniffing (misnamed containers) + honest skip codes.
24//   4. "cfix":2 schema tag in cbx.json -- the server treats a cbx.json WITHOUT the current tag as a cache
25//      MISS, so comics extracted by the OLD organ self-heal on next open (no remote rm; additive).
26//
27// SOVEREIGN: pure Nishi organs + raw syscalls. NO python/unzip/unrar/pdfium/mupdf/ghostscript/poppler/gcc/sh.
28// Image DISPLAY+zoom is the browser (last-mile); the viewer JS/CSS is hand-written (no 3rd-party libs).
29// NO-FAKE-GREEN: every written page's first bytes are a REAL image magic (JPEG/PNG/GIF/BMP/WEBP); skips are
30// COUNTED and printed, never silent. ADDITIVE (rule 13): output dir per slug; reruns overwrite atomically.
31// license_tier: ORIGINAL
32import "nx_syscalls.nx"
33import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
34import "nx_deflate.nx"
35const PAGE_MAGIC_1024: i64 = 1024
36const PAGE_MAGIC_1048576: i64 = 1048576
37
38// ---- io + string helpers (reused verbatim from nx_epub_read.nx) ----
39func er_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
40func er_w(fd: i64, s: *u8) -> i64 { sys_write(fd, s, er_slen(s)); return 0 }
41// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
42// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
43// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
44// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
45func er_n(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
46func er_p(s: *u8) -> i64 { er_w(1, s); return 0 }
47func er_pn(v: i64) -> i64 { er_n(1, v); return 0 }
48
49func le16(b: *u8, o: i64) -> i64 { return (b[o] as i64) | ((b[o+1] as i64) << 8) }
50func le32(b: *u8, o: i64) -> i64 { return (b[o] as i64) | ((b[o+1] as i64) << 8) | ((b[o+2] as i64) << 16) | ((b[o+3] as i64) << 24) }
51
52func er_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
53func er_catn(dst: *u8, off: i64, v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; var k: i64=0; if m==0{t[0]=48 as u8;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{dst[off+i]=t[k-1-i];i=i+1} return off+k }
54
55// substring search; return index in hay[0..hl] or -1 (verbatim from nx_epub_read.nx)
56func find_sub(hay: *u8, hl: i64, needle: *u8) -> i64 {
  let nl: i64 = er_slen(needle)
  if nl == 0 { return 0-1 }
  var i: i64 = 0
  while i + nl <= hl {
      var k: i64 = 0
      var hit: i64 = 1
      while k < nl { if hay[i+k] != needle[k] { hit = 0; k = nl } else { k = k + 1 } }
      if hit == 1 { return i }
      i = i + 1
  }
  return 0-1
68}
69
70// substring search starting at >= from; return index or -1
71func find_sub_from(hay: *u8, hl: i64, needle: *u8, from: i64) -> i64 {
  if from < 0 { return 0-1 }
  let r: i64 = find_sub((hay as i64 + from) as *u8, hl - from, needle)
  if r < 0 { return 0-1 }
  return from + r
76}
77
78// ---- ZIP central-directory walker (find_eocd / zip_enum reused from prior rev) ----
79func find_eocd(z: *u8, zl: i64) -> i64 {
  var i: i64 = zl - 22
  if i < 0 { return 0-1 }
  while i >= 0 {
      if z[i] == (0x50 as u8) { if z[i+1] == (0x4b as u8) { if z[i+2] == (0x05 as u8) { if z[i+3] == (0x06 as u8) { return i } } } }
      i = i - 1
  }
  return 0-1
87}
88
89// enumerate ALL zip entries: filenames into a NUL-separated arena (starts in noffs[]), parallel per-entry
90// {method,comp,dataoff} arrays (dataoff resolved from each entry's own LOCAL header). Returns entry count.
91func zip_enum(z: *u8, zl: i64, arena: *u8, acap: i64, noffs: *i64, meths: *i64, comps: *i64, doffs: *i64, ecap: i64) -> i64 {
  let eocd: i64 = find_eocd(z, zl)
  if eocd < 0 { return 0 }
  let count: i64 = le16(z, eocd + 10)
  var cd: i64 = le32(z, eocd + 16)
  var k: i64 = 0
  var n: i64 = 0
  var aoff: i64 = 0
  while k < count {
      if n >= ecap { return n }
      if cd + 46 > zl { return n }
      if z[cd] != (0x50 as u8) { return n }
      let method: i64 = le16(z, cd + 10)
      let comp: i64 = le32(z, cd + 20)
      let fnlen: i64 = le16(z, cd + 28)
      let extralen: i64 = le16(z, cd + 30)
      let commentlen: i64 = le16(z, cd + 32)
      let lho: i64 = le32(z, cd + 42)
      if aoff + fnlen + 1 < acap {
          noffs[n] = aoff
          var i: i64 = 0
          while i < fnlen { arena[aoff] = z[cd + 46 + i]; aoff = aoff + 1; i = i + 1 }
          arena[aoff] = 0 as u8; aoff = aoff + 1
          let lfn: i64 = le16(z, lho + 26)
          let lex: i64 = le16(z, lho + 28)
          meths[n] = method
          comps[n] = comp
          doffs[n] = lho + 30 + lfn + lex
          n = n + 1
      }
      cd = cd + 46 + fnlen + extralen + commentlen
      k = k + 1
  }
  return n
125}
126
127// ASCII byte-compare two NUL-terminated names: <0, 0, >0 (kept: extension matching in is_image_name)
128func name_cmp(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) {
      if b[i] == (0 as u8) { return 1 }
      let ca: i64 = a[i] as i64
      let cb: i64 = b[i] as i64
      if ca < cb { return 0-1 }
      if ca > cb { return 1 }
      i = i + 1
  }
  if b[i] != (0 as u8) { return 0-1 }
  return 0
140}
141
142// lowercase one ascii byte
143func lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
144
145// count of consecutive ascii digits in s starting at o
146func nat_digits(s: *u8, o: i64) -> i64 {
  var k: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let c: i64 = s[o+k] as i64
      var d: i64 = 0
      if c >= 48 { if c <= 57 { d = 1 } }
      if d == 1 { k = k + 1 } else { go = 0 }
  }
  return k
156}
157
158// NATURAL name compare (numeric-aware, case-insensitive): "page2" < "page10". The FORMAT-FIX for wild
159// comics whose page names are NOT zero-padded ("Page 2.jpg" vs "Page 10.jpg") -- ascii sort broke order.
160func name_cmp_nat(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  var j: i64 = 0
  var res: i64 = 2
  while res == 2 {
      let ca: i64 = a[i] as i64
      let cb: i64 = b[j] as i64
      if ca == 0 {
          if cb == 0 { res = 0 } else { res = 0-1 }
      } else {
          if cb == 0 { res = 1 } else {
              var da: i64 = 0
              if ca >= 48 { if ca <= 57 { da = 1 } }
              var db: i64 = 0
              if cb >= 48 { if cb <= 57 { db = 1 } }
              var both: i64 = 0
              if da == 1 { if db == 1 { both = 1 } }
              if both == 1 {
                  var zi: i64 = i
                  var zj: i64 = j
                  while a[zi] == (48 as u8) { zi = zi + 1 }
                  while b[zj] == (48 as u8) { zj = zj + 1 }
                  let li: i64 = nat_digits(a, zi)
                  let lj: i64 = nat_digits(b, zj)
                  if li < lj { res = 0-1 } else {
                      if li > lj { res = 1 } else {
                          var k: i64 = 0
                          while k < li {
                              if res == 2 {
                                  let xa: i64 = a[zi+k] as i64
                                  let xb: i64 = b[zj+k] as i64
                                  if xa < xb { res = 0-1 }
                                  if xa > xb { res = 1 }
                              }
                              k = k + 1
                          }
                          if res == 2 { i = zi + li; j = zj + lj }
                      }
                  }
              } else {
                  let la: i64 = lc(ca)
                  let lb2: i64 = lc(cb)
                  if la < lb2 { res = 0-1 }
                  if la > lb2 { res = 1 }
                  if res == 2 { i = i + 1; j = j + 1 }
              }
          }
      }
  }
  return res
210}
211
212// 1 if name ends with one of the image extensions .jpg/.jpeg/.png/.webp/.gif/.bmp (case-insensitive)
213func is_image_name(name: *u8) -> i64 {
  let n: i64 = er_slen(name)
  if n < 4 { return 0 }
  var dot: i64 = 0-1
  var i: i64 = 0
  while i < n { if name[i] == (0x2e as u8) { dot = i } i = i + 1 }
  if dot < 0 { return 0 }
  let extlen: i64 = n - dot - 1
  let e: *u8 = sys_mmap(16)
  var k: i64 = 0
  while k < extlen { if k < 12 { e[k] = lc(name[dot+1+k] as i64) as u8 } k = k + 1 }
  e[extlen] = 0 as u8
  if name_cmp(e, "jpg" as *u8) == 0 { return 1 }
  if name_cmp(e, "jpeg" as *u8) == 0 { return 1 }
  if name_cmp(e, "png" as *u8) == 0 { return 1 }
  if name_cmp(e, "webp" as *u8) == 0 { return 1 }
  if name_cmp(e, "gif" as *u8) == 0 { return 1 }
  if name_cmp(e, "bmp" as *u8) == 0 { return 1 }
  return 0
232}
233
234// 1 if the entry is archive JUNK, not a page: __MACOSX resource-fork dirs + AppleDouble "._" basenames.
235// These carry image EXTENSIONS (._0001.jpg) but their bytes are AppleDouble metadata -- they rendered as
236// BROKEN pages in the viewer before this gate. Bytes are ALSO magic-gated per page (belt + suspenders).
237func is_junk_name(name: *u8) -> i64 {
  let n: i64 = er_slen(name)
  if find_sub(name, n, "__MACOSX" as *u8) >= 0 { return 1 }
  var slash: i64 = 0-1
  var i: i64 = 0
  while i < n { if name[i] == (0x2f as u8) { slash = i } i = i + 1 }
  let b0: i64 = slash + 1
  if b0 + 1 < n { if name[b0] == (0x2e as u8) { if name[b0+1] == (0x5f as u8) { return 1 } } }
  return 0
246}
247
248// extension string (with leading dot) for a name, lowercased, into out; default ".jpg"
249func ext_of(name: *u8, out: *u8) -> i64 {
  let n: i64 = er_slen(name)
  var dot: i64 = 0-1
  var i: i64 = 0
  while i < n { if name[i] == (0x2e as u8) { dot = i } i = i + 1 }
  if dot < 0 { return er_cat(out, 0, ".jpg" as *u8) }
  var o: i64 = 0
  out[o] = 0x2e as u8; o = o + 1
  var k: i64 = dot + 1
  while k < n { out[o] = lc(name[k] as i64) as u8; o = o + 1; k = k + 1 }
  out[o] = 0 as u8
  return o
261}
262
263// mkdirat AT_FDCWD path 0755 (idempotent; -EEXIST harmless). x86_64 mkdirat=258, AT_FDCWD=-100.
264func ensure_dir(path: *u8) -> i64 { __syscall(258, 0-100, path, 0x1ed, 0, 0, 0); return 0 }
265
266const PAGE_CAP: i64 = 67108864   // 64 MiB per page buffer (largest cbz page << this)
267
268// build knowledge/staging/media/reader/<slug>/page<NNN><ext>, write n bytes; ret 0/-1
269func write_page(dir: *u8, idx: i64, ext: *u8, bytes: *u8, n: i64) -> i64 {
  let path: *u8 = sys_mmap(PAGE_MAGIC_1024)
  var o: i64 = er_cat(path, 0, dir as *u8)
  o = er_cat(path, o, "/page" as *u8)
  if idx < 100 { path[o] = 48 as u8; o = o + 1 }
  if idx < 10 { path[o] = 48 as u8; o = o + 1 }
  o = er_catn(path, o, idx)
  o = er_cat(path, o, ext as *u8); path[o] = 0 as u8
  let fd: i64 = sys_openat_wr(path, 0x1a4)
  if fd < 0 { return 0-1 }
  sys_write(fd, bytes, n)
  sys_close(fd)
  return 0
282}
283
284// write the page filename (page<NNN><ext>) into out (NUL-term); ret length
285func page_fname(out: *u8, idx: i64, ext: *u8) -> i64 {
  var o: i64 = er_cat(out, 0, "page" as *u8)
  if idx < 100 { out[o] = 48 as u8; o = o + 1 }
  if idx < 10 { out[o] = 48 as u8; o = o + 1 }
  o = er_catn(out, o, idx)
  o = er_cat(out, o, ext as *u8); out[o] = 0 as u8
  return o
292}
293
294// no-fake-green image signature check: 1 if the buffer begins with a REAL image magic among the formats
295// browsers render (JPEG ff d8 / PNG 89504e47 / GIF "GIF8" / BMP "BM" / WEBP "RIFF"...."WEBP").
296func img_sig_ok(b: *u8, n: i64) -> i64 {
  if n >= 2 { if (b[0] as i64) == 0xff { if (b[1] as i64) == 0xd8 { return 1 } } }                                                       // JPEG
  if n >= 4 { if (b[0] as i64) == 0x89 { if (b[1] as i64) == 0x50 { if (b[2] as i64) == 0x4e { if (b[3] as i64) == 0x47 { return 1 } } } } } // PNG
  if n >= 4 { if (b[0] as i64) == 0x47 { if (b[1] as i64) == 0x49 { if (b[2] as i64) == 0x46 { if (b[3] as i64) == 0x38 { return 1 } } } } } // GIF8
  if n >= 2 { if (b[0] as i64) == 0x42 { if (b[1] as i64) == 0x4d { return 1 } } }                                                        // BMP "BM"
  if n >= 12 { if (b[0] as i64) == 0x52 { if (b[1] as i64) == 0x49 { if (b[2] as i64) == 0x46 { if (b[3] as i64) == 0x46 { if (b[8] as i64) == 0x57 { if (b[9] as i64) == 0x45 { if (b[10] as i64) == 0x42 { if (b[11] as i64) == 0x50 { return 1 } } } } } } } } } // WEBP
  return 0
303}
304
305const ECAP: i64 = 4096   // max archive entries enumerated
306
307// ================= SHARED page emitter (zip + rar feed the same arrays) =================
308// meths[i]: 0 = raw copy at doffs[i] (zip stored / rar stored), 8 = zip deflate stream at doffs[i].
309// Filters junk names, NATURAL-sorts, extracts, PER-PAGE magic-gates, writes pages + cbx.json (with
310// "cfix":2 -- the server-side cache-heal tag; older cbx.json without it re-extracts on next open).
311// cover_only=1: extract ONLY the natural-first image to <dir>/cover.jpg (grid thumbnails must not pay
312// for a full 100MB+ unpack per cover request) -- no cbx.json, no page files.
313func emit_pages(z: *u8, slug: *u8, fmt: *u8, arena: *u8, noffs: *i64, meths: *i64, comps: *i64, doffs: *i64, nent: i64, cover_only: i64) -> i64 {
  let order: *i64 = sys_mmap(8 * ECAP) as *i64
  var nimg: i64 = 0
  var i: i64 = 0
  while i < nent {
      let nm: *u8 = (arena as i64 + noffs[i]) as *u8
      if is_junk_name(nm) == 0 { if is_image_name(nm) == 1 { order[nimg] = i; nimg = nimg + 1 } }
      i = i + 1
  }
  if nimg <= 0 { er_p("CBX-FAIL no-image-entries slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
323
  // insertion sort order[] by NATURAL name order = true page order
  var a: i64 = 1
  while a < nimg {
      let cur: i64 = order[a]
      let curnm: *u8 = (arena as i64 + noffs[cur]) as *u8
      var b: i64 = a - 1
      var stop: i64 = 0
      while stop == 0 {
          if b < 0 { stop = 1 }
          else {
              let bnm: *u8 = (arena as i64 + noffs[order[b]]) as *u8
              if name_cmp_nat(bnm, curnm) > 0 { order[b+1] = order[b]; b = b - 1 }
              else { stop = 1 }
          }
      }
      order[b+1] = cur
      a = a + 1
  }
342
  // per-slug output dir
  let dir: *u8 = sys_mmap(512)
  var dlen: i64 = er_cat(dir, 0, "knowledge/staging/media/reader/" as *u8)
  dlen = er_cat(dir, dlen, slug); dir[dlen] = 0 as u8
  ensure_dir(dir)
348
  // COVER-ONLY: write the natural-first REAL image as <dir>/cover.jpg and stop -- O(one page), not O(book)
  if cover_only == 1 {
      let cbuf: *u8 = sys_mmap(PAGE_CAP)
      var cw: i64 = 0
      var cp: i64 = 0
      while cp < nimg {
          if cw == 0 {
              let cei: i64 = order[cp]
              let cme: i64 = meths[cei]
              let cco: i64 = comps[cei]
              let cdo: i64 = doffs[cei]
              var cn: i64 = 0-1
              if cco <= PAGE_CAP {
                  if cme == 0 { var cc: i64 = 0; while cc < cco { cbuf[cc] = z[cdo + cc]; cc = cc + 1 } cn = cco }
                  if cme == 8 {
                      let cr: *NxDeflateResult = nx_deflate_inflate((z as i64 + cdo) as *u8, cco, PAGE_CAP)
                      if (cr as i64) != 0 { let crn: i64 = cr.output_size; var c3: i64 = 0; while c3 < crn { if c3 < PAGE_CAP { cbuf[c3] = cr.output_data[c3] } c3 = c3 + 1 } cn = crn }
                  }
              }
              if cn > 0 { if img_sig_ok(cbuf, cn) == 1 {
                  let cvp: *u8 = sys_mmap(600)
                  var cvo: i64 = er_cat(cvp, 0, dir as *u8); cvo = er_cat(cvp, cvo, "/cover.jpg" as *u8); cvp[cvo] = 0 as u8
                  let cvfd: i64 = sys_openat_wr(cvp, 0x1a4)
                  if cvfd >= 0 { sys_write(cvfd, cbuf, cn); sys_close(cvfd); cw = 1 }
              } }
          }
          cp = cp + 1
      }
      er_p("CBX-COVER slug=" as *u8); er_w(1, slug); er_p(" ok=" as *u8); er_pn(cw); er_p("\n" as *u8)
      if cw == 1 { return 0 }
      return 0-1
  }
381
  // cbx.json (pages array first; npages LAST = the post-gate truth, may be < nimg)
  let jp: *u8 = sys_mmap(512)
  var jo: i64 = er_cat(jp, 0, dir as *u8); jo = er_cat(jp, jo, "/cbx.json" as *u8); jp[jo] = 0 as u8
  let jfd: i64 = sys_openat_wr(jp, 0x1a4)
  if jfd < 0 { er_p("CBX-FAIL open-json slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
  er_w(jfd, "{\"title\":\"" as *u8); er_w(jfd, slug)
  er_w(jfd, "\",\"format\":\"" as *u8); er_w(jfd, fmt)
  er_w(jfd, "\",\"cfix\":2,\"pages\":[" as *u8)
390
  let pbuf: *u8 = sys_mmap(PAGE_CAP)
  let ext: *u8 = sys_mmap(16)
  let fname: *u8 = sys_mmap(64)
  var total_bytes: i64 = 0
  var written: i64 = 0
  var skip_magic: i64 = 0
  var skip_extract: i64 = 0
  var p: i64 = 0
  while p < nimg {
      let ei: i64 = order[p]
      let nm: *u8 = (arena as i64 + noffs[ei]) as *u8
      let method: i64 = meths[ei]
      let comp: i64 = comps[ei]
      let dataoff: i64 = doffs[ei]
      var pn: i64 = 0-1
      if comp > PAGE_CAP { pn = 0-1 } else {
          if method == 0 {
              var c: i64 = 0
              while c < comp { pbuf[c] = z[dataoff + c]; c = c + 1 }
              pn = comp
          }
          if method == 8 {
              let r: *NxDeflateResult = nx_deflate_inflate((z as i64 + dataoff) as *u8, comp, PAGE_CAP)
              if (r as i64) != 0 {
                  let rn: i64 = r.output_size
                  var c2: i64 = 0
                  while c2 < rn { if c2 < PAGE_CAP { pbuf[c2] = r.output_data[c2] } c2 = c2 + 1 }
                  pn = rn
              }
          }
      }
      if pn > 0 {
          // PER-PAGE no-fake-green gate: bytes must BE an image, or the entry is junk (AppleDouble,
          // stray text) -- skipped + counted, never written as a broken page.
          if img_sig_ok(pbuf, pn) == 1 {
              ext_of(nm, ext)
              write_page(dir, written, ext, pbuf, pn)
              page_fname(fname, written, ext)
              if written > 0 { er_w(jfd, "," as *u8) }
              er_w(jfd, "{\"idx\":" as *u8); er_n(jfd, written)
              er_w(jfd, ",\"file\":\"" as *u8); er_w(jfd, fname)
              er_w(jfd, "\",\"bytes\":" as *u8); er_n(jfd, pn)
              er_w(jfd, "}" as *u8)
              total_bytes = total_bytes + pn
              written = written + 1
          } else {
              skip_magic = skip_magic + 1
              er_p("CBX-SKIP nonimage-bytes entry=" as *u8); er_w(1, nm); er_p("\n" as *u8)
          }
      } else {
          skip_extract = skip_extract + 1
          er_p("CBX-SKIP extract-failed entry=" as *u8); er_w(1, nm); er_p("\n" as *u8)
      }
      p = p + 1
  }
  er_w(jfd, "],\"npages\":" as *u8); er_n(jfd, written)
  er_w(jfd, "}" as *u8)
  sys_close(jfd)
449
  er_p("CBX slug=" as *u8); er_w(1, slug)
  er_p(" source=" as *u8); er_w(1, fmt)
  er_p(" npages=" as *u8); er_pn(written)
  er_p(" skipped_nonimage=" as *u8); er_pn(skip_magic)
  er_p(" skipped_extract=" as *u8); er_pn(skip_extract)
  er_p(" total_bytes=" as *u8); er_pn(total_bytes)
  er_p("\n" as *u8)
457
  if written <= 0 { return 0-1 }
  return 0
460}
461
462// ================= RAR4 (Rar! 1a 07 00) STORE walker =================
463// Block: u16 crc, u8 type, u16 flags, u16 hsize. File header (type 0x74): packsize@7 unpsize@11
464// method@25 (0x30=stored) namesize@26 name@32 (@40 + high sizes if flags&0x100). Data at block+hsize.
465// Dirs (flags&0xe0==0xe0), encrypted (flags&4) and compressed (method!=0x30) entries are skip-counted.
466func rar4_enum(z: *u8, zl: i64, arena: *u8, acap: i64, noffs: *i64, meths: *i64, comps: *i64, doffs: *i64, ecap: i64, skipc: *i64) -> i64 {
  var pos: i64 = 7
  var n: i64 = 0
  var aoff: i64 = 0
  var go: i64 = 1
  while go == 1 {
      if pos + 7 > zl { go = 0 } else {
          let btype: i64 = z[pos+2] as i64
          let bflags: i64 = le16(z, pos+3)
          let hsize: i64 = le16(z, pos+5)
          if hsize < 7 { go = 0 } else {
              var datasz: i64 = 0
              if btype == 0x74 {
                  if pos + 32 > zl { go = 0 } else {
                      datasz = le32(z, pos+7)
                      let method: i64 = z[pos+25] as i64
                      let nmsz: i64 = le16(z, pos+26)
                      var nmoff: i64 = pos + 32
                      if (bflags & 0x100) != 0 { nmoff = pos + 40; datasz = datasz | (le32(z, pos+32) << 32) }
                      var isdir: i64 = 0
                      if (bflags & 0xe0) == 0xe0 { isdir = 1 }
                      var admit: i64 = 1
                      if isdir == 1 { admit = 0 }
                      if admit == 1 { if (bflags & 4) != 0 { admit = 0; skipc[0] = skipc[0] + 1 } }
                      if admit == 1 { if method != 0x30 { admit = 0; skipc[0] = skipc[0] + 1 } }
                      if n >= ecap { admit = 0 }
                      if nmoff + nmsz > zl { admit = 0 }
                      if datasz <= 0 { admit = 0 }
                      if admit == 1 { if aoff + nmsz + 1 < acap {
                          noffs[n] = aoff
                          var i: i64 = 0
                          var stopn: i64 = 0
                          while i < nmsz {
                              if stopn == 0 {
                                  let c: u8 = z[nmoff+i]
                                  if c == (0 as u8) { stopn = 1 } else { arena[aoff] = c; aoff = aoff + 1 }
                              }
                              i = i + 1
                          }
                          arena[aoff] = 0 as u8; aoff = aoff + 1
                          meths[n] = 0
                          comps[n] = datasz
                          doffs[n] = pos + hsize
                          n = n + 1
                      } }
                  }
              } else {
                  if btype == 0x7b { go = 0 }
                  if (bflags & 0x8000) != 0 { if pos + 11 <= zl { datasz = le32(z, pos+7) } }
              }
              if go == 1 {
                  let nxt: i64 = pos + hsize + datasz
                  if nxt <= pos { go = 0 } else { pos = nxt }
              }
          }
      }
  }
  return n
524}
525
526// RAR5 vint: 7-bit little-endian groups, bit7 = continuation. Advances pos[0]; bounded.
527func r5v(z: *u8, pos: *i64, zl: i64) -> i64 {
  var v: i64 = 0
  var sh: i64 = 0
  var go: i64 = 1
  while go == 1 {
      if pos[0] >= zl { go = 0 } else {
          if sh > 63 { go = 0 } else {
              let c: i64 = z[pos[0]] as i64
              pos[0] = pos[0] + 1
              v = v | ((c & 0x7f) << sh)
              sh = sh + 7
              if (c & 0x80) == 0 { go = 0 }
          }
      }
  }
  return v
543}
544
545// ================= RAR5 (Rar! 1a 07 01 00) STORE walker =================
546// Block: u32 crc, vint hsize, then header: vint type(2=file), vint flags(1=extra,2=data),
547// [vint extra_size], [vint data_size]; file: vint ff(1=dir,2=mtime,4=crc), vint unp, vint attr,
548// [u32 mtime], [u32 crc], vint cinfo (bits 7..9 = method, 0=store), vint host_os, vint namelen, name.
549// Data area follows the header (hstart+hsize); next block = data end.
550func rar5_enum(z: *u8, zl: i64, arena: *u8, acap: i64, noffs: *i64, meths: *i64, comps: *i64, doffs: *i64, ecap: i64, skipc: *i64) -> i64 {
  let pp: *i64 = sys_mmap(16) as *i64
  var pos: i64 = 8
  var n: i64 = 0
  var aoff: i64 = 0
  var go: i64 = 1
  while go == 1 {
      if pos + 8 > zl { go = 0 } else {
          pp[0] = pos + 4
          let hsz: i64 = r5v(z, pp, zl)
          let hstart: i64 = pp[0]
          if hsz <= 0 { go = 0 } else {
              if hstart + hsz > zl { go = 0 } else {
                  let htype: i64 = r5v(z, pp, zl)
                  let hflags: i64 = r5v(z, pp, zl)
                  var xsz: i64 = 0
                  if (hflags & 1) != 0 { xsz = r5v(z, pp, zl) }
                  var dsz: i64 = 0
                  if (hflags & 2) != 0 { dsz = r5v(z, pp, zl) }
                  if htype == 5 { go = 0 } else {
                      if htype == 2 {
                          let ff: i64 = r5v(z, pp, zl)
                          let unp: i64 = r5v(z, pp, zl)
                          let attr: i64 = r5v(z, pp, zl)
                          if (ff & 2) != 0 { pp[0] = pp[0] + 4 }
                          if (ff & 4) != 0 { pp[0] = pp[0] + 4 }
                          let cinfo: i64 = r5v(z, pp, zl)
                          let hos: i64 = r5v(z, pp, zl)
                          let nml: i64 = r5v(z, pp, zl)
                          let nmoff: i64 = pp[0]
                          let method: i64 = (cinfo >> 7) & 7
                          var admit: i64 = 1
                          if (ff & 1) != 0 { admit = 0 }
                          if admit == 1 { if method != 0 { admit = 0; skipc[0] = skipc[0] + 1 } }
                          if n >= ecap { admit = 0 }
                          if nmoff + nml > zl { admit = 0 }
                          if dsz <= 0 { admit = 0 }
                          if admit == 1 { if aoff + nml + 1 < acap {
                              noffs[n] = aoff
                              var i: i64 = 0
                              while i < nml { arena[aoff] = z[nmoff+i]; aoff = aoff + 1; i = i + 1 }
                              arena[aoff] = 0 as u8; aoff = aoff + 1
                              meths[n] = 0
                              comps[n] = dsz
                              doffs[n] = hstart + hsz
                              n = n + 1
                          } }
                      }
                      let nxt: i64 = hstart + hsz + dsz
                      if nxt <= pos { go = 0 } else { pos = nxt }
                  }
              }
          }
      }
  }
  return n
606}
607
608// ========================= PDF (DCTDecode) ===================
609func do_pdf(path: *u8, slug: *u8) -> i64 {
  let lb: *i64 = sys_mmap(16) as *i64
  let d: *u8 = sys_read_file(path, lb)
  if (d as i64) == 0 { er_p("CBX-FAIL pdf read slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
  let dl: i64 = lb[0]
614
  // count /DCTDecode -> if zero, HONEST unsupported (vector/text or FlateDecode-image)
  if find_sub(d, dl, "/DCTDecode" as *u8) < 0 {
      er_p("CBX-PDF-UNSUPPORTED slug=" as *u8); er_w(1, slug)
      er_p(" reason=no-DCTDecode-image-xobjects(vector/text-or-flate-image-pdf)\n" as *u8)
      return 0-2     // distinct from FAIL: honest skip
  }
621
  let dir: *u8 = sys_mmap(512)
  var dlen: i64 = er_cat(dir, 0, "knowledge/staging/media/reader/" as *u8)
  dlen = er_cat(dir, dlen, slug); dir[dlen] = 0 as u8
  ensure_dir(dir)
626
  let jp: *u8 = sys_mmap(512)
  var jo: i64 = er_cat(jp, 0, dir as *u8); jo = er_cat(jp, jo, "/cbx.json" as *u8); jp[jo] = 0 as u8
  let jfd: i64 = sys_openat_wr(jp, 0x1a4)
  if jfd < 0 { er_p("CBX-FAIL pdf open-json slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
  er_w(jfd, "{\"title\":\"" as *u8); er_w(jfd, slug)
  er_w(jfd, "\",\"format\":\"pdf-cbx\",\"cfix\":2,\"pages\":[" as *u8)
633
  let fname: *u8 = sys_mmap(64)
  var total_bytes: i64 = 0
  var any_ok: i64 = 0
  var written: i64 = 0
  var pos: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let dct: i64 = find_sub_from(d, dl, "/DCTDecode" as *u8, pos)
      if dct < 0 { go = 0 }
      else {
          let st: i64 = find_sub_from(d, dl, "stream" as *u8, dct)
          if st < 0 { go = 0 }
          else {
              var b: i64 = st + 6
              if b < dl { if (d[b] as i64) == 0x0d { b = b + 1 } }
              if b < dl { if (d[b] as i64) == 0x0a { b = b + 1 } }
              var is_jpeg: i64 = 0
              if b + 1 < dl { if (d[b] as i64) == 0xff { if (d[b+1] as i64) == 0xd8 { is_jpeg = 1 } } }
              let es: i64 = find_sub_from(d, dl, "endstream" as *u8, b)
              if is_jpeg == 1 { if es > b {
                  var e: i64 = es
                  if e > b { if (d[e-1] as i64) == 0x0a { e = e - 1 } }
                  if e > b { if (d[e-1] as i64) == 0x0d { e = e - 1 } }
                  let n: i64 = e - b
                  if n > 0 {
                      write_page(dir, written, ".jpg" as *u8, (d as i64 + b) as *u8, n)
                      page_fname(fname, written, ".jpg" as *u8)
                      if written > 0 { er_w(jfd, "," as *u8) }
                      er_w(jfd, "{\"idx\":" as *u8); er_n(jfd, written)
                      er_w(jfd, ",\"file\":\"" as *u8); er_w(jfd, fname)
                      er_w(jfd, "\",\"bytes\":" as *u8); er_n(jfd, n)
                      er_w(jfd, "}" as *u8)
                      total_bytes = total_bytes + n
                      if written == 0 { if (d[b] as i64) == 0xff { if (d[b+1] as i64) == 0xd8 { any_ok = 1 } } }
                      written = written + 1
                  }
              } }
              if es > 0 { pos = es + 9 } else { pos = b + 1 }
          }
      }
  }
  er_w(jfd, "],\"npages\":" as *u8); er_n(jfd, written)
  er_w(jfd, "}" as *u8)
  sys_close(jfd)
678
  er_p("CBX slug=" as *u8); er_w(1, slug)
  er_p(" source=pdf npages=" as *u8); er_pn(written)
  er_p(" total_bytes=" as *u8); er_pn(total_bytes)
  er_p(" first_magic_ok=" as *u8); er_pn(any_ok)
  er_p("\n" as *u8)
684
  if written <= 0 { return 0-1 }
  if any_ok != 1 { return 0-1 }
  return 0
688}
689
690// ========================= comic dispatcher (kept name do_cbz; now sniffs) ===============
691// Wild "cbz"/"cbr" files LIE about their container (zip renamed .cbr, rar renamed .cbz, pdf renamed
692// either) -- so route by MAGIC BYTES, never extension: PK->zip, Rar!->rar4/rar5 store, 7z->honest
693// unsupported, %PDF->the DCTDecode path. The extension only decided "is a comic candidate" upstream.
694func do_cbz(path: *u8, slug: *u8, cover_only: i64) -> i64 {
  let lb: *i64 = sys_mmap(16) as *i64
  let z: *u8 = sys_read_file(path, lb)
  if (z as i64) == 0 { er_p("CBX-FAIL comic read slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
  let zl: i64 = lb[0]
  if zl < 8 { er_p("CBX-FAIL comic tiny slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
700
  // %PDF -- a "comic" that is really a pdf
  if z[0] == (0x25 as u8) { if z[1] == (0x50 as u8) { if z[2] == (0x44 as u8) { if z[3] == (0x46 as u8) {
      return do_pdf(path, slug)
  } } } }
705
  // PK -- zip container (true cbz, or a renamed one)
  if z[0] == (0x50 as u8) { if z[1] == (0x4b as u8) {
      let arena: *u8 = sys_mmap(PAGE_MAGIC_1048576)
      let noffs: *i64 = sys_mmap(8 * ECAP) as *i64
      let meths: *i64 = sys_mmap(8 * ECAP) as *i64
      let comps: *i64 = sys_mmap(8 * ECAP) as *i64
      let doffs: *i64 = sys_mmap(8 * ECAP) as *i64
      let nent: i64 = zip_enum(z, zl, arena, PAGE_MAGIC_1048576, noffs, meths, comps, doffs, ECAP)
      if nent <= 0 { er_p("CBX-FAIL zip no-entries slug=" as *u8); er_w(1, slug); er_p("\n" as *u8); return 0-1 }
      return emit_pages(z, slug, "cbz" as *u8, arena, noffs, meths, comps, doffs, nent, cover_only)
  } }
717
  // Rar! 1a 07 -- rar4 (00) or rar5 (01 00)
  if z[0] == (0x52 as u8) { if z[1] == (0x61 as u8) { if z[2] == (0x72 as u8) { if z[3] == (0x21 as u8) {
      let arena: *u8 = sys_mmap(PAGE_MAGIC_1048576)
      let noffs: *i64 = sys_mmap(8 * ECAP) as *i64
      let meths: *i64 = sys_mmap(8 * ECAP) as *i64
      let comps: *i64 = sys_mmap(8 * ECAP) as *i64
      let doffs: *i64 = sys_mmap(8 * ECAP) as *i64
      let skipc: *i64 = sys_mmap(16) as *i64
      skipc[0] = 0
      var nent: i64 = 0
      var fmt: *u8 = "cbr-rar4" as *u8
      if (z[6] as i64) == 1 { fmt = "cbr-rar5" as *u8; nent = rar5_enum(z, zl, arena, PAGE_MAGIC_1048576, noffs, meths, comps, doffs, ECAP, skipc) }
      else { nent = rar4_enum(z, zl, arena, PAGE_MAGIC_1048576, noffs, meths, comps, doffs, ECAP, skipc) }
      if nent <= 0 {
          if skipc[0] > 0 {
              er_p("CBX-RAR-COMPRESSED-UNSUPPORTED slug=" as *u8); er_w(1, slug)
              er_p(" compressed_entries=" as *u8); er_pn(skipc[0])
              er_p(" (rar decompressor not built; STORED-jpeg rars extract fine)\n" as *u8)
              return 0-2
          }
          er_p("CBX-FAIL rar no-stored-entries slug=" as *u8); er_w(1, slug); er_p("\n" as *u8)
          return 0-1
      }
      if skipc[0] > 0 { er_p("CBX-NOTE rar compressed entries skipped=" as *u8); er_pn(skipc[0]); er_p("\n" as *u8) }
      return emit_pages(z, slug, fmt, arena, noffs, meths, comps, doffs, nent, cover_only)
  } } } }
744
  // 7z bc af 27 1c -- honest unsupported (no sovereign LZMA yet)
  if z[0] == (0x37 as u8) { if z[1] == (0x7a as u8) { if z[2] == (0xbc as u8) { if z[3] == (0xaf as u8) {
      er_p("CBX-7Z-UNSUPPORTED slug=" as *u8); er_w(1, slug); er_p(" reason=7z-lzma-not-built\n" as *u8)
      return 0-2
  } } } }
750
  er_p("CBX-FAIL comic unknown-container slug=" as *u8); er_w(1, slug)
  er_p(" magic0=" as *u8); er_pn(z[0] as i64)
  er_p(" magic1=" as *u8); er_pn(z[1] as i64)
  er_p("\n" as *u8)
  return 0-1
756}
757
758func main(argc: i64, argv: *i64) -> i64 {
  er_p("CBX: start (sniffed zip/rar4/rar5/pdf extract; natural sort; per-page magic gate; browser = last-mile zoom)\n" as *u8)
  ensure_dir("knowledge/staging/media/reader" as *u8)
761
  // argv-driven (additive): `nx_cbx_pages <path> <slug> [--cover]` extracts ANY comic/pdf on demand.
  // .pdf routes straight to do_pdf; .cbz/.cbr/.cb7 (any cb*) go to the MAGIC-sniffing dispatcher.
  // --cover extracts ONLY the natural-first image to <slug>/cover.jpg (fast grid thumbnails).
  if argc >= 3 {
      let apath: *u8 = argv[1] as *u8
      let aslug: *u8 = argv[2] as *u8
      var conly: i64 = 0
      if argc >= 4 {
          let a3: *u8 = argv[3] as *u8
          if a3[0] == (45 as u8) { if a3[1] == (45 as u8) { if a3[2] == (99 as u8) { conly = 1 } } }
      }
      var dot: i64 = 0 - 1; var ax: i64 = 0
      while apath[ax] != (0 as u8) { if apath[ax] == (46 as u8) { dot = ax } ax = ax + 1 }
      var ar: i64 = 0 - 1
      if dot >= 0 {
          if ((apath[dot+1] as i64) | 32) == 112 { if ((apath[dot+2] as i64) | 32) == 100 { if ((apath[dot+3] as i64) | 32) == 102 { ar = do_pdf(apath, aslug) } } }
          if ((apath[dot+1] as i64) | 32) == 99 { if ((apath[dot+2] as i64) | 32) == 98 { ar = do_cbz(apath, aslug, conly) } }
      }
      if ar == 0 { er_p("CBX-OK\n" as *u8); sys_exit(0); return 0 }
      er_p("CBX-FAIL\n" as *u8); sys_exit(1); return 1
  }
783
  var ok: i64 = 0
  var fail: i64 = 0
786
  // (A) the guaranteed win: a REAL comic archive
  let rc: i64 = do_cbz("/mnt/nas_homes/elderwesto/books/Unknown/dragonlancechronicles vol1 drag (1494)/dragonlancechronicles vol1 drag - Unknown.cbz" as *u8, "dragonlance_vol1" as *u8, 0)
  if rc == 0 { ok = ok + 1 } else { fail = fail + 1 }
790
  // (B) a REAL scanned/image PDF (honest: if vector/text -> UNSUPPORTED, not a fail)
  let rp: i64 = do_pdf("/mnt/nas_scans/insurance_home_and_car/20240713_181802_Raven_Scan.pdf" as *u8, "raven_insurance" as *u8)
  if rp == 0 { ok = ok + 1 }
  if rp == 0-1 { fail = fail + 1 }    // a real failure (UNSUPPORTED = 0-2 = honest skip, not a fail)
795
  er_p("CBX: produced=" as *u8); er_pn(ok); er_p(" failed=" as *u8); er_pn(fail); er_p("\n" as *u8)
  // GREEN requires the CBZ path to have rendered real zoomable pages (rc==0).
  if rc == 0 { if fail == 0 { er_p("CBX-OK\n" as *u8); sys_exit(0); return 0 } }
  er_p("CBX-FAIL\n" as *u8); sys_exit(1); return 1
800}
nx_cbx_pages.nx source

`nx_cbx_pages.nx` source
