code wiki / _hdl_build / nx_img_harvest.nx
`nx_img_harvest.nx` source

↩ module page · 195 lines · 9945 B
1// nx_img_harvest.nx -- S-CLASS-EXCEED image-URL harvester (operator 2026-07-03: "no we want to get s class
2// exceed" -- the generic <img src>-only extractor missed LAZY-LOADED + JS/JSON-embedded pages, which is how
3// modern manga readers actually ship their page lists; gallery-dl's edge is exactly this). MULTI-STRATEGY, raw
4// scan of the page bytes so it is agnostic to WHERE the URL lives:
5//   * <img src=..>              (the old floor)
6//   * lazy attrs               data-src / data-original / data-lazy-src / data-url  (quoted values are just strings)
7//   * srcset="a 1x, b 2x"      (each comma/space token checked -> highest set of candidates)
8//   * JS / JSON page arrays    "pages":["https:\/\/cdn\/01.jpg", ...]  window.__data / __NEXT_DATA__ / chapter.images
9// Mechanism: scan for every QUOTED region (" or '), tokenize its content on space/comma, UNESCAPE JSON "\/"->"/",
10// and if a token LOOKS like an image URL (path ends .jpg/.jpeg/.png/.webp/.gif/.avif before ?/#; NOT data:/.svg/
11// sprite/favicon UI-junk) resolve it (absolute passes through; relative via the proven nx_url_resolve) + DEDUP.
12// Order = appearance order (matches reading order for HTML galleries + JSON arrays). Returns the count; fills the
13// same offs/lens/url_buf interface nx_html_extract_imgs uses, so nx_manga_get swaps it in with no other change.
14// The completeness magic-guard in nx_manga_get still fetch-verifies each (valid FULL file), so a stray non-page
15// URL that slips the heuristic simply fails the fetch -- never a blank/false page. license_tier: ORIGINAL
16import "nx_syscalls.nx"
17import "nx_html_extract_imgs.nx"   // nx_url_resolve + NX_URL_RESOLVE_OK (proven relative->absolute resolver)
18const K_MAGIC_8192: i64 = 8192
19
20// lowercase a byte
21func ih_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
22
23// does tok[0..tn) (already the raw token, may contain ?query) name an image by extension? (path = up to ?/#)
24// returns 1/0. Rejects data: URIs, .svg, and obvious UI junk (favicon/sprite) even if they end in an image ext.
25func ih_looks_img(tok: *u8, tn: i64) -> i64 {
  if tn < 5 { return 0 }
  // reject data: URIs
  if tn >= 5 { if ih_lc(tok[0] as i64)==100 { if ih_lc(tok[1] as i64)==97 { if ih_lc(tok[2] as i64)==116 { if ih_lc(tok[3] as i64)==97 { if tok[4]==(58 as u8) { return 0 } } } } } }  // "data:"
  // path end = first ? or #
  var end: i64 = tn
  var i: i64 = 0
  while i < tn { let c: i64 = tok[i] as i64; if c == 63 { if end == tn { end = i } } if c == 35 { if end == tn { end = i } } i = i + 1 }
  if end < 5 { return 0 }
  // last '.'
  var dot: i64 = 0 - 1
  var j: i64 = 0
  while j < end { if tok[j] == (46 as u8) { dot = j } j = j + 1 }
  if dot < 0 { return 0 }
  let el: i64 = end - dot
  if el < 4 { return 0 }
  if el > 5 { return 0 }
  // build lowercased ext
  let e: *u8 = sys_mmap(8)
  var k: i64 = 0
  while dot + k < end { if k < 7 { e[k] = ih_lc(tok[dot + k] as i64) as u8 } k = k + 1 }
  e[k] = 0 as u8
  var okext: i64 = 0
  if ih_streq(e, ".jpg" as *u8)==1 { okext = 1 }
  if ih_streq(e, ".jpeg" as *u8)==1 { okext = 1 }
  if ih_streq(e, ".png" as *u8)==1 { okext = 1 }
  if ih_streq(e, ".webp" as *u8)==1 { okext = 1 }
  if ih_streq(e, ".gif" as *u8)==1 { okext = 1 }
  if ih_streq(e, ".avif" as *u8)==1 { okext = 1 }
  if okext == 0 { return 0 }
  // UI-junk denylist by path substring (favicon/sprite/logo) -- unambiguous non-page assets
  if ih_has_ci(tok, end, "favicon" as *u8)==1 { return 0 }
  if ih_has_ci(tok, end, "sprite" as *u8)==1 { return 0 }
  return 1
59}
60func ih_streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]==(0 as u8){return 1} return 0 }
61// case-insensitive substring within hay[0..hn)
62func ih_has_ci(hay: *u8, hn: i64, needle: *u8) -> i64 {
  var nl: i64 = 0; while needle[nl] != (0 as u8) { nl = nl + 1 }
  if nl == 0 { return 0 }
  var i: i64 = 0
  while i + nl <= hn {
      var m: i64 = 1; var jx: i64 = 0
      while jx < nl { if ih_lc(hay[i+jx] as i64) != ih_lc(needle[jx] as i64) { m = 0; jx = nl } else { jx = jx + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
73}
74
75// copy tok[0..tn) into out, converting JSON escape "\/"->"/" and stopping at a backslash-u or control; NUL-term.
76func ih_unescape(tok: *u8, tn: i64, out: *u8, cap: i64) -> i64 {
  var o: i64 = 0
  var i: i64 = 0
  while i < tn {
      if o >= cap - 1 { i = tn } else {
          let c: i64 = tok[i] as i64
          if c == 92 { if i + 1 < tn { if tok[i+1] == (47 as u8) { out[o] = 47 as u8; o = o + 1; i = i + 2 } else { out[o] = tok[i] as u8; o = o + 1; i = i + 1 } } else { i = tn } }
          else { out[o] = tok[i]; o = o + 1; i = i + 1 }
      }
  }
  out[o] = 0 as u8
  return o
88}
89
90// already-collected? linear scan of url_buf via offs/lens (n small)
91func ih_seen(url_buf: *u8, offs: *i64, lens: *i64, count: i64, cand: *u8, cn: i64) -> i64 {
  var i: i64 = 0
  while i < count {
      if lens[i] == cn {
          var eq: i64 = 1; var k: i64 = 0
          let bp: *u8 = ((url_buf as i64) + offs[i]) as *u8
          while k < cn { if bp[k] != cand[k] { eq = 0; k = cn } else { k = k + 1 } }
          if eq == 1 { return 1 }
      }
      i = i + 1
  }
  return 0
103}
104
105// resolve a candidate (unescaped) -> resolved into out; returns length or 0 on fail. Absolute http(s) passes
106// through; protocol-relative //h/p gets https:; else nx_url_resolve against base.
107func ih_resolve(base: *u8, blen: i64, cand: *u8, cn: i64, out: *u8, cap: i64) -> i64 {
  // http:// or https://
  if cn >= 7 { if ih_lc(cand[0] as i64)==104 { if ih_lc(cand[1] as i64)==116 { if ih_lc(cand[2] as i64)==116 { if ih_lc(cand[3] as i64)==112 {
      var s: i64 = 4
      if ih_lc(cand[4] as i64)==115 { s = 5 }
      if cand[s]==(58 as u8) { if cand[s+1]==(47 as u8) { if cand[s+2]==(47 as u8) {
          var o: i64 = 0; while o < cn { if o < cap - 1 { out[o] = cand[o] } o = o + 1 } out[cn] = 0 as u8; return cn
      } } }
  } } } } }
  // protocol-relative //host/path -> https:
  if cn >= 2 { if cand[0]==(47 as u8) { if cand[1]==(47 as u8) {
      let pfx: *u8 = "https:" as *u8
      var o: i64 = 0; while o < 6 { out[o] = pfx[o]; o = o + 1 }
      var q: i64 = 0; while q < cn { if o < cap - 1 { out[o] = cand[q]; o = o + 1 } q = q + 1 }
      out[o] = 0 as u8; return o
  } } }
  // relative -> proven resolver
  let olp: *i64 = sys_mmap(8) as *i64
  let rc: i64 = nx_url_resolve(base, blen, cand, cn, out, cap, olp)
  if rc == NX_URL_RESOLVE_OK { return olp[0] }
  return 0
128}
129
130// consider one raw token (a whitespace/comma-delimited piece of a quoted region): if it looks like an image URL,
131// unescape + resolve + dedup + append. Returns the NEW count. Shared by the quoted-region loop.
132func ih_consider(tok: *u8, tn: i64, base_url: *u8, base_url_len: i64,
               url_buf: *u8, url_buf_cap: i64, offs: *i64, lens: *i64, max_imgs: i64,
               count: i64, buf_pos_p: *i64, unesc: *u8, resolved: *u8) -> i64 {
  if count >= max_imgs { return count }
  if tn < 5 { return count }
  let un: i64 = ih_unescape(tok, tn, unesc, K_MAGIC_8192)
  if ih_looks_img(unesc, un) == 0 { return count }
  let rn: i64 = ih_resolve(base_url, base_url_len, unesc, un, resolved, K_MAGIC_8192)
  if rn <= 0 { return count }
  if ih_seen(url_buf, offs, lens, count, resolved, rn) == 1 { return count }
  if buf_pos_p[0] + rn >= url_buf_cap { return count }
  var k: i64 = 0
  while k < rn { url_buf[buf_pos_p[0] + k] = resolved[k]; k = k + 1 }
  offs[count] = buf_pos_p[0]
  lens[count] = rn
  buf_pos_p[0] = buf_pos_p[0] + rn
  return count + 1
149}
150
151// THE HARVEST. See header. Returns count; fills offs/lens (into url_buf) up to max_imgs.
152func nx_img_harvest(html: *u8, hlen: i64, base_url: *u8, base_url_len: i64,
                  url_buf: *u8, url_buf_cap: i64, offs: *i64, lens: *i64, max_imgs: i64) -> i64 {
  var count: i64 = 0
  let bpp: *i64 = sys_mmap(8) as *i64; bpp[0] = 0
  let tokbuf: *u8 = sys_mmap(K_MAGIC_8192)
  let unesc: *u8 = sys_mmap(K_MAGIC_8192)
  let resolved: *u8 = sys_mmap(K_MAGIC_8192)
  var i: i64 = 0
  while i < hlen {
      if count >= max_imgs { i = hlen } else {
          let ch: i64 = html[i] as i64
          var q: i64 = 0
          if ch == 34 { q = 1 }
          if ch == 39 { q = 1 }
          if q == 1 {
              // read the quoted region [i+1, close)
              var s: i64 = i + 1
              var e: i64 = s
              var go: i64 = 1
              while go == 1 { if e >= hlen { go = 0 } else { if (html[e] as i64) == ch { go = 0 } else { e = e + 1 } } }
              // tokenize [s,e) on space / comma / tab / newline into tokbuf; consider each token
              var ts: i64 = s
              var p: i64 = s
              while p <= e {
                  var brk: i64 = 0
                  if p == e { brk = 1 } else { let c2: i64 = html[p] as i64; if c2 == 32 { brk = 1 } if c2 == 9 { brk = 1 } if c2 == 10 { brk = 1 } if c2 == 13 { brk = 1 } if c2 == 44 { brk = 1 } }
                  if brk == 1 {
                      let tl: i64 = p - ts
                      if tl > 0 { if tl < K_MAGIC_8192 {
                          var w: i64 = 0
                          while w < tl { tokbuf[w] = html[ts + w]; w = w + 1 }
                          tokbuf[tl] = 0 as u8
                          count = ih_consider(tokbuf, tl, base_url, base_url_len, url_buf, url_buf_cap, offs, lens, max_imgs, count, bpp, unesc, resolved)
                      } }
                      ts = p + 1
                  }
                  p = p + 1
              }
              if e < hlen { i = e + 1 } else { i = hlen }
          } else { i = i + 1 }
      }
  }
  return count
195}
nx_img_harvest.nx source

`nx_img_harvest.nx` source
