`nx_arcmine.nx` source

↩ module page · 468 lines · 27072 B
1// nx_arcmine.nx -- INDEX-FIRST domain media harvester. The rung that makes a sovereign archive EXCEED
2// the Wayback Machine on media completeness (the 2026-06-29 goal: page3.com / megastar.co.uk day-by-day
3// WITH their media).
4//
5// module: nishi-core.archive.arcmine
6// depends: nx_medrec.nx (admission), nx_cdx_parse.nx (rows/fields), nx_https_fetch_follow.nx
7// usage: nx_arcmine <domain> [page] [min_bytes] [max_fetch] [mime]
8// capability: LIVE FETCH, read-only against the archive. Additive + IDEMPOTENT (Rule 10).
9//
10// *WHY INDEX-FIRST, AND WHY IT BEATS THE PAGE-WALK -- the whole point of this organ:
11// nx_archive_site_viewer discovers media by RENDERING a captured page and reading its <img> tags. That
12// ceiling is structural: it can only ever find media referenced by the pages it happens to render, at the
13// one timestamp it renders them. Every image on a page never rendered, on a page never captured, or
14// referenced only from a since-changed layout, is INVISIBLE to it -- and its absence is invisible too.
15// The CDX index knows every image the crawler EVER saw across the whole host and all time. Enumerating
16// from the index is exhaustive BY CONSTRUCTION rather than by luck of which page we happened to open.
17// That is the difference between browsing an archive and MINING one.
18//
19// *DEDUPE BY DIGEST, MEASURED 2026-08-06: captures of one URL are byte-identical. Real sample --
20// page3.com/3zine_pages/html_preview/01/ held 21 status-200 capture rows that collapse to just 6 unique
21// digests (01.jpg alone: 6 captures, ONE digest HSTRJJRQ.., sizes 27236..27245 = 9 bytes of WARC record
22// framing, not pixels). Fetching per-capture would spend 3.5x the bandwidth for byte-identical output.
23// collapse=digest is therefore applied at the QUERY, so the redundancy never crosses the network at all.
24// *AND THE COROLLARY THAT KILLED A PLANNED CHANGE: because captures are byte-identical, choosing the
25// LARGEST capture over the EARLIEST buys exactly nothing on this corpus. That tweak was measured, found
26// worthless, and NOT shipped.
27//
28// *TOMBSTONES ARE REJECTED FROM THE INDEX (nx_medrec), before a single byte is fetched -- a host's
29// "image not found" placeholder is a structurally valid JPEG that magic-byte validation cannot fail.
30//
31// *IDEMPOTENT BY CONSTRUCTION: the output filename IS the content digest, so a re-run rewrites identical
32// bytes and can never duplicate or corrupt. Safe to run twice, or to resume a partial page.
33// *PACING: max_fetch bounds one invocation deliberately. The archive is a shared resource and we have
34// rate-limited ourselves off hosts before by sweeping too fast; the operator advances pages at their own
35// cadence. OWED: wire pace_before(host) from nx_crawl_pace so politeness is structural, not procedural.
36// license_tier: ORIGINAL
37import "nx_medrec.nx"
38import "nx_x509_trust_store.nx"
39import "nx_trust_store_load_from_certdata.nx"
40import "nx_https_fetch_follow.nx"
41
42const AM_CAP:      i64 = 4194304
43const AM_ROWCAP:   i64 = 4000
44const AM_PATH:     i64 = 1024
45const AM_URLBUF:   i64 = 4096
46const AM_DIRMODE:  i64 = 0x1ed
47const AM_FILEMODE: i64 = 0x1a4
48const AM_DEF_MIN:  i64 = 2048
49const AM_DEF_MAX:  i64 = 40
50
51func aw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
52func an(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
53func aslice(buf: *u8, off: i64, len: i64) -> i64 { sys_write(1, ((buf as i64)+off) as *u8, len); return 0 }
54func ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
55func apsl(dst: *u8, off: i64, src: *u8, so: i64, sl: i64) -> i64 { var i: i64=0; while i<sl { dst[off+i]=src[so+i]; i=i+1 } return off+sl }
56func apn(buf: *u8, off: i64, v: i64) -> i64 { if v==0 { buf[off]=0x30 as u8; return off+1 } var m: i64=v; let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var o: i64=off; var q: i64=k-1; while q>=0 { buf[o]=t[q]; o=o+1; q=q-1 } return o }
57func s2i(s: *u8) -> i64 { var v: i64=0; var i: i64=0; var any: i64=0; while s[i]!=(0 as u8) { let c: i64=s[i] as i64; if c<0x30 { return 0-1 } if c>0x39 { return 0-1 } v=v*10+(c-0x30); any=1; i=i+1 } if any==0 { return 0-1 } return v }
58
59// real image magic -- the STRUCTURE check. Kept because a fetch can still return markup or a truncated
60// body even after the INDEX admitted the row; nx_medrec covers identity, this covers form.
61func is_image(buf: *u8, n: i64) -> i64 {
  if n<4 { return 0 }
  let a: i64=buf[0] as i64; let b: i64=buf[1] as i64; let c: i64=buf[2] as i64; let d: i64=buf[3] as i64
  if a==0x47 { if b==0x49 { if c==0x46 { return 1 } } }
  if a==0xff { if b==0xd8 { if c==0xff { return 1 } } }
  if a==0x89 { if b==0x50 { if c==0x4e { if d==0x47 { return 1 } } } }
  if a==0x42 { if b==0x4d { return 1 } }
  return 0
69}
70// *A PARTIAL WRITE MUST NEVER OCCUPY THE FINAL PATH. The skip check is a pure EXISTENCE test, so a
71// truncated file left by a failed write is skipped on every later run -- permanent corruption that
72// reports as "already had" forever and is invisible in the counters. Write to <path>.part and rename
73// into place only once the FULL byte count is down; a crash then leaves a .part (which the skip check
74// ignores and the next run overwrites), never half an image at the real name.
75// *Adding the skip optimisation silently changed the meaning of "the file is there" from "we fetched it"
76// to "something is at that path". This restores the first meaning, which is the one the skip relies on.
77func save_file(path: *u8, buf: *u8, n: i64) -> i64 {
  let tmp: *u8 = sys_mmap(AM_PATH)
  var t: i64 = ap(tmp, 0, path)
  t = ap(tmp, t, ".part" as *u8)
  tmp[t]=0 as u8
  let fd: i64=sys_openat_wr(tmp, AM_FILEMODE)
  if fd<0 { return 0 }
  var off: i64=0
  while off<n {
      let w: i64=sys_write(fd, ((buf as i64)+off) as *u8, n-off)
      if w<=0 { sys_close(fd); sys_unlinkat(tmp); return 0 }
      off=off+w
  }
  sys_close(fd)
  if off != n { sys_unlinkat(tmp); return 0 }
  if sys_renameat(tmp, path) < 0 { sys_unlinkat(tmp); return 0 }
  return 1
94}
95// *THE DIGEST IS KNOWN FROM THE INDEX BEFORE ANY FETCH, so an already-harvested image can be skipped
96// without spending a request. Measured need: re-running a page to continue past max_fetch was re-fetching
97// every image already stored -- idempotent, but it spends a shared rate-limited resource for zero gain.
98func have_file(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } sys_close(fd); return 1 }
99// *THE REVERSE INDEX THAT LETS THE PAGE RENDERER SKIP THE NETWORK ENTIRELY.
100// nx_archive_site_viewer rebuilds a captured page and must resolve each <img src> to bytes. Fetching them
101// is what makes it slow enough to be killed mid-run. But this harvester ALREADY stores those bytes,
102// content-addressed by digest -- the only thing missing is a URL -> digest lookup, because the sidecars are
103// keyed the other way. byurl/<canonhash>.txt closes that loop: the renderer becomes a local, network-free,
104// millisecond operation over whatever the harvester has already banked.
105//
106// *CANONICALISATION IS THE WHOLE TRICK. The CDX 'original' column reads http://www.page3.com:80/foo.jpg
107// while the page's own markup says /foo.jpg or http://www.page3.com/foo.jpg. Hashing either verbatim gives
108// two different keys and the join silently never matches -- it would look like "the archive doesn't have it"
109// rather than "we hashed it wrong". So both sides hash the SAME reduced form: scheme dropped, leading www.
110// dropped, :80/:443 dropped, host lowercased, query dropped, path kept verbatim.
111func canon_hash(buf: *u8, off: i64, len: i64) -> i64 {
  let end0: i64 = off + len
  var p: i64 = off
  var i: i64 = off
  var sch: i64 = 0 - 1
  while i + 2 < end0 {
      if buf[i]==(58 as u8) { if buf[i+1]==(47 as u8) { if buf[i+2]==(47 as u8) { if sch < 0 { sch = i + 3 } } } }
      i = i + 1
  }
  // *PATH ONLY -- the host is DELIBERATELY excluded. byurl/ already lives under
  // web_assets/archive/<host>/, so the host is established by the DIRECTORY and is redundant here.
  // Worse, including it is actively WRONG: the CDX 'original' column carries an absolute url
  // (http://www.page3.com:80/x.jpg) while a page's own markup usually carries a relative one (/x.jpg),
  // so a host+path key hashes those two forms DIFFERENTLY and the join never matches -- silently,
  // presenting as "the archive lacks this image" rather than "we hashed it wrong". Path-only makes
  // both forms agree by construction. (This replaced a host+path version that had exactly that bug,
  // caught by hand-trace before it ever ran.)
  if sch >= 0 {
      p = end0
      i = sch
      var found: i64 = 0
      while i < end0 {
          if found == 0 { if buf[i]==(47 as u8) { p = i; found = 1 } }
          i = i + 1
      }
  }
  var h: i64 = 1125899906842597
  i = p
  while i < end0 {
      let c: i64 = buf[i] as i64
      if c == 63 { i = end0 } else {
          // case folded: measured, megastar serves 3pxShadow.png whose CDX urlkey is 3pxshadow.png,
          // so case genuinely differs between the two sides of this join.
          h = (h * 131) + mr_lc(c)
          i = i + 1
      }
  }
  if h < 0 { h = 0 - h }
  return h & 0x7fffffffffffffff
150}
151func append_file(path: *u8, buf: *u8, n: i64) -> i64 {
  let fd: i64=sys_openat_append(path, AM_FILEMODE)
  if fd<0 { return 0 }
  var off: i64=0
  while off<n { let w: i64=sys_write(fd, ((buf as i64)+off) as *u8, n-off); if w<=0 { sys_close(fd); return 0 } off=off+w }
  sys_close(fd)
  return 1
158}
159func fetch_retry(url: *u8, store: *TrustStore, out: *u8, cap: i64, st: *i64, tries: i64) -> i64 {
  var t: i64=0; var n: i64=0-1
  while t<tries {
      n=nx_https_fetch_follow(url, store, out, cap, 6, st)
      if n>0 { if st[0]==200 { t=tries } else { t=t+1 } } else { t=t+1 }
  }
  return n
166}
167// *THE EXTENSION MUST COME FROM THE ROW, NOT FROM THE QUERY. Measured 2026-08-06: megastar.co.uk serves
168// babes-2.jpg with recorded mimetype image/PNG at 31,078 B -- a real content image whose extension and
169// content-type disagree. Deriving the stored extension from the query filter would name it .jpg, and
170// nx_archive_server sets Content-Type BY EXTENSION, so the browser would be handed image/jpeg for PNG
171// bytes and render nothing. Per-row mime is in the index, available BEFORE the fetch, so the skip check
172// still works. Era formats included (Rule 25) -- a 90s corpus is not all jpeg.
173func ext_for(mime: *u8) -> *u8 {
  if mr_ci_has(mime, "png")  == 1 { return ".png" }
  if mr_ci_has(mime, "gif")  == 1 { return ".gif" }
  if mr_ci_has(mime, "webp") == 1 { return ".webp" }
  if mr_ci_has(mime, "avif") == 1 { return ".avif" }
  if mr_ci_has(mime, "bmp")  == 1 { return ".bmp" }
  if mr_ci_has(mime, "icon") == 1 { return ".ico" }
  if mr_ci_has(mime, "svg")  == 1 { return ".svg" }
  if mr_ci_has(mime, "tiff") == 1 { return ".tif" }
  if mr_ci_has(mime, "xbitmap") == 1 { return ".xbm" }
  if mr_ci_has(mime, "pcx")  == 1 { return ".pcx" }
  return ".jpg"
185}
186// per-row extension: copy the mimetype slice out of the CDX row, then classify it.
187func ext_from_row(buf: *u8, off: i64, len: i64, scratch: *u8) -> *u8 {
  var i: i64 = 0
  while i < len { scratch[i] = buf[off+i]; i = i + 1 }
  scratch[len] = 0 as u8
  return ext_for(scratch)
192}
193
194// *THE SEARCHABLE SIGNAL LIVES IN THE FILENAME, AND CONTENT-ADDRESSING THROWS IT AWAY. Storing an image
195// as TEOIGMBT..jpg is exactly right for dedupe and catastrophic for retrieval: megastar.co.uk names its
196// files alex_simwise-1.jpg / alicia_keys.jpg / angels_and_demons-1.jpg -- person and subject names, the
197// precise tokens a query will use. A harvest nobody can query is not a search capability.
198// So each stored image also gets a TEXT SIDECAR carrying the original url, the TOKENISED filename
199// (separators -> spaces so BM25 can match "alex simwise"), host, capture timestamp, digest and size.
200// nx_corpus_ingest <metadir> <host> then ingests that directory into the domain's seg_store shard, and the
201// archive becomes queryable through the SAME front door as every other nishi-search corpus.
202// Rule 15/22: compose the proven ingester; do not write a second index writer.
203
204// THE WHOLE PATH, not just the filename -- query and extension stripped, separators folded to spaces.
205// *MEASURED CORRECTION 2026-08-06: the filename is the searchable signal only SOMETIMES. megastar.co.uk
206// mixes human-readable names (alex_simwise-1.jpg, alicia_keys.jpg, 280200_condoms.jpg) with opaque CMS ids
207// (sMEG01MTE0NDQxOTgxMzc.jpg, which tokenises to nothing a human would ever type). For that second class
208// the PATH is the only signal there is: /babenews/news/2006/04/07/ yields category and date. Tokenising the
209// last segment alone would have indexed a large slice of the corpus as unsearchable noise.
210func tok_name(dst: *u8, o: i64, src: *u8, so: i64, sl: i64) -> i64 {
  let end0: i64 = so + sl
  var sch: i64 = 0 - 1
  var i: i64 = so
  while i + 2 < end0 {
      if src[i]==(58 as u8) { if src[i+1]==(47 as u8) { if src[i+2]==(47 as u8) { if sch < 0 { sch = i + 3 } } } }
      i = i + 1
  }
  if sch < 0 { sch = so }
  var start: i64 = sch
  var found: i64 = 0
  i = sch
  while i < end0 {
      if found == 0 { if src[i]==(47 as u8) { start = i + 1; found = 1 } }
      i = i + 1
  }
  var end: i64 = end0
  i = start
  while i < end { if src[i]==(63 as u8) { end = i } else { i=i+1 } }
  var dot: i64 = 0-1
  i = start
  while i < end { if src[i]==(46 as u8) { dot = i } i=i+1 }
  if dot > start { end = dot }
  var p: i64 = o
  i = start
  while i < end {
      let c: i64 = src[i] as i64
      var w: i64 = c
      if c==95 { w=32 }
      if c==45 { w=32 }
      if c==43 { w=32 }
      if c==37 { w=32 }
      if c==47 { w=32 }
      dst[p]=w as u8
      p=p+1; i=i+1
  }
  return p
247}
248
249func main(argc: i64, argv: *i64) -> i64 {
  aw("=== nx_arcmine: INDEX-FIRST domain media harvest (exceed Wayback on completeness) ===\n" as *u8)
  if argc < 2 { aw("usage: nx_arcmine <domain> [page] [min_bytes] [max_fetch] [mime]\n" as *u8); return 2 }
  let host: *u8 = argv[1] as *u8
  var page: i64 = 0
  var minb: i64 = AM_DEF_MIN
  var maxf: i64 = AM_DEF_MAX
  var mime: *u8 = "image/.*" as *u8
  if argc >= 3 { let v: i64=s2i(argv[2] as *u8); if v>=0 { page=v } }
  if argc >= 4 { let v: i64=s2i(argv[3] as *u8); if v>=0 { minb=v } }
  if argc >= 5 { let v: i64=s2i(argv[4] as *u8); if v>=0 { maxf=v } }
  if argc >= 6 { mime = argv[5] as *u8 }
261
  aw("host=" as *u8); aw(host); aw("  page=" as *u8); an(page)
  aw("  min_bytes=" as *u8); an(minb); aw("  max_fetch=" as *u8); an(maxf)
  aw("  mime=" as *u8); aw(mime); aw("\n" as *u8)
265
  sys_mkdir("web_assets" as *u8, AM_DIRMODE)
  sys_mkdir("web_assets/archive" as *u8, AM_DIRMODE)
  let dir: *u8 = sys_mmap(AM_PATH)
  var dof: i64 = ap(dir, 0, "web_assets/archive/" as *u8)
  dof = ap(dir, dof, host); dir[dof]=0 as u8
  sys_mkdir(dir, AM_DIRMODE)
  let mdir: *u8 = sys_mmap(AM_PATH)
  var mof: i64 = ap(mdir, 0, dir); mof = ap(mdir, mof, "/media" as *u8); mdir[mof]=0 as u8
  sys_mkdir(mdir, AM_DIRMODE)
  let tdir: *u8 = sys_mmap(AM_PATH)
  var tof: i64 = ap(tdir, 0, dir); tof = ap(tdir, tof, "/meta" as *u8); tdir[tof]=0 as u8
  sys_mkdir(tdir, AM_DIRMODE)
  let bdir: *u8 = sys_mmap(AM_PATH)
  var bof: i64 = ap(bdir, 0, dir); bof = ap(bdir, bof, "/byurl" as *u8); bdir[bof]=0 as u8
  sys_mkdir(bdir, AM_DIRMODE)
  aw("media dir: " as *u8); aw(mdir); aw("\n" as *u8)
  aw("byurl dir: " as *u8); aw(bdir); aw("   (url->digest reverse index for the page renderer)\n" as *u8)
  aw("meta  dir: " as *u8); aw(tdir); aw("   (nx_corpus_ingest <metadir> " as *u8); aw(host); aw(" -> searchable)\n" as *u8)
284
  let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, AM_CAP)
  if r<=0 { aw("trust store load failed\n" as *u8); return 1 }
  let store: *TrustStore = r as *TrustStore
  let st: *i64 = sys_mmap(8) as *i64
289
  // ---- 1. ENUMERATE FROM THE INDEX. collapse=digest kills the byte-identical redundancy server-side.
  let cu: *u8 = sys_mmap(AM_URLBUF)
  var co: i64 = ap(cu, 0, "https://web.archive.org/cdx/search/cdx?url=" as *u8)
  co = ap(cu, co, host)
  co = ap(cu, co, "&matchType=domain&filter=statuscode:200&filter=mimetype:" as *u8)
  co = ap(cu, co, mime)
  co = ap(cu, co, "&collapse=digest&output=text&limit=300&page=" as *u8)
  co = apn(cu, co, page)
  cu[co]=0 as u8
  aw("\nCDX: " as *u8); aw(cu); aw("\n" as *u8)
300
  let cdxbuf: *u8 = sys_mmap(AM_CAP)
  let n: i64 = fetch_retry(cu, store, cdxbuf, AM_CAP, st, 4)
  if n<=0 { aw("CDX fetch failed (transport)\n" as *u8); return 1 }
  if st[0]!=200 { aw("CDX status " as *u8); an(st[0]); aw(" -- rate limited? back off and retry\n" as *u8); return 1 }
305
  let RS: *i64 = sys_mmap(AM_ROWCAP*8) as *i64
  let RE: *i64 = sys_mmap(AM_ROWCAP*8) as *i64
  let rows: i64 = cdx_rows(cdxbuf, n, RS, RE, AM_ROWCAP)
  aw("index rows (deduped by digest) = " as *u8); an(rows); aw("\n" as *u8)
  if rows<=0 { aw("page " as *u8); an(page); aw(" is EMPTY -- end of the index for this host/mime\n" as *u8); return 0 }
311
  // ---- 2. ADMIT FROM THE INDEX, then fetch only what survives.
  let o2: *i64 = sys_mmap(64) as *i64
  let ub: *u8 = sys_mmap(AM_URLBUF)
  let img: *u8 = sys_mmap(AM_CAP)
  let path: *u8 = sys_mmap(AM_PATH)
  let mpath: *u8 = sys_mmap(AM_PATH)
  let doc: *u8 = sys_mmap(AM_URLBUF)
  let mscratch: *u8 = sys_mmap(AM_PATH)
  let gpath: *u8 = sys_mmap(AM_PATH)
  let doc2: *u8 = sys_mmap(AM_URLBUF)
  let bpath: *u8 = sys_mmap(AM_PATH)
  let bref: *u8 = sys_mmap(AM_PATH)
324
  var seen: i64=0; var adm: i64=0; var rjt: i64=0; var rjs: i64=0; var rjz: i64=0; var rju: i64=0
  var got: i64=0; var bad: i64=0; var nonimg: i64=0; var bytes: i64=0; var skip: i64=0; var metas: i64=0; var byurls: i64=0
  var i: i64=0
  while i<rows {
      seen=seen+1
      let v: i64 = mr_row_admit(cdxbuf, RS[i], RE[i], minb, o2)
      if v == MR_REJ_TOMBSTONE { rjt=rjt+1 }
      if v == MR_REJ_STATUS    { rjs=rjs+1 }
      if v == MR_REJ_TOOSMALL  { rjz=rjz+1 }
      if v == MR_UNKNOWN       { rju=rju+1 }
      if v == MR_ADMIT {
          adm=adm+1
          // CONTENT-ADDRESS FIRST. The digest is an INDEX field, so the destination path is known
          // before any network decision -- which is what makes the skip below free.
          cdx_field(cdxbuf, RS[i], RE[i], MR_F_MIME, o2)
          let xt: *u8 = ext_from_row(cdxbuf, o2[0], o2[1], mscratch)
          cdx_field(cdxbuf, RS[i], RE[i], MR_F_DIGEST, o2)
          var p: i64 = ap(path, 0, mdir)
          p = ap(path, p, "/" as *u8)
          p = apsl(path, p, cdxbuf, o2[0], o2[1])
          p = ap(path, p, xt)
          path[p]=0 as u8
          // SEARCH SIDECAR, BUILT FROM INDEX DATA ONLY and written unconditionally. *This must not live
          // on the fetch path: images harvested before sidecars existed would then be SKIPPED and never
          // gain metadata -- permanently unsearchable, and silently so. Depending on no fetch makes the
          // skip path a BACKFILL instead of a dead end.
          var mp: i64 = ap(mpath, 0, tdir)
          mp = ap(mpath, mp, "/" as *u8)
          cdx_field(cdxbuf, RS[i], RE[i], MR_F_DIGEST, o2)
          mp = apsl(mpath, mp, cdxbuf, o2[0], o2[1])
          mp = ap(mpath, mp, ".txt" as *u8)
          mpath[mp]=0 as u8
          if have_file(mpath)==0 {
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_ORIG, o2)
              let suo: i64=o2[0]; let sul: i64=o2[1]
              var d: i64 = ap(doc, 0, "archived image\nurl: " as *u8)
              d = apsl(doc, d, cdxbuf, suo, sul)
              d = ap(doc, d, "\nname: " as *u8)
              d = tok_name(doc, d, cdxbuf, suo, sul)
              d = ap(doc, d, "\nhost: " as *u8)
              d = ap(doc, d, host)
              d = ap(doc, d, "\ncaptured: " as *u8)
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_TS, o2)
              d = apsl(doc, d, cdxbuf, o2[0], o2[1])
              d = ap(doc, d, "\ndigest: " as *u8)
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_DIGEST, o2)
              d = apsl(doc, d, cdxbuf, o2[0], o2[1])
              d = ap(doc, d, "\nindex_bytes: " as *u8)
              d = apn(doc, d, mr_row_size(cdxbuf, RS[i], RE[i], o2))
              d = ap(doc, d, "\nmedia: " as *u8)
              d = ap(doc, d, path)
              d = ap(doc, d, "\n" as *u8)
              if save_file(mpath, doc, d)==1 { metas=metas+1 }
              // BROWSABLE SURFACE, built incrementally. Appended inside the SAME have_file guard as the
              // sidecar, so exactly one tile exists per unique digest no matter how often a page is re-run.
              var g: i64 = ap(gpath, 0, dir); g = ap(gpath, g, "/index.html" as *u8); gpath[g]=0 as u8
              if have_file(gpath)==0 {
                  var h: i64 = ap(doc2, 0, "<!doctype html><meta charset=utf-8><title>archive: " as *u8)
                  h = ap(doc2, h, host)
                  h = ap(doc2, h, "</title><style>body{background:#111;color:#ddd;font:14px system-ui;margin:0;padding:16px}h1{font-size:16px;font-weight:600}a{color:#8bf}figure{display:inline-block;margin:6px;vertical-align:top;max-width:240px}img{max-width:240px;height:auto;display:block;background:#222}figcaption{font-size:11px;color:#999;word-break:break-all;padding-top:4px}</style>\n<h1>sovereign archive &mdash; " as *u8)
                  h = ap(doc2, h, host)
                  h = ap(doc2, h, " &mdash; recovered from the Wayback index</h1>\n" as *u8)
                  append_file(gpath, doc2, h)
              }
              var t: i64 = ap(doc2, 0, "<figure><a href=\"media/" as *u8)
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_DIGEST, o2)
              t = apsl(doc2, t, cdxbuf, o2[0], o2[1]); t = ap(doc2, t, xt)
              t = ap(doc2, t, "\"><img loading=lazy src=\"media/" as *u8)
              t = apsl(doc2, t, cdxbuf, o2[0], o2[1]); t = ap(doc2, t, xt)
              t = ap(doc2, t, "\"></a><figcaption>" as *u8)
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_ORIG, o2)
              t = tok_name(doc2, t, cdxbuf, o2[0], o2[1])
              t = ap(doc2, t, "<br>" as *u8)
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_TS, o2)
              t = apsl(doc2, t, cdxbuf, o2[0], o2[1])
              t = ap(doc2, t, "</figcaption></figure>\n" as *u8)
              append_file(gpath, doc2, t)
          }
          // *BYURL GETS ITS OWN GUARD -- it must NOT hang off the sidecar's.
          // Nested under have_file(mpath)==0 it could only ever be written when a SIDECAR WAS NEW, and
          // every one of the 727 megastar + 34 page3 sidecars already on disk predates byurl. The
          // reverse index would have been 0 of 761 while every run reported success, and the join it
          // exists to serve would have looked broken when it had simply never been populated.
          // *A SINGLE GUARD COVERING N INDEPENDENT ARTIFACTS MEANS N-1 OF THEM CAN NEVER BACKFILL.
          // The question is not "have I processed this row" but "does THIS artifact exist", and those
          // stop being the same question the moment a second artifact is added.
          cdx_field(cdxbuf, RS[i], RE[i], MR_F_ORIG, o2)
          var bp: i64 = ap(bpath, 0, bdir)
          bp = ap(bpath, bp, "/" as *u8)
          bp = apn(bpath, bp, mr_canon_hash(cdxbuf, o2[0], o2[1]))
          bp = ap(bpath, bp, ".txt" as *u8)
          bpath[bp]=0 as u8
          if have_file(bpath)==0 {
              cdx_field(cdxbuf, RS[i], RE[i], MR_F_DIGEST, o2)
              var bd: i64 = apsl(bref, 0, cdxbuf, o2[0], o2[1])
              bd = ap(bref, bd, xt)
              bref[bd]=0x0a as u8; bd=bd+1
              if save_file(bpath, bref, bd)==1 { byurls=byurls+1 }
          }
          if have_file(path)==1 { skip=skip+1 } else {
              if got < maxf {
                  // timestamp (field 1) + original url (field 2) -> the RAW archived bytes via id_
                  cdx_field(cdxbuf, RS[i], RE[i], MR_F_TS, o2)
                  let tso: i64=o2[0]; let tsl: i64=o2[1]
                  cdx_field(cdxbuf, RS[i], RE[i], MR_F_ORIG, o2)
                  let uo: i64=o2[0]; let ul: i64=o2[1]
                  var q: i64 = ap(ub, 0, "https://web.archive.org/web/" as *u8)
                  q = apsl(ub, q, cdxbuf, tso, tsl)
                  q = ap(ub, q, "id_/" as *u8)
                  q = apsl(ub, q, cdxbuf, uo, ul)
                  ub[q]=0 as u8
                  let m: i64 = fetch_retry(ub, store, img, AM_CAP, st, 3)
                  if m>0 {
                      if st[0]==200 {
                          if is_image(img, m)==1 {
                              if save_file(path, img, m)==1 { got=got+1; bytes=bytes+m } else { bad=bad+1 }
                          } else { nonimg=nonimg+1 }
                      } else { bad=bad+1 }
                  } else { bad=bad+1 }
              }
          }
      }
      i=i+1
  }
449
  aw("\n--- ARCMINE page " as *u8); an(page); aw(" ---\n" as *u8)
  aw("  index rows seen      = " as *u8); an(seen); aw("\n" as *u8)
  aw("  ADMITTED             = " as *u8); an(adm); aw("\n" as *u8)
  aw("  rejected tombstone   = " as *u8); an(rjt); aw("\n" as *u8)
  aw("  rejected status      = " as *u8); an(rjs); aw("\n" as *u8)
  aw("  rejected too-small   = " as *u8); an(rjz); aw("\n" as *u8)
  aw("  unknown (fail-closed)= " as *u8); an(rju); aw("\n" as *u8)
  aw("  FETCHED+STORED       = " as *u8); an(got); aw("\n" as *u8)
  aw("  fetch failed         = " as *u8); an(bad); aw("\n" as *u8)
  aw("  fetched-but-not-image= " as *u8); an(nonimg); aw("\n" as *u8)
  aw("  already had (skipped)= " as *u8); an(skip); aw("\n" as *u8)
  aw("  search sidecars      = " as *u8); an(metas); aw("\n" as *u8)
  aw("  byurl reverse-index  = " as *u8); an(byurls); aw("\n" as *u8)
  aw("  bytes stored         = " as *u8); an(bytes); aw("\n" as *u8)
  let remain: i64 = adm - skip - got
  if remain > 0 { aw("  NOTE: max_fetch capped this run -- " as *u8); an(remain); aw(" admitted rows NOT yet fetched. Re-run to continue.\n" as *u8) }
  aw("  next: nx_arcmine " as *u8); aw(host); aw(" " as *u8); an(page+1); aw("\n" as *u8)
  return 0
468}
nx_arcmine.nx source

`nx_arcmine.nx` source
