code wiki / (root) / nx_archive_media_fetch.nx
`nx_archive_media_fetch.nx` source

↩ module page · 171 lines · 10078 B
1// nx_archive_media_fetch.nx -- R6 step A (fetch half): pull the ACTUAL media bytes of the reconstructed page3.com
2// 1997 page and store them ourselves. Reads the cached CDX + page (web_assets/archive/_cache_*), extracts the
3// page3.com <img> URLs, fetches each archived image (web/<ts14>id_/<imgurl>), saves to
4// web_assets/archive/media/<basename>, content-addresses it (CID), and writes media/manifest.tsv
5// (name<TAB>origurl<TAB>bytes<TAB>cid). Cache-first per image (skip if already saved) so we never re-hammer the
6// archive. Fetch stack only -- decode/hash is the separate nx_archive_media_recovery (avoids the
7// syscalls.nx/nx_syscalls.nx clash). Distinct from the existing nx_media_fetch (MediaPool primitive). license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_x509_trust_store.nx"
10import "nx_trust_store_load_from_certdata.nx"
11import "nx_https_fetch_follow.nx"
12import "nx_cdx_parse.nx"
13import "nx_temporal_index.nx"
14const K_MAGIC_1000000007: i64 = 1000000007
15const K_MAGIC_4194304: i64 = 4194304
16const K_MAGIC_4096: i64 = 4096
17const K_MAGIC_19970601: i64 = 19970601
18const K_MAGIC_2048: i64 = 2048
19
20func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
21func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
22func ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
23func apn(buf: *u8, off: i64, v: i64) -> i64 { if v==0 { buf[off]=0x30 as u8; return off+1 } var m: i64=v; let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var o: i64=off; var q: i64=k-1; while q>=0 { buf[o]=t[q]; o=o+1; q=q-1 } return o }
24func apslice(dst: *u8, off: i64, src: *u8, so: i64, sl: i64) -> i64 { var i: i64=0; while i<sl { dst[off+i]=src[so+i]; i=i+1 } return off+sl }
25func cid_hash(buf: *u8, n: i64) -> i64 { var h: i64=0; var i: i64=0; while i<n { h=(h*131+(buf[i] as i64))%K_MAGIC_1000000007; i=i+1 } return h }
26func dstarts(buf: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 { if off+litlen>n { return 0 } var i: i64=0; while i<litlen { var a: i64=buf[off+i] as i64; if a>=0x41 { if a<=0x5a { a=a+0x20 } } if a != (lit[i] as i64) { return 0 } i=i+1 } return 1 }
27func contains_lit(buf: *u8, off: i64, len: i64, lit: *u8, litlen: i64) -> i64 { var i: i64=off; let e: i64=off+len-litlen; while i<=e { if dstarts(buf,i,off+len,lit,litlen)==1 { return 1 } i=i+1 } return 0 }
28// real-image magic check (rejects 404 HTML pages saved as .gif): GIF / JPEG / PNG / BMP.
29func is_image(buf: *u8, n: i64) -> i64 {
  if n<4 { return 0 }
  let a: i64=buf[0] as i64; let b: i64=buf[1] as i64; let c: i64=buf[2] as i64; let d: i64=buf[3] as i64
  if a==0x47 { if b==0x49 { if c==0x46 { return 1 } } }
  if a==0xff { if b==0xd8 { if c==0xff { return 1 } } }
  if a==0x89 { if b==0x50 { if c==0x4e { if d==0x47 { return 1 } } } }
  if a==0x42 { if b==0x4d { return 1 } }
  return 0
37}
38func save_file(path: *u8, buf: *u8, n: i64) -> i64 { let fd: i64=sys_openat_wr(path, 0x1a4); if fd<0 { return 0 } var off: i64=0; while off<n { let w: i64=sys_write(fd, ((buf as i64)+off) as *u8, n-off); if w<=0 { sys_close(fd); return 0 } off=off+w } sys_close(fd); return 1 }
39func read_file(path: *u8, buf: *u8, cap: i64, lenbox: *i64) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } let n: i64=sys_read(fd, buf, cap); sys_close(fd); if n<=0 { return 0 } lenbox[0]=n; return n }
40func fetch_retry(url: *u8, store: *TrustStore, out: *u8, cap: i64, st: *i64, tries: i64) -> i64 { var t: i64=0; var n: i64=0-1; while t<tries { n=nx_https_fetch_follow(url, store, out, cap, 6, st); if n>0 { if st[0]==200 { t=tries } else { gw("    [status " as *u8); gn(st[0]); gw(" retry]\n" as *u8); t=t+1 } } else { gw("    [err " as *u8); gn(n); gw(" retry]\n" as *u8); t=t+1 } } return n }
41func next_imgsrc(buf: *u8, n: i64, from: i64, out2: *i64) -> i64 {
  var i: i64=from
  while i<n {
      if dstarts(buf, i, n, "src=" as *u8, 4)==1 {
          if i+4<n { if (buf[i+4] as i64)==0x22 {
              let us: i64=i+5
              var ue: i64=us
              var go: i64=1
              while go==1 { if ue<n { if (buf[ue] as i64)==0x22 { go=0 } else { ue=ue+1 } } else { go=0 } }
              out2[0]=us; out2[1]=ue-us
              return ue+1
          } }
      }
      i=i+1
  }
  return 0-1
57}
58func basename_of(buf: *u8, off: i64, len: i64, dst: *u8) -> i64 {
  var last: i64=off-1
  var i: i64=off
  while i<off+len { if (buf[i] as i64)==0x2f { last=i } i=i+1 }
  var o: i64=0
  i=last+1
  while i<off+len {
      let c: i64=buf[i] as i64
      var keep: i64=0
      if c>=0x30 { if c<=0x39 { keep=1 } }
      if c>=0x41 { if c<=0x5a { keep=1 } }
      if c>=0x61 { if c<=0x7a { keep=1 } }
      if c==0x2e { keep=1 }
      if c==0x2d { keep=1 }
      if c==0x5f { keep=1 }
      if keep==1 { dst[o]=c as u8; o=o+1 }
      i=i+1
  }
  if o==0 { dst[0]=0x78 as u8; o=1 }
  dst[o]=0 as u8
  return o
79}
80
81func main() -> i64 {
  gw("=== nx_archive_media_fetch: pull + store the real page3.com 1997 media bytes ===\n" as *u8)
  sys_mkdir("web_assets" as *u8, 0x1ed)
  sys_mkdir("web_assets/archive" as *u8, 0x1ed)
  sys_mkdir("web_assets/archive/media" as *u8, 0x1ed)
  let r: i64=nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
  if r<=0 { gw("trust store load failed\n" as *u8); return 1 }
  let store: *TrustStore=r as *TrustStore
  let cap: i64=K_MAGIC_4194304
  let lb: *i64=sys_mmap(8) as *i64
  let st: *i64=sys_mmap(8) as *i64
92
  let cdxbuf: *u8=sys_mmap(cap)
  if read_file("web_assets/archive/_cache_cdx.txt" as *u8, cdxbuf, cap, lb)<=0 { gw("no cached CDX -- run nx_archive_viewer first\n" as *u8); return 1 }
  let ncdx: i64=lb[0]
  let RS: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let RE: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let nrows: i64=cdx_rows(cdxbuf, ncdx, RS, RE, 480)
  let keys: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let dates: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let tso: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let tsl: *i64=sys_mmap(K_MAGIC_4096) as *i64
  let b2: *i64=sys_mmap(16) as *i64
  var i: i64=0
  var mm: i64=0
  while i<nrows {
      cdx_field(cdxbuf, RS[i], RE[i], 4, b2)
      var is200: i64=0
      if b2[1]==3 { if (cdxbuf[b2[0]] as i64)==0x32 { if (cdxbuf[b2[0]+1] as i64)==0x30 { if (cdxbuf[b2[0]+2] as i64)==0x30 { is200=1 } } } }
      if is200==1 {
          cdx_field(cdxbuf, RS[i], RE[i], 0, b2)
          let kb: *u8=sys_mmap(256); var j: i64=0; while j<b2[1] { kb[j]=cdxbuf[b2[0]+j]; j=j+1 } kb[b2[1]]=0 as u8; keys[mm]=kb as i64
          cdx_field(cdxbuf, RS[i], RE[i], 1, b2); dates[mm]=cdx_ts8(cdxbuf, b2[0], b2[1]); tso[mm]=b2[0]; tsl[mm]=b2[1]
          mm=mm+1
      }
      i=i+1
  }
  let q: i64=ti_query(keys, dates, mm, "com,page3)/" as *u8, K_MAGIC_19970601)
  if q<0 { gw("no as-of-1997 capture\n" as *u8); return 1 }
  gw("capture " as *u8); gn(dates[q]); gw("\n" as *u8)
121
  let html: *u8=sys_mmap(cap)
  if read_file("web_assets/archive/_cache_page.txt" as *u8, html, cap, lb)<=0 { gw("no cached page -- run nx_archive_viewer first\n" as *u8); return 1 }
  let nhtml: i64=lb[0]
125
  let img: *u8=sys_mmap(cap)
  let man: *u8=sys_mmap(cap)
  var mo: i64=0
  let o2: *i64=sys_mmap(16) as *i64
  let nm: *u8=sys_mmap(256)
  let url: *u8=sys_mmap(K_MAGIC_2048)
  let path: *u8=sys_mmap(512)
  var from: i64=0
  var grabbed: i64=0
  var seen: i64=0
  var more: i64=1
  while more==1 {
      let p: i64=next_imgsrc(html, nhtml, from, o2)
      if p<0 { more=0 } else {
          from=p
          let uoff: i64=o2[0]; let ulen: i64=o2[1]
          seen=seen+1
          if contains_lit(html, uoff, ulen, "page3.com" as *u8, 9)==1 {
              if grabbed<8 {
                  let nl: i64=basename_of(html, uoff, ulen, nm)
                  var po: i64=0; po=ap(path, po, "web_assets/archive/media/" as *u8); po=apslice(path, po, nm, 0, nl); path[po]=0 as u8
                  var n3: i64=read_file(path, img, cap, lb)
                  var cached: i64=0
                  if n3>0 { n3=lb[0]; cached=1 } else {
                      var uo: i64=0; uo=ap(url, uo, "https://web.archive.org/web/" as *u8); uo=apslice(url, uo, cdxbuf, tso[q], tsl[q]); uo=ap(url, uo, "id_/" as *u8); uo=apslice(url, uo, html, uoff, ulen); url[uo]=0 as u8
                      gw("  fetch " as *u8); gw(nm); gw(" ...\n" as *u8)
                      n3=fetch_retry(url, store, img, cap, st, 6)
                  }
                  if n3>0 { if is_image(img, n3)==1 {
                      if cached==0 { save_file(path, img, n3) }
                      let cid: i64=cid_hash(img, n3)
                      mo=ap(man, mo, nm); man[mo]=0x09 as u8; mo=mo+1
                      mo=apslice(man, mo, html, uoff, ulen); man[mo]=0x09 as u8; mo=mo+1
                      mo=apn(man, mo, n3); man[mo]=0x09 as u8; mo=mo+1
                      mo=apn(man, mo, cid); man[mo]=0x0a as u8; mo=mo+1
                      gw("    " as *u8); gw(nm); gw("  bytes=" as *u8); gn(n3); gw("  cid=" as *u8); gn(cid); if cached==1 { gw("  [cache]" as *u8) } gw("\n" as *u8)
                      grabbed=grabbed+1
                  } else { gw("    " as *u8); gw(nm); gw("  [skip: not an image / 404 page, " as *u8); gn(n3); gw("B]\n" as *u8) } }
              }
          }
      }
  }
  save_file("web_assets/archive/media/manifest.tsv" as *u8, man, mo)
  gw("\nMEDIA-FETCH DONE: img-refs seen=" as *u8); gn(seen); gw(" page3 media grabbed=" as *u8); gn(grabbed); gw(" -> web_assets/archive/media/ + manifest.tsv\n" as *u8)
  return 0
171}
nx_archive_media_fetch.nx source

`nx_archive_media_fetch.nx` source
