code wiki / (root) / nx_archive_media_fetch.nx

nx_archive_media_fetch.nx

buildroot/runtime/nx_archive_media_fetch.nx

10078 B171 linesdepth 21pulls 148 transitivereach 0 importersview sourcekind tooltopic archive
docsdependenciesstructsconstsfunctions

about

nx_archive_media_fetch.nx -- R6 step A (fetch half): pull the ACTUAL media bytes of the reconstructed page3.com 1997 page and store them ourselves. Reads the cached CDX + page (web_assets/archive/_cache_*), extracts the page3.com <img> URLs, fetches each archived image (web/<ts14>id_/<imgurl>), saves to web_assets/archive/media/<basename>, content-addresses it (CID), and writes media/manifest.tsv (name<TAB>origurl<TAB>bytes<TAB>cid). Cache-first per image (skip if already saved) so we never re-hammer the archive. Fetch stack only -- decode/hash is the separate nx_archive_media_recovery (avoids the syscalls.nx/nx_syscalls.nx clash). Distinct from the existing nx_media_fetch (MediaPool primitive). license_tier: ORIGINAL

dependencies 6 imports · 0 importers

nx_syscalls.nx nx_x509_trust_store.nx nx_trust_store_load_from_certdata. nx_https_fetch_follow.nx nx_cdx_parse.nx nx_temporal_index.nx nx_archive_media_fetch.nx

imports: nx_syscalls.nxnx_x509_trust_store.nxnx_trust_store_load_from_certdata.nxnx_https_fetch_follow.nxnx_cdx_parse.nxnx_temporal_index.nx

imported by: nobody (leaf or entry point)

call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown

main gw sys_write sys_mkdir nx_trust_store_load_from_c sys_mmap sys_read_file sys_openat_rd sys_lseek sys_mmap ↻ sys_read sys_close nx_nss_certdata_parse sys_mmap ↻ _pat_class_cert _pat_value_octal _pat_end _find_newline _starts_with _parse_octal_line sys_mmap ↻ _is_space _decode_octal_escape _is_octal trust_store_alloc sys_mmap ↻ nx_x509_trust_store_load sys_mmap ↻ x509_parse sys_mmap ↻ asn1_cursor_init asn1_expect_tag sys_mmap ↻ asn1_read_tlv_header asn1_read_tag asn1_read_length asn1_read_length ↻ x509_read_tlv sys_mmap ↻ asn1_expect_tag ↻

structs

none

consts

14const K_MAGIC_1000000007: i64 = 1000000007
15const K_MAGIC_4194304: i64 = 4194304
16const K_MAGIC_4096: i64 = 4096
17const K_MAGIC_19970601: i64 = 19970601
18const K_MAGIC_2048: i64 = 2048

functions

20func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
called by 2: fetch_retrymain calls 1: sys_write
21func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
called by 2: fetch_retrymain calls 2: sys_writesys_mmap
22func ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
called by 1: main
23func apn(buf: *u8, off: i64, v: i64) -> i64 { if v==0 { buf[off]=0x30 as u8; return off+1 } var m: i64=v; let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var o: i64=off; var q: i64=k-1; while q>=0 { buf[o]=t[q]; o=o+1; q=q-1 } return o }
calls 1: sys_mmap
24func apslice(dst: *u8, off: i64, src: *u8, so: i64, sl: i64) -> i64 { var i: i64=0; while i<sl { dst[off+i]=src[so+i]; i=i+1 } return off+sl }
called by 1: main
25func cid_hash(buf: *u8, n: i64) -> i64 { var h: i64=0; var i: i64=0; while i<n { h=(h*131+(buf[i] as i64))%K_MAGIC_1000000007; i=i+1 } return h }
26func dstarts(buf: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 { if off+litlen>n { return 0 } var i: i64=0; while i<litlen { var a: i64=buf[off+i] as i64; if a>=0x41 { if a<=0x5a { a=a+0x20 } } if a != (lit[i] as i64) { return 0 } i=i+1 } return 1 }
called by 2: contains_litnext_imgsrc
27func contains_lit(buf: *u8, off: i64, len: i64, lit: *u8, litlen: i64) -> i64 { var i: i64=off; let e: i64=off+len-litlen; while i<=e { if dstarts(buf,i,off+len,lit,litlen)==1 { return 1 } i=i+1 } return 0 }
called by 1: main calls 1: dstarts
29func is_image(buf: *u8, n: i64) -> i64
38func save_file(path: *u8, buf: *u8, n: i64) -> i64 { let fd: i64=sys_openat_wr(path, 0x1a4); if fd<0 { return 0 } var off: i64=0; while off<n { let w: i64=sys_write(fd, ((buf as i64)+off) as *u8, n-off); if w<=0 { sys_close(fd); return 0 } off=off+w } sys_close(fd); return 1 }
calls 3: sys_openat_wrsys_writesys_close
39func read_file(path: *u8, buf: *u8, cap: i64, lenbox: *i64) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0 { return 0 } let n: i64=sys_read(fd, buf, cap); sys_close(fd); if n<=0 { return 0 } lenbox[0]=n; return n }
called by 1: main calls 3: sys_openat_rdsys_readsys_close
40func fetch_retry(url: *u8, store: *TrustStore, out: *u8, cap: i64, st: *i64, tries: i64) -> i64 { var t: i64=0; var n: i64=0-1; while t<tries { n=nx_https_fetch_follow(url, store, out, cap, 6, st); if n>0 { if st[0]==200 { t=tries } else { gw(" [status " as *u8); gn(st[0]); gw(" retry]\n" as *u8); t=t+1 } } else { gw(" [err " as *u8); gn(n); gw(" retry]\n" as *u8); t=t+1 } } return n }
calls 2: gwgn
41func next_imgsrc(buf: *u8, n: i64, from: i64, out2: *i64) -> i64
called by 1: main calls 1: dstarts
58func basename_of(buf: *u8, off: i64, len: i64, dst: *u8) -> i64
called by 1: main
81func main() -> i64
calls 15: gwsys_mkdirnx_trust_store_load_from_certdatasys_mmapread_filecdx_rows+9