nx_archive_capture_demo.nx source
↩ module page · 120 lines · 7408 B
1// nx_archive_capture_demo.nx -- R4 LIVE end-to-end: browse page3.com AS IT EXISTED in 1997, with its media list.
2// Pipeline: fetch page3.com's REAL Wayback CDX -> cdx_parse -> nx_temporal_index -> query as-of-1997 -> build the raw
3// archived-content URL (web.archive.org/web/<ts14>id_/<original>) -> fetch the ACTUAL 1997 page -> content-address it
4// (CID = polynomial hash, the seg_store key) -> scan its HTML for media references (the per-page media shopping-list).
5// Composes the VERIFIED nx_cdx_parse + nx_temporal_index + our sovereign TLS-1.3 fetch, with retry-on-failure
6// (graceful degradation -- web.archive.org occasionally drops a connection). Live, read-only. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11import "nx_cdx_parse.nx"
12import "nx_temporal_index.nx"
13const K_MAGIC_1000000007: i64 = 1000000007
14const K_MAGIC_4194304: i64 = 4194304
15const K_MAGIC_2048: i64 = 2048
16const K_MAGIC_19970601: i64 = 19970601
17
18func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
19func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
20func ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
21func apslice(dst: *u8, off: i64, src: *u8, so: i64, sl: i64) -> i64 { var i: i64=0; while i<sl { dst[off+i]=src[so+i]; i=i+1 } return off+sl }
22func phead(out: *u8, n: i64, lim: i64) -> i64 { var m: i64=n; if m<0 { m=0 } if m>lim { m=lim } if m>0 { sys_write(1,out,m) } sys_write(1,"\n" as *u8,1); return 0 }
23func cid_hash(buf: *u8, n: i64) -> i64 { var h: i64=0; var i: i64=0; while i<n { h = (h*131 + (buf[i] as i64)) % K_MAGIC_1000000007; i=i+1 } return h }
24func dstarts(buf: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 { if off+litlen>n { return 0 } var i: i64=0; while i<litlen { var a: i64=buf[off+i] as i64; if a>=0x41 { if a<=0x5a { a=a+0x20 } } if a != (lit[i] as i64) { return 0 } i=i+1 } return 1 }
25func count_img(buf: *u8, n: i64) -> i64 { var c: i64=0; var i: i64=0; while i+4<=n { if dstarts(buf,i,n,"<img" as *u8,4)==1 { c=c+1; i=i+4 } else { i=i+1 } } return c }
26// fetch with retry-on-failure (transient web.archive.org connection drops). Returns bytes (>0) or last error.
27func fetch_retry(url: *u8, store: *TrustStore, out: *u8, cap: i64, st: *i64, tries: i64) -> i64 {
28 var t: i64 = 0
29 var n: i64 = 0 - 1
30 while t < tries {
31 n = nx_https_fetch_follow(url, store, out, cap, 6, st)
32 if n > 0 { t = tries } else { gw(" [retry " as *u8); gn(t); gw(" got " as *u8); gn(n); gw("]\n" as *u8); t = t + 1 }
33 }
34 return n
35}
36
37func main() -> i64 {
38 gw("=== nx_archive_capture_demo: browse page3.com AS IT EXISTED in 1997 (live, sovereign) ===\n" as *u8)
39 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
40 if r <= 0 { gw("trust store load failed\n" as *u8); return 1 }
41 let store: *TrustStore = r as *TrustStore
42 let cap: i64 = K_MAGIC_4194304
43 let cdxbuf: *u8 = sys_mmap(cap)
44 let st: *i64 = sys_mmap(8) as *i64
45
46 // 1. fetch page3.com's REAL CDX (status-200 filtered in-code below)
47 gw("\n[1] fetch page3.com CDX (retry-on-drop) ...\n" as *u8)
48 let ncdx: i64 = fetch_retry("https://web.archive.org/cdx/search/cdx?url=page3.com&limit=80" as *u8, store, cdxbuf, cap, st, 5)
49 gw(" CDX status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(ncdx); gw("\n" as *u8)
50 if ncdx <= 0 { gw(" CDX fetch failed after retries -- abort\n" as *u8); return 1 }
51
52 // 2. parse rows -> temporal index, keeping only status-200 captures (+ ts14 / original spans)
53 let RS: *i64 = sys_mmap(K_MAGIC_2048) as *i64
54 let RE: *i64 = sys_mmap(K_MAGIC_2048) as *i64
55 let nrows: i64 = cdx_rows(cdxbuf, ncdx, RS, RE, 256)
56 let keys: *i64 = sys_mmap(K_MAGIC_2048) as *i64
57 let dates: *i64 = sys_mmap(K_MAGIC_2048) as *i64
58 let tso: *i64 = sys_mmap(K_MAGIC_2048) as *i64
59 let tsl: *i64 = sys_mmap(K_MAGIC_2048) as *i64
60 let oro: *i64 = sys_mmap(K_MAGIC_2048) as *i64
61 let orl: *i64 = sys_mmap(K_MAGIC_2048) as *i64
62 let b2: *i64 = sys_mmap(16) as *i64
63 var i: i64 = 0
64 var m: i64 = 0
65 while i < nrows {
66 cdx_field(cdxbuf, RS[i], RE[i], 4, b2)
67 var is200: i64 = 0
68 if b2[1]==3 { if (cdxbuf[b2[0]] as i64)==0x32 { if (cdxbuf[b2[0]+1] as i64)==0x30 { if (cdxbuf[b2[0]+2] as i64)==0x30 { is200=1 } } } }
69 if is200 == 1 {
70 cdx_field(cdxbuf, RS[i], RE[i], 0, b2)
71 let ko: i64 = b2[0]
72 let kl: i64 = b2[1]
73 let kb: *u8 = sys_mmap(256)
74 var j: i64 = 0
75 while j < kl { kb[j]=cdxbuf[ko+j]; j=j+1 }
76 kb[kl] = 0 as u8
77 keys[m] = kb as i64
78 cdx_field(cdxbuf, RS[i], RE[i], 1, b2); dates[m] = cdx_ts8(cdxbuf, b2[0], b2[1]); tso[m]=b2[0]; tsl[m]=b2[1]
79 cdx_field(cdxbuf, RS[i], RE[i], 2, b2); oro[m]=b2[0]; orl[m]=b2[1]
80 m = m + 1
81 }
82 i = i + 1
83 }
84 gw(" parsed rows=" as *u8); gn(nrows); gw(" status-200 captures=" as *u8); gn(m); gw("\n" as *u8)
85 if m <= 0 { gw(" no 200 captures -- abort\n" as *u8); return 1 }
86
87 // 3. browse as-of 1997-06 (most recent capture at-or-before)
88 let q: i64 = ti_query(keys, dates, m, "com,page3)/" as *u8, K_MAGIC_19970601)
89 if q < 0 { gw(" no capture as-of 1997 -- abort\n" as *u8); return 1 }
90 gw("\n[2] browse page3.com AS-OF 1997-06 -> capture date=" as *u8); gn(dates[q]); gw("\n" as *u8)
91
92 // 4. build the raw archived-content URL: web.archive.org/web/<ts14>id_/<original>
93 let url: *u8 = sys_mmap(K_MAGIC_2048)
94 var uo: i64 = 0
95 uo = ap(url, uo, "https://web.archive.org/web/" as *u8)
96 uo = apslice(url, uo, cdxbuf, tso[q], tsl[q])
97 uo = ap(url, uo, "id_/" as *u8)
98 uo = apslice(url, uo, cdxbuf, oro[q], orl[q])
99 url[uo] = 0 as u8
100 gw(" content URL: " as *u8); gw(url); gw("\n" as *u8)
101
102 // 5. fetch the ACTUAL archived page (retry-on-drop)
103 let cont: *u8 = sys_mmap(cap)
104 gw("\n[3] fetch archived content (retry-on-drop) ...\n" as *u8)
105 let ncont: i64 = fetch_retry(url, store, cont, cap, st, 5)
106 gw(" content status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(ncont); gw("\n" as *u8)
107 if ncont <= 0 { gw(" content fetch failed after retries -- abort\n" as *u8); return 1 }
108 gw(" --- archived page3.com head ---\n" as *u8)
109 phead(cont, ncont, 700)
110
111 // 6. content-address it + list its media references
112 let cid: i64 = cid_hash(cont, ncont)
113 let nimg: i64 = count_img(cont, ncont)
114 gw("\n[4] content CID (seg_store key, content-addressed)=" as *u8); gn(cid); gw("\n" as *u8)
115 gw(" media references in the archived HTML: <img> count=" as *u8); gn(nimg); gw("\n" as *u8)
116
117 gw("\nDEMO DONE -- fetched the ACTUAL archived page3.com page sovereignly, content-addressed it (CID), listed its media.\n" as *u8)
118 gw("(NEXT: store cont under CID in seg_store; resolve each <img> across archives + perceptual recovery = R6.)\n" as *u8)
119 return 0
120}