nx_crawl_preserve.nx source

1// nx_crawl_preserve.nx -- the CRAWL DRIVER: sweep a list of URLs, render-fetch each as a browser (browser headers 2// + redirect follow), and archive them all into ONE durable WARC -- then reload it and confirm every preserved 3// resource retrieves by URL. This is the "capture a dying site before it vanishes" step of the sovereign 4// Browsertrix, composing bf_fetch_follow (render front) + nx_web_archive (store). Non-200s are skipped + logged 5// (honest: hard hosts / access-control). Run from nxc2 root. argv[1..] = URLs (default = a small legal set). 6// license_tier: ORIGINAL 7import "nx_syscalls.nx" 8import "nx_x509_trust_store.nx" 9import "nx_trust_store_load_from_certdata.nx" 10import "nx_csprng.nx" 11import "nx_browser_fetch.nx" 12import "nx_web_archive.nx" 13const K_MAGIC_4194304: i64 = 4194304 14const K_MAGIC_8388608: i64 = 8388608 15const K_MAGIC_33554432: i64 = 33554432 16 17func pw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 18func pn(v: i64) -> i64 { let b: *u8=sys_mmap(24); var m: i64=v; if m<0{pw("-" as *u8);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 } 19func psl(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 20 21func main(argc: i64, argv: *i64) -> i64 { 22 let def: *i64 = sys_mmap(8*4) as *i64 23 def[0] = "https://example.com" as *u8 as i64 24 def[1] = "https://example.org" as *u8 as i64 25 def[2] = "https://download.samplelib.com/mp4/sample-5s.mp4" as *u8 as i64 26 var list: *i64 = def; var nurls: i64 = 3 27 if argc >= 2 { list = ((argv as i64) + 8) as *i64; nurls = argc - 1 } 28 29 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304) 30 if r <= 0 { pw("CRAWL: certdata load failed (run from nxc2 root)\n" as *u8); sys_exit(1); return 1 } 31 let store: *TrustStore = r as *TrustStore 32 let cr: *u8 = sys_mmap(32); let pk: *u8 = sys_mmap(32); nx_csprng_fill(cr, 32); nx_csprng_fill(pk, 32) 33 let now: i64 = sys_now_realtime_sec() 34 35 pw("== nx_crawl_preserve: sweep " as *u8); pn(nurls); pw(" URLs -> render-fetch -> archive -> durable WARC ==\n" as *u8) 36 let cap: i64 = K_MAGIC_8388608 37 let resp: *u8 = sys_mmap(cap) 38 let warc: *u8 = sys_mmap(K_MAGIC_33554432) // 32 MiB archive 39 var off: i64 = 0 40 var preserved: i64 = 0 41 42 var i: i64 = 0 43 while i < nurls { 44 let url: *u8 = (list[i]) as *u8 45 let n: i64 = bf_fetch_follow(url, store, cr, pk, now, resp, cap, 5) 46 let st: i64 = bf_status(resp, n) 47 let boff: i64 = bf_body_off(resp, n) 48 let blen: i64 = n - boff 49 pw(" [" as *u8); pn(st); pw("] " as *u8); pn(blen); pw("B " as *u8); pw(url); pw("\n" as *u8) 50 if st == 200 { if blen > 0 { let body: *u8 = ((resp as i64)+boff) as *u8; off = wa_write_resource(warc, off, url, "2026-07-11T00:00:00Z" as *u8, "application/octet-stream" as *u8, body, blen); preserved = preserved + 1 } } 51 i = i + 1 52 } 53 let path: *u8 = "nx_crawl.warc" as *u8 54 wa_save(path, warc, off) 55 pw("crawled " as *u8); pn(nurls); pw(", preserved " as *u8); pn(preserved); pw(" -> " as *u8); pw(path); pw(" (" as *u8); pn(off); pw(" bytes, " as *u8); pn(wa_count(warc, off)); pw(" records)\n" as *u8) 56 57 // reload the durable WARC + confirm every preserved URL retrieves 58 let warc2: *u8 = sys_mmap(K_MAGIC_33554432) 59 let n2: i64 = wa_load(path, warc2, K_MAGIC_33554432) 60 let op: *i64 = sys_mmap(8) as *i64; let ol: *i64 = sys_mmap(8) as *i64 61 var retrievable: i64 = 0 62 i = 0 63 while i < nurls { 64 let url: *u8 = (list[i]) as *u8 65 if wa_lookup(warc2, n2, url, psl(url), op, ol) == 1 { retrievable = retrievable + 1 } 66 i = i + 1 67 } 68 pw("reload durable WARC: " as *u8); pn(retrievable); pw("/" as *u8); pn(preserved); pw(" preserved resources retrievable by URL\n" as *u8) 69 if retrievable == preserved { if preserved > 0 { pw("verdict=GREEN (crawl->render-fetch->archive->durable WARC; every preserved resource survives the origin dying)\n" as *u8); sys_exit(0); return 0 } } 70 pw("verdict=RED (some preserved resource did not retrieve)\n" as *u8) 71 sys_exit(1); return 1 72}