nx_ia_resolve.nx source

1// nx_ia_resolve.nx -- resolve an archive.org /details/ page to the actual media file URL via the sanctioned 2// public /metadata/ API. A details page is an HTML viewer that loads the media by JS, so a static fetch archives 3// the PAGE, not the movie. This fetches https://archive.org/metadata/<id> (small JSON), reads the top-level 4// "server"+"dir" and the .mp4 (h.264 derivative) filename from "files", and builds the direct download URL. The 5// search engine / archive daemon calls this before preserving an archive.org item. license_tier: ORIGINAL 6import "nx_syscalls.nx" 7import "nx_x509_trust_store.nx" 8import "nx_browser_fetch.nx" 9const K_MAGIC_1024: i64 = 1024 10const K_MAGIC_4194304: i64 = 4194304 11const K_MAGIC_2048: i64 = 2048 12 13func ia_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 14func ia_puts(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off]=s[i];off=off+1;i=i+1} return off } 15func ia_find(hay: *u8, hl: i64, needle: *u8, nl: i64, from: i64) -> i64 { 16 if nl==0 { return from } var i: i64=from; let last: i64=hl-nl 17 while i<=last { var j: i64=0; var m: i64=1; while j<nl { if (hay[i+j]&0xff)!=(needle[j]&0xff){m=0;j=nl} else {j=j+1} } if m==1 {return i} i=i+1 } return 0-1 18} 19// identifier from a /details/<id> URL (to next '/' or end) 20func ia_identifier(url: *u8, ulen: i64, out: *u8, cap: i64) -> i64 { 21 let p: i64 = ia_find(url, ulen, "/details/" as *u8, 9, 0) 22 if p < 0 { out[0]=0 as u8; return 0 } 23 var i: i64 = p + 9; var o: i64 = 0 24 var go: i64 = 1 25 while go == 1 { if i>=ulen { go=0 } else { let c: i64=url[i]&0xff; if c==47 { go=0 } else { if c==63 { go=0 } else { if o<(cap-1){out[o]=c as u8;o=o+1} i=i+1 } } } } 26 out[o]=0 as u8; return o 27} 28// read a JSON string value that follows `keyq` (e.g. "\"server\":\"") into out, until the closing quote 29func ia_json_str(json: *u8, jlen: i64, keyq: *u8, out: *u8, cap: i64) -> i64 { 30 let kl: i64 = ia_slen(keyq) 31 let p: i64 = ia_find(json, jlen, keyq, kl, 0) 32 if p < 0 { out[0]=0 as u8; return 0 } 33 var i: i64 = p + kl; var o: i64 = 0 34 var go: i64 = 1 35 while go == 1 { if i>=jlen { go=0 } else { let c: i64=json[i]&0xff; if c==34 { go=0 } else { if c==92 { if o<(cap-1){out[o]=json[i+1];o=o+1} i=i+2 } else { if o<(cap-1){out[o]=c as u8;o=o+1} i=i+1 } } } } 36 out[o]=0 as u8; return o 37} 38// find the first "name":"...mp4" value (the h.264 derivative), into out 39func ia_find_mp4(json: *u8, jlen: i64, out: *u8, cap: i64) -> i64 { 40 var from: i64 = 0 41 var go: i64 = 1 42 while go == 1 { 43 let p: i64 = ia_find(json, jlen, "\"name\":\"" as *u8, 8, from) 44 if p < 0 { go = 0 } 45 else { 46 var i: i64 = p + 8; var o: i64 = 0 47 let nm: *u8 = out 48 var g2: i64 = 1 49 while g2 == 1 { if i>=jlen { g2=0 } else { let c: i64=json[i]&0xff; if c==34 { g2=0 } else { if o<(cap-1){nm[o]=c as u8;o=o+1} i=i+1 } } } 50 nm[o]=0 as u8 51 if o >= 4 { if (nm[o-4]&0xff)==46 { if (nm[o-3]&0xff)==109 { if (nm[o-2]&0xff)==112 { if (nm[o-1]&0xff)==52 { return o } } } } } // ends ".mp4" 52 from = i + 1 53 } 54 } 55 out[0]=0 as u8; return 0 56} 57// percent-encode a path segment (keep A-Za-z0-9 . _ - / ~ ; encode the rest incl space, comma, etc.) 58func ia_urlencode(s: *u8, slen: i64, out: *u8, cap: i64) -> i64 { 59 let hexd: *u8 = "0123456789ABCDEF" as *u8 60 var o: i64 = 0; var i: i64 = 0 61 while i < slen { 62 let c: i64 = s[i]&0xff 63 var safe: i64 = 0 64 if c>=48 { if c<=57 { safe=1 } } if c>=65 { if c<=90 { safe=1 } } if c>=97 { if c<=122 { safe=1 } } 65 if c==46 { safe=1 } if c==95 { safe=1 } if c==45 { safe=1 } if c==47 { safe=1 } if c==126 { safe=1 } 66 if safe==1 { if o<(cap-1){out[o]=c as u8;o=o+1} } else { if o<(cap-4){ out[o]=37 as u8; out[o+1]=hexd[(c>>4)&0xf]; out[o+2]=hexd[c&0xf]; o=o+3 } } 67 i=i+1 68 } 69 out[o]=0 as u8; return o 70} 71// RESOLVE a details URL -> the direct media download URL (fetches /metadata/ over TLS). Returns len (0 = fail). 72func ia_resolve(details_url: *u8, ulen: i64, store: *TrustStore, cr: *u8, pk: *u8, now: i64, out_url: *u8, cap: i64) -> i64 { 73 let id: *u8 = sys_mmap(K_MAGIC_1024) 74 let idl: i64 = ia_identifier(details_url, ulen, id, K_MAGIC_1024) 75 if idl == 0 { return 0 } 76 let meta: *u8 = sys_mmap(K_MAGIC_1024) 77 var mo: i64 = ia_puts(meta, 0, "https://archive.org/metadata/" as *u8); mo = ia_puts(meta, mo, id); meta[mo]=0 as u8 78 let resp: *u8 = sys_mmap(K_MAGIC_4194304) // metadata JSON is small (KBs) 79 let n: i64 = bf_fetch_follow(meta, store, cr, pk, now, resp, K_MAGIC_4194304, 5) 80 if n <= 0 { return 0 } 81 let boff: i64 = bf_body_off(resp, n) 82 let body: *u8 = ((resp as i64)+boff) as *u8 83 let blen: i64 = n - boff 84 let server: *u8 = sys_mmap(512); ia_json_str(body, blen, "\"server\":\"" as *u8, server, 512) 85 let dir: *u8 = sys_mmap(K_MAGIC_1024); ia_json_str(body, blen, "\"dir\":\"" as *u8, dir, K_MAGIC_1024) 86 let name: *u8 = sys_mmap(K_MAGIC_1024); let nl: i64 = ia_find_mp4(body, blen, name, K_MAGIC_1024) 87 if nl == 0 { return 0 } 88 if server[0] == (0 as u8) { return 0 } 89 var o: i64 = ia_puts(out_url, 0, "https://" as *u8) 90 o = ia_puts(out_url, o, server) 91 o = ia_puts(out_url, o, dir) 92 out_url[o]=47 as u8; o=o+1 // '/' 93 let enc: *u8 = sys_mmap(K_MAGIC_2048); ia_urlencode(name, nl, enc, K_MAGIC_2048) 94 o = ia_puts(out_url, o, enc) 95 out_url[o]=0 as u8 96 return o 97}