nx_download_research_fetch.nx source
↩ module page · 66 lines · 5019 B
1// nx_download_research_fetch.nx -- the Nishi RESEARCHER's sovereign fetch for DOWNLOAD ROBUSTNESS / SOTA
2// (operator: "downloads should keep alive + keep downloading so we don't get broken fragments -- get to SOTA").
3// Grounds the robust-download engine (segment retry, resume, integrity, keep-alive, backoff) in primary
4// sources. Mirrors nx_access_research_fetch (own TLS-1.3, Mozilla CA, idempotent have-skip). -> knowledge/
5// fetched/dl_*.raw. expect_exit: 0 license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_x509_trust_store.nx"
8import "nx_trust_store_load_from_certdata.nx"
9import "nx_https_fetch_follow.nx"
10const K_MAGIC_4194304: i64 = 4194304
11const K_MAGIC_8388608: i64 = 8388608
12
13func ff_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
14func ff_putn(v: i64) -> i64 { if v == 0 { sys_write(1, "0" as *u8, 1); return 0 } var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } let d: *u8 = sys_mmap(24); var k: i64 = 0; while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } let o: *u8 = sys_mmap(24); var q: i64 = k - 1; var x: i64 = 0; while q >= 0 { o[x] = d[q]; x = x + 1; q = q - 1 } sys_write(1, o, x); return 0 }
15func have_file(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
16func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
17 if have_file(opath) == 1 { ff_puts(opath); ff_puts(" [have-skip]\n" as *u8); return 1 }
18 let status: *i64 = sys_mmap(8) as *i64
19 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
20 ff_puts(url); ff_puts(" status=" as *u8); ff_putn(status[0]); ff_puts(" bytes=" as *u8); ff_putn(n)
21 if n <= 0 { ff_puts(" FETCH-FAIL\n" as *u8); return 0 }
22 var gz: i64 = 0
23 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
24 if gz == 1 { ff_puts(" [GZIP-skip]\n" as *u8); return 0 }
25 let fd: i64 = sys_openat_wr(opath, 0x1a4)
26 if fd < 0 { ff_puts(" SAVE-FAIL\n" as *u8); return 0 }
27 sys_write(fd, out, n)
28 sys_close(fd)
29 ff_puts(" SAVED\n" as *u8)
30 return 1
31}
32
33func main() -> i64 {
34 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
35 if r <= 0 { ff_puts("DL-RESEARCH: certdata load failed\n" as *u8); return 1 }
36 let store: *TrustStore = r as *TrustStore
37 let cap: i64 = K_MAGIC_8388608
38 let out: *u8 = sys_mmap(cap)
39 var ok: i64 = 0
40
41 ff_puts("== DOWNLOAD MANAGERS / SEGMENTED + RESUME ==\n" as *u8)
42 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Download_manager" as *u8, "knowledge/fetched/dl_dlmgr.raw" as *u8, store, out, cap)
43 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Segmented_file_transfer" as *u8, "knowledge/fetched/dl_segmented.raw" as *u8, store, out, cap)
44 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Aria2" as *u8, "knowledge/fetched/dl_aria2.raw" as *u8, store, out, cap)
45 ok = ok + fetch_save("https://en.wikipedia.org/wiki/YouTube-dl" as *u8, "knowledge/fetched/dl_ytdl.raw" as *u8, store, out, cap)
46 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Internet_Download_Manager" as *u8, "knowledge/fetched/dl_idm.raw" as *u8, store, out, cap)
47
48 ff_puts("== RESUME / RANGE / KEEP-ALIVE ==\n" as *u8)
49 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Byte_serving" as *u8, "knowledge/fetched/dl_byteserving.raw" as *u8, store, out, cap)
50 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HTTP_persistent_connection" as *u8, "knowledge/fetched/dl_keepalive.raw" as *u8, store, out, cap)
51 ok = ok + fetch_save("https://en.wikipedia.org/wiki/List_of_HTTP_header_fields" as *u8, "knowledge/fetched/dl_httphdr.raw" as *u8, store, out, cap)
52
53 ff_puts("== RETRY / BACKOFF / INTEGRITY ==\n" as *u8)
54 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Exponential_backoff" as *u8, "knowledge/fetched/dl_backoff.raw" as *u8, store, out, cap)
55 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Automatic_repeat_request" as *u8, "knowledge/fetched/dl_arq.raw" as *u8, store, out, cap)
56 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_integrity" as *u8, "knowledge/fetched/dl_integrity.raw" as *u8, store, out, cap)
57 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Checksum" as *u8, "knowledge/fetched/dl_checksum.raw" as *u8, store, out, cap)
58
59 ff_puts("== STREAMING FORMATS (segment integrity) ==\n" as *u8)
60 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HTTP_Live_Streaming" as *u8, "knowledge/fetched/dl_hls.raw" as *u8, store, out, cap)
61 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP" as *u8, "knowledge/fetched/dl_dash.raw" as *u8, store, out, cap)
62 ok = ok + fetch_save("https://en.wikipedia.org/wiki/MPEG_transport_stream" as *u8, "knowledge/fetched/dl_ts.raw" as *u8, store, out, cap)
63
64 ff_puts("DL-RESEARCH ok=" as *u8); ff_putn(ok); ff_puts("/15\n" as *u8)
65 return 0
66}