nx_mirror_research_fetch.nx source
↩ module page · 98 lines · 6889 B
1// nx_mirror_research_fetch.nx -- SOVEREIGN multi-source web-fetch to STUDY the S-CLASS "download & MIRROR a
2// published open artifact onto our own storage (the NAS)" category -- so the Nishi census (nx_mirror_census)
3// grades our download-to-NAS capability against a REAL sovereign-banked corpus, not hearsay. Same proven
4// pattern as nx_pub_research_fetch / nx_openpub_research_fetch: own TLS-1.3 (nx_https_fetch_follow) + Mozilla
5// CA store, each source saved to knowledge/fetched/mirror_*.raw (research COMPOUNDS + is greppable to CITE).
6// 100% sovereign (own TLS, nx_cc->nxasm, NO curl/wget/aria2/gcc). Idempotent have-skip + 4MB cap.
7//
8// Motivation (operator): apertvs.ai PUBLISHES downloadable software (Apertus weights/code/datasets); our
9// platform must be able to DOWNLOAD/MIRROR that published software onto the NAS as a first-class capability,
10// S-class-exceed -- and if it isn't yet, that census names the build targets. Research axes -> banked clusters:
11// A RESUMABLE/PARALLEL HTTP (aria2/wget/range) B BITTORRENT/P2P (swarm/DHT/magnet/academic-torrents)
12// C CONTENT-ADDRESSED/INTEGRITY (CAS/IPFS/checksum/SHA-2) D HUBS+MIRRORING (HuggingFace/rsync/mirror-site)
13// E STORAGE/DURABILITY (NAS/dedup/data-integrity) F PROVENANCE/SIGNING (digital-signature/supply-chain)
14// expect_exit: 0 license_tier: ORIGINAL
15import "nx_syscalls.nx"
16import "nx_x509_trust_store.nx"
17import "nx_trust_store_load_from_certdata.nx"
18import "nx_https_fetch_follow.nx"
19const K_MAGIC_4194304: i64 = 4194304
20
21func gf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
22func gf_putn(v: i64) -> i64 {
23 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
24 var m: i64 = v
25 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
26 let d: *u8 = sys_mmap(24); var k: i64 = 0
27 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
28 var j: i64 = k - 1
29 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
30 return 0
31}
32func have_file(path: *u8) -> i64 {
33 let fd: i64 = sys_openat_rd(path)
34 if fd < 0 { return 0 }
35 sys_close(fd)
36 return 1
37}
38func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
39 if have_file(opath) == 1 { gf_puts(opath); gf_puts(" [have-skip]\n"); return 1 }
40 let status: *i64 = sys_mmap(8) as *i64
41 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
42 gf_puts(url); gf_puts(" status="); gf_putn(status[0]); gf_puts(" bytes="); gf_putn(n)
43 if n <= 0 { gf_puts(" FETCH-FAIL\n"); return 0 }
44 var gz: i64 = 0
45 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
46 if gz == 1 { gf_puts(" [GZIP-skip]\n"); return 0 }
47 let fd: i64 = sys_openat_wr(opath, 0x1a4)
48 if fd < 0 { gf_puts(" SAVE-FAIL\n"); return 0 }
49 sys_write(fd, out, n)
50 sys_close(fd)
51 gf_puts(" SAVED\n")
52 return 1
53}
54
55func main() -> i64 {
56 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
57 if r <= 0 { gf_puts("MIRROR: certdata load failed\n"); return 1 }
58 let store: *TrustStore = r as *TrustStore
59 gf_puts("CA roots="); gf_putn(trust_store_count(store)); gf_puts("\n")
60 let cap: i64 = K_MAGIC_4194304
61 let out: *u8 = sys_mmap(cap)
62 var ok: i64 = 0
63
64 gf_puts("== A. RESUMABLE / PARALLEL HTTP DOWNLOAD (aria2/wget/range) ==\n")
65 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Download_manager" as *u8, "knowledge/fetched/mirror_a_download_manager.raw" as *u8, store, out, cap)
66 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Aria2" as *u8, "knowledge/fetched/mirror_a_aria2.raw" as *u8, store, out, cap)
67 ok = ok + fetch_save("https://en.wikipedia.org/wiki/GNU_Wget" as *u8, "knowledge/fetched/mirror_a_wget.raw" as *u8, store, out, cap)
68 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Byte_serving" as *u8, "knowledge/fetched/mirror_a_byte_serving.raw" as *u8, store, out, cap)
69
70 gf_puts("== B. BITTORRENT / P2P (swarm / DHT / magnet / academic-torrents) ==\n")
71 ok = ok + fetch_save("https://en.wikipedia.org/wiki/BitTorrent" as *u8, "knowledge/fetched/mirror_b_bittorrent.raw" as *u8, store, out, cap)
72 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Magnet_URI_scheme" as *u8, "knowledge/fetched/mirror_b_magnet.raw" as *u8, store, out, cap)
73 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Distributed_hash_table" as *u8, "knowledge/fetched/mirror_b_dht.raw" as *u8, store, out, cap)
74 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Academic_Torrents" as *u8, "knowledge/fetched/mirror_b_academic_torrents.raw" as *u8, store, out, cap)
75
76 gf_puts("== C. CONTENT-ADDRESSED / INTEGRITY (CAS / IPFS / checksum / SHA-2) ==\n")
77 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Content-addressable_storage" as *u8, "knowledge/fetched/mirror_c_cas.raw" as *u8, store, out, cap)
78 ok = ok + fetch_save("https://en.wikipedia.org/wiki/InterPlanetary_File_System" as *u8, "knowledge/fetched/mirror_c_ipfs.raw" as *u8, store, out, cap)
79 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Checksum" as *u8, "knowledge/fetched/mirror_c_checksum.raw" as *u8, store, out, cap)
80 ok = ok + fetch_save("https://en.wikipedia.org/wiki/SHA-2" as *u8, "knowledge/fetched/mirror_c_sha2.raw" as *u8, store, out, cap)
81
82 gf_puts("== D. HUBS + MIRRORING (HuggingFace / rsync / mirror-site) ==\n")
83 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Hugging_Face" as *u8, "knowledge/fetched/mirror_d_huggingface.raw" as *u8, store, out, cap)
84 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Rsync" as *u8, "knowledge/fetched/mirror_d_rsync.raw" as *u8, store, out, cap)
85 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Mirror_site" as *u8, "knowledge/fetched/mirror_d_mirror_site.raw" as *u8, store, out, cap)
86
87 gf_puts("== E. STORAGE / DURABILITY (NAS / dedup / data-integrity) ==\n")
88 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Network-attached_storage" as *u8, "knowledge/fetched/mirror_e_nas.raw" as *u8, store, out, cap)
89 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_deduplication" as *u8, "knowledge/fetched/mirror_e_dedup.raw" as *u8, store, out, cap)
90 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_integrity" as *u8, "knowledge/fetched/mirror_e_data_integrity.raw" as *u8, store, out, cap)
91
92 gf_puts("== F. PROVENANCE / SIGNING (digital-signature / supply-chain) ==\n")
93 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Digital_signature" as *u8, "knowledge/fetched/mirror_f_digital_signature.raw" as *u8, store, out, cap)
94 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Software_supply_chain" as *u8, "knowledge/fetched/mirror_f_supply_chain.raw" as *u8, store, out, cap)
95
96 gf_puts("READABLE SOURCES SAVED: "); gf_putn(ok); gf_puts(" / 21\n")
97 return 0
98}