nx_dlperf_research_fetch.nx source
↩ module page · 67 lines · 5508 B
1// nx_dlperf_research_fetch.nx -- SOVEREIGN researcher for S-CLASS download PERFORMANCE + RELIABILITY (operator:
2// "maximum speed, resume capabilities, error handling"): the aria2/axel/wget/curl bar for high-throughput,
3// resumable, fault-tolerant transfers. Banks each source to knowledge/fetched/dlperf_*.raw so nx_dlperf_census can
4// grade our streaming pull against a real corpus (>=2-source no-hearsay). Same proven pattern as the mirror
5// researcher: own TLS-1.3 + Mozilla CA, idempotent have-skip. Axes: A parallel/segmented, B resume(HTTP range),
6// C retry/backoff/fault-tolerance, D throughput/TCP, E integrity, F connection reuse. expect_exit:0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11const K_MAGIC_4194304: i64 = 4194304
12
13func gf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
14func gf_putn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let d:*u8=sys_mmap(24); var k:i64=0; while m>0{d[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=k-1; while i>=0{let o:*u8=sys_mmap(1);o[0]=d[i];sys_write(1,o,1);i=i-1} return 0 }
15func have_file(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0{return 0} sys_close(fd); return 1 }
16func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
17 if have_file(opath)==1 { gf_puts(opath); gf_puts(" [have-skip]\n"); return 1 }
18 let status: *i64 = sys_mmap(8) as *i64
19 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
20 gf_puts(url); gf_puts(" status="); gf_putn(status[0]); gf_puts(" bytes="); gf_putn(n)
21 if n<=0 { gf_puts(" FETCH-FAIL\n"); return 0 }
22 var gz: i64=0; if n>=2 { if out[0]==0x1f as u8 { if out[1]==0x8b as u8 { gz=1 } } }
23 if gz==1 { gf_puts(" [GZIP-skip]\n"); return 0 }
24 let fd: i64=sys_openat_wr(opath, 0x1a4); if fd<0 { gf_puts(" SAVE-FAIL\n"); return 0 }
25 sys_write(fd, out, n); sys_close(fd); gf_puts(" SAVED\n"); return 1
26}
27
28func main() -> i64 {
29 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
30 if r<=0 { gf_puts("DLPERF: certdata load failed\n"); return 1 }
31 let store: *TrustStore = r as *TrustStore
32 gf_puts("CA roots="); gf_putn(trust_store_count(store)); gf_puts("\n")
33 let cap: i64 = K_MAGIC_4194304
34 let out: *u8 = sys_mmap(cap)
35 var ok: i64 = 0
36
37 gf_puts("== A. PARALLEL / SEGMENTED (multi-connection speed) ==\n")
38 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Download_manager" as *u8, "knowledge/fetched/dlperf_a_download_manager.raw" as *u8, store, out, cap)
39 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comparison_of_download_managers" as *u8, "knowledge/fetched/dlperf_a_comparison.raw" as *u8, store, out, cap)
40 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Download_accelerator" as *u8, "knowledge/fetched/dlperf_a_accelerator.raw" as *u8, store, out, cap)
41
42 gf_puts("== B. RESUME (HTTP range / partial content) ==\n")
43 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Byte_serving" as *u8, "knowledge/fetched/dlperf_b_byte_serving.raw" as *u8, store, out, cap)
44 ok = ok + fetch_save("https://en.wikipedia.org/wiki/List_of_HTTP_header_fields" as *u8, "knowledge/fetched/dlperf_b_http_headers.raw" as *u8, store, out, cap)
45 ok = ok + fetch_save("https://en.wikipedia.org/wiki/List_of_HTTP_status_codes" as *u8, "knowledge/fetched/dlperf_b_http_status.raw" as *u8, store, out, cap)
46
47 gf_puts("== C. RETRY / BACKOFF / FAULT-TOLERANCE ==\n")
48 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Exponential_backoff" as *u8, "knowledge/fetched/dlperf_c_backoff.raw" as *u8, store, out, cap)
49 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Retransmission_(data_networks)" as *u8, "knowledge/fetched/dlperf_c_retransmission.raw" as *u8, store, out, cap)
50 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Fault_tolerance" as *u8, "knowledge/fetched/dlperf_c_fault_tolerance.raw" as *u8, store, out, cap)
51
52 gf_puts("== D. THROUGHPUT / TCP ==\n")
53 ok = ok + fetch_save("https://en.wikipedia.org/wiki/TCP_congestion_control" as *u8, "knowledge/fetched/dlperf_d_tcp_congestion.raw" as *u8, store, out, cap)
54 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Bandwidth_(computing)" as *u8, "knowledge/fetched/dlperf_d_bandwidth.raw" as *u8, store, out, cap)
55 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Transmission_Control_Protocol" as *u8, "knowledge/fetched/dlperf_d_tcp.raw" as *u8, store, out, cap)
56
57 gf_puts("== E. INTEGRITY / VERIFY ==\n")
58 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Data_integrity" as *u8, "knowledge/fetched/dlperf_e_data_integrity.raw" as *u8, store, out, cap)
59 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Error_detection_and_correction" as *u8, "knowledge/fetched/dlperf_e_error_detection.raw" as *u8, store, out, cap)
60
61 gf_puts("== F. CONNECTION REUSE / PIPELINING ==\n")
62 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HTTP_persistent_connection" as *u8, "knowledge/fetched/dlperf_f_persistent.raw" as *u8, store, out, cap)
63 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HTTP_pipelining" as *u8, "knowledge/fetched/dlperf_f_pipelining.raw" as *u8, store, out, cap)
64
65 gf_puts("READABLE SOURCES SAVED: "); gf_putn(ok); gf_puts(" / 16\n")
66 return 0
67}