nx_manga_research_fetch.nx source
↩ module page · 77 lines · 6979 B
1// nx_manga_research_fetch.nx -- SOVEREIGN researcher for the MANGA ACQUISITION + TRANSLATION exceed arc (operator
2// 2026-07-02: "build it all from the hardware rung up each rung so we can s-class-exceed ... nishi researcher and
3// census to beat other manga downloading tooling" + "[translate] native ... use the current [Python mm] as a
4// benchmark to far exceed"). Banks the LIVE July-2026 tooling landscape to knowledge/fetched/mga_*.raw so
5// nx_manga_exceed_census grades our native downloader+translator on a REAL corpus. Same proven pattern as the
6// reader/sharehub researchers (own TLS-1.3 + Mozilla CA, idempotent have-skip). GENERAL tooling knowledge only --
7// we build the general capability; the operator points it where they like on their sovereign box.
8// Axes: A DOWNLOAD TOOLING (gallery-dl/HakuNeko/FMD2/mangadex-dl/Mihon) B SOURCE APIs/STRUCTURE (MangaDex API,
9// gallery JSON, image CDN/referer) C METADATA/ARCHIVE (ComicInfo.xml, CBZ/CBR, OPDS) D TRANSLATE STACK
10// (manga-image-translator [the Python mm bar], Mokuro, manga-ocr, comic-text-detector, Magi) E POLITENESS
11// (rate-limit, robots, dedup/hash). expect_exit:0 license_tier: ORIGINAL
12import "nx_syscalls.nx"
13import "nx_x509_trust_store.nx"
14import "nx_trust_store_load_from_certdata.nx"
15import "nx_https_fetch_follow.nx"
16const K_MAGIC_4194304: i64 = 4194304
17
18func mf_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
19func mf_putn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let d:*u8=sys_mmap(24); var k:i64=0; while m>0{d[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i:i64=k-1; while i>=0{let o:*u8=sys_mmap(1);o[0]=d[i];sys_write(1,o,1);i=i-1} return 0 }
20func have_file(path: *u8) -> i64 { let fd: i64=sys_openat_rd(path); if fd<0{return 0} sys_close(fd); return 1 }
21func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
22 if have_file(opath)==1 { mf_puts(opath); mf_puts(" [have-skip]\n"); return 1 }
23 let status: *i64=sys_mmap(8) as *i64
24 let n: i64=nx_https_fetch_follow(url, store, out, cap, 6, status)
25 mf_puts(url); mf_puts(" status="); mf_putn(status[0]); mf_puts(" bytes="); mf_putn(n)
26 if n<=0 { mf_puts(" FETCH-FAIL\n"); return 0 }
27 var gz: i64=0; if n>=2 { if out[0]==0x1f as u8 { if out[1]==0x8b as u8 { gz=1 } } }
28 if gz==1 { mf_puts(" [GZIP-skip]\n"); return 0 }
29 let fd: i64=sys_openat_wr(opath, 0x1a4); if fd<0 { mf_puts(" SAVE-FAIL\n"); return 0 }
30 sys_write(fd, out, n); sys_close(fd); mf_puts(" SAVED\n"); return 1
31}
32
33func main() -> i64 {
34 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
35 if r<=0 { mf_puts("MANGA-RESEARCH: certdata load failed\n"); return 1 }
36 let store: *TrustStore = r as *TrustStore
37 mf_puts("CA roots="); mf_putn(trust_store_count(store)); mf_puts("\n")
38 let cap: i64=K_MAGIC_4194304
39 let out: *u8=sys_mmap(cap)
40 var ok: i64=0
41
42 mf_puts("== A. DOWNLOAD TOOLING (the bar to beat) ==\n")
43 ok = ok + fetch_save("https://raw.githubusercontent.com/mikf/gallery-dl/master/README.rst" as *u8, "knowledge/fetched/mga_a_gallerydl.raw" as *u8, store, out, cap)
44 ok = ok + fetch_save("https://raw.githubusercontent.com/mansuf/mangadex-downloader/main/README.md" as *u8, "knowledge/fetched/mga_a_mangadexdl.raw" as *u8, store, out, cap)
45 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HakuNeko" as *u8, "knowledge/fetched/mga_a_hakuneko.raw" as *u8, store, out, cap)
46 ok = ok + fetch_save("https://raw.githubusercontent.com/mihonapp/mihon/main/README.md" as *u8, "knowledge/fetched/mga_a_mihon.raw" as *u8, store, out, cap)
47 ok = ok + fetch_save("https://raw.githubusercontent.com/aria2/aria2/master/README.rst" as *u8, "knowledge/fetched/mga_a_aria2.raw" as *u8, store, out, cap)
48
49 mf_puts("== B. SOURCE APIs + STRUCTURE (how galleries expose pages) ==\n")
50 ok = ok + fetch_save("https://api.mangadex.org/docs/" as *u8, "knowledge/fetched/mga_b_mangadex_api.raw" as *u8, store, out, cap)
51 ok = ok + fetch_save("https://en.wikipedia.org/wiki/MangaDex" as *u8, "knowledge/fetched/mga_b_mangadex.raw" as *u8, store, out, cap)
52 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Content_delivery_network" as *u8, "knowledge/fetched/mga_b_cdn.raw" as *u8, store, out, cap)
53 ok = ok + fetch_save("https://en.wikipedia.org/wiki/HTTP_referer" as *u8, "knowledge/fetched/mga_b_referer.raw" as *u8, store, out, cap)
54
55 mf_puts("== C. METADATA + ARCHIVE FORMATS ==\n")
56 ok = ok + fetch_save("https://raw.githubusercontent.com/anansi-project/comicinfo/main/README.md" as *u8, "knowledge/fetched/mga_c_comicinfo.raw" as *u8, store, out, cap)
57 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comic_book_archive" as *u8, "knowledge/fetched/mga_c_cbz.raw" as *u8, store, out, cap)
58 ok = ok + fetch_save("https://en.wikipedia.org/wiki/OPDS" as *u8, "knowledge/fetched/mga_c_opds.raw" as *u8, store, out, cap)
59
60 mf_puts("== D. TRANSLATE STACK (the Python mm bar to FAR exceed, native) ==\n")
61 ok = ok + fetch_save("https://raw.githubusercontent.com/zyddnys/manga-image-translator/main/README.md" as *u8, "knowledge/fetched/mga_d_manga_translator.raw" as *u8, store, out, cap)
62 ok = ok + fetch_save("https://raw.githubusercontent.com/kha-white/manga-ocr/master/README.md" as *u8, "knowledge/fetched/mga_d_mangaocr.raw" as *u8, store, out, cap)
63 ok = ok + fetch_save("https://raw.githubusercontent.com/kha-white/mokuro/master/README.md" as *u8, "knowledge/fetched/mga_d_mokuro.raw" as *u8, store, out, cap)
64 ok = ok + fetch_save("https://raw.githubusercontent.com/dmMaze/comic-text-detector/main/README.md" as *u8, "knowledge/fetched/mga_d_textdetector.raw" as *u8, store, out, cap)
65 ok = ok + fetch_save("https://arxiv.org/abs/2401.10224" as *u8, "knowledge/fetched/mga_d_magi.raw" as *u8, store, out, cap)
66 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Optical_character_recognition" as *u8, "knowledge/fetched/mga_d_ocr.raw" as *u8, store, out, cap)
67 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Inpainting" as *u8, "knowledge/fetched/mga_d_inpainting.raw" as *u8, store, out, cap)
68 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Neural_machine_translation" as *u8, "knowledge/fetched/mga_d_nmt.raw" as *u8, store, out, cap)
69
70 mf_puts("== E. POLITENESS + INTEGRITY ==\n")
71 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Robots_exclusion_standard" as *u8, "knowledge/fetched/mga_e_robots.raw" as *u8, store, out, cap)
72 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Rate_limiting" as *u8, "knowledge/fetched/mga_e_ratelimit.raw" as *u8, store, out, cap)
73 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Content-addressable_storage" as *u8, "knowledge/fetched/mga_e_cas.raw" as *u8, store, out, cap)
74
75 mf_puts("READABLE SOURCES SAVED: "); mf_putn(ok); mf_puts(" / 24\n")
76 return 0
77}