nx_reader_research_fetch.nx source
↩ module page · 139 lines · 12093 B
1// nx_reader_research_fetch.nx -- the Nishi RESEARCHER's sovereign web-fetch for the S-CLASS READER / LIBRARY arc.
2// Operator 2026-06-21: "keep going getting it s class exceed on desktop and mobile -- mobile menu hides parts,
3// table-of-contents chapters shown wrong, and we have NO artwork/assets ingested to s-class-exceed Kindle.
4// use the nishi researcher and catalogue the gaps to get there."
5//
6// Mirrors the proven nx_brand_research_fetch / nx_fin_research_fetch pattern: own TLS-1.3 (nx_https_fetch_follow) +
7// Mozilla CA store, saves each cited source to knowledge/fetched/reader_*.raw so the research COMPOUNDS to the
8// Library and can be grepped to CITE the census yardsticks. The 2026-06-14 best-in-class research already covered
9// sync/aids/comics/audiobook; THIS pass deliberately targets the dimensions that were UNcovered and that the
10// operator just named: (1) EPUB structure + the navigation document / table-of-contents (the "TOC wrong" root),
11// (2) book COVER ART + embedded illustrations + image formats (the "no artwork ingested" root), (3) RESPONSIVE web
12// design + mobile/touch UX + desktop layout (the "mobile menu hides parts" + "desktop and mobile" root), plus the
13// e-reader exemplars (Kindle/Apple Books/Kobo/Calibre) and typography/accessibility/interop yardsticks.
14// 100% sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). Idempotent have-skip = re-run fills gaps (also the
15// per-fetch mmap-leak workaround -- each pass is a fresh process so memory resets).
16// EXTENDED 2026-07-02 (operator: "use the nishi researcher to really get where we are vs benchmarks vs july 2026
17// research on our library and how to beat kindle devices and apps and other s-class systems"): section Z adds the
18// KINDLE-DEVICE depth (Scribe/KF8/E-Ink), the manga/server cohort (KOReader/Mihon/Kavita/Komga/foliate/Mokuro),
19// the ZOOM bar we just shipped against (DeepZoom/OpenSeadragon/IIIF/ComiXology guided view), the LIVE doc-AI
20// research bar (olmOCR/MinerU/docling/Magi manga understanding), and modern compression (JXL/AVIF/DjVu) ->
21// knowledge/fetched/rdr_z*.raw, grounding nx_reader_exceed_census.
22// COVERAGE: 24 + 20 = 44 authoritative reader/format/UX/research topics. expect_exit: 0 license_tier: ORIGINAL
23import "nx_syscalls.nx"
24import "nx_x509_trust_store.nx"
25import "nx_trust_store_load_from_certdata.nx"
26import "nx_https_fetch_follow.nx"
27const K_MAGIC_4194304: i64 = 4194304
28const K_MAGIC_8388608: i64 = 8388608
29
30func ff_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
31func ff_putn(v: i64) -> i64 {
32 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
33 var m: i64 = v
34 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
35 let d: *u8 = sys_mmap(24); var k: i64 = 0
36 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
37 var j: i64 = k - 1
38 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
39 return 0
40}
41func have_file(path: *u8) -> i64 {
42 let fd: i64 = sys_openat_rd(path)
43 if fd < 0 { return 0 }
44 sys_close(fd)
45 return 1
46}
47func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
48 if have_file(opath) == 1 { ff_puts(opath); ff_puts(" [have-skip]\n" as *u8); return 1 }
49 let status: *i64 = sys_mmap(8) as *i64
50 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
51 ff_puts(url); ff_puts(" status=" as *u8); ff_putn(status[0]); ff_puts(" bytes=" as *u8); ff_putn(n)
52 if n <= 0 { ff_puts(" FETCH-FAIL\n" as *u8); return 0 }
53 var gz: i64 = 0
54 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
55 if gz == 1 { ff_puts(" [GZIP-skip]\n" as *u8); return 0 }
56 let fd: i64 = sys_openat_wr(opath, 0x1a4)
57 if fd < 0 { ff_puts(" SAVE-FAIL\n" as *u8); return 0 }
58 sys_write(fd, out, n)
59 sys_close(fd)
60 ff_puts(" SAVED\n" as *u8)
61 return 1
62}
63
64func main() -> i64 {
65 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
66 if r <= 0 { ff_puts("READER: certdata load failed\n" as *u8); return 1 }
67 let store: *TrustStore = r as *TrustStore
68 ff_puts("CA roots=" as *u8); ff_putn(trust_store_count(store)); ff_puts("\n" as *u8)
69 let cap: i64 = K_MAGIC_8388608
70 let out: *u8 = sys_mmap(cap)
71 var ok: i64 = 0
72
73 ff_puts("== EXEMPLARS + FORMATS: what an S-class reader must open / who to beat ==\n" as *u8)
74 ok = ok + fetch_save("https://en.wikipedia.org/wiki/E-book" as *u8, "knowledge/fetched/reader_x1_ebook.raw" as *u8, store, out, cap)
75 ok = ok + fetch_save("https://en.wikipedia.org/wiki/EPUB" as *u8, "knowledge/fetched/reader_x2_epub.raw" as *u8, store, out, cap)
76 ok = ok + fetch_save("https://en.wikipedia.org/wiki/E-reader" as *u8, "knowledge/fetched/reader_x3_ereader.raw" as *u8, store, out, cap)
77 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comparison_of_e-book_formats" as *u8, "knowledge/fetched/reader_x4_formats.raw" as *u8, store, out, cap)
78 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Amazon_Kindle" as *u8, "knowledge/fetched/reader_x5_kindle.raw" as *u8, store, out, cap)
79 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Apple_Books" as *u8, "knowledge/fetched/reader_x6_apple_books.raw" as *u8, store, out, cap)
80 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Kobo_eReader" as *u8, "knowledge/fetched/reader_x7_kobo.raw" as *u8, store, out, cap)
81 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Calibre_(software)" as *u8, "knowledge/fetched/reader_x8_calibre.raw" as *u8, store, out, cap)
82
83 ff_puts("== STRUCTURE + TABLE OF CONTENTS: the navigation document / TOC (the 'TOC wrong' root) ==\n" as *u8)
84 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Table_of_contents" as *u8, "knowledge/fetched/reader_t1_toc.raw" as *u8, store, out, cap)
85 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Book_design" as *u8, "knowledge/fetched/reader_t2_book_design.raw" as *u8, store, out, cap)
86
87 ff_puts("== ARTWORK: cover art + embedded illustrations + image formats (the 'no artwork ingested' root) ==\n" as *u8)
88 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Book_cover" as *u8, "knowledge/fetched/reader_a1_book_cover.raw" as *u8, store, out, cap)
89 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Cover_art" as *u8, "knowledge/fetched/reader_a2_cover_art.raw" as *u8, store, out, cap)
90 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Digital_image" as *u8, "knowledge/fetched/reader_a3_digital_image.raw" as *u8, store, out, cap)
91 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Illustration" as *u8, "knowledge/fetched/reader_a4_illustration.raw" as *u8, store, out, cap)
92 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Image_file_formats" as *u8, "knowledge/fetched/reader_a5_image_formats.raw" as *u8, store, out, cap)
93
94 ff_puts("== RESPONSIVE + MOBILE + DESKTOP UX (the 'mobile menu hides parts' + 'desktop and mobile' root) ==\n" as *u8)
95 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Responsive_web_design" as *u8, "knowledge/fetched/reader_r1_responsive.raw" as *u8, store, out, cap)
96 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Web_design" as *u8, "knowledge/fetched/reader_r2_web_design.raw" as *u8, store, out, cap)
97 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Mobile_web" as *u8, "knowledge/fetched/reader_r3_mobile_web.raw" as *u8, store, out, cap)
98 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Touchscreen" as *u8, "knowledge/fetched/reader_r4_touchscreen.raw" as *u8, store, out, cap)
99
100 ff_puts("== TYPOGRAPHY + ACCESSIBILITY + INTEROP yardsticks ==\n" as *u8)
101 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Web_typography" as *u8, "knowledge/fetched/reader_y1_web_typography.raw" as *u8, store, out, cap)
102 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Readability" as *u8, "knowledge/fetched/reader_y2_readability.raw" as *u8, store, out, cap)
103 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Web_Content_Accessibility_Guidelines" as *u8, "knowledge/fetched/reader_y3_wcag.raw" as *u8, store, out, cap)
104 ok = ok + fetch_save("https://en.wikipedia.org/wiki/OPDS" as *u8, "knowledge/fetched/reader_i1_opds.raw" as *u8, store, out, cap)
105 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comic_book_archive" as *u8, "knowledge/fetched/reader_i2_cbz.raw" as *u8, store, out, cap)
106
107 ff_puts("== Z1. KINDLE DEVICES (the named rival: Scribe / format / e-ink hardware) ==\n" as *u8)
108 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Kindle_Scribe" as *u8, "knowledge/fetched/rdr_z01_kindle_scribe.raw" as *u8, store, out, cap)
109 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Kindle_File_Format" as *u8, "knowledge/fetched/rdr_z02_kindle_format.raw" as *u8, store, out, cap)
110 ok = ok + fetch_save("https://en.wikipedia.org/wiki/E_Ink" as *u8, "knowledge/fetched/rdr_z03_eink.raw" as *u8, store, out, cap)
111
112 ff_puts("== Z2. MANGA READERS + LIBRARY SERVERS (the open-source cohort) ==\n" as *u8)
113 ok = ok + fetch_save("https://en.wikipedia.org/wiki/KOReader" as *u8, "knowledge/fetched/rdr_z04_koreader.raw" as *u8, store, out, cap)
114 ok = ok + fetch_save("https://raw.githubusercontent.com/mihonapp/mihon/main/README.md" as *u8, "knowledge/fetched/rdr_z05_mihon.raw" as *u8, store, out, cap)
115 ok = ok + fetch_save("https://raw.githubusercontent.com/Kareadita/Kavita/develop/README.md" as *u8, "knowledge/fetched/rdr_z06_kavita.raw" as *u8, store, out, cap)
116 ok = ok + fetch_save("https://raw.githubusercontent.com/gotson/komga/master/README.md" as *u8, "knowledge/fetched/rdr_z07_komga.raw" as *u8, store, out, cap)
117 ok = ok + fetch_save("https://raw.githubusercontent.com/johnfactotum/foliate-js/main/README.md" as *u8, "knowledge/fetched/rdr_z08_foliatejs.raw" as *u8, store, out, cap)
118 ok = ok + fetch_save("https://raw.githubusercontent.com/kha-white/mokuro/master/README.md" as *u8, "knowledge/fetched/rdr_z09_mokuro.raw" as *u8, store, out, cap)
119
120 ff_puts("== Z3. ZOOM BAR (tiled pyramid + guided view -- the exceed we just shipped) ==\n" as *u8)
121 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Deep_Zoom" as *u8, "knowledge/fetched/rdr_z10_deepzoom.raw" as *u8, store, out, cap)
122 ok = ok + fetch_save("https://raw.githubusercontent.com/openseadragon/openseadragon/master/README.md" as *u8, "knowledge/fetched/rdr_z11_openseadragon.raw" as *u8, store, out, cap)
123 ok = ok + fetch_save("https://en.wikipedia.org/wiki/International_Image_Interoperability_Framework" as *u8, "knowledge/fetched/rdr_z12_iiif.raw" as *u8, store, out, cap)
124 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comixology" as *u8, "knowledge/fetched/rdr_z13_comixology.raw" as *u8, store, out, cap)
125
126 ff_puts("== Z4. DOC-AI RESEARCH BAR (PDF structure / OCR / manga understanding -- live-current) ==\n" as *u8)
127 ok = ok + fetch_save("https://arxiv.org/abs/2502.18443" as *u8, "knowledge/fetched/rdr_z14_olmocr.raw" as *u8, store, out, cap)
128 ok = ok + fetch_save("https://raw.githubusercontent.com/opendatalab/MinerU/master/README.md" as *u8, "knowledge/fetched/rdr_z15_mineru.raw" as *u8, store, out, cap)
129 ok = ok + fetch_save("https://raw.githubusercontent.com/docling-project/docling/main/README.md" as *u8, "knowledge/fetched/rdr_z16_docling.raw" as *u8, store, out, cap)
130 ok = ok + fetch_save("https://arxiv.org/abs/2401.10224" as *u8, "knowledge/fetched/rdr_z17_magi.raw" as *u8, store, out, cap)
131
132 ff_puts("== Z5. MODERN COMPRESSION + SCAN FORMATS ==\n" as *u8)
133 ok = ok + fetch_save("https://en.wikipedia.org/wiki/JPEG_XL" as *u8, "knowledge/fetched/rdr_z18_jxl.raw" as *u8, store, out, cap)
134 ok = ok + fetch_save("https://en.wikipedia.org/wiki/AVIF" as *u8, "knowledge/fetched/rdr_z19_avif.raw" as *u8, store, out, cap)
135 ok = ok + fetch_save("https://en.wikipedia.org/wiki/DjVu" as *u8, "knowledge/fetched/rdr_z20_djvu.raw" as *u8, store, out, cap)
136
137 ff_puts("READABLE SOURCES SAVED THIS PASS+HAVE: " as *u8); ff_putn(ok); ff_puts(" / 44\n" as *u8)
138 return 0
139}