nx_docpub_research_fetch.nx source
↩ module page · 103 lines · 7945 B
1// nx_docpub_research_fetch.nx -- SOVEREIGN multi-source web-fetch to RESEARCH S-CLASS DOCUMENTATION
2// PUBLISHING (the operator's ask: a SELF-GENERATING LIVE documentation wiki "like GitBook or Javadocs",
3// measured S-class EXCEED). Mirrors the proven nx_pub_research_fetch / nx_ge_research_fetch pattern: own
4// TLS-1.3 (nx_https_fetch_follow) + Mozilla CA store, saves each source to knowledge/fetched/docpub_*.raw
5// so research COMPOUNDS to the Library and can be grepped to CITE (>=2-source no-hearsay rule). 100%
6// sovereign (own TLS, nx_cc->nxasm, no curl/wget/gcc). Idempotent have-skip = also the per-fetch mmap-leak
7// workaround (re-run fills gaps; wsl --shutdown between passes if the TLS certloop OOMs).
8// Research axes -> what "S-class doc publishing" must beat:
9// (A) DOCS-SITE / STATIC GENERATORS (the GitBook/Read-the-Docs class: a beautiful published docs site)
10// (B) API-REFERENCE-FROM-SOURCE (the Javadoc/Doxygen class: docs SELF-GENERATED from code)
11// (C) DOCS-AS-CODE / IA / STRUCTURE (Diataxis, single-sourcing, DITA: the typed structure)
12// (D) INTEGRITY / FRESHNESS / VERSION (link-rot, versioning, CI: the "evidence & freshness" axis)
13// (E) PUBLISHING DX / RENDER / SEARCH (markdown, syntax-highlight, full-text search: the reader UX)
14// expect_exit: 0 license_tier: ORIGINAL
15import "nx_syscalls.nx"
16import "nx_x509_trust_store.nx"
17import "nx_trust_store_load_from_certdata.nx"
18import "nx_https_fetch_follow.nx"
19const K_MAGIC_4194304: i64 = 4194304
20const K_MAGIC_8388608: i64 = 8388608
21
22func gf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
23func gf_putn(v: i64) -> i64 {
24 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
25 var m: i64 = v
26 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
27 let d: *u8 = sys_mmap(24); var k: i64 = 0
28 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
29 var j: i64 = k - 1
30 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
31 return 0
32}
33func have_file(path: *u8) -> i64 {
34 let fd: i64 = sys_openat_rd(path)
35 if fd < 0 { return 0 }
36 sys_close(fd)
37 return 1
38}
39func fetch_save(url: *u8, opath: *u8, store: *TrustStore, out: *u8, cap: i64) -> i64 {
40 if have_file(opath) == 1 { gf_puts(opath); gf_puts(" [have-skip]\n"); return 1 }
41 let status: *i64 = sys_mmap(8) as *i64
42 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 6, status)
43 gf_puts(url); gf_puts(" status="); gf_putn(status[0]); gf_puts(" bytes="); gf_putn(n)
44 if n <= 0 { gf_puts(" FETCH-FAIL\n"); return 0 }
45 var gz: i64 = 0
46 if n >= 2 { if out[0] == 0x1f as u8 { if out[1] == 0x8b as u8 { gz = 1 } } }
47 if gz == 1 { gf_puts(" [GZIP-skip]\n"); return 0 }
48 let fd: i64 = sys_openat_wr(opath, 0x1a4)
49 if fd < 0 { gf_puts(" SAVE-FAIL\n"); return 0 }
50 sys_write(fd, out, n)
51 sys_close(fd)
52 gf_puts(" SAVED\n")
53 return 1
54}
55
56func main() -> i64 {
57 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
58 if r <= 0 { gf_puts("DOCPUB: certdata load failed\n"); return 1 }
59 let store: *TrustStore = r as *TrustStore
60 gf_puts("CA roots="); gf_putn(trust_store_count(store)); gf_puts("\n")
61 let cap: i64 = K_MAGIC_8388608
62 let out: *u8 = sys_mmap(cap)
63 var ok: i64 = 0
64
65 gf_puts("== A. DOCS-SITE / STATIC GENERATORS (the GitBook / Read-the-Docs class) ==\n")
66 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Static_site_generator" as *u8, "knowledge/fetched/docpub_a_static_site_generator.raw" as *u8, store, out, cap)
67 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Read_the_Docs" as *u8, "knowledge/fetched/docpub_a_read_the_docs.raw" as *u8, store, out, cap)
68 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Sphinx_(documentation_generator)" as *u8, "knowledge/fetched/docpub_a_sphinx.raw" as *u8, store, out, cap)
69 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Jekyll_(software)" as *u8, "knowledge/fetched/docpub_a_jekyll.raw" as *u8, store, out, cap)
70 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Hugo_(software)" as *u8, "knowledge/fetched/docpub_a_hugo.raw" as *u8, store, out, cap)
71 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Help_authoring_tool" as *u8, "knowledge/fetched/docpub_a_help_authoring_tool.raw" as *u8, store, out, cap)
72
73 gf_puts("== B. API-REFERENCE-FROM-SOURCE (the Javadoc / Doxygen class -- SELF-generating) ==\n")
74 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Documentation_generator" as *u8, "knowledge/fetched/docpub_b_documentation_generator.raw" as *u8, store, out, cap)
75 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comparison_of_documentation_generators" as *u8, "knowledge/fetched/docpub_b_comparison_docgen.raw" as *u8, store, out, cap)
76 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Javadoc" as *u8, "knowledge/fetched/docpub_b_javadoc.raw" as *u8, store, out, cap)
77 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Doxygen" as *u8, "knowledge/fetched/docpub_b_doxygen.raw" as *u8, store, out, cap)
78 ok = ok + fetch_save("https://en.wikipedia.org/wiki/JSDoc" as *u8, "knowledge/fetched/docpub_b_jsdoc.raw" as *u8, store, out, cap)
79 ok = ok + fetch_save("https://en.wikipedia.org/wiki/OpenAPI_Specification" as *u8, "knowledge/fetched/docpub_b_openapi.raw" as *u8, store, out, cap)
80 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Comment_(computer_programming)" as *u8, "knowledge/fetched/docpub_b_doc_comment.raw" as *u8, store, out, cap)
81
82 gf_puts("== C. DOCS-AS-CODE / INFORMATION ARCHITECTURE / STRUCTURED AUTHORING ==\n")
83 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Software_documentation" as *u8, "knowledge/fetched/docpub_c_software_documentation.raw" as *u8, store, out, cap)
84 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Single-source_publishing" as *u8, "knowledge/fetched/docpub_c_single_source.raw" as *u8, store, out, cap)
85 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Darwin_Information_Typing_Architecture" as *u8, "knowledge/fetched/docpub_c_dita.raw" as *u8, store, out, cap)
86 ok = ok + fetch_save("https://en.wikipedia.org/wiki/DocBook" as *u8, "knowledge/fetched/docpub_c_docbook.raw" as *u8, store, out, cap)
87 ok = ok + fetch_save("https://en.wikipedia.org/wiki/ReStructuredText" as *u8, "knowledge/fetched/docpub_c_restructuredtext.raw" as *u8, store, out, cap)
88 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Information_architecture" as *u8, "knowledge/fetched/docpub_c_information_architecture.raw" as *u8, store, out, cap)
89 ok = ok + fetch_save("https://diataxis.fr/" as *u8, "knowledge/fetched/docpub_c_diataxis.raw" as *u8, store, out, cap)
90
91 gf_puts("== D. INTEGRITY / FRESHNESS / VERSIONING (evidence + no-rot axis) ==\n")
92 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Link_rot" as *u8, "knowledge/fetched/docpub_d_link_rot.raw" as *u8, store, out, cap)
93 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Software_versioning" as *u8, "knowledge/fetched/docpub_d_software_versioning.raw" as *u8, store, out, cap)
94 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Continuous_integration" as *u8, "knowledge/fetched/docpub_d_continuous_integration.raw" as *u8, store, out, cap)
95
96 gf_puts("== E. PUBLISHING DX / RENDER / SEARCH (the reader experience) ==\n")
97 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Markdown" as *u8, "knowledge/fetched/docpub_e_markdown.raw" as *u8, store, out, cap)
98 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Syntax_highlighting" as *u8, "knowledge/fetched/docpub_e_syntax_highlighting.raw" as *u8, store, out, cap)
99 ok = ok + fetch_save("https://en.wikipedia.org/wiki/Full-text_search" as *u8, "knowledge/fetched/docpub_e_full_text_search.raw" as *u8, store, out, cap)
100
101 gf_puts("READABLE SOURCES SAVED: "); gf_putn(ok); gf_puts(" / 26\n")
102 return 0
103}