nx_library_fetch.nx source
↩ module page · 144 lines · 6525 B
1// nx_library_fetch.nx -- SOVEREIGN digital-library best-practices research fetch.
2//
3// Rung L-R1a of the nishifamily.com/library arc: ground the library
4// capability-target + exceed benchmark on REAL sourced facts, not assumptions
5// (global Rule 4). Reuses the team's OWN proven sovereign HTTPS stack
6// (nx_tls13_client_session_run + nx_https_get_complete; no browser/node/curl),
7// validated against the real Mozilla CA store, NishiBot User-Agent -- the
8// identical path proven live in nx_night_fetch_test (W-NIGHT-1).
9//
10// Fetches six fact-dense OPEN pages (en.wikipedia.org, static, NishiBot allowed,
11// canonical/parens-free titles -> no redirect) -> raw HTTP responses:
12// /wiki/Open_Publication_Distribution_System -> lib_opds.raw (catalog interop)
13// /wiki/Digital_library -> lib_diglib.raw (architecture)
14// /wiki/Dublin_Core -> lib_dublin.raw (metadata schema)
15// /wiki/EPUB -> lib_epub.raw (open format)
16// /wiki/Okapi_BM25 -> lib_bm25.raw (search ranking)
17// /wiki/Faceted_search -> lib_facet.raw (browse UX)
18//
19// expect_exit: 0
20// license_tier: ORIGINAL
21
22import "nx_syscalls.nx"
23import "nx_csprng.nx"
24import "nx_x509_trust_store.nx"
25import "nx_trust_store_load_from_certdata.nx"
26import "nx_tls13_client_validate_certificate.nx"
27import "nx_tls13_client_session_run.nx"
28import "nx_https_url_for_fetch.nx"
29import "nx_https_url_connect.nx"
30import "nx_https_get.nx"
31import "nx_https_get_complete.nx"
32import "nx_http_response_parse.nx"
33const K_MAGIC_4194304: i64 = 4194304
34
35func lf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
36func lf_putn(v: i64) -> i64 {
37 let bb: *u8 = sys_mmap(28); var m: i64 = v
38 if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }
39 let t: *u8 = sys_mmap(28); var k: i64 = 0
40 if m == 0 { t[0] = 48 as u8; k = 1 }
41 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
42 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } sys_write(1, bb, k); return 0
43}
44func lf_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
45
46// Fetch one page over the sovereign TLS1.3 client, save the raw response.
47// Returns the HTTP status (200 ok), or a negative error code.
48func fetch_page(store: *TrustStore, full_url: *u8, path: *u8, path_len: i64, out_path: *u8) -> i64 {
49 lf_puts("--- fetch "); lf_puts(full_url); lf_puts("\n")
50
51 // ephemeral client keys
52 let cr: *u8 = sys_mmap(32)
53 var i: i64 = 0
54 nx_csprng_fill(cr, 32) // CWE-330 (debt 1785970852): was the constant 0xC0..0xDF
55 let priv: *u8 = sys_mmap(32)
56 i = 0
57 nx_csprng_fill(priv, 32) // CWE-330: the X25519 scalar was the constant 0xA0..0xBF on EVERY session
58
59 let url_p: *NxUrl = nx_url_new()
60 let target_raw: *u8 = sys_mmap(32)
61 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
62 target.url = url_p
63 target.port = 0
64 if nx_https_url_for_fetch(full_url, target) != NX_HTTPS_URL_OK { return 0 - 41 }
65
66 let fd_p: *i64 = sys_mmap(16) as *i64
67 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - 42 }
68 let fd: i64 = *fd_p
69
70 let val_ctx_raw: *u8 = sys_mmap(64)
71 let val_ctx: *TlsValidationContext = val_ctx_raw as *TlsValidationContext
72 val_ctx.store = store
73 val_ctx.sni_host = full_url + target.url.host_off
74 val_ctx.sni_host_len = target.url.host_len
75 val_ctx.now_epoch = sys_now_realtime_sec()
76
77 let sr: i64 = nx_tls13_client_session_run(
78 fd, full_url + target.url.host_off, target.url.host_len,
79 cr, priv, val_ctx
80 )
81 if sr <= 0 { sys_close(fd); return 0 - (200 + (0 - sr)) }
82
83 let session: *Tls13ClientSession = sr as *Tls13ClientSession
84 let buf: *u8 = sys_mmap(K_MAGIC_4194304)
85 let gc: i64 = nx_https_get_complete(
86 session, fd, path, path_len,
87 full_url + target.url.host_off, target.url.host_len,
88 buf, K_MAGIC_4194304
89 )
90 sys_close(fd)
91 if gc < 0 { return 0 - (100 + (0 - gc)) }
92
93 let rs: *i64 = sys_mmap(128) as *i64
94 nx_http_response_parse(buf, gc, rs)
95 let status: i64 = rs[1]
96
97 let ofd: i64 = sys_openat_wr(out_path, 0x1A4)
98 if ofd <= 0 { return 0 - 70 }
99 sys_write(ofd, buf, gc)
100 sys_close(ofd)
101
102 lf_puts(" ST="); lf_putn(status); lf_puts(" GC="); lf_putn(gc); lf_puts(" -> "); lf_puts(out_path); lf_puts("\n")
103 return status
104}
105
106func main() -> i64 {
107 let cpath: *u8 = "/tmp/mozilla_certdata.txt\x00"
108 let r: i64 = nx_trust_store_load_from_certdata(cpath, 512, K_MAGIC_4194304)
109 if r <= 0 { lf_puts("LIB-FETCH: certdata load failed\n"); return 1 }
110 let store: *TrustStore = r as *TrustStore
111 let n: i64 = trust_store_count(store)
112 if n < 50 { lf_puts("LIB-FETCH: too few CAs\n"); return 3 }
113 lf_puts("CA="); lf_putn(n); lf_puts("\n")
114
115 var ok: i64 = 0
116
117 let u1: *u8 = "https://en.wikipedia.org/wiki/Open_Publication_Distribution_System\x00"
118 let p1: *u8 = "/wiki/Open_Publication_Distribution_System\x00"
119 if fetch_page(store, u1, p1, lf_strlen(p1), "knowledge/fetched/lib_opds.raw\x00" as *u8) == 200 { ok = ok + 1 }
120
121 let u2: *u8 = "https://en.wikipedia.org/wiki/Digital_library\x00"
122 let p2: *u8 = "/wiki/Digital_library\x00"
123 if fetch_page(store, u2, p2, lf_strlen(p2), "knowledge/fetched/lib_diglib.raw\x00" as *u8) == 200 { ok = ok + 1 }
124
125 let u3: *u8 = "https://en.wikipedia.org/wiki/Dublin_Core\x00"
126 let p3: *u8 = "/wiki/Dublin_Core\x00"
127 if fetch_page(store, u3, p3, lf_strlen(p3), "knowledge/fetched/lib_dublin.raw\x00" as *u8) == 200 { ok = ok + 1 }
128
129 let u4: *u8 = "https://en.wikipedia.org/wiki/EPUB\x00"
130 let p4: *u8 = "/wiki/EPUB\x00"
131 if fetch_page(store, u4, p4, lf_strlen(p4), "knowledge/fetched/lib_epub.raw\x00" as *u8) == 200 { ok = ok + 1 }
132
133 let u5: *u8 = "https://en.wikipedia.org/wiki/Okapi_BM25\x00"
134 let p5: *u8 = "/wiki/Okapi_BM25\x00"
135 if fetch_page(store, u5, p5, lf_strlen(p5), "knowledge/fetched/lib_bm25.raw\x00" as *u8) == 200 { ok = ok + 1 }
136
137 let u6: *u8 = "https://en.wikipedia.org/wiki/Faceted_search\x00"
138 let p6: *u8 = "/wiki/Faceted_search\x00"
139 if fetch_page(store, u6, p6, lf_strlen(p6), "knowledge/fetched/lib_facet.raw\x00" as *u8) == 200 { ok = ok + 1 }
140
141 lf_puts("LIB-SOVEREIGN-FETCH-OK pages_200="); lf_putn(ok); lf_puts("/6\n")
142 if ok < 1 { return 51 }
143 return 0
144}