nx_live_https.nx source
↩ module page · 67 lines · 3312 B
1// nx_live_https.nx -- full VALIDATED live HTTPS crawl, bits-up end to end.
2// Loads the system CA bundle into our sovereign trust store (nx_pem_loader),
3// then nx_https_get with chain validation against a real HTTPS host. Every
4// layer ours: DNS + TCP + TLS1.3 + X.509 chain validation + HTTP + HTML->text
5// + index.
6import "nx_str.nx"
7import "nx_syscalls.nx"
8import "nx_csprng.nx"
9import "nx_x509_trust_store.nx"
10import "nx_pem_loader.nx"
11import "nx_https_get.nx"
12import "nx_html_to_text.nx"
13import "nx_bm25.nx"
14import "nx_search_inverted.nx" // nx_bm25_tf + nx_inv_hash_bytes_lower -- not carried by nx_bm25.nx
15
16func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
17func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
18func nx_pi(n: i64) -> i64 {
19 if n == 0 { nx_putc(0x30); return 0 }
20 var v: i64 = n; if v < 0 { nx_putc(0x2D); v = 0 - v }
21 let t: *u8 = sys_mmap(32); var k: i64 = 0
22 while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
23 while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
24 return 0
25}
26func nx_body_off(resp: *u8, n: i64) -> i64 {
27 var i: i64 = 0
28 while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
29 return 0
30}
31func nx_rep(text: *u8, tl: i64, q: *u8) -> i64 {
32 nx_puts(" '"); nx_puts(q); nx_puts("' x"); nx_pi(nx_bm25_tf(text, tl, nx_inv_hash_bytes_lower(q, nx_str_len(q)))); nx_putc(0x0A)
33 return 0
34}
35
36func main() -> i64 {
37 nx_puts("=== LIVE VALIDATED HTTPS crawl (bits-up CA store) ===\n")
38 let store: *TrustStore = trust_store_alloc(400)
39 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt\x00", store)
40 nx_puts(" loaded "); nx_pi(nroots); nx_puts(" CA roots into the sovereign trust store\n")
41 if nroots <= 0 { nx_puts(" no roots loaded -> cannot validate\n"); return 1 }
42
43 let cr: *u8 = sys_mmap(32); nx_csprng_fill(cr, 32)
44 let pk: *u8 = sys_mmap(32); nx_csprng_fill(pk, 32)
45 let out: *u8 = sys_mmap(131072)
46 let now: i64 = sys_now_realtime_sec()
47 let url: *u8 = "https://example.com/"
48 nx_puts(" fetching "); nx_puts(url); nx_puts(" WITH chain validation...\n")
49 let r: i64 = nx_https_get(url, cr, pk, store, now, out, 131072)
50 if r > 0 {
51 nx_puts(" VALIDATED HTTPS OK: fetched "); nx_pi(r); nx_puts(" bytes over a validated TLS 1.3 session\n")
52 let bo: i64 = nx_body_off(out, r)
53 let text: *u8 = sys_mmap(131072)
54 let tl: i64 = nx_html_to_text(((out as i64) + bo) as *u8, r - bo, text, 131072)
55 nx_puts(" extracted "); nx_pi(tl); nx_puts(" text chars; term frequencies:\n")
56 nx_rep(text, tl, "example")
57 nx_rep(text, tl, "domain")
58 nx_rep(text, tl, "information")
59 nx_puts("=> LIVE: sovereign DNS+TCP+TLS1.3+X509-validation+HTTP+extract+index against the real HTTPS web.\n")
60 return 0
61 }
62 let v: i64 = 0 - r
63 nx_puts(" HTTPS verdict (not 0): "); nx_pi(v)
64 nx_puts(" (3=connect 4=handshake/validation 5=fetch). CA store loaded ("); nx_pi(nroots); nx_puts(" roots);\n")
65 nx_puts(" the bounded remaining item is client X.509 chain-validation interop for this cert.\n")
66 return 1
67}