nx_crawl_https.nx source
↩ module page · 138 lines · 6660 B
1// nx_crawl_https.nx -- the hardened crawler on the REAL VALIDATED HTTPS web.
2// Governed (ISP-safe) + robots-respecting + content-deduped + indexed +
3// persisted, every fetch a VALIDATED TLS 1.3 session (nx_https_get + bits-up CA
4// store). The deployable pipeline, end-to-end, against the live HTTPS internet.
5import "nx_str.nx"
6import "nx_syscalls.nx"
7import "nx_csprng.nx"
8import "nx_x509_trust_store.nx"
9import "nx_pem_loader.nx"
10import "nx_https_get.nx"
11import "nx_html_to_text.nx"
12import "nx_simhash.nx"
13import "nx_bm25.nx"
14import "nx_robots.nx"
15import "nx_net_governor.nx"
16import "nx_docstore.nx"
17const K_MAGIC_2097152: i64 = 2097152
18const K_MAGIC_131072: i64 = 131072
19const K_MAGIC_262144: i64 = 262144
20
21func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
22func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
23func nx_pi(n: i64) -> i64 {
24 if n == 0 { nx_putc(0x30); return 0 }
25 var v: i64 = n; if v < 0 { nx_putc(0x2D); v = 0 - v }
26 let t: *u8 = sys_mmap(32); var k: i64 = 0
27 while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
28 while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
29 return 0
30}
31func nx_body_off(resp: *u8, n: i64) -> i64 {
32 var i: i64 = 0
33 while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
34 return 0
35}
36// path component of a full URL (after scheme://host), into out (null-term).
37func nx_pathof(url: *u8, out: *u8, cap: i64) -> i64 {
38 let n: i64 = nx_str_len(url)
39 var i: i64 = 0
40 var af: i64 = 0
41 var f: i64 = 0
42 while i < n - 2 { if f == 0 { if (url[i] as i64)==0x3A { if (url[i+1] as i64)==0x2F { if (url[i+2] as i64)==0x2F { af = i+3; f = 1 } } } } i = i + 1 }
43 var ps: i64 = 0 - 1
44 var p: i64 = af
45 while p < n { if ps < 0 { if (url[p] as i64)==0x2F { ps = p } } p = p + 1 }
46 if ps < 0 { out[0]=0x2F; out[1]=0; return 1 }
47 var o: i64 = 0
48 var q: i64 = ps
49 while q < n { if o < cap-1 { out[o]=url[q]; o=o+1 } q=q+1 }
50 out[o]=0
51 return o
52}
53
54func main() -> i64 {
55 nx_puts("=== Nishi LIVE HTTPS crawl: governed + robots + VALIDATED TLS ===\n")
56 let store: *TrustStore = trust_store_alloc(400)
57 let nroots: i64 = nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt\x00", store)
58 nx_puts(" CA roots loaded: "); nx_pi(nroots); nx_putc(0x0A)
59 if nroots <= 0 { return 1 }
60
61 let gov: *NxNetGov = sys_mmap(NX_NETGOV_BYTES) as *NxNetGov
62 nx_gov_init(gov, 1000, 10, K_MAGIC_2097152, K_MAGIC_131072) // 1s interval (ISP-safe), 10 reqs, 2MB, 128KB/resp
63 let uatok: *u8 = "NishiBot"
64 let cr: *u8 = sys_mmap(32)
65 let pk: *u8 = sys_mmap(32)
66
67 // robots.txt over validated HTTPS
68 nx_csprng_fill(cr, 32); nx_csprng_fill(pk, 32)
69 let rbuf: *u8 = sys_mmap(K_MAGIC_131072)
70 let rr: i64 = nx_https_get("https://example.com/robots.txt", cr, pk, store, sys_now_realtime_sec(), rbuf, K_MAGIC_131072)
71 nx_gov_record(gov, sys_now_ms(), 0)
72 var robots: *u8 = rbuf
73 var robolen: i64 = 0
74 if rr > 0 { let rbo: i64 = nx_body_off(rbuf, rr); robots = ((rbuf as i64) + rbo) as *u8; robolen = rr - rbo }
75 nx_puts(" robots.txt: "); if rr > 0 { nx_pi(rr); nx_puts(" bytes\n") } else { nx_puts("none (allow-all)\n") }
76
77 nx_docstore_reset("_offc/nx_crawl_https.store\x00")
78 let QCAP: i64 = 64
79 let queue: **u8 = sys_mmap(QCAP*8) as **u8
80 var qh: i64 = 0
81 var qt: i64 = 0
82 let visited: **u8 = sys_mmap(QCAP*8) as **u8
83 var nv: i64 = 0
84 queue[qt] = "https://example.com/"; qt = qt + 1
85 let idx: *NxInvIndex = nx_inv_new(32)
86 var ndoc: i64 = 0
87 let keptfp: *i64 = sys_mmap(32*8) as *i64
88
89 var running: i64 = 1
90 while running == 1 {
91 if qh >= qt { running = 0 }
92 if nx_gov_can_fetch(gov) == 0 { running = 0; nx_puts(" [governor budget reached]\n") }
93 if running == 1 {
94 let url: *u8 = queue[qh]; qh = qh + 1
95 var seen: i64 = 0
96 var vi: i64 = 0
97 while vi < nv { if nx_str_eq(url, visited[vi]) == 1 { seen = 1 } vi = vi + 1 }
98 if seen == 0 {
99 visited[nv] = url; nv = nv + 1
100 let pth: *u8 = sys_mmap(512)
101 nx_pathof(url, pth, 512)
102 if nx_robots_allowed(robots, robolen, uatok, nx_str_len(uatok), pth, nx_str_len(pth)) == 0 {
103 nx_puts(" robots-skip: "); nx_puts(url); nx_putc(0x0A)
104 } else {
105 let w: i64 = nx_gov_wait_ms(gov, sys_now_ms())
106 if w > 0 { sys_sleep_ms(w) }
107 nx_csprng_fill(cr, 32); nx_csprng_fill(pk, 32) // fresh key per connection
108 let resp: *u8 = sys_mmap(K_MAGIC_262144)
109 let n: i64 = nx_https_get(url, cr, pk, store, sys_now_realtime_sec(), resp, K_MAGIC_262144)
110 nx_gov_record(gov, sys_now_ms(), n)
111 if n > 0 {
112 let bo: i64 = nx_body_off(resp, n)
113 let text: *u8 = sys_mmap(K_MAGIC_262144)
114 let tlen: i64 = nx_html_to_text(((resp as i64)+bo) as *u8, n - bo, text, K_MAGIC_262144)
115 let fp: i64 = nx_simhash_fingerprint(text, tlen)
116 var dup: i64 = 0
117 var k: i64 = 0
118 while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
119 nx_puts(" VALIDATED-crawled "); nx_puts(url); nx_puts(" ("); nx_pi(n); nx_puts(" bytes -> "); nx_pi(tlen); nx_puts(" chars)")
120 if dup == 1 { nx_puts(" [dup]\n") } else {
121 keptfp[ndoc] = fp
122 nx_inv_index_row(idx, text, tlen, ndoc)
123 nx_docstore_append("_offc/nx_crawl_https.store\x00", url, nx_str_len(url), text, tlen)
124 ndoc = ndoc + 1
125 nx_puts(" [indexed+persisted]\n")
126 }
127 } else {
128 nx_puts(" fetch verdict "); nx_pi(0 - n); nx_puts(" for "); nx_puts(url); nx_putc(0x0A)
129 }
130 }
131 }
132 }
133 }
134 nx_inv_finalize_offsets(idx)
135 nx_puts("LIVE HTTPS crawl done: "); nx_pi(ndoc); nx_puts(" validated pages indexed+persisted; gov bytes="); nx_pi(gov.bytes_used); nx_puts(" reqs="); nx_pi(gov.n_requests); nx_putc(0x0A)
136 nx_puts("=> hardened pipeline (govern->robots->VALIDATED-TLS-fetch->extract->dedup->index->persist) on the REAL HTTPS web.\n")
137 return 0
138}