`nx_live_crawl.nx` source

1// nx_live_crawl.nx -- LIVE crawl of a real internet host via the sovereign
2// stack (our DNS resolver + our HTTP client + our HTML extractor + our index).
3// Target: info.cern.ch (the first website; serves plain HTTP on :80, so it
4// works with our no-TLS-yet client). Honest scope: HTTP-only until the crawler
5// gets the TLS layer the browser already has.
6import "fx.nx"
7import "nx_str.nx"
8import "nx_dns_resolve_a_record.nx"
9import "nx_http_client.nx"
10import "nx_html_to_text.nx"
11import "nx_bm25.nx"
12import "nx_search_inverted.nx"   // NxInvIndex + nx_inv_* + nx_bm25_tf -- same missing import as
13                                 // nx_search_neutrality_gate; nx_bm25.nx never carried these.
14
15func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
16func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
17func nx_pi(n: i64) -> i64 {
18    if n == 0 { nx_putc(0x30); return 0 }
19    var v: i64 = n; let t: *u8 = sys_mmap(32); var k: i64 = 0
20    while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
21    while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
22    return 0
23}
24func nx_body_off(resp: *u8, n: i64) -> i64 {
25    var i: i64 = 0
26    while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
27    return 0
28}
29func nx_report(text: *u8, tlen: i64, q: *u8) -> i64 {
30    let c: i64 = nx_bm25_tf(text, tlen, nx_inv_hash_bytes_lower(q, nx_str_len(q)))
31    nx_puts("    '"); nx_puts(q); nx_puts("' x"); nx_pi(c); nx_putc(0x0A)
32    return 0
33}
34
35func main() -> i64 {
36    let host: *u8 = "info.cern.ch"
37    let hlen: i64 = nx_str_len(host)
38    nx_puts("=== LIVE crawl via the sovereign stack -> "); nx_puts(host); nx_puts(" ===\n")
39
40    // 1. resolve via OUR DNS
41    let r: *DnsResolveResult = nx_dns_resolve_default(host, hlen, 1718452800)
42    if r.verdict != NX_DNS_R_OK { nx_puts("sovereign DNS resolve FAILED, verdict="); nx_pi(r.verdict); nx_putc(0x0A); return 1 }
43    let ipp: i64 = r.ipv4_packed
44    let a: i64 = (ipp >> 24) & 0xff
45    let b: i64 = (ipp >> 16) & 0xff
46    let c: i64 = (ipp >> 8) & 0xff
47    let d: i64 = ipp & 0xff
48    nx_puts("  sovereign DNS resolved -> "); nx_pi(a); nx_putc(0x2E); nx_pi(b); nx_putc(0x2E); nx_pi(c); nx_putc(0x2E); nx_pi(d); nx_putc(0x0A)
49
50    // 2. fetch over OUR HTTP client
51    let addr: *u8 = sys_mmap(16)
52    nx_http_client_sockaddr_ipv4(addr, a, b, c, d, 80)
53    let resp: *u8 = sys_mmap(131072)
54    let vb: *i64 = sys_mmap(8) as *i64
55    let n: i64 = nx_http_client_get(addr, "/", 1, host, hlen, resp, 131072, vb)
56    if n <= 0 { nx_puts("  HTTP fetch FAILED (verdict "); nx_pi(vb[0]); nx_puts(")\n"); return 2 }
57    nx_puts("  fetched "); nx_pi(n); nx_puts(" bytes over our HTTP client\n")
58
59    // 3. extract text via OUR HTML->text
60    let bo: i64 = nx_body_off(resp, n)
61    let text: *u8 = sys_mmap(131072)
62    let tlen: i64 = nx_html_to_text(((resp as i64) + bo) as *u8, n - bo, text, 131072)
63    nx_puts("  extracted "); nx_pi(tlen); nx_puts(" text chars\n")
64
65    // 4. index via OUR inverted index + query
66    let idx: *NxInvIndex = nx_inv_new(8)
67    nx_inv_index_row(idx, text, tlen, 0)
68    nx_inv_finalize_offsets(idx)
69    nx_inv_emit_row(idx, text, tlen, 0)
70    nx_puts("  indexed; term frequencies on the live page:\n")
71    nx_report(text, tlen, "hypertext")
72    nx_report(text, tlen, "web")
73    nx_report(text, tlen, "cern")
74    nx_report(text, tlen, "project")
75    nx_puts("=> LIVE: sovereign DNS + HTTP client crawled a REAL internet host, extracted + indexed, searchable.\n")
76    return 0
77}
nx_live_crawl.nx source

`nx_live_crawl.nx` source
