`nx_crawl_run.nx` source

↩ module page · 109 lines · 4633 B
1// nx_crawl_run.nx -- end-to-end crawl: fetch (real HTTP) -> extract -> dedup
2// -> index -> search, over the loopback mock-web. Proves the whole sovereign
3// loop AND that the engine finds uncensored content (the adult-aggregator
4// page) once crawled -- the answer to "did our search find it": it does, once
5// it crawls, with no prudish category-censorship.
6import "fx.nx"
7import "nx_str.nx"
8import "nx_http_client.nx"
9import "nx_html_to_text.nx"
10import "nx_simhash.nx"
11import "nx_bm25.nx"
12const K_MAGIC_8088: i64 = 8088
13const K_MAGIC_16384: i64 = 16384
14const K_MAGIC_8192: i64 = 8192
15
16func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
17func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
18func nx_put_i64(n: i64) -> i64 {
19    if n == 0 { nx_putc(0x30); return 0 }
20    var v: i64 = n; let t: *u8 = sys_mmap(32); var k: i64 = 0
21    while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
22    while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
23    return 0
24}
25
26// offset of the HTTP body (just past the first CRLFCRLF), or 0.
27func nx_body_off(resp: *u8, n: i64) -> i64 {
28    var i: i64 = 0
29    while i < n - 3 {
30        if (resp[i] as i64) == 0x0D { if (resp[i+1] as i64) == 0x0A { if (resp[i+2] as i64) == 0x0D { if (resp[i+3] as i64) == 0x0A { return i + 4 } } } }
31        i = i + 1
32    }
33    return 0
34}
35
36// print which crawled docs contain the query term
37func nx_crawl_search(urls: **u8, texts: **u8, tlens: *i64, n: i64, q: *u8) -> i64 {
38    let h: i64 = nx_inv_hash_bytes_lower(q, nx_str_len(q))
39    nx_puts("  q='"); nx_puts(q); nx_puts("' -> ")
40    var found: i64 = 0
41    var d: i64 = 0
42    while d < n {
43        if nx_bm25_tf(texts[d], tlens[d], h) > 0 {
44            if found > 0 { nx_puts(", ") }
45            nx_puts(urls[d]); found = found + 1
46        }
47        d = d + 1
48    }
49    if found == 0 { nx_puts("(no match)") }
50    nx_putc(0x0A)
51    return 0
52}
53
54func main() -> i64 {
55    let addr: *u8 = sys_mmap(16)
56    nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, K_MAGIC_8088)
57    let host: *u8 = "127.0.0.1"
58    let hlen: i64 = nx_str_len(host)
59
60    let NSEED: i64 = 4
61    let seeds: **u8 = sys_mmap(NSEED * 8) as **u8
62    seeds[0] = "/wiki"; seeds[1] = "/thothub"; seeds[2] = "/fan"; seeds[3] = "/thothub"
63
64    let docurl:  **u8 = sys_mmap(16 * 8) as **u8
65    let doctext: **u8 = sys_mmap(16 * 8) as **u8
66    let doctlen: *i64 = sys_mmap(16 * 8) as *i64
67    let keptfp:  *i64 = sys_mmap(16 * 8) as *i64
68    var ndoc: i64 = 0
69    let idx: *NxInvIndex = nx_inv_new(16)
70
71    nx_puts("=== Nishi crawler over loopback mock-web (REAL HTTP fetch on :8088) ===\n")
72    var s: i64 = 0
73    while s < NSEED {
74        let resp: *u8 = sys_mmap(K_MAGIC_16384)
75        let vb: *i64 = sys_mmap(8) as *i64
76        let n: i64 = nx_http_client_get(addr, seeds[s], nx_str_len(seeds[s]), host, hlen, resp, K_MAGIC_16384, vb)
77        if n > 0 {
78            let bo: i64 = nx_body_off(resp, n)
79            let text: *u8 = sys_mmap(K_MAGIC_8192)
80            let tlen: i64 = nx_html_to_text(((resp as i64) + bo) as *u8, n - bo, text, K_MAGIC_8192)
81            let fp: i64 = nx_simhash_fingerprint(text, tlen)
82            var dup: i64 = 0
83            var k: i64 = 0
84            while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
85            nx_puts("fetched "); nx_puts(seeds[s]); nx_puts(" ("); nx_put_i64(n); nx_puts(" bytes -> "); nx_put_i64(tlen); nx_puts(" text chars)")
86            if dup == 1 { nx_puts(" -> content NEAR-DUP, collapsed (not re-indexed)\n") }
87            else {
88                docurl[ndoc] = seeds[s]; doctext[ndoc] = text; doctlen[ndoc] = tlen; keptfp[ndoc] = fp
89                nx_inv_index_row(idx, text, tlen, ndoc)
90                nx_puts(" -> indexed as doc "); nx_put_i64(ndoc); nx_putc(0x0A)
91                ndoc = ndoc + 1
92            }
93        } else {
94            nx_puts("fetch FAILED for "); nx_puts(seeds[s]); nx_puts(" (mock-web up on :8088?)\n")
95        }
96        s = s + 1
97    }
98    nx_inv_finalize_offsets(idx)
99    var d: i64 = 0
100    while d < ndoc { nx_inv_emit_row(idx, doctext[d], doctlen[d], d); d = d + 1 }
101
102    nx_puts("\nindexed "); nx_put_i64(ndoc); nx_puts(" unique crawled docs. search over OUR crawl:\n")
103    nx_crawl_search(docurl, doctext, doctlen, ndoc, "diora")
104    nx_crawl_search(docurl, doctext, doctlen, ndoc, "leaked")
105    nx_crawl_search(docurl, doctext, doctlen, ndoc, "comic")
106    nx_crawl_search(docurl, doctext, doctlen, ndoc, "wikipedia")
107    nx_puts("=> crawler fetched real pages, deduped, indexed; the adult page IS found (uncensored).\n")
108    return 0
109}
nx_crawl_run.nx source

`nx_crawl_run.nx` source
