nx_crawl_bfs.nx source
↩ module page · 163 lines · 7070 B
1// nx_crawl_bfs.nx -- autonomous link-following crawler (frontier BFS).
2//
3// module: nishi-core.search.crawl_bfs
4// depends: nx_http_client, nx_html_to_text, nx_simhash, nx_bm25,
5// nx_crawl_frontier, fx, nx_str
6// capability: APP_RUNNABLE
7//
8// The real crawler: from ONE seed it FETCHES, extracts outbound links
9// (nx_crawl_frontier), enqueues the unseen ones (visited-set guards re-crawl),
10// and repeats -- discovering the site graph, not a hardcoded list. Each page is
11// content-deduped (nx_simhash) and indexed (nx_search_inverted). Proven on the
12// loopback mock-web: seed "/" -> discovers + crawls /wiki, /thothub, /fan.
13
14import "fx.nx"
15import "nx_str.nx"
16import "nx_http_client.nx"
17import "nx_html_to_text.nx"
18import "nx_simhash.nx"
19import "nx_bm25.nx"
20import "nx_crawl_frontier.nx"
21const K_MAGIC_8088: i64 = 8088
22const K_MAGIC_16384: i64 = 16384
23const K_MAGIC_8192: i64 = 8192
24
25func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
26func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
27func nx_put_i64(n: i64) -> i64 {
28 if n == 0 { nx_putc(0x30); return 0 }
29 var v: i64 = n; let t: *u8 = sys_mmap(32); var k: i64 = 0
30 while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
31 while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
32 return 0
33}
34func nx_body_off(resp: *u8, n: i64) -> i64 {
35 var i: i64 = 0
36 while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
37 return 0
38}
39
40// path component of a full URL (after scheme://host), or "/".
41func nx_url_path(url: *u8, out: *u8, outcap: i64) -> i64 {
42 let n: i64 = nx_str_len(url)
43 var i: i64 = 0
44 var afterscheme: i64 = 0
45 var found: i64 = 0
46 while i < n - 2 {
47 if found == 0 { if (url[i] as i64)==0x3A { if (url[i+1] as i64)==0x2F { if (url[i+2] as i64)==0x2F { afterscheme = i + 3; found = 1 } } } }
48 i = i + 1
49 }
50 var p: i64 = afterscheme
51 var pathstart: i64 = 0 - 1
52 while p < n { if pathstart < 0 { if (url[p] as i64) == 0x2F { pathstart = p } } p = p + 1 }
53 if pathstart < 0 { out[0] = 0x2F; out[1] = 0; return 1 }
54 var olen: i64 = 0
55 var q: i64 = pathstart
56 while q < n { if olen < outcap - 1 { out[olen] = url[q]; olen = olen + 1 } q = q + 1 }
57 out[olen] = 0
58 return olen
59}
60
61func nx_crawl_search(urls: **u8, texts: **u8, tlens: *i64, n: i64, q: *u8) -> i64 {
62 let h: i64 = nx_inv_hash_bytes_lower(q, nx_str_len(q))
63 nx_puts(" q='"); nx_puts(q); nx_puts("' -> ")
64 var found: i64 = 0
65 var d: i64 = 0
66 while d < n { if nx_bm25_tf(texts[d], tlens[d], h) > 0 { if found > 0 { nx_puts(", ") } nx_puts(urls[d]); found = found + 1 } d = d + 1 }
67 if found == 0 { nx_puts("(no match)") }
68 nx_putc(0x0A)
69 return 0
70}
71
72func main() -> i64 {
73 let addr: *u8 = sys_mmap(16)
74 nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, K_MAGIC_8088)
75 let host: *u8 = "127.0.0.1"
76 let hlen: i64 = nx_str_len(host)
77
78 let QCAP: i64 = 128
79 let queue: **u8 = sys_mmap(QCAP * 8) as **u8
80 var qh: i64 = 0
81 var qt: i64 = 0
82 let visited: **u8 = sys_mmap(QCAP * 8) as **u8
83 var nv: i64 = 0
84
85 let seed: *u8 = sys_mmap(8); seed[0] = 0x2F; seed[1] = 0 // "/"
86 queue[qt] = seed; qt = qt + 1
87
88 let docurl: **u8 = sys_mmap(64 * 8) as **u8
89 let doctext: **u8 = sys_mmap(64 * 8) as **u8
90 let doctlen: *i64 = sys_mmap(64 * 8) as *i64
91 let keptfp: *i64 = sys_mmap(64 * 8) as *i64
92 var ndoc: i64 = 0
93 let idx: *NxInvIndex = nx_inv_new(64)
94
95 nx_puts("=== Nishi BFS crawler (discovers by following links, seed '/') ===\n")
96 var budget: i64 = 20
97 var running: i64 = 1
98 while running == 1 {
99 if qh >= qt { running = 0 }
100 if ndoc >= budget { running = 0 }
101 if running == 1 {
102 let path: *u8 = queue[qh]
103 qh = qh + 1
104 var seen: i64 = 0
105 var vi: i64 = 0
106 while vi < nv { if nx_str_eq(path, visited[vi]) == 1 { seen = 1 } vi = vi + 1 }
107 if seen == 0 {
108 visited[nv] = path; nv = nv + 1
109 let resp: *u8 = sys_mmap(K_MAGIC_16384)
110 let vb: *i64 = sys_mmap(8) as *i64
111 let n: i64 = nx_http_client_get(addr, path, nx_str_len(path), host, hlen, resp, K_MAGIC_16384, vb)
112 if n > 0 {
113 let bo: i64 = nx_body_off(resp, n)
114 let bptr: *u8 = ((resp as i64) + bo) as *u8
115 let blen: i64 = n - bo
116 let text: *u8 = sys_mmap(K_MAGIC_8192)
117 let tlen: i64 = nx_html_to_text(bptr, blen, text, K_MAGIC_8192)
118 let fp: i64 = nx_simhash_fingerprint(text, tlen)
119 var dup: i64 = 0
120 var k: i64 = 0
121 while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
122 nx_puts("crawled "); nx_puts(path)
123 if dup == 1 { nx_puts(" (dup, skipped)\n") }
124 else {
125 docurl[ndoc] = path; doctext[ndoc] = text; doctlen[ndoc] = tlen; keptfp[ndoc] = fp
126 nx_inv_index_row(idx, text, tlen, ndoc)
127 ndoc = ndoc + 1
128 nx_puts(" (indexed)\n")
129 }
130 // discover links -> enqueue unseen
131 let lub: *u8 = sys_mmap(K_MAGIC_8192)
132 let offs: *i64 = sys_mmap(64 * 8) as *i64
133 let lens: *i64 = sys_mmap(64 * 8) as *i64
134 let nlinks: i64 = nx_crawl_extract_links(bptr, blen, lub, K_MAGIC_8192, offs, lens, 64)
135 var li: i64 = 0
136 while li < nlinks {
137 let linkurl: *u8 = sys_mmap(512)
138 var c: i64 = 0
139 while c < lens[li] { if c < 511 { linkurl[c] = lub[offs[li] + c] } c = c + 1 }
140 linkurl[lens[li]] = 0
141 let lp: *u8 = sys_mmap(256)
142 nx_url_path(linkurl, lp, 256)
143 var already: i64 = 0
144 var z: i64 = 0
145 while z < nv { if nx_str_eq(lp, visited[z]) == 1 { already = 1 } z = z + 1 }
146 if already == 0 { if qt < QCAP { queue[qt] = lp; qt = qt + 1 } }
147 li = li + 1
148 }
149 }
150 }
151 }
152 }
153 nx_inv_finalize_offsets(idx)
154 var d: i64 = 0
155 while d < ndoc { nx_inv_emit_row(idx, doctext[d], doctlen[d], d); d = d + 1 }
156
157 nx_puts("\ndiscovered + indexed "); nx_put_i64(ndoc); nx_puts(" pages from 1 seed. search:\n")
158 nx_crawl_search(docurl, doctext, doctlen, ndoc, "diora")
159 nx_crawl_search(docurl, doctext, doctlen, ndoc, "leaked")
160 nx_crawl_search(docurl, doctext, doctlen, ndoc, "comic")
161 nx_puts("=> autonomous link-following crawl: 1 seed -> whole site graph.\n")
162 return 0
163}