nx_crawl_polite.nx source
↩ module page · 162 lines · 8541 B
1// nx_crawl_polite.nx -- hardened, polite crawler (don't get blacklisted).
2//
3// module: nishi-core.search.crawl_polite
4// depends: nx_syscalls, nx_str, nx_http_client, nx_html_to_text, nx_simhash,
5// nx_bm25, nx_crawl_frontier, nx_robots
6// capability: APP_RUNNABLE
7//
8// nx_crawl_bfs + the anti-blacklist hardening (RFC 9309 + polite-crawling):
9// * fetch + honor robots.txt (skip Disallow paths) -- the host governs its
10// own content (the not-god cardinal), and honoring it avoids bans
11// * descriptive User-Agent header (identify ourselves, contactable)
12// * Crawl-delay pacing (sleep between requests; robots value or a default)
13// * 429/503 back-off (status-aware; honor rate-limit signals)
14// Proven on loopback: discovers the graph, SKIPS the robots-disallowed
15// /private, paces by Crawl-delay, sends our UA.
16
17import "nx_syscalls.nx"
18import "nx_str.nx"
19import "nx_http_client.nx"
20import "nx_http_get_ua.nx"
21import "nx_html_to_text.nx"
22import "nx_simhash.nx"
23import "nx_bm25.nx"
24import "nx_crawl_frontier.nx"
25import "nx_robots.nx"
26const K_MAGIC_8088: i64 = 8088
27const K_MAGIC_8192: i64 = 8192
28const K_MAGIC_16384: i64 = 16384
29
30func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 }
31func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 }
32func nx_put_i64(n: i64) -> i64 {
33 if n == 0 { nx_putc(0x30); return 0 }
34 var v: i64 = n; let t: *u8 = sys_mmap(32); var k: i64 = 0
35 while v > 0 { t[k] = 0x30 + (v - (v/10)*10); v = v/10; k = k+1 }
36 while k > 0 { k = k-1; sys_write(1, (((t as i64)+k) as *u8), 1) }
37 return 0
38}
39func nx_body_off(resp: *u8, n: i64) -> i64 {
40 var i: i64 = 0
41 while i < n - 3 { if (resp[i] as i64)==0x0D { if (resp[i+1] as i64)==0x0A { if (resp[i+2] as i64)==0x0D { if (resp[i+3] as i64)==0x0A { return i+4 } } } } i = i + 1 }
42 return 0
43}
44func bput(buf: *u8, p: *i64, cap: i64, s: *u8) -> i64 { let n: i64 = nx_str_len(s); var i: i64 = 0; while i < n { if p[0] < cap { buf[p[0]] = s[i]; p[0] = p[0]+1 } i = i+1 } return 0 }
45
46// GET with a descriptive User-Agent; drains the response; parses the status
47// code into out_status[0]. Returns bytes read (-1 on socket error).
48// thin wrapper over the shared primitive (DRY)
49func nx_polite_fetch(addr: *u8, path: *u8, plen: i64, host: *u8, hlen: i64,
50 ua: *u8, ualen: i64, out: *u8, cap: i64, out_status: *i64) -> i64 {
51 return nx_http_get_ua(addr, path, plen, host, hlen, ua, ualen, out, cap, out_status)
52}
53
54func main() -> i64 {
55 let addr: *u8 = sys_mmap(16)
56 nx_http_client_sockaddr_ipv4(addr, 127, 0, 0, 1, K_MAGIC_8088)
57 let host: *u8 = "127.0.0.1"
58 let hlen: i64 = nx_str_len(host)
59 let ua: *u8 = "NishiBot/1.0 (+http://nishi.example/bot)"
60 let ualen: i64 = nx_str_len(ua)
61 let uatok: *u8 = "NishiBot"
62 let uatoklen: i64 = nx_str_len(uatok)
63 let st: *i64 = sys_mmap(8) as *i64
64
65 // ---- 1. fetch + parse robots.txt ----
66 let rb: *u8 = sys_mmap(K_MAGIC_8192)
67 let rn: i64 = nx_polite_fetch(addr, "/robots.txt", 11, host, hlen, ua, ualen, rb, K_MAGIC_8192, st)
68 var robots: *u8 = rb
69 var robolen: i64 = 0
70 if rn > 0 { let rbo: i64 = nx_body_off(rb, rn); robots = ((rb as i64) + rbo) as *u8; robolen = rn - rbo }
71 let cd: i64 = nx_robots_crawl_delay(robots, robolen, uatok, uatoklen)
72 var delay_ms: i64 = cd * 1000
73 if delay_ms <= 0 { delay_ms = 500 }
74 nx_puts("robots.txt fetched ("); nx_put_i64(robolen); nx_puts(" bytes); Crawl-delay="); nx_put_i64(cd)
75 nx_puts("s; UA='"); nx_puts(ua); nx_puts("'\n")
76
77 // ---- 2. polite BFS ----
78 let QCAP: i64 = 128
79 let queue: **u8 = sys_mmap(QCAP * 8) as **u8
80 var qh: i64 = 0
81 var qt: i64 = 0
82 let visited: **u8 = sys_mmap(QCAP * 8) as **u8
83 var nv: i64 = 0
84 let seed: *u8 = sys_mmap(8); seed[0] = 0x2F; seed[1] = 0
85 queue[qt] = seed; qt = qt + 1
86
87 let idx: *NxInvIndex = nx_inv_new(64)
88 var ndoc: i64 = 0
89 let keptfp: *i64 = sys_mmap(64 * 8) as *i64
90
91 var running: i64 = 1
92 while running == 1 {
93 if qh >= qt { running = 0 }
94 if ndoc >= 20 { running = 0 }
95 if running == 1 {
96 let path: *u8 = queue[qh]; qh = qh + 1
97 var seen: i64 = 0
98 var vi: i64 = 0
99 while vi < nv { if nx_str_eq(path, visited[vi]) == 1 { seen = 1 } vi = vi + 1 }
100 if seen == 0 {
101 visited[nv] = path; nv = nv + 1
102 // ROBOTS: respect the host's rules
103 if nx_robots_allowed(robots, robolen, uatok, uatoklen, path, nx_str_len(path)) == 0 {
104 nx_puts(" SKIP (robots-disallowed): "); nx_puts(path); nx_putc(0x0A)
105 } else {
106 sys_sleep_ms(delay_ms) // politeness pacing
107 let resp: *u8 = sys_mmap(K_MAGIC_16384)
108 let n: i64 = nx_polite_fetch(addr, path, nx_str_len(path), host, hlen, ua, ualen, resp, K_MAGIC_16384, st)
109 if st[0] == 429 { nx_puts(" BACKOFF 429: "); nx_puts(path); nx_putc(0x0A); sys_sleep_ms(delay_ms * 4) }
110 else { if st[0] == 503 { nx_puts(" BACKOFF 503: "); nx_puts(path); nx_putc(0x0A); sys_sleep_ms(delay_ms * 4) }
111 else { if n > 0 {
112 let bo: i64 = nx_body_off(resp, n)
113 let bptr: *u8 = ((resp as i64) + bo) as *u8
114 let blen: i64 = n - bo
115 let text: *u8 = sys_mmap(K_MAGIC_8192)
116 let tlen: i64 = nx_html_to_text(bptr, blen, text, K_MAGIC_8192)
117 let fp: i64 = nx_simhash_fingerprint(text, tlen)
118 var dup: i64 = 0
119 var k: i64 = 0
120 while k < ndoc { if nx_simhash_hamming(fp, keptfp[k]) <= 6 { dup = 1 } k = k + 1 }
121 nx_puts(" crawled (status "); nx_put_i64(st[0]); nx_puts("): "); nx_puts(path)
122 if dup == 1 { nx_puts(" (dup)\n") } else {
123 keptfp[ndoc] = fp; nx_inv_index_row(idx, text, tlen, ndoc); ndoc = ndoc + 1; nx_puts(" (indexed)\n")
124 }
125 // discover links
126 let lub: *u8 = sys_mmap(K_MAGIC_8192)
127 let offs: *i64 = sys_mmap(64 * 8) as *i64
128 let lens: *i64 = sys_mmap(64 * 8) as *i64
129 let nlinks: i64 = nx_crawl_extract_links(bptr, blen, lub, K_MAGIC_8192, offs, lens, 64)
130 var li: i64 = 0
131 while li < nlinks {
132 let linkurl: *u8 = sys_mmap(512)
133 var c: i64 = 0
134 while c < lens[li] { if c < 511 { linkurl[c] = lub[offs[li] + c] } c = c + 1 }
135 linkurl[lens[li]] = 0
136 // path component (after scheme://host)
137 let lp: *u8 = sys_mmap(256)
138 let ln2: i64 = nx_str_len(linkurl)
139 var sp: i64 = 0
140 var afters: i64 = 0
141 var fnd: i64 = 0
142 while sp < ln2 - 2 { if fnd == 0 { if (linkurl[sp] as i64)==0x3A { if (linkurl[sp+1] as i64)==0x2F { if (linkurl[sp+2] as i64)==0x2F { afters = sp+3; fnd = 1 } } } } sp = sp + 1 }
143 var ps: i64 = 0 - 1
144 var pp: i64 = afters
145 while pp < ln2 { if ps < 0 { if (linkurl[pp] as i64)==0x2F { ps = pp } } pp = pp + 1 }
146 if ps < 0 { lp[0] = 0x2F; lp[1] = 0 } else { var o2: i64 = 0; var q2: i64 = ps; while q2 < ln2 { if o2 < 255 { lp[o2] = linkurl[q2]; o2 = o2 + 1 } q2 = q2 + 1 } lp[o2] = 0 }
147 var already: i64 = 0
148 var z: i64 = 0
149 while z < nv { if nx_str_eq(lp, visited[z]) == 1 { already = 1 } z = z + 1 }
150 if already == 0 { if qt < QCAP { queue[qt] = lp; qt = qt + 1 } }
151 li = li + 1
152 }
153 } } }
154 }
155 }
156 }
157 }
158 nx_inv_finalize_offsets(idx)
159 nx_puts("\npolitely crawled + indexed "); nx_put_i64(ndoc); nx_puts(" pages (robots-respected, UA-identified, "); nx_put_i64(delay_ms); nx_puts("ms paced)\n")
160 nx_puts("=> hardened: honors robots.txt + crawl-delay + identifies itself + backs off on 429/503.\n")
161 return 0
162}