nx_crawl_frontier.nx source
↩ module page · 115 lines · 4589 B
1// nx_crawl_frontier.nx -- outbound-link (frontier) extraction for the crawler.
2//
3// module: nishi-core.search.crawl_frontier
4// depends: syscalls.nx
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// The discovery half of the crawl loop: from a fetched HTML page, extract the
9// outbound href links to enqueue for crawling. Byte-scans for href="..." or
10// href='...', keeps absolute http(s) links, and dedups exact repeats. The
11// frontier is what lets the crawler walk an entity's real footprint (its own
12// links, official-site links, primary-source citations) rather than only
13// whatever ranks -- the knowledge-graph / link-graph approach to QUALITY
14// coverage, not SEO-bait. (Relative-URL resolution against a base href is a v2
15// refinement; absolute links are the bulk of a frontier and the safe v1.)
16
17import "syscalls.nx"
18
19func nx_cf_lc(c: i64) -> i64 {
20 if c >= 0x41 { if c <= 0x5A { return c + 0x20 } }
21 return c
22}
23
24// "href=" (case-insensitive on the name) at off, with at least one char after.
25func nx_cf_is_href_eq_at(html: *u8, html_len: i64, off: i64) -> i64 {
26 if off + 6 > html_len { return 0 }
27 if nx_cf_lc(html[off] as i64) != 0x68 { return 0 } // h
28 if nx_cf_lc(html[off + 1] as i64) != 0x72 { return 0 } // r
29 if nx_cf_lc(html[off + 2] as i64) != 0x65 { return 0 } // e
30 if nx_cf_lc(html[off + 3] as i64) != 0x66 { return 0 } // f
31 if (html[off + 4] as i64) != 0x3D { return 0 } // =
32 return 1
33}
34
35// does html[off..] begin with literal lit (lit_len bytes)? bounds-checked.
36func nx_cf_has_prefix(html: *u8, html_len: i64, off: i64, lit: *u8, lit_len: i64) -> i64 {
37 if off + lit_len > html_len { return 0 }
38 var k: i64 = 0
39 while k < lit_len {
40 if html[off + k] != lit[k] { return 0 }
41 k = k + 1
42 }
43 return 1
44}
45
46// byte-equal: html[cs..cs+clen) vs url_buf[bo..bo+blen) ?
47func nx_cf_eq(html: *u8, cs: i64, clen: i64, buf: *u8, bo: i64, blen: i64) -> i64 {
48 if clen != blen { return 0 }
49 var k: i64 = 0
50 while k < clen {
51 if html[cs + k] != buf[bo + k] { return 0 }
52 k = k + 1
53 }
54 return 1
55}
56
57// Extract unique absolute http(s) links into url_buf; offsets[]/lengths[] index
58// them. Returns the number of links written (<= max_links). Mirrors the
59// nx_html_extract_imgs contract.
60func nx_crawl_extract_links(html: *u8, html_len: i64,
61 url_buf: *u8, url_buf_cap: i64,
62 offsets: *i64, lengths: *i64, max_links: i64) -> i64 {
63 var count: i64 = 0
64 var used: i64 = 0
65 var i: i64 = 0
66 while i < html_len {
67 var step: i64 = 1
68 if nx_cf_is_href_eq_at(html, html_len, i) == 1 {
69 let q: i64 = html[i + 5] as i64
70 var is_quote: i64 = 0
71 if q == 0x22 { is_quote = 1 }
72 if q == 0x27 { is_quote = 1 }
73 if is_quote == 1 {
74 let cs: i64 = i + 6
75 var e: i64 = cs
76 var run: i64 = 1
77 while run == 1 {
78 run = 0
79 if e < html_len {
80 if (html[e] as i64) != q { e = e + 1; run = 1 }
81 }
82 }
83 let clen: i64 = e - cs
84 if clen > 0 {
85 var is_http: i64 = 0
86 if nx_cf_has_prefix(html, html_len, cs, "http://", 7) == 1 { is_http = 1 }
87 if nx_cf_has_prefix(html, html_len, cs, "https://", 8) == 1 { is_http = 1 }
88 if is_http == 1 {
89 var dup: i64 = 0
90 var m: i64 = 0
91 while m < count {
92 if nx_cf_eq(html, cs, clen, url_buf, offsets[m], lengths[m]) == 1 { dup = 1 }
93 m = m + 1
94 }
95 if dup == 0 {
96 if count < max_links {
97 if used + clen <= url_buf_cap {
98 var k: i64 = 0
99 while k < clen { url_buf[used + k] = html[cs + k]; k = k + 1 }
100 offsets[count] = used
101 lengths[count] = clen
102 used = used + clen
103 count = count + 1
104 }
105 }
106 }
107 }
108 }
109 step = (e - i) + 1
110 }
111 }
112 i = i + step
113 }
114 return count
115}