nx_html_links.nx source
↩ module page · 88 lines · 3624 B
1// nx_html_links.nx -- LINK EXTRACTION kernel: the crawler's discovery primitive.
2//
3// module: nishi-core.search.html_links
4// depends: (none beyond syscalls) -- pure scan, no allocation surprises
5// capability: CORE_COMPUTE
6//
7// Scans HTML for href= attribute values (case-insensitive, ' or " quoted) and
8// collects ABSOLUTE https:// URLs into a caller-provided flat store
9// (null-terminated each, offsets out). Relative/protocol-relative/http/
10// fragment links are skipped at this layer -- the crawler follows the
11// validated-HTTPS web only; duplicates are the CALLER's policy (dedupe is a
12// crawl decision, not a parse decision). KATs: nx_html_links_test.nx.
13
14import "nx_syscalls.nx"
15
16func hl_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
17
18// html[i..] starts with "href=" (case-insensitive)?
19func hl_is_href(html: *u8, n: i64, i: i64) -> i64 {
20 if i + 5 > n { return 0 }
21 if hl_lc(html[i] as i64) != 0x68 { return 0 } // h
22 if hl_lc(html[i+1] as i64) != 0x72 { return 0 } // r
23 if hl_lc(html[i+2] as i64) != 0x65 { return 0 } // e
24 if hl_lc(html[i+3] as i64) != 0x66 { return 0 } // f
25 if (html[i+4] as i64) != 0x3D { return 0 } // =
26 return 1
27}
28
29// value at html[p..] is "https://..." ?
30func hl_is_https(html: *u8, n: i64, p: i64) -> i64 {
31 if p + 8 > n { return 0 }
32 if hl_lc(html[p] as i64) != 0x68 { return 0 }
33 if hl_lc(html[p+1] as i64) != 0x74 { return 0 }
34 if hl_lc(html[p+2] as i64) != 0x74 { return 0 }
35 if hl_lc(html[p+3] as i64) != 0x70 { return 0 }
36 if hl_lc(html[p+4] as i64) != 0x73 { return 0 }
37 if (html[p+5] as i64) != 0x3A { return 0 }
38 if (html[p+6] as i64) != 0x2F { return 0 }
39 if (html[p+7] as i64) != 0x2F { return 0 }
40 return 1
41}
42
43// extract absolute-https hrefs. Returns link count; store gets each URL
44// null-terminated, offs[k] = offset of link k in store.
45func nx_html_links(html: *u8, n: i64, store: *u8, store_cap: i64,
46 offs: *i64, max_links: i64) -> i64 {
47 var count: i64 = 0
48 var w: i64 = 0
49 var i: i64 = 0
50 while i < n - 8 {
51 if hl_is_href(html, n, i) == 0 { i = i + 1 } else {
52 var p: i64 = i + 5
53 var q: i64 = 0
54 let c: i64 = html[p] as i64
55 if c == 0x22 { q = 0x22 }
56 if c == 0x27 { q = 0x27 }
57 if q == 0 { i = p } else {
58 p = p + 1
59 if hl_is_https(html, n, p) == 0 { i = p } else {
60 // copy until closing quote (bounded by caps; '<' aborts = malformed)
61 let start_w: i64 = w
62 var ok: i64 = 1
63 var scan: i64 = 1
64 while scan == 1 {
65 if p >= n { ok = 0; scan = 0 } else {
66 let ch: i64 = html[p] as i64
67 if ch == q { scan = 0 } else {
68 if ch == 0x3C { ok = 0; scan = 0 } else {
69 if w < store_cap - 1 { store[w] = ch; w = w + 1; p = p + 1 } else { ok = 0; scan = 0 }
70 }
71 }
72 }
73 }
74 if ok == 1 {
75 if count < max_links {
76 store[w] = 0 as u8
77 w = w + 1
78 offs[count] = start_w
79 count = count + 1
80 }
81 } else { w = start_w }
82 i = p
83 }
84 }
85 }
86 }
87 return count
88}