code wiki / _hdl_build / nx_sitemap_extract.nx
nx_sitemap_extract.nx source
↩ module page · 89 lines · 4389 B
1// nx_sitemap_extract.nx -- sitemap discovery (grow-capabilities 2026-07-06): the standard SOTA-crawler bulk
2// URL-discovery source. robots.txt "Sitemap:" lines point to sitemap XML; a sitemap is either a <urlset>
3// (page <loc>s) or a <sitemapindex> (sub-sitemap <loc>s, often .xml.gz). MEASURED: nyt/bbc/nhentai/wikipedia
4// all declare sitemaps. This extracts (a) Sitemap: URLs from robots.txt and (b) <loc> URLs from a sitemap --
5// both feed the crawler frontier so coverage isn't limited to <a>-linked pages. A parser, not a fetcher.
6// license_tier: ORIGINAL
7import "nx_syscalls.nx"
8const K_MAGIC_1000000: i64 = 1000000
9
10func sm_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
11// case-insensitive: does hay[off..off+blen) == b (b lowercase)?
12func sm_ci_at(hay: *u8, off: i64, hlen: i64, b: *u8, blen: i64) -> i64 {
13 if off + blen > hlen { return 0 }
14 var i: i64 = 0
15 while i < blen { if sm_lc(hay[off+i] as i64) != (b[i] as i64) { return 0 } i = i + 1 }
16 return 1
17}
18// copy a URL token (trim leading ws; stop at ws/CR/LF/'<') into slot; returns length (0 if empty/too long).
19func sm_copy_url(src: *u8, s: i64, e: i64, out: *u8, cap: i64) -> i64 {
20 var p: i64 = s
21 while p < e { let c: i64 = src[p] as i64; if c==32 { p=p+1 } else { if c==9 { p=p+1 } else { p=e+K_MAGIC_1000000 } } }
22 var q: i64 = s; var st: i64 = 0
23 while st == 0 { if q >= e { st=1 } else { let c: i64=src[q] as i64; if c==32 { q=q+1 } else { if c==9 { q=q+1 } else { st=1 } } } }
24 var o: i64 = 0
25 var run: i64 = 1
26 while run == 1 {
27 if q >= e { run = 0 } else {
28 let c: i64 = src[q] as i64
29 if c==32 { run=0 } else { if c==9 { run=0 } else { if c==10 { run=0 } else { if c==13 { run=0 } else { if c==60 { run=0 } else {
30 if o < cap-1 { out[o]=src[q] as u8; o=o+1 } q=q+1
31 } } } } }
32 }
33 }
34 out[o] = 0 as u8
35 return o
36}
37
38// robots.txt -> Sitemap: URLs, packed one per `slot` bytes. returns count (<=max).
39func sm_robots_sitemaps(rob: *u8, n: i64, out: *u8, slot: i64, max: i64) -> i64 {
40 let key: *u8 = "sitemap:" as *u8
41 var i: i64 = 0
42 var count: i64 = 0
43 while i < n {
44 // line start? (i==0 or prev char is \n)
45 var at_line: i64 = 0
46 if i == 0 { at_line = 1 } else { if rob[i-1]==(10 as u8) { at_line = 1 } }
47 if at_line == 1 { if sm_ci_at(rob, i, n, key, 8) == 1 {
48 if count < max {
49 let dst: *u8 = (out as i64 + count*slot) as *u8
50 // find end of line
51 var e: i64 = i + 8
52 var f: i64 = 0
53 while f == 0 { if e >= n { f=1 } else { if rob[e]==(10 as u8) { f=1 } else { e=e+1 } } }
54 if sm_copy_url(rob, i+8, e, dst, slot) > 0 { count = count + 1 }
55 }
56 } }
57 i = i + 1
58 }
59 return count
60}
61
62// sitemap XML -> <loc>...</loc> URLs, packed one per `slot` bytes. Works for BOTH <urlset> (page locs) and
63// <sitemapindex> (sub-sitemap locs). returns count (<=max).
64func sm_extract_locs(xml: *u8, n: i64, out: *u8, slot: i64, max: i64) -> i64 {
65 var i: i64 = 0
66 var count: i64 = 0
67 while i + 5 < n {
68 if xml[i]==(60 as u8) { if sm_ci_at(xml, i+1, n, "loc" as *u8, 3) == 1 {
69 let ac: i64 = xml[i+4] as i64
70 if ac==62 { if count < max { // "<loc>"
71 let s: i64 = i + 5
72 var e: i64 = s; var f: i64 = 0
73 while f == 0 { if e >= n { f=1 } else { if xml[e]==(60 as u8) { f=1 } else { e=e+1 } } }
74 let dst: *u8 = (out as i64 + count*slot) as *u8
75 if sm_copy_url(xml, s, e, dst, slot) > 0 { count = count + 1 }
76 i = e
77 } }
78 } }
79 i = i + 1
80 }
81 return count
82}
83
84// classify a loc: 1 if it is itself a sub-SITEMAP (ends .xml or .xml.gz), else 0 (a page to crawl).
85func sm_is_subsitemap(u: *u8, ul: i64) -> i64 {
86 if ul >= 4 { if u[ul-4]==(46 as u8) { if sm_lc(u[ul-3] as i64)==120 { if sm_lc(u[ul-2] as i64)==109 { if sm_lc(u[ul-1] as i64)==108 { return 1 } } } } }
87 if ul >= 7 { if u[ul-7]==(46 as u8) { if sm_lc(u[ul-6] as i64)==120 { if sm_lc(u[ul-5] as i64)==109 { if sm_lc(u[ul-4] as i64)==108 { if u[ul-3]==(46 as u8) { if sm_lc(u[ul-2] as i64)==103 { if sm_lc(u[ul-1] as i64)==122 { return 1 } } } } } } } } // ".xml.gz"
88 return 0
89}