nx_robots.nx source
↩ module page · 188 lines · 7384 B
1// nx_robots.nx -- robots.txt parser + matcher (RFC 9309).
2//
3// module: nishi-core.search.robots
4// depends: nx_str.nx
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// genealogy_id: rfc9309_robots_exclusion_protocol
9//
10// WHY (operator: don't get our IP blacklisted; and the not-god cardinal): a
11// host's robots.txt is how the HOST governs crawling of ITS content. Honoring
12// it is both polite (avoids bans) and exactly on-cardinal -- the host's rules
13// over the host's content, which we respect rather than override. RFC 9309
14// group selection (most-specific user-agent, else '*') + longest-match
15// Allow/Disallow precedence (Allow wins ties). Plus Crawl-delay extraction for
16// politeness pacing. Prefix + '$' end-anchor matching (the common forms);
17// full '*' wildcard is a refinement. Deterministic, integer-only.
18
19import "nx_str.nx"
20
21func _rb_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5A { return c + 0x20 } } return c }
22
23// exclusive end of the line starting at ls (index of '\n' or rlen).
24func _rb_line_end(robots: *u8, rlen: i64, ls: i64) -> i64 {
25 var e: i64 = ls
26 var run: i64 = 1
27 while run == 1 { run = 0; if e < rlen { if (robots[e] as i64) != 0x0A { e = e + 1; run = 1 } } }
28 return e
29}
30
31// if line [ls,le) begins with directive `dir` (lowercase, incl ':'),
32// case-insensitive, return the value-start offset (spaces skipped); else -1.
33func _rb_directive(robots: *u8, ls: i64, le: i64, dir: *u8) -> i64 {
34 let dl: i64 = nx_str_len(dir)
35 if le - ls < dl { return 0 - 1 }
36 var i: i64 = 0
37 while i < dl { if _rb_lc(robots[ls + i] as i64) != (dir[i] as i64) { return 0 - 1 } i = i + 1 }
38 var p: i64 = ls + dl
39 var run: i64 = 1
40 while run == 1 { run = 0; if p < le { if (robots[p] as i64) == 0x20 { p = p + 1; run = 1 } } }
41 return p
42}
43
44// trimmed value end (drop trailing CR / spaces) for [vs,le).
45func _rb_val_end(robots: *u8, vs: i64, le: i64) -> i64 {
46 var e: i64 = le
47 var run: i64 = 1
48 while run == 1 {
49 run = 0
50 if e > vs {
51 let c: i64 = robots[e - 1] as i64
52 if c == 0x0D { e = e - 1; run = 1 }
53 if c == 0x20 { e = e - 1; run = 1 }
54 }
55 }
56 return e
57}
58
59// is the robots user-agent token [vs,ve) a case-insensitive substring of `ua`?
60func _rb_ua_token_matches(robots: *u8, vs: i64, ve: i64, ua: *u8, ualen: i64) -> i64 {
61 let tlen: i64 = ve - vs
62 if tlen <= 0 { return 0 }
63 if tlen > ualen { return 0 }
64 var i: i64 = 0
65 while i <= ualen - tlen {
66 var j: i64 = 0
67 var ok: i64 = 1
68 while j < tlen { if _rb_lc(ua[i + j] as i64) != _rb_lc(robots[vs + j] as i64) { ok = 0; j = tlen } else { j = j + 1 } }
69 if ok == 1 { return 1 }
70 i = i + 1
71 }
72 return 0
73}
74
75func _rb_is_star(robots: *u8, vs: i64, ve: i64) -> i64 {
76 if ve - vs == 1 { if (robots[vs] as i64) == 0x2A { return 1 } }
77 return 0
78}
79
80// does `path` match rule-path [vs,ve)? returns matched length, or -1.
81// prefix match; trailing '$' = exact end-anchor; empty rule = no match (-1).
82func _rb_path_match(path: *u8, plen: i64, robots: *u8, vs: i64, ve: i64) -> i64 {
83 var rve: i64 = ve
84 var anchored: i64 = 0
85 if rve > vs { if (robots[rve - 1] as i64) == 0x24 { anchored = 1; rve = rve - 1 } } // '$'
86 let rlen: i64 = rve - vs
87 if rlen == 0 { return 0 - 1 }
88 if rlen > plen { return 0 - 1 }
89 var i: i64 = 0
90 while i < rlen { if (path[i] as i64) != (robots[vs + i] as i64) { return 0 - 1 } i = i + 1 }
91 if anchored == 1 { if plen != rlen { return 0 - 1 } }
92 return rlen
93}
94
95// is there a user-agent group specific to `ua` (a non-'*' token that matches)?
96func _rb_has_specific(robots: *u8, rlen: i64, ua: *u8, ualen: i64) -> i64 {
97 let d_ua: *u8 = "user-agent:"
98 var ls: i64 = 0
99 while ls < rlen {
100 let le: i64 = _rb_line_end(robots, rlen, ls)
101 let vs: i64 = _rb_directive(robots, ls, le, d_ua)
102 if vs >= 0 {
103 let ve: i64 = _rb_val_end(robots, vs, le)
104 if _rb_is_star(robots, vs, ve) == 0 { if _rb_ua_token_matches(robots, vs, ve, ua, ualen) == 1 { return 1 } }
105 }
106 ls = le + 1
107 }
108 return 0
109}
110
111// RFC 9309 verdict: 1 = allowed to crawl `path`, 0 = disallowed.
112func nx_robots_allowed(robots: *u8, rlen: i64, ua: *u8, ualen: i64, path: *u8, plen: i64) -> i64 {
113 var star_only: i64 = 1
114 if _rb_has_specific(robots, rlen, ua, ualen) == 1 { star_only = 0 }
115 let d_ua: *u8 = "user-agent:"
116 let d_dis: *u8 = "disallow:"
117 let d_allow: *u8 = "allow:"
118 var active: i64 = 0
119 var best_dis: i64 = 0 - 1
120 var best_allow: i64 = 0 - 1
121 var ls: i64 = 0
122 while ls < rlen {
123 let le: i64 = _rb_line_end(robots, rlen, ls)
124 let uavs: i64 = _rb_directive(robots, ls, le, d_ua)
125 if uavs >= 0 {
126 let ve: i64 = _rb_val_end(robots, uavs, le)
127 var match_grp: i64 = 0
128 if star_only == 1 { if _rb_is_star(robots, uavs, ve) == 1 { match_grp = 1 } }
129 else {
130 if _rb_is_star(robots, uavs, ve) == 0 { if _rb_ua_token_matches(robots, uavs, ve, ua, ualen) == 1 { match_grp = 1 } }
131 }
132 active = match_grp
133 } else {
134 if active == 1 {
135 let dvs: i64 = _rb_directive(robots, ls, le, d_dis)
136 if dvs >= 0 {
137 let dve: i64 = _rb_val_end(robots, dvs, le)
138 let ml: i64 = _rb_path_match(path, plen, robots, dvs, dve)
139 if ml > best_dis { best_dis = ml }
140 }
141 let avs: i64 = _rb_directive(robots, ls, le, d_allow)
142 if avs >= 0 {
143 let ave: i64 = _rb_val_end(robots, avs, le)
144 let ml2: i64 = _rb_path_match(path, plen, robots, avs, ave)
145 if ml2 > best_allow { best_allow = ml2 }
146 }
147 }
148 }
149 ls = le + 1
150 }
151 if best_dis < 0 { return 1 }
152 if best_allow >= best_dis { return 1 }
153 return 0
154}
155
156// Crawl-delay (seconds) for the applicable group, or 0 if unspecified.
157func nx_robots_crawl_delay(robots: *u8, rlen: i64, ua: *u8, ualen: i64) -> i64 {
158 var star_only: i64 = 1
159 if _rb_has_specific(robots, rlen, ua, ualen) == 1 { star_only = 0 }
160 let d_ua: *u8 = "user-agent:"
161 let d_cd: *u8 = "crawl-delay:"
162 var active: i64 = 0
163 var ls: i64 = 0
164 while ls < rlen {
165 let le: i64 = _rb_line_end(robots, rlen, ls)
166 let uavs: i64 = _rb_directive(robots, ls, le, d_ua)
167 if uavs >= 0 {
168 let ve: i64 = _rb_val_end(robots, uavs, le)
169 var match_grp: i64 = 0
170 if star_only == 1 { if _rb_is_star(robots, uavs, ve) == 1 { match_grp = 1 } }
171 else { if _rb_is_star(robots, uavs, ve) == 0 { if _rb_ua_token_matches(robots, uavs, ve, ua, ualen) == 1 { match_grp = 1 } } }
172 active = match_grp
173 } else {
174 if active == 1 {
175 let cvs: i64 = _rb_directive(robots, ls, le, d_cd)
176 if cvs >= 0 {
177 let cve: i64 = _rb_val_end(robots, cvs, le)
178 var v: i64 = 0
179 var p: i64 = cvs
180 while p < cve { let c: i64 = robots[p] as i64; if c >= 0x30 { if c <= 0x39 { v = v * 10 + (c - 0x30) } } p = p + 1 }
181 return v
182 }
183 }
184 }
185 ls = le + 1
186 }
187 return 0
188}