nx_antibot.nx source
↩ module page · 232 lines · 10595 B
1// nx_antibot.nx -- BOT-WALL CLASSIFICATION AS DATA: the /compare/webscraping R4 contract (symbol abt_classify).
2// Recognises a challenge / block / rate-limit response FROM CAPTURED SPECIMENS and reports a VERDICT. It never
3// solves, retries, rotates or escalates -- the plan row says so and so does the estate's ethics: a wall is a
4// host's answer, and the crawler's job is to record it honestly (outcome row) and move on, not to defeat it.
5// Concept source: crawl4ai antibot_detector.py (three tiers: structural vendor markers on any page size, generic
6// phrases only on short or error pages, structural integrity of the body). Sovereign form: the markers are ROWS in
7// knowledge/antibot_markers.conf (tier|vendor|reason|marker, literal bytes, no regex), the two scalars are rows in
8// knowledge/antibot.conf, and the classifier is a pure function over (status, body) plus those rows.
9// Codes:
10// ABT_CLEAN 0 nothing matched
11// ABT_RATE_LIMIT 1 HTTP 429 (always a rate limit; the pacer, not this organ, decides how long to wait)
12// ABT_STATUS_BLOCK 2 HTTP 403 / 503 carrying an HTML body (a refusal page, vendor unknown unless a marker names it)
13// ABT_VENDOR_WALL 3 a tier-1 marker matched (abt_last_vendor_g / abt_last_reason_g name it)
14// ABT_GENERIC_WALL 4 a tier-2 marker matched on a short or error body
15// ABT_EMPTY_SHELL 5 a 200 whose text is under shell_max_text: a JS-only page or a silent block -- its OWN class,
16// because an empty page is not evidence of a wall and must not be laundered into one
17// FALSE POSITIVES ARE NOT CHEAP HERE (the source says they are, because its fallback rescues them; ours has no
18// fallback by design): a wall verdict drops a page from the index and feeds the host-health streak, so the gate
19// carries a long-prose control that MENTIONS captcha, Cloudflare and Access Denied and must read CLEAN.
20// license_tier: ORIGINAL module: nishi-core.search.antibot No hw writes (Rule 26).
21import "nx_syscalls.nx"
22import "nx_lane_conf.nx"
23import "nx_html_to_text.nx"
24
25const ABT_MARKERS: *u8 = "knowledge/antibot_markers.conf"
26const ABT_CONF: *u8 = "knowledge/antibot.conf"
27const ABT_DEF_SMALL_PAGE_BYTES: i64 = 10240 // crawl4ai is_blocked: tier-2 terms only on pages < 10KB
28const ABT_DEF_SHELL_MAX_TEXT: i64 = 500 // == WC_MININDEX, the crawler's measured thin-page bar; one number for "empty"
29const ABT_CLEAN: i64 = 0
30const ABT_RATE_LIMIT: i64 = 1
31const ABT_STATUS_BLOCK: i64 = 2
32const ABT_VENDOR_WALL: i64 = 3
33const ABT_GENERIC_WALL: i64 = 4
34const ABT_EMPTY_SHELL: i64 = 5
35const ABT_HTTP_429: i64 = 429
36const ABT_HTTP_403: i64 = 403
37const ABT_HTTP_503: i64 = 503
38const ABT_HTTP_200: i64 = 200
39const ABT_HTTP_ERR_FLOOR: i64 = 400
40const ABT_PIPE: i64 = 124
41const ABT_NL: i64 = 10
42const ABT_CR: i64 = 13
43const ABT_HASH: i64 = 35
44// row table: per row 6 i64 slots: tier, vendor_off, vendor_len, reason_off, reason_len, marker_off ; marker_len in slot 6
45const ABT_ROW: i64 = 7
46const ABT_R_TIER: i64 = 0
47const ABT_R_VOFF: i64 = 1
48const ABT_R_VLEN: i64 = 2
49const ABT_R_ROFF: i64 = 3
50const ABT_R_RLEN: i64 = 4
51const ABT_R_MOFF: i64 = 5
52const ABT_R_MLEN: i64 = 6
53
54static abt_loaded_g: i64
55static abt_buf_g: *u8 // the conf bytes (rows point into it)
56static abt_buf_n_g: i64
57static abt_rows_g: *i64
58static abt_nrows_g: i64
59static abt_nrows_bad_g: i64 // rows that did not parse (announced, never silently skipped)
60static abt_small_bytes_g: i64
61static abt_shell_max_text_g: i64
62static abt_conf_src_g: i64
63// last verdict detail (pointers into the conf buffer; empty string when no row matched)
64static abt_last_vendor_g: *u8
65static abt_last_vendor_len_g: i64
66static abt_last_reason_g: *u8
67static abt_last_reason_len_g: i64
68static abt_last_row_g: i64
69static abt_text_scratch_g: *u8
70static abt_text_scratch_cap_g: i64
71
72func abt_count_nl(b: *u8, n: i64) -> i64 {
73 var i: i64 = 0
74 var c: i64 = 0
75 while i < n { if (b[i] as i64) == ABT_NL { c = c + 1 } i = i + 1 }
76 return c
77}
78// parse one row [s,e) into slot r; returns 1 ok, 0 malformed
79func abt_parse_row(b: *u8, s: i64, e: i64, rows: *i64, r: i64) -> i64 {
80 // fields: tier | vendor | reason | marker (the marker may itself contain '|', so it is field 4 TO END)
81 var p: i64 = s
82 var tier: i64 = 0
83 var got: i64 = 0
84 while p < e { let c: i64 = b[p] as i64; if c >= 48 { if c <= 57 { tier = tier * 10 + (c - 48); got = 1; p = p + 1 } else { p = e + 1 } } else { p = e + 1 } }
85 if got == 0 { return 0 }
86 // p ran to e+1 on the first non-digit; rescan for the pipes
87 var q: i64 = s
88 while q < e { if (b[q] as i64) == ABT_PIPE { q = e + q } else { q = q + 1 } }
89 if q < e { return 0 }
90 let p1: i64 = q - e // first pipe
91 if p1 <= s { return 0 }
92 var q2: i64 = p1 + 1
93 while q2 < e { if (b[q2] as i64) == ABT_PIPE { q2 = e + q2 } else { q2 = q2 + 1 } }
94 if q2 < e { return 0 }
95 let p2: i64 = q2 - e
96 var q3: i64 = p2 + 1
97 while q3 < e { if (b[q3] as i64) == ABT_PIPE { q3 = e + q3 } else { q3 = q3 + 1 } }
98 if q3 < e { return 0 }
99 let p3: i64 = q3 - e
100 if p3 + 1 >= e { return 0 } // empty marker
101 rows[r * ABT_ROW + ABT_R_TIER] = tier
102 rows[r * ABT_ROW + ABT_R_VOFF] = p1 + 1
103 rows[r * ABT_ROW + ABT_R_VLEN] = p2 - p1 - 1
104 rows[r * ABT_ROW + ABT_R_ROFF] = p2 + 1
105 rows[r * ABT_ROW + ABT_R_RLEN] = p3 - p2 - 1
106 rows[r * ABT_ROW + ABT_R_MOFF] = p3 + 1
107 rows[r * ABT_ROW + ABT_R_MLEN] = e - p3 - 1
108 return 1
109}
110// load the rows and the scalars once per process; returns the row count (0 when the file is absent)
111func abt_load() -> i64 {
112 if abt_loaded_g == 1 { return abt_nrows_g }
113 abt_loaded_g = 1
114 abt_conf_src_g = 0
115 let sb: i64 = lc_geti(ABT_CONF, "" as *u8, "small_page_bytes" as *u8, 0 - 1)
116 if sb < 0 { abt_small_bytes_g = ABT_DEF_SMALL_PAGE_BYTES } else { abt_small_bytes_g = sb; abt_conf_src_g = 1 }
117 let sm: i64 = lc_geti(ABT_CONF, "" as *u8, "shell_max_text" as *u8, 0 - 1)
118 if sm < 0 { abt_shell_max_text_g = ABT_DEF_SHELL_MAX_TEXT } else { abt_shell_max_text_g = sm; abt_conf_src_g = 1 }
119 let lb: *i64 = sys_mmap(16) as *i64
120 lb[0] = 0
121 let src: *u8 = sys_read_file(ABT_MARKERS, lb)
122 if (src as i64) == 0 { abt_nrows_g = 0; return 0 }
123 let n: i64 = lb[0]
124 // private copy: sys_read_file hands back a shared mapping; rows point into this copy for the process lifetime
125 let b: *u8 = sys_mmap(n + 1)
126 var i: i64 = 0
127 while i < n { b[i] = src[i]; i = i + 1 }
128 b[n] = 0 as u8
129 sys_free_file(src, n)
130 abt_buf_g = b
131 abt_buf_n_g = n
132 let cap: i64 = abt_count_nl(b, n) + 2
133 abt_rows_g = sys_mmap(cap * ABT_ROW * 8) as *i64
134 var nr: i64 = 0
135 var bad: i64 = 0
136 var s: i64 = 0
137 while s < n {
138 var e: i64 = s
139 while e < n { if (b[e] as i64) == ABT_NL { e = n + e } else { e = e + 1 } }
140 if e > n { e = e - n }
141 var e2: i64 = e
142 if e2 > s { if (b[e2 - 1] as i64) == ABT_CR { e2 = e2 - 1 } }
143 if e2 > s { if (b[s] as i64) != ABT_HASH {
144 if abt_parse_row(b, s, e2, abt_rows_g, nr) == 1 { nr = nr + 1 } else { bad = bad + 1 }
145 } }
146 s = e + 1
147 }
148 abt_nrows_g = nr
149 abt_nrows_bad_g = bad
150 return nr
151}
152func abt_reload() -> i64 { abt_loaded_g = 0; return abt_load() }
153
154// literal substring search
155func abt_find(hay: *u8, n: i64, needle: *u8, nl: i64) -> i64 {
156 if nl <= 0 { return 0 }
157 var i: i64 = 0
158 while i + nl <= n {
159 var k: i64 = 0
160 var m: i64 = 1
161 while k < nl { if hay[i + k] != needle[k] { m = 0; k = nl } else { k = k + 1 } }
162 if m == 1 { return 1 }
163 i = i + 1
164 }
165 return 0
166}
167func abt_set_last(r: i64) -> i64 {
168 abt_last_row_g = r
169 if r < 0 {
170 abt_last_vendor_g = "" as *u8; abt_last_vendor_len_g = 0
171 abt_last_reason_g = "" as *u8; abt_last_reason_len_g = 0
172 return 0
173 }
174 let rows: *i64 = abt_rows_g
175 abt_last_vendor_g = (abt_buf_g as i64 + rows[r * ABT_ROW + ABT_R_VOFF]) as *u8
176 abt_last_vendor_len_g = rows[r * ABT_ROW + ABT_R_VLEN]
177 abt_last_reason_g = (abt_buf_g as i64 + rows[r * ABT_ROW + ABT_R_ROFF]) as *u8
178 abt_last_reason_len_g = rows[r * ABT_ROW + ABT_R_RLEN]
179 return 0
180}
181// first matching row of the given tier, or -1
182func abt_scan_tier(html: *u8, n: i64, tier: i64) -> i64 {
183 let rows: *i64 = abt_rows_g
184 var r: i64 = 0
185 while r < abt_nrows_g {
186 if rows[r * ABT_ROW + ABT_R_TIER] == tier {
187 if abt_find(html, n, (abt_buf_g as i64 + rows[r * ABT_ROW + ABT_R_MOFF]) as *u8, rows[r * ABT_ROW + ABT_R_MLEN]) == 1 { return r }
188 }
189 r = r + 1
190 }
191 return 0 - 1
192}
193
194// THE CONTRACT. status = HTTP status (0 when unknown), html = body bytes.
195func abt_classify(status: i64, html: *u8, n: i64) -> i64 {
196 abt_load()
197 abt_set_last(0 - 1)
198 if status == ABT_HTTP_429 { return ABT_RATE_LIMIT }
199 // tier 1: structural vendor markers, any page size
200 let r1: i64 = abt_scan_tier(html, n, 1)
201 if r1 >= 0 { abt_set_last(r1); return ABT_VENDOR_WALL }
202 // tier 2: generic phrases, only on a short body or an error status
203 var short_or_err: i64 = 0
204 if n < abt_small_bytes_g { short_or_err = 1 }
205 if status >= ABT_HTTP_ERR_FLOOR { short_or_err = 1 }
206 if short_or_err == 1 {
207 let r2: i64 = abt_scan_tier(html, n, 2)
208 if r2 >= 0 { abt_set_last(r2); return ABT_GENERIC_WALL }
209 }
210 // error status with an HTML body and no marker: a refusal page from an unnamed vendor
211 if n > 0 { if status == ABT_HTTP_403 { return ABT_STATUS_BLOCK } if status == ABT_HTTP_503 { return ABT_STATUS_BLOCK } }
212 // tier 3: an empty shell on a 200 (its own class)
213 if status == ABT_HTTP_200 { if n > 0 {
214 if abt_text_scratch_cap_g < n + 16 {
215 if (abt_text_scratch_g as i64) != 0 { sys_munmap(abt_text_scratch_g, abt_text_scratch_cap_g) }
216 abt_text_scratch_cap_g = n + 16
217 abt_text_scratch_g = sys_mmap(abt_text_scratch_cap_g)
218 }
219 let tl: i64 = nx_html_to_text(html, n, abt_text_scratch_g, abt_text_scratch_cap_g)
220 if tl < abt_shell_max_text_g { return ABT_EMPTY_SHELL }
221 } }
222 return ABT_CLEAN
223}
224// a short name for an outcome row
225func abt_code_name(code: i64) -> *u8 {
226 if code == ABT_RATE_LIMIT { return "rate-limit" as *u8 }
227 if code == ABT_STATUS_BLOCK { return "status-block" as *u8 }
228 if code == ABT_VENDOR_WALL { return "vendor-wall" as *u8 }
229 if code == ABT_GENERIC_WALL { return "generic-wall" as *u8 }
230 if code == ABT_EMPTY_SHELL { return "empty-shell" as *u8 }
231 return "clean" as *u8
232}