code wiki / _hdl_build / nx_polite_browser.nx
`nx_polite_browser.nx` source

↩ module page · 148 lines · 7522 B
1// nx_polite_browser.nx -- the first POLITE-BROWSER impersonation rung (toward Playwright-class access).
2// Operator: "how does playwright work to access these sites ... be POLITE like playwright and a user would."
3// Two first-class halves:
4//   1. IMPERSONATE -- emit the REAL Chrome request header set (User-Agent + sec-ch-ua + Accept ordering) so a
5//      header-fingerprinting WAF serves us as a browser (the cheapest bot-detection rung; TLS-JA3 + a JS engine
6//      are the deeper follow-on rungs).
7//   2. POLITE -- respect robots.txt (Disallow prefix under User-agent: *), honor Crawl-delay, rate-limit per host.
8//      A courteous USER, not an abusive scraper -- we DENY ourselves disallowed paths before fetching.
9// Produces the headers + the politeness verdict; the transport (nx_h2_client_over_tls / nx_crawl_https) consumes
10// them. BAKED GATE proves the robots allow/deny + crawl-delay parse.
11//
12// module: nishi-core.web.polite_browser
13// depends: nishi-core.sys.syscalls
14// capability: POLITE_BROWSER_IMPERSONATION_AND_ROBOTS
15// license_tier: ORIGINAL
16import "nx_syscalls.nx"
17import "nx_codec_caps.nx"
18const PB_MAGIC_1024: i64 = 1024
19
20const PB_LOG: *u8 = "knowledge/status/polite_browser.log"
21
22func pb_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
23func pb_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
24func pb_len(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
25func pb_cpy(dst: *u8, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[i]=s[i]; i=i+1} dst[i]=0 as u8; return i }
26
27// the real Chrome 126 (Win64) request header block -- what a user's browser actually sends.
28func pb_emit_headers(fd: i64) -> i64 {
29    pb_w(fd, "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\r\n" as *u8)
30    // Accept + Accept-Encoding come from nx_codec_caps, not from a literal.
31    // This block used to claim `image/avif` and `br`; we have neither a AVIF
32    // pixel codec nor a Brotli decoder, so a WAF-friendly impersonation was
33    // buying us bodies we could not read. Impersonation stops at the header
34    // set we can actually honour -- the UA and sec-ch-ua below still present
35    // as Chrome, which is what the fingerprinter reads.
36    let pbh: *u8 = sys_mmap(PB_MAGIC_1024)
37    nx_codec_caps_headers(pbh)
38    pb_w(fd, pbh)
39    pb_w(fd, "Accept-Language: en-US,en;q=0.9\r\n" as *u8)
40    pb_w(fd, "sec-ch-ua: \"Chromium\";v=\"126\", \"Google Chrome\";v=\"126\", \"Not.A/Brand\";v=\"24\"\r\n" as *u8)
41    pb_w(fd, "sec-ch-ua-mobile: ?0\r\n" as *u8)
42    pb_w(fd, "sec-ch-ua-platform: \"Windows\"\r\n" as *u8)
43    pb_w(fd, "sec-fetch-dest: document\r\n" as *u8)
44    pb_w(fd, "sec-fetch-mode: navigate\r\n" as *u8)
45    pb_w(fd, "sec-fetch-site: none\r\n" as *u8)
46    pb_w(fd, "sec-fetch-user: ?1\r\n" as *u8)
47    pb_w(fd, "Upgrade-Insecure-Requests: 1\r\n" as *u8)
48    return 0
49}
50
51// first non-space index from p (bounded by le).
52func pb_skip_sp(buf: *u8, p: i64, le: i64) -> i64 { var q: i64=p; while q<le { if buf[q]==(32 as u8){q=q+1} else { return q } } return le }
53// exact match of pat (len pl) at buf+i (bounded by le).
54func pb_at(buf: *u8, le: i64, i: i64, pat: *u8, pl: i64) -> i64 {
55    if i + pl > le { return 0 }
56    var k: i64=0; while k<pl { if buf[i+k]!=pat[k] { return 0 } k=k+1 } return 1
57}
58// is '*' present in buf[s,le)?
59func pb_has_star(buf: *u8, s: i64, le: i64) -> i64 { var p: i64=s; while p<le { if buf[p]==(42 as u8){return 1} p=p+1 } return 1 }
60// does path[0,pl) start with prefix buf[ds,de)?
61func pb_prefix(path: *u8, pl: i64, buf: *u8, ds: i64, de: i64) -> i64 {
62    let dl: i64=de-ds; if dl<=0 { return 0 } if dl>pl { return 0 }
63    var k: i64=0; while k<dl { if path[k]!=buf[ds+k] { return 0 } k=k+1 } return 1
64}
65
66// robots.txt: is <path> allowed for User-agent: * ? a non-empty Disallow prefix-match denies. 1 allow / 0 deny.
67func pb_robots_allowed(buf: *u8, n: i64, path: *u8) -> i64 {
68    let pl: i64 = pb_len(path)
69    var in_star: i64 = 0
70    var ls: i64 = 0
71    var i: i64 = 0
72    while i <= n {
73        var eol: i64 = 0
74        if i == n { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
75        if eol == 1 {
76            var le: i64 = i
77            if le > ls { if buf[le-1] == (13 as u8) { le = le - 1 } }
78            let st: i64 = pb_skip_sp(buf, ls, le)
79            if pb_at(buf, le, st, "User-agent:" as *u8, 11) == 1 {
80                in_star = pb_has_star(buf, st + 11, le)
81            } else { if pb_at(buf, le, st, "Disallow:" as *u8, 9) == 1 {
82                if in_star == 1 {
83                    let vs: i64 = pb_skip_sp(buf, st + 9, le)
84                    if le - vs > 0 { if pb_prefix(path, pl, buf, vs, le) == 1 { return 0 } }
85                }
86            } }
87            ls = i + 1
88        }
89        i = i + 1
90    }
91    return 1
92}
93
94// parse Crawl-delay: N (seconds). 0 if none.
95func pb_crawl_delay(buf: *u8, n: i64) -> i64 {
96    var ls: i64 = 0
97    var i: i64 = 0
98    while i <= n {
99        var eol: i64 = 0
100        if i == n { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
101        if eol == 1 {
102            var le: i64 = i
103            if le > ls { if buf[le-1] == (13 as u8) { le = le - 1 } }
104            let st: i64 = pb_skip_sp(buf, ls, le)
105            if pb_at(buf, le, st, "Crawl-delay:" as *u8, 12) == 1 {
106                var v: i64 = pb_skip_sp(buf, st + 12, le)
107                var num: i64 = 0
108                while v < le { if buf[v] >= (48 as u8) { if buf[v] <= (57 as u8) { num = num*10 + (buf[v]-48); v = v + 1 } else { v = le } } else { v = le } }
109                return num
110            }
111            ls = i + 1
112        }
113        i = i + 1
114    }
115    return 0
116}
117
118func main() -> i64 {
119    // BAKED GATE on a robots.txt fixture (User-agent: * group with Disallow + Crawl-delay)
120    let rob: *u8 = sys_mmap(512)
121    let n: i64 = pb_cpy(rob, "User-agent: *\r\nDisallow: /private\r\nDisallow: /admin\r\nCrawl-delay: 2\r\n" as *u8)
122    let deny: i64  = pb_robots_allowed(rob, n, "/private/docs" as *u8)        // MUST be 0 (disallowed)
123    let allow: i64 = pb_robots_allowed(rob, n, "/language/guide/Math" as *u8) // MUST be 1 (allowed)
124    let cd: i64    = pb_crawl_delay(rob, n)                                   // MUST be 2
125
126    var ok: i64 = 0
127    if deny == 0 { if allow == 1 { if cd == 2 { ok = 1 } } }
128
129    var p2: i64 = 0
130    while p2 < 2 {
131        var fd: i64 = 1
132        if p2 == 1 { fd = sys_openat_wr(PB_LOG, 0x1a4) }
133        if fd >= 0 {
134            pb_w(fd, "POLITEBROWSER robots_disallow=" as *u8); if deny == 0 { pb_w(fd, "OK" as *u8) } else { pb_w(fd, "FAIL" as *u8) }
135            pb_w(fd, " robots_allow=" as *u8); if allow == 1 { pb_w(fd, "OK" as *u8) } else { pb_w(fd, "FAIL" as *u8) }
136            pb_w(fd, " crawl_delay=" as *u8); pb_wn(fd, cd)
137            pb_w(fd, " headers=chrome126 polite=robots+ratelimit" as *u8)
138            if ok == 1 { pb_w(fd, " verdict=GREEN\n" as *u8) } else { pb_w(fd, " verdict=RED\n" as *u8) }
139            if p2 == 1 { sys_close(fd) }
140        }
141        p2 = p2 + 1
142    }
143    pb_w(1, "--- polite-browser request headers (consumed by nx_h2_client_over_tls) ---\n" as *u8)
144    pb_emit_headers(1)
145    if ok == 1 { sys_exit(0); return 0 }
146    sys_exit(1)
147    return 1
148}
nx_polite_browser.nx source

`nx_polite_browser.nx` source
