nx_browser_fetch.nx source
↩ module page · 122 lines · 7096 B
1// nx_browser_fetch.nx -- the practical render-front fetch: browser headers (nx_https_get_spoof: real Firefox UA
2// + Accept) AND redirect-following, which neither existing fetch did together. This is what actually retrieves
3// media from real hosts -- they UA-gate non-browser clients AND 301 to a CDN. bf_fetch_follow loops the spoof
4// GET across 3xx Location redirects (bounded), returning the final raw response. For HARD hosts (Cloudflare-grade
5// TLS-fingerprint + JS challenges) the full Nishi-browser render is still needed -- but this covers the common
6// case (abandonware / niche sites, which are rarely behind advanced bot-walls). license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_https_get_spoof.nx"
10const K_MAGIC_8192: i64 = 8192
11
12func bf_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
13func bf_find(hay: *u8, hl: i64, needle: *u8, nl: i64, from: i64) -> i64 {
14 if nl == 0 { return from }
15 var i: i64 = from; let last: i64 = hl - nl
16 while i <= last { var j: i64=0; var m: i64=1; while j<nl { if (hay[i+j]&0xff)!=(needle[j]&0xff) { m=0; j=nl } else { j=j+1 } } if m==1 { return i } i=i+1 }
17 return 0 - 1
18}
19// parse the 3-digit status from "HTTP/1.1 XXX"
20func bf_status(resp: *u8, n: i64) -> i64 { if n < 12 { return 0 } return (((resp[9]&0xff)-48)*100) + (((resp[10]&0xff)-48)*10) + ((resp[11]&0xff)-48) }
21// extract the Location header value (case-tolerant) into out; returns len (0 if none). Bounded to the header block.
22func bf_location(resp: *u8, n: i64, out: *u8, cap: i64) -> i64 {
23 var hdr_end: i64 = bf_find(resp, n, "\r\n\r\n" as *u8, 4, 0); if hdr_end < 0 { hdr_end = n }
24 var p: i64 = bf_find(resp, hdr_end, "\r\nLocation: " as *u8, 12, 0)
25 if p < 0 { p = bf_find(resp, hdr_end, "\r\nlocation: " as *u8, 12, 0) }
26 if p < 0 { out[0] = 0 as u8; return 0 }
27 var i: i64 = p + 12; var o: i64 = 0
28 var go: i64 = 1
29 while go == 1 { if i >= hdr_end { go = 0 } else { let c: i64 = resp[i]&0xff; if c == 13 { go = 0 } else { if o < (cap-1) { out[o] = c as u8; o = o + 1 } i = i + 1 } } }
30 out[o] = 0 as u8
31 return o
32}
33// FETCH with browser headers + follow up to maxredir 3xx Location redirects. Returns the final raw response bytes.
34func bf_fetch_follow(url: *u8, store: *TrustStore, cr: *u8, pk: *u8, now: i64, out: *u8, cap: i64, maxredir: i64) -> i64 {
35 let cur: *u8 = sys_mmap(K_MAGIC_8192)
36 var cl: i64 = 0; while (url[cl]&0xff) != 0 { cur[cl] = url[cl]; cl = cl + 1 } cur[cl] = 0 as u8
37 var tries: i64 = 0
38 var n: i64 = 0
39 var go: i64 = 1
40 while go == 1 {
41 n = nx_https_get_spoof(cur, cr, pk, store, now, "" as *u8, 0, out, cap)
42 if n <= 0 { go = 0 }
43 else {
44 let st: i64 = bf_status(out, n)
45 var redir: i64 = 0
46 if st >= 300 { if st < 400 { redir = 1 } }
47 if redir == 0 { go = 0 }
48 else {
49 if tries >= maxredir { go = 0 }
50 else {
51 let loc: *u8 = sys_mmap(K_MAGIC_8192)
52 let ll: i64 = bf_location(out, n, loc, K_MAGIC_8192)
53 if ll == 0 { go = 0 }
54 else { var k: i64=0; while k < ll { cur[k]=loc[k]; k=k+1 } cur[ll]=0 as u8; tries = tries + 1 }
55 }
56 }
57 }
58 }
59 return n
60}
61// return the offset where the HTTP body begins (after the header block), or 0 if no header terminator
62func bf_body_off(resp: *u8, n: i64) -> i64 { let p: i64 = bf_find(resp, n, "\r\n\r\n" as *u8, 4, 0); if p < 0 { return 0 } return p + 4 }
63
64func bf_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
65// case-insensitive substring find
66func bf_ci_find(hay: *u8, hl: i64, needle: *u8, nl: i64, from: i64) -> i64 {
67 if nl == 0 { return from }
68 var i: i64 = from; let last: i64 = hl - nl
69 while i <= last { var j: i64=0; var m: i64=1; while j<nl { if bf_lc(hay[i+j]&0xff) != bf_lc(needle[j]&0xff) { m=0; j=nl } else { j=j+1 } } if m==1 { return i } i=i+1 }
70 return 0 - 1
71}
72// META-REFRESH: extract the redirect target from `<meta http-equiv="refresh" content="N;url=X">` in the HTML,
73// OR from a `Refresh: N; url=X` response header. Returns the (absolute-only) URL into out, or 0. Case-insensitive,
74// tolerant of quotes/spacing/attribute order. Only http(s) targets are followed (scam redirects are absolute).
75func bf_meta_refresh(html: *u8, hlen: i64, out: *u8, cap: i64) -> i64 {
76 // find a "refresh" occurrence, then a nearby "url=" within the same tag/header (bounded window)
77 var from: i64 = 0
78 var go: i64 = 1
79 while go == 1 {
80 let rp: i64 = bf_ci_find(html, hlen, "refresh" as *u8, 7, from)
81 if rp < 0 { go = 0 }
82 else {
83 var win: i64 = rp + 300; if win > hlen { win = hlen }
84 let up: i64 = bf_ci_find(html, win, "url=" as *u8, 4, rp)
85 if up >= 0 { if up < win {
86 var i: i64 = up + 4
87 if i < hlen { let q: i64 = html[i]&0xff; if q == 34 { i = i + 1 } else { if q == 39 { i = i + 1 } } } // skip opening quote
88 var o: i64 = 0; var scan: i64 = 1
89 while scan == 1 { if i >= hlen { scan = 0 } else { let c: i64 = html[i]&0xff; if c==34 { scan=0 } else { if c==39 { scan=0 } else { if c==62 { scan=0 } else { if c==32 { scan=0 } else { if c==13 { scan=0 } else { if c==10 { scan=0 } else { if o<(cap-1) { out[o]=c as u8; o=o+1 } i=i+1 } } } } } } } }
90 out[o] = 0 as u8
91 if o > 7 { if bf_find(out, o, "http" as *u8, 4, 0) == 0 { return o } } // absolute http(s) only
92 } }
93 from = rp + 7
94 }
95 }
96 out[0] = 0 as u8
97 return 0
98}
99// FETCH + follow 3xx (bf_fetch_follow) AND meta-refresh / Refresh-header redirects (bounded maxhops). Each hop's
100// target is only followed if it's an absolute http(s) URL; returns the final response. This reaches content that
101// hides behind interstitial / "you are leaving" / affiliate-wrap redirect chains that JS-only defenses miss.
102func bf_fetch_meta(url: *u8, store: *TrustStore, cr: *u8, pk: *u8, now: i64, out: *u8, cap: i64, maxhops: i64) -> i64 {
103 let cur: *u8 = sys_mmap(K_MAGIC_8192)
104 var cl: i64 = 0; while (url[cl]&0xff) != 0 { cur[cl] = url[cl]; cl = cl + 1 } cur[cl] = 0 as u8
105 var n: i64 = 0
106 var hops: i64 = 0
107 var go: i64 = 1
108 while go == 1 {
109 n = bf_fetch_follow(cur, store, cr, pk, now, out, cap, 5) // handles 3xx Location
110 if n <= 0 { go = 0 }
111 else {
112 let boff: i64 = bf_body_off(out, n)
113 let mr: *u8 = sys_mmap(K_MAGIC_8192)
114 // check the Refresh: header (in the header block) then a <meta refresh> in the body
115 var ml: i64 = bf_meta_refresh(out, boff, mr, K_MAGIC_8192)
116 if ml == 0 { ml = bf_meta_refresh(((out as i64)+boff) as *u8, n - boff, mr, K_MAGIC_8192) }
117 if ml == 0 { go = 0 }
118 else { if hops >= maxhops { go = 0 } else { var k: i64=0; while k < ml { cur[k]=mr[k]; k=k+1 } cur[ml]=0 as u8; hops = hops + 1 } }
119 }
120 }
121 return n
122}