code wiki / _hdl_build / nx_video_sniff_fetch.nx
nx_video_sniff_fetch.nx source
↩ module page · 320 lines · 18434 B
1// nx_video_sniff_fetch.nx -- the TLS-aware "sniff a URL" core, SHARED by the CLI and the API daemon (DRY;
2// consolidates the fetch+extract+guarded-sniff flow). Given a page URL + a loaded trust store: fetch the
3// page over sovereign TLS-1.3, fetch its external <script src> bundles (bounded), and run the whole thing
4// through the GUARDED sniff -> a "ROUTE=<r> URL=<u>" plan (VSK_* kind). No network happens in the guarded
5// child (page + bundles are fetched HERE; the child only runs the JS via pending-capture). license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_x509_trust_store.nx"
8import "nx_trust_store_load_from_certdata.nx"
9import "nx_https_fetch_follow.nx"
10import "nx_hls_resolve.nx"
11import "nx_video_sniff.nx"
12const VSF_MAGIC_65536: i64 = 65536
13const VSF_MAGIC_8192: i64 = 8192
14const VSF_MAGIC_8191: i64 = 8191
15const VSF_MAGIC_4096: i64 = 4096
16const VSF_MAGIC_4095: i64 = 4095
17const VSF_MAGIC_1024: i64 = 1024
18const VSF_MAGIC_2097152: i64 = 2097152
19
20const VSF_HTMLCAP: i64 = 4194304
21const VSF_EXTCAP: i64 = 4194304
22const VSF_BUNCAP: i64 = 2097152
23const VSF_MAXSRC: i64 = 8
24
25func vsf_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
26
27// Sniff `url` -> plan (into planbuf) + return the VSK_* kind. `store` = a loaded *TrustStore. cpu_secs bounds
28// the guarded JS run. Distinct route "fetch-failed" if the page can't be fetched.
29func vsf_sniff_url(url: *u8, ulen: i64, store: *TrustStore, planbuf: *u8, plancap: i64, cpu_secs: i64) -> i64 {
30 let status: *i64 = sys_mmap(8) as *i64
31 let html: *u8 = sys_mmap(VSF_HTMLCAP)
32 let hn: i64 = nx_https_fetch_follow(url, store, html, VSF_HTMLCAP, 6, status)
33 if hn <= 0 { var e: i64 = vs_puts(planbuf, 0, "ROUTE=fetch-failed URL=-" as *u8); planbuf[e] = 0 as u8; return VSK_NONE }
34
35 let srcs: *u8 = sys_mmap(VSF_MAGIC_65536)
36 let nsrc: i64 = vs_extract_script_srcs(html, hn, srcs, VSF_MAGIC_65536)
37 let extjs: *u8 = sys_mmap(VSF_EXTCAP)
38 let bundle: *u8 = sys_mmap(VSF_BUNCAP)
39 let absurl: *u8 = sys_mmap(VSF_MAGIC_8192)
40 let line: *u8 = sys_mmap(VSF_MAGIC_8192)
41 var extlen: i64 = 0
42 var fetched: i64 = 0
43 var li: i64 = 0
44 var walking: i64 = 1
45 while walking == 1 {
46 if fetched >= VSF_MAXSRC { walking = 0 }
47 else {
48 if (srcs[li] & 0xff) == 0 { walking = 0 }
49 else {
50 var q: i64 = 0
51 var inl: i64 = 1
52 while inl == 1 {
53 let c: i64 = srcs[li] & 0xff
54 if c == 0 { inl = 0 }
55 else { if c == 10 { inl = 0; li = li + 1 } else { if q < VSF_MAGIC_8191 { line[q] = c as u8; q = q + 1 } li = li + 1 } }
56 }
57 line[q] = 0 as u8
58 if q > 0 {
59 let rn: i64 = nx_hls_resolve_uri(url, ulen, line, q, absurl, VSF_MAGIC_8192)
60 if rn > 0 {
61 absurl[rn] = 0 as u8
62 let bn: i64 = nx_https_fetch_follow(absurl, store, bundle, VSF_BUNCAP, 6, status)
63 if bn > 0 {
64 var bp: i64 = 0
65 while bp < bn { if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = bundle[bp]; extlen = extlen + 1 } bp = bp + 1 }
66 if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = 59 as u8; extlen = extlen + 1; extjs[extlen] = 10 as u8; extlen = extlen + 1 }
67 fetched = fetched + 1
68 }
69 }
70 }
71 }
72 }
73 }
74 extjs[extlen] = 0 as u8
75 return vs_sniff_page_bundled(html, hn, extjs, extlen, planbuf, plancap, cpu_secs)
76}
77
78// PREAMBLE variant: identical fetch (page + external bundles over TLS) but a surrogate scriptlet PREAMBLE runs
79// first in the guarded sandbox, so anti-adblock/tracker-GATED media fetches fire and get captured. Used by the
80// clean-view /visit path to make hostile-gated sites yield their stream URL. license_tier: ORIGINAL
81func vsf_sniff_url_pre(url: *u8, ulen: i64, store: *TrustStore, preamble: *u8, plen: i64, planbuf: *u8, plancap: i64, cpu_secs: i64) -> i64 {
82 let status: *i64 = sys_mmap(8) as *i64
83 let html: *u8 = sys_mmap(VSF_HTMLCAP)
84 let hn: i64 = nx_https_fetch_follow(url, store, html, VSF_HTMLCAP, 6, status)
85 if hn <= 0 { var e: i64 = vs_puts(planbuf, 0, "ROUTE=fetch-failed URL=-" as *u8); planbuf[e] = 0 as u8; return VSK_NONE }
86 let srcs: *u8 = sys_mmap(VSF_MAGIC_65536)
87 let nsrc: i64 = vs_extract_script_srcs(html, hn, srcs, VSF_MAGIC_65536)
88 let extjs: *u8 = sys_mmap(VSF_EXTCAP)
89 let bundle: *u8 = sys_mmap(VSF_BUNCAP)
90 let absurl: *u8 = sys_mmap(VSF_MAGIC_8192)
91 let line: *u8 = sys_mmap(VSF_MAGIC_8192)
92 var extlen: i64 = 0
93 var fetched: i64 = 0
94 var li: i64 = 0
95 var walking: i64 = 1
96 while walking == 1 {
97 if fetched >= VSF_MAXSRC { walking = 0 }
98 else {
99 if (srcs[li] & 0xff) == 0 { walking = 0 }
100 else {
101 var q: i64 = 0
102 var inl: i64 = 1
103 while inl == 1 {
104 let c: i64 = srcs[li] & 0xff
105 if c == 0 { inl = 0 }
106 else { if c == 10 { inl = 0; li = li + 1 } else { if q < VSF_MAGIC_8191 { line[q] = c as u8; q = q + 1 } li = li + 1 } }
107 }
108 line[q] = 0 as u8
109 if q > 0 {
110 let rn: i64 = nx_hls_resolve_uri(url, ulen, line, q, absurl, VSF_MAGIC_8192)
111 if rn > 0 {
112 absurl[rn] = 0 as u8
113 let bn: i64 = nx_https_fetch_follow(absurl, store, bundle, VSF_BUNCAP, 6, status)
114 if bn > 0 {
115 var bp: i64 = 0
116 while bp < bn { if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = bundle[bp]; extlen = extlen + 1 } bp = bp + 1 }
117 if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = 59 as u8; extlen = extlen + 1; extjs[extlen] = 10 as u8; extlen = extlen + 1 }
118 fetched = fetched + 1
119 }
120 }
121 }
122 }
123 }
124 }
125 extjs[extlen] = 0 as u8
126 return vs_sniff_page_bundled_pre(preamble, plen, html, hn, extjs, extlen, planbuf, plancap, cpu_secs)
127}
128
129// ======== NETWORK-IN-THE-LOOP MULTI-ROUND SNIFF (WB-JS rung: token->URL round-trips) ====================
130// Real players resolve the stream via their OWN API calls (vk video API; javxxx wps ajax). Single-pass can't:
131// the guarded child has no network. Here we RUN the (DOM-seeded) page JS, then for each fetch/XHR it makes we
132// perform the REAL request (sovereign TLS + cookie jar), feed the response back via js_pending_service, and
133// el_drain the microtasks so the page's .then continuation runs (extract stream URL, issue next fetch, set src).
134// Proven primitives: js_pending_* + el_drain (crux test) + js_run_source_keep_doc (DOM seed, proven). SSRF-guarded.
135
136// --- SSRF guard (a network-driving sniff MUST refuse private/loopback targets; mirrors clean-serve csd_url_safe) ---
137func vn_lc(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c }
138func vn_pfx_ci(s: *u8, sl: i64, p: *u8) -> i64 { var i: i64=0; while p[i]!=(0 as u8) { if i>=sl { return 0 } if vn_lc(s[i]&0xff)!=vn_lc(p[i]&0xff) { return 0 } i=i+1 } return 1 }
139func vn_url_safe(url: *u8, ulen: i64) -> i64 {
140 var sl: i64 = 0
141 if vn_pfx_ci(url, ulen, "https://" as *u8)==1 { sl=8 } else { if vn_pfx_ci(url, ulen, "http://" as *u8)==1 { sl=7 } else { return 0 } }
142 let host: *u8 = sys_mmap(512); var h: i64=0; var i: i64=sl
143 while i<ulen { let c: i64=url[i]&0xff; if c==47 { i=ulen } else { if c==58 { i=ulen } else { if c==63 { i=ulen } else { if c==64 { h=0; i=i+1 } else { if h<511 { host[h]=c as u8; h=h+1 } i=i+1 } } } } }
144 host[h]=0 as u8
145 if h==0 { return 0 }
146 if vn_pfx_ci(host, h, "localhost" as *u8)==1 { return 0 }
147 if vn_pfx_ci(host, h, "127." as *u8)==1 { return 0 }
148 if vn_pfx_ci(host, h, "10." as *u8)==1 { return 0 }
149 if vn_pfx_ci(host, h, "192.168." as *u8)==1 { return 0 }
150 if vn_pfx_ci(host, h, "169.254." as *u8)==1 { return 0 }
151 if vn_pfx_ci(host, h, "172." as *u8)==1 { return 0 }
152 if vn_pfx_ci(host, h, "0." as *u8)==1 { return 0 }
153 if vn_pfx_ci(host, h, "[" as *u8)==1 { return 0 }
154 return 1
155}
156// URL terminator bytes (quote/space/angle/CR/LF/NUL) when scanning a URL out of a response body.
157func vn_term(c: i64) -> i64 { if c==34 { return 1 } if c==39 { return 1 } if c==32 { return 1 } if c==60 { return 1 } if c==62 { return 1 } if c==10 { return 1 } if c==13 { return 1 } if c==0 { return 1 } if c==92 { return 1 } return 0 }
158func vn_at(buf: *u8, pos: i64, n: i64, lit: *u8, litlen: i64) -> i64 {
159 if (pos + litlen) > n { return 0 }
160 var j: i64 = 0
161 while j < litlen { if (buf[pos+j] & 0xff) != (lit[j] & 0xff) { return 0 } j = j + 1 }
162 return 1
163}
164// Scan a fetched response BODY for http(s) URLs that are media (many APIs return the stream URL in their JSON,
165// and the page may set player.src WITHOUT re-fetching -> never captured as pending). Unescapes JSON \/ -> /.
166// Updates best/brank/bkind by rank if a higher-ranked media URL is found.
167func vn_scan_media(body: *u8, bn: i64, best: *u8, brankp: *i64, bkindp: *i64) -> i64 {
168 let u: *u8 = sys_mmap(VSF_MAGIC_4096)
169 var i: i64 = 0
170 while i < bn {
171 if vn_at(body, i, bn, "http" as *u8, 4) == 1 {
172 var q: i64 = 0
173 var j: i64 = i
174 var go: i64 = 1
175 while go == 1 {
176 if j >= bn { go = 0 }
177 else {
178 let c: i64 = body[j] & 0xff
179 if c == 92 { j = j + 1 } // JSON backslash-escape: drop the '\', keep next
180 else { if vn_term(c) == 1 { go = 0 } else { if q < VSF_MAGIC_4095 { u[q] = body[j] as u8; q = q + 1 } j = j + 1 } }
181 }
182 }
183 u[q] = 0 as u8
184 if q > 8 {
185 let mk: i64 = vs_url_kind(u, q)
186 if mk != VSK_NONE { let rr: i64 = vs_kind_rank(mk); if rr > brankp[0] { brankp[0] = rr; bkindp[0] = mk; var z: i64 = 0; while z < q { best[z] = u[z]; z = z + 1 } best[q] = 0 as u8 } }
187 }
188 if j > i { i = j } else { i = i + 1 }
189 } else { i = i + 1 }
190 }
191 return 0
192}
193// The multi-round engine: assemble (preamble + inline + bundles), run DOM-seeded, then service every fetch with a
194// REAL response + drain until a stream surfaces / no open pending / max_rounds. Writes "ROUTE=<r> URL=<u>".
195// ⚠runs untrusted JS WITH network -> the CALLER must isolate (fork+rlimit) for production; the CLI test runs it direct.
196func vs_sniff_net_bundled(preamble: *u8, plen: i64, html: *u8, hlen: i64, extjs: *u8, extlen: i64, store: *TrustStore, planbuf: *u8, plancap: i64, max_rounds: i64) -> i64 {
197 // DOM_TREE mode (real parsed nodes -> element.click() FIRES addEventListener handlers + getElementById
198 // resolves tree nodes) via js_render_page_pending_begin, which also KEEPS ctx/genv for the multi-round loop.
199 // Preamble runs FIRST (prepended as an inline <script> so it executes before the page's own scripts); the
200 // external bundles + the click-postamble are appended (run last, after handlers are registered).
201 let mcap: i64 = hlen + plen + 128
202 let modhtml: *u8 = sys_mmap(mcap)
203 var m: i64 = 0
204 let sopen: *u8 = "<script>" as *u8
205 var so: i64 = 0; while sopen[so] != (0 as u8) { modhtml[m] = sopen[so]; m = m + 1; so = so + 1 }
206 var a: i64 = 0; while a < plen { modhtml[m] = preamble[a]; m = m + 1; a = a + 1 }
207 let sclose: *u8 = "</script>\n" as *u8
208 var sc: i64 = 0; while sclose[sc] != (0 as u8) { modhtml[m] = sclose[sc]; m = m + 1; sc = sc + 1 }
209 var hi2: i64 = 0; while hi2 < hlen { modhtml[m] = html[hi2]; m = m + 1; hi2 = hi2 + 1 }
210 modhtml[m] = 0 as u8
211 let ecap: i64 = extlen + VSF_MAGIC_1024
212 let extras: *u8 = sys_mmap(ecap)
213 var e: i64 = 0
214 var p: i64 = 0; while p < extlen { extras[e] = extjs[p]; e = e + 1; p = p + 1 }
215 extras[e] = 59 as u8; e = e + 1; extras[e] = 10 as u8; e = e + 1
216 // POSTAMBLE: auto-trigger click-to-load players (javxxx #player-wrapper, jwplayer/videojs posters) so their
217 // decrypt/ajax handlers fire -- the multi-round loop then services the resulting fetches. Selectors id/class/tag
218 // only (querySelector supports #id/.class/tag). In DOM_TREE mode .click() now dispatches to the listeners.
219 let post: *u8 = "try{var __s=['#player-wrapper','#player','#video','.play-button','.play','.vjs-big-play-button','.jw-icon-display','.player','.video-player','video'];for(var __i=0;__i<__s.length;__i++){var __e=document.querySelector(__s[__i]);if(__e){if(__e.click){__e.click();}}}}catch(__z){};" as *u8
220 var pk: i64 = 0; while post[pk] != (0 as u8) { extras[e] = post[pk]; e = e + 1; pk = pk + 1 }
221 extras[e] = 0 as u8
222 let obx: *i64 = sys_mmap(32) as *i64
223 js_render_page_pending_begin(modhtml, m, extras, e, obx) // DOM_TREE run: inline + bundles + click, keep ctx/genv
224 let ctx: *i64 = (obx[0]) as *i64
225 let genv: *i64 = (obx[1]) as *i64
226 let best: *u8 = sys_mmap(VSF_MAGIC_4096); best[0] = 0 as u8
227 let brankp: *i64 = sys_mmap(16) as *i64; brankp[0] = 0
228 let bkindp: *i64 = sys_mmap(16) as *i64; bkindp[0] = VSK_NONE
229 if (genv as i64) != 0 {
230 let ub: *u8 = sys_mmap(VSF_MAGIC_4096)
231 let body: *u8 = sys_mmap(VSF_MAGIC_2097152)
232 let st: *i64 = sys_mmap(16) as *i64
233 var round: i64 = 0
234 var stop: i64 = 0
235 var postlen: i64 = 0
236 while post[postlen] != (0 as u8) { postlen = postlen + 1 }
237 while stop == 0 {
238 js_run_more_in_genv(genv, post, postlen) // re-click players each round: a jQuery-ready/DOMContentLoaded
239 var serviced: i64 = 0 // handler bound during the prior drain now gets its click
240 var fetches: i64 = 0
241 let total: i64 = js_pending_total(genv)
242 var i: i64 = 0
243 while i < total {
244 if js_pending_serviced(genv, i) == 0 {
245 let ul: i64 = js_pending_url(genv, i, ub, VSF_MAGIC_4096)
246 if ul > 0 {
247 let mk: i64 = vs_url_kind(ub, ul) // pending URL itself a stream?
248 if mk != VSK_NONE { let rr: i64 = vs_kind_rank(mk); if rr > brankp[0] { brankp[0] = rr; bkindp[0] = mk; var z: i64 = 0; while z < ul { best[z] = ub[z]; z = z + 1 } best[ul] = 0 as u8 } }
249 if fetches < 16 { // per-round fetch cap (abuse backstop)
250 if vn_url_safe(ub, ul) == 1 {
251 let bn: i64 = nx_https_fetch_follow(ub, store, body, VSF_MAGIC_2097152, 6, st)
252 fetches = fetches + 1
253 if bn > 0 { js_pending_service(ctx, genv, i, st[0], body, bn); serviced = serviced + 1; vn_scan_media(body, bn, best, brankp, bkindp) }
254 else { js_pending_service(ctx, genv, i, 0, "" as *u8, 0) }
255 } else { js_pending_service(ctx, genv, i, 0, "" as *u8, 0) }
256 } else { js_pending_service(ctx, genv, i, 0, "" as *u8, 0) }
257 } else { js_pending_service(ctx, genv, i, 0, "" as *u8, 0) }
258 }
259 i = i + 1
260 }
261 if bkindp[0] != VSK_NONE { stop = 1 }
262 else { if serviced == 0 { stop = 1 } else { el_drain(ctx, genv); round = round + 1; if round > max_rounds { stop = 1 } } }
263 }
264 }
265 var w: i64 = 0
266 w = vs_puts(planbuf, w, "ROUTE=" as *u8)
267 w = vs_puts(planbuf, w, vs_route_name(bkindp[0]))
268 w = vs_puts(planbuf, w, " URL=" as *u8)
269 if bkindp[0] != VSK_NONE { w = vs_puts(planbuf, w, best) } else { w = vs_puts(planbuf, w, "-" as *u8) }
270 planbuf[w] = 0 as u8
271 return bkindp[0]
272}
273// URL variant: fetch page + external bundles (mirrors vsf_sniff_url_pre), then multi-round network sniff.
274func vsf_sniff_url_net(url: *u8, ulen: i64, store: *TrustStore, preamble: *u8, plen: i64, planbuf: *u8, plancap: i64, max_rounds: i64) -> i64 {
275 let status: *i64 = sys_mmap(8) as *i64
276 let html: *u8 = sys_mmap(VSF_HTMLCAP)
277 let hn: i64 = nx_https_fetch_follow(url, store, html, VSF_HTMLCAP, 6, status)
278 if hn <= 0 { var e: i64 = vs_puts(planbuf, 0, "ROUTE=fetch-failed URL=-" as *u8); planbuf[e] = 0 as u8; return VSK_NONE }
279 let srcs: *u8 = sys_mmap(VSF_MAGIC_65536)
280 let nsrc: i64 = vs_extract_script_srcs(html, hn, srcs, VSF_MAGIC_65536)
281 let extjs: *u8 = sys_mmap(VSF_EXTCAP)
282 let bundle: *u8 = sys_mmap(VSF_BUNCAP)
283 let absurl: *u8 = sys_mmap(VSF_MAGIC_8192)
284 let line: *u8 = sys_mmap(VSF_MAGIC_8192)
285 var extlen: i64 = 0
286 var fetched: i64 = 0
287 var li: i64 = 0
288 var walking: i64 = 1
289 while walking == 1 {
290 if fetched >= VSF_MAXSRC { walking = 0 }
291 else {
292 if (srcs[li] & 0xff) == 0 { walking = 0 }
293 else {
294 var q: i64 = 0
295 var inl: i64 = 1
296 while inl == 1 {
297 let c: i64 = srcs[li] & 0xff
298 if c == 0 { inl = 0 }
299 else { if c == 10 { inl = 0; li = li + 1 } else { if q < VSF_MAGIC_8191 { line[q] = c as u8; q = q + 1 } li = li + 1 } }
300 }
301 line[q] = 0 as u8
302 if q > 0 {
303 let rn: i64 = nx_hls_resolve_uri(url, ulen, line, q, absurl, VSF_MAGIC_8192)
304 if rn > 0 {
305 absurl[rn] = 0 as u8
306 let bn: i64 = nx_https_fetch_follow(absurl, store, bundle, VSF_BUNCAP, 6, status)
307 if bn > 0 {
308 var bp: i64 = 0
309 while bp < bn { if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = bundle[bp]; extlen = extlen + 1 } bp = bp + 1 }
310 if extlen < (VSF_EXTCAP - 3) { extjs[extlen] = 59 as u8; extlen = extlen + 1; extjs[extlen] = 10 as u8; extlen = extlen + 1 }
311 fetched = fetched + 1
312 }
313 }
314 }
315 }
316 }
317 }
318 extjs[extlen] = 0 as u8
319 return vs_sniff_net_bundled(preamble, plen, html, hn, extjs, extlen, store, planbuf, plancap, max_rounds)
320}