nx_media_deepcrawl.nx source
↩ module page · 190 lines · 10999 B
1// nx_media_deepcrawl.nx -- the DEEP media harvester as a REUSABLE capability (operator 2026-07-08: "harvest the
2// media as part of the browser capabilities -- no manual network capture"). Crawls a page's resource graph over
3// sovereign TLS, following the CSS/JS it pulls in (incl. URLs the JS CONSTRUCTS from constants, reconstructed by
4// nx_js_urlfold WITHOUT executing anything), harvesting media from every hop into a MediaSet. Used by BOTH the
5// nx_media_inspect CLI and the in-browser 'm' key. Budget-bounded (maxfetch, maxdepth). Never runs page JS ->
6// robust on the minified ad bundles that segfault a tree-walker. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_https_fetch_follow.nx"
10import "nx_media_harvest.nx"
11import "nx_js_urlfold.nx"
12import "nx_url_resolve.nx"
13
14const MDC_CAP: i64 = 4194304
15const MDC_MAXN: i64 = 512
16const MDC_MAXFR: i64 = 192
17const MDC_MBUF: i64 = 1048576
18const MDC_FBUF: i64 = 262144
19const MDC_IFRAMES: i64 = 0 // iframe-following (temp-gated while debugging a hang)
20
21struct MediaSet { buf: *u8, off: *i64, len: *i64, kind: *i64, cnt: i64, used: i64 }
22struct Frontier { buf: *u8, off: *i64, len: *i64, depth: *i64, cnt: i64, head: i64, used: i64 }
23const MSET_BYTES: i64 = 48
24const FRNT_BYTES: i64 = 56
25
26func mdc_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
27func mdc_n(fd: i64, v0: i64) -> i64 { var v: i64=v0; if v<0 { sys_write(fd,"-" as *u8,1); v=0-v } let b: *u8=sys_mmap(24); var k: i64=0; if v==0{b[0]=48 as u8;k=1} while v>0{b[k]=(48+(v%10)) as u8;v=v/10;k=k+1} let o: *u8=sys_mmap(24); var j: i64=0; while j<k{o[j]=b[k-1-j];j=j+1} sys_write(fd,o,k); return 0 }
28func mdc_lc(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c }
29
30// allocate + init a MediaSet
31func nx_mediaset_new() -> *MediaSet {
32 let M: *MediaSet = sys_mmap(MSET_BYTES) as *MediaSet
33 M.buf = sys_mmap(MDC_MBUF); M.off = sys_mmap(8*MDC_MAXN) as *i64; M.len = sys_mmap(8*MDC_MAXN) as *i64; M.kind = sys_mmap(8*MDC_MAXN) as *i64
34 M.cnt = 0; M.used = 0
35 return M
36}
37
38func mdc_suf(u: *u8, end: i64, ext: *u8, el: i64) -> i64 { if end < el { return 0 } var i: i64=0; while i<el { if mdc_lc(u[end-el+i] as i64) != mdc_lc(ext[i] as i64) { return 0 } i=i+1 } return 1 }
39// path (up to ?/#) ends in .css/.js/.mjs/.json ?
40func mdc_follow_ext(u: *u8, ulen: i64) -> i64 {
41 var end: i64 = ulen; var i: i64 = 0
42 while i < ulen { let c: i64=u[i] as i64; if c==63 { if end==ulen { end=i } } if c==35 { if end==ulen { end=i } } i=i+1 }
43 if mdc_suf(u, end, ".css" as *u8, 4)==1 { return 1 }
44 if mdc_suf(u, end, ".js" as *u8, 3)==1 { return 1 }
45 if mdc_suf(u, end, ".mjs" as *u8, 4)==1 { return 1 }
46 if mdc_suf(u, end, ".json" as *u8, 5)==1 { return 1 }
47 return 0
48}
49func mdc_resolve(base: *u8, blen: i64, cand: *u8, clen: i64, out: *u8, cap: i64) -> i64 {
50 let olp: *i64 = sys_mmap(8) as *i64
51 let rc: i64 = nx_url_resolve(base, blen, cand, clen, out, cap, olp)
52 if rc == NX_URL_RESOLVE_OK { return olp[0] }
53 return 0
54}
55func mdc_add_media(M: *MediaSet, src: *u8, soff: i64, slen: i64, kind: i64) -> i64 {
56 if M.cnt >= MDC_MAXN { return 0 }
57 if slen <= 0 { return 0 }
58 let buf: *u8 = M.buf; let off: *i64 = M.off; let len: *i64 = M.len
59 var i: i64 = 0
60 while i < M.cnt { if len[i]==slen { var eq: i64=1; var k: i64=0; while k<slen { if buf[off[i]+k]!=src[soff+k] { eq=0; k=slen } else { k=k+1 } } if eq==1 { return 0 } } i=i+1 }
61 if M.used + slen >= MDC_MBUF { return 0 }
62 var m: i64 = 0
63 while m < slen { buf[M.used+m]=src[soff+m]; m=m+1 }
64 let c: i64 = M.cnt
65 off[c]=M.used; len[c]=slen; M.kind[c]=kind
66 M.used = M.used + slen; M.cnt = c + 1
67 return 1
68}
69func mdc_add_frontier(F: *Frontier, u: *u8, ulen0: i64, depth: i64) -> i64 {
70 if F.cnt >= MDC_MAXFR { return 0 }
71 // trim a #fragment -- never sent to the server, and it lets dedup collapse hash-variants
72 var ulen: i64 = ulen0; var fi: i64 = 0
73 while fi < ulen0 { if u[fi]==(35 as u8) { ulen=fi; fi=ulen0 } else { fi=fi+1 } }
74 if ulen <= 0 { return 0 }
75 let buf: *u8 = F.buf; let off: *i64 = F.off; let len: *i64 = F.len
76 var i: i64 = 0
77 while i < F.cnt { if len[i]==ulen { var eq: i64=1; var k: i64=0; while k<ulen { if buf[off[i]+k]!=u[k] { eq=0; k=ulen } else { k=k+1 } } if eq==1 { return 0 } } i=i+1 }
78 if F.used + ulen >= MDC_FBUF { return 0 }
79 var m: i64 = 0
80 while m < ulen { buf[F.used+m]=u[m]; m=m+1 }
81 let c: i64 = F.cnt
82 off[c]=F.used; len[c]=ulen; F.depth[c]=depth
83 F.used = F.used + ulen; F.cnt = c + 1
84 return 1
85}
86// find <iframe ...> tags and follow their src (embedded players/widgets are where video often lives)
87func mdc_scan_iframes(text: *u8, n: i64, base: *u8, blen: i64, F: *Frontier, depth: i64) -> i64 {
88 let res: *u8 = sys_mmap(8192); let tokp: *u8 = sys_mmap(8192)
89 var i: i64 = 0
90 while i + 7 < n {
91 var hit: i64 = 0
92 if text[i]==(60 as u8) { if mdc_lc(text[i+1] as i64)==105 { if mdc_lc(text[i+2] as i64)==102 { if mdc_lc(text[i+3] as i64)==114 { if mdc_lc(text[i+4] as i64)==97 { if mdc_lc(text[i+5] as i64)==109 { if mdc_lc(text[i+6] as i64)==101 { hit=1 } } } } } } }
93 if hit == 1 {
94 var te: i64 = i+7; var fe: i64=0
95 while fe==0 { if te>=n { fe=1 } else { if text[te]==(62 as u8) { fe=1 } else { te=te+1 } } }
96 var j: i64 = i+7; var done: i64 = 0
97 while j + 4 < te {
98 if done == 0 { if mdc_lc(text[j] as i64)==115 { if mdc_lc(text[j+1] as i64)==114 { if mdc_lc(text[j+2] as i64)==99 { if text[j+3]==(61 as u8) {
99 var v: i64 = j+4; var qc: i64 = 0
100 if v<te { let c: i64=text[v] as i64; if c==34 { qc=34; v=v+1 } else { if c==39 { qc=39; v=v+1 } } }
101 var ve: i64 = v; var ge: i64=0
102 while ge==0 { if ve>=te { ge=1 } else { let c2: i64=text[ve] as i64; if qc!=0 { if c2==qc { ge=1 } else { ve=ve+1 } } else { if c2==32 { ge=1 } else { if c2==62 { ge=1 } else { ve=ve+1 } } } } }
103 let vl: i64 = ve - v
104 if vl > 4 { if vl < 8000 { var w: i64=0; while w<vl { tokp[w]=text[v+w]; w=w+1 } tokp[vl]=0 as u8; let rl: i64 = mdc_resolve(base, blen, tokp, vl, res, 8192); if rl > 0 { mdc_add_frontier(F, res, rl, depth) } }
105 }
106 done = 1
107 } } } }
108 j = j + 1
109 }
110 i = te
111 } else { i = i + 1 }
112 }
113 return 0
114}
115// scan quoted regions for tokens ending .css/.js/.mjs/.json -> resolve -> frontier (STATIC <script src>/<link href>)
116func mdc_scan_follow(text: *u8, n: i64, base: *u8, blen: i64, F: *Frontier, depth: i64) -> i64 {
117 let tok: *u8 = sys_mmap(8192); let res: *u8 = sys_mmap(8192)
118 var i: i64 = 0
119 while i < n {
120 let ch: i64 = text[i] as i64
121 var q: i64 = 0
122 if ch==34 { q=1 } if ch==39 { q=1 }
123 if q == 1 {
124 var s: i64 = i+1; var e: i64 = s; var go: i64 = 1
125 while go==1 { if e>=n { go=0 } else { if (text[e] as i64)==ch { go=0 } else { e=e+1 } } }
126 let tl: i64 = e - s
127 if tl > 3 { if tl < 8192 {
128 var w: i64 = 0
129 while w < tl { tok[w]=text[s+w]; w=w+1 } tok[tl]=0 as u8
130 if mdc_follow_ext(tok, tl) == 1 { let rl: i64 = mdc_resolve(base, blen, tok, tl, res, 8192); if rl > 0 { mdc_add_frontier(F, res, rl, depth) } }
131 } }
132 if e < n { i = e + 1 } else { i = n }
133 } else { i = i + 1 }
134 }
135 return 0
136}
137
138// THE DEEP CRAWL. Fills M with every image/gif/video/stream/audio SOURCE URL reachable through the page's
139// resource graph (page -> css/js it references, incl. JS-constructed URLs). `verbose`=1 prints each hop to fd1.
140// `raw_path` (or 0) gets the first 200 page's bytes ("view source"). Returns the number of resources fetched.
141func nx_media_deepcrawl(start: *u8, ulen: i64, store: *TrustStore, M: *MediaSet, maxfetch: i64, maxdepth: i64, verbose: i64, raw_path: *u8) -> i64 {
142 let F: *Frontier = sys_mmap(FRNT_BYTES) as *Frontier
143 F.buf = sys_mmap(MDC_FBUF); F.off = sys_mmap(8*MDC_MAXFR) as *i64; F.len = sys_mmap(8*MDC_MAXFR) as *i64; F.depth = sys_mmap(8*MDC_MAXFR) as *i64
144 F.cnt = 0; F.head = 0; F.used = 0
145 mdc_add_frontier(F, start, ulen, 0)
146
147 let body: *u8 = sys_mmap(MDC_CAP)
148 let st: *i64 = sys_mmap(16) as *i64
149 let hub: *u8 = sys_mmap(MDC_MBUF)
150 let hoff: *i64 = sys_mmap(8*MDC_MAXN) as *i64
151 let hlen: *i64 = sys_mmap(8*MDC_MAXN) as *i64
152 let hkind: *i64 = sys_mmap(8*MDC_MAXN) as *i64
153 let fold: *u8 = sys_mmap(131072)
154 let foff: *i64 = sys_mmap(8*256) as *i64
155 let flen: *i64 = sys_mmap(8*256) as *i64
156 let res: *u8 = sys_mmap(8192)
157 let curbuf: *u8 = sys_mmap(4096)
158 var raw_saved: i64 = 0
159 var fetched: i64 = 0
160
161 while F.head < F.cnt {
162 if fetched >= maxfetch { F.head = F.cnt } else {
163 let coff: i64 = F.off[F.head]; let clen: i64 = F.len[F.head]; let cdep: i64 = F.depth[F.head]
164 var z: i64 = 0; while z < clen { curbuf[z]=F.buf[coff+z]; z=z+1 } curbuf[clen]=0 as u8
165 F.head = F.head + 1
166 let n: i64 = nx_https_fetch_follow_best(curbuf, store, body, MDC_CAP, 6, st)
167 if verbose == 1 { mdc_w(1, " [d" as *u8); mdc_n(1, cdep); mdc_w(1, "] " as *u8); mdc_n(1, st[0]); mdc_w(1, " " as *u8); mdc_n(1, n); mdc_w(1, "B " as *u8); sys_write(1, curbuf, clen); mdc_w(1, "\n" as *u8) }
168 if st[0] == 200 {
169 fetched = fetched + 1
170 if raw_saved == 0 { if (raw_path as i64) != 0 { let rfd: i64 = __syscall(257, 0-100, raw_path as i64, 577, 420, 0, 0); if rfd >= 0 { var dd: i64=0; while dd<n { let ww: i64=sys_write(rfd, ((body as i64)+dd) as *u8, n-dd); if ww<=0 { dd=n } else { dd=dd+ww } } __syscall(3, rfd, 0,0,0,0,0) } } raw_saved = 1 }
171 let hc: i64 = nx_media_harvest(body, n, curbuf, clen, hub, MDC_MBUF, hoff, hlen, hkind, MDC_MAXN)
172 var j: i64 = 0
173 while j < hc { let p: *u8 = ((hub as i64)+hoff[j]) as *u8; mdc_add_media(M, p, 0, hlen[j], hkind[j]); j=j+1 }
174 if cdep < maxdepth {
175 let fc: i64 = nx_js_fold_urls(body, n, fold, foff, flen, 256)
176 var g: i64 = 0
177 while g < fc {
178 let fp: *u8 = ((fold as i64)+foff[g]) as *u8
179 let rl: i64 = mdc_resolve(curbuf, clen, fp, flen[g], res, 8192)
180 if rl > 0 { let k: i64 = mh_classify(res, rl); if k >= 0 { mdc_add_media(M, res, 0, rl, k) } else { if mdc_follow_ext(res, rl)==1 { mdc_add_frontier(F, res, rl, cdep+1) } } }
181 g = g + 1
182 }
183 mdc_scan_follow(body, n, curbuf, clen, F, cdep+1)
184 if MDC_IFRAMES == 1 { mdc_scan_iframes(body, n, curbuf, clen, F, cdep+1) }
185 }
186 }
187 }
188 }
189 return fetched
190}