nx_hls_get.nx source
↩ module page · 240 lines · 16053 B
1// nx_hls_get.nx -- X-DLP rung 2: the HLS DOWNLOAD ORCHESTRATOR. Turns an .m3u8 URL into ONE stitched
2// .ts file the gallery can ingest + the operator can watch. Composes the gated rung 0 (nx_hls_parse)
3// + rung 1 (nx_hls_resolve) + the UA/Referer/Cookie-aware fetch (nx_video_get's nx_video_fetch):
4// fetch playlist -> if MASTER pick the highest-bandwidth variant -> fetch its media playlist ->
5// resolve EVERY segment URI to absolute -> fetch each in order -> CONCAT into outpath.
6// FORK-PER-SEGMENT: each segment is fetched in a child that exits, so the TLS stack's per-fetch mmap
7// leak dies with the child (no OOM over hundreds of segments) -- same isolation idiom as the connect
8// researcher. CLEAR + AES-128 (#EXT-X-KEY, rung 3 via nx_aes_cbc) HLS handled here; daemon wiring = rung 4;
9// fMP4/#EXT-X-MAP (init segment) = a later rung (MPEG-TS .ts concatenates cleanly, fMP4 does not).
10// argv[1]=m3u8 url argv[2]=outpath(.ts) argv[3]=referer argv[4]=cookie. license_tier: ORIGINAL
11// module: nishi-core.media.hls_get
12import "nx_hls_resolve.nx" // nx_hls_resolve_uri + (transitively) nx_hls_parse
13import "nx_video_get.nx" // nx_video_fetch (UA+Referer+Cookie GET -> file, follows redirects)
14import "nx_x509_trust_store.nx"
15import "nx_pem_loader.nx"
16import "nx_aes.nx" // aes128_expand_key
17import "nx_aes_cbc.nx" // aes128_cbc_decrypt + aes128_pkcs7_strip (NIST SP 800-38A KAT-gated)
18import "nx_syscalls.nx"
19import "nx_paced_fetch.nx" // R0 (/compare/mediaingest): pf_before_burst/pf_after around every playlist, key and segment fetch
20const HG_MAGIC_8192: i64 = 8192
21const HG_MAGIC_4096: i64 = 4096
22const HG_HTTP_OK: i64 = 200 // what the pacer is fed when the forked fetch succeeded (the child cannot hand back the real status)
23
24func hg_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
25func hg_wn(v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let b: *u8=sys_mmap(28); var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
26func hg_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
27
28// read the whole file at path into buf (bounded by cap); returns bytes read or -1 if open fails.
29func hg_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
30 let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 - 1 }
31 var o: i64 = 0; var go: i64 = 1
32 while go == 1 { if o >= cap { go = 0 } else { let r: i64 = sys_read(fd, (buf as i64 + o) as *u8, cap - o); if r <= 0 { go = 0 } else { o = o + r } } }
33 sys_close(fd); return o
34}
35// append n bytes to an open fd. returns 0 or -1.
36func hg_append(fd: i64, buf: *u8, n: i64) -> i64 { var o: i64 = 0; while o < n { let w: i64 = sys_write(fd, (buf as i64 + o) as *u8, n - o); if w <= 0 { return 0 - 1 } o = o + w } return 0 }
37
38// fork a child that fetches url -> partpath (UA/Referer/Cookie aware, redirect-following), parent waits.
39// returns the child's exit code (0 = ok). The per-fetch TLS mmap leak dies with the child.
40// R0 (/compare/mediaingest, 2026-08-30): paced in the PARENT around the fork. The BURST form is deliberate: segments
41// of one stream are not separate acts of politeness, so a healthy CDN streams at full speed and the wait is paid only
42// once the host has actively throttled us (nx_paced_fetch header). The child cannot hand back an HTTP status, so the
43// pacer is fed 200 on success and 0 on failure -- the same honest degradation nx_4chan uses, never a guessed number.
44func hg_fetch(url: *u8, referer: *u8, cookie: *u8, partpath: *u8, store: *TrustStore, now: i64) -> i64 {
45 pf_before_burst(url)
46 let pid: i64 = sys_fork()
47 if pid == 0 {
48 let n: i64 = nx_video_fetch(url, referer, cookie, partpath, store, now)
49 if n > 0 { sys_exit(0) }
50 sys_exit(1)
51 }
52 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
53 let rc: i64 = (st[0] >> 8) & 0xff
54 var fed: i64 = 0
55 if rc == 0 { fed = HG_HTTP_OK }
56 pf_after(url, fed, 0)
57 return rc
58}
59
60// ---- #EXT-X-KEY (AES-128) parsing + sequence-IV (rung 3) ----
61// scan buf[0..n) for the NUL-term literal; return the index just AFTER it, or -1.
62func hg_find_after(buf: *u8, n: i64, lit: *u8) -> i64 {
63 let ll: i64 = hg_slen(lit); if ll == 0 { return 0 - 1 }
64 var i: i64 = 0
65 while i + ll <= n { var j: i64 = 0; var hit: i64 = 1; while j < ll { if (buf[i+j] & 0xff) != (lit[j] & 0xff) { hit = 0; j = ll } else { j = j + 1 } } if hit == 1 { return i + ll } i = i + 1 }
66 return 0 - 1
67}
68func hg_hexv(c: i64) -> i64 { if c>=48 { if c<=57 { return c-48 } } if c>=97 { if c<=102 { return c-97+10 } } if c>=65 { if c<=70 { return c-65+10 } } return 0 - 1 }
69
70// parse the media playlist's #EXT-X-KEY (RFC 8216 ยง4.3.2.4). Returns 1 if METHOD=AES-128 (fills keyuri +
71// optional 16-byte iv, sets has_iv), else 0 (clear / METHOD=NONE / SAMPLE-AES unsupported).
72func hg_parse_key(pl: *u8, pllen: i64, keyuri: *u8, uricap: i64, ivout: *u8, has_iv: *i64) -> i64 {
73 has_iv[0] = 0
74 if hg_find_after(pl, pllen, "METHOD=AES-128\x00" as *u8) < 0 { return 0 }
75 let uo: i64 = hg_find_after(pl, pllen, "URI=\"\x00" as *u8)
76 if uo < 0 { return 0 }
77 var k: i64 = 0; var p: i64 = uo
78 while p < pllen { let c: i64 = pl[p] as i64; if c == 34 { p = pllen } else { if k < uricap - 1 { keyuri[k] = pl[p]; k = k + 1 } p = p + 1 } }
79 keyuri[k] = 0 as u8
80 let io: i64 = hg_find_after(pl, pllen, "IV=\x00" as *u8)
81 if io >= 0 {
82 var q: i64 = io
83 if q + 1 < pllen { if pl[q] == (48 as u8) { let c1: i64 = pl[q+1] as i64; if c1 == 120 { q = q + 2 } else { if c1 == 88 { q = q + 2 } } } } // skip 0x / 0X
84 var got: i64 = 0; var run: i64 = 1
85 while run == 1 {
86 if got >= 16 { run = 0 } else { if q + 1 >= pllen { run = 0 } else {
87 let hi: i64 = hg_hexv(pl[q] as i64); let lo: i64 = hg_hexv(pl[q+1] as i64)
88 if hi < 0 { run = 0 } else { if lo < 0 { run = 0 } else { ivout[got] = ((hi<<4)|lo) & 0xff; q = q + 2; got = got + 1 } }
89 } }
90 }
91 if got == 16 { has_iv[0] = 1 }
92 }
93 return 1
94}
95// #EXT-X-MEDIA-SEQUENCE (default 0) -- the base index for sequence-number IVs.
96func hg_media_seq(pl: *u8, pllen: i64) -> i64 {
97 let so: i64 = hg_find_after(pl, pllen, "#EXT-X-MEDIA-SEQUENCE:\x00" as *u8)
98 if so < 0 { return 0 }
99 var v: i64 = 0; var p: i64 = so
100 while p < pllen { let c: i64 = pl[p] as i64; if c >= 48 { if c <= 57 { v = v*10 + (c-48); p = p + 1 } else { p = pllen } } else { p = pllen } }
101 return v
102}
103// default IV when #EXT-X-KEY carries no IV= : the segment's media sequence number as a 128-bit big-endian int.
104func hg_seq_iv(seq: i64, iv: *u8) -> i64 {
105 var z: i64 = 0; while z < 16 { iv[z] = 0 as u8; z = z + 1 }
106 var v: i64 = seq; var b: i64 = 15
107 while b >= 8 { iv[b] = (v & 0xff) as u8; v = v >> 8; b = b - 1 }
108 return 0
109}
110
111// derive the .mp4 path from a .ts path: drop a trailing ".ts", then append ".mp4".
112func hg_mp4_path(ts: *u8, out: *u8) -> i64 {
113 var n: i64 = 0; while ts[n] != (0 as u8) { out[n] = ts[n]; n = n + 1 }
114 if n >= 3 { if out[n-3]==(46 as u8) { if out[n-2]==(116 as u8) { if out[n-1]==(115 as u8) { n = n - 3 } } } } // strip ".ts"
115 out[n]=46 as u8; out[n+1]=109 as u8; out[n+2]=112 as u8; out[n+3]=52 as u8; out[n+4]=0 as u8 // + ".mp4"
116 return n + 4
117}
118// best-effort: remux the stitched .ts -> a faststart .mp4 (Chrome plays it natively) via nx_ts2mp4.
119// Double-fork + exec; absent elf = harmless no-op (the .ts stays). nx_ts2mp4 assumes no-B-frames + ~96MB cap.
120func hg_remux(tspath: *u8, mp4path: *u8) -> i64 {
121 let wpath: *u8 = "/tmp/nx_ts2mp4.sov.elf" as *u8
122 let argv: *i64 = sys_mmap(32) as *i64; argv[0]=wpath as i64; argv[1]=tspath as i64; argv[2]=mp4path as i64; argv[3]=0
123 let envp: *i64 = sys_mmap(16) as *i64; envp[0]=0
124 let pid: i64 = sys_fork()
125 if pid == 0 { sys_execve(wpath, argv, envp); sys_exit(127) } // fork+WAIT: sequential post-processing (NOT detach)
126 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
127 return 0
128}
129
130// best-effort: re-index the gallery video roots so a just-stitched .ts shows up + streams (same idiom as
131// the torrent worker's post-extract refresh). Double-fork + exec the indexer; absent elf = harmless no-op.
132func hg_gallery_refresh() -> i64 {
133 let wpath: *u8 = "/tmp/nx_galx_vidindex.sov.elf" as *u8
134 let argv: *i64 = sys_mmap(16) as *i64; argv[0] = wpath as i64; argv[1] = 0
135 let envp: *i64 = sys_mmap(16) as *i64; envp[0] = 0
136 let pid: i64 = sys_fork()
137 if pid == 0 { sys_execve(wpath, argv, envp); sys_exit(127) } // fork+WAIT: sequential post-processing (NOT detach)
138 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
139 return 0
140}
141
142const HG_MAXENT: i64 = 20000 // max segments / variants
143const HG_PLCAP: i64 = 4194304 // 4 MiB playlist buffer
144const HG_SEGCAP: i64 = 33554432 // 32 MiB per-segment buffer (HLS .ts segments are typically a few MiB)
145
146func main(argc: i64, argv: *i64) -> i64 {
147 if argc < 3 { hg_w("usage: nx_hls_get <m3u8-url> <outpath.ts> [referer] [cookie]\n" as *u8); sys_exit(2); return 2 }
148 let url: *u8 = argv[1] as *u8; let outpath: *u8 = argv[2] as *u8
149 var referer: *u8 = "" as *u8; if argc >= 4 { referer = argv[3] as *u8 }
150 var cookie: *u8 = "" as *u8; if argc >= 5 { cookie = argv[4] as *u8 }
151
152 let store: *TrustStore = trust_store_alloc(400)
153 if nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt\x00", store) <= 0 { hg_w("HLS-GET no-ca\n" as *u8); sys_exit(1); return 1 }
154 let now: i64 = sys_now_realtime_sec()
155
156 let plpath: *u8 = "/tmp/nx_hls_playlist.m3u8\x00"
157 let partpath: *u8 = "/tmp/nx_hls_seg.part\x00"
158 let plbuf: *u8 = sys_mmap(HG_PLCAP)
159 let entries: *i64 = sys_mmap(8 * 3 * HG_MAXENT) as *i64
160 let meta: *i64 = sys_mmap(8 * 8) as *i64
161 let absurl: *u8 = sys_mmap(HG_MAGIC_8192) // resolved absolute URL scratch (reused per segment)
162
163 // ---- fetch the playlist ----
164 if hg_fetch(url, referer, cookie, plpath, store, now) != 0 { hg_w("HLS-GET playlist fetch FAILED\n" as *u8); sys_exit(1); return 1 }
165 var pllen: i64 = hg_readfile(plpath, plbuf, HG_PLCAP)
166 if pllen <= 0 { hg_w("HLS-GET empty playlist\n" as *u8); sys_exit(1); return 1 }
167 var cnt: i64 = nx_hls_parse(plbuf, pllen, entries, HG_MAXENT, meta)
168 if cnt < 0 { hg_w("HLS-GET not an m3u8 (server returned a non-playlist body)\n" as *u8); sys_exit(1); return 1 }
169
170 var baseurl: *u8 = url; var basen: i64 = hg_slen(url) // base for resolving child URIs
171
172 // ---- MASTER playlist: pick the highest-bandwidth variant, fetch its media playlist, re-parse ----
173 if meta[0] == 1 {
174 var best: i64 = 0 - 1; var bestbw: i64 = 0 - 1; var i: i64 = 0
175 while i < cnt { let bw: i64 = entries[i*3+2]; if bw > bestbw { bestbw = bw; best = i } i = i + 1 }
176 if best < 0 { hg_w("HLS-GET master with no variants\n" as *u8); sys_exit(1); return 1 }
177 let vs: i64 = entries[best*3+0]; let vl: i64 = entries[best*3+1]
178 let rn: i64 = nx_hls_resolve_uri(baseurl, basen, (plbuf as i64 + vs) as *u8, vl, absurl, HG_MAGIC_8192)
179 if rn < 0 { hg_w("HLS-GET variant resolve overflow\n" as *u8); sys_exit(1); return 1 }
180 absurl[rn] = 0 as u8
181 hg_w("HLS-GET master -> variant bw=" as *u8); hg_wn(bestbw); hg_w(" " as *u8); hg_w(absurl); hg_w("\n" as *u8)
182 let vbuf: *u8 = sys_mmap(HG_MAGIC_8192); var z: i64 = 0; while z <= rn { vbuf[z] = absurl[z]; z = z + 1 } // stable base (absurl is reused below)
183 baseurl = vbuf; basen = rn
184 if hg_fetch(baseurl, referer, cookie, plpath, store, now) != 0 { hg_w("HLS-GET media playlist fetch FAILED\n" as *u8); sys_exit(1); return 1 }
185 pllen = hg_readfile(plpath, plbuf, HG_PLCAP)
186 cnt = nx_hls_parse(plbuf, pllen, entries, HG_MAXENT, meta)
187 if cnt < 0 { hg_w("HLS-GET variant is not a media playlist\n" as *u8); sys_exit(1); return 1 }
188 }
189
190 if cnt == 0 { hg_w("HLS-GET 0 segments (encrypted-only or empty?)\n" as *u8); sys_exit(1); return 1 }
191 hg_w("HLS-GET media playlist: segments=" as *u8); hg_wn(cnt); hg_w("\n" as *u8)
192
193 // ---- rung 3: #EXT-X-KEY AES-128 -> fetch key + build schedule (else it stays a clear stream) ----
194 let keyuri: *u8 = sys_mmap(HG_MAGIC_4096); let ivexp: *u8 = sys_mmap(16); let has_iv: *i64 = sys_mmap(16) as *i64
195 let sched: *u8 = sys_mmap(176); var enc: i64 = 0; var media_seq: i64 = 0
196 if hg_parse_key(plbuf, pllen, keyuri, HG_MAGIC_4096, ivexp, has_iv) == 1 {
197 let krn: i64 = nx_hls_resolve_uri(baseurl, basen, keyuri, hg_slen(keyuri), absurl, HG_MAGIC_8192)
198 if krn < 0 { hg_w("HLS-GET key URI resolve overflow\n" as *u8); sys_exit(1); return 1 }
199 absurl[krn] = 0 as u8
200 if hg_fetch(absurl, referer, cookie, partpath, store, now) != 0 { hg_w("HLS-GET key fetch FAILED\n" as *u8); sys_exit(1); return 1 }
201 let keybuf: *u8 = sys_mmap(64); let kn: i64 = hg_readfile(partpath, keybuf, 64)
202 if kn < 16 { hg_w("HLS-GET key too short (got " as *u8); hg_wn(kn); hg_w(" bytes)\n" as *u8); sys_exit(1); return 1 }
203 aes128_expand_key(keybuf, sched)
204 media_seq = hg_media_seq(plbuf, pllen)
205 enc = 1
206 hg_w("HLS-GET AES-128 encrypted: key fetched, media_seq=" as *u8); hg_wn(media_seq); hg_w(" explicit_iv=" as *u8); hg_wn(has_iv[0]); hg_w("\n" as *u8)
207 }
208
209 // ---- fetch every segment in order, concat into outpath (.ts appends cleanly) ----
210 let out_fd: i64 = sys_openat_wr(outpath, 0x1a4); if out_fd < 0 { hg_w("HLS-GET cannot open output\n" as *u8); sys_exit(1); return 1 }
211 let segbuf: *u8 = sys_mmap(HG_SEGCAP)
212 var done: i64 = 0; var failed: i64 = 0; var bytes: i64 = 0; var i: i64 = 0
213 while i < cnt {
214 let ss: i64 = entries[i*3+0]; let sl: i64 = entries[i*3+1]
215 let rn: i64 = nx_hls_resolve_uri(baseurl, basen, (plbuf as i64 + ss) as *u8, sl, absurl, HG_MAGIC_8192)
216 if rn < 0 { failed = failed + 1 } else {
217 absurl[rn] = 0 as u8
218 if hg_fetch(absurl, referer, cookie, partpath, store, now) == 0 {
219 var sn: i64 = hg_readfile(partpath, segbuf, HG_SEGCAP)
220 if sn > 0 {
221 if enc == 1 {
222 let seg_iv: *u8 = sys_mmap(16)
223 if has_iv[0] == 1 { var c: i64 = 0; while c < 16 { seg_iv[c] = ivexp[c]; c = c + 1 } } else { hg_seq_iv(media_seq + i, seg_iv) }
224 let dn: i64 = aes128_cbc_decrypt(segbuf, sn, sched, seg_iv, segbuf) // in-place CBC decrypt
225 if dn > 0 { sn = aes128_pkcs7_strip(segbuf, dn) } // strip PKCS7 (per-segment)
226 }
227 hg_append(out_fd, segbuf, sn); bytes = bytes + sn; done = done + 1
228 } else { failed = failed + 1 }
229 } else { failed = failed + 1 }
230 }
231 if (i % 10) == 0 { hg_w(" seg " as *u8); hg_wn(i + 1); hg_w("/" as *u8); hg_wn(cnt); hg_w(" ok=" as *u8); hg_wn(done); hg_w(" bytes=" as *u8); hg_wn(bytes); hg_w("\n" as *u8) }
232 i = i + 1
233 }
234 sys_close(out_fd)
235 if done > 0 { hg_gallery_refresh() } // keep the .ts: the gallery fly-remuxes .ts->fMP4 (nx_ts2fmp4 + MSE, GB-capable) for playback -> no 96MB cap, no duplicate .mp4 entry. (hg_remux/hg_mp4_path kept for an explicit standalone-export feature.)
236 hg_w("HLS-GET done segments=" as *u8); hg_wn(done); hg_w("/" as *u8); hg_wn(cnt); hg_w(" failed=" as *u8); hg_wn(failed); hg_w(" bytes=" as *u8); hg_wn(bytes); hg_w(" -> " as *u8); hg_w(outpath); hg_w("\n" as *u8)
237 if done == cnt { hg_w("HLS-GET verdict=GREEN COMPLETE\n" as *u8); sys_exit(0); return 0 }
238 if done > 0 { hg_w("HLS-GET verdict=PARTIAL (some segments failed -- may be AES-encrypted [rung 3] or gated)\n" as *u8); sys_exit(0); return 0 }
239 hg_w("HLS-GET verdict=RED no-segments-fetched\n" as *u8); sys_exit(1); return 1
240}