nx_hls_get.nx source
↩ module page · 229 lines · 15183 B
1// nx_hls_get.nx -- X-DLP rung 2: the HLS DOWNLOAD ORCHESTRATOR. Turns an .m3u8 URL into ONE stitched
2// .ts file the gallery can ingest + the operator can watch. Composes the gated rung 0 (nx_hls_parse)
3// + rung 1 (nx_hls_resolve) + the UA/Referer/Cookie-aware fetch (nx_video_get's nx_video_fetch):
4// fetch playlist -> if MASTER pick the highest-bandwidth variant -> fetch its media playlist ->
5// resolve EVERY segment URI to absolute -> fetch each in order -> CONCAT into outpath.
6// FORK-PER-SEGMENT: each segment is fetched in a child that exits, so the TLS stack's per-fetch mmap
7// leak dies with the child (no OOM over hundreds of segments) -- same isolation idiom as the connect
8// researcher. CLEAR + AES-128 (#EXT-X-KEY, rung 3 via nx_aes_cbc) HLS handled here; daemon wiring = rung 4;
9// fMP4/#EXT-X-MAP (init segment) = a later rung (MPEG-TS .ts concatenates cleanly, fMP4 does not).
10// argv[1]=m3u8 url argv[2]=outpath(.ts) argv[3]=referer argv[4]=cookie. license_tier: ORIGINAL
11// module: nishi-core.media.hls_get
12import "nx_hls_resolve.nx" // nx_hls_resolve_uri + (transitively) nx_hls_parse
13import "nx_video_get.nx" // nx_video_fetch (UA+Referer+Cookie GET -> file, follows redirects)
14import "nx_x509_trust_store.nx"
15import "nx_pem_loader.nx"
16import "nx_aes.nx" // aes128_expand_key
17import "nx_aes_cbc.nx" // aes128_cbc_decrypt + aes128_pkcs7_strip (NIST SP 800-38A KAT-gated)
18import "nx_syscalls.nx"
19const HG_MAGIC_8192: i64 = 8192
20const HG_MAGIC_4096: i64 = 4096
21
22func hg_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
23func hg_wn(v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let b: *u8=sys_mmap(28); var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
24func hg_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
25
26// read the whole file at path into buf (bounded by cap); returns bytes read or -1 if open fails.
27func hg_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
28 let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 - 1 }
29 var o: i64 = 0; var go: i64 = 1
30 while go == 1 { if o >= cap { go = 0 } else { let r: i64 = sys_read(fd, (buf as i64 + o) as *u8, cap - o); if r <= 0 { go = 0 } else { o = o + r } } }
31 sys_close(fd); return o
32}
33// append n bytes to an open fd. returns 0 or -1.
34func hg_append(fd: i64, buf: *u8, n: i64) -> i64 { var o: i64 = 0; while o < n { let w: i64 = sys_write(fd, (buf as i64 + o) as *u8, n - o); if w <= 0 { return 0 - 1 } o = o + w } return 0 }
35
36// fork a child that fetches url -> partpath (UA/Referer/Cookie aware, redirect-following), parent waits.
37// returns the child's exit code (0 = ok). The per-fetch TLS mmap leak dies with the child.
38func hg_fetch(url: *u8, referer: *u8, cookie: *u8, partpath: *u8, store: *TrustStore, now: i64) -> i64 {
39 let pid: i64 = sys_fork()
40 if pid == 0 {
41 let n: i64 = nx_video_fetch(url, referer, cookie, partpath, store, now)
42 if n > 0 { sys_exit(0) }
43 sys_exit(1)
44 }
45 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
46 return (st[0] >> 8) & 0xff
47}
48
49// ---- #EXT-X-KEY (AES-128) parsing + sequence-IV (rung 3) ----
50// scan buf[0..n) for the NUL-term literal; return the index just AFTER it, or -1.
51func hg_find_after(buf: *u8, n: i64, lit: *u8) -> i64 {
52 let ll: i64 = hg_slen(lit); if ll == 0 { return 0 - 1 }
53 var i: i64 = 0
54 while i + ll <= n { var j: i64 = 0; var hit: i64 = 1; while j < ll { if (buf[i+j] & 0xff) != (lit[j] & 0xff) { hit = 0; j = ll } else { j = j + 1 } } if hit == 1 { return i + ll } i = i + 1 }
55 return 0 - 1
56}
57func hg_hexv(c: i64) -> i64 { if c>=48 { if c<=57 { return c-48 } } if c>=97 { if c<=102 { return c-97+10 } } if c>=65 { if c<=70 { return c-65+10 } } return 0 - 1 }
58
59// parse the media playlist's #EXT-X-KEY (RFC 8216 ยง4.3.2.4). Returns 1 if METHOD=AES-128 (fills keyuri +
60// optional 16-byte iv, sets has_iv), else 0 (clear / METHOD=NONE / SAMPLE-AES unsupported).
61func hg_parse_key(pl: *u8, pllen: i64, keyuri: *u8, uricap: i64, ivout: *u8, has_iv: *i64) -> i64 {
62 has_iv[0] = 0
63 if hg_find_after(pl, pllen, "METHOD=AES-128\x00" as *u8) < 0 { return 0 }
64 let uo: i64 = hg_find_after(pl, pllen, "URI=\"\x00" as *u8)
65 if uo < 0 { return 0 }
66 var k: i64 = 0; var p: i64 = uo
67 while p < pllen { let c: i64 = pl[p] as i64; if c == 34 { p = pllen } else { if k < uricap - 1 { keyuri[k] = pl[p]; k = k + 1 } p = p + 1 } }
68 keyuri[k] = 0 as u8
69 let io: i64 = hg_find_after(pl, pllen, "IV=\x00" as *u8)
70 if io >= 0 {
71 var q: i64 = io
72 if q + 1 < pllen { if pl[q] == (48 as u8) { let c1: i64 = pl[q+1] as i64; if c1 == 120 { q = q + 2 } else { if c1 == 88 { q = q + 2 } } } } // skip 0x / 0X
73 var got: i64 = 0; var run: i64 = 1
74 while run == 1 {
75 if got >= 16 { run = 0 } else { if q + 1 >= pllen { run = 0 } else {
76 let hi: i64 = hg_hexv(pl[q] as i64); let lo: i64 = hg_hexv(pl[q+1] as i64)
77 if hi < 0 { run = 0 } else { if lo < 0 { run = 0 } else { ivout[got] = ((hi<<4)|lo) & 0xff; q = q + 2; got = got + 1 } }
78 } }
79 }
80 if got == 16 { has_iv[0] = 1 }
81 }
82 return 1
83}
84// #EXT-X-MEDIA-SEQUENCE (default 0) -- the base index for sequence-number IVs.
85func hg_media_seq(pl: *u8, pllen: i64) -> i64 {
86 let so: i64 = hg_find_after(pl, pllen, "#EXT-X-MEDIA-SEQUENCE:\x00" as *u8)
87 if so < 0 { return 0 }
88 var v: i64 = 0; var p: i64 = so
89 while p < pllen { let c: i64 = pl[p] as i64; if c >= 48 { if c <= 57 { v = v*10 + (c-48); p = p + 1 } else { p = pllen } } else { p = pllen } }
90 return v
91}
92// default IV when #EXT-X-KEY carries no IV= : the segment's media sequence number as a 128-bit big-endian int.
93func hg_seq_iv(seq: i64, iv: *u8) -> i64 {
94 var z: i64 = 0; while z < 16 { iv[z] = 0 as u8; z = z + 1 }
95 var v: i64 = seq; var b: i64 = 15
96 while b >= 8 { iv[b] = (v & 0xff) as u8; v = v >> 8; b = b - 1 }
97 return 0
98}
99
100// derive the .mp4 path from a .ts path: drop a trailing ".ts", then append ".mp4".
101func hg_mp4_path(ts: *u8, out: *u8) -> i64 {
102 var n: i64 = 0; while ts[n] != (0 as u8) { out[n] = ts[n]; n = n + 1 }
103 if n >= 3 { if out[n-3]==(46 as u8) { if out[n-2]==(116 as u8) { if out[n-1]==(115 as u8) { n = n - 3 } } } } // strip ".ts"
104 out[n]=46 as u8; out[n+1]=109 as u8; out[n+2]=112 as u8; out[n+3]=52 as u8; out[n+4]=0 as u8 // + ".mp4"
105 return n + 4
106}
107// best-effort: remux the stitched .ts -> a faststart .mp4 (Chrome plays it natively) via nx_ts2mp4.
108// Double-fork + exec; absent elf = harmless no-op (the .ts stays). nx_ts2mp4 assumes no-B-frames + ~96MB cap.
109func hg_remux(tspath: *u8, mp4path: *u8) -> i64 {
110 let wpath: *u8 = "/tmp/nx_ts2mp4.sov.elf" as *u8
111 let argv: *i64 = sys_mmap(32) as *i64; argv[0]=wpath as i64; argv[1]=tspath as i64; argv[2]=mp4path as i64; argv[3]=0
112 let envp: *i64 = sys_mmap(16) as *i64; envp[0]=0
113 let pid: i64 = sys_fork()
114 if pid == 0 { sys_execve(wpath, argv, envp); sys_exit(127) } // fork+WAIT: sequential post-processing (NOT detach)
115 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
116 return 0
117}
118
119// best-effort: re-index the gallery video roots so a just-stitched .ts shows up + streams (same idiom as
120// the torrent worker's post-extract refresh). Double-fork + exec the indexer; absent elf = harmless no-op.
121func hg_gallery_refresh() -> i64 {
122 let wpath: *u8 = "/tmp/nx_galx_vidindex.sov.elf" as *u8
123 let argv: *i64 = sys_mmap(16) as *i64; argv[0] = wpath as i64; argv[1] = 0
124 let envp: *i64 = sys_mmap(16) as *i64; envp[0] = 0
125 let pid: i64 = sys_fork()
126 if pid == 0 { sys_execve(wpath, argv, envp); sys_exit(127) } // fork+WAIT: sequential post-processing (NOT detach)
127 let st: *i64 = sys_mmap(16) as *i64; sys_wait4(pid, st, 0)
128 return 0
129}
130
131const HG_MAXENT: i64 = 20000 // max segments / variants
132const HG_PLCAP: i64 = 4194304 // 4 MiB playlist buffer
133const HG_SEGCAP: i64 = 33554432 // 32 MiB per-segment buffer (HLS .ts segments are typically a few MiB)
134
135func main(argc: i64, argv: *i64) -> i64 {
136 if argc < 3 { hg_w("usage: nx_hls_get <m3u8-url> <outpath.ts> [referer] [cookie]\n" as *u8); sys_exit(2); return 2 }
137 let url: *u8 = argv[1] as *u8; let outpath: *u8 = argv[2] as *u8
138 var referer: *u8 = "" as *u8; if argc >= 4 { referer = argv[3] as *u8 }
139 var cookie: *u8 = "" as *u8; if argc >= 5 { cookie = argv[4] as *u8 }
140
141 let store: *TrustStore = trust_store_alloc(400)
142 if nx_pem_trust_load_file("/etc/ssl/certs/ca-certificates.crt\x00", store) <= 0 { hg_w("HLS-GET no-ca\n" as *u8); sys_exit(1); return 1 }
143 let now: i64 = sys_now_realtime_sec()
144
145 let plpath: *u8 = "/tmp/nx_hls_playlist.m3u8\x00"
146 let partpath: *u8 = "/tmp/nx_hls_seg.part\x00"
147 let plbuf: *u8 = sys_mmap(HG_PLCAP)
148 let entries: *i64 = sys_mmap(8 * 3 * HG_MAXENT) as *i64
149 let meta: *i64 = sys_mmap(8 * 8) as *i64
150 let absurl: *u8 = sys_mmap(HG_MAGIC_8192) // resolved absolute URL scratch (reused per segment)
151
152 // ---- fetch the playlist ----
153 if hg_fetch(url, referer, cookie, plpath, store, now) != 0 { hg_w("HLS-GET playlist fetch FAILED\n" as *u8); sys_exit(1); return 1 }
154 var pllen: i64 = hg_readfile(plpath, plbuf, HG_PLCAP)
155 if pllen <= 0 { hg_w("HLS-GET empty playlist\n" as *u8); sys_exit(1); return 1 }
156 var cnt: i64 = nx_hls_parse(plbuf, pllen, entries, HG_MAXENT, meta)
157 if cnt < 0 { hg_w("HLS-GET not an m3u8 (server returned a non-playlist body)\n" as *u8); sys_exit(1); return 1 }
158
159 var baseurl: *u8 = url; var basen: i64 = hg_slen(url) // base for resolving child URIs
160
161 // ---- MASTER playlist: pick the highest-bandwidth variant, fetch its media playlist, re-parse ----
162 if meta[0] == 1 {
163 var best: i64 = 0 - 1; var bestbw: i64 = 0 - 1; var i: i64 = 0
164 while i < cnt { let bw: i64 = entries[i*3+2]; if bw > bestbw { bestbw = bw; best = i } i = i + 1 }
165 if best < 0 { hg_w("HLS-GET master with no variants\n" as *u8); sys_exit(1); return 1 }
166 let vs: i64 = entries[best*3+0]; let vl: i64 = entries[best*3+1]
167 let rn: i64 = nx_hls_resolve_uri(baseurl, basen, (plbuf as i64 + vs) as *u8, vl, absurl, HG_MAGIC_8192)
168 if rn < 0 { hg_w("HLS-GET variant resolve overflow\n" as *u8); sys_exit(1); return 1 }
169 absurl[rn] = 0 as u8
170 hg_w("HLS-GET master -> variant bw=" as *u8); hg_wn(bestbw); hg_w(" " as *u8); hg_w(absurl); hg_w("\n" as *u8)
171 let vbuf: *u8 = sys_mmap(HG_MAGIC_8192); var z: i64 = 0; while z <= rn { vbuf[z] = absurl[z]; z = z + 1 } // stable base (absurl is reused below)
172 baseurl = vbuf; basen = rn
173 if hg_fetch(baseurl, referer, cookie, plpath, store, now) != 0 { hg_w("HLS-GET media playlist fetch FAILED\n" as *u8); sys_exit(1); return 1 }
174 pllen = hg_readfile(plpath, plbuf, HG_PLCAP)
175 cnt = nx_hls_parse(plbuf, pllen, entries, HG_MAXENT, meta)
176 if cnt < 0 { hg_w("HLS-GET variant is not a media playlist\n" as *u8); sys_exit(1); return 1 }
177 }
178
179 if cnt == 0 { hg_w("HLS-GET 0 segments (encrypted-only or empty?)\n" as *u8); sys_exit(1); return 1 }
180 hg_w("HLS-GET media playlist: segments=" as *u8); hg_wn(cnt); hg_w("\n" as *u8)
181
182 // ---- rung 3: #EXT-X-KEY AES-128 -> fetch key + build schedule (else it stays a clear stream) ----
183 let keyuri: *u8 = sys_mmap(HG_MAGIC_4096); let ivexp: *u8 = sys_mmap(16); let has_iv: *i64 = sys_mmap(16) as *i64
184 let sched: *u8 = sys_mmap(176); var enc: i64 = 0; var media_seq: i64 = 0
185 if hg_parse_key(plbuf, pllen, keyuri, HG_MAGIC_4096, ivexp, has_iv) == 1 {
186 let krn: i64 = nx_hls_resolve_uri(baseurl, basen, keyuri, hg_slen(keyuri), absurl, HG_MAGIC_8192)
187 if krn < 0 { hg_w("HLS-GET key URI resolve overflow\n" as *u8); sys_exit(1); return 1 }
188 absurl[krn] = 0 as u8
189 if hg_fetch(absurl, referer, cookie, partpath, store, now) != 0 { hg_w("HLS-GET key fetch FAILED\n" as *u8); sys_exit(1); return 1 }
190 let keybuf: *u8 = sys_mmap(64); let kn: i64 = hg_readfile(partpath, keybuf, 64)
191 if kn < 16 { hg_w("HLS-GET key too short (got " as *u8); hg_wn(kn); hg_w(" bytes)\n" as *u8); sys_exit(1); return 1 }
192 aes128_expand_key(keybuf, sched)
193 media_seq = hg_media_seq(plbuf, pllen)
194 enc = 1
195 hg_w("HLS-GET AES-128 encrypted: key fetched, media_seq=" as *u8); hg_wn(media_seq); hg_w(" explicit_iv=" as *u8); hg_wn(has_iv[0]); hg_w("\n" as *u8)
196 }
197
198 // ---- fetch every segment in order, concat into outpath (.ts appends cleanly) ----
199 let out_fd: i64 = sys_openat_wr(outpath, 0x1a4); if out_fd < 0 { hg_w("HLS-GET cannot open output\n" as *u8); sys_exit(1); return 1 }
200 let segbuf: *u8 = sys_mmap(HG_SEGCAP)
201 var done: i64 = 0; var failed: i64 = 0; var bytes: i64 = 0; var i: i64 = 0
202 while i < cnt {
203 let ss: i64 = entries[i*3+0]; let sl: i64 = entries[i*3+1]
204 let rn: i64 = nx_hls_resolve_uri(baseurl, basen, (plbuf as i64 + ss) as *u8, sl, absurl, HG_MAGIC_8192)
205 if rn < 0 { failed = failed + 1 } else {
206 absurl[rn] = 0 as u8
207 if hg_fetch(absurl, referer, cookie, partpath, store, now) == 0 {
208 var sn: i64 = hg_readfile(partpath, segbuf, HG_SEGCAP)
209 if sn > 0 {
210 if enc == 1 {
211 let seg_iv: *u8 = sys_mmap(16)
212 if has_iv[0] == 1 { var c: i64 = 0; while c < 16 { seg_iv[c] = ivexp[c]; c = c + 1 } } else { hg_seq_iv(media_seq + i, seg_iv) }
213 let dn: i64 = aes128_cbc_decrypt(segbuf, sn, sched, seg_iv, segbuf) // in-place CBC decrypt
214 if dn > 0 { sn = aes128_pkcs7_strip(segbuf, dn) } // strip PKCS7 (per-segment)
215 }
216 hg_append(out_fd, segbuf, sn); bytes = bytes + sn; done = done + 1
217 } else { failed = failed + 1 }
218 } else { failed = failed + 1 }
219 }
220 if (i % 10) == 0 { hg_w(" seg " as *u8); hg_wn(i + 1); hg_w("/" as *u8); hg_wn(cnt); hg_w(" ok=" as *u8); hg_wn(done); hg_w(" bytes=" as *u8); hg_wn(bytes); hg_w("\n" as *u8) }
221 i = i + 1
222 }
223 sys_close(out_fd)
224 if done > 0 { hg_gallery_refresh() } // keep the .ts: the gallery fly-remuxes .ts->fMP4 (nx_ts2fmp4 + MSE, GB-capable) for playback -> no 96MB cap, no duplicate .mp4 entry. (hg_remux/hg_mp4_path kept for an explicit standalone-export feature.)
225 hg_w("HLS-GET done segments=" as *u8); hg_wn(done); hg_w("/" as *u8); hg_wn(cnt); hg_w(" failed=" as *u8); hg_wn(failed); hg_w(" bytes=" as *u8); hg_wn(bytes); hg_w(" -> " as *u8); hg_w(outpath); hg_w("\n" as *u8)
226 if done == cnt { hg_w("HLS-GET verdict=GREEN COMPLETE\n" as *u8); sys_exit(0); return 0 }
227 if done > 0 { hg_w("HLS-GET verdict=PARTIAL (some segments failed -- may be AES-encrypted [rung 3] or gated)\n" as *u8); sys_exit(0); return 0 }
228 hg_w("HLS-GET verdict=RED no-segments-fetched\n" as *u8); sys_exit(1); return 1
229}