nx_yt_transcript.nx source
↩ module page · 274 lines · 16770 B
1// nx_yt_transcript.nx -- fetch a YouTube transcript into knowledge/fetched/, provenance-stamped, sovereign
2// from the first byte: the ANDROID_VR InnerTube player response (nx_yt_innertube) yields the captionTracks
3// baseUrl, the shared TLS-1.3 GET (nx_https_fetch_follow) pulls the timedtext body, and nx_yt_captions_lib
4// parses json3/srv1 into `startMs \t durMs \t text`. The WEB baseUrl is a proven dead end (empty 200, needs a
5// proof-of-origin token); the ANDROID client's baseUrl carries no such requirement.
6// nx_yt_transcript <url-or-id> [lang] -> write knowledge/fetched/yt_<id>.<lang>.transcript.txt
7// nx_yt_transcript measure <url-or-id> -> the decisive route (c): player captionTracks + both formats
8// nx_yt_transcript compare <ours> <theirs.json3> -> referee agreement (segment counts + text permil)
9// Refusals by NAME: REFUSED-NO-CAPTIONS, REFUSED-LANG-ABSENT (langs listed), REFUSED-EMPTY-BODY (the pot case).
10// license_tier: ORIGINAL No hw writes (Rule 26).
11import "nx_syscalls.nx"
12import "nx_yt_captions_lib.nx"
13import "nx_sha256.nx"
14
15func pw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
16func pn(v: i64) -> i64 {
17 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
18 var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
19 let d: *u8 = sys_mmap(24); var k: i64 = 0
20 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
21 let o: *u8 = sys_mmap(24); var i: i64 = 0
22 while i < k { o[i] = d[k - 1 - i]; i = i + 1 }
23 sys_write(1, o, k); return 0
24}
25func t_cat(dst: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[o] = s[i]; o = o + 1; i = i + 1 } return o }
26func t_hexnib(v: i64) -> i64 { if v < 10 { return 48 + v } return 87 + v }
27// emit "# <key>=" -- the '#' is written as byte 35 because the nx_cc lexer does not allow '#' in a string literal.
28func t_hk(doc: *u8, d0: i64, key: *u8) -> i64 {
29 var d: i64 = d0
30 doc[d] = 35 as u8; d = d + 1
31 doc[d] = 32 as u8; d = d + 1
32 var i: i64 = 0; while key[i] != (0 as u8) { doc[d] = key[i]; d = d + 1; i = i + 1 }
33 doc[d] = 61 as u8; d = d + 1
34 return d
35}
36
37// parse the &lang=XX value out of a timedtext baseUrl (NUL-terminated). Writes code, returns len.
38func t_url_lang(url: *u8, out: *u8, cap: i64) -> i64 {
39 var ul: i64 = 0; while url[ul] != (0 as u8) { ul = ul + 1 }
40 let p: i64 = ms_find(url, ul, "lang=" as *u8, 5, 0)
41 if p < 0 { out[0] = 0 as u8; return 0 }
42 var i: i64 = p + 5; var o: i64 = 0
43 while i < ul { let c: i64 = url[i]&0xff; if c == 38 { break } if o < cap - 1 { out[o] = c as u8; o = o + 1 } i = i + 1 }
44 out[o] = 0 as u8; return o
45}
46// normalise the input to a watch URL (a bare 11-char id -> https://www.youtube.com/watch?v=<id>).
47func t_norm_url(inp: *u8, out: *u8, cap: i64) -> i64 {
48 var n: i64 = 0; while inp[n] != (0 as u8) { n = n + 1 }
49 if ms_find(inp, n, "youtu" as *u8, 5, 0) >= 0 { var i: i64 = 0; while i < n { out[i] = inp[i]; i = i + 1 } out[n] = 0 as u8; return n }
50 var o: i64 = t_cat(out, 0, "https://www.youtube.com/watch?v=" as *u8)
51 var i: i64 = 0; while i < n { out[o] = inp[i]; o = o + 1; i = i + 1 }
52 out[o] = 0 as u8; return o
53}
54// title from videoDetails.
55func t_title(pj: *u8, jlen: i64, out: *u8, cap: i64) -> i64 {
56 let vd: i64 = ms_find(pj, jlen, "\"videoDetails\"" as *u8, 14, 0)
57 var from: i64 = 0; if vd >= 0 { from = vd }
58 let tp: i64 = ms_find(pj, jlen, "\"title\":\"" as *u8, 9, from)
59 if tp < 0 { out[0] = 0 as u8; return 0 }
60 let eidx: *i64 = sys_mmap(16) as *i64
61 let l: i64 = ytc_copy_jsonstr(pj, tp + 9, jlen, out, 0, cap, 1, eidx)
62 out[l] = 0 as u8; return l
63}
64
65// append a provenance row in nx_research_fetch's exact shape to the shared journal.
66// prov|<epoch>|<source-url>|<mirror>|<sha256hex>|<bytes>|<hops>
67func t_prov(src_url: *u8, mirror: *u8, body: *u8, n: i64) -> i64 {
68 let digest: *u8 = sys_mmap(32)
69 sha256_digest(body, n, digest)
70 let row: *u8 = sys_mmap(4096)
71 var p: i64 = t_cat(row, 0, "prov|" as *u8)
72 p = ytc_catn(row, p, sys_now_realtime_sec(), 4096)
73 row[p] = 124 as u8; p = p + 1
74 p = t_cat(row, p, src_url)
75 row[p] = 124 as u8; p = p + 1
76 p = t_cat(row, p, mirror)
77 row[p] = 124 as u8; p = p + 1
78 var i: i64 = 0
79 while i < 32 { let b: i64 = digest[i]&0xff; row[p] = t_hexnib((b >> 4)&0xf) as u8; row[p+1] = t_hexnib(b & 0xf) as u8; p = p + 2; i = i + 1 }
80 row[p] = 124 as u8; p = p + 1
81 p = ytc_catn(row, p, n, 4096)
82 row[p] = 124 as u8; p = p + 1
83 p = ytc_catn(row, p, 0, 4096) // hops: the caption GET is a single origin fetch
84 row[p] = 10 as u8; p = p + 1
85 let fd: i64 = sys_openat_append("knowledge/status/fetch_provenance.jrnl" as *u8, MODE_0644)
86 if fd < 0 { pw("YT-PROVENANCE-UNWRITTEN: cannot append the provenance journal -- this mirror reads UNPROVEN to the refs gate\n" as *u8); return 0 - 1 }
87 sys_write(fd, row, p); sys_close(fd)
88 pw("PROVENANCE "); sys_write(1, row, p)
89 return 0
90}
91
92// fetch the player response for a normalised url; prints diagnostics; returns bytes (player buffer via out_pp).
93func t_get_player(url: *u8, store: *TrustStore, out_pp: *i64) -> i64 {
94 let player: *u8 = sys_mmap(YTC_PLAYER_CAP)
95 var ul: i64 = 0; while url[ul] != (0 as u8) { ul = ul + 1 }
96 let pn2: i64 = ytc_player(url, ul, store, player, YTC_PLAYER_CAP)
97 out_pp[0] = player as i64
98 return pn2
99}
100
101func do_measure(idurl: *u8, store: *TrustStore) -> i64 {
102 let url: *u8 = sys_mmap(YTC_URL_CAP); t_norm_url(idurl, url, YTC_URL_CAP)
103 pw("== nx_yt_transcript measure ==\nurl: "); pw(url); pw("\n")
104 let pp: *i64 = sys_mmap(16) as *i64
105 let pnn: i64 = t_get_player(url, store, pp)
106 pw("player: bytes="); pn(pnn); pw("\n")
107 if pnn <= 0 { pw("YT-MEASURE: player fetch FAILED (yt_innertube code above)\n"); return 1 }
108 let player: *u8 = pp[0] as *u8
109 let arr: *u8 = sys_mmap(YTC_PLAYER_CAP)
110 let alen: i64 = ytc_ct_array(player, pnn, arr, YTC_PLAYER_CAP)
111 if alen < 0 { pw("captionTracks: ABSENT in the ANDROID_VR player response -> REFUSED-NO-CAPTIONS would fire\n"); pw("---player-response (raw)---\n"); sys_write(1, player, pnn); pw("\n---end player-response---\n"); return 0 }
112 let langs: *u8 = sys_mmap(4096); let nl: i64 = ytc_langs(arr, alen, langs, 4096)
113 pw("captionTracks: PRESENT n_langs="); pn(nl); pw(" langs="); pw(langs); pw("\n")
114 let baseurl: *u8 = sys_mmap(YTC_URL_CAP); let kind: *u8 = sys_mmap(64)
115 let pick: i64 = ytc_pick(arr, alen, "" as *u8, 0, baseurl, YTC_URL_CAP, kind, 64)
116 if pick != 1 { pw("YT-MEASURE: pick(first) returned "); pn(pick); pw("\n"); return 0 }
117 pw("track: kind="); pw(kind); pw(" baseUrl="); pw(baseurl); pw("\n")
118 let status: *i64 = sys_mmap(8) as *i64
119 // route (c) as-is (srv1/xml default)
120 let body: *u8 = sys_mmap(YTC_BODY_CAP)
121 let bn: i64 = ytc_fetch(baseurl, store, body, YTC_BODY_CAP, status)
122 pw("route(c) baseUrl as-is: status="); pn(status[0]); pw(" bytes="); pn(bn); pw(" class="); pn(ytc_classify_body(body, bn)); pw("\n")
123 // route (c) + &fmt=json3
124 let j3: *u8 = sys_mmap(YTC_URL_CAP); var o: i64 = 0; while baseurl[o] != (0 as u8) { j3[o] = baseurl[o]; o = o + 1 }
125 o = t_cat(j3, o, "&fmt=json3" as *u8); j3[o] = 0 as u8
126 let body2: *u8 = sys_mmap(YTC_BODY_CAP)
127 let bn2: i64 = ytc_fetch(j3, store, body2, YTC_BODY_CAP, status)
128 pw("route(c) &fmt=json3: status="); pn(status[0]); pw(" bytes="); pn(bn2); pw(" class="); pn(ytc_classify_body(body2, bn2)); pw("\n")
129 if bn2 > 0 { let ns: *i64 = sys_mmap(8) as *i64; let tsv: *u8 = sys_mmap(YTC_BODY_CAP); ytc_json3_to_tsv(body2, bn2, tsv, YTC_BODY_CAP, ns); pw("json3 parsed segments="); pn(ns[0]); pw("\n") }
130 return 0
131}
132
133func do_fetch(idurl: *u8, want: *u8, store: *TrustStore) -> i64 {
134 let url: *u8 = sys_mmap(YTC_URL_CAP); t_norm_url(idurl, url, YTC_URL_CAP)
135 // video id (for the filename): reuse yt_video_id from the innertube lib
136 let vid: *u8 = sys_mmap(64); var ul: i64 = 0; while url[ul] != (0 as u8) { ul = ul + 1 }
137 yt_video_id(url, ul, vid, 64)
138 let pp: *i64 = sys_mmap(16) as *i64
139 let pnn: i64 = t_get_player(url, store, pp)
140 if pnn <= 0 { pw("YT-TRANSCRIPT REFUSED: player fetch failed code="); pn(pnn); pw("\n"); return 1 }
141 let player: *u8 = pp[0] as *u8
142 let arr: *u8 = sys_mmap(YTC_PLAYER_CAP)
143 let alen: i64 = ytc_ct_array(player, pnn, arr, YTC_PLAYER_CAP)
144 if alen < 0 { pw("REFUSED-NO-CAPTIONS: the ANDROID_VR player response carries no captionTracks for "); pw(vid); pw("\n"); return 2 }
145 var wlen: i64 = 0; while want[wlen] != (0 as u8) { wlen = wlen + 1 }
146 let baseurl: *u8 = sys_mmap(YTC_URL_CAP); let kind: *u8 = sys_mmap(64)
147 let pick: i64 = ytc_pick(arr, alen, want, wlen, baseurl, YTC_URL_CAP, kind, 64)
148 if pick == 0 { pw("REFUSED-NO-CAPTIONS: captionTracks empty for "); pw(vid); pw("\n"); return 2 }
149 if pick != 1 {
150 let langs: *u8 = sys_mmap(4096); ytc_langs(arr, alen, langs, 4096)
151 pw("REFUSED-LANG-ABSENT: requested lang '"); pw(want); pw("' not among ["); pw(langs); pw("]\n"); return 3
152 }
153 // fetch json3 first (cleaner), fall back to srv1
154 let j3: *u8 = sys_mmap(YTC_URL_CAP); var o: i64 = 0; while baseurl[o] != (0 as u8) { j3[o] = baseurl[o]; o = o + 1 }
155 o = t_cat(j3, o, "&fmt=json3" as *u8); j3[o] = 0 as u8
156 let status: *i64 = sys_mmap(8) as *i64
157 let body: *u8 = sys_mmap(YTC_BODY_CAP)
158 var bn: i64 = ytc_fetch(j3, store, body, YTC_BODY_CAP, status)
159 var src_url: *u8 = j3
160 var fmt: *u8 = "json3" as *u8
161 var cls: i64 = ytc_classify_body(body, bn)
162 if cls == 2 { // empty json3 -> try srv1
163 bn = ytc_fetch(baseurl, store, body, YTC_BODY_CAP, status)
164 src_url = baseurl; fmt = "srv1" as *u8; cls = ytc_classify_body(body, bn)
165 }
166 if bn >= YTC_BODY_CAP - 1 { pw("REFUSED-BODY-FILLED-RESERVE: caption body filled the "); pn(YTC_BODY_CAP); pw("-byte reserve; refusing a possibly-truncated transcript\n"); return 5 }
167 if cls == 2 { pw("REFUSED-EMPTY-BODY: the timedtext endpoint returned status "); pn(status[0]); pw(" with an empty body for "); pw(vid); pw(" (proof-of-origin token required)\n"); return 4 }
168 let nsp: *i64 = sys_mmap(8) as *i64
169 let tsv: *u8 = sys_mmap(YTC_BODY_CAP)
170 var tlen: i64 = 0
171 if cls == 0 { tlen = ytc_json3_to_tsv(body, bn, tsv, YTC_BODY_CAP, nsp); fmt = "json3" as *u8 }
172 else { tlen = ytc_xml_to_tsv(body, bn, tsv, YTC_BODY_CAP, nsp); fmt = "srv1" as *u8 }
173 if nsp[0] <= 0 { pw("REFUSED-EMPTY-BODY: 0 usable segments parsed for "); pw(vid); pw("\n"); return 4 }
174 // language for the filename (authoritative: the baseUrl's own lang=)
175 let lang: *u8 = sys_mmap(64); let ll: i64 = t_url_lang(baseurl, lang, 64)
176 if ll == 0 { if wlen > 0 { var q: i64 = 0; while want[q] != (0 as u8) { lang[q] = want[q]; q = q + 1 } lang[wlen] = 0 as u8 } else { lang[0]=117 as u8; lang[1]=110 as u8; lang[2]=100 as u8; lang[3]=0 as u8 } }
177 let title: *u8 = sys_mmap(4096); t_title(player, pnn, title, 4096)
178 // build the transcript file: header (#) then rows.
179 let doc: *u8 = sys_mmap(YTC_BODY_CAP); var d: i64 = 0
180 doc[d] = 35 as u8; d = d + 1; doc[d] = 32 as u8; d = d + 1
181 d = t_cat(doc, d, "nx_yt_transcript v1" as *u8); doc[d] = 10 as u8; d = d + 1
182 d = t_hk(doc, d, "video_id" as *u8); d = t_cat(doc, d, vid); doc[d] = 10 as u8; d = d + 1
183 d = t_hk(doc, d, "title" as *u8); d = t_cat(doc, d, title); doc[d] = 10 as u8; d = d + 1
184 d = t_hk(doc, d, "lang" as *u8); d = t_cat(doc, d, lang); doc[d] = 10 as u8; d = d + 1
185 d = t_hk(doc, d, "kind" as *u8); d = t_cat(doc, d, kind); doc[d] = 10 as u8; d = d + 1
186 d = t_hk(doc, d, "format" as *u8); d = t_cat(doc, d, fmt); doc[d] = 10 as u8; d = d + 1
187 d = t_hk(doc, d, "segments" as *u8); d = ytc_catn(doc, d, nsp[0], YTC_BODY_CAP); doc[d] = 10 as u8; d = d + 1
188 // the CITED source is the watch page (durable); the caption endpoint below is a signed URL that expires in hours,
189 // so it stays in the header as the fetch detail and is NEVER the provenance url (refs gate: PROV-MISMATCH 2026-09-19)
190 let vurl: *u8 = sys_mmap(512); var vu: i64 = t_cat(vurl, 0, "https://www.youtube.com/watch?v=" as *u8); vu = t_cat(vurl, vu, vid); vurl[vu] = 0 as u8
191 d = t_hk(doc, d, "video_url" as *u8); d = t_cat(doc, d, vurl); doc[d] = 10 as u8; d = d + 1
192 d = t_hk(doc, d, "source_url" as *u8); d = t_cat(doc, d, src_url); doc[d] = 10 as u8; d = d + 1
193 d = t_hk(doc, d, "fetched_at" as *u8); d = ytc_catn(doc, d, sys_now_realtime_sec(), YTC_BODY_CAP); doc[d] = 10 as u8; d = d + 1
194 var k: i64 = 0
195 while k < tlen { doc[d] = tsv[k]; d = d + 1; k = k + 1 }
196 // path knowledge/fetched/yt_<id>.<lang>.transcript.txt
197 let path: *u8 = sys_mmap(512); var pi: i64 = t_cat(path, 0, "knowledge/fetched/yt_" as *u8)
198 pi = t_cat(path, pi, vid); path[pi] = 46 as u8; pi = pi + 1; pi = t_cat(path, pi, lang)
199 pi = t_cat(path, pi, ".transcript.txt" as *u8); path[pi] = 0 as u8
200 let fd: i64 = sys_openat_wr(path, MODE_0644)
201 if fd < 0 { pw("YT-TRANSCRIPT REFUSED: cannot open "); pw(path); pw(" for write\n"); return 1 }
202 var wr: i64 = 0
203 while wr < d { let w: i64 = sys_write(fd, ((doc as i64) + wr) as *u8, d - wr); if w <= 0 { break } wr = wr + w }
204 sys_close(fd)
205 t_prov(vurl, path, doc, d)
206 pw("YT-TRANSCRIPT OK video="); pw(vid); pw(" lang="); pw(lang); pw(" kind="); pw(kind); pw(" fmt="); pw(fmt)
207 pw(" segments="); pn(nsp[0]); pw(" bytes="); pn(d); pw(" file="); pw(path); pw("\n")
208 return 0
209}
210
211func do_compare(ours_path: *u8, theirs_path: *u8) -> i64 {
212 let ol: *i64 = sys_mmap(16) as *i64
213 let ours: *u8 = sys_read_file(ours_path, ol)
214 if (ours as i64) <= 0 { pw("YT-REFEREE: cannot read ours "); pw(ours_path); pw("\n"); return 1 }
215 let tl: *i64 = sys_mmap(16) as *i64
216 let theirs: *u8 = sys_read_file(theirs_path, tl)
217 if (theirs as i64) <= 0 { pw("YT-REFEREE: cannot read theirs "); pw(theirs_path); pw("\n"); return 1 }
218 let ns: *i64 = sys_mmap(8) as *i64
219 let tsv2: *u8 = sys_mmap(YTC_BODY_CAP)
220 let l2: i64 = ytc_json3_to_tsv(theirs, tl[0], tsv2, YTC_BODY_CAP, ns)
221 let theirs_n: i64 = ns[0]
222 // count ours data rows
223 let ours_n: i64 = ytc_row_count_ours(ours, ol[0])
224 var mn: i64 = ours_n; if theirs_n < mn { mn = theirs_n }
225 let a: *u8 = sys_mmap(65536); let b: *u8 = sys_mmap(65536)
226 let na: *u8 = sys_mmap(65536); let nb: *u8 = sys_mmap(65536)
227 var matched: i64 = 0; var i: i64 = 0
228 while i < mn {
229 let la: i64 = ytc_row_text(ours, ol[0], i, a, 65536)
230 let lb: i64 = ytc_row_text(tsv2, l2, i, b, 65536)
231 if la >= 0 { if lb >= 0 {
232 let xa: i64 = ytc_norm_ws(a, la, na, 65536)
233 let xb: i64 = ytc_norm_ws(b, lb, nb, 65536)
234 var eq: i64 = 1
235 if xa != xb { eq = 0 } else { var q: i64 = 0; while q < xa { if na[q] != nb[q] { eq = 0; break } q = q + 1 } }
236 if eq == 1 { matched = matched + 1 }
237 } }
238 i = i + 1
239 }
240 var permil: i64 = 0; if theirs_n > 0 { permil = matched * 1000 / theirs_n }
241 pw("YT-REFEREE ours_segments="); pn(ours_n); pw(" theirs_segments="); pn(theirs_n)
242 pw(" matched="); pn(matched); pw(" agreement_permil="); pn(permil); pw("\n")
243 return 0
244}
245// count data rows in an ours transcript (rows whose first byte is a digit).
246func ytc_row_count_ours(buf: *u8, len: i64) -> i64 {
247 var i: i64 = 0; var n: i64 = 0
248 while i < len {
249 let c0: i64 = buf[i]&0xff
250 if c0 >= 48 { if c0 <= 57 { n = n + 1 } }
251 while i < len { if (buf[i]&0xff) == 10 { break } i = i + 1 }
252 i = i + 1
253 }
254 return n
255}
256
257func main(argc: i64, argv: *i64) -> i64 {
258 if argc < 2 { pw("usage: nx_yt_transcript <url-or-id> [lang] | measure <url-or-id> | compare <ours> <theirs.json3>\n"); return 2 }
259 let a1: *u8 = argv[1] as *u8
260 // verb dispatch
261 if ms_find(a1, 7, "measure" as *u8, 7, 0) == 0 { if a1[7] == (0 as u8) {
262 if argc < 3 { pw("usage: nx_yt_transcript measure <url-or-id>\n"); return 2 }
263 let r: i64 = ytc_load_store(); if r <= 0 { pw("YT: certdata load failed (run from nxc2 root)\n"); return 1 }
264 return do_measure(argv[2] as *u8, r as *TrustStore)
265 } }
266 if ms_find(a1, 7, "compare" as *u8, 7, 0) == 0 { if a1[7] == (0 as u8) {
267 if argc < 4 { pw("usage: nx_yt_transcript compare <ours> <theirs.json3>\n"); return 2 }
268 return do_compare(argv[2] as *u8, argv[3] as *u8)
269 } }
270 let r: i64 = ytc_load_store(); if r <= 0 { pw("YT: certdata load failed (run from nxc2 root)\n"); return 1 }
271 var lang: *u8 = "" as *u8
272 if argc >= 3 { lang = argv[2] as *u8 }
273 return do_fetch(a1, lang, r as *TrustStore)
274}