nx_galx_durindex.nx source
↩ module page · 144 lines · 7674 B
1// nx_galx_durindex.nx -- build the per-recording DURATION index so the gallery can SORT BY RUNTIME for
2// EVERY format. IN-PROCESS extraction (no per-video fork): for each path it reads the header window and
3// derives the duration directly -- mp4/mov via moov->mvhd, mkv/webm via EBML Duration (nx_vid_dur_lib),
4// MPEG-TS via first/last PCR. Writes "size=<n> first_pcr90=<n> last_pcr90=<n> dur_ms=<n>" per line, line N =
5// recording N, then nx_galx_sortindex's reader parses dur_ms=. SEQUENTIAL, RESUMABLE (skips already-written
6// lines), IDEMPOTENT. (The old fork-nx_ts_dur design silently captured ZERO on this NAS volume; in-process
7// direct writes fix it.) usage: nx_galx_durindex <vid_paths.tsv> <unused> <out_dur.raw> [limit=0]
8// license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
11import "nx_vid_dur_lib.nx" // vd_sniff / vd_mp4_dur / vd_mkv_dur (gated container parsers)
12const K_MAGIC_8589934592: i64 = 8589934592
13const K_MAGIC_2048: i64 = 2048
14
15const WIN: i64 = 8388608
16
17func di_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
19// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
20// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
21// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
22func di_n(v: i64) -> i64 { nxi_out(v); return 0 }
23func di_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8) { let c: i64=s[i] as i64; if c<48 { return v } if c>57 { return v } v=v*10+(c-48); i=i+1 } return v }
24func di_bcat(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
25func di_bn(buf: *u8, off: i64, v: i64) -> i64 {
26 if v == 0 { buf[off] = 48 as u8; return off + 1 }
27 var m: i64 = v; var o: i64 = off
28 if m < 0 { buf[o] = 45 as u8; o = o + 1; m = 0 - m }
29 let t: *u8 = sys_mmap(28); var k: i64 = 0
30 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
31 var j: i64 = k; while j > 0 { j = j - 1; buf[o] = t[j]; o = o + 1 }
32 return o
33}
34// lines already written to out (the resume point); 0 if absent.
35func di_done_lines(path: *u8) -> i64 {
36 let szp: *i64 = sys_mmap(16) as *i64
37 let b: *u8 = sys_read_file(path, szp)
38 if (b as i64)==0 { return 0 }
39 let n: i64 = szp[0]; var c: i64=0; var i: i64=0
40 while i<n { if b[i]==(10 as u8) { c=c+1 } i=i+1 }
41 return c
42}
43func read_window(fd: i64, buf: *u8, cap: i64) -> i64 {
44 var got: i64 = 0; var go: i64 = 1
45 while go==1 { if got>=cap { go=0 } else { let r: i64=sys_read(fd, ((buf as i64)+got) as *u8, cap-got); if r<=0 { go=0 } else { got=got+r } } }
46 return got
47}
48// MPEG-TS: scan buf for PCRs (afc>=2, PCR_flag); set fp[0] to first, lp[0] to last.
49func scan_pcr(buf: *u8, n: i64, fp: *i64, lp: *i64) -> i64 {
50 var i: i64 = 0
51 while i + 188 <= n {
52 if buf[i] != (0x47 as u8) { i = i + 1 } else {
53 let afc: i64 = ((buf[i+3] as i64)>>4)&3
54 if afc>=2 { let afl: i64 = buf[i+4] as i64
55 if afl > 0 { let flags: i64 = buf[i+5] as i64
56 if ((flags>>4)&1)==1 {
57 let base: i64 = ((buf[i+6] as i64)<<25)|((buf[i+7] as i64)<<17)|((buf[i+8] as i64)<<9)|((buf[i+9] as i64)<<1)|((buf[i+10] as i64)>>7)
58 if fp[0] < 0 { fp[0] = base } lp[0] = base
59 } } }
60 i = i + 188
61 }
62 }
63 return 0
64}
65// derive duration (ms) for one video into `line` ("size=.. dur_ms=..\n"); returns line length. fp/lp = scratch.
66func extract_dur(path: *u8, buf: *u8, line: *u8, fp: *i64, lp: *i64) -> i64 {
67 var size: i64 = 0; var dms: i64 = 0
68 let fd: i64 = sys_openat_rd(path)
69 if fd >= 0 {
70 size = sys_lseek(fd, 0, 2)
71 sys_lseek(fd, 0, 0)
72 let g1: i64 = read_window(fd, buf, WIN)
73 let sf: i64 = vd_sniff(buf, g1)
74 if sf == 1 {
75 dms = vd_mp4_dur(buf, g1)
76 if dms < 0 { var es: i64 = size - WIN; if es < 0 { es = 0 } sys_lseek(fd, es, 0); let ge: i64 = read_window(fd, buf, WIN); dms = vd_mp4_dur(buf, ge) }
77 if dms < 0 { dms = 0 }
78 } else { if sf == 2 {
79 dms = vd_mkv_dur(buf, g1); if dms < 0 { dms = 0 }
80 } else {
81 fp[0] = 0 - 1; lp[0] = 0 - 1; scan_pcr(buf, g1, fp, lp); let first: i64 = fp[0]
82 fp[0] = 0 - 1; lp[0] = 0 - 1
83 var es: i64 = size - WIN; if es < 0 { es = 0 }
84 sys_lseek(fd, es, 0); let g2: i64 = read_window(fd, buf, WIN); scan_pcr(buf, g2, fp, lp); let last: i64 = lp[0]
85 if first >= 0 { if last >= 0 { var d90: i64 = last - first; if d90 < 0 { d90 = d90 + K_MAGIC_8589934592 } dms = d90 / 90 } }
86 } }
87 sys_close(fd)
88 }
89 var o: i64 = di_bcat(line, 0, "size=" as *u8); o = di_bn(line, o, size)
90 o = di_bcat(line, o, " first_pcr90=0 last_pcr90=0 dur_ms=" as *u8); o = di_bn(line, o, dms)
91 line[o] = 10 as u8; o = o + 1
92 return o
93}
94
95func main(argc: i64, argv: *i64) -> i64 {
96 if argc<4 { di_p("usage: nx_galx_durindex <vid_paths.tsv> <unused> <out_dur.raw> [limit]\n" as *u8); return 2 }
97 let vidpaths: *u8 = argv[1] as *u8
98 let outp: *u8 = argv[3] as *u8
99 var limit: i64 = 0
100 if argc>4 { limit = di_atoi(argv[4] as *u8) }
101 let szp: *i64 = sys_mmap(16) as *i64
102 let b: *u8 = sys_read_file(vidpaths, szp)
103 if (b as i64)==0 { di_p("durindex: cannot read vid_paths\n" as *u8); return 1 }
104 let sz: i64 = szp[0]
105 let resume: i64 = di_done_lines(outp)
106 di_p("durindex(in-proc): resume from id=" as *u8); di_n(resume); di_p("\n" as *u8)
107 // single-instance via a SEPARATE lock file (locking galx_dur.raw itself is mandatory on this NAS volume
108 // and blocks writes -- never lock the output we write to).
109 let lockfd: i64 = sys_openat_wr("knowledge/status/.galx_dur.lock" as *u8, 0x1a4)
110 if lockfd >= 0 { if sys_flock(lockfd, 6) != 0 { di_p("durindex: another instance holds the lock -> exit (resumable)\n" as *u8); return 0 } }
111 let outfd: i64 = sys_openat_append(outp, 0x1a4)
112 if outfd<0 { di_p("durindex: cannot open out\n" as *u8); return 1 }
113 let buf: *u8 = sys_mmap(WIN + 16)
114 let line: *u8 = sys_mmap(256)
115 let path: *u8 = sys_mmap(K_MAGIC_2048)
116 let fp: *i64 = sys_mmap(16) as *i64
117 let lp: *i64 = sys_mmap(16) as *i64
118 var id: i64=0; var done: i64=0; var stop: i64=0
119 var i: i64=0; var ls: i64=0
120 while i<=sz {
121 if stop==1 { i=sz+1 } else {
122 var nl: i64=0; if i==sz { nl=1 } else { if b[i]==(10 as u8) { nl=1 } }
123 if nl==1 {
124 if i>ls {
125 if id>=resume {
126 var ce: i64=i; if ce>ls { if b[ce-1]==(13 as u8) { ce=ce-1 } }
127 var o: i64=0; var k: i64=ls; while k<ce { path[o]=b[k]; o=o+1; k=k+1 } path[o]=0 as u8
128 let ll: i64 = extract_dur(path, buf, line, fp, lp)
129 sys_write(outfd, line, ll)
130 done=done+1
131 if done%500==0 { di_p(" indexed " as *u8); di_n(done); di_p(" (id " as *u8); di_n(id); di_p(")\n" as *u8) }
132 if limit>0 { if done>=limit { stop=1 } }
133 }
134 id=id+1
135 }
136 ls=i+1
137 }
138 i=i+1
139 }
140 }
141 sys_close(outfd)
142 di_p("DURINDEX DONE indexed=" as *u8); di_n(done); di_p(" next_id=" as *u8); di_n(resume+done); di_p("\n" as *u8)
143 sys_exit(0); return 0
144}