nx_ts_marks.nx source
↩ module page · 374 lines · 16610 B
1// nx_ts_marks.nx -- sovereign CLIP-ANALYSIS markers for a media asset (the first "clip analysis" payload
2// the operator asked for: DEAD-AIR/silence, extensible to scene/black-frame/behavior). Pure + gate-able.
3//
4// DEAD-AIR by DEMUX signal (works on ARBITRARY streaming content -- no decode, which is limited to our own
5// baseline test bitstreams): silence makes an audio encoder emit far fewer bytes. We bin audio payload bytes
6// into fixed time windows during the single TS scan, then a bin is "silent" if it holds < FRAC of the asset's
7// OWN median/mean bin (bitrate-AGNOSTIC by construction -- a 96k and a 320k stream both self-calibrate). Runs
8// of silent bins >= MIN_DEADAIR_MS become dead-air markers. The threshold is RELATIVE, so no magic byte count.
9//
10// NXMK sidecar section (appended AFTER the NXVI keyframe table; v2 readers read only up to nkf keyframes and
11// IGNORE this tail -> BACK-COMPAT, no format-version bump, no gallery seek regression). Marker type: 1=dead-air.
12// "NXMK"(4) nmarks(8) then nmarks * { type(4) start_ms(8) end_ms(8) score(4) }. license_tier: ORIGINAL
13import "nx_syscalls.nx"
14const TM_MAGIC_60000: i64 = 60000
15
16// tunables (research + real-data calibration refine these; the RELATIVE design keeps them robust):
17const TM_BIN_MS: i64 = 500 // audio-byte accumulation window
18const TM_SIL_NUM: i64 = 15 // silent if bin < median * 15/100 (15% of typical loudness)
19const TM_SIL_DEN: i64 = 100
20const TM_MIN_DEADAIR_MS: i64 = 1500 // a run must span >= this to count as dead air (not a normal pause)
21const TM_MOTION_NUM: i64 = 160 // high-motion if a video-byte window > 160% of the median window (P-frame bitrate = motion)
22const TM_MOTION_DEN: i64 = 100
23const TM_MIN_MOTION_MS: i64 = 1500 // a high-motion run must span >= this (sustained activity, not a single spike)
24const TM_SCENE_NUM: i64 = 200 // scene-cut if a window > 200% of the PREVIOUS window (prediction reset at a cut)
25const TM_SCENE_DEN: i64 = 100
26const TM_RHYTHM_NUM: i64 = 60 // rhythmic if the best autocorrelation peak >= 60% of lag-0 energy (strong periodicity)
27const TM_RHYTHM_DEN: i64 = 100
28const TM_MIN_RHYTHM_MS: i64 = 4000
29const TM_STATIC_PCT: i64 = 25 // nothing-happening threshold = this percentile of the file OWN video-byte distribution. Self-calibrating: the measured corpus caps peaks but lets bitrate COLLAPSE when the scene is static (real minima 8-33 percent of median).
30const TM_MIN_STATIC_MS: i64 = 3000 // a static run must span >= this to be worth cutting; shorter is a normal pause
31const TM_STATIC_GUARD_NUM: i64 = 70 // if p25 >= 70 percent of median the stream has NO downward range -- REFUSE rather than mark the whole file dead
32const TM_STATIC_GUARD_DEN: i64 = 100
33const TM_STATIC_NUM: i64 = 50 // a bin is static when it falls below this percent of the median. A FRACTION not a percentile: tm_percentile returns a bucket representative, so the bottom quartile is not strictly below p25 and a percentile threshold could never match.
34const TM_STATIC_DEN: i64 = 100
35const TM_MOTION_PCT: i64 = 90 // high-motion self-calibrates to the file p90. MEASURED 2026-07-31: the fixed 160-percent-of-median sat ABOVE THE MAXIMUM BIN on 8 of 10 real files, so ts_motion_marks could never fire on this corpus. // a rhythmic segment must span >= this (sustained beat, not a couple of pulses)
36
37func tm_w32(b: *u8, o: i64, v: i64) -> i64 { b[o]=((v>>24)&0xff) as u8; b[o+1]=((v>>16)&0xff) as u8; b[o+2]=((v>>8)&0xff) as u8; b[o+3]=(v&0xff) as u8; return o+4 }
38func tm_w64(b: *u8, o: i64, v: i64) -> i64 { o=tm_w32(b,o,(v>>32)&0xffffffff); o=tm_w32(b,o,v&0xffffffff); return o }
39
40// MEDIAN of bins[0..nb) via a bounded 256-bucket histogram over the [0,max] range (O(nb), no sort, no float).
41// Robust to loud outliers (a few loud bins can't drag the silence threshold the way a mean would).
42func tm_median(bins: *i64, nb: i64) -> i64 {
43 if nb <= 0 { return 0 }
44 var mx: i64 = 0; var i: i64 = 0
45 while i < nb { if bins[i] > mx { mx = bins[i] } i = i + 1 }
46 if mx <= 0 { return 0 }
47 let hist: *i64 = sys_mmap(8 * 257) as *i64
48 i = 0
49 while i < nb { var bk: i64 = (bins[i] * 256) / (mx + 1); if bk > 255 { bk = 255 } hist[bk] = hist[bk] + 1; i = i + 1 }
50 let half: i64 = nb / 2
51 var acc: i64 = 0; var mbkt: i64 = 255; var bkt: i64 = 0; var done: i64 = 0
52 while bkt < 256 { if done == 0 { acc = acc + hist[bkt]; if acc > half { mbkt = bkt; done = 1 } } bkt = bkt + 1 }
53 // bucket center back to a representative value
54 return (mbkt * (mx + 1)) / 256
55}
56
57// find dead-air runs. bins = audio bytes per TM_BIN_MS window, nb bins. Fills parallel out arrays; returns count.
58// DIAGNOSTIC (pure, no IO): characterise a bin series so a detector reporting 0
59// can be told apart from a detector that is structurally unable to fire.
60// out[0]=min out[1]=median out[2]=max out[3]=count below silence thresh
61// out[4]=count above motion thresh out[5]=LONGEST RUN above motion thresh
62// out[6]=count of exactly-zero bins
63func tm_bin_stats(bins: *i64, nb: i64, out: *i64) -> i64 {
64 if nb <= 0 { return 0 }
65 let med: i64 = tm_median(bins, nb)
66 var mn: i64 = bins[0]
67 var mx: i64 = bins[0]
68 var zc: i64 = 0
69 var i: i64 = 0
70 while i < nb {
71 if bins[i] < mn { mn = bins[i] }
72 if bins[i] > mx { mx = bins[i] }
73 if bins[i] == 0 { zc = zc + 1 }
74 i = i + 1
75 }
76 var st: i64 = (med * TM_SIL_NUM) / TM_SIL_DEN
77 if st < 1 { st = 1 }
78 var mt: i64 = (med * TM_MOTION_NUM) / TM_MOTION_DEN
79 if mt < 1 { mt = 1 }
80 var below: i64 = 0
81 var above: i64 = 0
82 var run: i64 = 0
83 var best: i64 = 0
84 var j: i64 = 0
85 while j < nb {
86 if bins[j] < st { below = below + 1 }
87 if bins[j] > mt {
88 above = above + 1
89 run = run + 1
90 if run > best { best = run }
91 } else {
92 run = 0
93 }
94 j = j + 1
95 }
96 out[0] = mn
97 out[1] = med
98 out[2] = mx
99 out[3] = below
100 out[4] = above
101 out[5] = best
102 out[6] = zc
103 return 1
104}
105func ts_silence_marks(bins: *i64, nb: i64, bin_ms: i64, mtype: *i64, mstart: *i64, mend: *i64, cap: i64) -> i64 {
106 if nb <= 0 { return 0 }
107 if bin_ms <= 0 { return 0 }
108 let med: i64 = tm_median(bins, nb)
109 var thresh: i64 = (med * TM_SIL_NUM) / TM_SIL_DEN
110 if thresh < 1 { thresh = 1 }
111 let minbins: i64 = TM_MIN_DEADAIR_MS / bin_ms
112 var nm: i64 = 0
113 var insil: i64 = 0
114 var runstart: i64 = 0
115 var i: i64 = 0
116 while i < nb {
117 var sil: i64 = 0
118 if bins[i] < thresh { sil = 1 }
119 if sil == 1 {
120 if insil == 0 { insil = 1; runstart = i }
121 } else {
122 if insil == 1 {
123 insil = 0
124 if (i - runstart) >= minbins {
125 if nm < cap { mtype[nm] = 1; mstart[nm] = runstart * bin_ms; mend[nm] = i * bin_ms; nm = nm + 1 }
126 }
127 }
128 }
129 i = i + 1
130 }
131 if insil == 1 {
132 if (nb - runstart) >= minbins {
133 if nm < cap { mtype[nm] = 1; mstart[nm] = runstart * bin_ms; mend[nm] = nb * bin_ms; nm = nm + 1 }
134 }
135 }
136 return nm
137}
138
139// HIGH-MOTION runs (type 3): video-byte windows above FRAC*median = high activity/motion. Runs of such windows
140// >= TM_MIN_MOTION_MS become markers (bitrate-RELATIVE, so a 2Mbps and a 20Mbps stream both self-calibrate).
141// Percentile of a bin series, same O(n) 256-bucket histogram tm_median uses -- no
142// sort, no copy. p in 0..100. Pure: no IO, no allocation beyond the histogram.
143func tm_percentile(bins: *i64, nb: i64, p: i64) -> i64 {
144 if nb <= 0 { return 0 }
145 if p < 0 { return 0 }
146 if p > 100 { return 0 }
147 var mx: i64 = 0
148 var i: i64 = 0
149 while i < nb { if bins[i] > mx { mx = bins[i] } i = i + 1 }
150 if mx <= 0 { return 0 }
151 let hist: *i64 = sys_mmap(8 * 257) as *i64
152 i = 0
153 while i < nb {
154 var bk: i64 = (bins[i] * 256) / (mx + 1)
155 if bk > 255 { bk = 255 }
156 hist[bk] = hist[bk] + 1
157 i = i + 1
158 }
159 let target: i64 = (nb * p) / 100
160 var acc: i64 = 0
161 var pbkt: i64 = 255
162 var bkt: i64 = 0
163 var done: i64 = 0
164 while bkt < 256 {
165 if done == 0 {
166 acc = acc + hist[bkt]
167 if acc > target { pbkt = bkt; done = 1 }
168 }
169 bkt = bkt + 1
170 }
171 return (pbkt * (mx + 1)) / 256
172}
173// STATIC / dead-air-on-the-VIDEO-side. The audio path (ts_silence_marks) cannot
174// see silence in this corpus: the audio is CBR, so byte count is flat regardless
175// of loudness and only a literal stream dropout ever falls below its threshold.
176// The video byte rate DOES collapse when nothing is happening, so that is where
177// the signal actually lives. Threshold is the file's own p25 -- self-calibrating
178// across 2Mbps and 20Mbps streams alike.
179func ts_static_marks(bins: *i64, nb: i64, bin_ms: i64, mtype: *i64, mstart: *i64, mend: *i64, cap: i64) -> i64 {
180 if nb <= 0 { return 0 }
181 if bin_ms <= 0 { return 0 }
182 let med: i64 = tm_median(bins, nb)
183 if med <= 0 { return 0 }
184 let p25: i64 = tm_percentile(bins, nb, TM_STATIC_PCT)
185 var thresh: i64 = (med * TM_STATIC_NUM) / TM_STATIC_DEN
186 if thresh < 1 { thresh = 1 }
187 // GUARD: a stream with no downward dynamic range would otherwise have its
188 // whole length marked dead. Refuse instead of emitting a confident lie.
189 if p25 * TM_STATIC_GUARD_DEN >= med * TM_STATIC_GUARD_NUM { return 0 }
190 let minbins: i64 = TM_MIN_STATIC_MS / bin_ms
191 var nm: i64 = 0
192 var inst: i64 = 0
193 var rs: i64 = 0
194 var i: i64 = 0
195 while i < nb {
196 var st: i64 = 0
197 if bins[i] < thresh { st = 1 }
198 if st == 1 {
199 if inst == 0 { inst = 1; rs = i }
200 } else {
201 if inst == 1 {
202 inst = 0
203 if (i - rs) >= minbins {
204 if nm < cap { mtype[nm] = 5; mstart[nm] = rs * bin_ms; mend[nm] = i * bin_ms; nm = nm + 1 }
205 }
206 }
207 }
208 i = i + 1
209 }
210 if inst == 1 {
211 if (nb - rs) >= minbins {
212 if nm < cap { mtype[nm] = 5; mstart[nm] = rs * bin_ms; mend[nm] = nb * bin_ms; nm = nm + 1 }
213 }
214 }
215 return nm
216}
217func ts_motion_marks(bins: *i64, nb: i64, bin_ms: i64, mtype: *i64, mstart: *i64, mend: *i64, cap: i64) -> i64 {
218 if nb <= 0 { return 0 }
219 if bin_ms <= 0 { return 0 }
220 let med: i64 = tm_median(bins, nb)
221 var thresh: i64 = tm_percentile(bins, nb, TM_MOTION_PCT)
222 if thresh <= med { thresh = (med * TM_MOTION_NUM) / TM_MOTION_DEN }
223 if thresh < 1 { thresh = 1 }
224 let minbins: i64 = TM_MIN_MOTION_MS / bin_ms
225 var nm: i64 = 0
226 var inhi: i64 = 0
227 var rs: i64 = 0
228 var i: i64 = 0
229 while i < nb {
230 var hi: i64 = 0
231 if bins[i] > thresh { hi = 1 }
232 if hi == 1 {
233 if inhi == 0 { inhi = 1; rs = i }
234 } else {
235 if inhi == 1 {
236 inhi = 0
237 if (i - rs) >= minbins {
238 if nm < cap { mtype[nm] = 3; mstart[nm] = rs * bin_ms; mend[nm] = i * bin_ms; nm = nm + 1 }
239 }
240 }
241 }
242 i = i + 1
243 }
244 if inhi == 1 {
245 if (nb - rs) >= minbins {
246 if nm < cap { mtype[nm] = 3; mstart[nm] = rs * bin_ms; mend[nm] = nb * bin_ms; nm = nm + 1 }
247 }
248 }
249 return nm
250}
251// SCENE-CUT points (type 2): a window whose video-byte volume rises sharply vs the PREVIOUS window (the encoder
252// spends far more bits when inter-prediction breaks at a cut) AND is above median = a scene change. Point markers.
253func ts_scene_marks(bins: *i64, nb: i64, bin_ms: i64, mtype: *i64, mstart: *i64, mend: *i64, cap: i64) -> i64 {
254 if nb <= 1 { return 0 }
255 if bin_ms <= 0 { return 0 }
256 let med: i64 = tm_median(bins, nb)
257 var nm: i64 = 0
258 var i: i64 = 1
259 while i < nb {
260 var cut: i64 = 0
261 if bins[i] > med { if bins[i] > (bins[i-1] * TM_SCENE_NUM) / TM_SCENE_DEN { cut = 1 } }
262 if cut == 1 {
263 if nm < cap { mtype[nm] = 2; mstart[nm] = i * bin_ms; mend[nm] = (i + 1) * bin_ms; nm = nm + 1 }
264 }
265 i = i + 1
266 }
267 return nm
268}
269// RHYTHM / CADENCE (type 4): sustained PERIODIC motion (repeated dance moves/phrases). Integer autocorrelation,
270// mean-subtracted, over a sliding window of the motion bins -- a strong peak at some lag k>=2 means the motion
271// REPEATS every k bins. NO float, NO FFT (same demux-relative idiom). tm_window_rhythmic returns 1 iff the best
272// lag-peak is >= FRAC of the lag-0 (variance) energy. Detects ~1-4s periods at 500ms bins (repeated moves/phrases),
273// NOT sub-bin musical beat -- that needs finer bins (the refinement rung). Helper keeps the 2-deep loop flat.
274func tm_window_rhythmic(bins: *i64, w: i64, W: i64) -> i64 {
275 var sum: i64 = 0
276 var i: i64 = 0
277 while i < W { sum = sum + bins[w+i]; i = i + 1 }
278 let mean: i64 = sum / W
279 var r0: i64 = 0
280 i = 0
281 while i < W { let d: i64 = bins[w+i] - mean; r0 = r0 + d*d; i = i + 1 }
282 if r0 <= 0 { return 0 }
283 var best: i64 = 0
284 let maxlag: i64 = W / 2
285 var k: i64 = 2
286 while k <= maxlag {
287 var rk: i64 = 0
288 var j: i64 = 0
289 while j + k < W { let a: i64 = bins[w+j] - mean; let b: i64 = bins[w+j+k] - mean; rk = rk + a*b; j = j + 1 }
290 if rk > best { best = rk }
291 k = k + 1
292 }
293 if best * TM_RHYTHM_DEN >= r0 * TM_RHYTHM_NUM { return 1 }
294 return 0
295}
296func ts_rhythm_marks(bins: *i64, nb: i64, bin_ms: i64, mtype: *i64, mstart: *i64, mend: *i64, cap: i64) -> i64 {
297 if bin_ms <= 0 { return 0 }
298 let W: i64 = 16
299 if nb < W { return 0 }
300 let S: i64 = 8
301 let minbins: i64 = TM_MIN_RHYTHM_MS / bin_ms
302 var nm: i64 = 0
303 var inr: i64 = 0
304 var rs: i64 = 0
305 var re: i64 = 0
306 var w: i64 = 0
307 while w + W <= nb {
308 var rhy: i64 = tm_window_rhythmic(bins, w, W)
309 if rhy == 1 {
310 if inr == 0 { inr = 1; rs = w }
311 re = w + W
312 } else {
313 if inr == 1 {
314 inr = 0
315 if (re - rs) >= minbins {
316 if nm < cap { mtype[nm] = 4; mstart[nm] = rs * bin_ms; mend[nm] = re * bin_ms; nm = nm + 1 }
317 }
318 }
319 }
320 w = w + S
321 }
322 if inr == 1 {
323 if (re - rs) >= minbins {
324 if nm < cap { mtype[nm] = 4; mstart[nm] = rs * bin_ms; mend[nm] = re * bin_ms; nm = nm + 1 }
325 }
326 }
327 return nm
328}
329// classify a clip from its NXMK markers + duration -> a coarse CONTENT LABEL (the FIRST classification rung;
330// rule-based over the T1 features -- trained fine-grained recognition is T3). Fills out[0..3] = quiet_ms, active_ms,
331// rhythmic_ms, n_scenes; returns label: 0=quiet/static 1=active/dynamic 2=rhythmic/dance-like 3=scene-heavy 4=mixed.
332// Flat priority ladder (no else-nesting; x86-lane safe): rhythm > scene-density > activity > quiet > mixed.
333func ts_classify(mtype: *i64, mstart: *i64, mend: *i64, nmarks: i64, dur_ms: i64, out: *i64) -> i64 {
334 var quiet: i64 = 0; var active: i64 = 0; var rhythmic: i64 = 0; var nscene: i64 = 0
335 var i: i64 = 0
336 while i < nmarks {
337 let t: i64 = mtype[i]
338 let span: i64 = mend[i] - mstart[i]
339 if t == 1 { quiet = quiet + span }
340 if t == 3 { active = active + span }
341 if t == 4 { rhythmic = rhythmic + span }
342 if t == 2 { nscene = nscene + 1 }
343 i = i + 1
344 }
345 out[0] = quiet; out[1] = active; out[2] = rhythmic; out[3] = nscene
346 var label: i64 = 4
347 if dur_ms > 0 {
348 let qf: i64 = (quiet * 100) / dur_ms
349 let af: i64 = (active * 100) / dur_ms
350 let rf: i64 = (rhythmic * 100) / dur_ms
351 let sd: i64 = (nscene * TM_MAGIC_60000) / dur_ms
352 var dec: i64 = 0
353 if dec == 0 { if rf >= 25 { label = 2; dec = 1 } }
354 if dec == 0 { if sd >= 12 { label = 3; dec = 1 } }
355 if dec == 0 { if af >= 40 { label = 1; dec = 1 } }
356 if dec == 0 { if qf >= 40 { label = 0; dec = 1 } }
357 }
358 out[4] = label
359 return label
360}
361// append the NXMK sidecar section at ob[o..]; returns new offset. Safe tail (v2 NXVI readers ignore it).
362func ts_marks_write(ob: *u8, o: i64, mtype: *i64, mstart: *i64, mend: *i64, nm: i64) -> i64 {
363 ob[o]=78 as u8; ob[o+1]=88 as u8; ob[o+2]=77 as u8; ob[o+3]=75 as u8; o = o + 4 // "NXMK"
364 o = tm_w64(ob, o, nm)
365 var i: i64 = 0
366 while i < nm {
367 o = tm_w32(ob, o, mtype[i])
368 o = tm_w64(ob, o, mstart[i])
369 o = tm_w64(ob, o, mend[i])
370 o = tm_w32(ob, o, 0) // score reserved (confidence, filled by later decode-based passes)
371 i = i + 1
372 }
373 return o
374}