code wiki / _hdl_build / nx_vcodec_i16ceil_gate.nx
nx_vcodec_i16ceil_gate.nx source
↩ module page · 307 lines · 15520 B
1import "nx_gate_base.nx"
2// nx_vcodec_i16ceil_gate.nx -- I_16x16 flat-intra CEILING PROBE (measure BEFORE building vcv-9). Question: do
3// FLAT keyframe MBs (the lv<=1 + vc_mb_t8_ok population that today codes as 4x 8x8 rich-intra) get CHEAPER as a
4// single 16x16-predicted MB (one mode, residual through the SAME 8x8 transform+entropy)? All bits are REAL: the
5// I16 variant codes through the live vc_enc_sub8 (DCT8+quant+RDOQ+range-coder, shared adaptive state), so the
6// reported bytes are actual coder output, not estimates. Method, per qp on ALL-INTRA /tmp/seq.y4m:
7// A = the real vc_enc_frame_packed_rct8 (luma), total bits + MSE.
8// B0 = OUR MIRROR of that frame loop, I16 disabled -> MUST equal A bit-for-bit (fidelity gate; if this is RED
9// the probe is invalid by construction and reports nothing).
10// B = the mirror with flat MBs coded I16: 16-edge gather (vc_gather_edges n=16), best-of-9 vc_pred_rich at
11// n=16 (a SUPERSET of H.264's 4 I16 modes), 4-bit mode, 4x vc_enc_sub8 on the quadrant residuals, recon
12// add-back. NO hierarchical-DC second transform (conservative: the measured win is a LOWER bound).
13// Verdict data: B vs A bytes (per-mille) + MSE + flat-MB share. license: ORIGINAL
14import "nx_syscalls.nx"
15import "nx_video_codec_wasm.nx"
16
17const SW: i64 = 352
18const SH: i64 = 288
19const NF: i64 = 48
20
21func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw("
22" as *u8); return ok }
23func gn(v: i64) -> i64 {
24 let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m}
25 let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}
26 var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
27
28func frames(yuv: *u8, hi: i64, total: i64, fp: *i64, fdata: i64) -> i64 {
29 var nf: i64=0; var p: i64=hi
30 while nf < NF { if p>=total { break }
31 var q: i64=p; while q<total { if (yuv[q]&0xff)==10 { break } q=q+1 } q=q+1
32 if q+fdata>total { break } fp[nf]=(yuv as i64)+q; nf=nf+1; p=q+fdata }
33 return nf }
34
35// I16 MB with a HIERARCHICAL-DC second transform (the H.264 I16 trick, on our 8x8 grid): 4 quadrant residuals
36// -> DCT8 each -> 2x2 Hadamard ACROSS the 4 DC coefficients (flat MBs: 3 diff-DCs -> 0, whole-MB mean rides ONE
37// coefficient) -> quant+RDOQ+entropy per quadrant (same live coders) -> dequant -> un-Hadamard DCs -> inverse
38// DCT8 -> recon. Isolates the TRANSFORM-stage gain from the prediction question: (B2 - B) = pure cross-quadrant
39// DC redundancy, same prediction. Encoder-side recon mirrors what a vcv-9 decoder would compute (dequant domain).
40func enc_i16_dchad(cur: *u8, recon: *u8, W: i64, cx: i64, cy: i64, aq: i64, buf: *u8, bp0: i64, rctx: *i64, t8c: *i64, T16: *i64, L16: *i64, eo: *i64, bestm: i64, myfq: *i64) -> i64 {
41 var bp: i64 = bp0
42 let corner: i64 = eo[0]
43 let dc: i64 = eo[1]
44 let M: *i64 = vc_t8p(t8c, VC_T8_M)
45 let work: *i64 = vc_t8p(t8c, VC_T8_WORK)
46 let freq: *i64 = vc_t8p(t8c, VC_T8_FREQ)
47 let scr: *i64 = vc_t8p(t8c, VC_T8_SCR)
48 let ta: *i64 = vc_t8p(t8c, VC_T8_TA)
49 let tb: *i64 = vc_t8p(t8c, VC_T8_TB)
50 let zz8: *i64 = vc_t8p(t8c, VC_T8_ZZ)
51 let csave: *i64 = vc_t8p(t8c, VC_T8_RES)
52 // stage 1: all 4 quadrant residuals -> DCT8 -> saved freq (I16 prediction is MB-external, so no
53 // intra-MB recon dependency blocks holding all 4 pre-quant blocks at once -- the property that makes
54 // a cross-quadrant transform possible at all).
55 var q: i64 = 0
56 while q < 4 {
57 let qi: i64 = q & 1
58 let qj: i64 = q >> 1
59 var ry: i64 = 0
60 while ry < 8 {
61 var rx: i64 = 0
62 while rx < 8 { work[ry*8+rx] = (cur[(cy+qj*8+ry)*W + cx+qi*8+rx] as i64) - vc_pred_rich(bestm, qi*8+rx, qj*8+ry, 16, T16, L16, corner, dc); rx = rx + 1 }
63 ry = ry + 1 }
64 nx_dct8_forward_2d(M, work, freq, scr, ta, tb)
65 var ci: i64 = 0
66 while ci < 64 { myfq[q*64 + ci] = freq[ci]; ci = ci + 1 }
67 q = q + 1
68 }
69 // stage 2: ORTHONORMAL 2x2 Hadamard across the 4 DCs (y = Hx/2, energy-preserving -- uniform quant then
70 // compares FAIRLY; the unnormalized version quadruples DC magnitude = accidental finer quant = an RD-point
71 // shift, not a win). Flat MB: 4 DCs of D -> one 2D + three 0s = the "one DC instead of 4" prize.
72 let d0: i64 = myfq[0]
73 let d1: i64 = myfq[64]
74 let d2: i64 = myfq[128]
75 let d3: i64 = myfq[192]
76 myfq[0] = (d0 + d1 + d2 + d3 + 1) >> 1
77 myfq[64] = (d0 - d1 + d2 - d3 + 1) >> 1
78 myfq[128] = (d0 + d1 - d2 - d3 + 1) >> 1
79 myfq[192] = (d0 - d1 - d2 + d3 + 1) >> 1
80 // stage 3: per quadrant quant + RDOQ + entropy (EXACT sub8 sequence) + dequant, saved back
81 q = 0
82 while q < 4 {
83 var ci2: i64 = 0
84 while ci2 < 64 { freq[ci2] = myfq[q*64 + ci2]; ci2 = ci2 + 1 }
85 ci2 = 0
86 while ci2 < 64 { csave[ci2] = freq[ci2] << 2; ci2 = ci2 + 1 }
87 vc_quant8(freq, aq)
88 vc_rdoq8(freq, csave, aq, zz8)
89 var rcmode: i64 = 0
90 if (rctx as i64) != 0 { if (rctx[0] & 1) == 1 { rcmode = 1 } }
91 if rcmode == 1 {
92 if (rctx[0] & 32) != 0 { rc_sig_encode(freq, 64, 8, rctx[1] as *i64, rctx[3] as *u8, rctx[2] as *i64, vc_t8p(t8c, VC_T8_RES)) }
93 else { rc_block64_encode(freq, rctx[1] as *i64, rctx[3] as *u8, rctx[2] as *i64, zz8) }
94 }
95 else { bp = ve64_encode_at(freq, buf, bp, zz8) }
96 vc_dequant8(freq, aq)
97 ci2 = 0
98 while ci2 < 64 { myfq[q*64 + ci2] = freq[ci2]; ci2 = ci2 + 1 }
99 q = q + 1
100 }
101 // stage 4: un-Hadamard the dequantized DCs (x = Hy/2, the orthonormal inverse)
102 let y0: i64 = myfq[0]
103 let y1: i64 = myfq[64]
104 let y2: i64 = myfq[128]
105 let y3: i64 = myfq[192]
106 myfq[0] = (y0 + y1 + y2 + y3 + 1) >> 1
107 myfq[64] = (y0 - y1 + y2 - y3 + 1) >> 1
108 myfq[128] = (y0 + y1 - y2 - y3 + 1) >> 1
109 myfq[192] = (y0 - y1 - y2 + y3 + 1) >> 1
110 // stage 5: inverse DCT8 per quadrant -> recon add
111 q = 0
112 while q < 4 {
113 let qi: i64 = q & 1
114 let qj: i64 = q >> 1
115 var ci3: i64 = 0
116 while ci3 < 64 { freq[ci3] = myfq[q*64 + ci3]; ci3 = ci3 + 1 }
117 nx_dct8_inverse_2d(M, freq, work, scr, ta, tb)
118 var ry: i64 = 0
119 while ry < 8 {
120 var rx: i64 = 0
121 while rx < 8 {
122 var v: i64 = vc_pred_rich(bestm, qi*8+rx, qj*8+ry, 16, T16, L16, corner, dc) + work[ry*8+rx]
123 if v < 0 { v = 0 }
124 if v > 255 { v = 255 }
125 recon[(cy+qj*8+ry)*W + cx+qi*8+rx] = v as u8
126 rx = rx + 1 }
127 ry = ry + 1 }
128 q = q + 1
129 }
130 return bp
131}
132
133// MIRROR of vc_enc_frame_packed_rct8's keyframe path (LUMA), plus the I16 branch when i16on>=1 and the MB is
134// flat (same lv<=1 + vc_mb_t8_ok population that picks 8x8 today). i16on=2 adds the DC-Hadamard second
135// transform (enc_i16_dchad). cnt[0] += I16 MBs coded.
136func enc_mirror(cur: *u8, prev: *u8, recon: *u8, W: i64, H: i64, qp: i64, buf: *u8, blk: *i64, mv: *i64, rctx: *i64, i16on: i64, cnt: *i64) -> i64 {
137 rctx[0] = rctx[0] | 8
138 if (rctx[0] & 16) != 0 { rctx[0] = (rctx[0] | 4) - 4 }
139 let est: *i64 = rctx[1] as *i64
140 let probs: *i64 = rctx[2] as *i64
141 rc_enc_init(est)
142 var ci: i64 = 0
143 while ci < RC_NCTX8 { probs[ci] = 2048; ci = ci + 1 }
144 var bp: i64 = nx_bw_put(buf, 16, 1, 1)
145 bp = nx_bw_put(buf, bp, 0, 1)
146 let BW: i64 = W / 16
147 let BH: i64 = H / 16
148 let t8c: *i64 = rctx[4] as *i64
149 let T16: *i64 = sys_mmap(32*8) as *i64
150 let L16: *i64 = sys_mmap(16*8) as *i64
151 let eo: *i64 = sys_mmap(4*8) as *i64
152 let myfq: *i64 = sys_mmap(4*64*8) as *i64
153 let work: *i64 = vc_t8p(t8c, VC_T8_WORK)
154 var by: i64 = 0
155 while by < BH {
156 var bx: i64 = 0
157 while bx < BW {
158 let cx: i64 = bx*16
159 let cy: i64 = by*16
160 let lv: i64 = vc_aq_level(vc_mb_mad(cur, W, cx, cy))
161 bp = nx_bw_put(buf, bp, lv, 2)
162 var t8: i64 = 0
163 if (rctx[0] & 16) == 0 { if lv <= 1 { t8 = vc_mb_t8_ok(cur, W, cx, cy) } }
164 var didi16: i64 = 0
165 if i16on >= 1 { if t8 == 1 {
166 rctx[5] = bp
167 bp = nx_bw_put(buf, bp, t8, 1)
168 var ar: i64 = 0
169 if by > 0 { if bx + 1 < BW { ar = 1 } }
170 vc_gather_edges(recon, W, cx, cy, 16, 0, ar, T16, L16, eo)
171 let corner: i64 = eo[0]
172 let dc: i64 = eo[1]
173 let aq: i64 = vc_aq_qp(qp, lv)
174 var bestm: i64 = 0
175 var bestsad: i64 = 2147483647
176 var m: i64 = 0
177 while m < 9 {
178 var sad: i64 = 0
179 if m != 0 { sad = aq }
180 if m != 0 { if m != 4 { sad = sad + (aq / 4) } }
181 var yy: i64 = 0
182 while yy < 16 {
183 var xx: i64 = 0
184 while xx < 16 {
185 let d: i64 = (cur[(cy+yy)*W + cx+xx] as i64) - vc_pred_rich(m, xx, yy, 16, T16, L16, corner, dc)
186 if d < 0 { sad = sad - d } else { sad = sad + d }
187 xx = xx + 1 }
188 yy = yy + 1 }
189 if sad < bestsad { bestsad = sad; bestm = m }
190 m = m + 1
191 }
192 bp = nx_bw_put(buf, bp, bestm, 4)
193 if i16on == 2 {
194 bp = enc_i16_dchad(cur, recon, W, cx, cy, aq, buf, bp, rctx, t8c, T16, L16, eo, bestm, myfq)
195 } else {
196 var qj: i64 = 0
197 while qj < 2 {
198 var qi: i64 = 0
199 while qi < 2 {
200 var ry: i64 = 0
201 while ry < 8 {
202 var rx: i64 = 0
203 while rx < 8 { work[ry*8+rx] = (cur[(cy+qj*8+ry)*W + cx+qi*8+rx] as i64) - vc_pred_rich(bestm, qi*8+rx, qj*8+ry, 16, T16, L16, corner, dc); rx = rx + 1 }
204 ry = ry + 1 }
205 bp = vc_enc_sub8(t8c, aq, buf, bp, rctx)
206 ry = 0
207 while ry < 8 {
208 var rx: i64 = 0
209 while rx < 8 {
210 var v: i64 = vc_pred_rich(bestm, qi*8+rx, qj*8+ry, 16, T16, L16, corner, dc) + work[ry*8+rx]
211 if v < 0 { v = 0 }
212 if v > 255 { v = 255 }
213 recon[(cy+qj*8+ry)*W + cx+qi*8+rx] = v as u8
214 rx = rx + 1 }
215 ry = ry + 1 }
216 qi = qi + 1 }
217 qj = qj + 1 }
218 }
219 cnt[0] = cnt[0] + 1
220 didi16 = 1
221 } }
222 if didi16 == 0 {
223 rctx[5] = bp
224 bp = nx_bw_put(buf, bp, t8, 1)
225 bp = vc_enc_block_packed_e(cur, prev, recon, W, H, bx, by, vc_aq_qp(qp, lv), 1, qp*30, buf, bp, blk, mv, 1 + (t8 << 1), rctx)
226 }
227 bx = bx + 1 }
228 by = by + 1 }
229 let rbytes: i64 = rc_enc_flush(est, rctx[3] as *u8)
230 let cavlc_end: i64 = (bp + 7) / 8
231 buf[0] = (cavlc_end & 0xff) as u8
232 buf[1] = ((cavlc_end >> 8) & 0xff) as u8
233 let rcbuf: *u8 = rctx[3] as *u8
234 var i: i64 = 0
235 while i < rbytes { buf[cavlc_end + i] = rcbuf[i]; i = i + 1 }
236 return (cavlc_end + rbytes) * 8
237}
238
239// one ALL-INTRA run over NF frames (luma plane): mode 0 = real frame fn (A), 1 = mirror no-I16 (B0), 2 = mirror+I16 (B).
240// out[0]=total bits, out[1]=luma SSE, out[2]=I16 MB count.
241func runv(fp: *i64, qp: i64, emode: i64, sz: i64, mode: i64, out: *i64) -> i64 {
242 let prevE: *u8=sys_mmap(sz+64); let reconE: *u8=sys_mmap(sz+64)
243 let buf: *u8=sys_mmap(4194304); let blk: *i64=sys_mmap(512) as *i64; let mv: *i64=sys_mmap(128) as *i64
244 let est: *i64=sys_mmap(64) as *i64; let probs: *i64=sys_mmap(32*8) as *i64; let rcbuf: *u8=sys_mmap(4194304)
245 let t8c: *i64=sys_mmap(5120) as *i64; let rctx: *i64=sys_mmap(64) as *i64
246 let cnt: *i64=sys_mmap(8) as *i64
247 vc_t8_init(t8c)
248 var z: i64=0; while z<sz { prevE[z]=0 as u8; z=z+1 }
249 rctx[1]=est as i64; rctx[2]=probs as i64; rctx[3]=rcbuf as i64; rctx[4]=t8c as i64; rctx[5]=0; rctx[6]=0; rctx[7]=0
250 cnt[0]=0
251 out[0]=0; out[1]=0; out[2]=0
252 let NL: i64 = SW*SH
253 var f: i64=0
254 while f < NF {
255 let cur: *u8=fp[f] as *u8
256 rctx[0]=emode
257 var bits: i64 = 0
258 if mode == 0 { bits = vc_enc_frame_packed_rct8(cur, prevE, reconE, SW, SH, qp, 1, qp*30, buf, blk, mv, 1, rctx) }
259 if mode == 1 { bits = enc_mirror(cur, prevE, reconE, SW, SH, qp, buf, blk, mv, rctx, 0, cnt) }
260 if mode == 2 { bits = enc_mirror(cur, prevE, reconE, SW, SH, qp, buf, blk, mv, rctx, 1, cnt) }
261 if mode == 3 { bits = enc_mirror(cur, prevE, reconE, SW, SH, qp, buf, blk, mv, rctx, 2, cnt) }
262 out[0] = out[0] + bits
263 var pxi: i64=0
264 while pxi<NL { let d: i64=(reconE[pxi]&0xff)-(cur[pxi]&0xff); out[1]=out[1]+d*d; pxi=pxi+1 }
265 f=f+1
266 }
267 out[2] = cnt[0]
268 return 0 }
269
270func sweepqp(fp: *i64, sz: i64, qp: i64) -> i64 {
271 gw(" --- qp=" as *u8); gn(qp); gw(" (ALL-INTRA, luma, emode 169) ---\n" as *u8)
272 let oA: *i64=sys_mmap(32) as *i64
273 let oB0: *i64=sys_mmap(32) as *i64
274 let oB: *i64=sys_mmap(32) as *i64
275 let oB2: *i64=sys_mmap(32) as *i64
276 runv(fp, qp, 169, sz, 0, oA)
277 runv(fp, qp, 169, sz, 1, oB0)
278 runv(fp, qp, 169, sz, 2, oB)
279 runv(fp, qp, 169, sz, 3, oB2)
280 let NL: i64 = NF*SW*SH
281 gw(" A real : bits=" as *u8); gn(oA[0]); gw(" MSEx1000=" as *u8); gn(oA[1]*1000/NL); gw("\n" as *u8)
282 gw(" B0 mirror : bits=" as *u8); gn(oB0[0]); gw(" fidelity=" as *u8)
283 if oB0[0] == oA[0] { gw("GREEN (bit-identical)" as *u8) } else { gw("RED -- PROBE INVALID" as *u8) }
284 gw("\n" as *u8)
285 gw(" B I16 : bits=" as *u8); gn(oB[0]); gw(" MSEx1000=" as *u8); gn(oB[1]*1000/NL)
286 gw(" i16MBs=" as *u8); gn(oB[2]); gw("/" as *u8); gn(NF*(SW/16)*(SH/16)); gw("\n" as *u8)
287 gw(" B2 I16+DCHAD: bits=" as *u8); gn(oB2[0]); gw(" MSEx1000=" as *u8); gn(oB2[1]*1000/NL); gw("\n" as *u8)
288 if oA[0] > 0 { gw(" --> I16 vs A = " as *u8); gn((oA[0]-oB[0])*1000/oA[0]); gw(" permille ยท I16+DCHAD vs A = " as *u8); gn((oA[0]-oB2[0])*1000/oA[0]); gw(" permille (positive = SMALLER)\n" as *u8) }
289 if oB[0] > 0 { gw(" --> TRANSFORM-STAGE gain (B2 vs B, same prediction) = " as *u8); gn((oB[0]-oB2[0])*1000/oB[0]); gw(" permille\n" as *u8) }
290 return 0 }
291
292func main() -> i64 {
293 gw("=== nx_vcodec_i16ceil_gate: I_16x16 flat-intra CEILING (real coders, all-intra) on /tmp/seq.y4m ===\n" as *u8)
294 let box: *i64 = sys_mmap(16) as *i64
295 let yuv: *u8 = sys_read_file("/tmp/seq.y4m" as *u8, box)
296 if (yuv as i64)==0 { gw("cannot read /tmp/seq.y4m -> RED\n" as *u8); return 1 }
297 let total: i64=box[0]
298 var hi: i64=0; while hi<total { if (yuv[hi]&0xff)==10 { break } hi=hi+1 } hi=hi+1
299 let C2: i64=(SW/2)*(SH/2); let fdata: i64=SW*SH+2*C2; let sz: i64=fdata
300 let fp: *i64=sys_mmap(128*8) as *i64
301 let nf: i64=frames(yuv, hi, total, fp, fdata)
302 if nf < NF { gw("too few frames -> RED\n" as *u8); return 1 }
303 sweepqp(fp, sz, 8)
304 sweepqp(fp, sz, 20)
305 sweepqp(fp, sz, 32)
306 gw("I16CEIL: DONE\n" as *u8)
307 return 0 }