code wiki / _hdl_build / nx_vcodec_tsize_rd_gate.nx
nx_vcodec_tsize_rd_gate.nx source
↩ module page · 103 lines · 6801 B
1import "nx_gate_base.nx"
2// nx_vcodec_tsize_rd_gate.nx -- RESEARCH-FIRST measurement for the SOTA climb rung 1 (larger transforms, the
3// HEVC/AV1 gain). Does an 8x8 transform (nx_dct8) beat our 4x4 (vt2) on real call content -- especially FLAT
4// backgrounds where energy compacts into few low-freq coeffs? Measures the CAVLC bit-cost (same run-length
5// syntax, so a fair proxy; the range coder improves both equally) vs reconstruction PSNR, swept over quant,
6// on FLAT and TEXTURED regions. If 8x8 is clearly cheaper at matched quality on flat content, the full
7// variable-transform-size integration is justified. license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_vtransform_dct2.nx"
10import "nx_dct8.nx"
11import "nx_ventropy.nx"
12
13func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw("
14" as *u8); return ok }
15func gn(v: i64) -> i64 {
16 let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m}
17 let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1}
18 var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
19// CAVLC bit-cost of a block's quantized coeffs, scanned in zz order (run-length: [1][run:4][level: 5+blen], EOB [1])
20func ve_bits_zz(coeffs: *i64, n: i64, zz: *i64) -> i64 {
21 var bits: i64=0; var run: i64=0; var k: i64=0
22 while k < n { let v: i64=coeffs[zz[k]]
23 if v==0 { run=run+1 } else { bits=bits+1+4+5+ve_blen(ve_zze(v)); run=0 } k=k+1 }
24 return bits+1 }
25func isqrt(v: i64) -> i64 { if v<=0 { return 0 } var x: i64=v; var y: i64=(x+1)/2; while y<x { x=y; y=(x+v/x)/2 } return x }
26// PSNR proxy: 10*log10(255^2/mse) approximated as an integer dB*10 via a small log table on mse
27func psnr10(sse: i64, npix: i64) -> i64 {
28 if sse<=0 { return 600 }
29 let mse: i64 = (sse*100)/npix // mse*100 for precision
30 // 10*log10(65025/(mse/100)) = 10*log10(6502500/mse). integer log10 via repeated /10.
31 var num: i64 = 6502500*100; var m: i64 = mse; if m<1 {m=1}
32 var ratio: i64 = num/m // = 10^(psnr/10) * 100
33 var db: i64 = 0; var t: i64 = ratio
34 while t >= 100 { db=db+10; t=t/10 } // each /10 = +10dB (integer decade)
35 // fractional dB from the remainder (t in [10,100) -> ~ +0..+10); no else-if chains (nx parser)
36 var fr: i64 = 0
37 if t>=13 { fr=1 } if t>=16 { fr=2 } if t>=20 { fr=3 } if t>=25 { fr=4 }
38 if t>=32 { fr=5 } if t>=40 { fr=6 } if t>=50 { fr=7 } if t>=63 { fr=8 } if t>=79 { fr=9 }
39 return db + fr }
40
41// encode WxH content with 4x4 (t=4) or 8x8 (t=8) at quant q; returns bits, writes psnr*10 to pr[0]
42func tscode(img: *u8, W: i64, H: i64, t: i64, q: i64, M: *i64, zz4: *i64, zz8: *i64, pr: *i64) -> i64 {
43 let blk: *i64=sys_mmap(64*8) as *i64; let o: *i64=sys_mmap(64*8) as *i64
44 let sc: *i64=sys_mmap(64*8) as *i64; let tmi: *i64=sys_mmap(8*8) as *i64; let tmo: *i64=sys_mmap(8*8) as *i64
45 var bits: i64=0; var sse: i64=0
46 var by: i64=0
47 while by < H/t { var bx: i64=0
48 while bx < W/t {
49 var i: i64=0; while i<t*t { let px: i64=(by*t+i/t)*W+bx*t+(i%t); blk[i]=(img[px]&0xff)-128; i=i+1 }
50 if t==4 { vt2_fwd(blk); vt2_quant(blk, q)
51 bits=bits+ve_bits_zz(blk, 16, zz4)
52 vt2_dequant(blk, q); vt2_inv(blk)
53 } else { nx_dct8_forward_2d(M, blk, o, sc, tmi, tmo)
54 i=0; while i<64 { var c: i64=o[i]; if c<0 { c=0-(((0-c)+q/2)/q)*q } else { c=((c+q/2)/q)*q } o[i]=c; i=i+1 } // deadzone quant+dequant on the 8x8 coeffs
55 var nzz: i64=0; i=0; while i<64 { blk[i]= o[i]/q * q; i=i+1 } // (blk reused for bit-count in real coeff units)
56 i=0; while i<64 { blk[i]=o[i]/q; i=i+1 } // quant indices for bit cost
57 bits=bits+ve_bits_zz(blk, 64, zz8)
58 nx_dct8_inverse_2d(M, o, blk, sc, tmi, tmo) // o holds dequantized coeffs -> spatial in blk
59 }
60 i=0; while i<t*t { let px2: i64=(by*t+i/t)*W+bx*t+(i%t); var v: i64=blk[i]+128; if v<0{v=0} if v>255{v=255}
61 let d: i64=v-(img[px2]&0xff); sse=sse+d*d; i=i+1 }
62 bx=bx+1 } by=by+1 }
63 let ps: i64 = psnr10(sse, W*H) // hoisted: call-in-array-store is a known nx_cc footgun
64 pr[0]=ps
65 return (bits+7)/8 }
66
67func main() -> i64 {
68 gw("=== nx_vcodec_tsize_rd_gate: 4x4 (vt2) vs 8x8 (dct8), matched-quality, flat vs textured ===\n" as *u8)
69 let M: *i64=sys_mmap(64*8) as *i64; nx_dct8_init(M)
70 let zz4: *i64=sys_mmap(16*8) as *i64
71 let z4d: *i64=sys_mmap(16*8) as *i64; var i: i64=0
72 while i<16 { zz4[i]=ve_zz_idx(i); i=i+1 }
73 let zz8: *i64=sys_mmap(64*8) as *i64
74 // JPEG 8x8 zig-zag, generated algorithmically (no literal -> no const-ptr-index footgun)
75 var zi: i64=0; var zr: i64=0; var zc: i64=0; var up: i64=1
76 while zi < 64 { zz8[zi]=zr*8+zc; zi=zi+1
77 if up==1 { if zc==7 { zr=zr+1; up=0 } else { if zr==0 { zc=zc+1; up=0 } else { zr=zr-1; zc=zc+1 } } }
78 else { if zr==7 { zc=zc+1; up=1 } else { if zc==0 { zr=zr+1; up=1 } else { zr=zr+1; zc=zc-1 } } } }
79 let W: i64=64; let H: i64=64
80 let flat: *u8=sys_mmap(W*H); let tex: *u8=sys_mmap(W*H)
81 var r: i64=0
82 while r<H { var c: i64=0
83 while c<W { flat[r*W+c]=(120 + (c+r)/3) as u8 // gentle gradient = FLAT background
84 var v: i64=128 + ((c*7+r*5)%37)*4 - 74; if v<0{v=0} if v>255{v=255}; tex[r*W+c]=v as u8 // high-freq = TEXTURED
85 c=c+1 } r=r+1 }
86 let pr: *i64=sys_mmap(16) as *i64
87 let qs: *i64=sys_mmap(8*8) as *i64; qs[0]=8; qs[1]=16; qs[2]=32; qs[3]=64
88 gw(" FLAT background (wall/skin) -- q sweep [bytes @ PSNR.dB]:\n" as *u8)
89 var qi: i64=0
90 while qi<4 { let q: i64=qs[qi]
91 let b4: i64=tscode(flat, W, H, 4, q, M, zz4, zz8, pr); let p4: i64=pr[0]
92 let b8: i64=tscode(flat, W, H, 8, q, M, zz4, zz8, pr); let p8: i64=pr[0]
93 gw(" q=" as *u8); gn(q); gw(" 4x4=" as *u8); gn(b4); gw("B@" as *u8); gn(p4/10); gw("." as *u8); gn(p4%10)
94 gw("dB 8x8=" as *u8); gn(b8); gw("B@" as *u8); gn(p8/10); gw("." as *u8); gn(p8%10); gw("dB\n" as *u8); qi=qi+1 }
95 gw(" TEXTURED (face detail):\n" as *u8)
96 qi=0
97 while qi<4 { let q: i64=qs[qi]
98 let b4: i64=tscode(tex, W, H, 4, q, M, zz4, zz8, pr); let p4: i64=pr[0]
99 let b8: i64=tscode(tex, W, H, 8, q, M, zz4, zz8, pr); let p8: i64=pr[0]
100 gw(" q=" as *u8); gn(q); gw(" 4x4=" as *u8); gn(b4); gw("B@" as *u8); gn(p4/10); gw("." as *u8); gn(p4%10)
101 gw("dB 8x8=" as *u8); gn(b8); gw("B@" as *u8); gn(p8/10); gw("." as *u8); gn(p8%10); gw("dB\n" as *u8); qi=qi+1 }
102 gw("READ: on FLAT, 8x8 should be MUCH fewer bytes at >= PSNR (energy compaction) -> validates variable transform size.\n" as *u8)
103 return 0 }