nx_vcodec_subpel_ceil.nx source

1// nx_vcodec_subpel_ceil.nx -- CEILING of the 6-tap sub-pel lever (the next real refinement after the magic-number 2// fixes). Our qp_pixel is BILINEAR; x264 uses a 6-tap [1,-5,20,20,-5,1]/32 half-pel filter. For each genuinely- 3// coded P-MB (non-skip, INTERIOR so the 6-tap has its +/-2..3 margin), find the best integer MV, then compare the 4// best fractional residual SAD under BILINEAR quarter-pel (what we do) vs 6-TAP half-pel. The reduction is the 5// prediction gain a 6-tap upgrade could realize (residual smaller -> fewer bits). Decides if the (vcv-bump, 6// bit-exact-critical) 6-tap build is worth it. Reads /tmp/seq.y4m. CIF. license: ORIGINAL 7import "nx_syscalls.nx" 8import "nx_video_codec_wasm.nx" 9import "nx_vmotion.nx" 10import "nx_vsubpel.nx" 11const K_MAGIC_2147483647: i64 = 2147483647 12const K_MAGIC_2026: i64 = 2026 13 14const SW: i64 = 352 15const SH: i64 = 288 16const NF: i64 = 48 17 18func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 19func gn(v: i64) -> i64 { 20 let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} 21 let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} 22 var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 } 23 24// 6-tap [1,-5,20,20,-5,1]/32 clamped 0..255 25func tap6(a: i64, b: i64, c: i64, d: i64, e: i64, f: i64) -> i64 { 26 var v: i64 = (a - 5*b + 20*c + 20*d - 5*e + f + 16) / 32 27 if v < 0 { v = 0 } if v > 255 { v = 255 } return v } 28// 6-tap luma sample at (x,y) with half-pel (hx,hy in {0,2}); interior only 29func h6(prev: *u8, W: i64, x: i64, y: i64, hx: i64, hy: i64) -> i64 { 30 if hx == 0 { if hy == 0 { return prev[y*W+x] as i64 } } 31 if hy == 0 { // horizontal half 32 return tap6(prev[y*W+x-2] as i64, prev[y*W+x-1] as i64, prev[y*W+x] as i64, prev[y*W+x+1] as i64, prev[y*W+x+2] as i64, prev[y*W+x+3] as i64) } 33 if hx == 0 { // vertical half 34 return tap6(prev[(y-2)*W+x] as i64, prev[(y-1)*W+x] as i64, prev[y*W+x] as i64, prev[(y+1)*W+x] as i64, prev[(y+2)*W+x] as i64, prev[(y+3)*W+x] as i64) } 35 // center (both half): vertical 6-tap of horizontal 6-taps 36 var c: *i64 = sys_mmap(64) as *i64; var j: i64 = 0-2 37 while j <= 3 { c[j+2] = tap6(prev[(y+j)*W+x-2] as i64, prev[(y+j)*W+x-1] as i64, prev[(y+j)*W+x] as i64, prev[(y+j)*W+x+1] as i64, prev[(y+j)*W+x+2] as i64, prev[(y+j)*W+x+3] as i64); j=j+1 } 38 var v: i64 = (c[0] - 5*c[1] + 20*c[2] + 20*c[3] - 5*c[4] + c[5] + 16) / 32 39 if v < 0 { v = 0 } if v > 255 { v = 255 } return v } 40 41func frames(yuv: *u8, hi: i64, total: i64, fp: *i64, fdata: i64) -> i64 { 42 var nf: i64=0; var p: i64=hi 43 while nf < NF { if p>=total { break } 44 var q: i64=p; while q<total { if (yuv[q]&0xff)==10 { break } q=q+1 } q=q+1 45 if q+fdata>total { break } fp[nf]=(yuv as i64)+q; nf=nf+1; p=q+fdata } 46 return nf } 47 48// best bilinear quarter-pel SAD for a 16x16 block at (cx,cy), integer MV (idx,idy) 49func bil_sad(cur: *u8, prev: *u8, W: i64, cx: i64, cy: i64, idx: i64, idy: i64) -> i64 { 50 let px: i64=cx+idx; let py: i64=cy+idy; var best: i64=K_MAGIC_2147483647 51 var qy: i64=0 52 while qy<=2 { var qx: i64=0 // FAIR: HALF-pel only ({0,2}^2) to match the 6-tap -> isolates the FILTER 53 while qx<=2 { var s: i64=0; var yy: i64=0 54 while yy<16 { var xx: i64=0 55 while xx<16 { let d: i64=(cur[(cy+yy)*W+cx+xx] as i64)-qp_pixel(prev,W,px+xx,py+yy,qx,qy); if d<0{s=s-d}else{s=s+d} xx=xx+1 } yy=yy+1 } 56 if s<best { best=s } qx=qx+2 } qy=qy+2 } 57 return best } 58// best 6-tap half-pel SAD (qx,qy in {0,2}) 59func t6_sad(cur: *u8, prev: *u8, W: i64, cx: i64, cy: i64, idx: i64, idy: i64) -> i64 { 60 let px: i64=cx+idx; let py: i64=cy+idy; var best: i64=K_MAGIC_2147483647 61 var hy: i64=0 62 while hy<=2 { var hx: i64=0 63 while hx<=2 { var s: i64=0; var yy: i64=0 64 while yy<16 { var xx: i64=0 65 while xx<16 { let d: i64=(cur[(cy+yy)*W+cx+xx] as i64)-h6(prev,W,px+xx,py+yy,hx,hy); if d<0{s=s-d}else{s=s+d} xx=xx+1 } yy=yy+1 } 66 if s<best { best=s } hx=hx+2 } hy=hy+2 } 67 return best } 68 69func main() -> i64 { 70 let vmscr3: *i64 = sys_mmap(32) as *i64 // vm_search scratch (K_MAGIC_2026-07-29: hoisted ONE alloc; the per-call sys_mmap inside vm_search was the wasm 0-stub + native map-leak class) 71 gw("=== nx_vcodec_subpel_ceil: 6-tap half-pel vs BILINEAR quarter-pel residual on coded P-MBs (/tmp/seq.y4m) ===\n" as *u8) 72 let box: *i64 = sys_mmap(16) as *i64 73 let yuv: *u8 = sys_read_file("/tmp/seq.y4m" as *u8, box) 74 if (yuv as i64)==0 { gw("cannot read /tmp/seq.y4m -> RED\n" as *u8); return 1 } 75 let total: i64=box[0] 76 var hi: i64=0; while hi<total { if (yuv[hi]&0xff)==10 { break } hi=hi+1 } hi=hi+1 77 let C2: i64=(SW/2)*(SH/2); let fdata: i64=SW*SH+2*C2 78 let fp: *i64=sys_mmap(128*8) as *i64 79 let nf: i64=frames(yuv, hi, total, fp, fdata); if nf<NF { gw("too few\n" as *u8); return 1 } 80 let mbx: i64=SW/16; let mby: i64=SH/16 81 var bilT: i64=0; var t6T: i64=0; var coded: i64=0 82 let mv: *i64=sys_mmap(32) as *i64 83 var f: i64=1 84 while f < NF { 85 let cur: *u8=fp[f] as *u8; let prev: *u8=fp[f-1] as *u8 86 var by: i64=1 87 while by<mby-1 { var bx: i64=1 // interior MBs only (6-tap margin) 88 while bx<mbx-1 { 89 let x0: i64=bx*16; let y0: i64=by*16 90 let zs: i64=vm_sad_zero(cur, prev, SW, bx, by, 16) 91 if zs >= 8*30 { // genuinely-coded (skip*30-consistent) 92 vm_search(cur, prev, SW, SH, bx, by, 16, 8, mv, vmscr3) 93 if x0+mv[0] >= 2 { if y0+mv[1] >= 2 { if x0+16+mv[0] <= SW-3 { if y0+16+mv[1] <= SH-3 { 94 coded=coded+1 95 bilT=bilT+bil_sad(cur, prev, SW, x0, y0, mv[0], mv[1]) 96 t6T=t6T+t6_sad(cur, prev, SW, x0, y0, mv[0], mv[1]) 97 } } } } 98 } 99 bx=bx+1 } by=by+1 } 100 f=f+4 101 } 102 if coded==0 { gw("no coded MBs\n" as *u8); return 0 } 103 gw(" coded interior MBs=" as *u8); gn(coded); gw("\n" as *u8) 104 gw(" residual SAD bilinear-qpel=" as *u8); gn(bilT); gw(" 6tap-halfpel=" as *u8); gn(t6T); gw("\n" as *u8) 105 if bilT>0 { gw(" 6-tap cuts residual " as *u8); gn((bilT-t6T)*1000/bilT); gw(" per-mille (big=build 6-tap; small=not worth the vcv bump)\n" as *u8) } 106 gw("SUBPEL_CEIL: DONE\n" as *u8) 107 return 0 }