nx_codec_real_rd2.nx source

1// nx_codec_real_rd2.nx -- the MODELED patent-free intra codec on a REAL frame's luma (576x1024), the 2// efficiency upgrade over nx_codec_real_rd (naive raw-coeff rANS). Per 8x8 block: level-shift -> 2D DCT 3// (nx_dct8, EXPIRED) -> PERCEPTUAL quant matrix (nx_qt_luma, JPEG Annex-K, EXPIRED) -> zigzag (nx_zigzag) 4// -> DC-differential + AC run/size SYMBOL stream (nx_jsym) separated from raw magnitude bits -> rANS the 5// symbols (nx_rans, PUBLIC DOMAIN); magnitude bits stored raw (near-incompressible). HONEST total = 6// 512B rANS table + 64B quant table + 8B header + rANS(symbols) + magbits. This is exactly the modeling 7// nx_codec_rd2_gate MEASURED at 3.5x over naive (at block level) -- now on the real frame, swept over 8// JPEG quality, to RE-MEASURE vs x264. Round-trip (rANS + jsym) asserted bit-exact each quality = liar-kill. 9// Every tool EXPIRED/PD; no CABAC; integer-only; nx_cc->nxasm, no gcc. license_tier: ORIGINAL 10import "nx_syscalls.nx" 11import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 12import "nx_h264_bitwriter.nx" 13import "nx_dct8.nx" 14import "nx_quant_table.nx" 15import "nx_zigzag.nx" 16import "nx_jpeg_huff_enc.nx" 17import "nx_jpeg_sym.nx" 18import "nx_rans.nx" 19const K_MAGIC_1024: i64 = 1024 20const K_MAGIC_9216: i64 = 9216 21const K_MAGIC_4096: i64 = 4096 22 23func g_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 24// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 25// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 26// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 27// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 28func g_num(v: i64) -> i64 { nxi_out(v); return 0 } 29func g_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 30// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 31// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 32// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 33// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 34func g_wn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 } 35func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v } 36func qrnd(v: i64, q: i64) -> i64 { if v>=0 { return (v + q/2)/q } return 0 - (((0-v) + q/2)/q) } 37 38func main() -> i64 { 39 g_puts("=== OURS-MODELED: patent-free intra codec (perceptual quant + zigzag + DC-diff/RLE + rANS) on REAL frame 576x1024 ===\n" as *u8) 40 let W: i64=576; let H: i64=K_MAGIC_1024; let sb: i64=12 41 let npx: i64=W*H 42 let bw8: i64=W/8; let bh8: i64=H/8; let nblk: i64=bw8*bh8 // 72*128 = K_MAGIC_9216 43 let ncoef: i64=nblk*64 44 45 let flen: *i64=sys_mmap(8) as *i64 46 let fb: *u8=sys_read_file("knowledge/staging/media/ref_frame0.yuv" as *u8, flen) 47 if flen[0] < npx { g_puts("FATAL: frame read short\n" as *u8); sys_exit(2); return 2 } 48 g_puts("-- read " as *u8); g_num(flen[0]); g_puts(" bytes; luma " as *u8); g_num(npx); g_puts(" blocks=" as *u8); g_num(nblk); g_puts("\n" as *u8) 49 50 let DM: *i64=sys_mmap(64*8) as *i64; nx_dct8_init(DM) 51 let to_zz: *i64=sys_mmap(64*8) as *i64; let from_zz: *i64=sys_mmap(64*8) as *i64; nx_zigzag_init(to_zz, from_zz) 52 let scratch: *i64=sys_mmap(64*8) as *i64; let ti: *i64=sys_mmap(8*8) as *i64; let to: *i64=sys_mmap(8*8) as *i64 53 let qt: *i64=sys_mmap(64*8) as *i64 54 let sh: *i64=sys_mmap(64*8) as *i64; let co: *i64=sys_mmap(64*8) as *i64; let q: *i64=sys_mmap(64*8) as *i64; let zz: *i64=sys_mmap(64*8) as *i64 55 let qall: *i64=sys_mmap(ncoef*8) as *i64 56 57 // entropy buffers (reused per quality) 58 let sym: *u8=sys_mmap(nblk*132 + K_MAGIC_4096) 59 let sp: *i64=sys_mmap(8) as *i64 60 let magw: *BitWriter=sys_mmap(64) as *BitWriter 61 let counts: *i64=sys_mmap(257*8) as *i64; let fr: *i64=sys_mmap(257*8) as *i64; let cu: *i64=sys_mmap(257*8) as *i64; let s2s: *i64=sys_mmap((1<<sb)*8) as *i64 62 let cap: i64=nblk*132 + K_MAGIC_4096; let senc: *u8=sys_mmap(cap); let sdec: *u8=sys_mmap(cap) 63 let sp2: *i64=sys_mmap(8) as *i64; let mpos: *i64=sys_mmap(8) as *i64 64 let zzm: *i64=sys_mmap(64*8) as *i64; let qnat: *i64=sys_mmap(64*8) as *i64; let dq: *i64=sys_mmap(64*8) as *i64; let rsh: *i64=sys_mmap(64*8) as *i64 65 let recon: *u8=sys_mmap(npx+16) 66 67 let qs: *i64=sys_mmap(16*8) as *i64 68 qs[0]=92; qs[1]=85; qs[2]=72; qs[3]=55; qs[4]=40; qs[5]=28; qs[6]=16; qs[7]=8 69 let nq: i64=8 70 71 let lg: i64=sys_openat_append("knowledge/status/codec_real_rd2.log" as *u8, 0x1a4) 72 var all_ok: i64=1 73 var qi: i64=0 74 while qi<nq { 75 let Q: i64=qs[qi] 76 nx_qt_luma(qt, Q) 77 // forward all blocks -> qall (zigzag order) 78 var by: i64=0 79 while by<bh8 { 80 var bx: i64=0 81 while bx<bw8 { 82 let bidx: i64=by*bw8+bx 83 var r: i64=0 84 while r<8 { var c: i64=0; while c<8 { sh[r*8+c]=(fb[(by*8+r)*W+(bx*8+c)] as i64)-128; c=c+1 } r=r+1 } 85 nx_dct8_forward_2d(DM, sh, co, scratch, ti, to) 86 var i: i64=0; while i<64 { q[i]=qrnd(co[i], qt[i]); i=i+1 } 87 nx_zigzag_scan(q, to_zz, zz) 88 i=0; while i<64 { qall[bidx*64+i]=zz[i]; i=i+1 } 89 bx=bx+1 90 } 91 by=by+1 92 } 93 // model: jsym -> sym stream + magnitude bits. FRESH (mmap-zeroed) magbuf each quality: the 94 // bitwriter OR-accumulates into the buffer, so a reused/dirty buffer would corrupt later qualities. 95 let magbuf: *u8=sys_mmap(npx*2 + K_MAGIC_4096) 96 sp[0]=0; bw_init(magw, magbuf, npx*2+K_MAGIC_4096) 97 var prevdc: i64=0; var b: i64=0 98 while b<nblk { prevdc=jsym_encode_block(sym, sp, magw, ((qall as i64)+(b*64*8)) as *i64, prevdc); b=b+1 } 99 while magw.bit_pos != 0 { bw_write_bit(magw, 0) } // byte-align so all mag bits are committed 100 let nsym: i64=sp[0] 101 let magbytes: i64=(bw_total_bits(magw)+7)/8 102 // rANS the symbol stream 103 rans_count(sym, nsym, counts); rans_normalize(counts, fr, sb); rans_cum(fr, cu); rans_build_slot2sym(fr, cu, s2s) 104 let sym_payload: i64=rans_encode(sym, nsym, fr, cu, sb, senc, cap) 105 let total: i64=512+64+8+sym_payload+magbytes 106 // decode: rANS -> symbols ; jsym -> coeffs ; reconstruct pixels 107 rans_decode(senc, sym_payload, nsym, fr, cu, sb, s2s, sdec) 108 var rans_ok: i64=1; var i2: i64=0; while i2<nsym { if sdec[i2]!=sym[i2] { rans_ok=0 } i2=i2+1 } 109 sp2[0]=0; mpos[0]=0; var mdc: i64=0; var coeff_ok: i64=1 110 var sse: i64=0; var maxe: i64=0 111 b=0 112 while b<nblk { 113 mdc=jsym_decode_block(sdec, sp2, magbuf, mpos, mdc, zzm) 114 var i: i64=0; while i<64 { if zzm[i]!=qall[b*64+i] { coeff_ok=0 } i=i+1 } 115 nx_zigzag_unscan(zzm, from_zz, qnat) 116 i=0; while i<64 { dq[i]=qnat[i]*qt[i]; i=i+1 } 117 nx_dct8_inverse_2d(DM, dq, rsh, scratch, ti, to) 118 let by2: i64=b/bw8; let bx2: i64=b%bw8 119 var r: i64=0 120 while r<8 { 121 var c: i64=0 122 while c<8 { 123 var p: i64=rsh[r*8+c]+128; if p<0 {p=0} if p>255 {p=255} 124 let idx: i64=(by2*8+r)*W+(bx2*8+c) 125 let d: i64=p-(fb[idx] as i64); sse=sse+d*d; let a: i64=iabs(d); if a>maxe {maxe=a} 126 recon[idx]=p as u8 127 c=c+1 128 } 129 r=r+1 130 } 131 b=b+1 132 } 133 if rans_ok==0 { all_ok=0 } 134 if coeff_ok==0 { all_ok=0 } 135 let okf: i64=rans_ok*coeff_ok 136 137 g_puts("OURS2 q=" as *u8); g_num(Q); g_puts(" total=" as *u8); g_num(total); g_puts(" sympay=" as *u8); g_num(sym_payload); g_puts(" magB=" as *u8); g_num(magbytes) 138 g_puts(" nsym=" as *u8); g_num(nsym); g_puts(" sse=" as *u8); g_num(sse); g_puts(" maxErr=" as *u8); g_num(maxe); g_puts(" ok=" as *u8); g_num(okf); g_puts("\n" as *u8) 139 if lg>=0 { g_w(lg,"OURS2 q=" as *u8); g_wn(lg,Q); g_w(lg," total=" as *u8); g_wn(lg,total); g_w(lg," sse=" as *u8); g_wn(lg,sse); g_w(lg," maxErr=" as *u8); g_wn(lg,maxe); g_w(lg," ok=" as *u8); g_wn(lg,okf); g_w(lg,"\n" as *u8) } 140 qi=qi+1 141 } 142 if lg>=0 { sys_close(lg) } 143 if all_ok==1 { g_puts("---- round-trip lossless ALL q (codec correct) verdict=GREEN\n" as *u8); sys_exit(0); return 0 } 144 g_puts("---- round-trip LOSSY (codec BUG) verdict=RED\n" as *u8); sys_exit(1); return 1 145}