nx_codec_rd2_gate.nx source

1// nx_codec_rd2_gate.nx -- SOVEREIGN MODELED rate-distortion head-to-head: rANS vs JPEG-Huffman on the 2// IDENTICAL SYMBOL STREAM (DC-diff categories + AC run/size + EOB) with IDENTICAL magnitude bits, at 3// IDENTICAL quality. This is the architectural fix the first head-to-head (nx_codec_rd_gate) demanded: 4// give rANS the same DC-diff + RLE MODELING JPEG uses, so the comparison is the pure entropy-coder 5// question (rANS vs Huffman on the same symbols) -- the magnitude bits are byte-identical in both and 6// cancel. No python / 3rd party. 7// T1 JPEG entropy lossless T2 modeled encode/decode lossless (jsym round-trips the coeffs) 8// T3 rANS(symbols) lossless T4 the modeling fix BEATS naive rANS (was 5571B) 9// T5 grade matches measured sizes (honest) T6 liar-kill: more bytes can't be a win 10// GREEN iff 6/6. knowledge/status/codec_rd2_gate.log. license_tier: ORIGINAL 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13import "nx_h264_bitwriter.nx" 14import "nx_dct8.nx" 15import "nx_quant_table.nx" 16import "nx_zigzag.nx" 17import "nx_jpeg_huff_enc.nx" 18import "nx_jpeg_block_enc.nx" 19import "nx_jpeg_sym.nx" 20import "nx_rans.nx" 21 22const GRADE_BEHIND: i64 = 1 23const GRADE_PARITY: i64 = 2 24const GRADE_EXCEEDS: i64 = 4 25 26func g_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } 27func g_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 28func g_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 } 29func qrnd(v: i64, q: i64) -> i64 { if v>=0 { return (v + q/2)/q } return 0 - (((0-v) + q/2)/q) } 30func rd_grade(our: i64, ref: i64) -> i64 { if our < ref { return GRADE_EXCEEDS } if our > ref { return GRADE_BEHIND } return GRADE_PARITY } 31func g_grade(gr: i64) -> i64 { if gr==4 { g_puts("EXCEEDS" as *u8) } if gr==2 { g_puts("PARITY" as *u8) } if gr==1 { g_puts("BEHIND" as *u8) } return 0 } 32 33func main() -> i64 { 34 g_puts("=== MODELED RD HEAD-TO-HEAD: rANS vs JPEG-Huffman on identical SYMBOLS (DC-diff+RLE) ===\n" as *u8) 35 let W: i64=256; let H: i64=256; let sb: i64=12 36 let npx: i64=W*H; let nblk: i64=(W/8)*(H/8); let ncoef: i64=nblk*64 37 38 let img: *i64=sys_mmap(npx*8) as *i64 39 var y: i64=0 40 while y<H { var x: i64=0; while x<W { var v: i64=(x+y)/2; if x>=84 { if x<172 { if y>=84 { if y<172 { v=210 } } } } img[y*W+x]=v; x=x+1 } y=y+1 } 41 42 let dcB: *i64=sys_mmap(20*8) as *i64; let dcV: *i64=sys_mmap(20*8) as *i64 43 let acB: *i64=sys_mmap(20*8) as *i64; let acV: *i64=sys_mmap(200*8) as *i64 44 let ndc: i64=jhe_dc_bits(dcB); jhe_dc_val(dcV); let nac: i64=jhe_ac_bits(acB); jhe_ac_val(acV) 45 let dcCO: *i64=sys_mmap(256*8) as *i64; let dcSI: *i64=sys_mmap(256*8) as *i64 46 let acCO: *i64=sys_mmap(256*8) as *i64; let acSI: *i64=sys_mmap(256*8) as *i64 47 jhe_gen(dcB, dcV, ndc, dcCO, dcSI); jhe_gen(acB, acV, nac, acCO, acSI) 48 let qt: *i64=sys_mmap(64*8) as *i64; nx_qt_luma(qt, 50) 49 let to_zz: *i64=sys_mmap(64*8) as *i64; let from_zz: *i64=sys_mmap(64*8) as *i64; nx_zigzag_init(to_zz, from_zz) 50 let DM: *i64=sys_mmap(64*8) as *i64; nx_dct8_init(DM) 51 let scratch: *i64=sys_mmap(64*8) as *i64; let ti: *i64=sys_mmap(8*8) as *i64; let to: *i64=sys_mmap(8*8) as *i64 52 53 // forward every block -> shared zigzag quantized coeffs qall 54 let qall: *i64=sys_mmap(ncoef*8) as *i64 55 let sh: *i64=sys_mmap(64*8) as *i64; let co: *i64=sys_mmap(64*8) as *i64; let q: *i64=sys_mmap(64*8) as *i64; let zz: *i64=sys_mmap(64*8) as *i64 56 var by: i64=0 57 while by < H/8 { var bx: i64=0; while bx < W/8 { 58 let bidx: i64=by*(W/8)+bx 59 var r: i64=0; while r<8 { var c: i64=0; while c<8 { sh[r*8+c]=img[(by*8+r)*W+(bx*8+c)]-128; c=c+1 } r=r+1 } 60 nx_dct8_forward_2d(DM, sh, co, scratch, ti, to) 61 var i: i64=0; while i<64 { q[i]=qrnd(co[i], qt[i]); i=i+1 } 62 nx_zigzag_scan(q, to_zz, zz) 63 i=0; while i<64 { qall[bidx*64+i]=zz[i]; i=i+1 } 64 bx=bx+1 } by=by+1 } 65 66 // --- JPEG-HUFFMAN (modeling + Huffman, interleaved) --- 67 let ebuf: *u8=sys_mmap(npx + 4096); let bw: *BitWriter=sys_mmap(64) as *BitWriter; bw_init(bw, ebuf, npx + 4096) 68 var prevdc: i64=0; var b: i64=0 69 while b<nblk { prevdc=jbe_encode_block(bw, dcCO, dcSI, acCO, acSI, ((qall as i64)+(b*64*8)) as *i64, prevdc); b=b+1 } 70 while bw.bit_pos != 0 { bw_write_bit(bw, 1) } 71 let jpeg_bytes: i64=bw.byte_pos 72 let pos: *i64=sys_mmap(8) as *i64; pos[0]=0; let zzd: *i64=sys_mmap(64*8) as *i64 73 var jpeg_ok: i64=1; var pdc: i64=0; b=0 74 while b<nblk { pdc=jbe_decode_block(ebuf, pos, dcCO, dcSI, acCO, acSI, pdc, zzd); var i: i64=0; while i<64 { if zzd[i]!=qall[b*64+i] { jpeg_ok=0 } i=i+1 } b=b+1 } 75 76 // --- MODELED rANS: same modeling -> symbol bytes + magnitude bits; rANS the symbols --- 77 let sym: *u8=sys_mmap(nblk*70 + 1024); let sp: *i64=sys_mmap(8) as *i64; sp[0]=0 78 let magw: *BitWriter=sys_mmap(64) as *BitWriter; let magbuf: *u8=sys_mmap(npx + 4096); bw_init(magw, magbuf, npx + 4096) 79 prevdc=0; b=0 80 while b<nblk { prevdc=jsym_encode_block(sym, sp, magw, ((qall as i64)+(b*64*8)) as *i64, prevdc); b=b+1 } 81 let nsym: i64=sp[0] 82 let magbytes: i64=(bw_total_bits(magw)+7)/8 83 // rANS the symbol stream 84 let counts: *i64=sys_mmap(257*8) as *i64; let fr: *i64=sys_mmap(257*8) as *i64; let cu: *i64=sys_mmap(257*8) as *i64; let s2s: *i64=sys_mmap((1<<sb)*8) as *i64 85 rans_count(sym, nsym, counts); rans_normalize(counts, fr, sb); rans_cum(fr, cu); rans_build_slot2sym(fr, cu, s2s) 86 let cap: i64=nsym + (nsym/2) + 1024; let senc: *u8=sys_mmap(cap) 87 let sym_payload: i64=rans_encode(sym, nsym, fr, cu, sb, senc, cap) 88 let modeled_total: i64=sym_payload + magbytes 89 // rANS lossless on symbols 90 let sdec: *u8=sys_mmap(nblk*70 + 1024); rans_decode(senc, sym_payload, nsym, fr, cu, sb, s2s, sdec) 91 var rans_ok: i64=1; var i3: i64=0; while i3<nsym { if sdec[i3]!=sym[i3] { rans_ok=0 } i3=i3+1 } 92 // modeled encode/decode lossless: jsym decode the symbols+magbits -> recover qall 93 let sp2: *i64=sys_mmap(8) as *i64; sp2[0]=0; let mpos: *i64=sys_mmap(8) as *i64; mpos[0]=0 94 var model_ok: i64=1; var mdc: i64=0; let zzm: *i64=sys_mmap(64*8) as *i64; b=0 95 while b<nblk { mdc=jsym_decode_block(sym, sp2, magbuf, mpos, mdc, zzm); var i: i64=0; while i<64 { if zzm[i]!=qall[b*64+i] { model_ok=0 } i=i+1 } b=b+1 } 96 97 let grade: i64=rd_grade(modeled_total, jpeg_bytes) 98 99 g_puts("-- image " as *u8); g_num(W); g_puts("x" as *u8); g_num(H); g_puts(" blocks=" as *u8); g_num(nblk); g_puts(" symbols=" as *u8); g_num(nsym); g_puts("\n" as *u8) 100 g_puts("-- IDENTICAL symbols + magnitude bits, equal quality. Entropy output bytes:\n" as *u8) 101 g_puts(" JPEG-Huffman : " as *u8); g_num(jpeg_bytes); g_puts("B\n" as *u8) 102 g_puts(" our rANS(model): " as *u8); g_num(modeled_total); g_puts("B (symbols " as *u8); g_num(sym_payload); g_puts(" + magbits " as *u8); g_num(magbytes); g_puts(")\n" as *u8) 103 g_puts(" vs naive rANS : 5571B (the un-modeled floor)\n" as *u8) 104 g_puts(" GRADE (rANS-modeled vs JPEG-Huffman, equal quality): " as *u8); g_grade(grade); g_puts("\n" as *u8) 105 106 var pass: i64=0; let rows: i64=6 107 pass=pass+g_check(" T1 JPEG entropy lossless" as *u8, jpeg_ok) 108 pass=pass+g_check(" T2 modeled encode/decode lossless (coeffs recovered)" as *u8, model_ok) 109 pass=pass+g_check(" T3 rANS(symbols) lossless" as *u8, rans_ok) 110 var t4: i64=0; if modeled_total < 5571 { t4=1 } 111 pass=pass+g_check(" T4 modeling fix BEATS naive rANS (<5571B)" as *u8, t4) 112 var t5: i64=0 113 if modeled_total < jpeg_bytes { if grade==GRADE_EXCEEDS { t5=1 } } 114 if modeled_total > jpeg_bytes { if grade==GRADE_BEHIND { t5=1 } } 115 if modeled_total == jpeg_bytes { if grade==GRADE_PARITY { t5=1 } } 116 pass=pass+g_check(" T5 grade matches measured sizes (honest)" as *u8, t5) 117 var t6: i64=0; if rd_grade(900, 800)==GRADE_BEHIND { t6=1 } 118 pass=pass+g_check(" T6 liar-kill: more bytes can NOT be graded a win" as *u8, t6) 119 120 g_puts("----\nRD2 rows=" as *u8); g_num(rows); g_puts(" pass=" as *u8); g_num(pass); g_puts("\n" as *u8) 121 let lg: i64=sys_openat_append("knowledge/status/codec_rd2_gate.log" as *u8, 0x1a4) 122 if lg>=0 { g_w(lg, "RD2 rows=" as *u8); g_wn(lg, rows); g_w(lg, " pass=" as *u8); g_wn(lg, pass); g_w(lg, " jpeg=" as *u8); g_wn(lg, jpeg_bytes); g_w(lg, " modeled=" as *u8); g_wn(lg, modeled_total); g_w(lg, " grade=" as *u8); g_wn(lg, grade); if pass==rows { g_w(lg, " verdict=GREEN\n" as *u8) } else { g_w(lg, " verdict=RED\n" as *u8) } sys_close(lg) } 123 if pass==rows { g_puts("RD2 GREEN (modeled rANS-vs-JPEG head-to-head, honest grade)\n" as *u8); sys_exit(0); return 0 } 124 g_puts("RD2 RED\n" as *u8); sys_exit(1); return 1 125}