nx_vtransform_dct.nx source
↩ module page · 139 lines · 6294 B
1// nx_vtransform_dct.nx -- Performs 4x4 integer DCT and position-dependent quantization for video compression.
2const K_MAGIC_13107: i64 = 13107
3const K_MAGIC_5243: i64 = 5243
4const K_MAGIC_8066: i64 = 8066
5const K_MAGIC_11916: i64 = 11916
6const K_MAGIC_4660: i64 = 4660
7const K_MAGIC_7490: i64 = 7490
8const K_MAGIC_10082: i64 = 10082
9const K_MAGIC_4194: i64 = 4194
10const K_MAGIC_6554: i64 = 6554
11const K_MAGIC_9362: i64 = 9362
12const K_MAGIC_3647: i64 = 3647
13const K_MAGIC_5825: i64 = 5825
14const K_MAGIC_8192: i64 = 8192
15const K_MAGIC_3355: i64 = 3355
16const K_MAGIC_7282: i64 = 7282
17const K_MAGIC_2893: i64 = 2893
18const K_MAGIC_4559: i64 = 4559
19// nx_vtransform_dct.nx -- sovereign 4x4 integer DCT (the H.264/AVC "core" transform) + its position-dependent
20// quantization. The current codec uses a Walsh-Hadamard transform (nx_vtransform); WHT is exactly invertible but
21// compacts natural-image energy worse than the DCT (RT-003 R1b names the DCT swap as the biggest remaining intra
22// lever). The H.264 core transform has non-uniform basis norms, so it needs the standard Mf (forward) / Vi (dequant)
23// scaling tables indexed by (qp%6, coefficient-position class) -- a scalar quant cannot normalize it. Pure integer,
24// no FPU/syscalls (stays wasm-clean so the codec can adopt it). Block = 16 i64 row-major (a 4x4 residual).
25// license_tier: ORIGINAL
26
27func vtd_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
28
29// ---- core transform butterflies (H.264 Cf forward / Ci inverse) ----
30// forward 1D on (b[base], b[base+st], b[base+2st], b[base+3st]) in place: y = Cf . x
31func vtd_fwd1(b: *i64, base: i64, st: i64) -> i64 {
32 let x0: i64 = b[base]; let x1: i64 = b[base+st]; let x2: i64 = b[base+2*st]; let x3: i64 = b[base+3*st]
33 let t0: i64 = x0 + x3; let t1: i64 = x1 + x2; let t2: i64 = x1 - x2; let t3: i64 = x0 - x3
34 b[base] = t0 + t1
35 b[base+st] = 2*t3 + t2
36 b[base+2*st] = t0 - t1
37 b[base+3*st] = t3 - 2*t2
38 return 0
39}
40// inverse 1D on the same 4 elements in place: x = Ci . y (note the >>1 on the odd basis, matched by 2* in forward)
41func vtd_inv1(b: *i64, base: i64, st: i64) -> i64 {
42 let y0: i64 = b[base]; let y1: i64 = b[base+st]; let y2: i64 = b[base+2*st]; let y3: i64 = b[base+3*st]
43 let t0: i64 = y0 + y2
44 let t1: i64 = y0 - y2
45 let t2: i64 = (y1 / 2) - y3
46 let t3: i64 = y1 + (y3 / 2)
47 b[base] = t0 + t3
48 b[base+st] = t1 + t2
49 b[base+2*st] = t1 - t2
50 b[base+3*st] = t0 - t3
51 return 0
52}
53// 2D forward: rows then columns.
54func vtd_fwd(blk: *i64) -> i64 {
55 var r: i64 = 0; while r < 4 { vtd_fwd1(blk, r*4, 1); r = r + 1 }
56 var c: i64 = 0; while c < 4 { vtd_fwd1(blk, c, 4); c = c + 1 }
57 return 0
58}
59// 2D inverse core transform: columns then rows (does NOT include the final >>6 -- the dequant path applies it).
60func vtd_inv(blk: *i64) -> i64 {
61 var c: i64 = 0; while c < 4 { vtd_inv1(blk, c, 4); c = c + 1 }
62 var r: i64 = 0; while r < 4 { vtd_inv1(blk, r*4, 1); r = r + 1 }
63 return 0
64}
65
66// ---- position class + scaling tables ----
67// class 0 = {(0,0),(0,2),(2,0),(2,2)} (even,even); class 1 = {(1,1),(1,3),(3,1),(3,3)} (odd,odd); class 2 = rest.
68func vtd_class(i: i64, j: i64) -> i64 {
69 if (i % 2) == 0 { if (j % 2) == 0 { return 0 } return 2 }
70 if (j % 2) == 1 { return 1 }
71 return 2
72}
73// H.264 MultFactor (forward quant), indexed idx = (qp%6)*3 + class.
74func vtd_mf(qm: i64, cls: i64) -> i64 {
75 let k: i64 = qm*3 + cls
76 if k==0 { return K_MAGIC_13107 } if k==1 { return K_MAGIC_5243 } if k==2 { return K_MAGIC_8066 }
77 if k==3 { return K_MAGIC_11916 } if k==4 { return K_MAGIC_4660 } if k==5 { return K_MAGIC_7490 }
78 if k==6 { return K_MAGIC_10082 } if k==7 { return K_MAGIC_4194 } if k==8 { return K_MAGIC_6554 }
79 if k==9 { return K_MAGIC_9362 } if k==10 { return K_MAGIC_3647 } if k==11 { return K_MAGIC_5825 }
80 if k==12 { return K_MAGIC_8192 } if k==13 { return K_MAGIC_3355 } if k==14 { return K_MAGIC_5243 }
81 if k==15 { return K_MAGIC_7282 } if k==16 { return K_MAGIC_2893 } if k==17 { return K_MAGIC_4559 }
82 return K_MAGIC_8192
83}
84// H.264 LevelScale (dequant), indexed idx = (qp%6)*3 + class.
85func vtd_vi(qm: i64, cls: i64) -> i64 {
86 let k: i64 = qm*3 + cls
87 if k==0 { return 10 } if k==1 { return 16 } if k==2 { return 13 }
88 if k==3 { return 11 } if k==4 { return 18 } if k==5 { return 14 }
89 if k==6 { return 13 } if k==7 { return 20 } if k==8 { return 16 }
90 if k==9 { return 14 } if k==10 { return 23 } if k==11 { return 18 }
91 if k==12 { return 16 } if k==13 { return 25 } if k==14 { return 20 }
92 if k==15 { return 18 } if k==16 { return 29 } if k==17 { return 23 }
93 return 16
94}
95
96// forward quant in place: blk holds the forward-transform coefficients W; replace with quantized levels Z.
97// Z = sign(W) * (|W|*Mf + f) >> qbits, qbits = 15 + qp/6, f = (1<<qbits)/3 (intra bias). returns nonzero count.
98func vtd_quant(blk: *i64, qp: i64) -> i64 {
99 let qm: i64 = qp % 6
100 let qsh: i64 = qp / 6
101 let qbits: i64 = 15 + qsh
102 let f: i64 = (1 << qbits) / 3
103 var nz: i64 = 0
104 var i: i64 = 0
105 while i < 4 {
106 var j: i64 = 0
107 while j < 4 {
108 let idx: i64 = i*4 + j
109 let cls: i64 = vtd_class(i, j)
110 let w: i64 = blk[idx]
111 let a: i64 = vtd_abs(w)
112 let z: i64 = (a * vtd_mf(qm, cls) + f) >> qbits
113 if w < 0 { blk[idx] = 0 - z } else { blk[idx] = z }
114 if z != 0 { nz = nz + 1 }
115 j = j + 1
116 }
117 i = i + 1
118 }
119 return nz
120}
121// dequant in place: blk holds quantized levels Z; replace with dequantized transform coeffs W' = Z*Vi << (qp/6).
122func vtd_dequant(blk: *i64, qp: i64) -> i64 {
123 let qm: i64 = qp % 6
124 let qsh: i64 = qp / 6
125 var i: i64 = 0
126 while i < 4 {
127 var j: i64 = 0
128 while j < 4 {
129 let idx: i64 = i*4 + j
130 let cls: i64 = vtd_class(i, j)
131 blk[idx] = (blk[idx] * vtd_vi(qm, cls)) << qsh
132 j = j + 1
133 }
134 i = i + 1
135 }
136 return 0
137}
138// final inverse scaling: after vtd_inv on dequantized coeffs, bring back to the pixel-residual domain: (e+32)>>6.
139func vtd_inv_scale(blk: *i64) -> i64 { var i: i64 = 0; while i < 16 { blk[i] = (blk[i] + 32) >> 6; i = i + 1 } return 0 }