nx_h264_dequant.nx source
↩ module page · 73 lines · 2711 B
1// nx_h264_dequant.nx -- H.264 inverse quantization / scaling for 4x4 residual
2// blocks (spec 8.5.12.1). Turns quantized levels c[] into the coefficient block
3// d[] that feeds the inverse transform (nx_h264_idct). Flat weightScale=16 (no
4// custom scaling list).
5//
6// m = QP % 6 ; shift = QP / 6
7// LevelScale = 16 * normAdjust4x4[m][posClass(i,j)]
8// QP >= 24: d = (c * LevelScale) << (shift - 4)
9// QP < 24: d = (c * LevelScale + (1 << (3 - shift))) >> (4 - shift)
10// posClass: 0 if (i,j both even), 1 if (both odd), else 2.
11//
12// normAdjust4x4 is the spec V matrix; verified by KAT (level 1 at QP0 -> 10/13/16
13// for the 3 position classes). Pure integer, table-driven.
14//
15// genealogy_id: itu_t_h264_sec8_5_12_1_scaling + normadjust4x4_matrix
16// lineage_id: qp_mod6_levelscale + round_shift_dequant
17// license_tier: ORIGINAL
18import "nx_syscalls.nx"
19
20func nx_dq_posclass(i: i64, j: i64) -> i64 {
21 if (i % 2) == 0 { if (j % 2) == 0 { return 0 } }
22 if (i % 2) == 1 { if (j % 2) == 1 { return 1 } }
23 return 2
24}
25
26// normAdjust4x4[m][posClass]
27// LEAK FIX 2026-08-01: this mmapped a page to build 18 CONSTANTS and returned ONE.
28// It runs 16x per nx_h264_dequant4x4 call, i.e. ~3.13 MILLION times for one 1080p
29// frame. sys_mmap here is page-granular with NO munmap, so that alone accounted for
30// ~12.8 GB of the measured 16 GB peak that near-wedged the host. A pure lookup must
31// not allocate. Same values, same m*3+pc indexing -- output is unchanged.
32func nx_dq_normadjust(m: i64, pc: i64) -> i64 {
33 let k: i64 = m * 3 + pc
34 if k == 0 { return 10 }
35 if k == 1 { return 16 }
36 if k == 2 { return 13 }
37 if k == 3 { return 11 }
38 if k == 4 { return 18 }
39 if k == 5 { return 14 }
40 if k == 6 { return 13 }
41 if k == 7 { return 20 }
42 if k == 8 { return 16 }
43 if k == 9 { return 14 }
44 if k == 10 { return 23 }
45 if k == 11 { return 18 }
46 if k == 12 { return 16 }
47 if k == 13 { return 25 }
48 if k == 14 { return 20 }
49 if k == 15 { return 18 }
50 if k == 16 { return 29 }
51 if k == 17 { return 23 }
52 return 0
53}
54
55// dequantize 16 levels c[] at quantization parameter qp into d[]
56func nx_h264_dequant4x4(c: *i64, qp: i64, d: *i64) -> i64 {
57 let m: i64 = qp % 6
58 let shift: i64 = qp / 6
59 var i: i64 = 0
60 while i < 4 {
61 var j: i64 = 0
62 while j < 4 {
63 let pc: i64 = nx_dq_posclass(i, j)
64 let ls: i64 = 16 * nx_dq_normadjust(m, pc)
65 let idx: i64 = i * 4 + j
66 if qp >= 24 { d[idx] = (c[idx] * ls) << (shift - 4) }
67 if qp < 24 { d[idx] = (c[idx] * ls + (1 << (3 - shift))) >> (4 - shift) }
68 j = j + 1
69 }
70 i = i + 1
71 }
72 return 0
73}