nx_av1_adst.nx source

1// nx_av1_adst.nx -- AV1/AV2 inverse ADST (asymmetric discrete sine transform). 2// 3// AV1 picks between DCT and ADST per block and per axis, because they have 4// opposite shapes at the block edge: the DCT's first basis is FLAT, the ADST's 5// is a RISING RAMP. Intra-predicted residuals are typically small next to the 6// predicted edge and grow with distance from it, which is exactly the ADST's 7// basis-0 shape -- that is the whole reason it exists in the codec. 8// 9// SUBSTITUTING ONE FOR THE OTHER DECODES. That is the danger: a block coded 10// with ADST and reconstructed with DCT produces a plausible image with wrong 11// gradients near block edges, which reads as banding or a soft grid rather 12// than as corruption. Nothing errors. 13// 14// THE SINPI CONSTANTS ARE NOT THE COSINE TABLE. ADST uses sin(k*pi/9) at 15// 12-bit -- 1321, 2482, 3344, 3803 -- which appear nowhere in the DCT's 16// quarter-period cosine table. Reaching for cos128() here silently builds a 17// transform with the wrong basis entirely. 18// 19// THE OUTPUT OF A BASIS-0 IMPULSE IS THE CONSTANT SET ITSELF. Feeding 20// [4096,0,0,0] returns exactly [1321, 2482, 3344, 3803] -- the four constants, 21// in order. That makes the transform exactly checkable without reference 22// vectors, and it is what the gate asserts. 23// 24// genealogy_id: av1_spec_7_13_2_6_inverse_adst 25// lineage_id: nx_av1_adst_v1 26// license_tier: ORIGINAL 27 28import "nx_syscalls.nx" 29import "nx_av1_txfm.nx" 30 31const NX_SINPI_1_9: i64 = 1321 32const NX_SINPI_2_9: i64 = 2482 33const NX_SINPI_3_9: i64 = 3344 34const NX_SINPI_4_9: i64 = 3803 35 36// ===== 4-point inverse ADST ======================================= 37// 38// Operates in place on four coefficients. Every local is declared once -- 39// nx_cc uses flat function scope. 40 41func nx_av1_iadst4(io: *i64) -> i64 { 42 let t0: i64 = io[0] 43 let t1: i64 = io[1] 44 let t2: i64 = io[2] 45 let t3: i64 = io[3] 46 47 var s0: i64 = NX_SINPI_1_9 * t0 48 var s1: i64 = NX_SINPI_2_9 * t0 49 let s2in: i64 = NX_SINPI_3_9 * t1 50 let s3in: i64 = NX_SINPI_4_9 * t2 51 let s4: i64 = NX_SINPI_1_9 * t2 52 let s5: i64 = NX_SINPI_2_9 * t3 53 let s6: i64 = NX_SINPI_4_9 * t3 54 55 // the cross term: this is what makes the basis asymmetric 56 let a7: i64 = t0 - t2 57 let b7: i64 = a7 + t3 58 59 s0 = s0 + s3in 60 s1 = s1 - s4 61 let s3: i64 = s2in 62 let s2: i64 = NX_SINPI_3_9 * b7 63 s0 = s0 + s5 64 s1 = s1 - s6 65 66 let x0: i64 = s0 + s3 67 let x1: i64 = s1 + s3 68 let x2: i64 = s2 69 let x3: i64 = s0 + s1 - s3 70 71 io[0] = nx_av1_round2(x0, NX_TX_COS_BITS) 72 io[1] = nx_av1_round2(x1, NX_TX_COS_BITS) 73 io[2] = nx_av1_round2(x2, NX_TX_COS_BITS) 74 io[3] = nx_av1_round2(x3, NX_TX_COS_BITS) 75 return 1 76} 77 78// ===== the transform-type selector ================================ 79// 80// AV1 chooses a transform PER AXIS: a block can be ADST vertically and DCT 81// horizontally. The pair is what the tx_type names, and getting the axes 82// swapped produces a transposed-looking error that survives on square, 83// symmetric content and shows up on everything else. 84 85const NX_TXT_DCT_DCT: i64 = 0 86const NX_TXT_ADST_DCT: i64 = 1 87const NX_TXT_DCT_ADST: i64 = 2 88const NX_TXT_ADST_ADST: i64 = 3 89 90// 1 if this transform type uses ADST on the ROW (horizontal) pass 91func nx_av1_txt_row_is_adst(tx_type: i64) -> i64 { 92 if tx_type == NX_TXT_DCT_ADST { return 1 } 93 if tx_type == NX_TXT_ADST_ADST { return 1 } 94 return 0 95} 96 97// 1 if this transform type uses ADST on the COLUMN (vertical) pass 98func nx_av1_txt_col_is_adst(tx_type: i64) -> i64 { 99 if tx_type == NX_TXT_ADST_DCT { return 1 } 100 if tx_type == NX_TXT_ADST_ADST { return 1 } 101 return 0 102}