nx_h264_deblock.nx source
↩ module page · 177 lines · 7445 B
1// nx_h264_deblock.nx -- H.264 in-loop deblocking, normal luma edge filter
2// (spec 8.7.2.3) + the alpha/beta (Table 8-16) and tc0 (Table 8-17) threshold
3// tables. Smooths block edges; this is the stage that was producing the <=1
4// pixel diffs vs ffmpeg at internal 4x4 edges.
5//
6// Edge line = [p3,p2,p1,p0,q0,q1,q2,q3] across the edge; the normal filter
7// (bS 1..3) modifies p1,p0,q0,q1 in place. indexA/indexB = qP (filter offsets 0).
8// filter if |p0-q0|<alpha && |p1-p0|<beta && |q1-q0|<beta
9// ap=|p2-p0| aq=|q2-q0| ; tc = tc0 + (ap<beta) + (aq<beta)
10// delta = Clip3(-tc,tc, ((q0-p0)*4 + (p1-q1) + 4) >> 3)
11// p0+=delta ; q0-=delta (clip 0..255)
12// if ap<beta: p1 += Clip3(-tc0,tc0,(p2 + ((p0o+q0o+1)>>1) - 2*p1)>>1)
13// if aq<beta: q1 += Clip3(-tc0,tc0,(q2 + ((p0o+q0o+1)>>1) - 2*q1)>>1)
14//
15// genealogy_id: itu_t_h264_sec8_7_deblocking + table8_16_alpha_beta + table8_17_tc0
16// lineage_id: normal_luma_edge_filter + threshold_tables
17// license_tier: ORIGINAL
18import "nx_syscalls.nx"
19
20func db_asr(x: i64, n: i64) -> i64 {
21 if x >= 0 { return x >> n }
22 let p: i64 = 0 - x
23 return 0 - ((p + ((1 << n) - 1)) >> n)
24}
25func db_clip3(lo: i64, hi: i64, v: i64) -> i64 {
26 if v < lo { return lo }
27 if v > hi { return hi }
28 return v
29}
30func db_clip1(v: i64) -> i64 {
31 if v < 0 { return 0 }
32 if v > 255 { return 255 }
33 return v
34}
35func db_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
36
37func nx_db_alpha(idx: i64) -> i64 {
38 let a: *i64 = sys_mmap(52 * 8) as *i64
39 a[0]=0;a[1]=0;a[2]=0;a[3]=0;a[4]=0;a[5]=0;a[6]=0;a[7]=0;a[8]=0;a[9]=0;a[10]=0;a[11]=0;a[12]=0;a[13]=0;a[14]=0;a[15]=0
40 a[16]=4;a[17]=4;a[18]=5;a[19]=6;a[20]=7;a[21]=8;a[22]=9;a[23]=10;a[24]=12;a[25]=13;a[26]=15;a[27]=17;a[28]=20;a[29]=22;a[30]=25;a[31]=28
41 a[32]=32;a[33]=36;a[34]=40;a[35]=45;a[36]=50;a[37]=56;a[38]=63;a[39]=71;a[40]=80;a[41]=90;a[42]=101;a[43]=113;a[44]=127;a[45]=144;a[46]=162;a[47]=182
42 a[48]=203;a[49]=226;a[50]=255;a[51]=255
43 if idx < 0 { return 0 }
44 if idx > 51 { return 255 }
45 return a[idx]
46}
47func nx_db_beta(idx: i64) -> i64 {
48 let b: *i64 = sys_mmap(52 * 8) as *i64
49 b[0]=0;b[1]=0;b[2]=0;b[3]=0;b[4]=0;b[5]=0;b[6]=0;b[7]=0;b[8]=0;b[9]=0;b[10]=0;b[11]=0;b[12]=0;b[13]=0;b[14]=0;b[15]=0
50 b[16]=2;b[17]=2;b[18]=2;b[19]=3;b[20]=3;b[21]=3;b[22]=3;b[23]=4;b[24]=4;b[25]=4;b[26]=6;b[27]=6;b[28]=7;b[29]=7;b[30]=8;b[31]=8
51 b[32]=9;b[33]=9;b[34]=10;b[35]=10;b[36]=11;b[37]=11;b[38]=12;b[39]=12;b[40]=13;b[41]=13;b[42]=14;b[43]=14;b[44]=15;b[45]=15;b[46]=16;b[47]=16
52 b[48]=17;b[49]=17;b[50]=18;b[51]=18
53 if idx < 0 { return 0 }
54 if idx > 51 { return 18 }
55 return b[idx]
56}
57// tc0 for bS in {1,2,3} (Table 8-17), indexA 0..51
58func nx_db_tc0(bs: i64, idx: i64) -> i64 {
59 let t: *i64 = sys_mmap(52 * 8) as *i64
60 var i: i64 = 0
61 while i < 52 { t[i] = 0; i = i + 1 }
62 if bs == 1 {
63 t[23]=1;t[24]=1;t[25]=1;t[26]=1;t[27]=1;t[28]=1;t[29]=1;t[30]=1;t[31]=1;t[32]=1;t[33]=2;t[34]=2;t[35]=2;t[36]=2;t[37]=3;t[38]=3;t[39]=3
64 t[40]=4;t[41]=4;t[42]=4;t[43]=5;t[44]=6;t[45]=6;t[46]=7;t[47]=8;t[48]=9;t[49]=10;t[50]=11;t[51]=13
65 }
66 if bs == 2 {
67 t[21]=1;t[22]=1;t[23]=1;t[24]=1;t[25]=1;t[26]=1;t[27]=1;t[28]=1;t[29]=1;t[30]=1;t[31]=2;t[32]=2;t[33]=2;t[34]=2;t[35]=3;t[36]=3;t[37]=3
68 t[38]=4;t[39]=4;t[40]=5;t[41]=5;t[42]=6;t[43]=7;t[44]=8;t[45]=8;t[46]=10;t[47]=11;t[48]=12;t[49]=13;t[50]=15;t[51]=17
69 }
70 if bs == 3 {
71 t[17]=1;t[18]=1;t[19]=1;t[20]=1;t[21]=1;t[22]=1;t[23]=1;t[24]=1;t[25]=1;t[26]=1;t[27]=2;t[28]=2;t[29]=2;t[30]=2;t[31]=3;t[32]=3;t[33]=3
72 t[34]=4;t[35]=4;t[36]=4;t[37]=5;t[38]=6;t[39]=6;t[40]=7;t[41]=8;t[42]=9;t[43]=10;t[44]=11;t[45]=13;t[46]=14;t[47]=16;t[48]=18;t[49]=20;t[50]=23;t[51]=25
73 }
74 if idx < 0 { return 0 }
75 if idx > 51 { return t[51] }
76 return t[idx]
77}
78
79// normal luma edge filter on s=[p3,p2,p1,p0,q0,q1,q2,q3] in place (bS 1..3).
80func nx_deblock_luma_edge(s: *i64, bs: i64, qp: i64) -> i64 {
81 if bs <= 0 { return 0 }
82 if bs >= 4 { return 0 } // strong filter (bS=4) not handled here
83 let p2: i64 = s[1]
84 let p1: i64 = s[2]
85 let p0: i64 = s[3]
86 let q0: i64 = s[4]
87 let q1: i64 = s[5]
88 let q2: i64 = s[6]
89 let alpha: i64 = nx_db_alpha(qp)
90 let beta: i64 = nx_db_beta(qp)
91 let tc0: i64 = nx_db_tc0(bs, qp)
92 if db_abs(p0 - q0) >= alpha { return 0 }
93 if db_abs(p1 - p0) >= beta { return 0 }
94 if db_abs(q1 - q0) >= beta { return 0 }
95 let ap: i64 = db_abs(p2 - p0)
96 let aq: i64 = db_abs(q2 - q0)
97 var tc: i64 = tc0
98 if ap < beta { tc = tc + 1 }
99 if aq < beta { tc = tc + 1 }
100 let delta: i64 = db_clip3(0 - tc, tc, db_asr((q0 - p0) * 4 + (p1 - q1) + 4, 3))
101 let avg: i64 = (p0 + q0 + 1) >> 1
102 s[3] = db_clip1(p0 + delta)
103 s[4] = db_clip1(q0 - delta)
104 if ap < beta { s[2] = p1 + db_clip3(0 - tc0, tc0, db_asr(p2 + avg - 2 * p1, 1)) }
105 if aq < beta { s[5] = q1 + db_clip3(0 - tc0, tc0, db_asr(q2 + avg - 2 * q1, 1)) }
106 return 0
107}
108
109// strong luma edge filter (bS=4, spec 8.7.2.4) on s=[p3,p2,p1,p0,q0,q1,q2,q3] in place.
110// All taps are convex combinations of valid samples -> already in [0,255].
111func nx_deblock_luma_bs4(s: *i64, qp: i64) -> i64 {
112 let p3: i64 = s[0]
113 let p2: i64 = s[1]
114 let p1: i64 = s[2]
115 let p0: i64 = s[3]
116 let q0: i64 = s[4]
117 let q1: i64 = s[5]
118 let q2: i64 = s[6]
119 let q3: i64 = s[7]
120 let alpha: i64 = nx_db_alpha(qp)
121 let beta: i64 = nx_db_beta(qp)
122 if db_abs(p0 - q0) >= alpha { return 0 }
123 if db_abs(p1 - p0) >= beta { return 0 }
124 if db_abs(q1 - q0) >= beta { return 0 }
125 let ap: i64 = db_abs(p2 - p0)
126 let aq: i64 = db_abs(q2 - q0)
127 let thr: i64 = (alpha >> 2) + 2
128 // p side
129 var pstrong: i64 = 0
130 if ap < beta { if db_abs(p0 - q0) < thr { pstrong = 1 } }
131 if pstrong == 1 {
132 s[3] = (p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3
133 s[2] = (p2 + p1 + p0 + q0 + 2) >> 2
134 s[1] = (2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3
135 }
136 if pstrong == 0 {
137 s[3] = (2 * p1 + p0 + q1 + 2) >> 2
138 }
139 // q side
140 var qstrong: i64 = 0
141 if aq < beta { if db_abs(p0 - q0) < thr { qstrong = 1 } }
142 if qstrong == 1 {
143 s[4] = (q2 + 2 * q1 + 2 * q0 + 2 * p0 + p1 + 4) >> 3
144 s[5] = (q2 + q1 + q0 + p0 + 2) >> 2
145 s[6] = (2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3
146 }
147 if qstrong == 0 {
148 s[4] = (2 * q1 + q0 + p1 + 2) >> 2
149 }
150 return 0
151}
152
153// chroma edge filter (spec 8.7.2.4) on s=[p1,p0,q0,q1] in place; only p0 (s[1]) and q0 (s[2]) change.
154// Normal (bS 1..3): tc = tc0(bS)+1, delta = clip3(-tc,tc,((q0-p0)*4+(p1-q1)+4)>>3). Strong (bS=4):
155// p0'=(2p1+p0+q1+2)>>2, q0'=(2q1+q0+p1+2)>>2. qp = chroma QPc. bS<=0 -> no filter.
156func nx_deblock_chroma_edge(s: *i64, bs: i64, qp: i64) -> i64 {
157 if bs <= 0 { return 0 }
158 let p1: i64 = s[0]
159 let p0: i64 = s[1]
160 let q0: i64 = s[2]
161 let q1: i64 = s[3]
162 let alpha: i64 = nx_db_alpha(qp)
163 let beta: i64 = nx_db_beta(qp)
164 if db_abs(p0 - q0) >= alpha { return 0 }
165 if db_abs(p1 - p0) >= beta { return 0 }
166 if db_abs(q1 - q0) >= beta { return 0 }
167 if bs >= 4 {
168 s[1] = (2 * p1 + p0 + q1 + 2) >> 2
169 s[2] = (2 * q1 + q0 + p1 + 2) >> 2
170 return 0
171 }
172 let tc: i64 = nx_db_tc0(bs, qp) + 1
173 let delta: i64 = db_clip3(0 - tc, tc, db_asr((q0 - p0) * 4 + (p1 - q1) + 4, 3))
174 s[1] = db_clip1(p0 + delta)
175 s[2] = db_clip1(q0 - delta)
176 return 0
177}