2 * Copyright (c) 2015 - 2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "libavutil/mips/generic_macros_msa.h"
22 #include "libavcodec/mips/hevcdsp_mips.h"
23 #include "libavcodec/mips/hevc_macros_msa.h"
25 static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = {
27 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
28 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
31 #define HEVC_BI_RND_CLIP2(in0, in1, vec0, vec1, rnd_val, out0, out1) \
33 ADDS_SH2_SH(vec0, in0, vec1, in1, out0, out1); \
34 SRARI_H2_SH(out0, out1, rnd_val); \
35 CLIP_SH2_0_255(out0, out1); \
38 #define HEVC_BI_RND_CLIP4(in0, in1, in2, in3, \
39 vec0, vec1, vec2, vec3, rnd_val, \
40 out0, out1, out2, out3) \
42 HEVC_BI_RND_CLIP2(in0, in1, vec0, vec1, rnd_val, out0, out1); \
43 HEVC_BI_RND_CLIP2(in2, in3, vec2, vec3, rnd_val, out2, out3); \
46 #define HEVC_BI_RND_CLIP2_MAX_SATU(in0, in1, vec0, vec1, rnd_val, \
49 ADDS_SH2_SH(vec0, in0, vec1, in1, out0, out1); \
50 SRARI_H2_SH(out0, out1, rnd_val); \
51 CLIP_SH2_0_255_MAX_SATU(out0, out1); \
54 #define HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, vec0, vec1, vec2, \
55 vec3, rnd_val, out0, out1, out2, out3) \
57 HEVC_BI_RND_CLIP2_MAX_SATU(in0, in1, vec0, vec1, rnd_val, out0, out1); \
58 HEVC_BI_RND_CLIP2_MAX_SATU(in2, in3, vec2, vec3, rnd_val, out2, out3); \
61 static void hevc_bi_copy_4w_msa(uint8_t *src0_ptr,
69 uint32_t loop_cnt, tp0, tp1, tp2, tp3;
70 uint64_t tpd0, tpd1, tpd2, tpd3;
71 v16i8 src0 = { 0 }, src1 = { 0 };
73 v8i16 in0 = { 0 }, in1 = { 0 }, in2 = { 0 }, in3 = { 0 };
74 v8i16 dst0, dst1, dst2, dst3;
77 LW2(src0_ptr, src_stride, tp0, tp1);
78 INSERT_W2_SB(tp0, tp1, src0);
79 LD2(src1_ptr, src2_stride, tpd0, tpd1);
80 INSERT_D2_SH(tpd0, tpd1, in0);
82 dst0 = (v8i16) __msa_ilvr_b(zero, src0);
85 dst0 = __msa_srari_h(dst0, 7);
86 dst0 = CLIP_SH_0_255_MAX_SATU(dst0);
88 dst0 = (v8i16) __msa_pckev_b((v16i8) dst0, (v16i8) dst0);
89 ST4x2_UB(dst0, dst, dst_stride);
90 } else if (4 == height) {
91 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
92 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
93 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
94 INSERT_D2_SH(tpd0, tpd1, in0);
95 INSERT_D2_SH(tpd2, tpd3, in1);
96 ILVRL_B2_SH(zero, src0, dst0, dst1);
97 SLLI_2V(dst0, dst1, 6);
98 HEVC_BI_RND_CLIP2_MAX_SATU(in0, in1, dst0, dst1, 7, dst0, dst1);
99 dst0 = (v8i16) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
100 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
101 } else if (0 == height % 8) {
102 for (loop_cnt = (height >> 3); loop_cnt--;) {
103 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
104 src0_ptr += 4 * src_stride;
105 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
106 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
107 src0_ptr += 4 * src_stride;
108 INSERT_W4_SB(tp0, tp1, tp2, tp3, src1);
109 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
110 src1_ptr += (4 * src2_stride);
111 INSERT_D2_SH(tpd0, tpd1, in0);
112 INSERT_D2_SH(tpd2, tpd3, in1);
113 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
114 src1_ptr += (4 * src2_stride);
115 INSERT_D2_SH(tpd0, tpd1, in2);
116 INSERT_D2_SH(tpd2, tpd3, in3);
117 ILVRL_B2_SH(zero, src0, dst0, dst1);
118 ILVRL_B2_SH(zero, src1, dst2, dst3);
119 SLLI_4V(dst0, dst1, dst2, dst3, 6);
120 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2,
121 dst3, 7, dst0, dst1, dst2, dst3);
122 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
123 ST4x8_UB(dst0, dst1, dst, dst_stride);
124 dst += (8 * dst_stride);
129 static void hevc_bi_copy_6w_msa(uint8_t *src0_ptr,
138 uint64_t tp0, tp1, tp2, tp3;
139 v16u8 out0, out1, out2, out3;
141 v16i8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 }, src3 = { 0 };
142 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
143 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
145 for (loop_cnt = (height >> 3); loop_cnt--;) {
146 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
147 src0_ptr += (4 * src_stride);
148 INSERT_D2_SB(tp0, tp1, src0);
149 INSERT_D2_SB(tp2, tp3, src1);
150 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
151 src0_ptr += (4 * src_stride);
152 INSERT_D2_SB(tp0, tp1, src2);
153 INSERT_D2_SB(tp2, tp3, src3);
154 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
155 src1_ptr += (8 * src2_stride);
156 ILVRL_B2_SH(zero, src0, dst0, dst1);
157 ILVRL_B2_SH(zero, src1, dst2, dst3);
158 ILVRL_B2_SH(zero, src2, dst4, dst5);
159 ILVRL_B2_SH(zero, src3, dst6, dst7);
160 SLLI_4V(dst0, dst1, dst2, dst3, 6);
161 SLLI_4V(dst4, dst5, dst6, dst7, 6);
162 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
163 7, dst0, dst1, dst2, dst3);
164 HEVC_BI_RND_CLIP4_MAX_SATU(in4, in5, in6, in7, dst4, dst5, dst6, dst7,
165 7, dst4, dst5, dst6, dst7);
166 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
167 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
168 ST6x4_UB(out0, out1, dst, dst_stride);
169 dst += (4 * dst_stride);
170 ST6x4_UB(out2, out3, dst, dst_stride);
171 dst += (4 * dst_stride);
175 static void hevc_bi_copy_8w_msa(uint8_t *src0_ptr,
183 uint64_t tp0, tp1, tp2, tp3;
184 v16u8 out0, out1, out2, out3;
185 v16i8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 }, src3 = { 0 };
187 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
188 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
191 LD2(src0_ptr, src_stride, tp0, tp1);
192 INSERT_D2_SB(tp0, tp1, src0);
193 LD_SH2(src1_ptr, src2_stride, in0, in1);
194 ILVRL_B2_SH(zero, src0, dst0, dst1);
195 SLLI_2V(dst0, dst1, 6);
196 HEVC_BI_RND_CLIP2_MAX_SATU(in0, in1, dst0, dst1, 7, dst0, dst1);
197 out0 = (v16u8) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
198 ST8x2_UB(out0, dst, dst_stride);
199 } else if (4 == height) {
200 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
201 INSERT_D2_SB(tp0, tp1, src0);
202 INSERT_D2_SB(tp2, tp3, src1);
203 ILVRL_B2_SH(zero, src0, dst0, dst1);
204 ILVRL_B2_SH(zero, src1, dst2, dst3);
205 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
206 SLLI_4V(dst0, dst1, dst2, dst3, 6);
207 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
208 7, dst0, dst1, dst2, dst3);
209 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
210 ST8x4_UB(out0, out1, dst, dst_stride);
211 } else if (6 == height) {
212 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
213 src0_ptr += 4 * src_stride;
214 INSERT_D2_SB(tp0, tp1, src0);
215 INSERT_D2_SB(tp2, tp3, src1);
216 LD2(src0_ptr, src_stride, tp0, tp1);
217 INSERT_D2_SB(tp0, tp1, src2);
218 ILVRL_B2_SH(zero, src0, dst0, dst1);
219 ILVRL_B2_SH(zero, src1, dst2, dst3);
220 ILVRL_B2_SH(zero, src2, dst4, dst5);
221 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
222 SLLI_4V(dst0, dst1, dst2, dst3, 6);
223 SLLI_2V(dst4, dst5, 6);
224 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
225 7, dst0, dst1, dst2, dst3);
226 HEVC_BI_RND_CLIP2_MAX_SATU(in4, in5, dst4, dst5, 7, dst4, dst5);
227 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
228 ST8x4_UB(out0, out1, dst, dst_stride);
229 dst += (4 * dst_stride);
230 ST8x2_UB(out2, dst, dst_stride);
231 } else if (0 == height % 8) {
234 for (loop_cnt = (height >> 3); loop_cnt--;) {
235 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
236 src0_ptr += 4 * src_stride;
237 INSERT_D2_SB(tp0, tp1, src0);
238 INSERT_D2_SB(tp2, tp3, src1);
239 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
240 src0_ptr += 4 * src_stride;
241 INSERT_D2_SB(tp0, tp1, src2);
242 INSERT_D2_SB(tp2, tp3, src3);
243 ILVRL_B2_SH(zero, src0, dst0, dst1);
244 ILVRL_B2_SH(zero, src1, dst2, dst3);
245 ILVRL_B2_SH(zero, src2, dst4, dst5);
246 ILVRL_B2_SH(zero, src3, dst6, dst7);
247 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6,
249 src1_ptr += (8 * src2_stride);
250 SLLI_4V(dst0, dst1, dst2, dst3, 6);
251 SLLI_4V(dst4, dst5, dst6, dst7, 6);
252 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2,
253 dst3, 7, dst0, dst1, dst2, dst3);
254 HEVC_BI_RND_CLIP4_MAX_SATU(in4, in5, in6, in7, dst4, dst5, dst6,
255 dst7, 7, dst4, dst5, dst6, dst7);
256 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
257 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
258 ST8x8_UB(out0, out1, out2, out3, dst, dst_stride);
259 dst += (8 * dst_stride);
264 static void hevc_bi_copy_12w_msa(uint8_t *src0_ptr,
274 v16u8 out0, out1, out2;
275 v16i8 src0, src1, src2, src3;
276 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
277 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
279 for (loop_cnt = 4; loop_cnt--;) {
280 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
281 src0_ptr += (4 * src_stride);
283 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
284 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
285 src1_ptr += (4 * src2_stride);
286 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
287 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, dst0, dst1,
289 SLLI_4V(dst0, dst1, dst2, dst3, 6);
290 ILVL_W2_SB(src1, src0, src3, src2, src0, src1);
291 ILVR_B2_SH(zero, src0, zero, src1, dst4, dst5);
292 SLLI_2V(dst4, dst5, 6);
293 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
294 7, dst0, dst1, dst2, dst3);
295 HEVC_BI_RND_CLIP2_MAX_SATU(in4, in5, dst4, dst5, 7, dst4, dst5);
296 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
297 ST12x4_UB(out0, out1, out2, dst, dst_stride);
298 dst += (4 * dst_stride);
302 static void hevc_bi_copy_16w_msa(uint8_t *src0_ptr,
311 v16u8 out0, out1, out2, out3;
312 v16i8 src0, src1, src2, src3;
313 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
314 v8i16 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
317 for (loop_cnt = (height >> 2); loop_cnt--;) {
318 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
319 src0_ptr += (4 * src_stride);
320 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
321 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
322 src1_ptr += (4 * src2_stride);
323 ILVRL_B2_SH(zero, src0, dst0_r, dst0_l);
324 ILVRL_B2_SH(zero, src1, dst1_r, dst1_l);
325 ILVRL_B2_SH(zero, src2, dst2_r, dst2_l);
326 ILVRL_B2_SH(zero, src3, dst3_r, dst3_l);
327 SLLI_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
328 SLLI_4V(dst0_l, dst1_l, dst2_l, dst3_l, 6);
329 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in4, in5, dst0_r, dst1_r, dst0_l,
330 dst1_l, 7, dst0_r, dst1_r, dst0_l, dst1_l);
331 HEVC_BI_RND_CLIP4_MAX_SATU(in2, in3, in6, in7, dst2_r, dst3_r, dst2_l,
332 dst3_l, 7, dst2_r, dst3_r, dst2_l, dst3_l);
333 PCKEV_B2_UB(dst0_l, dst0_r, dst1_l, dst1_r, out0, out1);
334 PCKEV_B2_UB(dst2_l, dst2_r, dst3_l, dst3_r, out2, out3);
335 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
336 dst += (4 * dst_stride);
340 static void hevc_bi_copy_24w_msa(uint8_t *src0_ptr,
349 v16u8 out0, out1, out2, out3, out4, out5;
350 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, zero = { 0 };
351 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
352 v8i16 in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, dst11;
354 for (loop_cnt = 8; loop_cnt--;) {
355 LD_SB4(src0_ptr, src_stride, src0, src1, src4, src5);
356 LD_SB4(src0_ptr + 16, src_stride, src2, src3, src6, src7);
357 src0_ptr += (4 * src_stride);
358 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
359 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
360 LD_SH4(src1_ptr + 16, src2_stride, in8, in9, in10, in11);
361 src1_ptr += (4 * src2_stride);
363 ILVRL_B2_SH(zero, src0, dst0, dst1);
364 ILVRL_B2_SH(zero, src1, dst2, dst3);
365 ILVR_B2_SH(zero, src2, zero, src3, dst4, dst5);
366 ILVRL_B2_SH(zero, src4, dst6, dst7);
367 ILVRL_B2_SH(zero, src5, dst8, dst9);
368 ILVR_B2_SH(zero, src6, zero, src7, dst10, dst11);
369 SLLI_4V(dst0, dst1, dst2, dst3, 6);
370 SLLI_4V(dst4, dst5, dst6, dst7, 6);
371 SLLI_4V(dst8, dst9, dst10, dst11, 6);
372 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in4, in1, in5, dst0, dst1, dst2, dst3,
373 7, dst0, dst1, dst2, dst3);
374 HEVC_BI_RND_CLIP4_MAX_SATU(in8, in9, in2, in6, dst4, dst5, dst6, dst7,
375 7, dst4, dst5, dst6, dst7);
376 HEVC_BI_RND_CLIP4_MAX_SATU(in3, in7, in10, in11, dst8, dst9, dst10,
377 dst11, 7, dst8, dst9, dst10, dst11);
378 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
379 PCKEV_B3_UB(dst7, dst6, dst9, dst8, dst11, dst10, out3, out4, out5);
380 ST_UB4(out0, out1, out3, out4, dst, dst_stride);
381 ST8x4_UB(out2, out5, dst + 16, dst_stride);
382 dst += (4 * dst_stride);
386 static void hevc_bi_copy_32w_msa(uint8_t *src0_ptr,
395 v16u8 out0, out1, out2, out3;
396 v16i8 src0, src1, src2, src3;
398 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
399 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
401 for (loop_cnt = (height >> 1); loop_cnt--;) {
402 LD_SB2(src0_ptr, 16, src0, src1);
403 src0_ptr += src_stride;
404 LD_SB2(src0_ptr, 16, src2, src3);
405 src0_ptr += src_stride;
406 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
407 src1_ptr += src2_stride;
408 LD_SH4(src1_ptr, 8, in4, in5, in6, in7);
409 src1_ptr += src2_stride;
411 ILVRL_B2_SH(zero, src0, dst0, dst1);
412 ILVRL_B2_SH(zero, src1, dst2, dst3);
413 ILVRL_B2_SH(zero, src2, dst4, dst5);
414 ILVRL_B2_SH(zero, src3, dst6, dst7);
415 SLLI_4V(dst0, dst1, dst2, dst3, 6);
416 SLLI_4V(dst4, dst5, dst6, dst7, 6);
417 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
418 7, dst0, dst1, dst2, dst3);
419 HEVC_BI_RND_CLIP4_MAX_SATU(in4, in5, in6, in7, dst4, dst5, dst6, dst7,
420 7, dst4, dst5, dst6, dst7);
421 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
422 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
423 ST_UB2(out0, out1, dst, 16);
425 ST_UB2(out2, out3, dst, 16);
430 static void hevc_bi_copy_48w_msa(uint8_t *src0_ptr,
439 v16u8 out0, out1, out2, out3, out4, out5;
440 v16i8 src0, src1, src2, src3, src4, src5;
442 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
443 v8i16 in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, dst11;
445 for (loop_cnt = (height >> 1); loop_cnt--;) {
446 LD_SB3(src0_ptr, 16, src0, src1, src2);
447 src0_ptr += src_stride;
448 LD_SB3(src0_ptr, 16, src3, src4, src5);
449 src0_ptr += src_stride;
451 LD_SH6(src1_ptr, 8, in0, in1, in2, in3, in4, in5);
452 src1_ptr += src2_stride;
453 LD_SH6(src1_ptr, 8, in6, in7, in8, in9, in10, in11);
454 src1_ptr += src2_stride;
456 ILVRL_B2_SH(zero, src0, dst0, dst1);
457 ILVRL_B2_SH(zero, src1, dst2, dst3);
458 ILVRL_B2_SH(zero, src2, dst4, dst5);
459 ILVRL_B2_SH(zero, src3, dst6, dst7);
460 ILVRL_B2_SH(zero, src4, dst8, dst9);
461 ILVRL_B2_SH(zero, src5, dst10, dst11);
463 SLLI_4V(dst0, dst1, dst2, dst3, 6);
464 SLLI_4V(dst4, dst5, dst6, dst7, 6);
465 SLLI_4V(dst8, dst9, dst10, dst11, 6);
467 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
468 7, dst0, dst1, dst2, dst3);
469 HEVC_BI_RND_CLIP4_MAX_SATU(in4, in5, in6, in7, dst4, dst5, dst6, dst7,
470 7, dst4, dst5, dst6, dst7);
471 HEVC_BI_RND_CLIP4_MAX_SATU(in8, in9, in10, in11, dst8, dst9, dst10,
472 dst11, 7, dst8, dst9, dst10, dst11);
473 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
474 PCKEV_B3_UB(dst7, dst6, dst9, dst8, dst11, dst10, out3, out4, out5);
475 ST_UB2(out0, out1, dst, 16);
476 ST_UB(out2, dst + 32);
478 ST_UB2(out3, out4, dst, 16);
479 ST_UB(out5, dst + 32);
484 static void hevc_bi_copy_64w_msa(uint8_t *src0_ptr,
493 v16u8 out0, out1, out2, out3;
494 v16i8 src0, src1, src2, src3;
496 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
497 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
499 for (loop_cnt = height; loop_cnt--;) {
500 LD_SB4(src0_ptr, 16, src0, src1, src2, src3);
501 src0_ptr += src_stride;
502 LD_SH8(src1_ptr, 8, in0, in1, in2, in3, in4, in5, in6, in7);
503 src1_ptr += src2_stride;
505 ILVRL_B2_SH(zero, src0, dst0, dst1);
506 ILVRL_B2_SH(zero, src1, dst2, dst3);
507 ILVRL_B2_SH(zero, src2, dst4, dst5);
508 ILVRL_B2_SH(zero, src3, dst6, dst7);
509 SLLI_4V(dst0, dst1, dst2, dst3, 6);
510 SLLI_4V(dst4, dst5, dst6, dst7, 6);
511 HEVC_BI_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, dst0, dst1, dst2, dst3,
512 7, dst0, dst1, dst2, dst3);
513 HEVC_BI_RND_CLIP4_MAX_SATU(in4, in5, in6, in7, dst4, dst5, dst6, dst7,
514 7, dst4, dst5, dst6, dst7);
515 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
516 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
518 ST_UB4(out0, out1, out2, out3, dst, 16);
523 static void hevc_hz_bi_8t_4w_msa(uint8_t *src0_ptr,
529 const int8_t *filter,
533 v8i16 filt0, filt1, filt2, filt3;
534 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
535 v16i8 mask1, mask2, mask3;
536 v16i8 vec0, vec1, vec2, vec3;
537 v8i16 dst0, dst1, dst2, dst3;
538 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
539 v8i16 filter_vec, const_vec;
540 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
544 /* rearranging filter */
545 filter_vec = LD_SH(filter);
546 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
552 const_vec = __msa_ldi_h(128);
555 for (loop_cnt = (height >> 3); loop_cnt--;) {
556 LD_SB8(src0_ptr, src_stride, src0, src1, src2, src3,
557 src4, src5, src6, src7);
558 src0_ptr += (8 * src_stride);
559 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
560 src1_ptr += (8 * src2_stride);
562 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
563 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
564 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
570 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
571 VSHF_B2_SB(src4, src5, src6, src7, mask0, mask0, vec2, vec3);
572 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
574 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec0, vec1);
575 VSHF_B2_SB(src4, src5, src6, src7, mask1, mask1, vec2, vec3);
576 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
578 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec0, vec1);
579 VSHF_B2_SB(src4, src5, src6, src7, mask2, mask2, vec2, vec3);
580 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
582 VSHF_B2_SB(src0, src1, src2, src3, mask3, mask3, vec0, vec1);
583 VSHF_B2_SB(src4, src5, src6, src7, mask3, mask3, vec2, vec3);
584 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
587 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
588 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
590 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
591 ST4x8_UB(dst0, dst1, dst, dst_stride);
592 dst += (8 * dst_stride);
596 static void hevc_hz_bi_8t_8w_msa(uint8_t *src0_ptr,
602 const int8_t *filter,
606 v8i16 filt0, filt1, filt2, filt3;
607 v16i8 src0, src1, src2, src3;
608 v16i8 mask1, mask2, mask3;
609 v16i8 vec0, vec1, vec2, vec3;
610 v8i16 dst0, dst1, dst2, dst3;
611 v8i16 in0, in1, in2, in3;
612 v8i16 filter_vec, const_vec;
613 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
617 const_vec = __msa_ldi_h(128);
620 filter_vec = LD_SH(filter);
621 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
627 for (loop_cnt = (height >> 2); loop_cnt--;) {
628 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
629 src0_ptr += (4 * src_stride);
630 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
631 src1_ptr += (4 * src2_stride);
632 XORI_B4_128_SB(src0, src1, src2, src3);
638 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
639 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
640 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
642 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
643 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
644 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
646 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
647 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
648 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
650 VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec0, vec1);
651 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec2, vec3);
652 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
655 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
656 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
658 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
659 ST8x4_UB(dst0, dst1, dst, dst_stride);
660 dst += (4 * dst_stride);
664 static void hevc_hz_bi_8t_12w_msa(uint8_t *src0_ptr,
670 const int8_t *filter,
676 v16i8 src0, src1, src2, src3;
677 v16i8 vec0, vec1, vec2;
678 v8i16 filt0, filt1, filt2, filt3;
679 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
680 v8i16 dst0, dst1, dst2;
681 v8i16 in0, in1, in2, in3;
682 v8i16 filter_vec, const_vec;
685 const_vec = __msa_ldi_h(128);
688 filter_vec = LD_SH(filter);
689 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
691 mask0 = LD_SB(ff_hevc_mask_arr);
695 mask4 = LD_SB(&ff_hevc_mask_arr[16]);
700 for (loop_cnt = 8; loop_cnt--;) {
701 LD_SB2(src0_ptr, 8, src0, src1);
702 src0_ptr += src_stride;
703 LD_SB2(src0_ptr, 8, src2, src3);
704 src0_ptr += src_stride;
705 LD_SH2(src1_ptr, 8, in0, in1);
706 src1_ptr += src2_stride;
707 LD_SH2(src1_ptr, 8, in2, in3);
708 src1_ptr += src2_stride;
709 XORI_B4_128_SB(src0, src1, src2, src3);
715 VSHF_B3_SB(src0, src0, src1, src3, src2, src2, mask0, mask4, mask0,
717 DPADD_SB2_SH(vec0, vec1, filt0, filt0, dst0, dst1);
718 dst2 = __msa_dpadd_s_h(dst2, vec2, (v16i8) filt0);
719 VSHF_B3_SB(src0, src0, src1, src3, src2, src2, mask1, mask5, mask1,
721 DPADD_SB2_SH(vec0, vec1, filt1, filt1, dst0, dst1);
722 dst2 = __msa_dpadd_s_h(dst2, vec2, (v16i8) filt1);
723 VSHF_B3_SB(src0, src0, src1, src3, src2, src2, mask2, mask6, mask2,
725 DPADD_SB2_SH(vec0, vec1, filt2, filt2, dst0, dst1);
726 dst2 = __msa_dpadd_s_h(dst2, vec2, (v16i8) filt2);
727 VSHF_B3_SB(src0, src0, src1, src3, src2, src2, mask3, mask7, mask3,
729 DPADD_SB2_SH(vec0, vec1, filt3, filt3, dst0, dst1);
730 dst2 = __msa_dpadd_s_h(dst2, vec2, (v16i8) filt3);
732 in1 = (v8i16) __msa_pckev_d((v2i64) in3, (v2i64) in1);
733 HEVC_BI_RND_CLIP2(in0, in1, dst0, dst1, 7, dst0, dst1);
734 dst2 = __msa_adds_s_h(in2, dst2);
735 dst2 = __msa_srari_h(dst2, 7);
736 dst2 = CLIP_SH_0_255(dst2);
737 PCKEV_B2_SH(dst1, dst0, dst2, dst2, dst0, dst1);
739 tmp2 = __msa_copy_s_d((v2i64) dst0, 0);
740 tmp0 = __msa_copy_s_w((v4i32) dst0, 2);
741 tmp3 = __msa_copy_s_d((v2i64) dst1, 0);
742 tmp1 = __msa_copy_s_w((v4i32) dst0, 3);
752 static void hevc_hz_bi_8t_16w_msa(uint8_t *src0_ptr,
758 const int8_t *filter,
762 v16i8 src0, src1, src2, src3;
763 v8i16 filt0, filt1, filt2, filt3;
764 v16i8 mask1, mask2, mask3;
765 v16i8 vec0, vec1, vec2, vec3;
766 v8i16 dst0, dst1, dst2, dst3;
767 v8i16 in0, in1, in2, in3;
768 v8i16 filter_vec, const_vec;
769 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
772 const_vec = __msa_ldi_h(128);
775 filter_vec = LD_SH(filter);
776 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
782 for (loop_cnt = (height >> 1); loop_cnt--;) {
783 LD_SB2(src0_ptr, 8, src0, src1);
784 src0_ptr += src_stride;
785 LD_SB2(src0_ptr, 8, src2, src3);
786 src0_ptr += src_stride;
787 LD_SH2(src1_ptr, 8, in0, in1);
788 src1_ptr += src2_stride;
789 LD_SH2(src1_ptr, 8, in2, in3);
790 src1_ptr += src2_stride;
791 XORI_B4_128_SB(src0, src1, src2, src3);
797 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
798 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
799 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
801 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
802 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
803 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
805 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
806 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
807 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
809 VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec0, vec1);
810 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec2, vec3);
811 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
814 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
815 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
817 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
818 ST_SH2(dst0, dst1, dst, dst_stride);
819 dst += (2 * dst_stride);
823 static void hevc_hz_bi_8t_24w_msa(uint8_t *src0_ptr,
829 const int8_t *filter,
834 v16i8 src0, src1, tmp0, tmp1;
835 v8i16 filt0, filt1, filt2, filt3;
836 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
837 v16i8 vec0, vec1, vec2, vec3;
838 v8i16 dst0, dst1, dst2;
840 v8i16 filter_vec, const_vec;
841 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
843 src0_ptr = src0_ptr - 3;
844 const_vec = __msa_ldi_h(128);
847 filter_vec = LD_SH(filter);
848 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
858 for (loop_cnt = height; loop_cnt--;) {
859 LD_SB2(src0_ptr, 16, src0, src1);
860 src0_ptr += src_stride;
861 LD_SH2(src1_ptr, 8, in0, in1);
862 in2 = LD_SH(src1_ptr + 16);
863 src1_ptr += src2_stride;
864 XORI_B2_128_SB(src0, src1);
869 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
870 VSHF_B2_SB(src1, src1, src0, src0, mask0, mask1, vec2, vec3);
871 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt1, dst0,
873 VSHF_B2_SB(src0, src1, src1, src1, mask5, mask1, vec0, vec1);
874 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec2, vec3);
875 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt2, filt2, dst1,
877 VSHF_B2_SB(src1, src1, src0, src0, mask2, mask3, vec0, vec1);
878 VSHF_B2_SB(src0, src1, src1, src1, mask7, mask3, vec2, vec3);
879 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt3, filt3, filt3, dst2,
882 HEVC_BI_RND_CLIP2(in0, in1, dst0, dst1, 7, dst0, dst1);
883 dst2 = __msa_adds_s_h(dst2, in2);
884 dst2 = __msa_srari_h(dst2, 7);
885 dst2 = CLIP_SH_0_255(dst2);
887 PCKEV_B2_SB(dst1, dst0, dst2, dst2, tmp0, tmp1);
888 dst_val0 = __msa_copy_u_d((v2i64) tmp1, 0);
890 SD(dst_val0, dst + 16);
895 static void hevc_hz_bi_8t_32w_msa(uint8_t *src0_ptr,
901 const int8_t *filter,
905 v16i8 src0, src1, src2, tmp0, tmp1;
906 v8i16 filt0, filt1, filt2, filt3;
907 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
908 v16i8 vec0, vec1, vec2, vec3;
909 v8i16 dst0, dst1, dst2, dst3;
910 v8i16 in0, in1, in2, in3;
911 v8i16 filter_vec, const_vec;
912 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
915 const_vec = __msa_ldi_h(128);
918 filter_vec = LD_SH(filter);
919 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
929 for (loop_cnt = height; loop_cnt--;) {
930 LD_SB2(src0_ptr, 16, src0, src1);
931 src2 = LD_SB(src0_ptr + 24);
932 src0_ptr += src_stride;
933 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
934 src1_ptr += src2_stride;
935 XORI_B3_128_SB(src0, src1, src2);
941 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
942 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
943 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
945 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
946 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
947 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
949 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
950 VSHF_B2_SB(src1, src1, src2, src2, mask2, mask2, vec2, vec3);
951 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
953 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
954 VSHF_B2_SB(src1, src1, src2, src2, mask3, mask3, vec2, vec3);
955 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
958 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
959 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
961 PCKEV_B2_SB(dst1, dst0, dst3, dst2, tmp0, tmp1);
962 ST_SB2(tmp0, tmp1, dst, 16);
967 static void hevc_hz_bi_8t_48w_msa(uint8_t *src0_ptr,
973 const int8_t *filter,
977 v16i8 src0, src1, src2, src3;
978 v16i8 tmp0, tmp1, tmp2;
979 v8i16 filt0, filt1, filt2, filt3;
980 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
981 v16i8 vec0, vec1, vec2, vec3;
982 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
983 v8i16 in0, in1, in2, in3, in4, in5;
984 v8i16 filter_vec, const_vec;
985 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
989 const_vec = __msa_ldi_h(128);
992 filter_vec = LD_SH(filter);
993 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1003 for (loop_cnt = 64; loop_cnt--;) {
1004 LD_SB3(src0_ptr, 16, src0, src1, src2);
1005 src3 = LD_SB(src0_ptr + 40);
1006 src0_ptr += src_stride;
1007 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
1008 XORI_B4_128_SB(src0, src1, src2, src3);
1015 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
1016 VSHF_B2_SB(src1, src1, src1, src2, mask0, mask4, vec2, vec3);
1017 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
1019 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
1020 VSHF_B2_SB(src1, src1, src1, src2, mask1, mask5, vec2, vec3);
1021 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
1023 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
1024 VSHF_B2_SB(src1, src1, src1, src2, mask2, mask6, vec2, vec3);
1025 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
1027 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
1028 VSHF_B2_SB(src1, src1, src1, src2, mask3, mask7, vec2, vec3);
1029 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
1031 HEVC_BI_RND_CLIP2(in0, in1, dst0, dst1, 7, dst0, dst1);
1032 HEVC_BI_RND_CLIP2(in2, in3, dst2, dst3, 7, dst2, dst3);
1033 PCKEV_B2_SB(dst1, dst0, dst3, dst2, tmp0, tmp1);
1035 ST_SB(tmp1, dst + 16);
1037 LD_SH2(src1_ptr + 32, 8, in4, in5);
1038 src1_ptr += src2_stride;
1042 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec0, vec1);
1043 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
1044 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt1, filt1, dst4,
1046 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec0, vec1);
1047 VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec2, vec3);
1048 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt3, filt3, dst4,
1051 HEVC_BI_RND_CLIP2(in4, in5, dst4, dst5, 7, dst4, dst5);
1053 tmp2 = __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
1054 ST_SB(tmp2, dst + 32);
1059 static void hevc_hz_bi_8t_64w_msa(uint8_t *src0_ptr,
1062 int32_t src2_stride,
1065 const int8_t *filter,
1069 v16i8 src0, src1, src2, src3, src4, src5, tmp0, tmp1;
1070 v8i16 filt0, filt1, filt2, filt3;
1071 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1072 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1073 v16i8 vec0, vec1, vec2, vec3;
1074 v8i16 dst0, dst1, dst2, dst3;
1075 v8i16 in0, in1, in2, in3;
1076 v8i16 filter_vec, const_vec;
1080 const_vec = __msa_ldi_h(128);
1083 filter_vec = LD_SH(filter);
1084 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1094 for (loop_cnt = height; loop_cnt--;) {
1095 LD_SB2(src0_ptr, 16, src0, src1);
1096 src2 = LD_SB(src0_ptr + 24);
1097 LD_SB2(src0_ptr + 32, 16, src3, src4);
1098 src5 = LD_SB(src0_ptr + 56);
1099 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
1100 XORI_B3_128_SB(src0, src1, src2);
1107 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
1108 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
1109 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
1111 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
1112 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
1113 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
1115 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
1116 VSHF_B2_SB(src1, src1, src2, src2, mask2, mask2, vec2, vec3);
1117 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
1119 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
1120 VSHF_B2_SB(src1, src1, src2, src2, mask3, mask3, vec2, vec3);
1121 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
1124 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1125 dst0, dst1, dst2, dst3, 7,
1126 dst0, dst1, dst2, dst3);
1128 PCKEV_B2_SB(dst1, dst0, dst3, dst2, tmp0, tmp1);
1129 ST_SB2(tmp0, tmp1, dst, 16);
1135 LD_SH4(src1_ptr + 32, 8, in0, in1, in2, in3);
1136 XORI_B3_128_SB(src0, src1, src2);
1142 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask4, vec0, vec1);
1143 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
1144 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
1146 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask5, vec0, vec1);
1147 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
1148 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
1150 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask6, vec0, vec1);
1151 VSHF_B2_SB(src1, src1, src2, src2, mask2, mask2, vec2, vec3);
1152 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, dst0,
1154 VSHF_B2_SB(src0, src0, src0, src1, mask3, mask7, vec0, vec1);
1155 VSHF_B2_SB(src1, src1, src2, src2, mask3, mask3, vec2, vec3);
1156 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt3, filt3, filt3, filt3, dst0,
1158 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1159 dst0, dst1, dst2, dst3, 7,
1160 dst0, dst1, dst2, dst3);
1161 PCKEV_B2_SB(dst1, dst0, dst3, dst2, tmp0, tmp1);
1162 ST_SB2(tmp0, tmp1, dst + 32, 16);
1163 src1_ptr += src2_stride;
1164 src0_ptr += src_stride;
1169 static void hevc_vt_bi_8t_4w_msa(uint8_t *src0_ptr,
1172 int32_t src2_stride,
1175 const int8_t *filter,
1179 v16i8 src0, src1, src2, src3, src4, src5;
1180 v16i8 src6, src7, src8, src9, src10;
1181 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
1182 v16i8 src11, src12, src13, src14;
1183 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1184 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1185 v16i8 src1110_r, src1211_r, src1312_r, src1413_r;
1186 v16i8 src2110, src4332, src6554, src8776, src10998;
1187 v16i8 src12111110, src14131312;
1188 v8i16 dst10, dst32, dst54, dst76;
1189 v8i16 filt0, filt1, filt2, filt3;
1190 v8i16 filter_vec, const_vec;
1192 src0_ptr -= (3 * src_stride);
1194 const_vec = __msa_ldi_h(128);
1197 filter_vec = LD_SH(filter);
1198 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1200 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1201 src0_ptr += (7 * src_stride);
1202 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1203 src10_r, src32_r, src54_r, src21_r);
1204 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1205 ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
1206 src2110, src4332, src6554);
1207 XORI_B3_128_SB(src2110, src4332, src6554);
1209 for (loop_cnt = (height >> 3); loop_cnt--;) {
1210 LD_SB8(src0_ptr, src_stride,
1211 src7, src8, src9, src10, src11, src12, src13, src14);
1212 src0_ptr += (8 * src_stride);
1213 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
1214 src1_ptr += (8 * src2_stride);
1216 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
1217 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
1218 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1219 src76_r, src87_r, src98_r, src109_r);
1220 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1221 src1110_r, src1211_r, src1312_r, src1413_r);
1222 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
1223 src1413_r, src1312_r,
1224 src8776, src10998, src12111110, src14131312);
1225 XORI_B4_128_SB(src8776, src10998, src12111110, src14131312);
1228 DPADD_SB4_SH(src2110, src4332, src6554, src8776,
1229 filt0, filt1, filt2, filt3, dst10, dst10, dst10, dst10);
1231 DPADD_SB4_SH(src4332, src6554, src8776, src10998,
1232 filt0, filt1, filt2, filt3, dst32, dst32, dst32, dst32);
1234 DPADD_SB4_SH(src6554, src8776, src10998, src12111110,
1235 filt0, filt1, filt2, filt3, dst54, dst54, dst54, dst54);
1237 DPADD_SB4_SH(src8776, src10998, src12111110, src14131312,
1238 filt0, filt1, filt2, filt3, dst76, dst76, dst76, dst76);
1240 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1241 dst10, dst32, dst54, dst76, 7,
1242 dst10, dst32, dst54, dst76);
1244 PCKEV_B2_SH(dst32, dst10, dst76, dst54, dst10, dst54);
1245 ST4x8_UB(dst10, dst54, dst, dst_stride);
1246 dst += (8 * dst_stride);
1249 src4332 = src12111110;
1250 src6554 = src14131312;
1255 static void hevc_vt_bi_8t_8w_msa(uint8_t *src0_ptr,
1258 int32_t src2_stride,
1261 const int8_t *filter,
1265 v16i8 src0, src1, src2, src3, src4, src5;
1266 v16i8 src6, src7, src8, src9, src10;
1267 v8i16 in0, in1, in2, in3;
1268 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1269 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1270 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
1271 v8i16 filt0, filt1, filt2, filt3;
1272 v8i16 filter_vec, const_vec;
1274 src0_ptr -= (3 * src_stride);
1275 const_vec = __msa_ldi_h(128);
1278 filter_vec = LD_SH(filter);
1279 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1281 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1282 src0_ptr += (7 * src_stride);
1283 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1284 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1285 src10_r, src32_r, src54_r, src21_r);
1286 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1288 for (loop_cnt = (height >> 2); loop_cnt--;) {
1289 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
1290 src0_ptr += (4 * src_stride);
1291 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
1292 src1_ptr += (4 * src2_stride);
1293 XORI_B4_128_SB(src7, src8, src9, src10);
1294 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1295 src76_r, src87_r, src98_r, src109_r);
1298 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1299 filt0, filt1, filt2, filt3,
1300 dst0_r, dst0_r, dst0_r, dst0_r);
1302 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1303 filt0, filt1, filt2, filt3,
1304 dst1_r, dst1_r, dst1_r, dst1_r);
1306 DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
1307 filt0, filt1, filt2, filt3,
1308 dst2_r, dst2_r, dst2_r, dst2_r);
1310 DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
1311 filt0, filt1, filt2, filt3,
1312 dst3_r, dst3_r, dst3_r, dst3_r);
1314 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1315 dst0_r, dst1_r, dst2_r, dst3_r, 7,
1316 dst0_r, dst1_r, dst2_r, dst3_r);
1318 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
1319 ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
1320 dst += (4 * dst_stride);
1333 static void hevc_vt_bi_8t_12w_msa(uint8_t *src0_ptr,
1336 int32_t src2_stride,
1339 const int8_t *filter,
1343 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1344 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
1345 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1346 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1347 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
1348 v16i8 src10_l, src32_l, src54_l, src76_l, src98_l;
1349 v16i8 src21_l, src43_l, src65_l, src87_l, src109_l;
1350 v16i8 src2110, src4332, src6554, src8776, src10998;
1351 v8i16 dst0_l, dst1_l;
1352 v8i16 filt0, filt1, filt2, filt3;
1353 v8i16 filter_vec, const_vec;
1355 src0_ptr -= (3 * src_stride);
1356 const_vec = __msa_ldi_h(128);
1359 filter_vec = LD_SH(filter);
1360 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1362 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1363 src0_ptr += (7 * src_stride);
1364 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1366 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1367 src10_r, src32_r, src54_r, src21_r);
1368 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1369 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1370 src10_l, src32_l, src54_l, src21_l);
1371 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1372 ILVR_D3_SB(src21_l, src10_l, src43_l, src32_l, src65_l, src54_l,
1373 src2110, src4332, src6554);
1375 for (loop_cnt = (height >> 2); loop_cnt--;) {
1376 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
1377 src0_ptr += (4 * src_stride);
1378 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
1379 LD_SH4((src1_ptr + 8), src2_stride, in4, in5, in6, in7);
1380 src1_ptr += (4 * src2_stride);
1382 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
1383 XORI_B4_128_SB(src7, src8, src9, src10);
1384 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1385 src76_r, src87_r, src98_r, src109_r);
1386 ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1387 src76_l, src87_l, src98_l, src109_l);
1388 ILVR_D2_SB(src87_l, src76_l, src109_l, src98_l, src8776, src10998);
1391 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1392 filt0, filt1, filt2, filt3,
1393 dst0_r, dst0_r, dst0_r, dst0_r);
1395 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1396 filt0, filt1, filt2, filt3,
1397 dst1_r, dst1_r, dst1_r, dst1_r);
1399 DPADD_SB4_SH(src32_r, src54_r, src76_r, src98_r,
1400 filt0, filt1, filt2, filt3,
1401 dst2_r, dst2_r, dst2_r, dst2_r);
1403 DPADD_SB4_SH(src43_r, src65_r, src87_r, src109_r,
1404 filt0, filt1, filt2, filt3,
1405 dst3_r, dst3_r, dst3_r, dst3_r);
1407 DPADD_SB4_SH(src2110, src4332, src6554, src8776,
1408 filt0, filt1, filt2, filt3,
1409 dst0_l, dst0_l, dst0_l, dst0_l);
1411 DPADD_SB4_SH(src4332, src6554, src8776, src10998,
1412 filt0, filt1, filt2, filt3,
1413 dst1_l, dst1_l, dst1_l, dst1_l);
1415 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1416 dst0_r, dst1_r, dst2_r, dst3_r, 7,
1417 dst0_r, dst1_r, dst2_r, dst3_r);
1418 HEVC_BI_RND_CLIP2(in4, in5, dst0_l, dst1_l, 7, dst0_l, dst1_l);
1421 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
1422 dst0_l = (v8i16) __msa_pckev_b((v16i8) dst1_l, (v16i8) dst0_l);
1423 ST12x4_UB(dst0_r, dst1_r, dst0_l, dst, dst_stride);
1424 dst += (4 * dst_stride);
1439 static void hevc_vt_bi_8t_16multx2mult_msa(uint8_t *src0_ptr,
1442 int32_t src2_stride,
1445 const int8_t *filter,
1446 int32_t height, int32_t width)
1448 uint8_t *src0_ptr_tmp;
1449 int16_t *src1_ptr_tmp;
1453 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1454 v8i16 in0, in1, in2, in3;
1455 v16i8 src10_r, src32_r, src54_r, src76_r;
1456 v16i8 src21_r, src43_r, src65_r, src87_r;
1457 v8i16 dst0_r, dst1_r;
1458 v16i8 src10_l, src32_l, src54_l, src76_l;
1459 v16i8 src21_l, src43_l, src65_l, src87_l;
1460 v8i16 dst0_l, dst1_l;
1461 v8i16 filt0, filt1, filt2, filt3;
1462 v8i16 filter_vec, const_vec;
1464 src0_ptr -= (3 * src_stride);
1465 const_vec = __msa_ldi_h(128);
1468 filter_vec = LD_SH(filter);
1469 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1471 for (cnt = (width >> 4); cnt--;) {
1472 src0_ptr_tmp = src0_ptr;
1473 src1_ptr_tmp = src1_ptr;
1476 LD_SB7(src0_ptr_tmp, src_stride,
1477 src0, src1, src2, src3, src4, src5, src6);
1478 src0_ptr_tmp += (7 * src_stride);
1479 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1481 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1482 src10_r, src32_r, src54_r, src21_r);
1483 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1484 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1485 src10_l, src32_l, src54_l, src21_l);
1486 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1488 for (loop_cnt = (height >> 1); loop_cnt--;) {
1489 LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
1490 src0_ptr_tmp += (2 * src_stride);
1491 LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
1492 LD_SH2((src1_ptr_tmp + 8), src2_stride, in2, in3);
1493 src1_ptr_tmp += (2 * src2_stride);
1494 XORI_B2_128_SB(src7, src8);
1496 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1497 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
1500 DPADD_SB4_SH(src10_r, src32_r, src54_r, src76_r,
1501 filt0, filt1, filt2, filt3,
1502 dst0_r, dst0_r, dst0_r, dst0_r);
1504 DPADD_SB4_SH(src21_r, src43_r, src65_r, src87_r,
1505 filt0, filt1, filt2, filt3,
1506 dst1_r, dst1_r, dst1_r, dst1_r);
1508 DPADD_SB4_SH(src10_l, src32_l, src54_l, src76_l,
1509 filt0, filt1, filt2, filt3,
1510 dst0_l, dst0_l, dst0_l, dst0_l);
1512 DPADD_SB4_SH(src21_l, src43_l, src65_l, src87_l,
1513 filt0, filt1, filt2, filt3,
1514 dst1_l, dst1_l, dst1_l, dst1_l);
1516 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
1517 dst0_r, dst1_r, dst0_l, dst1_l, 7,
1518 dst0_r, dst1_r, dst0_l, dst1_l);
1520 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
1521 ST_SH2(dst0_r, dst1_r, dst_tmp, dst_stride);
1522 dst_tmp += (2 * dst_stride);
1545 static void hevc_vt_bi_8t_16w_msa(uint8_t *src0_ptr,
1548 int32_t src2_stride,
1551 const int8_t *filter,
1554 hevc_vt_bi_8t_16multx2mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1555 dst, dst_stride, filter, height, 16);
1558 static void hevc_vt_bi_8t_24w_msa(uint8_t *src0_ptr,
1561 int32_t src2_stride,
1564 const int8_t *filter,
1567 hevc_vt_bi_8t_16multx2mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1568 dst, dst_stride, filter, height, 16);
1569 hevc_vt_bi_8t_8w_msa(src0_ptr + 16, src_stride, src1_ptr + 16, src2_stride,
1570 dst + 16, dst_stride, filter, height);
1573 static void hevc_vt_bi_8t_32w_msa(uint8_t *src0_ptr,
1576 int32_t src2_stride,
1579 const int8_t *filter,
1582 hevc_vt_bi_8t_16multx2mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1583 dst, dst_stride, filter, height, 32);
1586 static void hevc_vt_bi_8t_48w_msa(uint8_t *src0_ptr,
1589 int32_t src2_stride,
1592 const int8_t *filter,
1595 hevc_vt_bi_8t_16multx2mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1596 dst, dst_stride, filter, height, 48);
1599 static void hevc_vt_bi_8t_64w_msa(uint8_t *src0_ptr,
1602 int32_t src2_stride,
1605 const int8_t *filter,
1608 hevc_vt_bi_8t_16multx2mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1609 dst, dst_stride, filter, height, 64);
1612 static void hevc_hv_bi_8t_4w_msa(uint8_t *src0_ptr,
1615 int32_t src2_stride,
1618 const int8_t *filter_x,
1619 const int8_t *filter_y,
1625 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1626 v8i16 in0 = { 0 }, in1 = { 0 };
1627 v8i16 filt0, filt1, filt2, filt3;
1628 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1629 v16i8 mask1, mask2, mask3;
1630 v8i16 filter_vec, const_vec;
1631 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1632 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1634 v8i16 dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1635 v8i16 dst10, dst32, dst54, dst76, dst98, dst21, dst43, dst65, dst87, dst109;
1636 v4i32 dst0, dst1, dst2, dst3;
1637 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1639 src0_ptr -= ((3 * src_stride) + 3);
1640 filter_vec = LD_SH(filter_x);
1641 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1643 filter_vec = LD_SH(filter_y);
1644 UNPCK_R_SB_SH(filter_vec, filter_vec);
1646 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1652 const_vec = __msa_ldi_h(128);
1655 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1656 src0_ptr += (7 * src_stride);
1657 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1659 /* row 0 row 1 row 2 row 3 */
1660 VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1661 VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1662 VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
1663 vec8, vec9, vec10, vec11);
1664 VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
1665 vec12, vec13, vec14, vec15);
1667 dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1669 dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1671 dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1673 dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1676 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
1677 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
1678 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
1680 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
1682 for (loop_cnt = height >> 2; loop_cnt--;) {
1683 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
1684 src0_ptr += (4 * src_stride);
1685 XORI_B4_128_SB(src7, src8, src9, src10);
1687 LD2(src1_ptr, src2_stride, tp0, tp1);
1688 INSERT_D2_SH(tp0, tp1, in0);
1689 src1_ptr += (2 * src2_stride);
1690 LD2(src1_ptr, src2_stride, tp0, tp1);
1691 INSERT_D2_SH(tp0, tp1, in1);
1692 src1_ptr += (2 * src2_stride);
1694 VSHF_B4_SB(src7, src9, mask0, mask1, mask2, mask3,
1695 vec0, vec1, vec2, vec3);
1696 VSHF_B4_SB(src8, src10, mask0, mask1, mask2, mask3,
1697 vec4, vec5, vec6, vec7);
1698 dst97 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1700 dst108 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1703 dst76 = __msa_ilvr_h(dst97, dst66);
1704 ILVRL_H2_SH(dst108, dst97, dst87, dst109);
1705 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
1706 dst98 = __msa_ilvr_h(dst66, dst108);
1708 dst0 = HEVC_FILT_8TAP(dst10, dst32, dst54, dst76, filt_h0, filt_h1,
1710 dst1 = HEVC_FILT_8TAP(dst21, dst43, dst65, dst87, filt_h0, filt_h1,
1712 dst2 = HEVC_FILT_8TAP(dst32, dst54, dst76, dst98, filt_h0, filt_h1,
1714 dst3 = HEVC_FILT_8TAP(dst43, dst65, dst87, dst109, filt_h0, filt_h1,
1717 SRA_4V(dst0, dst1, dst2, dst3, 6);
1718 PCKEV_H2_SH(dst1, dst0, dst3, dst2, out0, out1);
1719 ADDS_SH2_SH(out0, in0, out1, in1, out0, out1);
1720 ADDS_SH2_SH(out0, const_vec, out1, const_vec, out0, out1);
1721 SRARI_H2_SH(out0, out1, 7);
1722 CLIP_SH2_0_255_MAX_SATU(out0, out1);
1723 out = (v16u8) __msa_pckev_b((v16i8) out1, (v16i8) out0);
1724 ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
1725 dst += (4 * dst_stride);
1733 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
1737 static void hevc_hv_bi_8t_8multx1mult_msa(uint8_t *src0_ptr,
1740 int32_t src2_stride,
1743 const int8_t *filter_x,
1744 const int8_t *filter_y,
1745 int32_t height, int32_t width)
1749 uint8_t *src0_ptr_tmp;
1750 int16_t *src1_ptr_tmp;
1753 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
1755 v8i16 filt0, filt1, filt2, filt3;
1756 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1757 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
1758 v16i8 mask1, mask2, mask3;
1759 v8i16 filter_vec, const_vec;
1760 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1761 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1762 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1763 v4i32 dst0_r, dst0_l;
1764 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
1765 v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
1767 src0_ptr -= ((3 * src_stride) + 3);
1768 const_vec = __msa_ldi_h(128);
1771 filter_vec = LD_SH(filter_x);
1772 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1774 filter_vec = LD_SH(filter_y);
1775 UNPCK_R_SB_SH(filter_vec, filter_vec);
1777 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1783 for (cnt = width >> 3; cnt--;) {
1784 src0_ptr_tmp = src0_ptr;
1786 src1_ptr_tmp = src1_ptr;
1788 LD_SB7(src0_ptr_tmp, src_stride,
1789 src0, src1, src2, src3, src4, src5, src6);
1790 src0_ptr_tmp += (7 * src_stride);
1791 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1793 /* row 0 row 1 row 2 row 3 */
1794 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1795 vec0, vec1, vec2, vec3);
1796 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1797 vec4, vec5, vec6, vec7);
1798 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1799 vec8, vec9, vec10, vec11);
1800 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1801 vec12, vec13, vec14, vec15);
1802 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1804 dst1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1806 dst2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1808 dst3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1811 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1812 vec0, vec1, vec2, vec3);
1813 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
1814 vec4, vec5, vec6, vec7);
1815 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
1816 vec8, vec9, vec10, vec11);
1817 dst4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1819 dst5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1821 dst6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1824 for (loop_cnt = height; loop_cnt--;) {
1825 src7 = LD_SB(src0_ptr_tmp);
1826 src7 = (v16i8) __msa_xori_b((v16u8) src7, 128);
1827 src0_ptr_tmp += src_stride;
1829 in0 = LD_SH(src1_ptr_tmp);
1830 src1_ptr_tmp += src2_stride;
1832 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
1833 vec0, vec1, vec2, vec3);
1834 dst7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1836 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
1837 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
1838 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
1839 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1840 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
1841 filt_h0, filt_h1, filt_h2, filt_h3);
1842 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
1843 filt_h0, filt_h1, filt_h2, filt_h3);
1847 tmp = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
1848 ADDS_SH2_SH(tmp, in0, tmp, const_vec, tmp, tmp);
1849 tmp = __msa_srari_h(tmp, 7);
1850 tmp = CLIP_SH_0_255_MAX_SATU(tmp);
1851 out = (v16u8) __msa_pckev_b((v16i8) tmp, (v16i8) tmp);
1852 ST8x1_UB(out, dst_tmp);
1853 dst_tmp += dst_stride;
1870 static void hevc_hv_bi_8t_8w_msa(uint8_t *src0_ptr,
1873 int32_t src2_stride,
1876 const int8_t *filter_x,
1877 const int8_t *filter_y,
1880 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
1881 dst, dst_stride, filter_x, filter_y,
1885 static void hevc_hv_bi_8t_12w_msa(uint8_t *src0_ptr,
1888 int32_t src2_stride,
1891 const int8_t *filter_x,
1892 const int8_t *filter_y,
1896 uint8_t *src0_ptr_tmp, *dst_tmp;
1897 int16_t *src1_ptr_tmp;
1900 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1901 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1902 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1903 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1904 v8i16 in0, in1 = { 0 }, out0, out1, tmp, filter_vec, const_vec;
1905 v8i16 filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
1906 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1907 v8i16 dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1908 v8i16 dst10, dst32, dst54, dst76, dst98, dst21, dst43, dst65, dst87, dst109;
1909 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
1910 v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
1911 v4i32 dst0_r, dst0_l, tmp0, tmp1, tmp2, tmp3;
1913 src0_ptr -= ((3 * src_stride) + 3);
1915 const_vec = __msa_ldi_h(128);
1918 filter_vec = LD_SH(filter_x);
1919 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1921 filter_vec = LD_SH(filter_y);
1922 UNPCK_R_SB_SH(filter_vec, filter_vec);
1924 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1926 mask0 = LD_SB(ff_hevc_mask_arr);
1931 src0_ptr_tmp = src0_ptr;
1933 src1_ptr_tmp = src1_ptr;
1935 LD_SB7(src0_ptr_tmp, src_stride, src0, src1, src2, src3, src4, src5,
1937 src0_ptr_tmp += (7 * src_stride);
1938 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1940 /* row 0 row 1 row 2 row 3 */
1941 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1943 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec4, vec5, vec6,
1945 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1947 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec12, vec13, vec14,
1949 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1951 dst1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1953 dst2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1955 dst3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1957 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1959 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3, vec4, vec5, vec6,
1961 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1963 dst4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1965 dst5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1967 dst6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1970 for (loop_cnt = 16; loop_cnt--;) {
1971 src7 = LD_SB(src0_ptr_tmp);
1972 src7 = (v16i8) __msa_xori_b((v16u8) src7, 128);
1973 src0_ptr_tmp += src_stride;
1975 in0 = LD_SH(src1_ptr_tmp);
1976 src1_ptr_tmp += src2_stride;
1978 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1980 dst7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1982 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
1983 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
1984 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
1985 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1986 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
1987 filt_h1, filt_h2, filt_h3);
1988 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l, filt_h0,
1989 filt_h1, filt_h2, filt_h3);
1993 tmp = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
1994 ADDS_SH2_SH(tmp, in0, tmp, const_vec, tmp, tmp);
1995 tmp = __msa_srari_h(tmp, 7);
1996 tmp = CLIP_SH_0_255_MAX_SATU(tmp);
1997 out = (v16u8) __msa_pckev_b((v16i8) tmp, (v16i8) tmp);
1998 ST8x1_UB(out, dst_tmp);
1999 dst_tmp += dst_stride;
2014 mask4 = LD_SB(ff_hevc_mask_arr + 16);
2019 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
2020 src0_ptr += (7 * src_stride);
2021 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
2023 /* row 0 row 1 row 2 row 3 */
2024 VSHF_B4_SB(src0, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
2025 VSHF_B4_SB(src1, src4, mask4, mask5, mask6, mask7, vec4, vec5, vec6, vec7);
2026 VSHF_B4_SB(src2, src5, mask4, mask5, mask6, mask7,
2027 vec8, vec9, vec10, vec11);
2028 VSHF_B4_SB(src3, src6, mask4, mask5, mask6, mask7,
2029 vec12, vec13, vec14, vec15);
2030 dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2032 dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2034 dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2036 dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
2039 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
2040 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
2041 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
2043 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
2045 for (loop_cnt = 4; loop_cnt--;) {
2046 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
2047 src0_ptr += (4 * src_stride);
2048 XORI_B4_128_SB(src7, src8, src9, src10);
2050 LD2(src1_ptr, src2_stride, tp0, tp1);
2051 INSERT_D2_SH(tp0, tp1, in0);
2052 src1_ptr += (2 * src2_stride);
2053 LD2(src1_ptr, src2_stride, tp0, tp1);
2054 INSERT_D2_SH(tp0, tp1, in1);
2055 src1_ptr += (2 * src2_stride);
2057 VSHF_B4_SB(src7, src9, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
2059 VSHF_B4_SB(src8, src10, mask4, mask5, mask6, mask7, vec4, vec5, vec6,
2061 dst97 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2063 dst108 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2066 dst76 = __msa_ilvr_h(dst97, dst66);
2067 ILVRL_H2_SH(dst108, dst97, dst87, dst109);
2068 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
2069 dst98 = __msa_ilvr_h(dst66, dst108);
2071 tmp0 = HEVC_FILT_8TAP(dst10, dst32, dst54, dst76, filt_h0, filt_h1,
2073 tmp1 = HEVC_FILT_8TAP(dst21, dst43, dst65, dst87, filt_h0, filt_h1,
2075 tmp2 = HEVC_FILT_8TAP(dst32, dst54, dst76, dst98, filt_h0, filt_h1,
2077 tmp3 = HEVC_FILT_8TAP(dst43, dst65, dst87, dst109, filt_h0, filt_h1,
2079 SRA_4V(tmp0, tmp1, tmp2, tmp3, 6);
2080 PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, out0, out1);
2081 ADDS_SH2_SH(out0, in0, out1, in1, out0, out1);
2082 ADDS_SH2_SH(out0, const_vec, out1, const_vec, out0, out1);
2083 SRARI_H2_SH(out0, out1, 7);
2084 CLIP_SH2_0_255_MAX_SATU(out0, out1);
2085 out = (v16u8) __msa_pckev_b((v16i8) out1, (v16i8) out0);
2086 ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2087 dst += (4 * dst_stride);
2095 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
2099 static void hevc_hv_bi_8t_16w_msa(uint8_t *src0_ptr,
2102 int32_t src2_stride,
2105 const int8_t *filter_x,
2106 const int8_t *filter_y,
2109 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2110 dst, dst_stride, filter_x, filter_y,
2114 static void hevc_hv_bi_8t_24w_msa(uint8_t *src0_ptr,
2117 int32_t src2_stride,
2120 const int8_t *filter_x,
2121 const int8_t *filter_y,
2124 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2125 dst, dst_stride, filter_x, filter_y,
2129 static void hevc_hv_bi_8t_32w_msa(uint8_t *src0_ptr,
2132 int32_t src2_stride,
2135 const int8_t *filter_x,
2136 const int8_t *filter_y,
2139 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2140 dst, dst_stride, filter_x, filter_y,
2144 static void hevc_hv_bi_8t_48w_msa(uint8_t *src0_ptr,
2147 int32_t src2_stride,
2150 const int8_t *filter_x,
2151 const int8_t *filter_y,
2154 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2155 dst, dst_stride, filter_x, filter_y,
2159 static void hevc_hv_bi_8t_64w_msa(uint8_t *src0_ptr,
2162 int32_t src2_stride,
2165 const int8_t *filter_x,
2166 const int8_t *filter_y,
2169 hevc_hv_bi_8t_8multx1mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2170 dst, dst_stride, filter_x, filter_y,
2174 static void hevc_hz_bi_4t_4x2_msa(uint8_t *src0_ptr,
2177 int32_t src2_stride,
2180 const int8_t *filter,
2184 v16i8 src0, src1, dst0, vec0, vec1;
2186 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2189 v8i16 filter_vec, const_vec;
2193 const_vec = __msa_ldi_h(128);
2196 filter_vec = LD_SH(filter);
2197 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2201 LD_SB2(src0_ptr, src_stride, src0, src1);
2202 LD_SH2(src1_ptr, src2_stride, in0, in1);
2203 in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
2204 XORI_B2_128_SB(src0, src1);
2205 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2207 DPADD_SB2_SH(vec0, vec1, filt0, filt1, tmp0, tmp0);
2209 tmp0 = __msa_adds_s_h(tmp0, in0);
2210 tmp0 = __msa_srari_h(tmp0, 7);
2211 tmp0 = CLIP_SH_0_255(tmp0);
2212 dst0 = __msa_pckev_b((v16i8) tmp0, (v16i8) tmp0);
2214 ST4x2_UB(dst0, dst, dst_stride);
2217 static void hevc_hz_bi_4t_4x4_msa(uint8_t *src0_ptr,
2220 int32_t src2_stride,
2223 const int8_t *filter,
2227 v16i8 src0, src1, src2, src3, dst0, vec0, vec1;
2228 v8i16 in0, in1, in2, in3;
2230 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2233 v8i16 filter_vec, const_vec;
2237 const_vec = __msa_ldi_h(128);
2240 filter_vec = LD_SH(filter);
2241 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2245 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2246 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2248 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
2249 XORI_B4_128_SB(src0, src1, src2, src3);
2253 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2254 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2255 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt1, filt1, tmp0, tmp1,
2257 HEVC_BI_RND_CLIP2(in0, in1, tmp0, tmp1, 7, tmp0, tmp1);
2258 dst0 = __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2260 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
2263 static void hevc_hz_bi_4t_4x8multiple_msa(uint8_t *src0_ptr,
2266 int32_t src2_stride,
2269 const int8_t *filter,
2274 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2276 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
2277 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2278 v16i8 mask1, vec0, vec1, vec2, vec3;
2279 v8i16 tmp0, tmp1, tmp2, tmp3;
2280 v8i16 filter_vec, const_vec;
2284 const_vec = __msa_ldi_h(128);
2287 filter_vec = LD_SH(filter);
2288 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2292 for (loop_cnt = (height >> 3); loop_cnt--;) {
2293 LD_SB8(src0_ptr, src_stride,
2294 src0, src1, src2, src3, src4, src5, src6, src7);
2295 src0_ptr += (8 * src_stride);
2296 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2297 src1_ptr += (4 * src2_stride);
2298 LD_SH4(src1_ptr, src2_stride, in4, in5, in6, in7);
2299 src1_ptr += (4 * src2_stride);
2300 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
2301 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
2302 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2308 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2309 VSHF_B2_SB(src4, src5, src6, src7, mask0, mask0, vec2, vec3);
2310 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, tmp0,
2312 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec0, vec1);
2313 VSHF_B2_SB(src4, src5, src6, src7, mask1, mask1, vec2, vec3);
2314 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, tmp0,
2317 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2318 tmp0, tmp1, tmp2, tmp3, 7, tmp0, tmp1, tmp2, tmp3);
2320 PCKEV_B2_SB(tmp1, tmp0, tmp3, tmp2, dst0, dst1);
2321 ST4x8_UB(dst0, dst1, dst, dst_stride);
2322 dst += (8 * dst_stride);
2326 static void hevc_hz_bi_4t_4w_msa(uint8_t *src0_ptr,
2329 int32_t src2_stride,
2332 const int8_t *filter,
2336 hevc_hz_bi_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2337 dst, dst_stride, filter, height);
2338 } else if (4 == height) {
2339 hevc_hz_bi_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2340 dst, dst_stride, filter, height);
2341 } else if (8 == height || 16 == height) {
2342 hevc_hz_bi_4t_4x8multiple_msa(src0_ptr, src_stride,
2343 src1_ptr, src2_stride,
2344 dst, dst_stride, filter, height);
2348 static void hevc_hz_bi_4t_6w_msa(uint8_t *src0_ptr,
2351 int32_t src2_stride,
2354 const int8_t *filter,
2359 v16i8 src0, src1, src2, src3;
2360 v8i16 in0, in1, in2, in3;
2361 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2363 v16i8 vec0, vec1, vec2, vec3;
2364 v8i16 dst0, dst1, dst2, dst3;
2365 v8i16 filter_vec, const_vec;
2369 const_vec = __msa_ldi_h(128);
2372 filter_vec = LD_SH(filter);
2373 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2377 for (loop_cnt = (height >> 2); loop_cnt--;) {
2378 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2379 src0_ptr += (4 * src_stride);
2380 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2381 src1_ptr += (4 * src2_stride);
2382 XORI_B4_128_SB(src0, src1, src2, src3);
2388 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2389 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2390 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2392 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
2393 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
2394 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2397 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2398 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2400 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2401 ST6x4_UB(dst0, dst1, dst, dst_stride);
2402 dst += (4 * dst_stride);
2406 static void hevc_hz_bi_4t_8x2_msa(uint8_t *src0_ptr,
2409 int32_t src2_stride,
2412 const int8_t *filter,
2418 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2419 v16i8 mask1, vec0, vec1, vec2, vec3;
2421 v8i16 filter_vec, const_vec;
2425 const_vec = __msa_ldi_h(128);
2428 filter_vec = LD_SH(filter);
2429 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2433 LD_SB2(src0_ptr, src_stride, src0, src1);
2434 LD_SH2(src1_ptr, src2_stride, in0, in1);
2435 XORI_B2_128_SB(src0, src1);
2439 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2440 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec2, vec3);
2441 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt1, filt1, dst0, dst1,
2443 HEVC_BI_RND_CLIP2(in0, in1, dst0, dst1, 7, dst0, dst1);
2445 dst0 = (v8i16) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
2446 ST8x2_UB(dst0, dst, dst_stride);
2449 static void hevc_hz_bi_4t_8x6_msa(uint8_t *src0_ptr,
2452 int32_t src2_stride,
2455 const int8_t *filter,
2459 v16i8 src0, src1, src2, src3, src4, src5;
2460 v8i16 in0, in1, in2, in3, in4, in5;
2461 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2463 v16i8 vec0, vec1, vec2, vec3;
2464 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
2465 v8i16 filter_vec, const_vec;
2469 const_vec = __msa_ldi_h(128);
2472 filter_vec = LD_SH(filter);
2473 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2477 LD_SB6(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5);
2478 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2479 src1_ptr += (4 * src2_stride);
2480 LD_SH2(src1_ptr, src2_stride, in4, in5);
2481 XORI_B6_128_SB(src0, src1, src2, src3, src4, src5);
2487 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2488 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2489 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0, dst1,
2491 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
2492 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
2493 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0, dst1,
2498 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2499 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec2, vec3);
2500 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt1, filt1, dst4, dst5,
2503 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2504 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2505 HEVC_BI_RND_CLIP2(in4, in5, dst4, dst5, 7, dst4, dst5);
2507 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2508 dst2 = (v8i16) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
2509 ST8x4_UB(dst0, dst1, dst, dst_stride);
2510 dst += (4 * dst_stride);
2511 ST8x2_UB(dst2, dst, dst_stride);
2514 static void hevc_hz_bi_4t_8x4multiple_msa(uint8_t *src0_ptr,
2517 int32_t src2_stride,
2520 const int8_t *filter,
2525 v16i8 src0, src1, src2, src3;
2526 v8i16 in0, in1, in2, in3;
2527 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2529 v16i8 vec0, vec1, vec2, vec3;
2530 v8i16 dst0, dst1, dst2, dst3;
2531 v8i16 filter_vec, const_vec;
2535 const_vec = __msa_ldi_h(128);
2538 filter_vec = LD_SH(filter);
2539 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2543 for (loop_cnt = (height >> 2); loop_cnt--;) {
2544 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2545 src0_ptr += (4 * src_stride);
2546 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2547 src1_ptr += (4 * src2_stride);
2548 XORI_B4_128_SB(src0, src1, src2, src3);
2554 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2555 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2556 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2558 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
2559 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
2560 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2563 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2564 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2566 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2567 ST8x4_UB(dst0, dst1, dst, dst_stride);
2568 dst += (4 * dst_stride);
2572 static void hevc_hz_bi_4t_8w_msa(uint8_t *src0_ptr,
2575 int32_t src2_stride,
2578 const int8_t *filter,
2582 hevc_hz_bi_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2583 dst, dst_stride, filter, height);
2584 } else if (6 == height) {
2585 hevc_hz_bi_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2586 dst, dst_stride, filter, height);
2587 } else if (0 == (height % 4)) {
2588 hevc_hz_bi_4t_8x4multiple_msa(src0_ptr, src_stride,
2589 src1_ptr, src2_stride,
2590 dst, dst_stride, filter, height);
2594 static void hevc_hz_bi_4t_12w_msa(uint8_t *src0_ptr,
2597 int32_t src2_stride,
2600 const int8_t *filter,
2605 v16i8 src0, src1, src2, src3;
2606 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
2607 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2609 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
2612 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2613 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
2614 v8i16 filter_vec, const_vec;
2618 const_vec = __msa_ldi_h(128);
2621 filter_vec = LD_SH(filter);
2622 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2627 for (loop_cnt = (height >> 2); loop_cnt--;) {
2628 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2629 src0_ptr += (4 * src_stride);
2630 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2631 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
2632 src1_ptr += (4 * src2_stride);
2634 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
2635 XORI_B4_128_SB(src0, src1, src2, src3);
2643 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2644 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2645 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2646 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2648 DPADD_SB2_SH(vec4, vec5, filt0, filt0, dst4, dst5);
2649 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
2650 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
2651 VSHF_B2_SB(src0, src1, src2, src3, mask3, mask3, vec4, vec5);
2652 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2654 DPADD_SB2_SH(vec4, vec5, filt1, filt1, dst4, dst5);
2656 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2657 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2658 HEVC_BI_RND_CLIP2(in4, in5, dst4, dst5, 7, dst4, dst5);
2660 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2661 dst2 = (v8i16) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
2662 ST12x4_UB(dst0, dst1, dst2, dst, dst_stride);
2663 dst += (4 * dst_stride);
2667 static void hevc_hz_bi_4t_16w_msa(uint8_t *src0_ptr,
2670 int32_t src2_stride,
2673 const int8_t *filter,
2677 v16i8 src0, src1, src2, src3, vec0, vec1, vec2, vec3;
2678 v8i16 in0, in1, in2, in3, dst0, dst1, dst2, dst3;
2680 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2682 v8i16 filter_vec, const_vec;
2686 const_vec = __msa_ldi_h(128);
2689 filter_vec = LD_SH(filter);
2690 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2694 for (loop_cnt = (height >> 1); loop_cnt--;) {
2695 LD_SB2(src0_ptr, src_stride, src0, src2);
2696 LD_SB2(src0_ptr + 8, src_stride, src1, src3);
2697 src0_ptr += (2 * src_stride);
2698 LD_SH2(src1_ptr, src2_stride, in0, in2);
2699 LD_SH2(src1_ptr + 8, src2_stride, in1, in3);
2700 src1_ptr += (2 * src2_stride);
2702 XORI_B4_128_SB(src0, src1, src2, src3);
2709 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2710 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2711 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2713 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0, vec1);
2714 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2, vec3);
2715 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2718 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2719 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2721 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2722 ST_SH2(dst0, dst1, dst, dst_stride);
2723 dst += (2 * dst_stride);
2727 static void hevc_hz_bi_4t_24w_msa(uint8_t *src0_ptr,
2730 int32_t src2_stride,
2733 const int8_t *filter,
2736 int16_t *src1_ptr_tmp;
2739 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2740 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
2742 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2743 v16i8 mask1, mask2, mask3;
2744 v16i8 vec0, vec1, vec2, vec3;
2745 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2746 v8i16 filter_vec, const_vec;
2750 const_vec = __msa_ldi_h(128);
2753 filter_vec = LD_SH(filter);
2754 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2761 src1_ptr_tmp = src1_ptr + 16;
2763 for (loop_cnt = (height >> 2); loop_cnt--;) {
2764 LD_SB4(src0_ptr, src_stride, src0, src2, src4, src6);
2765 LD_SB4(src0_ptr + 16, src_stride, src1, src3, src5, src7);
2766 src0_ptr += (4 * src_stride);
2767 LD_SH4(src1_ptr, src2_stride, in0, in2, in4, in6);
2768 LD_SH4(src1_ptr + 8, src2_stride, in1, in3, in5, in7);
2769 src1_ptr += (4 * src2_stride);
2770 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2776 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask2, vec0, vec1);
2777 VSHF_B2_SB(src2, src2, src2, src3, mask0, mask2, vec2, vec3);
2778 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2780 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask3, vec0, vec1);
2781 VSHF_B2_SB(src2, src2, src2, src3, mask1, mask3, vec2, vec3);
2782 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2789 VSHF_B2_SB(src4, src4, src4, src5, mask0, mask2, vec0, vec1);
2790 VSHF_B2_SB(src6, src6, src6, src7, mask0, mask2, vec2, vec3);
2791 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst4,
2793 VSHF_B2_SB(src4, src4, src4, src5, mask1, mask3, vec0, vec1);
2794 VSHF_B2_SB(src6, src6, src6, src7, mask1, mask3, vec2, vec3);
2795 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst4,
2798 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2799 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2800 HEVC_BI_RND_CLIP4(in4, in5, in6, in7,
2801 dst4, dst5, dst6, dst7, 7, dst4, dst5, dst6, dst7);
2803 PCKEV_B4_SH(dst1, dst0, dst3, dst2,
2804 dst5, dst4, dst7, dst6, dst0, dst1, dst2, dst3);
2805 ST_SH4(dst0, dst1, dst2, dst3, dst, dst_stride);
2806 dst += (4 * dst_stride);
2808 LD_SH4(src1_ptr_tmp, src2_stride, in0, in1, in2, in3);
2809 src1_ptr_tmp += (4 * src2_stride);
2815 VSHF_B2_SB(src1, src1, src3, src3, mask0, mask0, vec0, vec1);
2816 VSHF_B2_SB(src5, src5, src7, src7, mask0, mask0, vec2, vec3);
2817 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2819 VSHF_B2_SB(src1, src1, src3, src3, mask1, mask1, vec0, vec1);
2820 VSHF_B2_SB(src5, src5, src7, src7, mask1, mask1, vec2, vec3);
2821 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2824 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2825 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2827 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2828 ST8x4_UB(dst0, dst1, dst_tmp, dst_stride);
2829 dst_tmp += (4 * dst_stride);
2833 static void hevc_hz_bi_4t_32w_msa(uint8_t *src0_ptr,
2836 int32_t src2_stride,
2839 const int8_t *filter,
2843 v16i8 src0, src1, src2;
2844 v8i16 in0, in1, in2, in3;
2846 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2847 v16i8 mask1, mask2, mask3;
2848 v8i16 dst0, dst1, dst2, dst3;
2849 v16i8 vec0, vec1, vec2, vec3;
2850 v8i16 filter_vec, const_vec;
2854 const_vec = __msa_ldi_h(128);
2857 filter_vec = LD_SH(filter);
2858 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2864 for (loop_cnt = height; loop_cnt--;) {
2865 LD_SB2(src0_ptr, 16, src0, src1);
2866 src2 = LD_SB(src0_ptr + 24);
2867 src0_ptr += src_stride;
2868 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
2869 src1_ptr += src2_stride;
2870 XORI_B3_128_SB(src0, src1, src2);
2876 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask2, vec0, vec1);
2877 VSHF_B2_SB(src1, src1, src2, src2, mask0, mask0, vec2, vec3);
2878 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, dst0,
2880 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask3, vec0, vec1);
2881 VSHF_B2_SB(src1, src1, src2, src2, mask1, mask1, vec2, vec3);
2882 DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt1, filt1, filt1, filt1, dst0,
2885 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
2886 dst0, dst1, dst2, dst3, 7, dst0, dst1, dst2, dst3);
2888 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2889 ST_SH2(dst0, dst1, dst, 16);
2894 static void hevc_vt_bi_4t_4x2_msa(uint8_t *src0_ptr,
2897 int32_t src2_stride,
2900 const int8_t *filter,
2903 v16i8 src0, src1, src2, src3, src4;
2905 v16i8 src10_r, src32_r, src21_r, src43_r, src2110, src4332;
2908 v8i16 filter_vec, const_vec;
2910 src0_ptr -= src_stride;
2912 const_vec = __msa_ldi_h(128);
2915 filter_vec = LD_SH(filter);
2916 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2918 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
2919 src0_ptr += (3 * src_stride);
2921 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2922 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2923 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2925 LD_SB2(src0_ptr, src_stride, src3, src4);
2926 LD_SH2(src1_ptr, src2_stride, in0, in1);
2927 in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
2928 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2929 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_r, (v2i64) src32_r);
2930 src4332 = (v16i8) __msa_xori_b((v16u8) src4332, 128);
2933 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2934 dst10 = __msa_adds_s_h(dst10, in0);
2935 dst10 = __msa_srari_h(dst10, 7);
2936 dst10 = CLIP_SH_0_255(dst10);
2938 dst10 = (v8i16) __msa_pckev_b((v16i8) dst10, (v16i8) dst10);
2939 ST4x2_UB(dst10, dst, dst_stride);
2942 static void hevc_vt_bi_4t_4x4_msa(uint8_t *src0_ptr,
2945 int32_t src2_stride,
2948 const int8_t *filter,
2951 v16i8 src0, src1, src2, src3, src4, src5, src6;
2952 v8i16 in0, in1, in2, in3;
2953 v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
2954 v16i8 src2110, src4332, src6554;
2957 v8i16 filter_vec, const_vec;
2959 src0_ptr -= src_stride;
2961 const_vec = __msa_ldi_h(128);
2964 filter_vec = LD_SH(filter);
2965 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2967 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
2968 src0_ptr += (3 * src_stride);
2969 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2970 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2971 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2973 LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
2974 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2975 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
2976 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
2977 src32_r, src43_r, src54_r, src65_r);
2978 ILVR_D2_SB(src43_r, src32_r, src65_r, src54_r, src4332, src6554);
2979 XORI_B2_128_SB(src4332, src6554);
2982 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
2984 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
2985 HEVC_BI_RND_CLIP2(in0, in1, dst10, dst32, 7, dst10, dst32);
2987 dst10 = (v8i16) __msa_pckev_b((v16i8) dst32, (v16i8) dst10);
2988 ST4x4_UB(dst10, dst10, 0, 1, 2, 3, dst, dst_stride);
2991 static void hevc_vt_bi_4t_4x8multiple_msa(uint8_t *src0_ptr,
2994 int32_t src2_stride,
2997 const int8_t *filter,
3001 v16i8 src0, src1, src2, src3, src4, src5;
3002 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3003 v16i8 src6, src7, src8, src9;
3004 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
3005 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
3006 v16i8 src2110, src4332, src6554, src8776;
3007 v8i16 dst10, dst32, dst54, dst76;
3009 v8i16 filter_vec, const_vec;
3011 src0_ptr -= src_stride;
3013 const_vec = __msa_ldi_h(128);
3016 filter_vec = LD_SH(filter);
3017 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3019 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3020 src0_ptr += (3 * src_stride);
3021 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3022 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
3023 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3025 for (loop_cnt = (height >> 3); loop_cnt--;) {
3026 LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
3027 src0_ptr += (6 * src_stride);
3028 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
3029 src1_ptr += (8 * src2_stride);
3030 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
3031 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
3032 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3033 src32_r, src43_r, src54_r, src65_r);
3034 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3035 ILVR_D3_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r,
3036 src4332, src6554, src8776);
3037 XORI_B3_128_SB(src4332, src6554, src8776);
3040 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst10, dst10);
3042 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst32, dst32);
3044 DPADD_SB2_SH(src6554, src8776, filt0, filt1, dst54, dst54);
3046 LD_SB2(src0_ptr, src_stride, src9, src2);
3047 src0_ptr += (2 * src_stride);
3048 ILVR_B2_SB(src9, src8, src2, src9, src98_r, src109_r);
3049 src2110 = (v16i8) __msa_ilvr_d((v2i64) src109_r, (v2i64) src98_r);
3050 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3052 DPADD_SB2_SH(src8776, src2110, filt0, filt1, dst76, dst76);
3054 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3055 dst10, dst32, dst54, dst76, 7,
3056 dst10, dst32, dst54, dst76);
3058 PCKEV_B2_SH(dst32, dst10, dst76, dst54, dst10, dst54);
3059 ST4x8_UB(dst10, dst54, dst, dst_stride);
3060 dst += (8 * dst_stride);
3064 static void hevc_vt_bi_4t_4w_msa(uint8_t *src0_ptr,
3067 int32_t src2_stride,
3070 const int8_t *filter,
3074 hevc_vt_bi_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3075 dst, dst_stride, filter, height);
3076 } else if (4 == height) {
3077 hevc_vt_bi_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3078 dst, dst_stride, filter, height);
3080 hevc_vt_bi_4t_4x8multiple_msa(src0_ptr, src_stride,
3081 src1_ptr, src2_stride,
3082 dst, dst_stride, filter, height);
3086 static void hevc_vt_bi_4t_6w_msa(uint8_t *src0_ptr,
3089 int32_t src2_stride,
3092 const int8_t *filter,
3095 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3096 v8i16 in0, in1, in2, in3;
3097 v16i8 src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
3098 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3100 v8i16 filter_vec, const_vec;
3102 src0_ptr -= src_stride;
3104 const_vec = __msa_ldi_h(128);
3107 filter_vec = LD_SH(filter);
3108 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3110 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3111 src0_ptr += (3 * src_stride);
3112 LD_SB2(src0_ptr, src_stride, src3, src4);
3113 src0_ptr += (2 * src_stride);
3114 LD_SB2(src0_ptr, src_stride, src5, src6);
3115 src0_ptr += (2 * src_stride);
3116 LD_SB2(src0_ptr, src_stride, src7, src8);
3117 src0_ptr += (2 * src_stride);
3118 LD_SB2(src0_ptr, src_stride, src9, src10);
3119 src0_ptr += (2 * src_stride);
3121 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3122 src1_ptr += (4 * src2_stride);
3124 XORI_B3_128_SB(src0, src1, src2);
3125 XORI_B2_128_SB(src3, src4);
3126 XORI_B2_128_SB(src5, src6);
3127 XORI_B2_128_SB(src7, src8);
3128 XORI_B2_128_SB(src9, src10);
3130 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3131 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3134 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3136 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3138 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
3141 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
3143 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
3145 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3146 dst0_r, dst1_r, dst2_r, dst3_r, 7,
3147 dst0_r, dst1_r, dst2_r, dst3_r);
3149 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
3150 ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
3151 dst += (4 * dst_stride);
3153 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3154 src1_ptr += (4 * src2_stride);
3155 ILVR_B2_SB(src7, src6, src8, src7, src32_r, src43_r);
3158 DPADD_SB2_SH(src54_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3160 DPADD_SB2_SH(src65_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3162 ILVR_B2_SB(src9, src8, src10, src9, src54_r, src65_r);
3165 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
3167 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
3169 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3170 dst0_r, dst1_r, dst2_r, dst3_r, 7,
3171 dst0_r, dst1_r, dst2_r, dst3_r);
3173 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
3174 ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
3175 dst += (4 * dst_stride);
3178 static void hevc_vt_bi_4t_8x2_msa(uint8_t *src0_ptr,
3181 int32_t src2_stride,
3184 const int8_t *filter,
3187 v16i8 src0, src1, src2, src3, src4;
3188 v8i16 in0, in1, dst0_r, dst1_r;
3189 v16i8 src10_r, src32_r, src21_r, src43_r;
3191 v8i16 filter_vec, const_vec;
3193 src0_ptr -= src_stride;
3195 const_vec = __msa_ldi_h(128);
3198 filter_vec = LD_SH(filter);
3199 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3201 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3202 src0_ptr += (3 * src_stride);
3203 XORI_B3_128_SB(src0, src1, src2);
3204 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3206 LD_SB2(src0_ptr, src_stride, src3, src4);
3207 LD_SH2(src1_ptr, src2_stride, in0, in1);
3208 XORI_B2_128_SB(src3, src4);
3209 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3212 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3214 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3216 HEVC_BI_RND_CLIP2(in0, in1, dst0_r, dst1_r, 7, dst0_r, dst1_r);
3217 dst0_r = (v8i16) __msa_pckev_b((v16i8) dst1_r, (v16i8) dst0_r);
3219 ST8x2_UB(dst0_r, dst, dst_stride);
3222 static void hevc_vt_bi_4t_8x6_msa(uint8_t *src0_ptr,
3225 int32_t src2_stride,
3228 const int8_t *filter,
3231 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3232 v8i16 in0, in1, in2, in3, in4, in5;
3233 v16i8 src10_r, src32_r, src54_r, src76_r;
3234 v16i8 src21_r, src43_r, src65_r, src87_r;
3235 v8i16 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r;
3237 v8i16 filter_vec, const_vec;
3239 src0_ptr -= src_stride;
3241 const_vec = __msa_ldi_h(128);
3244 filter_vec = LD_SH(filter);
3245 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3247 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3248 src0_ptr += (3 * src_stride);
3249 XORI_B3_128_SB(src0, src1, src2);
3250 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3252 LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
3253 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
3254 XORI_B6_128_SB(src3, src4, src5, src6, src7, src8);
3255 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3256 src32_r, src43_r, src54_r, src65_r);
3257 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3260 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3262 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3264 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
3266 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
3268 DPADD_SB2_SH(src54_r, src76_r, filt0, filt1, dst4_r, dst4_r);
3270 DPADD_SB2_SH(src65_r, src87_r, filt0, filt1, dst5_r, dst5_r);
3271 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3272 dst0_r, dst1_r, dst2_r, dst3_r, 7,
3273 dst0_r, dst1_r, dst2_r, dst3_r);
3274 HEVC_BI_RND_CLIP2(in4, in5, dst4_r, dst5_r, 7, dst4_r, dst5_r);
3276 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
3277 dst2_r = (v8i16) __msa_pckev_b((v16i8) dst5_r, (v16i8) dst4_r);
3278 ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
3279 dst += (4 * dst_stride);
3280 ST8x2_UB(dst2_r, dst, dst_stride);
3283 static void hevc_vt_bi_4t_8x4multiple_msa(uint8_t *src0_ptr,
3286 int32_t src2_stride,
3289 const int8_t *filter,
3293 v16i8 src0, src1, src2, src3, src4, src5;
3294 v8i16 in0, in1, in2, in3;
3295 v16i8 src10_r, src32_r, src21_r, src43_r;
3296 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3298 v8i16 filter_vec, const_vec;
3300 src0_ptr -= src_stride;
3302 const_vec = __msa_ldi_h(128);
3305 filter_vec = LD_SH(filter);
3306 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3308 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3309 src0_ptr += (3 * src_stride);
3310 XORI_B3_128_SB(src0, src1, src2);
3311 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3313 for (loop_cnt = (height >> 2); loop_cnt--;) {
3314 LD_SB2(src0_ptr, src_stride, src3, src4);
3315 src0_ptr += (2 * src_stride);
3316 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3317 src1_ptr += (4 * src2_stride);
3318 XORI_B2_128_SB(src3, src4);
3319 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3322 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3324 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3326 LD_SB2(src0_ptr, src_stride, src5, src2);
3327 src0_ptr += (2 * src_stride);
3328 XORI_B2_128_SB(src5, src2);
3329 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3332 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst2_r, dst2_r);
3334 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst3_r, dst3_r);
3335 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3336 dst0_r, dst1_r, dst2_r, dst3_r, 7,
3337 dst0_r, dst1_r, dst2_r, dst3_r);
3339 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
3340 ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
3341 dst += (4 * dst_stride);
3345 static void hevc_vt_bi_4t_8w_msa(uint8_t *src0_ptr,
3348 int32_t src2_stride,
3351 const int8_t *filter,
3355 hevc_vt_bi_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3356 dst, dst_stride, filter, height);
3357 } else if (6 == height) {
3358 hevc_vt_bi_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3359 dst, dst_stride, filter, height);
3361 hevc_vt_bi_4t_8x4multiple_msa(src0_ptr, src_stride,
3362 src1_ptr, src2_stride,
3363 dst, dst_stride, filter, height);
3367 static void hevc_vt_bi_4t_12w_msa(uint8_t *src0_ptr,
3370 int32_t src2_stride,
3373 const int8_t *filter,
3377 v16i8 src0, src1, src2, src3, src4, src5, src6;
3378 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3379 v16i8 src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
3380 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3381 v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
3382 v16i8 src2110, src4332, src6554;
3383 v8i16 dst0_l, dst1_l, filt0, filt1;
3384 v8i16 filter_vec, const_vec;
3386 src0_ptr -= (1 * src_stride);
3388 const_vec = __msa_ldi_h(128);
3391 filter_vec = LD_SH(filter);
3392 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3394 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3395 src0_ptr += (3 * src_stride);
3396 XORI_B3_128_SB(src0, src1, src2);
3397 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3398 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3399 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
3401 for (loop_cnt = (height >> 2); loop_cnt--;) {
3402 LD_SB2(src0_ptr, src_stride, src3, src4);
3403 src0_ptr += (2 * src_stride);
3404 LD_SB2(src0_ptr, src_stride, src5, src6);
3405 src0_ptr += (2 * src_stride);
3406 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3407 LD_SH4((src1_ptr + 8), src2_stride, in4, in5, in6, in7);
3408 src1_ptr += (4 * src2_stride);
3409 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
3410 XORI_B2_128_SB(src3, src4);
3411 XORI_B2_128_SB(src5, src6);
3413 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3414 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3415 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
3416 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
3417 ILVL_B2_SB(src5, src4, src6, src5, src54_l, src65_l);
3418 src6554 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
3421 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3423 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3425 DPADD_SB2_SH(src2110, src4332, filt0, filt1, dst0_l, dst0_l);
3427 DPADD_SB2_SH(src32_r, src54_r, filt0, filt1, dst2_r, dst2_r);
3429 DPADD_SB2_SH(src43_r, src65_r, filt0, filt1, dst3_r, dst3_r);
3431 DPADD_SB2_SH(src4332, src6554, filt0, filt1, dst1_l, dst1_l);
3432 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3433 dst0_r, dst1_r, dst2_r, dst3_r, 7,
3434 dst0_r, dst1_r, dst2_r, dst3_r);
3435 HEVC_BI_RND_CLIP2(in4, in5, dst0_l, dst1_l, 7, dst0_l, dst1_l);
3437 PCKEV_B2_SH(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
3438 dst0_l = (v8i16) __msa_pckev_b((v16i8) dst1_l, (v16i8) dst0_l);
3439 ST12x4_UB(dst0_r, dst1_r, dst0_l, dst, dst_stride);
3440 dst += (4 * dst_stride);
3449 static void hevc_vt_bi_4t_16w_msa(uint8_t *src0_ptr,
3452 int32_t src2_stride,
3455 const int8_t *filter,
3459 v16i8 src0, src1, src2, src3, src4, src5;
3460 v8i16 in0, in1, in2, in3;
3461 v16i8 src10_r, src32_r, src21_r, src43_r;
3462 v16i8 src10_l, src32_l, src21_l, src43_l;
3463 v8i16 dst0_r, dst1_r, dst0_l, dst1_l;
3465 v8i16 filter_vec, const_vec;
3467 src0_ptr -= src_stride;
3469 const_vec = __msa_ldi_h(128);
3472 filter_vec = LD_SH(filter);
3473 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3475 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3476 src0_ptr += (3 * src_stride);
3477 XORI_B3_128_SB(src0, src1, src2);
3478 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3479 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3481 for (loop_cnt = (height >> 2); loop_cnt--;) {
3482 LD_SB2(src0_ptr, src_stride, src3, src4);
3483 src0_ptr += (2 * src_stride);
3484 LD_SH2(src1_ptr, src2_stride, in0, in1);
3485 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
3486 src1_ptr += (2 * src2_stride);
3487 XORI_B2_128_SB(src3, src4);
3488 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3489 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3492 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3494 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3496 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3498 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3499 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3500 dst0_r, dst1_r, dst0_l, dst1_l, 7,
3501 dst0_r, dst1_r, dst0_l, dst1_l);
3503 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3504 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
3505 dst += (2 * dst_stride);
3507 LD_SB2(src0_ptr, src_stride, src5, src2);
3508 src0_ptr += (2 * src_stride);
3509 LD_SH2(src1_ptr, src2_stride, in0, in1);
3510 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
3511 src1_ptr += (2 * src2_stride);
3512 XORI_B2_128_SB(src5, src2);
3513 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3514 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
3517 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
3519 DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, dst0_l, dst0_l);
3521 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
3523 DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, dst1_l, dst1_l);
3524 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3525 dst0_r, dst1_r, dst0_l, dst1_l, 7,
3526 dst0_r, dst1_r, dst0_l, dst1_l);
3528 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3529 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
3530 dst += (2 * dst_stride);
3534 static void hevc_vt_bi_4t_24w_msa(uint8_t *src0_ptr,
3537 int32_t src2_stride,
3540 const int8_t *filter,
3544 v16i8 src0, src1, src2, src3, src4, src5;
3545 v16i8 src6, src7, src8, src9, src10, src11;
3546 v8i16 in0, in1, in2, in3, in4, in5;
3547 v16i8 src10_r, src32_r, src76_r, src98_r;
3548 v16i8 src21_r, src43_r, src87_r, src109_r;
3549 v16i8 src10_l, src32_l, src21_l, src43_l;
3550 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3551 v8i16 dst0_l, dst1_l;
3553 v8i16 filter_vec, const_vec;
3555 src0_ptr -= src_stride;
3557 const_vec = __msa_ldi_h(128);
3560 filter_vec = LD_SH(filter);
3561 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3564 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3565 XORI_B3_128_SB(src0, src1, src2);
3566 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3567 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3569 LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
3570 src0_ptr += (3 * src_stride);
3571 XORI_B3_128_SB(src6, src7, src8);
3572 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3574 for (loop_cnt = (height >> 2); loop_cnt--;) {
3576 LD_SB2(src0_ptr, src_stride, src3, src4);
3577 LD_SH2(src1_ptr, src2_stride, in0, in1);
3578 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
3579 LD_SH2((src1_ptr + 16), src2_stride, in4, in5);
3580 src1_ptr += (2 * src2_stride);
3581 XORI_B2_128_SB(src3, src4);
3582 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3583 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3585 LD_SB2(src0_ptr + 16, src_stride, src9, src10);
3586 src0_ptr += (2 * src_stride);
3587 XORI_B2_128_SB(src9, src10);
3588 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3591 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3593 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3595 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3597 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3600 DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, dst2_r, dst2_r);
3602 DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, dst3_r, dst3_r);
3604 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3605 dst0_r, dst1_r, dst0_l, dst1_l, 7,
3606 dst0_r, dst1_r, dst0_l, dst1_l);
3608 HEVC_BI_RND_CLIP2(in4, in5, dst2_r, dst3_r, 7, dst2_r, dst3_r);
3610 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3611 dst2_r = (v8i16) __msa_pckev_b((v16i8) dst3_r, (v16i8) dst2_r);
3612 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
3613 ST8x2_UB(dst2_r, dst + 16, dst_stride);
3614 dst += (2 * dst_stride);
3617 LD_SB2(src0_ptr, src_stride, src5, src2);
3618 LD_SH2(src1_ptr, src2_stride, in0, in1);
3619 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
3620 LD_SH2((src1_ptr + 16), src2_stride, in4, in5);
3621 src1_ptr += (2 * src2_stride);
3622 XORI_B2_128_SB(src5, src2);
3623 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
3624 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
3626 LD_SB2(src0_ptr + 16, src_stride, src11, src8);
3627 src0_ptr += (2 * src_stride);
3628 XORI_B2_128_SB(src11, src8);
3629 ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
3632 DPADD_SB2_SH(src32_r, src10_r, filt0, filt1, dst0_r, dst0_r);
3634 DPADD_SB2_SH(src32_l, src10_l, filt0, filt1, dst0_l, dst0_l);
3636 DPADD_SB2_SH(src43_r, src21_r, filt0, filt1, dst1_r, dst1_r);
3638 DPADD_SB2_SH(src43_l, src21_l, filt0, filt1, dst1_l, dst1_l);
3641 DPADD_SB2_SH(src98_r, src76_r, filt0, filt1, dst2_r, dst2_r);
3643 DPADD_SB2_SH(src109_r, src87_r, filt0, filt1, dst3_r, dst3_r);
3645 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3646 dst0_r, dst1_r, dst0_l, dst1_l, 7,
3647 dst0_r, dst1_r, dst0_l, dst1_l);
3648 HEVC_BI_RND_CLIP2(in4, in5, dst2_r, dst3_r, 7, dst2_r, dst3_r);
3650 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3651 dst2_r = (v8i16) __msa_pckev_b((v16i8) dst3_r, (v16i8) dst2_r);
3652 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
3653 ST8x2_UB(dst2_r, dst + 16, dst_stride);
3654 dst += (2 * dst_stride);
3658 static void hevc_vt_bi_4t_32w_msa(uint8_t *src0_ptr,
3661 int32_t src2_stride,
3664 const int8_t *filter,
3668 uint8_t *dst_tmp = dst + 16;
3669 v16i8 src0, src1, src2, src3, src4, src6, src7, src8, src9, src10;
3670 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3671 v16i8 src10_r, src32_r, src76_r, src98_r;
3672 v16i8 src21_r, src43_r, src87_r, src109_r;
3673 v8i16 dst0_r, dst1_r, dst2_r, dst3_r;
3674 v16i8 src10_l, src32_l, src76_l, src98_l;
3675 v16i8 src21_l, src43_l, src87_l, src109_l;
3676 v8i16 dst0_l, dst1_l, dst2_l, dst3_l;
3678 v8i16 filter_vec, const_vec;
3680 src0_ptr -= src_stride;
3682 const_vec = __msa_ldi_h(128);
3685 filter_vec = LD_SH(filter);
3686 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3689 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3690 XORI_B3_128_SB(src0, src1, src2);
3691 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3692 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3695 LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
3696 src0_ptr += (3 * src_stride);
3697 XORI_B3_128_SB(src6, src7, src8);
3698 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3699 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
3701 for (loop_cnt = (height >> 1); loop_cnt--;) {
3703 LD_SB2(src0_ptr, src_stride, src3, src4);
3704 LD_SH2(src1_ptr, src2_stride, in0, in1);
3705 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
3706 LD_SH2((src1_ptr + 16), src2_stride, in4, in5);
3707 LD_SH2((src1_ptr + 24), src2_stride, in6, in7);
3708 src1_ptr += (2 * src2_stride);
3709 XORI_B2_128_SB(src3, src4);
3710 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3711 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3714 DPADD_SB2_SH(src10_r, src32_r, filt0, filt1, dst0_r, dst0_r);
3716 DPADD_SB2_SH(src10_l, src32_l, filt0, filt1, dst0_l, dst0_l);
3718 DPADD_SB2_SH(src21_r, src43_r, filt0, filt1, dst1_r, dst1_r);
3720 DPADD_SB2_SH(src21_l, src43_l, filt0, filt1, dst1_l, dst1_l);
3722 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
3723 dst0_r, dst1_r, dst0_l, dst1_l, 7,
3724 dst0_r, dst1_r, dst0_l, dst1_l);
3732 PCKEV_B2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
3733 ST_SH2(dst0_r, dst1_r, dst, dst_stride);
3734 dst += (2 * dst_stride);
3737 LD_SB2(src0_ptr + 16, src_stride, src9, src10);
3738 src0_ptr += (2 * src_stride);
3739 XORI_B2_128_SB(src9, src10);
3740 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3741 ILVL_B2_SB(src9, src8, src10, src9, src98_l, src109_l);
3744 DPADD_SB2_SH(src76_r, src98_r, filt0, filt1, dst2_r, dst2_r);
3746 DPADD_SB2_SH(src76_l, src98_l, filt0, filt1, dst2_l, dst2_l);
3748 DPADD_SB2_SH(src87_r, src109_r, filt0, filt1, dst3_r, dst3_r);
3750 DPADD_SB2_SH(src87_l, src109_l, filt0, filt1, dst3_l, dst3_l);
3752 HEVC_BI_RND_CLIP4(in4, in5, in6, in7,
3753 dst2_r, dst3_r, dst2_l, dst3_l, 7,
3754 dst2_r, dst3_r, dst2_l, dst3_l);
3756 PCKEV_B2_SH(dst2_l, dst2_r, dst3_l, dst3_r, dst2_r, dst3_r);
3757 ST_SH2(dst2_r, dst3_r, dst_tmp, dst_stride);
3758 dst_tmp += (2 * dst_stride);
3768 static void hevc_hv_bi_4t_4x2_msa(uint8_t *src0_ptr,
3771 int32_t src2_stride,
3774 const int8_t *filter_x,
3775 const int8_t *filter_y,
3779 v16i8 src0, src1, src2, src3, src4;
3781 v4i32 filt_h0, filt_h1;
3782 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
3784 v8i16 filter_vec, const_vec;
3785 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
3786 v8i16 dst0, dst1, dst2, dst3, dst4;
3787 v4i32 dst0_r, dst1_r;
3788 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3790 src0_ptr -= (src_stride + 1);
3792 filter_vec = LD_SH(filter_x);
3793 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3795 filter_vec = LD_SH(filter_y);
3796 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
3797 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
3799 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
3803 const_vec = __msa_ldi_h(128);
3806 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3807 src0_ptr += (3 * src_stride);
3808 XORI_B3_128_SB(src0, src1, src2);
3810 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3811 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3812 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3814 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3816 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3818 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3819 ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
3821 LD_SB2(src0_ptr, src_stride, src3, src4);
3822 LD_SH2(src1_ptr, src2_stride, in0, in1);
3823 in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
3824 XORI_B2_128_SB(src3, src4);
3826 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3828 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
3829 dst32_r = __msa_ilvr_h(dst3, dst2);
3830 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3833 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3835 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
3836 dst43_r = __msa_ilvr_h(dst4, dst3);
3837 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3839 dst0_r = (v4i32) __msa_pckev_h((v8i16) dst1_r, (v8i16) dst0_r);
3840 dst0_r = (v4i32) __msa_adds_s_h((v8i16) dst0_r, in0);
3841 dst0_r = (v4i32) __msa_srari_h((v8i16) dst0_r, 7);
3842 dst0_r = (v4i32) CLIP_SH_0_255(dst0_r);
3844 dst0_r = (v4i32) __msa_pckev_b((v16i8) dst0_r, (v16i8) dst0_r);
3845 ST4x2_UB(dst0_r, dst, dst_stride);
3848 static void hevc_hv_bi_4t_4x4_msa(uint8_t *src0_ptr,
3851 int32_t src2_stride,
3854 const int8_t *filter_x,
3855 const int8_t *filter_y,
3858 v8i16 in0, in1, in2, in3;
3859 v16i8 src0, src1, src2, src3, src4, src5, src6;
3861 v4i32 filt_h0, filt_h1;
3862 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
3864 v8i16 filter_vec, const_vec;
3865 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
3866 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
3867 v8i16 dst0_r, dst1_r;
3868 v4i32 tmp0, tmp1, tmp2, tmp3;
3869 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3871 src0_ptr -= (src_stride + 1);
3873 filter_vec = LD_SH(filter_x);
3874 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3876 filter_vec = LD_SH(filter_y);
3877 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
3878 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
3880 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
3884 const_vec = __msa_ldi_h(128);
3887 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3888 src0_ptr += (3 * src_stride);
3889 XORI_B3_128_SB(src0, src1, src2);
3891 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3892 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3893 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3895 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3897 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3899 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3900 ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
3902 LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
3903 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3904 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
3905 XORI_B4_128_SB(src3, src4, src5, src6);
3907 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3909 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
3910 dst32_r = __msa_ilvr_h(dst3, dst2);
3911 tmp0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3914 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3916 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
3917 dst43_r = __msa_ilvr_h(dst4, dst3);
3918 tmp1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3921 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
3923 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
3924 dst10_r = __msa_ilvr_h(dst5, dst4);
3925 tmp2 = HEVC_FILT_4TAP(dst32_r, dst10_r, filt_h0, filt_h1);
3928 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
3930 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
3931 dst21_r = __msa_ilvr_h(dst2, dst5);
3932 tmp3 = HEVC_FILT_4TAP(dst43_r, dst21_r, filt_h0, filt_h1);
3934 PCKEV_H2_SH(tmp1, tmp0, tmp3, tmp2, dst0_r, dst1_r);
3935 HEVC_BI_RND_CLIP2(in0, in1, dst0_r, dst1_r, 7, dst0_r, dst1_r);
3937 dst0_r = (v8i16) __msa_pckev_b((v16i8) dst1_r, (v16i8) dst0_r);
3938 ST4x4_UB(dst0_r, dst0_r, 0, 1, 2, 3, dst, dst_stride);
3939 dst += (4 * dst_stride);
3942 static void hevc_hv_bi_4t_4multx8mult_msa(uint8_t *src0_ptr,
3945 int32_t src2_stride,
3948 const int8_t *filter_x,
3949 const int8_t *filter_y,
3953 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3954 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3956 v4i32 filt_h0, filt_h1;
3957 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
3959 v8i16 filter_vec, const_vec;
3960 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
3961 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9;
3962 v8i16 tmp0, tmp1, tmp2, tmp3;
3963 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
3964 v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
3965 v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
3967 src0_ptr -= (src_stride + 1);
3969 filter_vec = LD_SH(filter_x);
3970 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3972 filter_vec = LD_SH(filter_y);
3973 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
3974 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
3976 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
3980 const_vec = __msa_ldi_h(128);
3983 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3984 src0_ptr += (3 * src_stride);
3985 XORI_B3_128_SB(src0, src1, src2);
3987 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3988 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3989 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3991 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
3993 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
3995 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
3996 ILVR_H2_SH(dst1, dst0, dst2, dst1, dst10_r, dst21_r);
3998 for (loop_cnt = height >> 3; loop_cnt--;) {
3999 LD_SB8(src0_ptr, src_stride,
4000 src3, src4, src5, src6, src7, src8, src9, src10);
4001 src0_ptr += (8 * src_stride);
4002 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
4003 src1_ptr += (8 * src2_stride);
4004 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
4005 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
4006 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
4008 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4010 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4011 dst32_r = __msa_ilvr_h(dst3, dst2);
4012 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4015 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
4017 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
4018 dst43_r = __msa_ilvr_h(dst4, dst3);
4019 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4022 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
4024 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
4025 dst54_r = __msa_ilvr_h(dst5, dst4);
4026 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4029 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
4031 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst6, dst6);
4032 dst65_r = __msa_ilvr_h(dst6, dst5);
4033 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4036 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
4038 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst7, dst7);
4039 dst76_r = __msa_ilvr_h(dst7, dst6);
4040 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
4043 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec0, vec1);
4045 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst8, dst8);
4046 dst87_r = __msa_ilvr_h(dst8, dst7);
4047 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
4050 VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec0, vec1);
4052 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst9, dst9);
4053 dst10_r = __msa_ilvr_h(dst9, dst8);
4054 dst6_r = HEVC_FILT_4TAP(dst76_r, dst10_r, filt_h0, filt_h1);
4057 VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec0, vec1);
4059 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
4060 dst21_r = __msa_ilvr_h(dst2, dst9);
4061 dst7_r = HEVC_FILT_4TAP(dst87_r, dst21_r, filt_h0, filt_h1);
4063 PCKEV_H4_SH(dst1_r, dst0_r, dst3_r, dst2_r,
4064 dst5_r, dst4_r, dst7_r, dst6_r, tmp0, tmp1, tmp2, tmp3);
4065 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
4066 tmp0, tmp1, tmp2, tmp3, 7, tmp0, tmp1, tmp2, tmp3);
4068 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
4069 ST4x8_UB(tmp0, tmp1, dst, dst_stride);
4070 dst += (8 * dst_stride);
4074 static void hevc_hv_bi_4t_4w_msa(uint8_t *src0_ptr,
4077 int32_t src2_stride,
4080 const int8_t *filter_x,
4081 const int8_t *filter_y,
4085 hevc_hv_bi_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4086 dst, dst_stride, filter_x, filter_y, height);
4087 } else if (4 == height) {
4088 hevc_hv_bi_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4089 dst, dst_stride, filter_x, filter_y, height);
4090 } else if (0 == (height % 8)) {
4091 hevc_hv_bi_4t_4multx8mult_msa(src0_ptr, src_stride,
4092 src1_ptr, src2_stride,
4094 filter_x, filter_y, height);
4098 static void hevc_hv_bi_4t_6w_msa(uint8_t *src0_ptr,
4101 int32_t src2_stride,
4104 const int8_t *filter_x,
4105 const int8_t *filter_y,
4109 v16i8 src0, src1, src2, src3, src4, src5, src6;
4110 v8i16 in0, in1, in2, in3;
4112 v4i32 filt_h0, filt_h1;
4113 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
4115 v8i16 filter_vec, const_vec;
4116 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4117 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
4118 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
4119 v8i16 tmp0, tmp1, tmp2, tmp3;
4120 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
4121 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
4123 src0_ptr -= (src_stride + 1);
4125 filter_vec = LD_SH(filter_x);
4126 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4128 filter_vec = LD_SH(filter_y);
4129 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
4130 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
4132 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
4136 const_vec = __msa_ldi_h(128);
4139 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4140 src0_ptr += (3 * src_stride);
4141 XORI_B3_128_SB(src0, src1, src2);
4143 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4144 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4145 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4147 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
4149 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
4151 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
4153 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
4154 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
4156 for (loop_cnt = height >> 2; loop_cnt--;) {
4157 LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
4158 src0_ptr += (4 * src_stride);
4159 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
4160 src1_ptr += (4 * src2_stride);
4161 XORI_B4_128_SB(src3, src4, src5, src6);
4163 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4165 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4167 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4168 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4169 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4173 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
4175 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
4177 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4178 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4179 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4183 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
4185 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
4187 ILVRL_H2_SH(dst5, dst4, dst10_r, dst10_l);
4188 dst2_r = HEVC_FILT_4TAP(dst32_r, dst10_r, filt_h0, filt_h1);
4189 dst2_l = HEVC_FILT_4TAP(dst32_l, dst10_l, filt_h0, filt_h1);
4193 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
4195 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
4197 ILVRL_H2_SH(dst2, dst5, dst21_r, dst21_l);
4198 dst3_r = HEVC_FILT_4TAP(dst43_r, dst21_r, filt_h0, filt_h1);
4199 dst3_l = HEVC_FILT_4TAP(dst43_l, dst21_l, filt_h0, filt_h1);
4202 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r,
4203 dst2_l, dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
4204 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
4205 tmp0, tmp1, tmp2, tmp3, 7, tmp0, tmp1, tmp2, tmp3);
4207 PCKEV_B2_SW(tmp1, tmp0, tmp3, tmp2, dst0_r, dst1_r);
4208 ST6x4_UB(dst0_r, dst1_r, dst, dst_stride);
4209 dst += (4 * dst_stride);
4213 static void hevc_hv_bi_4t_8x2_msa(uint8_t *src0_ptr,
4216 int32_t src2_stride,
4219 const int8_t *filter_x,
4220 const int8_t *filter_y,
4223 v16i8 src0, src1, src2, src3, src4;
4225 v4i32 filt_h0, filt_h1;
4226 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
4228 v8i16 filter_vec, const_vec;
4229 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4230 v8i16 dst0, dst1, dst2, dst3, dst4;
4231 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
4232 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
4233 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
4237 src0_ptr -= (src_stride + 1);
4239 filter_vec = LD_SH(filter_x);
4240 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4242 filter_vec = LD_SH(filter_y);
4243 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
4244 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
4246 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
4250 const_vec = __msa_ldi_h(128);
4253 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4254 src0_ptr += (3 * src_stride);
4255 XORI_B3_128_SB(src0, src1, src2);
4257 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4258 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4259 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4261 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
4263 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
4265 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
4267 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
4268 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
4270 LD_SB2(src0_ptr, src_stride, src3, src4);
4271 LD_SH2(src1_ptr, src2_stride, in0, in1);
4272 XORI_B2_128_SB(src3, src4);
4274 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4276 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4278 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4279 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4280 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4284 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
4286 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
4288 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4289 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4290 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4294 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1);
4295 HEVC_BI_RND_CLIP2(in0, in1, tmp0, tmp1, 7, tmp0, tmp1);
4297 dst0_r = (v4i32) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
4298 ST8x2_UB(dst0_r, dst, dst_stride);
4301 static void hevc_hv_bi_4t_8x6_msa(uint8_t *src0_ptr,
4304 int32_t src2_stride,
4307 const int8_t *filter_x,
4308 const int8_t *filter_y,
4311 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
4312 v8i16 in0, in1, in2, in3, in4, in5;
4314 v4i32 filt_h0, filt_h1;
4315 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
4317 v8i16 filter_vec, const_vec;
4318 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4319 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
4320 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
4321 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
4322 v4i32 dst4_r, dst4_l, dst5_r, dst5_l;
4323 v8i16 dst10_r, dst32_r, dst10_l, dst32_l;
4324 v8i16 dst21_r, dst43_r, dst21_l, dst43_l;
4325 v8i16 dst54_r, dst54_l, dst65_r, dst65_l;
4326 v8i16 dst76_r, dst76_l, dst87_r, dst87_l;
4328 src0_ptr -= (src_stride + 1);
4330 filter_vec = LD_SH(filter_x);
4331 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4333 filter_vec = LD_SH(filter_y);
4334 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
4335 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
4337 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
4341 const_vec = __msa_ldi_h(128);
4344 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4345 src0_ptr += (3 * src_stride);
4346 XORI_B3_128_SB(src0, src1, src2);
4347 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4348 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4349 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4351 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
4353 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
4355 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
4357 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
4358 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
4360 LD_SB2(src0_ptr, src_stride, src3, src4);
4361 src0_ptr += (2 * src_stride);
4362 XORI_B2_128_SB(src3, src4);
4363 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
4364 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4366 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4368 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4369 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4370 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4373 tmp0 = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
4375 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
4377 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
4379 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4380 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4381 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4384 tmp1 = __msa_pckev_h((v8i16) dst1_l, (v8i16) dst1_r);
4386 LD_SB2(src0_ptr, src_stride, src5, src6);
4387 src0_ptr += (2 * src_stride);
4388 XORI_B2_128_SB(src5, src6);
4390 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
4392 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
4394 ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
4395 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4396 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
4399 tmp2 = __msa_pckev_h((v8i16) dst2_l, (v8i16) dst2_r);
4402 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
4404 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst6, dst6);
4406 ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
4407 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4408 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
4411 tmp3 = __msa_pckev_h((v8i16) dst3_l, (v8i16) dst3_r);
4413 LD_SB2(src0_ptr, src_stride, src7, src8);
4414 XORI_B2_128_SB(src7, src8);
4416 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
4418 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst7, dst7);
4420 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
4421 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
4422 dst4_l = HEVC_FILT_4TAP(dst54_l, dst76_l, filt_h0, filt_h1);
4426 tmp4 = __msa_pckev_h((v8i16) dst4_l, (v8i16) dst4_r);
4428 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec0, vec1);
4430 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst8, dst8);
4432 ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
4433 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
4434 dst5_l = HEVC_FILT_4TAP(dst65_l, dst87_l, filt_h0, filt_h1);
4437 tmp5 = __msa_pckev_h((v8i16) dst5_l, (v8i16) dst5_r);
4439 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
4440 tmp0, tmp1, tmp2, tmp3, 7, tmp0, tmp1, tmp2, tmp3);
4441 HEVC_BI_RND_CLIP2(in4, in5, tmp4, tmp5, 7, tmp4, tmp5);
4443 PCKEV_B2_SW(tmp1, tmp0, tmp3, tmp2, dst0_r, dst1_r);
4444 dst2_r = (v4i32) __msa_pckev_b((v16i8) tmp5, (v16i8) tmp4);
4445 ST8x4_UB(dst0_r, dst1_r, dst, dst_stride);
4446 dst += (4 * dst_stride);
4447 ST8x2_UB(dst2_r, dst, dst_stride);
4450 static void hevc_hv_bi_4t_8multx4mult_msa(uint8_t *src0_ptr,
4453 int32_t src2_stride,
4456 const int8_t *filter_x,
4457 const int8_t *filter_y,
4461 uint32_t loop_cnt, cnt;
4462 uint8_t *src0_ptr_tmp;
4463 int16_t *src1_ptr_tmp;
4465 v16i8 src0, src1, src2, src3, src4, src5, src6;
4466 v8i16 in0, in1, in2, in3;
4468 v4i32 filt_h0, filt_h1;
4469 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
4471 v8i16 filter_vec, const_vec;
4472 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4473 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
4474 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
4475 v8i16 tmp0, tmp1, tmp2, tmp3;
4476 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
4477 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
4479 src0_ptr -= (src_stride + 1);
4481 filter_vec = LD_SH(filter_x);
4482 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4484 filter_vec = LD_SH(filter_y);
4485 vec0 = __msa_clti_s_b((v16i8) filter_vec, 0);
4486 filter_vec = (v8i16) __msa_ilvr_b(vec0, (v16i8) filter_vec);
4488 SPLATI_W2_SW(filter_vec, 0, filt_h0, filt_h1);
4492 const_vec = __msa_ldi_h(128);
4495 for (cnt = width >> 3; cnt--;) {
4496 src0_ptr_tmp = src0_ptr;
4498 src1_ptr_tmp = src1_ptr;
4500 LD_SB3(src0_ptr_tmp, src_stride, src0, src1, src2);
4501 src0_ptr_tmp += (3 * src_stride);
4502 XORI_B3_128_SB(src0, src1, src2);
4504 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4505 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4506 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4508 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst0, dst0);
4510 DPADD_SB2_SH(vec2, vec3, filt0, filt1, dst1, dst1);
4512 DPADD_SB2_SH(vec4, vec5, filt0, filt1, dst2, dst2);
4514 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
4515 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
4517 for (loop_cnt = height >> 2; loop_cnt--;) {
4518 LD_SB4(src0_ptr_tmp, src_stride, src3, src4, src5, src6);
4519 src0_ptr_tmp += (4 * src_stride);
4520 LD_SH4(src1_ptr_tmp, src2_stride, in0, in1, in2, in3);
4521 src1_ptr_tmp += (4 * src2_stride);
4522 XORI_B4_128_SB(src3, src4, src5, src6);
4524 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4526 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst3, dst3);
4528 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
4529 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4530 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
4534 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
4536 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst4, dst4);
4538 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
4539 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4540 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
4544 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
4546 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst5, dst5);
4548 ILVRL_H2_SH(dst5, dst4, dst10_r, dst10_l);
4549 dst2_r = HEVC_FILT_4TAP(dst32_r, dst10_r, filt_h0, filt_h1);
4550 dst2_l = HEVC_FILT_4TAP(dst32_l, dst10_l, filt_h0, filt_h1);
4554 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
4556 DPADD_SB2_SH(vec0, vec1, filt0, filt1, dst2, dst2);
4558 ILVRL_H2_SH(dst2, dst5, dst21_r, dst21_l);
4559 dst3_r = HEVC_FILT_4TAP(dst43_r, dst21_r, filt_h0, filt_h1);
4560 dst3_l = HEVC_FILT_4TAP(dst43_l, dst21_l, filt_h0, filt_h1);
4564 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r,
4565 dst2_l, dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
4566 HEVC_BI_RND_CLIP4(in0, in1, in2, in3,
4567 tmp0, tmp1, tmp2, tmp3, 7,
4568 tmp0, tmp1, tmp2, tmp3);
4570 PCKEV_B2_SW(tmp1, tmp0, tmp3, tmp2, dst0_r, dst1_r);
4571 ST8x4_UB(dst0_r, dst1_r, dst_tmp, dst_stride);
4572 dst_tmp += (4 * dst_stride);
4581 static void hevc_hv_bi_4t_8w_msa(uint8_t *src0_ptr,
4584 int32_t src2_stride,
4587 const int8_t *filter_x,
4588 const int8_t *filter_y,
4592 hevc_hv_bi_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4593 dst, dst_stride, filter_x, filter_y, height);
4594 } else if (6 == height) {
4595 hevc_hv_bi_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4596 dst, dst_stride, filter_x, filter_y, height);
4598 hevc_hv_bi_4t_8multx4mult_msa(src0_ptr, src_stride,
4599 src1_ptr, src2_stride,
4601 filter_x, filter_y, height, 8);
4605 static void hevc_hv_bi_4t_12w_msa(uint8_t *src0_ptr,
4608 int32_t src2_stride,
4611 const int8_t *filter_x,
4612 const int8_t *filter_y,
4615 hevc_hv_bi_4t_8multx4mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4616 dst, dst_stride, filter_x, filter_y,
4618 hevc_hv_bi_4t_4w_msa(src0_ptr + 8, src_stride, src1_ptr + 8, src2_stride,
4619 dst + 8, dst_stride, filter_x, filter_y, height);
4622 static void hevc_hv_bi_4t_16w_msa(uint8_t *src0_ptr,
4625 int32_t src2_stride,
4628 const int8_t *filter_x,
4629 const int8_t *filter_y,
4632 hevc_hv_bi_4t_8multx4mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4633 dst, dst_stride, filter_x, filter_y,
4637 static void hevc_hv_bi_4t_24w_msa(uint8_t *src0_ptr,
4640 int32_t src2_stride,
4643 const int8_t *filter_x,
4644 const int8_t *filter_y,
4647 hevc_hv_bi_4t_8multx4mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4648 dst, dst_stride, filter_x, filter_y,
4652 static void hevc_hv_bi_4t_32w_msa(uint8_t *src0_ptr,
4655 int32_t src2_stride,
4658 const int8_t *filter_x,
4659 const int8_t *filter_y,
4662 hevc_hv_bi_4t_8multx4mult_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4663 dst, dst_stride, filter_x, filter_y,
4667 #define BI_MC_COPY(WIDTH) \
4668 void ff_hevc_put_hevc_bi_pel_pixels##WIDTH##_8_msa(uint8_t *dst, \
4669 ptrdiff_t dst_stride, \
4671 ptrdiff_t src_stride, \
4672 int16_t *src_16bit, \
4678 hevc_bi_copy_##WIDTH##w_msa(src, src_stride, src_16bit, MAX_PB_SIZE, \
4679 dst, dst_stride, height); \
4694 #define BI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
4695 void ff_hevc_put_hevc_bi_##PEL##_##DIR##WIDTH##_8_msa(uint8_t *dst, \
4696 ptrdiff_t dst_stride, \
4698 ptrdiff_t src_stride, \
4699 int16_t *src_16bit, \
4705 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR - 1]; \
4707 hevc_##DIR1##_bi_##TAP##t_##WIDTH##w_msa(src, src_stride, src_16bit, \
4708 MAX_PB_SIZE, dst, dst_stride, \
4712 BI_MC(qpel, h, 4, 8, hz, mx);
4713 BI_MC(qpel, h, 8, 8, hz, mx);
4714 BI_MC(qpel, h, 12, 8, hz, mx);
4715 BI_MC(qpel, h, 16, 8, hz, mx);
4716 BI_MC(qpel, h, 24, 8, hz, mx);
4717 BI_MC(qpel, h, 32, 8, hz, mx);
4718 BI_MC(qpel, h, 48, 8, hz, mx);
4719 BI_MC(qpel, h, 64, 8, hz, mx);
4721 BI_MC(qpel, v, 4, 8, vt, my);
4722 BI_MC(qpel, v, 8, 8, vt, my);
4723 BI_MC(qpel, v, 12, 8, vt, my);
4724 BI_MC(qpel, v, 16, 8, vt, my);
4725 BI_MC(qpel, v, 24, 8, vt, my);
4726 BI_MC(qpel, v, 32, 8, vt, my);
4727 BI_MC(qpel, v, 48, 8, vt, my);
4728 BI_MC(qpel, v, 64, 8, vt, my);
4730 BI_MC(epel, h, 4, 4, hz, mx);
4731 BI_MC(epel, h, 8, 4, hz, mx);
4732 BI_MC(epel, h, 6, 4, hz, mx);
4733 BI_MC(epel, h, 12, 4, hz, mx);
4734 BI_MC(epel, h, 16, 4, hz, mx);
4735 BI_MC(epel, h, 24, 4, hz, mx);
4736 BI_MC(epel, h, 32, 4, hz, mx);
4738 BI_MC(epel, v, 4, 4, vt, my);
4739 BI_MC(epel, v, 8, 4, vt, my);
4740 BI_MC(epel, v, 6, 4, vt, my);
4741 BI_MC(epel, v, 12, 4, vt, my);
4742 BI_MC(epel, v, 16, 4, vt, my);
4743 BI_MC(epel, v, 24, 4, vt, my);
4744 BI_MC(epel, v, 32, 4, vt, my);
4748 #define BI_MC_HV(PEL, WIDTH, TAP) \
4749 void ff_hevc_put_hevc_bi_##PEL##_hv##WIDTH##_8_msa(uint8_t *dst, \
4750 ptrdiff_t dst_stride, \
4752 ptrdiff_t src_stride, \
4753 int16_t *src_16bit, \
4759 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \
4760 const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \
4762 hevc_hv_bi_##TAP##t_##WIDTH##w_msa(src, src_stride, src_16bit, \
4763 MAX_PB_SIZE, dst, dst_stride, \
4764 filter_x, filter_y, height); \
4767 BI_MC_HV(qpel, 4, 8);
4768 BI_MC_HV(qpel, 8, 8);
4769 BI_MC_HV(qpel, 12, 8);
4770 BI_MC_HV(qpel, 16, 8);
4771 BI_MC_HV(qpel, 24, 8);
4772 BI_MC_HV(qpel, 32, 8);
4773 BI_MC_HV(qpel, 48, 8);
4774 BI_MC_HV(qpel, 64, 8);
4776 BI_MC_HV(epel, 4, 4);
4777 BI_MC_HV(epel, 8, 4);
4778 BI_MC_HV(epel, 6, 4);
4779 BI_MC_HV(epel, 12, 4);
4780 BI_MC_HV(epel, 16, 4);
4781 BI_MC_HV(epel, 24, 4);
4782 BI_MC_HV(epel, 32, 4);