2 * Copyright (c) 2015 -2017 Parag Salasakar (Parag.Salasakar@imgtec.com)
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "libavutil/mips/generic_macros_msa.h"
22 #include "h264dsp_mips.h"
24 #define AVC_CALC_DPADD_H_6PIX_2COEFF_SH(in0, in1, in2, in3, in4, in5) \
26 v4i32 tmp0_m, tmp1_m; \
27 v8i16 out0_m, out1_m, out2_m, out3_m; \
28 v8i16 minus5h_m = __msa_ldi_h(-5); \
29 v8i16 plus20h_m = __msa_ldi_h(20); \
31 ILVRL_H2_SW(in5, in0, tmp0_m, tmp1_m); \
33 tmp0_m = __msa_hadd_s_w((v8i16) tmp0_m, (v8i16) tmp0_m); \
34 tmp1_m = __msa_hadd_s_w((v8i16) tmp1_m, (v8i16) tmp1_m); \
36 ILVRL_H2_SH(in1, in4, out0_m, out1_m); \
37 DPADD_SH2_SW(out0_m, out1_m, minus5h_m, minus5h_m, tmp0_m, tmp1_m); \
38 ILVRL_H2_SH(in2, in3, out2_m, out3_m); \
39 DPADD_SH2_SW(out2_m, out3_m, plus20h_m, plus20h_m, tmp0_m, tmp1_m); \
41 SRARI_W2_SW(tmp0_m, tmp1_m, 10); \
42 SAT_SW2_SW(tmp0_m, tmp1_m, 7); \
43 out0_m = __msa_pckev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \
48 static const uint8_t luma_mask_arr[16 * 8] = {
50 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7, 12,
51 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 9, 7, 10, 8, 11,
52 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
55 0, 5, 1, 6, 2, 7, 3, 8, 16, 21, 17, 22, 18, 23, 19, 24,
56 1, 4, 2, 5, 3, 6, 4, 7, 17, 20, 18, 21, 19, 22, 20, 23,
57 2, 3, 3, 4, 4, 5, 5, 6, 18, 19, 19, 20, 20, 21, 21, 22,
59 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 25,
60 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26
63 #define AVC_CALC_DPADD_B_6PIX_2COEFF_SH(vec0, vec1, vec2, vec3, vec4, vec5, \
66 v16i8 tmp0_m, tmp1_m; \
67 v16i8 minus5b_m = __msa_ldi_b(-5); \
68 v16i8 plus20b_m = __msa_ldi_b(20); \
70 ILVRL_B2_SB(vec5, vec0, tmp0_m, tmp1_m); \
71 HADD_SB2_SH(tmp0_m, tmp1_m, out1, out2); \
72 ILVRL_B2_SB(vec4, vec1, tmp0_m, tmp1_m); \
73 DPADD_SB2_SH(tmp0_m, tmp1_m, minus5b_m, minus5b_m, out1, out2); \
74 ILVRL_B2_SB(vec3, vec2, tmp0_m, tmp1_m); \
75 DPADD_SB2_SH(tmp0_m, tmp1_m, plus20b_m, plus20b_m, out1, out2); \
78 #define AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(vec0, vec1, vec2, vec3, vec4, vec5) \
81 v8i16 tmp2_m, tmp3_m; \
82 v8i16 minus5h_m = __msa_ldi_h(-5); \
83 v8i16 plus20h_m = __msa_ldi_h(20); \
85 tmp1_m = (v4i32) __msa_ilvr_h((v8i16) vec5, (v8i16) vec0); \
86 tmp1_m = __msa_hadd_s_w((v8i16) tmp1_m, (v8i16) tmp1_m); \
88 ILVR_H2_SH(vec1, vec4, vec2, vec3, tmp2_m, tmp3_m); \
89 DPADD_SH2_SW(tmp2_m, tmp3_m, minus5h_m, plus20h_m, tmp1_m, tmp1_m); \
91 tmp1_m = __msa_srari_w(tmp1_m, 10); \
92 tmp1_m = __msa_sat_s_w(tmp1_m, 7); \
94 tmp2_m = __msa_pckev_h((v8i16) tmp1_m, (v8i16) tmp1_m); \
99 #define AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1, \
100 mask0, mask1, mask2) \
103 v16i8 vec0_m, vec1_m, vec2_m; \
104 v16i8 minus5b_m = __msa_ldi_b(-5); \
105 v16i8 plus20b_m = __msa_ldi_b(20); \
107 vec0_m = __msa_vshf_b((v16i8) mask0, (v16i8) src1, (v16i8) src0); \
108 hz_out_m = __msa_hadd_s_h(vec0_m, vec0_m); \
110 VSHF_B2_SB(src0, src1, src0, src1, mask1, mask2, vec1_m, vec2_m); \
111 DPADD_SB2_SH(vec1_m, vec2_m, minus5b_m, plus20b_m, hz_out_m, hz_out_m); \
116 #define AVC_HORZ_FILTER_SH(in0, in1, mask0, mask1, mask2) \
120 v16i8 minus5b = __msa_ldi_b(-5); \
121 v16i8 plus20b = __msa_ldi_b(20); \
123 tmp0_m = __msa_vshf_b((v16i8) mask0, in1, in0); \
124 out0_m = __msa_hadd_s_h(tmp0_m, tmp0_m); \
126 tmp0_m = __msa_vshf_b((v16i8) mask1, in1, in0); \
127 out0_m = __msa_dpadd_s_h(out0_m, minus5b, tmp0_m); \
129 tmp0_m = __msa_vshf_b((v16i8) mask2, in1, in0); \
130 out0_m = __msa_dpadd_s_h(out0_m, plus20b, tmp0_m); \
135 #define AVC_DOT_SH3_SH(in0, in1, in2, coeff0, coeff1, coeff2) \
139 out0_m = __msa_dotp_s_h((v16i8) in0, (v16i8) coeff0); \
140 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in1, (v16i8) coeff1); \
141 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in2, (v16i8) coeff2); \
146 #define AVC_DOT_SW3_SW(in0, in1, in2, coeff0, coeff1, coeff2) \
150 out0_m = __msa_dotp_s_w((v8i16) in0, (v8i16) coeff0); \
151 out0_m = __msa_dpadd_s_w(out0_m, (v8i16) in1, (v8i16) coeff1); \
152 out0_m = __msa_dpadd_s_w(out0_m, (v8i16) in2, (v8i16) coeff2); \
153 out0_m = __msa_srari_w(out0_m, 10); \
154 out0_m = __msa_sat_s_w(out0_m, 7); \
158 static void avc_luma_hv_qrt_4x4_msa(const uint8_t *src_x, const uint8_t *src_y,
159 uint8_t *dst, int32_t stride)
161 const int16_t filt_const0 = 0xfb01;
162 const int16_t filt_const1 = 0x1414;
163 const int16_t filt_const2 = 0x1fb;
165 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, src_vt7, src_vt8;
166 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
167 v16i8 src_vt10_r, src_vt32_r, src_vt54_r, src_vt76_r;
168 v16i8 mask0, mask1, mask2, filt0, filt1, filt2;
169 v8i16 hz_out0, hz_out1, vt_out0, vt_out1, out0, out1;
171 filt0 = (v16i8) __msa_fill_h(filt_const0);
172 filt1 = (v16i8) __msa_fill_h(filt_const1);
173 filt2 = (v16i8) __msa_fill_h(filt_const2);
175 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
177 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
178 src_y += (5 * stride);
180 src_vt0 = (v16i8) __msa_insve_w((v4i32) src_vt0, 1, (v4i32) src_vt1);
181 src_vt1 = (v16i8) __msa_insve_w((v4i32) src_vt1, 1, (v4i32) src_vt2);
182 src_vt2 = (v16i8) __msa_insve_w((v4i32) src_vt2, 1, (v4i32) src_vt3);
183 src_vt3 = (v16i8) __msa_insve_w((v4i32) src_vt3, 1, (v4i32) src_vt4);
185 XORI_B4_128_SB(src_vt0, src_vt1, src_vt2, src_vt3);
187 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
188 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
189 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz1, mask0, mask1, mask2);
190 hz_out1 = AVC_HORZ_FILTER_SH(src_hz2, src_hz3, mask0, mask1, mask2);
192 SRARI_H2_SH(hz_out0, hz_out1, 5);
193 SAT_SH2_SH(hz_out0, hz_out1, 7);
195 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
197 src_vt4 = (v16i8) __msa_insve_w((v4i32) src_vt4, 1, (v4i32) src_vt5);
198 src_vt5 = (v16i8) __msa_insve_w((v4i32) src_vt5, 1, (v4i32) src_vt6);
199 src_vt6 = (v16i8) __msa_insve_w((v4i32) src_vt6, 1, (v4i32) src_vt7);
200 src_vt7 = (v16i8) __msa_insve_w((v4i32) src_vt7, 1, (v4i32) src_vt8);
202 XORI_B4_128_SB(src_vt4, src_vt5, src_vt6, src_vt7);
203 ILVR_B2_SB(src_vt1, src_vt0, src_vt3, src_vt2, src_vt10_r, src_vt32_r);
204 ILVR_B2_SB(src_vt5, src_vt4, src_vt7, src_vt6, src_vt54_r, src_vt76_r);
205 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
207 vt_out1 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
209 SRARI_H2_SH(vt_out0, vt_out1, 5);
210 SAT_SH2_SH(vt_out0, vt_out1, 7);
212 out0 = __msa_srari_h((hz_out0 + vt_out0), 1);
213 out1 = __msa_srari_h((hz_out1 + vt_out1), 1);
215 SAT_SH2_SH(out0, out1, 7);
216 out = PCKEV_XORI128_UB(out0, out1);
217 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
220 static void avc_luma_hv_qrt_8x8_msa(const uint8_t *src_x, const uint8_t *src_y,
221 uint8_t *dst, int32_t stride)
223 const int16_t filt_const0 = 0xfb01;
224 const int16_t filt_const1 = 0x1414;
225 const int16_t filt_const2 = 0x1fb;
227 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2;
228 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
229 v16i8 src_vt7, src_vt8, src_vt9, src_vt10, src_vt11, src_vt12;
230 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
231 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, src_vt98_r, src_vt109_r;
232 v16i8 src_vt1110_r, src_vt1211_r, filt0, filt1, filt2;
233 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
234 v8i16 vt_out3, tmp0, tmp1, tmp2, tmp3;
236 filt0 = (v16i8) __msa_fill_h(filt_const0);
237 filt1 = (v16i8) __msa_fill_h(filt_const1);
238 filt2 = (v16i8) __msa_fill_h(filt_const2);
240 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
241 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
242 src_y += (5 * stride);
244 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
246 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
247 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
248 src_x += (4 * stride);
250 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
251 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
252 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
253 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
255 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
256 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
258 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
259 src_y += (4 * stride);
260 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
262 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2, src_vt4,
263 src_vt3, src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r);
264 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6, src_vt8,
265 src_vt7, src_vt54_r, src_vt65_r, src_vt76_r, src_vt87_r);
266 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
268 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0, filt1,
270 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
272 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0, filt1,
274 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
275 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
277 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
278 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
279 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
280 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
282 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
283 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
285 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
286 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
287 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
288 ST8x4_UB(out0, out1, dst, stride);
291 LD_SB4(src_y, stride, src_vt9, src_vt10, src_vt11, src_vt12);
292 XORI_B4_128_SB(src_vt9, src_vt10, src_vt11, src_vt12);
294 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
295 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
296 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
297 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
299 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
300 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
302 ILVR_B4_SB(src_vt9, src_vt8, src_vt10, src_vt9, src_vt11, src_vt10,
303 src_vt12, src_vt11, src_vt98_r, src_vt109_r, src_vt1110_r,
305 vt_out0 = AVC_DOT_SH3_SH(src_vt54_r, src_vt76_r, src_vt98_r, filt0, filt1,
307 vt_out1 = AVC_DOT_SH3_SH(src_vt65_r, src_vt87_r, src_vt109_r, filt0, filt1,
309 vt_out2 = AVC_DOT_SH3_SH(src_vt76_r, src_vt98_r, src_vt1110_r, filt0, filt1,
311 vt_out3 = AVC_DOT_SH3_SH(src_vt87_r, src_vt109_r, src_vt1211_r, filt0,
313 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
314 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
316 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
317 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
318 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
319 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
321 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
322 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
323 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
324 ST8x4_UB(out0, out1, dst, stride);
328 static void avc_luma_hv_qrt_16x16_msa(const uint8_t *src_x,
329 const uint8_t *src_y, uint8_t *dst,
332 const int16_t filt_const0 = 0xfb01;
333 const int16_t filt_const1 = 0x1414;
334 const int16_t filt_const2 = 0x1fb;
335 const uint8_t *src_x_tmp = src_x;
336 const uint8_t *src_y_tmp = src_y;
337 uint8_t *dst_tmp = dst;
338 uint32_t multiple8_cnt, loop_cnt;
340 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2;
341 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
342 v16i8 src_vt7, src_vt8;
343 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
344 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, filt0, filt1, filt2;
345 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
346 v8i16 vt_out3, out0, out1, out2, out3;
348 filt0 = (v16i8) __msa_fill_h(filt_const0);
349 filt1 = (v16i8) __msa_fill_h(filt_const1);
350 filt2 = (v16i8) __msa_fill_h(filt_const2);
352 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
354 for (multiple8_cnt = 2; multiple8_cnt--;) {
359 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
360 src_y += (5 * stride);
362 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
364 for (loop_cnt = 4; loop_cnt--;) {
365 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
366 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
367 src_x += (4 * stride);
369 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
370 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
371 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
372 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
373 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
374 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
376 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
377 src_y += (4 * stride);
379 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
380 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2,
381 src_vt4, src_vt3, src_vt10_r, src_vt21_r, src_vt32_r,
383 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6,
384 src_vt8, src_vt7, src_vt54_r, src_vt65_r, src_vt76_r,
386 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0,
388 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0,
390 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0,
392 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0,
394 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
395 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
397 out0 = __msa_srari_h((hz_out0 + vt_out0), 1);
398 out1 = __msa_srari_h((hz_out1 + vt_out1), 1);
399 out2 = __msa_srari_h((hz_out2 + vt_out2), 1);
400 out3 = __msa_srari_h((hz_out3 + vt_out3), 1);
402 SAT_SH4_SH(out0, out1, out2, out3, 7);
403 tmp0 = PCKEV_XORI128_UB(out0, out1);
404 tmp1 = PCKEV_XORI128_UB(out2, out3);
405 ST8x4_UB(tmp0, tmp1, dst, stride);
421 static void avc_luma_hz_and_aver_dst_4x4_msa(const uint8_t *src,
423 uint8_t *dst, int32_t dst_stride)
425 v16i8 src0, src1, src2, src3;
426 v16u8 dst0, dst1, dst2, dst3, res;
428 v16i8 mask0, mask1, mask2;
429 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
430 v16i8 minus5b = __msa_ldi_b(-5);
431 v16i8 plus20b = __msa_ldi_b(20);
433 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
434 LD_SB4(src, src_stride, src0, src1, src2, src3);
436 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
437 XORI_B4_128_SB(src0, src1, src2, src3);
438 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
439 HADD_SB2_SH(vec0, vec1, res0, res1);
440 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
441 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
442 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
443 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
444 SRARI_H2_SH(res0, res1, 5);
445 SAT_SH2_SH(res0, res1, 7);
446 res = PCKEV_XORI128_UB(res0, res1);
447 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
449 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
450 res = __msa_aver_u_b(res, dst0);
452 ST4x4_UB(res, res, 0, 1, 2, 3, dst, dst_stride);
455 static void avc_luma_hz_and_aver_dst_8x8_msa(const uint8_t *src,
457 uint8_t *dst, int32_t dst_stride)
460 v16i8 src0, src1, src2, src3;
461 v16u8 dst0, dst1, dst2, dst3;
462 v8i16 res0, res1, res2, res3;
463 v16i8 mask0, mask1, mask2;
464 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
465 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
466 v16i8 minus5b = __msa_ldi_b(-5);
467 v16i8 plus20b = __msa_ldi_b(20);
469 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
471 for (loop_cnt = 2; loop_cnt--;) {
472 LD_SB4(src, src_stride, src0, src1, src2, src3);
473 src += (4 * src_stride);
475 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
477 XORI_B4_128_SB(src0, src1, src2, src3);
478 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
479 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
480 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
481 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
482 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
483 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
484 res0, res1, res2, res3);
485 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
486 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
487 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
488 plus20b, res0, res1, res2, res3);
489 SRARI_H4_SH(res0, res1, res2, res3, 5);
490 SAT_SH4_SH(res0, res1, res2, res3, 7);
491 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
492 CONVERT_UB_AVG_ST8x4_UB(res0, res1, res2, res3, dst0, dst1,
495 dst += (4 * dst_stride);
499 static void avc_luma_hz_and_aver_dst_16x16_msa(const uint8_t *src,
501 uint8_t *dst, int32_t dst_stride)
504 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
505 v16u8 dst0, dst1, dst2, dst3;
506 v16i8 mask0, mask1, mask2;
507 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
508 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
509 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
510 v16i8 minus5b = __msa_ldi_b(-5);
511 v16i8 plus20b = __msa_ldi_b(20);
513 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
515 for (loop_cnt = 4; loop_cnt--;) {
516 LD_SB2(src, 8, src0, src1);
518 LD_SB2(src, 8, src2, src3);
521 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
523 XORI_B4_128_SB(src0, src1, src2, src3);
524 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
525 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
526 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
527 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
528 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
529 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
530 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
531 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
532 minus5b, res0, res1, res2, res3);
533 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
534 plus20b, res0, res1, res2, res3);
535 LD_SB2(src, 8, src4, src5);
537 LD_SB2(src, 8, src6, src7);
539 XORI_B4_128_SB(src4, src5, src6, src7);
540 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec3);
541 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec6, vec9);
542 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec1, vec4);
543 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec7, vec10);
544 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec2, vec5);
545 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec8, vec11);
546 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
547 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
548 minus5b, res4, res5, res6, res7);
549 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
550 plus20b, res4, res5, res6, res7);
551 SRARI_H4_SH(res0, res1, res2, res3, 5);
552 SRARI_H4_SH(res4, res5, res6, res7, 5);
553 SAT_SH4_SH(res0, res1, res2, res3, 7);
554 SAT_SH4_SH(res4, res5, res6, res7, 7);
555 PCKEV_B4_SB(res1, res0, res3, res2, res5, res4, res7, res6,
556 vec0, vec1, vec2, vec3);
557 XORI_B4_128_SB(vec0, vec1, vec2, vec3);
558 AVER_UB4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3,
559 dst0, dst1, dst2, dst3);
560 ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
561 dst += (4 * dst_stride);
565 static void avc_luma_hz_qrt_and_aver_dst_4x4_msa(const uint8_t *src,
572 v16i8 src0, src1, src2, src3;
573 v16u8 dst0, dst1, dst2, dst3;
574 v16i8 mask0, mask1, mask2;
575 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
577 v16i8 minus5b = __msa_ldi_b(-5);
578 v16i8 plus20b = __msa_ldi_b(20);
581 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
589 LD_SB4(src, src_stride, src0, src1, src2, src3);
590 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
592 XORI_B4_128_SB(src0, src1, src2, src3);
593 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
594 HADD_SB2_SH(vec0, vec1, out0, out1);
595 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
596 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, out0, out1);
597 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
598 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, out0, out1);
599 SRARI_H2_SH(out0, out1, 5);
600 SAT_SH2_SH(out0, out1, 7);
602 PCKEV_B2_UB(out0, out0, out1, out1, res0, res1);
604 src0 = __msa_sld_b(src0, src0, slide);
605 src1 = __msa_sld_b(src1, src1, slide);
606 src2 = __msa_sld_b(src2, src2, slide);
607 src3 = __msa_sld_b(src3, src3, slide);
608 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
609 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
610 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src0);
611 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src1);
613 XORI_B2_128_UB(res0, res1);
615 dst0 = (v16u8) __msa_insve_w((v4i32) dst0, 1, (v4i32) dst1);
616 dst1 = (v16u8) __msa_insve_w((v4i32) dst2, 1, (v4i32) dst3);
618 AVER_UB2_UB(res0, dst0, res1, dst1, dst0, dst1);
620 ST4x4_UB(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
623 static void avc_luma_hz_qrt_and_aver_dst_8x8_msa(const uint8_t *src,
631 v16i8 src0, src1, src2, src3;
632 v16i8 mask0, mask1, mask2;
633 v16u8 dst0, dst1, dst2, dst3;
634 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
635 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
636 v8i16 out0, out1, out2, out3;
637 v16i8 minus5b = __msa_ldi_b(-5);
638 v16i8 plus20b = __msa_ldi_b(20);
639 v16i8 res0, res1, res2, res3;
641 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
649 for (loop_cnt = 2; loop_cnt--;) {
650 LD_SB4(src, src_stride, src0, src1, src2, src3);
651 src += (4 * src_stride);
653 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
655 XORI_B4_128_SB(src0, src1, src2, src3);
656 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
657 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
658 HADD_SB4_SH(vec0, vec1, vec2, vec3, out0, out1, out2, out3);
659 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
660 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
661 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
662 out0, out1, out2, out3);
663 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
664 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
665 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
666 plus20b, out0, out1, out2, out3);
668 src0 = __msa_sld_b(src0, src0, slide);
669 src1 = __msa_sld_b(src1, src1, slide);
670 src2 = __msa_sld_b(src2, src2, slide);
671 src3 = __msa_sld_b(src3, src3, slide);
673 SRARI_H4_SH(out0, out1, out2, out3, 5);
674 SAT_SH4_SH(out0, out1, out2, out3, 7);
676 PCKEV_B4_SB(out0, out0, out1, out1, out2, out2, out3, out3,
677 res0, res1, res2, res3);
679 res0 = __msa_aver_s_b(res0, src0);
680 res1 = __msa_aver_s_b(res1, src1);
681 res2 = __msa_aver_s_b(res2, src2);
682 res3 = __msa_aver_s_b(res3, src3);
684 XORI_B4_128_SB(res0, res1, res2, res3);
685 AVER_ST8x4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
688 dst += (4 * dst_stride);
692 static void avc_luma_hz_qrt_and_aver_dst_16x16_msa(const uint8_t *src,
700 v16i8 src0, src1, src2, src3;
701 v16i8 mask0, mask1, mask2, vshf;
703 v8i16 res0, res1, res2, res3;
704 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
705 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
706 v16i8 minus5b = __msa_ldi_b(-5);
707 v16i8 plus20b = __msa_ldi_b(20);
709 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
712 vshf = LD_SB(&luma_mask_arr[16 + 96]);
714 vshf = LD_SB(&luma_mask_arr[96]);
717 for (loop_cnt = 8; loop_cnt--;) {
718 LD_SB2(src, 8, src0, src1);
720 LD_SB2(src, 8, src2, src3);
723 LD_UB2(dst, dst_stride, dst0, dst1);
725 XORI_B4_128_SB(src0, src1, src2, src3);
726 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
727 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
728 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
729 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
730 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
731 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
732 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
733 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
734 minus5b, res0, res1, res2, res3);
735 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
736 plus20b, res0, res1, res2, res3);
737 VSHF_B2_SB(src0, src1, src2, src3, vshf, vshf, src0, src2);
738 SRARI_H4_SH(res0, res1, res2, res3, 5);
739 SAT_SH4_SH(res0, res1, res2, res3, 7);
740 PCKEV_B2_SB(res1, res0, res3, res2, out0, out1);
742 out0 = __msa_aver_s_b(out0, src0);
743 out1 = __msa_aver_s_b(out1, src2);
745 XORI_B2_128_SB(out0, out1);
746 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
747 ST_UB2(dst0, dst1, dst, dst_stride);
748 dst += (2 * dst_stride);
752 static void avc_luma_vt_and_aver_dst_4x4_msa(const uint8_t *src,
754 uint8_t *dst, int32_t dst_stride)
756 int16_t filt_const0 = 0xfb01;
757 int16_t filt_const1 = 0x1414;
758 int16_t filt_const2 = 0x1fb;
759 v16u8 dst0, dst1, dst2, dst3;
760 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
761 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
762 v16i8 src87_r, src2110, src4332, src6554, src8776;
764 v16i8 filt0, filt1, filt2;
767 filt0 = (v16i8) __msa_fill_h(filt_const0);
768 filt1 = (v16i8) __msa_fill_h(filt_const1);
769 filt2 = (v16i8) __msa_fill_h(filt_const2);
771 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
772 src += (5 * src_stride);
774 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
775 src10_r, src21_r, src32_r, src43_r);
776 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
777 XORI_B2_128_SB(src2110, src4332);
778 LD_SB4(src, src_stride, src5, src6, src7, src8);
779 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
780 src54_r, src65_r, src76_r, src87_r);
781 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
782 XORI_B2_128_SB(src6554, src8776);
783 out10 = DPADD_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
784 out32 = DPADD_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
785 SRARI_H2_SH(out10, out32, 5);
786 SAT_SH2_SH(out10, out32, 7);
787 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
788 res = PCKEV_XORI128_UB(out10, out32);
790 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
792 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
793 dst0 = __msa_aver_u_b(res, dst0);
795 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
798 static void avc_luma_vt_and_aver_dst_8x8_msa(const uint8_t *src,
800 uint8_t *dst, int32_t dst_stride)
803 int16_t filt_const0 = 0xfb01;
804 int16_t filt_const1 = 0x1414;
805 int16_t filt_const2 = 0x1fb;
806 v16u8 dst0, dst1, dst2, dst3;
807 v16i8 src0, src1, src2, src3, src4, src7, src8, src9, src10;
808 v16i8 src10_r, src32_r, src76_r, src98_r;
809 v16i8 src21_r, src43_r, src87_r, src109_r;
810 v8i16 out0, out1, out2, out3;
811 v16i8 filt0, filt1, filt2;
813 filt0 = (v16i8) __msa_fill_h(filt_const0);
814 filt1 = (v16i8) __msa_fill_h(filt_const1);
815 filt2 = (v16i8) __msa_fill_h(filt_const2);
817 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
818 src += (5 * src_stride);
820 XORI_B5_128_SB(src0, src1, src2, src3, src4);
821 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
822 src10_r, src21_r, src32_r, src43_r);
824 for (loop_cnt = 2; loop_cnt--;) {
825 LD_SB4(src, src_stride, src7, src8, src9, src10);
826 src += (4 * src_stride);
828 XORI_B4_128_SB(src7, src8, src9, src10);
829 ILVR_B4_SB(src7, src4, src8, src7, src9, src8, src10, src9,
830 src76_r, src87_r, src98_r, src109_r);
831 out0 = DPADD_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
832 out1 = DPADD_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
833 out2 = DPADD_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
834 out3 = DPADD_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
835 SRARI_H4_SH(out0, out1, out2, out3, 5);
836 SAT_SH4_SH(out0, out1, out2, out3, 7);
837 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
838 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
839 CONVERT_UB_AVG_ST8x4_UB(out0, out1, out2, out3, dst0, dst1,
841 dst += (4 * dst_stride);
851 static void avc_luma_vt_and_aver_dst_16x16_msa(const uint8_t *src,
853 uint8_t *dst, int32_t dst_stride)
856 int16_t filt_const0 = 0xfb01;
857 int16_t filt_const1 = 0x1414;
858 int16_t filt_const2 = 0x1fb;
859 v16u8 dst0, dst1, dst2, dst3;
860 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
861 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
862 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
863 v16i8 src65_l, src87_l;
864 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
865 v16i8 filt0, filt1, filt2;
866 v16u8 res0, res1, res2, res3;
868 filt0 = (v16i8) __msa_fill_h(filt_const0);
869 filt1 = (v16i8) __msa_fill_h(filt_const1);
870 filt2 = (v16i8) __msa_fill_h(filt_const2);
872 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
873 src += (5 * src_stride);
875 XORI_B5_128_SB(src0, src1, src2, src3, src4);
876 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
877 src10_r, src21_r, src32_r, src43_r);
878 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
879 src10_l, src21_l, src32_l, src43_l);
881 for (loop_cnt = 4; loop_cnt--;) {
882 LD_SB4(src, src_stride, src5, src6, src7, src8);
883 src += (4 * src_stride);
885 XORI_B4_128_SB(src5, src6, src7, src8);
886 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
887 src54_r, src65_r, src76_r, src87_r);
888 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
889 src54_l, src65_l, src76_l, src87_l);
890 out0_r = DPADD_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
891 out1_r = DPADD_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
892 out2_r = DPADD_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
893 out3_r = DPADD_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
894 out0_l = DPADD_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
895 out1_l = DPADD_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
896 out2_l = DPADD_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
897 out3_l = DPADD_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
898 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
899 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
900 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
901 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
902 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
903 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
904 out3_r, res0, res1, res2, res3);
905 XORI_B4_128_UB(res0, res1, res2, res3);
906 AVER_UB4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
907 res0, res1, res2, res3);
908 ST_UB4(res0, res1, res2, res3, dst, dst_stride);
909 dst += (4 * dst_stride);
923 static void avc_luma_vt_qrt_and_aver_dst_4x4_msa(const uint8_t *src,
929 int16_t filt_const0 = 0xfb01;
930 int16_t filt_const1 = 0x1414;
931 int16_t filt_const2 = 0x1fb;
932 v16u8 dst0, dst1, dst2, dst3;
933 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
934 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
935 v16i8 src87_r, src2110, src4332, src6554, src8776;
937 v16i8 filt0, filt1, filt2;
940 filt0 = (v16i8) __msa_fill_h(filt_const0);
941 filt1 = (v16i8) __msa_fill_h(filt_const1);
942 filt2 = (v16i8) __msa_fill_h(filt_const2);
944 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
945 src += (5 * src_stride);
947 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
948 src10_r, src21_r, src32_r, src43_r);
949 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
950 XORI_B2_128_SB(src2110, src4332);
951 LD_SB4(src, src_stride, src5, src6, src7, src8);
952 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
953 src54_r, src65_r, src76_r, src87_r);
954 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
955 XORI_B2_128_SB(src6554, src8776);
956 out10 = DPADD_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
957 out32 = DPADD_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
958 SRARI_H2_SH(out10, out32, 5);
959 SAT_SH2_SH(out10, out32, 7);
960 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
961 res = PCKEV_XORI128_UB(out10, out32);
964 src32_r = (v16i8) __msa_insve_w((v4i32) src3, 1, (v4i32) src4);
965 src54_r = (v16i8) __msa_insve_w((v4i32) src5, 1, (v4i32) src6);
967 src32_r = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
968 src54_r = (v16i8) __msa_insve_w((v4i32) src4, 1, (v4i32) src5);
971 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
972 res = __msa_aver_u_b(res, (v16u8) src32_r);
974 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
976 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
977 dst0 = __msa_aver_u_b(res, dst0);
979 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
982 static void avc_luma_vt_qrt_and_aver_dst_8x8_msa(const uint8_t *src,
989 int16_t filt_const0 = 0xfb01;
990 int16_t filt_const1 = 0x1414;
991 int16_t filt_const2 = 0x1fb;
992 v16u8 dst0, dst1, dst2, dst3;
993 v16i8 src0, src1, src2, src3, src4, src7, src8, src9, src10;
994 v16i8 src10_r, src32_r, src76_r, src98_r;
995 v16i8 src21_r, src43_r, src87_r, src109_r;
996 v8i16 out0_r, out1_r, out2_r, out3_r;
999 v16i8 filt0, filt1, filt2;
1001 filt0 = (v16i8) __msa_fill_h(filt_const0);
1002 filt1 = (v16i8) __msa_fill_h(filt_const1);
1003 filt2 = (v16i8) __msa_fill_h(filt_const2);
1005 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1006 src += (5 * src_stride);
1008 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1009 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1010 src10_r, src21_r, src32_r, src43_r);
1012 for (loop_cnt = 2; loop_cnt--;) {
1013 LD_SB4(src, src_stride, src7, src8, src9, src10);
1014 src += (4 * src_stride);
1016 XORI_B4_128_SB(src7, src8, src9, src10);
1017 ILVR_B4_SB(src7, src4, src8, src7, src9, src8, src10, src9,
1018 src76_r, src87_r, src98_r, src109_r);
1019 out0_r = DPADD_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
1020 out1_r = DPADD_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
1021 out2_r = DPADD_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
1022 out3_r = DPADD_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
1023 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
1024 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1025 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, res0, res1);
1028 PCKEV_D2_SB(src4, src3, src8, src7, src10_r, src32_r);
1030 PCKEV_D2_SB(src3, src2, src7, src4, src10_r, src32_r);
1033 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1034 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1036 vec0 = (v16u8) __msa_aver_s_b(res0, src10_r);
1037 vec1 = (v16u8) __msa_aver_s_b(res1, src32_r);
1039 XORI_B2_128_UB(vec0, vec1);
1040 AVER_UB2_UB(vec0, dst0, vec1, dst1, vec0, vec1);
1041 ST8x4_UB(vec0, vec1, dst, dst_stride);
1042 dst += (4 * dst_stride);
1054 static void avc_luma_vt_qrt_and_aver_dst_16x16_msa(const uint8_t *src,
1061 int16_t filt_const0 = 0xfb01;
1062 int16_t filt_const1 = 0x1414;
1063 int16_t filt_const2 = 0x1fb;
1064 v16u8 dst0, dst1, dst2, dst3;
1065 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1066 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
1067 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
1068 v16i8 src65_l, src87_l;
1069 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
1070 v16i8 out0, out1, out2, out3;
1071 v16i8 filt0, filt1, filt2;
1072 v16u8 res0, res1, res2, res3;
1074 filt0 = (v16i8) __msa_fill_h(filt_const0);
1075 filt1 = (v16i8) __msa_fill_h(filt_const1);
1076 filt2 = (v16i8) __msa_fill_h(filt_const2);
1078 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1079 src += (5 * src_stride);
1081 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1082 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1083 src10_r, src21_r, src32_r, src43_r);
1084 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1085 src10_l, src21_l, src32_l, src43_l);
1087 for (loop_cnt = 4; loop_cnt--;) {
1088 LD_SB4(src, src_stride, src5, src6, src7, src8);
1089 src += (4 * src_stride);
1091 XORI_B4_128_SB(src5, src6, src7, src8);
1092 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
1093 src54_r, src65_r, src76_r, src87_r);
1094 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
1095 src54_l, src65_l, src76_l, src87_l);
1096 out0_r = DPADD_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
1097 out1_r = DPADD_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
1098 out2_r = DPADD_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
1099 out3_r = DPADD_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
1100 out0_l = DPADD_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
1101 out1_l = DPADD_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
1102 out2_l = DPADD_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
1103 out3_l = DPADD_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
1104 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
1105 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
1106 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1107 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
1108 PCKEV_B4_SB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
1109 out3_r, out0, out1, out2, out3);
1110 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1113 res0 = (v16u8) __msa_aver_s_b(out0, src3);
1114 res1 = (v16u8) __msa_aver_s_b(out1, src4);
1115 res2 = (v16u8) __msa_aver_s_b(out2, src5);
1116 res3 = (v16u8) __msa_aver_s_b(out3, src6);
1118 res0 = (v16u8) __msa_aver_s_b(out0, src2);
1119 res1 = (v16u8) __msa_aver_s_b(out1, src3);
1120 res2 = (v16u8) __msa_aver_s_b(out2, src4);
1121 res3 = (v16u8) __msa_aver_s_b(out3, src5);
1124 XORI_B4_128_UB(res0, res1, res2, res3);
1125 AVER_UB4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
1126 dst0, dst1, dst2, dst3);
1127 ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
1128 dst += (4 * dst_stride);
1144 static void avc_luma_midh_qrt_and_aver_dst_4w_msa(const uint8_t *src,
1149 uint8_t horiz_offset)
1152 v16i8 src0, src1, src2, src3, src4, src5, src6;
1153 v16u8 dst0, dst1, res;
1154 v8i16 vt_res0, vt_res1, vt_res2, vt_res3;
1155 v4i32 hz_res0, hz_res1;
1157 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5;
1158 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
1159 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
1160 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
1161 v8i16 minus5h = __msa_ldi_h(-5);
1162 v8i16 plus20h = __msa_ldi_h(20);
1163 v8i16 zeros = { 0 };
1165 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1166 src += (5 * src_stride);
1168 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1170 for (row = (height >> 1); row--;) {
1171 LD_SB2(src, src_stride, src5, src6);
1172 src += (2 * src_stride);
1174 XORI_B2_128_SB(src5, src6);
1175 LD_UB2(dst, dst_stride, dst0, dst1);
1177 dst0 = (v16u8) __msa_ilvr_w((v4i32) dst1, (v4i32) dst0);
1179 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
1181 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
1183 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1,
1184 mask0, mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
1185 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3,
1186 mask0, mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
1188 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
1189 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
1191 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
1192 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
1194 SRARI_W2_SW(hz_res0, hz_res1, 10);
1195 SAT_SW2_SW(hz_res0, hz_res1, 7);
1197 res0 = __msa_srari_h(shf_vec2, 5);
1198 res1 = __msa_srari_h(shf_vec5, 5);
1200 SAT_SH2_SH(res0, res1, 7);
1203 res0 = __msa_ilvod_h(zeros, res0);
1204 res1 = __msa_ilvod_h(zeros, res1);
1206 ILVEV_H2_SH(res0, zeros, res1, zeros, res0, res1);
1208 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) res0);
1209 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) res1);
1210 res0 = __msa_pckev_h((v8i16) hz_res1, (v8i16) hz_res0);
1212 res = PCKEV_XORI128_UB(res0, res0);
1214 dst0 = __msa_aver_u_b(res, dst0);
1216 ST4x2_UB(dst0, dst, dst_stride);
1217 dst += (2 * dst_stride);
1227 static void avc_luma_midh_qrt_and_aver_dst_8w_msa(const uint8_t *src,
1232 uint8_t horiz_offset)
1234 uint32_t multiple8_cnt;
1236 for (multiple8_cnt = 2; multiple8_cnt--;) {
1237 avc_luma_midh_qrt_and_aver_dst_4w_msa(src, src_stride, dst, dst_stride,
1238 height, horiz_offset);
1245 static void avc_luma_midh_qrt_and_aver_dst_16w_msa(const uint8_t *src,
1250 uint8_t horiz_offset)
1252 uint32_t multiple8_cnt;
1254 for (multiple8_cnt = 4; multiple8_cnt--;) {
1255 avc_luma_midh_qrt_and_aver_dst_4w_msa(src, src_stride, dst, dst_stride,
1256 height, horiz_offset);
1263 static void avc_luma_midv_qrt_and_aver_dst_4w_msa(const uint8_t *src,
1272 v16i8 src0, src1, src2, src3, src4;
1274 v16i8 mask0, mask1, mask2;
1275 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1276 v8i16 hz_out4, hz_out5, hz_out6;
1277 v8i16 res0, res1, res2, res3;
1280 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
1281 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1282 src += (5 * src_stride);
1284 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1286 hz_out0 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1287 mask0, mask1, mask2);
1288 hz_out2 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src2, src3,
1289 mask0, mask1, mask2);
1291 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
1293 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1295 for (loop_cnt = (height >> 1); loop_cnt--;) {
1296 LD_SB2(src, src_stride, src0, src1);
1297 src += (2 * src_stride);
1299 XORI_B2_128_SB(src0, src1);
1300 LD_UB2(dst, dst_stride, dst0, dst1);
1301 hz_out5 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1304 hz_out6 = (v8i16) __msa_pckod_d((v2i64) hz_out5, (v2i64) hz_out5);
1305 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out0, hz_out1, hz_out2,
1306 hz_out3, hz_out4, hz_out5);
1307 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out1, hz_out2, hz_out3,
1308 hz_out4, hz_out5, hz_out6);
1311 res1 = __msa_srari_h(hz_out3, 5);
1312 res3 = __msa_srari_h(hz_out4, 5);
1314 res1 = __msa_srari_h(hz_out2, 5);
1315 res3 = __msa_srari_h(hz_out3, 5);
1318 SAT_SH2_SH(res1, res3, 7);
1320 res0 = __msa_aver_s_h(res0, res1);
1321 res1 = __msa_aver_s_h(res2, res3);
1323 vec0 = PCKEV_XORI128_UB(res0, res0);
1324 vec1 = PCKEV_XORI128_UB(res1, res1);
1326 AVER_UB2_UB(vec0, dst0, vec1, dst1, dst0, dst1);
1328 out0 = __msa_copy_u_w((v4i32) dst0, 0);
1329 out1 = __msa_copy_u_w((v4i32) dst1, 0);
1343 static void avc_luma_midv_qrt_and_aver_dst_8w_msa(const uint8_t *src,
1348 uint8_t vert_offset)
1351 v16i8 src0, src1, src2, src3, src4;
1352 v16u8 dst0, dst1, dst2, dst3;
1353 v16i8 mask0, mask1, mask2;
1354 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1355 v8i16 hz_out4, hz_out5, hz_out6, hz_out7, hz_out8;
1356 v8i16 res0, res1, res2, res3;
1357 v8i16 res4, res5, res6, res7;
1359 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1361 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1362 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1363 src += (5 * src_stride);
1365 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1366 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1367 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1368 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1369 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1371 for (loop_cnt = (height >> 2); loop_cnt--;) {
1372 LD_SB4(src, src_stride, src0, src1, src2, src3);
1373 XORI_B4_128_SB(src0, src1, src2, src3);
1374 src += (4 * src_stride);
1376 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1378 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1379 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1380 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1381 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1383 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out0, hz_out1, hz_out2,
1384 hz_out3, hz_out4, hz_out5);
1385 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out1, hz_out2, hz_out3,
1386 hz_out4, hz_out5, hz_out6);
1387 res4 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out2, hz_out3, hz_out4,
1388 hz_out5, hz_out6, hz_out7);
1389 res6 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out3, hz_out4, hz_out5,
1390 hz_out6, hz_out7, hz_out8);
1393 res1 = __msa_srari_h(hz_out3, 5);
1394 res3 = __msa_srari_h(hz_out4, 5);
1395 res5 = __msa_srari_h(hz_out5, 5);
1396 res7 = __msa_srari_h(hz_out6, 5);
1398 res1 = __msa_srari_h(hz_out2, 5);
1399 res3 = __msa_srari_h(hz_out3, 5);
1400 res5 = __msa_srari_h(hz_out4, 5);
1401 res7 = __msa_srari_h(hz_out5, 5);
1404 SAT_SH4_SH(res1, res3, res5, res7, 7);
1406 res0 = __msa_aver_s_h(res0, res1);
1407 res1 = __msa_aver_s_h(res2, res3);
1408 res2 = __msa_aver_s_h(res4, res5);
1409 res3 = __msa_aver_s_h(res6, res7);
1410 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1411 CONVERT_UB_AVG_ST8x4_UB(res0, res1, res2, res3, dst0, dst1,
1413 dst += (4 * dst_stride);
1423 static void avc_luma_midv_qrt_and_aver_dst_16w_msa(const uint8_t *src,
1428 uint8_t vert_offset)
1430 int32_t multiple8_cnt;
1432 for (multiple8_cnt = 2; multiple8_cnt--;) {
1433 avc_luma_midv_qrt_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride,
1434 height, vert_offset);
1441 static void avc_luma_hv_qrt_and_aver_dst_4x4_msa(const uint8_t *src_x,
1442 const uint8_t *src_y,
1446 uint32_t tp0, tp1, tp2, tp3;
1447 const int16_t filt_const0 = 0xfb01;
1448 const int16_t filt_const1 = 0x1414;
1449 const int16_t filt_const2 = 0x1fb;
1450 v16u8 res, dst0 = { 0 };
1451 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, src_vt7, src_vt8;
1452 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
1453 v16i8 src_vt10_r, src_vt32_r, src_vt54_r, src_vt76_r;
1454 v16i8 mask0, mask1, mask2, filt0, filt1, filt2;
1455 v8i16 hz_out0, hz_out1, vt_out0, vt_out1, res0, res1;
1457 filt0 = (v16i8) __msa_fill_h(filt_const0);
1458 filt1 = (v16i8) __msa_fill_h(filt_const1);
1459 filt2 = (v16i8) __msa_fill_h(filt_const2);
1461 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
1463 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1464 src_y += (5 * stride);
1466 src_vt0 = (v16i8) __msa_insve_w((v4i32) src_vt0, 1, (v4i32) src_vt1);
1467 src_vt1 = (v16i8) __msa_insve_w((v4i32) src_vt1, 1, (v4i32) src_vt2);
1468 src_vt2 = (v16i8) __msa_insve_w((v4i32) src_vt2, 1, (v4i32) src_vt3);
1469 src_vt3 = (v16i8) __msa_insve_w((v4i32) src_vt3, 1, (v4i32) src_vt4);
1471 XORI_B4_128_SB(src_vt0, src_vt1, src_vt2, src_vt3);
1473 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
1474 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1475 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz1, mask0, mask1, mask2);
1476 hz_out1 = AVC_HORZ_FILTER_SH(src_hz2, src_hz3, mask0, mask1, mask2);
1478 SRARI_H2_SH(hz_out0, hz_out1, 5);
1479 SAT_SH2_SH(hz_out0, hz_out1, 7);
1481 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
1483 src_vt4 = (v16i8) __msa_insve_w((v4i32) src_vt4, 1, (v4i32) src_vt5);
1484 src_vt5 = (v16i8) __msa_insve_w((v4i32) src_vt5, 1, (v4i32) src_vt6);
1485 src_vt6 = (v16i8) __msa_insve_w((v4i32) src_vt6, 1, (v4i32) src_vt7);
1486 src_vt7 = (v16i8) __msa_insve_w((v4i32) src_vt7, 1, (v4i32) src_vt8);
1488 XORI_B4_128_SB(src_vt4, src_vt5, src_vt6, src_vt7);
1489 ILVR_B2_SB(src_vt1, src_vt0, src_vt3, src_vt2, src_vt10_r, src_vt32_r);
1490 ILVR_B2_SB(src_vt5, src_vt4, src_vt7, src_vt6, src_vt54_r, src_vt76_r);
1491 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
1493 vt_out1 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
1495 SRARI_H2_SH(vt_out0, vt_out1, 5);
1496 SAT_SH2_SH(vt_out0, vt_out1, 7);
1497 LW4(dst, stride, tp0, tp1, tp2, tp3);
1498 INSERT_W4_UB(tp0, tp1, tp2, tp3, dst0);
1500 res1 = __msa_srari_h((hz_out1 + vt_out1), 1);
1501 res0 = __msa_srari_h((hz_out0 + vt_out0), 1);
1503 SAT_SH2_SH(res0, res1, 7);
1504 res = PCKEV_XORI128_UB(res0, res1);
1505 dst0 = __msa_aver_u_b(res, dst0);
1507 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, stride);
1510 static void avc_luma_hv_qrt_and_aver_dst_8x8_msa(const uint8_t *src_x,
1511 const uint8_t *src_y,
1515 const int16_t filt_const0 = 0xfb01;
1516 const int16_t filt_const1 = 0x1414;
1517 const int16_t filt_const2 = 0x1fb;
1518 uint64_t tp0, tp1, tp2, tp3;
1519 v16u8 out0, out1, dst0 = { 0 }, dst1 = { 0 };
1520 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, src_vt0, src_vt1, src_vt2;
1521 v16i8 src_vt3, src_vt4, src_vt5, src_vt6, src_vt7, src_vt8;
1522 v16i8 src_vt9, src_vt10, src_vt11, src_vt12, mask0, mask1, mask2;
1523 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
1524 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, src_vt98_r, src_vt109_r;
1525 v16i8 src_vt1110_r, src_vt1211_r, filt0, filt1, filt2;
1526 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
1527 v8i16 vt_out3, tmp0, tmp1, tmp2, tmp3;
1529 filt0 = (v16i8) __msa_fill_h(filt_const0);
1530 filt1 = (v16i8) __msa_fill_h(filt_const1);
1531 filt2 = (v16i8) __msa_fill_h(filt_const2);
1533 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1534 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1535 src_y += (5 * stride);
1537 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1539 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
1540 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1541 src_x += (4 * stride);
1543 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
1544 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
1545 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
1546 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
1548 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
1549 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
1551 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
1552 src_y += (4 * stride);
1553 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
1555 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2, src_vt4,
1556 src_vt3, src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r);
1557 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6, src_vt8,
1558 src_vt7, src_vt54_r, src_vt65_r, src_vt76_r, src_vt87_r);
1559 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
1561 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0, filt1,
1563 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
1565 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0, filt1,
1567 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
1568 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
1570 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
1571 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
1572 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
1573 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
1575 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
1576 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1578 LD4(dst, stride, tp0, tp1, tp2, tp3);
1579 INSERT_D2_UB(tp0, tp1, dst0);
1580 INSERT_D2_UB(tp2, tp3, dst1);
1582 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
1583 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
1584 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
1585 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
1586 ST8x4_UB(dst0, dst1, dst, stride);
1587 dst += (4 * stride);
1589 LD_SB4(src_y, stride, src_vt9, src_vt10, src_vt11, src_vt12);
1590 XORI_B4_128_SB(src_vt9, src_vt10, src_vt11, src_vt12);
1592 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
1593 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
1594 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
1595 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
1597 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
1598 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
1600 ILVR_B4_SB(src_vt9, src_vt8, src_vt10, src_vt9, src_vt11, src_vt10,
1601 src_vt12, src_vt11, src_vt98_r, src_vt109_r, src_vt1110_r,
1603 vt_out0 = AVC_DOT_SH3_SH(src_vt54_r, src_vt76_r, src_vt98_r, filt0, filt1,
1605 vt_out1 = AVC_DOT_SH3_SH(src_vt65_r, src_vt87_r, src_vt109_r, filt0, filt1,
1607 vt_out2 = AVC_DOT_SH3_SH(src_vt76_r, src_vt98_r, src_vt1110_r, filt0, filt1,
1609 vt_out3 = AVC_DOT_SH3_SH(src_vt87_r, src_vt109_r, src_vt1211_r, filt0,
1611 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
1612 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
1614 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
1615 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
1616 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
1617 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
1619 LD4(dst, stride, tp0, tp1, tp2, tp3);
1620 INSERT_D2_UB(tp0, tp1, dst0);
1621 INSERT_D2_UB(tp2, tp3, dst1);
1623 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
1624 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
1625 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
1626 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
1627 ST8x4_UB(dst0, dst1, dst, stride);
1628 dst += (4 * stride);
1631 static void avc_luma_hv_qrt_and_aver_dst_16x16_msa(const uint8_t *src_x,
1632 const uint8_t *src_y,
1636 const int16_t filt_const0 = 0xfb01;
1637 const int16_t filt_const1 = 0x1414;
1638 const int16_t filt_const2 = 0x1fb;
1639 const uint8_t *src_x_tmp = src_x;
1640 const uint8_t *src_y_tmp = src_y;
1641 uint8_t *dst_tmp = dst;
1642 uint32_t multiple8_cnt, loop_cnt;
1643 uint64_t tp0, tp1, tp2, tp3;
1644 v16u8 tmp0, tmp1, dst0 = { 0 }, dst1 = { 0 };
1645 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2;
1646 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
1647 v16i8 src_vt7, src_vt8;
1648 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
1649 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, filt0, filt1, filt2;
1650 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
1651 v8i16 vt_out3, out0, out1, out2, out3;
1653 filt0 = (v16i8) __msa_fill_h(filt_const0);
1654 filt1 = (v16i8) __msa_fill_h(filt_const1);
1655 filt2 = (v16i8) __msa_fill_h(filt_const2);
1657 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1659 for (multiple8_cnt = 2; multiple8_cnt--;) {
1664 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1665 src_y += (5 * stride);
1667 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1669 for (loop_cnt = 4; loop_cnt--;) {
1670 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
1671 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1672 src_x += (4 * stride);
1674 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
1675 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
1676 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
1677 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
1678 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
1679 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
1681 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
1682 src_y += (4 * stride);
1684 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
1685 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2,
1686 src_vt4, src_vt3, src_vt10_r, src_vt21_r, src_vt32_r,
1688 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6,
1689 src_vt8, src_vt7, src_vt54_r, src_vt65_r, src_vt76_r,
1691 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0,
1693 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0,
1695 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0,
1697 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0,
1699 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
1700 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
1702 out0 = __msa_srari_h((hz_out0 + vt_out0), 1);
1703 out1 = __msa_srari_h((hz_out1 + vt_out1), 1);
1704 out2 = __msa_srari_h((hz_out2 + vt_out2), 1);
1705 out3 = __msa_srari_h((hz_out3 + vt_out3), 1);
1707 LD4(dst, stride, tp0, tp1, tp2, tp3);
1708 INSERT_D2_UB(tp0, tp1, dst0);
1709 INSERT_D2_UB(tp2, tp3, dst1);
1711 SAT_SH4_SH(out0, out1, out2, out3, 7);
1712 tmp0 = PCKEV_XORI128_UB(out0, out1);
1713 tmp1 = PCKEV_XORI128_UB(out2, out3);
1714 AVER_UB2_UB(tmp0, dst0, tmp1, dst1, dst0, dst1);
1715 ST8x4_UB(dst0, dst1, dst, stride);
1716 dst += (4 * stride);
1731 void ff_put_h264_qpel16_mc00_msa(uint8_t *dst, const uint8_t *src,
1734 v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
1735 v16u8 src8, src9, src10, src11, src12, src13, src14, src15;
1737 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1738 src += (8 * stride);
1739 LD_UB8(src, stride, src8, src9, src10, src11, src12, src13, src14, src15);
1741 ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst, stride);
1742 dst += (8 * stride);
1743 ST_UB8(src8, src9, src10, src11, src12, src13, src14, src15, dst, stride);
1746 void ff_put_h264_qpel8_mc00_msa(uint8_t *dst, const uint8_t *src,
1749 uint64_t src0, src1, src2, src3, src4, src5, src6, src7;
1751 LD4(src, stride, src0, src1, src2, src3);
1753 LD4(src, stride, src4, src5, src6, src7);
1754 SD4(src0, src1, src2, src3, dst, stride);
1756 SD4(src4, src5, src6, src7, dst, stride);
1759 void ff_avg_h264_qpel16_mc00_msa(uint8_t *dst, const uint8_t *src,
1762 v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
1763 v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1765 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1766 src += (8 * stride);
1767 LD_UB8(dst, stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1769 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1771 AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5,
1773 ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, stride);
1774 dst += (8 * stride);
1776 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1777 LD_UB8(dst, stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1779 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1781 AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5,
1783 ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, stride);
1786 void ff_avg_h264_qpel8_mc00_msa(uint8_t *dst, const uint8_t *src,
1789 uint64_t tp0, tp1, tp2, tp3, tp4, tp5, tp6, tp7;
1790 v16u8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 }, src3 = { 0 };
1791 v16u8 dst0 = { 0 }, dst1 = { 0 }, dst2 = { 0 }, dst3 = { 0 };
1793 LD4(src, stride, tp0, tp1, tp2, tp3);
1795 LD4(src, stride, tp4, tp5, tp6, tp7);
1796 INSERT_D2_UB(tp0, tp1, src0);
1797 INSERT_D2_UB(tp2, tp3, src1);
1798 INSERT_D2_UB(tp4, tp5, src2);
1799 INSERT_D2_UB(tp6, tp7, src3);
1801 LD4(dst, stride, tp0, tp1, tp2, tp3);
1802 LD4(dst + 4 * stride, stride, tp4, tp5, tp6, tp7);
1803 INSERT_D2_UB(tp0, tp1, dst0);
1804 INSERT_D2_UB(tp2, tp3, dst1);
1805 INSERT_D2_UB(tp4, tp5, dst2);
1806 INSERT_D2_UB(tp6, tp7, dst3);
1808 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1811 ST8x8_UB(dst0, dst1, dst2, dst3, dst, stride);
1814 void ff_avg_h264_qpel4_mc00_msa(uint8_t *dst, const uint8_t *src,
1817 uint32_t tp0, tp1, tp2, tp3;
1818 v16u8 src0 = { 0 }, dst0 = { 0 };
1820 LW4(src, stride, tp0, tp1, tp2, tp3);
1821 INSERT_W4_UB(tp0, tp1, tp2, tp3, src0);
1822 LW4(dst, stride, tp0, tp1, tp2, tp3);
1823 INSERT_W4_UB(tp0, tp1, tp2, tp3, dst0);
1825 dst0 = __msa_aver_u_b(src0, dst0);
1827 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, stride);
1830 void ff_put_h264_qpel16_mc10_msa(uint8_t *dst, const uint8_t *src,
1834 v16i8 dst0, dst1, dst2, dst3, src0, src1, src2, src3, src4, src5, src6;
1835 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, src7, vec11;
1836 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1837 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1838 v16i8 minus5b = __msa_ldi_b(-5);
1839 v16i8 plus20b = __msa_ldi_b(20);
1841 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1847 for (loop_cnt = 4; loop_cnt--;) {
1848 LD_SB2(src, 16, src0, src1);
1850 LD_SB2(src, 16, src2, src3);
1852 LD_SB2(src, 16, src4, src5);
1854 LD_SB2(src, 16, src6, src7);
1857 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1858 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask3, vec0, vec3);
1859 VSHF_B2_SB(src2, src2, src2, src3, mask0, mask3, vec6, vec9);
1860 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask4, vec1, vec4);
1861 VSHF_B2_SB(src2, src2, src2, src3, mask1, mask4, vec7, vec10);
1862 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask5, vec2, vec5);
1863 VSHF_B2_SB(src2, src2, src2, src3, mask2, mask5, vec8, vec11);
1864 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
1865 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1866 minus5b, res0, res1, res2, res3);
1867 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1868 plus20b, res0, res1, res2, res3);
1869 VSHF_B2_SB(src4, src4, src4, src5, mask0, mask3, vec0, vec3);
1870 VSHF_B2_SB(src6, src6, src6, src7, mask0, mask3, vec6, vec9);
1871 VSHF_B2_SB(src4, src4, src4, src5, mask1, mask4, vec1, vec4);
1872 VSHF_B2_SB(src6, src6, src6, src7, mask1, mask4, vec7, vec10);
1873 VSHF_B2_SB(src4, src4, src4, src5, mask2, mask5, vec2, vec5);
1874 VSHF_B2_SB(src6, src6, src6, src7, mask2, mask5, vec8, vec11);
1875 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
1876 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1877 minus5b, res4, res5, res6, res7);
1878 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1879 plus20b, res4, res5, res6, res7);
1880 SLDI_B2_SB(src1, src3, src0, src2, src0, src2, 2);
1881 SLDI_B2_SB(src5, src7, src4, src6, src4, src6, 2);
1882 SRARI_H4_SH(res0, res1, res2, res3, 5);
1883 SRARI_H4_SH(res4, res5, res6, res7, 5);
1884 SAT_SH4_SH(res0, res1, res2, res3, 7);
1885 SAT_SH4_SH(res4, res5, res6, res7, 7);
1886 PCKEV_B2_SB(res1, res0, res3, res2, dst0, dst1);
1887 PCKEV_B2_SB(res5, res4, res7, res6, dst2, dst3);
1888 dst0 = __msa_aver_s_b(dst0, src0);
1889 dst1 = __msa_aver_s_b(dst1, src2);
1890 dst2 = __msa_aver_s_b(dst2, src4);
1891 dst3 = __msa_aver_s_b(dst3, src6);
1892 XORI_B4_128_SB(dst0, dst1, dst2, dst3);
1893 ST_SB4(dst0, dst1, dst2, dst3, dst, stride);
1894 dst += (4 * stride);
1898 void ff_put_h264_qpel16_mc30_msa(uint8_t *dst, const uint8_t *src,
1902 v16i8 dst0, dst1, dst2, dst3, src0, src1, src2, src3, src4, src5, src6;
1903 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, src7, vec11;
1904 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1905 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1906 v16i8 minus5b = __msa_ldi_b(-5);
1907 v16i8 plus20b = __msa_ldi_b(20);
1909 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1915 for (loop_cnt = 4; loop_cnt--;) {
1916 LD_SB2(src, 16, src0, src1);
1918 LD_SB2(src, 16, src2, src3);
1920 LD_SB2(src, 16, src4, src5);
1922 LD_SB2(src, 16, src6, src7);
1925 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1926 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask3, vec0, vec3);
1927 VSHF_B2_SB(src2, src2, src2, src3, mask0, mask3, vec6, vec9);
1928 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask4, vec1, vec4);
1929 VSHF_B2_SB(src2, src2, src2, src3, mask1, mask4, vec7, vec10);
1930 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask5, vec2, vec5);
1931 VSHF_B2_SB(src2, src2, src2, src3, mask2, mask5, vec8, vec11);
1932 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
1933 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1934 minus5b, res0, res1, res2, res3);
1935 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1936 plus20b, res0, res1, res2, res3);
1937 VSHF_B2_SB(src4, src4, src4, src5, mask0, mask3, vec0, vec3);
1938 VSHF_B2_SB(src6, src6, src6, src7, mask0, mask3, vec6, vec9);
1939 VSHF_B2_SB(src4, src4, src4, src5, mask1, mask4, vec1, vec4);
1940 VSHF_B2_SB(src6, src6, src6, src7, mask1, mask4, vec7, vec10);
1941 VSHF_B2_SB(src4, src4, src4, src5, mask2, mask5, vec2, vec5);
1942 VSHF_B2_SB(src6, src6, src6, src7, mask2, mask5, vec8, vec11);
1943 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
1944 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1945 minus5b, res4, res5, res6, res7);
1946 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1947 plus20b, res4, res5, res6, res7);
1948 SLDI_B2_SB(src1, src3, src0, src2, src0, src2, 3);
1949 SLDI_B2_SB(src5, src7, src4, src6, src4, src6, 3);
1950 SRARI_H4_SH(res0, res1, res2, res3, 5);
1951 SRARI_H4_SH(res4, res5, res6, res7, 5);
1952 SAT_SH4_SH(res0, res1, res2, res3, 7);
1953 SAT_SH4_SH(res4, res5, res6, res7, 7);
1954 PCKEV_B2_SB(res1, res0, res3, res2, dst0, dst1);
1955 PCKEV_B2_SB(res5, res4, res7, res6, dst2, dst3);
1956 dst0 = __msa_aver_s_b(dst0, src0);
1957 dst1 = __msa_aver_s_b(dst1, src2);
1958 dst2 = __msa_aver_s_b(dst2, src4);
1959 dst3 = __msa_aver_s_b(dst3, src6);
1960 XORI_B4_128_SB(dst0, dst1, dst2, dst3);
1961 ST_SB4(dst0, dst1, dst2, dst3, dst, stride);
1962 dst += (4 * stride);
1966 void ff_put_h264_qpel8_mc10_msa(uint8_t *dst, const uint8_t *src,
1969 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
1970 v16i8 tmp0, tmp1, tmp2, tmp3, vec11;
1971 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1972 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1973 v16i8 minus5b = __msa_ldi_b(-5);
1974 v16i8 plus20b = __msa_ldi_b(20);
1976 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1977 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1978 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1979 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
1980 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
1981 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
1982 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
1983 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
1984 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
1985 res0, res1, res2, res3);
1986 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
1987 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
1988 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
1989 res0, res1, res2, res3);
1990 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
1991 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
1992 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
1993 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
1994 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
1995 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
1996 res4, res5, res6, res7);
1997 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
1998 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
1999 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2000 res4, res5, res6, res7);
2001 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 2);
2002 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 2);
2003 SLDI_B2_SB(src4, src5, src4, src5, src4, src5, 2);
2004 SLDI_B2_SB(src6, src7, src6, src7, src6, src7, 2);
2005 PCKEV_D2_SB(src1, src0, src3, src2, src0, src1);
2006 PCKEV_D2_SB(src5, src4, src7, src6, src4, src5);
2007 SRARI_H4_SH(res0, res1, res2, res3, 5);
2008 SRARI_H4_SH(res4, res5, res6, res7, 5);
2009 SAT_SH4_SH(res0, res1, res2, res3, 7);
2010 SAT_SH4_SH(res4, res5, res6, res7, 7);
2011 PCKEV_B2_SB(res1, res0, res3, res2, tmp0, tmp1);
2012 PCKEV_B2_SB(res5, res4, res7, res6, tmp2, tmp3);
2013 tmp0 = __msa_aver_s_b(tmp0, src0);
2014 tmp1 = __msa_aver_s_b(tmp1, src1);
2015 tmp2 = __msa_aver_s_b(tmp2, src4);
2016 tmp3 = __msa_aver_s_b(tmp3, src5);
2017 XORI_B4_128_SB(tmp0, tmp1, tmp2, tmp3);
2018 ST8x8_UB(tmp0, tmp1, tmp2, tmp3, dst, stride);
2021 void ff_put_h264_qpel8_mc30_msa(uint8_t *dst, const uint8_t *src,
2024 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2025 v16i8 tmp0, tmp1, tmp2, tmp3, vec11;
2026 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2027 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2028 v16i8 minus5b = __msa_ldi_b(-5);
2029 v16i8 plus20b = __msa_ldi_b(20);
2031 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2032 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
2033 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2034 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2035 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2036 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
2037 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
2038 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
2039 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2040 res0, res1, res2, res3);
2041 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
2042 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
2043 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2044 res0, res1, res2, res3);
2045 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2046 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
2047 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
2048 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
2049 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
2050 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2051 res4, res5, res6, res7);
2052 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
2053 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
2054 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2055 res4, res5, res6, res7);
2056 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 3);
2057 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 3);
2058 SLDI_B2_SB(src4, src5, src4, src5, src4, src5, 3);
2059 SLDI_B2_SB(src6, src7, src6, src7, src6, src7, 3);
2060 PCKEV_D2_SB(src1, src0, src3, src2, src0, src1);
2061 PCKEV_D2_SB(src5, src4, src7, src6, src4, src5);
2062 SRARI_H4_SH(res0, res1, res2, res3, 5);
2063 SRARI_H4_SH(res4, res5, res6, res7, 5);
2064 SAT_SH4_SH(res0, res1, res2, res3, 7);
2065 SAT_SH4_SH(res4, res5, res6, res7, 7);
2066 PCKEV_B2_SB(res1, res0, res3, res2, tmp0, tmp1);
2067 PCKEV_B2_SB(res5, res4, res7, res6, tmp2, tmp3);
2068 tmp0 = __msa_aver_s_b(tmp0, src0);
2069 tmp1 = __msa_aver_s_b(tmp1, src1);
2070 tmp2 = __msa_aver_s_b(tmp2, src4);
2071 tmp3 = __msa_aver_s_b(tmp3, src5);
2072 XORI_B4_128_SB(tmp0, tmp1, tmp2, tmp3);
2073 ST8x8_UB(tmp0, tmp1, tmp2, tmp3, dst, stride);
2076 void ff_put_h264_qpel4_mc10_msa(uint8_t *dst, const uint8_t *src,
2079 v16i8 src0, src1, src2, src3, res, mask0, mask1, mask2;
2080 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2082 v16i8 minus5b = __msa_ldi_b(-5);
2083 v16i8 plus20b = __msa_ldi_b(20);
2085 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2086 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2087 XORI_B4_128_SB(src0, src1, src2, src3);
2088 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2089 HADD_SB2_SH(vec0, vec1, res0, res1);
2090 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2091 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2092 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2093 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2094 SRARI_H2_SH(res0, res1, 5);
2095 SAT_SH2_SH(res0, res1, 7);
2096 res = __msa_pckev_b((v16i8) res1, (v16i8) res0);
2097 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 2);
2098 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 2);
2099 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
2100 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2101 src0 = (v16i8) __msa_insve_d((v2i64) src0, 1, (v2i64) src1);
2102 res = __msa_aver_s_b(res, src0);
2103 res = (v16i8) __msa_xori_b((v16u8) res, 128);
2104 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
2107 void ff_put_h264_qpel4_mc30_msa(uint8_t *dst, const uint8_t *src,
2110 v16i8 src0, src1, src2, src3, res, mask0, mask1, mask2;
2111 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2113 v16i8 minus5b = __msa_ldi_b(-5);
2114 v16i8 plus20b = __msa_ldi_b(20);
2116 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2117 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2118 XORI_B4_128_SB(src0, src1, src2, src3);
2119 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2120 HADD_SB2_SH(vec0, vec1, res0, res1);
2121 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2122 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2123 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2124 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2125 SRARI_H2_SH(res0, res1, 5);
2126 SAT_SH2_SH(res0, res1, 7);
2127 res = __msa_pckev_b((v16i8) res1, (v16i8) res0);
2128 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 3);
2129 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 3);
2130 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
2131 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2132 src0 = (v16i8) __msa_insve_d((v2i64) src0, 1, (v2i64) src1);
2133 res = __msa_aver_s_b(res, src0);
2134 res = (v16i8) __msa_xori_b((v16u8) res, 128);
2135 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
2138 void ff_put_h264_qpel16_mc20_msa(uint8_t *dst, const uint8_t *src,
2142 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2143 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2145 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2146 v16i8 minus5b = __msa_ldi_b(-5);
2147 v16i8 plus20b = __msa_ldi_b(20);
2149 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2152 for (loop_cnt = 4; loop_cnt--;) {
2153 LD_SB2(src, 8, src0, src1);
2155 LD_SB2(src, 8, src2, src3);
2157 LD_SB2(src, 8, src4, src5);
2159 LD_SB2(src, 8, src6, src7);
2162 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2163 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
2164 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
2165 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
2166 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
2167 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
2168 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
2169 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
2170 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
2171 minus5b, res0, res1, res2, res3);
2172 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
2173 plus20b, res0, res1, res2, res3);
2174 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec3);
2175 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec6, vec9);
2176 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec1, vec4);
2177 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec7, vec10);
2178 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec2, vec5);
2179 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec8, vec11);
2180 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
2181 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
2182 minus5b, res4, res5, res6, res7);
2183 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
2184 plus20b, res4, res5, res6, res7);
2185 SRARI_H4_SH(res0, res1, res2, res3, 5);
2186 SRARI_H4_SH(res4, res5, res6, res7, 5);
2187 SAT_SH4_SH(res0, res1, res2, res3, 7);
2188 SAT_SH4_SH(res4, res5, res6, res7, 7);
2189 PCKEV_B4_SB(res1, res0, res3, res2, res5, res4, res7, res6, vec0, vec1,
2191 XORI_B4_128_SB(vec0, vec1, vec2, vec3);
2192 ST_SB4(vec0, vec1, vec2, vec3, dst, stride);
2193 dst += (4 * stride);
2197 void ff_put_h264_qpel8_mc20_msa(uint8_t *dst, const uint8_t *src,
2200 v16u8 out0, out1, out2, out3;
2201 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2202 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2204 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2205 v16i8 minus5b = __msa_ldi_b(-5);
2206 v16i8 plus20b = __msa_ldi_b(20);
2208 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2209 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
2210 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2211 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2212 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2213 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
2214 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
2215 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
2216 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2217 res0, res1, res2, res3);
2218 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
2219 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
2220 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
2221 plus20b, res0, res1, res2, res3);
2222 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2223 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
2224 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
2225 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
2226 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
2227 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2228 res4, res5, res6, res7);
2229 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
2230 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
2231 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
2232 plus20b, res4, res5, res6, res7);
2233 SRARI_H4_SH(res0, res1, res2, res3, 5);
2234 SRARI_H4_SH(res4, res5, res6, res7, 5);
2235 SAT_SH4_SH(res0, res1, res2, res3, 7);
2236 SAT_SH4_SH(res4, res5, res6, res7, 7);
2237 out0 = PCKEV_XORI128_UB(res0, res1);
2238 out1 = PCKEV_XORI128_UB(res2, res3);
2239 out2 = PCKEV_XORI128_UB(res4, res5);
2240 out3 = PCKEV_XORI128_UB(res6, res7);
2241 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2244 void ff_put_h264_qpel4_mc20_msa(uint8_t *dst, const uint8_t *src,
2248 v16i8 src0, src1, src2, src3, mask0, mask1, mask2;
2249 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2251 v16i8 minus5b = __msa_ldi_b(-5);
2252 v16i8 plus20b = __msa_ldi_b(20);
2254 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2255 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2256 XORI_B4_128_SB(src0, src1, src2, src3);
2257 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2258 HADD_SB2_SH(vec0, vec1, res0, res1);
2259 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2260 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2261 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2262 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2263 SRARI_H2_SH(res0, res1, 5);
2264 SAT_SH2_SH(res0, res1, 7);
2265 out = PCKEV_XORI128_UB(res0, res1);
2266 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2269 void ff_put_h264_qpel16_mc01_msa(uint8_t *dst, const uint8_t *src,
2273 int16_t filt_const0 = 0xfb01;
2274 int16_t filt_const1 = 0x1414;
2275 int16_t filt_const2 = 0x1fb;
2276 v16u8 res0, res1, res2, res3;
2277 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2278 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2279 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
2280 v16i8 src65_l, src87_l, filt0, filt1, filt2;
2281 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
2283 filt0 = (v16i8) __msa_fill_h(filt_const0);
2284 filt1 = (v16i8) __msa_fill_h(filt_const1);
2285 filt2 = (v16i8) __msa_fill_h(filt_const2);
2287 src -= (stride * 2);
2289 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2290 src += (5 * stride);
2292 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2293 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2295 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
2298 for (loop_cnt = 4; loop_cnt--;) {
2299 LD_SB4(src, stride, src5, src6, src7, src8);
2300 src += (4 * stride);
2302 XORI_B4_128_SB(src5, src6, src7, src8);
2303 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
2304 src65_r, src76_r, src87_r);
2305 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
2306 src65_l, src76_l, src87_l);
2307 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2308 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2309 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2310 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2311 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
2312 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
2313 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
2314 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
2315 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2316 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2317 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
2318 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
2319 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
2320 out3_r, res0, res1, res2, res3);
2321 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src2);
2322 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src3);
2323 res2 = (v16u8) __msa_aver_s_b((v16i8) res2, src4);
2324 res3 = (v16u8) __msa_aver_s_b((v16i8) res3, src5);
2325 XORI_B4_128_UB(res0, res1, res2, res3);
2326 ST_UB4(res0, res1, res2, res3, dst, stride);
2327 dst += (4 * stride);
2343 void ff_put_h264_qpel16_mc03_msa(uint8_t *dst, const uint8_t *src,
2347 int16_t filt_const0 = 0xfb01;
2348 int16_t filt_const1 = 0x1414;
2349 int16_t filt_const2 = 0x1fb;
2350 v16u8 res0, res1, res2, res3;
2351 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2352 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2353 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
2354 v16i8 src65_l, src87_l, filt0, filt1, filt2;
2355 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
2357 filt0 = (v16i8) __msa_fill_h(filt_const0);
2358 filt1 = (v16i8) __msa_fill_h(filt_const1);
2359 filt2 = (v16i8) __msa_fill_h(filt_const2);
2361 src -= (stride * 2);
2363 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2364 src += (5 * stride);
2366 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2367 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2369 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
2372 for (loop_cnt = 4; loop_cnt--;) {
2373 LD_SB4(src, stride, src5, src6, src7, src8);
2374 src += (4 * stride);
2376 XORI_B4_128_SB(src5, src6, src7, src8);
2377 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
2378 src65_r, src76_r, src87_r);
2379 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
2380 src65_l, src76_l, src87_l);
2381 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2382 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2383 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2384 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2385 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
2386 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
2387 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
2388 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
2389 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2390 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2391 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
2392 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
2393 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
2394 out3_r, res0, res1, res2, res3);
2395 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src3);
2396 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src4);
2397 res2 = (v16u8) __msa_aver_s_b((v16i8) res2, src5);
2398 res3 = (v16u8) __msa_aver_s_b((v16i8) res3, src6);
2399 XORI_B4_128_UB(res0, res1, res2, res3);
2400 ST_UB4(res0, res1, res2, res3, dst, stride);
2401 dst += (4 * stride);
2416 void ff_put_h264_qpel8_mc01_msa(uint8_t *dst, const uint8_t *src,
2419 const int16_t filt_const0 = 0xfb01;
2420 const int16_t filt_const1 = 0x1414;
2421 const int16_t filt_const2 = 0x1fb;
2422 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2423 v16i8 src11, src12, src10_r, src32_r, src54_r, src65_r, src76_r, src98_r;
2424 v16i8 src21_r, src43_r, src87_r, src109_r, src1211_r, src1110_r;
2425 v16i8 tmp0, tmp1, tmp2, tmp3, filt0, filt1, filt2, out0, out1, out2, out3;
2426 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
2428 filt0 = (v16i8) __msa_fill_h(filt_const0);
2429 filt1 = (v16i8) __msa_fill_h(filt_const1);
2430 filt2 = (v16i8) __msa_fill_h(filt_const2);
2432 src -= (stride * 2);
2434 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2435 src += (5 * stride);
2436 LD_SB8(src, stride, src5, src6, src7, src8, src9, src10, src11, src12);
2437 XORI_B8_128_SB(src5, src6, src7, src8, src9, src10, src11, src12);
2438 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2439 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2441 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2443 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src98_r,
2444 src109_r, src1110_r, src1211_r);
2445 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2446 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2447 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2448 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2449 out4_r = AVC_DOT_SH3_SH(src54_r, src76_r, src98_r, filt0, filt1, filt2);
2450 out5_r = AVC_DOT_SH3_SH(src65_r, src87_r, src109_r, filt0, filt1, filt2);
2451 out6_r = AVC_DOT_SH3_SH(src76_r, src98_r, src1110_r, filt0, filt1, filt2);
2452 out7_r = AVC_DOT_SH3_SH(src87_r, src109_r, src1211_r, filt0, filt1, filt2);
2453 PCKEV_D2_SB(src3, src2, src5, src4, tmp0, tmp1);
2454 PCKEV_D2_SB(src7, src6, src9, src8, tmp2, tmp3);
2455 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2456 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
2457 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2458 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
2459 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, out0, out1);
2460 PCKEV_B2_SB(out5_r, out4_r, out7_r, out6_r, out2, out3);
2461 out0 = __msa_aver_s_b(out0, tmp0);
2462 out1 = __msa_aver_s_b(out1, tmp1);
2463 out2 = __msa_aver_s_b(out2, tmp2);
2464 out3 = __msa_aver_s_b(out3, tmp3);
2465 XORI_B4_128_SB(out0, out1, out2, out3);
2466 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2469 void ff_put_h264_qpel8_mc03_msa(uint8_t *dst, const uint8_t *src,
2472 const int16_t filt_const0 = 0xfb01;
2473 const int16_t filt_const1 = 0x1414;
2474 const int16_t filt_const2 = 0x1fb;
2475 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2476 v16i8 src11, src12, src10_r, src32_r, src54_r, src65_r, src76_r, src98_r;
2477 v16i8 src21_r, src43_r, src87_r, src109_r, src1211_r, src1110_r;
2478 v16i8 filt0, filt1, filt2, out0, out1, out2, out3, tmp0, tmp1, tmp2, tmp3;
2479 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
2481 filt0 = (v16i8) __msa_fill_h(filt_const0);
2482 filt1 = (v16i8) __msa_fill_h(filt_const1);
2483 filt2 = (v16i8) __msa_fill_h(filt_const2);
2485 src -= (stride * 2);
2487 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2488 src += (5 * stride);
2489 LD_SB8(src, stride, src5, src6, src7, src8, src9, src10, src11, src12);
2490 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2491 XORI_B8_128_SB(src5, src6, src7, src8, src9, src10, src11, src12);
2492 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2494 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2496 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src98_r,
2497 src109_r, src1110_r, src1211_r);
2498 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2499 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2500 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2501 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2502 out4_r = AVC_DOT_SH3_SH(src54_r, src76_r, src98_r, filt0, filt1, filt2);
2503 out5_r = AVC_DOT_SH3_SH(src65_r, src87_r, src109_r, filt0, filt1, filt2);
2504 out6_r = AVC_DOT_SH3_SH(src76_r, src98_r, src1110_r, filt0, filt1, filt2);
2505 out7_r = AVC_DOT_SH3_SH(src87_r, src109_r, src1211_r, filt0, filt1, filt2);
2506 PCKEV_D2_SB(src4, src3, src6, src5, tmp0, tmp1);
2507 PCKEV_D2_SB(src8, src7, src10, src9, tmp2, tmp3);
2508 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2509 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
2510 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2511 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
2512 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, out0, out1);
2513 PCKEV_B2_SB(out5_r, out4_r, out7_r, out6_r, out2, out3);
2514 out0 = __msa_aver_s_b(out0, tmp0);
2515 out1 = __msa_aver_s_b(out1, tmp1);
2516 out2 = __msa_aver_s_b(out2, tmp2);
2517 out3 = __msa_aver_s_b(out3, tmp3);
2518 XORI_B4_128_SB(out0, out1, out2, out3);
2519 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2522 void ff_put_h264_qpel4_mc01_msa(uint8_t *dst, const uint8_t *src,
2525 int16_t filt_const0 = 0xfb01;
2526 int16_t filt_const1 = 0x1414;
2527 int16_t filt_const2 = 0x1fb;
2529 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2530 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2531 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
2534 filt0 = (v16i8) __msa_fill_h(filt_const0);
2535 filt1 = (v16i8) __msa_fill_h(filt_const1);
2536 filt2 = (v16i8) __msa_fill_h(filt_const2);
2538 src -= (stride * 2);
2540 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2541 src += (5 * stride);
2542 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2544 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
2545 XORI_B2_128_SB(src2110, src4332);
2546 LD_SB4(src, stride, src5, src6, src7, src8);
2547 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2549 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
2550 XORI_B2_128_SB(src6554, src8776);
2551 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
2552 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
2553 SRARI_H2_SH(out10, out32, 5);
2554 SAT_SH2_SH(out10, out32, 7);
2555 out = PCKEV_XORI128_UB(out10, out32);
2556 src32_r = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2557 src54_r = (v16i8) __msa_insve_w((v4i32) src4, 1, (v4i32) src5);
2558 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
2559 out = __msa_aver_u_b(out, (v16u8) src32_r);
2560 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2563 void ff_put_h264_qpel4_mc03_msa(uint8_t *dst, const uint8_t *src,
2566 int16_t filt_const0 = 0xfb01;
2567 int16_t filt_const1 = 0x1414;
2568 int16_t filt_const2 = 0x1fb;
2570 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2571 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2572 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
2575 filt0 = (v16i8) __msa_fill_h(filt_const0);
2576 filt1 = (v16i8) __msa_fill_h(filt_const1);
2577 filt2 = (v16i8) __msa_fill_h(filt_const2);
2579 src -= (stride * 2);
2581 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2582 src += (5 * stride);
2583 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2585 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
2586 XORI_B2_128_SB(src2110, src4332);
2587 LD_SB4(src, stride, src5, src6, src7, src8);
2588 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2590 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
2591 XORI_B2_128_SB(src6554, src8776);
2592 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
2593 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
2594 SRARI_H2_SH(out10, out32, 5);
2595 SAT_SH2_SH(out10, out32, 7);
2596 out = PCKEV_XORI128_UB(out10, out32);
2597 src32_r = (v16i8) __msa_insve_w((v4i32) src3, 1, (v4i32) src4);
2598 src54_r = (v16i8) __msa_insve_w((v4i32) src5, 1, (v4i32) src6);
2599 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
2600 out = __msa_aver_u_b(out, (v16u8) src32_r);
2601 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2604 void ff_put_h264_qpel16_mc11_msa(uint8_t *dst, const uint8_t *src,
2607 avc_luma_hv_qrt_16x16_msa(src - 2, src - (stride * 2), dst, stride);
2610 void ff_put_h264_qpel16_mc31_msa(uint8_t *dst, const uint8_t *src,
2613 avc_luma_hv_qrt_16x16_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2616 void ff_put_h264_qpel16_mc13_msa(uint8_t *dst, const uint8_t *src,
2619 avc_luma_hv_qrt_16x16_msa(src + stride - 2, src - (stride * 2), dst,
2623 void ff_put_h264_qpel16_mc33_msa(uint8_t *dst, const uint8_t *src,
2626 avc_luma_hv_qrt_16x16_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2630 void ff_put_h264_qpel8_mc11_msa(uint8_t *dst, const uint8_t *src,
2633 avc_luma_hv_qrt_8x8_msa(src - 2, src - (stride * 2), dst, stride);
2636 void ff_put_h264_qpel8_mc31_msa(uint8_t *dst, const uint8_t *src,
2639 avc_luma_hv_qrt_8x8_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2642 void ff_put_h264_qpel8_mc13_msa(uint8_t *dst, const uint8_t *src,
2645 avc_luma_hv_qrt_8x8_msa(src + stride - 2, src - (stride * 2), dst, stride);
2648 void ff_put_h264_qpel8_mc33_msa(uint8_t *dst, const uint8_t *src,
2651 avc_luma_hv_qrt_8x8_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2656 void ff_put_h264_qpel4_mc11_msa(uint8_t *dst, const uint8_t *src,
2659 avc_luma_hv_qrt_4x4_msa(src - 2, src - (stride * 2), dst, stride);
2662 void ff_put_h264_qpel4_mc31_msa(uint8_t *dst, const uint8_t *src,
2665 avc_luma_hv_qrt_4x4_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2668 void ff_put_h264_qpel4_mc13_msa(uint8_t *dst, const uint8_t *src,
2671 avc_luma_hv_qrt_4x4_msa(src + stride - 2, src - (stride * 2), dst, stride);
2674 void ff_put_h264_qpel4_mc33_msa(uint8_t *dst, const uint8_t *src,
2677 avc_luma_hv_qrt_4x4_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2681 void ff_put_h264_qpel16_mc21_msa(uint8_t *dst, const uint8_t *src,
2684 uint8_t *dst_tmp = dst;
2685 const uint8_t *src_tmp = src - (2 * stride) - 2;
2686 uint32_t multiple8_cnt, loop_cnt;
2687 const int32_t filt_const0 = 0xfffb0001;
2688 const int32_t filt_const1 = 0x140014;
2689 const int32_t filt_const2 = 0x1fffb;
2691 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask0, mask1;
2693 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2694 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2695 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2696 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
2697 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
2698 v8i16 hz_out87_l, filt0, filt1, filt2;
2701 filt0 = (v8i16) __msa_fill_w(filt_const0);
2702 filt1 = (v8i16) __msa_fill_w(filt_const1);
2703 filt2 = (v8i16) __msa_fill_w(filt_const2);
2705 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2707 for (multiple8_cnt = 2; multiple8_cnt--;) {
2711 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2712 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2713 src += (5 * stride);
2715 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2716 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2717 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2718 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2719 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2721 for (loop_cnt = 4; loop_cnt--;) {
2722 LD_SB4(src, stride, src5, src6, src7, src8);
2723 src += (4 * stride);
2725 XORI_B4_128_SB(src5, src6, src7, src8);
2727 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2728 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2729 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2730 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2732 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2733 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
2735 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2736 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
2738 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2739 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
2741 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2742 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
2745 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
2747 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
2749 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2750 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
2752 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
2754 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2755 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
2757 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
2759 dst4 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2760 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
2762 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
2764 dst6 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2766 dst1 = __msa_srari_h(hz_out2, 5);
2767 dst3 = __msa_srari_h(hz_out3, 5);
2768 dst5 = __msa_srari_h(hz_out4, 5);
2769 dst7 = __msa_srari_h(hz_out5, 5);
2770 SAT_SH4_SH(dst1, dst3, dst5, dst7, 7);
2772 dst0 = __msa_aver_s_h(dst0, dst1);
2773 dst1 = __msa_aver_s_h(dst2, dst3);
2774 dst2 = __msa_aver_s_h(dst4, dst5);
2775 dst3 = __msa_aver_s_h(dst6, dst7);
2777 out0 = PCKEV_XORI128_UB(dst0, dst1);
2778 out1 = PCKEV_XORI128_UB(dst2, dst3);
2779 ST8x4_UB(out0, out1, dst, stride);
2780 dst += (4 * stride);
2794 void ff_put_h264_qpel16_mc23_msa(uint8_t *dst, const uint8_t *src,
2797 uint8_t *dst_tmp = dst;
2798 const uint8_t *src_tmp = src - (2 * stride) - 2;
2799 uint32_t multiple8_cnt, loop_cnt;
2800 const int32_t filt_const0 = 0xfffb0001;
2801 const int32_t filt_const1 = 0x140014;
2802 const int32_t filt_const2 = 0x1fffb;
2804 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask0, mask1;
2806 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2807 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2808 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2809 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
2810 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
2811 v8i16 hz_out87_l, filt0, filt1, filt2;
2814 filt0 = (v8i16) __msa_fill_w(filt_const0);
2815 filt1 = (v8i16) __msa_fill_w(filt_const1);
2816 filt2 = (v8i16) __msa_fill_w(filt_const2);
2818 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2820 for (multiple8_cnt = 2; multiple8_cnt--;) {
2824 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2825 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2826 src += (5 * stride);
2828 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2829 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2830 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2831 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2832 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2834 for (loop_cnt = 4; loop_cnt--;) {
2835 LD_SB4(src, stride, src5, src6, src7, src8);
2836 src += (4 * stride);
2838 XORI_B4_128_SB(src5, src6, src7, src8);
2840 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2841 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2842 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2843 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2845 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2846 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
2848 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2849 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
2851 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2852 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
2854 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2855 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
2858 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
2860 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
2862 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2863 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
2865 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
2867 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2868 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
2870 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
2872 dst4 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2873 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
2875 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
2877 dst6 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2879 dst1 = __msa_srari_h(hz_out3, 5);
2880 dst3 = __msa_srari_h(hz_out4, 5);
2881 dst5 = __msa_srari_h(hz_out5, 5);
2882 dst7 = __msa_srari_h(hz_out6, 5);
2883 SAT_SH4_SH(dst1, dst3, dst5, dst7, 7);
2885 dst0 = __msa_aver_s_h(dst0, dst1);
2886 dst1 = __msa_aver_s_h(dst2, dst3);
2887 dst2 = __msa_aver_s_h(dst4, dst5);
2888 dst3 = __msa_aver_s_h(dst6, dst7);
2890 out0 = PCKEV_XORI128_UB(dst0, dst1);
2891 out1 = PCKEV_XORI128_UB(dst2, dst3);
2892 ST8x4_UB(out0, out1, dst, stride);
2893 dst += (4 * stride);
2907 void ff_put_h264_qpel8_mc21_msa(uint8_t *dst, const uint8_t *src,
2910 const int32_t filt_const0 = 0xfffb0001;
2911 const int32_t filt_const1 = 0x140014;
2912 const int32_t filt_const2 = 0x1fffb;
2914 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2915 v16i8 src11, src12, mask0, mask1, mask2;
2916 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2917 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
2918 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2919 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
2920 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
2921 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
2922 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
2923 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
2926 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2928 filt0 = (v8i16) __msa_fill_w(filt_const0);
2929 filt1 = (v8i16) __msa_fill_w(filt_const1);
2930 filt2 = (v8i16) __msa_fill_w(filt_const2);
2932 src -= ((2 * stride) + 2);
2934 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2935 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2936 src += (5 * stride);
2938 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2939 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2940 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2941 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2942 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2944 LD_SB4(src, stride, src5, src6, src7, src8);
2945 src += (4 * stride);
2946 XORI_B4_128_SB(src5, src6, src7, src8);
2948 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2949 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2950 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2951 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2953 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
2954 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
2955 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
2956 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
2957 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
2958 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
2959 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
2960 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
2962 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
2964 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
2966 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2967 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
2969 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
2971 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2972 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
2974 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
2976 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2977 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
2979 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
2981 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2983 SRARI_H4_SH(hz_out2, hz_out3, hz_out4, hz_out5, 5);
2984 SAT_SH4_SH(hz_out2, hz_out3, hz_out4, hz_out5, 7);
2986 dst0 = __msa_aver_s_h(dst0, hz_out2);
2987 dst1 = __msa_aver_s_h(dst1, hz_out3);
2988 dst2 = __msa_aver_s_h(dst2, hz_out4);
2989 dst3 = __msa_aver_s_h(dst3, hz_out5);
2991 out0 = PCKEV_XORI128_UB(dst0, dst1);
2992 out1 = PCKEV_XORI128_UB(dst2, dst3);
2993 ST8x4_UB(out0, out1, dst, stride);
2994 dst += (4 * stride);
2996 LD_SB4(src, stride, src9, src10, src11, src12);
2997 XORI_B4_128_SB(src9, src10, src11, src12);
2998 hz_out9 = AVC_HORZ_FILTER_SH(src9, src9, mask0, mask1, mask2);
2999 hz_out10 = AVC_HORZ_FILTER_SH(src10, src10, mask0, mask1, mask2);
3000 hz_out11 = AVC_HORZ_FILTER_SH(src11, src11, mask0, mask1, mask2);
3001 hz_out12 = AVC_HORZ_FILTER_SH(src12, src12, mask0, mask1, mask2);
3002 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3003 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
3005 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3006 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
3008 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
3010 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
3012 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3013 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
3015 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
3017 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3018 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
3020 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
3022 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3023 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
3025 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
3027 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3029 SRARI_H4_SH(hz_out6, hz_out7, hz_out8, hz_out9, 5);
3030 SAT_SH4_SH(hz_out6, hz_out7, hz_out8, hz_out9, 7);
3032 dst0 = __msa_aver_s_h(dst0, hz_out6);
3033 dst1 = __msa_aver_s_h(dst1, hz_out7);
3034 dst2 = __msa_aver_s_h(dst2, hz_out8);
3035 dst3 = __msa_aver_s_h(dst3, hz_out9);
3037 out0 = PCKEV_XORI128_UB(dst0, dst1);
3038 out1 = PCKEV_XORI128_UB(dst2, dst3);
3039 ST8x4_UB(out0, out1, dst, stride);
3042 void ff_put_h264_qpel8_mc23_msa(uint8_t *dst, const uint8_t *src,
3045 const int32_t filt_const0 = 0xfffb0001;
3046 const int32_t filt_const1 = 0x140014;
3047 const int32_t filt_const2 = 0x1fffb;
3049 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3050 v16i8 src11, src12, mask0, mask1, mask2;
3051 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3052 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
3053 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3054 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
3055 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
3056 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
3057 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
3058 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
3061 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
3063 filt0 = (v8i16) __msa_fill_w(filt_const0);
3064 filt1 = (v8i16) __msa_fill_w(filt_const1);
3065 filt2 = (v8i16) __msa_fill_w(filt_const2);
3067 src -= ((2 * stride) + 2);
3069 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3070 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3071 src += (5 * stride);
3073 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
3074 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
3075 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
3076 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
3077 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
3079 LD_SB4(src, stride, src5, src6, src7, src8);
3080 src += (4 * stride);
3081 XORI_B4_128_SB(src5, src6, src7, src8);
3083 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
3084 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
3085 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
3086 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3088 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3089 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3090 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3091 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
3092 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3093 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3094 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3095 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
3097 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3099 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
3101 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3102 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3104 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
3106 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3107 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3109 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
3111 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3112 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3114 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
3116 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3118 SRARI_H4_SH(hz_out3, hz_out4, hz_out5, hz_out6, 5);
3119 SAT_SH4_SH(hz_out3, hz_out4, hz_out5, hz_out6, 7);
3121 dst0 = __msa_aver_s_h(dst0, hz_out3);
3122 dst1 = __msa_aver_s_h(dst1, hz_out4);
3123 dst2 = __msa_aver_s_h(dst2, hz_out5);
3124 dst3 = __msa_aver_s_h(dst3, hz_out6);
3126 out0 = PCKEV_XORI128_UB(dst0, dst1);
3127 out1 = PCKEV_XORI128_UB(dst2, dst3);
3128 ST8x4_UB(out0, out1, dst, stride);
3129 dst += (4 * stride);
3131 LD_SB4(src, stride, src9, src10, src11, src12);
3132 XORI_B4_128_SB(src9, src10, src11, src12);
3133 hz_out9 = AVC_HORZ_FILTER_SH(src9, src9, mask0, mask1, mask2);
3134 hz_out10 = AVC_HORZ_FILTER_SH(src10, src10, mask0, mask1, mask2);
3135 hz_out11 = AVC_HORZ_FILTER_SH(src11, src11, mask0, mask1, mask2);
3136 hz_out12 = AVC_HORZ_FILTER_SH(src12, src12, mask0, mask1, mask2);
3137 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3138 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
3140 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3141 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
3143 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
3145 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
3147 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3148 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
3150 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
3152 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3153 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
3155 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
3157 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3158 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
3160 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
3162 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3164 SRARI_H4_SH(hz_out7, hz_out8, hz_out9, hz_out10, 5);
3165 SAT_SH4_SH(hz_out7, hz_out8, hz_out9, hz_out10, 7);
3167 dst0 = __msa_aver_s_h(dst0, hz_out7);
3168 dst1 = __msa_aver_s_h(dst1, hz_out8);
3169 dst2 = __msa_aver_s_h(dst2, hz_out9);
3170 dst3 = __msa_aver_s_h(dst3, hz_out10);
3172 out0 = PCKEV_XORI128_UB(dst0, dst1);
3173 out1 = PCKEV_XORI128_UB(dst2, dst3);
3174 ST8x4_UB(out0, out1, dst, stride);
3177 void ff_put_h264_qpel4_mc21_msa(uint8_t *dst, const uint8_t *src,
3180 const int32_t filt_const0 = 0xfffb0001;
3181 const int32_t filt_const1 = 0x140014;
3182 const int32_t filt_const2 = 0x1fffb;
3184 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3185 v16i8 mask0, mask1, mask2;
3186 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3187 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
3188 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3189 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
3192 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
3194 filt0 = (v8i16) __msa_fill_w(filt_const0);
3195 filt1 = (v8i16) __msa_fill_w(filt_const1);
3196 filt2 = (v8i16) __msa_fill_w(filt_const2);
3198 src -= ((2 * stride) + 2);
3200 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3201 src += (5 * stride);
3202 LD_SB4(src, stride, src5, src6, src7, src8);
3204 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3205 XORI_B4_128_SB(src5, src6, src7, src8);
3207 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
3208 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
3209 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
3210 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
3211 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3212 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
3213 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
3215 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3216 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3217 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3218 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3220 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3222 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3224 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3225 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3227 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3229 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3231 SRARI_H2_SH(hz_out2, hz_out4, 5);
3232 SAT_SH2_SH(hz_out2, hz_out4, 7);
3234 dst0 = __msa_aver_s_h(dst0, hz_out2);
3235 dst1 = __msa_aver_s_h(dst1, hz_out4);
3237 res = PCKEV_XORI128_UB(dst0, dst1);
3238 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
3241 void ff_put_h264_qpel4_mc23_msa(uint8_t *dst, const uint8_t *src,
3244 const int32_t filt_const0 = 0xfffb0001;
3245 const int32_t filt_const1 = 0x140014;
3246 const int32_t filt_const2 = 0x1fffb;
3248 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3249 v16i8 mask0, mask1, mask2;
3250 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3251 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
3252 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3253 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
3256 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
3258 filt0 = (v8i16) __msa_fill_w(filt_const0);
3259 filt1 = (v8i16) __msa_fill_w(filt_const1);
3260 filt2 = (v8i16) __msa_fill_w(filt_const2);
3262 src -= ((2 * stride) + 2);
3264 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3265 src += (5 * stride);
3266 LD_SB4(src, stride, src5, src6, src7, src8);
3268 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3269 XORI_B4_128_SB(src5, src6, src7, src8);
3271 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
3272 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
3273 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
3274 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
3275 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3276 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
3277 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
3279 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3280 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3281 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3282 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3284 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3286 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3288 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3289 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3291 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3293 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3295 PCKEV_D2_SH(hz_out4, hz_out3, hz_out6, hz_out5, hz_out0, hz_out1);
3296 SRARI_H2_SH(hz_out0, hz_out1, 5);
3297 SAT_SH2_SH(hz_out0, hz_out1, 7);
3299 dst0 = __msa_aver_s_h(dst0, hz_out0);
3300 dst1 = __msa_aver_s_h(dst1, hz_out1);
3302 res = PCKEV_XORI128_UB(dst0, dst1);
3303 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
3306 void ff_put_h264_qpel16_mc02_msa(uint8_t *dst, const uint8_t *src,
3310 int16_t filt_const0 = 0xfb01;
3311 int16_t filt_const1 = 0x1414;
3312 int16_t filt_const2 = 0x1fb;
3313 v16u8 res0, res1, res2, res3;
3314 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3315 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
3316 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
3317 v16i8 src65_l, src87_l, filt0, filt1, filt2;
3318 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
3320 filt0 = (v16i8) __msa_fill_h(filt_const0);
3321 filt1 = (v16i8) __msa_fill_h(filt_const1);
3322 filt2 = (v16i8) __msa_fill_h(filt_const2);
3323 src -= (stride * 2);
3325 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3326 src += (5 * stride);
3328 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3329 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3331 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3334 for (loop_cnt = 4; loop_cnt--;) {
3335 LD_SB4(src, stride, src5, src6, src7, src8);
3336 src += (4 * stride);
3338 XORI_B4_128_SB(src5, src6, src7, src8);
3339 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
3340 src65_r, src76_r, src87_r);
3341 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
3342 src65_l, src76_l, src87_l);
3343 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3344 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3345 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3346 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3347 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3348 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3349 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3350 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3351 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
3352 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3353 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
3354 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
3355 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
3356 out3_r, res0, res1, res2, res3);
3357 XORI_B4_128_UB(res0, res1, res2, res3);
3358 ST_UB4(res0, res1, res2, res3, dst, stride);
3359 dst += (4 * stride);
3373 void ff_put_h264_qpel8_mc02_msa(uint8_t *dst, const uint8_t *src,
3376 const int16_t filt_const0 = 0xfb01;
3377 const int16_t filt_const1 = 0x1414;
3378 const int16_t filt_const2 = 0x1fb;
3379 v16u8 out0, out1, out2, out3;
3380 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3381 v16i8 src11, src12, src10_r, src21_r, src32_r, src43_r, src76_r, src87_r;
3382 v16i8 src98_r, src109_r, src89_r, src910_r, src1110_r, src1211_r;
3383 v16i8 filt0, filt1, filt2;
3384 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
3386 filt0 = (v16i8) __msa_fill_h(filt_const0);
3387 filt1 = (v16i8) __msa_fill_h(filt_const1);
3388 filt2 = (v16i8) __msa_fill_h(filt_const2);
3390 src -= (stride * 2);
3392 LD_SB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
3393 src += (8 * stride);
3394 LD_SB5(src, stride, src8, src9, src10, src11, src12);
3395 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3397 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src76_r, src87_r,
3399 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src89_r,
3400 src910_r, src1110_r, src1211_r);
3401 XORI_B4_128_SB(src10_r, src21_r, src32_r, src43_r);
3402 XORI_B4_128_SB(src76_r, src87_r, src98_r, src109_r);
3403 XORI_B4_128_SB(src89_r, src910_r, src1110_r, src1211_r);
3404 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
3405 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
3406 out2_r = AVC_DOT_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
3407 out3_r = AVC_DOT_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
3408 out4_r = AVC_DOT_SH3_SH(src76_r, src98_r, src89_r, filt0, filt1, filt2);
3409 out5_r = AVC_DOT_SH3_SH(src87_r, src109_r, src910_r, filt0, filt1, filt2);
3410 out6_r = AVC_DOT_SH3_SH(src98_r, src89_r, src1110_r, filt0, filt1, filt2);
3411 out7_r = AVC_DOT_SH3_SH(src109_r, src910_r, src1211_r, filt0, filt1, filt2);
3412 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
3413 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
3414 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3415 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
3416 out0 = PCKEV_XORI128_UB(out0_r, out1_r);
3417 out1 = PCKEV_XORI128_UB(out2_r, out3_r);
3418 out2 = PCKEV_XORI128_UB(out4_r, out5_r);
3419 out3 = PCKEV_XORI128_UB(out6_r, out7_r);
3420 ST8x8_UB(out0, out1, out2, out3, dst, stride);
3423 void ff_put_h264_qpel4_mc02_msa(uint8_t *dst, const uint8_t *src,
3426 const int16_t filt_const0 = 0xfb01;
3427 const int16_t filt_const1 = 0x1414;
3428 const int16_t filt_const2 = 0x1fb;
3430 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3431 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
3432 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
3435 filt0 = (v16i8) __msa_fill_h(filt_const0);
3436 filt1 = (v16i8) __msa_fill_h(filt_const1);
3437 filt2 = (v16i8) __msa_fill_h(filt_const2);
3439 src -= (stride * 2);
3441 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3442 src += (5 * stride);
3443 LD_SB4(src, stride, src5, src6, src7, src8);
3445 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3447 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3449 ILVR_D4_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src87_r,
3450 src76_r, src2110, src4332, src6554, src8776);
3451 XORI_B4_128_SB(src2110, src4332, src6554, src8776);
3452 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
3453 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
3454 SRARI_H2_SH(out10, out32, 5);
3455 SAT_SH2_SH(out10, out32, 7);
3456 out = PCKEV_XORI128_UB(out10, out32);
3457 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3460 void ff_put_h264_qpel16_mc12_msa(uint8_t *dst, const uint8_t *src,
3465 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3467 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, mask3;
3468 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3469 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11, mask4, mask5;
3470 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3471 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3472 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3473 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3474 v8i16 minus5h = __msa_ldi_h(-5);
3475 v8i16 plus20h = __msa_ldi_h(20);
3481 src -= ((2 * stride) + 2);
3483 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3484 LD_SB5(src + 8, stride, src7, src8, src9, src10, src11);
3485 src += (5 * stride);
3486 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3487 XORI_B5_128_SB(src7, src8, src9, src10, src11);
3489 for (row = 16; row--;) {
3490 LD_SB2(src, 8, src5, src6);
3492 XORI_B2_128_SB(src5, src6);
3494 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3496 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src7, src8, src9, src10, src11, src6,
3498 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3499 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3500 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3501 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3502 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3503 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3504 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3505 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3506 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3507 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3508 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3509 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3510 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3511 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3512 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3513 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3514 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3515 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3516 dst0 = __msa_srari_h(shf_vec2, 5);
3517 dst1 = __msa_srari_h(shf_vec5, 5);
3518 dst2 = __msa_srari_h(shf_vec8, 5);
3519 dst3 = __msa_srari_h(shf_vec11, 5);
3520 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3521 PCKEV_H2_SH(dst2, dst0, dst3, dst1, dst0, dst1);
3522 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3523 dst0 = __msa_aver_s_h(dst2, dst0);
3524 dst1 = __msa_aver_s_h(dst3, dst1);
3525 out = PCKEV_XORI128_UB(dst0, dst1);
3542 void ff_put_h264_qpel16_mc32_msa(uint8_t *dst, const uint8_t *src,
3547 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3549 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, mask3;
3550 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3551 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11, mask4, mask5;
3552 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3553 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3554 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3555 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3556 v8i16 minus5h = __msa_ldi_h(-5);
3557 v8i16 plus20h = __msa_ldi_h(20);
3563 src -= ((2 * stride) + 2);
3565 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3566 LD_SB5(src + 8, stride, src7, src8, src9, src10, src11);
3567 src += (5 * stride);
3568 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3569 XORI_B5_128_SB(src7, src8, src9, src10, src11);
3571 for (row = 16; row--;) {
3572 LD_SB2(src, 8, src5, src6);
3574 XORI_B2_128_SB(src5, src6);
3576 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3578 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src7, src8, src9, src10, src11, src6,
3580 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3581 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3582 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3583 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3584 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3585 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3586 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3587 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3588 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3589 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3590 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3591 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3592 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3593 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3594 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3595 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3596 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3597 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3598 dst0 = __msa_srari_h(shf_vec2, 5);
3599 dst1 = __msa_srari_h(shf_vec5, 5);
3600 dst2 = __msa_srari_h(shf_vec8, 5);
3601 dst3 = __msa_srari_h(shf_vec11, 5);
3602 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3603 dst0 = __msa_pckod_h(dst2, dst0);
3604 dst1 = __msa_pckod_h(dst3, dst1);
3605 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3606 dst0 = __msa_aver_s_h(dst2, dst0);
3607 dst1 = __msa_aver_s_h(dst3, dst1);
3608 out = PCKEV_XORI128_UB(dst0, dst1);
3625 void ff_put_h264_qpel8_mc12_msa(uint8_t *dst, const uint8_t *src,
3630 v16i8 src0, src1, src2, src3, src4, src5, src6;
3631 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3;
3632 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3633 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11;
3634 v8i16 mask3, mask4, mask5;
3635 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3636 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3637 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3638 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3639 v8i16 minus5h = __msa_ldi_h(-5);
3640 v8i16 plus20h = __msa_ldi_h(20);
3646 src -= ((2 * stride) + 2);
3648 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3649 src += (5 * stride);
3650 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3652 for (row = 4; row--;) {
3653 LD_SB2(src, stride, src5, src6);
3654 src += (2 * stride);
3655 XORI_B2_128_SB(src5, src6);
3657 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3659 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
3661 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3662 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3663 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3664 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3665 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3666 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3667 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3668 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3669 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3670 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3671 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3672 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3673 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3674 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3675 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3676 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3677 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3678 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3679 dst0 = __msa_srari_h(shf_vec2, 5);
3680 dst1 = __msa_srari_h(shf_vec5, 5);
3681 dst2 = __msa_srari_h(shf_vec8, 5);
3682 dst3 = __msa_srari_h(shf_vec11, 5);
3683 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3684 PCKEV_H2_SH(dst2, dst0, dst3, dst1, dst0, dst1);
3685 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3686 dst0 = __msa_aver_s_h(dst2, dst0);
3687 dst1 = __msa_aver_s_h(dst3, dst1);
3688 out = PCKEV_XORI128_UB(dst0, dst1);
3689 ST8x2_UB(out, dst, stride);
3690 dst += (2 * stride);
3700 void ff_put_h264_qpel8_mc32_msa(uint8_t *dst, const uint8_t *src,
3705 v16i8 src0, src1, src2, src3, src4, src5, src6;
3706 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3;
3707 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3708 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11;
3709 v8i16 mask3, mask4, mask5;
3710 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3711 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3712 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3713 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3714 v8i16 minus5h = __msa_ldi_h(-5);
3715 v8i16 plus20h = __msa_ldi_h(20);
3721 src -= ((2 * stride) + 2);
3723 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3724 src += (5 * stride);
3725 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3727 for (row = 4; row--;) {
3728 LD_SB2(src, stride, src5, src6);
3729 src += (2 * stride);
3730 XORI_B2_128_SB(src5, src6);
3732 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3734 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
3736 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3737 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3738 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3739 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3740 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3741 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3742 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3743 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3744 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3745 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3746 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3747 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3748 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3749 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3750 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3751 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3752 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3753 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3754 dst0 = __msa_srari_h(shf_vec2, 5);
3755 dst1 = __msa_srari_h(shf_vec5, 5);
3756 dst2 = __msa_srari_h(shf_vec8, 5);
3757 dst3 = __msa_srari_h(shf_vec11, 5);
3758 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3759 dst0 = __msa_pckod_h(dst2, dst0);
3760 dst1 = __msa_pckod_h(dst3, dst1);
3761 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3762 dst0 = __msa_aver_s_h(dst2, dst0);
3763 dst1 = __msa_aver_s_h(dst3, dst1);
3764 out = PCKEV_XORI128_UB(dst0, dst1);
3765 ST8x2_UB(out, dst, stride);
3766 dst += (2 * stride);
3776 void ff_put_h264_qpel4_mc12_msa(uint8_t *dst, const uint8_t *src,
3779 const int16_t filt_const0 = 0xfb01;
3780 const int16_t filt_const1 = 0x1414;
3781 const int16_t filt_const2 = 0x1fb;
3783 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3784 v16i8 src10_r, src21_r, src32_r, src43_r, src54_r, src65_r, src76_r;
3785 v16i8 src87_r, src10_l, src21_l, src32_l, src43_l, src54_l, src65_l;
3786 v16i8 src76_l, src87_l, filt0, filt1, filt2;
3787 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, shf_vec7;
3788 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3789 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3790 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3791 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3792 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3793 v8i16 minus5h = __msa_ldi_h(-5);
3794 v8i16 plus20h = __msa_ldi_h(20);
3795 v8i16 zeros = { 0 };
3797 filt0 = (v16i8) __msa_fill_h(filt_const0);
3798 filt1 = (v16i8) __msa_fill_h(filt_const1);
3799 filt2 = (v16i8) __msa_fill_h(filt_const2);
3801 src -= ((2 * stride) + 2);
3803 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3804 src += (5 * stride);
3805 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3806 LD_SB4(src, stride, src5, src6, src7, src8);
3807 XORI_B4_128_SB(src5, src6, src7, src8);
3809 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3811 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3813 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3815 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l, src65_l,
3817 vt_res0 = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3818 vt_res1 = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3819 vt_res2 = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3820 vt_res3 = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3821 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3822 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3823 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3824 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3825 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3826 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3827 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3828 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3830 vt_res0 = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3831 vt_res1 = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3832 vt_res2 = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3833 vt_res3 = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3834 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3835 mask1, mask2, shf_vec0, shf_vec1, shf_vec6);
3836 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3837 mask1, mask2, shf_vec3, shf_vec4, shf_vec7);
3838 hz_res2 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3839 DPADD_SH2_SW(shf_vec1, shf_vec6, minus5h, plus20h, hz_res2, hz_res2);
3840 hz_res3 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3841 DPADD_SH2_SW(shf_vec4, shf_vec7, minus5h, plus20h, hz_res3, hz_res3);
3843 SRARI_W2_SW(hz_res0, hz_res1, 10);
3844 SAT_SW2_SW(hz_res0, hz_res1, 7);
3845 SRARI_W2_SW(hz_res2, hz_res3, 10);
3846 SAT_SW2_SW(hz_res2, hz_res3, 7);
3848 dst0 = __msa_srari_h(shf_vec2, 5);
3849 dst1 = __msa_srari_h(shf_vec5, 5);
3850 dst2 = __msa_srari_h(shf_vec6, 5);
3851 dst3 = __msa_srari_h(shf_vec7, 5);
3853 SAT_SH2_SH(dst0, dst1, 7);
3854 SAT_SH2_SH(dst2, dst3, 7);
3855 ILVEV_H2_SH(dst0, zeros, dst1, zeros, dst0, dst1);
3856 ILVEV_H2_SH(dst2, zeros, dst3, zeros, dst2, dst3);
3858 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) dst0);
3859 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) dst1);
3860 hz_res2 = __msa_aver_s_w(hz_res2, (v4i32) dst2);
3861 hz_res3 = __msa_aver_s_w(hz_res3, (v4i32) dst3);
3863 PCKEV_H2_SH(hz_res1, hz_res0, hz_res3, hz_res2, dst0, dst2);
3864 out = PCKEV_XORI128_UB(dst0, dst2);
3865 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3868 void ff_put_h264_qpel4_mc32_msa(uint8_t *dst, const uint8_t *src,
3871 const int16_t filt_const0 = 0xfb01;
3872 const int16_t filt_const1 = 0x1414;
3873 const int16_t filt_const2 = 0x1fb;
3875 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3876 v16i8 src10_r, src21_r, src32_r, src43_r, src54_r, src65_r, src76_r;
3877 v16i8 src87_r, src10_l, src21_l, src32_l, src43_l, src54_l, src65_l;
3878 v16i8 src76_l, src87_l, filt0, filt1, filt2;
3879 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, shf_vec7;
3880 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3881 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3882 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3883 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3884 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3885 v8i16 minus5h = __msa_ldi_h(-5);
3886 v8i16 plus20h = __msa_ldi_h(20);
3887 v8i16 zeros = { 0 };
3889 filt0 = (v16i8) __msa_fill_h(filt_const0);
3890 filt1 = (v16i8) __msa_fill_h(filt_const1);
3891 filt2 = (v16i8) __msa_fill_h(filt_const2);
3893 src -= ((2 * stride) + 2);
3895 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3896 src += (5 * stride);
3897 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3898 LD_SB4(src, stride, src5, src6, src7, src8);
3899 XORI_B4_128_SB(src5, src6, src7, src8);
3901 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3903 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3905 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3907 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l, src65_l,
3910 vt_res0 = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3911 vt_res1 = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3912 vt_res2 = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3913 vt_res3 = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3914 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3915 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3916 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3917 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3918 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3919 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3920 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3921 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3923 vt_res0 = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3924 vt_res1 = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3925 vt_res2 = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3926 vt_res3 = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3927 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3928 mask1, mask2, shf_vec0, shf_vec1, shf_vec6);
3929 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3930 mask1, mask2, shf_vec3, shf_vec4, shf_vec7);
3931 hz_res2 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3932 DPADD_SH2_SW(shf_vec1, shf_vec6, minus5h, plus20h, hz_res2, hz_res2);
3933 hz_res3 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3934 DPADD_SH2_SW(shf_vec4, shf_vec7, minus5h, plus20h, hz_res3, hz_res3);
3936 SRARI_W2_SW(hz_res0, hz_res1, 10);
3937 SAT_SW2_SW(hz_res0, hz_res1, 7);
3938 SRARI_W2_SW(hz_res2, hz_res3, 10);
3939 SAT_SW2_SW(hz_res2, hz_res3, 7);
3941 dst0 = __msa_srari_h(shf_vec2, 5);
3942 dst1 = __msa_srari_h(shf_vec5, 5);
3943 dst2 = __msa_srari_h(shf_vec6, 5);
3944 dst3 = __msa_srari_h(shf_vec7, 5);
3946 SAT_SH2_SH(dst0, dst1, 7);
3947 SAT_SH2_SH(dst2, dst3, 7);
3949 dst0 = __msa_ilvod_h(zeros, dst0);
3950 dst1 = __msa_ilvod_h(zeros, dst1);
3951 dst2 = __msa_ilvod_h(zeros, dst2);
3952 dst3 = __msa_ilvod_h(zeros, dst3);
3954 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) dst0);
3955 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) dst1);
3956 hz_res2 = __msa_aver_s_w(hz_res2, (v4i32) dst2);
3957 hz_res3 = __msa_aver_s_w(hz_res3, (v4i32) dst3);
3959 PCKEV_H2_SH(hz_res1, hz_res0, hz_res3, hz_res2, dst0, dst2);
3960 out = PCKEV_XORI128_UB(dst0, dst2);
3961 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3964 void ff_put_h264_qpel16_mc22_msa(uint8_t *dst, const uint8_t *src,
3967 const int32_t filt_const0 = 0xfffb0001;
3968 const int32_t filt_const1 = 0x140014;
3969 const int32_t filt_const2 = 0x1fffb;
3970 const uint8_t *src_tmp = src - (2 * stride) - 2;
3971 uint8_t *dst_tmp = dst;
3972 uint32_t multiple8_cnt, loop_cnt;
3974 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
3975 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3976 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3;
3977 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3978 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
3979 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
3980 v8i16 hz_out87_l, filt0, filt1, filt2;
3983 filt0 = (v8i16) __msa_fill_w(filt_const0);
3984 filt1 = (v8i16) __msa_fill_w(filt_const1);
3985 filt2 = (v8i16) __msa_fill_w(filt_const2);
3987 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
3989 for (multiple8_cnt = 2; multiple8_cnt--;) {
3993 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3994 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3995 src += (5 * stride);
3997 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
3998 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
3999 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4000 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4001 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4003 for (loop_cnt = 4; loop_cnt--;) {
4004 LD_SB4(src, stride, src0, src1, src2, src3);
4005 XORI_B4_128_SB(src0, src1, src2, src3);
4006 src += (4 * stride);
4008 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4009 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4010 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4011 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4013 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4014 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
4016 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4017 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
4019 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4020 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
4022 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4023 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
4026 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
4028 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
4030 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4031 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
4033 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
4035 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4036 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
4038 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
4040 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4041 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
4043 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
4045 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4047 out0 = PCKEV_XORI128_UB(dst0, dst1);
4048 out1 = PCKEV_XORI128_UB(dst2, dst3);
4049 ST8x4_UB(out0, out1, dst, stride);
4050 dst += (4 * stride);
4064 void ff_put_h264_qpel8_mc22_msa(uint8_t *dst, const uint8_t *src,
4067 const int32_t filt_const0 = 0xfffb0001;
4068 const int32_t filt_const1 = 0x140014;
4069 const int32_t filt_const2 = 0x1fffb;
4071 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
4072 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4073 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
4074 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4075 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
4076 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
4077 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
4078 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
4079 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
4082 filt0 = (v8i16) __msa_fill_w(filt_const0);
4083 filt1 = (v8i16) __msa_fill_w(filt_const1);
4084 filt2 = (v8i16) __msa_fill_w(filt_const2);
4086 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
4088 src -= ((2 * stride) + 2);
4089 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4090 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4091 src += (5 * stride);
4093 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4094 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4095 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4096 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4097 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4099 LD_SB4(src, stride, src0, src1, src2, src3);
4100 XORI_B4_128_SB(src0, src1, src2, src3);
4101 src += (4 * stride);
4102 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4103 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4104 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4105 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4106 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4107 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4108 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4109 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
4110 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4111 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4112 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4113 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
4115 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4117 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
4119 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4120 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4122 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
4124 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4125 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4127 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
4129 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4130 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4132 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
4134 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4135 out0 = PCKEV_XORI128_UB(dst0, dst1);
4136 out1 = PCKEV_XORI128_UB(dst2, dst3);
4137 ST8x4_UB(out0, out1, dst, stride);
4138 dst += (4 * stride);
4140 LD_SB4(src, stride, src0, src1, src2, src3);
4141 XORI_B4_128_SB(src0, src1, src2, src3);
4142 hz_out9 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4143 hz_out10 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4144 hz_out11 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4145 hz_out12 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4146 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4147 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
4149 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4150 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
4152 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
4154 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
4156 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4157 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
4159 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
4161 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4162 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
4164 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
4166 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4167 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
4169 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
4171 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4172 out0 = PCKEV_XORI128_UB(dst0, dst1);
4173 out1 = PCKEV_XORI128_UB(dst2, dst3);
4174 ST8x4_UB(out0, out1, dst, stride);
4177 void ff_put_h264_qpel4_mc22_msa(uint8_t *dst, const uint8_t *src,
4180 const int32_t filt_const0 = 0xfffb0001;
4181 const int32_t filt_const1 = 0x140014;
4182 const int32_t filt_const2 = 0x1fffb;
4184 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
4185 v16i8 mask0, mask1, mask2;
4186 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4187 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
4188 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4189 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
4192 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
4194 filt0 = (v8i16) __msa_fill_w(filt_const0);
4195 filt1 = (v8i16) __msa_fill_w(filt_const1);
4196 filt2 = (v8i16) __msa_fill_w(filt_const2);
4198 src -= ((2 * stride) + 2);
4200 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4201 src += (5 * stride);
4202 LD_SB4(src, stride, src5, src6, src7, src8);
4204 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4205 XORI_B4_128_SB(src5, src6, src7, src8);
4206 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
4207 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
4208 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
4209 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
4210 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
4211 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
4212 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
4213 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4214 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4215 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4216 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4218 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4220 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4222 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4223 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4225 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4227 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4228 res = PCKEV_XORI128_UB(dst0, dst1);
4229 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
4232 void ff_avg_h264_qpel16_mc10_msa(uint8_t *dst, const uint8_t *src,
4235 avc_luma_hz_qrt_and_aver_dst_16x16_msa(src - 2, stride, dst, stride, 0);
4238 void ff_avg_h264_qpel16_mc30_msa(uint8_t *dst, const uint8_t *src,
4241 avc_luma_hz_qrt_and_aver_dst_16x16_msa(src - 2, stride, dst, stride, 1);
4244 void ff_avg_h264_qpel8_mc10_msa(uint8_t *dst, const uint8_t *src,
4247 avc_luma_hz_qrt_and_aver_dst_8x8_msa(src - 2, stride, dst, stride, 0);
4250 void ff_avg_h264_qpel8_mc30_msa(uint8_t *dst, const uint8_t *src,
4253 avc_luma_hz_qrt_and_aver_dst_8x8_msa(src - 2, stride, dst, stride, 1);
4256 void ff_avg_h264_qpel4_mc10_msa(uint8_t *dst, const uint8_t *src,
4259 avc_luma_hz_qrt_and_aver_dst_4x4_msa(src - 2, stride, dst, stride, 0);
4262 void ff_avg_h264_qpel4_mc30_msa(uint8_t *dst, const uint8_t *src,
4265 avc_luma_hz_qrt_and_aver_dst_4x4_msa(src - 2, stride, dst, stride, 1);
4268 void ff_avg_h264_qpel16_mc20_msa(uint8_t *dst, const uint8_t *src,
4271 avc_luma_hz_and_aver_dst_16x16_msa(src - 2, stride, dst, stride);
4274 void ff_avg_h264_qpel8_mc20_msa(uint8_t *dst, const uint8_t *src,
4277 avc_luma_hz_and_aver_dst_8x8_msa(src - 2, stride, dst, stride);
4280 void ff_avg_h264_qpel4_mc20_msa(uint8_t *dst, const uint8_t *src,
4283 avc_luma_hz_and_aver_dst_4x4_msa(src - 2, stride, dst, stride);
4286 void ff_avg_h264_qpel16_mc01_msa(uint8_t *dst, const uint8_t *src,
4289 avc_luma_vt_qrt_and_aver_dst_16x16_msa(src - (stride * 2),
4290 stride, dst, stride, 0);
4293 void ff_avg_h264_qpel16_mc03_msa(uint8_t *dst, const uint8_t *src,
4296 avc_luma_vt_qrt_and_aver_dst_16x16_msa(src - (stride * 2),
4297 stride, dst, stride, 1);
4300 void ff_avg_h264_qpel8_mc01_msa(uint8_t *dst, const uint8_t *src,
4303 avc_luma_vt_qrt_and_aver_dst_8x8_msa(src - (stride * 2),
4304 stride, dst, stride, 0);
4307 void ff_avg_h264_qpel8_mc03_msa(uint8_t *dst, const uint8_t *src,
4310 avc_luma_vt_qrt_and_aver_dst_8x8_msa(src - (stride * 2),
4311 stride, dst, stride, 1);
4314 void ff_avg_h264_qpel4_mc01_msa(uint8_t *dst, const uint8_t *src,
4317 avc_luma_vt_qrt_and_aver_dst_4x4_msa(src - (stride * 2),
4318 stride, dst, stride, 0);
4321 void ff_avg_h264_qpel4_mc03_msa(uint8_t *dst, const uint8_t *src,
4324 avc_luma_vt_qrt_and_aver_dst_4x4_msa(src - (stride * 2),
4325 stride, dst, stride, 1);
4328 void ff_avg_h264_qpel16_mc11_msa(uint8_t *dst, const uint8_t *src,
4331 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src - 2,
4336 void ff_avg_h264_qpel16_mc31_msa(uint8_t *dst, const uint8_t *src,
4339 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src - 2,
4340 src - (stride * 2) +
4345 void ff_avg_h264_qpel16_mc13_msa(uint8_t *dst, const uint8_t *src,
4348 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src + stride - 2,
4353 void ff_avg_h264_qpel16_mc33_msa(uint8_t *dst, const uint8_t *src,
4356 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src + stride - 2,
4357 src - (stride * 2) +
4362 void ff_avg_h264_qpel8_mc11_msa(uint8_t *dst, const uint8_t *src,
4365 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src - 2,
4370 void ff_avg_h264_qpel8_mc31_msa(uint8_t *dst, const uint8_t *src,
4373 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src - 2,
4374 src - (stride * 2) +
4375 sizeof(uint8_t), dst, stride);
4378 void ff_avg_h264_qpel8_mc13_msa(uint8_t *dst, const uint8_t *src,
4381 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src + stride - 2,
4386 void ff_avg_h264_qpel8_mc33_msa(uint8_t *dst, const uint8_t *src,
4389 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src + stride - 2,
4390 src - (stride * 2) +
4391 sizeof(uint8_t), dst, stride);
4395 void ff_avg_h264_qpel4_mc11_msa(uint8_t *dst, const uint8_t *src,
4398 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src - 2,
4403 void ff_avg_h264_qpel4_mc31_msa(uint8_t *dst, const uint8_t *src,
4406 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src - 2,
4407 src - (stride * 2) +
4408 sizeof(uint8_t), dst, stride);
4411 void ff_avg_h264_qpel4_mc13_msa(uint8_t *dst, const uint8_t *src,
4414 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src + stride - 2,
4419 void ff_avg_h264_qpel4_mc33_msa(uint8_t *dst, const uint8_t *src,
4422 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src + stride - 2,
4423 src - (stride * 2) +
4424 sizeof(uint8_t), dst, stride);
4427 void ff_avg_h264_qpel16_mc21_msa(uint8_t *dst, const uint8_t *src,
4430 avc_luma_midv_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4431 stride, dst, stride, 16, 0);
4434 void ff_avg_h264_qpel16_mc23_msa(uint8_t *dst, const uint8_t *src,
4437 avc_luma_midv_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4438 stride, dst, stride, 16, 1);
4441 void ff_avg_h264_qpel8_mc21_msa(uint8_t *dst, const uint8_t *src,
4444 avc_luma_midv_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4445 stride, dst, stride, 8, 0);
4448 void ff_avg_h264_qpel8_mc23_msa(uint8_t *dst, const uint8_t *src,
4451 avc_luma_midv_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4452 stride, dst, stride, 8, 1);
4455 void ff_avg_h264_qpel4_mc21_msa(uint8_t *dst, const uint8_t *src,
4458 avc_luma_midv_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4459 stride, dst, stride, 4, 0);
4462 void ff_avg_h264_qpel4_mc23_msa(uint8_t *dst, const uint8_t *src,
4465 avc_luma_midv_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4466 stride, dst, stride, 4, 1);
4469 void ff_avg_h264_qpel16_mc02_msa(uint8_t *dst, const uint8_t *src,
4472 avc_luma_vt_and_aver_dst_16x16_msa(src - (stride * 2), stride, dst, stride);
4475 void ff_avg_h264_qpel8_mc02_msa(uint8_t *dst, const uint8_t *src,
4478 avc_luma_vt_and_aver_dst_8x8_msa(src - (stride * 2), stride, dst, stride);
4481 void ff_avg_h264_qpel4_mc02_msa(uint8_t *dst, const uint8_t *src,
4484 avc_luma_vt_and_aver_dst_4x4_msa(src - (stride * 2), stride, dst, stride);
4487 void ff_avg_h264_qpel16_mc12_msa(uint8_t *dst, const uint8_t *src,
4490 avc_luma_midh_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4491 stride, dst, stride, 16, 0);
4494 void ff_avg_h264_qpel16_mc32_msa(uint8_t *dst, const uint8_t *src,
4497 avc_luma_midh_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4498 stride, dst, stride, 16, 1);
4501 void ff_avg_h264_qpel8_mc12_msa(uint8_t *dst, const uint8_t *src,
4504 avc_luma_midh_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4505 stride, dst, stride, 8, 0);
4508 void ff_avg_h264_qpel8_mc32_msa(uint8_t *dst, const uint8_t *src,
4511 avc_luma_midh_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4512 stride, dst, stride, 8, 1);
4515 void ff_avg_h264_qpel4_mc12_msa(uint8_t *dst, const uint8_t *src,
4518 avc_luma_midh_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4519 stride, dst, stride, 4, 0);
4522 void ff_avg_h264_qpel4_mc32_msa(uint8_t *dst, const uint8_t *src,
4525 avc_luma_midh_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4526 stride, dst, stride, 4, 1);
4529 void ff_avg_h264_qpel16_mc22_msa(uint8_t *dst, const uint8_t *src,
4532 const int32_t filt_const0 = 0xfffb0001;
4533 const int32_t filt_const1 = 0x140014;
4534 const int32_t filt_const2 = 0x1fffb;
4535 const uint8_t *src_tmp = src - (2 * stride) - 2;
4536 uint8_t *dst_tmp = dst;
4537 uint64_t tp0, tp1, tp2, tp3;
4538 uint32_t multiple8_cnt, loop_cnt;
4539 v16u8 dst0, dst1, out0, out1;
4540 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
4541 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4542 v8i16 hz_out7, hz_out8, res0, res1, res2, res3;
4543 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4544 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
4545 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
4546 v8i16 hz_out87_l, filt0, filt1, filt2;
4549 filt0 = (v8i16) __msa_fill_w(filt_const0);
4550 filt1 = (v8i16) __msa_fill_w(filt_const1);
4551 filt2 = (v8i16) __msa_fill_w(filt_const2);
4553 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
4555 for (multiple8_cnt = 2; multiple8_cnt--;) {
4559 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4560 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4561 src += (5 * stride);
4563 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4564 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4565 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4566 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4567 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4569 for (loop_cnt = 4; loop_cnt--;) {
4570 LD_SB4(src, stride, src0, src1, src2, src3);
4571 XORI_B4_128_SB(src0, src1, src2, src3);
4572 src += (4 * stride);
4574 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4575 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4576 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4577 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4578 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4579 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
4581 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4582 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
4584 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4585 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
4587 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4588 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
4591 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
4593 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
4595 res0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4596 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
4598 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
4600 res1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4601 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
4603 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
4605 res2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4606 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
4608 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
4610 res3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4612 LD4(dst, stride, tp0, tp1, tp2, tp3);
4613 INSERT_D2_UB(tp0, tp1, dst0);
4614 INSERT_D2_UB(tp2, tp3, dst1);
4615 out0 = PCKEV_XORI128_UB(res0, res1);
4616 out1 = PCKEV_XORI128_UB(res2, res3);
4617 AVER_UB2_UB(out0, dst0, out1, dst1, out0, out1);
4618 ST8x4_UB(out0, out1, dst, stride);
4619 dst += (4 * stride);
4633 void ff_avg_h264_qpel8_mc22_msa(uint8_t *dst, const uint8_t *src,
4636 const int32_t filt_const0 = 0xfffb0001;
4637 const int32_t filt_const1 = 0x140014;
4638 const int32_t filt_const2 = 0x1fffb;
4639 uint64_t tp0, tp1, tp2, tp3;
4640 v16u8 out0, out1, dst0 = { 0 }, dst1 = { 0 };
4641 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
4642 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4643 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
4644 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4645 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
4646 v8i16 hz_out1110_r, hz_out1211_r, res0, res1, res2, res3;
4647 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
4648 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
4649 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
4652 filt0 = (v8i16) __msa_fill_w(filt_const0);
4653 filt1 = (v8i16) __msa_fill_w(filt_const1);
4654 filt2 = (v8i16) __msa_fill_w(filt_const2);
4656 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
4658 src -= ((2 * stride) + 2);
4659 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4660 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4661 src += (5 * stride);
4663 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4664 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4665 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4666 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4667 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4669 LD_SB4(src, stride, src0, src1, src2, src3);
4670 XORI_B4_128_SB(src0, src1, src2, src3);
4671 src += (4 * stride);
4672 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4673 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4674 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4675 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4676 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4677 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4678 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4679 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
4680 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4681 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4682 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4683 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
4685 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4687 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
4689 res0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4690 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4692 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
4694 res1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4695 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4697 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
4699 res2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4700 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4702 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
4704 res3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4705 LD4(dst, stride, tp0, tp1, tp2, tp3);
4706 INSERT_D2_UB(tp0, tp1, dst0);
4707 INSERT_D2_UB(tp2, tp3, dst1);
4708 out0 = PCKEV_XORI128_UB(res0, res1);
4709 out1 = PCKEV_XORI128_UB(res2, res3);
4710 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
4711 ST8x4_UB(dst0, dst1, dst, stride);
4712 dst += (4 * stride);
4714 LD_SB4(src, stride, src0, src1, src2, src3);
4715 XORI_B4_128_SB(src0, src1, src2, src3);
4716 hz_out9 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4717 hz_out10 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4718 hz_out11 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4719 hz_out12 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4720 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4721 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
4723 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4724 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
4726 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
4728 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
4730 res0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4731 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
4733 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
4735 res1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4736 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
4738 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
4740 res2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4741 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
4743 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
4745 res3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4746 LD4(dst, stride, tp0, tp1, tp2, tp3);
4747 INSERT_D2_UB(tp0, tp1, dst0);
4748 INSERT_D2_UB(tp2, tp3, dst1);
4749 out0 = PCKEV_XORI128_UB(res0, res1);
4750 out1 = PCKEV_XORI128_UB(res2, res3);
4751 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
4752 ST8x4_UB(dst0, dst1, dst, stride);
4755 void ff_avg_h264_qpel4_mc22_msa(uint8_t *dst, const uint8_t *src,
4758 const int32_t filt_const0 = 0xfffb0001;
4759 const int32_t filt_const1 = 0x140014;
4760 const int32_t filt_const2 = 0x1fffb;
4761 uint32_t tp0, tp1, tp2, tp3;
4762 v16u8 res, dst0 = { 0 };
4763 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
4764 v16i8 mask0, mask1, mask2;
4765 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4766 v8i16 hz_out7, hz_out8, res0, res1, filt0, filt1, filt2;
4767 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4768 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
4771 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
4773 filt0 = (v8i16) __msa_fill_w(filt_const0);
4774 filt1 = (v8i16) __msa_fill_w(filt_const1);
4775 filt2 = (v8i16) __msa_fill_w(filt_const2);
4777 src -= ((2 * stride) + 2);
4779 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4780 src += (5 * stride);
4781 LD_SB4(src, stride, src5, src6, src7, src8);
4783 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4784 XORI_B4_128_SB(src5, src6, src7, src8);
4785 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
4786 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
4787 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
4788 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
4789 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
4790 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
4791 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
4792 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4793 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4794 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4795 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4797 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4799 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4801 res0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4802 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4804 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4806 res1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4807 LW4(dst, stride, tp0, tp1, tp2, tp3);
4808 INSERT_W4_UB(tp0, tp1, tp2, tp3, dst0);
4809 res = PCKEV_XORI128_UB(res0, res1);
4810 res = __msa_aver_u_b(res, dst0);
4811 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);