2 * Copyright (c) 2015 -2017 Parag Salasakar (Parag.Salasakar@imgtec.com)
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "libavutil/mips/generic_macros_msa.h"
22 #include "h264dsp_mips.h"
24 #define AVC_CALC_DPADD_H_6PIX_2COEFF_SH(in0, in1, in2, in3, in4, in5) \
26 v4i32 tmp0_m, tmp1_m; \
27 v8i16 out0_m, out1_m, out2_m, out3_m; \
28 v8i16 minus5h_m = __msa_ldi_h(-5); \
29 v8i16 plus20h_m = __msa_ldi_h(20); \
31 ILVRL_H2_SW(in5, in0, tmp0_m, tmp1_m); \
33 tmp0_m = __msa_hadd_s_w((v8i16) tmp0_m, (v8i16) tmp0_m); \
34 tmp1_m = __msa_hadd_s_w((v8i16) tmp1_m, (v8i16) tmp1_m); \
36 ILVRL_H2_SH(in1, in4, out0_m, out1_m); \
37 DPADD_SH2_SW(out0_m, out1_m, minus5h_m, minus5h_m, tmp0_m, tmp1_m); \
38 ILVRL_H2_SH(in2, in3, out2_m, out3_m); \
39 DPADD_SH2_SW(out2_m, out3_m, plus20h_m, plus20h_m, tmp0_m, tmp1_m); \
41 SRARI_W2_SW(tmp0_m, tmp1_m, 10); \
42 SAT_SW2_SW(tmp0_m, tmp1_m, 7); \
43 out0_m = __msa_pckev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \
48 static const uint8_t luma_mask_arr[16 * 8] = {
50 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7, 12,
51 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 9, 7, 10, 8, 11,
52 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
55 0, 5, 1, 6, 2, 7, 3, 8, 16, 21, 17, 22, 18, 23, 19, 24,
56 1, 4, 2, 5, 3, 6, 4, 7, 17, 20, 18, 21, 19, 22, 20, 23,
57 2, 3, 3, 4, 4, 5, 5, 6, 18, 19, 19, 20, 20, 21, 21, 22,
59 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 25,
60 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26
63 #define AVC_CALC_DPADD_B_6PIX_2COEFF_SH(vec0, vec1, vec2, vec3, vec4, vec5, \
66 v16i8 tmp0_m, tmp1_m; \
67 v16i8 minus5b_m = __msa_ldi_b(-5); \
68 v16i8 plus20b_m = __msa_ldi_b(20); \
70 ILVRL_B2_SB(vec5, vec0, tmp0_m, tmp1_m); \
71 HADD_SB2_SH(tmp0_m, tmp1_m, out1, out2); \
72 ILVRL_B2_SB(vec4, vec1, tmp0_m, tmp1_m); \
73 DPADD_SB2_SH(tmp0_m, tmp1_m, minus5b_m, minus5b_m, out1, out2); \
74 ILVRL_B2_SB(vec3, vec2, tmp0_m, tmp1_m); \
75 DPADD_SB2_SH(tmp0_m, tmp1_m, plus20b_m, plus20b_m, out1, out2); \
78 #define AVC_CALC_DPADD_B_6PIX_2COEFF_R_SH(vec0, vec1, vec2, vec3, vec4, vec5) \
81 v16i8 tmp0_m, tmp2_m; \
82 v16i8 minus5b_m = __msa_ldi_b(-5); \
83 v16i8 plus20b_m = __msa_ldi_b(20); \
85 tmp1_m = (v8i16) __msa_ilvr_b((v16i8) vec5, (v16i8) vec0); \
86 tmp1_m = __msa_hadd_s_h((v16i8) tmp1_m, (v16i8) tmp1_m); \
88 ILVR_B2_SB(vec4, vec1, vec3, vec2, tmp0_m, tmp2_m); \
89 DPADD_SB2_SH(tmp0_m, tmp2_m, minus5b_m, plus20b_m, tmp1_m, tmp1_m); \
94 #define AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(vec0, vec1, vec2, vec3, vec4, vec5) \
97 v8i16 tmp2_m, tmp3_m; \
98 v8i16 minus5h_m = __msa_ldi_h(-5); \
99 v8i16 plus20h_m = __msa_ldi_h(20); \
101 tmp1_m = (v4i32) __msa_ilvr_h((v8i16) vec5, (v8i16) vec0); \
102 tmp1_m = __msa_hadd_s_w((v8i16) tmp1_m, (v8i16) tmp1_m); \
104 ILVR_H2_SH(vec1, vec4, vec2, vec3, tmp2_m, tmp3_m); \
105 DPADD_SH2_SW(tmp2_m, tmp3_m, minus5h_m, plus20h_m, tmp1_m, tmp1_m); \
107 tmp1_m = __msa_srari_w(tmp1_m, 10); \
108 tmp1_m = __msa_sat_s_w(tmp1_m, 7); \
110 tmp2_m = __msa_pckev_h((v8i16) tmp1_m, (v8i16) tmp1_m); \
115 #define AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1, \
116 mask0, mask1, mask2) \
119 v16i8 vec0_m, vec1_m, vec2_m; \
120 v16i8 minus5b_m = __msa_ldi_b(-5); \
121 v16i8 plus20b_m = __msa_ldi_b(20); \
123 vec0_m = __msa_vshf_b((v16i8) mask0, (v16i8) src1, (v16i8) src0); \
124 hz_out_m = __msa_hadd_s_h(vec0_m, vec0_m); \
126 VSHF_B2_SB(src0, src1, src0, src1, mask1, mask2, vec1_m, vec2_m); \
127 DPADD_SB2_SH(vec1_m, vec2_m, minus5b_m, plus20b_m, hz_out_m, hz_out_m); \
132 #define AVC_HORZ_FILTER_SH(in0, in1, mask0, mask1, mask2) \
136 v16i8 minus5b = __msa_ldi_b(-5); \
137 v16i8 plus20b = __msa_ldi_b(20); \
139 tmp0_m = __msa_vshf_b((v16i8) mask0, in1, in0); \
140 out0_m = __msa_hadd_s_h(tmp0_m, tmp0_m); \
142 tmp0_m = __msa_vshf_b((v16i8) mask1, in1, in0); \
143 out0_m = __msa_dpadd_s_h(out0_m, minus5b, tmp0_m); \
145 tmp0_m = __msa_vshf_b((v16i8) mask2, in1, in0); \
146 out0_m = __msa_dpadd_s_h(out0_m, plus20b, tmp0_m); \
151 #define AVC_DOT_SH3_SH(in0, in1, in2, coeff0, coeff1, coeff2) \
155 out0_m = __msa_dotp_s_h((v16i8) in0, (v16i8) coeff0); \
156 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in1, (v16i8) coeff1); \
157 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in2, (v16i8) coeff2); \
162 #define AVC_DOT_SW3_SW(in0, in1, in2, coeff0, coeff1, coeff2) \
166 out0_m = __msa_dotp_s_w((v8i16) in0, (v8i16) coeff0); \
167 out0_m = __msa_dpadd_s_w(out0_m, (v8i16) in1, (v8i16) coeff1); \
168 out0_m = __msa_dpadd_s_w(out0_m, (v8i16) in2, (v8i16) coeff2); \
169 out0_m = __msa_srari_w(out0_m, 10); \
170 out0_m = __msa_sat_s_w(out0_m, 7); \
174 static void avc_luma_hv_qrt_4x4_msa(const uint8_t *src_x, const uint8_t *src_y,
175 uint8_t *dst, int32_t stride)
177 const int16_t filt_const0 = 0xfb01;
178 const int16_t filt_const1 = 0x1414;
179 const int16_t filt_const2 = 0x1fb;
181 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, src_vt7, src_vt8;
182 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
183 v16i8 src_vt10_r, src_vt32_r, src_vt54_r, src_vt76_r;
184 v16i8 mask0, mask1, mask2, filt0, filt1, filt2;
185 v8i16 hz_out0, hz_out1, vt_out0, vt_out1, out0, out1;
187 filt0 = (v16i8) __msa_fill_h(filt_const0);
188 filt1 = (v16i8) __msa_fill_h(filt_const1);
189 filt2 = (v16i8) __msa_fill_h(filt_const2);
191 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
193 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
194 src_y += (5 * stride);
196 src_vt0 = (v16i8) __msa_insve_w((v4i32) src_vt0, 1, (v4i32) src_vt1);
197 src_vt1 = (v16i8) __msa_insve_w((v4i32) src_vt1, 1, (v4i32) src_vt2);
198 src_vt2 = (v16i8) __msa_insve_w((v4i32) src_vt2, 1, (v4i32) src_vt3);
199 src_vt3 = (v16i8) __msa_insve_w((v4i32) src_vt3, 1, (v4i32) src_vt4);
201 XORI_B4_128_SB(src_vt0, src_vt1, src_vt2, src_vt3);
203 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
204 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
205 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz1, mask0, mask1, mask2);
206 hz_out1 = AVC_HORZ_FILTER_SH(src_hz2, src_hz3, mask0, mask1, mask2);
208 SRARI_H2_SH(hz_out0, hz_out1, 5);
209 SAT_SH2_SH(hz_out0, hz_out1, 7);
211 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
213 src_vt4 = (v16i8) __msa_insve_w((v4i32) src_vt4, 1, (v4i32) src_vt5);
214 src_vt5 = (v16i8) __msa_insve_w((v4i32) src_vt5, 1, (v4i32) src_vt6);
215 src_vt6 = (v16i8) __msa_insve_w((v4i32) src_vt6, 1, (v4i32) src_vt7);
216 src_vt7 = (v16i8) __msa_insve_w((v4i32) src_vt7, 1, (v4i32) src_vt8);
218 XORI_B4_128_SB(src_vt4, src_vt5, src_vt6, src_vt7);
219 ILVR_B2_SB(src_vt1, src_vt0, src_vt3, src_vt2, src_vt10_r, src_vt32_r);
220 ILVR_B2_SB(src_vt5, src_vt4, src_vt7, src_vt6, src_vt54_r, src_vt76_r);
221 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
223 vt_out1 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
225 SRARI_H2_SH(vt_out0, vt_out1, 5);
226 SAT_SH2_SH(vt_out0, vt_out1, 7);
228 out0 = __msa_srari_h((hz_out0 + vt_out0), 1);
229 out1 = __msa_srari_h((hz_out1 + vt_out1), 1);
231 SAT_SH2_SH(out0, out1, 7);
232 out = PCKEV_XORI128_UB(out0, out1);
233 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
236 static void avc_luma_hv_qrt_8x8_msa(const uint8_t *src_x, const uint8_t *src_y,
237 uint8_t *dst, int32_t stride)
239 const int16_t filt_const0 = 0xfb01;
240 const int16_t filt_const1 = 0x1414;
241 const int16_t filt_const2 = 0x1fb;
243 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2;
244 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
245 v16i8 src_vt7, src_vt8, src_vt9, src_vt10, src_vt11, src_vt12;
246 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
247 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, src_vt98_r, src_vt109_r;
248 v16i8 src_vt1110_r, src_vt1211_r, filt0, filt1, filt2;
249 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
250 v8i16 vt_out3, tmp0, tmp1, tmp2, tmp3;
252 filt0 = (v16i8) __msa_fill_h(filt_const0);
253 filt1 = (v16i8) __msa_fill_h(filt_const1);
254 filt2 = (v16i8) __msa_fill_h(filt_const2);
256 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
257 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
258 src_y += (5 * stride);
260 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
262 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
263 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
264 src_x += (4 * stride);
266 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
267 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
268 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
269 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
271 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
272 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
274 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
275 src_y += (4 * stride);
276 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
278 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2, src_vt4,
279 src_vt3, src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r);
280 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6, src_vt8,
281 src_vt7, src_vt54_r, src_vt65_r, src_vt76_r, src_vt87_r);
282 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0, filt1,
284 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0, filt1,
286 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0, filt1,
288 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0, filt1,
290 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
291 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
293 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
294 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
295 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
296 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
298 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
299 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
301 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
302 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
303 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
304 ST8x4_UB(out0, out1, dst, stride);
307 LD_SB4(src_y, stride, src_vt9, src_vt10, src_vt11, src_vt12);
308 XORI_B4_128_SB(src_vt9, src_vt10, src_vt11, src_vt12);
310 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
311 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
312 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
313 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
315 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
316 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
318 ILVR_B4_SB(src_vt9, src_vt8, src_vt10, src_vt9, src_vt11, src_vt10,
319 src_vt12, src_vt11, src_vt98_r, src_vt109_r, src_vt1110_r,
321 vt_out0 = AVC_DOT_SH3_SH(src_vt54_r, src_vt76_r, src_vt98_r, filt0, filt1,
323 vt_out1 = AVC_DOT_SH3_SH(src_vt65_r, src_vt87_r, src_vt109_r, filt0, filt1,
325 vt_out2 = AVC_DOT_SH3_SH(src_vt76_r, src_vt98_r, src_vt1110_r, filt0, filt1,
327 vt_out3 = AVC_DOT_SH3_SH(src_vt87_r, src_vt109_r, src_vt1211_r, filt0,
329 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
330 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
332 tmp0 = __msa_srari_h((hz_out0 + vt_out0), 1);
333 tmp1 = __msa_srari_h((hz_out1 + vt_out1), 1);
334 tmp2 = __msa_srari_h((hz_out2 + vt_out2), 1);
335 tmp3 = __msa_srari_h((hz_out3 + vt_out3), 1);
337 SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
338 out0 = PCKEV_XORI128_UB(tmp0, tmp1);
339 out1 = PCKEV_XORI128_UB(tmp2, tmp3);
340 ST8x4_UB(out0, out1, dst, stride);
344 static void avc_luma_hv_qrt_16x16_msa(const uint8_t *src_x,
345 const uint8_t *src_y, uint8_t *dst,
348 const int16_t filt_const0 = 0xfb01;
349 const int16_t filt_const1 = 0x1414;
350 const int16_t filt_const2 = 0x1fb;
351 const uint8_t *src_x_tmp = src_x;
352 const uint8_t *src_y_tmp = src_y;
353 uint8_t *dst_tmp = dst;
354 uint32_t multiple8_cnt, loop_cnt;
356 v16i8 src_hz0, src_hz1, src_hz2, src_hz3, mask0, mask1, mask2;
357 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4, src_vt5, src_vt6;
358 v16i8 src_vt7, src_vt8;
359 v16i8 src_vt10_r, src_vt21_r, src_vt32_r, src_vt43_r, src_vt54_r;
360 v16i8 src_vt65_r, src_vt76_r, src_vt87_r, filt0, filt1, filt2;
361 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, vt_out0, vt_out1, vt_out2;
362 v8i16 vt_out3, out0, out1, out2, out3;
364 filt0 = (v16i8) __msa_fill_h(filt_const0);
365 filt1 = (v16i8) __msa_fill_h(filt_const1);
366 filt2 = (v16i8) __msa_fill_h(filt_const2);
368 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
370 for (multiple8_cnt = 2; multiple8_cnt--;) {
375 LD_SB5(src_y, stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
376 src_y += (5 * stride);
378 XORI_B5_128_SB(src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
380 for (loop_cnt = 4; loop_cnt--;) {
381 LD_SB4(src_x, stride, src_hz0, src_hz1, src_hz2, src_hz3);
382 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
383 src_x += (4 * stride);
385 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
386 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
387 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
388 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
389 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
390 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
392 LD_SB4(src_y, stride, src_vt5, src_vt6, src_vt7, src_vt8);
393 src_y += (4 * stride);
395 XORI_B4_128_SB(src_vt5, src_vt6, src_vt7, src_vt8);
396 ILVR_B4_SB(src_vt1, src_vt0, src_vt2, src_vt1, src_vt3, src_vt2,
397 src_vt4, src_vt3, src_vt10_r, src_vt21_r, src_vt32_r,
399 ILVR_B4_SB(src_vt5, src_vt4, src_vt6, src_vt5, src_vt7, src_vt6,
400 src_vt8, src_vt7, src_vt54_r, src_vt65_r, src_vt76_r,
402 vt_out0 = AVC_DOT_SH3_SH(src_vt10_r, src_vt32_r, src_vt54_r, filt0,
404 vt_out1 = AVC_DOT_SH3_SH(src_vt21_r, src_vt43_r, src_vt65_r, filt0,
406 vt_out2 = AVC_DOT_SH3_SH(src_vt32_r, src_vt54_r, src_vt76_r, filt0,
408 vt_out3 = AVC_DOT_SH3_SH(src_vt43_r, src_vt65_r, src_vt87_r, filt0,
410 SRARI_H4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 5);
411 SAT_SH4_SH(vt_out0, vt_out1, vt_out2, vt_out3, 7);
413 out0 = __msa_srari_h((hz_out0 + vt_out0), 1);
414 out1 = __msa_srari_h((hz_out1 + vt_out1), 1);
415 out2 = __msa_srari_h((hz_out2 + vt_out2), 1);
416 out3 = __msa_srari_h((hz_out3 + vt_out3), 1);
418 SAT_SH4_SH(out0, out1, out2, out3, 7);
419 tmp0 = PCKEV_XORI128_UB(out0, out1);
420 tmp1 = PCKEV_XORI128_UB(out2, out3);
421 ST8x4_UB(tmp0, tmp1, dst, stride);
437 static void avc_luma_hz_and_aver_dst_4x4_msa(const uint8_t *src,
439 uint8_t *dst, int32_t dst_stride)
441 v16i8 src0, src1, src2, src3;
442 v16u8 dst0, dst1, dst2, dst3, res;
444 v16i8 mask0, mask1, mask2;
445 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
446 v16i8 minus5b = __msa_ldi_b(-5);
447 v16i8 plus20b = __msa_ldi_b(20);
449 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
450 LD_SB4(src, src_stride, src0, src1, src2, src3);
452 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
453 XORI_B4_128_SB(src0, src1, src2, src3);
454 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
455 HADD_SB2_SH(vec0, vec1, res0, res1);
456 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
457 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
458 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
459 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
460 SRARI_H2_SH(res0, res1, 5);
461 SAT_SH2_SH(res0, res1, 7);
462 res = PCKEV_XORI128_UB(res0, res1);
463 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
465 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
466 res = __msa_aver_u_b(res, dst0);
468 ST4x4_UB(res, res, 0, 1, 2, 3, dst, dst_stride);
471 static void avc_luma_hz_and_aver_dst_8x8_msa(const uint8_t *src,
473 uint8_t *dst, int32_t dst_stride)
476 v16i8 src0, src1, src2, src3;
477 v16u8 dst0, dst1, dst2, dst3;
478 v8i16 res0, res1, res2, res3;
479 v16i8 mask0, mask1, mask2;
480 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
481 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
482 v16i8 minus5b = __msa_ldi_b(-5);
483 v16i8 plus20b = __msa_ldi_b(20);
485 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
487 for (loop_cnt = 2; loop_cnt--;) {
488 LD_SB4(src, src_stride, src0, src1, src2, src3);
489 src += (4 * src_stride);
491 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
493 XORI_B4_128_SB(src0, src1, src2, src3);
494 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
495 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
496 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
497 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
498 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
499 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
500 res0, res1, res2, res3);
501 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
502 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
503 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
504 plus20b, res0, res1, res2, res3);
505 SRARI_H4_SH(res0, res1, res2, res3, 5);
506 SAT_SH4_SH(res0, res1, res2, res3, 7);
507 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
508 CONVERT_UB_AVG_ST8x4_UB(res0, res1, res2, res3, dst0, dst1,
511 dst += (4 * dst_stride);
515 static void avc_luma_hz_and_aver_dst_16x16_msa(const uint8_t *src,
517 uint8_t *dst, int32_t dst_stride)
520 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
521 v16u8 dst0, dst1, dst2, dst3;
522 v16i8 mask0, mask1, mask2;
523 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
524 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
525 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
526 v16i8 minus5b = __msa_ldi_b(-5);
527 v16i8 plus20b = __msa_ldi_b(20);
529 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
531 for (loop_cnt = 4; loop_cnt--;) {
532 LD_SB2(src, 8, src0, src1);
534 LD_SB2(src, 8, src2, src3);
537 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
539 XORI_B4_128_SB(src0, src1, src2, src3);
540 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
541 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
542 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
543 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
544 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
545 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
546 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
547 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
548 minus5b, res0, res1, res2, res3);
549 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
550 plus20b, res0, res1, res2, res3);
551 LD_SB2(src, 8, src4, src5);
553 LD_SB2(src, 8, src6, src7);
555 XORI_B4_128_SB(src4, src5, src6, src7);
556 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec3);
557 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec6, vec9);
558 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec1, vec4);
559 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec7, vec10);
560 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec2, vec5);
561 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec8, vec11);
562 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
563 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
564 minus5b, res4, res5, res6, res7);
565 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
566 plus20b, res4, res5, res6, res7);
567 SRARI_H4_SH(res0, res1, res2, res3, 5);
568 SRARI_H4_SH(res4, res5, res6, res7, 5);
569 SAT_SH4_SH(res0, res1, res2, res3, 7);
570 SAT_SH4_SH(res4, res5, res6, res7, 7);
571 PCKEV_B4_SB(res1, res0, res3, res2, res5, res4, res7, res6,
572 vec0, vec1, vec2, vec3);
573 XORI_B4_128_SB(vec0, vec1, vec2, vec3);
574 AVER_UB4_UB(vec0, dst0, vec1, dst1, vec2, dst2, vec3, dst3,
575 dst0, dst1, dst2, dst3);
576 ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
577 dst += (4 * dst_stride);
581 static void avc_luma_hz_qrt_and_aver_dst_4x4_msa(const uint8_t *src,
588 v16i8 src0, src1, src2, src3;
589 v16u8 dst0, dst1, dst2, dst3;
590 v16i8 mask0, mask1, mask2;
591 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
593 v16i8 minus5b = __msa_ldi_b(-5);
594 v16i8 plus20b = __msa_ldi_b(20);
597 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
605 LD_SB4(src, src_stride, src0, src1, src2, src3);
606 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
608 XORI_B4_128_SB(src0, src1, src2, src3);
609 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
610 HADD_SB2_SH(vec0, vec1, out0, out1);
611 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
612 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, out0, out1);
613 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
614 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, out0, out1);
615 SRARI_H2_SH(out0, out1, 5);
616 SAT_SH2_SH(out0, out1, 7);
618 PCKEV_B2_UB(out0, out0, out1, out1, res0, res1);
620 src0 = __msa_sld_b(src0, src0, slide);
621 src1 = __msa_sld_b(src1, src1, slide);
622 src2 = __msa_sld_b(src2, src2, slide);
623 src3 = __msa_sld_b(src3, src3, slide);
624 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
625 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
626 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src0);
627 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src1);
629 XORI_B2_128_UB(res0, res1);
631 dst0 = (v16u8) __msa_insve_w((v4i32) dst0, 1, (v4i32) dst1);
632 dst1 = (v16u8) __msa_insve_w((v4i32) dst2, 1, (v4i32) dst3);
634 AVER_UB2_UB(res0, dst0, res1, dst1, dst0, dst1);
636 ST4x4_UB(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
639 static void avc_luma_hz_qrt_and_aver_dst_8x8_msa(const uint8_t *src,
647 v16i8 src0, src1, src2, src3;
648 v16i8 mask0, mask1, mask2;
649 v16u8 dst0, dst1, dst2, dst3;
650 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
651 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
652 v8i16 out0, out1, out2, out3;
653 v16i8 minus5b = __msa_ldi_b(-5);
654 v16i8 plus20b = __msa_ldi_b(20);
655 v16i8 res0, res1, res2, res3;
657 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
665 for (loop_cnt = 2; loop_cnt--;) {
666 LD_SB4(src, src_stride, src0, src1, src2, src3);
667 src += (4 * src_stride);
669 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
671 XORI_B4_128_SB(src0, src1, src2, src3);
672 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
673 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
674 HADD_SB4_SH(vec0, vec1, vec2, vec3, out0, out1, out2, out3);
675 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
676 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
677 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
678 out0, out1, out2, out3);
679 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
680 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
681 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
682 plus20b, out0, out1, out2, out3);
684 src0 = __msa_sld_b(src0, src0, slide);
685 src1 = __msa_sld_b(src1, src1, slide);
686 src2 = __msa_sld_b(src2, src2, slide);
687 src3 = __msa_sld_b(src3, src3, slide);
689 SRARI_H4_SH(out0, out1, out2, out3, 5);
690 SAT_SH4_SH(out0, out1, out2, out3, 7);
692 PCKEV_B4_SB(out0, out0, out1, out1, out2, out2, out3, out3,
693 res0, res1, res2, res3);
695 res0 = __msa_aver_s_b(res0, src0);
696 res1 = __msa_aver_s_b(res1, src1);
697 res2 = __msa_aver_s_b(res2, src2);
698 res3 = __msa_aver_s_b(res3, src3);
700 XORI_B4_128_SB(res0, res1, res2, res3);
701 AVER_ST8x4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
704 dst += (4 * dst_stride);
708 static void avc_luma_hz_qrt_and_aver_dst_16x16_msa(const uint8_t *src,
716 v16i8 src0, src1, src2, src3;
717 v16i8 mask0, mask1, mask2, vshf;
719 v8i16 res0, res1, res2, res3;
720 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
721 v16i8 vec6, vec7, vec8, vec9, vec10, vec11;
722 v16i8 minus5b = __msa_ldi_b(-5);
723 v16i8 plus20b = __msa_ldi_b(20);
725 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
728 vshf = LD_SB(&luma_mask_arr[16 + 96]);
730 vshf = LD_SB(&luma_mask_arr[96]);
733 for (loop_cnt = 8; loop_cnt--;) {
734 LD_SB2(src, 8, src0, src1);
736 LD_SB2(src, 8, src2, src3);
739 LD_UB2(dst, dst_stride, dst0, dst1);
741 XORI_B4_128_SB(src0, src1, src2, src3);
742 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
743 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
744 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
745 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
746 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
747 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
748 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
749 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
750 minus5b, res0, res1, res2, res3);
751 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
752 plus20b, res0, res1, res2, res3);
753 VSHF_B2_SB(src0, src1, src2, src3, vshf, vshf, src0, src2);
754 SRARI_H4_SH(res0, res1, res2, res3, 5);
755 SAT_SH4_SH(res0, res1, res2, res3, 7);
756 PCKEV_B2_SB(res1, res0, res3, res2, out0, out1);
758 out0 = __msa_aver_s_b(out0, src0);
759 out1 = __msa_aver_s_b(out1, src2);
761 XORI_B2_128_SB(out0, out1);
762 AVER_UB2_UB(out0, dst0, out1, dst1, dst0, dst1);
763 ST_UB2(dst0, dst1, dst, dst_stride);
764 dst += (2 * dst_stride);
768 static void avc_luma_vt_and_aver_dst_4x4_msa(const uint8_t *src,
770 uint8_t *dst, int32_t dst_stride)
772 int16_t filt_const0 = 0xfb01;
773 int16_t filt_const1 = 0x1414;
774 int16_t filt_const2 = 0x1fb;
775 v16u8 dst0, dst1, dst2, dst3;
776 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
777 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
778 v16i8 src87_r, src2110, src4332, src6554, src8776;
780 v16i8 filt0, filt1, filt2;
783 filt0 = (v16i8) __msa_fill_h(filt_const0);
784 filt1 = (v16i8) __msa_fill_h(filt_const1);
785 filt2 = (v16i8) __msa_fill_h(filt_const2);
787 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
788 src += (5 * src_stride);
790 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
791 src10_r, src21_r, src32_r, src43_r);
792 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
793 XORI_B2_128_SB(src2110, src4332);
794 LD_SB4(src, src_stride, src5, src6, src7, src8);
795 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
796 src54_r, src65_r, src76_r, src87_r);
797 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
798 XORI_B2_128_SB(src6554, src8776);
799 out10 = DPADD_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
800 out32 = DPADD_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
801 SRARI_H2_SH(out10, out32, 5);
802 SAT_SH2_SH(out10, out32, 7);
803 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
804 res = PCKEV_XORI128_UB(out10, out32);
806 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
808 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
809 dst0 = __msa_aver_u_b(res, dst0);
811 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
814 static void avc_luma_vt_and_aver_dst_8x8_msa(const uint8_t *src,
816 uint8_t *dst, int32_t dst_stride)
819 int16_t filt_const0 = 0xfb01;
820 int16_t filt_const1 = 0x1414;
821 int16_t filt_const2 = 0x1fb;
822 v16u8 dst0, dst1, dst2, dst3;
823 v16i8 src0, src1, src2, src3, src4, src7, src8, src9, src10;
824 v16i8 src10_r, src32_r, src76_r, src98_r;
825 v16i8 src21_r, src43_r, src87_r, src109_r;
826 v8i16 out0, out1, out2, out3;
827 v16i8 filt0, filt1, filt2;
829 filt0 = (v16i8) __msa_fill_h(filt_const0);
830 filt1 = (v16i8) __msa_fill_h(filt_const1);
831 filt2 = (v16i8) __msa_fill_h(filt_const2);
833 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
834 src += (5 * src_stride);
836 XORI_B5_128_SB(src0, src1, src2, src3, src4);
837 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
838 src10_r, src21_r, src32_r, src43_r);
840 for (loop_cnt = 2; loop_cnt--;) {
841 LD_SB4(src, src_stride, src7, src8, src9, src10);
842 src += (4 * src_stride);
844 XORI_B4_128_SB(src7, src8, src9, src10);
845 ILVR_B4_SB(src7, src4, src8, src7, src9, src8, src10, src9,
846 src76_r, src87_r, src98_r, src109_r);
847 out0 = DPADD_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
848 out1 = DPADD_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
849 out2 = DPADD_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
850 out3 = DPADD_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
851 SRARI_H4_SH(out0, out1, out2, out3, 5);
852 SAT_SH4_SH(out0, out1, out2, out3, 7);
853 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
854 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
855 CONVERT_UB_AVG_ST8x4_UB(out0, out1, out2, out3, dst0, dst1,
857 dst += (4 * dst_stride);
867 static void avc_luma_vt_and_aver_dst_16x16_msa(const uint8_t *src,
869 uint8_t *dst, int32_t dst_stride)
872 int16_t filt_const0 = 0xfb01;
873 int16_t filt_const1 = 0x1414;
874 int16_t filt_const2 = 0x1fb;
875 v16u8 dst0, dst1, dst2, dst3;
876 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
877 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
878 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
879 v16i8 src65_l, src87_l;
880 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
881 v16i8 filt0, filt1, filt2;
882 v16u8 res0, res1, res2, res3;
884 filt0 = (v16i8) __msa_fill_h(filt_const0);
885 filt1 = (v16i8) __msa_fill_h(filt_const1);
886 filt2 = (v16i8) __msa_fill_h(filt_const2);
888 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
889 src += (5 * src_stride);
891 XORI_B5_128_SB(src0, src1, src2, src3, src4);
892 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
893 src10_r, src21_r, src32_r, src43_r);
894 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
895 src10_l, src21_l, src32_l, src43_l);
897 for (loop_cnt = 4; loop_cnt--;) {
898 LD_SB4(src, src_stride, src5, src6, src7, src8);
899 src += (4 * src_stride);
901 XORI_B4_128_SB(src5, src6, src7, src8);
902 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
903 src54_r, src65_r, src76_r, src87_r);
904 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
905 src54_l, src65_l, src76_l, src87_l);
906 out0_r = DPADD_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
907 out1_r = DPADD_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
908 out2_r = DPADD_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
909 out3_r = DPADD_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
910 out0_l = DPADD_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
911 out1_l = DPADD_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
912 out2_l = DPADD_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
913 out3_l = DPADD_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
914 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
915 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
916 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
917 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
918 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
919 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
920 out3_r, res0, res1, res2, res3);
921 XORI_B4_128_UB(res0, res1, res2, res3);
922 AVER_UB4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
923 res0, res1, res2, res3);
924 ST_UB4(res0, res1, res2, res3, dst, dst_stride);
925 dst += (4 * dst_stride);
939 static void avc_luma_vt_qrt_and_aver_dst_4x4_msa(const uint8_t *src,
945 int16_t filt_const0 = 0xfb01;
946 int16_t filt_const1 = 0x1414;
947 int16_t filt_const2 = 0x1fb;
948 v16u8 dst0, dst1, dst2, dst3;
949 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
950 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
951 v16i8 src87_r, src2110, src4332, src6554, src8776;
953 v16i8 filt0, filt1, filt2;
956 filt0 = (v16i8) __msa_fill_h(filt_const0);
957 filt1 = (v16i8) __msa_fill_h(filt_const1);
958 filt2 = (v16i8) __msa_fill_h(filt_const2);
960 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
961 src += (5 * src_stride);
963 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
964 src10_r, src21_r, src32_r, src43_r);
965 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
966 XORI_B2_128_SB(src2110, src4332);
967 LD_SB4(src, src_stride, src5, src6, src7, src8);
968 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
969 src54_r, src65_r, src76_r, src87_r);
970 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
971 XORI_B2_128_SB(src6554, src8776);
972 out10 = DPADD_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
973 out32 = DPADD_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
974 SRARI_H2_SH(out10, out32, 5);
975 SAT_SH2_SH(out10, out32, 7);
976 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
977 res = PCKEV_XORI128_UB(out10, out32);
980 src32_r = (v16i8) __msa_insve_w((v4i32) src3, 1, (v4i32) src4);
981 src54_r = (v16i8) __msa_insve_w((v4i32) src5, 1, (v4i32) src6);
983 src32_r = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
984 src54_r = (v16i8) __msa_insve_w((v4i32) src4, 1, (v4i32) src5);
987 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
988 res = __msa_aver_u_b(res, (v16u8) src32_r);
990 ILVR_W2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
992 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0);
993 dst0 = __msa_aver_u_b(res, dst0);
995 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
998 static void avc_luma_vt_qrt_and_aver_dst_8x8_msa(const uint8_t *src,
1005 int16_t filt_const0 = 0xfb01;
1006 int16_t filt_const1 = 0x1414;
1007 int16_t filt_const2 = 0x1fb;
1008 v16u8 dst0, dst1, dst2, dst3;
1009 v16i8 src0, src1, src2, src3, src4, src7, src8, src9, src10;
1010 v16i8 src10_r, src32_r, src76_r, src98_r;
1011 v16i8 src21_r, src43_r, src87_r, src109_r;
1012 v8i16 out0_r, out1_r, out2_r, out3_r;
1015 v16i8 filt0, filt1, filt2;
1017 filt0 = (v16i8) __msa_fill_h(filt_const0);
1018 filt1 = (v16i8) __msa_fill_h(filt_const1);
1019 filt2 = (v16i8) __msa_fill_h(filt_const2);
1021 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1022 src += (5 * src_stride);
1024 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1025 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1026 src10_r, src21_r, src32_r, src43_r);
1028 for (loop_cnt = 2; loop_cnt--;) {
1029 LD_SB4(src, src_stride, src7, src8, src9, src10);
1030 src += (4 * src_stride);
1032 XORI_B4_128_SB(src7, src8, src9, src10);
1033 ILVR_B4_SB(src7, src4, src8, src7, src9, src8, src10, src9,
1034 src76_r, src87_r, src98_r, src109_r);
1035 out0_r = DPADD_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
1036 out1_r = DPADD_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
1037 out2_r = DPADD_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
1038 out3_r = DPADD_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
1039 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
1040 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1041 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, res0, res1);
1044 PCKEV_D2_SB(src4, src3, src8, src7, src10_r, src32_r);
1046 PCKEV_D2_SB(src3, src2, src7, src4, src10_r, src32_r);
1049 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1050 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1052 vec0 = (v16u8) __msa_aver_s_b(res0, src10_r);
1053 vec1 = (v16u8) __msa_aver_s_b(res1, src32_r);
1055 XORI_B2_128_UB(vec0, vec1);
1056 AVER_UB2_UB(vec0, dst0, vec1, dst1, vec0, vec1);
1057 ST8x4_UB(vec0, vec1, dst, dst_stride);
1058 dst += (4 * dst_stride);
1070 static void avc_luma_vt_qrt_and_aver_dst_16x16_msa(const uint8_t *src,
1077 int16_t filt_const0 = 0xfb01;
1078 int16_t filt_const1 = 0x1414;
1079 int16_t filt_const2 = 0x1fb;
1080 v16u8 dst0, dst1, dst2, dst3;
1081 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1082 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
1083 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
1084 v16i8 src65_l, src87_l;
1085 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
1086 v16i8 out0, out1, out2, out3;
1087 v16i8 filt0, filt1, filt2;
1088 v16u8 res0, res1, res2, res3;
1090 filt0 = (v16i8) __msa_fill_h(filt_const0);
1091 filt1 = (v16i8) __msa_fill_h(filt_const1);
1092 filt2 = (v16i8) __msa_fill_h(filt_const2);
1094 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1095 src += (5 * src_stride);
1097 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1098 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1099 src10_r, src21_r, src32_r, src43_r);
1100 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3,
1101 src10_l, src21_l, src32_l, src43_l);
1103 for (loop_cnt = 4; loop_cnt--;) {
1104 LD_SB4(src, src_stride, src5, src6, src7, src8);
1105 src += (4 * src_stride);
1107 XORI_B4_128_SB(src5, src6, src7, src8);
1108 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
1109 src54_r, src65_r, src76_r, src87_r);
1110 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7,
1111 src54_l, src65_l, src76_l, src87_l);
1112 out0_r = DPADD_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
1113 out1_r = DPADD_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
1114 out2_r = DPADD_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
1115 out3_r = DPADD_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
1116 out0_l = DPADD_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
1117 out1_l = DPADD_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
1118 out2_l = DPADD_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
1119 out3_l = DPADD_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
1120 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
1121 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
1122 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1123 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
1124 PCKEV_B4_SB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
1125 out3_r, out0, out1, out2, out3);
1126 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1129 res0 = (v16u8) __msa_aver_s_b(out0, src3);
1130 res1 = (v16u8) __msa_aver_s_b(out1, src4);
1131 res2 = (v16u8) __msa_aver_s_b(out2, src5);
1132 res3 = (v16u8) __msa_aver_s_b(out3, src6);
1134 res0 = (v16u8) __msa_aver_s_b(out0, src2);
1135 res1 = (v16u8) __msa_aver_s_b(out1, src3);
1136 res2 = (v16u8) __msa_aver_s_b(out2, src4);
1137 res3 = (v16u8) __msa_aver_s_b(out3, src5);
1140 XORI_B4_128_UB(res0, res1, res2, res3);
1141 AVER_UB4_UB(res0, dst0, res1, dst1, res2, dst2, res3, dst3,
1142 dst0, dst1, dst2, dst3);
1143 ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
1144 dst += (4 * dst_stride);
1160 static void avc_luma_mid_and_aver_dst_4x4_msa(const uint8_t *src,
1162 uint8_t *dst, int32_t dst_stride)
1164 v16i8 src0, src1, src2, src3, src4;
1165 v16i8 mask0, mask1, mask2;
1166 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1167 v8i16 hz_out4, hz_out5, hz_out6, hz_out7, hz_out8;
1168 v8i16 res0, res1, res2, res3;
1169 v16u8 dst0, dst1, dst2, dst3;
1170 v16u8 tmp0, tmp1, tmp2, tmp3;
1172 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
1173 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1174 src += (5 * src_stride);
1176 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1178 hz_out0 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1179 mask0, mask1, mask2);
1180 hz_out2 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src2, src3,
1181 mask0, mask1, mask2);
1183 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
1185 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1187 LD_SB4(src, src_stride, src0, src1, src2, src3);
1188 XORI_B4_128_SB(src0, src1, src2, src3);
1190 hz_out5 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1191 mask0, mask1, mask2);
1192 hz_out7 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src2, src3,
1193 mask0, mask1, mask2);
1195 PCKOD_D2_SH(hz_out5, hz_out5, hz_out7, hz_out7, hz_out6, hz_out8);
1197 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out0, hz_out1, hz_out2,
1198 hz_out3, hz_out4, hz_out5);
1199 res1 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out1, hz_out2, hz_out3,
1200 hz_out4, hz_out5, hz_out6);
1201 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out2, hz_out3, hz_out4,
1202 hz_out5, hz_out6, hz_out7);
1203 res3 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out3, hz_out4, hz_out5,
1204 hz_out6, hz_out7, hz_out8);
1205 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1206 tmp0 = PCKEV_XORI128_UB(res0, res1);
1207 tmp1 = PCKEV_XORI128_UB(res2, res3);
1208 PCKEV_D2_UB(dst1, dst0, dst3, dst2, tmp2, tmp3);
1209 AVER_UB2_UB(tmp0, tmp2, tmp1, tmp3, tmp0, tmp1);
1211 ST4x4_UB(tmp0, tmp1, 0, 2, 0, 2, dst, dst_stride);
1214 static void avc_luma_mid_and_aver_dst_8w_msa(const uint8_t *src,
1216 uint8_t *dst, int32_t dst_stride,
1220 v16i8 src0, src1, src2, src3, src4;
1221 v16i8 mask0, mask1, mask2;
1222 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1223 v8i16 hz_out4, hz_out5, hz_out6, hz_out7, hz_out8;
1224 v16u8 dst0, dst1, dst2, dst3;
1225 v8i16 res0, res1, res2, res3;
1227 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1229 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1230 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1231 src += (5 * src_stride);
1233 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1234 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1235 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1236 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1237 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1239 for (loop_cnt = (height >> 2); loop_cnt--;) {
1240 LD_SB4(src, src_stride, src0, src1, src2, src3);
1241 XORI_B4_128_SB(src0, src1, src2, src3);
1242 src += (4 * src_stride);
1244 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1245 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1246 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1247 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1249 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out0, hz_out1, hz_out2,
1250 hz_out3, hz_out4, hz_out5);
1251 res1 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out1, hz_out2, hz_out3,
1252 hz_out4, hz_out5, hz_out6);
1253 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out2, hz_out3, hz_out4,
1254 hz_out5, hz_out6, hz_out7);
1255 res3 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out3, hz_out4, hz_out5,
1256 hz_out6, hz_out7, hz_out8);
1257 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1258 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1259 CONVERT_UB_AVG_ST8x4_UB(res0, res1, res2, res3, dst0, dst1,
1262 dst += (4 * dst_stride);
1272 static void avc_luma_mid_and_aver_dst_16x16_msa(const uint8_t *src,
1277 avc_luma_mid_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride, 16);
1278 avc_luma_mid_and_aver_dst_8w_msa(src + 8, src_stride, dst + 8, dst_stride,
1282 static void avc_luma_midh_qrt_and_aver_dst_4w_msa(const uint8_t *src,
1287 uint8_t horiz_offset)
1290 v16i8 src0, src1, src2, src3, src4, src5, src6;
1291 v16u8 dst0, dst1, res;
1292 v8i16 vt_res0, vt_res1, vt_res2, vt_res3;
1293 v4i32 hz_res0, hz_res1;
1295 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5;
1296 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
1297 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
1298 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
1299 v8i16 minus5h = __msa_ldi_h(-5);
1300 v8i16 plus20h = __msa_ldi_h(20);
1301 v8i16 zeros = { 0 };
1303 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1304 src += (5 * src_stride);
1306 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1308 for (row = (height >> 1); row--;) {
1309 LD_SB2(src, src_stride, src5, src6);
1310 src += (2 * src_stride);
1312 XORI_B2_128_SB(src5, src6);
1313 LD_UB2(dst, dst_stride, dst0, dst1);
1315 dst0 = (v16u8) __msa_ilvr_w((v4i32) dst1, (v4i32) dst0);
1317 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
1319 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
1321 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1,
1322 mask0, mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
1323 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3,
1324 mask0, mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
1326 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
1327 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
1329 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
1330 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
1332 SRARI_W2_SW(hz_res0, hz_res1, 10);
1333 SAT_SW2_SW(hz_res0, hz_res1, 7);
1335 res0 = __msa_srari_h(shf_vec2, 5);
1336 res1 = __msa_srari_h(shf_vec5, 5);
1338 SAT_SH2_SH(res0, res1, 7);
1341 res0 = __msa_ilvod_h(zeros, res0);
1342 res1 = __msa_ilvod_h(zeros, res1);
1344 ILVEV_H2_SH(res0, zeros, res1, zeros, res0, res1);
1346 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) res0);
1347 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) res1);
1348 res0 = __msa_pckev_h((v8i16) hz_res1, (v8i16) hz_res0);
1350 res = PCKEV_XORI128_UB(res0, res0);
1352 dst0 = __msa_aver_u_b(res, dst0);
1354 ST4x2_UB(dst0, dst, dst_stride);
1355 dst += (2 * dst_stride);
1365 static void avc_luma_midh_qrt_and_aver_dst_8w_msa(const uint8_t *src,
1370 uint8_t horiz_offset)
1372 uint32_t multiple8_cnt;
1374 for (multiple8_cnt = 2; multiple8_cnt--;) {
1375 avc_luma_midh_qrt_and_aver_dst_4w_msa(src, src_stride, dst, dst_stride,
1376 height, horiz_offset);
1383 static void avc_luma_midh_qrt_and_aver_dst_16w_msa(const uint8_t *src,
1388 uint8_t horiz_offset)
1390 uint32_t multiple8_cnt;
1392 for (multiple8_cnt = 4; multiple8_cnt--;) {
1393 avc_luma_midh_qrt_and_aver_dst_4w_msa(src, src_stride, dst, dst_stride,
1394 height, horiz_offset);
1401 static void avc_luma_midv_qrt_and_aver_dst_4w_msa(const uint8_t *src,
1410 v16i8 src0, src1, src2, src3, src4;
1412 v16i8 mask0, mask1, mask2;
1413 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1414 v8i16 hz_out4, hz_out5, hz_out6;
1415 v8i16 res0, res1, res2, res3;
1418 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
1419 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1420 src += (5 * src_stride);
1422 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1424 hz_out0 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1425 mask0, mask1, mask2);
1426 hz_out2 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src2, src3,
1427 mask0, mask1, mask2);
1429 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
1431 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1433 for (loop_cnt = (height >> 1); loop_cnt--;) {
1434 LD_SB2(src, src_stride, src0, src1);
1435 src += (2 * src_stride);
1437 XORI_B2_128_SB(src0, src1);
1438 LD_UB2(dst, dst_stride, dst0, dst1);
1439 hz_out5 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src0, src1,
1442 hz_out6 = (v8i16) __msa_pckod_d((v2i64) hz_out5, (v2i64) hz_out5);
1443 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out0, hz_out1, hz_out2,
1444 hz_out3, hz_out4, hz_out5);
1445 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_R_SH(hz_out1, hz_out2, hz_out3,
1446 hz_out4, hz_out5, hz_out6);
1449 res1 = __msa_srari_h(hz_out3, 5);
1450 res3 = __msa_srari_h(hz_out4, 5);
1452 res1 = __msa_srari_h(hz_out2, 5);
1453 res3 = __msa_srari_h(hz_out3, 5);
1456 SAT_SH2_SH(res1, res3, 7);
1458 res0 = __msa_aver_s_h(res0, res1);
1459 res1 = __msa_aver_s_h(res2, res3);
1461 vec0 = PCKEV_XORI128_UB(res0, res0);
1462 vec1 = PCKEV_XORI128_UB(res1, res1);
1464 AVER_UB2_UB(vec0, dst0, vec1, dst1, dst0, dst1);
1466 out0 = __msa_copy_u_w((v4i32) dst0, 0);
1467 out1 = __msa_copy_u_w((v4i32) dst1, 0);
1481 static void avc_luma_midv_qrt_and_aver_dst_8w_msa(const uint8_t *src,
1486 uint8_t vert_offset)
1489 v16i8 src0, src1, src2, src3, src4;
1490 v16u8 dst0, dst1, dst2, dst3;
1491 v16i8 mask0, mask1, mask2;
1492 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1493 v8i16 hz_out4, hz_out5, hz_out6, hz_out7, hz_out8;
1494 v8i16 res0, res1, res2, res3;
1495 v8i16 res4, res5, res6, res7;
1497 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1499 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
1500 XORI_B5_128_SB(src0, src1, src2, src3, src4);
1501 src += (5 * src_stride);
1503 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1504 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1505 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1506 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1507 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
1509 for (loop_cnt = (height >> 2); loop_cnt--;) {
1510 LD_SB4(src, src_stride, src0, src1, src2, src3);
1511 XORI_B4_128_SB(src0, src1, src2, src3);
1512 src += (4 * src_stride);
1514 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1516 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
1517 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
1518 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
1519 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
1521 res0 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out0, hz_out1, hz_out2,
1522 hz_out3, hz_out4, hz_out5);
1523 res2 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out1, hz_out2, hz_out3,
1524 hz_out4, hz_out5, hz_out6);
1525 res4 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out2, hz_out3, hz_out4,
1526 hz_out5, hz_out6, hz_out7);
1527 res6 = AVC_CALC_DPADD_H_6PIX_2COEFF_SH(hz_out3, hz_out4, hz_out5,
1528 hz_out6, hz_out7, hz_out8);
1531 res1 = __msa_srari_h(hz_out3, 5);
1532 res3 = __msa_srari_h(hz_out4, 5);
1533 res5 = __msa_srari_h(hz_out5, 5);
1534 res7 = __msa_srari_h(hz_out6, 5);
1536 res1 = __msa_srari_h(hz_out2, 5);
1537 res3 = __msa_srari_h(hz_out3, 5);
1538 res5 = __msa_srari_h(hz_out4, 5);
1539 res7 = __msa_srari_h(hz_out5, 5);
1542 SAT_SH4_SH(res1, res3, res5, res7, 7);
1544 res0 = __msa_aver_s_h(res0, res1);
1545 res1 = __msa_aver_s_h(res2, res3);
1546 res2 = __msa_aver_s_h(res4, res5);
1547 res3 = __msa_aver_s_h(res6, res7);
1548 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1549 CONVERT_UB_AVG_ST8x4_UB(res0, res1, res2, res3, dst0, dst1,
1551 dst += (4 * dst_stride);
1561 static void avc_luma_midv_qrt_and_aver_dst_16w_msa(const uint8_t *src,
1566 uint8_t vert_offset)
1568 int32_t multiple8_cnt;
1570 for (multiple8_cnt = 2; multiple8_cnt--;) {
1571 avc_luma_midv_qrt_and_aver_dst_8w_msa(src, src_stride, dst, dst_stride,
1572 height, vert_offset);
1579 static void avc_luma_hv_qrt_and_aver_dst_4x4_msa(const uint8_t *src_x,
1580 const uint8_t *src_y,
1585 v16i8 src_hz0, src_hz1, src_hz2, src_hz3;
1586 v16u8 dst0, dst1, dst2, dst3;
1587 v16i8 src_vt0, src_vt1, src_vt2, src_vt3, src_vt4;
1588 v16i8 src_vt5, src_vt6, src_vt7, src_vt8;
1589 v16i8 mask0, mask1, mask2;
1590 v8i16 hz_out0, hz_out1, vert_out0, vert_out1;
1594 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
1595 LD_SB5(src_y, src_stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1596 src_y += (5 * src_stride);
1598 src_vt0 = (v16i8) __msa_insve_w((v4i32) src_vt0, 1, (v4i32) src_vt1);
1599 src_vt1 = (v16i8) __msa_insve_w((v4i32) src_vt1, 1, (v4i32) src_vt2);
1600 src_vt2 = (v16i8) __msa_insve_w((v4i32) src_vt2, 1, (v4i32) src_vt3);
1601 src_vt3 = (v16i8) __msa_insve_w((v4i32) src_vt3, 1, (v4i32) src_vt4);
1603 XORI_B4_128_SB(src_vt0, src_vt1, src_vt2, src_vt3);
1604 LD_SB4(src_x, src_stride, src_hz0, src_hz1, src_hz2, src_hz3);
1605 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1606 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1607 hz_out0 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src_hz0, src_hz1,
1608 mask0, mask1, mask2);
1609 hz_out1 = AVC_XOR_VSHF_B_AND_APPLY_6TAP_HORIZ_FILT_SH(src_hz2, src_hz3,
1610 mask0, mask1, mask2);
1611 SRARI_H2_SH(hz_out0, hz_out1, 5);
1612 SAT_SH2_SH(hz_out0, hz_out1, 7);
1613 LD_SB4(src_y, src_stride, src_vt5, src_vt6, src_vt7, src_vt8);
1615 src_vt4 = (v16i8) __msa_insve_w((v4i32) src_vt4, 1, (v4i32) src_vt5);
1616 src_vt5 = (v16i8) __msa_insve_w((v4i32) src_vt5, 1, (v4i32) src_vt6);
1617 src_vt6 = (v16i8) __msa_insve_w((v4i32) src_vt6, 1, (v4i32) src_vt7);
1618 src_vt7 = (v16i8) __msa_insve_w((v4i32) src_vt7, 1, (v4i32) src_vt8);
1620 XORI_B4_128_SB(src_vt4, src_vt5, src_vt6, src_vt7);
1623 vert_out0 = AVC_CALC_DPADD_B_6PIX_2COEFF_R_SH(src_vt0, src_vt1, src_vt2,
1624 src_vt3, src_vt4, src_vt5);
1625 vert_out1 = AVC_CALC_DPADD_B_6PIX_2COEFF_R_SH(src_vt2, src_vt3, src_vt4,
1626 src_vt5, src_vt6, src_vt7);
1627 SRARI_H2_SH(vert_out0, vert_out1, 5);
1628 SAT_SH2_SH(vert_out0, vert_out1, 7);
1630 res1 = __msa_srari_h((hz_out1 + vert_out1), 1);
1631 res0 = __msa_srari_h((hz_out0 + vert_out0), 1);
1633 SAT_SH2_SH(res0, res1, 7);
1634 res = PCKEV_XORI128_UB(res0, res1);
1636 dst0 = (v16u8) __msa_insve_w((v4i32) dst0, 1, (v4i32) dst1);
1637 dst1 = (v16u8) __msa_insve_w((v4i32) dst2, 1, (v4i32) dst3);
1638 dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
1639 dst0 = __msa_aver_u_b(res, dst0);
1641 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, dst_stride);
1644 static void avc_luma_hv_qrt_and_aver_dst_8x8_msa(const uint8_t *src_x,
1645 const uint8_t *src_y,
1651 v16i8 src_hz0, src_hz1, src_hz2, src_hz3;
1652 v16u8 dst0, dst1, dst2, dst3;
1653 v16i8 src_vt0, src_vt1, src_vt2, src_vt3;
1654 v16i8 src_vt4, src_vt5, src_vt6, src_vt7, src_vt8;
1655 v16i8 mask0, mask1, mask2;
1656 v8i16 hz_out0, hz_out1, hz_out2, hz_out3;
1657 v8i16 vert_out0, vert_out1, vert_out2, vert_out3;
1658 v8i16 out0, out1, out2, out3;
1660 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1662 LD_SB5(src_y, src_stride, src_vt0, src_vt1, src_vt2, src_vt3, src_vt4);
1663 src_y += (5 * src_stride);
1665 src_vt0 = (v16i8) __msa_insve_d((v2i64) src_vt0, 1, (v2i64) src_vt1);
1666 src_vt1 = (v16i8) __msa_insve_d((v2i64) src_vt1, 1, (v2i64) src_vt2);
1667 src_vt2 = (v16i8) __msa_insve_d((v2i64) src_vt2, 1, (v2i64) src_vt3);
1668 src_vt3 = (v16i8) __msa_insve_d((v2i64) src_vt3, 1, (v2i64) src_vt4);
1670 XORI_B4_128_SB(src_vt0, src_vt1, src_vt2, src_vt3);
1672 for (loop_cnt = 2; loop_cnt--;) {
1673 LD_SB4(src_x, src_stride, src_hz0, src_hz1, src_hz2, src_hz3);
1674 XORI_B4_128_SB(src_hz0, src_hz1, src_hz2, src_hz3);
1675 src_x += (4 * src_stride);
1677 LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
1678 hz_out0 = AVC_HORZ_FILTER_SH(src_hz0, src_hz0, mask0, mask1, mask2);
1679 hz_out1 = AVC_HORZ_FILTER_SH(src_hz1, src_hz1, mask0, mask1, mask2);
1680 hz_out2 = AVC_HORZ_FILTER_SH(src_hz2, src_hz2, mask0, mask1, mask2);
1681 hz_out3 = AVC_HORZ_FILTER_SH(src_hz3, src_hz3, mask0, mask1, mask2);
1682 SRARI_H4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 5);
1683 SAT_SH4_SH(hz_out0, hz_out1, hz_out2, hz_out3, 7);
1684 LD_SB4(src_y, src_stride, src_vt5, src_vt6, src_vt7, src_vt8);
1685 src_y += (4 * src_stride);
1687 src_vt4 = (v16i8) __msa_insve_d((v2i64) src_vt4, 1, (v2i64) src_vt5);
1688 src_vt5 = (v16i8) __msa_insve_d((v2i64) src_vt5, 1, (v2i64) src_vt6);
1689 src_vt6 = (v16i8) __msa_insve_d((v2i64) src_vt6, 1, (v2i64) src_vt7);
1690 src_vt7 = (v16i8) __msa_insve_d((v2i64) src_vt7, 1, (v2i64) src_vt8);
1692 XORI_B4_128_SB(src_vt4, src_vt5, src_vt6, src_vt7);
1693 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src_vt0, src_vt1, src_vt2, src_vt3,
1694 src_vt4, src_vt5, vert_out0, vert_out1);
1695 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src_vt2, src_vt3, src_vt4, src_vt5,
1696 src_vt6, src_vt7, vert_out2, vert_out3);
1697 SRARI_H4_SH(vert_out0, vert_out1, vert_out2, vert_out3, 5);
1698 SAT_SH4_SH(vert_out0, vert_out1, vert_out2, vert_out3, 7);
1700 out0 = __msa_srari_h((hz_out0 + vert_out0), 1);
1701 out1 = __msa_srari_h((hz_out1 + vert_out1), 1);
1702 out2 = __msa_srari_h((hz_out2 + vert_out2), 1);
1703 out3 = __msa_srari_h((hz_out3 + vert_out3), 1);
1705 SAT_SH4_SH(out0, out1, out2, out3, 7);
1706 ILVR_D2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
1707 CONVERT_UB_AVG_ST8x4_UB(out0, out1, out2, out3, dst0, dst1,
1709 dst += (4 * dst_stride);
1719 static void avc_luma_hv_qrt_and_aver_dst_16x16_msa(const uint8_t *src_x,
1720 const uint8_t *src_y,
1725 uint32_t multiple8_cnt;
1727 for (multiple8_cnt = 2; multiple8_cnt--;) {
1728 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src_x, src_y, src_stride,
1736 src_x += (8 * src_stride) - 16;
1737 src_y += (8 * src_stride) - 16;
1738 dst += (8 * dst_stride) - 16;
1740 for (multiple8_cnt = 2; multiple8_cnt--;) {
1741 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src_x, src_y, src_stride,
1750 void ff_put_h264_qpel16_mc00_msa(uint8_t *dst, const uint8_t *src,
1753 v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
1754 v16u8 src8, src9, src10, src11, src12, src13, src14, src15;
1756 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1757 src += (8 * stride);
1758 LD_UB8(src, stride, src8, src9, src10, src11, src12, src13, src14, src15);
1760 ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst, stride);
1761 dst += (8 * stride);
1762 ST_UB8(src8, src9, src10, src11, src12, src13, src14, src15, dst, stride);
1765 void ff_put_h264_qpel8_mc00_msa(uint8_t *dst, const uint8_t *src,
1768 uint64_t src0, src1, src2, src3, src4, src5, src6, src7;
1770 LD4(src, stride, src0, src1, src2, src3);
1772 LD4(src, stride, src4, src5, src6, src7);
1773 SD4(src0, src1, src2, src3, dst, stride);
1775 SD4(src4, src5, src6, src7, dst, stride);
1778 void ff_avg_h264_qpel16_mc00_msa(uint8_t *dst, const uint8_t *src,
1781 v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
1782 v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1784 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1785 src += (8 * stride);
1786 LD_UB8(dst, stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1788 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1790 AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5,
1792 ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, stride);
1793 dst += (8 * stride);
1795 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1796 LD_UB8(dst, stride, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1798 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1800 AVER_UB4_UB(src4, dst4, src5, dst5, src6, dst6, src7, dst7, dst4, dst5,
1802 ST_UB8(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst, stride);
1805 void ff_avg_h264_qpel8_mc00_msa(uint8_t *dst, const uint8_t *src,
1808 uint64_t tp0, tp1, tp2, tp3, tp4, tp5, tp6, tp7;
1809 v16u8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 }, src3 = { 0 };
1810 v16u8 dst0 = { 0 }, dst1 = { 0 }, dst2 = { 0 }, dst3 = { 0 };
1812 LD4(src, stride, tp0, tp1, tp2, tp3);
1814 LD4(src, stride, tp4, tp5, tp6, tp7);
1815 INSERT_D2_UB(tp0, tp1, src0);
1816 INSERT_D2_UB(tp2, tp3, src1);
1817 INSERT_D2_UB(tp4, tp5, src2);
1818 INSERT_D2_UB(tp6, tp7, src3);
1820 LD4(dst, stride, tp0, tp1, tp2, tp3);
1821 LD4(dst + 4 * stride, stride, tp4, tp5, tp6, tp7);
1822 INSERT_D2_UB(tp0, tp1, dst0);
1823 INSERT_D2_UB(tp2, tp3, dst1);
1824 INSERT_D2_UB(tp4, tp5, dst2);
1825 INSERT_D2_UB(tp6, tp7, dst3);
1827 AVER_UB4_UB(src0, dst0, src1, dst1, src2, dst2, src3, dst3, dst0, dst1,
1830 ST8x8_UB(dst0, dst1, dst2, dst3, dst, stride);
1833 void ff_avg_h264_qpel4_mc00_msa(uint8_t *dst, const uint8_t *src,
1836 uint32_t tp0, tp1, tp2, tp3;
1837 v16u8 src0 = { 0 }, dst0 = { 0 };
1839 LW4(src, stride, tp0, tp1, tp2, tp3);
1840 INSERT_W4_UB(tp0, tp1, tp2, tp3, src0);
1841 LW4(dst, stride, tp0, tp1, tp2, tp3);
1842 INSERT_W4_UB(tp0, tp1, tp2, tp3, dst0);
1844 dst0 = __msa_aver_u_b(src0, dst0);
1846 ST4x4_UB(dst0, dst0, 0, 1, 2, 3, dst, stride);
1849 void ff_put_h264_qpel16_mc10_msa(uint8_t *dst, const uint8_t *src,
1853 v16i8 dst0, dst1, dst2, dst3, src0, src1, src2, src3, src4, src5, src6;
1854 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, src7, vec11;
1855 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1856 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1857 v16i8 minus5b = __msa_ldi_b(-5);
1858 v16i8 plus20b = __msa_ldi_b(20);
1860 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1866 for (loop_cnt = 4; loop_cnt--;) {
1867 LD_SB2(src, 16, src0, src1);
1869 LD_SB2(src, 16, src2, src3);
1871 LD_SB2(src, 16, src4, src5);
1873 LD_SB2(src, 16, src6, src7);
1876 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1877 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask3, vec0, vec3);
1878 VSHF_B2_SB(src2, src2, src2, src3, mask0, mask3, vec6, vec9);
1879 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask4, vec1, vec4);
1880 VSHF_B2_SB(src2, src2, src2, src3, mask1, mask4, vec7, vec10);
1881 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask5, vec2, vec5);
1882 VSHF_B2_SB(src2, src2, src2, src3, mask2, mask5, vec8, vec11);
1883 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
1884 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1885 minus5b, res0, res1, res2, res3);
1886 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1887 plus20b, res0, res1, res2, res3);
1888 VSHF_B2_SB(src4, src4, src4, src5, mask0, mask3, vec0, vec3);
1889 VSHF_B2_SB(src6, src6, src6, src7, mask0, mask3, vec6, vec9);
1890 VSHF_B2_SB(src4, src4, src4, src5, mask1, mask4, vec1, vec4);
1891 VSHF_B2_SB(src6, src6, src6, src7, mask1, mask4, vec7, vec10);
1892 VSHF_B2_SB(src4, src4, src4, src5, mask2, mask5, vec2, vec5);
1893 VSHF_B2_SB(src6, src6, src6, src7, mask2, mask5, vec8, vec11);
1894 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
1895 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1896 minus5b, res4, res5, res6, res7);
1897 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1898 plus20b, res4, res5, res6, res7);
1899 SLDI_B2_SB(src1, src3, src0, src2, src0, src2, 2);
1900 SLDI_B2_SB(src5, src7, src4, src6, src4, src6, 2);
1901 SRARI_H4_SH(res0, res1, res2, res3, 5);
1902 SRARI_H4_SH(res4, res5, res6, res7, 5);
1903 SAT_SH4_SH(res0, res1, res2, res3, 7);
1904 SAT_SH4_SH(res4, res5, res6, res7, 7);
1905 PCKEV_B2_SB(res1, res0, res3, res2, dst0, dst1);
1906 PCKEV_B2_SB(res5, res4, res7, res6, dst2, dst3);
1907 dst0 = __msa_aver_s_b(dst0, src0);
1908 dst1 = __msa_aver_s_b(dst1, src2);
1909 dst2 = __msa_aver_s_b(dst2, src4);
1910 dst3 = __msa_aver_s_b(dst3, src6);
1911 XORI_B4_128_SB(dst0, dst1, dst2, dst3);
1912 ST_SB4(dst0, dst1, dst2, dst3, dst, stride);
1913 dst += (4 * stride);
1917 void ff_put_h264_qpel16_mc30_msa(uint8_t *dst, const uint8_t *src,
1921 v16i8 dst0, dst1, dst2, dst3, src0, src1, src2, src3, src4, src5, src6;
1922 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, src7, vec11;
1923 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1924 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1925 v16i8 minus5b = __msa_ldi_b(-5);
1926 v16i8 plus20b = __msa_ldi_b(20);
1928 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1934 for (loop_cnt = 4; loop_cnt--;) {
1935 LD_SB2(src, 16, src0, src1);
1937 LD_SB2(src, 16, src2, src3);
1939 LD_SB2(src, 16, src4, src5);
1941 LD_SB2(src, 16, src6, src7);
1944 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1945 VSHF_B2_SB(src0, src0, src0, src1, mask0, mask3, vec0, vec3);
1946 VSHF_B2_SB(src2, src2, src2, src3, mask0, mask3, vec6, vec9);
1947 VSHF_B2_SB(src0, src0, src0, src1, mask1, mask4, vec1, vec4);
1948 VSHF_B2_SB(src2, src2, src2, src3, mask1, mask4, vec7, vec10);
1949 VSHF_B2_SB(src0, src0, src0, src1, mask2, mask5, vec2, vec5);
1950 VSHF_B2_SB(src2, src2, src2, src3, mask2, mask5, vec8, vec11);
1951 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
1952 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1953 minus5b, res0, res1, res2, res3);
1954 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1955 plus20b, res0, res1, res2, res3);
1956 VSHF_B2_SB(src4, src4, src4, src5, mask0, mask3, vec0, vec3);
1957 VSHF_B2_SB(src6, src6, src6, src7, mask0, mask3, vec6, vec9);
1958 VSHF_B2_SB(src4, src4, src4, src5, mask1, mask4, vec1, vec4);
1959 VSHF_B2_SB(src6, src6, src6, src7, mask1, mask4, vec7, vec10);
1960 VSHF_B2_SB(src4, src4, src4, src5, mask2, mask5, vec2, vec5);
1961 VSHF_B2_SB(src6, src6, src6, src7, mask2, mask5, vec8, vec11);
1962 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
1963 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
1964 minus5b, res4, res5, res6, res7);
1965 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
1966 plus20b, res4, res5, res6, res7);
1967 SLDI_B2_SB(src1, src3, src0, src2, src0, src2, 3);
1968 SLDI_B2_SB(src5, src7, src4, src6, src4, src6, 3);
1969 SRARI_H4_SH(res0, res1, res2, res3, 5);
1970 SRARI_H4_SH(res4, res5, res6, res7, 5);
1971 SAT_SH4_SH(res0, res1, res2, res3, 7);
1972 SAT_SH4_SH(res4, res5, res6, res7, 7);
1973 PCKEV_B2_SB(res1, res0, res3, res2, dst0, dst1);
1974 PCKEV_B2_SB(res5, res4, res7, res6, dst2, dst3);
1975 dst0 = __msa_aver_s_b(dst0, src0);
1976 dst1 = __msa_aver_s_b(dst1, src2);
1977 dst2 = __msa_aver_s_b(dst2, src4);
1978 dst3 = __msa_aver_s_b(dst3, src6);
1979 XORI_B4_128_SB(dst0, dst1, dst2, dst3);
1980 ST_SB4(dst0, dst1, dst2, dst3, dst, stride);
1981 dst += (4 * stride);
1985 void ff_put_h264_qpel8_mc10_msa(uint8_t *dst, const uint8_t *src,
1988 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
1989 v16i8 tmp0, tmp1, tmp2, tmp3, vec11;
1990 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
1991 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
1992 v16i8 minus5b = __msa_ldi_b(-5);
1993 v16i8 plus20b = __msa_ldi_b(20);
1995 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
1996 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1997 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
1998 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
1999 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2000 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
2001 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
2002 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
2003 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2004 res0, res1, res2, res3);
2005 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
2006 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
2007 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2008 res0, res1, res2, res3);
2009 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2010 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
2011 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
2012 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
2013 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
2014 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2015 res4, res5, res6, res7);
2016 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
2017 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
2018 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2019 res4, res5, res6, res7);
2020 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 2);
2021 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 2);
2022 SLDI_B2_SB(src4, src5, src4, src5, src4, src5, 2);
2023 SLDI_B2_SB(src6, src7, src6, src7, src6, src7, 2);
2024 PCKEV_D2_SB(src1, src0, src3, src2, src0, src1);
2025 PCKEV_D2_SB(src5, src4, src7, src6, src4, src5);
2026 SRARI_H4_SH(res0, res1, res2, res3, 5);
2027 SRARI_H4_SH(res4, res5, res6, res7, 5);
2028 SAT_SH4_SH(res0, res1, res2, res3, 7);
2029 SAT_SH4_SH(res4, res5, res6, res7, 7);
2030 PCKEV_B2_SB(res1, res0, res3, res2, tmp0, tmp1);
2031 PCKEV_B2_SB(res5, res4, res7, res6, tmp2, tmp3);
2032 tmp0 = __msa_aver_s_b(tmp0, src0);
2033 tmp1 = __msa_aver_s_b(tmp1, src1);
2034 tmp2 = __msa_aver_s_b(tmp2, src4);
2035 tmp3 = __msa_aver_s_b(tmp3, src5);
2036 XORI_B4_128_SB(tmp0, tmp1, tmp2, tmp3);
2037 ST8x8_UB(tmp0, tmp1, tmp2, tmp3, dst, stride);
2040 void ff_put_h264_qpel8_mc30_msa(uint8_t *dst, const uint8_t *src,
2043 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2044 v16i8 tmp0, tmp1, tmp2, tmp3, vec11;
2045 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2046 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2047 v16i8 minus5b = __msa_ldi_b(-5);
2048 v16i8 plus20b = __msa_ldi_b(20);
2050 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2051 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
2052 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2053 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2054 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2055 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
2056 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
2057 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
2058 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2059 res0, res1, res2, res3);
2060 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
2061 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
2062 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2063 res0, res1, res2, res3);
2064 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2065 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
2066 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
2067 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
2068 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
2069 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2070 res4, res5, res6, res7);
2071 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
2072 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
2073 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b, plus20b,
2074 res4, res5, res6, res7);
2075 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 3);
2076 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 3);
2077 SLDI_B2_SB(src4, src5, src4, src5, src4, src5, 3);
2078 SLDI_B2_SB(src6, src7, src6, src7, src6, src7, 3);
2079 PCKEV_D2_SB(src1, src0, src3, src2, src0, src1);
2080 PCKEV_D2_SB(src5, src4, src7, src6, src4, src5);
2081 SRARI_H4_SH(res0, res1, res2, res3, 5);
2082 SRARI_H4_SH(res4, res5, res6, res7, 5);
2083 SAT_SH4_SH(res0, res1, res2, res3, 7);
2084 SAT_SH4_SH(res4, res5, res6, res7, 7);
2085 PCKEV_B2_SB(res1, res0, res3, res2, tmp0, tmp1);
2086 PCKEV_B2_SB(res5, res4, res7, res6, tmp2, tmp3);
2087 tmp0 = __msa_aver_s_b(tmp0, src0);
2088 tmp1 = __msa_aver_s_b(tmp1, src1);
2089 tmp2 = __msa_aver_s_b(tmp2, src4);
2090 tmp3 = __msa_aver_s_b(tmp3, src5);
2091 XORI_B4_128_SB(tmp0, tmp1, tmp2, tmp3);
2092 ST8x8_UB(tmp0, tmp1, tmp2, tmp3, dst, stride);
2095 void ff_put_h264_qpel4_mc10_msa(uint8_t *dst, const uint8_t *src,
2098 v16i8 src0, src1, src2, src3, res, mask0, mask1, mask2;
2099 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2101 v16i8 minus5b = __msa_ldi_b(-5);
2102 v16i8 plus20b = __msa_ldi_b(20);
2104 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2105 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2106 XORI_B4_128_SB(src0, src1, src2, src3);
2107 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2108 HADD_SB2_SH(vec0, vec1, res0, res1);
2109 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2110 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2111 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2112 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2113 SRARI_H2_SH(res0, res1, 5);
2114 SAT_SH2_SH(res0, res1, 7);
2115 res = __msa_pckev_b((v16i8) res1, (v16i8) res0);
2116 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 2);
2117 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 2);
2118 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
2119 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2120 src0 = (v16i8) __msa_insve_d((v2i64) src0, 1, (v2i64) src1);
2121 res = __msa_aver_s_b(res, src0);
2122 res = (v16i8) __msa_xori_b((v16u8) res, 128);
2123 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
2126 void ff_put_h264_qpel4_mc30_msa(uint8_t *dst, const uint8_t *src,
2129 v16i8 src0, src1, src2, src3, res, mask0, mask1, mask2;
2130 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2132 v16i8 minus5b = __msa_ldi_b(-5);
2133 v16i8 plus20b = __msa_ldi_b(20);
2135 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2136 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2137 XORI_B4_128_SB(src0, src1, src2, src3);
2138 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2139 HADD_SB2_SH(vec0, vec1, res0, res1);
2140 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2141 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2142 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2143 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2144 SRARI_H2_SH(res0, res1, 5);
2145 SAT_SH2_SH(res0, res1, 7);
2146 res = __msa_pckev_b((v16i8) res1, (v16i8) res0);
2147 SLDI_B2_SB(src0, src1, src0, src1, src0, src1, 3);
2148 SLDI_B2_SB(src2, src3, src2, src3, src2, src3, 3);
2149 src0 = (v16i8) __msa_insve_w((v4i32) src0, 1, (v4i32) src1);
2150 src1 = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2151 src0 = (v16i8) __msa_insve_d((v2i64) src0, 1, (v2i64) src1);
2152 res = __msa_aver_s_b(res, src0);
2153 res = (v16i8) __msa_xori_b((v16u8) res, 128);
2154 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
2157 void ff_put_h264_qpel16_mc20_msa(uint8_t *dst, const uint8_t *src,
2161 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2162 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2164 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2165 v16i8 minus5b = __msa_ldi_b(-5);
2166 v16i8 plus20b = __msa_ldi_b(20);
2168 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2171 for (loop_cnt = 4; loop_cnt--;) {
2172 LD_SB2(src, 8, src0, src1);
2174 LD_SB2(src, 8, src2, src3);
2176 LD_SB2(src, 8, src4, src5);
2178 LD_SB2(src, 8, src6, src7);
2181 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2182 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec3);
2183 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec9);
2184 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec1, vec4);
2185 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec7, vec10);
2186 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec2, vec5);
2187 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec8, vec11);
2188 HADD_SB4_SH(vec0, vec3, vec6, vec9, res0, res1, res2, res3);
2189 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
2190 minus5b, res0, res1, res2, res3);
2191 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
2192 plus20b, res0, res1, res2, res3);
2193 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec3);
2194 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec6, vec9);
2195 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec1, vec4);
2196 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec7, vec10);
2197 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec2, vec5);
2198 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec8, vec11);
2199 HADD_SB4_SH(vec0, vec3, vec6, vec9, res4, res5, res6, res7);
2200 DPADD_SB4_SH(vec1, vec4, vec7, vec10, minus5b, minus5b, minus5b,
2201 minus5b, res4, res5, res6, res7);
2202 DPADD_SB4_SH(vec2, vec5, vec8, vec11, plus20b, plus20b, plus20b,
2203 plus20b, res4, res5, res6, res7);
2204 SRARI_H4_SH(res0, res1, res2, res3, 5);
2205 SRARI_H4_SH(res4, res5, res6, res7, 5);
2206 SAT_SH4_SH(res0, res1, res2, res3, 7);
2207 SAT_SH4_SH(res4, res5, res6, res7, 7);
2208 PCKEV_B4_SB(res1, res0, res3, res2, res5, res4, res7, res6, vec0, vec1,
2210 XORI_B4_128_SB(vec0, vec1, vec2, vec3);
2211 ST_SB4(vec0, vec1, vec2, vec3, dst, stride);
2212 dst += (4 * stride);
2216 void ff_put_h264_qpel8_mc20_msa(uint8_t *dst, const uint8_t *src,
2219 v16u8 out0, out1, out2, out3;
2220 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, mask0, mask1, mask2;
2221 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
2223 v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
2224 v16i8 minus5b = __msa_ldi_b(-5);
2225 v16i8 plus20b = __msa_ldi_b(20);
2227 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2228 LD_SB8(src - 2, stride, src0, src1, src2, src3, src4, src5, src6, src7);
2229 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2230 VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2231 VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
2232 HADD_SB4_SH(vec0, vec1, vec2, vec3, res0, res1, res2, res3);
2233 VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
2234 VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
2235 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2236 res0, res1, res2, res3);
2237 VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec8, vec9);
2238 VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec10, vec11);
2239 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
2240 plus20b, res0, res1, res2, res3);
2241 VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
2242 VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
2243 HADD_SB4_SH(vec0, vec1, vec2, vec3, res4, res5, res6, res7);
2244 VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
2245 VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
2246 DPADD_SB4_SH(vec4, vec5, vec6, vec7, minus5b, minus5b, minus5b, minus5b,
2247 res4, res5, res6, res7);
2248 VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec8, vec9);
2249 VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec10, vec11);
2250 DPADD_SB4_SH(vec8, vec9, vec10, vec11, plus20b, plus20b, plus20b,
2251 plus20b, res4, res5, res6, res7);
2252 SRARI_H4_SH(res0, res1, res2, res3, 5);
2253 SRARI_H4_SH(res4, res5, res6, res7, 5);
2254 SAT_SH4_SH(res0, res1, res2, res3, 7);
2255 SAT_SH4_SH(res4, res5, res6, res7, 7);
2256 out0 = PCKEV_XORI128_UB(res0, res1);
2257 out1 = PCKEV_XORI128_UB(res2, res3);
2258 out2 = PCKEV_XORI128_UB(res4, res5);
2259 out3 = PCKEV_XORI128_UB(res6, res7);
2260 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2263 void ff_put_h264_qpel4_mc20_msa(uint8_t *dst, const uint8_t *src,
2267 v16i8 src0, src1, src2, src3, mask0, mask1, mask2;
2268 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
2270 v16i8 minus5b = __msa_ldi_b(-5);
2271 v16i8 plus20b = __msa_ldi_b(20);
2273 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
2274 LD_SB4(src - 2, stride, src0, src1, src2, src3);
2275 XORI_B4_128_SB(src0, src1, src2, src3);
2276 VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0, vec1);
2277 HADD_SB2_SH(vec0, vec1, res0, res1);
2278 VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2, vec3);
2279 DPADD_SB2_SH(vec2, vec3, minus5b, minus5b, res0, res1);
2280 VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4, vec5);
2281 DPADD_SB2_SH(vec4, vec5, plus20b, plus20b, res0, res1);
2282 SRARI_H2_SH(res0, res1, 5);
2283 SAT_SH2_SH(res0, res1, 7);
2284 out = PCKEV_XORI128_UB(res0, res1);
2285 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2288 void ff_put_h264_qpel16_mc01_msa(uint8_t *dst, const uint8_t *src,
2292 int16_t filt_const0 = 0xfb01;
2293 int16_t filt_const1 = 0x1414;
2294 int16_t filt_const2 = 0x1fb;
2295 v16u8 res0, res1, res2, res3;
2296 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2297 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2298 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
2299 v16i8 src65_l, src87_l, filt0, filt1, filt2;
2300 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
2302 filt0 = (v16i8) __msa_fill_h(filt_const0);
2303 filt1 = (v16i8) __msa_fill_h(filt_const1);
2304 filt2 = (v16i8) __msa_fill_h(filt_const2);
2306 src -= (stride * 2);
2308 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2309 src += (5 * stride);
2311 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2312 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2314 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
2317 for (loop_cnt = 4; loop_cnt--;) {
2318 LD_SB4(src, stride, src5, src6, src7, src8);
2319 src += (4 * stride);
2321 XORI_B4_128_SB(src5, src6, src7, src8);
2322 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
2323 src65_r, src76_r, src87_r);
2324 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
2325 src65_l, src76_l, src87_l);
2326 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2327 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2328 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2329 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2330 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
2331 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
2332 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
2333 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
2334 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2335 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2336 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
2337 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
2338 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
2339 out3_r, res0, res1, res2, res3);
2340 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src2);
2341 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src3);
2342 res2 = (v16u8) __msa_aver_s_b((v16i8) res2, src4);
2343 res3 = (v16u8) __msa_aver_s_b((v16i8) res3, src5);
2344 XORI_B4_128_UB(res0, res1, res2, res3);
2345 ST_UB4(res0, res1, res2, res3, dst, stride);
2346 dst += (4 * stride);
2362 void ff_put_h264_qpel16_mc03_msa(uint8_t *dst, const uint8_t *src,
2366 int16_t filt_const0 = 0xfb01;
2367 int16_t filt_const1 = 0x1414;
2368 int16_t filt_const2 = 0x1fb;
2369 v16u8 res0, res1, res2, res3;
2370 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2371 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2372 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
2373 v16i8 src65_l, src87_l, filt0, filt1, filt2;
2374 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
2376 filt0 = (v16i8) __msa_fill_h(filt_const0);
2377 filt1 = (v16i8) __msa_fill_h(filt_const1);
2378 filt2 = (v16i8) __msa_fill_h(filt_const2);
2380 src -= (stride * 2);
2382 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2383 src += (5 * stride);
2385 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2386 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2388 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
2391 for (loop_cnt = 4; loop_cnt--;) {
2392 LD_SB4(src, stride, src5, src6, src7, src8);
2393 src += (4 * stride);
2395 XORI_B4_128_SB(src5, src6, src7, src8);
2396 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
2397 src65_r, src76_r, src87_r);
2398 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
2399 src65_l, src76_l, src87_l);
2400 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2401 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2402 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2403 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2404 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
2405 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
2406 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
2407 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
2408 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2409 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2410 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
2411 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
2412 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
2413 out3_r, res0, res1, res2, res3);
2414 res0 = (v16u8) __msa_aver_s_b((v16i8) res0, src3);
2415 res1 = (v16u8) __msa_aver_s_b((v16i8) res1, src4);
2416 res2 = (v16u8) __msa_aver_s_b((v16i8) res2, src5);
2417 res3 = (v16u8) __msa_aver_s_b((v16i8) res3, src6);
2418 XORI_B4_128_UB(res0, res1, res2, res3);
2419 ST_UB4(res0, res1, res2, res3, dst, stride);
2420 dst += (4 * stride);
2435 void ff_put_h264_qpel8_mc01_msa(uint8_t *dst, const uint8_t *src,
2438 const int16_t filt_const0 = 0xfb01;
2439 const int16_t filt_const1 = 0x1414;
2440 const int16_t filt_const2 = 0x1fb;
2441 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2442 v16i8 src11, src12, src10_r, src32_r, src54_r, src65_r, src76_r, src98_r;
2443 v16i8 src21_r, src43_r, src87_r, src109_r, src1211_r, src1110_r;
2444 v16i8 tmp0, tmp1, tmp2, tmp3, filt0, filt1, filt2, out0, out1, out2, out3;
2445 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
2447 filt0 = (v16i8) __msa_fill_h(filt_const0);
2448 filt1 = (v16i8) __msa_fill_h(filt_const1);
2449 filt2 = (v16i8) __msa_fill_h(filt_const2);
2451 src -= (stride * 2);
2453 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2454 src += (5 * stride);
2455 LD_SB8(src, stride, src5, src6, src7, src8, src9, src10, src11, src12);
2456 XORI_B8_128_SB(src5, src6, src7, src8, src9, src10, src11, src12);
2457 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2458 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2460 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2462 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src98_r,
2463 src109_r, src1110_r, src1211_r);
2464 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2465 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2466 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2467 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2468 out4_r = AVC_DOT_SH3_SH(src54_r, src76_r, src98_r, filt0, filt1, filt2);
2469 out5_r = AVC_DOT_SH3_SH(src65_r, src87_r, src109_r, filt0, filt1, filt2);
2470 out6_r = AVC_DOT_SH3_SH(src76_r, src98_r, src1110_r, filt0, filt1, filt2);
2471 out7_r = AVC_DOT_SH3_SH(src87_r, src109_r, src1211_r, filt0, filt1, filt2);
2472 PCKEV_D2_SB(src3, src2, src5, src4, tmp0, tmp1);
2473 PCKEV_D2_SB(src7, src6, src9, src8, tmp2, tmp3);
2474 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2475 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
2476 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2477 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
2478 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, out0, out1);
2479 PCKEV_B2_SB(out5_r, out4_r, out7_r, out6_r, out2, out3);
2480 out0 = __msa_aver_s_b(out0, tmp0);
2481 out1 = __msa_aver_s_b(out1, tmp1);
2482 out2 = __msa_aver_s_b(out2, tmp2);
2483 out3 = __msa_aver_s_b(out3, tmp3);
2484 XORI_B4_128_SB(out0, out1, out2, out3);
2485 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2488 void ff_put_h264_qpel8_mc03_msa(uint8_t *dst, const uint8_t *src,
2491 const int16_t filt_const0 = 0xfb01;
2492 const int16_t filt_const1 = 0x1414;
2493 const int16_t filt_const2 = 0x1fb;
2494 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2495 v16i8 src11, src12, src10_r, src32_r, src54_r, src65_r, src76_r, src98_r;
2496 v16i8 src21_r, src43_r, src87_r, src109_r, src1211_r, src1110_r;
2497 v16i8 filt0, filt1, filt2, out0, out1, out2, out3, tmp0, tmp1, tmp2, tmp3;
2498 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
2500 filt0 = (v16i8) __msa_fill_h(filt_const0);
2501 filt1 = (v16i8) __msa_fill_h(filt_const1);
2502 filt2 = (v16i8) __msa_fill_h(filt_const2);
2504 src -= (stride * 2);
2506 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2507 src += (5 * stride);
2508 LD_SB8(src, stride, src5, src6, src7, src8, src9, src10, src11, src12);
2509 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2510 XORI_B8_128_SB(src5, src6, src7, src8, src9, src10, src11, src12);
2511 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2513 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2515 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src98_r,
2516 src109_r, src1110_r, src1211_r);
2517 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
2518 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
2519 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
2520 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
2521 out4_r = AVC_DOT_SH3_SH(src54_r, src76_r, src98_r, filt0, filt1, filt2);
2522 out5_r = AVC_DOT_SH3_SH(src65_r, src87_r, src109_r, filt0, filt1, filt2);
2523 out6_r = AVC_DOT_SH3_SH(src76_r, src98_r, src1110_r, filt0, filt1, filt2);
2524 out7_r = AVC_DOT_SH3_SH(src87_r, src109_r, src1211_r, filt0, filt1, filt2);
2525 PCKEV_D2_SB(src4, src3, src6, src5, tmp0, tmp1);
2526 PCKEV_D2_SB(src8, src7, src10, src9, tmp2, tmp3);
2527 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
2528 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
2529 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2530 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
2531 PCKEV_B2_SB(out1_r, out0_r, out3_r, out2_r, out0, out1);
2532 PCKEV_B2_SB(out5_r, out4_r, out7_r, out6_r, out2, out3);
2533 out0 = __msa_aver_s_b(out0, tmp0);
2534 out1 = __msa_aver_s_b(out1, tmp1);
2535 out2 = __msa_aver_s_b(out2, tmp2);
2536 out3 = __msa_aver_s_b(out3, tmp3);
2537 XORI_B4_128_SB(out0, out1, out2, out3);
2538 ST8x8_UB(out0, out1, out2, out3, dst, stride);
2541 void ff_put_h264_qpel4_mc01_msa(uint8_t *dst, const uint8_t *src,
2544 int16_t filt_const0 = 0xfb01;
2545 int16_t filt_const1 = 0x1414;
2546 int16_t filt_const2 = 0x1fb;
2548 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2549 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2550 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
2553 filt0 = (v16i8) __msa_fill_h(filt_const0);
2554 filt1 = (v16i8) __msa_fill_h(filt_const1);
2555 filt2 = (v16i8) __msa_fill_h(filt_const2);
2557 src -= (stride * 2);
2559 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2560 src += (5 * stride);
2561 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2563 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
2564 XORI_B2_128_SB(src2110, src4332);
2565 LD_SB4(src, stride, src5, src6, src7, src8);
2566 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2568 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
2569 XORI_B2_128_SB(src6554, src8776);
2570 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
2571 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
2572 SRARI_H2_SH(out10, out32, 5);
2573 SAT_SH2_SH(out10, out32, 7);
2574 out = PCKEV_XORI128_UB(out10, out32);
2575 src32_r = (v16i8) __msa_insve_w((v4i32) src2, 1, (v4i32) src3);
2576 src54_r = (v16i8) __msa_insve_w((v4i32) src4, 1, (v4i32) src5);
2577 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
2578 out = __msa_aver_u_b(out, (v16u8) src32_r);
2579 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2582 void ff_put_h264_qpel4_mc03_msa(uint8_t *dst, const uint8_t *src,
2585 int16_t filt_const0 = 0xfb01;
2586 int16_t filt_const1 = 0x1414;
2587 int16_t filt_const2 = 0x1fb;
2589 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2590 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
2591 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
2594 filt0 = (v16i8) __msa_fill_h(filt_const0);
2595 filt1 = (v16i8) __msa_fill_h(filt_const1);
2596 filt2 = (v16i8) __msa_fill_h(filt_const2);
2598 src -= (stride * 2);
2600 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2601 src += (5 * stride);
2602 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
2604 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
2605 XORI_B2_128_SB(src2110, src4332);
2606 LD_SB4(src, stride, src5, src6, src7, src8);
2607 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
2609 ILVR_D2_SB(src65_r, src54_r, src87_r, src76_r, src6554, src8776);
2610 XORI_B2_128_SB(src6554, src8776);
2611 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
2612 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
2613 SRARI_H2_SH(out10, out32, 5);
2614 SAT_SH2_SH(out10, out32, 7);
2615 out = PCKEV_XORI128_UB(out10, out32);
2616 src32_r = (v16i8) __msa_insve_w((v4i32) src3, 1, (v4i32) src4);
2617 src54_r = (v16i8) __msa_insve_w((v4i32) src5, 1, (v4i32) src6);
2618 src32_r = (v16i8) __msa_insve_d((v2i64) src32_r, 1, (v2i64) src54_r);
2619 out = __msa_aver_u_b(out, (v16u8) src32_r);
2620 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
2623 void ff_put_h264_qpel16_mc11_msa(uint8_t *dst, const uint8_t *src,
2626 avc_luma_hv_qrt_16x16_msa(src - 2, src - (stride * 2), dst, stride);
2629 void ff_put_h264_qpel16_mc31_msa(uint8_t *dst, const uint8_t *src,
2632 avc_luma_hv_qrt_16x16_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2635 void ff_put_h264_qpel16_mc13_msa(uint8_t *dst, const uint8_t *src,
2638 avc_luma_hv_qrt_16x16_msa(src + stride - 2, src - (stride * 2), dst,
2642 void ff_put_h264_qpel16_mc33_msa(uint8_t *dst, const uint8_t *src,
2645 avc_luma_hv_qrt_16x16_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2649 void ff_put_h264_qpel8_mc11_msa(uint8_t *dst, const uint8_t *src,
2652 avc_luma_hv_qrt_8x8_msa(src - 2, src - (stride * 2), dst, stride);
2655 void ff_put_h264_qpel8_mc31_msa(uint8_t *dst, const uint8_t *src,
2658 avc_luma_hv_qrt_8x8_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2661 void ff_put_h264_qpel8_mc13_msa(uint8_t *dst, const uint8_t *src,
2664 avc_luma_hv_qrt_8x8_msa(src + stride - 2, src - (stride * 2), dst, stride);
2667 void ff_put_h264_qpel8_mc33_msa(uint8_t *dst, const uint8_t *src,
2670 avc_luma_hv_qrt_8x8_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2675 void ff_put_h264_qpel4_mc11_msa(uint8_t *dst, const uint8_t *src,
2678 avc_luma_hv_qrt_4x4_msa(src - 2, src - (stride * 2), dst, stride);
2681 void ff_put_h264_qpel4_mc31_msa(uint8_t *dst, const uint8_t *src,
2684 avc_luma_hv_qrt_4x4_msa(src - 2, src - (stride * 2) + 1, dst, stride);
2687 void ff_put_h264_qpel4_mc13_msa(uint8_t *dst, const uint8_t *src,
2690 avc_luma_hv_qrt_4x4_msa(src + stride - 2, src - (stride * 2), dst, stride);
2693 void ff_put_h264_qpel4_mc33_msa(uint8_t *dst, const uint8_t *src,
2696 avc_luma_hv_qrt_4x4_msa(src + stride - 2, src - (stride * 2) + 1, dst,
2700 void ff_put_h264_qpel16_mc21_msa(uint8_t *dst, const uint8_t *src,
2703 uint8_t *dst_tmp = dst;
2704 const uint8_t *src_tmp = src - (2 * stride) - 2;
2705 uint32_t multiple8_cnt, loop_cnt;
2706 const int32_t filt_const0 = 0xfffb0001;
2707 const int32_t filt_const1 = 0x140014;
2708 const int32_t filt_const2 = 0x1fffb;
2710 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask0, mask1;
2712 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2713 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2714 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2715 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
2716 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
2717 v8i16 hz_out87_l, filt0, filt1, filt2;
2720 filt0 = (v8i16) __msa_fill_w(filt_const0);
2721 filt1 = (v8i16) __msa_fill_w(filt_const1);
2722 filt2 = (v8i16) __msa_fill_w(filt_const2);
2724 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2726 for (multiple8_cnt = 2; multiple8_cnt--;) {
2730 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2731 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2732 src += (5 * stride);
2734 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2735 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2736 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2737 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2738 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2740 for (loop_cnt = 4; loop_cnt--;) {
2741 LD_SB4(src, stride, src5, src6, src7, src8);
2742 src += (4 * stride);
2744 XORI_B4_128_SB(src5, src6, src7, src8);
2746 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2747 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2748 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2749 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2751 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2752 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
2754 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2755 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
2757 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2758 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
2760 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2761 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
2764 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
2766 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
2768 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2769 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
2771 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
2773 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2774 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
2776 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
2778 dst4 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2779 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
2781 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
2783 dst6 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2785 dst1 = __msa_srari_h(hz_out2, 5);
2786 dst3 = __msa_srari_h(hz_out3, 5);
2787 dst5 = __msa_srari_h(hz_out4, 5);
2788 dst7 = __msa_srari_h(hz_out5, 5);
2789 SAT_SH4_SH(dst1, dst3, dst5, dst7, 7);
2791 dst0 = __msa_aver_s_h(dst0, dst1);
2792 dst1 = __msa_aver_s_h(dst2, dst3);
2793 dst2 = __msa_aver_s_h(dst4, dst5);
2794 dst3 = __msa_aver_s_h(dst6, dst7);
2796 out0 = PCKEV_XORI128_UB(dst0, dst1);
2797 out1 = PCKEV_XORI128_UB(dst2, dst3);
2798 ST8x4_UB(out0, out1, dst, stride);
2799 dst += (4 * stride);
2813 void ff_put_h264_qpel16_mc23_msa(uint8_t *dst, const uint8_t *src,
2816 uint8_t *dst_tmp = dst;
2817 const uint8_t *src_tmp = src - (2 * stride) - 2;
2818 uint32_t multiple8_cnt, loop_cnt;
2819 const int32_t filt_const0 = 0xfffb0001;
2820 const int32_t filt_const1 = 0x140014;
2821 const int32_t filt_const2 = 0x1fffb;
2823 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, mask0, mask1;
2825 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2826 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2827 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2828 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
2829 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
2830 v8i16 hz_out87_l, filt0, filt1, filt2;
2833 filt0 = (v8i16) __msa_fill_w(filt_const0);
2834 filt1 = (v8i16) __msa_fill_w(filt_const1);
2835 filt2 = (v8i16) __msa_fill_w(filt_const2);
2837 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2839 for (multiple8_cnt = 2; multiple8_cnt--;) {
2843 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2844 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2845 src += (5 * stride);
2847 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2848 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2849 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2850 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2851 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2853 for (loop_cnt = 4; loop_cnt--;) {
2854 LD_SB4(src, stride, src5, src6, src7, src8);
2855 src += (4 * stride);
2857 XORI_B4_128_SB(src5, src6, src7, src8);
2859 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2860 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2861 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2862 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2864 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2865 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
2867 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
2868 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
2870 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2871 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
2873 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
2874 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
2877 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
2879 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
2881 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2882 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
2884 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
2886 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2887 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
2889 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
2891 dst4 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2892 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
2894 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
2896 dst6 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2898 dst1 = __msa_srari_h(hz_out3, 5);
2899 dst3 = __msa_srari_h(hz_out4, 5);
2900 dst5 = __msa_srari_h(hz_out5, 5);
2901 dst7 = __msa_srari_h(hz_out6, 5);
2902 SAT_SH4_SH(dst1, dst3, dst5, dst7, 7);
2904 dst0 = __msa_aver_s_h(dst0, dst1);
2905 dst1 = __msa_aver_s_h(dst2, dst3);
2906 dst2 = __msa_aver_s_h(dst4, dst5);
2907 dst3 = __msa_aver_s_h(dst6, dst7);
2909 out0 = PCKEV_XORI128_UB(dst0, dst1);
2910 out1 = PCKEV_XORI128_UB(dst2, dst3);
2911 ST8x4_UB(out0, out1, dst, stride);
2912 dst += (4 * stride);
2926 void ff_put_h264_qpel8_mc21_msa(uint8_t *dst, const uint8_t *src,
2929 const int32_t filt_const0 = 0xfffb0001;
2930 const int32_t filt_const1 = 0x140014;
2931 const int32_t filt_const2 = 0x1fffb;
2933 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2934 v16i8 src11, src12, mask0, mask1, mask2;
2935 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
2936 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
2937 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
2938 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
2939 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
2940 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
2941 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
2942 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
2945 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
2947 filt0 = (v8i16) __msa_fill_w(filt_const0);
2948 filt1 = (v8i16) __msa_fill_w(filt_const1);
2949 filt2 = (v8i16) __msa_fill_w(filt_const2);
2951 src -= ((2 * stride) + 2);
2953 LD_SB5(src, stride, src0, src1, src2, src3, src4);
2954 XORI_B5_128_SB(src0, src1, src2, src3, src4);
2955 src += (5 * stride);
2957 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
2958 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
2959 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
2960 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
2961 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
2963 LD_SB4(src, stride, src5, src6, src7, src8);
2964 src += (4 * stride);
2965 XORI_B4_128_SB(src5, src6, src7, src8);
2967 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
2968 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
2969 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
2970 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
2972 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
2973 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
2974 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
2975 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
2976 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
2977 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
2978 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
2979 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
2981 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
2983 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
2985 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2986 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
2988 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
2990 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2991 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
2993 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
2995 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
2996 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
2998 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
3000 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3002 SRARI_H4_SH(hz_out2, hz_out3, hz_out4, hz_out5, 5);
3003 SAT_SH4_SH(hz_out2, hz_out3, hz_out4, hz_out5, 7);
3005 dst0 = __msa_aver_s_h(dst0, hz_out2);
3006 dst1 = __msa_aver_s_h(dst1, hz_out3);
3007 dst2 = __msa_aver_s_h(dst2, hz_out4);
3008 dst3 = __msa_aver_s_h(dst3, hz_out5);
3010 out0 = PCKEV_XORI128_UB(dst0, dst1);
3011 out1 = PCKEV_XORI128_UB(dst2, dst3);
3012 ST8x4_UB(out0, out1, dst, stride);
3013 dst += (4 * stride);
3015 LD_SB4(src, stride, src9, src10, src11, src12);
3016 XORI_B4_128_SB(src9, src10, src11, src12);
3017 hz_out9 = AVC_HORZ_FILTER_SH(src9, src9, mask0, mask1, mask2);
3018 hz_out10 = AVC_HORZ_FILTER_SH(src10, src10, mask0, mask1, mask2);
3019 hz_out11 = AVC_HORZ_FILTER_SH(src11, src11, mask0, mask1, mask2);
3020 hz_out12 = AVC_HORZ_FILTER_SH(src12, src12, mask0, mask1, mask2);
3021 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3022 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
3024 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3025 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
3027 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
3029 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
3031 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3032 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
3034 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
3036 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3037 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
3039 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
3041 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3042 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
3044 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
3046 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3048 SRARI_H4_SH(hz_out6, hz_out7, hz_out8, hz_out9, 5);
3049 SAT_SH4_SH(hz_out6, hz_out7, hz_out8, hz_out9, 7);
3051 dst0 = __msa_aver_s_h(dst0, hz_out6);
3052 dst1 = __msa_aver_s_h(dst1, hz_out7);
3053 dst2 = __msa_aver_s_h(dst2, hz_out8);
3054 dst3 = __msa_aver_s_h(dst3, hz_out9);
3056 out0 = PCKEV_XORI128_UB(dst0, dst1);
3057 out1 = PCKEV_XORI128_UB(dst2, dst3);
3058 ST8x4_UB(out0, out1, dst, stride);
3061 void ff_put_h264_qpel8_mc23_msa(uint8_t *dst, const uint8_t *src,
3064 const int32_t filt_const0 = 0xfffb0001;
3065 const int32_t filt_const1 = 0x140014;
3066 const int32_t filt_const2 = 0x1fffb;
3068 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3069 v16i8 src11, src12, mask0, mask1, mask2;
3070 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3071 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
3072 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3073 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
3074 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
3075 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
3076 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
3077 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
3080 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
3082 filt0 = (v8i16) __msa_fill_w(filt_const0);
3083 filt1 = (v8i16) __msa_fill_w(filt_const1);
3084 filt2 = (v8i16) __msa_fill_w(filt_const2);
3086 src -= ((2 * stride) + 2);
3088 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3089 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3090 src += (5 * stride);
3092 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
3093 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
3094 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
3095 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
3096 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
3098 LD_SB4(src, stride, src5, src6, src7, src8);
3099 src += (4 * stride);
3100 XORI_B4_128_SB(src5, src6, src7, src8);
3102 hz_out5 = AVC_HORZ_FILTER_SH(src5, src5, mask0, mask1, mask2);
3103 hz_out6 = AVC_HORZ_FILTER_SH(src6, src6, mask0, mask1, mask2);
3104 hz_out7 = AVC_HORZ_FILTER_SH(src7, src7, mask0, mask1, mask2);
3105 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3107 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3108 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3109 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3110 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
3111 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3112 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3113 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3114 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
3116 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3118 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
3120 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3121 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3123 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
3125 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3126 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3128 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
3130 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3131 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3133 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
3135 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3137 SRARI_H4_SH(hz_out3, hz_out4, hz_out5, hz_out6, 5);
3138 SAT_SH4_SH(hz_out3, hz_out4, hz_out5, hz_out6, 7);
3140 dst0 = __msa_aver_s_h(dst0, hz_out3);
3141 dst1 = __msa_aver_s_h(dst1, hz_out4);
3142 dst2 = __msa_aver_s_h(dst2, hz_out5);
3143 dst3 = __msa_aver_s_h(dst3, hz_out6);
3145 out0 = PCKEV_XORI128_UB(dst0, dst1);
3146 out1 = PCKEV_XORI128_UB(dst2, dst3);
3147 ST8x4_UB(out0, out1, dst, stride);
3148 dst += (4 * stride);
3150 LD_SB4(src, stride, src9, src10, src11, src12);
3151 XORI_B4_128_SB(src9, src10, src11, src12);
3152 hz_out9 = AVC_HORZ_FILTER_SH(src9, src9, mask0, mask1, mask2);
3153 hz_out10 = AVC_HORZ_FILTER_SH(src10, src10, mask0, mask1, mask2);
3154 hz_out11 = AVC_HORZ_FILTER_SH(src11, src11, mask0, mask1, mask2);
3155 hz_out12 = AVC_HORZ_FILTER_SH(src12, src12, mask0, mask1, mask2);
3156 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3157 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
3159 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
3160 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
3162 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
3164 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
3166 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3167 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
3169 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
3171 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3172 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
3174 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
3176 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3177 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
3179 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
3181 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3183 SRARI_H4_SH(hz_out7, hz_out8, hz_out9, hz_out10, 5);
3184 SAT_SH4_SH(hz_out7, hz_out8, hz_out9, hz_out10, 7);
3186 dst0 = __msa_aver_s_h(dst0, hz_out7);
3187 dst1 = __msa_aver_s_h(dst1, hz_out8);
3188 dst2 = __msa_aver_s_h(dst2, hz_out9);
3189 dst3 = __msa_aver_s_h(dst3, hz_out10);
3191 out0 = PCKEV_XORI128_UB(dst0, dst1);
3192 out1 = PCKEV_XORI128_UB(dst2, dst3);
3193 ST8x4_UB(out0, out1, dst, stride);
3196 void ff_put_h264_qpel4_mc21_msa(uint8_t *dst, const uint8_t *src,
3199 const int32_t filt_const0 = 0xfffb0001;
3200 const int32_t filt_const1 = 0x140014;
3201 const int32_t filt_const2 = 0x1fffb;
3203 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3204 v16i8 mask0, mask1, mask2;
3205 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3206 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
3207 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3208 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
3211 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
3213 filt0 = (v8i16) __msa_fill_w(filt_const0);
3214 filt1 = (v8i16) __msa_fill_w(filt_const1);
3215 filt2 = (v8i16) __msa_fill_w(filt_const2);
3217 src -= ((2 * stride) + 2);
3219 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3220 src += (5 * stride);
3221 LD_SB4(src, stride, src5, src6, src7, src8);
3223 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3224 XORI_B4_128_SB(src5, src6, src7, src8);
3226 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
3227 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
3228 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
3229 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
3230 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3231 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
3232 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
3234 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3235 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3236 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3237 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3239 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3241 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3243 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3244 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3246 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3248 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3250 SRARI_H2_SH(hz_out2, hz_out4, 5);
3251 SAT_SH2_SH(hz_out2, hz_out4, 7);
3253 dst0 = __msa_aver_s_h(dst0, hz_out2);
3254 dst1 = __msa_aver_s_h(dst1, hz_out4);
3256 res = PCKEV_XORI128_UB(dst0, dst1);
3257 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
3260 void ff_put_h264_qpel4_mc23_msa(uint8_t *dst, const uint8_t *src,
3263 const int32_t filt_const0 = 0xfffb0001;
3264 const int32_t filt_const1 = 0x140014;
3265 const int32_t filt_const2 = 0x1fffb;
3267 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3268 v16i8 mask0, mask1, mask2;
3269 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3270 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
3271 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3272 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
3275 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
3277 filt0 = (v8i16) __msa_fill_w(filt_const0);
3278 filt1 = (v8i16) __msa_fill_w(filt_const1);
3279 filt2 = (v8i16) __msa_fill_w(filt_const2);
3281 src -= ((2 * stride) + 2);
3283 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3284 src += (5 * stride);
3285 LD_SB4(src, stride, src5, src6, src7, src8);
3287 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3288 XORI_B4_128_SB(src5, src6, src7, src8);
3290 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
3291 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
3292 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
3293 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
3294 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
3295 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
3296 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
3298 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
3299 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
3300 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
3301 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
3303 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
3305 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
3307 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3308 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
3310 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
3312 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
3314 PCKEV_D2_SH(hz_out4, hz_out3, hz_out6, hz_out5, hz_out0, hz_out1);
3315 SRARI_H2_SH(hz_out0, hz_out1, 5);
3316 SAT_SH2_SH(hz_out0, hz_out1, 7);
3318 dst0 = __msa_aver_s_h(dst0, hz_out0);
3319 dst1 = __msa_aver_s_h(dst1, hz_out1);
3321 res = PCKEV_XORI128_UB(dst0, dst1);
3322 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
3325 void ff_put_h264_qpel16_mc02_msa(uint8_t *dst, const uint8_t *src,
3329 int16_t filt_const0 = 0xfb01;
3330 int16_t filt_const1 = 0x1414;
3331 int16_t filt_const2 = 0x1fb;
3332 v16u8 res0, res1, res2, res3;
3333 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3334 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
3335 v16i8 src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l;
3336 v16i8 src65_l, src87_l, filt0, filt1, filt2;
3337 v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
3339 filt0 = (v16i8) __msa_fill_h(filt_const0);
3340 filt1 = (v16i8) __msa_fill_h(filt_const1);
3341 filt2 = (v16i8) __msa_fill_h(filt_const2);
3342 src -= (stride * 2);
3344 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3345 src += (5 * stride);
3347 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3348 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3350 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3353 for (loop_cnt = 4; loop_cnt--;) {
3354 LD_SB4(src, stride, src5, src6, src7, src8);
3355 src += (4 * stride);
3357 XORI_B4_128_SB(src5, src6, src7, src8);
3358 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r,
3359 src65_r, src76_r, src87_r);
3360 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l,
3361 src65_l, src76_l, src87_l);
3362 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3363 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3364 out2_r = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3365 out3_r = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3366 out0_l = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3367 out1_l = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3368 out2_l = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3369 out3_l = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3370 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
3371 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3372 SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 5);
3373 SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
3374 PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
3375 out3_r, res0, res1, res2, res3);
3376 XORI_B4_128_UB(res0, res1, res2, res3);
3377 ST_UB4(res0, res1, res2, res3, dst, stride);
3378 dst += (4 * stride);
3392 void ff_put_h264_qpel8_mc02_msa(uint8_t *dst, const uint8_t *src,
3395 const int16_t filt_const0 = 0xfb01;
3396 const int16_t filt_const1 = 0x1414;
3397 const int16_t filt_const2 = 0x1fb;
3398 v16u8 out0, out1, out2, out3;
3399 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3400 v16i8 src11, src12, src10_r, src21_r, src32_r, src43_r, src76_r, src87_r;
3401 v16i8 src98_r, src109_r, src89_r, src910_r, src1110_r, src1211_r;
3402 v16i8 filt0, filt1, filt2;
3403 v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r, out6_r, out7_r;
3405 filt0 = (v16i8) __msa_fill_h(filt_const0);
3406 filt1 = (v16i8) __msa_fill_h(filt_const1);
3407 filt2 = (v16i8) __msa_fill_h(filt_const2);
3409 src -= (stride * 2);
3411 LD_SB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
3412 src += (8 * stride);
3413 LD_SB5(src, stride, src8, src9, src10, src11, src12);
3414 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3416 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src76_r, src87_r,
3418 ILVR_B4_SB(src9, src8, src10, src9, src11, src10, src12, src11, src89_r,
3419 src910_r, src1110_r, src1211_r);
3420 XORI_B4_128_SB(src10_r, src21_r, src32_r, src43_r);
3421 XORI_B4_128_SB(src76_r, src87_r, src98_r, src109_r);
3422 XORI_B4_128_SB(src89_r, src910_r, src1110_r, src1211_r);
3423 out0_r = AVC_DOT_SH3_SH(src10_r, src32_r, src76_r, filt0, filt1, filt2);
3424 out1_r = AVC_DOT_SH3_SH(src21_r, src43_r, src87_r, filt0, filt1, filt2);
3425 out2_r = AVC_DOT_SH3_SH(src32_r, src76_r, src98_r, filt0, filt1, filt2);
3426 out3_r = AVC_DOT_SH3_SH(src43_r, src87_r, src109_r, filt0, filt1, filt2);
3427 out4_r = AVC_DOT_SH3_SH(src76_r, src98_r, src89_r, filt0, filt1, filt2);
3428 out5_r = AVC_DOT_SH3_SH(src87_r, src109_r, src910_r, filt0, filt1, filt2);
3429 out6_r = AVC_DOT_SH3_SH(src98_r, src89_r, src1110_r, filt0, filt1, filt2);
3430 out7_r = AVC_DOT_SH3_SH(src109_r, src910_r, src1211_r, filt0, filt1, filt2);
3431 SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 5);
3432 SRARI_H4_SH(out4_r, out5_r, out6_r, out7_r, 5);
3433 SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3434 SAT_SH4_SH(out4_r, out5_r, out6_r, out7_r, 7);
3435 out0 = PCKEV_XORI128_UB(out0_r, out1_r);
3436 out1 = PCKEV_XORI128_UB(out2_r, out3_r);
3437 out2 = PCKEV_XORI128_UB(out4_r, out5_r);
3438 out3 = PCKEV_XORI128_UB(out6_r, out7_r);
3439 ST8x8_UB(out0, out1, out2, out3, dst, stride);
3442 void ff_put_h264_qpel4_mc02_msa(uint8_t *dst, const uint8_t *src,
3445 const int16_t filt_const0 = 0xfb01;
3446 const int16_t filt_const1 = 0x1414;
3447 const int16_t filt_const2 = 0x1fb;
3449 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3450 v16i8 src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r;
3451 v16i8 src87_r, src2110, src4332, src6554, src8776, filt0, filt1, filt2;
3454 filt0 = (v16i8) __msa_fill_h(filt_const0);
3455 filt1 = (v16i8) __msa_fill_h(filt_const1);
3456 filt2 = (v16i8) __msa_fill_h(filt_const2);
3458 src -= (stride * 2);
3460 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3461 src += (5 * stride);
3462 LD_SB4(src, stride, src5, src6, src7, src8);
3464 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3466 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3468 ILVR_D4_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src87_r,
3469 src76_r, src2110, src4332, src6554, src8776);
3470 XORI_B4_128_SB(src2110, src4332, src6554, src8776);
3471 out10 = AVC_DOT_SH3_SH(src2110, src4332, src6554, filt0, filt1, filt2);
3472 out32 = AVC_DOT_SH3_SH(src4332, src6554, src8776, filt0, filt1, filt2);
3473 SRARI_H2_SH(out10, out32, 5);
3474 SAT_SH2_SH(out10, out32, 7);
3475 out = PCKEV_XORI128_UB(out10, out32);
3476 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3479 void ff_put_h264_qpel16_mc12_msa(uint8_t *dst, const uint8_t *src,
3484 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3486 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, mask3;
3487 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3488 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11, mask4, mask5;
3489 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3490 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3491 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3492 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3493 v8i16 minus5h = __msa_ldi_h(-5);
3494 v8i16 plus20h = __msa_ldi_h(20);
3500 src -= ((2 * stride) + 2);
3502 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3503 LD_SB5(src + 8, stride, src7, src8, src9, src10, src11);
3504 src += (5 * stride);
3505 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3506 XORI_B5_128_SB(src7, src8, src9, src10, src11);
3508 for (row = 16; row--;) {
3509 LD_SB2(src, 8, src5, src6);
3511 XORI_B2_128_SB(src5, src6);
3513 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3515 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src7, src8, src9, src10, src11, src6,
3517 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3518 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3519 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3520 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3521 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3522 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3523 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3524 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3525 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3526 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3527 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3528 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3529 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3530 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3531 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3532 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3533 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3534 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3535 dst0 = __msa_srari_h(shf_vec2, 5);
3536 dst1 = __msa_srari_h(shf_vec5, 5);
3537 dst2 = __msa_srari_h(shf_vec8, 5);
3538 dst3 = __msa_srari_h(shf_vec11, 5);
3539 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3540 PCKEV_H2_SH(dst2, dst0, dst3, dst1, dst0, dst1);
3541 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3542 dst0 = __msa_aver_s_h(dst2, dst0);
3543 dst1 = __msa_aver_s_h(dst3, dst1);
3544 out = PCKEV_XORI128_UB(dst0, dst1);
3561 void ff_put_h264_qpel16_mc32_msa(uint8_t *dst, const uint8_t *src,
3566 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3568 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, mask3;
3569 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3570 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11, mask4, mask5;
3571 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3572 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3573 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3574 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3575 v8i16 minus5h = __msa_ldi_h(-5);
3576 v8i16 plus20h = __msa_ldi_h(20);
3582 src -= ((2 * stride) + 2);
3584 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3585 LD_SB5(src + 8, stride, src7, src8, src9, src10, src11);
3586 src += (5 * stride);
3587 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3588 XORI_B5_128_SB(src7, src8, src9, src10, src11);
3590 for (row = 16; row--;) {
3591 LD_SB2(src, 8, src5, src6);
3593 XORI_B2_128_SB(src5, src6);
3595 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3597 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src7, src8, src9, src10, src11, src6,
3599 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3600 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3601 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3602 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3603 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3604 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3605 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3606 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3607 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3608 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3609 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3610 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3611 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3612 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3613 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3614 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3615 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3616 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3617 dst0 = __msa_srari_h(shf_vec2, 5);
3618 dst1 = __msa_srari_h(shf_vec5, 5);
3619 dst2 = __msa_srari_h(shf_vec8, 5);
3620 dst3 = __msa_srari_h(shf_vec11, 5);
3621 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3622 dst0 = __msa_pckod_h(dst2, dst0);
3623 dst1 = __msa_pckod_h(dst3, dst1);
3624 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3625 dst0 = __msa_aver_s_h(dst2, dst0);
3626 dst1 = __msa_aver_s_h(dst3, dst1);
3627 out = PCKEV_XORI128_UB(dst0, dst1);
3644 void ff_put_h264_qpel8_mc12_msa(uint8_t *dst, const uint8_t *src,
3649 v16i8 src0, src1, src2, src3, src4, src5, src6;
3650 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3;
3651 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3652 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11;
3653 v8i16 mask3, mask4, mask5;
3654 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3655 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3656 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3657 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3658 v8i16 minus5h = __msa_ldi_h(-5);
3659 v8i16 plus20h = __msa_ldi_h(20);
3665 src -= ((2 * stride) + 2);
3667 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3668 src += (5 * stride);
3669 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3671 for (row = 4; row--;) {
3672 LD_SB2(src, stride, src5, src6);
3673 src += (2 * stride);
3674 XORI_B2_128_SB(src5, src6);
3676 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3678 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
3680 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3681 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3682 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3683 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3684 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3685 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3686 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3687 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3688 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3689 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3690 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3691 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3692 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3693 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3694 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3695 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3696 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3697 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3698 dst0 = __msa_srari_h(shf_vec2, 5);
3699 dst1 = __msa_srari_h(shf_vec5, 5);
3700 dst2 = __msa_srari_h(shf_vec8, 5);
3701 dst3 = __msa_srari_h(shf_vec11, 5);
3702 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3703 PCKEV_H2_SH(dst2, dst0, dst3, dst1, dst0, dst1);
3704 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3705 dst0 = __msa_aver_s_h(dst2, dst0);
3706 dst1 = __msa_aver_s_h(dst3, dst1);
3707 out = PCKEV_XORI128_UB(dst0, dst1);
3708 ST8x2_UB(out, dst, stride);
3709 dst += (2 * stride);
3719 void ff_put_h264_qpel8_mc32_msa(uint8_t *dst, const uint8_t *src,
3724 v16i8 src0, src1, src2, src3, src4, src5, src6;
3725 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3;
3726 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3727 v8i16 shf_vec7, shf_vec8, shf_vec9, shf_vec10, shf_vec11;
3728 v8i16 mask3, mask4, mask5;
3729 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3730 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3731 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3732 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3733 v8i16 minus5h = __msa_ldi_h(-5);
3734 v8i16 plus20h = __msa_ldi_h(20);
3740 src -= ((2 * stride) + 2);
3742 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3743 src += (5 * stride);
3744 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3746 for (row = 4; row--;) {
3747 LD_SB2(src, stride, src5, src6);
3748 src += (2 * stride);
3749 XORI_B2_128_SB(src5, src6);
3751 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src0, src1, src2, src3, src4, src5,
3753 AVC_CALC_DPADD_B_6PIX_2COEFF_SH(src1, src2, src3, src4, src5, src6,
3755 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3756 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3757 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3758 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3759 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask3,
3760 mask4, mask5, shf_vec6, shf_vec7, shf_vec8);
3761 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask3,
3762 mask4, mask5, shf_vec9, shf_vec10, shf_vec11);
3763 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3764 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3765 hz_res2 = __msa_hadd_s_w(shf_vec6, shf_vec6);
3766 hz_res3 = __msa_hadd_s_w(shf_vec9, shf_vec9);
3767 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3768 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3769 DPADD_SH2_SW(shf_vec7, shf_vec8, minus5h, plus20h, hz_res2, hz_res2);
3770 DPADD_SH2_SW(shf_vec10, shf_vec11, minus5h, plus20h, hz_res3, hz_res3);
3771 SRARI_W4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 10);
3772 SAT_SW4_SW(hz_res0, hz_res1, hz_res2, hz_res3, 7);
3773 dst0 = __msa_srari_h(shf_vec2, 5);
3774 dst1 = __msa_srari_h(shf_vec5, 5);
3775 dst2 = __msa_srari_h(shf_vec8, 5);
3776 dst3 = __msa_srari_h(shf_vec11, 5);
3777 SAT_SH4_SH(dst0, dst1, dst2, dst3, 7);
3778 dst0 = __msa_pckod_h(dst2, dst0);
3779 dst1 = __msa_pckod_h(dst3, dst1);
3780 PCKEV_H2_SH(hz_res2, hz_res0, hz_res3, hz_res1, dst2, dst3);
3781 dst0 = __msa_aver_s_h(dst2, dst0);
3782 dst1 = __msa_aver_s_h(dst3, dst1);
3783 out = PCKEV_XORI128_UB(dst0, dst1);
3784 ST8x2_UB(out, dst, stride);
3785 dst += (2 * stride);
3795 void ff_put_h264_qpel4_mc12_msa(uint8_t *dst, const uint8_t *src,
3798 const int16_t filt_const0 = 0xfb01;
3799 const int16_t filt_const1 = 0x1414;
3800 const int16_t filt_const2 = 0x1fb;
3802 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3803 v16i8 src10_r, src21_r, src32_r, src43_r, src54_r, src65_r, src76_r;
3804 v16i8 src87_r, src10_l, src21_l, src32_l, src43_l, src54_l, src65_l;
3805 v16i8 src76_l, src87_l, filt0, filt1, filt2;
3806 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, shf_vec7;
3807 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3808 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3809 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3810 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3811 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3812 v8i16 minus5h = __msa_ldi_h(-5);
3813 v8i16 plus20h = __msa_ldi_h(20);
3814 v8i16 zeros = { 0 };
3816 filt0 = (v16i8) __msa_fill_h(filt_const0);
3817 filt1 = (v16i8) __msa_fill_h(filt_const1);
3818 filt2 = (v16i8) __msa_fill_h(filt_const2);
3820 src -= ((2 * stride) + 2);
3822 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3823 src += (5 * stride);
3824 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3825 LD_SB4(src, stride, src5, src6, src7, src8);
3826 XORI_B4_128_SB(src5, src6, src7, src8);
3828 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3830 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3832 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3834 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l, src65_l,
3836 vt_res0 = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3837 vt_res1 = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3838 vt_res2 = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3839 vt_res3 = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3840 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3841 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3842 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3843 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3844 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3845 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3846 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3847 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3849 vt_res0 = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3850 vt_res1 = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3851 vt_res2 = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3852 vt_res3 = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3853 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3854 mask1, mask2, shf_vec0, shf_vec1, shf_vec6);
3855 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3856 mask1, mask2, shf_vec3, shf_vec4, shf_vec7);
3857 hz_res2 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3858 DPADD_SH2_SW(shf_vec1, shf_vec6, minus5h, plus20h, hz_res2, hz_res2);
3859 hz_res3 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3860 DPADD_SH2_SW(shf_vec4, shf_vec7, minus5h, plus20h, hz_res3, hz_res3);
3862 SRARI_W2_SW(hz_res0, hz_res1, 10);
3863 SAT_SW2_SW(hz_res0, hz_res1, 7);
3864 SRARI_W2_SW(hz_res2, hz_res3, 10);
3865 SAT_SW2_SW(hz_res2, hz_res3, 7);
3867 dst0 = __msa_srari_h(shf_vec2, 5);
3868 dst1 = __msa_srari_h(shf_vec5, 5);
3869 dst2 = __msa_srari_h(shf_vec6, 5);
3870 dst3 = __msa_srari_h(shf_vec7, 5);
3872 SAT_SH2_SH(dst0, dst1, 7);
3873 SAT_SH2_SH(dst2, dst3, 7);
3874 ILVEV_H2_SH(dst0, zeros, dst1, zeros, dst0, dst1);
3875 ILVEV_H2_SH(dst2, zeros, dst3, zeros, dst2, dst3);
3877 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) dst0);
3878 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) dst1);
3879 hz_res2 = __msa_aver_s_w(hz_res2, (v4i32) dst2);
3880 hz_res3 = __msa_aver_s_w(hz_res3, (v4i32) dst3);
3882 PCKEV_H2_SH(hz_res1, hz_res0, hz_res3, hz_res2, dst0, dst2);
3883 out = PCKEV_XORI128_UB(dst0, dst2);
3884 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3887 void ff_put_h264_qpel4_mc32_msa(uint8_t *dst, const uint8_t *src,
3890 const int16_t filt_const0 = 0xfb01;
3891 const int16_t filt_const1 = 0x1414;
3892 const int16_t filt_const2 = 0x1fb;
3894 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3895 v16i8 src10_r, src21_r, src32_r, src43_r, src54_r, src65_r, src76_r;
3896 v16i8 src87_r, src10_l, src21_l, src32_l, src43_l, src54_l, src65_l;
3897 v16i8 src76_l, src87_l, filt0, filt1, filt2;
3898 v8i16 vt_res0, vt_res1, vt_res2, vt_res3, dst0, dst1, dst2, dst3, shf_vec7;
3899 v8i16 shf_vec0, shf_vec1, shf_vec2, shf_vec3, shf_vec4, shf_vec5, shf_vec6;
3900 v4i32 hz_res0, hz_res1, hz_res2, hz_res3;
3901 v8i16 mask0 = { 0, 5, 1, 6, 2, 7, 3, 8 };
3902 v8i16 mask1 = { 1, 4, 2, 5, 3, 6, 4, 7 };
3903 v8i16 mask2 = { 2, 3, 3, 4, 4, 5, 5, 6 };
3904 v8i16 minus5h = __msa_ldi_h(-5);
3905 v8i16 plus20h = __msa_ldi_h(20);
3906 v8i16 zeros = { 0 };
3908 filt0 = (v16i8) __msa_fill_h(filt_const0);
3909 filt1 = (v16i8) __msa_fill_h(filt_const1);
3910 filt2 = (v16i8) __msa_fill_h(filt_const2);
3912 src -= ((2 * stride) + 2);
3914 LD_SB5(src, stride, src0, src1, src2, src3, src4);
3915 src += (5 * stride);
3916 XORI_B5_128_SB(src0, src1, src2, src3, src4);
3917 LD_SB4(src, stride, src5, src6, src7, src8);
3918 XORI_B4_128_SB(src5, src6, src7, src8);
3920 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
3922 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
3924 ILVL_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_l, src21_l,
3926 ILVL_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_l, src65_l,
3929 vt_res0 = AVC_DOT_SH3_SH(src10_r, src32_r, src54_r, filt0, filt1, filt2);
3930 vt_res1 = AVC_DOT_SH3_SH(src10_l, src32_l, src54_l, filt0, filt1, filt2);
3931 vt_res2 = AVC_DOT_SH3_SH(src21_r, src43_r, src65_r, filt0, filt1, filt2);
3932 vt_res3 = AVC_DOT_SH3_SH(src21_l, src43_l, src65_l, filt0, filt1, filt2);
3933 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3934 mask1, mask2, shf_vec0, shf_vec1, shf_vec2);
3935 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3936 mask1, mask2, shf_vec3, shf_vec4, shf_vec5);
3937 hz_res0 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3938 DPADD_SH2_SW(shf_vec1, shf_vec2, minus5h, plus20h, hz_res0, hz_res0);
3939 hz_res1 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3940 DPADD_SH2_SW(shf_vec4, shf_vec5, minus5h, plus20h, hz_res1, hz_res1);
3942 vt_res0 = AVC_DOT_SH3_SH(src32_r, src54_r, src76_r, filt0, filt1, filt2);
3943 vt_res1 = AVC_DOT_SH3_SH(src32_l, src54_l, src76_l, filt0, filt1, filt2);
3944 vt_res2 = AVC_DOT_SH3_SH(src43_r, src65_r, src87_r, filt0, filt1, filt2);
3945 vt_res3 = AVC_DOT_SH3_SH(src43_l, src65_l, src87_l, filt0, filt1, filt2);
3946 VSHF_H3_SH(vt_res0, vt_res1, vt_res0, vt_res1, vt_res0, vt_res1, mask0,
3947 mask1, mask2, shf_vec0, shf_vec1, shf_vec6);
3948 VSHF_H3_SH(vt_res2, vt_res3, vt_res2, vt_res3, vt_res2, vt_res3, mask0,
3949 mask1, mask2, shf_vec3, shf_vec4, shf_vec7);
3950 hz_res2 = __msa_hadd_s_w(shf_vec0, shf_vec0);
3951 DPADD_SH2_SW(shf_vec1, shf_vec6, minus5h, plus20h, hz_res2, hz_res2);
3952 hz_res3 = __msa_hadd_s_w(shf_vec3, shf_vec3);
3953 DPADD_SH2_SW(shf_vec4, shf_vec7, minus5h, plus20h, hz_res3, hz_res3);
3955 SRARI_W2_SW(hz_res0, hz_res1, 10);
3956 SAT_SW2_SW(hz_res0, hz_res1, 7);
3957 SRARI_W2_SW(hz_res2, hz_res3, 10);
3958 SAT_SW2_SW(hz_res2, hz_res3, 7);
3960 dst0 = __msa_srari_h(shf_vec2, 5);
3961 dst1 = __msa_srari_h(shf_vec5, 5);
3962 dst2 = __msa_srari_h(shf_vec6, 5);
3963 dst3 = __msa_srari_h(shf_vec7, 5);
3965 SAT_SH2_SH(dst0, dst1, 7);
3966 SAT_SH2_SH(dst2, dst3, 7);
3968 dst0 = __msa_ilvod_h(zeros, dst0);
3969 dst1 = __msa_ilvod_h(zeros, dst1);
3970 dst2 = __msa_ilvod_h(zeros, dst2);
3971 dst3 = __msa_ilvod_h(zeros, dst3);
3973 hz_res0 = __msa_aver_s_w(hz_res0, (v4i32) dst0);
3974 hz_res1 = __msa_aver_s_w(hz_res1, (v4i32) dst1);
3975 hz_res2 = __msa_aver_s_w(hz_res2, (v4i32) dst2);
3976 hz_res3 = __msa_aver_s_w(hz_res3, (v4i32) dst3);
3978 PCKEV_H2_SH(hz_res1, hz_res0, hz_res3, hz_res2, dst0, dst2);
3979 out = PCKEV_XORI128_UB(dst0, dst2);
3980 ST4x4_UB(out, out, 0, 1, 2, 3, dst, stride);
3983 void ff_put_h264_qpel16_mc22_msa(uint8_t *dst, const uint8_t *src,
3986 const int32_t filt_const0 = 0xfffb0001;
3987 const int32_t filt_const1 = 0x140014;
3988 const int32_t filt_const2 = 0x1fffb;
3989 const uint8_t *src_tmp = src - (2 * stride) - 2;
3990 uint8_t *dst_tmp = dst;
3991 uint32_t multiple8_cnt, loop_cnt;
3993 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
3994 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
3995 v8i16 hz_out7, hz_out8, dst0, dst1, dst2, dst3;
3996 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
3997 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out10_l, hz_out21_l;
3998 v8i16 hz_out32_l, hz_out43_l, hz_out54_l, hz_out65_l, hz_out76_l;
3999 v8i16 hz_out87_l, filt0, filt1, filt2;
4002 filt0 = (v8i16) __msa_fill_w(filt_const0);
4003 filt1 = (v8i16) __msa_fill_w(filt_const1);
4004 filt2 = (v8i16) __msa_fill_w(filt_const2);
4006 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
4008 for (multiple8_cnt = 2; multiple8_cnt--;) {
4012 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4013 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4014 src += (5 * stride);
4016 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4017 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4018 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4019 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4020 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4022 for (loop_cnt = 4; loop_cnt--;) {
4023 LD_SB4(src, stride, src0, src1, src2, src3);
4024 XORI_B4_128_SB(src0, src1, src2, src3);
4025 src += (4 * stride);
4027 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4028 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4029 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4030 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4032 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4033 hz_out4, hz_out3, hz_out10_r, hz_out21_r, hz_out32_r,
4035 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2,
4036 hz_out4, hz_out3, hz_out10_l, hz_out21_l, hz_out32_l,
4038 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4039 hz_out8, hz_out7, hz_out54_r, hz_out65_r, hz_out76_r,
4041 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6,
4042 hz_out8, hz_out7, hz_out54_l, hz_out65_l, hz_out76_l,
4045 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0,
4047 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0,
4049 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4050 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0,
4052 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0,
4054 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4055 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0,
4057 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0,
4059 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4060 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0,
4062 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0,
4064 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4066 out0 = PCKEV_XORI128_UB(dst0, dst1);
4067 out1 = PCKEV_XORI128_UB(dst2, dst3);
4068 ST8x4_UB(out0, out1, dst, stride);
4069 dst += (4 * stride);
4083 void ff_put_h264_qpel8_mc22_msa(uint8_t *dst, const uint8_t *src,
4086 const int32_t filt_const0 = 0xfffb0001;
4087 const int32_t filt_const1 = 0x140014;
4088 const int32_t filt_const2 = 0x1fffb;
4090 v16i8 src0, src1, src2, src3, src4, mask0, mask1, mask2;
4091 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4092 v8i16 hz_out7, hz_out8, hz_out9, hz_out10, hz_out11, hz_out12;
4093 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4094 v8i16 hz_out65_r, hz_out76_r, hz_out87_r, hz_out89_r, hz_out910_r;
4095 v8i16 hz_out1110_r, hz_out1211_r, dst0, dst1, dst2, dst3;
4096 v8i16 hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l, hz_out54_l;
4097 v8i16 hz_out65_l, hz_out76_l, hz_out87_l, hz_out89_l, hz_out910_l;
4098 v8i16 hz_out1110_l, hz_out1211_l, filt0, filt1, filt2;
4101 filt0 = (v8i16) __msa_fill_w(filt_const0);
4102 filt1 = (v8i16) __msa_fill_w(filt_const1);
4103 filt2 = (v8i16) __msa_fill_w(filt_const2);
4105 LD_SB3(&luma_mask_arr[0], 16, mask0, mask1, mask2);
4107 src -= ((2 * stride) + 2);
4108 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4109 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4110 src += (5 * stride);
4112 hz_out0 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4113 hz_out1 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4114 hz_out2 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4115 hz_out3 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4116 hz_out4 = AVC_HORZ_FILTER_SH(src4, src4, mask0, mask1, mask2);
4118 LD_SB4(src, stride, src0, src1, src2, src3);
4119 XORI_B4_128_SB(src0, src1, src2, src3);
4120 src += (4 * stride);
4121 hz_out5 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4122 hz_out6 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4123 hz_out7 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4124 hz_out8 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4125 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4126 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4127 ILVL_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4128 hz_out3, hz_out10_l, hz_out21_l, hz_out32_l, hz_out43_l);
4129 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4130 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4131 ILVL_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4132 hz_out7, hz_out54_l, hz_out65_l, hz_out76_l, hz_out87_l);
4134 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4136 tmp1 = AVC_DOT_SW3_SW(hz_out10_l, hz_out32_l, hz_out54_l, filt0, filt1,
4138 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4139 tmp0 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4141 tmp1 = AVC_DOT_SW3_SW(hz_out21_l, hz_out43_l, hz_out65_l, filt0, filt1,
4143 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4144 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4146 tmp1 = AVC_DOT_SW3_SW(hz_out32_l, hz_out54_l, hz_out76_l, filt0, filt1,
4148 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4149 tmp0 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4151 tmp1 = AVC_DOT_SW3_SW(hz_out43_l, hz_out65_l, hz_out87_l, filt0, filt1,
4153 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4154 out0 = PCKEV_XORI128_UB(dst0, dst1);
4155 out1 = PCKEV_XORI128_UB(dst2, dst3);
4156 ST8x4_UB(out0, out1, dst, stride);
4157 dst += (4 * stride);
4159 LD_SB4(src, stride, src0, src1, src2, src3);
4160 XORI_B4_128_SB(src0, src1, src2, src3);
4161 hz_out9 = AVC_HORZ_FILTER_SH(src0, src0, mask0, mask1, mask2);
4162 hz_out10 = AVC_HORZ_FILTER_SH(src1, src1, mask0, mask1, mask2);
4163 hz_out11 = AVC_HORZ_FILTER_SH(src2, src2, mask0, mask1, mask2);
4164 hz_out12 = AVC_HORZ_FILTER_SH(src3, src3, mask0, mask1, mask2);
4165 ILVR_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4166 hz_out12, hz_out11, hz_out89_r, hz_out910_r, hz_out1110_r,
4168 ILVL_H4_SH(hz_out9, hz_out8, hz_out10, hz_out9, hz_out11, hz_out10,
4169 hz_out12, hz_out11, hz_out89_l, hz_out910_l, hz_out1110_l,
4171 tmp0 = AVC_DOT_SW3_SW(hz_out54_r, hz_out76_r, hz_out89_r, filt0, filt1,
4173 tmp1 = AVC_DOT_SW3_SW(hz_out54_l, hz_out76_l, hz_out89_l, filt0, filt1,
4175 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4176 tmp0 = AVC_DOT_SW3_SW(hz_out65_r, hz_out87_r, hz_out910_r, filt0, filt1,
4178 tmp1 = AVC_DOT_SW3_SW(hz_out65_l, hz_out87_l, hz_out910_l, filt0, filt1,
4180 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4181 tmp0 = AVC_DOT_SW3_SW(hz_out76_r, hz_out89_r, hz_out1110_r, filt0, filt1,
4183 tmp1 = AVC_DOT_SW3_SW(hz_out76_l, hz_out89_l, hz_out1110_l, filt0, filt1,
4185 dst2 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4186 tmp0 = AVC_DOT_SW3_SW(hz_out87_r, hz_out910_r, hz_out1211_r, filt0, filt1,
4188 tmp1 = AVC_DOT_SW3_SW(hz_out87_l, hz_out910_l, hz_out1211_l, filt0, filt1,
4190 dst3 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4191 out0 = PCKEV_XORI128_UB(dst0, dst1);
4192 out1 = PCKEV_XORI128_UB(dst2, dst3);
4193 ST8x4_UB(out0, out1, dst, stride);
4196 void ff_put_h264_qpel4_mc22_msa(uint8_t *dst, const uint8_t *src,
4199 const int32_t filt_const0 = 0xfffb0001;
4200 const int32_t filt_const1 = 0x140014;
4201 const int32_t filt_const2 = 0x1fffb;
4203 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
4204 v16i8 mask0, mask1, mask2;
4205 v8i16 hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6;
4206 v8i16 hz_out7, hz_out8, dst0, dst1, filt0, filt1, filt2;
4207 v8i16 hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r, hz_out54_r;
4208 v8i16 hz_out65_r, hz_out76_r, hz_out87_r;
4211 LD_SB3(&luma_mask_arr[48], 16, mask0, mask1, mask2);
4213 filt0 = (v8i16) __msa_fill_w(filt_const0);
4214 filt1 = (v8i16) __msa_fill_w(filt_const1);
4215 filt2 = (v8i16) __msa_fill_w(filt_const2);
4217 src -= ((2 * stride) + 2);
4219 LD_SB5(src, stride, src0, src1, src2, src3, src4);
4220 src += (5 * stride);
4221 LD_SB4(src, stride, src5, src6, src7, src8);
4223 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4224 XORI_B4_128_SB(src5, src6, src7, src8);
4225 hz_out0 = AVC_HORZ_FILTER_SH(src0, src1, mask0, mask1, mask2);
4226 hz_out2 = AVC_HORZ_FILTER_SH(src2, src3, mask0, mask1, mask2);
4227 hz_out4 = AVC_HORZ_FILTER_SH(src4, src5, mask0, mask1, mask2);
4228 hz_out6 = AVC_HORZ_FILTER_SH(src6, src7, mask0, mask1, mask2);
4229 hz_out8 = AVC_HORZ_FILTER_SH(src8, src8, mask0, mask1, mask2);
4230 PCKOD_D2_SH(hz_out0, hz_out0, hz_out2, hz_out2, hz_out1, hz_out3);
4231 PCKOD_D2_SH(hz_out4, hz_out4, hz_out6, hz_out6, hz_out5, hz_out7);
4232 ILVR_H4_SH(hz_out1, hz_out0, hz_out2, hz_out1, hz_out3, hz_out2, hz_out4,
4233 hz_out3, hz_out10_r, hz_out21_r, hz_out32_r, hz_out43_r);
4234 ILVR_H4_SH(hz_out5, hz_out4, hz_out6, hz_out5, hz_out7, hz_out6, hz_out8,
4235 hz_out7, hz_out54_r, hz_out65_r, hz_out76_r, hz_out87_r);
4237 tmp0 = AVC_DOT_SW3_SW(hz_out10_r, hz_out32_r, hz_out54_r, filt0, filt1,
4239 tmp1 = AVC_DOT_SW3_SW(hz_out21_r, hz_out43_r, hz_out65_r, filt0, filt1,
4241 dst0 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4242 tmp0 = AVC_DOT_SW3_SW(hz_out32_r, hz_out54_r, hz_out76_r, filt0, filt1,
4244 tmp1 = AVC_DOT_SW3_SW(hz_out43_r, hz_out65_r, hz_out87_r, filt0, filt1,
4246 dst1 = __msa_pckev_h((v8i16) tmp1, (v8i16) tmp0);
4247 res = PCKEV_XORI128_UB(dst0, dst1);
4248 ST4x4_UB(res, res, 0, 1, 2, 3, dst, stride);
4251 void ff_avg_h264_qpel16_mc10_msa(uint8_t *dst, const uint8_t *src,
4254 avc_luma_hz_qrt_and_aver_dst_16x16_msa(src - 2, stride, dst, stride, 0);
4257 void ff_avg_h264_qpel16_mc30_msa(uint8_t *dst, const uint8_t *src,
4260 avc_luma_hz_qrt_and_aver_dst_16x16_msa(src - 2, stride, dst, stride, 1);
4263 void ff_avg_h264_qpel8_mc10_msa(uint8_t *dst, const uint8_t *src,
4266 avc_luma_hz_qrt_and_aver_dst_8x8_msa(src - 2, stride, dst, stride, 0);
4269 void ff_avg_h264_qpel8_mc30_msa(uint8_t *dst, const uint8_t *src,
4272 avc_luma_hz_qrt_and_aver_dst_8x8_msa(src - 2, stride, dst, stride, 1);
4275 void ff_avg_h264_qpel4_mc10_msa(uint8_t *dst, const uint8_t *src,
4278 avc_luma_hz_qrt_and_aver_dst_4x4_msa(src - 2, stride, dst, stride, 0);
4281 void ff_avg_h264_qpel4_mc30_msa(uint8_t *dst, const uint8_t *src,
4284 avc_luma_hz_qrt_and_aver_dst_4x4_msa(src - 2, stride, dst, stride, 1);
4287 void ff_avg_h264_qpel16_mc20_msa(uint8_t *dst, const uint8_t *src,
4290 avc_luma_hz_and_aver_dst_16x16_msa(src - 2, stride, dst, stride);
4293 void ff_avg_h264_qpel8_mc20_msa(uint8_t *dst, const uint8_t *src,
4296 avc_luma_hz_and_aver_dst_8x8_msa(src - 2, stride, dst, stride);
4299 void ff_avg_h264_qpel4_mc20_msa(uint8_t *dst, const uint8_t *src,
4302 avc_luma_hz_and_aver_dst_4x4_msa(src - 2, stride, dst, stride);
4305 void ff_avg_h264_qpel16_mc01_msa(uint8_t *dst, const uint8_t *src,
4308 avc_luma_vt_qrt_and_aver_dst_16x16_msa(src - (stride * 2),
4309 stride, dst, stride, 0);
4312 void ff_avg_h264_qpel16_mc03_msa(uint8_t *dst, const uint8_t *src,
4315 avc_luma_vt_qrt_and_aver_dst_16x16_msa(src - (stride * 2),
4316 stride, dst, stride, 1);
4319 void ff_avg_h264_qpel8_mc01_msa(uint8_t *dst, const uint8_t *src,
4322 avc_luma_vt_qrt_and_aver_dst_8x8_msa(src - (stride * 2),
4323 stride, dst, stride, 0);
4326 void ff_avg_h264_qpel8_mc03_msa(uint8_t *dst, const uint8_t *src,
4329 avc_luma_vt_qrt_and_aver_dst_8x8_msa(src - (stride * 2),
4330 stride, dst, stride, 1);
4333 void ff_avg_h264_qpel4_mc01_msa(uint8_t *dst, const uint8_t *src,
4336 avc_luma_vt_qrt_and_aver_dst_4x4_msa(src - (stride * 2),
4337 stride, dst, stride, 0);
4340 void ff_avg_h264_qpel4_mc03_msa(uint8_t *dst, const uint8_t *src,
4343 avc_luma_vt_qrt_and_aver_dst_4x4_msa(src - (stride * 2),
4344 stride, dst, stride, 1);
4347 void ff_avg_h264_qpel16_mc11_msa(uint8_t *dst, const uint8_t *src,
4350 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src - 2,
4352 stride, dst, stride);
4355 void ff_avg_h264_qpel16_mc31_msa(uint8_t *dst, const uint8_t *src,
4358 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src - 2,
4359 src - (stride * 2) +
4360 sizeof(uint8_t), stride,
4364 void ff_avg_h264_qpel16_mc13_msa(uint8_t *dst, const uint8_t *src,
4367 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src + stride - 2,
4369 stride, dst, stride);
4372 void ff_avg_h264_qpel16_mc33_msa(uint8_t *dst, const uint8_t *src,
4375 avc_luma_hv_qrt_and_aver_dst_16x16_msa(src + stride - 2,
4376 src - (stride * 2) +
4377 sizeof(uint8_t), stride,
4381 void ff_avg_h264_qpel8_mc11_msa(uint8_t *dst, const uint8_t *src,
4384 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src - 2,
4386 stride, dst, stride);
4389 void ff_avg_h264_qpel8_mc31_msa(uint8_t *dst, const uint8_t *src,
4392 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src - 2,
4393 src - (stride * 2) +
4394 sizeof(uint8_t), stride, dst, stride);
4397 void ff_avg_h264_qpel8_mc13_msa(uint8_t *dst, const uint8_t *src,
4400 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src + stride - 2,
4402 stride, dst, stride);
4405 void ff_avg_h264_qpel8_mc33_msa(uint8_t *dst, const uint8_t *src,
4408 avc_luma_hv_qrt_and_aver_dst_8x8_msa(src + stride - 2,
4409 src - (stride * 2) +
4410 sizeof(uint8_t), stride, dst, stride);
4414 void ff_avg_h264_qpel4_mc11_msa(uint8_t *dst, const uint8_t *src,
4417 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src - 2,
4419 stride, dst, stride);
4422 void ff_avg_h264_qpel4_mc31_msa(uint8_t *dst, const uint8_t *src,
4425 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src - 2,
4426 src - (stride * 2) +
4427 sizeof(uint8_t), stride, dst, stride);
4430 void ff_avg_h264_qpel4_mc13_msa(uint8_t *dst, const uint8_t *src,
4433 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src + stride - 2,
4435 stride, dst, stride);
4438 void ff_avg_h264_qpel4_mc33_msa(uint8_t *dst, const uint8_t *src,
4441 avc_luma_hv_qrt_and_aver_dst_4x4_msa(src + stride - 2,
4442 src - (stride * 2) +
4443 sizeof(uint8_t), stride, dst, stride);
4446 void ff_avg_h264_qpel16_mc21_msa(uint8_t *dst, const uint8_t *src,
4449 avc_luma_midv_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4450 stride, dst, stride, 16, 0);
4453 void ff_avg_h264_qpel16_mc23_msa(uint8_t *dst, const uint8_t *src,
4456 avc_luma_midv_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4457 stride, dst, stride, 16, 1);
4460 void ff_avg_h264_qpel8_mc21_msa(uint8_t *dst, const uint8_t *src,
4463 avc_luma_midv_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4464 stride, dst, stride, 8, 0);
4467 void ff_avg_h264_qpel8_mc23_msa(uint8_t *dst, const uint8_t *src,
4470 avc_luma_midv_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4471 stride, dst, stride, 8, 1);
4474 void ff_avg_h264_qpel4_mc21_msa(uint8_t *dst, const uint8_t *src,
4477 avc_luma_midv_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4478 stride, dst, stride, 4, 0);
4481 void ff_avg_h264_qpel4_mc23_msa(uint8_t *dst, const uint8_t *src,
4484 avc_luma_midv_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4485 stride, dst, stride, 4, 1);
4488 void ff_avg_h264_qpel16_mc02_msa(uint8_t *dst, const uint8_t *src,
4491 avc_luma_vt_and_aver_dst_16x16_msa(src - (stride * 2), stride, dst, stride);
4494 void ff_avg_h264_qpel8_mc02_msa(uint8_t *dst, const uint8_t *src,
4497 avc_luma_vt_and_aver_dst_8x8_msa(src - (stride * 2), stride, dst, stride);
4500 void ff_avg_h264_qpel4_mc02_msa(uint8_t *dst, const uint8_t *src,
4503 avc_luma_vt_and_aver_dst_4x4_msa(src - (stride * 2), stride, dst, stride);
4506 void ff_avg_h264_qpel16_mc12_msa(uint8_t *dst, const uint8_t *src,
4509 avc_luma_midh_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4510 stride, dst, stride, 16, 0);
4513 void ff_avg_h264_qpel16_mc32_msa(uint8_t *dst, const uint8_t *src,
4516 avc_luma_midh_qrt_and_aver_dst_16w_msa(src - (2 * stride) - 2,
4517 stride, dst, stride, 16, 1);
4520 void ff_avg_h264_qpel8_mc12_msa(uint8_t *dst, const uint8_t *src,
4523 avc_luma_midh_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4524 stride, dst, stride, 8, 0);
4527 void ff_avg_h264_qpel8_mc32_msa(uint8_t *dst, const uint8_t *src,
4530 avc_luma_midh_qrt_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4531 stride, dst, stride, 8, 1);
4534 void ff_avg_h264_qpel4_mc12_msa(uint8_t *dst, const uint8_t *src,
4537 avc_luma_midh_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4538 stride, dst, stride, 4, 0);
4541 void ff_avg_h264_qpel4_mc32_msa(uint8_t *dst, const uint8_t *src,
4544 avc_luma_midh_qrt_and_aver_dst_4w_msa(src - (2 * stride) - 2,
4545 stride, dst, stride, 4, 1);
4548 void ff_avg_h264_qpel16_mc22_msa(uint8_t *dst, const uint8_t *src,
4551 avc_luma_mid_and_aver_dst_16x16_msa(src - (2 * stride) - 2,
4552 stride, dst, stride);
4555 void ff_avg_h264_qpel8_mc22_msa(uint8_t *dst, const uint8_t *src,
4558 avc_luma_mid_and_aver_dst_8w_msa(src - (2 * stride) - 2,
4559 stride, dst, stride, 8);
4562 void ff_avg_h264_qpel4_mc22_msa(uint8_t *dst, const uint8_t *src,
4565 avc_luma_mid_and_aver_dst_4x4_msa(src - (2 * stride) - 2,
4566 stride, dst, stride);