2 * MMX optimized DSP utils
3 * Copyright (c) 2000, 2001 Fabrice Bellard
4 * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
6 * This file is part of FFmpeg.
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 * MMX optimization by Nick Kurshev <nickols_k@mail.ru>
25 #include "libavutil/attributes.h"
26 #include "libavutil/cpu.h"
27 #include "libavutil/x86/asm.h"
28 #include "libavcodec/dsputil.h"
29 #include "libavcodec/h264dsp.h"
30 #include "libavcodec/mpegvideo.h"
31 #include "libavcodec/simple_idct.h"
32 #include "dsputil_mmx.h"
33 #include "idct_xvid.h"
34 #include "diracdsp_mmx.h"
39 /* pixel operations */
40 DECLARE_ALIGNED(8, const uint64_t, ff_bone) = 0x0101010101010101ULL;
41 DECLARE_ALIGNED(8, const uint64_t, ff_wtwo) = 0x0002000200020002ULL;
43 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_1) = { 0x0001000100010001ULL, 0x0001000100010001ULL };
44 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_2) = { 0x0002000200020002ULL, 0x0002000200020002ULL };
45 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_3) = { 0x0003000300030003ULL, 0x0003000300030003ULL };
46 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_4) = { 0x0004000400040004ULL, 0x0004000400040004ULL };
47 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_5) = { 0x0005000500050005ULL, 0x0005000500050005ULL };
48 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_8) = { 0x0008000800080008ULL, 0x0008000800080008ULL };
49 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_9) = { 0x0009000900090009ULL, 0x0009000900090009ULL };
50 DECLARE_ALIGNED(8, const uint64_t, ff_pw_15) = 0x000F000F000F000FULL;
51 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_16) = { 0x0010001000100010ULL, 0x0010001000100010ULL };
52 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_17) = { 0x0011001100110011ULL, 0x0011001100110011ULL };
53 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_18) = { 0x0012001200120012ULL, 0x0012001200120012ULL };
54 DECLARE_ALIGNED(8, const uint64_t, ff_pw_20) = 0x0014001400140014ULL;
55 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_27) = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };
56 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_28) = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };
57 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_32) = { 0x0020002000200020ULL, 0x0020002000200020ULL };
58 DECLARE_ALIGNED(8, const uint64_t, ff_pw_42) = 0x002A002A002A002AULL;
59 DECLARE_ALIGNED(8, const uint64_t, ff_pw_53) = 0x0035003500350035ULL;
60 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_63) = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };
61 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_64) = { 0x0040004000400040ULL, 0x0040004000400040ULL };
62 DECLARE_ALIGNED(8, const uint64_t, ff_pw_96) = 0x0060006000600060ULL;
63 DECLARE_ALIGNED(8, const uint64_t, ff_pw_128) = 0x0080008000800080ULL;
64 DECLARE_ALIGNED(8, const uint64_t, ff_pw_255) = 0x00ff00ff00ff00ffULL;
65 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_512) = { 0x0200020002000200ULL, 0x0200020002000200ULL };
66 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };
68 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_0) = { 0x0000000000000000ULL, 0x0000000000000000ULL };
69 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_1) = { 0x0101010101010101ULL, 0x0101010101010101ULL };
70 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_3) = { 0x0303030303030303ULL, 0x0303030303030303ULL };
71 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_4) = { 0x0404040404040404ULL, 0x0404040404040404ULL };
72 DECLARE_ALIGNED(8, const uint64_t, ff_pb_7) = 0x0707070707070707ULL;
73 DECLARE_ALIGNED(8, const uint64_t, ff_pb_1F) = 0x1F1F1F1F1F1F1F1FULL;
74 DECLARE_ALIGNED(8, const uint64_t, ff_pb_3F) = 0x3F3F3F3F3F3F3F3FULL;
75 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_80) = { 0x8080808080808080ULL, 0x8080808080808080ULL };
76 DECLARE_ALIGNED(8, const uint64_t, ff_pb_81) = 0x8181818181818181ULL;
77 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_A1) = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };
78 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_F8) = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };
79 DECLARE_ALIGNED(8, const uint64_t, ff_pb_FC) = 0xFCFCFCFCFCFCFCFCULL;
80 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_FE) = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };
82 DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };
83 DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };
87 void ff_put_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
88 ptrdiff_t line_size, int h);
89 void ff_put_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
90 ptrdiff_t line_size, int h);
91 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
92 int dstStride, int src1Stride, int h);
93 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
94 uint8_t *src2, int dstStride,
95 int src1Stride, int h);
96 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
97 int dstStride, int src1Stride, int h);
98 void ff_put_pixels16_x2_mmxext(uint8_t *block, const uint8_t *pixels,
99 ptrdiff_t line_size, int h);
100 void ff_put_pixels16_x2_3dnow(uint8_t *block, const uint8_t *pixels,
101 ptrdiff_t line_size, int h);
102 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
103 int dstStride, int src1Stride, int h);
104 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
105 int dstStride, int src1Stride, int h);
106 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
107 int dstStride, int src1Stride, int h);
108 void ff_put_no_rnd_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
109 ptrdiff_t line_size, int h);
110 void ff_put_no_rnd_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
111 ptrdiff_t line_size, int h);
112 void ff_put_no_rnd_pixels8_x2_exact_mmxext(uint8_t *block,
113 const uint8_t *pixels,
114 ptrdiff_t line_size, int h);
115 void ff_put_no_rnd_pixels8_x2_exact_3dnow(uint8_t *block,
116 const uint8_t *pixels,
117 ptrdiff_t line_size, int h);
118 void ff_put_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
119 ptrdiff_t line_size, int h);
120 void ff_put_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
121 ptrdiff_t line_size, int h);
122 void ff_put_no_rnd_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
123 ptrdiff_t line_size, int h);
124 void ff_put_no_rnd_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
125 ptrdiff_t line_size, int h);
126 void ff_put_no_rnd_pixels8_y2_exact_mmxext(uint8_t *block,
127 const uint8_t *pixels,
128 ptrdiff_t line_size, int h);
129 void ff_put_no_rnd_pixels8_y2_exact_3dnow(uint8_t *block,
130 const uint8_t *pixels,
131 ptrdiff_t line_size, int h);
132 void ff_avg_pixels8_mmxext(uint8_t *block, const uint8_t *pixels,
133 ptrdiff_t line_size, int h);
134 void ff_avg_pixels8_3dnow(uint8_t *block, const uint8_t *pixels,
135 ptrdiff_t line_size, int h);
136 void ff_avg_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
137 ptrdiff_t line_size, int h);
138 void ff_avg_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
139 ptrdiff_t line_size, int h);
140 void ff_avg_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
141 ptrdiff_t line_size, int h);
142 void ff_avg_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
143 ptrdiff_t line_size, int h);
144 void ff_avg_pixels8_xy2_mmxext(uint8_t *block, const uint8_t *pixels,
145 ptrdiff_t line_size, int h);
146 void ff_avg_pixels8_xy2_3dnow(uint8_t *block, const uint8_t *pixels,
147 ptrdiff_t line_size, int h);
149 void ff_put_pixels8_mmxext(uint8_t *block, const uint8_t *pixels, ptrdiff_t line_size, int h);
150 static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,
151 int line_size, int h)
153 ff_put_pixels8_mmxext(block, pixels, line_size, h);
154 ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);
157 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
158 int dstStride, int srcStride, int h);
159 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
160 int dstStride, int srcStride, int h);
161 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
162 int dstStride, int srcStride,
164 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
165 int dstStride, int srcStride, int h);
166 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
167 int dstStride, int srcStride, int h);
168 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
169 int dstStride, int srcStride,
171 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
172 int dstStride, int srcStride);
173 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
174 int dstStride, int srcStride);
175 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
176 int dstStride, int srcStride);
177 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
178 int dstStride, int srcStride);
179 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
180 int dstStride, int srcStride);
181 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
182 int dstStride, int srcStride);
183 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
184 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
185 #endif /* HAVE_YASM */
190 #define JUMPALIGN() __asm__ volatile (".p2align 3"::)
191 #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)
193 #define MOVQ_BFE(regd) \
195 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
196 "paddb %%"#regd", %%"#regd" \n\t" ::)
199 #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))
200 #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))
202 // for shared library it's better to use this way for accessing constants
204 #define MOVQ_BONE(regd) \
206 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
207 "psrlw $15, %%"#regd" \n\t" \
208 "packuswb %%"#regd", %%"#regd" \n\t" ::)
210 #define MOVQ_WTWO(regd) \
212 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
213 "psrlw $15, %%"#regd" \n\t" \
214 "psllw $1, %%"#regd" \n\t"::)
218 // using regr as temporary and for the output result
219 // first argument is unmodifed and second is trashed
220 // regfe is supposed to contain 0xfefefefefefefefe
221 #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe) \
222 "movq "#rega", "#regr" \n\t" \
223 "pand "#regb", "#regr" \n\t" \
224 "pxor "#rega", "#regb" \n\t" \
225 "pand "#regfe", "#regb" \n\t" \
226 "psrlq $1, "#regb" \n\t" \
227 "paddb "#regb", "#regr" \n\t"
229 #define PAVGB_MMX(rega, regb, regr, regfe) \
230 "movq "#rega", "#regr" \n\t" \
231 "por "#regb", "#regr" \n\t" \
232 "pxor "#rega", "#regb" \n\t" \
233 "pand "#regfe", "#regb" \n\t" \
234 "psrlq $1, "#regb" \n\t" \
235 "psubb "#regb", "#regr" \n\t"
237 // mm6 is supposed to contain 0xfefefefefefefefe
238 #define PAVGBP_MMX_NO_RND(rega, regb, regr, regc, regd, regp) \
239 "movq "#rega", "#regr" \n\t" \
240 "movq "#regc", "#regp" \n\t" \
241 "pand "#regb", "#regr" \n\t" \
242 "pand "#regd", "#regp" \n\t" \
243 "pxor "#rega", "#regb" \n\t" \
244 "pxor "#regc", "#regd" \n\t" \
245 "pand %%mm6, "#regb" \n\t" \
246 "pand %%mm6, "#regd" \n\t" \
247 "psrlq $1, "#regb" \n\t" \
248 "psrlq $1, "#regd" \n\t" \
249 "paddb "#regb", "#regr" \n\t" \
250 "paddb "#regd", "#regp" \n\t"
252 #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp) \
253 "movq "#rega", "#regr" \n\t" \
254 "movq "#regc", "#regp" \n\t" \
255 "por "#regb", "#regr" \n\t" \
256 "por "#regd", "#regp" \n\t" \
257 "pxor "#rega", "#regb" \n\t" \
258 "pxor "#regc", "#regd" \n\t" \
259 "pand %%mm6, "#regb" \n\t" \
260 "pand %%mm6, "#regd" \n\t" \
261 "psrlq $1, "#regd" \n\t" \
262 "psrlq $1, "#regb" \n\t" \
263 "psubb "#regb", "#regr" \n\t" \
264 "psubb "#regd", "#regp" \n\t"
266 /***********************************/
267 /* MMX no rounding */
269 #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx
270 #define SET_RND MOVQ_WONE
271 #define PAVGBP(a, b, c, d, e, f) PAVGBP_MMX_NO_RND(a, b, c, d, e, f)
272 #define PAVGB(a, b, c, e) PAVGB_MMX_NO_RND(a, b, c, e)
273 #define OP_AVG(a, b, c, e) PAVGB_MMX(a, b, c, e)
275 #include "dsputil_rnd_template.c"
282 /***********************************/
285 #define DEF(x, y) x ## _ ## y ## _mmx
286 #define SET_RND MOVQ_WTWO
287 #define PAVGBP(a, b, c, d, e, f) PAVGBP_MMX(a, b, c, d, e, f)
288 #define PAVGB(a, b, c, e) PAVGB_MMX(a, b, c, e)
290 #include "dsputil_rnd_template.c"
298 #endif /* HAVE_INLINE_ASM */
302 #define ff_put_pixels8_mmx ff_put_pixels8_mmxext
304 /***********************************/
307 #define DEF(x) x ## _3dnow
309 #include "dsputil_avg_template.c"
313 /***********************************/
314 /* MMXEXT specific */
316 #define DEF(x) x ## _mmxext
318 #include "dsputil_avg_template.c"
322 #endif /* HAVE_YASM */
326 #define put_no_rnd_pixels16_mmx put_pixels16_mmx
327 #define put_no_rnd_pixels8_mmx put_pixels8_mmx
328 #define put_pixels16_mmxext put_pixels16_mmx
329 #define put_pixels8_mmxext put_pixels8_mmx
330 #define put_pixels4_mmxext put_pixels4_mmx
331 #define put_no_rnd_pixels16_mmxext put_no_rnd_pixels16_mmx
332 #define put_no_rnd_pixels8_mmxext put_no_rnd_pixels8_mmx
334 /***********************************/
337 void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
343 /* read the pixels */
348 "movq (%3), %%mm0 \n\t"
349 "movq 8(%3), %%mm1 \n\t"
350 "movq 16(%3), %%mm2 \n\t"
351 "movq 24(%3), %%mm3 \n\t"
352 "movq 32(%3), %%mm4 \n\t"
353 "movq 40(%3), %%mm5 \n\t"
354 "movq 48(%3), %%mm6 \n\t"
355 "movq 56(%3), %%mm7 \n\t"
356 "packuswb %%mm1, %%mm0 \n\t"
357 "packuswb %%mm3, %%mm2 \n\t"
358 "packuswb %%mm5, %%mm4 \n\t"
359 "packuswb %%mm7, %%mm6 \n\t"
360 "movq %%mm0, (%0) \n\t"
361 "movq %%mm2, (%0, %1) \n\t"
362 "movq %%mm4, (%0, %1, 2) \n\t"
363 "movq %%mm6, (%0, %2) \n\t"
364 :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),
367 pix += line_size * 4;
370 // if here would be an exact copy of the code above
371 // compiler would generate some very strange code
374 "movq (%3), %%mm0 \n\t"
375 "movq 8(%3), %%mm1 \n\t"
376 "movq 16(%3), %%mm2 \n\t"
377 "movq 24(%3), %%mm3 \n\t"
378 "movq 32(%3), %%mm4 \n\t"
379 "movq 40(%3), %%mm5 \n\t"
380 "movq 48(%3), %%mm6 \n\t"
381 "movq 56(%3), %%mm7 \n\t"
382 "packuswb %%mm1, %%mm0 \n\t"
383 "packuswb %%mm3, %%mm2 \n\t"
384 "packuswb %%mm5, %%mm4 \n\t"
385 "packuswb %%mm7, %%mm6 \n\t"
386 "movq %%mm0, (%0) \n\t"
387 "movq %%mm2, (%0, %1) \n\t"
388 "movq %%mm4, (%0, %1, 2) \n\t"
389 "movq %%mm6, (%0, %2) \n\t"
390 :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)
394 #define put_signed_pixels_clamped_mmx_half(off) \
395 "movq "#off"(%2), %%mm1 \n\t" \
396 "movq 16 + "#off"(%2), %%mm2 \n\t" \
397 "movq 32 + "#off"(%2), %%mm3 \n\t" \
398 "movq 48 + "#off"(%2), %%mm4 \n\t" \
399 "packsswb 8 + "#off"(%2), %%mm1 \n\t" \
400 "packsswb 24 + "#off"(%2), %%mm2 \n\t" \
401 "packsswb 40 + "#off"(%2), %%mm3 \n\t" \
402 "packsswb 56 + "#off"(%2), %%mm4 \n\t" \
403 "paddb %%mm0, %%mm1 \n\t" \
404 "paddb %%mm0, %%mm2 \n\t" \
405 "paddb %%mm0, %%mm3 \n\t" \
406 "paddb %%mm0, %%mm4 \n\t" \
407 "movq %%mm1, (%0) \n\t" \
408 "movq %%mm2, (%0, %3) \n\t" \
409 "movq %%mm3, (%0, %3, 2) \n\t" \
410 "movq %%mm4, (%0, %1) \n\t"
412 void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
415 x86_reg line_skip = line_size;
419 "movq "MANGLE(ff_pb_80)", %%mm0 \n\t"
420 "lea (%3, %3, 2), %1 \n\t"
421 put_signed_pixels_clamped_mmx_half(0)
422 "lea (%0, %3, 4), %0 \n\t"
423 put_signed_pixels_clamped_mmx_half(64)
424 : "+&r"(pixels), "=&r"(line_skip3)
425 : "r"(block), "r"(line_skip)
429 void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
436 /* read the pixels */
443 "movq (%2), %%mm0 \n\t"
444 "movq 8(%2), %%mm1 \n\t"
445 "movq 16(%2), %%mm2 \n\t"
446 "movq 24(%2), %%mm3 \n\t"
447 "movq %0, %%mm4 \n\t"
448 "movq %1, %%mm6 \n\t"
449 "movq %%mm4, %%mm5 \n\t"
450 "punpcklbw %%mm7, %%mm4 \n\t"
451 "punpckhbw %%mm7, %%mm5 \n\t"
452 "paddsw %%mm4, %%mm0 \n\t"
453 "paddsw %%mm5, %%mm1 \n\t"
454 "movq %%mm6, %%mm5 \n\t"
455 "punpcklbw %%mm7, %%mm6 \n\t"
456 "punpckhbw %%mm7, %%mm5 \n\t"
457 "paddsw %%mm6, %%mm2 \n\t"
458 "paddsw %%mm5, %%mm3 \n\t"
459 "packuswb %%mm1, %%mm0 \n\t"
460 "packuswb %%mm3, %%mm2 \n\t"
461 "movq %%mm0, %0 \n\t"
462 "movq %%mm2, %1 \n\t"
463 : "+m"(*pix), "+m"(*(pix + line_size))
466 pix += line_size * 2;
471 static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,
472 ptrdiff_t line_size, int h)
475 "lea (%3, %3), %%"REG_a" \n\t"
478 "movq (%1 ), %%mm0 \n\t"
479 "movq (%1, %3), %%mm1 \n\t"
480 "movq %%mm0, (%2) \n\t"
481 "movq %%mm1, (%2, %3) \n\t"
482 "add %%"REG_a", %1 \n\t"
483 "add %%"REG_a", %2 \n\t"
484 "movq (%1 ), %%mm0 \n\t"
485 "movq (%1, %3), %%mm1 \n\t"
486 "movq %%mm0, (%2) \n\t"
487 "movq %%mm1, (%2, %3) \n\t"
488 "add %%"REG_a", %1 \n\t"
489 "add %%"REG_a", %2 \n\t"
492 : "+g"(h), "+r"(pixels), "+r"(block)
493 : "r"((x86_reg)line_size)
498 static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,
499 ptrdiff_t line_size, int h)
502 "lea (%3, %3), %%"REG_a" \n\t"
505 "movq (%1 ), %%mm0 \n\t"
506 "movq 8(%1 ), %%mm4 \n\t"
507 "movq (%1, %3), %%mm1 \n\t"
508 "movq 8(%1, %3), %%mm5 \n\t"
509 "movq %%mm0, (%2) \n\t"
510 "movq %%mm4, 8(%2) \n\t"
511 "movq %%mm1, (%2, %3) \n\t"
512 "movq %%mm5, 8(%2, %3) \n\t"
513 "add %%"REG_a", %1 \n\t"
514 "add %%"REG_a", %2 \n\t"
515 "movq (%1 ), %%mm0 \n\t"
516 "movq 8(%1 ), %%mm4 \n\t"
517 "movq (%1, %3), %%mm1 \n\t"
518 "movq 8(%1, %3), %%mm5 \n\t"
519 "movq %%mm0, (%2) \n\t"
520 "movq %%mm4, 8(%2) \n\t"
521 "movq %%mm1, (%2, %3) \n\t"
522 "movq %%mm5, 8(%2, %3) \n\t"
523 "add %%"REG_a", %1 \n\t"
524 "add %%"REG_a", %2 \n\t"
527 : "+g"(h), "+r"(pixels), "+r"(block)
528 : "r"((x86_reg)line_size)
533 #define CLEAR_BLOCKS(name, n) \
534 static void name(int16_t *blocks) \
537 "pxor %%mm7, %%mm7 \n\t" \
538 "mov %1, %%"REG_a" \n\t" \
540 "movq %%mm7, (%0, %%"REG_a") \n\t" \
541 "movq %%mm7, 8(%0, %%"REG_a") \n\t" \
542 "movq %%mm7, 16(%0, %%"REG_a") \n\t" \
543 "movq %%mm7, 24(%0, %%"REG_a") \n\t" \
544 "add $32, %%"REG_a" \n\t" \
546 :: "r"(((uint8_t *)blocks) + 128 * n), \
551 CLEAR_BLOCKS(clear_blocks_mmx, 6)
552 CLEAR_BLOCKS(clear_block_mmx, 1)
554 static void clear_block_sse(int16_t *block)
557 "xorps %%xmm0, %%xmm0 \n"
558 "movaps %%xmm0, (%0) \n"
559 "movaps %%xmm0, 16(%0) \n"
560 "movaps %%xmm0, 32(%0) \n"
561 "movaps %%xmm0, 48(%0) \n"
562 "movaps %%xmm0, 64(%0) \n"
563 "movaps %%xmm0, 80(%0) \n"
564 "movaps %%xmm0, 96(%0) \n"
565 "movaps %%xmm0, 112(%0) \n"
571 static void clear_blocks_sse(int16_t *blocks)
574 "xorps %%xmm0, %%xmm0 \n"
575 "mov %1, %%"REG_a" \n"
577 "movaps %%xmm0, (%0, %%"REG_a") \n"
578 "movaps %%xmm0, 16(%0, %%"REG_a") \n"
579 "movaps %%xmm0, 32(%0, %%"REG_a") \n"
580 "movaps %%xmm0, 48(%0, %%"REG_a") \n"
581 "movaps %%xmm0, 64(%0, %%"REG_a") \n"
582 "movaps %%xmm0, 80(%0, %%"REG_a") \n"
583 "movaps %%xmm0, 96(%0, %%"REG_a") \n"
584 "movaps %%xmm0, 112(%0, %%"REG_a") \n"
585 "add $128, %%"REG_a" \n"
587 :: "r"(((uint8_t *)blocks) + 128 * 6),
593 static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)
599 "movq (%1, %0), %%mm0 \n\t"
600 "movq (%2, %0), %%mm1 \n\t"
601 "paddb %%mm0, %%mm1 \n\t"
602 "movq %%mm1, (%2, %0) \n\t"
603 "movq 8(%1, %0), %%mm0 \n\t"
604 "movq 8(%2, %0), %%mm1 \n\t"
605 "paddb %%mm0, %%mm1 \n\t"
606 "movq %%mm1, 8(%2, %0) \n\t"
612 : "r"(src), "r"(dst), "r"((x86_reg)w - 15)
615 dst[i + 0] += src[i + 0];
619 static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,
620 const uint8_t *diff, int w,
621 int *left, int *left_top)
625 int l = *left & 0xff;
626 int tl = *left_top & 0xff;
631 "movzbl (%3, %4), %2 \n"
644 "add (%6, %4), %b0 \n"
645 "mov %b0, (%5, %4) \n"
648 : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)
649 : "r"(dst + w), "r"(diff + w), "rm"(top + w)
656 static inline void transpose4x4(uint8_t *dst, uint8_t *src, x86_reg dst_stride, x86_reg src_stride){
657 __asm__ volatile( //FIXME could save 1 instruction if done as 8x4 ...
658 "movd (%1), %%mm0 \n\t"
660 "movd (%1), %%mm1 \n\t"
661 "movd (%1,%3,1), %%mm2 \n\t"
662 "movd (%1,%3,2), %%mm3 \n\t"
663 "punpcklbw %%mm1, %%mm0 \n\t"
664 "punpcklbw %%mm3, %%mm2 \n\t"
665 "movq %%mm0, %%mm1 \n\t"
666 "punpcklwd %%mm2, %%mm0 \n\t"
667 "punpckhwd %%mm2, %%mm1 \n\t"
668 "movd %%mm0, (%0) \n\t"
670 "punpckhdq %%mm0, %%mm0 \n\t"
671 "movd %%mm0, (%0) \n\t"
672 "movd %%mm1, (%0,%2,1) \n\t"
673 "punpckhdq %%mm1, %%mm1 \n\t"
674 "movd %%mm1, (%0,%2,2) \n\t"
684 #define H263_LOOP_FILTER \
685 "pxor %%mm7, %%mm7 \n\t" \
686 "movq %0, %%mm0 \n\t" \
687 "movq %0, %%mm1 \n\t" \
688 "movq %3, %%mm2 \n\t" \
689 "movq %3, %%mm3 \n\t" \
690 "punpcklbw %%mm7, %%mm0 \n\t" \
691 "punpckhbw %%mm7, %%mm1 \n\t" \
692 "punpcklbw %%mm7, %%mm2 \n\t" \
693 "punpckhbw %%mm7, %%mm3 \n\t" \
694 "psubw %%mm2, %%mm0 \n\t" \
695 "psubw %%mm3, %%mm1 \n\t" \
696 "movq %1, %%mm2 \n\t" \
697 "movq %1, %%mm3 \n\t" \
698 "movq %2, %%mm4 \n\t" \
699 "movq %2, %%mm5 \n\t" \
700 "punpcklbw %%mm7, %%mm2 \n\t" \
701 "punpckhbw %%mm7, %%mm3 \n\t" \
702 "punpcklbw %%mm7, %%mm4 \n\t" \
703 "punpckhbw %%mm7, %%mm5 \n\t" \
704 "psubw %%mm2, %%mm4 \n\t" \
705 "psubw %%mm3, %%mm5 \n\t" \
706 "psllw $2, %%mm4 \n\t" \
707 "psllw $2, %%mm5 \n\t" \
708 "paddw %%mm0, %%mm4 \n\t" \
709 "paddw %%mm1, %%mm5 \n\t" \
710 "pxor %%mm6, %%mm6 \n\t" \
711 "pcmpgtw %%mm4, %%mm6 \n\t" \
712 "pcmpgtw %%mm5, %%mm7 \n\t" \
713 "pxor %%mm6, %%mm4 \n\t" \
714 "pxor %%mm7, %%mm5 \n\t" \
715 "psubw %%mm6, %%mm4 \n\t" \
716 "psubw %%mm7, %%mm5 \n\t" \
717 "psrlw $3, %%mm4 \n\t" \
718 "psrlw $3, %%mm5 \n\t" \
719 "packuswb %%mm5, %%mm4 \n\t" \
720 "packsswb %%mm7, %%mm6 \n\t" \
721 "pxor %%mm7, %%mm7 \n\t" \
722 "movd %4, %%mm2 \n\t" \
723 "punpcklbw %%mm2, %%mm2 \n\t" \
724 "punpcklbw %%mm2, %%mm2 \n\t" \
725 "punpcklbw %%mm2, %%mm2 \n\t" \
726 "psubusb %%mm4, %%mm2 \n\t" \
727 "movq %%mm2, %%mm3 \n\t" \
728 "psubusb %%mm4, %%mm3 \n\t" \
729 "psubb %%mm3, %%mm2 \n\t" \
730 "movq %1, %%mm3 \n\t" \
731 "movq %2, %%mm4 \n\t" \
732 "pxor %%mm6, %%mm3 \n\t" \
733 "pxor %%mm6, %%mm4 \n\t" \
734 "paddusb %%mm2, %%mm3 \n\t" \
735 "psubusb %%mm2, %%mm4 \n\t" \
736 "pxor %%mm6, %%mm3 \n\t" \
737 "pxor %%mm6, %%mm4 \n\t" \
738 "paddusb %%mm2, %%mm2 \n\t" \
739 "packsswb %%mm1, %%mm0 \n\t" \
740 "pcmpgtb %%mm0, %%mm7 \n\t" \
741 "pxor %%mm7, %%mm0 \n\t" \
742 "psubb %%mm7, %%mm0 \n\t" \
743 "movq %%mm0, %%mm1 \n\t" \
744 "psubusb %%mm2, %%mm0 \n\t" \
745 "psubb %%mm0, %%mm1 \n\t" \
746 "pand %5, %%mm1 \n\t" \
747 "psrlw $2, %%mm1 \n\t" \
748 "pxor %%mm7, %%mm1 \n\t" \
749 "psubb %%mm7, %%mm1 \n\t" \
750 "movq %0, %%mm5 \n\t" \
751 "movq %3, %%mm6 \n\t" \
752 "psubb %%mm1, %%mm5 \n\t" \
753 "paddb %%mm1, %%mm6 \n\t"
755 static void h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale)
757 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
758 const int strength = ff_h263_loop_filter_strength[qscale];
763 "movq %%mm3, %1 \n\t"
764 "movq %%mm4, %2 \n\t"
765 "movq %%mm5, %0 \n\t"
766 "movq %%mm6, %3 \n\t"
767 : "+m"(*(uint64_t*)(src - 2 * stride)),
768 "+m"(*(uint64_t*)(src - 1 * stride)),
769 "+m"(*(uint64_t*)(src + 0 * stride)),
770 "+m"(*(uint64_t*)(src + 1 * stride))
771 : "g"(2 * strength), "m"(ff_pb_FC)
776 static void h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale)
778 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
779 const int strength = ff_h263_loop_filter_strength[qscale];
780 DECLARE_ALIGNED(8, uint64_t, temp)[4];
781 uint8_t *btemp = (uint8_t*)temp;
785 transpose4x4(btemp, src, 8, stride);
786 transpose4x4(btemp + 4, src + 4 * stride, 8, stride);
788 H263_LOOP_FILTER // 5 3 4 6
794 : "g"(2 * strength), "m"(ff_pb_FC)
798 "movq %%mm5, %%mm1 \n\t"
799 "movq %%mm4, %%mm0 \n\t"
800 "punpcklbw %%mm3, %%mm5 \n\t"
801 "punpcklbw %%mm6, %%mm4 \n\t"
802 "punpckhbw %%mm3, %%mm1 \n\t"
803 "punpckhbw %%mm6, %%mm0 \n\t"
804 "movq %%mm5, %%mm3 \n\t"
805 "movq %%mm1, %%mm6 \n\t"
806 "punpcklwd %%mm4, %%mm5 \n\t"
807 "punpcklwd %%mm0, %%mm1 \n\t"
808 "punpckhwd %%mm4, %%mm3 \n\t"
809 "punpckhwd %%mm0, %%mm6 \n\t"
810 "movd %%mm5, (%0) \n\t"
811 "punpckhdq %%mm5, %%mm5 \n\t"
812 "movd %%mm5, (%0, %2) \n\t"
813 "movd %%mm3, (%0, %2, 2) \n\t"
814 "punpckhdq %%mm3, %%mm3 \n\t"
815 "movd %%mm3, (%0, %3) \n\t"
816 "movd %%mm1, (%1) \n\t"
817 "punpckhdq %%mm1, %%mm1 \n\t"
818 "movd %%mm1, (%1, %2) \n\t"
819 "movd %%mm6, (%1, %2, 2) \n\t"
820 "punpckhdq %%mm6, %%mm6 \n\t"
821 "movd %%mm6, (%1, %3) \n\t"
823 "r"(src + 4 * stride),
824 "r"((x86_reg)stride),
825 "r"((x86_reg)(3 * stride))
830 /* Draw the edges of width 'w' of an image of size width, height
831 * this MMX version can only handle w == 8 || w == 16. */
832 static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,
833 int w, int h, int sides)
835 uint8_t *ptr, *last_line;
838 last_line = buf + (height - 1) * wrap;
844 "movd (%0), %%mm0 \n\t"
845 "punpcklbw %%mm0, %%mm0 \n\t"
846 "punpcklwd %%mm0, %%mm0 \n\t"
847 "punpckldq %%mm0, %%mm0 \n\t"
848 "movq %%mm0, -8(%0) \n\t"
849 "movq -8(%0, %2), %%mm1 \n\t"
850 "punpckhbw %%mm1, %%mm1 \n\t"
851 "punpckhwd %%mm1, %%mm1 \n\t"
852 "punpckhdq %%mm1, %%mm1 \n\t"
853 "movq %%mm1, (%0, %2) \n\t"
858 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
863 "movd (%0), %%mm0 \n\t"
864 "punpcklbw %%mm0, %%mm0 \n\t"
865 "punpcklwd %%mm0, %%mm0 \n\t"
866 "punpckldq %%mm0, %%mm0 \n\t"
867 "movq %%mm0, -8(%0) \n\t"
868 "movq %%mm0, -16(%0) \n\t"
869 "movq -8(%0, %2), %%mm1 \n\t"
870 "punpckhbw %%mm1, %%mm1 \n\t"
871 "punpckhwd %%mm1, %%mm1 \n\t"
872 "punpckhdq %%mm1, %%mm1 \n\t"
873 "movq %%mm1, (%0, %2) \n\t"
874 "movq %%mm1, 8(%0, %2) \n\t"
879 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
885 "movd (%0), %%mm0 \n\t"
886 "punpcklbw %%mm0, %%mm0 \n\t"
887 "punpcklwd %%mm0, %%mm0 \n\t"
888 "movd %%mm0, -4(%0) \n\t"
889 "movd -4(%0, %2), %%mm1 \n\t"
890 "punpcklbw %%mm1, %%mm1 \n\t"
891 "punpckhwd %%mm1, %%mm1 \n\t"
892 "punpckhdq %%mm1, %%mm1 \n\t"
893 "movd %%mm1, (%0, %2) \n\t"
898 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
902 /* top and bottom (and hopefully also the corners) */
903 if (sides & EDGE_TOP) {
904 for (i = 0; i < h; i += 4) {
905 ptr = buf - (i + 1) * wrap - w;
908 "movq (%1, %0), %%mm0 \n\t"
909 "movq %%mm0, (%0) \n\t"
910 "movq %%mm0, (%0, %2) \n\t"
911 "movq %%mm0, (%0, %2, 2) \n\t"
912 "movq %%mm0, (%0, %3) \n\t"
917 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),
918 "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)
923 if (sides & EDGE_BOTTOM) {
924 for (i = 0; i < h; i += 4) {
925 ptr = last_line + (i + 1) * wrap - w;
928 "movq (%1, %0), %%mm0 \n\t"
929 "movq %%mm0, (%0) \n\t"
930 "movq %%mm0, (%0, %2) \n\t"
931 "movq %%mm0, (%0, %2, 2) \n\t"
932 "movq %%mm0, (%0, %3) \n\t"
937 : "r"((x86_reg)last_line - (x86_reg)ptr - w),
938 "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),
939 "r"(ptr + width + 2 * w)
944 #endif /* HAVE_INLINE_ASM */
948 #define QPEL_OP(OPNAME, ROUNDER, RND, OP, MMX) \
949 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src, \
952 ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8); \
955 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src, \
959 uint8_t * const half = (uint8_t*)temp; \
960 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8, \
962 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half, \
963 stride, stride, 8); \
966 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src, \
969 ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride, \
973 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src, \
977 uint8_t * const half = (uint8_t*)temp; \
978 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8, \
980 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride, \
984 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src, \
988 uint8_t * const half = (uint8_t*)temp; \
989 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, \
991 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half, \
992 stride, stride, 8); \
995 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src, \
998 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src, \
1002 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src, \
1006 uint8_t * const half = (uint8_t*)temp; \
1007 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, \
1009 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
1013 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src, \
1016 uint64_t half[8 + 9]; \
1017 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1018 uint8_t * const halfHV = ((uint8_t*)half); \
1019 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1021 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, \
1023 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1024 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1028 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src, \
1031 uint64_t half[8 + 9]; \
1032 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1033 uint8_t * const halfHV = ((uint8_t*)half); \
1034 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1036 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1038 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1039 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1043 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src, \
1046 uint64_t half[8 + 9]; \
1047 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1048 uint8_t * const halfHV = ((uint8_t*)half); \
1049 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1051 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, \
1053 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1054 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1058 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src, \
1061 uint64_t half[8 + 9]; \
1062 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1063 uint8_t * const halfHV = ((uint8_t*)half); \
1064 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1066 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1068 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1069 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1073 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src, \
1076 uint64_t half[8 + 9]; \
1077 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1078 uint8_t * const halfHV = ((uint8_t*)half); \
1079 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1081 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1082 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1086 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src, \
1089 uint64_t half[8 + 9]; \
1090 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1091 uint8_t * const halfHV = ((uint8_t*)half); \
1092 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1094 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1095 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1099 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src, \
1102 uint64_t half[8 + 9]; \
1103 uint8_t * const halfH = ((uint8_t*)half); \
1104 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1106 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, \
1108 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1112 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src, \
1115 uint64_t half[8 + 9]; \
1116 uint8_t * const halfH = ((uint8_t*)half); \
1117 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1119 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1121 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1125 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src, \
1129 uint8_t * const halfH = ((uint8_t*)half); \
1130 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1132 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1136 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src, \
1139 ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16); \
1142 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src, \
1145 uint64_t temp[32]; \
1146 uint8_t * const half = (uint8_t*)temp; \
1147 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16, \
1149 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, \
1153 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src, \
1156 ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src, \
1157 stride, stride, 16);\
1160 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src, \
1163 uint64_t temp[32]; \
1164 uint8_t * const half = (uint8_t*)temp; \
1165 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16, \
1167 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half, \
1168 stride, stride, 16); \
1171 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src, \
1174 uint64_t temp[32]; \
1175 uint8_t * const half = (uint8_t*)temp; \
1176 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16, \
1178 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, \
1182 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src, \
1185 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src, \
1189 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src, \
1192 uint64_t temp[32]; \
1193 uint8_t * const half = (uint8_t*)temp; \
1194 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16, \
1196 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half, \
1197 stride, stride, 16); \
1200 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src, \
1203 uint64_t half[16 * 2 + 17 * 2]; \
1204 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1205 uint8_t * const halfHV = ((uint8_t*)half); \
1206 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1208 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1210 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1212 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1216 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src, \
1219 uint64_t half[16 * 2 + 17 * 2]; \
1220 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1221 uint8_t * const halfHV = ((uint8_t*)half); \
1222 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1224 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1226 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1228 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1232 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src, \
1235 uint64_t half[16 * 2 + 17 * 2]; \
1236 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1237 uint8_t * const halfHV = ((uint8_t*)half); \
1238 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1240 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1242 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1244 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1248 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src, \
1251 uint64_t half[16 * 2 + 17 * 2]; \
1252 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1253 uint8_t * const halfHV = ((uint8_t*)half); \
1254 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1256 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1258 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1260 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1264 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src, \
1267 uint64_t half[16 * 2 + 17 * 2]; \
1268 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1269 uint8_t * const halfHV = ((uint8_t*)half); \
1270 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1272 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1274 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1278 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src, \
1281 uint64_t half[16 * 2 + 17 * 2]; \
1282 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1283 uint8_t * const halfHV = ((uint8_t*)half); \
1284 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1286 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1288 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1292 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src, \
1295 uint64_t half[17 * 2]; \
1296 uint8_t * const halfH = ((uint8_t*)half); \
1297 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1299 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1301 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1305 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src, \
1308 uint64_t half[17 * 2]; \
1309 uint8_t * const halfH = ((uint8_t*)half); \
1310 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1312 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1314 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1318 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src, \
1321 uint64_t half[17 * 2]; \
1322 uint8_t * const halfH = ((uint8_t*)half); \
1323 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1325 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1329 #define PUT_OP(a, b, temp, size) \
1330 "mov"#size" "#a", "#b" \n\t"
1332 #define AVG_MMXEXT_OP(a, b, temp, size) \
1333 "mov"#size" "#b", "#temp" \n\t" \
1334 "pavgb "#temp", "#a" \n\t" \
1335 "mov"#size" "#a", "#b" \n\t"
1337 QPEL_OP(put_, ff_pw_16, _, PUT_OP, mmxext)
1338 QPEL_OP(avg_, ff_pw_16, _, AVG_MMXEXT_OP, mmxext)
1339 QPEL_OP(put_no_rnd_, ff_pw_15, _no_rnd_, PUT_OP, mmxext)
1340 #endif /* HAVE_YASM */
1344 void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1346 put_pixels8_xy2_mmx(dst, src, stride, 8);
1348 void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1350 put_pixels16_xy2_mmx(dst, src, stride, 16);
1352 void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1354 avg_pixels8_xy2_mmx(dst, src, stride, 8);
1356 void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1358 avg_pixels16_xy2_mmx(dst, src, stride, 16);
1361 typedef void emulated_edge_mc_func(uint8_t *dst, const uint8_t *src,
1362 ptrdiff_t linesize, int block_w, int block_h,
1363 int src_x, int src_y, int w, int h);
1365 static av_always_inline void gmc(uint8_t *dst, uint8_t *src,
1366 int stride, int h, int ox, int oy,
1367 int dxx, int dxy, int dyx, int dyy,
1368 int shift, int r, int width, int height,
1369 emulated_edge_mc_func *emu_edge_fn)
1372 const int ix = ox >> (16 + shift);
1373 const int iy = oy >> (16 + shift);
1374 const int oxs = ox >> 4;
1375 const int oys = oy >> 4;
1376 const int dxxs = dxx >> 4;
1377 const int dxys = dxy >> 4;
1378 const int dyxs = dyx >> 4;
1379 const int dyys = dyy >> 4;
1380 const uint16_t r4[4] = { r, r, r, r };
1381 const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };
1382 const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };
1383 const uint64_t shift2 = 2 * shift;
1384 #define MAX_STRIDE 4096U
1386 uint8_t edge_buf[(MAX_H + 1) * MAX_STRIDE];
1389 const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);
1390 const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);
1391 const int dxh = dxy * (h - 1);
1392 const int dyw = dyx * (w - 1);
1393 int need_emu = (unsigned)ix >= width - w ||
1394 (unsigned)iy >= height - h;
1396 if ( // non-constant fullpel offset (3% of blocks)
1397 ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |
1398 (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)
1399 // uses more than 16 bits of subpel mv (only at huge resolution)
1400 || (dxx | dxy | dyx | dyy) & 15
1401 || (need_emu && (h > MAX_H || stride > MAX_STRIDE))) {
1402 // FIXME could still use mmx for some of the rows
1403 ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,
1404 shift, r, width, height);
1408 src += ix + iy * stride;
1410 emu_edge_fn(edge_buf, src, stride, w + 1, h + 1, ix, iy, width, height);
1415 "movd %0, %%mm6 \n\t"
1416 "pxor %%mm7, %%mm7 \n\t"
1417 "punpcklwd %%mm6, %%mm6 \n\t"
1418 "punpcklwd %%mm6, %%mm6 \n\t"
1422 for (x = 0; x < w; x += 4) {
1423 uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),
1424 oxs - dxys + dxxs * (x + 1),
1425 oxs - dxys + dxxs * (x + 2),
1426 oxs - dxys + dxxs * (x + 3) };
1427 uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),
1428 oys - dyys + dyxs * (x + 1),
1429 oys - dyys + dyxs * (x + 2),
1430 oys - dyys + dyxs * (x + 3) };
1432 for (y = 0; y < h; y++) {
1434 "movq %0, %%mm4 \n\t"
1435 "movq %1, %%mm5 \n\t"
1436 "paddw %2, %%mm4 \n\t"
1437 "paddw %3, %%mm5 \n\t"
1438 "movq %%mm4, %0 \n\t"
1439 "movq %%mm5, %1 \n\t"
1440 "psrlw $12, %%mm4 \n\t"
1441 "psrlw $12, %%mm5 \n\t"
1442 : "+m"(*dx4), "+m"(*dy4)
1443 : "m"(*dxy4), "m"(*dyy4)
1447 "movq %%mm6, %%mm2 \n\t"
1448 "movq %%mm6, %%mm1 \n\t"
1449 "psubw %%mm4, %%mm2 \n\t"
1450 "psubw %%mm5, %%mm1 \n\t"
1451 "movq %%mm2, %%mm0 \n\t"
1452 "movq %%mm4, %%mm3 \n\t"
1453 "pmullw %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)
1454 "pmullw %%mm5, %%mm3 \n\t" // dx * dy
1455 "pmullw %%mm5, %%mm2 \n\t" // (s - dx) * dy
1456 "pmullw %%mm4, %%mm1 \n\t" // dx * (s - dy)
1458 "movd %4, %%mm5 \n\t"
1459 "movd %3, %%mm4 \n\t"
1460 "punpcklbw %%mm7, %%mm5 \n\t"
1461 "punpcklbw %%mm7, %%mm4 \n\t"
1462 "pmullw %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy
1463 "pmullw %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy
1465 "movd %2, %%mm5 \n\t"
1466 "movd %1, %%mm4 \n\t"
1467 "punpcklbw %%mm7, %%mm5 \n\t"
1468 "punpcklbw %%mm7, %%mm4 \n\t"
1469 "pmullw %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)
1470 "pmullw %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)
1471 "paddw %5, %%mm1 \n\t"
1472 "paddw %%mm3, %%mm2 \n\t"
1473 "paddw %%mm1, %%mm0 \n\t"
1474 "paddw %%mm2, %%mm0 \n\t"
1476 "psrlw %6, %%mm0 \n\t"
1477 "packuswb %%mm0, %%mm0 \n\t"
1478 "movd %%mm0, %0 \n\t"
1480 : "=m"(dst[x + y * stride])
1481 : "m"(src[0]), "m"(src[1]),
1482 "m"(src[stride]), "m"(src[stride + 1]),
1483 "m"(*r4), "m"(shift2)
1487 src += 4 - h * stride;
1494 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1495 int stride, int h, int ox, int oy,
1496 int dxx, int dxy, int dyx, int dyy,
1497 int shift, int r, int width, int height)
1499 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1500 width, height, &ff_emulated_edge_mc_8);
1503 static void gmc_sse(uint8_t *dst, uint8_t *src,
1504 int stride, int h, int ox, int oy,
1505 int dxx, int dxy, int dyx, int dyy,
1506 int shift, int r, int width, int height)
1508 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1509 width, height, &ff_emulated_edge_mc_8);
1512 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1513 int stride, int h, int ox, int oy,
1514 int dxx, int dxy, int dyx, int dyy,
1515 int shift, int r, int width, int height)
1517 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1518 width, height, &ff_emulated_edge_mc_8);
1523 #endif /* HAVE_INLINE_ASM */
1525 void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1526 ptrdiff_t line_size, int h);
1527 void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1528 ptrdiff_t line_size, int h);
1530 void ff_put_h264_chroma_mc8_rnd_mmx (uint8_t *dst, uint8_t *src,
1531 int stride, int h, int x, int y);
1532 void ff_avg_h264_chroma_mc8_rnd_mmxext(uint8_t *dst, uint8_t *src,
1533 int stride, int h, int x, int y);
1534 void ff_avg_h264_chroma_mc8_rnd_3dnow(uint8_t *dst, uint8_t *src,
1535 int stride, int h, int x, int y);
1537 void ff_put_h264_chroma_mc4_mmx (uint8_t *dst, uint8_t *src,
1538 int stride, int h, int x, int y);
1539 void ff_avg_h264_chroma_mc4_mmxext (uint8_t *dst, uint8_t *src,
1540 int stride, int h, int x, int y);
1541 void ff_avg_h264_chroma_mc4_3dnow (uint8_t *dst, uint8_t *src,
1542 int stride, int h, int x, int y);
1544 void ff_put_h264_chroma_mc2_mmxext (uint8_t *dst, uint8_t *src,
1545 int stride, int h, int x, int y);
1546 void ff_avg_h264_chroma_mc2_mmxext (uint8_t *dst, uint8_t *src,
1547 int stride, int h, int x, int y);
1549 void ff_put_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,
1550 int stride, int h, int x, int y);
1551 void ff_put_h264_chroma_mc4_ssse3 (uint8_t *dst, uint8_t *src,
1552 int stride, int h, int x, int y);
1554 void ff_avg_h264_chroma_mc8_rnd_ssse3(uint8_t *dst, uint8_t *src,
1555 int stride, int h, int x, int y);
1556 void ff_avg_h264_chroma_mc4_ssse3 (uint8_t *dst, uint8_t *src,
1557 int stride, int h, int x, int y);
1559 #define CHROMA_MC(OP, NUM, DEPTH, OPT) \
1560 void ff_ ## OP ## _h264_chroma_mc ## NUM ## _ ## DEPTH ## _ ## OPT \
1561 (uint8_t *dst, uint8_t *src, \
1562 int stride, int h, int x, int y);
1564 CHROMA_MC(put, 2, 10, mmxext)
1565 CHROMA_MC(avg, 2, 10, mmxext)
1566 CHROMA_MC(put, 4, 10, mmxext)
1567 CHROMA_MC(avg, 4, 10, mmxext)
1568 CHROMA_MC(put, 8, 10, sse2)
1569 CHROMA_MC(avg, 8, 10, sse2)
1570 CHROMA_MC(put, 8, 10, avx)
1571 CHROMA_MC(avg, 8, 10, avx)
1576 void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1578 put_pixels8_mmx(dst, src, stride, 8);
1581 void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1583 avg_pixels8_mmx(dst, src, stride, 8);
1586 void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1588 put_pixels16_mmx(dst, src, stride, 16);
1591 void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1593 avg_pixels16_mmx(dst, src, stride, 16);
1595 #endif /* HAVE_INLINE_ASM */
1599 void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,
1600 int stride, int rnd)
1602 ff_put_pixels8_mmx(dst, src, stride, 8);
1605 void ff_avg_vc1_mspel_mc00_mmxext(uint8_t *dst, const uint8_t *src,
1606 int stride, int rnd)
1608 ff_avg_pixels8_mmxext(dst, src, stride, 8);
1610 #endif /* HAVE_YASM */
1612 #if CONFIG_DIRAC_DECODER
1613 #define DIRAC_PIXOP(OPNAME2, OPNAME, EXT)\
1614 void ff_ ## OPNAME2 ## _dirac_pixels8_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1617 ff_ ## OPNAME2 ## _dirac_pixels8_c(dst, src, stride, h);\
1619 OPNAME ## _pixels8_ ## EXT(dst, src[0], stride, h);\
1621 void ff_ ## OPNAME2 ## _dirac_pixels16_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1624 ff_ ## OPNAME2 ## _dirac_pixels16_c(dst, src, stride, h);\
1626 OPNAME ## _pixels16_ ## EXT(dst, src[0], stride, h);\
1628 void ff_ ## OPNAME2 ## _dirac_pixels32_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1631 ff_ ## OPNAME2 ## _dirac_pixels32_c(dst, src, stride, h);\
1633 OPNAME ## _pixels16_ ## EXT(dst , src[0] , stride, h);\
1634 OPNAME ## _pixels16_ ## EXT(dst+16, src[0]+16, stride, h);\
1639 DIRAC_PIXOP(put, put, mmx)
1640 DIRAC_PIXOP(avg, avg, mmx)
1644 DIRAC_PIXOP(avg, ff_avg, mmxext)
1646 void ff_put_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1649 ff_put_dirac_pixels16_c(dst, src, stride, h);
1651 ff_put_pixels16_sse2(dst, src[0], stride, h);
1653 void ff_avg_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1656 ff_avg_dirac_pixels16_c(dst, src, stride, h);
1658 ff_avg_pixels16_sse2(dst, src[0], stride, h);
1660 void ff_put_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1663 ff_put_dirac_pixels32_c(dst, src, stride, h);
1665 ff_put_pixels16_sse2(dst , src[0] , stride, h);
1666 ff_put_pixels16_sse2(dst+16, src[0]+16, stride, h);
1669 void ff_avg_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1672 ff_avg_dirac_pixels32_c(dst, src, stride, h);
1674 ff_avg_pixels16_sse2(dst , src[0] , stride, h);
1675 ff_avg_pixels16_sse2(dst+16, src[0]+16, stride, h);
1681 /* XXX: Those functions should be suppressed ASAP when all IDCTs are
1684 static void ff_libmpeg2mmx_idct_put(uint8_t *dest, int line_size,
1688 ff_put_pixels_clamped_mmx(block, dest, line_size);
1691 static void ff_libmpeg2mmx_idct_add(uint8_t *dest, int line_size,
1695 ff_add_pixels_clamped_mmx(block, dest, line_size);
1698 static void ff_libmpeg2mmx2_idct_put(uint8_t *dest, int line_size,
1701 ff_mmxext_idct(block);
1702 ff_put_pixels_clamped_mmx(block, dest, line_size);
1705 static void ff_libmpeg2mmx2_idct_add(uint8_t *dest, int line_size,
1708 ff_mmxext_idct(block);
1709 ff_add_pixels_clamped_mmx(block, dest, line_size);
1714 static void vector_clipf_sse(float *dst, const float *src,
1715 float min, float max, int len)
1717 x86_reg i = (len - 16) * 4;
1719 "movss %3, %%xmm4 \n\t"
1720 "movss %4, %%xmm5 \n\t"
1721 "shufps $0, %%xmm4, %%xmm4 \n\t"
1722 "shufps $0, %%xmm5, %%xmm5 \n\t"
1724 "movaps (%2, %0), %%xmm0 \n\t" // 3/1 on intel
1725 "movaps 16(%2, %0), %%xmm1 \n\t"
1726 "movaps 32(%2, %0), %%xmm2 \n\t"
1727 "movaps 48(%2, %0), %%xmm3 \n\t"
1728 "maxps %%xmm4, %%xmm0 \n\t"
1729 "maxps %%xmm4, %%xmm1 \n\t"
1730 "maxps %%xmm4, %%xmm2 \n\t"
1731 "maxps %%xmm4, %%xmm3 \n\t"
1732 "minps %%xmm5, %%xmm0 \n\t"
1733 "minps %%xmm5, %%xmm1 \n\t"
1734 "minps %%xmm5, %%xmm2 \n\t"
1735 "minps %%xmm5, %%xmm3 \n\t"
1736 "movaps %%xmm0, (%1, %0) \n\t"
1737 "movaps %%xmm1, 16(%1, %0) \n\t"
1738 "movaps %%xmm2, 32(%1, %0) \n\t"
1739 "movaps %%xmm3, 48(%1, %0) \n\t"
1743 : "r"(dst), "r"(src), "m"(min), "m"(max)
1748 #endif /* HAVE_INLINE_ASM */
1750 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
1752 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
1754 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
1756 int order, int mul);
1757 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
1759 int order, int mul);
1760 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
1762 int order, int mul);
1764 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
1765 const int16_t *window, unsigned int len);
1766 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
1767 const int16_t *window, unsigned int len);
1768 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
1769 const int16_t *window, unsigned int len);
1770 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
1771 const int16_t *window, unsigned int len);
1772 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
1773 const int16_t *window, unsigned int len);
1774 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
1775 const int16_t *window, unsigned int len);
1777 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
1778 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
1780 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
1781 const uint8_t *diff, int w,
1782 int *left, int *left_top);
1783 int ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
1785 int ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
1788 void ff_vector_clip_int32_mmx (int32_t *dst, const int32_t *src,
1789 int32_t min, int32_t max, unsigned int len);
1790 void ff_vector_clip_int32_sse2 (int32_t *dst, const int32_t *src,
1791 int32_t min, int32_t max, unsigned int len);
1792 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
1793 int32_t min, int32_t max, unsigned int len);
1794 void ff_vector_clip_int32_sse4 (int32_t *dst, const int32_t *src,
1795 int32_t min, int32_t max, unsigned int len);
1797 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX) \
1799 c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
1800 c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
1801 c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
1802 c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
1803 c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
1804 c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
1805 c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
1806 c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
1807 c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
1808 c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
1809 c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
1810 c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
1811 c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
1812 c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
1813 c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
1814 c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
1817 #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU) \
1819 c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _ ## CPU; \
1820 c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_ ## CPU; \
1821 c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_ ## CPU; \
1822 c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \
1825 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
1828 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1831 c->put_pixels_clamped = ff_put_pixels_clamped_mmx;
1832 c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
1833 c->add_pixels_clamped = ff_add_pixels_clamped_mmx;
1835 if (!high_bit_depth) {
1836 c->clear_block = clear_block_mmx;
1837 c->clear_blocks = clear_blocks_mmx;
1838 c->draw_edges = draw_edges_mmx;
1840 SET_HPEL_FUNCS(put, [0], 16, mmx);
1841 SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);
1842 SET_HPEL_FUNCS(avg, [0], 16, mmx);
1843 SET_HPEL_FUNCS(avg_no_rnd, , 16, mmx);
1844 SET_HPEL_FUNCS(put, [1], 8, mmx);
1845 SET_HPEL_FUNCS(put_no_rnd, [1], 8, mmx);
1846 SET_HPEL_FUNCS(avg, [1], 8, mmx);
1849 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
1853 c->add_bytes = add_bytes_mmx;
1855 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
1856 c->h263_v_loop_filter = h263_v_loop_filter_mmx;
1857 c->h263_h_loop_filter = h263_h_loop_filter_mmx;
1859 #endif /* HAVE_INLINE_ASM */
1862 if (!high_bit_depth && CONFIG_H264CHROMA) {
1863 c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_mmx;
1864 c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_mmx;
1867 c->vector_clip_int32 = ff_vector_clip_int32_mmx;
1872 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
1875 const int bit_depth = avctx->bits_per_raw_sample;
1876 const int high_bit_depth = bit_depth > 8;
1879 SET_QPEL_FUNCS(avg_qpel, 0, 16, mmxext, );
1880 SET_QPEL_FUNCS(avg_qpel, 1, 8, mmxext, );
1882 SET_QPEL_FUNCS(put_qpel, 0, 16, mmxext, );
1883 SET_QPEL_FUNCS(put_qpel, 1, 8, mmxext, );
1884 SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
1885 SET_QPEL_FUNCS(put_no_rnd_qpel, 1, 8, mmxext, );
1887 if (!high_bit_depth) {
1888 c->put_pixels_tab[0][1] = ff_put_pixels16_x2_mmxext;
1889 c->put_pixels_tab[0][2] = ff_put_pixels16_y2_mmxext;
1891 c->avg_pixels_tab[0][0] = ff_avg_pixels16_mmxext;
1892 c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_mmxext;
1893 c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_mmxext;
1895 c->put_pixels_tab[1][1] = ff_put_pixels8_x2_mmxext;
1896 c->put_pixels_tab[1][2] = ff_put_pixels8_y2_mmxext;
1898 c->avg_pixels_tab[1][0] = ff_avg_pixels8_mmxext;
1899 c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_mmxext;
1900 c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_mmxext;
1903 if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {
1904 if (!high_bit_depth) {
1905 c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_mmxext;
1906 c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_mmxext;
1907 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_mmxext;
1908 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_mmxext;
1910 c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_mmxext;
1911 c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_mmxext;
1914 #endif /* HAVE_YASM */
1916 #if HAVE_MMXEXT_EXTERNAL
1917 if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1918 avctx->codec_id == AV_CODEC_ID_THEORA)) {
1919 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_mmxext;
1920 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_mmxext;
1923 if (!high_bit_depth && CONFIG_H264CHROMA) {
1924 c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_mmxext;
1925 c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_mmxext;
1926 c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_mmxext;
1927 c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_mmxext;
1929 if (bit_depth == 10 && CONFIG_H264CHROMA) {
1930 c->put_h264_chroma_pixels_tab[2] = ff_put_h264_chroma_mc2_10_mmxext;
1931 c->avg_h264_chroma_pixels_tab[2] = ff_avg_h264_chroma_mc2_10_mmxext;
1932 c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_10_mmxext;
1933 c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_10_mmxext;
1936 /* slower than cmov version on AMD */
1937 if (!(mm_flags & AV_CPU_FLAG_3DNOW))
1938 c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
1940 c->scalarproduct_int16 = ff_scalarproduct_int16_mmxext;
1941 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
1943 if (avctx->flags & CODEC_FLAG_BITEXACT) {
1944 c->apply_window_int16 = ff_apply_window_int16_mmxext;
1946 c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
1948 #endif /* HAVE_MMXEXT_EXTERNAL */
1951 static av_cold void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,
1954 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1957 if (!high_bit_depth) {
1958 c->put_pixels_tab[0][1] = ff_put_pixels16_x2_3dnow;
1959 c->put_pixels_tab[0][2] = ff_put_pixels16_y2_3dnow;
1961 c->avg_pixels_tab[0][0] = ff_avg_pixels16_3dnow;
1962 c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_3dnow;
1963 c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_3dnow;
1965 c->put_pixels_tab[1][1] = ff_put_pixels8_x2_3dnow;
1966 c->put_pixels_tab[1][2] = ff_put_pixels8_y2_3dnow;
1968 c->avg_pixels_tab[1][0] = ff_avg_pixels8_3dnow;
1969 c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_3dnow;
1970 c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_3dnow;
1972 if (!(avctx->flags & CODEC_FLAG_BITEXACT)){
1973 c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_3dnow;
1974 c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_3dnow;
1975 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_3dnow;
1976 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_3dnow;
1978 c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_3dnow;
1979 c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_3dnow;
1983 if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1984 avctx->codec_id == AV_CODEC_ID_THEORA)) {
1985 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_3dnow;
1986 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_3dnow;
1989 if (!high_bit_depth && CONFIG_H264CHROMA) {
1990 c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_3dnow;
1991 c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_3dnow;
1993 #endif /* HAVE_YASM */
1996 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
1999 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
2002 if (!high_bit_depth) {
2003 if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
2004 /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
2005 c->clear_block = clear_block_sse;
2006 c->clear_blocks = clear_blocks_sse;
2010 c->vector_clipf = vector_clipf_sse;
2011 #endif /* HAVE_INLINE_ASM */
2014 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
2017 #endif /* HAVE_YASM */
2020 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
2023 const int bit_depth = avctx->bits_per_raw_sample;
2024 const int high_bit_depth = bit_depth > 8;
2026 #if HAVE_SSE2_INLINE
2027 if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
2028 c->idct_put = ff_idct_xvid_sse2_put;
2029 c->idct_add = ff_idct_xvid_sse2_add;
2030 c->idct = ff_idct_xvid_sse2;
2031 c->idct_permutation_type = FF_SSE2_IDCT_PERM;
2033 #endif /* HAVE_SSE2_INLINE */
2035 #if HAVE_SSE2_EXTERNAL
2036 if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
2037 // these functions are slower than mmx on AMD, but faster on Intel
2038 if (!high_bit_depth) {
2039 c->put_pixels_tab[0][0] = ff_put_pixels16_sse2;
2040 c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;
2041 c->avg_pixels_tab[0][0] = ff_avg_pixels16_sse2;
2045 if (bit_depth == 10) {
2046 if (CONFIG_H264CHROMA) {
2047 c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_sse2;
2048 c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_sse2;
2052 c->scalarproduct_int16 = ff_scalarproduct_int16_sse2;
2053 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
2054 if (mm_flags & AV_CPU_FLAG_ATOM) {
2055 c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
2057 c->vector_clip_int32 = ff_vector_clip_int32_sse2;
2059 if (avctx->flags & CODEC_FLAG_BITEXACT) {
2060 c->apply_window_int16 = ff_apply_window_int16_sse2;
2061 } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
2062 c->apply_window_int16 = ff_apply_window_int16_round_sse2;
2064 c->bswap_buf = ff_bswap32_buf_sse2;
2065 #endif /* HAVE_SSE2_EXTERNAL */
2068 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
2071 #if HAVE_SSSE3_EXTERNAL
2072 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
2074 if (!high_bit_depth && CONFIG_H264CHROMA) {
2075 c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_rnd_ssse3;
2076 c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_rnd_ssse3;
2077 c->put_h264_chroma_pixels_tab[1] = ff_put_h264_chroma_mc4_ssse3;
2078 c->avg_h264_chroma_pixels_tab[1] = ff_avg_h264_chroma_mc4_ssse3;
2080 c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
2081 if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
2082 c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
2084 if (mm_flags & AV_CPU_FLAG_ATOM)
2085 c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
2087 c->apply_window_int16 = ff_apply_window_int16_ssse3;
2088 if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
2089 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
2090 c->bswap_buf = ff_bswap32_buf_ssse3;
2091 #endif /* HAVE_SSSE3_EXTERNAL */
2094 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
2097 #if HAVE_SSE4_EXTERNAL
2098 c->vector_clip_int32 = ff_vector_clip_int32_sse4;
2099 #endif /* HAVE_SSE4_EXTERNAL */
2102 static av_cold void dsputil_init_avx(DSPContext *c, AVCodecContext *avctx, int mm_flags)
2104 #if HAVE_AVX_EXTERNAL
2105 const int bit_depth = avctx->bits_per_raw_sample;
2107 if (bit_depth == 10) {
2108 if (CONFIG_H264CHROMA) {
2109 c->put_h264_chroma_pixels_tab[0] = ff_put_h264_chroma_mc8_10_avx;
2110 c->avg_h264_chroma_pixels_tab[0] = ff_avg_h264_chroma_mc8_10_avx;
2113 #endif /* HAVE_AVX_EXTERNAL */
2116 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
2118 int mm_flags = av_get_cpu_flags();
2120 #if HAVE_7REGS && HAVE_INLINE_ASM
2121 if (mm_flags & AV_CPU_FLAG_CMOV)
2122 c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;
2125 if (mm_flags & AV_CPU_FLAG_MMX) {
2127 const int idct_algo = avctx->idct_algo;
2129 if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
2130 if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
2131 c->idct_put = ff_simple_idct_put_mmx;
2132 c->idct_add = ff_simple_idct_add_mmx;
2133 c->idct = ff_simple_idct_mmx;
2134 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
2136 } else if (idct_algo == FF_IDCT_LIBMPEG2MMX) {
2137 if (mm_flags & AV_CPU_FLAG_MMX2) {
2138 c->idct_put = ff_libmpeg2mmx2_idct_put;
2139 c->idct_add = ff_libmpeg2mmx2_idct_add;
2140 c->idct = ff_mmxext_idct;
2142 c->idct_put = ff_libmpeg2mmx_idct_put;
2143 c->idct_add = ff_libmpeg2mmx_idct_add;
2144 c->idct = ff_mmx_idct;
2146 c->idct_permutation_type = FF_LIBMPEG2_IDCT_PERM;
2148 } else if (idct_algo == FF_IDCT_XVIDMMX) {
2149 if (mm_flags & AV_CPU_FLAG_SSE2) {
2150 c->idct_put = ff_idct_xvid_sse2_put;
2151 c->idct_add = ff_idct_xvid_sse2_add;
2152 c->idct = ff_idct_xvid_sse2;
2153 c->idct_permutation_type = FF_SSE2_IDCT_PERM;
2154 } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
2155 c->idct_put = ff_idct_xvid_mmxext_put;
2156 c->idct_add = ff_idct_xvid_mmxext_add;
2157 c->idct = ff_idct_xvid_mmxext;
2159 c->idct_put = ff_idct_xvid_mmx_put;
2160 c->idct_add = ff_idct_xvid_mmx_add;
2161 c->idct = ff_idct_xvid_mmx;
2165 #endif /* HAVE_INLINE_ASM */
2167 dsputil_init_mmx(c, avctx, mm_flags);
2170 if (mm_flags & AV_CPU_FLAG_MMXEXT)
2171 dsputil_init_mmxext(c, avctx, mm_flags);
2173 if (mm_flags & AV_CPU_FLAG_3DNOW)
2174 dsputil_init_3dnow(c, avctx, mm_flags);
2176 if (mm_flags & AV_CPU_FLAG_SSE)
2177 dsputil_init_sse(c, avctx, mm_flags);
2179 if (mm_flags & AV_CPU_FLAG_SSE2)
2180 dsputil_init_sse2(c, avctx, mm_flags);
2182 if (mm_flags & AV_CPU_FLAG_SSSE3)
2183 dsputil_init_ssse3(c, avctx, mm_flags);
2185 if (mm_flags & AV_CPU_FLAG_SSE4)
2186 dsputil_init_sse4(c, avctx, mm_flags);
2188 if (mm_flags & AV_CPU_FLAG_AVX)
2189 dsputil_init_avx(c, avctx, mm_flags);
2191 if (CONFIG_ENCODERS)
2192 ff_dsputilenc_init_mmx(c, avctx);