2 * MMX optimized DSP utils
3 * Copyright (c) 2000, 2001 Fabrice Bellard
4 * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
6 * This file is part of FFmpeg.
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 * MMX optimization by Nick Kurshev <nickols_k@mail.ru>
25 #include "libavutil/attributes.h"
26 #include "libavutil/cpu.h"
27 #include "libavutil/x86/asm.h"
28 #include "libavcodec/dsputil.h"
29 #include "libavcodec/h264dsp.h"
30 #include "libavcodec/mpegvideo.h"
31 #include "libavcodec/simple_idct.h"
32 #include "dsputil_mmx.h"
33 #include "idct_xvid.h"
34 #include "diracdsp_mmx.h"
39 /* pixel operations */
40 DECLARE_ALIGNED(8, const uint64_t, ff_bone) = 0x0101010101010101ULL;
41 DECLARE_ALIGNED(8, const uint64_t, ff_wtwo) = 0x0002000200020002ULL;
43 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_1) = { 0x0001000100010001ULL, 0x0001000100010001ULL };
44 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_2) = { 0x0002000200020002ULL, 0x0002000200020002ULL };
45 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_3) = { 0x0003000300030003ULL, 0x0003000300030003ULL };
46 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_4) = { 0x0004000400040004ULL, 0x0004000400040004ULL };
47 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_5) = { 0x0005000500050005ULL, 0x0005000500050005ULL };
48 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_8) = { 0x0008000800080008ULL, 0x0008000800080008ULL };
49 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_9) = { 0x0009000900090009ULL, 0x0009000900090009ULL };
50 DECLARE_ALIGNED(8, const uint64_t, ff_pw_15) = 0x000F000F000F000FULL;
51 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_16) = { 0x0010001000100010ULL, 0x0010001000100010ULL };
52 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_17) = { 0x0011001100110011ULL, 0x0011001100110011ULL };
53 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_18) = { 0x0012001200120012ULL, 0x0012001200120012ULL };
54 DECLARE_ALIGNED(8, const uint64_t, ff_pw_20) = 0x0014001400140014ULL;
55 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_27) = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };
56 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_28) = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };
57 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_32) = { 0x0020002000200020ULL, 0x0020002000200020ULL };
58 DECLARE_ALIGNED(8, const uint64_t, ff_pw_42) = 0x002A002A002A002AULL;
59 DECLARE_ALIGNED(8, const uint64_t, ff_pw_53) = 0x0035003500350035ULL;
60 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_63) = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };
61 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_64) = { 0x0040004000400040ULL, 0x0040004000400040ULL };
62 DECLARE_ALIGNED(8, const uint64_t, ff_pw_96) = 0x0060006000600060ULL;
63 DECLARE_ALIGNED(8, const uint64_t, ff_pw_128) = 0x0080008000800080ULL;
64 DECLARE_ALIGNED(8, const uint64_t, ff_pw_255) = 0x00ff00ff00ff00ffULL;
65 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_512) = { 0x0200020002000200ULL, 0x0200020002000200ULL };
66 DECLARE_ALIGNED(16, const xmm_reg, ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };
68 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_0) = { 0x0000000000000000ULL, 0x0000000000000000ULL };
69 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_1) = { 0x0101010101010101ULL, 0x0101010101010101ULL };
70 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_3) = { 0x0303030303030303ULL, 0x0303030303030303ULL };
71 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_4) = { 0x0404040404040404ULL, 0x0404040404040404ULL };
72 DECLARE_ALIGNED(8, const uint64_t, ff_pb_7) = 0x0707070707070707ULL;
73 DECLARE_ALIGNED(8, const uint64_t, ff_pb_1F) = 0x1F1F1F1F1F1F1F1FULL;
74 DECLARE_ALIGNED(8, const uint64_t, ff_pb_3F) = 0x3F3F3F3F3F3F3F3FULL;
75 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_80) = { 0x8080808080808080ULL, 0x8080808080808080ULL };
76 DECLARE_ALIGNED(8, const uint64_t, ff_pb_81) = 0x8181818181818181ULL;
77 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_A1) = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };
78 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_F8) = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };
79 DECLARE_ALIGNED(8, const uint64_t, ff_pb_FC) = 0xFCFCFCFCFCFCFCFCULL;
80 DECLARE_ALIGNED(16, const xmm_reg, ff_pb_FE) = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };
82 DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };
83 DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };
87 void ff_put_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
88 ptrdiff_t line_size, int h);
89 void ff_put_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
90 ptrdiff_t line_size, int h);
91 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
92 int dstStride, int src1Stride, int h);
93 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
94 uint8_t *src2, int dstStride,
95 int src1Stride, int h);
96 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
97 int dstStride, int src1Stride, int h);
98 void ff_put_pixels16_x2_mmxext(uint8_t *block, const uint8_t *pixels,
99 ptrdiff_t line_size, int h);
100 void ff_put_pixels16_x2_3dnow(uint8_t *block, const uint8_t *pixels,
101 ptrdiff_t line_size, int h);
102 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
103 int dstStride, int src1Stride, int h);
104 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
105 int dstStride, int src1Stride, int h);
106 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
107 int dstStride, int src1Stride, int h);
108 void ff_put_no_rnd_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
109 ptrdiff_t line_size, int h);
110 void ff_put_no_rnd_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
111 ptrdiff_t line_size, int h);
112 void ff_put_no_rnd_pixels8_x2_exact_mmxext(uint8_t *block,
113 const uint8_t *pixels,
114 ptrdiff_t line_size, int h);
115 void ff_put_no_rnd_pixels8_x2_exact_3dnow(uint8_t *block,
116 const uint8_t *pixels,
117 ptrdiff_t line_size, int h);
118 void ff_put_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
119 ptrdiff_t line_size, int h);
120 void ff_put_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
121 ptrdiff_t line_size, int h);
122 void ff_put_no_rnd_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
123 ptrdiff_t line_size, int h);
124 void ff_put_no_rnd_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
125 ptrdiff_t line_size, int h);
126 void ff_put_no_rnd_pixels8_y2_exact_mmxext(uint8_t *block,
127 const uint8_t *pixels,
128 ptrdiff_t line_size, int h);
129 void ff_put_no_rnd_pixels8_y2_exact_3dnow(uint8_t *block,
130 const uint8_t *pixels,
131 ptrdiff_t line_size, int h);
132 void ff_avg_pixels8_mmxext(uint8_t *block, const uint8_t *pixels,
133 ptrdiff_t line_size, int h);
134 void ff_avg_pixels8_3dnow(uint8_t *block, const uint8_t *pixels,
135 ptrdiff_t line_size, int h);
136 void ff_avg_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
137 ptrdiff_t line_size, int h);
138 void ff_avg_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
139 ptrdiff_t line_size, int h);
140 void ff_avg_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
141 ptrdiff_t line_size, int h);
142 void ff_avg_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
143 ptrdiff_t line_size, int h);
144 void ff_avg_pixels8_xy2_mmxext(uint8_t *block, const uint8_t *pixels,
145 ptrdiff_t line_size, int h);
146 void ff_avg_pixels8_xy2_3dnow(uint8_t *block, const uint8_t *pixels,
147 ptrdiff_t line_size, int h);
149 void ff_put_pixels8_mmxext(uint8_t *block, const uint8_t *pixels, ptrdiff_t line_size, int h);
150 static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,
151 int line_size, int h)
153 ff_put_pixels8_mmxext(block, pixels, line_size, h);
154 ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);
157 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
158 int dstStride, int srcStride, int h);
159 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
160 int dstStride, int srcStride, int h);
161 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
162 int dstStride, int srcStride,
164 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
165 int dstStride, int srcStride, int h);
166 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
167 int dstStride, int srcStride, int h);
168 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
169 int dstStride, int srcStride,
171 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
172 int dstStride, int srcStride);
173 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
174 int dstStride, int srcStride);
175 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
176 int dstStride, int srcStride);
177 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
178 int dstStride, int srcStride);
179 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
180 int dstStride, int srcStride);
181 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
182 int dstStride, int srcStride);
183 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
184 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
185 #endif /* HAVE_YASM */
190 #define JUMPALIGN() __asm__ volatile (".p2align 3"::)
191 #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)
193 #define MOVQ_BFE(regd) \
195 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
196 "paddb %%"#regd", %%"#regd" \n\t" ::)
199 #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))
200 #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))
202 // for shared library it's better to use this way for accessing constants
204 #define MOVQ_BONE(regd) \
206 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
207 "psrlw $15, %%"#regd" \n\t" \
208 "packuswb %%"#regd", %%"#regd" \n\t" ::)
210 #define MOVQ_WTWO(regd) \
212 "pcmpeqd %%"#regd", %%"#regd" \n\t" \
213 "psrlw $15, %%"#regd" \n\t" \
214 "psllw $1, %%"#regd" \n\t"::)
218 // using regr as temporary and for the output result
219 // first argument is unmodifed and second is trashed
220 // regfe is supposed to contain 0xfefefefefefefefe
221 #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe) \
222 "movq "#rega", "#regr" \n\t" \
223 "pand "#regb", "#regr" \n\t" \
224 "pxor "#rega", "#regb" \n\t" \
225 "pand "#regfe", "#regb" \n\t" \
226 "psrlq $1, "#regb" \n\t" \
227 "paddb "#regb", "#regr" \n\t"
229 #define PAVGB_MMX(rega, regb, regr, regfe) \
230 "movq "#rega", "#regr" \n\t" \
231 "por "#regb", "#regr" \n\t" \
232 "pxor "#rega", "#regb" \n\t" \
233 "pand "#regfe", "#regb" \n\t" \
234 "psrlq $1, "#regb" \n\t" \
235 "psubb "#regb", "#regr" \n\t"
237 // mm6 is supposed to contain 0xfefefefefefefefe
238 #define PAVGBP_MMX_NO_RND(rega, regb, regr, regc, regd, regp) \
239 "movq "#rega", "#regr" \n\t" \
240 "movq "#regc", "#regp" \n\t" \
241 "pand "#regb", "#regr" \n\t" \
242 "pand "#regd", "#regp" \n\t" \
243 "pxor "#rega", "#regb" \n\t" \
244 "pxor "#regc", "#regd" \n\t" \
245 "pand %%mm6, "#regb" \n\t" \
246 "pand %%mm6, "#regd" \n\t" \
247 "psrlq $1, "#regb" \n\t" \
248 "psrlq $1, "#regd" \n\t" \
249 "paddb "#regb", "#regr" \n\t" \
250 "paddb "#regd", "#regp" \n\t"
252 #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp) \
253 "movq "#rega", "#regr" \n\t" \
254 "movq "#regc", "#regp" \n\t" \
255 "por "#regb", "#regr" \n\t" \
256 "por "#regd", "#regp" \n\t" \
257 "pxor "#rega", "#regb" \n\t" \
258 "pxor "#regc", "#regd" \n\t" \
259 "pand %%mm6, "#regb" \n\t" \
260 "pand %%mm6, "#regd" \n\t" \
261 "psrlq $1, "#regd" \n\t" \
262 "psrlq $1, "#regb" \n\t" \
263 "psubb "#regb", "#regr" \n\t" \
264 "psubb "#regd", "#regp" \n\t"
266 /***********************************/
267 /* MMX no rounding */
269 #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx
270 #define SET_RND MOVQ_WONE
271 #define PAVGBP(a, b, c, d, e, f) PAVGBP_MMX_NO_RND(a, b, c, d, e, f)
272 #define PAVGB(a, b, c, e) PAVGB_MMX_NO_RND(a, b, c, e)
273 #define OP_AVG(a, b, c, e) PAVGB_MMX(a, b, c, e)
275 #include "dsputil_rnd_template.c"
282 /***********************************/
285 #define DEF(x, y) x ## _ ## y ## _mmx
286 #define SET_RND MOVQ_WTWO
287 #define PAVGBP(a, b, c, d, e, f) PAVGBP_MMX(a, b, c, d, e, f)
288 #define PAVGB(a, b, c, e) PAVGB_MMX(a, b, c, e)
290 #include "dsputil_rnd_template.c"
298 #endif /* HAVE_INLINE_ASM */
302 #define ff_put_pixels8_mmx ff_put_pixels8_mmxext
304 /***********************************/
307 #define DEF(x) x ## _3dnow
309 #include "dsputil_avg_template.c"
313 /***********************************/
314 /* MMXEXT specific */
316 #define DEF(x) x ## _mmxext
318 #include "dsputil_avg_template.c"
322 #endif /* HAVE_YASM */
326 #define put_no_rnd_pixels16_mmx put_pixels16_mmx
327 #define put_no_rnd_pixels8_mmx put_pixels8_mmx
328 #define put_pixels16_mmxext put_pixels16_mmx
329 #define put_pixels8_mmxext put_pixels8_mmx
330 #define put_pixels4_mmxext put_pixels4_mmx
331 #define put_no_rnd_pixels16_mmxext put_no_rnd_pixels16_mmx
332 #define put_no_rnd_pixels8_mmxext put_no_rnd_pixels8_mmx
334 /***********************************/
337 void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
343 /* read the pixels */
348 "movq (%3), %%mm0 \n\t"
349 "movq 8(%3), %%mm1 \n\t"
350 "movq 16(%3), %%mm2 \n\t"
351 "movq 24(%3), %%mm3 \n\t"
352 "movq 32(%3), %%mm4 \n\t"
353 "movq 40(%3), %%mm5 \n\t"
354 "movq 48(%3), %%mm6 \n\t"
355 "movq 56(%3), %%mm7 \n\t"
356 "packuswb %%mm1, %%mm0 \n\t"
357 "packuswb %%mm3, %%mm2 \n\t"
358 "packuswb %%mm5, %%mm4 \n\t"
359 "packuswb %%mm7, %%mm6 \n\t"
360 "movq %%mm0, (%0) \n\t"
361 "movq %%mm2, (%0, %1) \n\t"
362 "movq %%mm4, (%0, %1, 2) \n\t"
363 "movq %%mm6, (%0, %2) \n\t"
364 :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),
367 pix += line_size * 4;
370 // if here would be an exact copy of the code above
371 // compiler would generate some very strange code
374 "movq (%3), %%mm0 \n\t"
375 "movq 8(%3), %%mm1 \n\t"
376 "movq 16(%3), %%mm2 \n\t"
377 "movq 24(%3), %%mm3 \n\t"
378 "movq 32(%3), %%mm4 \n\t"
379 "movq 40(%3), %%mm5 \n\t"
380 "movq 48(%3), %%mm6 \n\t"
381 "movq 56(%3), %%mm7 \n\t"
382 "packuswb %%mm1, %%mm0 \n\t"
383 "packuswb %%mm3, %%mm2 \n\t"
384 "packuswb %%mm5, %%mm4 \n\t"
385 "packuswb %%mm7, %%mm6 \n\t"
386 "movq %%mm0, (%0) \n\t"
387 "movq %%mm2, (%0, %1) \n\t"
388 "movq %%mm4, (%0, %1, 2) \n\t"
389 "movq %%mm6, (%0, %2) \n\t"
390 :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)
394 #define put_signed_pixels_clamped_mmx_half(off) \
395 "movq "#off"(%2), %%mm1 \n\t" \
396 "movq 16 + "#off"(%2), %%mm2 \n\t" \
397 "movq 32 + "#off"(%2), %%mm3 \n\t" \
398 "movq 48 + "#off"(%2), %%mm4 \n\t" \
399 "packsswb 8 + "#off"(%2), %%mm1 \n\t" \
400 "packsswb 24 + "#off"(%2), %%mm2 \n\t" \
401 "packsswb 40 + "#off"(%2), %%mm3 \n\t" \
402 "packsswb 56 + "#off"(%2), %%mm4 \n\t" \
403 "paddb %%mm0, %%mm1 \n\t" \
404 "paddb %%mm0, %%mm2 \n\t" \
405 "paddb %%mm0, %%mm3 \n\t" \
406 "paddb %%mm0, %%mm4 \n\t" \
407 "movq %%mm1, (%0) \n\t" \
408 "movq %%mm2, (%0, %3) \n\t" \
409 "movq %%mm3, (%0, %3, 2) \n\t" \
410 "movq %%mm4, (%0, %1) \n\t"
412 void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
415 x86_reg line_skip = line_size;
419 "movq "MANGLE(ff_pb_80)", %%mm0 \n\t"
420 "lea (%3, %3, 2), %1 \n\t"
421 put_signed_pixels_clamped_mmx_half(0)
422 "lea (%0, %3, 4), %0 \n\t"
423 put_signed_pixels_clamped_mmx_half(64)
424 : "+&r"(pixels), "=&r"(line_skip3)
425 : "r"(block), "r"(line_skip)
429 void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
436 /* read the pixels */
443 "movq (%2), %%mm0 \n\t"
444 "movq 8(%2), %%mm1 \n\t"
445 "movq 16(%2), %%mm2 \n\t"
446 "movq 24(%2), %%mm3 \n\t"
447 "movq %0, %%mm4 \n\t"
448 "movq %1, %%mm6 \n\t"
449 "movq %%mm4, %%mm5 \n\t"
450 "punpcklbw %%mm7, %%mm4 \n\t"
451 "punpckhbw %%mm7, %%mm5 \n\t"
452 "paddsw %%mm4, %%mm0 \n\t"
453 "paddsw %%mm5, %%mm1 \n\t"
454 "movq %%mm6, %%mm5 \n\t"
455 "punpcklbw %%mm7, %%mm6 \n\t"
456 "punpckhbw %%mm7, %%mm5 \n\t"
457 "paddsw %%mm6, %%mm2 \n\t"
458 "paddsw %%mm5, %%mm3 \n\t"
459 "packuswb %%mm1, %%mm0 \n\t"
460 "packuswb %%mm3, %%mm2 \n\t"
461 "movq %%mm0, %0 \n\t"
462 "movq %%mm2, %1 \n\t"
463 : "+m"(*pix), "+m"(*(pix + line_size))
466 pix += line_size * 2;
471 static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,
472 ptrdiff_t line_size, int h)
475 "lea (%3, %3), %%"REG_a" \n\t"
478 "movq (%1 ), %%mm0 \n\t"
479 "movq (%1, %3), %%mm1 \n\t"
480 "movq %%mm0, (%2) \n\t"
481 "movq %%mm1, (%2, %3) \n\t"
482 "add %%"REG_a", %1 \n\t"
483 "add %%"REG_a", %2 \n\t"
484 "movq (%1 ), %%mm0 \n\t"
485 "movq (%1, %3), %%mm1 \n\t"
486 "movq %%mm0, (%2) \n\t"
487 "movq %%mm1, (%2, %3) \n\t"
488 "add %%"REG_a", %1 \n\t"
489 "add %%"REG_a", %2 \n\t"
492 : "+g"(h), "+r"(pixels), "+r"(block)
493 : "r"((x86_reg)line_size)
498 static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,
499 ptrdiff_t line_size, int h)
502 "lea (%3, %3), %%"REG_a" \n\t"
505 "movq (%1 ), %%mm0 \n\t"
506 "movq 8(%1 ), %%mm4 \n\t"
507 "movq (%1, %3), %%mm1 \n\t"
508 "movq 8(%1, %3), %%mm5 \n\t"
509 "movq %%mm0, (%2) \n\t"
510 "movq %%mm4, 8(%2) \n\t"
511 "movq %%mm1, (%2, %3) \n\t"
512 "movq %%mm5, 8(%2, %3) \n\t"
513 "add %%"REG_a", %1 \n\t"
514 "add %%"REG_a", %2 \n\t"
515 "movq (%1 ), %%mm0 \n\t"
516 "movq 8(%1 ), %%mm4 \n\t"
517 "movq (%1, %3), %%mm1 \n\t"
518 "movq 8(%1, %3), %%mm5 \n\t"
519 "movq %%mm0, (%2) \n\t"
520 "movq %%mm4, 8(%2) \n\t"
521 "movq %%mm1, (%2, %3) \n\t"
522 "movq %%mm5, 8(%2, %3) \n\t"
523 "add %%"REG_a", %1 \n\t"
524 "add %%"REG_a", %2 \n\t"
527 : "+g"(h), "+r"(pixels), "+r"(block)
528 : "r"((x86_reg)line_size)
533 #define CLEAR_BLOCKS(name, n) \
534 static void name(int16_t *blocks) \
537 "pxor %%mm7, %%mm7 \n\t" \
538 "mov %1, %%"REG_a" \n\t" \
540 "movq %%mm7, (%0, %%"REG_a") \n\t" \
541 "movq %%mm7, 8(%0, %%"REG_a") \n\t" \
542 "movq %%mm7, 16(%0, %%"REG_a") \n\t" \
543 "movq %%mm7, 24(%0, %%"REG_a") \n\t" \
544 "add $32, %%"REG_a" \n\t" \
546 :: "r"(((uint8_t *)blocks) + 128 * n), \
551 CLEAR_BLOCKS(clear_blocks_mmx, 6)
552 CLEAR_BLOCKS(clear_block_mmx, 1)
554 static void clear_block_sse(int16_t *block)
557 "xorps %%xmm0, %%xmm0 \n"
558 "movaps %%xmm0, (%0) \n"
559 "movaps %%xmm0, 16(%0) \n"
560 "movaps %%xmm0, 32(%0) \n"
561 "movaps %%xmm0, 48(%0) \n"
562 "movaps %%xmm0, 64(%0) \n"
563 "movaps %%xmm0, 80(%0) \n"
564 "movaps %%xmm0, 96(%0) \n"
565 "movaps %%xmm0, 112(%0) \n"
571 static void clear_blocks_sse(int16_t *blocks)
574 "xorps %%xmm0, %%xmm0 \n"
575 "mov %1, %%"REG_a" \n"
577 "movaps %%xmm0, (%0, %%"REG_a") \n"
578 "movaps %%xmm0, 16(%0, %%"REG_a") \n"
579 "movaps %%xmm0, 32(%0, %%"REG_a") \n"
580 "movaps %%xmm0, 48(%0, %%"REG_a") \n"
581 "movaps %%xmm0, 64(%0, %%"REG_a") \n"
582 "movaps %%xmm0, 80(%0, %%"REG_a") \n"
583 "movaps %%xmm0, 96(%0, %%"REG_a") \n"
584 "movaps %%xmm0, 112(%0, %%"REG_a") \n"
585 "add $128, %%"REG_a" \n"
587 :: "r"(((uint8_t *)blocks) + 128 * 6),
593 static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)
599 "movq (%1, %0), %%mm0 \n\t"
600 "movq (%2, %0), %%mm1 \n\t"
601 "paddb %%mm0, %%mm1 \n\t"
602 "movq %%mm1, (%2, %0) \n\t"
603 "movq 8(%1, %0), %%mm0 \n\t"
604 "movq 8(%2, %0), %%mm1 \n\t"
605 "paddb %%mm0, %%mm1 \n\t"
606 "movq %%mm1, 8(%2, %0) \n\t"
612 : "r"(src), "r"(dst), "r"((x86_reg)w - 15)
615 dst[i + 0] += src[i + 0];
619 static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,
620 const uint8_t *diff, int w,
621 int *left, int *left_top)
625 int l = *left & 0xff;
626 int tl = *left_top & 0xff;
631 "movzbl (%3, %4), %2 \n"
644 "add (%6, %4), %b0 \n"
645 "mov %b0, (%5, %4) \n"
648 : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)
649 : "r"(dst + w), "r"(diff + w), "rm"(top + w)
656 static inline void transpose4x4(uint8_t *dst, uint8_t *src, x86_reg dst_stride, x86_reg src_stride){
657 __asm__ volatile( //FIXME could save 1 instruction if done as 8x4 ...
658 "movd (%1), %%mm0 \n\t"
660 "movd (%1), %%mm1 \n\t"
661 "movd (%1,%3,1), %%mm2 \n\t"
662 "movd (%1,%3,2), %%mm3 \n\t"
663 "punpcklbw %%mm1, %%mm0 \n\t"
664 "punpcklbw %%mm3, %%mm2 \n\t"
665 "movq %%mm0, %%mm1 \n\t"
666 "punpcklwd %%mm2, %%mm0 \n\t"
667 "punpckhwd %%mm2, %%mm1 \n\t"
668 "movd %%mm0, (%0) \n\t"
670 "punpckhdq %%mm0, %%mm0 \n\t"
671 "movd %%mm0, (%0) \n\t"
672 "movd %%mm1, (%0,%2,1) \n\t"
673 "punpckhdq %%mm1, %%mm1 \n\t"
674 "movd %%mm1, (%0,%2,2) \n\t"
684 #define H263_LOOP_FILTER \
685 "pxor %%mm7, %%mm7 \n\t" \
686 "movq %0, %%mm0 \n\t" \
687 "movq %0, %%mm1 \n\t" \
688 "movq %3, %%mm2 \n\t" \
689 "movq %3, %%mm3 \n\t" \
690 "punpcklbw %%mm7, %%mm0 \n\t" \
691 "punpckhbw %%mm7, %%mm1 \n\t" \
692 "punpcklbw %%mm7, %%mm2 \n\t" \
693 "punpckhbw %%mm7, %%mm3 \n\t" \
694 "psubw %%mm2, %%mm0 \n\t" \
695 "psubw %%mm3, %%mm1 \n\t" \
696 "movq %1, %%mm2 \n\t" \
697 "movq %1, %%mm3 \n\t" \
698 "movq %2, %%mm4 \n\t" \
699 "movq %2, %%mm5 \n\t" \
700 "punpcklbw %%mm7, %%mm2 \n\t" \
701 "punpckhbw %%mm7, %%mm3 \n\t" \
702 "punpcklbw %%mm7, %%mm4 \n\t" \
703 "punpckhbw %%mm7, %%mm5 \n\t" \
704 "psubw %%mm2, %%mm4 \n\t" \
705 "psubw %%mm3, %%mm5 \n\t" \
706 "psllw $2, %%mm4 \n\t" \
707 "psllw $2, %%mm5 \n\t" \
708 "paddw %%mm0, %%mm4 \n\t" \
709 "paddw %%mm1, %%mm5 \n\t" \
710 "pxor %%mm6, %%mm6 \n\t" \
711 "pcmpgtw %%mm4, %%mm6 \n\t" \
712 "pcmpgtw %%mm5, %%mm7 \n\t" \
713 "pxor %%mm6, %%mm4 \n\t" \
714 "pxor %%mm7, %%mm5 \n\t" \
715 "psubw %%mm6, %%mm4 \n\t" \
716 "psubw %%mm7, %%mm5 \n\t" \
717 "psrlw $3, %%mm4 \n\t" \
718 "psrlw $3, %%mm5 \n\t" \
719 "packuswb %%mm5, %%mm4 \n\t" \
720 "packsswb %%mm7, %%mm6 \n\t" \
721 "pxor %%mm7, %%mm7 \n\t" \
722 "movd %4, %%mm2 \n\t" \
723 "punpcklbw %%mm2, %%mm2 \n\t" \
724 "punpcklbw %%mm2, %%mm2 \n\t" \
725 "punpcklbw %%mm2, %%mm2 \n\t" \
726 "psubusb %%mm4, %%mm2 \n\t" \
727 "movq %%mm2, %%mm3 \n\t" \
728 "psubusb %%mm4, %%mm3 \n\t" \
729 "psubb %%mm3, %%mm2 \n\t" \
730 "movq %1, %%mm3 \n\t" \
731 "movq %2, %%mm4 \n\t" \
732 "pxor %%mm6, %%mm3 \n\t" \
733 "pxor %%mm6, %%mm4 \n\t" \
734 "paddusb %%mm2, %%mm3 \n\t" \
735 "psubusb %%mm2, %%mm4 \n\t" \
736 "pxor %%mm6, %%mm3 \n\t" \
737 "pxor %%mm6, %%mm4 \n\t" \
738 "paddusb %%mm2, %%mm2 \n\t" \
739 "packsswb %%mm1, %%mm0 \n\t" \
740 "pcmpgtb %%mm0, %%mm7 \n\t" \
741 "pxor %%mm7, %%mm0 \n\t" \
742 "psubb %%mm7, %%mm0 \n\t" \
743 "movq %%mm0, %%mm1 \n\t" \
744 "psubusb %%mm2, %%mm0 \n\t" \
745 "psubb %%mm0, %%mm1 \n\t" \
746 "pand %5, %%mm1 \n\t" \
747 "psrlw $2, %%mm1 \n\t" \
748 "pxor %%mm7, %%mm1 \n\t" \
749 "psubb %%mm7, %%mm1 \n\t" \
750 "movq %0, %%mm5 \n\t" \
751 "movq %3, %%mm6 \n\t" \
752 "psubb %%mm1, %%mm5 \n\t" \
753 "paddb %%mm1, %%mm6 \n\t"
755 static void h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale)
757 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
758 const int strength = ff_h263_loop_filter_strength[qscale];
763 "movq %%mm3, %1 \n\t"
764 "movq %%mm4, %2 \n\t"
765 "movq %%mm5, %0 \n\t"
766 "movq %%mm6, %3 \n\t"
767 : "+m"(*(uint64_t*)(src - 2 * stride)),
768 "+m"(*(uint64_t*)(src - 1 * stride)),
769 "+m"(*(uint64_t*)(src + 0 * stride)),
770 "+m"(*(uint64_t*)(src + 1 * stride))
771 : "g"(2 * strength), "m"(ff_pb_FC)
776 static void h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale)
778 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
779 const int strength = ff_h263_loop_filter_strength[qscale];
780 DECLARE_ALIGNED(8, uint64_t, temp)[4];
781 uint8_t *btemp = (uint8_t*)temp;
785 transpose4x4(btemp, src, 8, stride);
786 transpose4x4(btemp + 4, src + 4 * stride, 8, stride);
788 H263_LOOP_FILTER // 5 3 4 6
794 : "g"(2 * strength), "m"(ff_pb_FC)
798 "movq %%mm5, %%mm1 \n\t"
799 "movq %%mm4, %%mm0 \n\t"
800 "punpcklbw %%mm3, %%mm5 \n\t"
801 "punpcklbw %%mm6, %%mm4 \n\t"
802 "punpckhbw %%mm3, %%mm1 \n\t"
803 "punpckhbw %%mm6, %%mm0 \n\t"
804 "movq %%mm5, %%mm3 \n\t"
805 "movq %%mm1, %%mm6 \n\t"
806 "punpcklwd %%mm4, %%mm5 \n\t"
807 "punpcklwd %%mm0, %%mm1 \n\t"
808 "punpckhwd %%mm4, %%mm3 \n\t"
809 "punpckhwd %%mm0, %%mm6 \n\t"
810 "movd %%mm5, (%0) \n\t"
811 "punpckhdq %%mm5, %%mm5 \n\t"
812 "movd %%mm5, (%0, %2) \n\t"
813 "movd %%mm3, (%0, %2, 2) \n\t"
814 "punpckhdq %%mm3, %%mm3 \n\t"
815 "movd %%mm3, (%0, %3) \n\t"
816 "movd %%mm1, (%1) \n\t"
817 "punpckhdq %%mm1, %%mm1 \n\t"
818 "movd %%mm1, (%1, %2) \n\t"
819 "movd %%mm6, (%1, %2, 2) \n\t"
820 "punpckhdq %%mm6, %%mm6 \n\t"
821 "movd %%mm6, (%1, %3) \n\t"
823 "r"(src + 4 * stride),
824 "r"((x86_reg)stride),
825 "r"((x86_reg)(3 * stride))
830 /* Draw the edges of width 'w' of an image of size width, height
831 * this MMX version can only handle w == 8 || w == 16. */
832 static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,
833 int w, int h, int sides)
835 uint8_t *ptr, *last_line;
838 last_line = buf + (height - 1) * wrap;
844 "movd (%0), %%mm0 \n\t"
845 "punpcklbw %%mm0, %%mm0 \n\t"
846 "punpcklwd %%mm0, %%mm0 \n\t"
847 "punpckldq %%mm0, %%mm0 \n\t"
848 "movq %%mm0, -8(%0) \n\t"
849 "movq -8(%0, %2), %%mm1 \n\t"
850 "punpckhbw %%mm1, %%mm1 \n\t"
851 "punpckhwd %%mm1, %%mm1 \n\t"
852 "punpckhdq %%mm1, %%mm1 \n\t"
853 "movq %%mm1, (%0, %2) \n\t"
858 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
863 "movd (%0), %%mm0 \n\t"
864 "punpcklbw %%mm0, %%mm0 \n\t"
865 "punpcklwd %%mm0, %%mm0 \n\t"
866 "punpckldq %%mm0, %%mm0 \n\t"
867 "movq %%mm0, -8(%0) \n\t"
868 "movq %%mm0, -16(%0) \n\t"
869 "movq -8(%0, %2), %%mm1 \n\t"
870 "punpckhbw %%mm1, %%mm1 \n\t"
871 "punpckhwd %%mm1, %%mm1 \n\t"
872 "punpckhdq %%mm1, %%mm1 \n\t"
873 "movq %%mm1, (%0, %2) \n\t"
874 "movq %%mm1, 8(%0, %2) \n\t"
879 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
885 "movd (%0), %%mm0 \n\t"
886 "punpcklbw %%mm0, %%mm0 \n\t"
887 "punpcklwd %%mm0, %%mm0 \n\t"
888 "movd %%mm0, -4(%0) \n\t"
889 "movd -4(%0, %2), %%mm1 \n\t"
890 "punpcklbw %%mm1, %%mm1 \n\t"
891 "punpckhwd %%mm1, %%mm1 \n\t"
892 "punpckhdq %%mm1, %%mm1 \n\t"
893 "movd %%mm1, (%0, %2) \n\t"
898 : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
902 /* top and bottom (and hopefully also the corners) */
903 if (sides & EDGE_TOP) {
904 for (i = 0; i < h; i += 4) {
905 ptr = buf - (i + 1) * wrap - w;
908 "movq (%1, %0), %%mm0 \n\t"
909 "movq %%mm0, (%0) \n\t"
910 "movq %%mm0, (%0, %2) \n\t"
911 "movq %%mm0, (%0, %2, 2) \n\t"
912 "movq %%mm0, (%0, %3) \n\t"
917 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),
918 "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)
923 if (sides & EDGE_BOTTOM) {
924 for (i = 0; i < h; i += 4) {
925 ptr = last_line + (i + 1) * wrap - w;
928 "movq (%1, %0), %%mm0 \n\t"
929 "movq %%mm0, (%0) \n\t"
930 "movq %%mm0, (%0, %2) \n\t"
931 "movq %%mm0, (%0, %2, 2) \n\t"
932 "movq %%mm0, (%0, %3) \n\t"
937 : "r"((x86_reg)last_line - (x86_reg)ptr - w),
938 "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),
939 "r"(ptr + width + 2 * w)
944 #endif /* HAVE_INLINE_ASM */
948 #define QPEL_OP(OPNAME, ROUNDER, RND, OP, MMX) \
949 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src, \
952 ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8); \
955 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src, \
959 uint8_t * const half = (uint8_t*)temp; \
960 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8, \
962 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half, \
963 stride, stride, 8); \
966 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src, \
969 ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride, \
973 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src, \
977 uint8_t * const half = (uint8_t*)temp; \
978 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8, \
980 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride, \
984 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src, \
988 uint8_t * const half = (uint8_t*)temp; \
989 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, \
991 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half, \
992 stride, stride, 8); \
995 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src, \
998 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src, \
1002 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src, \
1006 uint8_t * const half = (uint8_t*)temp; \
1007 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src, \
1009 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
1013 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src, \
1016 uint64_t half[8 + 9]; \
1017 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1018 uint8_t * const halfHV = ((uint8_t*)half); \
1019 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1021 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, \
1023 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1024 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1028 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src, \
1031 uint64_t half[8 + 9]; \
1032 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1033 uint8_t * const halfHV = ((uint8_t*)half); \
1034 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1036 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1038 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1039 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1043 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src, \
1046 uint64_t half[8 + 9]; \
1047 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1048 uint8_t * const halfHV = ((uint8_t*)half); \
1049 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1051 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8, \
1053 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1054 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1058 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src, \
1061 uint64_t half[8 + 9]; \
1062 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1063 uint8_t * const halfHV = ((uint8_t*)half); \
1064 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1066 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1068 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1069 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1073 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src, \
1076 uint64_t half[8 + 9]; \
1077 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1078 uint8_t * const halfHV = ((uint8_t*)half); \
1079 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1081 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1082 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV, \
1086 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src, \
1089 uint64_t half[8 + 9]; \
1090 uint8_t * const halfH = ((uint8_t*)half) + 64; \
1091 uint8_t * const halfHV = ((uint8_t*)half); \
1092 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1094 ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1095 ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV, \
1099 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src, \
1102 uint64_t half[8 + 9]; \
1103 uint8_t * const halfH = ((uint8_t*)half); \
1104 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1106 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, \
1108 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1112 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src, \
1115 uint64_t half[8 + 9]; \
1116 uint8_t * const halfH = ((uint8_t*)half); \
1117 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1119 ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8, \
1121 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1125 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src, \
1129 uint8_t * const halfH = ((uint8_t*)half); \
1130 ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8, \
1132 ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH, \
1136 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src, \
1139 ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16); \
1142 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src, \
1145 uint64_t temp[32]; \
1146 uint8_t * const half = (uint8_t*)temp; \
1147 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16, \
1149 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, \
1153 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src, \
1156 ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src, \
1157 stride, stride, 16);\
1160 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src, \
1163 uint64_t temp[32]; \
1164 uint8_t * const half = (uint8_t*)temp; \
1165 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16, \
1167 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half, \
1168 stride, stride, 16); \
1171 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src, \
1174 uint64_t temp[32]; \
1175 uint8_t * const half = (uint8_t*)temp; \
1176 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16, \
1178 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride, \
1182 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src, \
1185 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src, \
1189 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src, \
1192 uint64_t temp[32]; \
1193 uint8_t * const half = (uint8_t*)temp; \
1194 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16, \
1196 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half, \
1197 stride, stride, 16); \
1200 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src, \
1203 uint64_t half[16 * 2 + 17 * 2]; \
1204 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1205 uint8_t * const halfHV = ((uint8_t*)half); \
1206 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1208 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1210 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1212 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1216 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src, \
1219 uint64_t half[16 * 2 + 17 * 2]; \
1220 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1221 uint8_t * const halfHV = ((uint8_t*)half); \
1222 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1224 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1226 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1228 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1232 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src, \
1235 uint64_t half[16 * 2 + 17 * 2]; \
1236 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1237 uint8_t * const halfHV = ((uint8_t*)half); \
1238 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1240 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1242 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1244 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1248 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src, \
1251 uint64_t half[16 * 2 + 17 * 2]; \
1252 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1253 uint8_t * const halfHV = ((uint8_t*)half); \
1254 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1256 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1258 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1260 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1264 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src, \
1267 uint64_t half[16 * 2 + 17 * 2]; \
1268 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1269 uint8_t * const halfHV = ((uint8_t*)half); \
1270 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1272 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1274 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV, \
1278 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src, \
1281 uint64_t half[16 * 2 + 17 * 2]; \
1282 uint8_t * const halfH = ((uint8_t*)half) + 256; \
1283 uint8_t * const halfHV = ((uint8_t*)half); \
1284 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1286 ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH, \
1288 ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV, \
1292 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src, \
1295 uint64_t half[17 * 2]; \
1296 uint8_t * const halfH = ((uint8_t*)half); \
1297 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1299 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16, \
1301 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1305 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src, \
1308 uint64_t half[17 * 2]; \
1309 uint8_t * const halfH = ((uint8_t*)half); \
1310 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1312 ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16, \
1314 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1318 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src, \
1321 uint64_t half[17 * 2]; \
1322 uint8_t * const halfH = ((uint8_t*)half); \
1323 ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16, \
1325 ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH, \
1329 #define PUT_OP(a, b, temp, size) \
1330 "mov"#size" "#a", "#b" \n\t"
1332 #define AVG_MMXEXT_OP(a, b, temp, size) \
1333 "mov"#size" "#b", "#temp" \n\t" \
1334 "pavgb "#temp", "#a" \n\t" \
1335 "mov"#size" "#a", "#b" \n\t"
1337 QPEL_OP(put_, ff_pw_16, _, PUT_OP, mmxext)
1338 QPEL_OP(avg_, ff_pw_16, _, AVG_MMXEXT_OP, mmxext)
1339 QPEL_OP(put_no_rnd_, ff_pw_15, _no_rnd_, PUT_OP, mmxext)
1340 #endif /* HAVE_YASM */
1344 void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1346 put_pixels8_xy2_mmx(dst, src, stride, 8);
1348 void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1350 put_pixels16_xy2_mmx(dst, src, stride, 16);
1352 void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1354 avg_pixels8_xy2_mmx(dst, src, stride, 8);
1356 void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1358 avg_pixels16_xy2_mmx(dst, src, stride, 16);
1361 typedef void emulated_edge_mc_func(uint8_t *dst, const uint8_t *src,
1362 ptrdiff_t linesize, int block_w, int block_h,
1363 int src_x, int src_y, int w, int h);
1365 static av_always_inline void gmc(uint8_t *dst, uint8_t *src,
1366 int stride, int h, int ox, int oy,
1367 int dxx, int dxy, int dyx, int dyy,
1368 int shift, int r, int width, int height,
1369 emulated_edge_mc_func *emu_edge_fn)
1372 const int ix = ox >> (16 + shift);
1373 const int iy = oy >> (16 + shift);
1374 const int oxs = ox >> 4;
1375 const int oys = oy >> 4;
1376 const int dxxs = dxx >> 4;
1377 const int dxys = dxy >> 4;
1378 const int dyxs = dyx >> 4;
1379 const int dyys = dyy >> 4;
1380 const uint16_t r4[4] = { r, r, r, r };
1381 const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };
1382 const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };
1383 const uint64_t shift2 = 2 * shift;
1384 #define MAX_STRIDE 4096U
1386 uint8_t edge_buf[(MAX_H + 1) * MAX_STRIDE];
1389 const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);
1390 const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);
1391 const int dxh = dxy * (h - 1);
1392 const int dyw = dyx * (w - 1);
1393 int need_emu = (unsigned)ix >= width - w ||
1394 (unsigned)iy >= height - h;
1396 if ( // non-constant fullpel offset (3% of blocks)
1397 ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |
1398 (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)
1399 // uses more than 16 bits of subpel mv (only at huge resolution)
1400 || (dxx | dxy | dyx | dyy) & 15
1401 || (need_emu && (h > MAX_H || stride > MAX_STRIDE))) {
1402 // FIXME could still use mmx for some of the rows
1403 ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,
1404 shift, r, width, height);
1408 src += ix + iy * stride;
1410 emu_edge_fn(edge_buf, src, stride, w + 1, h + 1, ix, iy, width, height);
1415 "movd %0, %%mm6 \n\t"
1416 "pxor %%mm7, %%mm7 \n\t"
1417 "punpcklwd %%mm6, %%mm6 \n\t"
1418 "punpcklwd %%mm6, %%mm6 \n\t"
1422 for (x = 0; x < w; x += 4) {
1423 uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),
1424 oxs - dxys + dxxs * (x + 1),
1425 oxs - dxys + dxxs * (x + 2),
1426 oxs - dxys + dxxs * (x + 3) };
1427 uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),
1428 oys - dyys + dyxs * (x + 1),
1429 oys - dyys + dyxs * (x + 2),
1430 oys - dyys + dyxs * (x + 3) };
1432 for (y = 0; y < h; y++) {
1434 "movq %0, %%mm4 \n\t"
1435 "movq %1, %%mm5 \n\t"
1436 "paddw %2, %%mm4 \n\t"
1437 "paddw %3, %%mm5 \n\t"
1438 "movq %%mm4, %0 \n\t"
1439 "movq %%mm5, %1 \n\t"
1440 "psrlw $12, %%mm4 \n\t"
1441 "psrlw $12, %%mm5 \n\t"
1442 : "+m"(*dx4), "+m"(*dy4)
1443 : "m"(*dxy4), "m"(*dyy4)
1447 "movq %%mm6, %%mm2 \n\t"
1448 "movq %%mm6, %%mm1 \n\t"
1449 "psubw %%mm4, %%mm2 \n\t"
1450 "psubw %%mm5, %%mm1 \n\t"
1451 "movq %%mm2, %%mm0 \n\t"
1452 "movq %%mm4, %%mm3 \n\t"
1453 "pmullw %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)
1454 "pmullw %%mm5, %%mm3 \n\t" // dx * dy
1455 "pmullw %%mm5, %%mm2 \n\t" // (s - dx) * dy
1456 "pmullw %%mm4, %%mm1 \n\t" // dx * (s - dy)
1458 "movd %4, %%mm5 \n\t"
1459 "movd %3, %%mm4 \n\t"
1460 "punpcklbw %%mm7, %%mm5 \n\t"
1461 "punpcklbw %%mm7, %%mm4 \n\t"
1462 "pmullw %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy
1463 "pmullw %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy
1465 "movd %2, %%mm5 \n\t"
1466 "movd %1, %%mm4 \n\t"
1467 "punpcklbw %%mm7, %%mm5 \n\t"
1468 "punpcklbw %%mm7, %%mm4 \n\t"
1469 "pmullw %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)
1470 "pmullw %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)
1471 "paddw %5, %%mm1 \n\t"
1472 "paddw %%mm3, %%mm2 \n\t"
1473 "paddw %%mm1, %%mm0 \n\t"
1474 "paddw %%mm2, %%mm0 \n\t"
1476 "psrlw %6, %%mm0 \n\t"
1477 "packuswb %%mm0, %%mm0 \n\t"
1478 "movd %%mm0, %0 \n\t"
1480 : "=m"(dst[x + y * stride])
1481 : "m"(src[0]), "m"(src[1]),
1482 "m"(src[stride]), "m"(src[stride + 1]),
1483 "m"(*r4), "m"(shift2)
1487 src += 4 - h * stride;
1494 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1495 int stride, int h, int ox, int oy,
1496 int dxx, int dxy, int dyx, int dyy,
1497 int shift, int r, int width, int height)
1499 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1500 width, height, &ff_emulated_edge_mc_8);
1503 static void gmc_sse(uint8_t *dst, uint8_t *src,
1504 int stride, int h, int ox, int oy,
1505 int dxx, int dxy, int dyx, int dyy,
1506 int shift, int r, int width, int height)
1508 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1509 width, height, &ff_emulated_edge_mc_8);
1512 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1513 int stride, int h, int ox, int oy,
1514 int dxx, int dxy, int dyx, int dyy,
1515 int shift, int r, int width, int height)
1517 gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1518 width, height, &ff_emulated_edge_mc_8);
1523 #endif /* HAVE_INLINE_ASM */
1525 void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1526 ptrdiff_t line_size, int h);
1527 void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1528 ptrdiff_t line_size, int h);
1533 void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1535 put_pixels8_mmx(dst, src, stride, 8);
1538 void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1540 avg_pixels8_mmx(dst, src, stride, 8);
1543 void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1545 put_pixels16_mmx(dst, src, stride, 16);
1548 void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1550 avg_pixels16_mmx(dst, src, stride, 16);
1552 #endif /* HAVE_INLINE_ASM */
1556 void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,
1557 int stride, int rnd)
1559 ff_put_pixels8_mmx(dst, src, stride, 8);
1562 void ff_avg_vc1_mspel_mc00_mmxext(uint8_t *dst, const uint8_t *src,
1563 int stride, int rnd)
1565 ff_avg_pixels8_mmxext(dst, src, stride, 8);
1567 #endif /* HAVE_YASM */
1569 #if CONFIG_DIRAC_DECODER
1570 #define DIRAC_PIXOP(OPNAME2, OPNAME, EXT)\
1571 void ff_ ## OPNAME2 ## _dirac_pixels8_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1574 ff_ ## OPNAME2 ## _dirac_pixels8_c(dst, src, stride, h);\
1576 OPNAME ## _pixels8_ ## EXT(dst, src[0], stride, h);\
1578 void ff_ ## OPNAME2 ## _dirac_pixels16_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1581 ff_ ## OPNAME2 ## _dirac_pixels16_c(dst, src, stride, h);\
1583 OPNAME ## _pixels16_ ## EXT(dst, src[0], stride, h);\
1585 void ff_ ## OPNAME2 ## _dirac_pixels32_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1588 ff_ ## OPNAME2 ## _dirac_pixels32_c(dst, src, stride, h);\
1590 OPNAME ## _pixels16_ ## EXT(dst , src[0] , stride, h);\
1591 OPNAME ## _pixels16_ ## EXT(dst+16, src[0]+16, stride, h);\
1596 DIRAC_PIXOP(put, put, mmx)
1597 DIRAC_PIXOP(avg, avg, mmx)
1601 DIRAC_PIXOP(avg, ff_avg, mmxext)
1603 void ff_put_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1606 ff_put_dirac_pixels16_c(dst, src, stride, h);
1608 ff_put_pixels16_sse2(dst, src[0], stride, h);
1610 void ff_avg_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1613 ff_avg_dirac_pixels16_c(dst, src, stride, h);
1615 ff_avg_pixels16_sse2(dst, src[0], stride, h);
1617 void ff_put_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1620 ff_put_dirac_pixels32_c(dst, src, stride, h);
1622 ff_put_pixels16_sse2(dst , src[0] , stride, h);
1623 ff_put_pixels16_sse2(dst+16, src[0]+16, stride, h);
1626 void ff_avg_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1629 ff_avg_dirac_pixels32_c(dst, src, stride, h);
1631 ff_avg_pixels16_sse2(dst , src[0] , stride, h);
1632 ff_avg_pixels16_sse2(dst+16, src[0]+16, stride, h);
1638 /* XXX: Those functions should be suppressed ASAP when all IDCTs are
1641 static void ff_libmpeg2mmx_idct_put(uint8_t *dest, int line_size,
1645 ff_put_pixels_clamped_mmx(block, dest, line_size);
1648 static void ff_libmpeg2mmx_idct_add(uint8_t *dest, int line_size,
1652 ff_add_pixels_clamped_mmx(block, dest, line_size);
1655 static void ff_libmpeg2mmx2_idct_put(uint8_t *dest, int line_size,
1658 ff_mmxext_idct(block);
1659 ff_put_pixels_clamped_mmx(block, dest, line_size);
1662 static void ff_libmpeg2mmx2_idct_add(uint8_t *dest, int line_size,
1665 ff_mmxext_idct(block);
1666 ff_add_pixels_clamped_mmx(block, dest, line_size);
1671 static void vector_clipf_sse(float *dst, const float *src,
1672 float min, float max, int len)
1674 x86_reg i = (len - 16) * 4;
1676 "movss %3, %%xmm4 \n\t"
1677 "movss %4, %%xmm5 \n\t"
1678 "shufps $0, %%xmm4, %%xmm4 \n\t"
1679 "shufps $0, %%xmm5, %%xmm5 \n\t"
1681 "movaps (%2, %0), %%xmm0 \n\t" // 3/1 on intel
1682 "movaps 16(%2, %0), %%xmm1 \n\t"
1683 "movaps 32(%2, %0), %%xmm2 \n\t"
1684 "movaps 48(%2, %0), %%xmm3 \n\t"
1685 "maxps %%xmm4, %%xmm0 \n\t"
1686 "maxps %%xmm4, %%xmm1 \n\t"
1687 "maxps %%xmm4, %%xmm2 \n\t"
1688 "maxps %%xmm4, %%xmm3 \n\t"
1689 "minps %%xmm5, %%xmm0 \n\t"
1690 "minps %%xmm5, %%xmm1 \n\t"
1691 "minps %%xmm5, %%xmm2 \n\t"
1692 "minps %%xmm5, %%xmm3 \n\t"
1693 "movaps %%xmm0, (%1, %0) \n\t"
1694 "movaps %%xmm1, 16(%1, %0) \n\t"
1695 "movaps %%xmm2, 32(%1, %0) \n\t"
1696 "movaps %%xmm3, 48(%1, %0) \n\t"
1700 : "r"(dst), "r"(src), "m"(min), "m"(max)
1705 #endif /* HAVE_INLINE_ASM */
1707 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
1709 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
1711 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
1713 int order, int mul);
1714 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
1716 int order, int mul);
1717 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
1719 int order, int mul);
1721 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
1722 const int16_t *window, unsigned int len);
1723 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
1724 const int16_t *window, unsigned int len);
1725 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
1726 const int16_t *window, unsigned int len);
1727 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
1728 const int16_t *window, unsigned int len);
1729 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
1730 const int16_t *window, unsigned int len);
1731 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
1732 const int16_t *window, unsigned int len);
1734 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
1735 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
1737 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
1738 const uint8_t *diff, int w,
1739 int *left, int *left_top);
1740 int ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
1742 int ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
1745 void ff_vector_clip_int32_mmx (int32_t *dst, const int32_t *src,
1746 int32_t min, int32_t max, unsigned int len);
1747 void ff_vector_clip_int32_sse2 (int32_t *dst, const int32_t *src,
1748 int32_t min, int32_t max, unsigned int len);
1749 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
1750 int32_t min, int32_t max, unsigned int len);
1751 void ff_vector_clip_int32_sse4 (int32_t *dst, const int32_t *src,
1752 int32_t min, int32_t max, unsigned int len);
1754 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX) \
1756 c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
1757 c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
1758 c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
1759 c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
1760 c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
1761 c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
1762 c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
1763 c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
1764 c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
1765 c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
1766 c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
1767 c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
1768 c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
1769 c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
1770 c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
1771 c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
1774 #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU) \
1776 c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _ ## CPU; \
1777 c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_ ## CPU; \
1778 c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_ ## CPU; \
1779 c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \
1782 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
1785 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1788 c->put_pixels_clamped = ff_put_pixels_clamped_mmx;
1789 c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
1790 c->add_pixels_clamped = ff_add_pixels_clamped_mmx;
1792 if (!high_bit_depth) {
1793 c->clear_block = clear_block_mmx;
1794 c->clear_blocks = clear_blocks_mmx;
1795 c->draw_edges = draw_edges_mmx;
1797 SET_HPEL_FUNCS(put, [0], 16, mmx);
1798 SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);
1799 SET_HPEL_FUNCS(avg, [0], 16, mmx);
1800 SET_HPEL_FUNCS(avg_no_rnd, , 16, mmx);
1801 SET_HPEL_FUNCS(put, [1], 8, mmx);
1802 SET_HPEL_FUNCS(put_no_rnd, [1], 8, mmx);
1803 SET_HPEL_FUNCS(avg, [1], 8, mmx);
1806 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
1810 c->add_bytes = add_bytes_mmx;
1812 if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
1813 c->h263_v_loop_filter = h263_v_loop_filter_mmx;
1814 c->h263_h_loop_filter = h263_h_loop_filter_mmx;
1816 #endif /* HAVE_INLINE_ASM */
1819 c->vector_clip_int32 = ff_vector_clip_int32_mmx;
1824 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
1827 const int bit_depth = avctx->bits_per_raw_sample;
1828 const int high_bit_depth = bit_depth > 8;
1831 SET_QPEL_FUNCS(avg_qpel, 0, 16, mmxext, );
1832 SET_QPEL_FUNCS(avg_qpel, 1, 8, mmxext, );
1834 SET_QPEL_FUNCS(put_qpel, 0, 16, mmxext, );
1835 SET_QPEL_FUNCS(put_qpel, 1, 8, mmxext, );
1836 SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
1837 SET_QPEL_FUNCS(put_no_rnd_qpel, 1, 8, mmxext, );
1839 if (!high_bit_depth) {
1840 c->put_pixels_tab[0][1] = ff_put_pixels16_x2_mmxext;
1841 c->put_pixels_tab[0][2] = ff_put_pixels16_y2_mmxext;
1843 c->avg_pixels_tab[0][0] = ff_avg_pixels16_mmxext;
1844 c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_mmxext;
1845 c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_mmxext;
1847 c->put_pixels_tab[1][1] = ff_put_pixels8_x2_mmxext;
1848 c->put_pixels_tab[1][2] = ff_put_pixels8_y2_mmxext;
1850 c->avg_pixels_tab[1][0] = ff_avg_pixels8_mmxext;
1851 c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_mmxext;
1852 c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_mmxext;
1855 if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {
1856 if (!high_bit_depth) {
1857 c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_mmxext;
1858 c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_mmxext;
1859 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_mmxext;
1860 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_mmxext;
1862 c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_mmxext;
1863 c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_mmxext;
1866 #endif /* HAVE_YASM */
1868 #if HAVE_MMXEXT_EXTERNAL
1869 if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1870 avctx->codec_id == AV_CODEC_ID_THEORA)) {
1871 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_mmxext;
1872 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_mmxext;
1875 /* slower than cmov version on AMD */
1876 if (!(mm_flags & AV_CPU_FLAG_3DNOW))
1877 c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
1879 c->scalarproduct_int16 = ff_scalarproduct_int16_mmxext;
1880 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
1882 if (avctx->flags & CODEC_FLAG_BITEXACT) {
1883 c->apply_window_int16 = ff_apply_window_int16_mmxext;
1885 c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
1887 #endif /* HAVE_MMXEXT_EXTERNAL */
1890 static av_cold void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,
1893 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1896 if (!high_bit_depth) {
1897 c->put_pixels_tab[0][1] = ff_put_pixels16_x2_3dnow;
1898 c->put_pixels_tab[0][2] = ff_put_pixels16_y2_3dnow;
1900 c->avg_pixels_tab[0][0] = ff_avg_pixels16_3dnow;
1901 c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_3dnow;
1902 c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_3dnow;
1904 c->put_pixels_tab[1][1] = ff_put_pixels8_x2_3dnow;
1905 c->put_pixels_tab[1][2] = ff_put_pixels8_y2_3dnow;
1907 c->avg_pixels_tab[1][0] = ff_avg_pixels8_3dnow;
1908 c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_3dnow;
1909 c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_3dnow;
1911 if (!(avctx->flags & CODEC_FLAG_BITEXACT)){
1912 c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_3dnow;
1913 c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_3dnow;
1914 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_3dnow;
1915 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_3dnow;
1917 c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_3dnow;
1918 c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_3dnow;
1922 if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1923 avctx->codec_id == AV_CODEC_ID_THEORA)) {
1924 c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_3dnow;
1925 c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_3dnow;
1927 #endif /* HAVE_YASM */
1930 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
1933 const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1936 if (!high_bit_depth) {
1937 if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
1938 /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
1939 c->clear_block = clear_block_sse;
1940 c->clear_blocks = clear_blocks_sse;
1944 c->vector_clipf = vector_clipf_sse;
1945 #endif /* HAVE_INLINE_ASM */
1948 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
1951 #endif /* HAVE_YASM */
1954 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
1957 const int bit_depth = avctx->bits_per_raw_sample;
1958 const int high_bit_depth = bit_depth > 8;
1960 #if HAVE_SSE2_INLINE
1961 if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
1962 c->idct_put = ff_idct_xvid_sse2_put;
1963 c->idct_add = ff_idct_xvid_sse2_add;
1964 c->idct = ff_idct_xvid_sse2;
1965 c->idct_permutation_type = FF_SSE2_IDCT_PERM;
1967 #endif /* HAVE_SSE2_INLINE */
1969 #if HAVE_SSE2_EXTERNAL
1970 if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
1971 // these functions are slower than mmx on AMD, but faster on Intel
1972 if (!high_bit_depth) {
1973 c->put_pixels_tab[0][0] = ff_put_pixels16_sse2;
1974 c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;
1975 c->avg_pixels_tab[0][0] = ff_avg_pixels16_sse2;
1979 c->scalarproduct_int16 = ff_scalarproduct_int16_sse2;
1980 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
1981 if (mm_flags & AV_CPU_FLAG_ATOM) {
1982 c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
1984 c->vector_clip_int32 = ff_vector_clip_int32_sse2;
1986 if (avctx->flags & CODEC_FLAG_BITEXACT) {
1987 c->apply_window_int16 = ff_apply_window_int16_sse2;
1988 } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
1989 c->apply_window_int16 = ff_apply_window_int16_round_sse2;
1991 c->bswap_buf = ff_bswap32_buf_sse2;
1992 #endif /* HAVE_SSE2_EXTERNAL */
1995 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
1998 #if HAVE_SSSE3_EXTERNAL
1999 c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
2000 if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
2001 c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
2003 if (mm_flags & AV_CPU_FLAG_ATOM)
2004 c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
2006 c->apply_window_int16 = ff_apply_window_int16_ssse3;
2007 if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
2008 c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
2009 c->bswap_buf = ff_bswap32_buf_ssse3;
2010 #endif /* HAVE_SSSE3_EXTERNAL */
2013 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
2016 #if HAVE_SSE4_EXTERNAL
2017 c->vector_clip_int32 = ff_vector_clip_int32_sse4;
2018 #endif /* HAVE_SSE4_EXTERNAL */
2021 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
2023 int mm_flags = av_get_cpu_flags();
2025 #if HAVE_7REGS && HAVE_INLINE_ASM
2026 if (mm_flags & AV_CPU_FLAG_CMOV)
2027 c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;
2030 if (mm_flags & AV_CPU_FLAG_MMX) {
2032 const int idct_algo = avctx->idct_algo;
2034 if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
2035 if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
2036 c->idct_put = ff_simple_idct_put_mmx;
2037 c->idct_add = ff_simple_idct_add_mmx;
2038 c->idct = ff_simple_idct_mmx;
2039 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
2041 } else if (idct_algo == FF_IDCT_LIBMPEG2MMX) {
2042 if (mm_flags & AV_CPU_FLAG_MMX2) {
2043 c->idct_put = ff_libmpeg2mmx2_idct_put;
2044 c->idct_add = ff_libmpeg2mmx2_idct_add;
2045 c->idct = ff_mmxext_idct;
2047 c->idct_put = ff_libmpeg2mmx_idct_put;
2048 c->idct_add = ff_libmpeg2mmx_idct_add;
2049 c->idct = ff_mmx_idct;
2051 c->idct_permutation_type = FF_LIBMPEG2_IDCT_PERM;
2053 } else if (idct_algo == FF_IDCT_XVIDMMX) {
2054 if (mm_flags & AV_CPU_FLAG_SSE2) {
2055 c->idct_put = ff_idct_xvid_sse2_put;
2056 c->idct_add = ff_idct_xvid_sse2_add;
2057 c->idct = ff_idct_xvid_sse2;
2058 c->idct_permutation_type = FF_SSE2_IDCT_PERM;
2059 } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
2060 c->idct_put = ff_idct_xvid_mmxext_put;
2061 c->idct_add = ff_idct_xvid_mmxext_add;
2062 c->idct = ff_idct_xvid_mmxext;
2064 c->idct_put = ff_idct_xvid_mmx_put;
2065 c->idct_add = ff_idct_xvid_mmx_add;
2066 c->idct = ff_idct_xvid_mmx;
2070 #endif /* HAVE_INLINE_ASM */
2072 dsputil_init_mmx(c, avctx, mm_flags);
2075 if (mm_flags & AV_CPU_FLAG_MMXEXT)
2076 dsputil_init_mmxext(c, avctx, mm_flags);
2078 if (mm_flags & AV_CPU_FLAG_3DNOW)
2079 dsputil_init_3dnow(c, avctx, mm_flags);
2081 if (mm_flags & AV_CPU_FLAG_SSE)
2082 dsputil_init_sse(c, avctx, mm_flags);
2084 if (mm_flags & AV_CPU_FLAG_SSE2)
2085 dsputil_init_sse2(c, avctx, mm_flags);
2087 if (mm_flags & AV_CPU_FLAG_SSSE3)
2088 dsputil_init_ssse3(c, avctx, mm_flags);
2090 if (mm_flags & AV_CPU_FLAG_SSE4)
2091 dsputil_init_sse4(c, avctx, mm_flags);
2093 if (CONFIG_ENCODERS)
2094 ff_dsputilenc_init_mmx(c, avctx);