git.sesse.net Git - ffmpeg/blob - libavcodec/x86/dsputil_mmx.c

   1 /*
   2  * MMX optimized DSP utils
   3  * Copyright (c) 2000, 2001 Fabrice Bellard
   4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * This file is part of FFmpeg.
   7  *
   8  * FFmpeg is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * FFmpeg is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with FFmpeg; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  *
  22  * MMX optimization by Nick Kurshev <nickols_k@mail.ru>
  23  */
  24
  25 #include "libavutil/attributes.h"
  26 #include "libavutil/cpu.h"
  27 #include "libavutil/x86/asm.h"
  28 #include "libavcodec/dsputil.h"
  29 #include "libavcodec/h264dsp.h"
  30 #include "libavcodec/mpegvideo.h"
  31 #include "libavcodec/simple_idct.h"
  32 #include "dsputil_mmx.h"
  33 #include "idct_xvid.h"
  34 #include "diracdsp_mmx.h"
  35
  36 //#undef NDEBUG
  37 //#include <assert.h>
  38
  39 /* pixel operations */
  40 DECLARE_ALIGNED(8,  const uint64_t, ff_bone) = 0x0101010101010101ULL;
  41 DECLARE_ALIGNED(8,  const uint64_t, ff_wtwo) = 0x0002000200020002ULL;
  42
  43 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1)    = { 0x0001000100010001ULL, 0x0001000100010001ULL };
  44 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_2)    = { 0x0002000200020002ULL, 0x0002000200020002ULL };
  45 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_3)    = { 0x0003000300030003ULL, 0x0003000300030003ULL };
  46 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_4)    = { 0x0004000400040004ULL, 0x0004000400040004ULL };
  47 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_5)    = { 0x0005000500050005ULL, 0x0005000500050005ULL };
  48 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_8)    = { 0x0008000800080008ULL, 0x0008000800080008ULL };
  49 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_9)    = { 0x0009000900090009ULL, 0x0009000900090009ULL };
  50 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_15)   =   0x000F000F000F000FULL;
  51 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_16)   = { 0x0010001000100010ULL, 0x0010001000100010ULL };
  52 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_17)   = { 0x0011001100110011ULL, 0x0011001100110011ULL };
  53 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_18)   = { 0x0012001200120012ULL, 0x0012001200120012ULL };
  54 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_20)   =   0x0014001400140014ULL;
  55 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_27)   = { 0x001B001B001B001BULL, 0x001B001B001B001BULL };
  56 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_28)   = { 0x001C001C001C001CULL, 0x001C001C001C001CULL };
  57 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_32)   = { 0x0020002000200020ULL, 0x0020002000200020ULL };
  58 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_42)   =   0x002A002A002A002AULL;
  59 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_53)   =   0x0035003500350035ULL;
  60 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_63)   = { 0x003F003F003F003FULL, 0x003F003F003F003FULL };
  61 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_64)   = { 0x0040004000400040ULL, 0x0040004000400040ULL };
  62 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_96)   =   0x0060006000600060ULL;
  63 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_128)  =   0x0080008000800080ULL;
  64 DECLARE_ALIGNED(8,  const uint64_t, ff_pw_255)  =   0x00ff00ff00ff00ffULL;
  65 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_512)  = { 0x0200020002000200ULL, 0x0200020002000200ULL };
  66 DECLARE_ALIGNED(16, const xmm_reg,  ff_pw_1019) = { 0x03FB03FB03FB03FBULL, 0x03FB03FB03FB03FBULL };
  67
  68 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_0)    = { 0x0000000000000000ULL, 0x0000000000000000ULL };
  69 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_1)    = { 0x0101010101010101ULL, 0x0101010101010101ULL };
  70 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_3)    = { 0x0303030303030303ULL, 0x0303030303030303ULL };
  71 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_4)    = { 0x0404040404040404ULL, 0x0404040404040404ULL };
  72 DECLARE_ALIGNED(8,  const uint64_t, ff_pb_7)    =   0x0707070707070707ULL;
  73 DECLARE_ALIGNED(8,  const uint64_t, ff_pb_1F)   =   0x1F1F1F1F1F1F1F1FULL;
  74 DECLARE_ALIGNED(8,  const uint64_t, ff_pb_3F)   =   0x3F3F3F3F3F3F3F3FULL;
  75 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_80)   = { 0x8080808080808080ULL, 0x8080808080808080ULL };
  76 DECLARE_ALIGNED(8,  const uint64_t, ff_pb_81)   =   0x8181818181818181ULL;
  77 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_A1)   = { 0xA1A1A1A1A1A1A1A1ULL, 0xA1A1A1A1A1A1A1A1ULL };
  78 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_F8)   = { 0xF8F8F8F8F8F8F8F8ULL, 0xF8F8F8F8F8F8F8F8ULL };
  79 DECLARE_ALIGNED(8,  const uint64_t, ff_pb_FC)   =   0xFCFCFCFCFCFCFCFCULL;
  80 DECLARE_ALIGNED(16, const xmm_reg,  ff_pb_FE)   = { 0xFEFEFEFEFEFEFEFEULL, 0xFEFEFEFEFEFEFEFEULL };
  81
  82 DECLARE_ALIGNED(16, const double, ff_pd_1)[2] = { 1.0, 1.0 };
  83 DECLARE_ALIGNED(16, const double, ff_pd_2)[2] = { 2.0, 2.0 };
  84
  85
  86 #if HAVE_YASM
  87 void ff_put_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
  88                               ptrdiff_t line_size, int h);
  89 void ff_put_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
  90                              ptrdiff_t line_size, int h);
  91 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  92                               int dstStride, int src1Stride, int h);
  93 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
  94                                      uint8_t *src2, int dstStride,
  95                                      int src1Stride, int h);
  96 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  97                               int dstStride, int src1Stride, int h);
  98 void ff_put_pixels16_x2_mmxext(uint8_t *block, const uint8_t *pixels,
  99                                ptrdiff_t line_size, int h);
 100 void ff_put_pixels16_x2_3dnow(uint8_t *block, const uint8_t *pixels,
 101                               ptrdiff_t line_size, int h);
 102 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
 103                                int dstStride, int src1Stride, int h);
 104 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
 105                                int dstStride, int src1Stride, int h);
 106 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
 107                                       int dstStride, int src1Stride, int h);
 108 void ff_put_no_rnd_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
 109                                      ptrdiff_t line_size, int h);
 110 void ff_put_no_rnd_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
 111                                     ptrdiff_t line_size, int h);
 112 void ff_put_no_rnd_pixels8_x2_exact_mmxext(uint8_t *block,
 113                                            const uint8_t *pixels,
 114                                            ptrdiff_t line_size, int h);
 115 void ff_put_no_rnd_pixels8_x2_exact_3dnow(uint8_t *block,
 116                                           const uint8_t *pixels,
 117                                           ptrdiff_t line_size, int h);
 118 void ff_put_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
 119                               ptrdiff_t line_size, int h);
 120 void ff_put_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
 121                              ptrdiff_t line_size, int h);
 122 void ff_put_no_rnd_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
 123                                      ptrdiff_t line_size, int h);
 124 void ff_put_no_rnd_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
 125                                     ptrdiff_t line_size, int h);
 126 void ff_put_no_rnd_pixels8_y2_exact_mmxext(uint8_t *block,
 127                                            const uint8_t *pixels,
 128                                            ptrdiff_t line_size, int h);
 129 void ff_put_no_rnd_pixels8_y2_exact_3dnow(uint8_t *block,
 130                                           const uint8_t *pixels,
 131                                           ptrdiff_t line_size, int h);
 132 void ff_avg_pixels8_mmxext(uint8_t *block, const uint8_t *pixels,
 133                            ptrdiff_t line_size, int h);
 134 void ff_avg_pixels8_3dnow(uint8_t *block, const uint8_t *pixels,
 135                           ptrdiff_t line_size, int h);
 136 void ff_avg_pixels8_x2_mmxext(uint8_t *block, const uint8_t *pixels,
 137                               ptrdiff_t line_size, int h);
 138 void ff_avg_pixels8_x2_3dnow(uint8_t *block, const uint8_t *pixels,
 139                              ptrdiff_t line_size, int h);
 140 void ff_avg_pixels8_y2_mmxext(uint8_t *block, const uint8_t *pixels,
 141                               ptrdiff_t line_size, int h);
 142 void ff_avg_pixels8_y2_3dnow(uint8_t *block, const uint8_t *pixels,
 143                              ptrdiff_t line_size, int h);
 144 void ff_avg_pixels8_xy2_mmxext(uint8_t *block, const uint8_t *pixels,
 145                                ptrdiff_t line_size, int h);
 146 void ff_avg_pixels8_xy2_3dnow(uint8_t *block, const uint8_t *pixels,
 147                               ptrdiff_t line_size, int h);
 148
 149 void ff_put_pixels8_mmxext(uint8_t *block, const uint8_t *pixels, ptrdiff_t line_size, int h);
 150 static void ff_put_pixels16_mmxext(uint8_t *block, const uint8_t *pixels,
 151                                    int line_size, int h)
 152 {
 153     ff_put_pixels8_mmxext(block,     pixels,     line_size, h);
 154     ff_put_pixels8_mmxext(block + 8, pixels + 8, line_size, h);
 155 }
 156
 157 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 158                                          int dstStride, int srcStride, int h);
 159 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 160                                          int dstStride, int srcStride, int h);
 161 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 162                                                  int dstStride, int srcStride,
 163                                                  int h);
 164 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 165                                         int dstStride, int srcStride, int h);
 166 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 167                                         int dstStride, int srcStride, int h);
 168 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 169                                                 int dstStride, int srcStride,
 170                                                 int h);
 171 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 172                                          int dstStride, int srcStride);
 173 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 174                                          int dstStride, int srcStride);
 175 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 176                                                  int dstStride, int srcStride);
 177 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 178                                         int dstStride, int srcStride);
 179 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 180                                         int dstStride, int srcStride);
 181 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
 182                                                 int dstStride, int srcStride);
 183 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
 184 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
 185 #endif /* HAVE_YASM */
 186
 187
 188 #if HAVE_INLINE_ASM
 189
 190 #define JUMPALIGN()     __asm__ volatile (".p2align 3"::)
 191 #define MOVQ_ZERO(regd) __asm__ volatile ("pxor %%"#regd", %%"#regd ::)
 192
 193 #define MOVQ_BFE(regd)                                  \
 194     __asm__ volatile (                                  \
 195         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \
 196         "paddb   %%"#regd", %%"#regd"   \n\t" ::)
 197
 198 #ifndef PIC
 199 #define MOVQ_BONE(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_bone))
 200 #define MOVQ_WTWO(regd) __asm__ volatile ("movq %0, %%"#regd" \n\t" :: "m"(ff_wtwo))
 201 #else
 202 // for shared library it's better to use this way for accessing constants
 203 // pcmpeqd -> -1
 204 #define MOVQ_BONE(regd)                                 \
 205     __asm__ volatile (                                  \
 206         "pcmpeqd  %%"#regd", %%"#regd"  \n\t"           \
 207         "psrlw          $15, %%"#regd"  \n\t"           \
 208         "packuswb %%"#regd", %%"#regd"  \n\t" ::)
 209
 210 #define MOVQ_WTWO(regd)                                 \
 211     __asm__ volatile (                                  \
 212         "pcmpeqd %%"#regd", %%"#regd"   \n\t"           \
 213         "psrlw         $15, %%"#regd"   \n\t"           \
 214         "psllw          $1, %%"#regd"   \n\t"::)
 215
 216 #endif
 217
 218 // using regr as temporary and for the output result
 219 // first argument is unmodifed and second is trashed
 220 // regfe is supposed to contain 0xfefefefefefefefe
 221 #define PAVGB_MMX_NO_RND(rega, regb, regr, regfe)                \
 222     "movq   "#rega", "#regr"            \n\t"                    \
 223     "pand   "#regb", "#regr"            \n\t"                    \
 224     "pxor   "#rega", "#regb"            \n\t"                    \
 225     "pand  "#regfe", "#regb"            \n\t"                    \
 226     "psrlq       $1, "#regb"            \n\t"                    \
 227     "paddb  "#regb", "#regr"            \n\t"
 228
 229 #define PAVGB_MMX(rega, regb, regr, regfe)                       \
 230     "movq   "#rega", "#regr"            \n\t"                    \
 231     "por    "#regb", "#regr"            \n\t"                    \
 232     "pxor   "#rega", "#regb"            \n\t"                    \
 233     "pand  "#regfe", "#regb"            \n\t"                    \
 234     "psrlq       $1, "#regb"            \n\t"                    \
 235     "psubb  "#regb", "#regr"            \n\t"
 236
 237 // mm6 is supposed to contain 0xfefefefefefefefe
 238 #define PAVGBP_MMX_NO_RND(rega, regb, regr,  regc, regd, regp)   \
 239     "movq  "#rega", "#regr"             \n\t"                    \
 240     "movq  "#regc", "#regp"             \n\t"                    \
 241     "pand  "#regb", "#regr"             \n\t"                    \
 242     "pand  "#regd", "#regp"             \n\t"                    \
 243     "pxor  "#rega", "#regb"             \n\t"                    \
 244     "pxor  "#regc", "#regd"             \n\t"                    \
 245     "pand    %%mm6, "#regb"             \n\t"                    \
 246     "pand    %%mm6, "#regd"             \n\t"                    \
 247     "psrlq      $1, "#regb"             \n\t"                    \
 248     "psrlq      $1, "#regd"             \n\t"                    \
 249     "paddb "#regb", "#regr"             \n\t"                    \
 250     "paddb "#regd", "#regp"             \n\t"
 251
 252 #define PAVGBP_MMX(rega, regb, regr, regc, regd, regp)           \
 253     "movq  "#rega", "#regr"             \n\t"                    \
 254     "movq  "#regc", "#regp"             \n\t"                    \
 255     "por   "#regb", "#regr"             \n\t"                    \
 256     "por   "#regd", "#regp"             \n\t"                    \
 257     "pxor  "#rega", "#regb"             \n\t"                    \
 258     "pxor  "#regc", "#regd"             \n\t"                    \
 259     "pand    %%mm6, "#regb"             \n\t"                    \
 260     "pand    %%mm6, "#regd"             \n\t"                    \
 261     "psrlq      $1, "#regd"             \n\t"                    \
 262     "psrlq      $1, "#regb"             \n\t"                    \
 263     "psubb "#regb", "#regr"             \n\t"                    \
 264     "psubb "#regd", "#regp"             \n\t"
 265
 266 /***********************************/
 267 /* MMX no rounding */
 268 #define NO_RND 1
 269 #define DEF(x, y) x ## _no_rnd_ ## y ## _mmx
 270 #define SET_RND  MOVQ_WONE
 271 #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX_NO_RND(a, b, c, d, e, f)
 272 #define PAVGB(a, b, c, e)               PAVGB_MMX_NO_RND(a, b, c, e)
 273 #define OP_AVG(a, b, c, e)              PAVGB_MMX(a, b, c, e)
 274
 275 #include "dsputil_rnd_template.c"
 276
 277 #undef DEF
 278 #undef SET_RND
 279 #undef PAVGBP
 280 #undef PAVGB
 281 #undef NO_RND
 282 /***********************************/
 283 /* MMX rounding */
 284
 285 #define DEF(x, y) x ## _ ## y ## _mmx
 286 #define SET_RND  MOVQ_WTWO
 287 #define PAVGBP(a, b, c, d, e, f)        PAVGBP_MMX(a, b, c, d, e, f)
 288 #define PAVGB(a, b, c, e)               PAVGB_MMX(a, b, c, e)
 289
 290 #include "dsputil_rnd_template.c"
 291
 292 #undef DEF
 293 #undef SET_RND
 294 #undef PAVGBP
 295 #undef PAVGB
 296 #undef OP_AVG
 297
 298 #endif /* HAVE_INLINE_ASM */
 299
 300
 301 #if HAVE_YASM
 302 #define ff_put_pixels8_mmx ff_put_pixels8_mmxext
 303
 304 /***********************************/
 305 /* 3Dnow specific */
 306
 307 #define DEF(x) x ## _3dnow
 308
 309 #include "dsputil_avg_template.c"
 310
 311 #undef DEF
 312
 313 /***********************************/
 314 /* MMXEXT specific */
 315
 316 #define DEF(x) x ## _mmxext
 317
 318 #include "dsputil_avg_template.c"
 319
 320 #undef DEF
 321
 322 #endif /* HAVE_YASM */
 323
 324
 325 #if HAVE_INLINE_ASM
 326 #define put_no_rnd_pixels16_mmx put_pixels16_mmx
 327 #define put_no_rnd_pixels8_mmx put_pixels8_mmx
 328 #define put_pixels16_mmxext put_pixels16_mmx
 329 #define put_pixels8_mmxext put_pixels8_mmx
 330 #define put_pixels4_mmxext put_pixels4_mmx
 331 #define put_no_rnd_pixels16_mmxext put_no_rnd_pixels16_mmx
 332 #define put_no_rnd_pixels8_mmxext put_no_rnd_pixels8_mmx
 333
 334 /***********************************/
 335 /* standard MMX */
 336
 337 void ff_put_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
 338                                int line_size)
 339 {
 340     const int16_t *p;
 341     uint8_t *pix;
 342
 343     /* read the pixels */
 344     p   = block;
 345     pix = pixels;
 346     /* unrolled loop */
 347     __asm__ volatile (
 348         "movq      (%3), %%mm0          \n\t"
 349         "movq     8(%3), %%mm1          \n\t"
 350         "movq    16(%3), %%mm2          \n\t"
 351         "movq    24(%3), %%mm3          \n\t"
 352         "movq    32(%3), %%mm4          \n\t"
 353         "movq    40(%3), %%mm5          \n\t"
 354         "movq    48(%3), %%mm6          \n\t"
 355         "movq    56(%3), %%mm7          \n\t"
 356         "packuswb %%mm1, %%mm0          \n\t"
 357         "packuswb %%mm3, %%mm2          \n\t"
 358         "packuswb %%mm5, %%mm4          \n\t"
 359         "packuswb %%mm7, %%mm6          \n\t"
 360         "movq     %%mm0, (%0)           \n\t"
 361         "movq     %%mm2, (%0, %1)       \n\t"
 362         "movq     %%mm4, (%0, %1, 2)    \n\t"
 363         "movq     %%mm6, (%0, %2)       \n\t"
 364         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3),
 365            "r"(p)
 366         : "memory");
 367     pix += line_size * 4;
 368     p   += 32;
 369
 370     // if here would be an exact copy of the code above
 371     // compiler would generate some very strange code
 372     // thus using "r"
 373     __asm__ volatile (
 374         "movq       (%3), %%mm0         \n\t"
 375         "movq      8(%3), %%mm1         \n\t"
 376         "movq     16(%3), %%mm2         \n\t"
 377         "movq     24(%3), %%mm3         \n\t"
 378         "movq     32(%3), %%mm4         \n\t"
 379         "movq     40(%3), %%mm5         \n\t"
 380         "movq     48(%3), %%mm6         \n\t"
 381         "movq     56(%3), %%mm7         \n\t"
 382         "packuswb  %%mm1, %%mm0         \n\t"
 383         "packuswb  %%mm3, %%mm2         \n\t"
 384         "packuswb  %%mm5, %%mm4         \n\t"
 385         "packuswb  %%mm7, %%mm6         \n\t"
 386         "movq      %%mm0, (%0)          \n\t"
 387         "movq      %%mm2, (%0, %1)      \n\t"
 388         "movq      %%mm4, (%0, %1, 2)   \n\t"
 389         "movq      %%mm6, (%0, %2)      \n\t"
 390         :: "r"(pix), "r"((x86_reg)line_size), "r"((x86_reg)line_size * 3), "r"(p)
 391         : "memory");
 392 }
 393
 394 #define put_signed_pixels_clamped_mmx_half(off)             \
 395     "movq          "#off"(%2), %%mm1        \n\t"           \
 396     "movq     16 + "#off"(%2), %%mm2        \n\t"           \
 397     "movq     32 + "#off"(%2), %%mm3        \n\t"           \
 398     "movq     48 + "#off"(%2), %%mm4        \n\t"           \
 399     "packsswb  8 + "#off"(%2), %%mm1        \n\t"           \
 400     "packsswb 24 + "#off"(%2), %%mm2        \n\t"           \
 401     "packsswb 40 + "#off"(%2), %%mm3        \n\t"           \
 402     "packsswb 56 + "#off"(%2), %%mm4        \n\t"           \
 403     "paddb              %%mm0, %%mm1        \n\t"           \
 404     "paddb              %%mm0, %%mm2        \n\t"           \
 405     "paddb              %%mm0, %%mm3        \n\t"           \
 406     "paddb              %%mm0, %%mm4        \n\t"           \
 407     "movq               %%mm1, (%0)         \n\t"           \
 408     "movq               %%mm2, (%0, %3)     \n\t"           \
 409     "movq               %%mm3, (%0, %3, 2)  \n\t"           \
 410     "movq               %%mm4, (%0, %1)     \n\t"
 411
 412 void ff_put_signed_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
 413                                       int line_size)
 414 {
 415     x86_reg line_skip = line_size;
 416     x86_reg line_skip3;
 417
 418     __asm__ volatile (
 419         "movq "MANGLE(ff_pb_80)", %%mm0     \n\t"
 420         "lea         (%3, %3, 2), %1        \n\t"
 421         put_signed_pixels_clamped_mmx_half(0)
 422         "lea         (%0, %3, 4), %0        \n\t"
 423         put_signed_pixels_clamped_mmx_half(64)
 424         : "+&r"(pixels), "=&r"(line_skip3)
 425         : "r"(block), "r"(line_skip)
 426         : "memory");
 427 }
 428
 429 void ff_add_pixels_clamped_mmx(const int16_t *block, uint8_t *pixels,
 430                                int line_size)
 431 {
 432     const int16_t *p;
 433     uint8_t *pix;
 434     int i;
 435
 436     /* read the pixels */
 437     p   = block;
 438     pix = pixels;
 439     MOVQ_ZERO(mm7);
 440     i = 4;
 441     do {
 442         __asm__ volatile (
 443             "movq        (%2), %%mm0    \n\t"
 444             "movq       8(%2), %%mm1    \n\t"
 445             "movq      16(%2), %%mm2    \n\t"
 446             "movq      24(%2), %%mm3    \n\t"
 447             "movq          %0, %%mm4    \n\t"
 448             "movq          %1, %%mm6    \n\t"
 449             "movq       %%mm4, %%mm5    \n\t"
 450             "punpcklbw  %%mm7, %%mm4    \n\t"
 451             "punpckhbw  %%mm7, %%mm5    \n\t"
 452             "paddsw     %%mm4, %%mm0    \n\t"
 453             "paddsw     %%mm5, %%mm1    \n\t"
 454             "movq       %%mm6, %%mm5    \n\t"
 455             "punpcklbw  %%mm7, %%mm6    \n\t"
 456             "punpckhbw  %%mm7, %%mm5    \n\t"
 457             "paddsw     %%mm6, %%mm2    \n\t"
 458             "paddsw     %%mm5, %%mm3    \n\t"
 459             "packuswb   %%mm1, %%mm0    \n\t"
 460             "packuswb   %%mm3, %%mm2    \n\t"
 461             "movq       %%mm0, %0       \n\t"
 462             "movq       %%mm2, %1       \n\t"
 463             : "+m"(*pix), "+m"(*(pix + line_size))
 464             : "r"(p)
 465             : "memory");
 466         pix += line_size * 2;
 467         p   += 16;
 468     } while (--i);
 469 }
 470
 471 static void put_pixels8_mmx(uint8_t *block, const uint8_t *pixels,
 472                             ptrdiff_t line_size, int h)
 473 {
 474     __asm__ volatile (
 475         "lea   (%3, %3), %%"REG_a"      \n\t"
 476         ".p2align     3                 \n\t"
 477         "1:                             \n\t"
 478         "movq  (%1    ), %%mm0          \n\t"
 479         "movq  (%1, %3), %%mm1          \n\t"
 480         "movq     %%mm0, (%2)           \n\t"
 481         "movq     %%mm1, (%2, %3)       \n\t"
 482         "add  %%"REG_a", %1             \n\t"
 483         "add  %%"REG_a", %2             \n\t"
 484         "movq  (%1    ), %%mm0          \n\t"
 485         "movq  (%1, %3), %%mm1          \n\t"
 486         "movq     %%mm0, (%2)           \n\t"
 487         "movq     %%mm1, (%2, %3)       \n\t"
 488         "add  %%"REG_a", %1             \n\t"
 489         "add  %%"REG_a", %2             \n\t"
 490         "subl        $4, %0             \n\t"
 491         "jnz         1b                 \n\t"
 492         : "+g"(h), "+r"(pixels),  "+r"(block)
 493         : "r"((x86_reg)line_size)
 494         : "%"REG_a, "memory"
 495         );
 496 }
 497
 498 static void put_pixels16_mmx(uint8_t *block, const uint8_t *pixels,
 499                              ptrdiff_t line_size, int h)
 500 {
 501     __asm__ volatile (
 502         "lea   (%3, %3), %%"REG_a"      \n\t"
 503         ".p2align     3                 \n\t"
 504         "1:                             \n\t"
 505         "movq  (%1    ), %%mm0          \n\t"
 506         "movq 8(%1    ), %%mm4          \n\t"
 507         "movq  (%1, %3), %%mm1          \n\t"
 508         "movq 8(%1, %3), %%mm5          \n\t"
 509         "movq     %%mm0,  (%2)          \n\t"
 510         "movq     %%mm4, 8(%2)          \n\t"
 511         "movq     %%mm1,  (%2, %3)      \n\t"
 512         "movq     %%mm5, 8(%2, %3)      \n\t"
 513         "add  %%"REG_a", %1             \n\t"
 514         "add  %%"REG_a", %2             \n\t"
 515         "movq  (%1    ), %%mm0          \n\t"
 516         "movq 8(%1    ), %%mm4          \n\t"
 517         "movq  (%1, %3), %%mm1          \n\t"
 518         "movq 8(%1, %3), %%mm5          \n\t"
 519         "movq     %%mm0,  (%2)          \n\t"
 520         "movq     %%mm4, 8(%2)          \n\t"
 521         "movq     %%mm1,  (%2, %3)      \n\t"
 522         "movq     %%mm5, 8(%2, %3)      \n\t"
 523         "add  %%"REG_a", %1             \n\t"
 524         "add  %%"REG_a", %2             \n\t"
 525         "subl        $4, %0             \n\t"
 526         "jnz         1b                 \n\t"
 527         : "+g"(h), "+r"(pixels),  "+r"(block)
 528         : "r"((x86_reg)line_size)
 529         : "%"REG_a, "memory"
 530         );
 531 }
 532
 533 #define CLEAR_BLOCKS(name, n)                           \
 534 static void name(int16_t *blocks)                       \
 535 {                                                       \
 536     __asm__ volatile (                                  \
 537         "pxor %%mm7, %%mm7              \n\t"           \
 538         "mov     %1,        %%"REG_a"   \n\t"           \
 539         "1:                             \n\t"           \
 540         "movq %%mm7,   (%0, %%"REG_a")  \n\t"           \
 541         "movq %%mm7,  8(%0, %%"REG_a")  \n\t"           \
 542         "movq %%mm7, 16(%0, %%"REG_a")  \n\t"           \
 543         "movq %%mm7, 24(%0, %%"REG_a")  \n\t"           \
 544         "add    $32, %%"REG_a"          \n\t"           \
 545         "js      1b                     \n\t"           \
 546         :: "r"(((uint8_t *)blocks) + 128 * n),          \
 547            "i"(-128 * n)                                \
 548         : "%"REG_a                                      \
 549         );                                              \
 550 }
 551 CLEAR_BLOCKS(clear_blocks_mmx, 6)
 552 CLEAR_BLOCKS(clear_block_mmx, 1)
 553
 554 static void clear_block_sse(int16_t *block)
 555 {
 556     __asm__ volatile (
 557         "xorps  %%xmm0, %%xmm0          \n"
 558         "movaps %%xmm0,    (%0)         \n"
 559         "movaps %%xmm0,  16(%0)         \n"
 560         "movaps %%xmm0,  32(%0)         \n"
 561         "movaps %%xmm0,  48(%0)         \n"
 562         "movaps %%xmm0,  64(%0)         \n"
 563         "movaps %%xmm0,  80(%0)         \n"
 564         "movaps %%xmm0,  96(%0)         \n"
 565         "movaps %%xmm0, 112(%0)         \n"
 566         :: "r"(block)
 567         : "memory"
 568     );
 569 }
 570
 571 static void clear_blocks_sse(int16_t *blocks)
 572 {
 573     __asm__ volatile (
 574         "xorps  %%xmm0, %%xmm0              \n"
 575         "mov        %1,         %%"REG_a"   \n"
 576         "1:                                 \n"
 577         "movaps %%xmm0,    (%0, %%"REG_a")  \n"
 578         "movaps %%xmm0,  16(%0, %%"REG_a")  \n"
 579         "movaps %%xmm0,  32(%0, %%"REG_a")  \n"
 580         "movaps %%xmm0,  48(%0, %%"REG_a")  \n"
 581         "movaps %%xmm0,  64(%0, %%"REG_a")  \n"
 582         "movaps %%xmm0,  80(%0, %%"REG_a")  \n"
 583         "movaps %%xmm0,  96(%0, %%"REG_a")  \n"
 584         "movaps %%xmm0, 112(%0, %%"REG_a")  \n"
 585         "add      $128,         %%"REG_a"   \n"
 586         "js         1b                      \n"
 587         :: "r"(((uint8_t *)blocks) + 128 * 6),
 588            "i"(-128 * 6)
 589         : "%"REG_a
 590     );
 591 }
 592
 593 static void add_bytes_mmx(uint8_t *dst, uint8_t *src, int w)
 594 {
 595     x86_reg i = 0;
 596     __asm__ volatile (
 597         "jmp          2f                \n\t"
 598         "1:                             \n\t"
 599         "movq   (%1, %0), %%mm0         \n\t"
 600         "movq   (%2, %0), %%mm1         \n\t"
 601         "paddb     %%mm0, %%mm1         \n\t"
 602         "movq      %%mm1, (%2, %0)      \n\t"
 603         "movq  8(%1, %0), %%mm0         \n\t"
 604         "movq  8(%2, %0), %%mm1         \n\t"
 605         "paddb     %%mm0, %%mm1         \n\t"
 606         "movq      %%mm1, 8(%2, %0)     \n\t"
 607         "add         $16, %0            \n\t"
 608         "2:                             \n\t"
 609         "cmp          %3, %0            \n\t"
 610         "js           1b                \n\t"
 611         : "+r"(i)
 612         : "r"(src), "r"(dst), "r"((x86_reg)w - 15)
 613     );
 614     for ( ; i < w; i++)
 615         dst[i + 0] += src[i + 0];
 616 }
 617
 618 #if HAVE_7REGS
 619 static void add_hfyu_median_prediction_cmov(uint8_t *dst, const uint8_t *top,
 620                                             const uint8_t *diff, int w,
 621                                             int *left, int *left_top)
 622 {
 623     x86_reg w2 = -w;
 624     x86_reg x;
 625     int l  = *left     & 0xff;
 626     int tl = *left_top & 0xff;
 627     int t;
 628     __asm__ volatile (
 629         "mov          %7, %3            \n"
 630         "1:                             \n"
 631         "movzbl (%3, %4), %2            \n"
 632         "mov          %2, %k3           \n"
 633         "sub         %b1, %b3           \n"
 634         "add         %b0, %b3           \n"
 635         "mov          %2, %1            \n"
 636         "cmp          %0, %2            \n"
 637         "cmovg        %0, %2            \n"
 638         "cmovg        %1, %0            \n"
 639         "cmp         %k3, %0            \n"
 640         "cmovg       %k3, %0            \n"
 641         "mov          %7, %3            \n"
 642         "cmp          %2, %0            \n"
 643         "cmovl        %2, %0            \n"
 644         "add    (%6, %4), %b0           \n"
 645         "mov         %b0, (%5, %4)      \n"
 646         "inc          %4                \n"
 647         "jl           1b                \n"
 648         : "+&q"(l), "+&q"(tl), "=&r"(t), "=&q"(x), "+&r"(w2)
 649         : "r"(dst + w), "r"(diff + w), "rm"(top + w)
 650     );
 651     *left     = l;
 652     *left_top = tl;
 653 }
 654 #endif
 655
 656 static inline void transpose4x4(uint8_t *dst, uint8_t *src, x86_reg dst_stride, x86_reg src_stride){
 657     __asm__ volatile( //FIXME could save 1 instruction if done as 8x4 ...
 658         "movd  (%1), %%mm0              \n\t"
 659         "add   %3, %1                   \n\t"
 660         "movd  (%1), %%mm1              \n\t"
 661         "movd  (%1,%3,1), %%mm2         \n\t"
 662         "movd  (%1,%3,2), %%mm3         \n\t"
 663         "punpcklbw %%mm1, %%mm0         \n\t"
 664         "punpcklbw %%mm3, %%mm2         \n\t"
 665         "movq %%mm0, %%mm1              \n\t"
 666         "punpcklwd %%mm2, %%mm0         \n\t"
 667         "punpckhwd %%mm2, %%mm1         \n\t"
 668         "movd  %%mm0, (%0)              \n\t"
 669         "add   %2, %0                   \n\t"
 670         "punpckhdq %%mm0, %%mm0         \n\t"
 671         "movd  %%mm0, (%0)              \n\t"
 672         "movd  %%mm1, (%0,%2,1)         \n\t"
 673         "punpckhdq %%mm1, %%mm1         \n\t"
 674         "movd  %%mm1, (%0,%2,2)         \n\t"
 675
 676         :  "+&r" (dst),
 677            "+&r" (src)
 678         :  "r" (dst_stride),
 679            "r" (src_stride)
 680         :  "memory"
 681     );
 682 }
 683
 684 #define H263_LOOP_FILTER                        \
 685     "pxor      %%mm7, %%mm7             \n\t"   \
 686     "movq         %0, %%mm0             \n\t"   \
 687     "movq         %0, %%mm1             \n\t"   \
 688     "movq         %3, %%mm2             \n\t"   \
 689     "movq         %3, %%mm3             \n\t"   \
 690     "punpcklbw %%mm7, %%mm0             \n\t"   \
 691     "punpckhbw %%mm7, %%mm1             \n\t"   \
 692     "punpcklbw %%mm7, %%mm2             \n\t"   \
 693     "punpckhbw %%mm7, %%mm3             \n\t"   \
 694     "psubw     %%mm2, %%mm0             \n\t"   \
 695     "psubw     %%mm3, %%mm1             \n\t"   \
 696     "movq         %1, %%mm2             \n\t"   \
 697     "movq         %1, %%mm3             \n\t"   \
 698     "movq         %2, %%mm4             \n\t"   \
 699     "movq         %2, %%mm5             \n\t"   \
 700     "punpcklbw %%mm7, %%mm2             \n\t"   \
 701     "punpckhbw %%mm7, %%mm3             \n\t"   \
 702     "punpcklbw %%mm7, %%mm4             \n\t"   \
 703     "punpckhbw %%mm7, %%mm5             \n\t"   \
 704     "psubw     %%mm2, %%mm4             \n\t"   \
 705     "psubw     %%mm3, %%mm5             \n\t"   \
 706     "psllw        $2, %%mm4             \n\t"   \
 707     "psllw        $2, %%mm5             \n\t"   \
 708     "paddw     %%mm0, %%mm4             \n\t"   \
 709     "paddw     %%mm1, %%mm5             \n\t"   \
 710     "pxor      %%mm6, %%mm6             \n\t"   \
 711     "pcmpgtw   %%mm4, %%mm6             \n\t"   \
 712     "pcmpgtw   %%mm5, %%mm7             \n\t"   \
 713     "pxor      %%mm6, %%mm4             \n\t"   \
 714     "pxor      %%mm7, %%mm5             \n\t"   \
 715     "psubw     %%mm6, %%mm4             \n\t"   \
 716     "psubw     %%mm7, %%mm5             \n\t"   \
 717     "psrlw        $3, %%mm4             \n\t"   \
 718     "psrlw        $3, %%mm5             \n\t"   \
 719     "packuswb  %%mm5, %%mm4             \n\t"   \
 720     "packsswb  %%mm7, %%mm6             \n\t"   \
 721     "pxor      %%mm7, %%mm7             \n\t"   \
 722     "movd         %4, %%mm2             \n\t"   \
 723     "punpcklbw %%mm2, %%mm2             \n\t"   \
 724     "punpcklbw %%mm2, %%mm2             \n\t"   \
 725     "punpcklbw %%mm2, %%mm2             \n\t"   \
 726     "psubusb   %%mm4, %%mm2             \n\t"   \
 727     "movq      %%mm2, %%mm3             \n\t"   \
 728     "psubusb   %%mm4, %%mm3             \n\t"   \
 729     "psubb     %%mm3, %%mm2             \n\t"   \
 730     "movq         %1, %%mm3             \n\t"   \
 731     "movq         %2, %%mm4             \n\t"   \
 732     "pxor      %%mm6, %%mm3             \n\t"   \
 733     "pxor      %%mm6, %%mm4             \n\t"   \
 734     "paddusb   %%mm2, %%mm3             \n\t"   \
 735     "psubusb   %%mm2, %%mm4             \n\t"   \
 736     "pxor      %%mm6, %%mm3             \n\t"   \
 737     "pxor      %%mm6, %%mm4             \n\t"   \
 738     "paddusb   %%mm2, %%mm2             \n\t"   \
 739     "packsswb  %%mm1, %%mm0             \n\t"   \
 740     "pcmpgtb   %%mm0, %%mm7             \n\t"   \
 741     "pxor      %%mm7, %%mm0             \n\t"   \
 742     "psubb     %%mm7, %%mm0             \n\t"   \
 743     "movq      %%mm0, %%mm1             \n\t"   \
 744     "psubusb   %%mm2, %%mm0             \n\t"   \
 745     "psubb     %%mm0, %%mm1             \n\t"   \
 746     "pand         %5, %%mm1             \n\t"   \
 747     "psrlw        $2, %%mm1             \n\t"   \
 748     "pxor      %%mm7, %%mm1             \n\t"   \
 749     "psubb     %%mm7, %%mm1             \n\t"   \
 750     "movq         %0, %%mm5             \n\t"   \
 751     "movq         %3, %%mm6             \n\t"   \
 752     "psubb     %%mm1, %%mm5             \n\t"   \
 753     "paddb     %%mm1, %%mm6             \n\t"
 754
 755 static void h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale)
 756 {
 757     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
 758         const int strength = ff_h263_loop_filter_strength[qscale];
 759
 760         __asm__ volatile (
 761             H263_LOOP_FILTER
 762
 763             "movq %%mm3, %1             \n\t"
 764             "movq %%mm4, %2             \n\t"
 765             "movq %%mm5, %0             \n\t"
 766             "movq %%mm6, %3             \n\t"
 767             : "+m"(*(uint64_t*)(src - 2 * stride)),
 768               "+m"(*(uint64_t*)(src - 1 * stride)),
 769               "+m"(*(uint64_t*)(src + 0 * stride)),
 770               "+m"(*(uint64_t*)(src + 1 * stride))
 771             : "g"(2 * strength), "m"(ff_pb_FC)
 772             );
 773     }
 774 }
 775
 776 static void h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale)
 777 {
 778     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
 779         const int strength = ff_h263_loop_filter_strength[qscale];
 780         DECLARE_ALIGNED(8, uint64_t, temp)[4];
 781         uint8_t *btemp = (uint8_t*)temp;
 782
 783         src -= 2;
 784
 785         transpose4x4(btemp,     src,              8, stride);
 786         transpose4x4(btemp + 4, src + 4 * stride, 8, stride);
 787         __asm__ volatile (
 788             H263_LOOP_FILTER // 5 3 4 6
 789
 790             : "+m"(temp[0]),
 791               "+m"(temp[1]),
 792               "+m"(temp[2]),
 793               "+m"(temp[3])
 794             : "g"(2 * strength), "m"(ff_pb_FC)
 795             );
 796
 797         __asm__ volatile (
 798             "movq      %%mm5, %%mm1         \n\t"
 799             "movq      %%mm4, %%mm0         \n\t"
 800             "punpcklbw %%mm3, %%mm5         \n\t"
 801             "punpcklbw %%mm6, %%mm4         \n\t"
 802             "punpckhbw %%mm3, %%mm1         \n\t"
 803             "punpckhbw %%mm6, %%mm0         \n\t"
 804             "movq      %%mm5, %%mm3         \n\t"
 805             "movq      %%mm1, %%mm6         \n\t"
 806             "punpcklwd %%mm4, %%mm5         \n\t"
 807             "punpcklwd %%mm0, %%mm1         \n\t"
 808             "punpckhwd %%mm4, %%mm3         \n\t"
 809             "punpckhwd %%mm0, %%mm6         \n\t"
 810             "movd      %%mm5, (%0)          \n\t"
 811             "punpckhdq %%mm5, %%mm5         \n\t"
 812             "movd      %%mm5, (%0, %2)      \n\t"
 813             "movd      %%mm3, (%0, %2, 2)   \n\t"
 814             "punpckhdq %%mm3, %%mm3         \n\t"
 815             "movd      %%mm3, (%0, %3)      \n\t"
 816             "movd      %%mm1, (%1)          \n\t"
 817             "punpckhdq %%mm1, %%mm1         \n\t"
 818             "movd      %%mm1, (%1, %2)      \n\t"
 819             "movd      %%mm6, (%1, %2, 2)   \n\t"
 820             "punpckhdq %%mm6, %%mm6         \n\t"
 821             "movd      %%mm6, (%1, %3)      \n\t"
 822             :: "r"(src),
 823                "r"(src + 4 * stride),
 824                "r"((x86_reg)stride),
 825                "r"((x86_reg)(3 * stride))
 826             );
 827     }
 828 }
 829
 830 /* Draw the edges of width 'w' of an image of size width, height
 831  * this MMX version can only handle w == 8 || w == 16. */
 832 static void draw_edges_mmx(uint8_t *buf, int wrap, int width, int height,
 833                            int w, int h, int sides)
 834 {
 835     uint8_t *ptr, *last_line;
 836     int i;
 837
 838     last_line = buf + (height - 1) * wrap;
 839     /* left and right */
 840     ptr = buf;
 841     if (w == 8) {
 842         __asm__ volatile (
 843             "1:                             \n\t"
 844             "movd            (%0), %%mm0    \n\t"
 845             "punpcklbw      %%mm0, %%mm0    \n\t"
 846             "punpcklwd      %%mm0, %%mm0    \n\t"
 847             "punpckldq      %%mm0, %%mm0    \n\t"
 848             "movq           %%mm0, -8(%0)   \n\t"
 849             "movq      -8(%0, %2), %%mm1    \n\t"
 850             "punpckhbw      %%mm1, %%mm1    \n\t"
 851             "punpckhwd      %%mm1, %%mm1    \n\t"
 852             "punpckhdq      %%mm1, %%mm1    \n\t"
 853             "movq           %%mm1, (%0, %2) \n\t"
 854             "add               %1, %0       \n\t"
 855             "cmp               %3, %0       \n\t"
 856             "jb                1b           \n\t"
 857             : "+r"(ptr)
 858             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
 859             );
 860     } else if(w==16){
 861         __asm__ volatile (
 862             "1:                                 \n\t"
 863             "movd            (%0), %%mm0        \n\t"
 864             "punpcklbw      %%mm0, %%mm0        \n\t"
 865             "punpcklwd      %%mm0, %%mm0        \n\t"
 866             "punpckldq      %%mm0, %%mm0        \n\t"
 867             "movq           %%mm0, -8(%0)       \n\t"
 868             "movq           %%mm0, -16(%0)      \n\t"
 869             "movq      -8(%0, %2), %%mm1        \n\t"
 870             "punpckhbw      %%mm1, %%mm1        \n\t"
 871             "punpckhwd      %%mm1, %%mm1        \n\t"
 872             "punpckhdq      %%mm1, %%mm1        \n\t"
 873             "movq           %%mm1,  (%0, %2)    \n\t"
 874             "movq           %%mm1, 8(%0, %2)    \n\t"
 875             "add               %1, %0           \n\t"
 876             "cmp               %3, %0           \n\t"
 877             "jb                1b               \n\t"
 878             : "+r"(ptr)
 879             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
 880             );
 881     } else {
 882         av_assert1(w == 4);
 883         __asm__ volatile (
 884             "1:                             \n\t"
 885             "movd            (%0), %%mm0    \n\t"
 886             "punpcklbw      %%mm0, %%mm0    \n\t"
 887             "punpcklwd      %%mm0, %%mm0    \n\t"
 888             "movd           %%mm0, -4(%0)   \n\t"
 889             "movd      -4(%0, %2), %%mm1    \n\t"
 890             "punpcklbw      %%mm1, %%mm1    \n\t"
 891             "punpckhwd      %%mm1, %%mm1    \n\t"
 892             "punpckhdq      %%mm1, %%mm1    \n\t"
 893             "movd           %%mm1, (%0, %2) \n\t"
 894             "add               %1, %0       \n\t"
 895             "cmp               %3, %0       \n\t"
 896             "jb                1b           \n\t"
 897             : "+r"(ptr)
 898             : "r"((x86_reg)wrap), "r"((x86_reg)width), "r"(ptr + wrap * height)
 899             );
 900     }
 901
 902     /* top and bottom (and hopefully also the corners) */
 903     if (sides & EDGE_TOP) {
 904         for (i = 0; i < h; i += 4) {
 905             ptr = buf - (i + 1) * wrap - w;
 906             __asm__ volatile (
 907                 "1:                             \n\t"
 908                 "movq (%1, %0), %%mm0           \n\t"
 909                 "movq    %%mm0, (%0)            \n\t"
 910                 "movq    %%mm0, (%0, %2)        \n\t"
 911                 "movq    %%mm0, (%0, %2, 2)     \n\t"
 912                 "movq    %%mm0, (%0, %3)        \n\t"
 913                 "add        $8, %0              \n\t"
 914                 "cmp        %4, %0              \n\t"
 915                 "jb         1b                  \n\t"
 916                 : "+r"(ptr)
 917                 : "r"((x86_reg)buf - (x86_reg)ptr - w), "r"((x86_reg) -wrap),
 918                   "r"((x86_reg) -wrap * 3), "r"(ptr + width + 2 * w)
 919                 );
 920         }
 921     }
 922
 923     if (sides & EDGE_BOTTOM) {
 924         for (i = 0; i < h; i += 4) {
 925             ptr = last_line + (i + 1) * wrap - w;
 926             __asm__ volatile (
 927                 "1:                             \n\t"
 928                 "movq (%1, %0), %%mm0           \n\t"
 929                 "movq    %%mm0, (%0)            \n\t"
 930                 "movq    %%mm0, (%0, %2)        \n\t"
 931                 "movq    %%mm0, (%0, %2, 2)     \n\t"
 932                 "movq    %%mm0, (%0, %3)        \n\t"
 933                 "add        $8, %0              \n\t"
 934                 "cmp        %4, %0              \n\t"
 935                 "jb         1b                  \n\t"
 936                 : "+r"(ptr)
 937                 : "r"((x86_reg)last_line - (x86_reg)ptr - w),
 938                   "r"((x86_reg)wrap), "r"((x86_reg)wrap * 3),
 939                   "r"(ptr + width + 2 * w)
 940                 );
 941         }
 942     }
 943 }
 944 #endif /* HAVE_INLINE_ASM */
 945
 946
 947 #if HAVE_YASM
 948 #define QPEL_OP(OPNAME, ROUNDER, RND, OP, MMX)                          \
 949 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
 950                                           int stride)                   \
 951 {                                                                       \
 952     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
 953 }                                                                       \
 954                                                                         \
 955 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
 956                                          int stride)                    \
 957 {                                                                       \
 958     uint64_t temp[8];                                                   \
 959     uint8_t * const half = (uint8_t*)temp;                              \
 960     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 961                                                    stride, 8);          \
 962     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 963                                         stride, stride, 8);             \
 964 }                                                                       \
 965                                                                         \
 966 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
 967                                          int stride)                    \
 968 {                                                                       \
 969     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
 970                                                    stride, 8);          \
 971 }                                                                       \
 972                                                                         \
 973 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
 974                                          int stride)                    \
 975 {                                                                       \
 976     uint64_t temp[8];                                                   \
 977     uint8_t * const half = (uint8_t*)temp;                              \
 978     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 979                                                    stride, 8);          \
 980     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
 981                                         stride, 8);                     \
 982 }                                                                       \
 983                                                                         \
 984 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
 985                                          int stride)                    \
 986 {                                                                       \
 987     uint64_t temp[8];                                                   \
 988     uint8_t * const half = (uint8_t*)temp;                              \
 989     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 990                                                    8, stride);          \
 991     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 992                                         stride, stride, 8);             \
 993 }                                                                       \
 994                                                                         \
 995 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
 996                                          int stride)                    \
 997 {                                                                       \
 998     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
 999                                                    stride, stride);     \
1000 }                                                                       \
1001                                                                         \
1002 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
1003                                          int stride)                    \
1004 {                                                                       \
1005     uint64_t temp[8];                                                   \
1006     uint8_t * const half = (uint8_t*)temp;                              \
1007     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
1008                                                    8, stride);          \
1009     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
1010                                         stride, 8);                     \
1011 }                                                                       \
1012                                                                         \
1013 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
1014                                          int stride)                    \
1015 {                                                                       \
1016     uint64_t half[8 + 9];                                               \
1017     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1018     uint8_t * const halfHV = ((uint8_t*)half);                          \
1019     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1020                                                    stride, 9);          \
1021     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
1022                                         stride, 9);                     \
1023     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1024     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
1025                                         stride, 8, 8);                  \
1026 }                                                                       \
1027                                                                         \
1028 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
1029                                          int stride)                    \
1030 {                                                                       \
1031     uint64_t half[8 + 9];                                               \
1032     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1033     uint8_t * const halfHV = ((uint8_t*)half);                          \
1034     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1035                                                    stride, 9);          \
1036     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
1037                                         stride, 9);                     \
1038     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1039     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
1040                                         stride, 8, 8);                  \
1041 }                                                                       \
1042                                                                         \
1043 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
1044                                          int stride)                    \
1045 {                                                                       \
1046     uint64_t half[8 + 9];                                               \
1047     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1048     uint8_t * const halfHV = ((uint8_t*)half);                          \
1049     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1050                                                    stride, 9);          \
1051     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
1052                                         stride, 9);                     \
1053     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1054     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
1055                                         stride, 8, 8);                  \
1056 }                                                                       \
1057                                                                         \
1058 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
1059                                          int stride)                    \
1060 {                                                                       \
1061     uint64_t half[8 + 9];                                               \
1062     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1063     uint8_t * const halfHV = ((uint8_t*)half);                          \
1064     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1065                                                    stride, 9);          \
1066     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
1067                                         stride, 9);                     \
1068     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1069     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
1070                                         stride, 8, 8);                  \
1071 }                                                                       \
1072                                                                         \
1073 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
1074                                          int stride)                    \
1075 {                                                                       \
1076     uint64_t half[8 + 9];                                               \
1077     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1078     uint8_t * const halfHV = ((uint8_t*)half);                          \
1079     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1080                                                    stride, 9);          \
1081     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1082     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
1083                                         stride, 8, 8);                  \
1084 }                                                                       \
1085                                                                         \
1086 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
1087                                          int stride)                    \
1088 {                                                                       \
1089     uint64_t half[8 + 9];                                               \
1090     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
1091     uint8_t * const halfHV = ((uint8_t*)half);                          \
1092     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1093                                                    stride, 9);          \
1094     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
1095     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
1096                                         stride, 8, 8);                  \
1097 }                                                                       \
1098                                                                         \
1099 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
1100                                          int stride)                    \
1101 {                                                                       \
1102     uint64_t half[8 + 9];                                               \
1103     uint8_t * const halfH = ((uint8_t*)half);                           \
1104     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1105                                                    stride, 9);          \
1106     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
1107                                         8, stride, 9);                  \
1108     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
1109                                                    stride, 8);          \
1110 }                                                                       \
1111                                                                         \
1112 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
1113                                          int stride)                    \
1114 {                                                                       \
1115     uint64_t half[8 + 9];                                               \
1116     uint8_t * const halfH = ((uint8_t*)half);                           \
1117     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1118                                                    stride, 9);          \
1119     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
1120                                         stride, 9);                     \
1121     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
1122                                                    stride, 8);          \
1123 }                                                                       \
1124                                                                         \
1125 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
1126                                          int stride)                    \
1127 {                                                                       \
1128     uint64_t half[9];                                                   \
1129     uint8_t * const halfH = ((uint8_t*)half);                           \
1130     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
1131                                                    stride, 9);          \
1132     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
1133                                                    stride, 8);          \
1134 }                                                                       \
1135                                                                         \
1136 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
1137                                            int stride)                  \
1138 {                                                                       \
1139     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
1140 }                                                                       \
1141                                                                         \
1142 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
1143                                           int stride)                   \
1144 {                                                                       \
1145     uint64_t temp[32];                                                  \
1146     uint8_t * const half = (uint8_t*)temp;                              \
1147     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
1148                                                     stride, 16);        \
1149     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
1150                                          stride, 16);                   \
1151 }                                                                       \
1152                                                                         \
1153 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
1154                                           int stride)                   \
1155 {                                                                       \
1156     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
1157                                                     stride, stride, 16);\
1158 }                                                                       \
1159                                                                         \
1160 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
1161                                           int stride)                   \
1162 {                                                                       \
1163     uint64_t temp[32];                                                  \
1164     uint8_t * const half = (uint8_t*)temp;                              \
1165     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
1166                                                     stride, 16);        \
1167     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
1168                                          stride, stride, 16);           \
1169 }                                                                       \
1170                                                                         \
1171 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
1172                                           int stride)                   \
1173 {                                                                       \
1174     uint64_t temp[32];                                                  \
1175     uint8_t * const half = (uint8_t*)temp;                              \
1176     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
1177                                                     stride);            \
1178     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
1179                                          stride, 16);                   \
1180 }                                                                       \
1181                                                                         \
1182 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
1183                                           int stride)                   \
1184 {                                                                       \
1185     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
1186                                                     stride, stride);    \
1187 }                                                                       \
1188                                                                         \
1189 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
1190                                           int stride)                   \
1191 {                                                                       \
1192     uint64_t temp[32];                                                  \
1193     uint8_t * const half = (uint8_t*)temp;                              \
1194     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
1195                                                     stride);            \
1196     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
1197                                          stride, stride, 16);           \
1198 }                                                                       \
1199                                                                         \
1200 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
1201                                           int stride)                   \
1202 {                                                                       \
1203     uint64_t half[16 * 2 + 17 * 2];                                     \
1204     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1205     uint8_t * const halfHV = ((uint8_t*)half);                          \
1206     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1207                                                     stride, 17);        \
1208     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
1209                                          stride, 17);                   \
1210     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1211                                                     16, 16);            \
1212     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
1213                                          stride, 16, 16);               \
1214 }                                                                       \
1215                                                                         \
1216 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
1217                                           int stride)                   \
1218 {                                                                       \
1219     uint64_t half[16 * 2 + 17 * 2];                                     \
1220     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1221     uint8_t * const halfHV = ((uint8_t*)half);                          \
1222     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1223                                                     stride, 17);        \
1224     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
1225                                          stride, 17);                   \
1226     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1227                                                     16, 16);            \
1228     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
1229                                          stride, 16, 16);               \
1230 }                                                                       \
1231                                                                         \
1232 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
1233                                           int stride)                   \
1234 {                                                                       \
1235     uint64_t half[16 * 2 + 17 * 2];                                     \
1236     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1237     uint8_t * const halfHV = ((uint8_t*)half);                          \
1238     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1239                                                     stride, 17);        \
1240     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
1241                                          stride, 17);                   \
1242     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1243                                                     16, 16);            \
1244     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
1245                                          stride, 16, 16);               \
1246 }                                                                       \
1247                                                                         \
1248 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
1249                                           int stride)                   \
1250 {                                                                       \
1251     uint64_t half[16 * 2 + 17 * 2];                                     \
1252     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1253     uint8_t * const halfHV = ((uint8_t*)half);                          \
1254     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1255                                                     stride, 17);        \
1256     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
1257                                          stride, 17);                   \
1258     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1259                                                     16, 16);            \
1260     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
1261                                          stride, 16, 16);               \
1262 }                                                                       \
1263                                                                         \
1264 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
1265                                           int stride)                   \
1266 {                                                                       \
1267     uint64_t half[16 * 2 + 17 * 2];                                     \
1268     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1269     uint8_t * const halfHV = ((uint8_t*)half);                          \
1270     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1271                                                     stride, 17);        \
1272     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1273                                                     16, 16);            \
1274     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
1275                                          stride, 16, 16);               \
1276 }                                                                       \
1277                                                                         \
1278 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
1279                                           int stride)                   \
1280 {                                                                       \
1281     uint64_t half[16 * 2 + 17 * 2];                                     \
1282     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
1283     uint8_t * const halfHV = ((uint8_t*)half);                          \
1284     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1285                                                     stride, 17);        \
1286     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
1287                                                     16, 16);            \
1288     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
1289                                          stride, 16, 16);               \
1290 }                                                                       \
1291                                                                         \
1292 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
1293                                           int stride)                   \
1294 {                                                                       \
1295     uint64_t half[17 * 2];                                              \
1296     uint8_t * const halfH = ((uint8_t*)half);                           \
1297     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1298                                                     stride, 17);        \
1299     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
1300                                          stride, 17);                   \
1301     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
1302                                                     stride, 16);        \
1303 }                                                                       \
1304                                                                         \
1305 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
1306                                           int stride)                   \
1307 {                                                                       \
1308     uint64_t half[17 * 2];                                              \
1309     uint8_t * const halfH = ((uint8_t*)half);                           \
1310     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1311                                                     stride, 17);        \
1312     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
1313                                          stride, 17);                   \
1314     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
1315                                                     stride, 16);        \
1316 }                                                                       \
1317                                                                         \
1318 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
1319                                           int stride)                   \
1320 {                                                                       \
1321     uint64_t half[17 * 2];                                              \
1322     uint8_t * const halfH = ((uint8_t*)half);                           \
1323     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
1324                                                     stride, 17);        \
1325     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
1326                                                     stride, 16);        \
1327 }
1328
1329 #define PUT_OP(a, b, temp, size)                \
1330     "mov"#size"        "#a", "#b"       \n\t"
1331
1332 #define AVG_MMXEXT_OP(a, b, temp, size)         \
1333     "mov"#size"        "#b", "#temp"    \n\t"   \
1334     "pavgb          "#temp", "#a"       \n\t"   \
1335     "mov"#size"        "#a", "#b"       \n\t"
1336
1337 QPEL_OP(put_,          ff_pw_16, _,        PUT_OP,        mmxext)
1338 QPEL_OP(avg_,          ff_pw_16, _,        AVG_MMXEXT_OP, mmxext)
1339 QPEL_OP(put_no_rnd_,   ff_pw_15, _no_rnd_, PUT_OP,        mmxext)
1340 #endif /* HAVE_YASM */
1341
1342
1343 #if HAVE_INLINE_ASM
1344 void ff_put_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1345 {
1346   put_pixels8_xy2_mmx(dst, src, stride, 8);
1347 }
1348 void ff_put_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1349 {
1350   put_pixels16_xy2_mmx(dst, src, stride, 16);
1351 }
1352 void ff_avg_rv40_qpel8_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1353 {
1354   avg_pixels8_xy2_mmx(dst, src, stride, 8);
1355 }
1356 void ff_avg_rv40_qpel16_mc33_mmx(uint8_t *dst, uint8_t *src, int stride)
1357 {
1358   avg_pixels16_xy2_mmx(dst, src, stride, 16);
1359 }
1360
1361 typedef void emulated_edge_mc_func(uint8_t *dst, const uint8_t *src,
1362                                    ptrdiff_t linesize, int block_w, int block_h,
1363                                    int src_x, int src_y, int w, int h);
1364
1365 static av_always_inline void gmc(uint8_t *dst, uint8_t *src,
1366                                  int stride, int h, int ox, int oy,
1367                                  int dxx, int dxy, int dyx, int dyy,
1368                                  int shift, int r, int width, int height,
1369                                  emulated_edge_mc_func *emu_edge_fn)
1370 {
1371     const int w    = 8;
1372     const int ix   = ox  >> (16 + shift);
1373     const int iy   = oy  >> (16 + shift);
1374     const int oxs  = ox  >> 4;
1375     const int oys  = oy  >> 4;
1376     const int dxxs = dxx >> 4;
1377     const int dxys = dxy >> 4;
1378     const int dyxs = dyx >> 4;
1379     const int dyys = dyy >> 4;
1380     const uint16_t r4[4]   = { r, r, r, r };
1381     const uint16_t dxy4[4] = { dxys, dxys, dxys, dxys };
1382     const uint16_t dyy4[4] = { dyys, dyys, dyys, dyys };
1383     const uint64_t shift2 = 2 * shift;
1384 #define MAX_STRIDE 4096U
1385 #define MAX_H 8U
1386     uint8_t edge_buf[(MAX_H + 1) * MAX_STRIDE];
1387     int x, y;
1388
1389     const int dxw = (dxx - (1 << (16 + shift))) * (w - 1);
1390     const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);
1391     const int dxh = dxy * (h - 1);
1392     const int dyw = dyx * (w - 1);
1393     int need_emu =  (unsigned)ix >= width  - w ||
1394                     (unsigned)iy >= height - h;
1395
1396     if ( // non-constant fullpel offset (3% of blocks)
1397         ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |
1398          (oy ^ (oy + dyw)) | (oy ^ (oy + dyh)) | (oy ^ (oy + dyw + dyh))) >> (16 + shift)
1399         // uses more than 16 bits of subpel mv (only at huge resolution)
1400         || (dxx | dxy | dyx | dyy) & 15
1401         || (need_emu && (h > MAX_H || stride > MAX_STRIDE))) {
1402         // FIXME could still use mmx for some of the rows
1403         ff_gmc_c(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy,
1404                  shift, r, width, height);
1405         return;
1406     }
1407
1408     src += ix + iy * stride;
1409     if (need_emu) {
1410         emu_edge_fn(edge_buf, src, stride, w + 1, h + 1, ix, iy, width, height);
1411         src = edge_buf;
1412     }
1413
1414     __asm__ volatile (
1415         "movd         %0, %%mm6         \n\t"
1416         "pxor      %%mm7, %%mm7         \n\t"
1417         "punpcklwd %%mm6, %%mm6         \n\t"
1418         "punpcklwd %%mm6, %%mm6         \n\t"
1419         :: "r"(1<<shift)
1420     );
1421
1422     for (x = 0; x < w; x += 4) {
1423         uint16_t dx4[4] = { oxs - dxys + dxxs * (x + 0),
1424                             oxs - dxys + dxxs * (x + 1),
1425                             oxs - dxys + dxxs * (x + 2),
1426                             oxs - dxys + dxxs * (x + 3) };
1427         uint16_t dy4[4] = { oys - dyys + dyxs * (x + 0),
1428                             oys - dyys + dyxs * (x + 1),
1429                             oys - dyys + dyxs * (x + 2),
1430                             oys - dyys + dyxs * (x + 3) };
1431
1432         for (y = 0; y < h; y++) {
1433             __asm__ volatile (
1434                 "movq      %0, %%mm4    \n\t"
1435                 "movq      %1, %%mm5    \n\t"
1436                 "paddw     %2, %%mm4    \n\t"
1437                 "paddw     %3, %%mm5    \n\t"
1438                 "movq   %%mm4, %0       \n\t"
1439                 "movq   %%mm5, %1       \n\t"
1440                 "psrlw    $12, %%mm4    \n\t"
1441                 "psrlw    $12, %%mm5    \n\t"
1442                 : "+m"(*dx4), "+m"(*dy4)
1443                 : "m"(*dxy4), "m"(*dyy4)
1444             );
1445
1446             __asm__ volatile (
1447                 "movq      %%mm6, %%mm2 \n\t"
1448                 "movq      %%mm6, %%mm1 \n\t"
1449                 "psubw     %%mm4, %%mm2 \n\t"
1450                 "psubw     %%mm5, %%mm1 \n\t"
1451                 "movq      %%mm2, %%mm0 \n\t"
1452                 "movq      %%mm4, %%mm3 \n\t"
1453                 "pmullw    %%mm1, %%mm0 \n\t" // (s - dx) * (s - dy)
1454                 "pmullw    %%mm5, %%mm3 \n\t" // dx * dy
1455                 "pmullw    %%mm5, %%mm2 \n\t" // (s - dx) * dy
1456                 "pmullw    %%mm4, %%mm1 \n\t" // dx * (s - dy)
1457
1458                 "movd         %4, %%mm5 \n\t"
1459                 "movd         %3, %%mm4 \n\t"
1460                 "punpcklbw %%mm7, %%mm5 \n\t"
1461                 "punpcklbw %%mm7, %%mm4 \n\t"
1462                 "pmullw    %%mm5, %%mm3 \n\t" // src[1, 1] * dx * dy
1463                 "pmullw    %%mm4, %%mm2 \n\t" // src[0, 1] * (s - dx) * dy
1464
1465                 "movd         %2, %%mm5 \n\t"
1466                 "movd         %1, %%mm4 \n\t"
1467                 "punpcklbw %%mm7, %%mm5 \n\t"
1468                 "punpcklbw %%mm7, %%mm4 \n\t"
1469                 "pmullw    %%mm5, %%mm1 \n\t" // src[1, 0] * dx * (s - dy)
1470                 "pmullw    %%mm4, %%mm0 \n\t" // src[0, 0] * (s - dx) * (s - dy)
1471                 "paddw        %5, %%mm1 \n\t"
1472                 "paddw     %%mm3, %%mm2 \n\t"
1473                 "paddw     %%mm1, %%mm0 \n\t"
1474                 "paddw     %%mm2, %%mm0 \n\t"
1475
1476                 "psrlw        %6, %%mm0 \n\t"
1477                 "packuswb  %%mm0, %%mm0 \n\t"
1478                 "movd      %%mm0, %0    \n\t"
1479
1480                 : "=m"(dst[x + y * stride])
1481                 : "m"(src[0]), "m"(src[1]),
1482                   "m"(src[stride]), "m"(src[stride + 1]),
1483                   "m"(*r4), "m"(shift2)
1484             );
1485             src += stride;
1486         }
1487         src += 4 - h * stride;
1488     }
1489 }
1490
1491 #if CONFIG_VIDEODSP
1492 #if HAVE_YASM
1493 #if ARCH_X86_32
1494 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1495                     int stride, int h, int ox, int oy,
1496                     int dxx, int dxy, int dyx, int dyy,
1497                     int shift, int r, int width, int height)
1498 {
1499     gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1500         width, height, &ff_emulated_edge_mc_8);
1501 }
1502 #endif
1503 static void gmc_sse(uint8_t *dst, uint8_t *src,
1504                     int stride, int h, int ox, int oy,
1505                     int dxx, int dxy, int dyx, int dyy,
1506                     int shift, int r, int width, int height)
1507 {
1508     gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1509         width, height, &ff_emulated_edge_mc_8);
1510 }
1511 #else
1512 static void gmc_mmx(uint8_t *dst, uint8_t *src,
1513                     int stride, int h, int ox, int oy,
1514                     int dxx, int dxy, int dyx, int dyy,
1515                     int shift, int r, int width, int height)
1516 {
1517     gmc(dst, src, stride, h, ox, oy, dxx, dxy, dyx, dyy, shift, r,
1518         width, height, &ff_emulated_edge_mc_8);
1519 }
1520 #endif
1521 #endif
1522
1523 #endif /* HAVE_INLINE_ASM */
1524
1525 void ff_put_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1526                           ptrdiff_t line_size, int h);
1527 void ff_avg_pixels16_sse2(uint8_t *block, const uint8_t *pixels,
1528                           ptrdiff_t line_size, int h);
1529
1530 #if HAVE_INLINE_ASM
1531
1532 /* CAVS-specific */
1533 void ff_put_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1534 {
1535     put_pixels8_mmx(dst, src, stride, 8);
1536 }
1537
1538 void ff_avg_cavs_qpel8_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1539 {
1540     avg_pixels8_mmx(dst, src, stride, 8);
1541 }
1542
1543 void ff_put_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1544 {
1545     put_pixels16_mmx(dst, src, stride, 16);
1546 }
1547
1548 void ff_avg_cavs_qpel16_mc00_mmxext(uint8_t *dst, uint8_t *src, int stride)
1549 {
1550     avg_pixels16_mmx(dst, src, stride, 16);
1551 }
1552 #endif /* HAVE_INLINE_ASM */
1553
1554 #if HAVE_YASM
1555 /* VC-1-specific */
1556 void ff_put_vc1_mspel_mc00_mmx(uint8_t *dst, const uint8_t *src,
1557                                int stride, int rnd)
1558 {
1559     ff_put_pixels8_mmx(dst, src, stride, 8);
1560 }
1561
1562 void ff_avg_vc1_mspel_mc00_mmxext(uint8_t *dst, const uint8_t *src,
1563                                   int stride, int rnd)
1564 {
1565     ff_avg_pixels8_mmxext(dst, src, stride, 8);
1566 }
1567 #endif /* HAVE_YASM */
1568
1569 #if CONFIG_DIRAC_DECODER
1570 #define DIRAC_PIXOP(OPNAME2, OPNAME, EXT)\
1571 void ff_ ## OPNAME2 ## _dirac_pixels8_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1572 {\
1573     if (h&3)\
1574         ff_ ## OPNAME2 ## _dirac_pixels8_c(dst, src, stride, h);\
1575     else\
1576         OPNAME ## _pixels8_ ## EXT(dst, src[0], stride, h);\
1577 }\
1578 void ff_ ## OPNAME2 ## _dirac_pixels16_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1579 {\
1580     if (h&3)\
1581         ff_ ## OPNAME2 ## _dirac_pixels16_c(dst, src, stride, h);\
1582     else\
1583         OPNAME ## _pixels16_ ## EXT(dst, src[0], stride, h);\
1584 }\
1585 void ff_ ## OPNAME2 ## _dirac_pixels32_ ## EXT(uint8_t *dst, const uint8_t *src[5], int stride, int h)\
1586 {\
1587     if (h&3) {\
1588         ff_ ## OPNAME2 ## _dirac_pixels32_c(dst, src, stride, h);\
1589     } else {\
1590         OPNAME ## _pixels16_ ## EXT(dst   , src[0]   , stride, h);\
1591         OPNAME ## _pixels16_ ## EXT(dst+16, src[0]+16, stride, h);\
1592     }\
1593 }
1594
1595 #if HAVE_MMX_INLINE
1596 DIRAC_PIXOP(put, put, mmx)
1597 DIRAC_PIXOP(avg, avg, mmx)
1598 #endif
1599
1600 #if HAVE_YASM
1601 DIRAC_PIXOP(avg, ff_avg, mmxext)
1602
1603 void ff_put_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1604 {
1605     if (h&3)
1606         ff_put_dirac_pixels16_c(dst, src, stride, h);
1607     else
1608     ff_put_pixels16_sse2(dst, src[0], stride, h);
1609 }
1610 void ff_avg_dirac_pixels16_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1611 {
1612     if (h&3)
1613         ff_avg_dirac_pixels16_c(dst, src, stride, h);
1614     else
1615     ff_avg_pixels16_sse2(dst, src[0], stride, h);
1616 }
1617 void ff_put_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1618 {
1619     if (h&3) {
1620         ff_put_dirac_pixels32_c(dst, src, stride, h);
1621     } else {
1622     ff_put_pixels16_sse2(dst   , src[0]   , stride, h);
1623     ff_put_pixels16_sse2(dst+16, src[0]+16, stride, h);
1624     }
1625 }
1626 void ff_avg_dirac_pixels32_sse2(uint8_t *dst, const uint8_t *src[5], int stride, int h)
1627 {
1628     if (h&3) {
1629         ff_avg_dirac_pixels32_c(dst, src, stride, h);
1630     } else {
1631     ff_avg_pixels16_sse2(dst   , src[0]   , stride, h);
1632     ff_avg_pixels16_sse2(dst+16, src[0]+16, stride, h);
1633     }
1634 }
1635 #endif
1636 #endif
1637
1638 /* XXX: Those functions should be suppressed ASAP when all IDCTs are
1639  * converted. */
1640 #if CONFIG_GPL
1641 static void ff_libmpeg2mmx_idct_put(uint8_t *dest, int line_size,
1642                                     int16_t *block)
1643 {
1644     ff_mmx_idct(block);
1645     ff_put_pixels_clamped_mmx(block, dest, line_size);
1646 }
1647
1648 static void ff_libmpeg2mmx_idct_add(uint8_t *dest, int line_size,
1649                                     int16_t *block)
1650 {
1651     ff_mmx_idct(block);
1652     ff_add_pixels_clamped_mmx(block, dest, line_size);
1653 }
1654
1655 static void ff_libmpeg2mmx2_idct_put(uint8_t *dest, int line_size,
1656                                      int16_t *block)
1657 {
1658     ff_mmxext_idct(block);
1659     ff_put_pixels_clamped_mmx(block, dest, line_size);
1660 }
1661
1662 static void ff_libmpeg2mmx2_idct_add(uint8_t *dest, int line_size,
1663                                      int16_t *block)
1664 {
1665     ff_mmxext_idct(block);
1666     ff_add_pixels_clamped_mmx(block, dest, line_size);
1667 }
1668 #endif
1669
1670 #if HAVE_INLINE_ASM
1671 static void vector_clipf_sse(float *dst, const float *src,
1672                              float min, float max, int len)
1673 {
1674     x86_reg i = (len - 16) * 4;
1675     __asm__ volatile (
1676         "movss          %3, %%xmm4      \n\t"
1677         "movss          %4, %%xmm5      \n\t"
1678         "shufps $0, %%xmm4, %%xmm4      \n\t"
1679         "shufps $0, %%xmm5, %%xmm5      \n\t"
1680         "1:                             \n\t"
1681         "movaps   (%2, %0), %%xmm0      \n\t" // 3/1 on intel
1682         "movaps 16(%2, %0), %%xmm1      \n\t"
1683         "movaps 32(%2, %0), %%xmm2      \n\t"
1684         "movaps 48(%2, %0), %%xmm3      \n\t"
1685         "maxps      %%xmm4, %%xmm0      \n\t"
1686         "maxps      %%xmm4, %%xmm1      \n\t"
1687         "maxps      %%xmm4, %%xmm2      \n\t"
1688         "maxps      %%xmm4, %%xmm3      \n\t"
1689         "minps      %%xmm5, %%xmm0      \n\t"
1690         "minps      %%xmm5, %%xmm1      \n\t"
1691         "minps      %%xmm5, %%xmm2      \n\t"
1692         "minps      %%xmm5, %%xmm3      \n\t"
1693         "movaps     %%xmm0,   (%1, %0)  \n\t"
1694         "movaps     %%xmm1, 16(%1, %0)  \n\t"
1695         "movaps     %%xmm2, 32(%1, %0)  \n\t"
1696         "movaps     %%xmm3, 48(%1, %0)  \n\t"
1697         "sub           $64, %0          \n\t"
1698         "jge            1b              \n\t"
1699         : "+&r"(i)
1700         : "r"(dst), "r"(src), "m"(min), "m"(max)
1701         : "memory"
1702     );
1703 }
1704
1705 #endif /* HAVE_INLINE_ASM */
1706
1707 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
1708                                       int order);
1709 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
1710                                     int order);
1711 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
1712                                                const int16_t *v3,
1713                                                int order, int mul);
1714 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
1715                                              const int16_t *v3,
1716                                              int order, int mul);
1717 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
1718                                               const int16_t *v3,
1719                                               int order, int mul);
1720
1721 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
1722                                         const int16_t *window, unsigned int len);
1723 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
1724                                       const int16_t *window, unsigned int len);
1725 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
1726                                   const int16_t *window, unsigned int len);
1727 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
1728                                 const int16_t *window, unsigned int len);
1729 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
1730                                  const int16_t *window, unsigned int len);
1731 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
1732                                       const int16_t *window, unsigned int len);
1733
1734 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
1735 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
1736
1737 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
1738                                           const uint8_t *diff, int w,
1739                                           int *left, int *left_top);
1740 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
1741                                        int w, int left);
1742 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
1743                                       int w, int left);
1744
1745 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
1746                                    int32_t min, int32_t max, unsigned int len);
1747 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
1748                                    int32_t min, int32_t max, unsigned int len);
1749 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
1750                                    int32_t min, int32_t max, unsigned int len);
1751 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
1752                                    int32_t min, int32_t max, unsigned int len);
1753
1754 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
1755     do {                                                                     \
1756     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
1757     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
1758     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
1759     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
1760     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
1761     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
1762     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
1763     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
1764     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
1765     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
1766     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
1767     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
1768     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
1769     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
1770     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
1771     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
1772     } while (0)
1773
1774 #define SET_HPEL_FUNCS(PFX, IDX, SIZE, CPU)                                     \
1775     do {                                                                        \
1776         c->PFX ## _pixels_tab IDX [0] = PFX ## _pixels ## SIZE ## _     ## CPU; \
1777         c->PFX ## _pixels_tab IDX [1] = PFX ## _pixels ## SIZE ## _x2_  ## CPU; \
1778         c->PFX ## _pixels_tab IDX [2] = PFX ## _pixels ## SIZE ## _y2_  ## CPU; \
1779         c->PFX ## _pixels_tab IDX [3] = PFX ## _pixels ## SIZE ## _xy2_ ## CPU; \
1780     } while (0)
1781
1782 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
1783                                      int mm_flags)
1784 {
1785     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1786
1787 #if HAVE_INLINE_ASM
1788     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
1789     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
1790     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
1791
1792     if (!high_bit_depth) {
1793         c->clear_block  = clear_block_mmx;
1794         c->clear_blocks = clear_blocks_mmx;
1795         c->draw_edges   = draw_edges_mmx;
1796
1797         SET_HPEL_FUNCS(put,        [0], 16, mmx);
1798         SET_HPEL_FUNCS(put_no_rnd, [0], 16, mmx);
1799         SET_HPEL_FUNCS(avg,        [0], 16, mmx);
1800         SET_HPEL_FUNCS(avg_no_rnd,    , 16, mmx);
1801         SET_HPEL_FUNCS(put,        [1],  8, mmx);
1802         SET_HPEL_FUNCS(put_no_rnd, [1],  8, mmx);
1803         SET_HPEL_FUNCS(avg,        [1],  8, mmx);
1804     }
1805
1806 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
1807     c->gmc = gmc_mmx;
1808 #endif
1809
1810     c->add_bytes = add_bytes_mmx;
1811
1812     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
1813         c->h263_v_loop_filter = h263_v_loop_filter_mmx;
1814         c->h263_h_loop_filter = h263_h_loop_filter_mmx;
1815     }
1816 #endif /* HAVE_INLINE_ASM */
1817
1818 #if HAVE_YASM
1819     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
1820 #endif
1821
1822 }
1823
1824 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
1825                                         int mm_flags)
1826 {
1827     const int bit_depth      = avctx->bits_per_raw_sample;
1828     const int high_bit_depth = bit_depth > 8;
1829
1830 #if HAVE_YASM
1831     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
1832     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
1833
1834     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
1835     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
1836     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
1837     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
1838
1839     if (!high_bit_depth) {
1840         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_mmxext;
1841         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_mmxext;
1842
1843         c->avg_pixels_tab[0][0] = ff_avg_pixels16_mmxext;
1844         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_mmxext;
1845         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_mmxext;
1846
1847         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_mmxext;
1848         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_mmxext;
1849
1850         c->avg_pixels_tab[1][0] = ff_avg_pixels8_mmxext;
1851         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_mmxext;
1852         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_mmxext;
1853     }
1854
1855     if (!(avctx->flags & CODEC_FLAG_BITEXACT)) {
1856         if (!high_bit_depth) {
1857             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_mmxext;
1858             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_mmxext;
1859             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_mmxext;
1860             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_mmxext;
1861
1862             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_mmxext;
1863             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_mmxext;
1864         }
1865     }
1866 #endif /* HAVE_YASM */
1867
1868 #if HAVE_MMXEXT_EXTERNAL
1869     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1870                                avctx->codec_id == AV_CODEC_ID_THEORA)) {
1871         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_mmxext;
1872         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_mmxext;
1873     }
1874
1875     /* slower than cmov version on AMD */
1876     if (!(mm_flags & AV_CPU_FLAG_3DNOW))
1877         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
1878
1879     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
1880     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
1881
1882     if (avctx->flags & CODEC_FLAG_BITEXACT) {
1883         c->apply_window_int16 = ff_apply_window_int16_mmxext;
1884     } else {
1885         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
1886     }
1887 #endif /* HAVE_MMXEXT_EXTERNAL */
1888 }
1889
1890 static av_cold void dsputil_init_3dnow(DSPContext *c, AVCodecContext *avctx,
1891                                        int mm_flags)
1892 {
1893     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1894
1895 #if HAVE_YASM
1896     if (!high_bit_depth) {
1897         c->put_pixels_tab[0][1] = ff_put_pixels16_x2_3dnow;
1898         c->put_pixels_tab[0][2] = ff_put_pixels16_y2_3dnow;
1899
1900         c->avg_pixels_tab[0][0] = ff_avg_pixels16_3dnow;
1901         c->avg_pixels_tab[0][1] = ff_avg_pixels16_x2_3dnow;
1902         c->avg_pixels_tab[0][2] = ff_avg_pixels16_y2_3dnow;
1903
1904         c->put_pixels_tab[1][1] = ff_put_pixels8_x2_3dnow;
1905         c->put_pixels_tab[1][2] = ff_put_pixels8_y2_3dnow;
1906
1907         c->avg_pixels_tab[1][0] = ff_avg_pixels8_3dnow;
1908         c->avg_pixels_tab[1][1] = ff_avg_pixels8_x2_3dnow;
1909         c->avg_pixels_tab[1][2] = ff_avg_pixels8_y2_3dnow;
1910
1911         if (!(avctx->flags & CODEC_FLAG_BITEXACT)){
1912             c->put_no_rnd_pixels_tab[0][1] = ff_put_no_rnd_pixels16_x2_3dnow;
1913             c->put_no_rnd_pixels_tab[0][2] = ff_put_no_rnd_pixels16_y2_3dnow;
1914             c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_3dnow;
1915             c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_3dnow;
1916
1917             c->avg_pixels_tab[0][3] = ff_avg_pixels16_xy2_3dnow;
1918             c->avg_pixels_tab[1][3] = ff_avg_pixels8_xy2_3dnow;
1919         }
1920     }
1921
1922     if (CONFIG_VP3_DECODER && (avctx->codec_id == AV_CODEC_ID_VP3 ||
1923                                avctx->codec_id == AV_CODEC_ID_THEORA)) {
1924         c->put_no_rnd_pixels_tab[1][1] = ff_put_no_rnd_pixels8_x2_exact_3dnow;
1925         c->put_no_rnd_pixels_tab[1][2] = ff_put_no_rnd_pixels8_y2_exact_3dnow;
1926     }
1927 #endif /* HAVE_YASM */
1928 }
1929
1930 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
1931                                      int mm_flags)
1932 {
1933     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
1934
1935 #if HAVE_INLINE_ASM
1936     if (!high_bit_depth) {
1937         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
1938             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
1939             c->clear_block  = clear_block_sse;
1940             c->clear_blocks = clear_blocks_sse;
1941         }
1942     }
1943
1944     c->vector_clipf = vector_clipf_sse;
1945 #endif /* HAVE_INLINE_ASM */
1946
1947 #if HAVE_YASM
1948 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
1949     c->gmc = gmc_sse;
1950 #endif
1951 #endif /* HAVE_YASM */
1952 }
1953
1954 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
1955                                       int mm_flags)
1956 {
1957     const int bit_depth      = avctx->bits_per_raw_sample;
1958     const int high_bit_depth = bit_depth > 8;
1959
1960 #if HAVE_SSE2_INLINE
1961     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
1962         c->idct_put              = ff_idct_xvid_sse2_put;
1963         c->idct_add              = ff_idct_xvid_sse2_add;
1964         c->idct                  = ff_idct_xvid_sse2;
1965         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
1966     }
1967 #endif /* HAVE_SSE2_INLINE */
1968
1969 #if HAVE_SSE2_EXTERNAL
1970     if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
1971         // these functions are slower than mmx on AMD, but faster on Intel
1972         if (!high_bit_depth) {
1973             c->put_pixels_tab[0][0]        = ff_put_pixels16_sse2;
1974             c->put_no_rnd_pixels_tab[0][0] = ff_put_pixels16_sse2;
1975             c->avg_pixels_tab[0][0]        = ff_avg_pixels16_sse2;
1976         }
1977     }
1978
1979     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
1980     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
1981     if (mm_flags & AV_CPU_FLAG_ATOM) {
1982         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
1983     } else {
1984         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
1985     }
1986     if (avctx->flags & CODEC_FLAG_BITEXACT) {
1987         c->apply_window_int16 = ff_apply_window_int16_sse2;
1988     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
1989         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
1990     }
1991     c->bswap_buf = ff_bswap32_buf_sse2;
1992 #endif /* HAVE_SSE2_EXTERNAL */
1993 }
1994
1995 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
1996                                        int mm_flags)
1997 {
1998 #if HAVE_SSSE3_EXTERNAL
1999     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
2000     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
2001         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
2002
2003     if (mm_flags & AV_CPU_FLAG_ATOM)
2004         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
2005     else
2006         c->apply_window_int16 = ff_apply_window_int16_ssse3;
2007     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
2008         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
2009     c->bswap_buf = ff_bswap32_buf_ssse3;
2010 #endif /* HAVE_SSSE3_EXTERNAL */
2011 }
2012
2013 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
2014                                       int mm_flags)
2015 {
2016 #if HAVE_SSE4_EXTERNAL
2017     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
2018 #endif /* HAVE_SSE4_EXTERNAL */
2019 }
2020
2021 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
2022 {
2023     int mm_flags = av_get_cpu_flags();
2024
2025 #if HAVE_7REGS && HAVE_INLINE_ASM
2026     if (mm_flags & AV_CPU_FLAG_CMOV)
2027         c->add_hfyu_median_prediction = add_hfyu_median_prediction_cmov;
2028 #endif
2029
2030     if (mm_flags & AV_CPU_FLAG_MMX) {
2031 #if HAVE_INLINE_ASM
2032         const int idct_algo = avctx->idct_algo;
2033
2034         if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
2035             if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
2036                 c->idct_put              = ff_simple_idct_put_mmx;
2037                 c->idct_add              = ff_simple_idct_add_mmx;
2038                 c->idct                  = ff_simple_idct_mmx;
2039                 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
2040 #if CONFIG_GPL
2041             } else if (idct_algo == FF_IDCT_LIBMPEG2MMX) {
2042                 if (mm_flags & AV_CPU_FLAG_MMX2) {
2043                     c->idct_put = ff_libmpeg2mmx2_idct_put;
2044                     c->idct_add = ff_libmpeg2mmx2_idct_add;
2045                     c->idct     = ff_mmxext_idct;
2046                 } else {
2047                     c->idct_put = ff_libmpeg2mmx_idct_put;
2048                     c->idct_add = ff_libmpeg2mmx_idct_add;
2049                     c->idct     = ff_mmx_idct;
2050                 }
2051                 c->idct_permutation_type = FF_LIBMPEG2_IDCT_PERM;
2052 #endif
2053             } else if (idct_algo == FF_IDCT_XVIDMMX) {
2054                 if (mm_flags & AV_CPU_FLAG_SSE2) {
2055                     c->idct_put              = ff_idct_xvid_sse2_put;
2056                     c->idct_add              = ff_idct_xvid_sse2_add;
2057                     c->idct                  = ff_idct_xvid_sse2;
2058                     c->idct_permutation_type = FF_SSE2_IDCT_PERM;
2059                 } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
2060                     c->idct_put              = ff_idct_xvid_mmxext_put;
2061                     c->idct_add              = ff_idct_xvid_mmxext_add;
2062                     c->idct                  = ff_idct_xvid_mmxext;
2063                 } else {
2064                     c->idct_put              = ff_idct_xvid_mmx_put;
2065                     c->idct_add              = ff_idct_xvid_mmx_add;
2066                     c->idct                  = ff_idct_xvid_mmx;
2067                 }
2068             }
2069         }
2070 #endif /* HAVE_INLINE_ASM */
2071
2072         dsputil_init_mmx(c, avctx, mm_flags);
2073     }
2074
2075     if (mm_flags & AV_CPU_FLAG_MMXEXT)
2076         dsputil_init_mmxext(c, avctx, mm_flags);
2077
2078     if (mm_flags & AV_CPU_FLAG_3DNOW)
2079         dsputil_init_3dnow(c, avctx, mm_flags);
2080
2081     if (mm_flags & AV_CPU_FLAG_SSE)
2082         dsputil_init_sse(c, avctx, mm_flags);
2083
2084     if (mm_flags & AV_CPU_FLAG_SSE2)
2085         dsputil_init_sse2(c, avctx, mm_flags);
2086
2087     if (mm_flags & AV_CPU_FLAG_SSSE3)
2088         dsputil_init_ssse3(c, avctx, mm_flags);
2089
2090     if (mm_flags & AV_CPU_FLAG_SSE4)
2091         dsputil_init_sse4(c, avctx, mm_flags);
2092
2093     if (CONFIG_ENCODERS)
2094         ff_dsputilenc_init_mmx(c, avctx);
2095 }