git.sesse.net Git - ffmpeg/blob - libavcodec/x86/dsputil_init.c

   1 /*
   2  * This file is part of Libav.
   3  *
   4  * Libav is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * Libav is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with Libav; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  17  */
  18
  19 #include "config.h"
  20 #include "libavutil/attributes.h"
  21 #include "libavutil/cpu.h"
  22 #include "libavutil/internal.h"
  23 #include "libavutil/x86/asm.h"
  24 #include "libavutil/x86/cpu.h"
  25 #include "libavcodec/avcodec.h"
  26 #include "libavcodec/dsputil.h"
  27 #include "libavcodec/simple_idct.h"
  28 #include "libavcodec/version.h"
  29 #include "dsputil_x86.h"
  30 #include "idct_xvid.h"
  31
  32 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  33                               int dstStride, int src1Stride, int h);
  34 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
  35                                      uint8_t *src2, int dstStride,
  36                                      int src1Stride, int h);
  37 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  38                               int dstStride, int src1Stride, int h);
  39 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  40                                int dstStride, int src1Stride, int h);
  41 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  42                                int dstStride, int src1Stride, int h);
  43 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  44                                       int dstStride, int src1Stride, int h);
  45 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  46                                           int dstStride, int srcStride, int h);
  47 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  48                                           int dstStride, int srcStride, int h);
  49 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  50                                                  int dstStride, int srcStride,
  51                                                  int h);
  52 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  53                                          int dstStride, int srcStride, int h);
  54 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  55                                          int dstStride, int srcStride, int h);
  56 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  57                                                 int dstStride, int srcStride,
  58                                                 int h);
  59 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  60                                           int dstStride, int srcStride);
  61 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  62                                           int dstStride, int srcStride);
  63 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  64                                                  int dstStride, int srcStride);
  65 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  66                                          int dstStride, int srcStride);
  67 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  68                                          int dstStride, int srcStride);
  69 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  70                                                 int dstStride, int srcStride);
  71 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
  72 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
  73
  74 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
  75                                       int order);
  76 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
  77                                     int order);
  78 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
  79                                                const int16_t *v3,
  80                                                int order, int mul);
  81 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
  82                                              const int16_t *v3,
  83                                              int order, int mul);
  84 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
  85                                               const int16_t *v3,
  86                                               int order, int mul);
  87
  88 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
  89 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
  90
  91 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
  92                                           const uint8_t *diff, int w,
  93                                           int *left, int *left_top);
  94 int ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
  95                                       int w, int left);
  96 int ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
  97                                      int w, int left);
  98
  99 void ff_vector_clip_int32_mmx(int32_t *dst, const int32_t *src,
 100                               int32_t min, int32_t max, unsigned int len);
 101 void ff_vector_clip_int32_sse2(int32_t *dst, const int32_t *src,
 102                                int32_t min, int32_t max, unsigned int len);
 103 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
 104                                    int32_t min, int32_t max, unsigned int len);
 105 void ff_vector_clip_int32_sse4(int32_t *dst, const int32_t *src,
 106                                int32_t min, int32_t max, unsigned int len);
 107
 108 #if HAVE_YASM
 109
 110 PIXELS16(static, ff_avg, , , _mmxext)
 111 PIXELS16(static, ff_put, , , _mmxext)
 112
 113 #define QPEL_OP(OPNAME, RND, MMX)                                       \
 114 static void OPNAME ## qpel8_mc00_ ## MMX(uint8_t *dst, uint8_t *src,    \
 115                                          ptrdiff_t stride)              \
 116 {                                                                       \
 117     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
 118 }                                                                       \
 119                                                                         \
 120 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
 121                                          ptrdiff_t stride)              \
 122 {                                                                       \
 123     uint64_t temp[8];                                                   \
 124     uint8_t *const half = (uint8_t *) temp;                             \
 125     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 126                                                    stride, 8);          \
 127     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 128                                         stride, stride, 8);             \
 129 }                                                                       \
 130                                                                         \
 131 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
 132                                          ptrdiff_t stride)              \
 133 {                                                                       \
 134     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
 135                                                    stride, 8);          \
 136 }                                                                       \
 137                                                                         \
 138 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
 139                                          ptrdiff_t stride)              \
 140 {                                                                       \
 141     uint64_t temp[8];                                                   \
 142     uint8_t *const half = (uint8_t *) temp;                             \
 143     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 144                                                    stride, 8);          \
 145     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
 146                                         stride, 8);                     \
 147 }                                                                       \
 148                                                                         \
 149 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
 150                                          ptrdiff_t stride)              \
 151 {                                                                       \
 152     uint64_t temp[8];                                                   \
 153     uint8_t *const half = (uint8_t *) temp;                             \
 154     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 155                                                    8, stride);          \
 156     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 157                                         stride, stride, 8);             \
 158 }                                                                       \
 159                                                                         \
 160 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
 161                                          ptrdiff_t stride)              \
 162 {                                                                       \
 163     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
 164                                                    stride, stride);     \
 165 }                                                                       \
 166                                                                         \
 167 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
 168                                          ptrdiff_t stride)              \
 169 {                                                                       \
 170     uint64_t temp[8];                                                   \
 171     uint8_t *const half = (uint8_t *) temp;                             \
 172     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 173                                                    8, stride);          \
 174     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
 175                                         stride, 8);                     \
 176 }                                                                       \
 177                                                                         \
 178 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
 179                                          ptrdiff_t stride)              \
 180 {                                                                       \
 181     uint64_t half[8 + 9];                                               \
 182     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 183     uint8_t *const halfHV = ((uint8_t *) half);                         \
 184     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 185                                                    stride, 9);          \
 186     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 187                                         stride, 9);                     \
 188     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 189     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 190                                         stride, 8, 8);                  \
 191 }                                                                       \
 192                                                                         \
 193 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
 194                                          ptrdiff_t stride)              \
 195 {                                                                       \
 196     uint64_t half[8 + 9];                                               \
 197     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 198     uint8_t *const halfHV = ((uint8_t *) half);                         \
 199     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 200                                                    stride, 9);          \
 201     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 202                                         stride, 9);                     \
 203     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 204     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 205                                         stride, 8, 8);                  \
 206 }                                                                       \
 207                                                                         \
 208 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
 209                                          ptrdiff_t stride)              \
 210 {                                                                       \
 211     uint64_t half[8 + 9];                                               \
 212     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 213     uint8_t *const halfHV = ((uint8_t *) half);                         \
 214     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 215                                                    stride, 9);          \
 216     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 217                                         stride, 9);                     \
 218     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 219     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 220                                         stride, 8, 8);                  \
 221 }                                                                       \
 222                                                                         \
 223 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
 224                                          ptrdiff_t stride)              \
 225 {                                                                       \
 226     uint64_t half[8 + 9];                                               \
 227     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 228     uint8_t *const halfHV = ((uint8_t *) half);                         \
 229     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 230                                                    stride, 9);          \
 231     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 232                                         stride, 9);                     \
 233     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 234     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 235                                         stride, 8, 8);                  \
 236 }                                                                       \
 237                                                                         \
 238 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
 239                                          ptrdiff_t stride)              \
 240 {                                                                       \
 241     uint64_t half[8 + 9];                                               \
 242     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 243     uint8_t *const halfHV = ((uint8_t *) half);                         \
 244     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 245                                                    stride, 9);          \
 246     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 247     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 248                                         stride, 8, 8);                  \
 249 }                                                                       \
 250                                                                         \
 251 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
 252                                          ptrdiff_t stride)              \
 253 {                                                                       \
 254     uint64_t half[8 + 9];                                               \
 255     uint8_t *const halfH  = ((uint8_t *) half) + 64;                    \
 256     uint8_t *const halfHV = ((uint8_t *) half);                         \
 257     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 258                                                    stride, 9);          \
 259     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 260     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 261                                         stride, 8, 8);                  \
 262 }                                                                       \
 263                                                                         \
 264 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
 265                                          ptrdiff_t stride)              \
 266 {                                                                       \
 267     uint64_t half[8 + 9];                                               \
 268     uint8_t *const halfH = ((uint8_t *) half);                          \
 269     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 270                                                    stride, 9);          \
 271     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
 272                                         8, stride, 9);                  \
 273     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 274                                                    stride, 8);          \
 275 }                                                                       \
 276                                                                         \
 277 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
 278                                          ptrdiff_t stride)              \
 279 {                                                                       \
 280     uint64_t half[8 + 9];                                               \
 281     uint8_t *const halfH = ((uint8_t *) half);                          \
 282     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 283                                                    stride, 9);          \
 284     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 285                                         stride, 9);                     \
 286     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 287                                                    stride, 8);          \
 288 }                                                                       \
 289                                                                         \
 290 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
 291                                          ptrdiff_t stride)              \
 292 {                                                                       \
 293     uint64_t half[9];                                                   \
 294     uint8_t *const halfH = ((uint8_t *) half);                          \
 295     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 296                                                    stride, 9);          \
 297     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 298                                                    stride, 8);          \
 299 }                                                                       \
 300                                                                         \
 301 static void OPNAME ## qpel16_mc00_ ## MMX(uint8_t *dst, uint8_t *src,   \
 302                                           ptrdiff_t stride)             \
 303 {                                                                       \
 304     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
 305 }                                                                       \
 306                                                                         \
 307 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
 308                                           ptrdiff_t stride)             \
 309 {                                                                       \
 310     uint64_t temp[32];                                                  \
 311     uint8_t *const half = (uint8_t *) temp;                             \
 312     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 313                                                     stride, 16);        \
 314     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 315                                          stride, 16);                   \
 316 }                                                                       \
 317                                                                         \
 318 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
 319                                           ptrdiff_t stride)             \
 320 {                                                                       \
 321     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
 322                                                     stride, stride, 16);\
 323 }                                                                       \
 324                                                                         \
 325 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
 326                                           ptrdiff_t stride)             \
 327 {                                                                       \
 328     uint64_t temp[32];                                                  \
 329     uint8_t *const half = (uint8_t*) temp;                              \
 330     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 331                                                     stride, 16);        \
 332     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
 333                                          stride, stride, 16);           \
 334 }                                                                       \
 335                                                                         \
 336 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
 337                                           ptrdiff_t stride)             \
 338 {                                                                       \
 339     uint64_t temp[32];                                                  \
 340     uint8_t *const half = (uint8_t *) temp;                             \
 341     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 342                                                     stride);            \
 343     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 344                                          stride, 16);                   \
 345 }                                                                       \
 346                                                                         \
 347 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
 348                                           ptrdiff_t stride)             \
 349 {                                                                       \
 350     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
 351                                                     stride, stride);    \
 352 }                                                                       \
 353                                                                         \
 354 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
 355                                           ptrdiff_t stride)             \
 356 {                                                                       \
 357     uint64_t temp[32];                                                  \
 358     uint8_t *const half = (uint8_t *) temp;                             \
 359     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 360                                                     stride);            \
 361     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
 362                                          stride, stride, 16);           \
 363 }                                                                       \
 364                                                                         \
 365 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
 366                                           ptrdiff_t stride)             \
 367 {                                                                       \
 368     uint64_t half[16 * 2 + 17 * 2];                                     \
 369     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 370     uint8_t *const halfHV = ((uint8_t *) half);                         \
 371     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 372                                                     stride, 17);        \
 373     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 374                                          stride, 17);                   \
 375     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 376                                                     16, 16);            \
 377     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 378                                          stride, 16, 16);               \
 379 }                                                                       \
 380                                                                         \
 381 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
 382                                           ptrdiff_t stride)             \
 383 {                                                                       \
 384     uint64_t half[16 * 2 + 17 * 2];                                     \
 385     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 386     uint8_t *const halfHV = ((uint8_t *) half);                         \
 387     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 388                                                     stride, 17);        \
 389     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 390                                          stride, 17);                   \
 391     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 392                                                     16, 16);            \
 393     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 394                                          stride, 16, 16);               \
 395 }                                                                       \
 396                                                                         \
 397 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
 398                                           ptrdiff_t stride)             \
 399 {                                                                       \
 400     uint64_t half[16 * 2 + 17 * 2];                                     \
 401     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 402     uint8_t *const halfHV = ((uint8_t *) half);                         \
 403     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 404                                                     stride, 17);        \
 405     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 406                                          stride, 17);                   \
 407     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 408                                                     16, 16);            \
 409     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 410                                          stride, 16, 16);               \
 411 }                                                                       \
 412                                                                         \
 413 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
 414                                           ptrdiff_t stride)             \
 415 {                                                                       \
 416     uint64_t half[16 * 2 + 17 * 2];                                     \
 417     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 418     uint8_t *const halfHV = ((uint8_t *) half);                         \
 419     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 420                                                     stride, 17);        \
 421     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 422                                          stride, 17);                   \
 423     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 424                                                     16, 16);            \
 425     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 426                                          stride, 16, 16);               \
 427 }                                                                       \
 428                                                                         \
 429 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
 430                                           ptrdiff_t stride)             \
 431 {                                                                       \
 432     uint64_t half[16 * 2 + 17 * 2];                                     \
 433     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 434     uint8_t *const halfHV = ((uint8_t *) half);                         \
 435     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 436                                                     stride, 17);        \
 437     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 438                                                     16, 16);            \
 439     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 440                                          stride, 16, 16);               \
 441 }                                                                       \
 442                                                                         \
 443 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
 444                                           ptrdiff_t stride)             \
 445 {                                                                       \
 446     uint64_t half[16 * 2 + 17 * 2];                                     \
 447     uint8_t *const halfH  = ((uint8_t *) half) + 256;                   \
 448     uint8_t *const halfHV = ((uint8_t *) half);                         \
 449     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 450                                                     stride, 17);        \
 451     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 452                                                     16, 16);            \
 453     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 454                                          stride, 16, 16);               \
 455 }                                                                       \
 456                                                                         \
 457 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
 458                                           ptrdiff_t stride)             \
 459 {                                                                       \
 460     uint64_t half[17 * 2];                                              \
 461     uint8_t *const halfH = ((uint8_t *) half);                          \
 462     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 463                                                     stride, 17);        \
 464     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 465                                          stride, 17);                   \
 466     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 467                                                     stride, 16);        \
 468 }                                                                       \
 469                                                                         \
 470 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
 471                                           ptrdiff_t stride)             \
 472 {                                                                       \
 473     uint64_t half[17 * 2];                                              \
 474     uint8_t *const halfH = ((uint8_t *) half);                          \
 475     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 476                                                     stride, 17);        \
 477     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 478                                          stride, 17);                   \
 479     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 480                                                     stride, 16);        \
 481 }                                                                       \
 482                                                                         \
 483 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
 484                                           ptrdiff_t stride)             \
 485 {                                                                       \
 486     uint64_t half[17 * 2];                                              \
 487     uint8_t *const halfH = ((uint8_t *) half);                          \
 488     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 489                                                     stride, 17);        \
 490     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 491                                                     stride, 16);        \
 492 }
 493
 494 QPEL_OP(put_,        _,        mmxext)
 495 QPEL_OP(avg_,        _,        mmxext)
 496 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
 497
 498 #endif /* HAVE_YASM */
 499
 500 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
 501 do {                                                                         \
 502     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
 503     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
 504     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
 505     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
 506     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
 507     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
 508     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
 509     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
 510     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
 511     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
 512     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
 513     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
 514     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
 515     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
 516     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
 517     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
 518 } while (0)
 519
 520 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
 521                                      int cpu_flags)
 522 {
 523 #if HAVE_MMX_INLINE
 524     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 525
 526     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
 527     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
 528     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
 529
 530     if (!high_bit_depth) {
 531         c->clear_block  = ff_clear_block_mmx;
 532         c->clear_blocks = ff_clear_blocks_mmx;
 533         c->draw_edges   = ff_draw_edges_mmx;
 534
 535         switch (avctx->idct_algo) {
 536         case FF_IDCT_AUTO:
 537         case FF_IDCT_SIMPLEMMX:
 538             c->idct_put              = ff_simple_idct_put_mmx;
 539             c->idct_add              = ff_simple_idct_add_mmx;
 540             c->idct                  = ff_simple_idct_mmx;
 541             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
 542             break;
 543         case FF_IDCT_XVIDMMX:
 544             c->idct_put              = ff_idct_xvid_mmx_put;
 545             c->idct_add              = ff_idct_xvid_mmx_add;
 546             c->idct                  = ff_idct_xvid_mmx;
 547             break;
 548         }
 549     }
 550
 551     c->gmc = ff_gmc_mmx;
 552
 553     c->add_bytes = ff_add_bytes_mmx;
 554 #endif /* HAVE_MMX_INLINE */
 555
 556 #if HAVE_MMX_EXTERNAL
 557     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
 558 #endif /* HAVE_MMX_EXTERNAL */
 559 }
 560
 561 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
 562                                         int cpu_flags)
 563 {
 564 #if HAVE_MMXEXT_INLINE
 565     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 566
 567     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
 568         c->idct_put = ff_idct_xvid_mmxext_put;
 569         c->idct_add = ff_idct_xvid_mmxext_add;
 570         c->idct     = ff_idct_xvid_mmxext;
 571     }
 572 #endif /* HAVE_MMXEXT_INLINE */
 573
 574 #if HAVE_MMXEXT_EXTERNAL
 575     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
 576     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
 577
 578     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
 579     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
 580     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
 581     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
 582
 583     /* slower than cmov version on AMD */
 584     if (!(cpu_flags & AV_CPU_FLAG_3DNOW))
 585         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
 586
 587     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
 588     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
 589 #endif /* HAVE_MMXEXT_EXTERNAL */
 590 }
 591
 592 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
 593                                      int cpu_flags)
 594 {
 595 #if HAVE_SSE_INLINE
 596     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 597
 598     c->vector_clipf = ff_vector_clipf_sse;
 599
 600 #if FF_API_XVMC
 601 FF_DISABLE_DEPRECATION_WARNINGS
 602     /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
 603     if (CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)
 604         return;
 605 FF_ENABLE_DEPRECATION_WARNINGS
 606 #endif /* FF_API_XVMC */
 607
 608     if (!high_bit_depth) {
 609         c->clear_block  = ff_clear_block_sse;
 610         c->clear_blocks = ff_clear_blocks_sse;
 611     }
 612 #endif /* HAVE_SSE_INLINE */
 613 }
 614
 615 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
 616                                       int cpu_flags)
 617 {
 618 #if HAVE_SSE2_INLINE
 619     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 620
 621     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
 622         c->idct_put              = ff_idct_xvid_sse2_put;
 623         c->idct_add              = ff_idct_xvid_sse2_add;
 624         c->idct                  = ff_idct_xvid_sse2;
 625         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
 626     }
 627 #endif /* HAVE_SSE2_INLINE */
 628
 629 #if HAVE_SSE2_EXTERNAL
 630     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
 631     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
 632     if (cpu_flags & AV_CPU_FLAG_ATOM) {
 633         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
 634     } else {
 635         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
 636     }
 637     c->bswap_buf = ff_bswap32_buf_sse2;
 638 #endif /* HAVE_SSE2_EXTERNAL */
 639 }
 640
 641 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
 642                                        int cpu_flags)
 643 {
 644 #if HAVE_SSSE3_EXTERNAL
 645     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
 646     if (cpu_flags & AV_CPU_FLAG_SSE4) // not really SSE4, just slow on Conroe
 647         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
 648
 649     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
 650         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
 651     c->bswap_buf = ff_bswap32_buf_ssse3;
 652 #endif /* HAVE_SSSE3_EXTERNAL */
 653 }
 654
 655 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
 656                                       int cpu_flags)
 657 {
 658 #if HAVE_SSE4_EXTERNAL
 659     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
 660 #endif /* HAVE_SSE4_EXTERNAL */
 661 }
 662
 663 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx)
 664 {
 665     int cpu_flags = av_get_cpu_flags();
 666
 667 #if HAVE_7REGS && HAVE_INLINE_ASM
 668     if (cpu_flags & AV_CPU_FLAG_CMOV)
 669         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
 670 #endif
 671
 672     if (X86_MMX(cpu_flags))
 673         dsputil_init_mmx(c, avctx, cpu_flags);
 674
 675     if (X86_MMXEXT(cpu_flags))
 676         dsputil_init_mmxext(c, avctx, cpu_flags);
 677
 678     if (X86_SSE(cpu_flags))
 679         dsputil_init_sse(c, avctx, cpu_flags);
 680
 681     if (X86_SSE2(cpu_flags))
 682         dsputil_init_sse2(c, avctx, cpu_flags);
 683
 684     if (EXTERNAL_SSSE3(cpu_flags))
 685         dsputil_init_ssse3(c, avctx, cpu_flags);
 686
 687     if (EXTERNAL_SSE4(cpu_flags))
 688         dsputil_init_sse4(c, avctx, cpu_flags);
 689
 690     if (CONFIG_ENCODERS)
 691         ff_dsputilenc_init_mmx(c, avctx);
 692 }