git.sesse.net Git - ffmpeg/blob - libavcodec/x86/dsputil_init.c

   1 /*
   2  * Copyright (c) 2000, 2001 Fabrice Bellard
   3  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   4  *
   5  * This file is part of FFmpeg.
   6  *
   7  * FFmpeg is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * FFmpeg is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with FFmpeg; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include "config.h"
  23 #include "libavutil/attributes.h"
  24 #include "libavutil/cpu.h"
  25 #include "libavutil/internal.h"
  26 #include "libavutil/x86/asm.h"
  27 #include "libavutil/x86/cpu.h"
  28 #include "libavcodec/dsputil.h"
  29 #include "libavcodec/simple_idct.h"
  30 #include "dsputil_x86.h"
  31 #include "idct_xvid.h"
  32
  33 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  34                               int dstStride, int src1Stride, int h);
  35 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
  36                                      uint8_t *src2, int dstStride,
  37                                      int src1Stride, int h);
  38 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  39                               int dstStride, int src1Stride, int h);
  40 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  41                                int dstStride, int src1Stride, int h);
  42 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  43                                int dstStride, int src1Stride, int h);
  44 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  45                                       int dstStride, int src1Stride, int h);
  46 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  47                                          int dstStride, int srcStride, int h);
  48 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  49                                          int dstStride, int srcStride, int h);
  50 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  51                                                  int dstStride, int srcStride,
  52                                                  int h);
  53 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  54                                         int dstStride, int srcStride, int h);
  55 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  56                                         int dstStride, int srcStride, int h);
  57 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  58                                                 int dstStride, int srcStride,
  59                                                 int h);
  60 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  61                                          int dstStride, int srcStride);
  62 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  63                                          int dstStride, int srcStride);
  64 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  65                                                  int dstStride, int srcStride);
  66 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  67                                         int dstStride, int srcStride);
  68 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  69                                         int dstStride, int srcStride);
  70 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  71                                                 int dstStride, int srcStride);
  72 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
  73 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
  74
  75 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
  76                                       int order);
  77 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
  78                                     int order);
  79 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
  80                                                const int16_t *v3,
  81                                                int order, int mul);
  82 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
  83                                              const int16_t *v3,
  84                                              int order, int mul);
  85 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
  86                                               const int16_t *v3,
  87                                               int order, int mul);
  88
  89 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
  90                                         const int16_t *window, unsigned int len);
  91 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
  92                                       const int16_t *window, unsigned int len);
  93 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
  94                                   const int16_t *window, unsigned int len);
  95 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
  96                                 const int16_t *window, unsigned int len);
  97 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
  98                                  const int16_t *window, unsigned int len);
  99 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
 100                                       const int16_t *window, unsigned int len);
 101
 102 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
 103 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
 104
 105 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
 106                                           const uint8_t *diff, int w,
 107                                           int *left, int *left_top);
 108 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
 109                                        int w, int left);
 110 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
 111                                       int w, int left);
 112
 113 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
 114                                    int32_t min, int32_t max, unsigned int len);
 115 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
 116                                    int32_t min, int32_t max, unsigned int len);
 117 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
 118                                    int32_t min, int32_t max, unsigned int len);
 119 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
 120                                    int32_t min, int32_t max, unsigned int len);
 121
 122 #if HAVE_YASM
 123
 124 PIXELS16(static, ff_avg, , , _mmxext)
 125 PIXELS16(static, ff_put, , , _mmxext)
 126
 127 #define QPEL_OP(OPNAME, RND, MMX)                                       \
 128 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
 129                                           ptrdiff_t stride)             \
 130 {                                                                       \
 131     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
 132 }                                                                       \
 133                                                                         \
 134 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
 135                                          ptrdiff_t stride)              \
 136 {                                                                       \
 137     uint64_t temp[8];                                                   \
 138     uint8_t * const half = (uint8_t*)temp;                              \
 139     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 140                                                    stride, 8);          \
 141     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 142                                         stride, stride, 8);             \
 143 }                                                                       \
 144                                                                         \
 145 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
 146                                          ptrdiff_t stride)              \
 147 {                                                                       \
 148     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
 149                                                    stride, 8);          \
 150 }                                                                       \
 151                                                                         \
 152 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
 153                                          ptrdiff_t stride)              \
 154 {                                                                       \
 155     uint64_t temp[8];                                                   \
 156     uint8_t * const half = (uint8_t*)temp;                              \
 157     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 158                                                    stride, 8);          \
 159     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
 160                                         stride, 8);                     \
 161 }                                                                       \
 162                                                                         \
 163 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
 164                                          ptrdiff_t stride)              \
 165 {                                                                       \
 166     uint64_t temp[8];                                                   \
 167     uint8_t * const half = (uint8_t*)temp;                              \
 168     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 169                                                    8, stride);          \
 170     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 171                                         stride, stride, 8);             \
 172 }                                                                       \
 173                                                                         \
 174 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
 175                                          ptrdiff_t stride)              \
 176 {                                                                       \
 177     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
 178                                                    stride, stride);     \
 179 }                                                                       \
 180                                                                         \
 181 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
 182                                          ptrdiff_t stride)              \
 183 {                                                                       \
 184     uint64_t temp[8];                                                   \
 185     uint8_t * const half = (uint8_t*)temp;                              \
 186     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 187                                                    8, stride);          \
 188     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
 189                                         stride, 8);                     \
 190 }                                                                       \
 191                                                                         \
 192 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
 193                                          ptrdiff_t stride)              \
 194 {                                                                       \
 195     uint64_t half[8 + 9];                                               \
 196     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 197     uint8_t * const halfHV = ((uint8_t*)half);                          \
 198     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 199                                                    stride, 9);          \
 200     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 201                                         stride, 9);                     \
 202     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 203     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 204                                         stride, 8, 8);                  \
 205 }                                                                       \
 206                                                                         \
 207 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
 208                                          ptrdiff_t stride)              \
 209 {                                                                       \
 210     uint64_t half[8 + 9];                                               \
 211     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 212     uint8_t * const halfHV = ((uint8_t*)half);                          \
 213     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 214                                                    stride, 9);          \
 215     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 216                                         stride, 9);                     \
 217     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 218     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 219                                         stride, 8, 8);                  \
 220 }                                                                       \
 221                                                                         \
 222 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
 223                                          ptrdiff_t stride)              \
 224 {                                                                       \
 225     uint64_t half[8 + 9];                                               \
 226     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 227     uint8_t * const halfHV = ((uint8_t*)half);                          \
 228     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 229                                                    stride, 9);          \
 230     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 231                                         stride, 9);                     \
 232     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 233     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 234                                         stride, 8, 8);                  \
 235 }                                                                       \
 236                                                                         \
 237 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
 238                                          ptrdiff_t stride)              \
 239 {                                                                       \
 240     uint64_t half[8 + 9];                                               \
 241     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 242     uint8_t * const halfHV = ((uint8_t*)half);                          \
 243     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 244                                                    stride, 9);          \
 245     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 246                                         stride, 9);                     \
 247     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 248     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 249                                         stride, 8, 8);                  \
 250 }                                                                       \
 251                                                                         \
 252 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
 253                                          ptrdiff_t stride)              \
 254 {                                                                       \
 255     uint64_t half[8 + 9];                                               \
 256     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 257     uint8_t * const halfHV = ((uint8_t*)half);                          \
 258     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 259                                                    stride, 9);          \
 260     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 261     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 262                                         stride, 8, 8);                  \
 263 }                                                                       \
 264                                                                         \
 265 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
 266                                          ptrdiff_t stride)              \
 267 {                                                                       \
 268     uint64_t half[8 + 9];                                               \
 269     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 270     uint8_t * const halfHV = ((uint8_t*)half);                          \
 271     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 272                                                    stride, 9);          \
 273     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 274     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 275                                         stride, 8, 8);                  \
 276 }                                                                       \
 277                                                                         \
 278 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
 279                                          ptrdiff_t stride)              \
 280 {                                                                       \
 281     uint64_t half[8 + 9];                                               \
 282     uint8_t * const halfH = ((uint8_t*)half);                           \
 283     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 284                                                    stride, 9);          \
 285     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
 286                                         8, stride, 9);                  \
 287     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 288                                                    stride, 8);          \
 289 }                                                                       \
 290                                                                         \
 291 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
 292                                          ptrdiff_t stride)              \
 293 {                                                                       \
 294     uint64_t half[8 + 9];                                               \
 295     uint8_t * const halfH = ((uint8_t*)half);                           \
 296     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 297                                                    stride, 9);          \
 298     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 299                                         stride, 9);                     \
 300     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 301                                                    stride, 8);          \
 302 }                                                                       \
 303                                                                         \
 304 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
 305                                          ptrdiff_t stride)              \
 306 {                                                                       \
 307     uint64_t half[9];                                                   \
 308     uint8_t * const halfH = ((uint8_t*)half);                           \
 309     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 310                                                    stride, 9);          \
 311     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 312                                                    stride, 8);          \
 313 }                                                                       \
 314                                                                         \
 315 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
 316                                            ptrdiff_t stride)            \
 317 {                                                                       \
 318     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
 319 }                                                                       \
 320                                                                         \
 321 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
 322                                           ptrdiff_t stride)             \
 323 {                                                                       \
 324     uint64_t temp[32];                                                  \
 325     uint8_t * const half = (uint8_t*)temp;                              \
 326     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 327                                                     stride, 16);        \
 328     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 329                                          stride, 16);                   \
 330 }                                                                       \
 331                                                                         \
 332 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
 333                                           ptrdiff_t stride)             \
 334 {                                                                       \
 335     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
 336                                                     stride, stride, 16);\
 337 }                                                                       \
 338                                                                         \
 339 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
 340                                           ptrdiff_t stride)             \
 341 {                                                                       \
 342     uint64_t temp[32];                                                  \
 343     uint8_t * const half = (uint8_t*)temp;                              \
 344     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 345                                                     stride, 16);        \
 346     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
 347                                          stride, stride, 16);           \
 348 }                                                                       \
 349                                                                         \
 350 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
 351                                           ptrdiff_t stride)             \
 352 {                                                                       \
 353     uint64_t temp[32];                                                  \
 354     uint8_t * const half = (uint8_t*)temp;                              \
 355     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 356                                                     stride);            \
 357     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 358                                          stride, 16);                   \
 359 }                                                                       \
 360                                                                         \
 361 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
 362                                           ptrdiff_t stride)             \
 363 {                                                                       \
 364     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
 365                                                     stride, stride);    \
 366 }                                                                       \
 367                                                                         \
 368 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
 369                                           ptrdiff_t stride)             \
 370 {                                                                       \
 371     uint64_t temp[32];                                                  \
 372     uint8_t * const half = (uint8_t*)temp;                              \
 373     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 374                                                     stride);            \
 375     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
 376                                          stride, stride, 16);           \
 377 }                                                                       \
 378                                                                         \
 379 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
 380                                           ptrdiff_t stride)             \
 381 {                                                                       \
 382     uint64_t half[16 * 2 + 17 * 2];                                     \
 383     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 384     uint8_t * const halfHV = ((uint8_t*)half);                          \
 385     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 386                                                     stride, 17);        \
 387     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 388                                          stride, 17);                   \
 389     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 390                                                     16, 16);            \
 391     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 392                                          stride, 16, 16);               \
 393 }                                                                       \
 394                                                                         \
 395 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
 396                                           ptrdiff_t stride)             \
 397 {                                                                       \
 398     uint64_t half[16 * 2 + 17 * 2];                                     \
 399     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 400     uint8_t * const halfHV = ((uint8_t*)half);                          \
 401     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 402                                                     stride, 17);        \
 403     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 404                                          stride, 17);                   \
 405     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 406                                                     16, 16);            \
 407     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 408                                          stride, 16, 16);               \
 409 }                                                                       \
 410                                                                         \
 411 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
 412                                           ptrdiff_t stride)             \
 413 {                                                                       \
 414     uint64_t half[16 * 2 + 17 * 2];                                     \
 415     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 416     uint8_t * const halfHV = ((uint8_t*)half);                          \
 417     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 418                                                     stride, 17);        \
 419     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 420                                          stride, 17);                   \
 421     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 422                                                     16, 16);            \
 423     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 424                                          stride, 16, 16);               \
 425 }                                                                       \
 426                                                                         \
 427 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
 428                                           ptrdiff_t stride)             \
 429 {                                                                       \
 430     uint64_t half[16 * 2 + 17 * 2];                                     \
 431     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 432     uint8_t * const halfHV = ((uint8_t*)half);                          \
 433     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 434                                                     stride, 17);        \
 435     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 436                                          stride, 17);                   \
 437     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 438                                                     16, 16);            \
 439     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 440                                          stride, 16, 16);               \
 441 }                                                                       \
 442                                                                         \
 443 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
 444                                           ptrdiff_t stride)             \
 445 {                                                                       \
 446     uint64_t half[16 * 2 + 17 * 2];                                     \
 447     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 448     uint8_t * const halfHV = ((uint8_t*)half);                          \
 449     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 450                                                     stride, 17);        \
 451     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 452                                                     16, 16);            \
 453     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 454                                          stride, 16, 16);               \
 455 }                                                                       \
 456                                                                         \
 457 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
 458                                           ptrdiff_t stride)             \
 459 {                                                                       \
 460     uint64_t half[16 * 2 + 17 * 2];                                     \
 461     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 462     uint8_t * const halfHV = ((uint8_t*)half);                          \
 463     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 464                                                     stride, 17);        \
 465     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 466                                                     16, 16);            \
 467     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 468                                          stride, 16, 16);               \
 469 }                                                                       \
 470                                                                         \
 471 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
 472                                           ptrdiff_t stride)             \
 473 {                                                                       \
 474     uint64_t half[17 * 2];                                              \
 475     uint8_t * const halfH = ((uint8_t*)half);                           \
 476     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 477                                                     stride, 17);        \
 478     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 479                                          stride, 17);                   \
 480     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 481                                                     stride, 16);        \
 482 }                                                                       \
 483                                                                         \
 484 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
 485                                           ptrdiff_t stride)             \
 486 {                                                                       \
 487     uint64_t half[17 * 2];                                              \
 488     uint8_t * const halfH = ((uint8_t*)half);                           \
 489     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 490                                                     stride, 17);        \
 491     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 492                                          stride, 17);                   \
 493     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 494                                                     stride, 16);        \
 495 }                                                                       \
 496                                                                         \
 497 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
 498                                           ptrdiff_t stride)             \
 499 {                                                                       \
 500     uint64_t half[17 * 2];                                              \
 501     uint8_t * const halfH = ((uint8_t*)half);                           \
 502     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 503                                                     stride, 17);        \
 504     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 505                                                     stride, 16);        \
 506 }
 507
 508 QPEL_OP(put_,        _,        mmxext)
 509 QPEL_OP(avg_,        _,        mmxext)
 510 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
 511 #endif /* HAVE_YASM */
 512
 513 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
 514     do {                                                                     \
 515     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
 516     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
 517     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
 518     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
 519     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
 520     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
 521     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
 522     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
 523     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
 524     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
 525     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
 526     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
 527     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
 528     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
 529     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
 530     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
 531     } while (0)
 532
 533 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
 534                                      int cpu_flags)
 535 {
 536 #if HAVE_MMX_INLINE
 537     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 538
 539     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
 540     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
 541     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
 542
 543     if (!high_bit_depth) {
 544         c->clear_block  = ff_clear_block_mmx;
 545         c->clear_blocks = ff_clear_blocks_mmx;
 546         c->draw_edges   = ff_draw_edges_mmx;
 547     }
 548
 549 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
 550     c->gmc = ff_gmc_mmx;
 551 #endif
 552
 553     c->add_bytes = ff_add_bytes_mmx;
 554 #endif /* HAVE_MMX_INLINE */
 555
 556 #if HAVE_MMX_EXTERNAL
 557     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
 558 #endif /* HAVE_MMX_EXTERNAL */
 559 }
 560
 561 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
 562                                         int cpu_flags)
 563 {
 564 #if HAVE_MMXEXT_INLINE
 565     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 566
 567     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX && avctx->lowres == 0) {
 568         c->idct_put = ff_idct_xvid_mmxext_put;
 569         c->idct_add = ff_idct_xvid_mmxext_add;
 570         c->idct     = ff_idct_xvid_mmxext;
 571     }
 572 #endif /* HAVE_MMXEXT_INLINE */
 573
 574 #if HAVE_MMXEXT_EXTERNAL
 575     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
 576     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
 577
 578     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
 579     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
 580     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
 581     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
 582
 583     /* slower than cmov version on AMD */
 584     if (!(cpu_flags & AV_CPU_FLAG_3DNOW))
 585         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
 586
 587     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
 588     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
 589
 590     if (avctx->flags & CODEC_FLAG_BITEXACT) {
 591         c->apply_window_int16 = ff_apply_window_int16_mmxext;
 592     } else {
 593         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
 594     }
 595 #endif /* HAVE_MMXEXT_EXTERNAL */
 596 }
 597
 598 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
 599                                      int cpu_flags)
 600 {
 601 #if HAVE_SSE_INLINE
 602     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 603
 604     if (!high_bit_depth) {
 605 #if FF_API_XVMC
 606 FF_DISABLE_DEPRECATION_WARNINGS
 607         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
 608             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
 609 FF_ENABLE_DEPRECATION_WARNINGS
 610 #endif /* FF_API_XVMC */
 611         c->clear_block  = ff_clear_block_sse;
 612         c->clear_blocks = ff_clear_blocks_sse;
 613 #if FF_API_XVMC
 614         }
 615 #endif /* FF_API_XVMC */
 616     }
 617
 618     c->vector_clipf = ff_vector_clipf_sse;
 619 #endif /* HAVE_SSE_INLINE */
 620
 621 #if HAVE_YASM
 622 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
 623     c->gmc = ff_gmc_sse;
 624 #endif
 625 #endif /* HAVE_YASM */
 626 }
 627
 628 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
 629                                       int cpu_flags)
 630 {
 631 #if HAVE_SSE2_INLINE
 632     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 633
 634     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX && avctx->lowres == 0) {
 635         c->idct_put              = ff_idct_xvid_sse2_put;
 636         c->idct_add              = ff_idct_xvid_sse2_add;
 637         c->idct                  = ff_idct_xvid_sse2;
 638         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
 639     }
 640 #endif /* HAVE_SSE2_INLINE */
 641
 642 #if HAVE_SSE2_EXTERNAL
 643     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
 644     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
 645     if (cpu_flags & AV_CPU_FLAG_ATOM) {
 646         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
 647     } else {
 648         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
 649     }
 650     if (avctx->flags & CODEC_FLAG_BITEXACT) {
 651         c->apply_window_int16 = ff_apply_window_int16_sse2;
 652     } else if (!(cpu_flags & AV_CPU_FLAG_SSE2SLOW)) {
 653         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
 654     }
 655     c->bswap_buf = ff_bswap32_buf_sse2;
 656 #endif /* HAVE_SSE2_EXTERNAL */
 657 }
 658
 659 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
 660                                        int cpu_flags)
 661 {
 662 #if HAVE_SSSE3_EXTERNAL
 663     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
 664     if (cpu_flags & AV_CPU_FLAG_SSE4) // not really SSE4, just slow on Conroe
 665         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
 666
 667     if (cpu_flags & AV_CPU_FLAG_ATOM)
 668         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
 669     else
 670         c->apply_window_int16 = ff_apply_window_int16_ssse3;
 671     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
 672         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
 673     c->bswap_buf = ff_bswap32_buf_ssse3;
 674 #endif /* HAVE_SSSE3_EXTERNAL */
 675 }
 676
 677 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
 678                                       int cpu_flags)
 679 {
 680 #if HAVE_SSE4_EXTERNAL
 681     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
 682 #endif /* HAVE_SSE4_EXTERNAL */
 683 }
 684
 685 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx)
 686 {
 687     int cpu_flags = av_get_cpu_flags();
 688
 689 #if HAVE_7REGS && HAVE_INLINE_ASM
 690     if (HAVE_MMX && cpu_flags & AV_CPU_FLAG_CMOV)
 691         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
 692 #endif
 693
 694     if (X86_MMX(cpu_flags)) {
 695 #if HAVE_INLINE_ASM
 696         const int idct_algo = avctx->idct_algo;
 697
 698         if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
 699             if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
 700                 c->idct_put              = ff_simple_idct_put_mmx;
 701                 c->idct_add              = ff_simple_idct_add_mmx;
 702                 c->idct                  = ff_simple_idct_mmx;
 703                 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
 704             } else if (idct_algo == FF_IDCT_XVIDMMX) {
 705                 c->idct_put              = ff_idct_xvid_mmx_put;
 706                 c->idct_add              = ff_idct_xvid_mmx_add;
 707                 c->idct                  = ff_idct_xvid_mmx;
 708             }
 709         }
 710 #endif /* HAVE_INLINE_ASM */
 711
 712         dsputil_init_mmx(c, avctx, cpu_flags);
 713     }
 714
 715     if (X86_MMXEXT(cpu_flags))
 716         dsputil_init_mmxext(c, avctx, cpu_flags);
 717
 718     if (X86_SSE(cpu_flags))
 719         dsputil_init_sse(c, avctx, cpu_flags);
 720
 721     if (X86_SSE2(cpu_flags))
 722         dsputil_init_sse2(c, avctx, cpu_flags);
 723
 724     if (EXTERNAL_SSSE3(cpu_flags))
 725         dsputil_init_ssse3(c, avctx, cpu_flags);
 726
 727     if (EXTERNAL_SSE4(cpu_flags))
 728         dsputil_init_sse4(c, avctx, cpu_flags);
 729
 730     if (CONFIG_ENCODERS)
 731         ff_dsputilenc_init_mmx(c, avctx);
 732 }