git.sesse.net Git - ffmpeg/blob - libavcodec/x86/dsputil_init.c

   1 /*
   2  * Copyright (c) 2000, 2001 Fabrice Bellard
   3  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   4  *
   5  * This file is part of FFmpeg.
   6  *
   7  * FFmpeg is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * FFmpeg is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with FFmpeg; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include "config.h"
  23 #include "libavutil/attributes.h"
  24 #include "libavutil/cpu.h"
  25 #include "libavutil/x86/asm.h"
  26 #include "libavcodec/dsputil.h"
  27 #include "libavcodec/simple_idct.h"
  28 #include "dsputil_x86.h"
  29 #include "idct_xvid.h"
  30
  31 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  32                               int dstStride, int src1Stride, int h);
  33 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
  34                                      uint8_t *src2, int dstStride,
  35                                      int src1Stride, int h);
  36 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  37                               int dstStride, int src1Stride, int h);
  38 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  39                                int dstStride, int src1Stride, int h);
  40 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  41                                int dstStride, int src1Stride, int h);
  42 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
  43                                       int dstStride, int src1Stride, int h);
  44 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  45                                          int dstStride, int srcStride, int h);
  46 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  47                                          int dstStride, int srcStride, int h);
  48 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  49                                                  int dstStride, int srcStride,
  50                                                  int h);
  51 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  52                                         int dstStride, int srcStride, int h);
  53 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  54                                         int dstStride, int srcStride, int h);
  55 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  56                                                 int dstStride, int srcStride,
  57                                                 int h);
  58 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  59                                          int dstStride, int srcStride);
  60 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  61                                          int dstStride, int srcStride);
  62 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  63                                                  int dstStride, int srcStride);
  64 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  65                                         int dstStride, int srcStride);
  66 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  67                                         int dstStride, int srcStride);
  68 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
  69                                                 int dstStride, int srcStride);
  70 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
  71 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
  72
  73 void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);
  74 void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);
  75
  76 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
  77                                       int order);
  78 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
  79                                     int order);
  80 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
  81                                                const int16_t *v3,
  82                                                int order, int mul);
  83 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
  84                                              const int16_t *v3,
  85                                              int order, int mul);
  86 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
  87                                               const int16_t *v3,
  88                                               int order, int mul);
  89
  90 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
  91                                         const int16_t *window, unsigned int len);
  92 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
  93                                       const int16_t *window, unsigned int len);
  94 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
  95                                   const int16_t *window, unsigned int len);
  96 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
  97                                 const int16_t *window, unsigned int len);
  98 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
  99                                  const int16_t *window, unsigned int len);
 100 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
 101                                       const int16_t *window, unsigned int len);
 102
 103 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
 104 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
 105
 106 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
 107                                           const uint8_t *diff, int w,
 108                                           int *left, int *left_top);
 109 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
 110                                        int w, int left);
 111 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
 112                                       int w, int left);
 113
 114 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
 115                                    int32_t min, int32_t max, unsigned int len);
 116 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
 117                                    int32_t min, int32_t max, unsigned int len);
 118 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
 119                                    int32_t min, int32_t max, unsigned int len);
 120 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
 121                                    int32_t min, int32_t max, unsigned int len);
 122
 123 #if HAVE_YASM
 124
 125 PIXELS16(static, ff_avg, , , _mmxext)
 126 PIXELS16(static, ff_put, , , _mmxext)
 127
 128 #define QPEL_OP(OPNAME, RND, MMX)                                       \
 129 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
 130                                           ptrdiff_t stride)             \
 131 {                                                                       \
 132     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
 133 }                                                                       \
 134                                                                         \
 135 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
 136                                          ptrdiff_t stride)              \
 137 {                                                                       \
 138     uint64_t temp[8];                                                   \
 139     uint8_t * const half = (uint8_t*)temp;                              \
 140     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 141                                                    stride, 8);          \
 142     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 143                                         stride, stride, 8);             \
 144 }                                                                       \
 145                                                                         \
 146 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
 147                                          ptrdiff_t stride)              \
 148 {                                                                       \
 149     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
 150                                                    stride, 8);          \
 151 }                                                                       \
 152                                                                         \
 153 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
 154                                          ptrdiff_t stride)              \
 155 {                                                                       \
 156     uint64_t temp[8];                                                   \
 157     uint8_t * const half = (uint8_t*)temp;                              \
 158     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
 159                                                    stride, 8);          \
 160     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
 161                                         stride, 8);                     \
 162 }                                                                       \
 163                                                                         \
 164 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
 165                                          ptrdiff_t stride)              \
 166 {                                                                       \
 167     uint64_t temp[8];                                                   \
 168     uint8_t * const half = (uint8_t*)temp;                              \
 169     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 170                                                    8, stride);          \
 171     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
 172                                         stride, stride, 8);             \
 173 }                                                                       \
 174                                                                         \
 175 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
 176                                          ptrdiff_t stride)              \
 177 {                                                                       \
 178     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
 179                                                    stride, stride);     \
 180 }                                                                       \
 181                                                                         \
 182 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
 183                                          ptrdiff_t stride)              \
 184 {                                                                       \
 185     uint64_t temp[8];                                                   \
 186     uint8_t * const half = (uint8_t*)temp;                              \
 187     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
 188                                                    8, stride);          \
 189     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
 190                                         stride, 8);                     \
 191 }                                                                       \
 192                                                                         \
 193 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
 194                                          ptrdiff_t stride)              \
 195 {                                                                       \
 196     uint64_t half[8 + 9];                                               \
 197     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 198     uint8_t * const halfHV = ((uint8_t*)half);                          \
 199     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 200                                                    stride, 9);          \
 201     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 202                                         stride, 9);                     \
 203     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 204     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 205                                         stride, 8, 8);                  \
 206 }                                                                       \
 207                                                                         \
 208 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
 209                                          ptrdiff_t stride)              \
 210 {                                                                       \
 211     uint64_t half[8 + 9];                                               \
 212     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 213     uint8_t * const halfHV = ((uint8_t*)half);                          \
 214     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 215                                                    stride, 9);          \
 216     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 217                                         stride, 9);                     \
 218     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 219     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 220                                         stride, 8, 8);                  \
 221 }                                                                       \
 222                                                                         \
 223 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
 224                                          ptrdiff_t stride)              \
 225 {                                                                       \
 226     uint64_t half[8 + 9];                                               \
 227     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 228     uint8_t * const halfHV = ((uint8_t*)half);                          \
 229     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 230                                                    stride, 9);          \
 231     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
 232                                         stride, 9);                     \
 233     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 234     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 235                                         stride, 8, 8);                  \
 236 }                                                                       \
 237                                                                         \
 238 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
 239                                          ptrdiff_t stride)              \
 240 {                                                                       \
 241     uint64_t half[8 + 9];                                               \
 242     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 243     uint8_t * const halfHV = ((uint8_t*)half);                          \
 244     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 245                                                    stride, 9);          \
 246     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 247                                         stride, 9);                     \
 248     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 249     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 250                                         stride, 8, 8);                  \
 251 }                                                                       \
 252                                                                         \
 253 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
 254                                          ptrdiff_t stride)              \
 255 {                                                                       \
 256     uint64_t half[8 + 9];                                               \
 257     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 258     uint8_t * const halfHV = ((uint8_t*)half);                          \
 259     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 260                                                    stride, 9);          \
 261     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 262     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
 263                                         stride, 8, 8);                  \
 264 }                                                                       \
 265                                                                         \
 266 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
 267                                          ptrdiff_t stride)              \
 268 {                                                                       \
 269     uint64_t half[8 + 9];                                               \
 270     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
 271     uint8_t * const halfHV = ((uint8_t*)half);                          \
 272     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 273                                                    stride, 9);          \
 274     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
 275     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
 276                                         stride, 8, 8);                  \
 277 }                                                                       \
 278                                                                         \
 279 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
 280                                          ptrdiff_t stride)              \
 281 {                                                                       \
 282     uint64_t half[8 + 9];                                               \
 283     uint8_t * const halfH = ((uint8_t*)half);                           \
 284     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 285                                                    stride, 9);          \
 286     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
 287                                         8, stride, 9);                  \
 288     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 289                                                    stride, 8);          \
 290 }                                                                       \
 291                                                                         \
 292 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
 293                                          ptrdiff_t stride)              \
 294 {                                                                       \
 295     uint64_t half[8 + 9];                                               \
 296     uint8_t * const halfH = ((uint8_t*)half);                           \
 297     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 298                                                    stride, 9);          \
 299     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
 300                                         stride, 9);                     \
 301     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 302                                                    stride, 8);          \
 303 }                                                                       \
 304                                                                         \
 305 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
 306                                          ptrdiff_t stride)              \
 307 {                                                                       \
 308     uint64_t half[9];                                                   \
 309     uint8_t * const halfH = ((uint8_t*)half);                           \
 310     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
 311                                                    stride, 9);          \
 312     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
 313                                                    stride, 8);          \
 314 }                                                                       \
 315                                                                         \
 316 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
 317                                            ptrdiff_t stride)            \
 318 {                                                                       \
 319     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
 320 }                                                                       \
 321                                                                         \
 322 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
 323                                           ptrdiff_t stride)             \
 324 {                                                                       \
 325     uint64_t temp[32];                                                  \
 326     uint8_t * const half = (uint8_t*)temp;                              \
 327     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 328                                                     stride, 16);        \
 329     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 330                                          stride, 16);                   \
 331 }                                                                       \
 332                                                                         \
 333 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
 334                                           ptrdiff_t stride)             \
 335 {                                                                       \
 336     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
 337                                                     stride, stride, 16);\
 338 }                                                                       \
 339                                                                         \
 340 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
 341                                           ptrdiff_t stride)             \
 342 {                                                                       \
 343     uint64_t temp[32];                                                  \
 344     uint8_t * const half = (uint8_t*)temp;                              \
 345     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
 346                                                     stride, 16);        \
 347     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
 348                                          stride, stride, 16);           \
 349 }                                                                       \
 350                                                                         \
 351 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
 352                                           ptrdiff_t stride)             \
 353 {                                                                       \
 354     uint64_t temp[32];                                                  \
 355     uint8_t * const half = (uint8_t*)temp;                              \
 356     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 357                                                     stride);            \
 358     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
 359                                          stride, 16);                   \
 360 }                                                                       \
 361                                                                         \
 362 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
 363                                           ptrdiff_t stride)             \
 364 {                                                                       \
 365     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
 366                                                     stride, stride);    \
 367 }                                                                       \
 368                                                                         \
 369 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
 370                                           ptrdiff_t stride)             \
 371 {                                                                       \
 372     uint64_t temp[32];                                                  \
 373     uint8_t * const half = (uint8_t*)temp;                              \
 374     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
 375                                                     stride);            \
 376     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
 377                                          stride, stride, 16);           \
 378 }                                                                       \
 379                                                                         \
 380 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
 381                                           ptrdiff_t stride)             \
 382 {                                                                       \
 383     uint64_t half[16 * 2 + 17 * 2];                                     \
 384     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 385     uint8_t * const halfHV = ((uint8_t*)half);                          \
 386     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 387                                                     stride, 17);        \
 388     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 389                                          stride, 17);                   \
 390     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 391                                                     16, 16);            \
 392     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 393                                          stride, 16, 16);               \
 394 }                                                                       \
 395                                                                         \
 396 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
 397                                           ptrdiff_t stride)             \
 398 {                                                                       \
 399     uint64_t half[16 * 2 + 17 * 2];                                     \
 400     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 401     uint8_t * const halfHV = ((uint8_t*)half);                          \
 402     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 403                                                     stride, 17);        \
 404     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 405                                          stride, 17);                   \
 406     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 407                                                     16, 16);            \
 408     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 409                                          stride, 16, 16);               \
 410 }                                                                       \
 411                                                                         \
 412 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
 413                                           ptrdiff_t stride)             \
 414 {                                                                       \
 415     uint64_t half[16 * 2 + 17 * 2];                                     \
 416     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 417     uint8_t * const halfHV = ((uint8_t*)half);                          \
 418     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 419                                                     stride, 17);        \
 420     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 421                                          stride, 17);                   \
 422     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 423                                                     16, 16);            \
 424     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 425                                          stride, 16, 16);               \
 426 }                                                                       \
 427                                                                         \
 428 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
 429                                           ptrdiff_t stride)             \
 430 {                                                                       \
 431     uint64_t half[16 * 2 + 17 * 2];                                     \
 432     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 433     uint8_t * const halfHV = ((uint8_t*)half);                          \
 434     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 435                                                     stride, 17);        \
 436     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 437                                          stride, 17);                   \
 438     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 439                                                     16, 16);            \
 440     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 441                                          stride, 16, 16);               \
 442 }                                                                       \
 443                                                                         \
 444 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
 445                                           ptrdiff_t stride)             \
 446 {                                                                       \
 447     uint64_t half[16 * 2 + 17 * 2];                                     \
 448     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 449     uint8_t * const halfHV = ((uint8_t*)half);                          \
 450     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 451                                                     stride, 17);        \
 452     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 453                                                     16, 16);            \
 454     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
 455                                          stride, 16, 16);               \
 456 }                                                                       \
 457                                                                         \
 458 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
 459                                           ptrdiff_t stride)             \
 460 {                                                                       \
 461     uint64_t half[16 * 2 + 17 * 2];                                     \
 462     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
 463     uint8_t * const halfHV = ((uint8_t*)half);                          \
 464     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 465                                                     stride, 17);        \
 466     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
 467                                                     16, 16);            \
 468     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
 469                                          stride, 16, 16);               \
 470 }                                                                       \
 471                                                                         \
 472 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
 473                                           ptrdiff_t stride)             \
 474 {                                                                       \
 475     uint64_t half[17 * 2];                                              \
 476     uint8_t * const halfH = ((uint8_t*)half);                           \
 477     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 478                                                     stride, 17);        \
 479     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
 480                                          stride, 17);                   \
 481     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 482                                                     stride, 16);        \
 483 }                                                                       \
 484                                                                         \
 485 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
 486                                           ptrdiff_t stride)             \
 487 {                                                                       \
 488     uint64_t half[17 * 2];                                              \
 489     uint8_t * const halfH = ((uint8_t*)half);                           \
 490     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 491                                                     stride, 17);        \
 492     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
 493                                          stride, 17);                   \
 494     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 495                                                     stride, 16);        \
 496 }                                                                       \
 497                                                                         \
 498 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
 499                                           ptrdiff_t stride)             \
 500 {                                                                       \
 501     uint64_t half[17 * 2];                                              \
 502     uint8_t * const halfH = ((uint8_t*)half);                           \
 503     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
 504                                                     stride, 17);        \
 505     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
 506                                                     stride, 16);        \
 507 }
 508
 509 QPEL_OP(put_,        _,        mmxext)
 510 QPEL_OP(avg_,        _,        mmxext)
 511 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
 512 #endif /* HAVE_YASM */
 513
 514 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
 515     do {                                                                     \
 516     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
 517     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
 518     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
 519     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
 520     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
 521     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
 522     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
 523     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
 524     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
 525     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
 526     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
 527     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
 528     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
 529     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
 530     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
 531     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
 532     } while (0)
 533
 534 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
 535                                      int mm_flags)
 536 {
 537 #if HAVE_MMX_INLINE
 538     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 539
 540     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
 541     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
 542     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
 543
 544     if (!high_bit_depth) {
 545         c->clear_block  = ff_clear_block_mmx;
 546         c->clear_blocks = ff_clear_blocks_mmx;
 547         c->draw_edges   = ff_draw_edges_mmx;
 548     }
 549
 550 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
 551     c->gmc = ff_gmc_mmx;
 552 #endif
 553
 554     c->add_bytes = ff_add_bytes_mmx;
 555 #endif /* HAVE_MMX_INLINE */
 556
 557 #if HAVE_MMX_EXTERNAL
 558     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
 559         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;
 560         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;
 561     }
 562
 563     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
 564 #endif /* HAVE_MMX_EXTERNAL */
 565 }
 566
 567 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
 568                                         int mm_flags)
 569 {
 570 #if HAVE_MMXEXT_EXTERNAL
 571     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
 572     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
 573
 574     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
 575     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
 576     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
 577     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
 578
 579     /* slower than cmov version on AMD */
 580     if (!(mm_flags & AV_CPU_FLAG_3DNOW))
 581         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
 582
 583     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
 584     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
 585
 586     if (avctx->flags & CODEC_FLAG_BITEXACT) {
 587         c->apply_window_int16 = ff_apply_window_int16_mmxext;
 588     } else {
 589         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
 590     }
 591 #endif /* HAVE_MMXEXT_EXTERNAL */
 592 }
 593
 594 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
 595                                      int mm_flags)
 596 {
 597 #if HAVE_SSE_INLINE
 598     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 599
 600     if (!high_bit_depth) {
 601         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
 602             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
 603             c->clear_block  = ff_clear_block_sse;
 604             c->clear_blocks = ff_clear_blocks_sse;
 605         }
 606     }
 607
 608     c->vector_clipf = ff_vector_clipf_sse;
 609 #endif /* HAVE_SSE_INLINE */
 610
 611 #if HAVE_YASM
 612 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
 613     c->gmc = ff_gmc_sse;
 614 #endif
 615 #endif /* HAVE_YASM */
 616 }
 617
 618 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
 619                                       int mm_flags)
 620 {
 621 #if HAVE_SSE2_INLINE
 622     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
 623
 624     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
 625         c->idct_put              = ff_idct_xvid_sse2_put;
 626         c->idct_add              = ff_idct_xvid_sse2_add;
 627         c->idct                  = ff_idct_xvid_sse2;
 628         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
 629     }
 630 #endif /* HAVE_SSE2_INLINE */
 631
 632 #if HAVE_SSE2_EXTERNAL
 633     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
 634     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
 635     if (mm_flags & AV_CPU_FLAG_ATOM) {
 636         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
 637     } else {
 638         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
 639     }
 640     if (avctx->flags & CODEC_FLAG_BITEXACT) {
 641         c->apply_window_int16 = ff_apply_window_int16_sse2;
 642     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
 643         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
 644     }
 645     c->bswap_buf = ff_bswap32_buf_sse2;
 646 #endif /* HAVE_SSE2_EXTERNAL */
 647 }
 648
 649 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
 650                                        int mm_flags)
 651 {
 652 #if HAVE_SSSE3_EXTERNAL
 653     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
 654     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
 655         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
 656
 657     if (mm_flags & AV_CPU_FLAG_ATOM)
 658         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
 659     else
 660         c->apply_window_int16 = ff_apply_window_int16_ssse3;
 661     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
 662         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
 663     c->bswap_buf = ff_bswap32_buf_ssse3;
 664 #endif /* HAVE_SSSE3_EXTERNAL */
 665 }
 666
 667 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
 668                                       int mm_flags)
 669 {
 670 #if HAVE_SSE4_EXTERNAL
 671     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
 672 #endif /* HAVE_SSE4_EXTERNAL */
 673 }
 674
 675 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
 676 {
 677     int mm_flags = av_get_cpu_flags();
 678
 679 #if HAVE_7REGS && HAVE_INLINE_ASM
 680     if (mm_flags & AV_CPU_FLAG_CMOV)
 681         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
 682 #endif
 683
 684     if (mm_flags & AV_CPU_FLAG_MMX) {
 685 #if HAVE_INLINE_ASM
 686         const int idct_algo = avctx->idct_algo;
 687
 688         if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
 689             if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
 690                 c->idct_put              = ff_simple_idct_put_mmx;
 691                 c->idct_add              = ff_simple_idct_add_mmx;
 692                 c->idct                  = ff_simple_idct_mmx;
 693                 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
 694             } else if (idct_algo == FF_IDCT_XVIDMMX) {
 695                 if (mm_flags & AV_CPU_FLAG_SSE2) {
 696                     c->idct_put              = ff_idct_xvid_sse2_put;
 697                     c->idct_add              = ff_idct_xvid_sse2_add;
 698                     c->idct                  = ff_idct_xvid_sse2;
 699                     c->idct_permutation_type = FF_SSE2_IDCT_PERM;
 700                 } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
 701                     c->idct_put              = ff_idct_xvid_mmxext_put;
 702                     c->idct_add              = ff_idct_xvid_mmxext_add;
 703                     c->idct                  = ff_idct_xvid_mmxext;
 704                 } else {
 705                     c->idct_put              = ff_idct_xvid_mmx_put;
 706                     c->idct_add              = ff_idct_xvid_mmx_add;
 707                     c->idct                  = ff_idct_xvid_mmx;
 708                 }
 709             }
 710         }
 711 #endif /* HAVE_INLINE_ASM */
 712
 713         dsputil_init_mmx(c, avctx, mm_flags);
 714     }
 715
 716     if (mm_flags & AV_CPU_FLAG_MMXEXT)
 717         dsputil_init_mmxext(c, avctx, mm_flags);
 718
 719     if (mm_flags & AV_CPU_FLAG_SSE)
 720         dsputil_init_sse(c, avctx, mm_flags);
 721
 722     if (mm_flags & AV_CPU_FLAG_SSE2)
 723         dsputil_init_sse2(c, avctx, mm_flags);
 724
 725     if (mm_flags & AV_CPU_FLAG_SSSE3)
 726         dsputil_init_ssse3(c, avctx, mm_flags);
 727
 728     if (mm_flags & AV_CPU_FLAG_SSE4)
 729         dsputil_init_sse4(c, avctx, mm_flags);
 730
 731     if (CONFIG_ENCODERS)
 732         ff_dsputilenc_init_mmx(c, avctx);
 733 }