git.sesse.net Git - ffmpeg/blob - libswscale/x86/swscale_template.c

   1 /*
   2  * Copyright (C) 2001-2011 Michael Niedermayer <michaelni@gmx.at>
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include <stdint.h>
  22
  23 #include "libavutil/x86/asm.h"
  24 #include "libswscale/swscale_internal.h"
  25
  26 #undef REAL_MOVNTQ
  27 #undef MOVNTQ
  28 #undef MOVNTQ2
  29 #undef PREFETCH
  30
  31 #if COMPILE_TEMPLATE_MMXEXT
  32 #define PREFETCH "prefetchnta"
  33 #else
  34 #define PREFETCH  " # nop"
  35 #endif
  36
  37 #if COMPILE_TEMPLATE_MMXEXT
  38 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
  39 #define MOVNTQ2 "movntq "
  40 #else
  41 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
  42 #define MOVNTQ2 "movq "
  43 #endif
  44 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
  45
  46 #if !COMPILE_TEMPLATE_MMXEXT
  47 static av_always_inline void
  48 dither_8to16(const uint8_t *srcDither, int rot)
  49 {
  50     if (rot) {
  51         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
  52                          "movq       (%0), %%mm3\n\t"
  53                          "movq      %%mm3, %%mm4\n\t"
  54                          "psrlq       $24, %%mm3\n\t"
  55                          "psllq       $40, %%mm4\n\t"
  56                          "por       %%mm4, %%mm3\n\t"
  57                          "movq      %%mm3, %%mm4\n\t"
  58                          "punpcklbw %%mm0, %%mm3\n\t"
  59                          "punpckhbw %%mm0, %%mm4\n\t"
  60                          :: "r"(srcDither)
  61                          );
  62     } else {
  63         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
  64                          "movq       (%0), %%mm3\n\t"
  65                          "movq      %%mm3, %%mm4\n\t"
  66                          "punpcklbw %%mm0, %%mm3\n\t"
  67                          "punpckhbw %%mm0, %%mm4\n\t"
  68                          :: "r"(srcDither)
  69                          );
  70     }
  71 }
  72 #endif
  73
  74 static void RENAME(yuv2yuvX)(const int16_t *filter, int filterSize,
  75                            const int16_t **src, uint8_t *dest, int dstW,
  76                            const uint8_t *dither, int offset)
  77 {
  78     dither_8to16(dither, offset);
  79     filterSize--;
  80     __asm__ volatile(
  81         "movd %0, %%mm1\n\t"
  82         "punpcklwd %%mm1, %%mm1\n\t"
  83         "punpckldq %%mm1, %%mm1\n\t"
  84         "psllw        $3, %%mm1\n\t"
  85         "paddw     %%mm1, %%mm3\n\t"
  86         "paddw     %%mm1, %%mm4\n\t"
  87         "psraw        $4, %%mm3\n\t"
  88         "psraw        $4, %%mm4\n\t"
  89         ::"m"(filterSize)
  90      );
  91
  92     __asm__ volatile(\
  93         "movq    %%mm3, %%mm6\n\t"
  94         "movq    %%mm4, %%mm7\n\t"
  95         "movl %3, %%ecx\n\t"
  96         "mov                                 %0, %%"REG_d"  \n\t"\
  97         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
  98         ".p2align                             4             \n\t" /* FIXME Unroll? */\
  99         "1:                                                 \n\t"\
 100         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
 101         "movq                (%%"REG_S", %%"REG_c", 2), %%mm2      \n\t" /* srcData */\
 102         "movq               8(%%"REG_S", %%"REG_c", 2), %%mm5      \n\t" /* srcData */\
 103         "add                                $16, %%"REG_d"  \n\t"\
 104         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
 105         "test                         %%"REG_S", %%"REG_S"  \n\t"\
 106         "pmulhw                           %%mm0, %%mm2      \n\t"\
 107         "pmulhw                           %%mm0, %%mm5      \n\t"\
 108         "paddw                            %%mm2, %%mm3      \n\t"\
 109         "paddw                            %%mm5, %%mm4      \n\t"\
 110         " jnz                                1b             \n\t"\
 111         "psraw                               $3, %%mm3      \n\t"\
 112         "psraw                               $3, %%mm4      \n\t"\
 113         "packuswb                         %%mm4, %%mm3      \n\t"
 114         MOVNTQ2 "                         %%mm3, (%1, %%"REG_c")\n\t"
 115         "add                          $8, %%"REG_c"         \n\t"\
 116         "cmp                          %2, %%"REG_c"         \n\t"\
 117         "movq    %%mm6, %%mm3\n\t"
 118         "movq    %%mm7, %%mm4\n\t"
 119         "mov                                 %0, %%"REG_d"  \n\t"\
 120         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
 121         "jb                                  1b             \n\t"\
 122         :: "g" (filter),
 123            "r" (dest-offset), "g" ((x86_reg)(dstW+offset)), "m" (offset)
 124         : "%"REG_d, "%"REG_S, "%"REG_c
 125     );
 126 }
 127
 128 #define YSCALEYUV2PACKEDX_UV \
 129     __asm__ volatile(\
 130         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
 131         ".p2align                      4                \n\t"\
 132         "nop                                            \n\t"\
 133         "1:                                             \n\t"\
 134         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
 135         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 136         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
 137         "movq                      %%mm3, %%mm4         \n\t"\
 138         ".p2align                      4                \n\t"\
 139         "2:                                             \n\t"\
 140         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
 141         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
 142         "add                          %6, %%"REG_S"     \n\t" \
 143         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
 144         "add                         $16, %%"REG_d"     \n\t"\
 145         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 146         "pmulhw                    %%mm0, %%mm2         \n\t"\
 147         "pmulhw                    %%mm0, %%mm5         \n\t"\
 148         "paddw                     %%mm2, %%mm3         \n\t"\
 149         "paddw                     %%mm5, %%mm4         \n\t"\
 150         "test                  %%"REG_S", %%"REG_S"     \n\t"\
 151         " jnz                         2b                \n\t"\
 152
 153 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
 154     "lea                "offset"(%0), %%"REG_d"     \n\t"\
 155     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 156     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
 157     "movq                    "#dst1", "#dst2"       \n\t"\
 158     ".p2align                      4                \n\t"\
 159     "2:                                             \n\t"\
 160     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
 161     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
 162     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
 163     "add                         $16, %%"REG_d"            \n\t"\
 164     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 165     "pmulhw                 "#coeff", "#src1"       \n\t"\
 166     "pmulhw                 "#coeff", "#src2"       \n\t"\
 167     "paddw                   "#src1", "#dst1"       \n\t"\
 168     "paddw                   "#src2", "#dst2"       \n\t"\
 169     "test                  %%"REG_S", %%"REG_S"     \n\t"\
 170     " jnz                         2b                \n\t"\
 171
 172 #define YSCALEYUV2PACKEDX \
 173     YSCALEYUV2PACKEDX_UV \
 174     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
 175
 176 #define YSCALEYUV2PACKEDX_END                     \
 177         :: "r" (&c->redDither),                   \
 178             "m" (dummy), "m" (dummy), "m" (dummy),\
 179             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
 180             NAMED_CONSTRAINTS_ADD(bF8,bFC) \
 181         : "%"REG_a, "%"REG_d, "%"REG_S            \
 182     );
 183
 184 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
 185     __asm__ volatile(\
 186         "xor %%"REG_a", %%"REG_a"                       \n\t"\
 187         ".p2align                      4                \n\t"\
 188         "nop                                            \n\t"\
 189         "1:                                             \n\t"\
 190         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
 191         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 192         "pxor                      %%mm4, %%mm4         \n\t"\
 193         "pxor                      %%mm5, %%mm5         \n\t"\
 194         "pxor                      %%mm6, %%mm6         \n\t"\
 195         "pxor                      %%mm7, %%mm7         \n\t"\
 196         ".p2align                      4                \n\t"\
 197         "2:                                             \n\t"\
 198         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
 199         "add                          %6, %%"REG_S"      \n\t" \
 200         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
 201         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
 202         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
 203         "movq                      %%mm0, %%mm3         \n\t"\
 204         "punpcklwd                 %%mm1, %%mm0         \n\t"\
 205         "punpckhwd                 %%mm1, %%mm3         \n\t"\
 206         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
 207         "pmaddwd                   %%mm1, %%mm0         \n\t"\
 208         "pmaddwd                   %%mm1, %%mm3         \n\t"\
 209         "paddd                     %%mm0, %%mm4         \n\t"\
 210         "paddd                     %%mm3, %%mm5         \n\t"\
 211         "add                          %6, %%"REG_S"      \n\t" \
 212         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
 213         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
 214         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
 215         "test                  %%"REG_S", %%"REG_S"     \n\t"\
 216         "movq                      %%mm2, %%mm0         \n\t"\
 217         "punpcklwd                 %%mm3, %%mm2         \n\t"\
 218         "punpckhwd                 %%mm3, %%mm0         \n\t"\
 219         "pmaddwd                   %%mm1, %%mm2         \n\t"\
 220         "pmaddwd                   %%mm1, %%mm0         \n\t"\
 221         "paddd                     %%mm2, %%mm6         \n\t"\
 222         "paddd                     %%mm0, %%mm7         \n\t"\
 223         " jnz                         2b                \n\t"\
 224         "psrad                       $16, %%mm4         \n\t"\
 225         "psrad                       $16, %%mm5         \n\t"\
 226         "psrad                       $16, %%mm6         \n\t"\
 227         "psrad                       $16, %%mm7         \n\t"\
 228         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
 229         "packssdw                  %%mm5, %%mm4         \n\t"\
 230         "packssdw                  %%mm7, %%mm6         \n\t"\
 231         "paddw                     %%mm0, %%mm4         \n\t"\
 232         "paddw                     %%mm0, %%mm6         \n\t"\
 233         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
 234         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
 235
 236 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
 237     "lea                "offset"(%0), %%"REG_d"     \n\t"\
 238     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 239     "pxor                      %%mm1, %%mm1         \n\t"\
 240     "pxor                      %%mm5, %%mm5         \n\t"\
 241     "pxor                      %%mm7, %%mm7         \n\t"\
 242     "pxor                      %%mm6, %%mm6         \n\t"\
 243     ".p2align                      4                \n\t"\
 244     "2:                                             \n\t"\
 245     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
 246     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
 247     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
 248     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
 249     "movq                      %%mm0, %%mm3         \n\t"\
 250     "punpcklwd                 %%mm4, %%mm0         \n\t"\
 251     "punpckhwd                 %%mm4, %%mm3         \n\t"\
 252     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
 253     "pmaddwd                   %%mm4, %%mm0         \n\t"\
 254     "pmaddwd                   %%mm4, %%mm3         \n\t"\
 255     "paddd                     %%mm0, %%mm1         \n\t"\
 256     "paddd                     %%mm3, %%mm5         \n\t"\
 257     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
 258     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
 259     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
 260     "test                  %%"REG_S", %%"REG_S"     \n\t"\
 261     "movq                      %%mm2, %%mm0         \n\t"\
 262     "punpcklwd                 %%mm3, %%mm2         \n\t"\
 263     "punpckhwd                 %%mm3, %%mm0         \n\t"\
 264     "pmaddwd                   %%mm4, %%mm2         \n\t"\
 265     "pmaddwd                   %%mm4, %%mm0         \n\t"\
 266     "paddd                     %%mm2, %%mm7         \n\t"\
 267     "paddd                     %%mm0, %%mm6         \n\t"\
 268     " jnz                         2b                \n\t"\
 269     "psrad                       $16, %%mm1         \n\t"\
 270     "psrad                       $16, %%mm5         \n\t"\
 271     "psrad                       $16, %%mm7         \n\t"\
 272     "psrad                       $16, %%mm6         \n\t"\
 273     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
 274     "packssdw                  %%mm5, %%mm1         \n\t"\
 275     "packssdw                  %%mm6, %%mm7         \n\t"\
 276     "paddw                     %%mm0, %%mm1         \n\t"\
 277     "paddw                     %%mm0, %%mm7         \n\t"\
 278     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
 279     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
 280
 281 #define YSCALEYUV2PACKEDX_ACCURATE \
 282     YSCALEYUV2PACKEDX_ACCURATE_UV \
 283     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
 284
 285 #define YSCALEYUV2RGBX \
 286     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
 287     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
 288     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
 289     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
 290     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
 291     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
 292     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
 293     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
 294     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
 295     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
 296     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
 297     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
 298     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
 299     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
 300     "paddw           %%mm3, %%mm4       \n\t"\
 301     "movq            %%mm2, %%mm0       \n\t"\
 302     "movq            %%mm5, %%mm6       \n\t"\
 303     "movq            %%mm4, %%mm3       \n\t"\
 304     "punpcklwd       %%mm2, %%mm2       \n\t"\
 305     "punpcklwd       %%mm5, %%mm5       \n\t"\
 306     "punpcklwd       %%mm4, %%mm4       \n\t"\
 307     "paddw           %%mm1, %%mm2       \n\t"\
 308     "paddw           %%mm1, %%mm5       \n\t"\
 309     "paddw           %%mm1, %%mm4       \n\t"\
 310     "punpckhwd       %%mm0, %%mm0       \n\t"\
 311     "punpckhwd       %%mm6, %%mm6       \n\t"\
 312     "punpckhwd       %%mm3, %%mm3       \n\t"\
 313     "paddw           %%mm7, %%mm0       \n\t"\
 314     "paddw           %%mm7, %%mm6       \n\t"\
 315     "paddw           %%mm7, %%mm3       \n\t"\
 316     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
 317     "packuswb        %%mm0, %%mm2       \n\t"\
 318     "packuswb        %%mm6, %%mm5       \n\t"\
 319     "packuswb        %%mm3, %%mm4       \n\t"\
 320
 321 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
 322     "movq       "#b", "#q2"     \n\t" /* B */\
 323     "movq       "#r", "#t"      \n\t" /* R */\
 324     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
 325     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
 326     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
 327     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
 328     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
 329     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
 330     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
 331     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
 332     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
 333     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
 334 \
 335     MOVNTQ(   q0,   (dst, index, 4))\
 336     MOVNTQ(    b,  8(dst, index, 4))\
 337     MOVNTQ(   q2, 16(dst, index, 4))\
 338     MOVNTQ(   q3, 24(dst, index, 4))\
 339 \
 340     "add      $8, "#index"      \n\t"\
 341     "cmp "#dstw", "#index"      \n\t"\
 342     " jb      1b                \n\t"
 343 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
 344
 345 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
 346                                    const int16_t **lumSrc, int lumFilterSize,
 347                                    const int16_t *chrFilter, const int16_t **chrUSrc,
 348                                    const int16_t **chrVSrc,
 349                                    int chrFilterSize, const int16_t **alpSrc,
 350                                    uint8_t *dest, int dstW, int dstY)
 351 {
 352     x86_reg dummy=0;
 353     x86_reg dstW_reg = dstW;
 354     x86_reg uv_off = c->uv_offx2;
 355
 356     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 357         YSCALEYUV2PACKEDX_ACCURATE
 358         YSCALEYUV2RGBX
 359         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
 360         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
 361         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
 362         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
 363         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
 364         "psraw                        $3, %%mm1         \n\t"
 365         "psraw                        $3, %%mm7         \n\t"
 366         "packuswb                  %%mm7, %%mm1         \n\t"
 367         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
 368         YSCALEYUV2PACKEDX_END
 369     } else {
 370         YSCALEYUV2PACKEDX_ACCURATE
 371         YSCALEYUV2RGBX
 372         "pcmpeqd %%mm7, %%mm7 \n\t"
 373         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 374         YSCALEYUV2PACKEDX_END
 375     }
 376 }
 377
 378 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
 379                                 const int16_t **lumSrc, int lumFilterSize,
 380                                 const int16_t *chrFilter, const int16_t **chrUSrc,
 381                                 const int16_t **chrVSrc,
 382                                 int chrFilterSize, const int16_t **alpSrc,
 383                                 uint8_t *dest, int dstW, int dstY)
 384 {
 385     x86_reg dummy=0;
 386     x86_reg dstW_reg = dstW;
 387     x86_reg uv_off = c->uv_offx2;
 388
 389     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 390         YSCALEYUV2PACKEDX
 391         YSCALEYUV2RGBX
 392         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
 393         "psraw                        $3, %%mm1         \n\t"
 394         "psraw                        $3, %%mm7         \n\t"
 395         "packuswb                  %%mm7, %%mm1         \n\t"
 396         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 397         YSCALEYUV2PACKEDX_END
 398     } else {
 399         YSCALEYUV2PACKEDX
 400         YSCALEYUV2RGBX
 401         "pcmpeqd %%mm7, %%mm7 \n\t"
 402         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 403         YSCALEYUV2PACKEDX_END
 404     }
 405 }
 406
 407 #define REAL_WRITERGB16(dst, dstw, index) \
 408     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
 409     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
 410     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
 411     "psrlq           $3, %%mm2  \n\t"\
 412 \
 413     "movq         %%mm2, %%mm1  \n\t"\
 414     "movq         %%mm4, %%mm3  \n\t"\
 415 \
 416     "punpcklbw    %%mm7, %%mm3  \n\t"\
 417     "punpcklbw    %%mm5, %%mm2  \n\t"\
 418     "punpckhbw    %%mm7, %%mm4  \n\t"\
 419     "punpckhbw    %%mm5, %%mm1  \n\t"\
 420 \
 421     "psllq           $3, %%mm3  \n\t"\
 422     "psllq           $3, %%mm4  \n\t"\
 423 \
 424     "por          %%mm3, %%mm2  \n\t"\
 425     "por          %%mm4, %%mm1  \n\t"\
 426 \
 427     MOVNTQ(%%mm2,  (dst, index, 2))\
 428     MOVNTQ(%%mm1, 8(dst, index, 2))\
 429 \
 430     "add             $8, "#index"   \n\t"\
 431     "cmp        "#dstw", "#index"   \n\t"\
 432     " jb             1b             \n\t"
 433 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
 434
 435 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
 436                                     const int16_t **lumSrc, int lumFilterSize,
 437                                     const int16_t *chrFilter, const int16_t **chrUSrc,
 438                                     const int16_t **chrVSrc,
 439                                     int chrFilterSize, const int16_t **alpSrc,
 440                                     uint8_t *dest, int dstW, int dstY)
 441 {
 442     x86_reg dummy=0;
 443     x86_reg dstW_reg = dstW;
 444     x86_reg uv_off = c->uv_offx2;
 445
 446     YSCALEYUV2PACKEDX_ACCURATE
 447     YSCALEYUV2RGBX
 448     "pxor %%mm7, %%mm7 \n\t"
 449     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 450 #ifdef DITHER1XBPP
 451     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
 452     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
 453     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
 454 #endif
 455     WRITERGB16(%4, %5, %%REGa)
 456     YSCALEYUV2PACKEDX_END
 457 }
 458
 459 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
 460                                  const int16_t **lumSrc, int lumFilterSize,
 461                                  const int16_t *chrFilter, const int16_t **chrUSrc,
 462                                  const int16_t **chrVSrc,
 463                                  int chrFilterSize, const int16_t **alpSrc,
 464                                  uint8_t *dest, int dstW, int dstY)
 465 {
 466     x86_reg dummy=0;
 467     x86_reg dstW_reg = dstW;
 468     x86_reg uv_off = c->uv_offx2;
 469
 470     YSCALEYUV2PACKEDX
 471     YSCALEYUV2RGBX
 472     "pxor %%mm7, %%mm7 \n\t"
 473     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 474 #ifdef DITHER1XBPP
 475     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
 476     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
 477     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
 478 #endif
 479     WRITERGB16(%4, %5, %%REGa)
 480     YSCALEYUV2PACKEDX_END
 481 }
 482
 483 #define REAL_WRITERGB15(dst, dstw, index) \
 484     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
 485     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
 486     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
 487     "psrlq           $3, %%mm2  \n\t"\
 488     "psrlq           $1, %%mm5  \n\t"\
 489 \
 490     "movq         %%mm2, %%mm1  \n\t"\
 491     "movq         %%mm4, %%mm3  \n\t"\
 492 \
 493     "punpcklbw    %%mm7, %%mm3  \n\t"\
 494     "punpcklbw    %%mm5, %%mm2  \n\t"\
 495     "punpckhbw    %%mm7, %%mm4  \n\t"\
 496     "punpckhbw    %%mm5, %%mm1  \n\t"\
 497 \
 498     "psllq           $2, %%mm3  \n\t"\
 499     "psllq           $2, %%mm4  \n\t"\
 500 \
 501     "por          %%mm3, %%mm2  \n\t"\
 502     "por          %%mm4, %%mm1  \n\t"\
 503 \
 504     MOVNTQ(%%mm2,  (dst, index, 2))\
 505     MOVNTQ(%%mm1, 8(dst, index, 2))\
 506 \
 507     "add             $8, "#index"   \n\t"\
 508     "cmp        "#dstw", "#index"   \n\t"\
 509     " jb             1b             \n\t"
 510 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
 511
 512 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
 513                                     const int16_t **lumSrc, int lumFilterSize,
 514                                     const int16_t *chrFilter, const int16_t **chrUSrc,
 515                                     const int16_t **chrVSrc,
 516                                     int chrFilterSize, const int16_t **alpSrc,
 517                                     uint8_t *dest, int dstW, int dstY)
 518 {
 519     x86_reg dummy=0;
 520     x86_reg dstW_reg = dstW;
 521     x86_reg uv_off = c->uv_offx2;
 522
 523     YSCALEYUV2PACKEDX_ACCURATE
 524     YSCALEYUV2RGBX
 525     "pxor %%mm7, %%mm7 \n\t"
 526     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 527 #ifdef DITHER1XBPP
 528     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
 529     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
 530     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
 531 #endif
 532     WRITERGB15(%4, %5, %%REGa)
 533     YSCALEYUV2PACKEDX_END
 534 }
 535
 536 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
 537                                  const int16_t **lumSrc, int lumFilterSize,
 538                                  const int16_t *chrFilter, const int16_t **chrUSrc,
 539                                  const int16_t **chrVSrc,
 540                                  int chrFilterSize, const int16_t **alpSrc,
 541                                  uint8_t *dest, int dstW, int dstY)
 542 {
 543     x86_reg dummy=0;
 544     x86_reg dstW_reg = dstW;
 545     x86_reg uv_off = c->uv_offx2;
 546
 547     YSCALEYUV2PACKEDX
 548     YSCALEYUV2RGBX
 549     "pxor %%mm7, %%mm7 \n\t"
 550     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 551 #ifdef DITHER1XBPP
 552     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
 553     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
 554     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
 555 #endif
 556     WRITERGB15(%4, %5, %%REGa)
 557     YSCALEYUV2PACKEDX_END
 558 }
 559
 560 #define WRITEBGR24MMX(dst, dstw, index) \
 561     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
 562     "movq      %%mm2, %%mm1     \n\t" /* B */\
 563     "movq      %%mm5, %%mm6     \n\t" /* R */\
 564     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
 565     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
 566     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
 567     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
 568     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
 569     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
 570     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
 571     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
 572     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
 573     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
 574 \
 575     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
 576     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
 577     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
 578     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
 579 \
 580     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
 581     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
 582     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
 583     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
 584 \
 585     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
 586     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
 587     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
 588     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
 589 \
 590     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
 591     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
 592     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
 593     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
 594     MOVNTQ(%%mm0, (dst))\
 595 \
 596     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
 597     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
 598     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
 599     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
 600     MOVNTQ(%%mm6, 8(dst))\
 601 \
 602     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
 603     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
 604     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
 605     MOVNTQ(%%mm5, 16(dst))\
 606 \
 607     "add         $24, "#dst"    \n\t"\
 608 \
 609     "add          $8, "#index"  \n\t"\
 610     "cmp     "#dstw", "#index"  \n\t"\
 611     " jb          1b            \n\t"
 612
 613 #define WRITEBGR24MMXEXT(dst, dstw, index) \
 614     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
 615     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
 616     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
 617     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
 618     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
 619     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
 620 \
 621     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
 622     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
 623     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
 624 \
 625     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
 626     "por    %%mm1, %%mm6        \n\t"\
 627     "por    %%mm3, %%mm6        \n\t"\
 628     MOVNTQ(%%mm6, (dst))\
 629 \
 630     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
 631     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
 632     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
 633     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
 634 \
 635     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
 636     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
 637     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
 638 \
 639     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
 640     "por    %%mm3, %%mm6        \n\t"\
 641     MOVNTQ(%%mm6, 8(dst))\
 642 \
 643     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
 644     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
 645     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
 646 \
 647     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
 648     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
 649     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
 650 \
 651     "por    %%mm1, %%mm3        \n\t"\
 652     "por    %%mm3, %%mm6        \n\t"\
 653     MOVNTQ(%%mm6, 16(dst))\
 654 \
 655     "add      $24, "#dst"       \n\t"\
 656 \
 657     "add       $8, "#index"     \n\t"\
 658     "cmp  "#dstw", "#index"     \n\t"\
 659     " jb       1b               \n\t"
 660
 661 #if COMPILE_TEMPLATE_MMXEXT
 662 #undef WRITEBGR24
 663 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMXEXT(dst, dstw, index)
 664 #else
 665 #undef WRITEBGR24
 666 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
 667 #endif
 668
 669 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
 670                                    const int16_t **lumSrc, int lumFilterSize,
 671                                    const int16_t *chrFilter, const int16_t **chrUSrc,
 672                                    const int16_t **chrVSrc,
 673                                    int chrFilterSize, const int16_t **alpSrc,
 674                                    uint8_t *dest, int dstW, int dstY)
 675 {
 676     x86_reg dummy=0;
 677     x86_reg dstW_reg = dstW;
 678     x86_reg uv_off = c->uv_offx2;
 679
 680     YSCALEYUV2PACKEDX_ACCURATE
 681     YSCALEYUV2RGBX
 682     "pxor %%mm7, %%mm7 \n\t"
 683     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
 684     "add %4, %%"REG_c"                        \n\t"
 685     WRITEBGR24(%%REGc, %5, %%REGa)
 686     :: "r" (&c->redDither),
 687        "m" (dummy), "m" (dummy), "m" (dummy),
 688        "r" (dest), "m" (dstW_reg), "m"(uv_off)
 689        NAMED_CONSTRAINTS_ADD(ff_M24A,ff_M24C,ff_M24B)
 690     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
 691     );
 692 }
 693
 694 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
 695                                 const int16_t **lumSrc, int lumFilterSize,
 696                                 const int16_t *chrFilter, const int16_t **chrUSrc,
 697                                 const int16_t **chrVSrc,
 698                                 int chrFilterSize, const int16_t **alpSrc,
 699                                 uint8_t *dest, int dstW, int dstY)
 700 {
 701     x86_reg dummy=0;
 702     x86_reg dstW_reg = dstW;
 703     x86_reg uv_off = c->uv_offx2;
 704
 705     YSCALEYUV2PACKEDX
 706     YSCALEYUV2RGBX
 707     "pxor                    %%mm7, %%mm7       \n\t"
 708     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
 709     "add                        %4, %%"REG_c"   \n\t"
 710     WRITEBGR24(%%REGc, %5, %%REGa)
 711     :: "r" (&c->redDither),
 712        "m" (dummy), "m" (dummy), "m" (dummy),
 713        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
 714        NAMED_CONSTRAINTS_ADD(ff_M24A,ff_M24C,ff_M24B)
 715     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
 716     );
 717 }
 718
 719 #define REAL_WRITEYUY2(dst, dstw, index) \
 720     "packuswb  %%mm3, %%mm3     \n\t"\
 721     "packuswb  %%mm4, %%mm4     \n\t"\
 722     "packuswb  %%mm7, %%mm1     \n\t"\
 723     "punpcklbw %%mm4, %%mm3     \n\t"\
 724     "movq      %%mm1, %%mm7     \n\t"\
 725     "punpcklbw %%mm3, %%mm1     \n\t"\
 726     "punpckhbw %%mm3, %%mm7     \n\t"\
 727 \
 728     MOVNTQ(%%mm1, (dst, index, 2))\
 729     MOVNTQ(%%mm7, 8(dst, index, 2))\
 730 \
 731     "add          $8, "#index"  \n\t"\
 732     "cmp     "#dstw", "#index"  \n\t"\
 733     " jb          1b            \n\t"
 734 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
 735
 736 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
 737                                      const int16_t **lumSrc, int lumFilterSize,
 738                                      const int16_t *chrFilter, const int16_t **chrUSrc,
 739                                      const int16_t **chrVSrc,
 740                                      int chrFilterSize, const int16_t **alpSrc,
 741                                      uint8_t *dest, int dstW, int dstY)
 742 {
 743     x86_reg dummy=0;
 744     x86_reg dstW_reg = dstW;
 745     x86_reg uv_off = c->uv_offx2;
 746
 747     YSCALEYUV2PACKEDX_ACCURATE
 748     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 749     "psraw $3, %%mm3    \n\t"
 750     "psraw $3, %%mm4    \n\t"
 751     "psraw $3, %%mm1    \n\t"
 752     "psraw $3, %%mm7    \n\t"
 753     WRITEYUY2(%4, %5, %%REGa)
 754     YSCALEYUV2PACKEDX_END
 755 }
 756
 757 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
 758                                   const int16_t **lumSrc, int lumFilterSize,
 759                                   const int16_t *chrFilter, const int16_t **chrUSrc,
 760                                   const int16_t **chrVSrc,
 761                                   int chrFilterSize, const int16_t **alpSrc,
 762                                   uint8_t *dest, int dstW, int dstY)
 763 {
 764     x86_reg dummy=0;
 765     x86_reg dstW_reg = dstW;
 766     x86_reg uv_off = c->uv_offx2;
 767
 768     YSCALEYUV2PACKEDX
 769     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 770     "psraw $3, %%mm3    \n\t"
 771     "psraw $3, %%mm4    \n\t"
 772     "psraw $3, %%mm1    \n\t"
 773     "psraw $3, %%mm7    \n\t"
 774     WRITEYUY2(%4, %5, %%REGa)
 775     YSCALEYUV2PACKEDX_END
 776 }
 777
 778 #define REAL_YSCALEYUV2RGB_UV(index, c) \
 779     "xor            "#index", "#index"  \n\t"\
 780     ".p2align              4            \n\t"\
 781     "1:                                 \n\t"\
 782     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
 783     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
 784     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
 785     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
 786     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
 787     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
 788     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
 789     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
 790     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
 791     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
 792     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
 793     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
 794     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
 795     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
 796     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
 797     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
 798     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
 799     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
 800     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
 801     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
 802     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
 803     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
 804
 805 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
 806     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
 807     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
 808     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
 809     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
 810     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
 811     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
 812     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 813     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 814     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 815     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 816     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 817     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 818
 819 #define REAL_YSCALEYUV2RGB_COEFF(c) \
 820     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
 821     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
 822     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
 823     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
 824     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
 825     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
 826     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
 827     "paddw             %%mm3, %%mm4     \n\t"\
 828     "movq              %%mm2, %%mm0     \n\t"\
 829     "movq              %%mm5, %%mm6     \n\t"\
 830     "movq              %%mm4, %%mm3     \n\t"\
 831     "punpcklwd         %%mm2, %%mm2     \n\t"\
 832     "punpcklwd         %%mm5, %%mm5     \n\t"\
 833     "punpcklwd         %%mm4, %%mm4     \n\t"\
 834     "paddw             %%mm1, %%mm2     \n\t"\
 835     "paddw             %%mm1, %%mm5     \n\t"\
 836     "paddw             %%mm1, %%mm4     \n\t"\
 837     "punpckhwd         %%mm0, %%mm0     \n\t"\
 838     "punpckhwd         %%mm6, %%mm6     \n\t"\
 839     "punpckhwd         %%mm3, %%mm3     \n\t"\
 840     "paddw             %%mm7, %%mm0     \n\t"\
 841     "paddw             %%mm7, %%mm6     \n\t"\
 842     "paddw             %%mm7, %%mm3     \n\t"\
 843     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
 844     "packuswb          %%mm0, %%mm2     \n\t"\
 845     "packuswb          %%mm6, %%mm5     \n\t"\
 846     "packuswb          %%mm3, %%mm4     \n\t"\
 847
 848 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
 849
 850 #define YSCALEYUV2RGB(index, c) \
 851     REAL_YSCALEYUV2RGB_UV(index, c) \
 852     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
 853     REAL_YSCALEYUV2RGB_COEFF(c)
 854
 855 /**
 856  * vertical bilinear scale YV12 to RGB
 857  */
 858 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
 859                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
 860                                 const int16_t *abuf[2], uint8_t *dest,
 861                                 int dstW, int yalpha, int uvalpha, int y)
 862 {
 863     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 864                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 865
 866     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 867         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
 868 #if ARCH_X86_64
 869         __asm__ volatile(
 870             YSCALEYUV2RGB(%%r8, %5)
 871             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
 872             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 873             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 874             "packuswb            %%mm7, %%mm1       \n\t"
 875             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 876             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
 877                "a" (&c->redDither),
 878                "r" (abuf0), "r" (abuf1)
 879             : "%r8"
 880         );
 881 #else
 882         c->u_temp=(intptr_t)abuf0;
 883         c->v_temp=(intptr_t)abuf1;
 884         __asm__ volatile(
 885             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 886             "mov        %4, %%"REG_b"               \n\t"
 887             "push %%"REG_BP"                        \n\t"
 888             YSCALEYUV2RGB(%%REGBP, %5)
 889             "push                   %0              \n\t"
 890             "push                   %1              \n\t"
 891             "mov          "U_TEMP"(%5), %0          \n\t"
 892             "mov          "V_TEMP"(%5), %1          \n\t"
 893             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
 894             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 895             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 896             "packuswb            %%mm7, %%mm1       \n\t"
 897             "pop                    %1              \n\t"
 898             "pop                    %0              \n\t"
 899             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 900             "pop %%"REG_BP"                         \n\t"
 901             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 902             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 903                "a" (&c->redDither)
 904         );
 905 #endif
 906     } else {
 907         __asm__ volatile(
 908             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 909             "mov        %4, %%"REG_b"               \n\t"
 910             "push %%"REG_BP"                        \n\t"
 911             YSCALEYUV2RGB(%%REGBP, %5)
 912             "pcmpeqd %%mm7, %%mm7                   \n\t"
 913             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 914             "pop %%"REG_BP"                         \n\t"
 915             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 916             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 917                "a" (&c->redDither)
 918         );
 919     }
 920 }
 921
 922 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
 923                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
 924                                 const int16_t *abuf[2], uint8_t *dest,
 925                                 int dstW, int yalpha, int uvalpha, int y)
 926 {
 927     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 928                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 929
 930     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 931     __asm__ volatile(
 932         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 933         "mov        %4, %%"REG_b"               \n\t"
 934         "push %%"REG_BP"                        \n\t"
 935         YSCALEYUV2RGB(%%REGBP, %5)
 936         "pxor    %%mm7, %%mm7                   \n\t"
 937         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
 938         "pop %%"REG_BP"                         \n\t"
 939         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 940         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 941            "a" (&c->redDither)
 942            NAMED_CONSTRAINTS_ADD(ff_M24A,ff_M24C,ff_M24B)
 943     );
 944 }
 945
 946 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
 947                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
 948                                  const int16_t *abuf[2], uint8_t *dest,
 949                                  int dstW, int yalpha, int uvalpha, int y)
 950 {
 951     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 952                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 953
 954     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 955     __asm__ volatile(
 956         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 957         "mov        %4, %%"REG_b"               \n\t"
 958         "push %%"REG_BP"                        \n\t"
 959         YSCALEYUV2RGB(%%REGBP, %5)
 960         "pxor    %%mm7, %%mm7                   \n\t"
 961         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 962 #ifdef DITHER1XBPP
 963         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
 964         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
 965         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
 966 #endif
 967         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
 968         "pop %%"REG_BP"                         \n\t"
 969         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 970         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 971            "a" (&c->redDither)
 972            NAMED_CONSTRAINTS_ADD(bF8)
 973     );
 974 }
 975
 976 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
 977                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
 978                                  const int16_t *abuf[2], uint8_t *dest,
 979                                  int dstW, int yalpha, int uvalpha, int y)
 980 {
 981     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 982                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 983
 984     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 985     __asm__ volatile(
 986         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 987         "mov        %4, %%"REG_b"               \n\t"
 988         "push %%"REG_BP"                        \n\t"
 989         YSCALEYUV2RGB(%%REGBP, %5)
 990         "pxor    %%mm7, %%mm7                   \n\t"
 991         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 992 #ifdef DITHER1XBPP
 993         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
 994         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
 995         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
 996 #endif
 997         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
 998         "pop %%"REG_BP"                         \n\t"
 999         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1000         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1001            "a" (&c->redDither)
1002            NAMED_CONSTRAINTS_ADD(bF8,bFC)
1003     );
1004 }
1005
1006 #define REAL_YSCALEYUV2PACKED(index, c) \
1007     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1008     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1009     "psraw                $3, %%mm0                           \n\t"\
1010     "psraw                $3, %%mm1                           \n\t"\
1011     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1012     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1013     "xor            "#index", "#index"                        \n\t"\
1014     ".p2align              4            \n\t"\
1015     "1:                                 \n\t"\
1016     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1017     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1018     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1019     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1020     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1021     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1022     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1023     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1024     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1025     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1026     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1027     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1028     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1029     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1030     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1031     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1032     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1033     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1034     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1035     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1036     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1037     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1038     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1039     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1040     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1041     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1042     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1043
1044 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1045
1046 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
1047                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
1048                                   const int16_t *abuf[2], uint8_t *dest,
1049                                   int dstW, int yalpha, int uvalpha, int y)
1050 {
1051     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1052                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1053
1054     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1055     __asm__ volatile(
1056         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1057         "mov %4, %%"REG_b"                        \n\t"
1058         "push %%"REG_BP"                        \n\t"
1059         YSCALEYUV2PACKED(%%REGBP, %5)
1060         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1061         "pop %%"REG_BP"                         \n\t"
1062         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1063         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1064            "a" (&c->redDither)
1065     );
1066 }
1067
1068 #define REAL_YSCALEYUV2RGB1(index, c) \
1069     "xor            "#index", "#index"  \n\t"\
1070     ".p2align              4            \n\t"\
1071     "1:                                 \n\t"\
1072     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1073     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1074     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1075     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1076     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1077     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1078     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1079     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1080     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1081     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1082     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1083     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1084     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1085     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1086     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1087     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1088     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1089     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1090     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1091     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1092     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1093     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1094     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1095     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1096     "paddw             %%mm3, %%mm4     \n\t"\
1097     "movq              %%mm2, %%mm0     \n\t"\
1098     "movq              %%mm5, %%mm6     \n\t"\
1099     "movq              %%mm4, %%mm3     \n\t"\
1100     "punpcklwd         %%mm2, %%mm2     \n\t"\
1101     "punpcklwd         %%mm5, %%mm5     \n\t"\
1102     "punpcklwd         %%mm4, %%mm4     \n\t"\
1103     "paddw             %%mm1, %%mm2     \n\t"\
1104     "paddw             %%mm1, %%mm5     \n\t"\
1105     "paddw             %%mm1, %%mm4     \n\t"\
1106     "punpckhwd         %%mm0, %%mm0     \n\t"\
1107     "punpckhwd         %%mm6, %%mm6     \n\t"\
1108     "punpckhwd         %%mm3, %%mm3     \n\t"\
1109     "paddw             %%mm7, %%mm0     \n\t"\
1110     "paddw             %%mm7, %%mm6     \n\t"\
1111     "paddw             %%mm7, %%mm3     \n\t"\
1112     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1113     "packuswb          %%mm0, %%mm2     \n\t"\
1114     "packuswb          %%mm6, %%mm5     \n\t"\
1115     "packuswb          %%mm3, %%mm4     \n\t"\
1116
1117 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1118
1119 // do vertical chrominance interpolation
1120 #define REAL_YSCALEYUV2RGB1b(index, c) \
1121     "xor            "#index", "#index"  \n\t"\
1122     ".p2align              4            \n\t"\
1123     "1:                                 \n\t"\
1124     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1125     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1126     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1127     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1128     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1129     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1130     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1131     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1132     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1133     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1134     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1135     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1136     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1137     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1138     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1139     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1140     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1141     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1142     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1143     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1144     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1145     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1146     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1147     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1148     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1149     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1150     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1151     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1152     "paddw             %%mm3, %%mm4     \n\t"\
1153     "movq              %%mm2, %%mm0     \n\t"\
1154     "movq              %%mm5, %%mm6     \n\t"\
1155     "movq              %%mm4, %%mm3     \n\t"\
1156     "punpcklwd         %%mm2, %%mm2     \n\t"\
1157     "punpcklwd         %%mm5, %%mm5     \n\t"\
1158     "punpcklwd         %%mm4, %%mm4     \n\t"\
1159     "paddw             %%mm1, %%mm2     \n\t"\
1160     "paddw             %%mm1, %%mm5     \n\t"\
1161     "paddw             %%mm1, %%mm4     \n\t"\
1162     "punpckhwd         %%mm0, %%mm0     \n\t"\
1163     "punpckhwd         %%mm6, %%mm6     \n\t"\
1164     "punpckhwd         %%mm3, %%mm3     \n\t"\
1165     "paddw             %%mm7, %%mm0     \n\t"\
1166     "paddw             %%mm7, %%mm6     \n\t"\
1167     "paddw             %%mm7, %%mm3     \n\t"\
1168     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1169     "packuswb          %%mm0, %%mm2     \n\t"\
1170     "packuswb          %%mm6, %%mm5     \n\t"\
1171     "packuswb          %%mm3, %%mm4     \n\t"\
1172
1173 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1174
1175 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1176     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1177     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1178     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1179     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1180     "packuswb          %%mm1, %%mm7     \n\t"
1181 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1182
1183 /**
1184  * YV12 to RGB without scaling or interpolating
1185  */
1186 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1187                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
1188                                 const int16_t *abuf0, uint8_t *dest,
1189                                 int dstW, int uvalpha, int y)
1190 {
1191     const int16_t *ubuf0 = ubuf[0];
1192     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1193
1194     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1195         const int16_t *ubuf1 = ubuf[0];
1196         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1197             __asm__ volatile(
1198                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1199                 "mov        %4, %%"REG_b"               \n\t"
1200                 "push %%"REG_BP"                        \n\t"
1201                 YSCALEYUV2RGB1(%%REGBP, %5)
1202                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1203                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1204                 "pop %%"REG_BP"                         \n\t"
1205                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1206                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1207                    "a" (&c->redDither)
1208             );
1209         } else {
1210             __asm__ volatile(
1211                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1212                 "mov        %4, %%"REG_b"               \n\t"
1213                 "push %%"REG_BP"                        \n\t"
1214                 YSCALEYUV2RGB1(%%REGBP, %5)
1215                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1216                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1217                 "pop %%"REG_BP"                         \n\t"
1218                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1219                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1220                    "a" (&c->redDither)
1221             );
1222         }
1223     } else {
1224         const int16_t *ubuf1 = ubuf[1];
1225         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1226             __asm__ volatile(
1227                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1228                 "mov        %4, %%"REG_b"               \n\t"
1229                 "push %%"REG_BP"                        \n\t"
1230                 YSCALEYUV2RGB1b(%%REGBP, %5)
1231                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1232                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1233                 "pop %%"REG_BP"                         \n\t"
1234                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1235                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1236                    "a" (&c->redDither)
1237             );
1238         } else {
1239             __asm__ volatile(
1240                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1241                 "mov        %4, %%"REG_b"               \n\t"
1242                 "push %%"REG_BP"                        \n\t"
1243                 YSCALEYUV2RGB1b(%%REGBP, %5)
1244                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1245                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1246                 "pop %%"REG_BP"                         \n\t"
1247                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1248                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1249                    "a" (&c->redDither)
1250             );
1251         }
1252     }
1253 }
1254
1255 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1256                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
1257                                 const int16_t *abuf0, uint8_t *dest,
1258                                 int dstW, int uvalpha, int y)
1259 {
1260     const int16_t *ubuf0 = ubuf[0];
1261     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1262
1263     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1264         const int16_t *ubuf1 = ubuf[0];
1265         __asm__ volatile(
1266             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1267             "mov        %4, %%"REG_b"               \n\t"
1268             "push %%"REG_BP"                        \n\t"
1269             YSCALEYUV2RGB1(%%REGBP, %5)
1270             "pxor    %%mm7, %%mm7                   \n\t"
1271             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1272             "pop %%"REG_BP"                         \n\t"
1273             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1274             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1275                "a" (&c->redDither)
1276                NAMED_CONSTRAINTS_ADD(ff_M24A,ff_M24C,ff_M24B)
1277         );
1278     } else {
1279         const int16_t *ubuf1 = ubuf[1];
1280         __asm__ volatile(
1281             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1282             "mov        %4, %%"REG_b"               \n\t"
1283             "push %%"REG_BP"                        \n\t"
1284             YSCALEYUV2RGB1b(%%REGBP, %5)
1285             "pxor    %%mm7, %%mm7                   \n\t"
1286             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1287             "pop %%"REG_BP"                         \n\t"
1288             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1289             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1290                "a" (&c->redDither)
1291                NAMED_CONSTRAINTS_ADD(ff_M24A,ff_M24C,ff_M24B)
1292         );
1293     }
1294 }
1295
1296 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1297                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1298                                  const int16_t *abuf0, uint8_t *dest,
1299                                  int dstW, int uvalpha, int y)
1300 {
1301     const int16_t *ubuf0 = ubuf[0];
1302     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1303
1304     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1305         const int16_t *ubuf1 = ubuf[0];
1306         __asm__ volatile(
1307             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1308             "mov        %4, %%"REG_b"               \n\t"
1309             "push %%"REG_BP"                        \n\t"
1310             YSCALEYUV2RGB1(%%REGBP, %5)
1311             "pxor    %%mm7, %%mm7                   \n\t"
1312             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1313 #ifdef DITHER1XBPP
1314             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1315             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1316             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1317 #endif
1318             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1319             "pop %%"REG_BP"                         \n\t"
1320             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1321             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1322                "a" (&c->redDither)
1323                NAMED_CONSTRAINTS_ADD(bF8)
1324         );
1325     } else {
1326         const int16_t *ubuf1 = ubuf[1];
1327         __asm__ volatile(
1328             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1329             "mov        %4, %%"REG_b"               \n\t"
1330             "push %%"REG_BP"                        \n\t"
1331             YSCALEYUV2RGB1b(%%REGBP, %5)
1332             "pxor    %%mm7, %%mm7                   \n\t"
1333             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1334 #ifdef DITHER1XBPP
1335             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1336             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1337             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1338 #endif
1339             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1340             "pop %%"REG_BP"                         \n\t"
1341             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1342             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1343                "a" (&c->redDither)
1344                NAMED_CONSTRAINTS_ADD(bF8)
1345         );
1346     }
1347 }
1348
1349 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1350                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1351                                  const int16_t *abuf0, uint8_t *dest,
1352                                  int dstW, int uvalpha, int y)
1353 {
1354     const int16_t *ubuf0 = ubuf[0];
1355     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1356
1357     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1358         const int16_t *ubuf1 = ubuf[0];
1359         __asm__ volatile(
1360             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1361             "mov        %4, %%"REG_b"               \n\t"
1362             "push %%"REG_BP"                        \n\t"
1363             YSCALEYUV2RGB1(%%REGBP, %5)
1364             "pxor    %%mm7, %%mm7                   \n\t"
1365             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1366 #ifdef DITHER1XBPP
1367             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1368             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1369             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1370 #endif
1371             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1372             "pop %%"REG_BP"                         \n\t"
1373             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1374             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1375                "a" (&c->redDither)
1376                NAMED_CONSTRAINTS_ADD(bF8,bFC)
1377         );
1378     } else {
1379         const int16_t *ubuf1 = ubuf[1];
1380         __asm__ volatile(
1381             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1382             "mov        %4, %%"REG_b"               \n\t"
1383             "push %%"REG_BP"                        \n\t"
1384             YSCALEYUV2RGB1b(%%REGBP, %5)
1385             "pxor    %%mm7, %%mm7                   \n\t"
1386             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1387 #ifdef DITHER1XBPP
1388             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1389             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1390             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1391 #endif
1392             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1393             "pop %%"REG_BP"                         \n\t"
1394             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1395             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1396                "a" (&c->redDither)
1397                NAMED_CONSTRAINTS_ADD(bF8,bFC)
1398         );
1399     }
1400 }
1401
1402 #define REAL_YSCALEYUV2PACKED1(index, c) \
1403     "xor            "#index", "#index"  \n\t"\
1404     ".p2align              4            \n\t"\
1405     "1:                                 \n\t"\
1406     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1407     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1408     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1409     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1410     "psraw                $7, %%mm3     \n\t" \
1411     "psraw                $7, %%mm4     \n\t" \
1412     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1413     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1414     "psraw                $7, %%mm1     \n\t" \
1415     "psraw                $7, %%mm7     \n\t" \
1416
1417 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1418
1419 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1420     "xor "#index", "#index"             \n\t"\
1421     ".p2align              4            \n\t"\
1422     "1:                                 \n\t"\
1423     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1424     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1425     "add "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1426     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1427     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1428     "sub "UV_OFF_BYTE"("#c"), "#index"  \n\t" \
1429     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1430     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1431     "psrlw                $8, %%mm3     \n\t" \
1432     "psrlw                $8, %%mm4     \n\t" \
1433     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1434     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1435     "psraw                $7, %%mm1     \n\t" \
1436     "psraw                $7, %%mm7     \n\t"
1437 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1438
1439 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1440                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
1441                                   const int16_t *abuf0, uint8_t *dest,
1442                                   int dstW, int uvalpha, int y)
1443 {
1444     const int16_t *ubuf0 = ubuf[0];
1445     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1446
1447     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1448         const int16_t *ubuf1 = ubuf[0];
1449         __asm__ volatile(
1450             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1451             "mov        %4, %%"REG_b"               \n\t"
1452             "push %%"REG_BP"                        \n\t"
1453             YSCALEYUV2PACKED1(%%REGBP, %5)
1454             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1455             "pop %%"REG_BP"                         \n\t"
1456             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1457             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1458                "a" (&c->redDither)
1459         );
1460     } else {
1461         const int16_t *ubuf1 = ubuf[1];
1462         __asm__ volatile(
1463             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1464             "mov        %4, %%"REG_b"               \n\t"
1465             "push %%"REG_BP"                        \n\t"
1466             YSCALEYUV2PACKED1b(%%REGBP, %5)
1467             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1468             "pop %%"REG_BP"                         \n\t"
1469             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1470             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1471                "a" (&c->redDither)
1472         );
1473     }
1474 }
1475
1476 #if COMPILE_TEMPLATE_MMXEXT
1477 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1478                                  int dstWidth, const uint8_t *src,
1479                                  int srcW, int xInc)
1480 {
1481     int32_t *filterPos = c->hLumFilterPos;
1482     int16_t *filter    = c->hLumFilter;
1483     void    *mmxextFilterCode = c->lumMmxextFilterCode;
1484     int i;
1485 #if defined(PIC)
1486     uint64_t ebxsave;
1487 #endif
1488 #if ARCH_X86_64
1489     uint64_t retsave;
1490 #endif
1491
1492     __asm__ volatile(
1493 #if defined(PIC)
1494         "mov               %%"REG_b", %5        \n\t"
1495 #if ARCH_X86_64
1496         "mov               -8(%%rsp), %%"REG_a" \n\t"
1497         "mov               %%"REG_a", %6        \n\t"
1498 #endif
1499 #else
1500 #if ARCH_X86_64
1501         "mov               -8(%%rsp), %%"REG_a" \n\t"
1502         "mov               %%"REG_a", %5        \n\t"
1503 #endif
1504 #endif
1505         "pxor                  %%mm7, %%mm7     \n\t"
1506         "mov                      %0, %%"REG_c" \n\t"
1507         "mov                      %1, %%"REG_D" \n\t"
1508         "mov                      %2, %%"REG_d" \n\t"
1509         "mov                      %3, %%"REG_b" \n\t"
1510         "xor               %%"REG_a", %%"REG_a" \n\t" // i
1511         PREFETCH"        (%%"REG_c")            \n\t"
1512         PREFETCH"      32(%%"REG_c")            \n\t"
1513         PREFETCH"      64(%%"REG_c")            \n\t"
1514
1515 #if ARCH_X86_64
1516 #define CALL_MMXEXT_FILTER_CODE \
1517         "movl            (%%"REG_b"), %%esi     \n\t"\
1518         "call                    *%4            \n\t"\
1519         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
1520         "add               %%"REG_S", %%"REG_c" \n\t"\
1521         "add               %%"REG_a", %%"REG_D" \n\t"\
1522         "xor               %%"REG_a", %%"REG_a" \n\t"\
1523
1524 #else
1525 #define CALL_MMXEXT_FILTER_CODE \
1526         "movl (%%"REG_b"), %%esi        \n\t"\
1527         "call         *%4                       \n\t"\
1528         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
1529         "add               %%"REG_a", %%"REG_D" \n\t"\
1530         "xor               %%"REG_a", %%"REG_a" \n\t"\
1531
1532 #endif /* ARCH_X86_64 */
1533
1534         CALL_MMXEXT_FILTER_CODE
1535         CALL_MMXEXT_FILTER_CODE
1536         CALL_MMXEXT_FILTER_CODE
1537         CALL_MMXEXT_FILTER_CODE
1538         CALL_MMXEXT_FILTER_CODE
1539         CALL_MMXEXT_FILTER_CODE
1540         CALL_MMXEXT_FILTER_CODE
1541         CALL_MMXEXT_FILTER_CODE
1542
1543 #if defined(PIC)
1544         "mov                      %5, %%"REG_b" \n\t"
1545 #if ARCH_X86_64
1546         "mov                      %6, %%"REG_a" \n\t"
1547         "mov               %%"REG_a", -8(%%rsp) \n\t"
1548 #endif
1549 #else
1550 #if ARCH_X86_64
1551         "mov                      %5, %%"REG_a" \n\t"
1552         "mov               %%"REG_a", -8(%%rsp) \n\t"
1553 #endif
1554 #endif
1555         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
1556            "m" (mmxextFilterCode)
1557 #if defined(PIC)
1558           ,"m" (ebxsave)
1559 #endif
1560 #if ARCH_X86_64
1561           ,"m"(retsave)
1562 #endif
1563         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1564 #if !defined(PIC)
1565          ,"%"REG_b
1566 #endif
1567     );
1568
1569     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
1570         dst[i] = src[srcW-1]*128;
1571 }
1572
1573 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
1574                                  int dstWidth, const uint8_t *src1,
1575                                  const uint8_t *src2, int srcW, int xInc)
1576 {
1577     int32_t *filterPos = c->hChrFilterPos;
1578     int16_t *filter    = c->hChrFilter;
1579     void    *mmxextFilterCode = c->chrMmxextFilterCode;
1580     int i;
1581 #if defined(PIC)
1582     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1583 #endif
1584 #if ARCH_X86_64
1585     DECLARE_ALIGNED(8, uint64_t, retsave);
1586 #endif
1587
1588     __asm__ volatile(
1589 #if defined(PIC)
1590         "mov          %%"REG_b", %7         \n\t"
1591 #if ARCH_X86_64
1592         "mov          -8(%%rsp), %%"REG_a"  \n\t"
1593         "mov          %%"REG_a", %8         \n\t"
1594 #endif
1595 #else
1596 #if ARCH_X86_64
1597         "mov          -8(%%rsp), %%"REG_a"  \n\t"
1598         "mov          %%"REG_a", %7         \n\t"
1599 #endif
1600 #endif
1601         "pxor             %%mm7, %%mm7      \n\t"
1602         "mov                 %0, %%"REG_c"  \n\t"
1603         "mov                 %1, %%"REG_D"  \n\t"
1604         "mov                 %2, %%"REG_d"  \n\t"
1605         "mov                 %3, %%"REG_b"  \n\t"
1606         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1607         PREFETCH"   (%%"REG_c")             \n\t"
1608         PREFETCH" 32(%%"REG_c")             \n\t"
1609         PREFETCH" 64(%%"REG_c")             \n\t"
1610
1611         CALL_MMXEXT_FILTER_CODE
1612         CALL_MMXEXT_FILTER_CODE
1613         CALL_MMXEXT_FILTER_CODE
1614         CALL_MMXEXT_FILTER_CODE
1615         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1616         "mov                 %5, %%"REG_c"  \n\t" // src
1617         "mov                 %6, %%"REG_D"  \n\t" // buf2
1618         PREFETCH"   (%%"REG_c")             \n\t"
1619         PREFETCH" 32(%%"REG_c")             \n\t"
1620         PREFETCH" 64(%%"REG_c")             \n\t"
1621
1622         CALL_MMXEXT_FILTER_CODE
1623         CALL_MMXEXT_FILTER_CODE
1624         CALL_MMXEXT_FILTER_CODE
1625         CALL_MMXEXT_FILTER_CODE
1626
1627 #if defined(PIC)
1628         "mov %7, %%"REG_b"    \n\t"
1629 #if ARCH_X86_64
1630         "mov                 %8, %%"REG_a"  \n\t"
1631         "mov          %%"REG_a", -8(%%rsp)  \n\t"
1632 #endif
1633 #else
1634 #if ARCH_X86_64
1635         "mov                 %7, %%"REG_a"  \n\t"
1636         "mov          %%"REG_a", -8(%%rsp)  \n\t"
1637 #endif
1638 #endif
1639         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
1640            "m" (mmxextFilterCode), "m" (src2), "m"(dst2)
1641 #if defined(PIC)
1642           ,"m" (ebxsave)
1643 #endif
1644 #if ARCH_X86_64
1645           ,"m"(retsave)
1646 #endif
1647         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1648 #if !defined(PIC)
1649          ,"%"REG_b
1650 #endif
1651     );
1652
1653     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
1654         dst1[i] = src1[srcW-1]*128;
1655         dst2[i] = src2[srcW-1]*128;
1656     }
1657 }
1658 #endif /* COMPILE_TEMPLATE_MMXEXT */
1659
1660 static av_cold void RENAME(sws_init_swscale)(SwsContext *c)
1661 {
1662     enum AVPixelFormat dstFormat = c->dstFormat;
1663
1664     c->use_mmx_vfilter= 0;
1665     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) && dstFormat != AV_PIX_FMT_NV12
1666         && dstFormat != AV_PIX_FMT_NV21 && !(c->flags & SWS_BITEXACT)) {
1667             if (c->flags & SWS_ACCURATE_RND) {
1668                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1669                     switch (c->dstFormat) {
1670                     case AV_PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
1671                     case AV_PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
1672                     case AV_PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
1673                     case AV_PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
1674                     case AV_PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
1675                     default: break;
1676                     }
1677                 }
1678             } else {
1679                 c->use_mmx_vfilter= 1;
1680                 c->yuv2planeX = RENAME(yuv2yuvX    );
1681                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1682                     switch (c->dstFormat) {
1683                     case AV_PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
1684                     case AV_PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
1685                     case AV_PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
1686                     case AV_PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
1687                     case AV_PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
1688                     default: break;
1689                     }
1690                 }
1691             }
1692         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1693             switch (c->dstFormat) {
1694             case AV_PIX_FMT_RGB32:
1695                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
1696                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
1697                 break;
1698             case AV_PIX_FMT_BGR24:
1699                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
1700                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
1701                 break;
1702             case AV_PIX_FMT_RGB555:
1703                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
1704                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
1705                 break;
1706             case AV_PIX_FMT_RGB565:
1707                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
1708                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
1709                 break;
1710             case AV_PIX_FMT_YUYV422:
1711                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
1712                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
1713                 break;
1714             default:
1715                 break;
1716             }
1717         }
1718     }
1719
1720     if (c->srcBpc == 8 && c->dstBpc <= 14) {
1721     // Use the new MMX scaler if the MMXEXT one can't be used (it is faster than the x86 ASM one).
1722 #if COMPILE_TEMPLATE_MMXEXT
1723     if (c->flags & SWS_FAST_BILINEAR && c->canMMXEXTBeUsed) {
1724         c->hyscale_fast = RENAME(hyscale_fast);
1725         c->hcscale_fast = RENAME(hcscale_fast);
1726     } else {
1727 #endif /* COMPILE_TEMPLATE_MMXEXT */
1728         c->hyscale_fast = NULL;
1729         c->hcscale_fast = NULL;
1730 #if COMPILE_TEMPLATE_MMXEXT
1731     }
1732 #endif /* COMPILE_TEMPLATE_MMXEXT */
1733     }
1734 }