git.sesse.net Git - ffmpeg/blob - libswscale/x86/swscale_template.c

   1 /*
   2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
   3  *
   4  * This file is part of Libav.
   5  *
   6  * Libav is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * Libav is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with Libav; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #undef REAL_MOVNTQ
  22 #undef MOVNTQ
  23 #undef PREFETCH
  24
  25 #if COMPILE_TEMPLATE_MMX2
  26 #define PREFETCH "prefetchnta"
  27 #else
  28 #define PREFETCH  " # nop"
  29 #endif
  30
  31 #if COMPILE_TEMPLATE_MMX2
  32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
  33 #else
  34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
  35 #endif
  36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
  37
  38 #define YSCALEYUV2PACKEDX_UV \
  39     __asm__ volatile(\
  40         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
  41         ".p2align                      4                \n\t"\
  42         "nop                                            \n\t"\
  43         "1:                                             \n\t"\
  44         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
  45         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
  46         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
  47         "movq                      %%mm3, %%mm4         \n\t"\
  48         ".p2align                      4                \n\t"\
  49         "2:                                             \n\t"\
  50         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
  51         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
  52         "add                          %6, %%"REG_S"     \n\t" \
  53         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
  54         "add                         $16, %%"REG_d"     \n\t"\
  55         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
  56         "pmulhw                    %%mm0, %%mm2         \n\t"\
  57         "pmulhw                    %%mm0, %%mm5         \n\t"\
  58         "paddw                     %%mm2, %%mm3         \n\t"\
  59         "paddw                     %%mm5, %%mm4         \n\t"\
  60         "test                  %%"REG_S", %%"REG_S"     \n\t"\
  61         " jnz                         2b                \n\t"\
  62
  63 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
  64     "lea                "offset"(%0), %%"REG_d"     \n\t"\
  65     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
  66     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
  67     "movq                    "#dst1", "#dst2"       \n\t"\
  68     ".p2align                      4                \n\t"\
  69     "2:                                             \n\t"\
  70     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
  71     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
  72     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
  73     "add                         $16, %%"REG_d"            \n\t"\
  74     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
  75     "pmulhw                 "#coeff", "#src1"       \n\t"\
  76     "pmulhw                 "#coeff", "#src2"       \n\t"\
  77     "paddw                   "#src1", "#dst1"       \n\t"\
  78     "paddw                   "#src2", "#dst2"       \n\t"\
  79     "test                  %%"REG_S", %%"REG_S"     \n\t"\
  80     " jnz                         2b                \n\t"\
  81
  82 #define YSCALEYUV2PACKEDX \
  83     YSCALEYUV2PACKEDX_UV \
  84     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
  85
  86 #define YSCALEYUV2PACKEDX_END                     \
  87         :: "r" (&c->redDither),                   \
  88             "m" (dummy), "m" (dummy), "m" (dummy),\
  89             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
  90         : "%"REG_a, "%"REG_d, "%"REG_S            \
  91     );
  92
  93 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
  94     __asm__ volatile(\
  95         "xor %%"REG_a", %%"REG_a"                       \n\t"\
  96         ".p2align                      4                \n\t"\
  97         "nop                                            \n\t"\
  98         "1:                                             \n\t"\
  99         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
 100         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 101         "pxor                      %%mm4, %%mm4         \n\t"\
 102         "pxor                      %%mm5, %%mm5         \n\t"\
 103         "pxor                      %%mm6, %%mm6         \n\t"\
 104         "pxor                      %%mm7, %%mm7         \n\t"\
 105         ".p2align                      4                \n\t"\
 106         "2:                                             \n\t"\
 107         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
 108         "add                          %6, %%"REG_S"      \n\t" \
 109         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
 110         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
 111         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
 112         "movq                      %%mm0, %%mm3         \n\t"\
 113         "punpcklwd                 %%mm1, %%mm0         \n\t"\
 114         "punpckhwd                 %%mm1, %%mm3         \n\t"\
 115         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
 116         "pmaddwd                   %%mm1, %%mm0         \n\t"\
 117         "pmaddwd                   %%mm1, %%mm3         \n\t"\
 118         "paddd                     %%mm0, %%mm4         \n\t"\
 119         "paddd                     %%mm3, %%mm5         \n\t"\
 120         "add                          %6, %%"REG_S"      \n\t" \
 121         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
 122         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
 123         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
 124         "test                  %%"REG_S", %%"REG_S"     \n\t"\
 125         "movq                      %%mm2, %%mm0         \n\t"\
 126         "punpcklwd                 %%mm3, %%mm2         \n\t"\
 127         "punpckhwd                 %%mm3, %%mm0         \n\t"\
 128         "pmaddwd                   %%mm1, %%mm2         \n\t"\
 129         "pmaddwd                   %%mm1, %%mm0         \n\t"\
 130         "paddd                     %%mm2, %%mm6         \n\t"\
 131         "paddd                     %%mm0, %%mm7         \n\t"\
 132         " jnz                         2b                \n\t"\
 133         "psrad                       $16, %%mm4         \n\t"\
 134         "psrad                       $16, %%mm5         \n\t"\
 135         "psrad                       $16, %%mm6         \n\t"\
 136         "psrad                       $16, %%mm7         \n\t"\
 137         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
 138         "packssdw                  %%mm5, %%mm4         \n\t"\
 139         "packssdw                  %%mm7, %%mm6         \n\t"\
 140         "paddw                     %%mm0, %%mm4         \n\t"\
 141         "paddw                     %%mm0, %%mm6         \n\t"\
 142         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
 143         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
 144
 145 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
 146     "lea                "offset"(%0), %%"REG_d"     \n\t"\
 147     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
 148     "pxor                      %%mm1, %%mm1         \n\t"\
 149     "pxor                      %%mm5, %%mm5         \n\t"\
 150     "pxor                      %%mm7, %%mm7         \n\t"\
 151     "pxor                      %%mm6, %%mm6         \n\t"\
 152     ".p2align                      4                \n\t"\
 153     "2:                                             \n\t"\
 154     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
 155     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
 156     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
 157     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
 158     "movq                      %%mm0, %%mm3         \n\t"\
 159     "punpcklwd                 %%mm4, %%mm0         \n\t"\
 160     "punpckhwd                 %%mm4, %%mm3         \n\t"\
 161     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
 162     "pmaddwd                   %%mm4, %%mm0         \n\t"\
 163     "pmaddwd                   %%mm4, %%mm3         \n\t"\
 164     "paddd                     %%mm0, %%mm1         \n\t"\
 165     "paddd                     %%mm3, %%mm5         \n\t"\
 166     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
 167     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
 168     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
 169     "test                  %%"REG_S", %%"REG_S"     \n\t"\
 170     "movq                      %%mm2, %%mm0         \n\t"\
 171     "punpcklwd                 %%mm3, %%mm2         \n\t"\
 172     "punpckhwd                 %%mm3, %%mm0         \n\t"\
 173     "pmaddwd                   %%mm4, %%mm2         \n\t"\
 174     "pmaddwd                   %%mm4, %%mm0         \n\t"\
 175     "paddd                     %%mm2, %%mm7         \n\t"\
 176     "paddd                     %%mm0, %%mm6         \n\t"\
 177     " jnz                         2b                \n\t"\
 178     "psrad                       $16, %%mm1         \n\t"\
 179     "psrad                       $16, %%mm5         \n\t"\
 180     "psrad                       $16, %%mm7         \n\t"\
 181     "psrad                       $16, %%mm6         \n\t"\
 182     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
 183     "packssdw                  %%mm5, %%mm1         \n\t"\
 184     "packssdw                  %%mm6, %%mm7         \n\t"\
 185     "paddw                     %%mm0, %%mm1         \n\t"\
 186     "paddw                     %%mm0, %%mm7         \n\t"\
 187     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
 188     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
 189
 190 #define YSCALEYUV2PACKEDX_ACCURATE \
 191     YSCALEYUV2PACKEDX_ACCURATE_UV \
 192     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
 193
 194 #define YSCALEYUV2RGBX \
 195     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
 196     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
 197     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
 198     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
 199     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
 200     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
 201     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
 202     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
 203     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
 204     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
 205     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
 206     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
 207     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
 208     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
 209     "paddw           %%mm3, %%mm4       \n\t"\
 210     "movq            %%mm2, %%mm0       \n\t"\
 211     "movq            %%mm5, %%mm6       \n\t"\
 212     "movq            %%mm4, %%mm3       \n\t"\
 213     "punpcklwd       %%mm2, %%mm2       \n\t"\
 214     "punpcklwd       %%mm5, %%mm5       \n\t"\
 215     "punpcklwd       %%mm4, %%mm4       \n\t"\
 216     "paddw           %%mm1, %%mm2       \n\t"\
 217     "paddw           %%mm1, %%mm5       \n\t"\
 218     "paddw           %%mm1, %%mm4       \n\t"\
 219     "punpckhwd       %%mm0, %%mm0       \n\t"\
 220     "punpckhwd       %%mm6, %%mm6       \n\t"\
 221     "punpckhwd       %%mm3, %%mm3       \n\t"\
 222     "paddw           %%mm7, %%mm0       \n\t"\
 223     "paddw           %%mm7, %%mm6       \n\t"\
 224     "paddw           %%mm7, %%mm3       \n\t"\
 225     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
 226     "packuswb        %%mm0, %%mm2       \n\t"\
 227     "packuswb        %%mm6, %%mm5       \n\t"\
 228     "packuswb        %%mm3, %%mm4       \n\t"\
 229
 230 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
 231     "movq       "#b", "#q2"     \n\t" /* B */\
 232     "movq       "#r", "#t"      \n\t" /* R */\
 233     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
 234     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
 235     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
 236     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
 237     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
 238     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
 239     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
 240     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
 241     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
 242     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
 243 \
 244     MOVNTQ(   q0,   (dst, index, 4))\
 245     MOVNTQ(    b,  8(dst, index, 4))\
 246     MOVNTQ(   q2, 16(dst, index, 4))\
 247     MOVNTQ(   q3, 24(dst, index, 4))\
 248 \
 249     "add      $8, "#index"      \n\t"\
 250     "cmp "#dstw", "#index"      \n\t"\
 251     " jb      1b                \n\t"
 252 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
 253
 254 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
 255                                    const int16_t **lumSrc, int lumFilterSize,
 256                                    const int16_t *chrFilter, const int16_t **chrUSrc,
 257                                    const int16_t **chrVSrc,
 258                                    int chrFilterSize, const int16_t **alpSrc,
 259                                    uint8_t *dest, int dstW, int dstY)
 260 {
 261     x86_reg dummy=0;
 262     x86_reg dstW_reg = dstW;
 263     x86_reg uv_off = c->uv_off_byte;
 264
 265     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 266         YSCALEYUV2PACKEDX_ACCURATE
 267         YSCALEYUV2RGBX
 268         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
 269         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
 270         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
 271         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
 272         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
 273         "psraw                        $3, %%mm1         \n\t"
 274         "psraw                        $3, %%mm7         \n\t"
 275         "packuswb                  %%mm7, %%mm1         \n\t"
 276         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
 277         YSCALEYUV2PACKEDX_END
 278     } else {
 279         YSCALEYUV2PACKEDX_ACCURATE
 280         YSCALEYUV2RGBX
 281         "pcmpeqd %%mm7, %%mm7 \n\t"
 282         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 283         YSCALEYUV2PACKEDX_END
 284     }
 285 }
 286
 287 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
 288                                 const int16_t **lumSrc, int lumFilterSize,
 289                                 const int16_t *chrFilter, const int16_t **chrUSrc,
 290                                 const int16_t **chrVSrc,
 291                                 int chrFilterSize, const int16_t **alpSrc,
 292                                 uint8_t *dest, int dstW, int dstY)
 293 {
 294     x86_reg dummy=0;
 295     x86_reg dstW_reg = dstW;
 296     x86_reg uv_off = c->uv_off_byte;
 297
 298     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 299         YSCALEYUV2PACKEDX
 300         YSCALEYUV2RGBX
 301         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
 302         "psraw                        $3, %%mm1         \n\t"
 303         "psraw                        $3, %%mm7         \n\t"
 304         "packuswb                  %%mm7, %%mm1         \n\t"
 305         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 306         YSCALEYUV2PACKEDX_END
 307     } else {
 308         YSCALEYUV2PACKEDX
 309         YSCALEYUV2RGBX
 310         "pcmpeqd %%mm7, %%mm7 \n\t"
 311         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 312         YSCALEYUV2PACKEDX_END
 313     }
 314 }
 315
 316 #define REAL_WRITERGB16(dst, dstw, index) \
 317     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
 318     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
 319     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
 320     "psrlq           $3, %%mm2  \n\t"\
 321 \
 322     "movq         %%mm2, %%mm1  \n\t"\
 323     "movq         %%mm4, %%mm3  \n\t"\
 324 \
 325     "punpcklbw    %%mm7, %%mm3  \n\t"\
 326     "punpcklbw    %%mm5, %%mm2  \n\t"\
 327     "punpckhbw    %%mm7, %%mm4  \n\t"\
 328     "punpckhbw    %%mm5, %%mm1  \n\t"\
 329 \
 330     "psllq           $3, %%mm3  \n\t"\
 331     "psllq           $3, %%mm4  \n\t"\
 332 \
 333     "por          %%mm3, %%mm2  \n\t"\
 334     "por          %%mm4, %%mm1  \n\t"\
 335 \
 336     MOVNTQ(%%mm2,  (dst, index, 2))\
 337     MOVNTQ(%%mm1, 8(dst, index, 2))\
 338 \
 339     "add             $8, "#index"   \n\t"\
 340     "cmp        "#dstw", "#index"   \n\t"\
 341     " jb             1b             \n\t"
 342 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
 343
 344 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
 345                                     const int16_t **lumSrc, int lumFilterSize,
 346                                     const int16_t *chrFilter, const int16_t **chrUSrc,
 347                                     const int16_t **chrVSrc,
 348                                     int chrFilterSize, const int16_t **alpSrc,
 349                                     uint8_t *dest, int dstW, int dstY)
 350 {
 351     x86_reg dummy=0;
 352     x86_reg dstW_reg = dstW;
 353     x86_reg uv_off = c->uv_off_byte;
 354
 355     YSCALEYUV2PACKEDX_ACCURATE
 356     YSCALEYUV2RGBX
 357     "pxor %%mm7, %%mm7 \n\t"
 358     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 359 #ifdef DITHER1XBPP
 360     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
 361     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
 362     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
 363 #endif
 364     WRITERGB16(%4, %5, %%REGa)
 365     YSCALEYUV2PACKEDX_END
 366 }
 367
 368 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
 369                                  const int16_t **lumSrc, int lumFilterSize,
 370                                  const int16_t *chrFilter, const int16_t **chrUSrc,
 371                                  const int16_t **chrVSrc,
 372                                  int chrFilterSize, const int16_t **alpSrc,
 373                                  uint8_t *dest, int dstW, int dstY)
 374 {
 375     x86_reg dummy=0;
 376     x86_reg dstW_reg = dstW;
 377     x86_reg uv_off = c->uv_off_byte;
 378
 379     YSCALEYUV2PACKEDX
 380     YSCALEYUV2RGBX
 381     "pxor %%mm7, %%mm7 \n\t"
 382     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 383 #ifdef DITHER1XBPP
 384     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
 385     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
 386     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
 387 #endif
 388     WRITERGB16(%4, %5, %%REGa)
 389     YSCALEYUV2PACKEDX_END
 390 }
 391
 392 #define REAL_WRITERGB15(dst, dstw, index) \
 393     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
 394     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
 395     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
 396     "psrlq           $3, %%mm2  \n\t"\
 397     "psrlq           $1, %%mm5  \n\t"\
 398 \
 399     "movq         %%mm2, %%mm1  \n\t"\
 400     "movq         %%mm4, %%mm3  \n\t"\
 401 \
 402     "punpcklbw    %%mm7, %%mm3  \n\t"\
 403     "punpcklbw    %%mm5, %%mm2  \n\t"\
 404     "punpckhbw    %%mm7, %%mm4  \n\t"\
 405     "punpckhbw    %%mm5, %%mm1  \n\t"\
 406 \
 407     "psllq           $2, %%mm3  \n\t"\
 408     "psllq           $2, %%mm4  \n\t"\
 409 \
 410     "por          %%mm3, %%mm2  \n\t"\
 411     "por          %%mm4, %%mm1  \n\t"\
 412 \
 413     MOVNTQ(%%mm2,  (dst, index, 2))\
 414     MOVNTQ(%%mm1, 8(dst, index, 2))\
 415 \
 416     "add             $8, "#index"   \n\t"\
 417     "cmp        "#dstw", "#index"   \n\t"\
 418     " jb             1b             \n\t"
 419 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
 420
 421 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
 422                                     const int16_t **lumSrc, int lumFilterSize,
 423                                     const int16_t *chrFilter, const int16_t **chrUSrc,
 424                                     const int16_t **chrVSrc,
 425                                     int chrFilterSize, const int16_t **alpSrc,
 426                                     uint8_t *dest, int dstW, int dstY)
 427 {
 428     x86_reg dummy=0;
 429     x86_reg dstW_reg = dstW;
 430     x86_reg uv_off = c->uv_off_byte;
 431
 432     YSCALEYUV2PACKEDX_ACCURATE
 433     YSCALEYUV2RGBX
 434     "pxor %%mm7, %%mm7 \n\t"
 435     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 436 #ifdef DITHER1XBPP
 437     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
 438     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
 439     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
 440 #endif
 441     WRITERGB15(%4, %5, %%REGa)
 442     YSCALEYUV2PACKEDX_END
 443 }
 444
 445 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
 446                                  const int16_t **lumSrc, int lumFilterSize,
 447                                  const int16_t *chrFilter, const int16_t **chrUSrc,
 448                                  const int16_t **chrVSrc,
 449                                  int chrFilterSize, const int16_t **alpSrc,
 450                                  uint8_t *dest, int dstW, int dstY)
 451 {
 452     x86_reg dummy=0;
 453     x86_reg dstW_reg = dstW;
 454     x86_reg uv_off = c->uv_off_byte;
 455
 456     YSCALEYUV2PACKEDX
 457     YSCALEYUV2RGBX
 458     "pxor %%mm7, %%mm7 \n\t"
 459     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 460 #ifdef DITHER1XBPP
 461     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
 462     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
 463     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
 464 #endif
 465     WRITERGB15(%4, %5, %%REGa)
 466     YSCALEYUV2PACKEDX_END
 467 }
 468
 469 #define WRITEBGR24MMX(dst, dstw, index) \
 470     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
 471     "movq      %%mm2, %%mm1     \n\t" /* B */\
 472     "movq      %%mm5, %%mm6     \n\t" /* R */\
 473     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
 474     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
 475     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
 476     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
 477     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
 478     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
 479     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
 480     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
 481     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
 482     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
 483 \
 484     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
 485     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
 486     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
 487     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
 488 \
 489     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
 490     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
 491     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
 492     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
 493 \
 494     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
 495     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
 496     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
 497     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
 498 \
 499     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
 500     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
 501     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
 502     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
 503     MOVNTQ(%%mm0, (dst))\
 504 \
 505     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
 506     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
 507     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
 508     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
 509     MOVNTQ(%%mm6, 8(dst))\
 510 \
 511     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
 512     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
 513     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
 514     MOVNTQ(%%mm5, 16(dst))\
 515 \
 516     "add         $24, "#dst"    \n\t"\
 517 \
 518     "add          $8, "#index"  \n\t"\
 519     "cmp     "#dstw", "#index"  \n\t"\
 520     " jb          1b            \n\t"
 521
 522 #define WRITEBGR24MMX2(dst, dstw, index) \
 523     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
 524     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
 525     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
 526     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
 527     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
 528     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
 529 \
 530     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
 531     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
 532     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
 533 \
 534     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
 535     "por    %%mm1, %%mm6        \n\t"\
 536     "por    %%mm3, %%mm6        \n\t"\
 537     MOVNTQ(%%mm6, (dst))\
 538 \
 539     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
 540     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
 541     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
 542     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
 543 \
 544     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
 545     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
 546     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
 547 \
 548     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
 549     "por    %%mm3, %%mm6        \n\t"\
 550     MOVNTQ(%%mm6, 8(dst))\
 551 \
 552     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
 553     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
 554     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
 555 \
 556     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
 557     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
 558     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
 559 \
 560     "por    %%mm1, %%mm3        \n\t"\
 561     "por    %%mm3, %%mm6        \n\t"\
 562     MOVNTQ(%%mm6, 16(dst))\
 563 \
 564     "add      $24, "#dst"       \n\t"\
 565 \
 566     "add       $8, "#index"     \n\t"\
 567     "cmp  "#dstw", "#index"     \n\t"\
 568     " jb       1b               \n\t"
 569
 570 #if COMPILE_TEMPLATE_MMX2
 571 #undef WRITEBGR24
 572 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
 573 #else
 574 #undef WRITEBGR24
 575 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
 576 #endif
 577
 578 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
 579                                    const int16_t **lumSrc, int lumFilterSize,
 580                                    const int16_t *chrFilter, const int16_t **chrUSrc,
 581                                    const int16_t **chrVSrc,
 582                                    int chrFilterSize, const int16_t **alpSrc,
 583                                    uint8_t *dest, int dstW, int dstY)
 584 {
 585     x86_reg dummy=0;
 586     x86_reg dstW_reg = dstW;
 587     x86_reg uv_off = c->uv_off_byte;
 588
 589     YSCALEYUV2PACKEDX_ACCURATE
 590     YSCALEYUV2RGBX
 591     "pxor %%mm7, %%mm7 \n\t"
 592     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
 593     "add %4, %%"REG_c"                        \n\t"
 594     WRITEBGR24(%%REGc, %5, %%REGa)
 595     :: "r" (&c->redDither),
 596        "m" (dummy), "m" (dummy), "m" (dummy),
 597        "r" (dest), "m" (dstW_reg), "m"(uv_off)
 598     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
 599     );
 600 }
 601
 602 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
 603                                 const int16_t **lumSrc, int lumFilterSize,
 604                                 const int16_t *chrFilter, const int16_t **chrUSrc,
 605                                 const int16_t **chrVSrc,
 606                                 int chrFilterSize, const int16_t **alpSrc,
 607                                 uint8_t *dest, int dstW, int dstY)
 608 {
 609     x86_reg dummy=0;
 610     x86_reg dstW_reg = dstW;
 611     x86_reg uv_off = c->uv_off_byte;
 612
 613     YSCALEYUV2PACKEDX
 614     YSCALEYUV2RGBX
 615     "pxor                    %%mm7, %%mm7       \n\t"
 616     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
 617     "add                        %4, %%"REG_c"   \n\t"
 618     WRITEBGR24(%%REGc, %5, %%REGa)
 619     :: "r" (&c->redDither),
 620        "m" (dummy), "m" (dummy), "m" (dummy),
 621        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
 622     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
 623     );
 624 }
 625
 626 #define REAL_WRITEYUY2(dst, dstw, index) \
 627     "packuswb  %%mm3, %%mm3     \n\t"\
 628     "packuswb  %%mm4, %%mm4     \n\t"\
 629     "packuswb  %%mm7, %%mm1     \n\t"\
 630     "punpcklbw %%mm4, %%mm3     \n\t"\
 631     "movq      %%mm1, %%mm7     \n\t"\
 632     "punpcklbw %%mm3, %%mm1     \n\t"\
 633     "punpckhbw %%mm3, %%mm7     \n\t"\
 634 \
 635     MOVNTQ(%%mm1, (dst, index, 2))\
 636     MOVNTQ(%%mm7, 8(dst, index, 2))\
 637 \
 638     "add          $8, "#index"  \n\t"\
 639     "cmp     "#dstw", "#index"  \n\t"\
 640     " jb          1b            \n\t"
 641 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
 642
 643 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
 644                                      const int16_t **lumSrc, int lumFilterSize,
 645                                      const int16_t *chrFilter, const int16_t **chrUSrc,
 646                                      const int16_t **chrVSrc,
 647                                      int chrFilterSize, const int16_t **alpSrc,
 648                                      uint8_t *dest, int dstW, int dstY)
 649 {
 650     x86_reg dummy=0;
 651     x86_reg dstW_reg = dstW;
 652     x86_reg uv_off = c->uv_off_byte;
 653
 654     YSCALEYUV2PACKEDX_ACCURATE
 655     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 656     "psraw $3, %%mm3    \n\t"
 657     "psraw $3, %%mm4    \n\t"
 658     "psraw $3, %%mm1    \n\t"
 659     "psraw $3, %%mm7    \n\t"
 660     WRITEYUY2(%4, %5, %%REGa)
 661     YSCALEYUV2PACKEDX_END
 662 }
 663
 664 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
 665                                   const int16_t **lumSrc, int lumFilterSize,
 666                                   const int16_t *chrFilter, const int16_t **chrUSrc,
 667                                   const int16_t **chrVSrc,
 668                                   int chrFilterSize, const int16_t **alpSrc,
 669                                   uint8_t *dest, int dstW, int dstY)
 670 {
 671     x86_reg dummy=0;
 672     x86_reg dstW_reg = dstW;
 673     x86_reg uv_off = c->uv_off_byte;
 674
 675     YSCALEYUV2PACKEDX
 676     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 677     "psraw $3, %%mm3    \n\t"
 678     "psraw $3, %%mm4    \n\t"
 679     "psraw $3, %%mm1    \n\t"
 680     "psraw $3, %%mm7    \n\t"
 681     WRITEYUY2(%4, %5, %%REGa)
 682     YSCALEYUV2PACKEDX_END
 683 }
 684
 685 #define REAL_YSCALEYUV2RGB_UV(index, c) \
 686     "xor            "#index", "#index"  \n\t"\
 687     ".p2align              4            \n\t"\
 688     "1:                                 \n\t"\
 689     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
 690     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
 691     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 692     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
 693     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
 694     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 695     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
 696     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
 697     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
 698     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
 699     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
 700     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
 701     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
 702     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
 703     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
 704     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
 705     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
 706     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
 707     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
 708     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
 709     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
 710     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
 711
 712 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
 713     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
 714     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
 715     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
 716     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
 717     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
 718     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
 719     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 720     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 721     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 722     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 723     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 724     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 725
 726 #define REAL_YSCALEYUV2RGB_COEFF(c) \
 727     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
 728     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
 729     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
 730     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
 731     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
 732     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
 733     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
 734     "paddw             %%mm3, %%mm4     \n\t"\
 735     "movq              %%mm2, %%mm0     \n\t"\
 736     "movq              %%mm5, %%mm6     \n\t"\
 737     "movq              %%mm4, %%mm3     \n\t"\
 738     "punpcklwd         %%mm2, %%mm2     \n\t"\
 739     "punpcklwd         %%mm5, %%mm5     \n\t"\
 740     "punpcklwd         %%mm4, %%mm4     \n\t"\
 741     "paddw             %%mm1, %%mm2     \n\t"\
 742     "paddw             %%mm1, %%mm5     \n\t"\
 743     "paddw             %%mm1, %%mm4     \n\t"\
 744     "punpckhwd         %%mm0, %%mm0     \n\t"\
 745     "punpckhwd         %%mm6, %%mm6     \n\t"\
 746     "punpckhwd         %%mm3, %%mm3     \n\t"\
 747     "paddw             %%mm7, %%mm0     \n\t"\
 748     "paddw             %%mm7, %%mm6     \n\t"\
 749     "paddw             %%mm7, %%mm3     \n\t"\
 750     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
 751     "packuswb          %%mm0, %%mm2     \n\t"\
 752     "packuswb          %%mm6, %%mm5     \n\t"\
 753     "packuswb          %%mm3, %%mm4     \n\t"\
 754
 755 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
 756
 757 #define YSCALEYUV2RGB(index, c) \
 758     REAL_YSCALEYUV2RGB_UV(index, c) \
 759     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
 760     REAL_YSCALEYUV2RGB_COEFF(c)
 761
 762 /**
 763  * vertical bilinear scale YV12 to RGB
 764  */
 765 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
 766                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
 767                                 const int16_t *abuf[2], uint8_t *dest,
 768                                 int dstW, int yalpha, int uvalpha, int y)
 769 {
 770     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 771                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 772
 773     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
 774         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
 775 #if ARCH_X86_64
 776         __asm__ volatile(
 777             YSCALEYUV2RGB(%%r8, %5)
 778             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
 779             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 780             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 781             "packuswb            %%mm7, %%mm1       \n\t"
 782             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 783             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
 784                "a" (&c->redDither),
 785                "r" (abuf0), "r" (abuf1)
 786             : "%r8"
 787         );
 788 #else
 789         *(const uint16_t **)(&c->u_temp)=abuf0;
 790         *(const uint16_t **)(&c->v_temp)=abuf1;
 791         __asm__ volatile(
 792             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 793             "mov        %4, %%"REG_b"               \n\t"
 794             "push %%"REG_BP"                        \n\t"
 795             YSCALEYUV2RGB(%%REGBP, %5)
 796             "push                   %0              \n\t"
 797             "push                   %1              \n\t"
 798             "mov          "U_TEMP"(%5), %0          \n\t"
 799             "mov          "V_TEMP"(%5), %1          \n\t"
 800             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
 801             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 802             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
 803             "packuswb            %%mm7, %%mm1       \n\t"
 804             "pop                    %1              \n\t"
 805             "pop                    %0              \n\t"
 806             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
 807             "pop %%"REG_BP"                         \n\t"
 808             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 809             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 810                "a" (&c->redDither)
 811         );
 812 #endif
 813     } else {
 814         __asm__ volatile(
 815             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 816             "mov        %4, %%"REG_b"               \n\t"
 817             "push %%"REG_BP"                        \n\t"
 818             YSCALEYUV2RGB(%%REGBP, %5)
 819             "pcmpeqd %%mm7, %%mm7                   \n\t"
 820             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 821             "pop %%"REG_BP"                         \n\t"
 822             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 823             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 824                "a" (&c->redDither)
 825         );
 826     }
 827 }
 828
 829 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
 830                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
 831                                 const int16_t *abuf[2], uint8_t *dest,
 832                                 int dstW, int yalpha, int uvalpha, int y)
 833 {
 834     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 835                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 836
 837     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 838     __asm__ volatile(
 839         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 840         "mov        %4, %%"REG_b"               \n\t"
 841         "push %%"REG_BP"                        \n\t"
 842         YSCALEYUV2RGB(%%REGBP, %5)
 843         "pxor    %%mm7, %%mm7                   \n\t"
 844         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
 845         "pop %%"REG_BP"                         \n\t"
 846         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 847         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 848            "a" (&c->redDither)
 849     );
 850 }
 851
 852 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
 853                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
 854                                  const int16_t *abuf[2], uint8_t *dest,
 855                                  int dstW, int yalpha, int uvalpha, int y)
 856 {
 857     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 858                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 859
 860     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 861     __asm__ volatile(
 862         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 863         "mov        %4, %%"REG_b"               \n\t"
 864         "push %%"REG_BP"                        \n\t"
 865         YSCALEYUV2RGB(%%REGBP, %5)
 866         "pxor    %%mm7, %%mm7                   \n\t"
 867         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 868 #ifdef DITHER1XBPP
 869         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
 870         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
 871         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
 872 #endif
 873         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
 874         "pop %%"REG_BP"                         \n\t"
 875         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 876         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 877            "a" (&c->redDither)
 878     );
 879 }
 880
 881 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
 882                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
 883                                  const int16_t *abuf[2], uint8_t *dest,
 884                                  int dstW, int yalpha, int uvalpha, int y)
 885 {
 886     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 887                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 888
 889     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 890     __asm__ volatile(
 891         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 892         "mov        %4, %%"REG_b"               \n\t"
 893         "push %%"REG_BP"                        \n\t"
 894         YSCALEYUV2RGB(%%REGBP, %5)
 895         "pxor    %%mm7, %%mm7                   \n\t"
 896         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
 897 #ifdef DITHER1XBPP
 898         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
 899         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
 900         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
 901 #endif
 902         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
 903         "pop %%"REG_BP"                         \n\t"
 904         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 905         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 906            "a" (&c->redDither)
 907     );
 908 }
 909
 910 #define REAL_YSCALEYUV2PACKED(index, c) \
 911     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
 912     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
 913     "psraw                $3, %%mm0                           \n\t"\
 914     "psraw                $3, %%mm1                           \n\t"\
 915     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
 916     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
 917     "xor            "#index", "#index"                        \n\t"\
 918     ".p2align              4            \n\t"\
 919     "1:                                 \n\t"\
 920     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
 921     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
 922     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 923     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
 924     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
 925     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 926     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
 927     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
 928     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
 929     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
 930     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
 931     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
 932     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
 933     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
 934     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
 935     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
 936     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
 937     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
 938     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
 939     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
 940     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
 941     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 942     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
 943     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 944     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 945     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 946     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
 947
 948 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
 949
 950 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
 951                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
 952                                   const int16_t *abuf[2], uint8_t *dest,
 953                                   int dstW, int yalpha, int uvalpha, int y)
 954 {
 955     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
 956                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
 957
 958     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
 959     __asm__ volatile(
 960         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
 961         "mov %4, %%"REG_b"                        \n\t"
 962         "push %%"REG_BP"                        \n\t"
 963         YSCALEYUV2PACKED(%%REGBP, %5)
 964         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
 965         "pop %%"REG_BP"                         \n\t"
 966         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
 967         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
 968            "a" (&c->redDither)
 969     );
 970 }
 971
 972 #define REAL_YSCALEYUV2RGB1(index, c) \
 973     "xor            "#index", "#index"  \n\t"\
 974     ".p2align              4            \n\t"\
 975     "1:                                 \n\t"\
 976     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
 977     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 978     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
 979     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
 980     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
 981     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
 982     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
 983     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
 984     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
 985     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
 986     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
 987     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
 988     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
 989     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
 990     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
 991     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 992     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
 993     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
 994     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
 995     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
 996     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
 997     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
 998     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
 999     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1000     "paddw             %%mm3, %%mm4     \n\t"\
1001     "movq              %%mm2, %%mm0     \n\t"\
1002     "movq              %%mm5, %%mm6     \n\t"\
1003     "movq              %%mm4, %%mm3     \n\t"\
1004     "punpcklwd         %%mm2, %%mm2     \n\t"\
1005     "punpcklwd         %%mm5, %%mm5     \n\t"\
1006     "punpcklwd         %%mm4, %%mm4     \n\t"\
1007     "paddw             %%mm1, %%mm2     \n\t"\
1008     "paddw             %%mm1, %%mm5     \n\t"\
1009     "paddw             %%mm1, %%mm4     \n\t"\
1010     "punpckhwd         %%mm0, %%mm0     \n\t"\
1011     "punpckhwd         %%mm6, %%mm6     \n\t"\
1012     "punpckhwd         %%mm3, %%mm3     \n\t"\
1013     "paddw             %%mm7, %%mm0     \n\t"\
1014     "paddw             %%mm7, %%mm6     \n\t"\
1015     "paddw             %%mm7, %%mm3     \n\t"\
1016     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1017     "packuswb          %%mm0, %%mm2     \n\t"\
1018     "packuswb          %%mm6, %%mm5     \n\t"\
1019     "packuswb          %%mm3, %%mm4     \n\t"\
1020
1021 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1022
1023 // do vertical chrominance interpolation
1024 #define REAL_YSCALEYUV2RGB1b(index, c) \
1025     "xor            "#index", "#index"  \n\t"\
1026     ".p2align              4            \n\t"\
1027     "1:                                 \n\t"\
1028     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1029     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1030     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1031     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1032     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1033     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1034     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1035     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1036     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1037     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1038     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1039     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1040     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1041     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1042     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1043     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1044     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1045     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1046     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1047     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1048     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1049     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1050     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1051     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1052     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1053     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1054     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1055     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1056     "paddw             %%mm3, %%mm4     \n\t"\
1057     "movq              %%mm2, %%mm0     \n\t"\
1058     "movq              %%mm5, %%mm6     \n\t"\
1059     "movq              %%mm4, %%mm3     \n\t"\
1060     "punpcklwd         %%mm2, %%mm2     \n\t"\
1061     "punpcklwd         %%mm5, %%mm5     \n\t"\
1062     "punpcklwd         %%mm4, %%mm4     \n\t"\
1063     "paddw             %%mm1, %%mm2     \n\t"\
1064     "paddw             %%mm1, %%mm5     \n\t"\
1065     "paddw             %%mm1, %%mm4     \n\t"\
1066     "punpckhwd         %%mm0, %%mm0     \n\t"\
1067     "punpckhwd         %%mm6, %%mm6     \n\t"\
1068     "punpckhwd         %%mm3, %%mm3     \n\t"\
1069     "paddw             %%mm7, %%mm0     \n\t"\
1070     "paddw             %%mm7, %%mm6     \n\t"\
1071     "paddw             %%mm7, %%mm3     \n\t"\
1072     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1073     "packuswb          %%mm0, %%mm2     \n\t"\
1074     "packuswb          %%mm6, %%mm5     \n\t"\
1075     "packuswb          %%mm3, %%mm4     \n\t"\
1076
1077 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1078
1079 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1080     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1081     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1082     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1083     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1084     "packuswb          %%mm1, %%mm7     \n\t"
1085 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1086
1087 /**
1088  * YV12 to RGB without scaling or interpolating
1089  */
1090 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1091                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1092                                 const int16_t *abuf0, uint8_t *dest,
1093                                 int dstW, int uvalpha, int y)
1094 {
1095     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1096     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1097
1098     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1099         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1100             __asm__ volatile(
1101                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1102                 "mov        %4, %%"REG_b"               \n\t"
1103                 "push %%"REG_BP"                        \n\t"
1104                 YSCALEYUV2RGB1(%%REGBP, %5)
1105                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1106                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1107                 "pop %%"REG_BP"                         \n\t"
1108                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1109                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1110                    "a" (&c->redDither)
1111             );
1112         } else {
1113             __asm__ volatile(
1114                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1115                 "mov        %4, %%"REG_b"               \n\t"
1116                 "push %%"REG_BP"                        \n\t"
1117                 YSCALEYUV2RGB1(%%REGBP, %5)
1118                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1119                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1120                 "pop %%"REG_BP"                         \n\t"
1121                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1122                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1123                    "a" (&c->redDither)
1124             );
1125         }
1126     } else {
1127         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1128             __asm__ volatile(
1129                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1130                 "mov        %4, %%"REG_b"               \n\t"
1131                 "push %%"REG_BP"                        \n\t"
1132                 YSCALEYUV2RGB1b(%%REGBP, %5)
1133                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1134                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1135                 "pop %%"REG_BP"                         \n\t"
1136                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1137                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1138                    "a" (&c->redDither)
1139             );
1140         } else {
1141             __asm__ volatile(
1142                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1143                 "mov        %4, %%"REG_b"               \n\t"
1144                 "push %%"REG_BP"                        \n\t"
1145                 YSCALEYUV2RGB1b(%%REGBP, %5)
1146                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1147                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1148                 "pop %%"REG_BP"                         \n\t"
1149                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1150                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1151                    "a" (&c->redDither)
1152             );
1153         }
1154     }
1155 }
1156
1157 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1158                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1159                                 const int16_t *abuf0, uint8_t *dest,
1160                                 int dstW, int uvalpha, int y)
1161 {
1162     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1163     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1164
1165     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1166         __asm__ volatile(
1167             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1168             "mov        %4, %%"REG_b"               \n\t"
1169             "push %%"REG_BP"                        \n\t"
1170             YSCALEYUV2RGB1(%%REGBP, %5)
1171             "pxor    %%mm7, %%mm7                   \n\t"
1172             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1173             "pop %%"REG_BP"                         \n\t"
1174             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1175             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1176                "a" (&c->redDither)
1177         );
1178     } else {
1179         __asm__ volatile(
1180             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1181             "mov        %4, %%"REG_b"               \n\t"
1182             "push %%"REG_BP"                        \n\t"
1183             YSCALEYUV2RGB1b(%%REGBP, %5)
1184             "pxor    %%mm7, %%mm7                   \n\t"
1185             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1186             "pop %%"REG_BP"                         \n\t"
1187             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1188             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1189                "a" (&c->redDither)
1190         );
1191     }
1192 }
1193
1194 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1195                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1196                                  const int16_t *abuf0, uint8_t *dest,
1197                                  int dstW, int uvalpha, int y)
1198 {
1199     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1200     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1201
1202     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1203         __asm__ volatile(
1204             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1205             "mov        %4, %%"REG_b"               \n\t"
1206             "push %%"REG_BP"                        \n\t"
1207             YSCALEYUV2RGB1(%%REGBP, %5)
1208             "pxor    %%mm7, %%mm7                   \n\t"
1209             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1210 #ifdef DITHER1XBPP
1211             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1212             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1213             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1214 #endif
1215             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1216             "pop %%"REG_BP"                         \n\t"
1217             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1218             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1219                "a" (&c->redDither)
1220         );
1221     } else {
1222         __asm__ volatile(
1223             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1224             "mov        %4, %%"REG_b"               \n\t"
1225             "push %%"REG_BP"                        \n\t"
1226             YSCALEYUV2RGB1b(%%REGBP, %5)
1227             "pxor    %%mm7, %%mm7                   \n\t"
1228             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1229 #ifdef DITHER1XBPP
1230             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1231             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1232             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1233 #endif
1234             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1235             "pop %%"REG_BP"                         \n\t"
1236             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1237             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1238                "a" (&c->redDither)
1239         );
1240     }
1241 }
1242
1243 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1244                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1245                                  const int16_t *abuf0, uint8_t *dest,
1246                                  int dstW, int uvalpha, int y)
1247 {
1248     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1249     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1250
1251     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1252         __asm__ volatile(
1253             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1254             "mov        %4, %%"REG_b"               \n\t"
1255             "push %%"REG_BP"                        \n\t"
1256             YSCALEYUV2RGB1(%%REGBP, %5)
1257             "pxor    %%mm7, %%mm7                   \n\t"
1258             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1259 #ifdef DITHER1XBPP
1260             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1261             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1262             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1263 #endif
1264             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1265             "pop %%"REG_BP"                         \n\t"
1266             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1267             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1268                "a" (&c->redDither)
1269         );
1270     } else {
1271         __asm__ volatile(
1272             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1273             "mov        %4, %%"REG_b"               \n\t"
1274             "push %%"REG_BP"                        \n\t"
1275             YSCALEYUV2RGB1b(%%REGBP, %5)
1276             "pxor    %%mm7, %%mm7                   \n\t"
1277             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1278 #ifdef DITHER1XBPP
1279             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1280             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1281             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1282 #endif
1283             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1284             "pop %%"REG_BP"                         \n\t"
1285             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1286             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1287                "a" (&c->redDither)
1288         );
1289     }
1290 }
1291
1292 #define REAL_YSCALEYUV2PACKED1(index, c) \
1293     "xor            "#index", "#index"  \n\t"\
1294     ".p2align              4            \n\t"\
1295     "1:                                 \n\t"\
1296     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1297     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1298     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1299     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1300     "psraw                $7, %%mm3     \n\t" \
1301     "psraw                $7, %%mm4     \n\t" \
1302     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1303     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1304     "psraw                $7, %%mm1     \n\t" \
1305     "psraw                $7, %%mm7     \n\t" \
1306
1307 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1308
1309 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1310     "xor "#index", "#index"             \n\t"\
1311     ".p2align              4            \n\t"\
1312     "1:                                 \n\t"\
1313     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1314     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1315     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1316     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1317     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1318     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1319     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1320     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1321     "psrlw                $8, %%mm3     \n\t" \
1322     "psrlw                $8, %%mm4     \n\t" \
1323     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1324     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1325     "psraw                $7, %%mm1     \n\t" \
1326     "psraw                $7, %%mm7     \n\t"
1327 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1328
1329 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1330                                   const int16_t *ubuf[2], const int16_t *bguf[2],
1331                                   const int16_t *abuf0, uint8_t *dest,
1332                                   int dstW, int uvalpha, int y)
1333 {
1334     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1335     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1336
1337     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1338         __asm__ volatile(
1339             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1340             "mov        %4, %%"REG_b"               \n\t"
1341             "push %%"REG_BP"                        \n\t"
1342             YSCALEYUV2PACKED1(%%REGBP, %5)
1343             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1344             "pop %%"REG_BP"                         \n\t"
1345             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1346             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1347                "a" (&c->redDither)
1348         );
1349     } else {
1350         __asm__ volatile(
1351             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1352             "mov        %4, %%"REG_b"               \n\t"
1353             "push %%"REG_BP"                        \n\t"
1354             YSCALEYUV2PACKED1b(%%REGBP, %5)
1355             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1356             "pop %%"REG_BP"                         \n\t"
1357             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1358             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1359                "a" (&c->redDither)
1360         );
1361     }
1362 }
1363
1364 static av_always_inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src,
1365                                                   int width, enum PixelFormat srcFormat)
1366 {
1367
1368     if(srcFormat == PIX_FMT_BGR24) {
1369         __asm__ volatile(
1370             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1371             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1372             :
1373         );
1374     } else {
1375         __asm__ volatile(
1376             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1377             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1378             :
1379         );
1380     }
1381
1382     __asm__ volatile(
1383         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1384         "mov                        %2, %%"REG_a"   \n\t"
1385         "pxor                    %%mm7, %%mm7       \n\t"
1386         "1:                                         \n\t"
1387         PREFETCH"               64(%0)              \n\t"
1388         "movd                     (%0), %%mm0       \n\t"
1389         "movd                    2(%0), %%mm1       \n\t"
1390         "movd                    6(%0), %%mm2       \n\t"
1391         "movd                    8(%0), %%mm3       \n\t"
1392         "add                       $12, %0          \n\t"
1393         "punpcklbw               %%mm7, %%mm0       \n\t"
1394         "punpcklbw               %%mm7, %%mm1       \n\t"
1395         "punpcklbw               %%mm7, %%mm2       \n\t"
1396         "punpcklbw               %%mm7, %%mm3       \n\t"
1397         "pmaddwd                 %%mm5, %%mm0       \n\t"
1398         "pmaddwd                 %%mm6, %%mm1       \n\t"
1399         "pmaddwd                 %%mm5, %%mm2       \n\t"
1400         "pmaddwd                 %%mm6, %%mm3       \n\t"
1401         "paddd                   %%mm1, %%mm0       \n\t"
1402         "paddd                   %%mm3, %%mm2       \n\t"
1403         "paddd                   %%mm4, %%mm0       \n\t"
1404         "paddd                   %%mm4, %%mm2       \n\t"
1405         "psrad                     $15, %%mm0       \n\t"
1406         "psrad                     $15, %%mm2       \n\t"
1407         "packssdw                %%mm2, %%mm0       \n\t"
1408         "packuswb                %%mm0, %%mm0       \n\t"
1409         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1410         "add                        $4, %%"REG_a"   \n\t"
1411         " js                        1b              \n\t"
1412     : "+r" (src)
1413     : "r" (dst+width), "g" ((x86_reg)-width)
1414     : "%"REG_a
1415     );
1416 }
1417
1418 static void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src,
1419                              int width, uint32_t *unused)
1420 {
1421     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1422 }
1423
1424 static void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src,
1425                              int width, uint32_t *unused)
1426 {
1427     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1428 }
1429
1430 static av_always_inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV,
1431                                                    const uint8_t *src, int width,
1432                                                    enum PixelFormat srcFormat)
1433 {
1434     __asm__ volatile(
1435         "movq                    24(%4), %%mm6       \n\t"
1436         "mov                        %3, %%"REG_a"   \n\t"
1437         "pxor                    %%mm7, %%mm7       \n\t"
1438         "1:                                         \n\t"
1439         PREFETCH"               64(%0)              \n\t"
1440         "movd                     (%0), %%mm0       \n\t"
1441         "movd                    2(%0), %%mm1       \n\t"
1442         "punpcklbw               %%mm7, %%mm0       \n\t"
1443         "punpcklbw               %%mm7, %%mm1       \n\t"
1444         "movq                    %%mm0, %%mm2       \n\t"
1445         "movq                    %%mm1, %%mm3       \n\t"
1446         "pmaddwd                  (%4), %%mm0       \n\t"
1447         "pmaddwd                 8(%4), %%mm1       \n\t"
1448         "pmaddwd                16(%4), %%mm2       \n\t"
1449         "pmaddwd                 %%mm6, %%mm3       \n\t"
1450         "paddd                   %%mm1, %%mm0       \n\t"
1451         "paddd                   %%mm3, %%mm2       \n\t"
1452
1453         "movd                    6(%0), %%mm1       \n\t"
1454         "movd                    8(%0), %%mm3       \n\t"
1455         "add                       $12, %0          \n\t"
1456         "punpcklbw               %%mm7, %%mm1       \n\t"
1457         "punpcklbw               %%mm7, %%mm3       \n\t"
1458         "movq                    %%mm1, %%mm4       \n\t"
1459         "movq                    %%mm3, %%mm5       \n\t"
1460         "pmaddwd                  (%4), %%mm1       \n\t"
1461         "pmaddwd                 8(%4), %%mm3       \n\t"
1462         "pmaddwd                16(%4), %%mm4       \n\t"
1463         "pmaddwd                 %%mm6, %%mm5       \n\t"
1464         "paddd                   %%mm3, %%mm1       \n\t"
1465         "paddd                   %%mm5, %%mm4       \n\t"
1466
1467         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1468         "paddd                   %%mm3, %%mm0       \n\t"
1469         "paddd                   %%mm3, %%mm2       \n\t"
1470         "paddd                   %%mm3, %%mm1       \n\t"
1471         "paddd                   %%mm3, %%mm4       \n\t"
1472         "psrad                     $15, %%mm0       \n\t"
1473         "psrad                     $15, %%mm2       \n\t"
1474         "psrad                     $15, %%mm1       \n\t"
1475         "psrad                     $15, %%mm4       \n\t"
1476         "packssdw                %%mm1, %%mm0       \n\t"
1477         "packssdw                %%mm4, %%mm2       \n\t"
1478         "packuswb                %%mm0, %%mm0       \n\t"
1479         "packuswb                %%mm2, %%mm2       \n\t"
1480         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1481         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1482         "add                        $4, %%"REG_a"   \n\t"
1483         " js                        1b              \n\t"
1484     : "+r" (src)
1485     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1486     : "%"REG_a
1487     );
1488 }
1489
1490 static void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV,
1491                               const uint8_t *src1, const uint8_t *src2,
1492                               int width, uint32_t *unused)
1493 {
1494     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1495     assert(src1 == src2);
1496 }
1497
1498 static void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV,
1499                               const uint8_t *src1, const uint8_t *src2,
1500                               int width, uint32_t *unused)
1501 {
1502     assert(src1==src2);
1503     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1504 }
1505
1506 #if COMPILE_TEMPLATE_MMX2
1507 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1508                                  int dstWidth, const uint8_t *src,
1509                                  int srcW, int xInc)
1510 {
1511     int16_t *filterPos = c->hLumFilterPos;
1512     int16_t *filter    = c->hLumFilter;
1513     void    *mmx2FilterCode= c->lumMmx2FilterCode;
1514     int i;
1515 #if defined(PIC)
1516     uint64_t ebxsave;
1517 #endif
1518 #if ARCH_X86_64
1519     uint64_t retsave;
1520 #endif
1521
1522     __asm__ volatile(
1523 #if defined(PIC)
1524         "mov               %%"REG_b", %5        \n\t"
1525 #if ARCH_X86_64
1526         "mov               -8(%%rsp), %%"REG_a" \n\t"
1527         "mov               %%"REG_a", %6        \n\t"
1528 #endif
1529 #else
1530 #if ARCH_X86_64
1531         "mov               -8(%%rsp), %%"REG_a" \n\t"
1532         "mov               %%"REG_a", %5        \n\t"
1533 #endif
1534 #endif
1535         "pxor                  %%mm7, %%mm7     \n\t"
1536         "mov                      %0, %%"REG_c" \n\t"
1537         "mov                      %1, %%"REG_D" \n\t"
1538         "mov                      %2, %%"REG_d" \n\t"
1539         "mov                      %3, %%"REG_b" \n\t"
1540         "xor               %%"REG_a", %%"REG_a" \n\t" // i
1541         PREFETCH"        (%%"REG_c")            \n\t"
1542         PREFETCH"      32(%%"REG_c")            \n\t"
1543         PREFETCH"      64(%%"REG_c")            \n\t"
1544
1545 #if ARCH_X86_64
1546 #define CALL_MMX2_FILTER_CODE \
1547         "movl            (%%"REG_b"), %%esi     \n\t"\
1548         "call                    *%4            \n\t"\
1549         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
1550         "add               %%"REG_S", %%"REG_c" \n\t"\
1551         "add               %%"REG_a", %%"REG_D" \n\t"\
1552         "xor               %%"REG_a", %%"REG_a" \n\t"\
1553
1554 #else
1555 #define CALL_MMX2_FILTER_CODE \
1556         "movl (%%"REG_b"), %%esi        \n\t"\
1557         "call         *%4                       \n\t"\
1558         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
1559         "add               %%"REG_a", %%"REG_D" \n\t"\
1560         "xor               %%"REG_a", %%"REG_a" \n\t"\
1561
1562 #endif /* ARCH_X86_64 */
1563
1564         CALL_MMX2_FILTER_CODE
1565         CALL_MMX2_FILTER_CODE
1566         CALL_MMX2_FILTER_CODE
1567         CALL_MMX2_FILTER_CODE
1568         CALL_MMX2_FILTER_CODE
1569         CALL_MMX2_FILTER_CODE
1570         CALL_MMX2_FILTER_CODE
1571         CALL_MMX2_FILTER_CODE
1572
1573 #if defined(PIC)
1574         "mov                      %5, %%"REG_b" \n\t"
1575 #if ARCH_X86_64
1576         "mov                      %6, %%"REG_a" \n\t"
1577         "mov               %%"REG_a", -8(%%rsp) \n\t"
1578 #endif
1579 #else
1580 #if ARCH_X86_64
1581         "mov                      %5, %%"REG_a" \n\t"
1582         "mov               %%"REG_a", -8(%%rsp) \n\t"
1583 #endif
1584 #endif
1585         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
1586            "m" (mmx2FilterCode)
1587 #if defined(PIC)
1588           ,"m" (ebxsave)
1589 #endif
1590 #if ARCH_X86_64
1591           ,"m"(retsave)
1592 #endif
1593         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1594 #if !defined(PIC)
1595          ,"%"REG_b
1596 #endif
1597     );
1598
1599     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
1600         dst[i] = src[srcW-1]*128;
1601 }
1602
1603 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
1604                                  int dstWidth, const uint8_t *src1,
1605                                  const uint8_t *src2, int srcW, int xInc)
1606 {
1607     int16_t *filterPos = c->hChrFilterPos;
1608     int16_t *filter    = c->hChrFilter;
1609     void    *mmx2FilterCode= c->chrMmx2FilterCode;
1610     int i;
1611 #if defined(PIC)
1612     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1613 #endif
1614 #if ARCH_X86_64
1615     DECLARE_ALIGNED(8, uint64_t, retsave);
1616 #endif
1617
1618     __asm__ volatile(
1619 #if defined(PIC)
1620         "mov          %%"REG_b", %7         \n\t"
1621 #if ARCH_X86_64
1622         "mov          -8(%%rsp), %%"REG_a"  \n\t"
1623         "mov          %%"REG_a", %8         \n\t"
1624 #endif
1625 #else
1626 #if ARCH_X86_64
1627         "mov          -8(%%rsp), %%"REG_a"  \n\t"
1628         "mov          %%"REG_a", %7         \n\t"
1629 #endif
1630 #endif
1631         "pxor             %%mm7, %%mm7      \n\t"
1632         "mov                 %0, %%"REG_c"  \n\t"
1633         "mov                 %1, %%"REG_D"  \n\t"
1634         "mov                 %2, %%"REG_d"  \n\t"
1635         "mov                 %3, %%"REG_b"  \n\t"
1636         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1637         PREFETCH"   (%%"REG_c")             \n\t"
1638         PREFETCH" 32(%%"REG_c")             \n\t"
1639         PREFETCH" 64(%%"REG_c")             \n\t"
1640
1641         CALL_MMX2_FILTER_CODE
1642         CALL_MMX2_FILTER_CODE
1643         CALL_MMX2_FILTER_CODE
1644         CALL_MMX2_FILTER_CODE
1645         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1646         "mov                 %5, %%"REG_c"  \n\t" // src
1647         "mov                 %6, %%"REG_D"  \n\t" // buf2
1648         PREFETCH"   (%%"REG_c")             \n\t"
1649         PREFETCH" 32(%%"REG_c")             \n\t"
1650         PREFETCH" 64(%%"REG_c")             \n\t"
1651
1652         CALL_MMX2_FILTER_CODE
1653         CALL_MMX2_FILTER_CODE
1654         CALL_MMX2_FILTER_CODE
1655         CALL_MMX2_FILTER_CODE
1656
1657 #if defined(PIC)
1658         "mov %7, %%"REG_b"    \n\t"
1659 #if ARCH_X86_64
1660         "mov                 %8, %%"REG_a"  \n\t"
1661         "mov          %%"REG_a", -8(%%rsp)  \n\t"
1662 #endif
1663 #else
1664 #if ARCH_X86_64
1665         "mov                 %7, %%"REG_a"  \n\t"
1666         "mov          %%"REG_a", -8(%%rsp)  \n\t"
1667 #endif
1668 #endif
1669         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
1670            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
1671 #if defined(PIC)
1672           ,"m" (ebxsave)
1673 #endif
1674 #if ARCH_X86_64
1675           ,"m"(retsave)
1676 #endif
1677         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1678 #if !defined(PIC)
1679          ,"%"REG_b
1680 #endif
1681     );
1682
1683     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
1684         dst1[i] = src1[srcW-1]*128;
1685         dst2[i] = src2[srcW-1]*128;
1686     }
1687 }
1688 #endif /* COMPILE_TEMPLATE_MMX2 */
1689
1690 static av_cold void RENAME(sws_init_swScale)(SwsContext *c)
1691 {
1692     enum PixelFormat srcFormat = c->srcFormat,
1693                      dstFormat = c->dstFormat;
1694
1695     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) &&
1696         dstFormat != PIX_FMT_NV12 && dstFormat != PIX_FMT_NV21) {
1697         if (!(c->flags & SWS_BITEXACT)) {
1698             if (c->flags & SWS_ACCURATE_RND) {
1699                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1700                     switch (c->dstFormat) {
1701                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
1702                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
1703                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
1704                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
1705                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
1706                     default: break;
1707                     }
1708                 }
1709             } else {
1710                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1711                     switch (c->dstFormat) {
1712                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
1713                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
1714                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
1715                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
1716                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
1717                     default: break;
1718                     }
1719                 }
1720             }
1721         }
1722         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1723             switch (c->dstFormat) {
1724             case PIX_FMT_RGB32:
1725                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
1726                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
1727                 break;
1728             case PIX_FMT_BGR24:
1729                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
1730                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
1731                 break;
1732             case PIX_FMT_RGB555:
1733                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
1734                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
1735                 break;
1736             case PIX_FMT_RGB565:
1737                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
1738                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
1739                 break;
1740             case PIX_FMT_YUYV422:
1741                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
1742                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
1743                 break;
1744             default:
1745                 break;
1746             }
1747         }
1748     }
1749
1750     if (c->srcBpc == 8 && c->dstBpc <= 10) {
1751     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
1752 #if COMPILE_TEMPLATE_MMX2
1753     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
1754     {
1755         c->hyscale_fast = RENAME(hyscale_fast);
1756         c->hcscale_fast = RENAME(hcscale_fast);
1757     } else {
1758 #endif /* COMPILE_TEMPLATE_MMX2 */
1759         c->hyscale_fast = NULL;
1760         c->hcscale_fast = NULL;
1761 #if COMPILE_TEMPLATE_MMX2
1762     }
1763 #endif /* COMPILE_TEMPLATE_MMX2 */
1764     }
1765
1766     if (!c->chrSrcHSubSample) {
1767         switch(srcFormat) {
1768         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
1769         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
1770         default: break;
1771         }
1772     }
1773
1774     switch (srcFormat) {
1775     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
1776     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
1777     default: break;
1778     }
1779 }