git.sesse.net Git - vlc/blob - modules/video_chroma/i420_rgb_mmx.h

   1 /*****************************************************************************
   2  * transforms_yuvmmx.h: MMX YUV transformation assembly
   3  *****************************************************************************
   4  * Copyright (C) 1999-2007 the VideoLAN team
   5  * $Id$
   6  *
   7  * Authors: Olie Lho <ollie@sis.com.tw>
   8  *          Gaël Hendryckx <jimmy@via.ecp.fr>
   9  *          Samuel Hocevar <sam@zoy.org>
  10  *          Damien Fouilleul <damienf@videolan.org>
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with this program; if not, write to the Free Software
  24  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  25  *****************************************************************************/
  26
  27 /* hope these constant values are cache line aligned */
  28 #if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 3)
  29 #define USED_U64(foo) \
  30     static const uint64_t foo __asm__ (#foo) __attribute__((used))
  31 #else
  32 #define USED_U64(foo) \
  33     static const uint64_t foo __asm__ (#foo) __attribute__((unused))
  34 #endif
  35 USED_U64(mmx_80w)     = 0x0080008000800080ULL;
  36 USED_U64(mmx_10w)     = 0x1010101010101010ULL;
  37 USED_U64(mmx_00ffw)   = 0x00ff00ff00ff00ffULL;
  38 USED_U64(mmx_Y_coeff) = 0x253f253f253f253fULL;
  39
  40 USED_U64(mmx_U_green) = 0xf37df37df37df37dULL;
  41 USED_U64(mmx_U_blue)  = 0x4093409340934093ULL;
  42 USED_U64(mmx_V_red)   = 0x3312331233123312ULL;
  43 USED_U64(mmx_V_green) = 0xe5fce5fce5fce5fcULL;
  44
  45 USED_U64(mmx_mask_f8) = 0xf8f8f8f8f8f8f8f8ULL;
  46 USED_U64(mmx_mask_fc) = 0xfcfcfcfcfcfcfcfcULL;
  47 #undef USED_U64
  48
  49 /* Use RIP-relative code in PIC mode on amd64 */
  50 #if defined(__x86_64__) && defined(__PIC__)
  51 #   define G "(%%rip)"
  52 #else
  53 #   define G
  54 #endif
  55
  56 #define MMX_INIT_16 "                                                       \n\
  57 movd       (%1), %%mm0      # Load 4 Cb       00 00 00 00 u3 u2 u1 u0       \n\
  58 movd       (%2), %%mm1      # Load 4 Cr       00 00 00 00 v3 v2 v1 v0       \n\
  59 pxor      %%mm4, %%mm4      # zero mm4                                      \n\
  60 movq       (%0), %%mm6      # Load 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
  61 "
  62
  63 #define SSE2_INIT_16_ALIGNED "                                              \n\
  64 movq        (%1), %%xmm0    # Load 8 Cb       00 00 00 00 u3 u2 u1 u0       \n\
  65 movq        (%2), %%xmm1    # Load 8 Cr       00 00 00 00 v3 v2 v1 v0       \n\
  66 pxor      %%xmm4, %%xmm4    # zero mm4                                      \n\
  67 movdqa      (%0), %%xmm6    # Load 16 Y       Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
  68 "
  69
  70 #define SSE2_INIT_16_UNALIGNED "                                            \n\
  71 movq        (%1), %%xmm0    # Load 8 Cb       00 00 00 00 u3 u2 u1 u0       \n\
  72 movq        (%2), %%xmm1    # Load 8 Cr       00 00 00 00 v3 v2 v1 v0       \n\
  73 pxor      %%xmm4, %%xmm4    # zero mm4                                      \n\
  74 movdqu      (%0), %%xmm6    # Load 16 Y       Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
  75 prefetchnta (%3)            # Tell CPU not to cache output RGB data         \n\
  76 "
  77
  78 #define MMX_INTRINSICS_INIT_16      \
  79     tmp64 = *(uint32_t *)p_u;       \
  80     mm0 = (__m64)tmp64;             \
  81     tmp64 = *(uint32_t *)p_v;       \
  82     mm1 = (__m64)tmp64;             \
  83     mm4 = _mm_setzero_si64();       \
  84     mm6 = (__m64)*(uint64_t *)p_y;  \
  85
  86 #define SSE2_INTRINSICS_INIT_16_ALIGNED     \
  87     xmm0 = _mm_loadl_epi64((__m128i *)p_u); \
  88     xmm1 = _mm_loadl_epi64((__m128i *)p_v); \
  89     xmm4 = _mm_setzero_si128();             \
  90     xmm6 = _mm_load_si128((__m128i *)p_y);  \
  91
  92 #define SSE2_INTRINSICS_INIT_16_UNALIGNED   \
  93     xmm0 = _mm_loadl_epi64((__m128i *)p_u); \
  94     xmm1 = _mm_loadl_epi64((__m128i *)p_v); \
  95     xmm4 = _mm_setzero_si128();             \
  96     xmm6 = _mm_loadu_si128((__m128i *)p_y); \
  97     _mm_prefetch(p_buffer, _MM_HINT_NTA);    \
  98
  99 #define MMX_INIT_16_GRAY "                                                  \n\
 100 movq      (%0), %%mm6       # Load 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
 101 #movl      $0, (%3)         # cache preload for image                       \n\
 102 "
 103
 104 #define MMX_INIT_32 "                                                       \n\
 105 movd      (%1), %%mm0       # Load 4 Cb       00 00 00 00 u3 u2 u1 u0       \n\
 106 movl        $0, (%3)        # cache preload for image                       \n\
 107 movd      (%2), %%mm1       # Load 4 Cr       00 00 00 00 v3 v2 v1 v0       \n\
 108 pxor     %%mm4, %%mm4       # zero mm4                                      \n\
 109 movq      (%0), %%mm6       # Load 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
 110 "
 111
 112 #define SSE2_INIT_32_ALIGNED "                                              \n\
 113 movq        (%1), %%xmm0    # Load 8 Cb       00 00 00 00 u3 u2 u1 u0       \n\
 114 movq        (%2), %%xmm1    # Load 8 Cr       00 00 00 00 v3 v2 v1 v0       \n\
 115 pxor      %%xmm4, %%xmm4    # zero mm4                                      \n\
 116 movdqa      (%0), %%xmm6    # Load 16 Y       Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
 117 "
 118
 119 #define SSE2_INIT_32_UNALIGNED "                                            \n\
 120 movq        (%1), %%xmm0    # Load 8 Cb       00 00 00 00 u3 u2 u1 u0       \n\
 121 movq        (%2), %%xmm1    # Load 8 Cr       00 00 00 00 v3 v2 v1 v0       \n\
 122 pxor      %%xmm4, %%xmm4    # zero mm4                                      \n\
 123 movdqu      (%0), %%xmm6    # Load 16 Y       Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0       \n\
 124 prefetchnta (%3)            # Tell CPU not to cache output RGB data         \n\
 125 "
 126
 127 #define MMX_INTRINSICS_INIT_32      \
 128     tmp64 = *(uint32_t *)p_u;       \
 129     mm0 = (__m64)tmp64;             \
 130     *(uint16_t *)p_buffer = 0;      \
 131     tmp64 = *(uint32_t *)p_v;       \
 132     mm1 = (__m64)tmp64;             \
 133     mm4 = _mm_setzero_si64();       \
 134     mm6 = (__m64)*(uint64_t *)p_y;
 135
 136 #define SSE2_INTRINSICS_INIT_32_ALIGNED      \
 137     xmm0 = _mm_loadl_epi64((__m128i *)p_u); \
 138     xmm1 = _mm_loadl_epi64((__m128i *)p_v); \
 139     xmm4 = _mm_setzero_si128();             \
 140     xmm6 = _mm_load_si128((__m128i *)p_y);  \
 141
 142 #define SSE2_INTRINSICS_INIT_32_UNALIGNED    \
 143     xmm0 = _mm_loadl_epi64((__m128i *)p_u); \
 144     xmm1 = _mm_loadl_epi64((__m128i *)p_v); \
 145     xmm4 = _mm_setzero_si128();             \
 146     xmm6 = _mm_loadu_si128((__m128i *)p_y); \
 147     _mm_prefetch(p_buffer, _MM_HINT_NTA); \
 148
 149 /*
 150  * Do the multiply part of the conversion for even and odd pixels,
 151  * register usage:
 152  * mm0 -> Cblue, mm1 -> Cred, mm2 -> Cgreen even pixels,
 153  * mm3 -> Cblue, mm4 -> Cred, mm5 -> Cgreen odd  pixels,
 154  * mm6 -> Y even, mm7 -> Y odd
 155  */
 156
 157 #define MMX_YUV_MUL "                                                       \n\
 158 # convert the chroma part                                                   \n\
 159 punpcklbw %%mm4, %%mm0          # scatter 4 Cb    00 u3 00 u2 00 u1 00 u0   \n\
 160 punpcklbw %%mm4, %%mm1          # scatter 4 Cr    00 v3 00 v2 00 v1 00 v0   \n\
 161 psubsw    mmx_80w"G", %%mm0     # Cb -= 128                                 \n\
 162 psubsw    mmx_80w"G", %%mm1     # Cr -= 128                                 \n\
 163 psllw     $3, %%mm0             # Promote precision                         \n\
 164 psllw     $3, %%mm1             # Promote precision                         \n\
 165 movq      %%mm0, %%mm2          # Copy 4 Cb       00 u3 00 u2 00 u1 00 u0   \n\
 166 movq      %%mm1, %%mm3          # Copy 4 Cr       00 v3 00 v2 00 v1 00 v0   \n\
 167 pmulhw    mmx_U_green"G", %%mm2 # Mul Cb with green coeff -> Cb green       \n\
 168 pmulhw    mmx_V_green"G", %%mm3 # Mul Cr with green coeff -> Cr green       \n\
 169 pmulhw    mmx_U_blue"G", %%mm0  # Mul Cb -> Cblue 00 b3 00 b2 00 b1 00 b0   \n\
 170 pmulhw    mmx_V_red"G", %%mm1   # Mul Cr -> Cred  00 r3 00 r2 00 r1 00 r0   \n\
 171 paddsw    %%mm3, %%mm2          # Cb green + Cr green -> Cgreen             \n\
 172                                                                             \n\
 173 # convert the luma part                                                     \n\
 174 psubusb   mmx_10w"G", %%mm6     # Y -= 16                                   \n\
 175 movq      %%mm6, %%mm7          # Copy 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0   \n\
 176 pand      mmx_00ffw"G", %%mm6   # get Y even      00 Y6 00 Y4 00 Y2 00 Y0   \n\
 177 psrlw     $8, %%mm7             # get Y odd       00 Y7 00 Y5 00 Y3 00 Y1   \n\
 178 psllw     $3, %%mm6             # Promote precision                         \n\
 179 psllw     $3, %%mm7             # Promote precision                         \n\
 180 pmulhw    mmx_Y_coeff"G", %%mm6 # Mul 4 Y even    00 y6 00 y4 00 y2 00 y0   \n\
 181 pmulhw    mmx_Y_coeff"G", %%mm7 # Mul 4 Y odd     00 y7 00 y5 00 y3 00 y1   \n\
 182 "
 183
 184 #define SSE2_YUV_MUL "                                                      \n\
 185 # convert the chroma part                                                   \n\
 186 punpcklbw %%xmm4, %%xmm0        # scatter 8 Cb    00 u3 00 u2 00 u1 00 u0   \n\
 187 punpcklbw %%xmm4, %%xmm1        # scatter 8 Cr    00 v3 00 v2 00 v1 00 v0   \n\
 188 movl      $0x00800080, %%eax    #                                           \n\
 189 movd      %%eax, %%xmm5         #                                           \n\
 190 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to     0080 0080 ... 0080 0080   \n\
 191 psubsw    %%xmm5, %%xmm0        # Cb -= 128                                 \n\
 192 psubsw    %%xmm5, %%xmm1        # Cr -= 128                                 \n\
 193 psllw     $3, %%xmm0            # Promote precision                         \n\
 194 psllw     $3, %%xmm1            # Promote precision                         \n\
 195 movdqa    %%xmm0, %%xmm2        # Copy 8 Cb       00 u3 00 u2 00 u1 00 u0   \n\
 196 movdqa    %%xmm1, %%xmm3        # Copy 8 Cr       00 v3 00 v2 00 v1 00 v0   \n\
 197 movl      $0xf37df37d, %%eax    #                                           \n\
 198 movd      %%eax, %%xmm5         #                                           \n\
 199 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to     f37d f37d ... f37d f37d   \n\
 200 pmulhw    %%xmm5, %%xmm2        # Mul Cb with green coeff -> Cb green       \n\
 201 movl      $0xe5fce5fc, %%eax    #                                           \n\
 202 movd      %%eax, %%xmm5         #                                           \n\
 203 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to     e5fc e5fc ... e5fc e5fc   \n\
 204 pmulhw    %%xmm5, %%xmm3        # Mul Cr with green coeff -> Cr green       \n\
 205 movl      $0x40934093, %%eax    #                                           \n\
 206 movd      %%eax, %%xmm5         #                                           \n\
 207 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to     4093 4093 ... 4093 4093   \n\
 208 pmulhw    %%xmm5, %%xmm0        # Mul Cb -> Cblue 00 b3 00 b2 00 b1 00 b0   \n\
 209 movl      $0x33123312, %%eax    #                                           \n\
 210 movd      %%eax, %%xmm5         #                                           \n\
 211 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to     3312 3312 ... 3312 3312   \n\
 212 pmulhw    %%xmm5, %%xmm1        # Mul Cr -> Cred  00 r3 00 r2 00 r1 00 r0   \n\
 213 paddsw    %%xmm3, %%xmm2        # Cb green + Cr green -> Cgreen             \n\
 214                                                                             \n\
 215 # convert the luma part                                                     \n\
 216 movl      $0x10101010, %%eax    #                                           \n\
 217 movd      %%eax, %%xmm5         #                                           \n\
 218 pshufd    $0, %%xmm5, %%xmm5    # Set xmm5 to   1010 1010 ... 1010 1010     \n\
 219 psubusb   %%xmm5, %%xmm6        # Y -= 16                                   \n\
 220 movdqa    %%xmm6, %%xmm7        # Copy 16 Y       Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0   \n\
 221 movl      $0x00ff00ff, %%eax    #                                           \n\
 222 movd      %%eax, %%xmm5         #                                           \n\
 223 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     00ff 00ff ... 00ff 00ff   \n\
 224 pand      %%xmm5, %%xmm6        # get Y even      00 Y6 00 Y4 00 Y2 00 Y0   \n\
 225 psrlw     $8, %%xmm7            # get Y odd       00 Y7 00 Y5 00 Y3 00 Y1   \n\
 226 psllw     $3, %%xmm6            # Promote precision                         \n\
 227 psllw     $3, %%xmm7            # Promote precision                         \n\
 228 movl      $0x253f253f, %%eax    #                                           \n\
 229 movd      %%eax, %%xmm5         #                                           \n\
 230 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     253f 253f ... 253f 253f   \n\
 231 pmulhw    %%xmm5, %%xmm6        # Mul 8 Y even    00 y6 00 y4 00 y2 00 y0   \n\
 232 pmulhw    %%xmm5, %%xmm7        # Mul 8 Y odd     00 y7 00 y5 00 y3 00 y1   \n\
 233 "
 234
 235 #define MMX_INTRINSICS_YUV_MUL \
 236     mm0 = _mm_unpacklo_pi8(mm0, mm4); \
 237     mm1 = _mm_unpacklo_pi8(mm1, mm4); \
 238     mm0 = _mm_subs_pi16(mm0, (__m64)mmx_80w); \
 239     mm1 = _mm_subs_pi16(mm1, (__m64)mmx_80w); \
 240     mm0 = _mm_slli_pi16(mm0, 3); \
 241     mm1 = _mm_slli_pi16(mm1, 3); \
 242     mm2 = mm0; \
 243     mm3 = mm1; \
 244     mm2 = _mm_mulhi_pi16(mm2, (__m64)mmx_U_green); \
 245     mm3 = _mm_mulhi_pi16(mm3, (__m64)mmx_V_green); \
 246     mm0 = _mm_mulhi_pi16(mm0, (__m64)mmx_U_blue); \
 247     mm1 = _mm_mulhi_pi16(mm1, (__m64)mmx_V_red); \
 248     mm2 = _mm_adds_pi16(mm2, mm3); \
 249     \
 250     mm6 = _mm_subs_pu8(mm6, (__m64)mmx_10w); \
 251     mm7 = mm6; \
 252     mm6 = _mm_and_si64(mm6, (__m64)mmx_00ffw); \
 253     mm7 = _mm_srli_pi16(mm7, 8); \
 254     mm6 = _mm_slli_pi16(mm6, 3); \
 255     mm7 = _mm_slli_pi16(mm7, 3); \
 256     mm6 = _mm_mulhi_pi16(mm6, (__m64)mmx_Y_coeff); \
 257     mm7 = _mm_mulhi_pi16(mm7, (__m64)mmx_Y_coeff);
 258
 259 #define SSE2_INTRINSICS_YUV_MUL \
 260     xmm0 = _mm_unpacklo_epi8(xmm0, xmm4); \
 261     xmm1 = _mm_unpacklo_epi8(xmm1, xmm4); \
 262     xmm5 = _mm_set1_epi32(0x00800080UL); \
 263     xmm0 = _mm_subs_epi16(xmm0, xmm5); \
 264     xmm1 = _mm_subs_epi16(xmm1, xmm5); \
 265     xmm0 = _mm_slli_epi16(xmm0, 3); \
 266     xmm1 = _mm_slli_epi16(xmm1, 3); \
 267     xmm2 = xmm0; \
 268     xmm3 = xmm1; \
 269     xmm5 = _mm_set1_epi32(0xf37df37dUL); \
 270     xmm2 = _mm_mulhi_epi16(xmm2, xmm5); \
 271     xmm5 = _mm_set1_epi32(0xe5fce5fcUL); \
 272     xmm3 = _mm_mulhi_epi16(xmm3, xmm5); \
 273     xmm5 = _mm_set1_epi32(0x40934093UL); \
 274     xmm0 = _mm_mulhi_epi16(xmm0, xmm5); \
 275     xmm5 = _mm_set1_epi32(0x33123312UL); \
 276     xmm1 = _mm_mulhi_epi16(xmm1, xmm5); \
 277     xmm2 = _mm_adds_epi16(xmm2, xmm3); \
 278     \
 279     xmm5 = _mm_set1_epi32(0x10101010UL); \
 280     xmm6 = _mm_subs_epu8(xmm6, xmm5); \
 281     xmm7 = xmm6; \
 282     xmm5 = _mm_set1_epi32(0x00ff00ffUL); \
 283     xmm6 = _mm_and_si128(xmm6, xmm5); \
 284     xmm7 = _mm_srli_epi16(xmm7, 8); \
 285     xmm6 = _mm_slli_epi16(xmm6, 3); \
 286     xmm7 = _mm_slli_epi16(xmm7, 3); \
 287     xmm5 = _mm_set1_epi32(0x253f253fUL); \
 288     xmm6 = _mm_mulhi_epi16(xmm6, xmm5); \
 289     xmm7 = _mm_mulhi_epi16(xmm7, xmm5);
 290
 291 /*
 292  * Do the addition part of the conversion for even and odd pixels,
 293  * register usage:
 294  * mm0 -> Cblue, mm1 -> Cred, mm2 -> Cgreen even pixels,
 295  * mm3 -> Cblue, mm4 -> Cred, mm5 -> Cgreen odd  pixels,
 296  * mm6 -> Y even, mm7 -> Y odd
 297  */
 298
 299 #define MMX_YUV_ADD "                                                       \n\
 300 # Do horizontal and vertical scaling                                        \n\
 301 movq      %%mm0, %%mm3          # Copy Cblue                                \n\
 302 movq      %%mm1, %%mm4          # Copy Cred                                 \n\
 303 movq      %%mm2, %%mm5          # Copy Cgreen                               \n\
 304 paddsw    %%mm6, %%mm0          # Y even + Cblue  00 B6 00 B4 00 B2 00 B0   \n\
 305 paddsw    %%mm7, %%mm3          # Y odd  + Cblue  00 B7 00 B5 00 B3 00 B1   \n\
 306 paddsw    %%mm6, %%mm1          # Y even + Cred   00 R6 00 R4 00 R2 00 R0   \n\
 307 paddsw    %%mm7, %%mm4          # Y odd  + Cred   00 R7 00 R5 00 R3 00 R1   \n\
 308 paddsw    %%mm6, %%mm2          # Y even + Cgreen 00 G6 00 G4 00 G2 00 G0   \n\
 309 paddsw    %%mm7, %%mm5          # Y odd  + Cgreen 00 G7 00 G5 00 G3 00 G1   \n\
 310                                                                             \n\
 311 # Limit RGB even to 0..255                                                  \n\
 312 packuswb  %%mm0, %%mm0          # B6 B4 B2 B0 / B6 B4 B2 B0                 \n\
 313 packuswb  %%mm1, %%mm1          # R6 R4 R2 R0 / R6 R4 R2 R0                 \n\
 314 packuswb  %%mm2, %%mm2          # G6 G4 G2 G0 / G6 G4 G2 G0                 \n\
 315                                                                             \n\
 316 # Limit RGB odd to 0..255                                                   \n\
 317 packuswb  %%mm3, %%mm3          # B7 B5 B3 B1 / B7 B5 B3 B1                 \n\
 318 packuswb  %%mm4, %%mm4          # R7 R5 R3 R1 / R7 R5 R3 R1                 \n\
 319 packuswb  %%mm5, %%mm5          # G7 G5 G3 G1 / G7 G5 G3 G1                 \n\
 320                                                                             \n\
 321 # Interleave RGB even and odd                                               \n\
 322 punpcklbw %%mm3, %%mm0          #                 B7 B6 B5 B4 B3 B2 B1 B0   \n\
 323 punpcklbw %%mm4, %%mm1          #                 R7 R6 R5 R4 R3 R2 R1 R0   \n\
 324 punpcklbw %%mm5, %%mm2          #                 G7 G6 G5 G4 G3 G2 G1 G0   \n\
 325 "
 326
 327 #define SSE2_YUV_ADD "                                                      \n\
 328 # Do horizontal and vertical scaling                                        \n\
 329 movdqa    %%xmm0, %%xmm3        # Copy Cblue                                \n\
 330 movdqa    %%xmm1, %%xmm4        # Copy Cred                                 \n\
 331 movdqa    %%xmm2, %%xmm5        # Copy Cgreen                               \n\
 332 paddsw    %%xmm6, %%xmm0        # Y even + Cblue  00 B6 00 B4 00 B2 00 B0   \n\
 333 paddsw    %%xmm7, %%xmm3        # Y odd  + Cblue  00 B7 00 B5 00 B3 00 B1   \n\
 334 paddsw    %%xmm6, %%xmm1        # Y even + Cred   00 R6 00 R4 00 R2 00 R0   \n\
 335 paddsw    %%xmm7, %%xmm4        # Y odd  + Cred   00 R7 00 R5 00 R3 00 R1   \n\
 336 paddsw    %%xmm6, %%xmm2        # Y even + Cgreen 00 G6 00 G4 00 G2 00 G0   \n\
 337 paddsw    %%xmm7, %%xmm5        # Y odd  + Cgreen 00 G7 00 G5 00 G3 00 G1   \n\
 338                                                                             \n\
 339 # Limit RGB even to 0..255                                                  \n\
 340 packuswb  %%xmm0, %%xmm0        # B6 B4 B2 B0 / B6 B4 B2 B0                 \n\
 341 packuswb  %%xmm1, %%xmm1        # R6 R4 R2 R0 / R6 R4 R2 R0                 \n\
 342 packuswb  %%xmm2, %%xmm2        # G6 G4 G2 G0 / G6 G4 G2 G0                 \n\
 343                                                                             \n\
 344 # Limit RGB odd to 0..255                                                   \n\
 345 packuswb  %%xmm3, %%xmm3        # B7 B5 B3 B1 / B7 B5 B3 B1                 \n\
 346 packuswb  %%xmm4, %%xmm4        # R7 R5 R3 R1 / R7 R5 R3 R1                 \n\
 347 packuswb  %%xmm5, %%xmm5        # G7 G5 G3 G1 / G7 G5 G3 G1                 \n\
 348                                                                             \n\
 349 # Interleave RGB even and odd                                               \n\
 350 punpcklbw %%xmm3, %%xmm0        #                 B7 B6 B5 B4 B3 B2 B1 B0   \n\
 351 punpcklbw %%xmm4, %%xmm1        #                 R7 R6 R5 R4 R3 R2 R1 R0   \n\
 352 punpcklbw %%xmm5, %%xmm2        #                 G7 G6 G5 G4 G3 G2 G1 G0   \n\
 353 "
 354
 355 #define MMX_INTRINSICS_YUV_ADD \
 356     mm3 = mm0; \
 357     mm4 = mm1; \
 358     mm5 = mm2; \
 359     mm0 = _mm_adds_pi16(mm0, mm6); \
 360     mm3 = _mm_adds_pi16(mm3, mm7); \
 361     mm1 = _mm_adds_pi16(mm1, mm6); \
 362     mm4 = _mm_adds_pi16(mm4, mm7); \
 363     mm2 = _mm_adds_pi16(mm2, mm6); \
 364     mm5 = _mm_adds_pi16(mm5, mm7); \
 365     \
 366     mm0 = _mm_packs_pu16(mm0, mm0); \
 367     mm1 = _mm_packs_pu16(mm1, mm1); \
 368     mm2 = _mm_packs_pu16(mm2, mm2); \
 369     \
 370     mm3 = _mm_packs_pu16(mm3, mm3); \
 371     mm4 = _mm_packs_pu16(mm4, mm4); \
 372     mm5 = _mm_packs_pu16(mm5, mm5); \
 373     \
 374     mm0 = _mm_unpacklo_pi8(mm0, mm3); \
 375     mm1 = _mm_unpacklo_pi8(mm1, mm4); \
 376     mm2 = _mm_unpacklo_pi8(mm2, mm5);
 377
 378 #define SSE2_INTRINSICS_YUV_ADD \
 379     xmm3 = xmm0; \
 380     xmm4 = xmm1; \
 381     xmm5 = xmm2; \
 382     xmm0 = _mm_adds_epi16(xmm0, xmm6); \
 383     xmm3 = _mm_adds_epi16(xmm3, xmm7); \
 384     xmm1 = _mm_adds_epi16(xmm1, xmm6); \
 385     xmm4 = _mm_adds_epi16(xmm4, xmm7); \
 386     xmm2 = _mm_adds_epi16(xmm2, xmm6); \
 387     xmm5 = _mm_adds_epi16(xmm5, xmm7); \
 388     \
 389     xmm0 = _mm_packus_epi16(xmm0, xmm0); \
 390     xmm1 = _mm_packus_epi16(xmm1, xmm1); \
 391     xmm2 = _mm_packus_epi16(xmm2, xmm2); \
 392     \
 393     xmm3 = _mm_packus_epi16(xmm3, xmm3); \
 394     xmm4 = _mm_packus_epi16(xmm4, xmm4); \
 395     xmm5 = _mm_packus_epi16(xmm5, xmm5); \
 396     \
 397     xmm0 = _mm_unpacklo_epi8(xmm0, xmm3); \
 398     xmm1 = _mm_unpacklo_epi8(xmm1, xmm4); \
 399     xmm2 = _mm_unpacklo_epi8(xmm2, xmm5);
 400
 401 /*
 402  * Grayscale case, only use Y
 403  */
 404
 405 #define MMX_YUV_GRAY "                                                      \n\
 406 # convert the luma part                                                     \n\
 407 psubusb   mmx_10w"G", %%mm6                                                 \n\
 408 movq      %%mm6, %%mm7                                                      \n\
 409 pand      mmx_00ffw"G", %%mm6                                               \n\
 410 psrlw     $8, %%mm7                                                         \n\
 411 psllw     $3, %%mm6                                                         \n\
 412 psllw     $3, %%mm7                                                         \n\
 413 pmulhw    mmx_Y_coeff"G", %%mm6                                             \n\
 414 pmulhw    mmx_Y_coeff"G", %%mm7                                             \n\
 415 packuswb  %%mm6, %%mm6                                                      \n\
 416 packuswb  %%mm7, %%mm7                                                      \n\
 417 punpcklbw %%mm7, %%mm6                                                      \n\
 418 "
 419
 420 #define MMX_UNPACK_16_GRAY "                                                \n\
 421 movq      %%mm6, %%mm5                                                      \n\
 422 pand      mmx_mask_f8"G", %%mm6                                             \n\
 423 pand      mmx_mask_fc"G", %%mm5                                             \n\
 424 movq      %%mm6, %%mm7                                                      \n\
 425 psrlw     $3, %%mm7                                                         \n\
 426 pxor      %%mm3, %%mm3                                                      \n\
 427 movq      %%mm7, %%mm2                                                      \n\
 428 movq      %%mm5, %%mm0                                                      \n\
 429 punpcklbw %%mm3, %%mm5                                                      \n\
 430 punpcklbw %%mm6, %%mm7                                                      \n\
 431 psllw     $3, %%mm5                                                         \n\
 432 por       %%mm5, %%mm7                                                      \n\
 433 movq      %%mm7, (%3)                                                       \n\
 434 punpckhbw %%mm3, %%mm0                                                      \n\
 435 punpckhbw %%mm6, %%mm2                                                      \n\
 436 psllw     $3, %%mm0                                                         \n\
 437 movq      8(%0), %%mm6                                                      \n\
 438 por       %%mm0, %%mm2                                                      \n\
 439 movq      %%mm2, 8(%3)                                                      \n\
 440 "
 441
 442
 443 /*
 444  * convert RGB plane to RGB 15 bits,
 445  * mm0 -> B, mm1 -> R, mm2 -> G,
 446  * mm4 -> GB, mm5 -> AR pixel 4-7,
 447  * mm6 -> GB, mm7 -> AR pixel 0-3
 448  */
 449
 450 #define MMX_UNPACK_15 "                                                     \n\
 451 # mask unneeded bits off                                                    \n\
 452 pand      mmx_mask_f8"G", %%mm0 # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 453 psrlw     $3,%%mm0              # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 454 pand      mmx_mask_f8"G", %%mm2 # g7g6g5g4 g3______ g7g6g5g4 g3______       \n\
 455 pand      mmx_mask_f8"G", %%mm1 # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 456 psrlw     $1,%%mm1              # __r7r6r5 r4r3____ __r7r6r5 r4r3____       \n\
 457 pxor      %%mm4, %%mm4          # zero mm4                                  \n\
 458 movq      %%mm0, %%mm5          # Copy B7-B0                                \n\
 459 movq      %%mm2, %%mm7          # Copy G7-G0                                \n\
 460                                                                             \n\
 461 # convert rgb24 plane to rgb15 pack for pixel 0-3                           \n\
 462 punpcklbw %%mm4, %%mm2          # ________ ________ g7g6g5g4 g3______       \n\
 463 punpcklbw %%mm1, %%mm0          # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 464 psllw     $2,%%mm2              # ________ ____g7g6 g5g4g3__ ________       \n\
 465 por       %%mm2, %%mm0          # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 466 movq      8(%0), %%mm6          # Load 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0   \n\
 467 movq      %%mm0, (%3)           # store pixel 0-3                           \n\
 468                                                                             \n\
 469 # convert rgb24 plane to rgb16 pack for pixel 0-3                           \n\
 470 punpckhbw %%mm4, %%mm7          # ________ ________ g7g6g5g4 g3______       \n\
 471 punpckhbw %%mm1, %%mm5          # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 472 psllw     $2,%%mm7              # ________ ____g7g6 g5g4g3__ ________       \n\
 473 movd      4(%1), %%mm0          # Load 4 Cb       __ __ __ __ u3 u2 u1 u0   \n\
 474 por       %%mm7, %%mm5          # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 475 movd      4(%2), %%mm1          # Load 4 Cr       __ __ __ __ v3 v2 v1 v0   \n\
 476 movq      %%mm5, 8(%3)          # store pixel 4-7                           \n\
 477 "
 478
 479 #define SSE2_UNPACK_15_ALIGNED "                                            \n\
 480 # mask unneeded bits off                                                    \n\
 481 movl      $0xf8f8f8f8, %%eax    #                                           \n\
 482 movd      %%eax, %%xmm5         #                                           \n\
 483 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 484 pand      %%xmm5, %%xmm0        # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 485 psrlw     $3,%%xmm0             # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 486 pand      %%xmm5, %%xmm2        # g7g6g5g4 g3______ g7g6g5g4 g3______       \n\
 487 pand      %%xmm5, %%xmm1        # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 488 psrlw     $1,%%xmm1             # __r7r6r5 r4r3____ __r7r6r5 r4r3____       \n\
 489 pxor      %%xmm4, %%xmm4        # zero mm4                                  \n\
 490 movdqa    %%xmm0, %%xmm5        # Copy B15-B0                               \n\
 491 movdqa    %%xmm2, %%xmm7        # Copy G15-G0                               \n\
 492                                                                             \n\
 493 # convert rgb24 plane to rgb15 pack for pixel 0-7                           \n\
 494 punpcklbw %%xmm4, %%xmm2        # ________ ________ g7g6g5g4 g3______       \n\
 495 punpcklbw %%xmm1, %%xmm0        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 496 psllw     $2,%%xmm2             # ________ ____g7g6 g5g4g3__ ________       \n\
 497 por       %%xmm2, %%xmm0        # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 498 movntdq   %%xmm0, (%3)          # store pixel 0-7                           \n\
 499                                                                             \n\
 500 # convert rgb24 plane to rgb15 pack for pixel 8-15                          \n\
 501 punpckhbw %%xmm4, %%xmm7        # ________ ________ g7g6g5g4 g3______       \n\
 502 punpckhbw %%xmm1, %%xmm5        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 503 psllw     $2,%%xmm7             # ________ ____g7g6 g5g4g3__ ________       \n\
 504 por       %%xmm7, %%xmm5        # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 505 movntdq   %%xmm5, 16(%3)        # store pixel 4-7                           \n\
 506 "
 507
 508 #define SSE2_UNPACK_15_UNALIGNED "                                          \n\
 509 # mask unneeded bits off                                                    \n\
 510 movl      $0xf8f8f8f8, %%eax    #                                           \n\
 511 movd      %%eax, %%xmm5         #                                           \n\
 512 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 513 pand      %%xmm5, %%xmm0        # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 514 psrlw     $3,%%xmm0             # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 515 pand      %%xmm5, %%xmm2        # g7g6g5g4 g3______ g7g6g5g4 g3______       \n\
 516 pand      %%xmm5, %%xmm1        # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 517 psrlw     $1,%%xmm1             # __r7r6r5 r4r3____ __r7r6r5 r4r3____       \n\
 518 pxor      %%xmm4, %%xmm4        # zero mm4                                  \n\
 519 movdqa    %%xmm0, %%xmm5        # Copy B15-B0                               \n\
 520 movdqa    %%xmm2, %%xmm7        # Copy G15-G0                               \n\
 521                                                                             \n\
 522 # convert rgb24 plane to rgb15 pack for pixel 0-7                           \n\
 523 punpcklbw %%xmm4, %%xmm2        # ________ ________ g7g6g5g4 g3______       \n\
 524 punpcklbw %%xmm1, %%xmm0        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 525 psllw     $2,%%xmm2             # ________ ____g7g6 g5g4g3__ ________       \n\
 526 por       %%xmm2, %%xmm0        # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 527 movdqu    %%xmm0, (%3)          # store pixel 0-7                           \n\
 528                                                                             \n\
 529 # convert rgb24 plane to rgb15 pack for pixel 8-15                          \n\
 530 punpckhbw %%xmm4, %%xmm7        # ________ ________ g7g6g5g4 g3______       \n\
 531 punpckhbw %%xmm1, %%xmm5        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 532 psllw     $2,%%xmm7             # ________ ____g7g6 g5g4g3__ ________       \n\
 533 por       %%xmm7, %%xmm5        # r7r6r5r4 r3__g7g6 g5g4g3b7 b6b5b4b3       \n\
 534 movdqu    %%xmm5, 16(%3)        # store pixel 4-7                           \n\
 535 "
 536
 537 #define MMX_INTRINSICS_UNPACK_15 \
 538     mm0 = _mm_and_si64(mm0, (__m64)mmx_mask_f8); \
 539     mm0 = _mm_srli_pi16(mm0, 3); \
 540     mm2 = _mm_and_si64(mm2, (__m64)mmx_mask_f8); \
 541     mm1 = _mm_and_si64(mm1, (__m64)mmx_mask_f8); \
 542     mm1 = _mm_srli_pi16(mm1, 1); \
 543     mm4 = _mm_setzero_si64(); \
 544     mm5 = mm0; \
 545     mm7 = mm2; \
 546     \
 547     mm2 = _mm_unpacklo_pi8(mm2, mm4); \
 548     mm0 = _mm_unpacklo_pi8(mm0, mm1); \
 549     mm2 = _mm_slli_pi16(mm2, 2); \
 550     mm0 = _mm_or_si64(mm0, mm2); \
 551     tmp64 = *(uint64_t *)(p_y + 8); \
 552     mm6 = (__m64)tmp64; \
 553     *(uint64_t *)p_buffer = (uint64_t)mm0; \
 554     \
 555     mm7 = _mm_unpackhi_pi8(mm7, mm4); \
 556     mm5 = _mm_unpackhi_pi8(mm5, mm1); \
 557     mm7 = _mm_slli_pi16(mm7, 2); \
 558     tmp64 = (uint64_t)*(uint32_t *)(p_u + 4); \
 559     mm0 = (__m64)tmp64; \
 560     mm5 = _mm_or_si64(mm5, mm7); \
 561     tmp64 = (uint64_t)*(uint32_t *)(p_v + 4); \
 562     mm1 = (__m64)tmp64; \
 563     *(uint64_t *)(p_buffer + 4) = (uint64_t)mm5;
 564
 565 #define SSE2_INTRINSICS_UNPACK_15_ALIGNED \
 566     xmm5 = _mm_set1_epi32(0xf8f8f8f8UL); \
 567     xmm0 = _mm_and_si128(xmm0, xmm5); \
 568     xmm0 = _mm_srli_epi16(xmm0, 3); \
 569     xmm2 = _mm_and_si128(xmm2, xmm5); \
 570     xmm1 = _mm_and_si128(xmm1, xmm5); \
 571     xmm1 = _mm_srli_epi16(xmm1, 1); \
 572     xmm4 = _mm_setzero_si128();          \
 573     xmm5 = xmm0; \
 574     xmm7 = xmm2; \
 575     \
 576     xmm2 = _mm_unpacklo_epi8(xmm2, xmm4); \
 577     xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
 578     xmm2 = _mm_slli_epi16(xmm2, 2); \
 579     xmm0 = _mm_or_si128(xmm0, xmm2); \
 580     _mm_stream_si128((__m128i*)p_buffer, xmm0); \
 581     \
 582     xmm7 = _mm_unpackhi_epi8(xmm7, xmm4); \
 583     xmm5 = _mm_unpackhi_epi8(xmm5, xmm1); \
 584     xmm7 = _mm_slli_epi16(xmm7, 2); \
 585     xmm5 = _mm_or_si128(xmm5, xmm7); \
 586     _mm_stream_si128((__m128i*)(p_buffer+8), xmm5);
 587
 588 #define SSE2_INTRINSICS_UNPACK_15_UNALIGNED \
 589     xmm5 = _mm_set1_epi32(0xf8f8f8f8UL); \
 590     xmm0 = _mm_and_si128(xmm0, xmm5); \
 591     xmm0 = _mm_srli_epi16(xmm0, 3); \
 592     xmm2 = _mm_and_si128(xmm2, xmm5); \
 593     xmm1 = _mm_and_si128(xmm1, xmm5); \
 594     xmm1 = _mm_srli_epi16(xmm1, 1); \
 595     xmm4 = _mm_setzero_si128();          \
 596     xmm5 = xmm0; \
 597     xmm7 = xmm2; \
 598     \
 599     xmm2 = _mm_unpacklo_epi8(xmm2, xmm4); \
 600     xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
 601     xmm2 = _mm_slli_epi16(xmm2, 2); \
 602     xmm0 = _mm_or_si128(xmm0, xmm2); \
 603     _mm_storeu_si128((__m128i*)p_buffer, xmm0); \
 604     \
 605     xmm7 = _mm_unpackhi_epi8(xmm7, xmm4); \
 606     xmm5 = _mm_unpackhi_epi8(xmm5, xmm1); \
 607     xmm7 = _mm_slli_epi16(xmm7, 2); \
 608     xmm5 = _mm_or_si128(xmm5, xmm7); \
 609     _mm_storeu_si128((__m128i*)(p_buffer+16), xmm5);
 610
 611 /*
 612  * convert RGB plane to RGB 16 bits,
 613  * mm0 -> B, mm1 -> R, mm2 -> G,
 614  * mm4 -> GB, mm5 -> AR pixel 4-7,
 615  * mm6 -> GB, mm7 -> AR pixel 0-3
 616  */
 617
 618 #define MMX_UNPACK_16 "                                                     \n\
 619 # mask unneeded bits off                                                    \n\
 620 pand      mmx_mask_f8"G", %%mm0 # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 621 pand      mmx_mask_fc"G", %%mm2 # g7g6g5g4 g3g2____ g7g6g5g4 g3g2____       \n\
 622 pand      mmx_mask_f8"G", %%mm1 # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 623 psrlw     $3,%%mm0              # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 624 pxor      %%mm4, %%mm4          # zero mm4                                  \n\
 625 movq      %%mm0, %%mm5          # Copy B7-B0                                \n\
 626 movq      %%mm2, %%mm7          # Copy G7-G0                                \n\
 627                                                                             \n\
 628 # convert rgb24 plane to rgb16 pack for pixel 0-3                           \n\
 629 punpcklbw %%mm4, %%mm2          # ________ ________ g7g6g5g4 g3g2____       \n\
 630 punpcklbw %%mm1, %%mm0          # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 631 psllw     $3,%%mm2              # ________ __g7g6g5 g4g3g2__ ________       \n\
 632 por       %%mm2, %%mm0          # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 633 movq      8(%0), %%mm6          # Load 8 Y        Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y0   \n\
 634 movq      %%mm0, (%3)           # store pixel 0-3                           \n\
 635                                                                             \n\
 636 # convert rgb24 plane to rgb16 pack for pixel 0-3                           \n\
 637 punpckhbw %%mm4, %%mm7          # ________ ________ g7g6g5g4 g3g2____       \n\
 638 punpckhbw %%mm1, %%mm5          # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 639 psllw     $3,%%mm7              # ________ __g7g6g5 g4g3g2__ ________       \n\
 640 movd      4(%1), %%mm0          # Load 4 Cb       __ __ __ __ u3 u2 u1 u0   \n\
 641 por       %%mm7, %%mm5          # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 642 movd      4(%2), %%mm1          # Load 4 Cr       __ __ __ __ v3 v2 v1 v0   \n\
 643 movq      %%mm5, 8(%3)          # store pixel 4-7                           \n\
 644 "
 645
 646 #define SSE2_UNPACK_16_ALIGNED "                                            \n\
 647 # mask unneeded bits off                                                    \n\
 648 movl      $0xf8f8f8f8, %%eax    #                                           \n\
 649 movd      %%eax, %%xmm5         #                                           \n\
 650 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 651 pand      %%xmm5, %%xmm0        # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 652 pand      %%xmm5, %%xmm1        # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 653 movl      $0xfcfcfcfc, %%eax    #                                           \n\
 654 movd      %%eax, %%xmm5         #                                           \n\
 655 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 656 pand      %%xmm5, %%xmm2        # g7g6g5g4 g3g2____ g7g6g5g4 g3g2____       \n\
 657 psrlw     $3,%%xmm0             # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 658 pxor      %%xmm4, %%xmm4        # zero mm4                                  \n\
 659 movdqa    %%xmm0, %%xmm5        # Copy B15-B0                               \n\
 660 movdqa    %%xmm2, %%xmm7        # Copy G15-G0                               \n\
 661                                                                             \n\
 662 # convert rgb24 plane to rgb16 pack for pixel 0-7                           \n\
 663 punpcklbw %%xmm4, %%xmm2        # ________ ________ g7g6g5g4 g3g2____       \n\
 664 punpcklbw %%xmm1, %%xmm0        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 665 psllw     $3,%%xmm2             # ________ __g7g6g5 g4g3g2__ ________       \n\
 666 por       %%xmm2, %%xmm0        # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 667 movntdq   %%xmm0, (%3)          # store pixel 0-7                           \n\
 668                                                                             \n\
 669 # convert rgb24 plane to rgb16 pack for pixel 8-15                          \n\
 670 punpckhbw %%xmm4, %%xmm7        # ________ ________ g7g6g5g4 g3g2____       \n\
 671 punpckhbw %%xmm1, %%xmm5        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 672 psllw     $3,%%xmm7             # ________ __g7g6g5 g4g3g2__ ________       \n\
 673 por       %%xmm7, %%xmm5        # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 674 movntdq   %%xmm5, 16(%3)        # store pixel 4-7                           \n\
 675 "
 676
 677 #define SSE2_UNPACK_16_UNALIGNED "                                          \n\
 678 # mask unneeded bits off                                                    \n\
 679 movl      $0xf8f8f8f8, %%eax    #                                           \n\
 680 movd      %%eax, %%xmm5         #                                           \n\
 681 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 682 pand      %%xmm5, %%xmm0        # b7b6b5b4 b3______ b7b6b5b4 b3______       \n\
 683 pand      %%xmm5, %%xmm1        # r7r6r5r4 r3______ r7r6r5r4 r3______       \n\
 684 movl      $0xfcfcfcfc, %%eax    #                                           \n\
 685 movd      %%eax, %%xmm5         #                                           \n\
 686 pshufd    $0, %%xmm5, %%xmm5    # set xmm5 to     f8f8 f8f8 ... f8f8 f8f8   \n\
 687 pand      %%xmm5, %%xmm2        # g7g6g5g4 g3g2____ g7g6g5g4 g3g2____       \n\
 688 psrlw     $3,%%xmm0             # ______b7 b6b5b4b3 ______b7 b6b5b4b3       \n\
 689 pxor      %%xmm4, %%xmm4        # zero mm4                                  \n\
 690 movdqa    %%xmm0, %%xmm5        # Copy B15-B0                               \n\
 691 movdqa    %%xmm2, %%xmm7        # Copy G15-G0                               \n\
 692                                                                             \n\
 693 # convert rgb24 plane to rgb16 pack for pixel 0-7                           \n\
 694 punpcklbw %%xmm4, %%xmm2        # ________ ________ g7g6g5g4 g3g2____       \n\
 695 punpcklbw %%xmm1, %%xmm0        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 696 psllw     $3,%%xmm2             # ________ __g7g6g5 g4g3g2__ ________       \n\
 697 por       %%xmm2, %%xmm0        # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 698 movdqu    %%xmm0, (%3)          # store pixel 0-7                           \n\
 699                                                                             \n\
 700 # convert rgb24 plane to rgb16 pack for pixel 8-15                          \n\
 701 punpckhbw %%xmm4, %%xmm7        # ________ ________ g7g6g5g4 g3g2____       \n\
 702 punpckhbw %%xmm1, %%xmm5        # r7r6r5r4 r3______ ______b7 b6b5b4b3       \n\
 703 psllw     $3,%%xmm7             # ________ __g7g6g5 g4g3g2__ ________       \n\
 704 por       %%xmm7, %%xmm5        # r7r6r5r4 r3g7g6g5 g4g3g2b7 b6b5b4b3       \n\
 705 movdqu    %%xmm5, 16(%3)        # store pixel 4-7                           \n\
 706 "
 707
 708 #define MMX_INTRINSICS_UNPACK_16 \
 709     mm0 = _mm_and_si64(mm0, (__m64)mmx_mask_f8); \
 710     mm2 = _mm_and_si64(mm2, (__m64)mmx_mask_fc); \
 711     mm1 = _mm_and_si64(mm1, (__m64)mmx_mask_f8); \
 712     mm0 = _mm_srli_pi16(mm0, 3); \
 713     mm4 = _mm_setzero_si64(); \
 714     mm5 = mm0; \
 715     mm7 = mm2; \
 716     \
 717     mm2 = _mm_unpacklo_pi8(mm2, mm4); \
 718     mm0 = _mm_unpacklo_pi8(mm0, mm1); \
 719     mm2 = _mm_slli_pi16(mm2, 3); \
 720     mm0 = _mm_or_si64(mm0, mm2); \
 721     tmp64 = *(uint64_t *)(p_y + 8); \
 722     mm6 = (__m64)tmp64; \
 723     *(uint64_t *)p_buffer = (uint64_t)mm0; \
 724     \
 725     mm7 = _mm_unpackhi_pi8(mm7, mm4); \
 726     mm5 = _mm_unpackhi_pi8(mm5, mm1); \
 727     mm7 = _mm_slli_pi16(mm7, 3); \
 728     tmp64 = (uint64_t)*(uint32_t *)(p_u + 4); \
 729     mm0 = (__m64)tmp64; \
 730     mm5 = _mm_or_si64(mm5, mm7); \
 731     tmp64 = (uint64_t)*(uint32_t *)(p_v + 4); \
 732     mm1 = (__m64)tmp64; \
 733     *(uint64_t *)(p_buffer + 4) = (uint64_t)mm5;
 734
 735 #define SSE2_INTRINSICS_UNPACK_16_ALIGNED \
 736     xmm5 = _mm_set1_epi32(0xf8f8f8f8UL); \
 737     xmm0 = _mm_and_si128(xmm0, xmm5); \
 738     xmm1 = _mm_and_si128(xmm1, xmm5); \
 739     xmm5 = _mm_set1_epi32(0xfcfcfcfcUL); \
 740     xmm2 = _mm_and_si128(xmm2, xmm5); \
 741     xmm0 = _mm_srli_epi16(xmm0, 3); \
 742     xmm4 = _mm_setzero_si128();          \
 743     xmm5 = xmm0; \
 744     xmm7 = xmm2; \
 745     \
 746     xmm2 = _mm_unpacklo_epi8(xmm2, xmm4); \
 747     xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
 748     xmm2 = _mm_slli_epi16(xmm2, 3); \
 749     xmm0 = _mm_or_si128(xmm0, xmm2); \
 750     _mm_stream_si128((__m128i*)p_buffer, xmm0); \
 751     \
 752     xmm7 = _mm_unpackhi_epi8(xmm7, xmm4); \
 753     xmm5 = _mm_unpackhi_epi8(xmm5, xmm1); \
 754     xmm7 = _mm_slli_epi16(xmm7, 3); \
 755     xmm5 = _mm_or_si128(xmm5, xmm7); \
 756     _mm_stream_si128((__m128i*)(p_buffer+8), xmm5);
 757
 758 #define SSE2_INTRINSICS_UNPACK_16_UNALIGNED \
 759     xmm5 = _mm_set1_epi32(0xf8f8f8f8UL); \
 760     xmm0 = _mm_and_si128(xmm0, xmm5); \
 761     xmm1 = _mm_and_si128(xmm1, xmm5); \
 762     xmm5 = _mm_set1_epi32(0xfcfcfcfcUL); \
 763     xmm2 = _mm_and_si128(xmm2, xmm5); \
 764     xmm0 = _mm_srli_epi16(xmm0, 3); \
 765     xmm4 = _mm_setzero_si128();          \
 766     xmm5 = xmm0; \
 767     xmm7 = xmm2; \
 768     \
 769     xmm2 = _mm_unpacklo_epi8(xmm2, xmm4); \
 770     xmm0 = _mm_unpacklo_epi8(xmm0, xmm1); \
 771     xmm2 = _mm_slli_epi16(xmm2, 3); \
 772     xmm0 = _mm_or_si128(xmm0, xmm2); \
 773     _mm_storeu_si128((__m128i*)p_buffer, xmm0); \
 774     \
 775     xmm7 = _mm_unpackhi_epi8(xmm7, xmm4); \
 776     xmm5 = _mm_unpackhi_epi8(xmm5, xmm1); \
 777     xmm7 = _mm_slli_epi16(xmm7, 3); \
 778     xmm5 = _mm_or_si128(xmm5, xmm7); \
 779     _mm_storeu_si128((__m128i*)(p_buffer+8), xmm5);
 780
 781 /*
 782  * convert RGB plane to RGB packed format,
 783  * mm0 -> B, mm1 -> R, mm2 -> G
 784  */
 785
 786 #define MMX_UNPACK_32_ARGB "                                                \n\
 787 pxor      %%mm3, %%mm3  # zero mm3                                          \n\
 788 movq      %%mm0, %%mm4  #                 B7 B6 B5 B4 B3 B2 B1 B0           \n\
 789 punpcklbw %%mm2, %%mm4  #                 G3 B3 G2 B2 G1 B1 G0 B0           \n\
 790 movq      %%mm1, %%mm5  #                 R7 R6 R5 R4 R3 R2 R1 R0           \n\
 791 punpcklbw %%mm3, %%mm5  #                 00 R3 00 R2 00 R1 00 R0           \n\
 792 movq      %%mm4, %%mm6  #                 G3 B3 G2 B2 G1 B1 G0 B0           \n\
 793 punpcklwd %%mm5, %%mm4  #                 00 R1 B1 G1 00 R0 B0 G0           \n\
 794 movq      %%mm4, (%3)   # Store ARGB1 ARGB0                                 \n\
 795 punpckhwd %%mm5, %%mm6  #                 00 R3 B3 G3 00 R2 B2 G2           \n\
 796 movq      %%mm6, 8(%3)  # Store ARGB3 ARGB2                                 \n\
 797 punpckhbw %%mm2, %%mm0  #                 G7 B7 G6 B6 G5 B5 G4 B4           \n\
 798 punpckhbw %%mm3, %%mm1  #                 00 R7 00 R6 00 R5 00 R4           \n\
 799 movq      %%mm0, %%mm5  #                 G7 B7 G6 B6 G5 B5 G4 B4           \n\
 800 punpcklwd %%mm1, %%mm5  #                 00 R5 B5 G5 00 R4 B4 G4           \n\
 801 movq      %%mm5, 16(%3) # Store ARGB5 ARGB4                                 \n\
 802 punpckhwd %%mm1, %%mm0  #                 00 R7 B7 G7 00 R6 B6 G6           \n\
 803 movq      %%mm0, 24(%3) # Store ARGB7 ARGB6                                 \n\
 804 "
 805
 806 #define SSE2_UNPACK_32_ARGB_ALIGNED "                                       \n\
 807 pxor      %%xmm3, %%xmm3  # zero xmm3                                       \n\
 808 movdqa    %%xmm0, %%xmm4  #               B7 B6 B5 B4 B3 B2 B1 B0           \n\
 809 punpcklbw %%xmm2, %%xmm4  #               G3 B3 G2 B2 G1 B1 G0 B0           \n\
 810 movdqa    %%xmm1, %%xmm5  #               R7 R6 R5 R4 R3 R2 R1 R0           \n\
 811 punpcklbw %%xmm3, %%xmm5  #               00 R3 00 R2 00 R1 00 R0           \n\
 812 movdqa    %%xmm4, %%xmm6  #               G3 B3 G2 B2 G1 B1 G0 B0           \n\
 813 punpcklwd %%xmm5, %%xmm4  #               00 R1 B1 G1 00 R0 B0 G0           \n\
 814 movntdq   %%xmm4, (%3)    # Store ARGB3 ARGB2 ARGB1 ARGB0                   \n\
 815 punpckhwd %%xmm5, %%xmm6  #               00 R3 B3 G3 00 R2 B2 G2           \n\
 816 movntdq   %%xmm6, 16(%3)  # Store ARGB7 ARGB6 ARGB5 ARGB4                   \n\
 817 punpckhbw %%xmm2, %%xmm0  #               G7 B7 G6 B6 G5 B5 G4 B4           \n\
 818 punpckhbw %%xmm3, %%xmm1  #               00 R7 00 R6 00 R5 00 R4           \n\
 819 movdqa    %%xmm0, %%xmm5  #               G7 B7 G6 B6 G5 B5 G4 B4           \n\
 820 punpcklwd %%xmm1, %%xmm5  #               00 R5 B5 G5 00 R4 B4 G4           \n\
 821 movntdq   %%xmm5, 32(%3)  # Store ARGB11 ARGB10 ARGB9 ARGB8                 \n\
 822 punpckhwd %%xmm1, %%xmm0  #               00 R7 B7 G7 00 R6 B6 G6           \n\
 823 movntdq   %%xmm0, 48(%3)  # Store ARGB15 ARGB14 ARGB13 ARGB12               \n\
 824 "
 825
 826 #define SSE2_UNPACK_32_ARGB_UNALIGNED "                                     \n\
 827 pxor      %%xmm3, %%xmm3  # zero xmm3                                       \n\
 828 movdqa    %%xmm0, %%xmm4  #               B7 B6 B5 B4 B3 B2 B1 B0           \n\
 829 punpcklbw %%xmm2, %%xmm4  #               G3 B3 G2 B2 G1 B1 G0 B0           \n\
 830 movdqa    %%xmm1, %%xmm5  #               R7 R6 R5 R4 R3 R2 R1 R0           \n\
 831 punpcklbw %%xmm3, %%xmm5  #               00 R3 00 R2 00 R1 00 R0           \n\
 832 movdqa    %%xmm4, %%xmm6  #               G3 B3 G2 B2 G1 B1 G0 B0           \n\
 833 punpcklwd %%xmm5, %%xmm4  #               00 R1 B1 G1 00 R0 B0 G0           \n\
 834 movdqu    %%xmm4, (%3)    # Store ARGB3 ARGB2 ARGB1 ARGB0                   \n\
 835 punpckhwd %%xmm5, %%xmm6  #               00 R3 B3 G3 00 R2 B2 G2           \n\
 836 movdqu    %%xmm6, 16(%3)  # Store ARGB7 ARGB6 ARGB5 ARGB4                   \n\
 837 punpckhbw %%xmm2, %%xmm0  #               G7 B7 G6 B6 G5 B5 G4 B4           \n\
 838 punpckhbw %%xmm3, %%xmm1  #               00 R7 00 R6 00 R5 00 R4           \n\
 839 movdqa    %%xmm0, %%xmm5  #               G7 B7 G6 B6 G5 B5 G4 B4           \n\
 840 punpcklwd %%xmm1, %%xmm5  #               00 R5 B5 G5 00 R4 B4 G4           \n\
 841 movdqu    %%xmm5, 32(%3)  # Store ARGB11 ARGB10 ARGB9 ARGB8                 \n\
 842 punpckhwd %%xmm1, %%xmm0  #               00 R7 B7 G7 00 R6 B6 G6           \n\
 843 movdqu    %%xmm0, 48(%3)  # Store ARGB15 ARGB14 ARGB13 ARGB12               \n\
 844 "
 845
 846 #define MMX_INTRINSICS_UNPACK_32_ARGB \
 847     mm3 = _mm_setzero_si64(); \
 848     mm4 = mm0; \
 849     mm4 = _mm_unpacklo_pi8(mm4, mm2); \
 850     mm5 = mm1; \
 851     mm5 = _mm_unpacklo_pi8(mm5, mm3); \
 852     mm6 = mm4; \
 853     mm4 = _mm_unpacklo_pi16(mm4, mm5); \
 854     *(uint64_t *)p_buffer = (uint64_t)mm4; \
 855     mm6 = _mm_unpackhi_pi16(mm6, mm5); \
 856     *(uint64_t *)(p_buffer + 2) = (uint64_t)mm6; \
 857     mm0 = _mm_unpackhi_pi8(mm0, mm2); \
 858     mm1 = _mm_unpackhi_pi8(mm1, mm3); \
 859     mm5 = mm0; \
 860     mm5 = _mm_unpacklo_pi16(mm5, mm1); \
 861     *(uint64_t *)(p_buffer + 4) = (uint64_t)mm5; \
 862     mm0 = _mm_unpackhi_pi16(mm0, mm1); \
 863     *(uint64_t *)(p_buffer + 6) = (uint64_t)mm0;
 864
 865 #define SSE2_INTRINSICS_UNPACK_32_ARGB_ALIGNED \
 866     xmm3 = _mm_setzero_si128();          \
 867     xmm4 = xmm0; \
 868     xmm4 = _mm_unpacklo_epi8(xmm4, xmm2); \
 869     xmm5 = xmm1; \
 870     xmm5 = _mm_unpacklo_epi8(xmm5, xmm3); \
 871     xmm6 = xmm4; \
 872     xmm4 = _mm_unpacklo_epi16(xmm4, xmm5); \
 873     _mm_stream_si128((__m128i*)(p_buffer), xmm4); \
 874     xmm6 = _mm_unpackhi_epi16(xmm6, xmm5); \
 875     _mm_stream_si128((__m128i*)(p_buffer+4), xmm6); \
 876     xmm0 = _mm_unpackhi_epi8(xmm0, xmm2); \
 877     xmm1 = _mm_unpackhi_epi8(xmm1, xmm3); \
 878     xmm5 = xmm0; \
 879     xmm5 = _mm_unpacklo_epi16(xmm5, xmm1); \
 880     _mm_stream_si128((__m128i*)(p_buffer+8), xmm5); \
 881     xmm0 = _mm_unpackhi_epi16(xmm0, xmm1); \
 882     _mm_stream_si128((__m128i*)(p_buffer+12), xmm0);
 883
 884 #define SSE2_INTRINSICS_UNPACK_32_ARGB_UNALIGNED \
 885     xmm3 = _mm_setzero_si128();          \
 886     xmm4 = xmm0; \
 887     xmm4 = _mm_unpacklo_epi8(xmm4, xmm2); \
 888     xmm5 = xmm1; \
 889     xmm5 = _mm_unpacklo_epi8(xmm5, xmm3); \
 890     xmm6 = xmm4; \
 891     xmm4 = _mm_unpacklo_epi16(xmm4, xmm5); \
 892     _mm_storeu_si128((__m128i*)(p_buffer), xmm4); \
 893     xmm6 = _mm_unpackhi_epi16(xmm6, xmm5); \
 894     _mm_storeu_si128((__m128i*)(p_buffer+4), xmm6); \
 895     xmm0 = _mm_unpackhi_epi8(xmm0, xmm2); \
 896     xmm1 = _mm_unpackhi_epi8(xmm1, xmm3); \
 897     xmm5 = xmm0; \
 898     xmm5 = _mm_unpacklo_epi16(xmm5, xmm1); \
 899     _mm_storeu_si128((__m128i*)(p_buffer+8), xmm5); \
 900     xmm0 = _mm_unpackhi_epi16(xmm0, xmm1); \
 901     _mm_storeu_si128((__m128i*)(p_buffer+12), xmm0);
 902
 903 #define MMX_UNPACK_32_BGRA "                                                \n\
 904 pxor      %%mm3, %%mm3  # zero mm3                                          \n\
 905 movq      %%mm2, %%mm4  #                 G7 G6 G5 G4 G3 G2 G1 G0           \n\
 906 punpcklbw %%mm0, %%mm4  #                 B3 G3 B2 G2 B1 G1 B0 G0           \n\
 907 punpcklbw %%mm1, %%mm3  #                 R3 00 R2 00 R1 00 R0 00           \n\
 908 movq      %%mm3, %%mm5  #                 R3 00 R2 00 R1 00 R0 00           \n\
 909 punpcklwd %%mm4, %%mm3  #                 B1 G1 R1 00 B0 G0 R0 00           \n\
 910 movq      %%mm3, (%3)   # Store BGRA1 BGRA0                                 \n\
 911 punpckhwd %%mm4, %%mm5  #                 B3 G3 R3 00 B2 G2 R2 00           \n\
 912 movq      %%mm5, 8(%3)  # Store BGRA3 BGRA2                                 \n\
 913 pxor      %%mm3, %%mm3  # zero mm3                                          \n\
 914 movq      %%mm2, %%mm4  #                 G7 G6 G5 G4 G3 G2 G1 G0           \n\
 915 punpckhbw %%mm0, %%mm4  #                 B7 G7 B6 G6 B5 G5 B4 G4           \n\
 916 punpckhbw %%mm1, %%mm3  #                 R7 00 R6 00 R5 00 R4 00           \n\
 917 movq      %%mm3, %%mm5  #                 R7 00 R6 00 R5 00 R4 00           \n\
 918 punpcklwd %%mm1, %%mm3  #                 B5 G5 R5 00 B4 G4 R4 00           \n\
 919 movq      %%mm3, 16(%3) # Store BGRA5 BGRA4                                 \n\
 920 punpckhwd %%mm4, %%mm5  #                 B7 G7 R7 00 B6 G6 R6 00           \n\
 921 movq      %%mm5, 24(%3) # Store BGRA7 BGRA6                                 \n\
 922 "
 923
 924 #define SSE2_UNPACK_32_BGRA_ALIGNED "                                       \n\
 925 pxor      %%xmm3, %%xmm3  # zero mm3                                        \n\
 926 movdqa    %%xmm2, %%xmm4  #                 G7 G6 G5 G4 G3 G2 G1 G0         \n\
 927 punpcklbw %%xmm0, %%xmm4  #                 B3 G3 B2 G2 B1 G1 B0 G0         \n\
 928 punpcklbw %%xmm1, %%xmm3  #                 R3 00 R2 00 R1 00 R0 00         \n\
 929 movdqa    %%xmm3, %%xmm5  #                 R3 00 R2 00 R1 00 R0 00         \n\
 930 punpcklwd %%xmm4, %%xmm3  #                 B1 G1 R1 00 B0 G0 R0 00         \n\
 931 movntdq   %%xmm3, (%3)    # Store BGRA3 BGRA2 BGRA1 BGRA0                   \n\
 932 punpckhwd %%xmm4, %%xmm5  #                 B3 G3 R3 00 B2 G2 R2 00         \n\
 933 movntdq   %%xmm5, 8(%3)   # Store BGRA7 BGRA6 BGRA5 BGRA4                   \n\
 934 pxor      %%xmm3, %%xmm3  # zero mm3                                        \n\
 935 movdqa    %%xmm2, %%xmm4  #                 G7 G6 G5 G4 G3 G2 G1 G0         \n\
 936 punpckhbw %%xmm0, %%xmm4  #                 B7 G7 B6 G6 B5 G5 B4 G4         \n\
 937 punpckhbw %%xmm1, %%xmm3  #                 R7 00 R6 00 R5 00 R4 00         \n\
 938 movdqa    %%xmm3, %%xmm5  #                 R7 00 R6 00 R5 00 R4 00         \n\
 939 punpcklwd %%xmm1, %%xmm3  #                 B5 G5 R5 00 B4 G4 R4 00         \n\
 940 movntdq   %%xmm3, 16(%3)  # Store BGRA11 BGRA10 BGRA9 BGRA8                 \n\
 941 punpckhwd %%xmm4, %%xmm5  #                 B7 G7 R7 00 B6 G6 R6 00         \n\
 942 movntdq   %%xmm5, 24(%3)  # Store BGRA15 BGRA14 BGRA13 BGRA12               \n\
 943 "
 944
 945 #define SSE2_UNPACK_32_BGRA_UNALIGNED "                                     \n\
 946 pxor      %%xmm3, %%xmm3  # zero mm3                                        \n\
 947 movdqa    %%xmm2, %%xmm4  #                 G7 G6 G5 G4 G3 G2 G1 G0         \n\
 948 punpcklbw %%xmm0, %%xmm4  #                 B3 G3 B2 G2 B1 G1 B0 G0         \n\
 949 punpcklbw %%xmm1, %%xmm3  #                 R3 00 R2 00 R1 00 R0 00         \n\
 950 movdqa    %%xmm3, %%xmm5  #                 R3 00 R2 00 R1 00 R0 00         \n\
 951 punpcklwd %%xmm4, %%xmm3  #                 B1 G1 R1 00 B0 G0 R0 00         \n\
 952 movdqu    %%xmm3, (%3)    # Store BGRA3 BGRA2 BGRA1 BGRA0                   \n\
 953 punpckhwd %%xmm4, %%xmm5  #                 B3 G3 R3 00 B2 G2 R2 00         \n\
 954 movdqu    %%xmm5, 8(%3)   # Store BGRA7 BGRA6 BGRA5 BGRA4                   \n\
 955 pxor      %%xmm3, %%xmm3  # zero mm3                                        \n\
 956 movdqa    %%xmm2, %%xmm4  #                 G7 G6 G5 G4 G3 G2 G1 G0         \n\
 957 punpckhbw %%xmm0, %%xmm4  #                 B7 G7 B6 G6 B5 G5 B4 G4         \n\
 958 punpckhbw %%xmm1, %%xmm3  #                 R7 00 R6 00 R5 00 R4 00         \n\
 959 movdqa    %%xmm3, %%xmm5  #                 R7 00 R6 00 R5 00 R4 00         \n\
 960 punpcklwd %%xmm1, %%xmm3  #                 B5 G5 R5 00 B4 G4 R4 00         \n\
 961 movdqu    %%xmm3, 16(%3)  # Store BGRA11 BGRA10 BGRA9 BGRA8                 \n\
 962 punpckhwd %%xmm4, %%xmm5  #                 B7 G7 R7 00 B6 G6 R6 00         \n\
 963 movdqu    %%xmm5, 24(%3)  # Store BGRA15 BGRA14 BGRA13 BGRA12               \n\
 964 "
 965
 966 #define MMX_INTRINSICS_UNPACK_32_BGRA \
 967     mm3 = _mm_setzero_si64(); \
 968     mm4 = mm2; \
 969     mm4 = _mm_unpacklo_pi8(mm4, mm0); \
 970     mm1 = _mm_unpacklo_pi8(mm1, mm3); \
 971     mm5 = mm3; \
 972     mm3 = _mm_unpacklo_pi16(mm3, mm4); \
 973     *(uint64_t *)p_buffer = (uint64_t)mm3; \
 974     mm5 = _mm_unpackhi_pi16(mm5, mm4); \
 975     *(uint64_t *)(p_buffer + 2) = (uint64_t)mm5; \
 976     mm3 = _mm_setzero_si64(); \
 977     mm4 = mm2; \
 978     mm0 = _mm_unpackhi_pi8(mm0, mm4); \
 979     mm1 = _mm_unpackhi_pi8(mm1, mm3); \
 980     mm5 = mm3; \
 981     mm3 = _mm_unpacklo_pi16(mm3, mm1); \
 982     *(uint64_t *)(p_buffer + 4) = (uint64_t)mm3; \
 983     mm5 = _mm_unpackhi_pi16(mm5, mm4); \
 984     *(uint64_t *)(p_buffer + 6) = (uint64_t)mm5; \
 985
 986 #define SSE2_INTRINSICS_UNPACK_32_BGRA_ALIGNED \
 987     xmm3 = _mm_setzero_si128(); \
 988     xmm4 = xmm2; \
 989     xmm4 = _mm_unpacklo_epi8(xmm4, xmm0); \
 990     xmm1 = _mm_unpacklo_epi8(xmm1, xmm3); \
 991     xmm5 = xmm3; \
 992     xmm3 = _mm_unpacklo_epi16(xmm3, xmm4); \
 993     _mm_stream_si128((__m128i*)(p_buffer), xmm3); \
 994     xmm5 = _mm_unpackhi_epi16(xmm5, xmm4); \
 995     _mm_stream_si128((__m128i*)(p_buffer+4), xmm5); \
 996     xmm3 = _mm_setzero_si128(); \
 997     xmm4 = xmm2; \
 998     xmm0 = _mm_unpackhi_epi8(xmm0, xmm4); \
 999     xmm1 = _mm_unpackhi_epi8(xmm1, xmm3); \
1000     xmm5 = xmm3; \
1001     xmm3 = _mm_unpacklo_epi16(xmm3, xmm1); \
1002     _mm_stream_si128((__m128i*)(p_buffer+8), xmm3); \
1003     xmm5 = _mm_unpackhi_epi16(xmm5, xmm4); \
1004     _mm_stream_si128((__m128i*)(p_buffer+12), xmm5); \
1005
1006 #define SSE2_INTRINSICS_UNPACK_32_BGRA_UNALIGNED \
1007     xmm3 = _mm_setzero_si128(); \
1008     xmm4 = xmm2; \
1009     xmm4 = _mm_unpacklo_epi8(xmm4, xmm0); \
1010     xmm1 = _mm_unpacklo_epi8(xmm1, xmm3); \
1011     xmm5 = xmm3; \
1012     xmm3 = _mm_unpacklo_epi16(xmm3, xmm4); \
1013     _mm_storeu_si128((__m128i*)(p_buffer), xmm3); \
1014     xmm5 = _mm_unpackhi_epi16(xmm5, xmm4); \
1015     _mm_storeu_si128((__m128i*)(p_buffer+4), xmm5); \
1016     xmm3 = _mm_setzero_si128(); \
1017     xmm4 = xmm2; \
1018     xmm0 = _mm_unpackhi_epi8(xmm0, xmm4); \
1019     xmm1 = _mm_unpackhi_epi8(xmm1, xmm3); \
1020     xmm5 = xmm3; \
1021     xmm3 = _mm_unpacklo_epi16(xmm3, xmm1); \
1022     _mm_storeu_si128((__m128i*)(p_buffer+8), xmm3); \
1023     xmm5 = _mm_unpackhi_epi16(xmm5, xmm4); \
1024     _mm_storeu_si128((__m128i*)(p_buffer+12), xmm5); \
1025