Free user supplied data when deleting a frame

[x264] / common / x86 / predict-a.asm

diff --git a/common/x86/predict-a.asm b/common/x86/predict-a.asm
index 1876d96a982c2f507495780ee62e56b43f91063d..460ecb75e04c05a991d940a47aa08c8ac490612f 100644 (file)
--- a/common/x86/predict-a.asm
+++ b/common/x86/predict-a.asm
@@ -1,7 +1,7 @@
 ;*****************************************************************************
 ;* predict-a.asm: x86 intra prediction
 ;*****************************************************************************
 ;*****************************************************************************
 ;* predict-a.asm: x86 intra prediction
 ;*****************************************************************************

-;* Copyright (C) 2005-2011 x264 project
+;* Copyright (C) 2005-2012 x264 project

 ;*
 ;* Authors: Loren Merritt <lorenm@u.washington.edu>
 ;*          Holger Lubitz <holger@lubitz.org>
 ;*
 ;* Authors: Loren Merritt <lorenm@u.washington.edu>
 ;*          Holger Lubitz <holger@lubitz.org>


@@ -31,10 +31,18 @@
 SECTION_RODATA
 
 pw_76543210:
 SECTION_RODATA
 
 pw_76543210:

-pw_3210:    dw 0, 1, 2, 3, 4, 5, 6, 7
-pb_00s_ff:  times 8 db 0
-pb_0s_ff:   times 7 db 0
-            db 0xff
+pw_3210:     dw 0, 1, 2, 3, 4, 5, 6, 7
+pw_43210123: dw -3, -2, -1, 0, 1, 2, 3, 4
+pw_m3:       times 8 dw -3
+pw_m7:       times 8 dw -7
+pb_00s_ff:   times 8 db 0
+pb_0s_ff:    times 7 db 0
+             db 0xff
+shuf_fixtr:  db 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7
+shuf_nop:    db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+shuf_hu:     db 7,6,5,4,3,2,1,0,0,0,0,0,0,0,0,0
+shuf_vr:     db 2,4,6,8,9,10,11,12,13,14,15,0,1,3,5,7
+pw_reverse:  db 14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1

 
 SECTION .text
 
 
 SECTION .text
 


@@ -45,11 +53,11 @@ cextern pw_1
 cextern pw_2
 cextern pw_4
 cextern pw_8
 cextern pw_2
 cextern pw_4
 cextern pw_8

-cextern pw_ff00
-cextern pb_reverse
+cextern pw_16
+cextern pw_00ff

 cextern pw_pixel_max
 
 cextern pw_pixel_max
 

-%macro STORE8x8 2
+%macro STORE8x8 2-4

     add r0, 4*FDEC_STRIDEB
     mova        [r0 + -4*FDEC_STRIDEB], %1
     mova        [r0 + -3*FDEC_STRIDEB], %1
     add r0, 4*FDEC_STRIDEB
     mova        [r0 + -4*FDEC_STRIDEB], %1
     mova        [r0 + -3*FDEC_STRIDEB], %1


@@ -61,6 +69,28 @@ cextern pw_pixel_max
     mova        [r0 +  3*FDEC_STRIDEB], %2
 %endmacro
 
     mova        [r0 +  3*FDEC_STRIDEB], %2
 %endmacro
 

+%macro STORE8x16 4
+    add r0, 4*FDEC_STRIDEB
+    mova        [r0 + -4*FDEC_STRIDEB], %1
+    mova        [r0 + -3*FDEC_STRIDEB], %1
+    mova        [r0 + -2*FDEC_STRIDEB], %1
+    mova        [r0 + -1*FDEC_STRIDEB], %1
+    add r0, 4*FDEC_STRIDEB
+    mova        [r0 + -4*FDEC_STRIDEB], %2
+    mova        [r0 + -3*FDEC_STRIDEB], %2
+    mova        [r0 + -2*FDEC_STRIDEB], %2
+    mova        [r0 + -1*FDEC_STRIDEB], %2
+    add r0, 4*FDEC_STRIDEB
+    mova        [r0 + -4*FDEC_STRIDEB], %3
+    mova        [r0 + -3*FDEC_STRIDEB], %3
+    mova        [r0 + -2*FDEC_STRIDEB], %3
+    mova        [r0 + -1*FDEC_STRIDEB], %3
+    mova        [r0 +  0*FDEC_STRIDEB], %4
+    mova        [r0 +  1*FDEC_STRIDEB], %4
+    mova        [r0 +  2*FDEC_STRIDEB], %4
+    mova        [r0 +  3*FDEC_STRIDEB], %4
+%endmacro
+

 %macro STORE16x16 2-4
 %ifidn %0, 4
     mov         r1d, 8
 %macro STORE16x16 2-4
 %ifidn %0, 4
     mov         r1d, 8


@@ -132,403 +162,404 @@ cextern pw_pixel_max
 
 ; dest, left, right, src, tmp
 ; output: %1 = (t[n-1] + t[n]*2 + t[n+1] + 2) >> 2
 
 ; dest, left, right, src, tmp
 ; output: %1 = (t[n-1] + t[n]*2 + t[n+1] + 2) >> 2

-%macro PRED8x8_LOWPASS 5-6
-%ifidn %1, w
-    paddw       %3, %4
-    psrlw       %3, 1
-    pavgw       %2, %5, %3
+%macro PRED8x8_LOWPASS 4-5
+%if HIGH_BIT_DEPTH
+    paddw       %2, %3
+    psrlw       %2, 1
+    pavgw       %1, %4, %2

 %else
 %else

-    mova        %6, %3
-    pavgb       %3, %4
-    pxor        %4, %6
-    pand        %4, [pb_1]
-    psubusb     %3, %4
-    pavgb       %2, %5, %3
-%endif
-%endmacro
-
-%macro LOAD_PLANE_ARGS 0
-%ifdef ARCH_X86_64
-    movd        mm0, r1d
-    movd        mm2, r2d
-    movd        mm4, r3d
-    pshufw      mm0, mm0, 0
-    pshufw      mm2, mm2, 0
-    pshufw      mm4, mm4, 0
-%else
-    pshufw      mm0, r1m, 0
-    pshufw      mm2, r2m, 0
-    pshufw      mm4, r3m, 0
+    mova        %5, %2
+    pavgb       %2, %3
+    pxor        %3, %5
+    pand        %3, [pb_1]
+    psubusb     %2, %3
+    pavgb       %1, %4, %2

 %endif
 %endmacro
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_ddl( pixel *src )
 ;-----------------------------------------------------------------------------
 %endif
 %endmacro
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_ddl( pixel *src )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_4x4_DDL 4
-cglobal predict_4x4_ddl_%1, 1,1
+%macro PREDICT_4x4_DDL 0
+cglobal predict_4x4_ddl, 1,1

     movu    m1, [r0-FDEC_STRIDEB]
     movu    m1, [r0-FDEC_STRIDEB]

-    psll%2  m2, m1, %3
-    mova    m3, m1
-    mova    m4, m1
+    PSLLPIX m2, m1, 1
+    mova    m0, m1
+%if HIGH_BIT_DEPTH
+    PSRLPIX m1, m1, 1
+    pshufhw m1, m1, q2210
+%else

     pxor    m1, m2
     pxor    m1, m2

-    psrl%2  m1, %3
-    pxor    m3, m1
-    PRED8x8_LOWPASS %4, m0, m2, m3, m4, m5
+    PSRLPIX m1, m1, 1
+    pxor    m1, m0
+%endif
+    PRED8x8_LOWPASS m0, m2, m1, m0, m3

 
 %assign Y 0
 %rep 4
 
 %assign Y 0
 %rep 4

-    psrl%2      m0, %3
-    movh        [r0+Y*FDEC_STRIDEB], m0
+    PSRLPIX m0, m0, 1
+    movh   [r0+Y*FDEC_STRIDEB], m0

 %assign Y (Y+1)
 %endrep
 
     RET
 %endmacro
 
 %assign Y (Y+1)
 %endrep
 
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_4x4_DDL sse2, dq, 2, w
-INIT_AVX
-PREDICT_4x4_DDL avx , dq, 2, w
-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-cglobal predict_4x4_ddl_mmxext, 1,2
-    mova    m1, [r0-2*FDEC_STRIDE+4]
-    mova    m2, [r0-2*FDEC_STRIDE+0]
-    mova    m3, [r0-2*FDEC_STRIDE+2]
-    PRED8x8_LOWPASS w, m0, m1, m2, m3
-    mova    [r0+0*FDEC_STRIDE], m0
-
-    mova    m5, [r0-2*FDEC_STRIDE+6]
-    mova    m6, [r0-2*FDEC_STRIDE+8]
-    pshufw  m7, m6, 0xF9
-    PRED8x8_LOWPASS w, m4, m7, m5, m6
-    mova    [r0+6*FDEC_STRIDE], m4
-
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_4x4_DDL
+INIT_XMM avx
+PREDICT_4x4_DDL
+INIT_MMX mmx2
+cglobal predict_4x4_ddl, 1,2
+    movu    m1, [r0-FDEC_STRIDEB+4]
+    PRED8x8_LOWPASS m0, m1, [r0-FDEC_STRIDEB+0], [r0-FDEC_STRIDEB+2]
+    mova    m3, [r0-FDEC_STRIDEB+8]
+    mova    [r0+0*FDEC_STRIDEB], m0
+    pshufw  m4, m3, q3321
+    PRED8x8_LOWPASS m2, m4, [r0-FDEC_STRIDEB+6], m3
+    mova    [r0+3*FDEC_STRIDEB], m2
+    pshufw  m1, m0, q0021
+    punpckldq m1, m2
+    mova    [r0+1*FDEC_STRIDEB], m1

     psllq   m0, 16
     psllq   m0, 16

-    PALIGNR m4, m0, 6, m1
-    mova    [r0+4*FDEC_STRIDE], m4
-
-    psllq   m0, 16
-    PALIGNR m4, m0, 6, m0
-    mova    [r0+2*FDEC_STRIDE], m4
+    PALIGNR m2, m0, 6, m0
+    mova    [r0+2*FDEC_STRIDEB], m2

     RET
     RET

-%else
-INIT_MMX
-PREDICT_4x4_DDL mmxext, q , 8, b
+%else ; !HIGH_BIT_DEPTH
+INIT_MMX mmx2
+PREDICT_4x4_DDL

 %endif
 
 %endif
 

+;-----------------------------------------------------------------------------
+; void predict_4x4_vr( pixel *src )
+;-----------------------------------------------------------------------------
+%if HIGH_BIT_DEPTH == 0
+INIT_MMX ssse3
+cglobal predict_4x4_vr, 1,1
+    movd    m1, [r0-1*FDEC_STRIDEB]        ; ........t3t2t1t0
+    mova    m4, m1
+    palignr m1, [r0-1*FDEC_STRIDEB-8], 7   ; ......t3t2t1t0lt
+    pavgb   m4, m1
+    palignr m1, [r0+0*FDEC_STRIDEB-8], 7   ; ....t3t2t1t0ltl0
+    mova    m0, m1
+    palignr m1, [r0+1*FDEC_STRIDEB-8], 7   ; ..t3t2t1t0ltl0l1
+    mova    m2, m1
+    palignr m1, [r0+2*FDEC_STRIDEB-8], 7   ; t3t2t1t0ltl0l1l2
+    PRED8x8_LOWPASS m2, m0, m1, m2, m3
+    pshufw  m0, m2, 0
+    psrlq   m2, 16
+    movd    [r0+0*FDEC_STRIDEB], m4
+    palignr m4, m0, 7
+    movd    [r0+1*FDEC_STRIDEB], m2
+    psllq   m0, 8
+    movd    [r0+2*FDEC_STRIDEB], m4
+    palignr m2, m0, 7
+    movd    [r0+3*FDEC_STRIDEB], m2
+    RET
+%endif ; !HIGH_BIT_DEPTH
+

 ;-----------------------------------------------------------------------------
 ; void predict_4x4_ddr( pixel *src )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_ddr( pixel *src )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_4x4 7
-cglobal predict_4x4_ddr_%1, 1,1
-    movu      m1, [r0+1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
-    movq      m2, [r0+0*FDEC_STRIDEB-8]
-%ifdef HIGH_BIT_DEPTH
-    movh      m4, [r0-1*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    punpckl%2 m2, m4
-    movh      m3, [r0-1*FDEC_STRIDEB]
-    punpckh%3 m1, m2
-    PALIGNR   m3, m1, 5*SIZEOF_PIXEL, m1
-    mova      m1, m3
-    movhps    m4, [r0+2*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR   m3, m4, 7*SIZEOF_PIXEL, m4
-    mova      m2, m3
-    movhps    m4, [r0+3*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR   m3, m4, 7*SIZEOF_PIXEL, m4
+%macro PREDICT_4x4 4
+cglobal predict_4x4_ddr, 1,1
+%if HIGH_BIT_DEPTH
+    movu      m2, [r0-1*FDEC_STRIDEB-8]
+    pinsrw    m2, [r0+0*FDEC_STRIDEB-2], 2
+    pinsrw    m2, [r0+1*FDEC_STRIDEB-2], 1
+    pinsrw    m2, [r0+2*FDEC_STRIDEB-2], 0
+    movhps    m3, [r0+3*FDEC_STRIDEB-8]
+%else ; !HIGH_BIT_DEPTH
+    movd      m0, [r0+2*FDEC_STRIDEB-4]
+    movd      m1, [r0+0*FDEC_STRIDEB-4]
+    punpcklbw m0, [r0+1*FDEC_STRIDEB-4]
+    punpcklbw m1, [r0-1*FDEC_STRIDEB-4]
+    punpckhwd m0, m1
+    movd      m2, [r0-1*FDEC_STRIDEB]
+%if cpuflag(ssse3)
+    palignr   m2, m0, 4

 %else
 %else

-    punpckh%2 m2, [r0-1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
-    movh      m3, [r0-1*FDEC_STRIDEB]
-    punpckh%3 m1, m2
-    PALIGNR   m3, m1, 5*SIZEOF_PIXEL, m1
-    mova      m1, m3
-    PALIGNR   m3, [r0+2*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m4
-    mova      m2, m3
-    PALIGNR   m3, [r0+3*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m4
+    psllq     m2, 32
+    punpckhdq m0, m2
+    SWAP       2, 0

 %endif
 %endif

-    PRED8x8_LOWPASS %5, m0, m3, m1, m2, m4
+    movd      m3, [r0+3*FDEC_STRIDEB-4]
+    psllq     m3, 32
+%endif ; !HIGH_BIT_DEPTH
+
+    PSRLPIX   m1, m2, 1
+    mova      m0, m2
+    PALIGNR   m2, m3, 7*SIZEOF_PIXEL, m3
+    PRED8x8_LOWPASS m0, m2, m1, m0, m3

 %assign Y 3
     movh      [r0+Y*FDEC_STRIDEB], m0
 %rep 3
 %assign Y (Y-1)
 %assign Y 3
     movh      [r0+Y*FDEC_STRIDEB], m0
 %rep 3
 %assign Y (Y-1)

-    psrl%4    m0, %7
+    PSRLPIX   m0, m0, 1

     movh      [r0+Y*FDEC_STRIDEB], m0
 %endrep
     RET
 
     movh      [r0+Y*FDEC_STRIDEB], m0
 %endrep
     RET
 

-cglobal predict_4x4_vr_%1, 1,1,6*(mmsize/16)
-    movh    m0, [r0-1*FDEC_STRIDEB]                                       ; ........t3t2t1t0
-    mova    m5, m0
-%ifdef HIGH_BIT_DEPTH
-    movhps  m1, [r0-1*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR m0, m1, 7*SIZEOF_PIXEL, m1                                    ; ......t3t2t1t0lt
-    pavg%5  m5, m0
-    movhps  m1, [r0+0*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR m0, m1, 7*SIZEOF_PIXEL, m1                                    ; ....t3t2t1t0ltl0
-    mova    m1, m0
-    movhps  m2, [r0+1*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR m0, m2, 7*SIZEOF_PIXEL, m2                                    ; ..t3t2t1t0ltl0l1
-    mova    m2, m0
-    movhps  m3, [r0+2*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    PALIGNR m0, m3, 7*SIZEOF_PIXEL, m3                                    ; t3t2t1t0ltl0l1l2
+;-----------------------------------------------------------------------------
+; void predict_4x4_vr( pixel *src )
+;-----------------------------------------------------------------------------
+cglobal predict_4x4_vr, 1,1
+%if HIGH_BIT_DEPTH
+    movu      m1, [r0-1*FDEC_STRIDEB-8]
+    pinsrw    m1, [r0+0*FDEC_STRIDEB-2], 2
+    pinsrw    m1, [r0+1*FDEC_STRIDEB-2], 1
+    pinsrw    m1, [r0+2*FDEC_STRIDEB-2], 0
+%else ; !HIGH_BIT_DEPTH
+    movd      m0, [r0+2*FDEC_STRIDEB-4]
+    movd      m1, [r0+0*FDEC_STRIDEB-4]
+    punpcklbw m0, [r0+1*FDEC_STRIDEB-4]
+    punpcklbw m1, [r0-1*FDEC_STRIDEB-4]
+    punpckhwd m0, m1
+    movd      m1, [r0-1*FDEC_STRIDEB]
+%if cpuflag(ssse3)
+    palignr   m1, m0, 4

 %else
 %else

-    PALIGNR m0, [r0-1*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m1    ; ......t3t2t1t0lt
-    pavg%5  m5, m0
-    PALIGNR m0, [r0+0*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m1    ; ....t3t2t1t0ltl0
-    mova    m1, m0
-    PALIGNR m0, [r0+1*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m2    ; ..t3t2t1t0ltl0l1
-    mova    m2, m0
-    PALIGNR m0, [r0+2*FDEC_STRIDEB-8*SIZEOF_PIXEL], 7*SIZEOF_PIXEL, m3    ; t3t2t1t0ltl0l1l2
+    psllq     m1, 32
+    punpckhdq m0, m1
+    SWAP       1, 0

 %endif
 %endif

-    PRED8x8_LOWPASS %5, m3, m1, m0, m2, m4
-    psll%4  m1, m3, %7*6
-    psrl%4  m3, %7*2
-    movh    [r0+0*FDEC_STRIDEB], m5
-    movh    [r0+1*FDEC_STRIDEB], m3
-    PALIGNR m5, m1, 7*SIZEOF_PIXEL, m2
-    psll%4  m1, %7
-    movh    [r0+2*FDEC_STRIDEB], m5
-    PALIGNR m3, m1, 7*SIZEOF_PIXEL, m1
-    movh    [r0+3*FDEC_STRIDEB], m3
+%endif ; !HIGH_BIT_DEPTH
+    PSRLPIX   m2, m1, 1
+    PSRLPIX   m0, m1, 2
+    pavg%1    m4, m1, m2
+    PSRLPIX   m4, m4, 3
+    PRED8x8_LOWPASS m2, m0, m1, m2, m3
+    PSLLPIX   m0, m2, 6
+    PSRLPIX   m2, m2, 2
+    movh      [r0+0*FDEC_STRIDEB], m4
+    PALIGNR   m4, m0, 7*SIZEOF_PIXEL, m3
+    movh      [r0+1*FDEC_STRIDEB], m2
+    PSLLPIX   m0, m0, 1
+    movh      [r0+2*FDEC_STRIDEB], m4
+    PALIGNR   m2, m0, 7*SIZEOF_PIXEL, m0
+    movh      [r0+3*FDEC_STRIDEB], m2

     RET
 
     RET
 

-cglobal predict_4x4_hd_%1, 1,1,6*(mmsize/16)
-    movh      m0, [r0-1*FDEC_STRIDEB-4*SIZEOF_PIXEL] ; lt ..
-%ifdef HIGH_BIT_DEPTH
-    movh      m1, [r0-1*FDEC_STRIDEB]
-    punpckl%6 m0, m1                                 ; t3 t2 t1 t0 lt .. .. ..
-    psll%4    m0, %7                                 ; t2 t1 t0 lt .. .. .. ..
-    movh      m1, [r0+3*FDEC_STRIDEB-4*SIZEOF_PIXEL] ; l3
-    movh      m2, [r0+2*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    punpckl%2 m1, m2                                 ; l2 l3
-    movh      m2, [r0+1*FDEC_STRIDEB-4*SIZEOF_PIXEL] ; l1
-    movh      m3, [r0+0*FDEC_STRIDEB-4*SIZEOF_PIXEL]
-    punpckl%2 m2, m3                                 ; l0 l1
+;-----------------------------------------------------------------------------
+; void predict_4x4_hd( pixel *src )
+;-----------------------------------------------------------------------------
+cglobal predict_4x4_hd, 1,1
+%if HIGH_BIT_DEPTH
+    movu      m1, [r0-1*FDEC_STRIDEB-8]
+    PSLLPIX   m1, m1, 1
+    pinsrw    m1, [r0+0*FDEC_STRIDEB-2], 3
+    pinsrw    m1, [r0+1*FDEC_STRIDEB-2], 2
+    pinsrw    m1, [r0+2*FDEC_STRIDEB-2], 1
+    pinsrw    m1, [r0+3*FDEC_STRIDEB-2], 0

 %else
 %else

-    punpckl%6 m0, [r0-1*FDEC_STRIDEB]                ; t3 t2 t1 t0 lt .. .. ..
-    psll%4    m0, %7                                 ; t2 t1 t0 lt .. .. .. ..
-    movu      m1, [r0+3*FDEC_STRIDEB-8*SIZEOF_PIXEL] ; l3
-    punpckh%2 m1, [r0+2*FDEC_STRIDEB-8*SIZEOF_PIXEL] ; l2 l3
-    movu      m2, [r0+1*FDEC_STRIDEB-8*SIZEOF_PIXEL] ; l1
-    punpckh%2 m2, [r0+0*FDEC_STRIDEB-8*SIZEOF_PIXEL] ; l0 l1
+    movd      m0, [r0-1*FDEC_STRIDEB-4] ; lt ..
+    punpckldq m0, [r0-1*FDEC_STRIDEB]   ; t3 t2 t1 t0 lt .. .. ..
+    PSLLPIX   m0, m0, 1                 ; t2 t1 t0 lt .. .. .. ..
+    movd      m1, [r0+3*FDEC_STRIDEB-4] ; l3
+    punpcklbw m1, [r0+2*FDEC_STRIDEB-4] ; l2 l3
+    movd      m2, [r0+1*FDEC_STRIDEB-4] ; l1
+    punpcklbw m2, [r0+0*FDEC_STRIDEB-4] ; l0 l1
+    punpckh%3 m1, m2                    ; l0 l1 l2 l3
+    punpckh%4 m1, m0                    ; t2 t1 t0 lt l0 l1 l2 l3

 %endif
 %endif

-    punpckh%3 m1, m2                                 ; l0 l1 l2 l3
-    punpckh%6 m1, m0                                 ; t2 t1 t0 lt l0 l1 l2 l3
-    psrl%4    m2, m1, %7                             ; .. t2 t1 t0 lt l0 l1 l2
-    psrl%4    m0, m1, %7*2                           ; .. .. t2 t1 t0 lt l0 l1
-    pavg%5    m5, m1, m2
-    PRED8x8_LOWPASS %5, m3, m1, m0, m2, m4
+    PSRLPIX   m2, m1, 1                 ; .. t2 t1 t0 lt l0 l1 l2
+    PSRLPIX   m0, m1, 2                 ; .. .. t2 t1 t0 lt l0 l1
+    pavg%1    m5, m1, m2
+    PRED8x8_LOWPASS m3, m1, m0, m2, m4

     punpckl%2 m5, m3
     punpckl%2 m5, m3

-    psrl%4    m3, %7*4
+    PSRLPIX   m3, m3, 4

     PALIGNR   m3, m5, 6*SIZEOF_PIXEL, m4
 %assign Y 3
     movh      [r0+Y*FDEC_STRIDEB], m5
 %rep 2
 %assign Y (Y-1)
     PALIGNR   m3, m5, 6*SIZEOF_PIXEL, m4
 %assign Y 3
     movh      [r0+Y*FDEC_STRIDEB], m5
 %rep 2
 %assign Y (Y-1)

-    psrl%4    m5, %7*2
+    PSRLPIX   m5, m5, 2

     movh      [r0+Y*FDEC_STRIDEB], m5
 %endrep
     movh      [r0+0*FDEC_STRIDEB], m3
     RET
     movh      [r0+Y*FDEC_STRIDEB], m5
 %endrep
     movh      [r0+0*FDEC_STRIDEB], m3
     RET

-%endmacro
+%endmacro ; PREDICT_4x4

 
 

-%ifdef HIGH_BIT_DEPTH
-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-cglobal predict_4x4_ddr_mmxext, 1,1
-    movq    m3, [r0+3*FDEC_STRIDEB-8]
-    psrlq   m3, 48
-    PALIGNR m3, [r0+2*FDEC_STRIDEB-8], 6, m6
-    PALIGNR m3, [r0+1*FDEC_STRIDEB-8], 6, m7
-    movq    m6, [r0+0*FDEC_STRIDEB-8]
-    PALIGNR m3, m6, 6, m5
-
-    movq    m4, [r0-1*FDEC_STRIDEB-8]
-    movq    m2, m3
-    movq    m1, m3
-    PALIGNR m2, m4, 6, m5
-    movq    m1, m2
-    psllq   m1, 16
-    PRED8x8_LOWPASS w, m0, m3, m1, m2
-    pshufw  m0, m0, 0x1B
+;-----------------------------------------------------------------------------
+; void predict_4x4_ddr( pixel *src )
+;-----------------------------------------------------------------------------
+%if HIGH_BIT_DEPTH
+INIT_MMX mmx2
+cglobal predict_4x4_ddr, 1,1
+    mova      m0, [r0+1*FDEC_STRIDEB-8]
+    punpckhwd m0, [r0+0*FDEC_STRIDEB-8]
+    mova      m3, [r0+3*FDEC_STRIDEB-8]
+    punpckhwd m3, [r0+2*FDEC_STRIDEB-8]
+    punpckhdq m3, m0
+
+    pshufw  m0, m3, q3321
+    pinsrw  m0, [r0-1*FDEC_STRIDEB-2], 3
+    pshufw  m1, m0, q3321
+    PRED8x8_LOWPASS m0, m1, m3, m0

     movq    [r0+3*FDEC_STRIDEB], m0
 
     movq    m2, [r0-1*FDEC_STRIDEB-0]
     movq    [r0+3*FDEC_STRIDEB], m0
 
     movq    m2, [r0-1*FDEC_STRIDEB-0]

-    movq    m5, m2
-    PALIGNR m5, m4, 6, m4
-    movq    m3, m5
-    PALIGNR m5, m6, 6, m6
-    PRED8x8_LOWPASS w, m1, m5, m2, m3
+    pshufw  m4, m2, q2100
+    pinsrw  m4, [r0-1*FDEC_STRIDEB-2], 0
+    movq    m1, m4
+    PALIGNR m4, m3, 6, m3
+    PRED8x8_LOWPASS m1, m4, m2, m1

     movq    [r0+0*FDEC_STRIDEB], m1
 
     movq    [r0+0*FDEC_STRIDEB], m1
 

-    psllq   m0, 16
-    PALIGNR m1, m0, 6, m2
-    movq    [r0+1*FDEC_STRIDEB], m1
+    pshufw  m2, m0, q3321
+    punpckldq m2, m1

     psllq   m0, 16
     PALIGNR m1, m0, 6, m0
     psllq   m0, 16
     PALIGNR m1, m0, 6, m0

-    movq    [r0+2*FDEC_STRIDEB], m1
-    psrlq   m1, 16
+    movq    [r0+1*FDEC_STRIDEB], m1
+    movq    [r0+2*FDEC_STRIDEB], m2

     movd    [r0+3*FDEC_STRIDEB+4], m1
     RET
 
     movd    [r0+3*FDEC_STRIDEB+4], m1
     RET
 

-cglobal predict_4x4_hd_mmxext, 1,1
+;-----------------------------------------------------------------------------
+; void predict_4x4_hd( pixel *src )
+;-----------------------------------------------------------------------------
+cglobal predict_4x4_hd, 1,1

     mova      m0, [r0+1*FDEC_STRIDEB-8]
     punpckhwd m0, [r0+0*FDEC_STRIDEB-8]
     mova      m1, [r0+3*FDEC_STRIDEB-8]
     punpckhwd m1, [r0+2*FDEC_STRIDEB-8]
     punpckhdq m1, m0
     mova      m0, m1
     mova      m0, [r0+1*FDEC_STRIDEB-8]
     punpckhwd m0, [r0+0*FDEC_STRIDEB-8]
     mova      m1, [r0+3*FDEC_STRIDEB-8]
     punpckhwd m1, [r0+2*FDEC_STRIDEB-8]
     punpckhdq m1, m0
     mova      m0, m1

-    mova      m4, m1

 
     movu      m3, [r0-1*FDEC_STRIDEB-2]
 
     movu      m3, [r0-1*FDEC_STRIDEB-2]

+    pshufw    m4, m1, q0032

     mova      m7, m3
     mova      m7, m3

-    punpckhdq m4, [r0-1*FDEC_STRIDEB-6]
+    punpckldq m4, m3

     PALIGNR   m3, m1, 2, m2
     PALIGNR   m3, m1, 2, m2

-    PRED8x8_LOWPASS w, m2, m4, m1, m3, m6
+    PRED8x8_LOWPASS m2, m4, m1, m3

 
     pavgw     m0, m3
 
     pavgw     m0, m3

-    mova      m5, m0
-    punpcklwd m5, m2
-    mova      m4, m0
-    punpckhwd m4, m2
+    punpcklwd m5, m0, m2
+    punpckhwd m4, m0, m2

     mova      [r0+3*FDEC_STRIDEB], m5
     mova      [r0+1*FDEC_STRIDEB], m4
     mova      [r0+3*FDEC_STRIDEB], m5
     mova      [r0+1*FDEC_STRIDEB], m4

+    psrlq     m5, 32
+    punpckldq m5, m4
+    mova      [r0+2*FDEC_STRIDEB], m5

 
 

-    mova      m4, m7
+    pshufw    m4, m7, q2100

     mova      m6, [r0-1*FDEC_STRIDEB+0]
     mova      m6, [r0-1*FDEC_STRIDEB+0]

-    PALIGNR   m7, [r0+0*FDEC_STRIDEB-8], 6, m5
-    PRED8x8_LOWPASS w, m3, m7, m6, m4, m1
-
-    PALIGNR   m3, m0, 6, m5
+    pinsrw    m4, [r0+0*FDEC_STRIDEB-2], 0
+    PRED8x8_LOWPASS m3, m4, m6, m7
+    PALIGNR   m3, m0, 6, m0

     mova      [r0+0*FDEC_STRIDEB], m3
     mova      [r0+0*FDEC_STRIDEB], m3

-    psrlq     m0, 16
-    psrlq     m2, 16
-    punpcklwd m0, m2
-    mova      [r0+2*FDEC_STRIDEB], m0

     RET
 
     RET
 

-INIT_XMM
-%define PALIGNR PALIGNR_MMX
-PREDICT_4x4 sse2  , wd, dq, dq, w, qdq, 2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_4x4 ssse3 , wd, dq, dq, w, qdq, 2
-INIT_AVX
-PREDICT_4x4 avx   , wd, dq, dq, w, qdq, 2
-%else
-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-PREDICT_4x4 mmxext, bw, wd, q , b, dq , 8
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_4x4 ssse3 , bw, wd, q , b, dq , 8
+INIT_XMM sse2
+PREDICT_4x4 w, wd, dq, qdq
+INIT_XMM ssse3
+PREDICT_4x4 w, wd, dq, qdq
+INIT_XMM avx
+PREDICT_4x4 w, wd, dq, qdq
+%else ; !HIGH_BIT_DEPTH
+INIT_MMX mmx2
+PREDICT_4x4 b, bw, wd, dq
+INIT_MMX ssse3
+%define predict_4x4_vr_ssse3 predict_4x4_vr_ssse3_cache64
+PREDICT_4x4 b, bw, wd, dq

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_hu( pixel *src )
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_hu( pixel *src )
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

 INIT_MMX
 INIT_MMX

-cglobal predict_4x4_hu_mmxext, 1,1
-    movq      m0, [r0+0*FDEC_STRIDEB-4*2]
-    punpckhwd m0, [r0+1*FDEC_STRIDEB-4*2]
-    movq      m1, [r0+2*FDEC_STRIDEB-4*2]
-    punpckhwd m1, [r0+3*FDEC_STRIDEB-4*2]
+cglobal predict_4x4_hu_mmx2, 1,1
+    movq      m0, [r0+0*FDEC_STRIDEB-8]
+    punpckhwd m0, [r0+1*FDEC_STRIDEB-8]
+    movq      m1, [r0+2*FDEC_STRIDEB-8]
+    punpckhwd m1, [r0+3*FDEC_STRIDEB-8]

     punpckhdq m0, m1
     punpckhdq m0, m1

-    pshufw    m1, m1, 0xFF
+    pshufw    m1, m1, q3333

     movq      [r0+3*FDEC_STRIDEB], m1
     movq      [r0+3*FDEC_STRIDEB], m1

-    movd      [r0+2*FDEC_STRIDEB+4], m1
-    mova      m2, m0
-    psrlq     m2, 16
-    pavgw     m2, m0
-
-    pshufw    m1, m0, 11111001b
-    pshufw    m5, m0, 11111110b
-    PRED8x8_LOWPASS w, m3, m0, m5, m1, m7
-    movq      m6, m2
-    punpcklwd m6, m3
-    mova      [r0+0*FDEC_STRIDEB], m6
+    pshufw    m3, m0, q3321
+    pshufw    m4, m0, q3332
+    pavgw     m2, m0, m3
+    PRED8x8_LOWPASS m3, m0, m4, m3
+    punpcklwd m4, m2, m3
+    mova      [r0+0*FDEC_STRIDEB], m4

     psrlq     m2, 16
     psrlq     m3, 16
     punpcklwd m2, m3
     mova      [r0+1*FDEC_STRIDEB], m2
     psrlq     m2, 16
     psrlq     m3, 16
     punpcklwd m2, m3
     mova      [r0+1*FDEC_STRIDEB], m2

-    psrlq     m2, 32
-    movd      [r0+2*FDEC_STRIDEB+0], m2
+    punpckhdq m2, m1
+    mova      [r0+2*FDEC_STRIDEB], m2

     RET
 
 %else ; !HIGH_BIT_DEPTH
 INIT_MMX
     RET
 
 %else ; !HIGH_BIT_DEPTH
 INIT_MMX

-cglobal predict_4x4_hu_mmxext, 1,1
-    movq      mm0, [r0+0*FDEC_STRIDE-8]
-    punpckhbw mm0, [r0+1*FDEC_STRIDE-8]
-    movq      mm1, [r0+2*FDEC_STRIDE-8]
-    punpckhbw mm1, [r0+3*FDEC_STRIDE-8]
-    punpckhwd mm0, mm1
-    movq      mm1, mm0
-    punpckhbw mm1, mm1
-    pshufw    mm1, mm1, 0xFF
-    punpckhdq mm0, mm1
-    movq      mm2, mm0
-    movq      mm3, mm0
-    movq      mm7, mm0
-    psrlq     mm2, 16
-    psrlq     mm3, 8
-    pavgb     mm7, mm3
-    PRED8x8_LOWPASS b, mm4, mm0, mm2, mm3, mm5
-    punpcklbw mm7, mm4
-%assign Y 0
-    movd    [r0+Y*FDEC_STRIDE], mm7
-%rep 2
-%assign Y (Y+1)
-    psrlq    mm7, 16
-    movd    [r0+Y*FDEC_STRIDE], mm7
-%endrep
-    movd    [r0+3*FDEC_STRIDE], mm1
+cglobal predict_4x4_hu_mmx2, 1,1
+    movd      m1, [r0+0*FDEC_STRIDEB-4]
+    punpcklbw m1, [r0+1*FDEC_STRIDEB-4]
+    movd      m0, [r0+2*FDEC_STRIDEB-4]
+    punpcklbw m0, [r0+3*FDEC_STRIDEB-4]
+    punpckhwd m1, m0
+    movq      m0, m1
+    punpckhbw m1, m1
+    pshufw    m1, m1, q3333
+    punpckhdq m0, m1
+    movq      m2, m0
+    movq      m3, m0
+    movq      m5, m0
+    psrlq     m3, 8
+    psrlq     m2, 16
+    pavgb     m5, m3
+    PRED8x8_LOWPASS m3, m0, m2, m3, m4
+    movd      [r0+3*FDEC_STRIDEB], m1
+    punpcklbw m5, m3
+    movd      [r0+0*FDEC_STRIDEB], m5
+    psrlq     m5, 16
+    movd      [r0+1*FDEC_STRIDEB], m5
+    psrlq     m5, 16
+    movd      [r0+2*FDEC_STRIDEB], m5

     RET
 %endif ; HIGH_BIT_DEPTH
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_vl( pixel *src )
 ;-----------------------------------------------------------------------------
     RET
 %endif ; HIGH_BIT_DEPTH
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_vl( pixel *src )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_4x4_V1 4
-cglobal predict_4x4_vl_%1, 1,1,6*(mmsize/16)
+%macro PREDICT_4x4_V1 1
+cglobal predict_4x4_vl, 1,1

     movu        m1, [r0-FDEC_STRIDEB]
     movu        m1, [r0-FDEC_STRIDEB]

-    psrl%2      m3, m1, %3
-    psrl%2      m2, m1, %3*2
-    pavg%4      m4, m3, m1
-    PRED8x8_LOWPASS %4, m0, m1, m2, m3, m5
+    PSRLPIX     m3, m1, 1
+    PSRLPIX     m2, m1, 2
+    pavg%1      m4, m3, m1
+    PRED8x8_LOWPASS m0, m1, m2, m3, m5

 
     movh        [r0+0*FDEC_STRIDEB], m4
     movh        [r0+1*FDEC_STRIDEB], m0
 
     movh        [r0+0*FDEC_STRIDEB], m4
     movh        [r0+1*FDEC_STRIDEB], m0

-    psrl%2      m4, %3
-    psrl%2      m0, %3
+    PSRLPIX     m4, m4, 1
+    PSRLPIX     m0, m0, 1

     movh        [r0+2*FDEC_STRIDEB], m4
     movh        [r0+3*FDEC_STRIDEB], m0
     RET
 %endmacro
 
     movh        [r0+2*FDEC_STRIDEB], m4
     movh        [r0+3*FDEC_STRIDEB], m0
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_4x4_V1 sse2, dq, 2, w
-%ifdef ARCH_X86_64
-INIT_AVX
-PREDICT_4x4_V1 avx , dq, 2, w
-%endif
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_4x4_V1 w
+INIT_XMM avx
+PREDICT_4x4_V1 w

 
 

-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-cglobal predict_4x4_vl_mmxext, 1,4
+INIT_MMX mmx2
+cglobal predict_4x4_vl, 1,4

     mova    m1, [r0-FDEC_STRIDEB+0]
     mova    m2, [r0-FDEC_STRIDEB+8]
     mova    m1, [r0-FDEC_STRIDEB+0]
     mova    m2, [r0-FDEC_STRIDEB+8]

-    mova    m3, m2
-    PALIGNR m2, m1, 4, m6
-    PALIGNR m3, m1, 2, m5
-    mova    m4, m3
-    pavgw   m4, m1
-    mova    [r0+0*FDEC_STRIDEB], m4
-    psrlq   m4, 16
-    mova    [r0+2*FDEC_STRIDEB], m4
-    PRED8x8_LOWPASS w, m0, m1, m2, m3, m6
+    mova    m0, m2
+    PALIGNR m2, m1, 4, m4
+    PALIGNR m0, m1, 2, m4
+    mova    m3, m0
+    pavgw   m3, m1
+    mova    [r0+0*FDEC_STRIDEB], m3
+    psrlq   m3, 16
+    mova    [r0+2*FDEC_STRIDEB], m3
+    PRED8x8_LOWPASS m0, m1, m2, m0

     mova    [r0+1*FDEC_STRIDEB], m0
     psrlq   m0, 16
     mova    [r0+3*FDEC_STRIDEB], m0
     mova    [r0+1*FDEC_STRIDEB], m0
     psrlq   m0, 16
     mova    [r0+3*FDEC_STRIDEB], m0


@@ -544,17 +575,17 @@ cglobal predict_4x4_vl_mmxext, 1,4
     mov     [r0+2*FDEC_STRIDEB+6], r1w
     mov     [r0+3*FDEC_STRIDEB+6], r3w
     RET
     mov     [r0+2*FDEC_STRIDEB+6], r1w
     mov     [r0+3*FDEC_STRIDEB+6], r3w
     RET

-%else
-INIT_MMX
-PREDICT_4x4_V1 mmxext, q , 8, b
+%else ; !HIGH_BIT_DEPTH
+INIT_MMX mmx2
+PREDICT_4x4_V1 b

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_dc( pixel *src )
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_4x4_dc( pixel *src )
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
-INIT_MMX
-cglobal predict_4x4_dc_mmxext, 1,1
+INIT_MMX mmx2
+%if HIGH_BIT_DEPTH
+cglobal predict_4x4_dc, 1,1

     mova   m2, [r0+0*FDEC_STRIDEB-4*SIZEOF_PIXEL]
     paddw  m2, [r0+1*FDEC_STRIDEB-4*SIZEOF_PIXEL]
     paddw  m2, [r0+2*FDEC_STRIDEB-4*SIZEOF_PIXEL]
     mova   m2, [r0+0*FDEC_STRIDEB-4*SIZEOF_PIXEL]
     paddw  m2, [r0+1*FDEC_STRIDEB-4*SIZEOF_PIXEL]
     paddw  m2, [r0+2*FDEC_STRIDEB-4*SIZEOF_PIXEL]


@@ -572,38 +603,37 @@ cglobal predict_4x4_dc_mmxext, 1,1
     mova   [r0+3*FDEC_STRIDEB], m0
     RET
 
     mova   [r0+3*FDEC_STRIDEB], m0
     RET
 

-%else
-INIT_MMX
-cglobal predict_4x4_dc_mmxext, 1,4
+%else ; !HIGH_BIT_DEPTH
+cglobal predict_4x4_dc, 1,4

     pxor   mm7, mm7
     pxor   mm7, mm7

-    movd   mm0, [r0-FDEC_STRIDE]
+    movd   mm0, [r0-FDEC_STRIDEB]

     psadbw mm0, mm7
     movd   r3d, mm0
     movzx  r1d, byte [r0-1]
     psadbw mm0, mm7
     movd   r3d, mm0
     movzx  r1d, byte [r0-1]

-%assign n 1
+%assign Y 1

 %rep 3
 %rep 3

-    movzx  r2d, byte [r0+FDEC_STRIDE*n-1]
+    movzx  r2d, byte [r0+FDEC_STRIDEB*Y-1]

     add    r1d, r2d
     add    r1d, r2d

-%assign n n+1
+%assign Y Y+1

 %endrep
     lea    r1d, [r1+r3+4]
     shr    r1d, 3
     imul   r1d, 0x01010101
 %endrep
     lea    r1d, [r1+r3+4]
     shr    r1d, 3
     imul   r1d, 0x01010101

-    mov   [r0+FDEC_STRIDE*0], r1d
-    mov   [r0+FDEC_STRIDE*1], r1d
-    mov   [r0+FDEC_STRIDE*2], r1d
-    mov   [r0+FDEC_STRIDE*3], r1d
+    mov   [r0+FDEC_STRIDEB*0], r1d
+    mov   [r0+FDEC_STRIDEB*1], r1d
+    mov   [r0+FDEC_STRIDEB*2], r1d
+    mov   [r0+FDEC_STRIDEB*3], r1d

     RET
 %endif ; HIGH_BIT_DEPTH
 
     RET
 %endif ; HIGH_BIT_DEPTH
 

-%macro PREDICT_FILTER 6
+%macro PREDICT_FILTER 4

 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------

-;void predict_8x8_filter( pixel *src, pixel edge[33], int i_neighbor, int i_filters )
+;void predict_8x8_filter( pixel *src, pixel edge[36], int i_neighbor, int i_filters )

 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_filter_%1, 4,5,7*(mmsize/16)
+cglobal predict_8x8_filter, 4,6,6

     add          r0, 0x58*SIZEOF_PIXEL
 %define src r0-0x58*SIZEOF_PIXEL
     add          r0, 0x58*SIZEOF_PIXEL
 %define src r0-0x58*SIZEOF_PIXEL

-%ifndef ARCH_X86_64
+%if ARCH_X86_64 == 0

     mov          r4, r1
 %define t1 r4
 %define t4 r1
     mov          r4, r1
 %define t1 r4
 %define t4 r1


@@ -611,162 +641,165 @@ cglobal predict_8x8_filter_%1, 4,5,7*(mmsize/16)
 %define t1 r1
 %define t4 r4
 %endif
 %define t1 r1
 %define t4 r4
 %endif

-    test       r3b, 0x01
+    test       r3b, 1

     je .check_top
     je .check_top

+    mov        t4d, r2d
+    and        t4d, 8
+    neg         t4

     mova        m0, [src+0*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m0, [src+0*FDEC_STRIDEB-8*SIZEOF_PIXEL]

-    punpckh%2%3 m0, [src-1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
+    punpckh%1%2 m0, [src+0*FDEC_STRIDEB-8*SIZEOF_PIXEL+t4*(FDEC_STRIDEB/8)]

     mova        m1, [src+2*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m1, [src+2*FDEC_STRIDEB-8*SIZEOF_PIXEL]

-    punpckh%2%3 m1, [src+1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
-    punpckh%3%4 m1, m0
+    punpckh%1%2 m1, [src+1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
+    punpckh%2%3 m1, m0

     mova        m2, [src+4*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m2, [src+4*FDEC_STRIDEB-8*SIZEOF_PIXEL]

-    punpckh%2%3 m2, [src+3*FDEC_STRIDEB-8*SIZEOF_PIXEL]
+    punpckh%1%2 m2, [src+3*FDEC_STRIDEB-8*SIZEOF_PIXEL]

     mova        m3, [src+6*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m3, [src+6*FDEC_STRIDEB-8*SIZEOF_PIXEL]

-    punpckh%2%3 m3, [src+5*FDEC_STRIDEB-8*SIZEOF_PIXEL]
-    punpckh%3%4 m3, m2
-    punpckh%4%5 m3, m1
+    punpckh%1%2 m3, [src+5*FDEC_STRIDEB-8*SIZEOF_PIXEL]
+    punpckh%2%3 m3, m2
+    punpckh%3%4 m3, m1

     mova        m0, [src+7*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m1, [src-1*FDEC_STRIDEB]
     mova        m0, [src+7*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m1, [src-1*FDEC_STRIDEB]

-    mova        m4, m3
-    mova        m2, m3
-    PALIGNR     m4, m0, 7*SIZEOF_PIXEL, m0
-    PALIGNR     m1, m2, 1*SIZEOF_PIXEL, m2
-    test       r2b, 0x08
-    je .fix_lt_1
-.do_left:
-    mova        m0, m4
-    PRED8x8_LOWPASS %2, m2, m1, m4, m3, m5
-    mova        [t1+8*SIZEOF_PIXEL], m2
-    mova        m4, m0
-    PRED8x8_LOWPASS %2, m1, m3, m0, m4, m5
-    movd        t4, m1
-    mov         [t1+7*SIZEOF_PIXEL], t4%2
-.check_top:
-    test        r3b, 0x02
+    PALIGNR     m4, m3, m0, 7*SIZEOF_PIXEL, m0
+    PALIGNR     m1, m1, m3, 1*SIZEOF_PIXEL, m2
+    PRED8x8_LOWPASS m3, m1, m4, m3, m5
+    mova        [t1+8*SIZEOF_PIXEL], m3
+    movzx      t4d, pixel [src+7*FDEC_STRIDEB-1*SIZEOF_PIXEL]
+    movzx      r5d, pixel [src+6*FDEC_STRIDEB-1*SIZEOF_PIXEL]
+    lea        t4d, [t4*3+2]
+    add        t4d, r5d
+    shr        t4d, 2
+    mov         [t1+7*SIZEOF_PIXEL], t4%1
+    mov         [t1+6*SIZEOF_PIXEL], t4%1
+    test       r3b, 2

     je .done
     je .done

+.check_top:
+%if SIZEOF_PIXEL==1 && cpuflag(ssse3)
+INIT_XMM cpuname
+    movu        m3, [src-1*FDEC_STRIDEB]
+    movhps      m0, [src-1*FDEC_STRIDEB-8]
+    test       r2b, 8
+    je .fix_lt_2
+.do_top:
+    and        r2d, 4
+%ifdef PIC
+    lea         r3, [shuf_fixtr]
+    pshufb      m3, [r3+r2*4]
+%else
+    pshufb      m3, [shuf_fixtr+r2*4] ; neighbor&MB_TOPRIGHT ? shuf_nop : shuf_fixtr
+%endif
+    psrldq      m1, m3, 15
+    PALIGNR     m2, m3, m0, 15, m0
+    PALIGNR     m1, m3, 1, m5
+    PRED8x8_LOWPASS m0, m2, m1, m3, m5
+    mova        [t1+16*SIZEOF_PIXEL], m0
+    psrldq      m0, 15
+    movd        [t1+32*SIZEOF_PIXEL], m0
+.done:
+    REP_RET
+.fix_lt_2:
+    pslldq      m0, m3, 15
+    jmp .do_top
+
+%else

     mova        m0, [src-1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m3, [src-1*FDEC_STRIDEB]
     mova        m1, [src-1*FDEC_STRIDEB+8*SIZEOF_PIXEL]
     mova        m0, [src-1*FDEC_STRIDEB-8*SIZEOF_PIXEL]
     mova        m3, [src-1*FDEC_STRIDEB]
     mova        m1, [src-1*FDEC_STRIDEB+8*SIZEOF_PIXEL]

-    mova        m2, m3
-    mova        m4, m3
-    PALIGNR     m2, m0, 7*SIZEOF_PIXEL, m0
-    PALIGNR     m1, m4, 1*SIZEOF_PIXEL, m4
-    test        r2b, 0x08
+    test       r2b, 8

     je .fix_lt_2
     je .fix_lt_2

-    test        r2b, 0x04
+    test       r2b, 4

     je .fix_tr_1
 .do_top:
     je .fix_tr_1
 .do_top:

-    PRED8x8_LOWPASS %2, m4, m2, m1, m3, m5
+    PALIGNR     m2, m3, m0, 7*SIZEOF_PIXEL, m0
+    PALIGNR     m0, m1, m3, 1*SIZEOF_PIXEL, m5
+    PRED8x8_LOWPASS m4, m2, m0, m3, m5

     mova        [t1+16*SIZEOF_PIXEL], m4
     mova        [t1+16*SIZEOF_PIXEL], m4

-    test        r3b, 0x04
+    test       r3b, 4

     je .done
     je .done

-    test        r2b, 0x04
-    je .fix_tr_2
-    mova        m0, [src-1*FDEC_STRIDEB+8*SIZEOF_PIXEL]
-    mova        m2, m0
-    mova        m4, m0
-    psrl%5      m5, m0, 7*%6
-    PALIGNR     m2, m3, 7*SIZEOF_PIXEL, m3
-    PALIGNR     m5, m4, 1*SIZEOF_PIXEL, m4
-    PRED8x8_LOWPASS %2, m1, m2, m5, m0, m4
-    jmp .do_topright
-.fix_tr_2:
-    punpckh%2%3 m3, m3
-    pshuf%3     m1, m3, 0xFF
-.do_topright:
-    mova        [t1+24*SIZEOF_PIXEL], m1
-    psrl%5      m1, 7*%6
-    movd        t4, m1
-    mov         [t1+32*SIZEOF_PIXEL], t4%2
+    PSRLPIX     m5, m1, 7
+    PALIGNR     m2, m1, m3, 7*SIZEOF_PIXEL, m3
+    PALIGNR     m5, m1, 1*SIZEOF_PIXEL, m4
+    PRED8x8_LOWPASS m0, m2, m5, m1, m4
+    mova        [t1+24*SIZEOF_PIXEL], m0
+    PSRLPIX     m0, m0, 7
+    movd        [t1+32*SIZEOF_PIXEL], m0

 .done:
     REP_RET
 .done:
     REP_RET

-.fix_lt_1:
-    pxor        m5, m3, m4
-    psrl%5      m5, 7*%6
-    psll%5      m5, 6*%6
-    pxor        m1, m5
-    jmp .do_left

 .fix_lt_2:
 .fix_lt_2:

-    pxor        m5, m3, m2
-    psll%5      m5, 7*%6
-    psrl%5      m5, 7*%6
-    pxor        m2, m5
-    test       r2b, 0x04
+    PSLLPIX     m0, m3, 7
+    test       r2b, 4

     jne .do_top
 .fix_tr_1:
     jne .do_top
 .fix_tr_1:

-    pxor        m5, m3, m1
-    psrl%5      m5, 7*%6
-    psll%5      m5, 7*%6
-    pxor        m1, m5
+    punpckh%1%2 m1, m3, m3
+    pshuf%2     m1, m1, q3333

     jmp .do_top
     jmp .do_top

+%endif

 %endmacro
 
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-%define PALIGNR PALIGNR_MMX
-PREDICT_FILTER sse2  , w, d, q, dq, 2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_FILTER ssse3 , w, d, q, dq, 2
-INIT_AVX
-PREDICT_FILTER avx   , w, d, q, dq, 2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_FILTER w, d, q, dq
+INIT_XMM ssse3
+PREDICT_FILTER w, d, q, dq
+INIT_XMM avx
+PREDICT_FILTER w, d, q, dq

 %else
 %else

-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-PREDICT_FILTER mmxext, b, w, d, q , 8
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_FILTER ssse3 , b, w, d, q , 8
+INIT_MMX mmx2
+PREDICT_FILTER b, w, d, q
+INIT_MMX ssse3
+PREDICT_FILTER b, w, d, q

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_v( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_v( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_V 1
-cglobal predict_8x8_v_%1, 2,2
+%macro PREDICT_8x8_V 0
+cglobal predict_8x8_v, 2,2

     mova        m0, [r1+16*SIZEOF_PIXEL]
     STORE8x8    m0, m0
     RET
 %endmacro
 
     mova        m0, [r1+16*SIZEOF_PIXEL]
     STORE8x8    m0, m0
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_8x8_V sse2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_V

 %else
 %else

-INIT_MMX
-PREDICT_8x8_V mmxext
+INIT_MMX mmx2
+PREDICT_8x8_V

 %endif
 
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------

-; void predict_8x8_h( pixel *src, pixel edge[33] )
+; void predict_8x8_h( pixel *src, pixel edge[36] )

 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_H 3
-cglobal predict_8x8_h_%1, 2,2
+%macro PREDICT_8x8_H 2
+cglobal predict_8x8_h, 2,2

     movu      m1, [r1+7*SIZEOF_PIXEL]
     add       r0, 4*FDEC_STRIDEB
     movu      m1, [r1+7*SIZEOF_PIXEL]
     add       r0, 4*FDEC_STRIDEB

-    punpckl%2 m2, m1, m1
-    punpckh%2 m1, m1
-%assign n 0
+    punpckl%1 m2, m1, m1
+    punpckh%1 m1, m1
+%assign Y 0

 %rep 8
 %rep 8

-%assign i 1+n/4
-    SPLAT%3 m0, m %+ i, (3-n)&3
-    mova [r0+(n-4)*FDEC_STRIDEB], m0
-%assign n n+1
+%assign i 1+Y/4
+    SPLAT%2 m0, m %+ i, (3-Y)&3
+    mova [r0+(Y-4)*FDEC_STRIDEB], m0
+%assign Y Y+1

 %endrep
     RET
 %endmacro
 
 %endrep
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_8x8_H sse2  , wd, D
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_H wd, D

 %else
 %else

-INIT_MMX
-PREDICT_8x8_H mmxext, bw, W
+INIT_MMX mmx2
+PREDICT_8x8_H bw, W

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_dc( pixel *src, pixel *edge );
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_dc( pixel *src, pixel *edge );
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-cglobal predict_8x8_dc_sse2, 2,2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+cglobal predict_8x8_dc, 2,2

     movu        m0, [r1+14]
     paddw       m0, [r1+32]
     HADDW       m0, m1
     movu        m0, [r1+14]
     paddw       m0, [r1+32]
     HADDW       m0, m1


@@ -776,9 +809,9 @@ cglobal predict_8x8_dc_sse2, 2,2
     STORE8x8    m0, m0
     REP_RET
 
     STORE8x8    m0, m0
     REP_RET
 

-%else
-INIT_MMX
-cglobal predict_8x8_dc_mmxext, 2,2
+%else ; !HIGH_BIT_DEPTH
+INIT_MMX mmx2
+cglobal predict_8x8_dc, 2,2

     pxor        mm0, mm0
     pxor        mm1, mm1
     psadbw      mm0, [r1+7]
     pxor        mm0, mm0
     pxor        mm1, mm1
     psadbw      mm0, [r1+7]


@@ -796,7 +829,7 @@ cglobal predict_8x8_dc_mmxext, 2,2
 ; void predict_8x8_dc_top ( pixel *src, pixel *edge );
 ; void predict_8x8_dc_left( pixel *src, pixel *edge );
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_dc_top ( pixel *src, pixel *edge );
 ; void predict_8x8_dc_left( pixel *src, pixel *edge );
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

 %macro PREDICT_8x8_DC 3
 cglobal %1, 2,2
     %3          m0, [r1+%2]
 %macro PREDICT_8x8_DC 3
 cglobal %1, 2,2
     %3          m0, [r1+%2]


@@ -807,11 +840,11 @@ cglobal %1, 2,2
     STORE8x8    m0, m0
     RET
 %endmacro
     STORE8x8    m0, m0
     RET
 %endmacro

-INIT_XMM
-PREDICT_8x8_DC predict_8x8_dc_top_sse2 , 32, mova
-PREDICT_8x8_DC predict_8x8_dc_left_sse2, 14, movu
+INIT_XMM sse2
+PREDICT_8x8_DC predict_8x8_dc_top , 32, mova
+PREDICT_8x8_DC predict_8x8_dc_left, 14, movu

 
 

-%else
+%else ; !HIGH_BIT_DEPTH

 %macro PREDICT_8x8_DC 2
 cglobal %1, 2,2
     pxor        mm0, mm0
 %macro PREDICT_8x8_DC 2
 cglobal %1, 2,2
     pxor        mm0, mm0


@@ -824,122 +857,138 @@ cglobal %1, 2,2
     RET
 %endmacro
 INIT_MMX
     RET
 %endmacro
 INIT_MMX

-PREDICT_8x8_DC predict_8x8_dc_top_mmxext, 16
-PREDICT_8x8_DC predict_8x8_dc_left_mmxext, 7
+PREDICT_8x8_DC predict_8x8_dc_top_mmx2, 16
+PREDICT_8x8_DC predict_8x8_dc_left_mmx2, 7

 %endif ; HIGH_BIT_DEPTH
 
 ; sse2 is faster even on amd for 8-bit, so there's no sense in spending exe
 ; size on the 8-bit mmx functions below if we know sse2 is available.
 %endif ; HIGH_BIT_DEPTH
 
 ; sse2 is faster even on amd for 8-bit, so there's no sense in spending exe
 ; size on the 8-bit mmx functions below if we know sse2 is available.

-%macro PREDICT_8x8 4
+%macro PREDICT_8x8_DDLR 0

 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddl( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddl( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_ddl_%1, 2,2,8*(mmsize/16)
-    mova        m5, [r1+16*SIZEOF_PIXEL]
-    movu        m2, [r1+17*SIZEOF_PIXEL]
-    movu        m3, [r1+23*SIZEOF_PIXEL]
-    movu        m4, [r1+25*SIZEOF_PIXEL]
-    psll%3      m1, m5, %4
-    add         r0, FDEC_STRIDEB*4
-    PRED8x8_LOWPASS %2, m0, m1, m2, m5, m7
-%if avx_enabled == 1
-    INIT_XMM
-    PRED8x8_LOWPASS %2, m1, m3, m4, [r1+24*SIZEOF_PIXEL], m6
-    INIT_AVX
+cglobal predict_8x8_ddl, 2,2,7
+    mova        m0, [r1+16*SIZEOF_PIXEL]
+    mova        m1, [r1+24*SIZEOF_PIXEL]
+%if cpuflag(cache64)
+    movd        m5, [r1+32*SIZEOF_PIXEL]
+    palignr     m3, m1, m0, 1*SIZEOF_PIXEL
+    palignr     m5, m5, m1, 1*SIZEOF_PIXEL
+    palignr     m4, m1, m0, 7*SIZEOF_PIXEL

 %else
 %else

-    PRED8x8_LOWPASS %2, m1, m3, m4, [r1+24*SIZEOF_PIXEL], m6
+    movu        m3, [r1+17*SIZEOF_PIXEL]
+    movu        m4, [r1+23*SIZEOF_PIXEL]
+    movu        m5, [r1+25*SIZEOF_PIXEL]

 %endif
 %endif

-%assign Y 3
+    PSLLPIX     m2, m0, 1
+    add         r0, FDEC_STRIDEB*4
+    PRED8x8_LOWPASS m0, m2, m3, m0, m6
+    PRED8x8_LOWPASS m1, m4, m5, m1, m6
+    mova        [r0+3*FDEC_STRIDEB], m1
+%assign Y 2

 %rep 6
 %rep 6

+    PALIGNR     m1, m0, 7*SIZEOF_PIXEL, m2
+    PSLLPIX     m0, m0, 1

     mova        [r0+Y*FDEC_STRIDEB], m1
     mova        [r0+Y*FDEC_STRIDEB], m1

-    psll%3      m1, %4
-    psrl%3      m2, m0, 7*%4
-    psll%3      m0, %4
-    por         m1, m2

 %assign Y (Y-1)
 %endrep
 %assign Y (Y-1)
 %endrep

-    mova        [r0+Y*FDEC_STRIDEB], m1
-    psll%3      m1, %4
-    psrl%3      m0, 7*%4
-    por         m1, m0
-%assign Y (Y-1)
+    PALIGNR     m1, m0, 7*SIZEOF_PIXEL, m0

     mova        [r0+Y*FDEC_STRIDEB], m1
     RET
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddr( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
     mova        [r0+Y*FDEC_STRIDEB], m1
     RET
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddr( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-%if avx_enabled == 0
-cglobal predict_8x8_ddr_%1, 2,2,7*(mmsize/16)
-    movu        m1, [r1+ 7*SIZEOF_PIXEL]
-    movu        m2, [r1+ 9*SIZEOF_PIXEL]
-    movu        m3, [r1+15*SIZEOF_PIXEL]
-    movu        m4, [r1+17*SIZEOF_PIXEL]
+cglobal predict_8x8_ddr, 2,2,7

     add         r0, FDEC_STRIDEB*4
     add         r0, FDEC_STRIDEB*4

-    PRED8x8_LOWPASS %2, m0, m1, m2, [r1+ 8*SIZEOF_PIXEL], m5
-    PRED8x8_LOWPASS %2, m1, m3, m4, [r1+16*SIZEOF_PIXEL], m6
-%assign Y 3
+    mova        m0, [r1+ 8*SIZEOF_PIXEL]
+    mova        m1, [r1+16*SIZEOF_PIXEL]
+    ; edge[] is 32byte aligned, so some of the unaligned loads are known to be not cachesplit
+    movu        m2, [r1+ 7*SIZEOF_PIXEL]
+    movu        m5, [r1+17*SIZEOF_PIXEL]
+%if cpuflag(cache64)
+    palignr     m3, m1, m0, 1*SIZEOF_PIXEL
+    palignr     m4, m1, m0, 7*SIZEOF_PIXEL
+%else
+    movu        m3, [r1+ 9*SIZEOF_PIXEL]
+    movu        m4, [r1+15*SIZEOF_PIXEL]
+%endif
+    PRED8x8_LOWPASS m0, m2, m3, m0, m6
+    PRED8x8_LOWPASS m1, m4, m5, m1, m6
+    mova        [r0+3*FDEC_STRIDEB], m0
+%assign Y -4

 %rep 6
 %rep 6

-    mova        [r0+Y*FDEC_STRIDEB], m0
-    psrl%3      m0, %4
-    psll%3      m2, m1, 7*%4
-    psrl%3      m1, %4
-    por         m0, m2
-%assign Y (Y-1)
+    PALIGNR     m1, m0, 7*SIZEOF_PIXEL, m2
+    PSLLPIX     m0, m0, 1
+    mova        [r0+Y*FDEC_STRIDEB], m1
+%assign Y (Y+1)

 %endrep
 %endrep

-    mova        [r0+Y*FDEC_STRIDEB], m0
-    psrl%3      m0, %4
-    psll%3      m1, 7*%4
-    por         m0, m1
-%assign Y (Y-1)
-    mova        [r0+Y*FDEC_STRIDEB], m0
+    PALIGNR     m1, m0, 7*SIZEOF_PIXEL, m0
+    mova        [r0+Y*FDEC_STRIDEB], m1

     RET
     RET

-%endif
-%endmacro ; PREDICT_8x8
-
-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_8x8 sse2  , w, dq, 2
-INIT_AVX
-PREDICT_8x8 avx   , w, dq, 2
-%elifndef ARCH_X86_64
-INIT_MMX
-PREDICT_8x8 mmxext, b, q , 8
+%endmacro ; PREDICT_8x8_DDLR
+
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_DDLR
+INIT_XMM ssse3
+PREDICT_8x8_DDLR
+INIT_XMM ssse3, cache64
+PREDICT_8x8_DDLR
+%elif ARCH_X86_64 == 0
+INIT_MMX mmx2
+PREDICT_8x8_DDLR

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hu( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hu( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_HU 6
-cglobal predict_8x8_hu_%1, 2,2,8*(mmsize/16)
-    movu      m1, [r1+7*SIZEOF_PIXEL] ; l0 l1 l2 l3 l4 l5 l6 l7
+%macro PREDICT_8x8_HU 2
+cglobal predict_8x8_hu, 2,2,8

     add       r0, 4*FDEC_STRIDEB
     add       r0, 4*FDEC_STRIDEB

-    pshuf%4   m0, m1, 00011011b       ; l6 l7 l4 l5 l2 l3 l0 l1
-    psll%3    m1, 7*%6                ; l7 .. .. .. .. .. .. ..
+%if HIGH_BIT_DEPTH
+%if cpuflag(ssse3)
+    movu      m5, [r1+7*SIZEOF_PIXEL]
+    pshufb    m5, [pw_reverse]
+%else
+    movq      m6, [r1+7*SIZEOF_PIXEL]
+    movq      m5, [r1+11*SIZEOF_PIXEL]
+    pshuflw   m6, m6, q0123
+    pshuflw   m5, m5, q0123
+    movlhps   m5, m6
+%endif ; cpuflag
+    psrldq    m2, m5, 2
+    pshufd    m3, m5, q0321
+    pshufhw   m2, m2, q2210
+    pshufhw   m3, m3, q1110
+    pavgw     m4, m5, m2
+%else ; !HIGH_BIT_DEPTH
+    movu      m1, [r1+7*SIZEOF_PIXEL] ; l0 l1 l2 l3 l4 l5 l6 l7
+    pshufw    m0, m1, q0123           ; l6 l7 l4 l5 l2 l3 l0 l1
+    psllq     m1, 56                  ; l7 .. .. .. .. .. .. ..

     mova      m2, m0
     mova      m2, m0

-    psll%4    m0, 8*SIZEOF_PIXEL
-    psrl%4    m2, 8*SIZEOF_PIXEL
-    por       m2, m0                  ; l7 l6 l5 l4 l3 l2 l1 l0
+    psllw     m0, 8
+    psrlw     m2, 8
+    por       m2, m0
+    mova      m3, m2

     mova      m4, m2
     mova      m4, m2

-    mova      m5, m2
-    psrl%3    m3, m2, 2*%6
-    psrl%3    m2, %6
+    mova      m5, m2                  ; l7 l6 l5 l4 l3 l2 l1 l0
+    psrlq     m3, 16
+    psrlq     m2, 8

     por       m2, m1                  ; l7 l7 l6 l5 l4 l3 l2 l1
     por       m2, m1                  ; l7 l7 l6 l5 l4 l3 l2 l1

-    punpckh%5 m1, m1
+    punpckhbw m1, m1

     por       m3, m1                  ; l7 l7 l7 l6 l5 l4 l3 l2
     por       m3, m1                  ; l7 l7 l7 l6 l5 l4 l3 l2

-    pavg%2    m4, m2
-    PRED8x8_LOWPASS %2, m1, m3, m5, m2, m6
-    punpckh%5 m5, m4, m1              ; p8 p7 p6 p5
-    punpckl%5 m4, m1                  ; p4 p3 p2 p1
-    mova      m6, m5
-    mova      m7, m5
-    mova      m0, m5
-    PALIGNR   m5, m4, 2*SIZEOF_PIXEL, m1
-    pshuf%4   m1, m6, 11111001b
-    PALIGNR   m6, m4, 4*SIZEOF_PIXEL, m2
-    pshuf%4   m2, m7, 11111110b
-    PALIGNR   m7, m4, 6*SIZEOF_PIXEL, m3
-    pshuf%4   m3, m0, 11111111b
+    pavgb     m4, m2
+%endif ; !HIGH_BIT_DEPTH
+    PRED8x8_LOWPASS m2, m3, m5, m2, m6
+    punpckh%2 m0, m4, m2              ; p8 p7 p6 p5
+    punpckl%2 m4, m2                  ; p4 p3 p2 p1
+    PALIGNR   m5, m0, m4, 2*SIZEOF_PIXEL, m3
+    pshuf%1   m1, m0, q3321
+    PALIGNR   m6, m0, m4, 4*SIZEOF_PIXEL, m3
+    pshuf%1   m2, m0, q3332
+    PALIGNR   m7, m0, m4, 6*SIZEOF_PIXEL, m3
+    pshuf%1   m3, m0, q3333

     mova      [r0-4*FDEC_STRIDEB], m4
     mova      [r0-3*FDEC_STRIDEB], m5
     mova      [r0-2*FDEC_STRIDEB], m6
     mova      [r0-4*FDEC_STRIDEB], m4
     mova      [r0-3*FDEC_STRIDEB], m5
     mova      [r0-2*FDEC_STRIDEB], m6


@@ -951,130 +1000,160 @@ cglobal predict_8x8_hu_%1, 2,2,8*(mmsize/16)
     RET
 %endmacro
 
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_HU sse2  , w, dq, d, wd, 2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_8x8_HU ssse3 , w, dq, d, wd, 2
-INIT_AVX
-PREDICT_8x8_HU avx   , w, dq, d, wd, 2
-%elifndef ARCH_X86_64
-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_HU mmxext, b, q , w, bw, 8
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_HU d, wd
+INIT_XMM ssse3
+PREDICT_8x8_HU d, wd
+INIT_XMM avx
+PREDICT_8x8_HU d, wd
+%elif ARCH_X86_64 == 0
+INIT_MMX mmx2
+PREDICT_8x8_HU w, bw

 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vr( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vr( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_VR 4
-cglobal predict_8x8_vr_%1, 2,3,7*(mmsize/16)
+%macro PREDICT_8x8_VR 1
+cglobal predict_8x8_vr, 2,3

     mova        m2, [r1+16*SIZEOF_PIXEL]
     mova        m2, [r1+16*SIZEOF_PIXEL]

+%ifidn cpuname, ssse3
+    mova        m0, [r1+8*SIZEOF_PIXEL]
+    palignr     m3, m2, m0, 7*SIZEOF_PIXEL
+    palignr     m1, m2, m0, 6*SIZEOF_PIXEL
+%else

     movu        m3, [r1+15*SIZEOF_PIXEL]
     movu        m1, [r1+14*SIZEOF_PIXEL]
     movu        m3, [r1+15*SIZEOF_PIXEL]
     movu        m1, [r1+14*SIZEOF_PIXEL]

-    pavg%2      m4, m3, m2
+%endif
+    pavg%1      m4, m3, m2

     add         r0, FDEC_STRIDEB*4
     add         r0, FDEC_STRIDEB*4

-    PRED8x8_LOWPASS %2, m0, m1, m2, m3, m5
+    PRED8x8_LOWPASS m3, m1, m2, m3, m5

     mova        [r0-4*FDEC_STRIDEB], m4
     mova        [r0-4*FDEC_STRIDEB], m4

-    mova        [r0-3*FDEC_STRIDEB], m0
-    mova        m5, m0
-    mova        m6, m4
+    mova        [r0-3*FDEC_STRIDEB], m3

     mova        m1, [r1+8*SIZEOF_PIXEL]
     mova        m1, [r1+8*SIZEOF_PIXEL]

-    mova        m2, m1
-    psll%3      m2, %4
-    mova        m3, m1
-    psll%3      m3, 2*%4
-    PRED8x8_LOWPASS %2, m0, m1, m3, m2, m4
+    PSLLPIX     m0, m1, 1
+    PSLLPIX     m2, m1, 2
+    PRED8x8_LOWPASS m0, m1, m2, m0, m6

 
 %assign Y -2
 %rep 5
 
 %assign Y -2
 %rep 5

-    %assign i (5 + ((Y+3)&1))
-    PALIGNR     m %+ i, m0, 7*SIZEOF_PIXEL, m2
-    mova        [r0+Y*FDEC_STRIDEB], m %+ i
-    psll%3      m0, %4
+    PALIGNR     m4, m0, 7*SIZEOF_PIXEL, m5
+    mova        [r0+Y*FDEC_STRIDEB], m4
+    PSLLPIX     m0, m0, 1
+    SWAP 3, 4

 %assign Y (Y+1)
 %endrep
 %assign Y (Y+1)
 %endrep

-    PALIGNR     m5, m0, 7*SIZEOF_PIXEL, m0
-    mova        [r0+Y*FDEC_STRIDEB], m5
+    PALIGNR     m4, m0, 7*SIZEOF_PIXEL, m0
+    mova        [r0+Y*FDEC_STRIDEB], m4

     RET
 %endmacro
 
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_VR sse2  , w, dq, 2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_8x8_VR ssse3 , w, dq, 2
-INIT_AVX
-PREDICT_8x8_VR avx   , w, dq, 2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_VR w
+INIT_XMM ssse3
+PREDICT_8x8_VR w
+INIT_XMM avx
+PREDICT_8x8_VR w
+%elif ARCH_X86_64 == 0
+INIT_MMX mmx2
+PREDICT_8x8_VR b
+%endif
+
+%macro LOAD_PLANE_ARGS 0
+%if ARCH_X86_64
+    movd        mm0, r1d
+    movd        mm2, r2d
+    movd        mm4, r3d
+    pshufw      mm0, mm0, 0
+    pshufw      mm2, mm2, 0
+    pshufw      mm4, mm4, 0

 %else
 %else

-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_VR mmxext, b, q , 8
+    pshufw      mm0, r1m, 0
+    pshufw      mm2, r2m, 0
+    pshufw      mm4, r3m, 0

 %endif
 %endif

+%endmacro

 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_p_core( uint8_t *src, int i00, int b, int c )
 ;-----------------------------------------------------------------------------
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_p_core( uint8_t *src, int i00, int b, int c )
 ;-----------------------------------------------------------------------------

-%ifndef ARCH_X86_64
-INIT_MMX
-cglobal predict_8x8c_p_core_mmxext, 1,2
+%if ARCH_X86_64 == 0 && HIGH_BIT_DEPTH == 0
+%macro PREDICT_CHROMA_P_MMX 1
+cglobal predict_8x%1c_p_core, 1,2

     LOAD_PLANE_ARGS
     LOAD_PLANE_ARGS

-    movq        mm1, mm2
-    pmullw      mm2, [pw_3210]
-    psllw       mm1, 2
-    paddsw      mm0, mm2        ; mm0 = {i+0*b, i+1*b, i+2*b, i+3*b}
-    paddsw      mm1, mm0        ; mm1 = {i+4*b, i+5*b, i+6*b, i+7*b}
-
-    mov         r1d, 8
+    movq        m1, m2
+    pmullw      m2, [pw_3210]
+    psllw       m1, 2
+    paddsw      m0, m2        ; m0 = {i+0*b, i+1*b, i+2*b, i+3*b}
+    paddsw      m1, m0        ; m1 = {i+4*b, i+5*b, i+6*b, i+7*b}
+    mov         r1d, %1

 ALIGN 4
 .loop:
 ALIGN 4
 .loop:

-    movq        mm5, mm0
-    movq        mm6, mm1
-    psraw       mm5, 5
-    psraw       mm6, 5
-    packuswb    mm5, mm6
-    movq        [r0], mm5
+    movq        m5, m0
+    movq        m6, m1
+    psraw       m5, 5
+    psraw       m6, 5
+    packuswb    m5, m6
+    movq        [r0], m5

 
 

-    paddsw      mm0, mm4
-    paddsw      mm1, mm4
+    paddsw      m0, m4
+    paddsw      m1, m4

     add         r0, FDEC_STRIDE
     dec         r1d
     add         r0, FDEC_STRIDE
     dec         r1d

-    jg          .loop
+    jg .loop

     REP_RET
     REP_RET

-%endif ; !ARCH_X86_64
+%endmacro ; PREDICT_CHROMA_P_MMX
+
+INIT_MMX mmx2
+PREDICT_CHROMA_P_MMX 8
+PREDICT_CHROMA_P_MMX 16
+%endif ; !ARCH_X86_64 && !HIGH_BIT_DEPTH

 
 

-%macro PREDICT_8x8C_P 1
-cglobal predict_8x8c_p_core_%1, 1,1
+%macro PREDICT_CHROMA_P_XMM 1
+%if HIGH_BIT_DEPTH
+cglobal predict_8x%1c_p_core, 1,2,7

     movd        m0, r1m
     movd        m2, r2m
     movd        m4, r3m
     movd        m0, r1m
     movd        m2, r2m
     movd        m4, r3m

-%ifdef HIGH_BIT_DEPTH

     mova        m3, [pw_pixel_max]
     pxor        m1, m1
     mova        m3, [pw_pixel_max]
     pxor        m1, m1

-%endif

     SPLATW      m0, m0, 0
     SPLATW      m2, m2, 0
     SPLATW      m4, m4, 0
     SPLATW      m0, m0, 0
     SPLATW      m2, m2, 0
     SPLATW      m4, m4, 0

-    pmullw      m2, [pw_76543210]
-%ifdef HIGH_BIT_DEPTH
-    mov        r1d, 8
+    pmullw      m2, [pw_43210123] ; b
+%if %1 == 16
+    pmullw      m5, m4, [pw_m7]   ; c
+%else
+    pmullw      m5, m4, [pw_m3]
+%endif
+    paddw       m5, [pw_16]
+    mov        r1d, %1

 .loop:
 .loop:

-    paddsw      m5, m0, m2
-    psraw       m5, 5
-    CLIPW       m5, m1, m3
-    mova      [r0], m5
-    paddw       m2, m4
+    paddsw      m6, m2, m5
+    paddsw      m6, m0
+    psraw       m6, 5
+    CLIPW       m6, m1, m3
+    mova      [r0], m6
+    paddw       m5, m4

     add         r0, FDEC_STRIDEB
     add         r0, FDEC_STRIDEB

-    dec r1d
+    dec        r1d

     jg .loop
     jg .loop

-%else ;!HIGH_BIT_DEPTH
-    paddsw      m0, m2        ; m0 = {i+0*b, i+1*b, i+2*b, i+3*b, i+4*b, i+5*b, i+6*b, i+7*b}
+    REP_RET
+%else ; !HIGH_BIT_DEPTH
+cglobal predict_8x%1c_p_core, 1,2
+    movd        m0, r1m
+    movd        m2, r2m
+    movd        m4, r3m
+    SPLATW      m0, m0, 0
+    SPLATW      m2, m2, 0
+    SPLATW      m4, m4, 0
+    pmullw      m2, [pw_76543210]
+    paddsw      m0, m2            ; m0 = {i+0*b, i+1*b, i+2*b, i+3*b, i+4*b, i+5*b, i+6*b, i+7*b}

     paddsw      m3, m0, m4
     paddsw      m4, m4
     paddsw      m3, m0, m4
     paddsw      m4, m4

-call .loop
-    add         r0, FDEC_STRIDE*4
+    mov         r1d, %1/4

 .loop:
     paddsw      m1, m3, m4
     paddsw      m5, m0, m4
 .loop:
     paddsw      m1, m3, m4
     paddsw      m5, m0, m4


@@ -1090,18 +1169,26 @@ call .loop
     packuswb    m5, m1
     movq        [r0+FDEC_STRIDE*2], m5
     movhps      [r0+FDEC_STRIDE*3], m5
     packuswb    m5, m1
     movq        [r0+FDEC_STRIDE*2], m5
     movhps      [r0+FDEC_STRIDE*3], m5

-%endif ;!HIGH_BIT_DEPTH
+    add         r0, FDEC_STRIDE*4
+    dec        r1d
+    jg .loop

     RET
     RET

-%endmacro ; PREDICT_8x8C_P
+%endif ; HIGH_BIT_DEPTH
+%endmacro ; PREDICT_CHROMA_P_XMM

 
 

-INIT_XMM
-PREDICT_8x8C_P sse2
+INIT_XMM sse2
+PREDICT_CHROMA_P_XMM 8
+PREDICT_CHROMA_P_XMM 16
+INIT_XMM avx
+PREDICT_CHROMA_P_XMM 8
+PREDICT_CHROMA_P_XMM 16

 
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_p_core( uint8_t *src, int i00, int b, int c )
 ;-----------------------------------------------------------------------------
 
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_p_core( uint8_t *src, int i00, int b, int c )
 ;-----------------------------------------------------------------------------

-%ifndef ARCH_X86_64
-cglobal predict_16x16_p_core_mmxext, 1,2
+%if ARCH_X86_64 == 0
+INIT_MMX mmx2
+cglobal predict_16x16_p_core, 1,2

     LOAD_PLANE_ARGS
     movq        mm5, mm2
     movq        mm1, mm2
     LOAD_PLANE_ARGS
     movq        mm5, mm2
     movq        mm1, mm2


@@ -1141,21 +1228,18 @@ ALIGN 4
     REP_RET
 %endif ; !ARCH_X86_64
 
     REP_RET
 %endif ; !ARCH_X86_64
 

-%macro PREDICT_16x16_P 1
-cglobal predict_16x16_p_core_%1, 1,2,8
+%macro PREDICT_16x16_P 0
+cglobal predict_16x16_p_core, 1,2,8

     movd     m0, r1m
     movd     m1, r2m
     movd     m2, r3m
     movd     m0, r1m
     movd     m1, r2m
     movd     m2, r3m

-%ifdef HIGH_BIT_DEPTH
-    pxor     m6, m6
-    pxor     m7, m7
-%endif

     SPLATW   m0, m0, 0
     SPLATW   m1, m1, 0
     SPLATW   m2, m2, 0
     pmullw   m3, m1, [pw_76543210]
     psllw    m1, 3
     SPLATW   m0, m0, 0
     SPLATW   m1, m1, 0
     SPLATW   m2, m2, 0
     pmullw   m3, m1, [pw_76543210]
     psllw    m1, 3

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH
+    pxor     m6, m6

     mov     r1d, 16
 .loop:
     mova     m4, m0
     mov     r1d, 16
 .loop:
     mova     m4, m0


@@ -1201,246 +1285,280 @@ ALIGN 4
     REP_RET
 %endmacro ; PREDICT_16x16_P
 
     REP_RET
 %endmacro ; PREDICT_16x16_P
 

-INIT_XMM
-PREDICT_16x16_P sse2
-%ifndef HIGH_BIT_DEPTH
-INIT_AVX
-PREDICT_16x16_P avx
+INIT_XMM sse2
+PREDICT_16x16_P
+%if HIGH_BIT_DEPTH == 0
+INIT_XMM avx
+PREDICT_16x16_P

 %endif
 
 %endif
 

-%ifndef HIGH_BIT_DEPTH
-%macro PREDICT_8x8 1
+%if HIGH_BIT_DEPTH == 0
+%macro PREDICT_8x8 0

 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddl( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddl( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_ddl_%1, 2,2
-    movdqa      xmm3, [r1+16]
-    movdqu      xmm2, [r1+17]
-    pslldq      xmm1, xmm3, 1
-    add          r0, FDEC_STRIDE*4
-    PRED8x8_LOWPASS b, xmm0, xmm1, xmm2, xmm3, xmm4
+cglobal predict_8x8_ddl, 2,2
+    mova        m0, [r1+16]
+%ifidn cpuname, ssse3
+    movd        m2, [r1+32]
+    palignr     m2, m0, 1
+%else
+    movu        m2, [r1+17]
+%endif
+    pslldq      m1, m0, 1
+    add        r0, FDEC_STRIDE*4
+    PRED8x8_LOWPASS m0, m1, m2, m0, m3

 
 %assign Y -4
 %rep 8
 
 %assign Y -4
 %rep 8

-    psrldq      xmm0, 1
-    movq        [r0+Y*FDEC_STRIDE], xmm0
+    psrldq      m0, 1
+    movq        [r0+Y*FDEC_STRIDE], m0

 %assign Y (Y+1)
 %endrep
     RET
 
 %assign Y (Y+1)
 %endrep
     RET
 

+%ifnidn cpuname, ssse3

 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddr( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_ddr( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_ddr_%1, 2,2
-    movdqu      xmm3, [r1+8]
-    movdqu      xmm1, [r1+7]
-    psrldq      xmm2, xmm3, 1
-    add           r0, FDEC_STRIDE*4
-    PRED8x8_LOWPASS b, xmm0, xmm1, xmm2, xmm3, xmm4
+cglobal predict_8x8_ddr, 2,2
+    movu        m0, [r1+8]
+    movu        m1, [r1+7]
+    psrldq      m2, m0, 1
+    add         r0, FDEC_STRIDE*4
+    PRED8x8_LOWPASS m0, m1, m2, m0, m3

 
 

-    psrldq      xmm1, xmm0, 1
+    psrldq      m1, m0, 1

 %assign Y 3
 %rep 3
 %assign Y 3
 %rep 3

-    movq        [r0+Y*FDEC_STRIDE], xmm0
-    movq        [r0+(Y-1)*FDEC_STRIDE], xmm1
-    psrldq      xmm0, 2
-    psrldq      xmm1, 2
+    movq        [r0+Y*FDEC_STRIDE], m0
+    movq        [r0+(Y-1)*FDEC_STRIDE], m1
+    psrldq      m0, 2
+    psrldq      m1, 2

 %assign Y (Y-2)
 %endrep
 %assign Y (Y-2)
 %endrep

-    movq        [r0-3*FDEC_STRIDE], xmm0
-    movq        [r0-4*FDEC_STRIDE], xmm1
+    movq        [r0-3*FDEC_STRIDE], m0
+    movq        [r0-4*FDEC_STRIDE], m1

     RET
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vl( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
     RET
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vl( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_vl_%1, 2,2
-    movdqa      xmm4, [r1+16]
-    pslldq      xmm1, xmm4, 1
-    psrldq      xmm2, xmm4, 1
-    pavgb       xmm3, xmm4, xmm2
-    add           r0, FDEC_STRIDE*4
-    PRED8x8_LOWPASS b, xmm0, xmm1, xmm2, xmm4, xmm5
-; xmm0: (t0 + 2*t1 + t2 + 2) >> 2
-; xmm3: (t0 + t1 + 1) >> 1
+cglobal predict_8x8_vl, 2,2
+    mova        m0, [r1+16]
+    pslldq      m1, m0, 1
+    psrldq      m2, m0, 1
+    pavgb       m3, m0, m2
+    add         r0, FDEC_STRIDE*4
+    PRED8x8_LOWPASS m0, m1, m2, m0, m5
+; m0: (t0 + 2*t1 + t2 + 2) >> 2
+; m3: (t0 + t1 + 1) >> 1

 
 %assign Y -4
 %rep 3
 
 %assign Y -4
 %rep 3

-    psrldq      xmm0, 1
-    movq        [r0+ Y   *FDEC_STRIDE], xmm3
-    movq        [r0+(Y+1)*FDEC_STRIDE], xmm0
-    psrldq      xmm3, 1
+    psrldq      m0, 1
+    movq        [r0+ Y   *FDEC_STRIDE], m3
+    movq        [r0+(Y+1)*FDEC_STRIDE], m0
+    psrldq      m3, 1

 %assign Y (Y+2)
 %endrep
 %assign Y (Y+2)
 %endrep

-    psrldq      xmm0, 1
-    movq        [r0+ Y   *FDEC_STRIDE], xmm3
-    movq        [r0+(Y+1)*FDEC_STRIDE], xmm0
-
+    psrldq      m0, 1
+    movq        [r0+ Y   *FDEC_STRIDE], m3
+    movq        [r0+(Y+1)*FDEC_STRIDE], m0

     RET
     RET

+%endif ; !ssse3

 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vr( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_vr( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-cglobal predict_8x8_vr_%1, 2,2,7
-    movdqu      xmm0, [r1+8]
-    movdqa      xmm6, [pw_ff00]
+cglobal predict_8x8_vr, 2,2
+    movu        m2, [r1+8]

     add         r0, 4*FDEC_STRIDE
     add         r0, 4*FDEC_STRIDE

-    movdqa      xmm2, xmm0
-    movdqa      xmm3, xmm0
-    pslldq      xmm1, xmm0, 2
-    pslldq      xmm0, 1
-    pavgb       xmm2, xmm0
-    PRED8x8_LOWPASS b, xmm4, xmm3, xmm1, xmm0, xmm5
-    pandn       xmm6, xmm4
-    movdqa      xmm5, xmm4
-    psrlw       xmm4, 8
-    packuswb    xmm6, xmm4
-    movhlps     xmm4, xmm6
-    movhps [r0-3*FDEC_STRIDE], xmm5
-    movhps [r0-4*FDEC_STRIDE], xmm2
-    psrldq      xmm5, 4
-    movss       xmm5, xmm6
-    psrldq      xmm2, 4
-    movss       xmm2, xmm4
-%assign Y 3
-%rep 3
-    psrldq      xmm5, 1
-    psrldq      xmm2, 1
-    movq        [r0+Y*FDEC_STRIDE], xmm5
-    movq        [r0+(Y-1)*FDEC_STRIDE], xmm2
-%assign Y (Y-2)
-%endrep
+    pslldq      m1, m2, 2
+    pslldq      m0, m2, 1
+    pavgb       m3, m2, m0
+    PRED8x8_LOWPASS m0, m2, m1, m0, m4
+    movhps      [r0-4*FDEC_STRIDE], m3
+    movhps      [r0-3*FDEC_STRIDE], m0
+%if cpuflag(ssse3)
+    movhlps     m3, m3
+    pshufb      m0, [shuf_vr]
+    palignr     m3, m0, 13
+%else
+    mova        m2, m0
+    mova        m1, [pw_00ff]
+    pand        m1, m0
+    psrlw       m0, 8
+    packuswb    m1, m0
+    pslldq      m1, 4
+    movhlps     m3, m1
+    shufps      m1, m2, q3210
+    psrldq      m3, 5
+    psrldq      m1, 5
+    SWAP         0, 1
+%endif
+    movq        [r0+3*FDEC_STRIDE], m0
+    movq        [r0+2*FDEC_STRIDE], m3
+    psrldq      m0, 1
+    psrldq      m3, 1
+    movq        [r0+1*FDEC_STRIDE], m0
+    movq        [r0+0*FDEC_STRIDE], m3
+    psrldq      m0, 1
+    psrldq      m3, 1
+    movq        [r0-1*FDEC_STRIDE], m0
+    movq        [r0-2*FDEC_STRIDE], m3

     RET
 %endmacro ; PREDICT_8x8
 
     RET
 %endmacro ; PREDICT_8x8
 

-INIT_XMM
-PREDICT_8x8 sse2
-INIT_AVX
-PREDICT_8x8 avx
+INIT_XMM sse2
+PREDICT_8x8
+INIT_XMM ssse3
+PREDICT_8x8
+INIT_XMM avx
+PREDICT_8x8

 
 %endif ; !HIGH_BIT_DEPTH
 
 
 %endif ; !HIGH_BIT_DEPTH
 

+;-----------------------------------------------------------------------------
+; void predict_8x8_vl( pixel *src, pixel *edge )
+;-----------------------------------------------------------------------------
+%macro PREDICT_8x8_VL_10 1
+cglobal predict_8x8_vl, 2,2,8
+    mova         m0, [r1+16*SIZEOF_PIXEL]
+    mova         m1, [r1+24*SIZEOF_PIXEL]
+    PALIGNR      m2, m1, m0, SIZEOF_PIXEL*1, m4
+    PSRLPIX      m4, m1, 1
+    pavg%1       m6, m0, m2
+    pavg%1       m7, m1, m4
+    add          r0, FDEC_STRIDEB*4
+    mova         [r0-4*FDEC_STRIDEB], m6
+    PALIGNR      m3, m7, m6, SIZEOF_PIXEL*1, m5
+    mova         [r0-2*FDEC_STRIDEB], m3
+    PALIGNR      m3, m7, m6, SIZEOF_PIXEL*2, m5
+    mova         [r0+0*FDEC_STRIDEB], m3
+    PALIGNR      m7, m7, m6, SIZEOF_PIXEL*3, m5
+    mova         [r0+2*FDEC_STRIDEB], m7
+    PALIGNR      m3, m1, m0, SIZEOF_PIXEL*7, m6
+    PSLLPIX      m5, m0, 1
+    PRED8x8_LOWPASS m0, m5, m2, m0, m7
+    PRED8x8_LOWPASS m1, m3, m4, m1, m7
+    PALIGNR      m4, m1, m0, SIZEOF_PIXEL*1, m2
+    mova         [r0-3*FDEC_STRIDEB], m4
+    PALIGNR      m4, m1, m0, SIZEOF_PIXEL*2, m2
+    mova         [r0-1*FDEC_STRIDEB], m4
+    PALIGNR      m4, m1, m0, SIZEOF_PIXEL*3, m2
+    mova         [r0+1*FDEC_STRIDEB], m4
+    PALIGNR      m1, m1, m0, SIZEOF_PIXEL*4, m2
+    mova         [r0+3*FDEC_STRIDEB], m1
+    RET
+%endmacro
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_VL_10 w
+INIT_XMM ssse3
+PREDICT_8x8_VL_10 w
+INIT_XMM avx
+PREDICT_8x8_VL_10 w
+%else
+INIT_MMX mmx2
+PREDICT_8x8_VL_10 b
+%endif
+

 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hd( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hd( pixel *src, pixel *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_HD 5
-cglobal predict_8x8_hd_%1, 2,2,8*(mmsize/16)
+%macro PREDICT_8x8_HD 2
+cglobal predict_8x8_hd, 2,2

     add       r0, 4*FDEC_STRIDEB
     add       r0, 4*FDEC_STRIDEB

-    mova      m0, [r1]                     ; l7 .. .. .. .. .. .. ..
-    mova      m1, [r1+ 8*SIZEOF_PIXEL]     ; lt l0 l1 l2 l3 l4 l5 l6
+    mova      m0, [r1+ 8*SIZEOF_PIXEL]     ; lt l0 l1 l2 l3 l4 l5 l6
+    movu      m1, [r1+ 7*SIZEOF_PIXEL]     ; l0 l1 l2 l3 l4 l5 l6 l7
+%ifidn cpuname, ssse3

     mova      m2, [r1+16*SIZEOF_PIXEL]     ; t7 t6 t5 t4 t3 t2 t1 t0
     mova      m2, [r1+16*SIZEOF_PIXEL]     ; t7 t6 t5 t4 t3 t2 t1 t0

-    mova      m3, m1                       ; lt l0 l1 l2 l3 l4 l5 l6

     mova      m4, m2                       ; t7 t6 t5 t4 t3 t2 t1 t0
     mova      m4, m2                       ; t7 t6 t5 t4 t3 t2 t1 t0

-    PALIGNR   m2, m1, 7*SIZEOF_PIXEL, m5   ; t6 t5 t4 t3 t2 t1 t0 lt
-    PALIGNR   m1, m0, 7*SIZEOF_PIXEL, m6   ; l0 l1 l2 l3 l4 l5 l6 l7
-    PALIGNR   m4, m3, 1*SIZEOF_PIXEL, m7   ; t0 lt l0 l1 l2 l3 l4 l5
-    mova      m5, m3
-    pavg%2    m3, m1
-    PRED8x8_LOWPASS %2, m0, m4, m1, m5, m7
-    psrl%3    m4, m2, 2*%5                 ; .. .. t6 t5 t4 t3 t2 t1
-    psrl%3    m1, m2, %5                   ; .. t6 t5 t4 t3 t2 t1 t0
-    PRED8x8_LOWPASS %2, m6, m4, m2, m1, m5
+    palignr   m2, m0, 7*SIZEOF_PIXEL       ; t6 t5 t4 t3 t2 t1 t0 lt
+    palignr   m4, m0, 1*SIZEOF_PIXEL       ; t0 lt l0 l1 l2 l3 l4 l5
+%else
+    movu      m2, [r1+15*SIZEOF_PIXEL]
+    movu      m4, [r1+ 9*SIZEOF_PIXEL]
+%endif ; cpuflag
+    pavg%1    m3, m0, m1
+    PRED8x8_LOWPASS m0, m4, m1, m0, m5
+    PSRLPIX   m4, m2, 2                    ; .. .. t6 t5 t4 t3 t2 t1
+    PSRLPIX   m1, m2, 1                    ; .. t6 t5 t4 t3 t2 t1 t0
+    PRED8x8_LOWPASS m1, m4, m2, m1, m5

                                            ; .. p11 p10 p9
                                            ; .. p11 p10 p9

-    punpckh%4 m7, m3, m0                   ; p8 p7 p6 p5
-    punpckl%4 m3, m0                       ; p4 p3 p2 p1
-    mova      m1, m7
-    mova      m0, m7
-    mova      m4, m7
+    punpckh%2 m2, m3, m0                   ; p8 p7 p6 p5
+    punpckl%2 m3, m0                       ; p4 p3 p2 p1

     mova      [r0+3*FDEC_STRIDEB], m3
     mova      [r0+3*FDEC_STRIDEB], m3

-    PALIGNR   m7, m3, 2*SIZEOF_PIXEL, m5
-    mova      [r0+2*FDEC_STRIDEB], m7
-    PALIGNR   m1, m3, 4*SIZEOF_PIXEL, m5
-    mova      [r0+1*FDEC_STRIDEB], m1
-    PALIGNR   m0, m3, 6*SIZEOF_PIXEL, m3
+    PALIGNR   m0, m2, m3, 2*SIZEOF_PIXEL, m5
+    mova      [r0+2*FDEC_STRIDEB], m0
+    PALIGNR   m0, m2, m3, 4*SIZEOF_PIXEL, m5
+    mova      [r0+1*FDEC_STRIDEB], m0
+    PALIGNR   m0, m2, m3, 6*SIZEOF_PIXEL, m3

     mova      [r0+0*FDEC_STRIDEB], m0
     mova      [r0+0*FDEC_STRIDEB], m0

-    mova      m2, m6
-    mova      m3, m6
-    mova      [r0-1*FDEC_STRIDEB], m4
-    PALIGNR   m6, m4, 2*SIZEOF_PIXEL, m5
-    mova      [r0-2*FDEC_STRIDEB], m6
-    PALIGNR   m2, m4, 4*SIZEOF_PIXEL, m5
-    mova      [r0-3*FDEC_STRIDEB], m2
-    PALIGNR   m3, m4, 6*SIZEOF_PIXEL, m4
-    mova      [r0-4*FDEC_STRIDEB], m3
+    mova      [r0-1*FDEC_STRIDEB], m2
+    PALIGNR   m0, m1, m2, 2*SIZEOF_PIXEL, m5
+    mova      [r0-2*FDEC_STRIDEB], m0
+    PALIGNR   m0, m1, m2, 4*SIZEOF_PIXEL, m5
+    mova      [r0-3*FDEC_STRIDEB], m0
+    PALIGNR   m1, m1, m2, 6*SIZEOF_PIXEL, m2
+    mova      [r0-4*FDEC_STRIDEB], m1

     RET
 %endmacro
 
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_HD sse2  , w, dq, wd, 2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_8x8_HD ssse3 , w, dq, wd, 2
-INIT_AVX
-PREDICT_8x8_HD avx   , w, dq, wd, 2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8_HD w, wd
+INIT_XMM ssse3
+PREDICT_8x8_HD w, wd
+INIT_XMM avx
+PREDICT_8x8_HD w, wd

 %else
 %else

-INIT_MMX
-%define PALIGNR PALIGNR_MMX
-PREDICT_8x8_HD mmxext, b, q , bw, 8
+INIT_MMX mmx2
+PREDICT_8x8_HD b, bw

 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hd( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hd( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_HD 1
-cglobal predict_8x8_hd_%1, 2,2
-    add       r0, 4*FDEC_STRIDE
-    movdqa  xmm0, [r1]
-    movdqa  xmm1, [r1+16]
-    movdqa  xmm2, xmm1
-    movdqa  xmm3, xmm1
-    PALIGNR xmm1, xmm0, 7, xmm4
-    PALIGNR xmm2, xmm0, 9, xmm5
-    PALIGNR xmm3, xmm0, 8, xmm0
-    pavgb   xmm4, xmm1, xmm3
-    PRED8x8_LOWPASS b, xmm0, xmm1, xmm2, xmm3, xmm5
-    punpcklbw xmm4, xmm0
-    movhlps xmm0, xmm4
+%macro PREDICT_8x8_HD 0
+cglobal predict_8x8_hd, 2,2
+    add     r0, 4*FDEC_STRIDE
+    movu    m1, [r1+7]
+    movu    m3, [r1+8]
+    movu    m2, [r1+9]
+    pavgb   m4, m1, m3
+    PRED8x8_LOWPASS m0, m1, m2, m3, m5
+    punpcklbw m4, m0
+    movhlps m0, m4

 
 %assign Y 3
 %rep 3
 
 %assign Y 3
 %rep 3

-    movq   [r0+(Y)*FDEC_STRIDE], xmm4
-    movq   [r0+(Y-4)*FDEC_STRIDE], xmm0
-    psrldq xmm4, 2
-    psrldq xmm0, 2
+    movq   [r0+(Y)*FDEC_STRIDE], m4
+    movq   [r0+(Y-4)*FDEC_STRIDE], m0
+    psrldq m4, 2
+    psrldq m0, 2

 %assign Y (Y-1)
 %endrep
 %assign Y (Y-1)
 %endrep

-    movq   [r0+(Y)*FDEC_STRIDE], xmm4
-    movq   [r0+(Y-4)*FDEC_STRIDE], xmm0
+    movq   [r0+(Y)*FDEC_STRIDE], m4
+    movq   [r0+(Y-4)*FDEC_STRIDE], m0

     RET
 %endmacro
 
     RET
 %endmacro
 

-INIT_XMM
-PREDICT_8x8_HD sse2
-%define PALIGNR PALIGNR_SSSE3
-PREDICT_8x8_HD ssse3
-INIT_AVX
-PREDICT_8x8_HD avx
-INIT_MMX
-%define PALIGNR PALIGNR_MMX
+INIT_XMM sse2
+PREDICT_8x8_HD
+INIT_XMM avx
+PREDICT_8x8_HD

 %endif ; HIGH_BIT_DEPTH
 
 %endif ; HIGH_BIT_DEPTH
 

-INIT_MMX
+%if HIGH_BIT_DEPTH == 0

 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hu( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8_hu( uint8_t *src, uint8_t *edge )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_8x8_HU 1
-cglobal predict_8x8_hu_%1, 2,2
+INIT_MMX
+cglobal predict_8x8_hu_sse2, 2,2

     add        r0, 4*FDEC_STRIDE
     add        r0, 4*FDEC_STRIDE

-%ifidn %1, ssse3
-    movq      mm5, [r1+7]
-    movq      mm6, [pb_reverse]
-    movq      mm1, mm5
-    movq      mm2, mm5
-    movq      mm3, mm5
-    pshufb    mm5, mm6
-    psrlq     mm6, 8
-    pshufb    mm2, mm6
-    psrlq     mm6, 8
-    pshufb    mm3, mm6
-    movq      mm4, mm5
-%else

     movq      mm1, [r1+7]           ; l0 l1 l2 l3 l4 l5 l6 l7
     movq      mm1, [r1+7]           ; l0 l1 l2 l3 l4 l5 l6 l7

-    pshufw    mm0, mm1, 00011011b   ; l6 l7 l4 l5 l2 l3 l0 l1
+    pshufw    mm0, mm1, q0123       ; l6 l7 l4 l5 l2 l3 l0 l1

     movq      mm2, mm0
     psllw     mm0, 8
     psrlw     mm2, 8
     movq      mm2, mm0
     psllw     mm0, 8
     psrlw     mm2, 8


@@ -1454,9 +1572,8 @@ cglobal predict_8x8_hu_%1, 2,2
     por       mm2, mm1              ; l7 l7 l6 l5 l4 l3 l2 l1
     punpckhbw mm1, mm1
     por       mm3, mm1              ; l7 l7 l7 l6 l5 l4 l3 l2
     por       mm2, mm1              ; l7 l7 l6 l5 l4 l3 l2 l1
     punpckhbw mm1, mm1
     por       mm3, mm1              ; l7 l7 l7 l6 l5 l4 l3 l2

-%endif

     pavgb     mm4, mm2
     pavgb     mm4, mm2

-    PRED8x8_LOWPASS b, mm1, mm3, mm5, mm2, mm6
+    PRED8x8_LOWPASS mm1, mm3, mm5, mm2, mm6

 
     movq2dq   xmm0, mm4
     movq2dq   xmm1, mm1
 
     movq2dq   xmm0, mm4
     movq2dq   xmm1, mm1


@@ -1468,119 +1585,179 @@ cglobal predict_8x8_hu_%1, 2,2
     psrldq    xmm0, 2
 %assign Y (Y+1)
 %endrep
     psrldq    xmm0, 2
 %assign Y (Y+1)
 %endrep

-    pshufw     mm5, mm4, 11111001b
-    pshufw     mm6, mm4, 11111110b
-    pshufw     mm7, mm4, 11111111b
+    pshufw     mm5, mm4, q3321
+    pshufw     mm6, mm4, q3332
+    pshufw     mm7, mm4, q3333

     movq     [r0+Y*FDEC_STRIDE], xmm0
     movq     [r0+0*FDEC_STRIDE], mm4
     movq     [r0+1*FDEC_STRIDE], mm5
     movq     [r0+2*FDEC_STRIDE], mm6
     movq     [r0+3*FDEC_STRIDE], mm7
     RET
     movq     [r0+Y*FDEC_STRIDE], xmm0
     movq     [r0+0*FDEC_STRIDE], mm4
     movq     [r0+1*FDEC_STRIDE], mm5
     movq     [r0+2*FDEC_STRIDE], mm6
     movq     [r0+3*FDEC_STRIDE], mm7
     RET

-%endmacro

 
 

-%ifndef HIGH_BIT_DEPTH
-PREDICT_8x8_HU sse2
-PREDICT_8x8_HU ssse3
-%endif
+INIT_XMM
+cglobal predict_8x8_hu_ssse3, 2,2
+    add       r0, 4*FDEC_STRIDE
+    movq      m3, [r1+7]
+    pshufb    m3, [shuf_hu]
+    psrldq    m1, m3, 1
+    psrldq    m2, m3, 2
+    pavgb     m0, m1, m3
+    PRED8x8_LOWPASS m1, m3, m2, m1, m4
+    punpcklbw m0, m1
+%assign Y -4
+%rep 3
+    movq   [r0+ Y   *FDEC_STRIDE], m0
+    movhps [r0+(Y+4)*FDEC_STRIDE], m0
+    psrldq    m0, 2
+    pshufhw   m0, m0, q2210
+%assign Y (Y+1)
+%endrep
+    movq   [r0+ Y   *FDEC_STRIDE], m0
+    movhps [r0+(Y+4)*FDEC_STRIDE], m0
+    RET
+%endif ; !HIGH_BIT_DEPTH

 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_v( uint8_t *src )
 ;-----------------------------------------------------------------------------
 
 
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_v( uint8_t *src )
 ;-----------------------------------------------------------------------------
 

-%macro PREDICT_8x8C_V 1
-cglobal predict_8x8c_v_%1, 1,1
+%macro PREDICT_8x8C_V 0
+cglobal predict_8x8c_v, 1,1

     mova        m0, [r0 - FDEC_STRIDEB]
     STORE8x8    m0, m0
     RET
 %endmacro
 
     mova        m0, [r0 - FDEC_STRIDEB]
     STORE8x8    m0, m0
     RET
 %endmacro
 

-%ifdef HIGH_BIT_DEPTH
-INIT_XMM
-PREDICT_8x8C_V sse2
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x8C_V

 %else
 %else

-INIT_MMX
-PREDICT_8x8C_V mmx
+INIT_MMX mmx
+PREDICT_8x8C_V

 %endif
 
 %endif
 

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

 
 INIT_MMX
 cglobal predict_8x8c_v_mmx, 1,1
     mova        m0, [r0 - FDEC_STRIDEB]
     mova        m1, [r0 - FDEC_STRIDEB + 8]
 
 INIT_MMX
 cglobal predict_8x8c_v_mmx, 1,1
     mova        m0, [r0 - FDEC_STRIDEB]
     mova        m1, [r0 - FDEC_STRIDEB + 8]

-%assign n 0
+%assign Y 0

 %rep 8
 %rep 8

-    mova        [r0 + (n&1)*FDEC_STRIDEB], m0
-    mova        [r0 + (n&1)*FDEC_STRIDEB + 8], m1
-%if (n&1) && (n!=7)
+    mova        [r0 + (Y&1)*FDEC_STRIDEB], m0
+    mova        [r0 + (Y&1)*FDEC_STRIDEB + 8], m1
+%if (Y&1) && (Y!=7)

     add         r0, FDEC_STRIDEB*2
 %endif
     add         r0, FDEC_STRIDEB*2
 %endif

-%assign n n+1
+%assign Y Y+1

 %endrep
     RET
 
 %endif
 
 %endrep
     RET
 
 %endif
 

+%macro PREDICT_8x16C_V 0
+cglobal predict_8x16c_v, 1,1
+    mova        m0, [r0 - FDEC_STRIDEB]
+    STORE8x16    m0, m0, m0, m0
+    RET
+%endmacro
+
+%if HIGH_BIT_DEPTH
+INIT_XMM sse2
+PREDICT_8x16C_V
+%else
+INIT_MMX mmx
+PREDICT_8x16C_V
+%endif
+

 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_h( uint8_t *src )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_h( uint8_t *src )
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
-%macro PREDICT_8x8C_H 1
+%if HIGH_BIT_DEPTH

 
 

-cglobal predict_8x8c_h_%1, 1,1
+%macro PREDICT_C_H 1
+cglobal predict_8x%1c_h, 1,1

     add        r0, FDEC_STRIDEB*4
     add        r0, FDEC_STRIDEB*4

-%assign n -4
-%rep 8
-    movd       m0, [r0+FDEC_STRIDEB*n-SIZEOF_PIXEL*2]
+%assign Y -4
+%rep %1
+    movd       m0, [r0+FDEC_STRIDEB*Y-SIZEOF_PIXEL*2]

     SPLATW     m0, m0, 1
     SPLATW     m0, m0, 1

-    mova [r0+FDEC_STRIDEB*n], m0
-%assign n n+1
+    mova [r0+FDEC_STRIDEB*Y], m0
+%if mmsize == 8
+    mova [r0+FDEC_STRIDEB*Y+8], m0
+%endif
+%assign Y Y+1

 %endrep
     RET
 %endrep
     RET

-

 %endmacro
 
 %endmacro
 

-INIT_XMM
-PREDICT_8x8C_H sse2
+INIT_MMX mmx2
+PREDICT_C_H 8
+PREDICT_C_H 16
+INIT_XMM sse2
+PREDICT_C_H 8
+PREDICT_C_H 16

 
 

-%else
+%else ; !HIGH_BIT_DEPTH

 
 

-%macro PREDICT_8x8C_H 1
-cglobal predict_8x8c_h_%1, 1,1
-%ifidn %1, ssse3
+%macro PREDICT_C_H_CORE 1
+%assign Y %1
+%rep 4
+    SPLATB_LOAD m0, r0+FDEC_STRIDE*Y-1, m1
+    mova [r0+FDEC_STRIDE*Y], m0
+%assign Y Y+1
+%endrep
+%endmacro
+
+%macro PREDICT_C_H 1
+cglobal predict_8x%1c_h, 1,1
+%if cpuflag(ssse3)

     mova   m1, [pb_3]
 %endif
     mova   m1, [pb_3]
 %endif

+%if %1==16

     add    r0, FDEC_STRIDE*4
     add    r0, FDEC_STRIDE*4

-%assign n -4
-%rep 8
-    SPLATB m0, r0+FDEC_STRIDE*n-1, m1
-    mova [r0+FDEC_STRIDE*n], m0
-%assign n n+1
-%endrep
+    PREDICT_C_H_CORE -4
+    add    r0, FDEC_STRIDE*4
+    PREDICT_C_H_CORE -4
+%endif
+    add    r0, FDEC_STRIDE*4
+    PREDICT_C_H_CORE -4
+    PREDICT_C_H_CORE 0

     RET
 %endmacro
 
     RET
 %endmacro
 

-INIT_MMX
-%define SPLATB SPLATB_MMX
-PREDICT_8x8C_H mmxext
-%define SPLATB SPLATB_SSSE3
-PREDICT_8x8C_H ssse3
+INIT_MMX mmx2
+PREDICT_C_H 8
+PREDICT_C_H 16
+INIT_MMX ssse3
+PREDICT_C_H 8
+PREDICT_C_H 16

 
 %endif
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_dc( pixel *src )
 ;-----------------------------------------------------------------------------
 
 
 %endif
 ;-----------------------------------------------------------------------------
 ; void predict_8x8c_dc( pixel *src )
 ;-----------------------------------------------------------------------------
 

-%macro PREDICT_8x8C_DC 1
-cglobal predict_8x8c_dc_%1, 1,3
+%macro LOAD_LEFT 1
+    movzx    r1d, pixel [r0+FDEC_STRIDEB*(%1-4)-SIZEOF_PIXEL]
+    movzx    r2d, pixel [r0+FDEC_STRIDEB*(%1-3)-SIZEOF_PIXEL]
+    add      r1d, r2d
+    movzx    r2d, pixel [r0+FDEC_STRIDEB*(%1-2)-SIZEOF_PIXEL]
+    add      r1d, r2d
+    movzx    r2d, pixel [r0+FDEC_STRIDEB*(%1-1)-SIZEOF_PIXEL]
+    add      r1d, r2d
+%endmacro
+
+%macro PREDICT_8x8C_DC 0
+cglobal predict_8x8c_dc, 1,3

     pxor      m7, m7
     pxor      m7, m7

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

     movq      m0, [r0-FDEC_STRIDEB+0]
     movq      m1, [r0-FDEC_STRIDEB+8]
     HADDW     m0, m2
     HADDW     m1, m2
     movq      m0, [r0-FDEC_STRIDEB+0]
     movq      m1, [r0-FDEC_STRIDEB+8]
     HADDW     m0, m2
     HADDW     m1, m2

-%else
+%else ; !HIGH_BIT_DEPTH

     movd      m0, [r0-FDEC_STRIDEB+0]
     movd      m1, [r0-FDEC_STRIDEB+4]
     psadbw    m0, m7            ; s0
     movd      m0, [r0-FDEC_STRIDEB+0]
     movd      m1, [r0-FDEC_STRIDEB+4]
     psadbw    m0, m7            ; s0


@@ -1588,99 +1765,204 @@ cglobal predict_8x8c_dc_%1, 1,3
 %endif
     add       r0, FDEC_STRIDEB*4
 
 %endif
     add       r0, FDEC_STRIDEB*4
 

-    movzx    r1d, pixel [r0-FDEC_STRIDEB*4-SIZEOF_PIXEL]
-    movzx    r2d, pixel [r0-FDEC_STRIDEB*3-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movzx    r2d, pixel [r0-FDEC_STRIDEB*2-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movzx    r2d, pixel [r0-FDEC_STRIDEB*1-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movd      m2, r1d            ; s2
-
-    movzx    r1d, pixel [r0+FDEC_STRIDEB*0-SIZEOF_PIXEL]
-    movzx    r2d, pixel [r0+FDEC_STRIDEB*1-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movzx    r2d, pixel [r0+FDEC_STRIDEB*2-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movzx    r2d, pixel [r0+FDEC_STRIDEB*3-SIZEOF_PIXEL]
-    add      r1d, r2d
-    movd      m3, r1d            ; s3
+    LOAD_LEFT 0                 ; s2
+    movd      m2, r1d
+    LOAD_LEFT 4                 ; s3
+    movd      m3, r1d

 
     punpcklwd m0, m1
     punpcklwd m2, m3
     punpckldq m0, m2            ; s0, s1, s2, s3
 
     punpcklwd m0, m1
     punpcklwd m2, m3
     punpckldq m0, m2            ; s0, s1, s2, s3

-    pshufw    m3, m0, 11110110b ; s2, s1, s3, s3
-    pshufw    m0, m0, 01110100b ; s0, s1, s3, s1
+    pshufw    m3, m0, q3312     ; s2, s1, s3, s3
+    pshufw    m0, m0, q1310     ; s0, s1, s3, s1

     paddw     m0, m3
     psrlw     m0, 2
     pavgw     m0, m7            ; s0+s2, s1, s3, s1+s3
     paddw     m0, m3
     psrlw     m0, 2
     pavgw     m0, m7            ; s0+s2, s1, s3, s1+s3

-%ifdef HIGH_BIT_DEPTH
-%ifidn %1, sse2
+%if HIGH_BIT_DEPTH
+%if cpuflag(sse2)

     movq2dq   xmm0, m0
     punpcklwd xmm0, xmm0
     movq2dq   xmm0, m0
     punpcklwd xmm0, xmm0

-    pshufd    xmm1, xmm0, 11111010b
+    pshufd    xmm1, xmm0, q3322

     punpckldq xmm0, xmm0
     punpckldq xmm0, xmm0

-%assign n 0
+%assign Y 0

 %rep 8
 %rep 8

-%assign i (0 + (n/4))
-    movdqa [r0+FDEC_STRIDEB*(n-4)+0], xmm %+ i
-%assign n n+1
+%assign i (0 + (Y/4))
+    movdqa [r0+FDEC_STRIDEB*(Y-4)+0], xmm %+ i
+%assign Y Y+1

 %endrep
 %endrep

-%else
-    pshufw    m1, m0, 0x00
-    pshufw    m2, m0, 0x55
-    pshufw    m3, m0, 0xaa
-    pshufw    m4, m0, 0xff
-%assign n 0
+%else ; !sse2
+    pshufw    m1, m0, q0000
+    pshufw    m2, m0, q1111
+    pshufw    m3, m0, q2222
+    pshufw    m4, m0, q3333
+%assign Y 0

 %rep 8
 %rep 8

-%assign i (1 + (n/4)*2)
-%assign j (2 + (n/4)*2)
-    movq [r0+FDEC_STRIDEB*(n-4)+0], m %+ i
-    movq [r0+FDEC_STRIDEB*(n-4)+8], m %+ j
-%assign n n+1
+%assign i (1 + (Y/4)*2)
+%assign j (2 + (Y/4)*2)
+    movq [r0+FDEC_STRIDEB*(Y-4)+0], m %+ i
+    movq [r0+FDEC_STRIDEB*(Y-4)+8], m %+ j
+%assign Y Y+1

 %endrep
 %endif
 %endrep
 %endif

-%else
+%else ; !HIGH_BIT_DEPTH

     packuswb  m0, m0
     punpcklbw m0, m0
     movq      m1, m0
     punpcklbw m0, m0
     punpckhbw m1, m1
     packuswb  m0, m0
     punpcklbw m0, m0
     movq      m1, m0
     punpcklbw m0, m0
     punpckhbw m1, m1

-%assign n 0
+%assign Y 0

 %rep 8
 %rep 8

-%assign i (0 + (n/4))
-    movq [r0+FDEC_STRIDEB*(n-4)], m %+ i
-%assign n n+1
+%assign i (0 + (Y/4))
+    movq [r0+FDEC_STRIDEB*(Y-4)], m %+ i
+%assign Y Y+1

 %endrep
 %endif
     RET
 %endmacro
 
 %endrep
 %endif
     RET
 %endmacro
 

-INIT_MMX
-PREDICT_8x8C_DC mmxext
-%ifdef HIGH_BIT_DEPTH
-PREDICT_8x8C_DC sse2
+INIT_MMX mmx2
+PREDICT_8x8C_DC
+%if HIGH_BIT_DEPTH
+INIT_MMX sse2
+PREDICT_8x8C_DC

 %endif
 
 %endif
 

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH
+%macro STORE_4LINES 3
+%if cpuflag(sse2)
+    movdqa [r0+FDEC_STRIDEB*(%3-4)], %1
+    movdqa [r0+FDEC_STRIDEB*(%3-3)], %1
+    movdqa [r0+FDEC_STRIDEB*(%3-2)], %1
+    movdqa [r0+FDEC_STRIDEB*(%3-1)], %1
+%else
+    movq [r0+FDEC_STRIDEB*(%3-4)+0], %1
+    movq [r0+FDEC_STRIDEB*(%3-4)+8], %2
+    movq [r0+FDEC_STRIDEB*(%3-3)+0], %1
+    movq [r0+FDEC_STRIDEB*(%3-3)+8], %2
+    movq [r0+FDEC_STRIDEB*(%3-2)+0], %1
+    movq [r0+FDEC_STRIDEB*(%3-2)+8], %2
+    movq [r0+FDEC_STRIDEB*(%3-1)+0], %1
+    movq [r0+FDEC_STRIDEB*(%3-1)+8], %2
+%endif
+%endmacro
+%else
+%macro STORE_4LINES 2
+    movq [r0+FDEC_STRIDEB*(%2-4)], %1
+    movq [r0+FDEC_STRIDEB*(%2-3)], %1
+    movq [r0+FDEC_STRIDEB*(%2-2)], %1
+    movq [r0+FDEC_STRIDEB*(%2-1)], %1
+%endmacro
+%endif

 
 

+%macro PREDICT_8x16C_DC 0
+cglobal predict_8x16c_dc, 1,3
+    pxor      m7, m7
+%if HIGH_BIT_DEPTH
+    movq      m0, [r0-FDEC_STRIDEB+0]
+    movq      m1, [r0-FDEC_STRIDEB+8]
+    HADDW     m0, m2
+    HADDW     m1, m2
+%else
+    movd      m0, [r0-FDEC_STRIDEB+0]
+    movd      m1, [r0-FDEC_STRIDEB+4]
+    psadbw    m0, m7            ; s0
+    psadbw    m1, m7            ; s1
+%endif
+    punpcklwd m0, m1            ; s0, s1
+
+    add       r0, FDEC_STRIDEB*4
+    LOAD_LEFT 0                 ; s2
+    pinsrw    m0, r1d, 2
+    LOAD_LEFT 4                 ; s3
+    pinsrw    m0, r1d, 3        ; s0, s1, s2, s3
+    add       r0, FDEC_STRIDEB*8
+    LOAD_LEFT 0                 ; s4
+    pinsrw    m1, r1d, 2
+    LOAD_LEFT 4                 ; s5
+    pinsrw    m1, r1d, 3        ; s1, __, s4, s5
+    sub       r0, FDEC_STRIDEB*8
+
+    pshufw    m2, m0, q1310     ; s0, s1, s3, s1
+    pshufw    m0, m0, q3312     ; s2, s1, s3, s3
+    pshufw    m3, m1, q0302     ; s4, s1, s5, s1
+    pshufw    m1, m1, q3322     ; s4, s4, s5, s5
+    paddw     m0, m2
+    paddw     m1, m3
+    psrlw     m0, 2
+    psrlw     m1, 2
+    pavgw     m0, m7
+    pavgw     m1, m7
+%if HIGH_BIT_DEPTH
+%if cpuflag(sse2)
+    movq2dq xmm0, m0
+    movq2dq xmm1, m1
+    punpcklwd xmm0, xmm0
+    punpcklwd xmm1, xmm1
+    pshufd    xmm2, xmm0, q3322
+    pshufd    xmm3, xmm1, q3322
+    punpckldq xmm0, xmm0
+    punpckldq xmm1, xmm1
+    STORE_4LINES xmm0, xmm0, 0
+    STORE_4LINES xmm2, xmm2, 4
+    STORE_4LINES xmm1, xmm1, 8
+    STORE_4LINES xmm3, xmm3, 12
+%else
+    pshufw    m2, m0, q0000
+    pshufw    m3, m0, q1111
+    pshufw    m4, m0, q2222
+    pshufw    m5, m0, q3333
+    STORE_4LINES m2, m3, 0
+    STORE_4LINES m4, m5, 4
+    pshufw    m2, m1, q0000
+    pshufw    m3, m1, q1111
+    pshufw    m4, m1, q2222
+    pshufw    m5, m1, q3333
+    STORE_4LINES m2, m3, 8
+    STORE_4LINES m4, m5, 12
+%endif
+%else
+    packuswb  m0, m0            ; dc0, dc1, dc2, dc3
+    packuswb  m1, m1            ; dc4, dc5, dc6, dc7
+    punpcklbw m0, m0
+    punpcklbw m1, m1
+    pshufw    m2, m0, q1100
+    pshufw    m3, m0, q3322
+    pshufw    m4, m1, q1100
+    pshufw    m5, m1, q3322
+    STORE_4LINES m2, 0
+    STORE_4LINES m3, 4
+    add       r0, FDEC_STRIDEB*8
+    STORE_4LINES m4, 0
+    STORE_4LINES m5, 4
+%endif
+    RET
+%endmacro
+
+INIT_MMX mmx2
+PREDICT_8x16C_DC
+%if HIGH_BIT_DEPTH
+INIT_MMX sse2
+PREDICT_8x16C_DC
+%endif
+
+%macro PREDICT_C_DC_TOP 1
+%if HIGH_BIT_DEPTH

 INIT_XMM
 INIT_XMM

-cglobal predict_8x8c_dc_top_sse2, 1,1
+cglobal predict_8x%1c_dc_top_sse2, 1,1

     pxor        m2, m2
     mova        m0, [r0 - FDEC_STRIDEB]
     pxor        m2, m2
     mova        m0, [r0 - FDEC_STRIDEB]

-    pmaddwd     m0, [pw_1]
-    pshufd      m1, m0, 0x31
-    paddd       m0, m1
+    pshufd      m1, m0, q2301
+    paddw       m0, m1
+    pshuflw     m1, m0, q2301
+    pshufhw     m1, m1, q2301
+    paddw       m0, m1

     psrlw       m0, 1
     pavgw       m0, m2
     psrlw       m0, 1
     pavgw       m0, m2

-    pshuflw     m0, m0, 0
-    pshufhw     m0, m0, 0
-    STORE8x8    m0, m0
+    STORE8x%1   m0, m0, m0, m0

     RET
     RET

-
-%else
-
-cglobal predict_8x8c_dc_top_mmxext, 1,1
+%else ; !HIGH_BIT_DEPTH
+INIT_MMX
+cglobal predict_8x%1c_dc_top_mmx2, 1,1

     movq        mm0, [r0 - FDEC_STRIDE]
     pxor        mm1, mm1
     pxor        mm2, mm2
     movq        mm0, [r0 - FDEC_STRIDE]
     pxor        mm1, mm1
     pxor        mm2, mm2


@@ -1695,17 +1977,20 @@ cglobal predict_8x8c_dc_top_mmxext, 1,1
     pshufw      mm1, mm1, 0
     pshufw      mm0, mm0, 0     ; dc0 (w)
     packuswb    mm0, mm1        ; dc0,dc1 (b)
     pshufw      mm1, mm1, 0
     pshufw      mm0, mm0, 0     ; dc0 (w)
     packuswb    mm0, mm1        ; dc0,dc1 (b)

-    STORE8x8    mm0, mm0
+    STORE8x%1   mm0, mm0, mm0, mm0

     RET
     RET

-

 %endif
 %endif

+%endmacro
+
+PREDICT_C_DC_TOP 8
+PREDICT_C_DC_TOP 16

 
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_v( pixel *src )
 ;-----------------------------------------------------------------------------
 
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_v( pixel *src )
 ;-----------------------------------------------------------------------------

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

 INIT_MMX
 INIT_MMX

-cglobal predict_16x16_v_mmx, 1,2
+cglobal predict_16x16_v_mmx2, 1,2

     mova        m0, [r0 - FDEC_STRIDEB+ 0]
     mova        m1, [r0 - FDEC_STRIDEB+ 8]
     mova        m2, [r0 - FDEC_STRIDEB+16]
     mova        m0, [r0 - FDEC_STRIDEB+ 0]
     mova        m1, [r0 - FDEC_STRIDEB+ 8]
     mova        m2, [r0 - FDEC_STRIDEB+16]


@@ -1718,9 +2003,9 @@ cglobal predict_16x16_v_sse2, 2,2
     mova      m1, [r0 - FDEC_STRIDEB+16]
     STORE16x16_SSE2 m0, m1
     REP_RET
     mova      m1, [r0 - FDEC_STRIDEB+16]
     STORE16x16_SSE2 m0, m1
     REP_RET

-%else
+%else ; !HIGH_BIT_DEPTH

 INIT_MMX
 INIT_MMX

-cglobal predict_16x16_v_mmx, 1,2
+cglobal predict_16x16_v_mmx2, 1,2

     movq        m0, [r0 - FDEC_STRIDE + 0]
     movq        m1, [r0 - FDEC_STRIDE + 8]
     STORE16x16  m0, m1
     movq        m0, [r0 - FDEC_STRIDE + 0]
     movq        m1, [r0 - FDEC_STRIDE + 8]
     STORE16x16  m0, m1


@@ -1735,37 +2020,37 @@ cglobal predict_16x16_v_sse2, 1,1
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_h( pixel *src )
 ;-----------------------------------------------------------------------------
 ;-----------------------------------------------------------------------------
 ; void predict_16x16_h( pixel *src )
 ;-----------------------------------------------------------------------------

-%macro PREDICT_16x16_H 1
-cglobal predict_16x16_h_%1, 1,2
+%macro PREDICT_16x16_H 0
+cglobal predict_16x16_h, 1,2

     mov r1, 12*FDEC_STRIDEB
     mov r1, 12*FDEC_STRIDEB

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

 .vloop:
 .vloop:

-%assign n 0
+%assign Y 0

 %rep 4
 %rep 4

-    movd        m0, [r0+r1+n*FDEC_STRIDEB-2*SIZEOF_PIXEL]
+    movd        m0, [r0+r1+Y*FDEC_STRIDEB-2*SIZEOF_PIXEL]

     SPLATW      m0, m0, 1
     SPLATW      m0, m0, 1

-    mova [r0+r1+n*FDEC_STRIDEB+ 0], m0
-    mova [r0+r1+n*FDEC_STRIDEB+16], m0
+    mova [r0+r1+Y*FDEC_STRIDEB+ 0], m0
+    mova [r0+r1+Y*FDEC_STRIDEB+16], m0

 %if mmsize==8
 %if mmsize==8

-    mova [r0+r1+n*FDEC_STRIDEB+ 8], m0
-    mova [r0+r1+n*FDEC_STRIDEB+24], m0
+    mova [r0+r1+Y*FDEC_STRIDEB+ 8], m0
+    mova [r0+r1+Y*FDEC_STRIDEB+24], m0

 %endif
 %endif

-%assign n n+1
+%assign Y Y+1

 %endrep
 
 %endrep
 

-%else
-%ifidn %1, ssse3
+%else ; !HIGH_BIT_DEPTH
+%if cpuflag(ssse3)

     mova   m1, [pb_3]
 %endif
 .vloop:
     mova   m1, [pb_3]
 %endif
 .vloop:

-%assign n 0
+%assign Y 0

 %rep 4
 %rep 4

-    SPLATB m0, r0+r1+FDEC_STRIDE*n-1, m1
-    mova [r0+r1+FDEC_STRIDE*n], m0
+    SPLATB_LOAD m0, r0+r1+FDEC_STRIDE*Y-1, m1
+    mova [r0+r1+FDEC_STRIDE*Y], m0

 %if mmsize==8
 %if mmsize==8

-    mova [r0+r1+FDEC_STRIDE*n+8], m0
+    mova [r0+r1+FDEC_STRIDE*Y+8], m0

 %endif
 %endif

-%assign n n+1
+%assign Y Y+1

 %endrep
 %endif ; HIGH_BIT_DEPTH
     sub r1, 4*FDEC_STRIDEB
 %endrep
 %endif ; HIGH_BIT_DEPTH
     sub r1, 4*FDEC_STRIDEB


@@ -1773,16 +2058,15 @@ cglobal predict_16x16_h_%1, 1,2
     REP_RET
 %endmacro
 
     REP_RET
 %endmacro
 

-INIT_MMX
-%define SPLATB SPLATB_MMX
-PREDICT_16x16_H mmxext
-INIT_XMM
-%ifdef HIGH_BIT_DEPTH
-PREDICT_16x16_H sse2
+INIT_MMX mmx2
+PREDICT_16x16_H
+INIT_XMM sse2
+%if HIGH_BIT_DEPTH
+PREDICT_16x16_H

 %else
 ;no SSE2 for 8-bit, it's slower than MMX on all systems that don't support SSSE3
 %else
 ;no SSE2 for 8-bit, it's slower than MMX on all systems that don't support SSSE3

-%define SPLATB SPLATB_SSSE3
-PREDICT_16x16_H ssse3
+INIT_XMM ssse3
+PREDICT_16x16_H

 %endif
 
 ;-----------------------------------------------------------------------------
 %endif
 
 ;-----------------------------------------------------------------------------


@@ -1790,7 +2074,7 @@ PREDICT_16x16_H ssse3
 ;-----------------------------------------------------------------------------
 
 %macro PRED16x16_DC 2
 ;-----------------------------------------------------------------------------
 
 %macro PRED16x16_DC 2

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

     mova       m0, [r0 - FDEC_STRIDEB+ 0]
     paddw      m0, [r0 - FDEC_STRIDEB+ 8]
     paddw      m0, [r0 - FDEC_STRIDEB+16]
     mova       m0, [r0 - FDEC_STRIDEB+ 0]
     paddw      m0, [r0 - FDEC_STRIDEB+ 8]
     paddw      m0, [r0 - FDEC_STRIDEB+16]


@@ -1800,7 +2084,7 @@ PREDICT_16x16_H ssse3
     psrlw      m0, %2
     SPLATW     m0, m0
     STORE16x16 m0, m0, m0, m0
     psrlw      m0, %2
     SPLATW     m0, m0
     STORE16x16 m0, m0, m0, m0

-%else
+%else ; !HIGH_BIT_DEPTH

     pxor        m0, m0
     pxor        m1, m1
     psadbw      m0, [r0 - FDEC_STRIDE]
     pxor        m0, m0
     pxor        m1, m1
     psadbw      m0, [r0 - FDEC_STRIDE]


@@ -1814,9 +2098,9 @@ PREDICT_16x16_H ssse3
 %endif
 %endmacro
 
 %endif
 %endmacro
 

-INIT_MMX
-cglobal predict_16x16_dc_core_mmxext, 1,2
-%ifdef ARCH_X86_64
+INIT_MMX mmx2
+cglobal predict_16x16_dc_core, 1,2
+%if ARCH_X86_64

     movd         m6, r1d
     PRED16x16_DC m6, 5
 %else
     movd         m6, r1d
     PRED16x16_DC m6, 5
 %else


@@ -1824,20 +2108,20 @@ cglobal predict_16x16_dc_core_mmxext, 1,2
 %endif
     REP_RET
 
 %endif
     REP_RET
 

-INIT_MMX
-cglobal predict_16x16_dc_top_mmxext, 1,2
+INIT_MMX mmx2
+cglobal predict_16x16_dc_top, 1,2

     PRED16x16_DC [pw_8], 4
     REP_RET
 
     PRED16x16_DC [pw_8], 4
     REP_RET
 

-INIT_MMX
-%ifdef HIGH_BIT_DEPTH
-cglobal predict_16x16_dc_left_core_mmxext, 1,2
+INIT_MMX mmx2
+%if HIGH_BIT_DEPTH
+cglobal predict_16x16_dc_left_core, 1,2

     movd       m0, r1m
     SPLATW     m0, m0
     STORE16x16 m0, m0, m0, m0
     REP_RET
     movd       m0, r1m
     SPLATW     m0, m0
     STORE16x16 m0, m0, m0, m0
     REP_RET

-%else
-cglobal predict_16x16_dc_left_core_mmxext, 1,1
+%else ; !HIGH_BIT_DEPTH
+cglobal predict_16x16_dc_left_core, 1,1

     movd       m0, r1m
     pshufw     m0, m0, 0
     packuswb   m0, m0
     movd       m0, r1m
     pshufw     m0, m0, 0
     packuswb   m0, m0


@@ -1850,7 +2134,7 @@ cglobal predict_16x16_dc_left_core_mmxext, 1,1
 ;-----------------------------------------------------------------------------
 
 %macro PRED16x16_DC_SSE2 2
 ;-----------------------------------------------------------------------------
 
 %macro PRED16x16_DC_SSE2 2

-%ifdef HIGH_BIT_DEPTH
+%if HIGH_BIT_DEPTH

     mova       m0, [r0 - FDEC_STRIDEB+ 0]
     paddw      m0, [r0 - FDEC_STRIDEB+16]
     HADDW      m0, m2
     mova       m0, [r0 - FDEC_STRIDEB+ 0]
     paddw      m0, [r0 - FDEC_STRIDEB+16]
     HADDW      m0, m2


@@ -1858,7 +2142,7 @@ cglobal predict_16x16_dc_left_core_mmxext, 1,1
     psrlw      m0, %2
     SPLATW     m0, m0
     STORE16x16_SSE2 m0, m0
     psrlw      m0, %2
     SPLATW     m0, m0
     STORE16x16_SSE2 m0, m0

-%else
+%else ; !HIGH_BIT_DEPTH

     pxor        m0, m0
     psadbw      m0, [r0 - FDEC_STRIDE]
     movhlps     m1, m0
     pxor        m0, m0
     psadbw      m0, [r0 - FDEC_STRIDE]
     movhlps     m1, m0


@@ -1871,25 +2155,25 @@ cglobal predict_16x16_dc_left_core_mmxext, 1,1
 %endif
 %endmacro
 
 %endif
 %endmacro
 

-INIT_XMM
-cglobal predict_16x16_dc_core_sse2, 2,2,4
+INIT_XMM sse2
+cglobal predict_16x16_dc_core, 2,2,4

     movd       m3, r1m
     PRED16x16_DC_SSE2 m3, 5
     REP_RET
 
     movd       m3, r1m
     PRED16x16_DC_SSE2 m3, 5
     REP_RET
 

-cglobal predict_16x16_dc_top_sse2, 1,2
+cglobal predict_16x16_dc_top, 1,2

     PRED16x16_DC_SSE2 [pw_8], 4
     REP_RET
 
     PRED16x16_DC_SSE2 [pw_8], 4
     REP_RET
 

-INIT_XMM
-%ifdef HIGH_BIT_DEPTH
-cglobal predict_16x16_dc_left_core_sse2, 1,2
+INIT_XMM sse2
+%if HIGH_BIT_DEPTH
+cglobal predict_16x16_dc_left_core, 1,2

     movd       m0, r1m
     SPLATW     m0, m0
     STORE16x16_SSE2 m0, m0
     REP_RET
     movd       m0, r1m
     SPLATW     m0, m0
     STORE16x16_SSE2 m0, m0
     REP_RET

-%else
-cglobal predict_16x16_dc_left_core_sse2, 1,1
+%else ; !HIGH_BIT_DEPTH
+cglobal predict_16x16_dc_left_core, 1,1

     movd       m0, r1m
     SPLATW     m0, m0
     packuswb   m0, m0
     movd       m0, r1m
     SPLATW     m0, m0
     packuswb   m0, m0