;*****************************************************************************
-;* quant-a.asm: h264 encoder library
+;* quant-a.asm: x86 quantization and level-run
;*****************************************************************************
-;* Copyright (C) 2005-2008 x264 project
+;* Copyright (C) 2005-2011 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
+;* Fiona Glaser <fiona@x264.com>
;* Christian Heine <sennindemokrit@gmx.net>
+;* Oskar Arvidsson <oskar@irock.se>
+;* Henrik Gramner <hengar-6@student.ltu.se>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
+;*
+;* This program is also available under a commercial proprietary license.
+;* For more information, contact us at licensing@x264.com.
;*****************************************************************************
%include "x86inc.asm"
+%include "x86util.asm"
SECTION_RODATA
-pw_1: times 8 dw 1
-pd_1: times 4 dd 1
%macro DQM4 3
dw %1, %2, %1, %2, %2, %3, %2, %3
DQM8 32, 28, 51, 30, 40, 38
DQM8 36, 32, 58, 34, 46, 43
+decimate_mask_table4:
+ db 0,3,2,6,2,5,5,9,1,5,4,8,5,8,8,12,1,4,4,8,4,7,7,11,4,8,7,11,8,11,11,15,1,4
+ db 3,7,4,7,7,11,3,7,6,10,7,10,10,14,4,7,7,11,7,10,10,14,7,11,10,14,11,14,14
+ db 18,0,4,3,7,3,6,6,10,3,7,6,10,7,10,10,14,3,6,6,10,6,9,9,13,6,10,9,13,10,13
+ db 13,17,4,7,6,10,7,10,10,14,6,10,9,13,10,13,13,17,7,10,10,14,10,13,13,17,10
+ db 14,13,17,14,17,17,21,0,3,3,7,3,6,6,10,2,6,5,9,6,9,9,13,3,6,6,10,6,9,9,13
+ db 6,10,9,13,10,13,13,17,3,6,5,9,6,9,9,13,5,9,8,12,9,12,12,16,6,9,9,13,9,12
+ db 12,16,9,13,12,16,13,16,16,20,3,7,6,10,6,9,9,13,6,10,9,13,10,13,13,17,6,9
+ db 9,13,9,12,12,16,9,13,12,16,13,16,16,20,7,10,9,13,10,13,13,17,9,13,12,16
+ db 13,16,16,20,10,13,13,17,13,16,16,20,13,17,16,20,17,20,20,24
+
+chroma_dc_dct_mask_mmx: dw 0, 0,-1,-1, 0, 0,-1,-1
+chroma_dc_dmf_mask_mmx: dw 0, 0,-1,-1, 0,-1,-1, 0
+chroma_dc_dct_mask: dw 1, 1,-1,-1, 1, 1,-1,-1
+chroma_dc_dmf_mask: dw 1, 1,-1,-1, 1,-1,-1, 1
+
SECTION .text
-%macro QUANT_DC_START 0
+cextern pb_1
+cextern pw_1
+cextern pd_1
+cextern pb_01
+cextern pd_1024
+
+%macro QUANT_DC_START_MMX 0
movd m6, r1m ; mf
movd m7, r2m ; bias
-%ifidn m0, mm0
- pshufw m6, m6, 0
- pshufw m7, m7, 0
+%ifdef HIGH_BIT_DEPTH
+ SPLATD m6, m6
+ SPLATD m7, m7
%else
- pshuflw m6, m6, 0
- pshuflw m7, m7, 0
- punpcklqdq m6, m6
- punpcklqdq m7, m7
-%endif
+ SPLATW m6, m6
+ SPLATW m7, m7
+%endif ; HIGH_BIT_DEPTH
+%endmacro
+
+%macro QUANT_DC_START_SSSE3 0
+ movdqa m5, [pb_01]
+ movd m6, r1m ; mf
+ movd m7, r2m ; bias
+ pshufb m6, m5
+ pshufb m7, m5
%endmacro
%macro PABSW_MMX 2
psignw %1, %2
%endmacro
-%macro QUANT_ONE 3
+%macro PSIGND_MMX 2-3
+%if %0==3
+ mova %1, %2
+ pxor %1, %3
+ psubd %1, %3
+%else
+ pxor %1, %2
+ psubd %1, %2
+%endif
+%endmacro
+
+%macro PSIGND_SSSE3 2+
+ psignd %1, %2
+%endmacro
+
+%macro PABSD_MMX 2
+ pxor %1, %1
+ pcmpgtd %1, %2
+ pxor %2, %1
+ psubd %2, %1
+ SWAP %1, %2
+%endmacro
+
+%macro PABSD_SSSE3 2
+ pabsd %1, %2
+%endmacro
+
+%macro QUANT_END_MMX 0
+ xor eax, eax
+%ifdef ARCH_X86_64
+%if mmsize == 16
+ packsswb m5, m5
+%endif
+ movq rcx, m5
+ test rcx, rcx
+%else
+%if mmsize == 16
+ pxor m4, m4
+ pcmpeqb m5, m4
+ pmovmskb ecx, m5
+ cmp ecx, (1<<mmsize)-1
+%else
+ packsswb m5, m5
+ movd ecx, m5
+ test ecx, ecx
+%endif
+%endif
+ setne al
+%endmacro
+
+%macro QUANT_END_SSE4 0
+ xor eax, eax
+ ptest m5, m5
+ setne al
+%endmacro
+
+%ifdef HIGH_BIT_DEPTH
+%macro QUANT_ONE_DC_MMX 4
+ mova m0, [%1]
+ PABSD m1, m0
+ paddd m1, %3
+ mova m2, m1
+ psrlq m2, 32
+ pmuludq m1, %2
+ pmuludq m2, %2
+ psllq m2, 32
+ paddd m1, m2
+ psrld m1, 16
+ PSIGND m1, m0
+ mova [%1], m1
+%if %4
+ por m5, m1
+%else
+ SWAP 5, 1
+%endif
+%endmacro
+
+%macro QUANT_TWO_DC_MMX 4
+ QUANT_ONE_DC_MMX %1, %2, %3, %4
+ QUANT_ONE_DC_MMX %1+mmsize, %2, %3, %4+mmsize
+%endmacro
+
+%macro QUANT_ONE_DC_SSE4 4
+ mova m0, [%1]
+ PABSD m1, m0
+ paddd m1, %3
+ pmulld m1, %2
+ psrad m1, 16
+ PSIGND m1, m0
+ mova [%1], m1
+%if %4
+ por m5, m1
+%else
+ SWAP 5, 1
+%endif
+%endmacro
+
+%macro QUANT_TWO_DC_SSE4 4
+ mova m0, [%1]
+ mova m1, [%1+mmsize]
+ PABSD m2, m0
+ PABSD m3, m1
+ paddd m2, %3
+ paddd m3, %3
+ pmulld m2, %2
+ pmulld m3, %2
+ psrad m2, 16
+ psrad m3, 16
+ PSIGND m2, m0
+ PSIGND m3, m1
+ mova [%1], m2
+ mova [%1+mmsize], m3
+%if %4
+ por m5, m2
+%else
+ SWAP 5, 2
+%endif
+ por m5, m3
+%endmacro
+
+%macro QUANT_ONE_AC_MMX 4
+ mova m0, [%1]
+ mova m2, [%2]
+ PABSD m1, m0
+ mova m4, m2
+ paddd m1, [%3]
+ mova m3, m1
+ psrlq m4, 32
+ psrlq m3, 32
+ pmuludq m1, m2
+ pmuludq m3, m4
+ psllq m3, 32
+ paddd m1, m3
+ psrad m1, 16
+ PSIGND m1, m0
+ mova [%1], m1
+%if %4
+ por m5, m1
+%else
+ SWAP 5, 1
+%endif
+%endmacro
+
+%macro QUANT_TWO_AC_MMX 4
+ QUANT_ONE_AC_MMX %1, %2, %3, %4
+ QUANT_ONE_AC_MMX %1+mmsize, %2+mmsize, %3+mmsize, %4+mmsize
+%endmacro
+
+%macro QUANT_TWO_AC_SSE4 4
+ mova m0, [%1]
+ mova m1, [%1+mmsize]
+ PABSD m2, m0
+ PABSD m3, m1
+ paddd m2, [%3]
+ paddd m3, [%3+mmsize]
+ pmulld m2, [%2]
+ pmulld m3, [%2+mmsize]
+ psrad m2, 16
+ psrad m3, 16
+ PSIGND m2, m0
+ PSIGND m3, m1
+ mova [%1], m2
+ mova [%1+mmsize], m3
+%if %4
+ por m5, m2
+%else
+ SWAP 5, 2
+%endif
+ por m5, m3
+%endmacro
+
+;-----------------------------------------------------------------------------
+; int quant_2x2( int32_t dct[M*N], int mf, int bias )
+;-----------------------------------------------------------------------------
+%macro QUANT_DC 3
+cglobal quant_%1x%2_dc_%3, 3,3,8*(mmsize/16)
+ QUANT_DC_START_MMX
+%if %1*%2 <= mmsize/4
+ QUANT_ONE_DC r0, m6, m7, 0
+%else
+%assign x 0
+%rep %1*%2/(mmsize/2)
+ QUANT_TWO_DC r0+x, m6, m7, x
+%assign x x+mmsize*2
+%endrep
+%endif
+ QUANT_END
+ RET
+%endmacro
+
+;-----------------------------------------------------------------------------
+; int quant_MxN( int32_t dct[M*N], uint32_t mf[M*N], uint32_t bias[M*N] )
+;-----------------------------------------------------------------------------
+%macro QUANT_AC 3
+cglobal quant_%1x%2_%3, 3,3,8*(mmsize/16)
+%assign x 0
+%rep %1*%2/(mmsize/2)
+ QUANT_TWO_AC r0+x, r1+x, r2+x, x
+%assign x x+mmsize*2
+%endrep
+ QUANT_END
+ RET
+%endmacro
+
+%define QUANT_TWO_AC QUANT_TWO_AC_MMX
+%define QUANT_ONE_DC QUANT_ONE_DC_MMX
+%define QUANT_TWO_DC QUANT_TWO_DC_MMX
+%define QUANT_END QUANT_END_MMX
+%define PABSD PABSD_MMX
+%define PSIGND PSIGND_MMX
+INIT_XMM
+QUANT_DC 2, 2, sse2
+QUANT_DC 4, 4, sse2
+QUANT_AC 4, 4, sse2
+QUANT_AC 8, 8, sse2
+
+%define PABSD PABSD_SSSE3
+%define PSIGND PSIGND_SSSE3
+QUANT_DC 2, 2, ssse3
+QUANT_DC 4, 4, ssse3
+QUANT_AC 4, 4, ssse3
+QUANT_AC 8, 8, ssse3
+
+%define QUANT_TWO_AC QUANT_TWO_AC_SSE4
+%define QUANT_ONE_DC QUANT_ONE_DC_SSE4
+%define QUANT_TWO_DC QUANT_TWO_DC_SSE4
+%define QUANT_END QUANT_END_SSE4
+QUANT_DC 2, 2, sse4
+QUANT_DC 4, 4, sse4
+QUANT_AC 4, 4, sse4
+QUANT_AC 8, 8, sse4
+
+%undef SIGND
+%undef PABSD
+%undef QUANT_END
+%undef QUANT_TWO_AC
+%undef QUANT_ONE_DC
+%undef QUANT_TWO_DC
+%endif ; HIGH_BIT_DEPTH
+
+%ifndef HIGH_BIT_DEPTH
+%macro QUANT_ONE 4
;;; %1 (m64) dct[y][x]
;;; %2 (m64/mmx) mf[y][x] or mf[0][0] (as uint16_t)
;;; %3 (m64/mmx) bias[y][x] or bias[0][0] (as uint16_t)
pmulhuw m0, %2 ; divide
PSIGNW m0, m1 ; restore sign
mova %1, m0 ; store
+%if %4
+ por m5, m0
+%else
+ SWAP 5, 0
+%endif
+%endmacro
+
+%macro QUANT_TWO 7
+ mova m1, %1
+ mova m3, %2
+ PABSW m0, m1
+ PABSW m2, m3
+ paddusw m0, %5
+ paddusw m2, %6
+ pmulhuw m0, %3
+ pmulhuw m2, %4
+ PSIGNW m0, m1
+ PSIGNW m2, m3
+ mova %1, m0
+ mova %2, m2
+%if %7
+ por m5, m0
+ por m5, m2
+%else
+ SWAP 5, 0
+ por m5, m2
+%endif
%endmacro
;-----------------------------------------------------------------------------
-; void x264_quant_4x4_dc_mmxext( int16_t dct[16], int mf, int bias )
+; void quant_4x4_dc( int16_t dct[16], int mf, int bias )
;-----------------------------------------------------------------------------
-%macro QUANT_DC 2
-cglobal %1, 1,1
+%macro QUANT_DC 2-3 0
+cglobal %1, 1,1,%3
QUANT_DC_START
+%if %2==1
+ QUANT_ONE [r0], m6, m7, 0
+%else
%assign x 0
-%rep %2
- QUANT_ONE [r0+x], m6, m7
-%assign x x+regsize
+%rep %2/2
+ QUANT_TWO [r0+x], [r0+x+mmsize], m6, m6, m7, m7, x
+%assign x x+mmsize*2
%endrep
+%endif
+ QUANT_END
RET
%endmacro
;-----------------------------------------------------------------------------
-; void x264_quant_4x4_mmx( int16_t dct[16], uint16_t mf[16], uint16_t bias[16] )
+; int quant_4x4( int16_t dct[16], uint16_t mf[16], uint16_t bias[16] )
;-----------------------------------------------------------------------------
%macro QUANT_AC 2
cglobal %1, 3,3
%assign x 0
-%rep %2
- QUANT_ONE [r0+x], [r1+x], [r2+x]
-%assign x x+regsize
+%rep %2/2
+ QUANT_TWO [r0+x], [r0+x+mmsize], [r1+x], [r1+x+mmsize], [r2+x], [r2+x+mmsize], x
+%assign x x+mmsize*2
%endrep
+ QUANT_END
RET
%endmacro
INIT_MMX
+%define QUANT_END QUANT_END_MMX
%define PABSW PABSW_MMX
%define PSIGNW PSIGNW_MMX
-QUANT_DC x264_quant_2x2_dc_mmxext, 1
+%define QUANT_DC_START QUANT_DC_START_MMX
+QUANT_DC quant_2x2_dc_mmxext, 1
%ifndef ARCH_X86_64 ; not needed because sse2 is faster
-QUANT_DC x264_quant_4x4_dc_mmxext, 4
-QUANT_AC x264_quant_4x4_mmx, 4
-QUANT_AC x264_quant_8x8_mmx, 16
+QUANT_DC quant_4x4_dc_mmxext, 4
+QUANT_AC quant_4x4_mmx, 4
+QUANT_AC quant_8x8_mmx, 16
%endif
INIT_XMM
-QUANT_DC x264_quant_4x4_dc_sse2, 2
-QUANT_AC x264_quant_4x4_sse2, 2
-QUANT_AC x264_quant_8x8_sse2, 8
+QUANT_DC quant_4x4_dc_sse2, 2, 8
+QUANT_AC quant_4x4_sse2, 2
+QUANT_AC quant_8x8_sse2, 8
%define PABSW PABSW_SSSE3
%define PSIGNW PSIGNW_SSSE3
-QUANT_DC x264_quant_4x4_dc_ssse3, 2
-QUANT_AC x264_quant_4x4_ssse3, 2
-QUANT_AC x264_quant_8x8_ssse3, 8
+QUANT_DC quant_4x4_dc_ssse3, 2, 8
+QUANT_AC quant_4x4_ssse3, 2
+QUANT_AC quant_8x8_ssse3, 8
INIT_MMX
-QUANT_DC x264_quant_2x2_dc_ssse3, 1
+QUANT_DC quant_2x2_dc_ssse3, 1
+%define QUANT_END QUANT_END_SSE4
+;Not faster on Conroe, so only used in SSE4 versions
+%define QUANT_DC_START QUANT_DC_START_SSSE3
+INIT_XMM
+QUANT_DC quant_4x4_dc_sse4, 2, 8
+QUANT_AC quant_4x4_sse4, 2
+QUANT_AC quant_8x8_sse4, 8
+%endif ; !HIGH_BIT_DEPTH
%macro DEQUANT16_L 3
;;; %1 dct[y][x]
;;; %2,%3 dequant_mf[i_mf][y][x]
-;;; m5 i_qbits
-
+;;; m2 i_qbits
mova m0, %2
+%ifdef HIGH_BIT_DEPTH
+ pmaddwd m0, %1
+ pslld m0, m2
+%else
packssdw m0, %3
pmullw m0, %1
- psllw m0, m5
+ psllw m0, m2
+%endif
mova %1, m0
%endmacro
%macro DEQUANT32_R 3
;;; %1 dct[y][x]
;;; %2,%3 dequant_mf[i_mf][y][x]
-;;; m5 -i_qbits
-;;; m6 f
-;;; m7 0
-
+;;; m2 -i_qbits
+;;; m3 f
+;;; m4 0
mova m0, %1
- mova m1, m0
- punpcklwd m0, m7
- punpckhwd m1, m7
+%ifdef HIGH_BIT_DEPTH
+ pmaddwd m0, %2
+ paddd m0, m3
+ psrad m0, m2
+%else
+ punpckhwd m1, m0, m4
+ punpcklwd m0, m4
pmaddwd m0, %2
pmaddwd m1, %3
- paddd m0, m6
- paddd m1, m6
- psrad m0, m5
- psrad m1, m5
+ paddd m0, m3
+ paddd m1, m3
+ psrad m0, m2
+ psrad m1, m2
packssdw m0, m1
+%endif
mova %1, m0
%endmacro
%if 8*(%2-2*%3)
mov t0d, 8*(%2-2*%3)
%%loop:
- %1 [r0+t0+8*%3], [r1+t0*2+16*%3], [r1+t0*2+24*%3]
- %1 [r0+t0 ], [r1+t0*2 ], [r1+t0*2+ 8*%3]
+ %1 [r0+(t0 )*SIZEOF_PIXEL], [r1+t0*2 ], [r1+t0*2+ 8*%3]
+ %1 [r0+(t0+8*%3)*SIZEOF_PIXEL], [r1+t0*2+16*%3], [r1+t0*2+24*%3]
sub t0d, 16*%3
jge %%loop
- rep ret
+ REP_RET
%else
- %1 [r0+8*%3], [r1+16*%3], [r1+24*%3]
- %1 [r0 ], [r1 ], [r1+ 8*%3]
- ret
+ %1 [r0+(8*%3)*SIZEOF_PIXEL], [r1+16*%3], [r1+24*%3]
+ %1 [r0+(0 )*SIZEOF_PIXEL], [r1+0 ], [r1+ 8*%3]
+ RET
%endif
%endmacro
-%macro DEQUANT16_FLAT 2-8
+%macro DEQUANT16_FLAT 2-5
mova m0, %1
+ psllw m0, m4
%assign i %0-2
%rep %0-1
%if i
%else
pmullw m0, [r0+%2]
%endif
- psllw m %+ i, m7
mova [r0+%2], m %+ i
%assign i i-1
%rotate 1
%endrep
%endmacro
-%ifdef ARCH_X86_64
- %define t0 r4
- %define t0d r4d
- %define t1 r3
- %define t1d r3d
- %define t2 r2
- %define t2d r2d
+%ifdef WIN64
+ DECLARE_REG_TMP 6,3,2
+%elifdef ARCH_X86_64
+ DECLARE_REG_TMP 4,3,2
%else
- %define t0 r2
- %define t0d r2d
- %define t1 r0
- %define t1d r0d
- %define t2 r1
- %define t2d r1d
+ DECLARE_REG_TMP 2,0,1
%endif
-;-----------------------------------------------------------------------------
-; void x264_dequant_4x4_mmx( int16_t dct[4][4], int dequant_mf[6][4][4], int i_qp )
-;-----------------------------------------------------------------------------
-%macro DEQUANT 4
-cglobal x264_dequant_%2x%2_%1, 0,3
+%macro DEQUANT_START 2
movifnidn t2d, r2m
imul t0d, t2d, 0x2b
shr t0d, 8 ; i_qbits = i_qp / 6
lea t1, [t0*3]
sub t2d, t1d
sub t2d, t1d ; i_mf = i_qp % 6
- shl t2d, %3+2
+ shl t2d, %1
%ifdef ARCH_X86_64
add r1, t2 ; dequant_mf[i_mf]
%else
- add r1, r1m ; dequant_mf[i_mf]
- mov r0, r0m ; dct
+ add r1, r1mp ; dequant_mf[i_mf]
+ mov r0, r0mp ; dct
%endif
- sub t0d, %3
+ sub t0d, %2
jl .rshift32 ; negative qbits => rightshift
+%endmacro
+
+;-----------------------------------------------------------------------------
+; void dequant_4x4( dctcoef dct[4][4], int dequant_mf[6][4][4], int i_qp )
+;-----------------------------------------------------------------------------
+%macro DEQUANT 4
+cglobal dequant_%2x%2_%1, 0,3,6*(mmsize/16)
+.skip_prologue:
+ DEQUANT_START %3+2, %3
.lshift:
- movd m5, t0d
+ movd m2, t0d
DEQUANT_LOOP DEQUANT16_L, %2*%2/4, %4
.rshift32:
neg t0d
- movd m5, t0d
- picgetgot t0d
- mova m6, [pd_1 GLOBAL]
- pxor m7, m7
- pslld m6, m5
- psrld m6, 1
+ movd m2, t0d
+ mova m3, [pd_1]
+ pxor m4, m4
+ pslld m3, m2
+ psrld m3, 1
DEQUANT_LOOP DEQUANT32_R, %2*%2/4, %4
-cglobal x264_dequant_%2x%2_flat16_%1, 0,3
+%ifnidn %1, avx
+cglobal dequant_%2x%2_flat16_%1, 0,3
movifnidn t2d, r2m
%if %2 == 8
cmp t2d, 12
- jl x264_dequant_%2x%2_%1
+ jl dequant_%2x%2_%1.skip_prologue
sub t2d, 12
%endif
imul t0d, t2d, 0x2b
sub t2d, t1d
sub t2d, t1d ; i_mf = i_qp % 6
shl t2d, %3
-%ifdef PIC64
- lea r1, [dequant%2_scale GLOBAL]
+%ifdef PIC
+ lea r1, [dequant%2_scale]
add r1, t2
%else
- picgetgot r0
- lea r1, [t2 + dequant%2_scale GLOBAL]
+ lea r1, [dequant%2_scale + t2]
%endif
- movifnidn r0d, r0m
- movd m7, t0d
+ movifnidn r0, r0mp
+ movd m4, t0d
%if %2 == 4
%ifidn %1, mmx
DEQUANT16_FLAT [r1], 0, 16
DEQUANT16_FLAT [r1+16], 16, 48, 80, 112
DEQUANT16_FLAT [r1+32], 32, 96
%endif
- ret
+ RET
+%endif ; !AVX
%endmacro ; DEQUANT
+%ifdef HIGH_BIT_DEPTH
+INIT_XMM
+DEQUANT sse2, 4, 4, 1
+DEQUANT sse4, 4, 4, 1
+DEQUANT sse2, 8, 6, 1
+DEQUANT sse4, 8, 6, 1
+%else
%ifndef ARCH_X86_64
INIT_MMX
DEQUANT mmx, 4, 4, 1
INIT_XMM
DEQUANT sse2, 4, 4, 2
DEQUANT sse2, 8, 6, 2
+INIT_AVX
+DEQUANT avx, 4, 4, 2
+DEQUANT avx, 8, 6, 2
+%endif
+
+%macro DEQUANT_DC 2
+cglobal dequant_4x4dc_%1, 0,3,6*(mmsize/16)
+ DEQUANT_START 6, 6
+
+.lshift:
+ movd m3, [r1]
+ movd m2, t0d
+ pslld m3, m2
+%ifdef HIGH_BIT_DEPTH
+ pshufd m3, m3, 0
+%assign x 0
+%rep SIZEOF_PIXEL*16/mmsize
+ mova m0, [r0+mmsize*0+x]
+ mova m1, [r0+mmsize*1+x]
+ pmaddwd m0, m3
+ pmaddwd m1, m3
+ mova [r0+mmsize*0+x], m0
+ mova [r0+mmsize*1+x], m1
+%assign x x+mmsize*2
+%endrep
+
+%else ; !HIGH_BIT_DEPTH
+%if mmsize==16
+ pshuflw m3, m3, 0
+ punpcklqdq m3, m3
+%else
+ pshufw m3, m3, 0
+%endif
+%assign x 0
+%rep SIZEOF_PIXEL*16/mmsize
+ mova m0, [r0+mmsize*0+x]
+ mova m1, [r0+mmsize*1+x]
+ pmullw m0, m3
+ pmullw m1, m3
+ mova [r0+mmsize*0+x], m0
+ mova [r0+mmsize*1+x], m1
+%assign x x+mmsize*2
+%endrep
+%endif ; HIGH_BIT_DEPTH
+ RET
+.rshift32:
+ neg t0d
+ movd m3, t0d
+ mova m4, [p%2_1]
+ mova m5, m4
+ pslld m4, m3
+ psrld m4, 1
+ movd m2, [r1]
+%assign x 0
+%ifdef HIGH_BIT_DEPTH
+ pshufd m2, m2, 0
+%rep SIZEOF_PIXEL*32/mmsize
+ mova m0, [r0+x]
+ pmaddwd m0, m2
+ paddd m0, m4
+ psrad m0, m3
+ mova [r0+x], m0
+%assign x x+mmsize
+%endrep
+%else
+%if mmsize==8
+ punpcklwd m2, m2
+%else
+ pshuflw m2, m2, 0
+%endif
+ punpcklwd m2, m4
+%rep SIZEOF_PIXEL*32/mmsize
+ mova m0, [r0+x]
+ punpckhwd m1, m0, m5
+ punpcklwd m0, m5
+ pmaddwd m0, m2
+ pmaddwd m1, m2
+ psrad m0, m3
+ psrad m1, m3
+ packssdw m0, m1
+ mova [r0+x], m0
+%assign x x+mmsize
+%endrep
+%endif
+ RET
+%endmacro
+
+%ifdef HIGH_BIT_DEPTH
+INIT_XMM
+DEQUANT_DC sse2 , d
+DEQUANT_DC sse4 , d
+INIT_AVX
+DEQUANT_DC avx , d
+%else
+INIT_MMX
+DEQUANT_DC mmxext, w
+INIT_XMM
+DEQUANT_DC sse2 , w
+INIT_AVX
+DEQUANT_DC avx , w
+%endif
+
+; t4 is eax for return value.
+%ifdef ARCH_X86_64
+ DECLARE_REG_TMP 0,1,2,3,6,4 ; Identical for both Windows and *NIX
+%else
+ DECLARE_REG_TMP 4,1,2,3,0,5
+%endif
;-----------------------------------------------------------------------------
-; void x264_denoise_dct_core_mmx( int16_t *dct, uint32_t *sum, uint16_t *offset, int size )
+; x264_optimize_chroma_dc( dctcoef dct[4], int dequant_mf )
;-----------------------------------------------------------------------------
-%macro DENOISE_DCT 1
-cglobal x264_denoise_dct_core_%1, 4,5
- movzx r4d, word [r0] ; backup DC coefficient
- pxor m7, m7
+
+; %2 == 1 for sse2 or ssse3, 0 for sse4/avx
+%macro OPTIMIZE_CHROMA_DC 2
+%assign %%regs 4+%2
+%ifndef ARCH_X86_64
+ %assign %%regs %%regs+1 ; t0-t4 are volatile on x86-64
+%endif
+cglobal optimize_chroma_dc_%1, 0,%%regs,7
+ movifnidn t0, r0mp
+ movd m2, r1m
+ movq m1, [t0]
+%if %2
+ pxor m4, m4
+%else ; sse4, avx
+ pcmpeqb m4, m4
+ pslld m4, 11
+%endif
+%ifidn %1, sse2
+ mova m3, [chroma_dc_dct_mask_mmx]
+ mova m5, [chroma_dc_dmf_mask_mmx]
+%else
+ mova m3, [chroma_dc_dct_mask]
+ mova m5, [chroma_dc_dmf_mask]
+%endif
+ pshuflw m2, m2, 0
+ pshufd m0, m1, 00010001b ; 1 0 3 2 1 0 3 2
+ punpcklqdq m2, m2
+ punpcklqdq m1, m1 ; 3 2 1 0 3 2 1 0
+ mova m6, [pd_1024] ; 32<<5, elements are shifted 5 bits to the left
+ PSIGNW m0, m3 ; -1 -0 3 2 -1 -0 3 2
+ PSIGNW m2, m5 ; + - - + - - + +
+ paddw m0, m1 ; -1+3 -0+2 1+3 0+2 -1+3 -0+2 1+3 0+2
+ pmaddwd m0, m2 ; 0-1-2+3 0-1+2-3 0+1-2-3 0+1+2+3 * dmf
+ punpcklwd m1, m1
+ psrad m2, 16 ; + - - +
+ mov t1d, 3
+ paddd m0, m6
+ xor t4d, t4d
+%ifidn %1, sse2
+ psrad m1, 31 ; has to be 0 or -1 in order for PSIGND_MMX to work correctly
+%endif
+%if %2
+ mova m6, m0
+ SWAP 0, 6
+ psrad m6, 11
+ pcmpeqd m6, m4
+ pmovmskb t5d, m6
+ cmp t5d, 0xffff
+%else ; sse4, avx
+ ptest m0, m4
+%endif
+ jz .ret ; if the DC coefficients already round to zero, terminate early
+ mova m3, m0
+.outer_loop:
+ movsx t3d, word [t0+2*t1] ; dct[coeff]
+ pshufd m6, m1, 11111111b
+ pshufd m1, m1, 10010000b ; move the next element to high dword
+ PSIGND m5, m2, m6
+ test t3d, t3d
+ jz .loop_end
+.outer_loop_0:
+ mov t2d, t3d
+ sar t3d, 31
+ or t3d, 1
+.inner_loop:
+ psubd m3, m5 ; coeff -= sign
+ pxor m6, m0, m3
+%if %2
+ psrad m6, 11
+ pcmpeqd m6, m4
+ pmovmskb t5d, m6
+ cmp t5d, 0xffff
+%else ; sse4, avx
+ ptest m6, m4
+%endif
+ jz .round_coeff
+ paddd m3, m5 ; coeff += sign
+ mov t4d, 1
+.loop_end:
+ dec t1d
+ jz .last_coeff
+ pshufd m2, m2, 01111000b ; - + - + / - - + +
+ jg .outer_loop
+.ret:
+ REP_RET
+.round_coeff:
+ sub t2d, t3d
+ mov [t0+2*t1], t2w
+ jnz .inner_loop
+ jmp .loop_end
+.last_coeff:
+ movsx t3d, word [t0]
+ punpcklqdq m2, m2 ; + + + +
+ PSIGND m5, m2, m1
+ test t3d, t3d
+ jnz .outer_loop_0
+ REP_RET
+%endmacro
+
+INIT_XMM
+%define PSIGNW PSIGNW_MMX
+%define PSIGND PSIGND_MMX
+OPTIMIZE_CHROMA_DC sse2, 1
+%define PSIGNW PSIGNW_SSSE3
+%define PSIGND PSIGND_SSSE3
+OPTIMIZE_CHROMA_DC ssse3, 1
+OPTIMIZE_CHROMA_DC sse4, 0
+INIT_AVX
+OPTIMIZE_CHROMA_DC avx, 0
+
+%ifdef HIGH_BIT_DEPTH
+;-----------------------------------------------------------------------------
+; void denoise_dct( int32_t *dct, uint32_t *sum, uint32_t *offset, int size )
+;-----------------------------------------------------------------------------
+%macro DENOISE_DCT 1-2 0
+cglobal denoise_dct_%1, 4,4,%2
+ pxor m6, m6
.loop:
- sub r3, regsize
- mova m2, [r0+r3*2+0*regsize]
- mova m3, [r0+r3*2+1*regsize]
- PABSW m0, m2
- PABSW m1, m3
+ sub r3, mmsize/2
+ mova m2, [r0+r3*4+0*mmsize]
+ mova m3, [r0+r3*4+1*mmsize]
+ PABSD m0, m2
+ PABSD m1, m3
mova m4, m0
mova m5, m1
- psubusw m0, [r2+r3*2+0*regsize]
- psubusw m1, [r2+r3*2+1*regsize]
- PSIGNW m0, m2
- PSIGNW m1, m3
- mova [r0+r3*2+0*regsize], m0
- mova [r0+r3*2+1*regsize], m1
- mova m2, m4
- mova m3, m5
- punpcklwd m4, m7
- punpckhwd m2, m7
- punpcklwd m5, m7
- punpckhwd m3, m7
- paddd m4, [r1+r3*4+0*regsize]
- paddd m2, [r1+r3*4+1*regsize]
- paddd m5, [r1+r3*4+2*regsize]
- paddd m3, [r1+r3*4+3*regsize]
- mova [r1+r3*4+0*regsize], m4
- mova [r1+r3*4+1*regsize], m2
- mova [r1+r3*4+2*regsize], m5
- mova [r1+r3*4+3*regsize], m3
+ psubd m0, [r2+r3*4+0*mmsize]
+ psubd m1, [r2+r3*4+1*mmsize]
+ pcmpgtd m7, m0, m6
+ pand m0, m7
+ pcmpgtd m7, m1, m6
+ pand m1, m7
+ PSIGND m0, m2
+ PSIGND m1, m3
+ mova [r0+r3*4+0*mmsize], m0
+ mova [r0+r3*4+1*mmsize], m1
+ paddd m4, [r1+r3*4+0*mmsize]
+ paddd m5, [r1+r3*4+1*mmsize]
+ mova [r1+r3*4+0*mmsize], m4
+ mova [r1+r3*4+1*mmsize], m5
jg .loop
- mov [r0], r4w ; restore DC coefficient
- RET
+ REP_RET
+%endmacro
+
+%define PABSD PABSD_MMX
+%define PSIGND PSIGND_MMX
+%ifndef ARCH_X86_64
+INIT_MMX
+DENOISE_DCT mmx
+%endif
+INIT_XMM
+DENOISE_DCT sse2, 8
+%define PABSD PABSD_SSSE3
+%define PSIGND PSIGND_SSSE3
+DENOISE_DCT ssse3, 8
+INIT_AVX
+DENOISE_DCT avx , 8
+
+%else ; !HIGH_BIT_DEPTH
+
+;-----------------------------------------------------------------------------
+; void denoise_dct( int16_t *dct, uint32_t *sum, uint16_t *offset, int size )
+;-----------------------------------------------------------------------------
+%macro DENOISE_DCT 1-2 0
+cglobal denoise_dct_%1, 4,4,%2
+ pxor m6, m6
+.loop:
+ sub r3, mmsize
+ mova m2, [r0+r3*2+0*mmsize]
+ mova m3, [r0+r3*2+1*mmsize]
+ PABSW m0, m2
+ PABSW m1, m3
+ psubusw m4, m0, [r2+r3*2+0*mmsize]
+ psubusw m5, m1, [r2+r3*2+1*mmsize]
+ PSIGNW m4, m2
+ PSIGNW m5, m3
+ mova [r0+r3*2+0*mmsize], m4
+ mova [r0+r3*2+1*mmsize], m5
+ punpcklwd m2, m0, m6
+ punpcklwd m3, m1, m6
+ punpckhwd m0, m6
+ punpckhwd m1, m6
+ paddd m2, [r1+r3*4+0*mmsize]
+ paddd m0, [r1+r3*4+1*mmsize]
+ paddd m3, [r1+r3*4+2*mmsize]
+ paddd m1, [r1+r3*4+3*mmsize]
+ mova [r1+r3*4+0*mmsize], m2
+ mova [r1+r3*4+1*mmsize], m0
+ mova [r1+r3*4+2*mmsize], m3
+ mova [r1+r3*4+3*mmsize], m1
+ jg .loop
+ REP_RET
%endmacro
%define PABSW PABSW_MMX
DENOISE_DCT mmx
%endif
INIT_XMM
-DENOISE_DCT sse2
+DENOISE_DCT sse2, 7
%define PABSW PABSW_SSSE3
%define PSIGNW PSIGNW_SSSE3
-DENOISE_DCT ssse3
+DENOISE_DCT ssse3, 7
+INIT_AVX
+DENOISE_DCT avx, 7
+
+%endif ; !HIGH_BIT_DEPTH
+
+;-----------------------------------------------------------------------------
+; int decimate_score( dctcoef *dct )
+;-----------------------------------------------------------------------------
+
+%macro DECIMATE_MASK_SSE2 7
+%ifdef HIGH_BIT_DEPTH
+ movdqa xmm0, [%3+ 0]
+ movdqa xmm1, [%3+32]
+ packssdw xmm0, [%3+16]
+ packssdw xmm1, [%3+48]
+%if %7
+ pabsw xmm0, xmm0
+ pabsw xmm1, xmm1
+%else
+ ABS2_MMX xmm0, xmm1, xmm3, xmm4
+%endif
+%else
+%if %7
+ pabsw xmm0, [%3+ 0]
+ pabsw xmm1, [%3+16]
+%else
+ movdqa xmm0, [%3+ 0]
+ movdqa xmm1, [%3+16]
+ ABS2_MMX xmm0, xmm1, xmm3, xmm4
+%endif
+%endif
+ packsswb xmm0, xmm1
+ pxor xmm2, xmm2
+ pcmpeqb xmm2, xmm0
+ pcmpgtb xmm0, %4
+ pmovmskb %1, xmm2
+ pmovmskb %2, xmm0
+%endmacro
+
+%macro DECIMATE_MASK_MMX 7
+%ifdef HIGH_BIT_DEPTH
+ movq mm0, [%3+ 0]
+ movq mm1, [%3+16]
+ movq mm2, [%3+32]
+ movq mm3, [%3+48]
+ packssdw mm0, [%3+ 8]
+ packssdw mm1, [%3+24]
+ packssdw mm2, [%3+40]
+ packssdw mm3, [%3+56]
+%else
+ movq mm0, [%3+ 0]
+ movq mm1, [%3+ 8]
+ movq mm2, [%3+16]
+ movq mm3, [%3+24]
+%endif
+ ABS2_MMX mm0, mm1, mm6, mm7
+ ABS2_MMX mm2, mm3, mm6, mm7
+ packsswb mm0, mm1
+ packsswb mm2, mm3
+ pxor mm4, mm4
+ pxor mm6, mm6
+ pcmpeqb mm4, mm0
+ pcmpeqb mm6, mm2
+ pcmpgtb mm0, %4
+ pcmpgtb mm2, %4
+ pmovmskb %6, mm4
+ pmovmskb %1, mm6
+ shl %1, 8
+ or %1, %6
+ pmovmskb %6, mm0
+ pmovmskb %2, mm2
+ shl %2, 8
+ or %2, %6
+%endmacro
+
+cextern decimate_table4
+cextern decimate_table8
+
+%macro DECIMATE4x4 4
+
+;A LUT is faster than bsf on AMD processors.
+;This is not true for score64.
+cglobal decimate_score%1_%2, 1,3
+%ifdef PIC
+ lea r10, [decimate_table4]
+ lea r11, [decimate_mask_table4]
+ %define table r10
+ %define mask_table r11
+%else
+ %define table decimate_table4
+ %define mask_table decimate_mask_table4
+%endif
+ DECIMATE_MASK edx, eax, r0, [pb_1], %2, ecx, %4
+ xor edx, 0xffff
+ je .ret
+ test eax, eax
+ jne .ret9
+%if %1==15
+ shr edx, 1
+%endif
+%if %3==1
+ movzx ecx, dl
+ movzx eax, byte [mask_table + rcx]
+ cmp edx, ecx
+ je .ret
+ bsr ecx, ecx
+ shr edx, 1
+ shr edx, cl
+ bsf ecx, edx
+ shr edx, 1
+ shr edx, cl
+ add al, byte [table + rcx]
+ add al, byte [mask_table + rdx]
+%else
+.loop:
+ bsf ecx, edx
+ shr edx, cl
+ add al, byte [table + rcx]
+ shr edx, 1
+ jne .loop
+%endif
+.ret:
+ RET
+.ret9:
+ mov eax, 9
+ RET
+
+%endmacro
+
+%ifndef ARCH_X86_64
+INIT_MMX
+%define DECIMATE_MASK DECIMATE_MASK_MMX
+DECIMATE4x4 15, mmxext, 0, 0
+DECIMATE4x4 16, mmxext, 0, 0
+DECIMATE4x4 15, mmxext_slowctz, 1, 0
+DECIMATE4x4 16, mmxext_slowctz, 1, 0
+%endif
+INIT_XMM
+%define DECIMATE_MASK DECIMATE_MASK_SSE2
+DECIMATE4x4 15, sse2, 0, 0
+DECIMATE4x4 16, sse2, 0, 0
+DECIMATE4x4 15, sse2_slowctz, 1, 0
+DECIMATE4x4 16, sse2_slowctz, 1, 0
+DECIMATE4x4 15, ssse3, 0, 1
+DECIMATE4x4 16, ssse3, 0, 1
+DECIMATE4x4 15, ssse3_slowctz, 1, 1
+DECIMATE4x4 16, ssse3_slowctz, 1, 1
+
+%macro DECIMATE8x8 2
+
+%ifdef ARCH_X86_64
+cglobal decimate_score64_%1, 1,4
+%ifdef PIC
+ lea r10, [decimate_table8]
+ %define table r10
+%else
+ %define table decimate_table8
+%endif
+ mova m5, [pb_1]
+ DECIMATE_MASK r1d, eax, r0+SIZEOF_DCTCOEF* 0, m5, %1, null, %2
+ test eax, eax
+ jne .ret9
+ DECIMATE_MASK r2d, eax, r0+SIZEOF_DCTCOEF*16, m5, %1, null, %2
+ shl r2d, 16
+ or r1d, r2d
+ DECIMATE_MASK r2d, r3d, r0+SIZEOF_DCTCOEF*32, m5, %1, null, %2
+ shl r2, 32
+ or eax, r3d
+ or r1, r2
+ DECIMATE_MASK r2d, r3d, r0+SIZEOF_DCTCOEF*48, m5, %1, null, %2
+ shl r2, 48
+ or r1, r2
+ xor r1, -1
+ je .ret
+ or eax, r3d
+ jne .ret9
+.loop:
+ bsf rcx, r1
+ shr r1, cl
+ add al, byte [table + rcx]
+ shr r1, 1
+ jne .loop
+.ret:
+ REP_RET
+.ret9:
+ mov eax, 9
+ RET
+
+%else ; ARCH
+%ifidn %1, mmxext
+cglobal decimate_score64_%1, 1,6
+%else
+cglobal decimate_score64_%1, 1,5
+%endif
+ mova m5, [pb_1]
+ DECIMATE_MASK r3, r2, r0+SIZEOF_DCTCOEF* 0, m5, %1, r5, %2
+ test r2, r2
+ jne .ret9
+ DECIMATE_MASK r4, r2, r0+SIZEOF_DCTCOEF*16, m5, %1, r5, %2
+ shl r4, 16
+ or r3, r4
+ DECIMATE_MASK r4, r1, r0+SIZEOF_DCTCOEF*32, m5, %1, r5, %2
+ or r2, r1
+ DECIMATE_MASK r1, r0, r0+SIZEOF_DCTCOEF*48, m5, %1, r5, %2
+ shl r1, 16
+ or r4, r1
+ xor r3, -1
+ je .tryret
+ xor r4, -1
+.cont:
+ or r0, r2
+ jne .ret9 ;r0 is zero at this point, so we don't need to zero it
+.loop:
+ bsf ecx, r3
+ test r3, r3
+ je .largerun
+ shrd r3, r4, cl
+ shr r4, cl
+ add r0b, byte [decimate_table8 + ecx]
+ shrd r3, r4, 1
+ shr r4, 1
+ cmp r0, 6 ;score64's threshold is never higher than 6
+ jge .ret9 ;this early termination is only useful on 32-bit because it can be done in the latency after shrd
+ test r3, r3
+ jne .loop
+ test r4, r4
+ jne .loop
+.ret:
+ REP_RET
+.tryret:
+ xor r4, -1
+ jne .cont
+ REP_RET
+.ret9:
+ mov eax, 9
+ RET
+.largerun:
+ mov r3, r4
+ xor r4, r4
+ bsf ecx, r3
+ shr r3, cl
+ shr r3, 1
+ jne .loop
+ REP_RET
+%endif ; ARCH
+
+%endmacro
+
+%ifndef ARCH_X86_64
+INIT_MMX
+%define DECIMATE_MASK DECIMATE_MASK_MMX
+DECIMATE8x8 mmxext, 0
+%endif
+INIT_XMM
+%define DECIMATE_MASK DECIMATE_MASK_SSE2
+DECIMATE8x8 sse2, 0
+DECIMATE8x8 ssse3, 1
+
+;-----------------------------------------------------------------------------
+; int coeff_last( dctcoef *dct )
+;-----------------------------------------------------------------------------
+
+%macro LAST_X86 3
+ bsr %1, %2
+%endmacro
+
+%macro LAST_SSE4A 3
+ lzcnt %1, %2
+ xor %1, %3
+%endmacro
+
+%ifdef HIGH_BIT_DEPTH
+%macro LAST_MASK4_MMX 2-3
+ movq mm0, [%2]
+ packssdw mm0, [%2+8]
+ packsswb mm0, mm0
+ pcmpeqb mm0, mm2
+ pmovmskb %1, mm0
+%endmacro
+
+%macro LAST_MASK_SSE2 2-3
+ movdqa xmm0, [%2+ 0]
+ movdqa xmm1, [%2+32]
+ packssdw xmm0, [%2+16]
+ packssdw xmm1, [%2+48]
+ packsswb xmm0, xmm1
+ pcmpeqb xmm0, xmm2
+ pmovmskb %1, xmm0
+%endmacro
+
+%macro LAST_MASK_MMX 3
+ movq mm0, [%2+ 0]
+ movq mm1, [%2+16]
+ packssdw mm0, [%2+ 8]
+ packssdw mm1, [%2+24]
+ movq mm3, [%2+32]
+ movq mm4, [%2+48]
+ packssdw mm3, [%2+40]
+ packssdw mm4, [%2+56]
+ packsswb mm0, mm1
+ packsswb mm3, mm4
+ pcmpeqb mm0, mm2
+ pcmpeqb mm3, mm2
+ pmovmskb %1, mm0
+ pmovmskb %3, mm3
+ shl %3, 8
+ or %1, %3
+%endmacro
+
+%macro COEFF_LAST4 1
+cglobal coeff_last4_%1, 1,3
+ pxor mm2, mm2
+ LAST_MASK4_MMX r1d, r0
+ xor r1d, 0xff
+ shr r1d, 4
+ LAST eax, r1d, 0x1f
+ RET
+%endmacro
+
+%define LAST LAST_X86
+COEFF_LAST4 mmxext
+%define LAST LAST_SSE4A
+COEFF_LAST4 mmxext_lzcnt
+
+%else ; !HIGH_BIT_DEPTH
+%macro LAST_MASK4_MMX 2-3
+ movq mm0, [%2]
+ packsswb mm0, mm0
+ pcmpeqb mm0, mm2
+ pmovmskb %1, mm0
+%endmacro
+
+%macro LAST_MASK_SSE2 2-3
+ movdqa xmm0, [%2+ 0]
+ packsswb xmm0, [%2+16]
+ pcmpeqb xmm0, xmm2
+ pmovmskb %1, xmm0
+%endmacro
+
+%macro LAST_MASK_MMX 3
+ movq mm0, [%2+ 0]
+ movq mm1, [%2+16]
+ packsswb mm0, [%2+ 8]
+ packsswb mm1, [%2+24]
+ pcmpeqb mm0, mm2
+ pcmpeqb mm1, mm2
+ pmovmskb %1, mm0
+ pmovmskb %3, mm1
+ shl %3, 8
+ or %1, %3
+%endmacro
+
+%macro COEFF_LAST4 1
+%ifdef ARCH_X86_64
+cglobal coeff_last4_%1, 1,1
+ LAST rax, [r0], 0x3f
+ shr eax, 4
+ RET
+%else
+cglobal coeff_last4_%1, 0,3
+ mov edx, r0mp
+ mov eax, [edx+4]
+ xor ecx, ecx
+ test eax, eax
+ cmovz eax, [edx]
+ setnz cl
+ LAST eax, eax, 0x1f
+ shr eax, 4
+ lea eax, [eax+ecx*2]
+ RET
+%endif
+%endmacro
+
+%define LAST LAST_X86
+COEFF_LAST4 mmxext
+%define LAST LAST_SSE4A
+COEFF_LAST4 mmxext_lzcnt
+%endif ; HIGH_BIT_DEPTH
+
+%macro COEFF_LAST 1
+cglobal coeff_last15_%1, 1,3
+ pxor m2, m2
+ LAST_MASK r1d, r0-SIZEOF_DCTCOEF, r2d
+ xor r1d, 0xffff
+ LAST eax, r1d, 0x1f
+ dec eax
+ RET
+
+cglobal coeff_last16_%1, 1,3
+ pxor m2, m2
+ LAST_MASK r1d, r0, r2d
+ xor r1d, 0xffff
+ LAST eax, r1d, 0x1f
+ RET
+
+%ifndef ARCH_X86_64
+cglobal coeff_last64_%1, 1, 5-mmsize/16
+ pxor m2, m2
+ LAST_MASK r2d, r0+SIZEOF_DCTCOEF* 32, r4d
+ LAST_MASK r3d, r0+SIZEOF_DCTCOEF* 48, r4d
+ shl r3d, 16
+ or r2d, r3d
+ xor r2d, -1
+ jne .secondhalf
+ LAST_MASK r1d, r0+SIZEOF_DCTCOEF* 0, r4d
+ LAST_MASK r3d, r0+SIZEOF_DCTCOEF*16, r4d
+ shl r3d, 16
+ or r1d, r3d
+ not r1d
+ LAST eax, r1d, 0x1f
+ RET
+.secondhalf:
+ LAST eax, r2d, 0x1f
+ add eax, 32
+ RET
+%else
+cglobal coeff_last64_%1, 1,4
+ pxor m2, m2
+ LAST_MASK_SSE2 r1d, r0+SIZEOF_DCTCOEF* 0
+ LAST_MASK_SSE2 r2d, r0+SIZEOF_DCTCOEF*16
+ LAST_MASK_SSE2 r3d, r0+SIZEOF_DCTCOEF*32
+ LAST_MASK_SSE2 r0d, r0+SIZEOF_DCTCOEF*48
+ shl r2d, 16
+ shl r0d, 16
+ or r1d, r2d
+ or r3d, r0d
+ shl r3, 32
+ or r1, r3
+ not r1
+ LAST rax, r1, 0x3f
+ RET
+%endif
+%endmacro
+
+%define LAST LAST_X86
+%ifndef ARCH_X86_64
+INIT_MMX
+%define LAST_MASK LAST_MASK_MMX
+COEFF_LAST mmxext
+%endif
+INIT_XMM
+%define LAST_MASK LAST_MASK_SSE2
+COEFF_LAST sse2
+%define LAST LAST_SSE4A
+COEFF_LAST sse2_lzcnt
+
+;-----------------------------------------------------------------------------
+; int coeff_level_run( dctcoef *dct, run_level_t *runlevel )
+;-----------------------------------------------------------------------------
+
+%macro LZCOUNT_X86 3
+ bsr %1, %2
+ xor %1, %3
+%endmacro
+
+%macro LZCOUNT_SSE4A 3
+ lzcnt %1, %2
+%endmacro
+
+; t6 = eax for return, t3 = ecx for shift, t[01] = r[01] for x86_64 args
+%ifdef WIN64
+ DECLARE_REG_TMP 3,1,2,0,4,5,6
+%elifdef ARCH_X86_64
+ DECLARE_REG_TMP 0,1,2,3,4,5,6
+%else
+ DECLARE_REG_TMP 6,3,2,1,4,5,0
+%endif
+
+%macro COEFF_LEVELRUN 2
+cglobal coeff_level_run%2_%1,0,7
+ movifnidn t0, r0mp
+ movifnidn t1, r1mp
+ pxor m2, m2
+ LAST_MASK t5d, t0-(%2&1)*SIZEOF_DCTCOEF, t4d
+ not t5d
+ shl t5d, 32-((%2+1)&~1)
+ mov t4d, %2-1
+ LZCOUNT t3d, t5d, 0x1f
+ xor t6d, t6d
+ add t5d, t5d
+ sub t4d, t3d
+ shl t5d, t3b
+ mov [t1], t4d
+.loop:
+ LZCOUNT t3d, t5d, 0x1f
+%ifdef HIGH_BIT_DEPTH
+ mov t2d, [t0+t4*4]
+ mov [t1+t6 +4+16*4], t3b
+ mov [t1+t6*4+ 4], t2d
+%else
+ mov t2w, [t0+t4*2]
+ mov [t1+t6 +4+16*2], t3b
+ mov [t1+t6*2+ 4], t2w
+%endif
+ inc t3d
+ shl t5d, t3b
+ inc t6d
+ sub t4d, t3d
+ jge .loop
+ REP_RET
+%endmacro
+
+INIT_MMX
+%define LZCOUNT LZCOUNT_X86
+%ifndef ARCH_X86_64
+%define LAST_MASK LAST_MASK_MMX
+COEFF_LEVELRUN mmxext, 15
+COEFF_LEVELRUN mmxext, 16
+%endif
+%define LAST_MASK LAST_MASK4_MMX
+COEFF_LEVELRUN mmxext, 4
+INIT_XMM
+%define LAST_MASK LAST_MASK_SSE2
+COEFF_LEVELRUN sse2, 15
+COEFF_LEVELRUN sse2, 16
+%define LZCOUNT LZCOUNT_SSE4A
+COEFF_LEVELRUN sse2_lzcnt, 15
+COEFF_LEVELRUN sse2_lzcnt, 16
+INIT_MMX
+%define LAST_MASK LAST_MASK4_MMX
+COEFF_LEVELRUN mmxext_lzcnt, 4
projects / x264 / blobdiff