;***************************************************************************** ;* x86-optimized HEVC MC ;* Copyright 2015 Anton Khirnov ;* ;* This file is part of Libav. ;* ;* Libav is free software; you can redistribute it and/or ;* modify it under the terms of the GNU Lesser General Public ;* License as published by the Free Software Foundation; either ;* version 2.1 of the License, or (at your option) any later version. ;* ;* Libav is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ;* Lesser General Public License for more details. ;* ;* You should have received a copy of the GNU Lesser General Public ;* License along with Libav; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** %include "libavutil/x86/x86util.asm" SECTION_RODATA pw_1023: times 8 dw 1023 cextern hevc_qpel_coeffs cextern hevc_qpel_coeffs8 cextern hevc_epel_coeffs cextern hevc_epel_coeffs8 cextern pw_8 cextern pw_16 cextern pw_32 cextern pw_64 SECTION .text ; %1: width ; %2: bit depth %macro COMMON_DEFS 2 %assign blocksize 8 %assign nb_blocks ((%1 + blocksize - 1) / blocksize) %define last_block_truncated (blocksize * nb_blocks > %1) %if %2 > 8 %define LOAD_BLOCK movu %define LOAD_HALFBLOCK movq %assign pixelsize 2 %else %define LOAD_BLOCK movq %define LOAD_HALFBLOCK movd %assign pixelsize 1 %endif %define STORE_BLOCK mova %define STORE_HALFBLOCK movq %endmacro ; %1: block index %macro BLOCK_DEFS 1 %if last_block_truncated && %1 == nb_blocks - 1 %define block_truncated 1 %define LOAD LOAD_HALFBLOCK %define STORE STORE_HALFBLOCK %else %define block_truncated 0 %define LOAD LOAD_BLOCK %define STORE STORE_BLOCK %endif %endmacro ; hevc_get_pixels__(int16_t *dst, ptrdiff_t dststride, ; pixel *src, ptrdiff_t srcstride, ; int height, int mx, int my, int *mcbuffer) ; %1: block width ; %2: bit depth ; %3: log2 of height unroll %macro GET_PIXELS 3 cglobal hevc_get_pixels_ %+ %1 %+ _ %+ %2, 5, 5, 2, dst, dststride, src, srcstride, height ; rest of the args unused %assign shift 14 - %2 COMMON_DEFS %1, %2 %if pixelsize == 1 pxor m0, m0 %endif shr heightd, %3 .loop: %assign i 0 %rep (1 << %3) %assign j 0 %rep nb_blocks BLOCK_DEFS j LOAD m1, [srcq + j * pixelsize * blocksize] %if pixelsize == 1 punpcklbw m1, m0 %endif psllw m1, shift STORE [dstq + j * 2 * blocksize], m1 %assign j (j + 1) %endrep add dstq, dststrideq add srcq, srcstrideq %assign i (i + 1) %endrep dec heightd jg .loop RET %endmacro INIT_XMM sse2 GET_PIXELS 4, 8, 1 GET_PIXELS 8, 8, 1 GET_PIXELS 12, 8, 3 GET_PIXELS 16, 8, 2 GET_PIXELS 24, 8, 3 GET_PIXELS 32, 8, 3 GET_PIXELS 48, 8, 3 GET_PIXELS 64, 8, 3 GET_PIXELS 4, 10, 1 GET_PIXELS 8, 10, 1 GET_PIXELS 12, 10, 3 GET_PIXELS 16, 10, 2 GET_PIXELS 24, 10, 3 GET_PIXELS 32, 10, 3 GET_PIXELS 48, 10, 3 GET_PIXELS 64, 10, 3 ; hevc_qpel_h/v__8(int16_t *dst, ptrdiff_t dststride, ; uint8_t *src, ptrdiff_t srcstride, ; int height, int mx, int my, int *mcbuffer) ; 8-bit qpel interpolation ; %1: block width ; %2: 0 - horizontal; 1 - vertical %macro QPEL_8 2 %if %2 %define postfix v %define mvfrac myq %define coeffsaddr r5q %define pixstride srcstrideq %define pixstride3 r5q %define src_m3 r6q %else %define postfix h %define mvfrac mxq %define coeffsaddr r6q %define pixstride 1 %define pixstride3 3 %define src_m3 (srcq - 3) %endif COMMON_DEFS %1, 8 cglobal hevc_qpel_ %+ postfix %+ _ %+ %1 %+ _8, 7, 7, 7, dst, dststride, src, srcstride, height, mx, my and mvfrac, 0x3 dec mvfrac shl mvfrac, 4 lea coeffsaddr, [hevc_qpel_coeffs8] mova m0, [coeffsaddr + mvfrac] SPLATW m1, m0, 1 SPLATW m2, m0, 2 SPLATW m3, m0, 3 SPLATW m0, m0, 0 %if %2 lea pixstride3, [srcstrideq + 2 * srcstrideq] mov src_m3, srcq sub src_m3, pixstride3 %endif .loop: %assign i 0 %rep nb_blocks BLOCK_DEFS i LOAD m4, [src_m3 + i * blocksize] LOAD m5, [src_m3 + i * blocksize + 1 * pixstride] punpcklbw m4, m5 pmaddubsw m4, m0 LOAD m5, [src_m3 + i * blocksize + 2 * pixstride] LOAD m6, [srcq + i * blocksize] punpcklbw m5, m6 pmaddubsw m5, m1 paddsw m4, m5 LOAD m5, [srcq + i * blocksize + 1 * pixstride] LOAD m6, [srcq + i * blocksize + 2 * pixstride] punpcklbw m5, m6 pmaddubsw m5, m2 paddsw m4, m5 LOAD m5, [srcq + i * blocksize + pixstride3] LOAD m6, [srcq + i * blocksize + 4 * pixstride] punpcklbw m5, m6 pmaddubsw m5, m3 paddsw m4, m5 STORE [dstq + i * 2 * blocksize], m4 %assign i (i + 1) %endrep add dstq, dststrideq add srcq, srcstrideq %if %2 add src_m3, srcstrideq %endif dec heightd jg .loop RET %endmacro INIT_XMM ssse3 QPEL_8 4, 0 QPEL_8 8, 0 QPEL_8 12, 0 QPEL_8 16, 0 QPEL_8 24, 0 QPEL_8 32, 0 QPEL_8 48, 0 QPEL_8 64, 0 QPEL_8 4, 1 QPEL_8 8, 1 QPEL_8 12, 1 QPEL_8 16, 1 QPEL_8 24, 1 QPEL_8 32, 1 QPEL_8 48, 1 QPEL_8 64, 1 ; 16-bit qpel interpolation ; %1: block width ; %2: shift applied to the result ; %3: 0 - horizontal; 1 - vertical %macro QPEL_16 3 %if %3 %define mvfrac myq %define pixstride srcstrideq %define pixstride3 sstride3q %define src_m3 srcm3q %else %define mvfrac mxq %define pixstride 2 %define pixstride3 6 %define src_m3 (srcq - 6) %endif COMMON_DEFS %1, 16 and mvfrac, 0x3 dec mvfrac shl mvfrac, 4 lea coeffsregq, [hevc_qpel_coeffs] mova m0, [coeffsregq + mvfrac] pshufd m1, m0, 0x55 pshufd m2, m0, 0xaa pshufd m3, m0, 0xff pshufd m0, m0, 0x00 %if %3 lea sstride3q, [srcstrideq + 2 * srcstrideq] mov srcm3q, srcq sub srcm3q, sstride3q %endif .loop: %assign i 0 %rep nb_blocks BLOCK_DEFS i LOAD m4, [src_m3 + i * 2 * blocksize] LOAD m5, [src_m3 + i * 2 * blocksize + 1 * pixstride] LOAD m6, [src_m3 + i * 2 * blocksize + 2 * pixstride] LOAD m7, [srcq + i * 2 * blocksize + 0 * pixstride] LOAD m8, [srcq + i * 2 * blocksize + 1 * pixstride] LOAD m9, [srcq + i * 2 * blocksize + 2 * pixstride] LOAD m10, [srcq + i * 2 * blocksize + pixstride3] LOAD m11, [srcq + i * 2 * blocksize + 4 * pixstride] punpcklwd m12, m4, m5 pmaddwd m12, m0 punpcklwd m13, m6, m7 pmaddwd m13, m1 paddd m12, m13 punpcklwd m13, m8, m9 pmaddwd m13, m2 paddd m12, m13 punpcklwd m13, m10, m11 pmaddwd m13, m3 paddd m12, m13 psrad m12, %2 %if block_truncated == 0 punpckhwd m4, m5 pmaddwd m4, m0 punpckhwd m6, m7 pmaddwd m6, m1 paddd m4, m6 punpckhwd m8, m9 pmaddwd m8, m2 paddd m4, m8 punpckhwd m10, m11 pmaddwd m10, m3 paddd m4, m10 psrad m4, %2 %endif packssdw m12, m4 STORE [dstq + i * 2 * blocksize], m12 %assign i (i + 1) %endrep add dstq, dststrideq add srcq, srcstrideq %if %3 add srcm3q, srcstrideq %endif dec heightd jg .loop RET %endmacro %if ARCH_X86_64 %macro QPEL_H_10 1 cglobal hevc_qpel_h_ %+ %1 %+ _10, 7, 9, 14, dst, dststride, src, srcstride, height, mx, my, mcbuffer, coeffsreg QPEL_16 %1, 2, 0 %endmacro INIT_XMM avx QPEL_H_10 4 QPEL_H_10 8 QPEL_H_10 12 QPEL_H_10 16 QPEL_H_10 24 QPEL_H_10 32 QPEL_H_10 48 QPEL_H_10 64 %macro QPEL_V_10 1 cglobal hevc_qpel_v_ %+ %1 %+ _10, 7, 10, 14, dst, dststride, src, srcstride, height, mx, my, sstride3, srcm3, coeffsreg QPEL_16 %1, 2, 1 %endmacro INIT_XMM avx QPEL_V_10 4 QPEL_V_10 8 QPEL_V_10 12 QPEL_V_10 16 QPEL_V_10 24 QPEL_V_10 32 QPEL_V_10 48 QPEL_V_10 64 ; hevc_qpel_hv_(int16_t *dst, ptrdiff_t dststride, ; uint8_t *src, ptrdiff_t srcstride, ; int height, int mx, int my, int *mcbuffer) %macro QPEL_HV 1 cglobal hevc_qpel_hv_ %+ %1, 7, 10, 14, dst, dststride, src, srcstride, height, mx, my, sstride3, srcm3, coeffsreg QPEL_16 %1, 6, 1 %endmacro INIT_XMM avx QPEL_HV 4 QPEL_HV 8 QPEL_HV 12 QPEL_HV 16 QPEL_HV 24 QPEL_HV 32 QPEL_HV 48 QPEL_HV 64 %endif ; ARCH_X86_64 ; hevc_epel_h/v__8(int16_t *dst, ptrdiff_t dststride, ; uint8_t *src, ptrdiff_t srcstride, ; int height, int mx, int my, int *mcbuffer) ; 8-bit epel interpolation ; %1: block width ; %2: 0 - horizontal; 1 - vertical %macro EPEL_8 2 %if %2 %define postfix v %define mvfrac myq %define coeffsaddr r5q %define pixstride srcstrideq %define pixstride3 r5q %else %define postfix h %define mvfrac mxq %define coeffsaddr r6q %define pixstride 1 %define pixstride3 3 %endif COMMON_DEFS %1, 8 cglobal hevc_epel_ %+ postfix %+ _ %+ %1 %+ _8, 7, 7, 6, dst, dststride, src, srcstride, height, mx, my and mvfrac, 0x7 dec mvfrac shl mvfrac, 4 lea coeffsaddr, [hevc_epel_coeffs8] movq m0, [coeffsaddr + mvfrac] SPLATW m1, m0, 1 SPLATW m0, m0, 0 %if %2 lea pixstride3, [srcstrideq + 2 * srcstrideq] %endif sub srcq, pixstride .loop: %assign i 0 %rep nb_blocks BLOCK_DEFS i LOAD m2, [srcq + i * blocksize + 0 * pixstride] LOAD m3, [srcq + i * blocksize + 1 * pixstride] LOAD m4, [srcq + i * blocksize + 2 * pixstride] LOAD m5, [srcq + i * blocksize + pixstride3] punpcklbw m2, m3 punpcklbw m4, m5 pmaddubsw m2, m0 pmaddubsw m4, m1 paddsw m2, m4 STORE [dstq + i * 2 * blocksize], m2 %assign i (i + 1) %endrep add dstq, dststrideq add srcq, srcstrideq dec heightd jg .loop RET %endmacro INIT_XMM ssse3 EPEL_8 4, 0 EPEL_8 8, 0 EPEL_8 12, 0 EPEL_8 16, 0 EPEL_8 24, 0 EPEL_8 32, 0 EPEL_8 4, 1 EPEL_8 8, 1 EPEL_8 12, 1 EPEL_8 16, 1 EPEL_8 24, 1 EPEL_8 32, 1 %macro EPEL_16 3 %if %3 %define mvfrac myq %define pixstride srcstrideq %define pixstride3 sstride3q %else %define mvfrac mxq %define pixstride 2 %define pixstride3 6 %endif COMMON_DEFS %1, 16 and mvfrac, 0x7 dec mvfrac shl mvfrac, 5 lea coeffsregq, [hevc_epel_coeffs] mova m0, [coeffsregq + mvfrac] pshufd m1, m0, 0x55 pshufd m0, m0, 0x00 %if %3 lea sstride3q, [srcstrideq + 2 * srcstrideq] %endif sub srcq, pixstride .loop: %assign i 0 %rep nb_blocks BLOCK_DEFS i LOAD m2, [srcq + i * 2 * blocksize + 0 * pixstride] LOAD m3, [srcq + i * 2 * blocksize + 1 * pixstride] LOAD m4, [srcq + i * 2 * blocksize + 2 * pixstride] LOAD m5, [srcq + i * 2 * blocksize + pixstride3] punpcklwd m6, m2, m3 punpcklwd m7, m4, m5 pmaddwd m6, m0 pmaddwd m7, m1 paddd m6, m7 psrad m6, %2 %if block_truncated == 0 punpckhwd m2, m3 punpckhwd m4, m5 pmaddwd m2, m0 pmaddwd m4, m1 paddd m2, m4 psrad m2, %2 %endif packssdw m6, m2 STORE [dstq + i * 2 * blocksize], m6 %assign i (i + 1) %endrep add dstq, dststrideq add srcq, srcstrideq dec heightd jg .loop RET %endmacro %if ARCH_X86_64 %macro EPEL_H_10 1 cglobal hevc_epel_h_ %+ %1 %+ _10, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg EPEL_16 %1, 2, 0 %endmacro INIT_XMM avx EPEL_H_10 4 EPEL_H_10 8 EPEL_H_10 12 EPEL_H_10 16 EPEL_H_10 24 EPEL_H_10 32 %macro EPEL_V_10 1 cglobal hevc_epel_v_ %+ %1 %+ _10, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg EPEL_16 %1, 2, 1 %endmacro INIT_XMM avx EPEL_V_10 4 EPEL_V_10 8 EPEL_V_10 12 EPEL_V_10 16 EPEL_V_10 24 EPEL_V_10 32 ; hevc_epel_hv__8(int16_t *dst, ptrdiff_t dststride, ; int16_t *src, ptrdiff_t srcstride, ; int height, int mx, int my, int *mcbuffer) %macro EPEL_HV 1 cglobal hevc_epel_hv_ %+ %1, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg EPEL_16 %1, 6, 1 %endmacro INIT_XMM avx EPEL_HV 4 EPEL_HV 8 EPEL_HV 12 EPEL_HV 16 EPEL_HV 24 EPEL_HV 32 %endif ; ARCH_X86_64 ; hevc_put_unweighted_pred__(pixel *dst, ptrdiff_t dststride, ; int16_t *src, ptrdiff_t srcstride, ; int height) %macro AVG 5 %if %3 %if %4 == 4 movq %5, %2 paddsw %1, %5 %else paddsw %1, %2 %endif %endif %endmacro ; %1: 0 - one source; 1 - two sources ; %2: width ; %3: bit depth %macro PUT_PRED 3 %if %1 cglobal hevc_put_unweighted_pred_avg_ %+ %2 %+ _ %+ %3, 6, 6, 4, dst, dststride, src, src2, srcstride, height %else cglobal hevc_put_unweighted_pred_ %+ %2 %+ _ %+ %3, 5, 5, 4, dst, dststride, src, srcstride, height %endif %assign shift 14 + %1 - %3 %assign offset (1 << (shift - 1)) %define offset_data pw_ %+ offset mova m0, [offset_data] %if %3 > 8 %define STORE_BLOCK movu %define STORE_HALF movq %assign pixel_max ((1 << %3) - 1) %define pw_pixel_max pw_ %+ pixel_max pxor m1, m1 mova m2, [pw_pixel_max] %else %define STORE_BLOCK movq %define STORE_HALF movd %endif .loop: %assign i 0 %rep (%2 + 7) / 8 %if (i + 1) * 8 > %2 %define LOAD movq %define STORE STORE_HALF %else %define LOAD mova %define STORE STORE_BLOCK %endif LOAD m3, [srcq + 16 * i] AVG m3, [src2q + 16 * i], %1, %3 - i * 8, m4 paddsw m3, m0 psraw m3, shift %if %3 == 8 packuswb m3, m3 STORE [dstq + 8 * i], m3 %else CLIPW m3, m1, m2 STORE [dstq + 16 * i], m3 %endif %assign i (i + 1) %endrep add dstq, dststrideq add srcq, srcstrideq %if %1 add src2q, srcstrideq %endif dec heightd jg .loop RET %endmacro INIT_XMM sse2 PUT_PRED 0, 4, 8 PUT_PRED 1, 4, 8 PUT_PRED 0, 8, 8 PUT_PRED 1, 8, 8 PUT_PRED 0, 12, 8 PUT_PRED 1, 12, 8 PUT_PRED 0, 16, 8 PUT_PRED 1, 16, 8 PUT_PRED 0, 24, 8 PUT_PRED 1, 24, 8 PUT_PRED 0, 32, 8 PUT_PRED 1, 32, 8 PUT_PRED 0, 48, 8 PUT_PRED 1, 48, 8 PUT_PRED 0, 64, 8 PUT_PRED 1, 64, 8 PUT_PRED 0, 4, 10 PUT_PRED 1, 4, 10 PUT_PRED 0, 8, 10 PUT_PRED 1, 8, 10 PUT_PRED 0, 12, 10 PUT_PRED 1, 12, 10 PUT_PRED 0, 16, 10 PUT_PRED 1, 16, 10 PUT_PRED 0, 24, 10 PUT_PRED 1, 24, 10 PUT_PRED 0, 32, 10 PUT_PRED 1, 32, 10 PUT_PRED 0, 48, 10 PUT_PRED 1, 48, 10 PUT_PRED 0, 64, 10 PUT_PRED 1, 64, 10 %macro PUT_WEIGHTED_PRED 3 %if %1 cglobal hevc_put_weighted_pred_avg_ %+ %2 %+ _ %+ %3, 11, 11, 8, denom, weight0, weight1, offset0, offset1, dst, dststride, src0, src1, srcstride, height %else cglobal hevc_put_weighted_pred_ %+ %2 %+ _ %+ %3, 8, 8, 8, denom, weight0, offset0, dst, dststride, src0, srcstride, height %endif and denomd, 0xff movsx weight0d, weight0w movsx offset0d, offset0w %if %1 movsx weight1d, weight1w movsx offset1d, offset1w %endif add denomd, 14 + %1 - %3 movd m0, denomd %if %3 > 8 %assign pixel_max ((1 << %3) - 1) %define pw_pixel_max pw_ %+ pixel_max pxor m4, m4 mova m5, [pw_pixel_max] shl offset0d, %3 - 8 %if %1 shl offset1d, %3 - 8 %endif %endif %if %1 lea offset0d, [offset0d + offset1d + 1] %else lea offset0d, [2 * offset0d + 1] %endif movd m1, offset0d SPLATD m1 pslld m1, m0 psrad m1, 1 movd m2, weight0d SPLATD m2 %if %1 movd m3, weight1d SPLATD m3 %endif .loop: %assign i 0 %rep (%2 + 3) / 4 pmovsxwd m6, [src0q + 8 * i] pmulld m6, m2 %if %1 pmovsxwd m7, [src1q + 8 * i] pmulld m7, m3 paddd m6, m7 %endif paddd m6, m1 psrad m6, m0 packssdw m6, m6 %if %3 > 8 CLIPW m6, m4, m5 movq [dstq + 8 * i], m6 %else packuswb m6, m6 movd [dstq + 4 * i], m6 %endif %assign i (i + 1) %endrep add dstq, dststrideq add src0q, srcstrideq %if %1 add src1q, srcstrideq %endif dec heightd jg .loop RET %endmacro %if ARCH_X86_64 INIT_XMM sse4 PUT_WEIGHTED_PRED 0, 4, 8 PUT_WEIGHTED_PRED 1, 4, 8 PUT_WEIGHTED_PRED 0, 8, 8 PUT_WEIGHTED_PRED 1, 8, 8 PUT_WEIGHTED_PRED 0, 12, 8 PUT_WEIGHTED_PRED 1, 12, 8 PUT_WEIGHTED_PRED 0, 16, 8 PUT_WEIGHTED_PRED 1, 16, 8 PUT_WEIGHTED_PRED 0, 24, 8 PUT_WEIGHTED_PRED 1, 24, 8 PUT_WEIGHTED_PRED 0, 32, 8 PUT_WEIGHTED_PRED 1, 32, 8 PUT_WEIGHTED_PRED 0, 48, 8 PUT_WEIGHTED_PRED 1, 48, 8 PUT_WEIGHTED_PRED 0, 64, 8 PUT_WEIGHTED_PRED 1, 64, 8 PUT_WEIGHTED_PRED 0, 4, 10 PUT_WEIGHTED_PRED 1, 4, 10 PUT_WEIGHTED_PRED 0, 8, 10 PUT_WEIGHTED_PRED 1, 8, 10 PUT_WEIGHTED_PRED 0, 12, 10 PUT_WEIGHTED_PRED 1, 12, 10 PUT_WEIGHTED_PRED 0, 16, 10 PUT_WEIGHTED_PRED 1, 16, 10 PUT_WEIGHTED_PRED 0, 24, 10 PUT_WEIGHTED_PRED 1, 24, 10 PUT_WEIGHTED_PRED 0, 32, 10 PUT_WEIGHTED_PRED 1, 32, 10 PUT_WEIGHTED_PRED 0, 48, 10 PUT_WEIGHTED_PRED 1, 48, 10 PUT_WEIGHTED_PRED 0, 64, 10 PUT_WEIGHTED_PRED 1, 64, 10 %endif ; ARCH_X86_64