2 ; * Provide SSE luma and chroma mc functions for HEVC decoding
3 ; * Copyright (c) 2013 Pierre-Edouard LEPERE
5 ; * This file is part of FFmpeg.
7 ; * FFmpeg is free software; you can redistribute it and/or
8 ; * modify it under the terms of the GNU Lesser General Public
9 ; * License as published by the Free Software Foundation; either
10 ; * version 2.1 of the License, or (at your option) any later version.
12 ; * FFmpeg is distributed in the hope that it will be useful,
13 ; * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 ; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ; * Lesser General Public License for more details.
17 ; * You should have received a copy of the GNU Lesser General Public
18 ; * License along with FFmpeg; if not, write to the Free Software
19 ; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 %include "libavutil/x86/x86util.asm"
24 pw_8: times 16 dw (1 << 9)
25 pw_10: times 16 dw (1 << 11)
26 pw_12: times 16 dw (1 << 13)
27 pw_bi_8: times 16 dw (1 << 8)
28 pw_bi_10: times 16 dw (1 << 10)
29 pw_bi_12: times 16 dw (1 << 12)
30 max_pixels_8: times 16 dw ((1 << 8)-1)
31 max_pixels_10: times 16 dw ((1 << 10)-1)
32 max_pixels_12: times 16 dw ((1 << 12)-1)
38 hevc_epel_filters_%4_%1 times %2 d%3 -2, 58
55 EPEL_TABLE 8,16, b, avx2
56 EPEL_TABLE 10, 8, w, avx2
58 EPEL_TABLE 8, 8, b, sse4
59 EPEL_TABLE 10, 4, w, sse4
60 EPEL_TABLE 12, 4, w, sse4
63 hevc_qpel_filters_%4_%1 times %2 d%3 -1, 4
77 QPEL_TABLE 8, 8, b, sse4
78 QPEL_TABLE 10, 4, w, sse4
79 QPEL_TABLE 12, 4, w, sse4
81 QPEL_TABLE 8,16, b, avx2
82 QPEL_TABLE 10, 8, w, avx2
84 %define MAX_PB_SIZE 64
86 %define hevc_qpel_filters_sse4_14 hevc_qpel_filters_sse4_10
88 %define hevc_qpel_filters_avx2_14 hevc_qpel_filters_avx2_10
92 %macro SIMPLE_BILOAD 4 ;width, tab, r1, r2
94 movq %3, [%2] ; load data from source2
96 movdqa %3, [%2] ; load data from source2
101 movdqa %3, [%2] ; load data from source2
102 movq %4, [%2+16] ; load data from source2
108 movdqa %3, [%2] ; load data from source2
109 movdqa %4, [%2+16] ; load data from source2
117 %macro SIMPLE_LOAD 4 ;width, bitd, tab, r1
118 %if %1 == 2 || (%2 == 8 && %1 <= 4)
119 movd %4, [%3] ; load data from source
120 %elif %1 == 4 || (%2 == 8 && %1 <= 8)
121 movq %4, [%3] ; load data from source
122 %elif notcpuflag(avx)
123 movu %4, [%3] ; load data from source
124 %elif %1 <= 8 || (%2 == 8 && %1 <= 16)
132 %macro EPEL_FILTER 2-4 ; bit depth, filter index
136 lea rfilterq, [hevc_epel_filters_avx2_%1]
138 %define rfilterq hevc_epel_filters_avx2_%1
143 lea rfilterq, [hevc_epel_filters_sse4_%1]
145 %define rfilterq hevc_epel_filters_sse4_%1
147 %endif ;cpuflag(avx2)
150 shl %2q, 6 ; multiply by 64
152 shl %2q, 5 ; multiply by 32
155 mova m14, [rfilterq + %2q] ; get 2 first values of filters
156 mova m15, [rfilterq + %2q+%%offset] ; get 2 last values of filters
158 mova %3, [rfilterq + %2q] ; get 2 first values of filters
159 mova %4, [rfilterq + %2q+%%offset] ; get 2 last values of filters
163 %macro EPEL_HV_FILTER 1
167 %define %%table hevc_epel_filters_avx2_%1
171 %define %%table hevc_epel_filters_sse4_%1
175 lea rfilterq, [%%table]
177 %define rfilterq %%table
181 shl mxq, %%shift ; multiply by 32
182 shl myq, %%shift ; multiply by 32
183 mova m14, [rfilterq + mxq] ; get 2 first values of filters
184 mova m15, [rfilterq + mxq+%%offset] ; get 2 last values of filters
185 lea r3srcq, [srcstrideq*3]
188 %define %%table hevc_epel_filters_avx2_10
190 %define %%table hevc_epel_filters_sse4_10
193 lea rfilterq, [%%table]
195 %define rfilterq %%table
197 mova m12, [rfilterq + myq] ; get 2 first values of filters
198 mova m13, [rfilterq + myq+%%offset] ; get 2 last values of filters
206 %define %%table hevc_qpel_filters_avx2_%1
210 %define %%table hevc_qpel_filters_sse4_%1
214 lea rfilterq, [%%table]
216 %define rfilterq %%table
219 shl %2q, %%shift ; multiply by 32
220 mova m12, [rfilterq + %2q] ; get 4 first values of filters
221 mova m13, [rfilterq + %2q + %%offset] ; get 4 first values of filters
222 mova m14, [rfilterq + %2q + 2*%%offset] ; get 4 first values of filters
223 mova m15, [rfilterq + %2q + 3*%%offset] ; get 4 first values of filters
232 %if (%1 == 8 && %4 <= 4)
234 %elif (%1 == 8 && %4 <= 8) || (%1 > 8 && %4 <= 4)
237 %define %%load movdqu
240 %%load m0, [rfilterq ]
242 %%load m1, [rfilterq+ %3]
243 %%load m2, [rfilterq+2*%3]
244 %%load m3, [rfilterq+3*%3]
246 %%load m1, [rfilterq+ %3q]
247 %%load m2, [rfilterq+2*%3q]
248 %%load m3, [rfilterq+r3srcq]
252 SBUTTERFLY bw, 0, 1, 7
253 SBUTTERFLY bw, 2, 3, 7
260 SBUTTERFLY wd, 0, 1, 7
261 SBUTTERFLY wd, 2, 3, 7
271 %assign %%stride (%1+7)/8
289 %%load m0, [%2-3*%%stride] ;load data from source
290 %%load m1, [%2-2*%%stride]
291 %%load m2, [%2-%%stride ]
293 %%load m4, [%2+%%stride ]
294 %%load m5, [%2+2*%%stride]
295 %%load m6, [%2+3*%%stride]
296 %%load m7, [%2+4*%%stride]
300 SBUTTERFLY wd, 0, 1, %4
301 SBUTTERFLY wd, 2, 3, %4
302 SBUTTERFLY wd, 4, 5, %4
303 SBUTTERFLY wd, 6, 7, %4
312 SBUTTERFLY dq, 0, 1, %4
313 SBUTTERFLY dq, 2, 3, %4
314 SBUTTERFLY dq, 4, 5, %4
315 SBUTTERFLY dq, 6, 7, %4
328 movu m0, [%5q ] ;load x- 3*srcstride
329 movu m1, [%5q+ %3q ] ;load x- 2*srcstride
330 movu m2, [%5q+ 2*%3q ] ;load x-srcstride
331 movu m3, [%2 ] ;load x
332 movu m4, [%2+ %3q] ;load x+stride
333 movu m5, [%2+ 2*%3q] ;load x+2*stride
334 movu m6, [%2+r3srcq] ;load x+3*stride
335 movu m7, [%2+ 4*%3q] ;load x+4*stride
338 SBUTTERFLY bw, 0, 1, 8
339 SBUTTERFLY bw, 2, 3, 8
340 SBUTTERFLY bw, 4, 5, 8
341 SBUTTERFLY bw, 6, 7, 8
350 SBUTTERFLY wd, 0, 1, 8
351 SBUTTERFLY wd, 2, 3, 8
352 SBUTTERFLY wd, 4, 5, 8
353 SBUTTERFLY wd, 6, 7, 8
363 %macro PEL_12STORE2 3
366 %macro PEL_12STORE4 3
369 %macro PEL_12STORE6 3
374 %macro PEL_12STORE8 3
377 %macro PEL_12STORE12 3
381 %macro PEL_12STORE16 3
382 PEL_12STORE8 %1, %2, %3
386 %macro PEL_10STORE2 3
389 %macro PEL_10STORE4 3
392 %macro PEL_10STORE6 3
397 %macro PEL_10STORE8 3
400 %macro PEL_10STORE12 3
404 %macro PEL_10STORE16 3
408 PEL_10STORE8 %1, %2, %3
413 %macro PEL_10STORE32 3
414 PEL_10STORE16 %1, %2, %3
431 %macro PEL_8STORE12 3
436 %macro PEL_8STORE16 3
443 %macro PEL_8STORE32 3
448 add %1q, 2*MAX_PB_SIZE ; dst += dststride
449 add %2q, %3q ; src += srcstride
450 dec heightd ; cmp height
451 jnz .loop ; height loop
455 %macro MC_PIXEL_COMPUTE 2-3 ;width, bitdepth
457 %if cpuflag(avx2) && %0 ==3
459 vextracti128 xm1, m0, 1
475 %macro EPEL_COMPUTE 4-8 ; bitdepth, width, filter1, filter2, HV/m0, m2, m1, m3
488 %if cpuflag(avx2) && (%0 == 5)
490 vextracti128 xm10, m0, 1
491 vinserti128 m10, m1, xm10, 0
493 vinserti128 m0, m0, xm1, 1
496 vextracti128 xm10, m2, 1
497 vinserti128 m10, m3, xm10, 0
499 vinserti128 m2, m2, xm3, 1
502 pmaddubsw %%reg0, %3 ;x1*c1+x2*c2
503 pmaddubsw %%reg2, %4 ;x3*c3+x4*c4
525 packssdw %%reg0, %%reg1
529 %macro QPEL_HV_COMPUTE 4 ; width, bitdepth, filter idx
533 %define %%table hevc_qpel_filters_avx2_%2
536 %define %%table hevc_qpel_filters_sse4_%2
540 lea rfilterq, [%%table]
542 %define rfilterq %%table
546 pmaddubsw m0, [rfilterq + %3q*8 ] ;x1*c1+x2*c2
547 pmaddubsw m2, [rfilterq + %3q*8+%%offset] ;x3*c3+x4*c4
548 pmaddubsw m4, [rfilterq + %3q*8+2*%%offset] ;x5*c5+x6*c6
549 pmaddubsw m6, [rfilterq + %3q*8+3*%%offset] ;x7*c7+x8*c8
554 pmaddwd m0, [rfilterq + %3q*8 ]
555 pmaddwd m2, [rfilterq + %3q*8+%%offset]
556 pmaddwd m4, [rfilterq + %3q*8+2*%%offset]
557 pmaddwd m6, [rfilterq + %3q*8+3*%%offset]
565 pmaddwd m1, [rfilterq + %3q*8 ]
566 pmaddwd m3, [rfilterq + %3q*8+%%offset]
567 pmaddwd m5, [rfilterq + %3q*8+2*%%offset]
568 pmaddwd m7, [rfilterq + %3q*8+3*%%offset]
580 %macro QPEL_COMPUTE 2-3 ; width, bitdepth
582 %if cpuflag(avx2) && (%0 == 3)
584 vextracti128 xm10, m0, 1
585 vinserti128 m10, m1, xm10, 0
586 vinserti128 m0, m0, xm1, 1
589 vextracti128 xm10, m2, 1
590 vinserti128 m10, m3, xm10, 0
591 vinserti128 m2, m2, xm3, 1
595 vextracti128 xm10, m4, 1
596 vinserti128 m10, m5, xm10, 0
597 vinserti128 m4, m4, xm5, 1
600 vextracti128 xm10, m6, 1
601 vinserti128 m10, m7, xm10, 0
602 vinserti128 m6, m6, xm7, 1
606 pmaddubsw m0, m12 ;x1*c1+x2*c2
607 pmaddubsw m2, m13 ;x3*c3+x4*c4
608 pmaddubsw m4, m14 ;x5*c5+x6*c6
609 pmaddubsw m6, m15 ;x7*c7+x8*c8
648 %macro BI_COMPUTE 7-8 ; width, bitd, src1l, src1h, scr2l, scr2h, pw
653 UNI_COMPUTE %1, %2, %3, %4, %7
654 %if %0 == 8 && cpuflag(avx2) && (%2 == 8)
662 %if %1 > 8 || (%2 > 8 && %1 > 4)
668 CLIPW %3, [pb_0], [max_pixels_%2]
669 %if (%1 > 8 && notcpuflag(avx)) || %1 > 16
670 CLIPW %4, [pb_0], [max_pixels_%2]
676 ; ******************************
677 ; void put_hevc_mc_pixels(int16_t *dst, ptrdiff_t dststride,
678 ; uint8_t *_src, ptrdiff_t _srcstride,
679 ; int height, int mx, int my)
680 ; ******************************
682 %macro HEVC_PUT_HEVC_PEL_PIXELS 2
683 HEVC_PEL_PIXELS %1, %2
684 HEVC_UNI_PEL_PIXELS %1, %2
685 HEVC_BI_PEL_PIXELS %1, %2
688 %macro HEVC_PEL_PIXELS 2
689 cglobal hevc_put_hevc_pel_pixels%1_%2, 4, 4, 3, dst, src, srcstride,height
692 SIMPLE_LOAD %1, %2, srcq, m0
693 MC_PIXEL_COMPUTE %1, %2, 1
694 PEL_10STORE%1 dstq, m0, m1
695 LOOP_END dst, src, srcstride
699 %macro HEVC_UNI_PEL_PIXELS 2
700 cglobal hevc_put_hevc_uni_pel_pixels%1_%2, 5, 5, 2, dst, dststride, src, srcstride,height
702 SIMPLE_LOAD %1, %2, srcq, m0
703 PEL_%2STORE%1 dstq, m0, m1
704 add dstq, dststrideq ; dst += dststride
705 add srcq, srcstrideq ; src += srcstride
706 dec heightd ; cmp height
707 jnz .loop ; height loop
711 %macro HEVC_BI_PEL_PIXELS 2
712 cglobal hevc_put_hevc_bi_pel_pixels%1_%2, 6, 6, 6, dst, dststride, src, srcstride, src2, height
714 movdqa m5, [pw_bi_%2]
716 SIMPLE_LOAD %1, %2, srcq, m0
717 SIMPLE_BILOAD %1, src2q, m3, m4
718 MC_PIXEL_COMPUTE %1, %2, 1
719 BI_COMPUTE %1, %2, m0, m1, m3, m4, m5, 1
720 PEL_%2STORE%1 dstq, m0, m1
721 add dstq, dststrideq ; dst += dststride
722 add srcq, srcstrideq ; src += srcstride
723 add src2q, 2*MAX_PB_SIZE ; src += srcstride
724 dec heightd ; cmp height
725 jnz .loop ; height loop
730 ; ******************************
731 ; void put_hevc_epel_hX(int16_t *dst, ptrdiff_t dststride,
732 ; uint8_t *_src, ptrdiff_t _srcstride,
733 ; int width, int height, int mx, int my,
735 ; ******************************
738 %macro HEVC_PUT_HEVC_EPEL 2
739 cglobal hevc_put_hevc_epel_h%1_%2, 5, 6, 11, dst, src, srcstride, height, mx, rfilter
740 %assign %%stride ((%2 + 7)/8)
741 EPEL_FILTER %2, mx, m4, m5
743 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
744 EPEL_COMPUTE %2, %1, m4, m5, 1
745 PEL_10STORE%1 dstq, m0, m1
746 LOOP_END dst, src, srcstride
749 cglobal hevc_put_hevc_uni_epel_h%1_%2, 6, 7, 11, dst, dststride, src, srcstride, height, mx, rfilter
750 %assign %%stride ((%2 + 7)/8)
752 EPEL_FILTER %2, mx, m4, m5
754 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
755 EPEL_COMPUTE %2, %1, m4, m5
756 UNI_COMPUTE %1, %2, m0, m1, m6
757 PEL_%2STORE%1 dstq, m0, m1
758 add dstq, dststrideq ; dst += dststride
759 add srcq, srcstrideq ; src += srcstride
760 dec heightd ; cmp height
761 jnz .loop ; height loop
764 cglobal hevc_put_hevc_bi_epel_h%1_%2, 7, 8, 11, dst, dststride, src, srcstride, src2, height, mx, rfilter
765 movdqa m6, [pw_bi_%2]
766 EPEL_FILTER %2, mx, m4, m5
768 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
769 EPEL_COMPUTE %2, %1, m4, m5, 1
770 SIMPLE_BILOAD %1, src2q, m2, m3
771 BI_COMPUTE %1, %2, m0, m1, m2, m3, m6, 1
772 PEL_%2STORE%1 dstq, m0, m1
773 add dstq, dststrideq ; dst += dststride
774 add srcq, srcstrideq ; src += srcstride
775 add src2q, 2*MAX_PB_SIZE ; src += srcstride
776 dec heightd ; cmp height
777 jnz .loop ; height loop
780 ; ******************************
781 ; void put_hevc_epel_v(int16_t *dst, ptrdiff_t dststride,
782 ; uint8_t *_src, ptrdiff_t _srcstride,
783 ; int width, int height, int mx, int my,
785 ; ******************************
787 cglobal hevc_put_hevc_epel_v%1_%2, 6, 7, 11, dst, src, srcstride, height, r3src, my, rfilter
788 lea r3srcq, [srcstrideq*3]
790 EPEL_FILTER %2, my, m4, m5
792 EPEL_LOAD %2, srcq, srcstride, %1
793 EPEL_COMPUTE %2, %1, m4, m5, 1
794 PEL_10STORE%1 dstq, m0, m1
795 LOOP_END dst, src, srcstride
798 cglobal hevc_put_hevc_uni_epel_v%1_%2, 7, 8, 11, dst, dststride, src, srcstride, height, r3src, my, rfilter
799 lea r3srcq, [srcstrideq*3]
802 EPEL_FILTER %2, my, m4, m5
804 EPEL_LOAD %2, srcq, srcstride, %1
805 EPEL_COMPUTE %2, %1, m4, m5
806 UNI_COMPUTE %1, %2, m0, m1, m6
807 PEL_%2STORE%1 dstq, m0, m1
808 add dstq, dststrideq ; dst += dststride
809 add srcq, srcstrideq ; src += srcstride
810 dec heightd ; cmp height
811 jnz .loop ; height loop
815 cglobal hevc_put_hevc_bi_epel_v%1_%2, 8, 9, 11, dst, dststride, src, srcstride, src2, height, r3src, my, rfilter
816 lea r3srcq, [srcstrideq*3]
817 movdqa m6, [pw_bi_%2]
819 EPEL_FILTER %2, my, m4, m5
821 EPEL_LOAD %2, srcq, srcstride, %1
822 EPEL_COMPUTE %2, %1, m4, m5, 1
823 SIMPLE_BILOAD %1, src2q, m2, m3
824 BI_COMPUTE %1, %2, m0, m1, m2, m3, m6, 1
825 PEL_%2STORE%1 dstq, m0, m1
826 add dstq, dststrideq ; dst += dststride
827 add srcq, srcstrideq ; src += srcstride
828 add src2q, 2*MAX_PB_SIZE ; src += srcstride
829 dec heightd ; cmp height
830 jnz .loop ; height loop
835 ; ******************************
836 ; void put_hevc_epel_hv(int16_t *dst, ptrdiff_t dststride,
837 ; uint8_t *_src, ptrdiff_t _srcstride,
838 ; int width, int height, int mx, int my)
839 ; ******************************
841 %macro HEVC_PUT_HEVC_EPEL_HV 2
842 cglobal hevc_put_hevc_epel_hv%1_%2, 6, 8, 16 , dst, src, srcstride, height, mx, my, r3src, rfilter
843 %assign %%stride ((%2 + 7)/8)
846 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
847 EPEL_COMPUTE %2, %1, m14, m15
848 %if (%1 > 8 && (%2 == 8))
853 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
854 EPEL_COMPUTE %2, %1, m14, m15
855 %if (%1 > 8 && (%2 == 8))
860 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
861 EPEL_COMPUTE %2, %1, m14, m15
862 %if (%1 > 8 && (%2 == 8))
868 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
869 EPEL_COMPUTE %2, %1, m14, m15
870 %if (%1 > 8 && (%2 == 8))
880 EPEL_COMPUTE 14, %1, m12, m13
881 %if (%1 > 8 && (%2 == 8))
883 punpcklwd m2, m10, m11
885 punpckhwd m3, m10, m11
886 EPEL_COMPUTE 14, %1, m12, m13, m4, m2, m8, m3
888 vinserti128 m2, m0, xm4, 1
889 vextracti128 xm3, m0, 1
890 vinserti128 m3, m4, xm3, 0
891 PEL_10STORE%1 dstq, m2, m3
893 PEL_10STORE%1 dstq, m0, m4
896 PEL_10STORE%1 dstq, m0, m1
901 %if (%1 > 8 && (%2 == 8))
906 LOOP_END dst, src, srcstride
909 cglobal hevc_put_hevc_uni_epel_hv%1_%2, 7, 9, 16 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter
910 %assign %%stride ((%2 + 7)/8)
913 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
914 EPEL_COMPUTE %2, %1, m14, m15
915 %if (%1 > 8 && (%2 == 8))
920 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
921 EPEL_COMPUTE %2, %1, m14, m15
922 %if (%1 > 8 && (%2 == 8))
927 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
928 EPEL_COMPUTE %2, %1, m14, m15
929 %if (%1 > 8 && (%2 == 8))
935 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
936 EPEL_COMPUTE %2, %1, m14, m15
937 %if (%1 > 8 && (%2 == 8))
947 EPEL_COMPUTE 14, %1, m12, m13
948 %if (%1 > 8 && (%2 == 8))
950 punpcklwd m2, m10, m11
952 punpckhwd m3, m10, m11
953 EPEL_COMPUTE 14, %1, m12, m13, m4, m2, m8, m3
954 UNI_COMPUTE %1, %2, m0, m4, [pw_%2]
956 UNI_COMPUTE %1, %2, m0, m1, [pw_%2]
958 PEL_%2STORE%1 dstq, m0, m1
962 %if (%1 > 8 && (%2 == 8))
967 add dstq, dststrideq ; dst += dststride
968 add srcq, srcstrideq ; src += srcstride
969 dec heightd ; cmp height
970 jnz .loop ; height loop
973 cglobal hevc_put_hevc_bi_epel_hv%1_%2, 8, 10, 16, dst, dststride, src, srcstride, src2, height, mx, my, r3src, rfilter
974 %assign %%stride ((%2 + 7)/8)
977 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
978 EPEL_COMPUTE %2, %1, m14, m15
979 %if (%1 > 8 && (%2 == 8))
984 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
985 EPEL_COMPUTE %2, %1, m14, m15
986 %if (%1 > 8 && (%2 == 8))
991 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
992 EPEL_COMPUTE %2, %1, m14, m15
993 %if (%1 > 8 && (%2 == 8))
999 EPEL_LOAD %2, srcq-%%stride, %%stride, %1
1000 EPEL_COMPUTE %2, %1, m14, m15
1001 %if (%1 > 8 && (%2 == 8))
1005 punpcklwd m0, m4, m5
1006 punpcklwd m2, m6, m7
1008 punpckhwd m1, m4, m5
1009 punpckhwd m3, m6, m7
1011 EPEL_COMPUTE 14, %1, m12, m13
1012 %if (%1 > 8 && (%2 == 8))
1013 punpcklwd m4, m8, m9
1014 punpcklwd m2, m10, m11
1015 punpckhwd m8, m8, m9
1016 punpckhwd m3, m10, m11
1017 EPEL_COMPUTE 14, %1, m12, m13, m4, m2, m8, m3
1018 SIMPLE_BILOAD %1, src2q, m8, m3
1020 vinserti128 m1, m8, xm3, 1
1021 vextracti128 xm8, m8, 1
1022 vinserti128 m2, m3, xm8, 0
1023 BI_COMPUTE %1, %2, m0, m4, m1, m2, [pw_bi_%2]
1025 BI_COMPUTE %1, %2, m0, m4, m8, m3, [pw_bi_%2]
1028 SIMPLE_BILOAD %1, src2q, m8, m9
1029 BI_COMPUTE %1, %2, m0, m1, m8, m9, [pw_bi_%2]
1031 PEL_%2STORE%1 dstq, m0, m4
1035 %if (%1 > 8 && (%2 == 8))
1040 add dstq, dststrideq ; dst += dststride
1041 add srcq, srcstrideq ; src += srcstride
1042 add src2q, 2*MAX_PB_SIZE ; src += srcstride
1043 dec heightd ; cmp height
1044 jnz .loop ; height loop
1048 ; ******************************
1049 ; void put_hevc_qpel_hX_X_X(int16_t *dst, ptrdiff_t dststride,
1050 ; uint8_t *_src, ptrdiff_t _srcstride,
1051 ; int width, int height, int mx, int my)
1052 ; ******************************
1054 %macro HEVC_PUT_HEVC_QPEL 2
1055 cglobal hevc_put_hevc_qpel_h%1_%2, 5, 6, 16, dst, src, srcstride, height, mx, rfilter
1058 QPEL_H_LOAD %2, srcq, %1, 10
1059 QPEL_COMPUTE %1, %2, 1
1063 PEL_10STORE%1 dstq, m0, m1
1064 LOOP_END dst, src, srcstride
1067 cglobal hevc_put_hevc_uni_qpel_h%1_%2, 6, 7, 16 , dst, dststride, src, srcstride, height, mx, rfilter
1071 QPEL_H_LOAD %2, srcq, %1, 10
1076 UNI_COMPUTE %1, %2, m0, m1, m9
1077 PEL_%2STORE%1 dstq, m0, m1
1078 add dstq, dststrideq ; dst += dststride
1079 add srcq, srcstrideq ; src += srcstride
1080 dec heightd ; cmp height
1081 jnz .loop ; height loop
1084 cglobal hevc_put_hevc_bi_qpel_h%1_%2, 7, 8, 16 , dst, dststride, src, srcstride, src2, height, mx, rfilter
1085 movdqa m9, [pw_bi_%2]
1088 QPEL_H_LOAD %2, srcq, %1, 10
1089 QPEL_COMPUTE %1, %2, 1
1093 SIMPLE_BILOAD %1, src2q, m10, m11
1094 BI_COMPUTE %1, %2, m0, m1, m10, m11, m9, 1
1095 PEL_%2STORE%1 dstq, m0, m1
1096 add dstq, dststrideq ; dst += dststride
1097 add srcq, srcstrideq ; src += srcstride
1098 add src2q, 2*MAX_PB_SIZE ; src += srcstride
1099 dec heightd ; cmp height
1100 jnz .loop ; height loop
1104 ; ******************************
1105 ; void put_hevc_qpel_vX_X_X(int16_t *dst, ptrdiff_t dststride,
1106 ; uint8_t *_src, ptrdiff_t _srcstride,
1107 ; int width, int height, int mx, int my)
1108 ; ******************************
1110 cglobal hevc_put_hevc_qpel_v%1_%2, 6, 8, 16, dst, src, srcstride, height, r3src, my, rfilter
1111 lea r3srcq, [srcstrideq*3]
1114 QPEL_V_LOAD %2, srcq, srcstride, %1, r7
1115 QPEL_COMPUTE %1, %2, 1
1119 PEL_10STORE%1 dstq, m0, m1
1120 LOOP_END dst, src, srcstride
1123 cglobal hevc_put_hevc_uni_qpel_v%1_%2, 7, 9, 16, dst, dststride, src, srcstride, height, r3src, my, rfilter
1125 lea r3srcq, [srcstrideq*3]
1128 QPEL_V_LOAD %2, srcq, srcstride, %1, r8
1133 UNI_COMPUTE %1, %2, m0, m1, m9
1134 PEL_%2STORE%1 dstq, m0, m1
1135 add dstq, dststrideq ; dst += dststride
1136 add srcq, srcstrideq ; src += srcstride
1137 dec heightd ; cmp height
1138 jnz .loop ; height loop
1141 cglobal hevc_put_hevc_bi_qpel_v%1_%2, 8, 10, 16, dst, dststride, src, srcstride, src2, height, r3src, my, rfilter
1142 movdqa m9, [pw_bi_%2]
1143 lea r3srcq, [srcstrideq*3]
1146 QPEL_V_LOAD %2, srcq, srcstride, %1, r9
1147 QPEL_COMPUTE %1, %2, 1
1151 SIMPLE_BILOAD %1, src2q, m10, m11
1152 BI_COMPUTE %1, %2, m0, m1, m10, m11, m9, 1
1153 PEL_%2STORE%1 dstq, m0, m1
1154 add dstq, dststrideq ; dst += dststride
1155 add srcq, srcstrideq ; src += srcstride
1156 add src2q, 2*MAX_PB_SIZE ; src += srcstride
1157 dec heightd ; cmp height
1158 jnz .loop ; height loop
1163 ; ******************************
1164 ; void put_hevc_qpel_hvX_X(int16_t *dst, ptrdiff_t dststride,
1165 ; uint8_t *_src, ptrdiff_t _srcstride,
1166 ; int height, int mx, int my)
1167 ; ******************************
1168 %macro HEVC_PUT_HEVC_QPEL_HV 2
1169 cglobal hevc_put_hevc_qpel_hv%1_%2, 6, 8, 16, dst, src, srcstride, height, mx, my, r3src, rfilter
1177 shl mxq, %%shift ; multiply by 32
1178 shl myq, %%shift ; multiply by 32
1179 lea r3srcq, [srcstrideq*3]
1181 QPEL_H_LOAD %2, srcq, %1, 15
1182 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1184 add srcq, srcstrideq
1185 QPEL_H_LOAD %2, srcq, %1, 15
1186 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1188 add srcq, srcstrideq
1189 QPEL_H_LOAD %2, srcq, %1, 15
1190 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1192 add srcq, srcstrideq
1193 QPEL_H_LOAD %2, srcq, %1, 15
1194 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1196 add srcq, srcstrideq
1197 QPEL_H_LOAD %2, srcq, %1, 15
1198 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1200 add srcq, srcstrideq
1201 QPEL_H_LOAD %2, srcq, %1, 15
1202 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1204 add srcq, srcstrideq
1205 QPEL_H_LOAD %2, srcq, %1, 15
1206 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1208 add srcq, srcstrideq
1210 QPEL_H_LOAD %2, srcq, %1, 15
1211 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1213 punpcklwd m0, m8, m9
1214 punpcklwd m2, m10, m11
1215 punpcklwd m4, m12, m13
1216 punpcklwd m6, m14, m15
1218 punpckhwd m1, m8, m9
1219 punpckhwd m3, m10, m11
1220 punpckhwd m5, m12, m13
1221 punpckhwd m7, m14, m15
1223 QPEL_HV_COMPUTE %1, 14, my, ackssdw
1224 PEL_10STORE%1 dstq, m0, m1
1242 LOOP_END dst, src, srcstride
1245 cglobal hevc_put_hevc_uni_qpel_hv%1_%2, 7, 9, 16 , dst, dststride, src, srcstride, height, mx, my, r3src, rfilter
1253 shl mxq, %%shift ; multiply by 32
1254 shl myq, %%shift ; multiply by 32
1255 lea r3srcq, [srcstrideq*3]
1257 QPEL_H_LOAD %2, srcq, %1, 15
1258 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1260 add srcq, srcstrideq
1261 QPEL_H_LOAD %2, srcq, %1, 15
1262 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1264 add srcq, srcstrideq
1265 QPEL_H_LOAD %2, srcq, %1, 15
1266 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1268 add srcq, srcstrideq
1269 QPEL_H_LOAD %2, srcq, %1, 15
1270 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1272 add srcq, srcstrideq
1273 QPEL_H_LOAD %2, srcq, %1, 15
1274 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1276 add srcq, srcstrideq
1277 QPEL_H_LOAD %2, srcq, %1, 15
1278 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1280 add srcq, srcstrideq
1281 QPEL_H_LOAD %2, srcq, %1, 15
1282 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1284 add srcq, srcstrideq
1286 QPEL_H_LOAD %2, srcq, %1, 15
1287 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1289 punpcklwd m0, m8, m9
1290 punpcklwd m2, m10, m11
1291 punpcklwd m4, m12, m13
1292 punpcklwd m6, m14, m15
1294 punpckhwd m1, m8, m9
1295 punpckhwd m3, m10, m11
1296 punpckhwd m5, m12, m13
1297 punpckhwd m7, m14, m15
1299 QPEL_HV_COMPUTE %1, 14, my, ackusdw
1300 UNI_COMPUTE %1, %2, m0, m1, [pw_%2]
1301 PEL_%2STORE%1 dstq, m0, m1
1320 add dstq, dststrideq ; dst += dststride
1321 add srcq, srcstrideq ; src += srcstride
1322 dec heightd ; cmp height
1323 jnz .loop ; height loop
1326 cglobal hevc_put_hevc_bi_qpel_hv%1_%2, 8, 10, 16, dst, dststride, src, srcstride, src2, height, mx, my, r3src, rfilter
1334 shl mxq, %%shift ; multiply by 32
1335 shl myq, %%shift ; multiply by 32
1336 lea r3srcq, [srcstrideq*3]
1338 QPEL_H_LOAD %2, srcq, %1, 15
1339 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1341 add srcq, srcstrideq
1342 QPEL_H_LOAD %2, srcq, %1, 15
1343 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1345 add srcq, srcstrideq
1346 QPEL_H_LOAD %2, srcq, %1, 15
1347 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1349 add srcq, srcstrideq
1350 QPEL_H_LOAD %2, srcq, %1, 15
1351 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1353 add srcq, srcstrideq
1354 QPEL_H_LOAD %2, srcq, %1, 15
1355 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1357 add srcq, srcstrideq
1358 QPEL_H_LOAD %2, srcq, %1, 15
1359 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1361 add srcq, srcstrideq
1362 QPEL_H_LOAD %2, srcq, %1, 15
1363 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1365 add srcq, srcstrideq
1367 QPEL_H_LOAD %2, srcq, %1, 15
1368 QPEL_HV_COMPUTE %1, %2, mx, ackssdw
1370 punpcklwd m0, m8, m9
1371 punpcklwd m2, m10, m11
1372 punpcklwd m4, m12, m13
1373 punpcklwd m6, m14, m15
1375 punpckhwd m1, m8, m9
1376 punpckhwd m3, m10, m11
1377 punpckhwd m5, m12, m13
1378 punpckhwd m7, m14, m15
1380 QPEL_HV_COMPUTE %1, 14, my, ackssdw
1381 SIMPLE_BILOAD %1, src2q, m8, m9 ;m9 not used in this case
1382 BI_COMPUTE %1, %2, m0, m1, m8, m9, [pw_bi_%2]
1383 PEL_%2STORE%1 dstq, m0, m1
1402 add dstq, dststrideq ; dst += dststride
1403 add srcq, srcstrideq ; src += srcstride
1404 add src2q, 2*MAX_PB_SIZE ; src += srcstride
1405 dec heightd ; cmp height
1406 jnz .loop ; height loop
1410 %macro WEIGHTING_FUNCS 2
1411 %if WIN64 || ARCH_X86_32
1412 cglobal hevc_put_hevc_uni_w%1_%2, 4, 5, 7, dst, dststride, src, srcstride, height, denom, wx, ox
1416 cglobal hevc_put_hevc_uni_w%1_%2, 6, 6, 7, dst, dststride, src, srcstride, height, denom, wx, ox
1417 %define SHIFT denomd
1419 lea SHIFT, [SHIFT+14-%2] ; shift = 14 - bitd + denom
1424 movd m4, SHIFT ; shift
1437 shl SHIFT, %2-8 ; ox << (bitd - 8)
1441 %if WIN64 || ARCH_X86_32
1445 SIMPLE_LOAD %1, 10, srcq, m0
1455 punpckhwd m1, m0, m6
1468 CLIPW m0, [pb_0], [max_pixels_%2]
1470 PEL_%2STORE%1 dstq, m0, m1
1471 add dstq, dststrideq ; dst += dststride
1472 add srcq, 2*MAX_PB_SIZE ; src += srcstride
1473 dec heightd ; cmp height
1474 jnz .loop ; height loop
1477 cglobal hevc_put_hevc_bi_w%1_%2, 5, 7, 10, dst, dststride, src, srcstride, src2, height, denom, wx0, wx1, ox0, ox1
1483 lea r6d, [r6d+14-%2] ; shift = 14 - bitd + denom
1485 movd m0, r6d ; shift
1494 movd m5, r6d ; shift+1
1500 shl r6d, %2-8 ; ox << (bitd - 8)
1503 movd m4, r6d ; offset
1509 SIMPLE_LOAD %1, 10, srcq, m0
1510 SIMPLE_LOAD %1, 10, src2q, m8
1524 punpckhwd m1, m0, m6
1526 punpckhwd m9, m8, m7
1539 CLIPW m0, [pb_0], [max_pixels_%2]
1541 PEL_%2STORE%1 dstq, m0, m1
1542 add dstq, dststrideq ; dst += dststride
1543 add srcq, 2*MAX_PB_SIZE ; src += srcstride
1544 add src2q, 2*MAX_PB_SIZE ; src2 += srcstride
1545 dec r6d ; cmp height
1546 jnz .loop ; height loop
1550 INIT_XMM sse4 ; adds ff_ and _sse4 to function name
1552 WEIGHTING_FUNCS 2, 8
1553 WEIGHTING_FUNCS 4, 8
1554 WEIGHTING_FUNCS 6, 8
1555 WEIGHTING_FUNCS 8, 8
1557 WEIGHTING_FUNCS 2, 10
1558 WEIGHTING_FUNCS 4, 10
1559 WEIGHTING_FUNCS 6, 10
1560 WEIGHTING_FUNCS 8, 10
1562 WEIGHTING_FUNCS 2, 12
1563 WEIGHTING_FUNCS 4, 12
1564 WEIGHTING_FUNCS 6, 12
1565 WEIGHTING_FUNCS 8, 12
1567 HEVC_PUT_HEVC_PEL_PIXELS 2, 8
1568 HEVC_PUT_HEVC_PEL_PIXELS 4, 8
1569 HEVC_PUT_HEVC_PEL_PIXELS 6, 8
1570 HEVC_PUT_HEVC_PEL_PIXELS 8, 8
1571 HEVC_PUT_HEVC_PEL_PIXELS 12, 8
1572 HEVC_PUT_HEVC_PEL_PIXELS 16, 8
1574 HEVC_PUT_HEVC_PEL_PIXELS 2, 10
1575 HEVC_PUT_HEVC_PEL_PIXELS 4, 10
1576 HEVC_PUT_HEVC_PEL_PIXELS 6, 10
1577 HEVC_PUT_HEVC_PEL_PIXELS 8, 10
1579 HEVC_PUT_HEVC_PEL_PIXELS 2, 12
1580 HEVC_PUT_HEVC_PEL_PIXELS 4, 12
1581 HEVC_PUT_HEVC_PEL_PIXELS 6, 12
1582 HEVC_PUT_HEVC_PEL_PIXELS 8, 12
1584 HEVC_PUT_HEVC_EPEL 2, 8
1585 HEVC_PUT_HEVC_EPEL 4, 8
1586 HEVC_PUT_HEVC_EPEL 6, 8
1587 HEVC_PUT_HEVC_EPEL 8, 8
1588 HEVC_PUT_HEVC_EPEL 12, 8
1589 HEVC_PUT_HEVC_EPEL 16, 8
1592 HEVC_PUT_HEVC_EPEL 2, 10
1593 HEVC_PUT_HEVC_EPEL 4, 10
1594 HEVC_PUT_HEVC_EPEL 6, 10
1595 HEVC_PUT_HEVC_EPEL 8, 10
1597 HEVC_PUT_HEVC_EPEL 2, 12
1598 HEVC_PUT_HEVC_EPEL 4, 12
1599 HEVC_PUT_HEVC_EPEL 6, 12
1600 HEVC_PUT_HEVC_EPEL 8, 12
1602 HEVC_PUT_HEVC_EPEL_HV 2, 8
1603 HEVC_PUT_HEVC_EPEL_HV 4, 8
1604 HEVC_PUT_HEVC_EPEL_HV 6, 8
1605 HEVC_PUT_HEVC_EPEL_HV 8, 8
1606 HEVC_PUT_HEVC_EPEL_HV 16, 8
1608 HEVC_PUT_HEVC_EPEL_HV 2, 10
1609 HEVC_PUT_HEVC_EPEL_HV 4, 10
1610 HEVC_PUT_HEVC_EPEL_HV 6, 10
1611 HEVC_PUT_HEVC_EPEL_HV 8, 10
1613 HEVC_PUT_HEVC_EPEL_HV 2, 12
1614 HEVC_PUT_HEVC_EPEL_HV 4, 12
1615 HEVC_PUT_HEVC_EPEL_HV 6, 12
1616 HEVC_PUT_HEVC_EPEL_HV 8, 12
1618 HEVC_PUT_HEVC_QPEL 4, 8
1619 HEVC_PUT_HEVC_QPEL 8, 8
1620 HEVC_PUT_HEVC_QPEL 12, 8
1621 HEVC_PUT_HEVC_QPEL 16, 8
1623 HEVC_PUT_HEVC_QPEL 4, 10
1624 HEVC_PUT_HEVC_QPEL 8, 10
1626 HEVC_PUT_HEVC_QPEL 4, 12
1627 HEVC_PUT_HEVC_QPEL 8, 12
1629 HEVC_PUT_HEVC_QPEL_HV 2, 8
1630 HEVC_PUT_HEVC_QPEL_HV 4, 8
1631 HEVC_PUT_HEVC_QPEL_HV 6, 8
1632 HEVC_PUT_HEVC_QPEL_HV 8, 8
1634 HEVC_PUT_HEVC_QPEL_HV 2, 10
1635 HEVC_PUT_HEVC_QPEL_HV 4, 10
1636 HEVC_PUT_HEVC_QPEL_HV 6, 10
1637 HEVC_PUT_HEVC_QPEL_HV 8, 10
1639 HEVC_PUT_HEVC_QPEL_HV 2, 12
1640 HEVC_PUT_HEVC_QPEL_HV 4, 12
1641 HEVC_PUT_HEVC_QPEL_HV 6, 12
1642 HEVC_PUT_HEVC_QPEL_HV 8, 12
1644 %if HAVE_AVX2_EXTERNAL
1645 INIT_YMM avx2 ; adds ff_ and _avx2 to function name & enables 256b registers : m0 for 256b, xm0 for 128b. cpuflag(avx2) = 1 / notcpuflag(avx) = 0
1647 HEVC_PUT_HEVC_PEL_PIXELS 32, 8
1648 HEVC_PUT_HEVC_PEL_PIXELS 16, 10
1650 HEVC_PUT_HEVC_EPEL 32, 8
1651 HEVC_PUT_HEVC_EPEL 16, 10
1653 HEVC_PUT_HEVC_EPEL_HV 16, 10
1654 HEVC_PUT_HEVC_EPEL_HV 32, 8
1656 HEVC_PUT_HEVC_QPEL 32, 8
1658 HEVC_PUT_HEVC_QPEL 16, 10
1660 HEVC_PUT_HEVC_QPEL_HV 16, 10
1663 %endif ; ARCH_X86_64