1 ;******************************************************************************
2 ;* Copyright (c) 2012 Michael Niedermayer
4 ;* This file is part of FFmpeg.
6 ;* FFmpeg is free software; you can redistribute it and/or
7 ;* modify it under the terms of the GNU Lesser General Public
8 ;* License as published by the Free Software Foundation; either
9 ;* version 2.1 of the License, or (at your option) any later version.
11 ;* FFmpeg is distributed in the hope that it will be useful,
12 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 ;* Lesser General Public License for more details.
16 ;* You should have received a copy of the GNU Lesser General Public
17 ;* License along with FFmpeg; if not, write to the Free Software
18 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 ;******************************************************************************
21 %include "libavutil/x86/x86util.asm"
24 flt2pm31: times 8 dd 4.6566129e-10
25 flt2p31 : times 8 dd 2147483648.0
26 flt2p15 : times 8 dd 32768.0
28 word_unpack_shuf : db 0, 1, 4, 5, 8, 9,12,13, 2, 3, 6, 7,10,11,14,15
33 ;to, from, a/u, log2_outsize, log_intsize, const
35 cglobal pack_2ch_%2_to_%1_%3, 3, 4, 6, dst, src, len, src2
36 mov src2q , [srcq+gprsize]
41 jne pack_2ch_%2_to_%1_u_int %+ SUFFIX
43 jne pack_2ch_%2_to_%1_u_int %+ SUFFIX
45 jne pack_2ch_%2_to_%1_u_int %+ SUFFIX
47 pack_2ch_%2_to_%1_u_int %+ SUFFIX
49 lea srcq , [srcq + (1<<%5)*lenq]
50 lea src2q, [src2q + (1<<%5)*lenq]
51 lea dstq , [dstq + (2<<%4)*lenq]
56 mov%3 m0, [ srcq +(1<<%5)*lenq]
58 mov%3 m2, [ src2q+(1<<%5)*lenq]
68 mov%3 m0, [ srcq +(1<<%5)*lenq]
69 mov%3 m1, [mmsize + srcq +(1<<%5)*lenq]
70 mov%3 m2, [ src2q+(1<<%5)*lenq]
71 mov%3 m3, [mmsize + src2q+(1<<%5)*lenq]
78 mov%3 [ dstq+(2<<%4)*lenq], m0
79 mov%3 [ mmsize + dstq+(2<<%4)*lenq], m1
81 mov%3 [2*mmsize + dstq+(2<<%4)*lenq], m2
82 mov%3 [3*mmsize + dstq+(2<<%4)*lenq], m3
83 add lenq, 4*mmsize/(2<<%4)
85 add lenq, 2*mmsize/(2<<%4)
92 cglobal unpack_2ch_%2_to_%1_%3, 3, 4, 7, dst, src, len, dst2
93 mov dst2q , [dstq+gprsize]
98 jne unpack_2ch_%2_to_%1_u_int %+ SUFFIX
100 jne unpack_2ch_%2_to_%1_u_int %+ SUFFIX
102 jne unpack_2ch_%2_to_%1_u_int %+ SUFFIX
104 unpack_2ch_%2_to_%1_u_int %+ SUFFIX
106 lea srcq , [srcq + (2<<%5)*lenq]
107 lea dstq , [dstq + (1<<%4)*lenq]
108 lea dst2q, [dst2q + (1<<%4)*lenq]
111 mova m6, [word_unpack_shuf]
113 mov%3 m0, [ srcq +(2<<%5)*lenq]
114 mov%3 m2, [ mmsize + srcq +(2<<%5)*lenq]
116 %ifidn SUFFIX, _ssse3
137 shufps m0, m2, 10001000b
138 shufps m1, m2, 11011101b
141 mov%3 m2, [2*mmsize + srcq +(2<<%5)*lenq]
143 mov%3 m4, [3*mmsize + srcq +(2<<%5)*lenq]
144 shufps m2, m4, 10001000b
145 shufps m3, m4, 11011101b
149 mov%3 [ dstq+(1<<%4)*lenq], m0
151 mov%3 [ dst2q+(1<<%4)*lenq], m2
152 mov%3 [ mmsize + dstq+(1<<%4)*lenq], m1
153 mov%3 [ mmsize + dst2q+(1<<%4)*lenq], m3
154 add lenq, 2*mmsize/(1<<%4)
156 mov%3 [ dst2q+(1<<%4)*lenq], m1
157 add lenq, mmsize/(1<<%4)
164 cglobal %2_to_%1_%3, 3, 3, 6, dst, src, len
169 jne %2_to_%1_u_int %+ SUFFIX
171 jne %2_to_%1_u_int %+ SUFFIX
173 %2_to_%1_u_int %+ SUFFIX
175 lea srcq , [srcq + (1<<%5)*lenq]
176 lea dstq , [dstq + (1<<%4)*lenq]
180 mov%3 m0, [ srcq +(1<<%5)*lenq]
181 mov%3 m1, [ mmsize + srcq +(1<<%5)*lenq]
183 mov%3 m2, [2*mmsize + srcq +(1<<%5)*lenq]
184 mov%3 m3, [3*mmsize + srcq +(1<<%5)*lenq]
187 mov%3 [ dstq+(1<<%4)*lenq], m0
188 mov%3 [ mmsize + dstq+(1<<%4)*lenq], m1
190 mov%3 [2*mmsize + dstq+(1<<%4)*lenq], m2
191 mov%3 [3*mmsize + dstq+(1<<%4)*lenq], m3
192 add lenq, 4*mmsize/(1<<%4)
194 add lenq, 2*mmsize/(1<<%4)
206 cglobal pack_6ch_%2_to_%1_%3, 2,8,7, dst, src, src1, src2, src3, src4, src5, len
210 %define lend dword r2m
212 mov src1q, [srcq+1*gprsize]
213 mov src2q, [srcq+2*gprsize]
214 mov src3q, [srcq+3*gprsize]
215 mov src4q, [srcq+4*gprsize]
216 mov src5q, [srcq+5*gprsize]
221 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
223 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
225 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
227 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
229 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
231 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
233 jne pack_6ch_%2_to_%1_u_int %+ SUFFIX
235 pack_6ch_%2_to_%1_u_int %+ SUFFIX
245 mov%3 m1, [srcq+src1q]
246 mov%3 m2, [srcq+src2q]
247 mov%3 m3, [srcq+src3q]
248 mov%3 m4, [srcq+src4q]
249 mov%3 m5, [srcq+src5q]
256 blendps m6, m4, m0, 1100b
265 blendps m2, m5, m1, 1100b
278 mov %+ %3 %+ ps [dstq ], m0
279 mov %+ %3 %+ ps [dstq+16], m6
280 mov %+ %3 %+ ps [dstq+32], m4
281 mov %+ %3 %+ ps [dstq+48], m1
282 mov %+ %3 %+ ps [dstq+64], m2
283 mov %+ %3 %+ ps [dstq+80], m5
285 SBUTTERFLY dq, 0, 1, 6
286 SBUTTERFLY dq, 2, 3, 6
287 SBUTTERFLY dq, 4, 5, 6
308 %macro UNPACK_6CH 5-7
309 cglobal unpack_6ch_%2_to_%1_%3, 2, 8, 8, dst, src, dst1, dst2, dst3, dst4, dst5, len
313 %define lend dword r2m
315 mov dst1q, [dstq+1*gprsize]
316 mov dst2q, [dstq+2*gprsize]
317 mov dst3q, [dstq+3*gprsize]
318 mov dst4q, [dstq+4*gprsize]
319 mov dst5q, [dstq+5*gprsize]
324 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
326 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
328 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
330 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
332 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
334 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
336 jne unpack_6ch_%2_to_%1_u_int %+ SUFFIX
338 unpack_6ch_%2_to_%1_u_int %+ SUFFIX
367 mov %+ %3 %+ ps [dstq ], m0
368 mov %+ %3 %+ ps [dstq+dst1q], m1
369 mov %+ %3 %+ ps [dstq+dst2q], m2
370 mov %+ %3 %+ ps [dstq+dst3q], m3
371 mov %+ %3 %+ ps [dstq+dst4q], m4
372 mov %+ %3 %+ ps [dstq+dst5q], m5
381 %define PACK_8CH_GPRS (10 * ARCH_X86_64) + ((6 + HAVE_ALIGNED_STACK) * ARCH_X86_32)
384 cglobal pack_8ch_%2_to_%1_%3, 2,PACK_8CH_GPRS,10, ARCH_X86_32*48, dst, src, len, src1, src2, src3, src4, src5, src6, src7
387 DEFINE_ARGS dst, src, src2, src3, src4, src5, src6
388 %define lend dword r2m
390 %define src1m dword [rsp+32]
391 %if HAVE_ALIGNED_STACK == 0
392 DEFINE_ARGS dst, src, src2, src3, src5, src6
394 %define src4m dword [rsp+36]
397 %define src7m dword [rsp+40]
400 mov src7q, [srcq+7*gprsize]
401 mov src6q, [srcq+6*gprsize]
405 mov src5q, [srcq+5*gprsize]
406 mov src4q, [srcq+4*gprsize]
407 mov src3q, [srcq+3*gprsize]
408 %if ARCH_X86_32 && HAVE_ALIGNED_STACK == 0
411 mov src2q, [srcq+2*gprsize]
412 mov src1q, [srcq+1*gprsize]
420 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
422 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
424 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
426 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
428 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
429 %if ARCH_X86_32 && HAVE_ALIGNED_STACK == 0
434 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
436 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
438 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
444 jne pack_8ch_%2_to_%1_u_int %+ SUFFIX
446 pack_8ch_%2_to_%1_u_int %+ SUFFIX
451 %if ARCH_X86_64 || HAVE_ALIGNED_STACK
470 %define m9 [flt2pm31]
475 mov%3 m1, [srcq+src1q]
476 mov%3 m2, [srcq+src2q]
477 %if ARCH_X86_32 && HAVE_ALIGNED_STACK == 0
480 mov%3 m3, [srcq+src3q]
481 mov%3 m4, [srcq+src4q]
482 mov%3 m5, [srcq+src5q]
486 mov%3 m6, [srcq+src6q]
487 mov%3 m7, [srcq+src7q]
490 TRANSPOSE8x4D 0, 1, 2, 3, 4, 5, 6, 7, 8
501 TRANSPOSE8x4D 0, 1, 2, 3, 4, 5, 6, 7, [rsp], [rsp+16], 1
531 %macro INT16_TO_INT32_N 6
543 %macro INT32_TO_INT16_N 6
553 %macro INT32_TO_FLOAT_INIT 6
556 %macro INT32_TO_FLOAT_N 6
563 %macro FLOAT_TO_INT32_INIT 6
566 %macro FLOAT_TO_INT32_N 6
577 %macro INT16_TO_FLOAT_INIT 6
580 %macro INT16_TO_FLOAT_N 6
581 INT16_TO_INT32_N %1,%2,%3,%4,%5,%6
592 %macro FLOAT_TO_INT16_INIT 6
595 %macro FLOAT_TO_INT16_N 6
612 CONV int32, int16, u, 2, 1, INT16_TO_INT32_N, NOP_N
613 CONV int32, int16, a, 2, 1, INT16_TO_INT32_N, NOP_N
614 CONV int16, int32, u, 1, 2, INT32_TO_INT16_N, NOP_N
615 CONV int16, int32, a, 1, 2, INT32_TO_INT16_N, NOP_N
617 PACK_6CH float, float, u, 2, 2, NOP_N, NOP_N
618 PACK_6CH float, float, a, 2, 2, NOP_N, NOP_N
621 PACK_6CH float, float, u, 2, 2, NOP_N, NOP_N
622 PACK_6CH float, float, a, 2, 2, NOP_N, NOP_N
624 UNPACK_6CH float, float, u, 2, 2, NOP_N, NOP_N
625 UNPACK_6CH float, float, a, 2, 2, NOP_N, NOP_N
628 CONV int32, int16, u, 2, 1, INT16_TO_INT32_N, NOP_N
629 CONV int32, int16, a, 2, 1, INT16_TO_INT32_N, NOP_N
630 CONV int16, int32, u, 1, 2, INT32_TO_INT16_N, NOP_N
631 CONV int16, int32, a, 1, 2, INT32_TO_INT16_N, NOP_N
633 PACK_2CH int16, int16, u, 1, 1, NOP_N, NOP_N
634 PACK_2CH int16, int16, a, 1, 1, NOP_N, NOP_N
635 PACK_2CH int32, int32, u, 2, 2, NOP_N, NOP_N
636 PACK_2CH int32, int32, a, 2, 2, NOP_N, NOP_N
637 PACK_2CH int32, int16, u, 2, 1, INT16_TO_INT32_N, NOP_N
638 PACK_2CH int32, int16, a, 2, 1, INT16_TO_INT32_N, NOP_N
639 PACK_2CH int16, int32, u, 1, 2, INT32_TO_INT16_N, NOP_N
640 PACK_2CH int16, int32, a, 1, 2, INT32_TO_INT16_N, NOP_N
642 UNPACK_2CH int16, int16, u, 1, 1, NOP_N, NOP_N
643 UNPACK_2CH int16, int16, a, 1, 1, NOP_N, NOP_N
644 UNPACK_2CH int32, int32, u, 2, 2, NOP_N, NOP_N
645 UNPACK_2CH int32, int32, a, 2, 2, NOP_N, NOP_N
646 UNPACK_2CH int32, int16, u, 2, 1, INT16_TO_INT32_N, NOP_N
647 UNPACK_2CH int32, int16, a, 2, 1, INT16_TO_INT32_N, NOP_N
648 UNPACK_2CH int16, int32, u, 1, 2, INT32_TO_INT16_N, NOP_N
649 UNPACK_2CH int16, int32, a, 1, 2, INT32_TO_INT16_N, NOP_N
651 CONV float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
652 CONV float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
653 CONV int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
654 CONV int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
655 CONV float, int16, u, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
656 CONV float, int16, a, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
657 CONV int16, float, u, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
658 CONV int16, float, a, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
660 PACK_2CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
661 PACK_2CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
662 PACK_2CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
663 PACK_2CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
664 PACK_2CH float, int16, u, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
665 PACK_2CH float, int16, a, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
666 PACK_2CH int16, float, u, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
667 PACK_2CH int16, float, a, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
669 UNPACK_2CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
670 UNPACK_2CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
671 UNPACK_2CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
672 UNPACK_2CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
673 UNPACK_2CH float, int16, u, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
674 UNPACK_2CH float, int16, a, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
675 UNPACK_2CH int16, float, u, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
676 UNPACK_2CH int16, float, a, 1, 2, FLOAT_TO_INT16_N, FLOAT_TO_INT16_INIT
678 PACK_6CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
679 PACK_6CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
680 PACK_6CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
681 PACK_6CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
683 UNPACK_6CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
684 UNPACK_6CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
685 UNPACK_6CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
686 UNPACK_6CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
688 PACK_8CH float, float, u, 2, 2, NOP_N, NOP_N
689 PACK_8CH float, float, a, 2, 2, NOP_N, NOP_N
691 PACK_8CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
692 PACK_8CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
693 PACK_8CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
694 PACK_8CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
697 UNPACK_2CH int16, int16, u, 1, 1, NOP_N, NOP_N
698 UNPACK_2CH int16, int16, a, 1, 1, NOP_N, NOP_N
699 UNPACK_2CH int32, int16, u, 2, 1, INT16_TO_INT32_N, NOP_N
700 UNPACK_2CH int32, int16, a, 2, 1, INT16_TO_INT32_N, NOP_N
701 UNPACK_2CH float, int16, u, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
702 UNPACK_2CH float, int16, a, 2, 1, INT16_TO_FLOAT_N, INT16_TO_FLOAT_INIT
704 %if HAVE_AVX_EXTERNAL
706 PACK_6CH float, float, u, 2, 2, NOP_N, NOP_N
707 PACK_6CH float, float, a, 2, 2, NOP_N, NOP_N
709 UNPACK_6CH float, float, u, 2, 2, NOP_N, NOP_N
710 UNPACK_6CH float, float, a, 2, 2, NOP_N, NOP_N
712 PACK_6CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
713 PACK_6CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
714 PACK_6CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
715 PACK_6CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
717 UNPACK_6CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
718 UNPACK_6CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
719 UNPACK_6CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
720 UNPACK_6CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
722 PACK_8CH float, float, u, 2, 2, NOP_N, NOP_N
723 PACK_8CH float, float, a, 2, 2, NOP_N, NOP_N
725 PACK_8CH float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
726 PACK_8CH float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
727 PACK_8CH int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
728 PACK_8CH int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
731 CONV float, int32, u, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
732 CONV float, int32, a, 2, 2, INT32_TO_FLOAT_N, INT32_TO_FLOAT_INIT
735 %if HAVE_AVX2_EXTERNAL
737 CONV int32, float, u, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT
738 CONV int32, float, a, 2, 2, FLOAT_TO_INT32_N, FLOAT_TO_INT32_INIT