1 ;******************************************************************************
2 ;* x86-optimized horizontal/vertical line scaling functions
3 ;* Copyright (c) 2011 Ronald S. Bultje <rsbultje@gmail.com>
4 ;* Kieran Kunhya <kieran@kunhya.com>
6 ;* This file is part of Libav.
8 ;* Libav is free software; you can redistribute it and/or
9 ;* modify it under the terms of the GNU Lesser General Public
10 ;* License as published by the Free Software Foundation; either
11 ;* version 2.1 of the License, or (at your option) any later version.
13 ;* Libav is distributed in the hope that it will be useful,
14 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 ;* Lesser General Public License for more details.
18 ;* You should have received a copy of the GNU Lesser General Public
19 ;* License along with Libav; if not, write to the Free Software
20 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 ;******************************************************************************
24 %include "x86util.asm"
28 max_19bit_int: times 4 dd 0x7ffff
29 max_19bit_flt: times 4 dd 524287.0
30 minshort: times 8 dw 0x8000
31 unicoeff: times 4 dd 0x20000000
32 yuv2yuvX_16_start: times 4 dd 0x4000 - 0x40000000
33 yuv2yuvX_10_start: times 4 dd 0x10000
34 yuv2yuvX_9_start: times 4 dd 0x20000
35 yuv2yuvX_10_upper: times 8 dw 0x3ff
36 yuv2yuvX_9_upper: times 8 dw 0x1ff
38 pd_4min0x40000:times 4 dd 4 - (0x40000)
41 pw_512: times 8 dw 512
42 pw_1024: times 8 dw 1024
46 ;-----------------------------------------------------------------------------
47 ; horizontal line scaling
49 ; void hscale<source_width>to<intermediate_nbits>_<filterSize>_<opt>
50 ; (SwsContext *c, int{16,32}_t *dst,
51 ; int dstW, const uint{8,16}_t *src,
52 ; const int16_t *filter,
53 ; const int16_t *filterPos, int filterSize);
55 ; Scale one horizontal line. Input is either 8-bits width or 16-bits width
56 ; ($source_width can be either 8, 9, 10 or 16, difference is whether we have to
57 ; downscale before multiplying). Filter is 14-bits. Output is either 15bits
58 ; (in int16_t) or 19bits (in int32_t), as given in $intermediate_nbits. Each
59 ; output pixel is generated from $filterSize input pixels, the position of
60 ; the first pixel is given in filterPos[nOutputPixel].
61 ;-----------------------------------------------------------------------------
63 ; SCALE_FUNC source_width, intermediate_nbits, filtersize, filtersuffix, opt, n_args, n_xmm
65 cglobal hscale%1to%2_%4_%5, %6, 7, %7
71 mova m2, [max_19bit_int]
73 mova m2, [max_19bit_int]
75 mova m2, [max_19bit_flt]
76 %endif ; mmx/sse2/ssse3/sse4
99 shl r2, 1 ; this allows *16 (i.e. now *8) in lea instructions for the 8-tap filter
106 lea r1, [r1+r2*(2>>r2shr)]
108 lea r1, [r1+r2*(4>>r2shr)]
110 lea r5, [r5+r2*(2>>r2shr)]
114 %if %3 == 4 ; filterSize == 4 scaling
115 ; load 2x4 or 4x4 source pixels into m0/m1
116 movsx r0, word [r5+r2*2+0] ; filterPos[0]
117 movsx r6, word [r5+r2*2+2] ; filterPos[1]
118 movlh m0, [r3+r0*srcmul] ; src[filterPos[0] + {0,1,2,3}]
120 movlh m1, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}]
123 movhps m0, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}]
125 movd m4, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}]
127 movsx r0, word [r5+r2*2+4] ; filterPos[2]
128 movsx r6, word [r5+r2*2+6] ; filterPos[3]
129 movlh m1, [r3+r0*srcmul] ; src[filterPos[2] + {0,1,2,3}]
131 movhps m1, [r3+r6*srcmul] ; src[filterPos[3] + {0,1,2,3}]
133 movd m5, [r3+r6*srcmul] ; src[filterPos[3] + {0,1,2,3}]
136 %endif ; %1 == 8 && %5 <= ssse
137 %endif ; mmsize == 8/16
139 punpcklbw m0, m3 ; byte -> word
140 punpcklbw m1, m3 ; byte -> word
143 ; multiply with filter coefficients
144 %if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll
145 ; add back 0x8000 * sum(coeffs) after the horizontal add
149 pmaddwd m0, [r4+r2*8+mmsize*0] ; *= filter[{0,1,..,6,7}]
150 pmaddwd m1, [r4+r2*8+mmsize*1] ; *= filter[{8,9,..,14,15}]
152 ; add up horizontally (4 srcpix * 4 coefficients -> 1 dstpix)
153 %if mmsize == 8 ; mmx
160 shufps m0, m1, 10001000b
161 shufps m4, m1, 11011101b
164 phaddd m0, m1 ; filter[{ 0, 1, 2, 3}]*src[filterPos[0]+{0,1,2,3}],
165 ; filter[{ 4, 5, 6, 7}]*src[filterPos[1]+{0,1,2,3}],
166 ; filter[{ 8, 9,10,11}]*src[filterPos[2]+{0,1,2,3}],
167 ; filter[{12,13,14,15}]*src[filterPos[3]+{0,1,2,3}]
168 %endif ; mmx/sse2/ssse3/sse4
169 %else ; %3 == 8, i.e. filterSize == 8 scaling
170 ; load 2x8 or 4x8 source pixels into m0, m1, m4 and m5
171 movsx r0, word [r5+r2*1+0] ; filterPos[0]
172 movsx r6, word [r5+r2*1+2] ; filterPos[1]
173 movbh m0, [r3+ r0 *srcmul] ; src[filterPos[0] + {0,1,2,3,4,5,6,7}]
175 movbh m1, [r3+(r0+4)*srcmul] ; src[filterPos[0] + {4,5,6,7}]
176 movbh m4, [r3+ r6 *srcmul] ; src[filterPos[1] + {0,1,2,3}]
177 movbh m5, [r3+(r6+4)*srcmul] ; src[filterPos[1] + {4,5,6,7}]
179 movbh m1, [r3+ r6 *srcmul] ; src[filterPos[1] + {0,1,2,3,4,5,6,7}]
180 movsx r0, word [r5+r2*1+4] ; filterPos[2]
181 movsx r6, word [r5+r2*1+6] ; filterPos[3]
182 movbh m4, [r3+ r0 *srcmul] ; src[filterPos[2] + {0,1,2,3,4,5,6,7}]
183 movbh m5, [r3+ r6 *srcmul] ; src[filterPos[3] + {0,1,2,3,4,5,6,7}]
184 %endif ; mmsize == 8/16
186 punpcklbw m0, m3 ; byte -> word
187 punpcklbw m1, m3 ; byte -> word
188 punpcklbw m4, m3 ; byte -> word
189 punpcklbw m5, m3 ; byte -> word
193 %if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll
194 ; add back 0x8000 * sum(coeffs) after the horizontal add
200 pmaddwd m0, [r4+r2*8+mmsize*0] ; *= filter[{0,1,..,6,7}]
201 pmaddwd m1, [r4+r2*8+mmsize*1] ; *= filter[{8,9,..,14,15}]
202 pmaddwd m4, [r4+r2*8+mmsize*2] ; *= filter[{16,17,..,22,23}]
203 pmaddwd m5, [r4+r2*8+mmsize*3] ; *= filter[{24,25,..,30,31}]
205 ; add up horizontally (8 srcpix * 8 coefficients -> 1 dstpix)
219 ; emulate horizontal add as transpose + vertical add
233 ; FIXME if we rearrange the filter in pairs of 4, we can
234 ; load pixels likewise and use 2 x paddd + phaddd instead
235 ; of 3 x phaddd here, faster on older cpus
238 phaddd m0, m4 ; filter[{ 0, 1,..., 6, 7}]*src[filterPos[0]+{0,1,...,6,7}],
239 ; filter[{ 8, 9,...,14,15}]*src[filterPos[1]+{0,1,...,6,7}],
240 ; filter[{16,17,...,22,23}]*src[filterPos[2]+{0,1,...,6,7}],
241 ; filter[{24,25,...,30,31}]*src[filterPos[3]+{0,1,...,6,7}]
242 %endif ; mmx/sse2/ssse3/sse4
245 %else ; %3 == X, i.e. any filterSize scaling
249 %else ; %4 == X || %4 == X8
254 movsxd r6, r6d ; filterSize
255 lea r12, [r3+(r6-r6sub)*srcmul] ; &src[filterSize&~4]
260 lea r0, [r3+(r6-r6sub)*srcmul] ; &src[filterSize&~4]
276 movsx r0, word [r5+r2*2+0] ; filterPos[0]
277 movsx r1x, word [r5+r2*2+2] ; filterPos[1]
278 ; FIXME maybe do 4px/iteration on x86-64 (x86-32 wouldn't have enough regs)?
284 ; load 2x4 (mmx) or 2x8 (sse) source pixels into m0/m1 -> m4/m5
285 movbh m0, [src_reg+r0 *srcmul] ; src[filterPos[0] + {0,1,2,3(,4,5,6,7)}]
286 movbh m1, [src_reg+(r1x+r6sub)*srcmul] ; src[filterPos[1] + {0,1,2,3(,4,5,6,7)}]
293 %if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll
294 ; add back 0x8000 * sum(coeffs) after the horizontal add
298 pmaddwd m0, [r4 ] ; filter[{0,1,2,3(,4,5,6,7)}]
299 pmaddwd m1, [r4+(r6+r6sub)*2] ; filter[filtersize+{0,1,2,3(,4,5,6,7)}]
303 add src_reg, srcmul*mmsize/2
304 cmp src_reg, filter2 ; while (src += 4) < &src[filterSize]
308 movsx r1x, word [r5+r2*2+2] ; filterPos[1]
309 movlh m0, [src_reg+r0 *srcmul] ; split last 4 srcpx of dstpx[0]
310 sub r1x, r6 ; and first 4 srcpx of dstpx[1]
312 movhps m0, [src_reg+(r1x+r6sub)*srcmul]
314 movd m1, [src_reg+(r1x+r6sub)*srcmul]
316 %endif ; %1 == 8 && %5 <= ssse
320 %if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll
321 ; add back 0x8000 * sum(coeffs) after the horizontal add
327 lea r4, [r4+(r6+r6sub)*2]
329 %if mmsize == 8 ; mmx
342 %endif ; sse2/ssse3/sse4
347 pshufd m4, m4, 11011000b
353 %endif ; sse2/ssse3/sse4
354 %endif ; mmsize == 8/16
357 %if %1 == 16 ; add 0x8000 * sum(coeffs), i.e. back from signed -> unsigned
362 psrad m0, 14 + %1 - %2
369 movh [r1+r2*(2>>r2shr)], m0
375 PMINSD_MMX m0, m2, m4
382 %endif ; mmx/sse2/ssse3/sse4
384 mova [r1+r2*(4>>r2shr)], m0
390 add r2, (mmsize<<r2shr)/4 ; both 8tap and 4tap really only do 4 pixels (or for mmx: 2 pixels)
391 ; per iteration. see "shl r2,1" above as for why we do this
408 ; SCALE_FUNCS source_width, intermediate_nbits, opt, n_xmm
410 SCALE_FUNC %1, %2, 4, 4, %3, 6, %4
411 SCALE_FUNC %1, %2, 8, 8, %3, 6, %4
413 SCALE_FUNC %1, %2, X, X, %3, 7, %4
415 SCALE_FUNC %1, %2, X, X4, %3, 7, %4
416 SCALE_FUNC %1, %2, X, X8, %3, 7, %4
420 ; SCALE_FUNCS2 opt, 8_xmm_args, 9to10_xmm_args, 16_xmm_args
421 %macro SCALE_FUNCS2 4
423 SCALE_FUNCS 8, 15, %1, %2
424 SCALE_FUNCS 9, 15, %1, %3
425 SCALE_FUNCS 10, 15, %1, %3
426 SCALE_FUNCS 14, 15, %1, %3
427 SCALE_FUNCS 16, 15, %1, %4
429 SCALE_FUNCS 8, 19, %1, %2
430 SCALE_FUNCS 9, 19, %1, %3
431 SCALE_FUNCS 10, 19, %1, %3
432 SCALE_FUNCS 14, 19, %1, %3
433 SCALE_FUNCS 16, 19, %1, %4
438 SCALE_FUNCS2 mmx, 0, 0, 0
441 SCALE_FUNCS2 sse2, 6, 7, 8
442 SCALE_FUNCS2 ssse3, 6, 6, 8
443 SCALE_FUNCS2 sse4, 6, 6, 8
445 ;-----------------------------------------------------------------------------
446 ; vertical line scaling
448 ; void yuv2plane1_<output_size>_<opt>(const int16_t *src, uint8_t *dst, int dstW,
449 ; const uint8_t *dither, int offset)
451 ; void yuv2planeX_<output_size>_<opt>(const int16_t *filter, int filterSize,
452 ; const int16_t **src, uint8_t *dst, int dstW,
453 ; const uint8_t *dither, int offset)
455 ; Scale one or $filterSize lines of source data to generate one line of output
456 ; data. The input is 15-bit in int16_t if $output_size is [8,10] and 19-bit in
457 ; int32_t if $output_size is 16. $filter is 12-bits. $filterSize is a multiple
458 ; of 2. $offset is either 0 or 3. $dither holds 8 values.
459 ;-----------------------------------------------------------------------------
461 %macro yuv2planeX_fn 4
471 cglobal yuv2planeX_%2_%1, %4, 7, %3
472 %if %2 == 8 || %2 == 9 || %2 == 10
474 %endif ; %2 == 8/9/10
478 %assign pad 0x2c - (stack_offset & 15)
485 ; create registers holding dither
486 movq m_dith, [r5] ; dither
490 punpcklqdq m_dith, m_dith
491 %endif ; mmsize == 16
492 PALIGNR m_dith, m_dith, 3, m0
497 punpcklwd m8, m_dith, m6
500 punpcklwd m5, m_dith, m6
507 mova [rsp+16], m_dith
510 punpcklbw m5, m_dith, m6
514 punpcklwd m3, m_dith, m6
523 mova [rsp+24], m_dith
524 %endif ; mmsize == 8/16
531 ; the rep here is for the 8bit output mmx case, where dither covers
532 ; 8 pixels but we can only handle 2 pixels per register, and thus 4
533 ; pixels per iteration. In order to not have to keep track of where
534 ; we are w.r.t. dithering, we unroll the mmx/8bit loop x2.
541 mova m2, [rsp+mmsize*(0+%%i)]
542 mova m1, [rsp+mmsize*(1+%%i)]
547 %else ; %2 == 9/10/16
548 mova m1, [yuv2yuvX_%2_start]
550 %endif ; %2 == 8/9/10/16
554 mov r6, [r2+gprsize*cntr_reg-2*gprsize]
557 mova m5, [r6+r5*4+mmsize]
560 %endif ; %2 == 8/9/10/16
561 mov r6, [r2+gprsize*cntr_reg-gprsize]
564 mova m6, [r6+r5*4+mmsize]
567 %endif ; %2 == 8/9/10/16
570 movd m0, [r0+2*cntr_reg-4]; coeff[0], coeff[1]
572 pshuflw m7, m0, 0 ; coeff[0]
573 pshuflw m0, m0, 0x55 ; coeff[1]
574 pmovsxwd m7, m7 ; word -> dword
575 pmovsxwd m0, m0 ; word -> dword
596 %endif ; %2 == 8/9/10/16
599 jg .filterloop_ %+ %%i
607 %endif ; %2 == 8/9/10/16
613 %else ; %2 == 9/10/16
625 %endif ; mmx2/sse2/sse4/avx
626 pminsw m2, [yuv2yuvX_%2_upper]
627 %endif ; %2 == 9/10/16
629 %endif ; %2 == 8/9/10/16
646 %else ; %2 == 9/10/16
648 %endif ; %2 == 8/9/10/16
651 %define PALIGNR PALIGNR_MMX
654 yuv2planeX_fn mmx2, 8, 0, 7
655 yuv2planeX_fn mmx2, 9, 0, 5
656 yuv2planeX_fn mmx2, 10, 0, 5
660 yuv2planeX_fn sse2, 8, 10, 7
661 yuv2planeX_fn sse2, 9, 7, 5
662 yuv2planeX_fn sse2, 10, 7, 5
664 %define PALIGNR PALIGNR_SSSE3
665 yuv2planeX_fn sse4, 8, 10, 7
666 yuv2planeX_fn sse4, 9, 7, 5
667 yuv2planeX_fn sse4, 10, 7, 5
668 yuv2planeX_fn sse4, 16, 8, 5
671 yuv2planeX_fn avx, 8, 10, 7
672 yuv2planeX_fn avx, 9, 7, 5
673 yuv2planeX_fn avx, 10, 7, 5
675 ; %1=outout-bpc, %2=alignment (u/a)
676 %macro yuv2plane1_mainloop 2
679 paddsw m0, m2, [r0+r2*2+mmsize*0]
680 paddsw m1, m3, [r0+r2*2+mmsize*1]
686 paddd m0, m4, [r0+r2*4+mmsize*0]
687 paddd m1, m4, [r0+r2*4+mmsize*1]
688 paddd m2, m4, [r0+r2*4+mmsize*2]
689 paddd m3, m4, [r0+r2*4+mmsize*3]
694 %if cpuflag(sse4) ; avx/sse4
702 %endif ; mmx/sse2/sse4/avx
704 mov%2 [r1+r2*2+mmsize], m2
706 paddsw m0, m2, [r0+r2*2+mmsize*0]
707 paddsw m1, m2, [r0+r2*2+mmsize*1]
715 mov%2 [r1+r2*2+mmsize], m1
721 %macro yuv2plane1_fn 3
722 cglobal yuv2plane1_%1, %3, %3, %2
724 and r2, ~(mmsize - 1)
740 ; create registers holding dither
741 movq m3, [r3] ; dither
746 %endif ; mmsize == 16
747 PALIGNR_MMX m3, m3, 3, m2
751 punpckhbw m3, m4 ; byte->word
752 punpcklbw m2, m4 ; byte->word
766 %if cpuflag(sse4) ; sse4/avx
769 mova m4, [pd_4min0x40000]
771 %endif ; mmx/sse2/sse4/avx
774 ; actual pixel scaling
776 yuv2plane1_mainloop %1, a
780 yuv2plane1_mainloop %1, a
783 yuv2plane1_mainloop %1, u
784 %endif ; mmsize == 8/16
790 yuv2plane1_fn 8, 0, 5
791 yuv2plane1_fn 16, 0, 3
794 yuv2plane1_fn 9, 0, 3
795 yuv2plane1_fn 10, 0, 3
799 yuv2plane1_fn 8, 5, 5
800 yuv2plane1_fn 9, 5, 3
801 yuv2plane1_fn 10, 5, 3
802 yuv2plane1_fn 16, 6, 3
805 yuv2plane1_fn 16, 5, 3
808 yuv2plane1_fn 8, 5, 5
809 yuv2plane1_fn 9, 5, 3
810 yuv2plane1_fn 10, 5, 3
811 yuv2plane1_fn 16, 5, 3