1 ;******************************************************************************
2 ;* Copyright (c) 2012 Michael Niedermayer
3 ;* Copyright (c) 2014 James Almer <jamrial <at> gmail.com>
4 ;* Copyright (c) 2014 Ronald S. Bultje <rsbultje@gmail.com>
6 ;* This file is part of FFmpeg.
8 ;* FFmpeg is free software; you can redistribute it and/or
9 ;* modify it under the terms of the GNU Lesser General Public
10 ;* License as published by the Free Software Foundation; either
11 ;* version 2.1 of the License, or (at your option) any later version.
13 ;* FFmpeg is distributed in the hope that it will be useful,
14 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 ;* Lesser General Public License for more details.
18 ;* You should have received a copy of the GNU Lesser General Public
19 ;* License along with FFmpeg; if not, write to the Free Software
20 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 ;******************************************************************************
23 %include "libavutil/x86/x86util.asm"
33 .filter_bank: pointer 1
34 .filter_length: resd 1
36 .ideal_dst_incr: resd 1
43 .compensation_distance: resd 1
47 ; there's a few more here but we only care about the first few
58 %macro RESAMPLE_FNS 3-5 ; format [float or int16], bps, log2_bps, float op suffix [s or d], 1.0 constant
59 ; int resample_common_$format(ResampleContext *ctx, $format *dst,
60 ; const $format *src, int size, int update_ctx)
61 %if ARCH_X86_64 ; unix64 and win64
62 cglobal resample_common_%1, 0, 15, 2, ctx, dst, src, phase_shift, index, frac, \
63 dst_incr_mod, size, min_filter_count_x4, \
64 min_filter_len_x4, dst_incr_div, src_incr, \
65 phase_mask, dst_end, filter_bank
67 ; use red-zone for variable storage
68 %define ctx_stackq [rsp-0x8]
69 %define src_stackq [rsp-0x10]
71 %define update_context_stackd r4m
73 %define update_context_stackd [rsp-0x14]
76 ; load as many variables in registers as possible; for the rest, store
77 ; on stack so that we have 'ctx' available as one extra register
79 mov phase_maskd, [ctxq+ResampleContext.phase_mask]
81 mov update_context_stackd, r4d
83 mov indexd, [ctxq+ResampleContext.index]
84 mov fracd, [ctxq+ResampleContext.frac]
85 mov dst_incr_modd, [ctxq+ResampleContext.dst_incr_mod]
86 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
87 mov src_incrd, [ctxq+ResampleContext.src_incr]
89 mov min_filter_len_x4d, [ctxq+ResampleContext.filter_length]
90 mov dst_incr_divd, [ctxq+ResampleContext.dst_incr_div]
91 shl min_filter_len_x4d, %3
92 lea dst_endq, [dstq+sizeq*%2]
95 mov ecx, [ctxq+ResampleContext.phase_shift]
96 mov edi, [ctxq+ResampleContext.filter_alloc]
98 DEFINE_ARGS filter_alloc, dst, src, phase_shift, index, frac, dst_incr_mod, \
99 filter, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
100 src_incr, phase_mask, dst_end, filter_bank
102 mov R9d, [ctxq+ResampleContext.filter_alloc]
103 mov ecx, [ctxq+ResampleContext.phase_shift]
105 DEFINE_ARGS phase_shift, dst, src, filter_alloc, index, frac, dst_incr_mod, \
106 filter, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
107 src_incr, phase_mask, dst_end, filter_bank
110 neg min_filter_len_x4q
111 sub filter_bankq, min_filter_len_x4q
112 sub srcq, min_filter_len_x4q
115 cglobal resample_common_%1, 1, 7, 2, ctx, phase_shift, dst, frac, \
116 index, min_filter_length_x4, filter_bank
118 ; push temp variables to stack
119 %define ctx_stackq r0mp
120 %define src_stackq r2mp
121 %define update_context_stackd r4m
126 PUSH dword [ctxq+ResampleContext.dst_incr_div]
127 PUSH dword [ctxq+ResampleContext.dst_incr_mod]
128 PUSH dword [ctxq+ResampleContext.filter_alloc]
130 PUSH dword [ctxq+ResampleContext.phase_mask]
131 PUSH dword [ctxq+ResampleContext.src_incr]
132 mov min_filter_length_x4d, [ctxq+ResampleContext.filter_length]
133 mov indexd, [ctxq+ResampleContext.index]
134 shl min_filter_length_x4d, %3
135 mov fracd, [ctxq+ResampleContext.frac]
136 neg min_filter_length_x4q
137 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
138 sub r2mp, min_filter_length_x4q
139 sub filter_bankq, min_filter_length_x4q
140 PUSH min_filter_length_x4q
142 mov phase_shiftd, [ctxq+ResampleContext.phase_shift]
144 DEFINE_ARGS src, phase_shift, dst, frac, index, min_filter_count_x4, filter
146 %define filter_bankq dword [rsp+0x0]
147 %define min_filter_length_x4q dword [rsp+0x4]
148 %define src_incrd dword [rsp+0x8]
149 %define phase_maskd dword [rsp+0xc]
150 %define dst_endq dword [rsp+0x10]
151 %define filter_allocd dword [rsp+0x14]
152 %define dst_incr_modd dword [rsp+0x18]
153 %define dst_incr_divd dword [rsp+0x1c]
159 mov filterd, filter_allocd
162 mov min_filter_count_x4q, min_filter_len_x4q
163 lea filterq, [filter_bankq+filterq*%2]
165 mov min_filter_count_x4q, filter_bankq
166 lea filterq, [min_filter_count_x4q+filterq*%2]
167 mov min_filter_count_x4q, min_filter_length_x4q
177 movu m1, [srcq+min_filter_count_x4q*1]
180 vpmadcswd m0, m1, [filterq+min_filter_count_x4q*1], m0
182 pmaddwd m1, [filterq+min_filter_count_x4q*1]
186 %if cpuflag(fma4) || cpuflag(fma3)
187 fmaddp%4 m0, m1, [filterq+min_filter_count_x4q*1], m0
189 mulp%4 m1, m1, [filterq+min_filter_count_x4q*1]
193 add min_filter_count_x4q, mmsize
199 add fracd, dst_incr_modd
201 add indexd, dst_incr_divd
204 ; horizontal sum & store
206 vextractf128 xm1, m0, 0x1
212 shufps xm1, xm0, xm0, q0001
214 add fracd, dst_incr_modd
216 add indexd, dst_incr_divd
225 DEFINE_ARGS filter_alloc, dst, src, phase_shift, index, frac, dst_incr_mod, \
226 index_incr, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
227 src_incr, phase_mask, dst_end, filter_bank
229 DEFINE_ARGS phase_shift, dst, src, filter_alloc, index, frac, dst_incr_mod, \
230 index_incr, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
231 src_incr, phase_mask, dst_end, filter_bank
233 DEFINE_ARGS src, phase_shift, dst, frac, index, index_incr
237 mov index_incrd, indexd
239 and indexd, phase_maskd
240 sar index_incrd, phase_shiftb
241 lea srcq, [srcq+index_incrq*%2]
246 DEFINE_ARGS ctx, dst, src, phase_shift, index, frac
248 DEFINE_ARGS src, ctx, update_context, frac, index
251 cmp dword update_context_stackd, 0
253 ; strictly speaking, the function should always return the consumed
254 ; number of bytes; however, we only use the value if update_context
255 ; is true, so let's just leave it uninitialized otherwise
258 mov [ctxq+ResampleContext.frac ], fracd
260 mov [ctxq+ResampleContext.index], indexd
269 ; int resample_linear_$format(ResampleContext *ctx, float *dst,
270 ; const float *src, int size, int update_ctx)
271 %if ARCH_X86_64 ; unix64 and win64
273 cglobal resample_linear_%1, 0, 15, 5, ctx, dst, phase_mask, phase_shift, index, frac, \
274 size, dst_incr_mod, min_filter_count_x4, \
275 min_filter_len_x4, dst_incr_div, src_incr, \
276 src, dst_end, filter_bank
280 cglobal resample_linear_%1, 0, 15, 5, ctx, phase_mask, src, phase_shift, index, frac, \
281 size, dst_incr_mod, min_filter_count_x4, \
282 min_filter_len_x4, dst_incr_div, src_incr, \
283 dst, dst_end, filter_bank
288 ; use red-zone for variable storage
289 %define ctx_stackq [rsp-0x8]
290 %define src_stackq [rsp-0x10]
291 %define phase_mask_stackd [rsp-0x14]
293 %define update_context_stackd r4m
295 %define update_context_stackd [rsp-0x18]
298 ; load as many variables in registers as possible; for the rest, store
299 ; on stack so that we have 'ctx' available as one extra register
301 mov phase_maskd, [ctxq+ResampleContext.phase_mask]
303 mov update_context_stackd, r4d
305 mov indexd, [ctxq+ResampleContext.index]
306 mov fracd, [ctxq+ResampleContext.frac]
307 mov dst_incr_modd, [ctxq+ResampleContext.dst_incr_mod]
308 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
309 mov src_incrd, [ctxq+ResampleContext.src_incr]
311 mov phase_mask_stackd, phase_maskd
312 mov min_filter_len_x4d, [ctxq+ResampleContext.filter_length]
316 cvtsi2s%4 xm0, src_incrd
320 mov dst_incr_divd, [ctxq+ResampleContext.dst_incr_div]
321 shl min_filter_len_x4d, %3
322 lea dst_endq, [dstq+sizeq*%2]
325 mov ecx, [ctxq+ResampleContext.phase_shift]
326 mov edi, [ctxq+ResampleContext.filter_alloc]
328 DEFINE_ARGS filter_alloc, dst, filter2, phase_shift, index, frac, filter1, \
329 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
330 dst_incr_div, src_incr, src, dst_end, filter_bank
332 mov R9d, [ctxq+ResampleContext.filter_alloc]
333 mov ecx, [ctxq+ResampleContext.phase_shift]
335 DEFINE_ARGS phase_shift, filter2, src, filter_alloc, index, frac, filter1, \
336 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
337 dst_incr_div, src_incr, dst, dst_end, filter_bank
340 neg min_filter_len_x4q
341 sub filter_bankq, min_filter_len_x4q
342 sub srcq, min_filter_len_x4q
345 cglobal resample_linear_%1, 1, 7, 5, ctx, min_filter_length_x4, filter2, \
346 frac, index, dst, filter_bank
348 ; push temp variables to stack
349 %define ctx_stackq r0mp
350 %define src_stackq r2mp
351 %define update_context_stackd r4m
356 PUSH dword [ctxq+ResampleContext.dst_incr_div]
358 mov r3, dword [ctxq+ResampleContext.filter_alloc]
359 PUSH dword [ctxq+ResampleContext.dst_incr_mod]
363 mov r3, dword [ctxq+ResampleContext.src_incr]
364 PUSH dword [ctxq+ResampleContext.phase_mask]
373 mov min_filter_length_x4d, [ctxq+ResampleContext.filter_length]
374 mov indexd, [ctxq+ResampleContext.index]
375 shl min_filter_length_x4d, %3
376 mov fracd, [ctxq+ResampleContext.frac]
377 neg min_filter_length_x4q
378 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
379 sub r2mp, min_filter_length_x4q
380 sub filter_bankq, min_filter_length_x4q
381 PUSH min_filter_length_x4q
383 PUSH dword [ctxq+ResampleContext.phase_shift]
385 DEFINE_ARGS filter1, min_filter_count_x4, filter2, frac, index, dst, src
387 %define phase_shift_stackd dword [rsp+0x0]
388 %define filter_bankq dword [rsp+0x4]
389 %define min_filter_length_x4q dword [rsp+0x8]
390 %define src_incrd dword [rsp+0xc]
391 %define phase_mask_stackd dword [rsp+0x10]
392 %define filter_alloc_x4q dword [rsp+0x14]
393 %define filter_allocd dword [rsp+0x18]
394 %define dst_incr_modd dword [rsp+0x1c]
395 %define dst_endq dword [rsp+0x20]
396 %define dst_incr_divd dword [rsp+0x24]
402 mov filter1d, filter_allocd
403 imul filter1d, indexd
405 mov min_filter_count_x4q, min_filter_len_x4q
406 lea filter1q, [filter_bankq+filter1q*%2]
407 lea filter2q, [filter1q+filter_allocq*%2]
409 mov min_filter_count_x4q, filter_bankq
410 lea filter1q, [min_filter_count_x4q+filter1q*%2]
411 mov min_filter_count_x4q, min_filter_length_x4q
412 mov filter2q, filter1q
413 add filter2q, filter_alloc_x4q
425 movu m1, [srcq+min_filter_count_x4q*1]
428 vpmadcswd m2, m1, [filter2q+min_filter_count_x4q*1], m2
429 vpmadcswd m0, m1, [filter1q+min_filter_count_x4q*1], m0
431 pmaddwd m3, m1, [filter2q+min_filter_count_x4q*1]
432 pmaddwd m1, [filter1q+min_filter_count_x4q*1]
437 %if cpuflag(fma4) || cpuflag(fma3)
438 fmaddp%4 m2, m1, [filter2q+min_filter_count_x4q*1], m2
439 fmaddp%4 m0, m1, [filter1q+min_filter_count_x4q*1], m0
441 mulp%4 m3, m1, [filter2q+min_filter_count_x4q*1]
442 mulp%4 m1, m1, [filter1q+min_filter_count_x4q*1]
447 add min_filter_count_x4q, mmsize
462 PSHUFLW m3, m2, q0032
463 PSHUFLW m1, m0, q0032
468 ; This is probably a really bad idea on atom and other machines with a
469 ; long transfer latency between GPRs and XMMs (atom). However, it does
470 ; make the clip a lot simpler...
472 add indexd, dst_incr_divd
476 add fracd, dst_incr_modd
482 ; note that for imul/idiv, I need to move filter to edx/eax for each:
483 ; - 32bit: eax=r0[filter1], edx=r2[filter2]
484 ; - win64: eax=r6[filter1], edx=r1[todo]
485 ; - unix64: eax=r6[filter1], edx=r2[todo]
487 ; val += (v2 - val) * (FELEML) frac / c->src_incr;
489 vextractf128 xm1, m0, 0x1
490 vextractf128 xm3, m2, 0x1
497 shufp%4 xm1, xm1, q0000
498 %if cpuflag(fma4) || cpuflag(fma3)
499 fmaddp%4 xm0, xm2, xm1, xm0
505 ; horizontal sum & store
509 shufps xm1, xm0, xm0, q0001
511 add fracd, dst_incr_modd
513 add indexd, dst_incr_divd
522 DEFINE_ARGS filter_alloc, dst, filter2, phase_shift, index, frac, index_incr, \
523 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
524 dst_incr_div, src_incr, src, dst_end, filter_bank
526 DEFINE_ARGS phase_shift, filter2, src, filter_alloc, index, frac, index_incr, \
527 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
528 dst_incr_div, src_incr, dst, dst_end, filter_bank
530 DEFINE_ARGS filter1, phase_shift, index_incr, frac, index, dst, src
535 mov phase_shiftd, phase_shift_stackd
537 mov index_incrd, indexd
539 and indexd, phase_mask_stackd
540 sar index_incrd, phase_shiftb
541 lea srcq, [srcq+index_incrq*%2]
546 DEFINE_ARGS ctx, dst, filter2, phase_shift, index, frac, index_incr, \
547 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
548 dst_incr_div, src_incr, src, dst_end, filter_bank
550 DEFINE_ARGS ctx, filter2, src, phase_shift, index, frac, index_incr, \
551 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
552 dst_incr_div, src_incr, dst, dst_end, filter_bank
554 DEFINE_ARGS filter1, ctx, update_context, frac, index, dst, src
557 cmp dword update_context_stackd, 0
559 ; strictly speaking, the function should always return the consumed
560 ; number of bytes; however, we only use the value if update_context
561 ; is true, so let's just leave it uninitialized otherwise
564 mov [ctxq+ResampleContext.frac ], fracd
566 mov [ctxq+ResampleContext.index], indexd
577 RESAMPLE_FNS float, 4, 2, s, pf_1
579 %if HAVE_AVX_EXTERNAL
581 RESAMPLE_FNS float, 4, 2, s, pf_1
583 %if HAVE_FMA3_EXTERNAL
585 RESAMPLE_FNS float, 4, 2, s, pf_1
587 %if HAVE_FMA4_EXTERNAL
589 RESAMPLE_FNS float, 4, 2, s, pf_1
594 RESAMPLE_FNS int16, 2, 1
598 RESAMPLE_FNS int16, 2, 1
599 %if HAVE_XOP_EXTERNAL
601 RESAMPLE_FNS int16, 2, 1
605 RESAMPLE_FNS double, 8, 3, d, pdbl_1