1 ;******************************************************************************
2 ;* Copyright (c) 2012 Michael Niedermayer
3 ;* Copyright (c) 2014 James Almer <jamrial <at> gmail.com>
4 ;* Copyright (c) 2014 Ronald S. Bultje <rsbultje@gmail.com>
6 ;* This file is part of FFmpeg.
8 ;* FFmpeg is free software; you can redistribute it and/or
9 ;* modify it under the terms of the GNU Lesser General Public
10 ;* License as published by the Free Software Foundation; either
11 ;* version 2.1 of the License, or (at your option) any later version.
13 ;* FFmpeg is distributed in the hope that it will be useful,
14 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 ;* Lesser General Public License for more details.
18 ;* You should have received a copy of the GNU Lesser General Public
19 ;* License along with FFmpeg; if not, write to the Free Software
20 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 ;******************************************************************************
23 %include "libavutil/x86/x86util.asm"
33 .filter_bank: pointer 1
34 .filter_length: resd 1
36 .ideal_dst_incr: resd 1
43 .compensation_distance: resd 1
46 ; there's a few more here but we only care about the first few
57 ; FIXME remove unneeded variables (index_incr, phase_mask)
58 %macro RESAMPLE_FNS 3-5 ; format [float or int16], bps, log2_bps, float op suffix [s or d], 1.0 constant
59 ; int resample_common_$format(ResampleContext *ctx, $format *dst,
60 ; const $format *src, int size, int update_ctx)
61 %if ARCH_X86_64 ; unix64 and win64
62 cglobal resample_common_%1, 0, 15, 2, ctx, dst, src, phase_count, index, frac, \
63 dst_incr_mod, size, min_filter_count_x4, \
64 min_filter_len_x4, dst_incr_div, src_incr, \
65 phase_mask, dst_end, filter_bank
67 ; use red-zone for variable storage
68 %define ctx_stackq [rsp-0x8]
69 %define src_stackq [rsp-0x10]
71 %define update_context_stackd r4m
73 %define update_context_stackd [rsp-0x14]
76 ; load as many variables in registers as possible; for the rest, store
77 ; on stack so that we have 'ctx' available as one extra register
80 mov update_context_stackd, r4d
82 mov indexd, [ctxq+ResampleContext.index]
83 mov fracd, [ctxq+ResampleContext.frac]
84 mov dst_incr_modd, [ctxq+ResampleContext.dst_incr_mod]
85 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
86 mov src_incrd, [ctxq+ResampleContext.src_incr]
88 mov min_filter_len_x4d, [ctxq+ResampleContext.filter_length]
89 mov dst_incr_divd, [ctxq+ResampleContext.dst_incr_div]
90 shl min_filter_len_x4d, %3
91 lea dst_endq, [dstq+sizeq*%2]
94 mov ecx, [ctxq+ResampleContext.phase_count]
95 mov edi, [ctxq+ResampleContext.filter_alloc]
97 DEFINE_ARGS filter_alloc, dst, src, phase_count, index, frac, dst_incr_mod, \
98 filter, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
99 src_incr, phase_mask, dst_end, filter_bank
101 mov R9d, [ctxq+ResampleContext.filter_alloc]
102 mov ecx, [ctxq+ResampleContext.phase_count]
104 DEFINE_ARGS phase_count, dst, src, filter_alloc, index, frac, dst_incr_mod, \
105 filter, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
106 src_incr, phase_mask, dst_end, filter_bank
109 neg min_filter_len_x4q
110 sub filter_bankq, min_filter_len_x4q
111 sub srcq, min_filter_len_x4q
114 cglobal resample_common_%1, 1, 7, 2, ctx, phase_count, dst, frac, \
115 index, min_filter_length_x4, filter_bank
117 ; push temp variables to stack
118 %define ctx_stackq r0mp
119 %define src_stackq r2mp
120 %define update_context_stackd r4m
125 PUSH dword [ctxq+ResampleContext.dst_incr_div]
126 PUSH dword [ctxq+ResampleContext.dst_incr_mod]
127 PUSH dword [ctxq+ResampleContext.filter_alloc]
129 PUSH dword [ctxq+ResampleContext.phase_count] ; unneeded replacement for phase_mask
130 PUSH dword [ctxq+ResampleContext.src_incr]
131 mov min_filter_length_x4d, [ctxq+ResampleContext.filter_length]
132 mov indexd, [ctxq+ResampleContext.index]
133 shl min_filter_length_x4d, %3
134 mov fracd, [ctxq+ResampleContext.frac]
135 neg min_filter_length_x4q
136 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
137 sub r2mp, min_filter_length_x4q
138 sub filter_bankq, min_filter_length_x4q
139 PUSH min_filter_length_x4q
141 mov phase_countd, [ctxq+ResampleContext.phase_count]
143 DEFINE_ARGS src, phase_count, dst, frac, index, min_filter_count_x4, filter
145 %define filter_bankq dword [rsp+0x0]
146 %define min_filter_length_x4q dword [rsp+0x4]
147 %define src_incrd dword [rsp+0x8]
148 %define phase_maskd dword [rsp+0xc]
149 %define dst_endq dword [rsp+0x10]
150 %define filter_allocd dword [rsp+0x14]
151 %define dst_incr_modd dword [rsp+0x18]
152 %define dst_incr_divd dword [rsp+0x1c]
158 mov filterd, filter_allocd
161 mov min_filter_count_x4q, min_filter_len_x4q
162 lea filterq, [filter_bankq+filterq*%2]
164 mov min_filter_count_x4q, filter_bankq
165 lea filterq, [min_filter_count_x4q+filterq*%2]
166 mov min_filter_count_x4q, min_filter_length_x4q
176 movu m1, [srcq+min_filter_count_x4q*1]
179 vpmadcswd m0, m1, [filterq+min_filter_count_x4q*1], m0
181 pmaddwd m1, [filterq+min_filter_count_x4q*1]
185 %if cpuflag(fma4) || cpuflag(fma3)
186 fmaddp%4 m0, m1, [filterq+min_filter_count_x4q*1], m0
188 mulp%4 m1, m1, [filterq+min_filter_count_x4q*1]
192 add min_filter_count_x4q, mmsize
198 add fracd, dst_incr_modd
200 add indexd, dst_incr_divd
203 ; horizontal sum & store
205 vextractf128 xm1, m0, 0x1
211 shufps xm1, xm0, xm0, q0001
213 add fracd, dst_incr_modd
215 add indexd, dst_incr_divd
224 DEFINE_ARGS filter_alloc, dst, src, phase_count, index, frac, dst_incr_mod, \
225 index_incr, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
226 src_incr, phase_mask, dst_end, filter_bank
228 DEFINE_ARGS phase_count, dst, src, filter_alloc, index, frac, dst_incr_mod, \
229 index_incr, min_filter_count_x4, min_filter_len_x4, dst_incr_div, \
230 src_incr, phase_mask, dst_end, filter_bank
232 DEFINE_ARGS src, phase_count, dst, frac, index, index_incr
237 cmp indexd, phase_countd
240 sub indexd, phase_countd
242 cmp indexd, phase_countd
249 DEFINE_ARGS ctx, dst, src, phase_count, index, frac
251 DEFINE_ARGS src, ctx, update_context, frac, index
254 cmp dword update_context_stackd, 0
256 ; strictly speaking, the function should always return the consumed
257 ; number of bytes; however, we only use the value if update_context
258 ; is true, so let's just leave it uninitialized otherwise
261 mov [ctxq+ResampleContext.frac ], fracd
263 mov [ctxq+ResampleContext.index], indexd
272 ; int resample_linear_$format(ResampleContext *ctx, float *dst,
273 ; const float *src, int size, int update_ctx)
274 %if ARCH_X86_64 ; unix64 and win64
276 cglobal resample_linear_%1, 0, 15, 5, ctx, dst, phase_mask, phase_count, index, frac, \
277 size, dst_incr_mod, min_filter_count_x4, \
278 min_filter_len_x4, dst_incr_div, src_incr, \
279 src, dst_end, filter_bank
283 cglobal resample_linear_%1, 0, 15, 5, ctx, phase_mask, src, phase_count, index, frac, \
284 size, dst_incr_mod, min_filter_count_x4, \
285 min_filter_len_x4, dst_incr_div, src_incr, \
286 dst, dst_end, filter_bank
291 ; use red-zone for variable storage
292 %define ctx_stackq [rsp-0x8]
293 %define src_stackq [rsp-0x10]
294 %define phase_mask_stackd [rsp-0x14]
296 %define update_context_stackd r4m
298 %define update_context_stackd [rsp-0x18]
301 ; load as many variables in registers as possible; for the rest, store
302 ; on stack so that we have 'ctx' available as one extra register
305 mov update_context_stackd, r4d
307 mov indexd, [ctxq+ResampleContext.index]
308 mov fracd, [ctxq+ResampleContext.frac]
309 mov dst_incr_modd, [ctxq+ResampleContext.dst_incr_mod]
310 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
311 mov src_incrd, [ctxq+ResampleContext.src_incr]
313 mov min_filter_len_x4d, [ctxq+ResampleContext.filter_length]
317 cvtsi2s%4 xm0, src_incrd
321 mov dst_incr_divd, [ctxq+ResampleContext.dst_incr_div]
322 shl min_filter_len_x4d, %3
323 lea dst_endq, [dstq+sizeq*%2]
326 mov ecx, [ctxq+ResampleContext.phase_count]
327 mov edi, [ctxq+ResampleContext.filter_alloc]
329 DEFINE_ARGS filter_alloc, dst, filter2, phase_count, index, frac, filter1, \
330 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
331 dst_incr_div, src_incr, src, dst_end, filter_bank
333 mov R9d, [ctxq+ResampleContext.filter_alloc]
334 mov ecx, [ctxq+ResampleContext.phase_count]
336 DEFINE_ARGS phase_count, filter2, src, filter_alloc, index, frac, filter1, \
337 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
338 dst_incr_div, src_incr, dst, dst_end, filter_bank
341 neg min_filter_len_x4q
342 sub filter_bankq, min_filter_len_x4q
343 sub srcq, min_filter_len_x4q
346 cglobal resample_linear_%1, 1, 7, 5, ctx, min_filter_length_x4, filter2, \
347 frac, index, dst, filter_bank
349 ; push temp variables to stack
350 %define ctx_stackq r0mp
351 %define src_stackq r2mp
352 %define update_context_stackd r4m
357 PUSH dword [ctxq+ResampleContext.dst_incr_div]
359 mov r3, dword [ctxq+ResampleContext.filter_alloc]
360 PUSH dword [ctxq+ResampleContext.dst_incr_mod]
364 mov r3, dword [ctxq+ResampleContext.src_incr]
365 PUSH dword [ctxq+ResampleContext.phase_count] ; unneeded replacement of phase_mask
374 mov min_filter_length_x4d, [ctxq+ResampleContext.filter_length]
375 mov indexd, [ctxq+ResampleContext.index]
376 shl min_filter_length_x4d, %3
377 mov fracd, [ctxq+ResampleContext.frac]
378 neg min_filter_length_x4q
379 mov filter_bankq, [ctxq+ResampleContext.filter_bank]
380 sub r2mp, min_filter_length_x4q
381 sub filter_bankq, min_filter_length_x4q
382 PUSH min_filter_length_x4q
384 PUSH dword [ctxq+ResampleContext.phase_count]
386 DEFINE_ARGS filter1, min_filter_count_x4, filter2, frac, index, dst, src
388 %define phase_count_stackd dword [rsp+0x0]
389 %define filter_bankq dword [rsp+0x4]
390 %define min_filter_length_x4q dword [rsp+0x8]
391 %define src_incrd dword [rsp+0xc]
392 %define phase_mask_stackd dword [rsp+0x10]
393 %define filter_alloc_x4q dword [rsp+0x14]
394 %define filter_allocd dword [rsp+0x18]
395 %define dst_incr_modd dword [rsp+0x1c]
396 %define dst_endq dword [rsp+0x20]
397 %define dst_incr_divd dword [rsp+0x24]
403 mov filter1d, filter_allocd
404 imul filter1d, indexd
406 mov min_filter_count_x4q, min_filter_len_x4q
407 lea filter1q, [filter_bankq+filter1q*%2]
408 lea filter2q, [filter1q+filter_allocq*%2]
410 mov min_filter_count_x4q, filter_bankq
411 lea filter1q, [min_filter_count_x4q+filter1q*%2]
412 mov min_filter_count_x4q, min_filter_length_x4q
413 mov filter2q, filter1q
414 add filter2q, filter_alloc_x4q
426 movu m1, [srcq+min_filter_count_x4q*1]
429 vpmadcswd m2, m1, [filter2q+min_filter_count_x4q*1], m2
430 vpmadcswd m0, m1, [filter1q+min_filter_count_x4q*1], m0
432 pmaddwd m3, m1, [filter2q+min_filter_count_x4q*1]
433 pmaddwd m1, [filter1q+min_filter_count_x4q*1]
438 %if cpuflag(fma4) || cpuflag(fma3)
439 fmaddp%4 m2, m1, [filter2q+min_filter_count_x4q*1], m2
440 fmaddp%4 m0, m1, [filter1q+min_filter_count_x4q*1], m0
442 mulp%4 m3, m1, [filter2q+min_filter_count_x4q*1]
443 mulp%4 m1, m1, [filter1q+min_filter_count_x4q*1]
448 add min_filter_count_x4q, mmsize
463 PSHUFLW m3, m2, q0032
464 PSHUFLW m1, m0, q0032
469 ; This is probably a really bad idea on atom and other machines with a
470 ; long transfer latency between GPRs and XMMs (atom). However, it does
471 ; make the clip a lot simpler...
473 add indexd, dst_incr_divd
477 add fracd, dst_incr_modd
483 ; note that for imul/idiv, I need to move filter to edx/eax for each:
484 ; - 32bit: eax=r0[filter1], edx=r2[filter2]
485 ; - win64: eax=r6[filter1], edx=r1[todo]
486 ; - unix64: eax=r6[filter1], edx=r2[todo]
488 ; val += (v2 - val) * (FELEML) frac / c->src_incr;
490 vextractf128 xm1, m0, 0x1
491 vextractf128 xm3, m2, 0x1
498 shufp%4 xm1, xm1, q0000
499 %if cpuflag(fma4) || cpuflag(fma3)
500 fmaddp%4 xm0, xm2, xm1, xm0
506 ; horizontal sum & store
510 shufps xm1, xm0, xm0, q0001
512 add fracd, dst_incr_modd
514 add indexd, dst_incr_divd
523 DEFINE_ARGS filter_alloc, dst, filter2, phase_count, index, frac, index_incr, \
524 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
525 dst_incr_div, src_incr, src, dst_end, filter_bank
527 DEFINE_ARGS phase_count, filter2, src, filter_alloc, index, frac, index_incr, \
528 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
529 dst_incr_div, src_incr, dst, dst_end, filter_bank
531 DEFINE_ARGS filter1, phase_count, index_incr, frac, index, dst, src
536 mov phase_countd, phase_count_stackd
539 cmp indexd, phase_countd
542 sub indexd, phase_countd
544 cmp indexd, phase_countd
551 DEFINE_ARGS ctx, dst, filter2, phase_count, index, frac, index_incr, \
552 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
553 dst_incr_div, src_incr, src, dst_end, filter_bank
555 DEFINE_ARGS ctx, filter2, src, phase_count, index, frac, index_incr, \
556 dst_incr_mod, min_filter_count_x4, min_filter_len_x4, \
557 dst_incr_div, src_incr, dst, dst_end, filter_bank
559 DEFINE_ARGS filter1, ctx, update_context, frac, index, dst, src
562 cmp dword update_context_stackd, 0
564 ; strictly speaking, the function should always return the consumed
565 ; number of bytes; however, we only use the value if update_context
566 ; is true, so let's just leave it uninitialized otherwise
569 mov [ctxq+ResampleContext.frac ], fracd
571 mov [ctxq+ResampleContext.index], indexd
582 RESAMPLE_FNS float, 4, 2, s, pf_1
584 %if HAVE_AVX_EXTERNAL
586 RESAMPLE_FNS float, 4, 2, s, pf_1
588 %if HAVE_FMA3_EXTERNAL
590 RESAMPLE_FNS float, 4, 2, s, pf_1
592 %if HAVE_FMA4_EXTERNAL
594 RESAMPLE_FNS float, 4, 2, s, pf_1
599 RESAMPLE_FNS int16, 2, 1
603 RESAMPLE_FNS int16, 2, 1
604 %if HAVE_XOP_EXTERNAL
606 RESAMPLE_FNS int16, 2, 1
610 RESAMPLE_FNS double, 8, 3, d, pdbl_1