git.sesse.net Git - ffmpeg/blob - libavcodec/arm/h264dsp_neon.S

   1 /*
   2  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include "asm.S"
  22
  23         .macro transpose_8x8 r0 r1 r2 r3 r4 r5 r6 r7
  24         vtrn.32         \r0, \r4
  25         vtrn.32         \r1, \r5
  26         vtrn.32         \r2, \r6
  27         vtrn.32         \r3, \r7
  28         vtrn.16         \r0, \r2
  29         vtrn.16         \r1, \r3
  30         vtrn.16         \r4, \r6
  31         vtrn.16         \r5, \r7
  32         vtrn.8          \r0, \r1
  33         vtrn.8          \r2, \r3
  34         vtrn.8          \r4, \r5
  35         vtrn.8          \r6, \r7
  36         .endm
  37
  38         .macro transpose_4x4 r0 r1 r2 r3
  39         vtrn.16         \r0, \r2
  40         vtrn.16         \r1, \r3
  41         vtrn.8          \r0, \r1
  42         vtrn.8          \r2, \r3
  43         .endm
  44
  45         .macro swap4 r0 r1 r2 r3 r4 r5 r6 r7
  46         vswp            \r0, \r4
  47         vswp            \r1, \r5
  48         vswp            \r2, \r6
  49         vswp            \r3, \r7
  50         .endm
  51
  52         .macro transpose16_4x4 r0 r1 r2 r3 r4 r5 r6 r7
  53         vtrn.32         \r0, \r2
  54         vtrn.32         \r1, \r3
  55         vtrn.32         \r4, \r6
  56         vtrn.32         \r5, \r7
  57         vtrn.16         \r0, \r1
  58         vtrn.16         \r2, \r3
  59         vtrn.16         \r4, \r5
  60         vtrn.16         \r6, \r7
  61         .endm
  62
  63 /* chroma_mc8(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
  64         .macro  h264_chroma_mc8 type
  65 function ff_\type\()_h264_chroma_mc8_neon, export=1
  66         push            {r4-r7, lr}
  67         ldrd            r4,  [sp, #20]
  68 .ifc \type,avg
  69         mov             lr,  r0
  70 .endif
  71         pld             [r1]
  72         pld             [r1, r2]
  73
  74         muls            r7,  r4,  r5
  75         rsb             r6,  r7,  r5,  lsl #3
  76         rsb             ip,  r7,  r4,  lsl #3
  77         sub             r4,  r7,  r4,  lsl #3
  78         sub             r4,  r4,  r5,  lsl #3
  79         add             r4,  r4,  #64
  80
  81         beq             2f
  82
  83         add             r5,  r1,  r2
  84
  85         vdup.8          d0,  r4
  86         lsl             r4,  r2,  #1
  87         vdup.8          d1,  ip
  88         vld1.64         {d4, d5}, [r1], r4
  89         vdup.8          d2,  r6
  90         vld1.64         {d6, d7}, [r5], r4
  91         vdup.8          d3,  r7
  92
  93         vext.8          d5,  d4,  d5,  #1
  94         vext.8          d7,  d6,  d7,  #1
  95
  96 1:      pld             [r5]
  97         vmull.u8        q8,  d4,  d0
  98         vmlal.u8        q8,  d5,  d1
  99         vld1.64         {d4, d5}, [r1], r4
 100         vmlal.u8        q8,  d6,  d2
 101         vext.8          d5,  d4,  d5,  #1
 102         vmlal.u8        q8,  d7,  d3
 103         vmull.u8        q9,  d6,  d0
 104         subs            r3,  r3,  #2
 105         vmlal.u8        q9,  d7,  d1
 106         vmlal.u8        q9,  d4,  d2
 107         vmlal.u8        q9,  d5,  d3
 108         vrshrn.u16      d16, q8,  #6
 109         vld1.64         {d6, d7}, [r5], r4
 110         pld             [r1]
 111         vrshrn.u16      d17, q9,  #6
 112 .ifc \type,avg
 113         vld1.64         {d20}, [lr,:64], r2
 114         vld1.64         {d21}, [lr,:64], r2
 115         vrhadd.u8       q8,  q8,  q10
 116 .endif
 117         vext.8          d7,  d6,  d7,  #1
 118         vst1.64         {d16}, [r0,:64], r2
 119         vst1.64         {d17}, [r0,:64], r2
 120         bgt             1b
 121
 122         pop             {r4-r7, pc}
 123
 124 2:      tst             r6,  r6
 125         add             ip,  ip,  r6
 126         vdup.8          d0,  r4
 127         vdup.8          d1,  ip
 128
 129         beq             4f
 130
 131         add             r5,  r1,  r2
 132         lsl             r4,  r2,  #1
 133         vld1.64         {d4}, [r1], r4
 134         vld1.64         {d6}, [r5], r4
 135
 136 3:      pld             [r5]
 137         vmull.u8        q8,  d4,  d0
 138         vmlal.u8        q8,  d6,  d1
 139         vld1.64         {d4}, [r1], r4
 140         vmull.u8        q9,  d6,  d0
 141         vmlal.u8        q9,  d4,  d1
 142         vld1.64         {d6}, [r5], r4
 143         vrshrn.u16      d16, q8,  #6
 144         vrshrn.u16      d17, q9,  #6
 145 .ifc \type,avg
 146         vld1.64         {d20}, [lr,:64], r2
 147         vld1.64         {d21}, [lr,:64], r2
 148         vrhadd.u8       q8,  q8,  q10
 149 .endif
 150         subs            r3,  r3,  #2
 151         pld             [r1]
 152         vst1.64         {d16}, [r0,:64], r2
 153         vst1.64         {d17}, [r0,:64], r2
 154         bgt             3b
 155
 156         pop             {r4-r7, pc}
 157
 158 4:      vld1.64         {d4, d5}, [r1], r2
 159         vld1.64         {d6, d7}, [r1], r2
 160         vext.8          d5,  d4,  d5,  #1
 161         vext.8          d7,  d6,  d7,  #1
 162
 163 5:      pld             [r1]
 164         subs            r3,  r3,  #2
 165         vmull.u8        q8,  d4,  d0
 166         vmlal.u8        q8,  d5,  d1
 167         vld1.64         {d4, d5}, [r1], r2
 168         vmull.u8        q9,  d6,  d0
 169         vmlal.u8        q9,  d7,  d1
 170         pld             [r1]
 171         vext.8          d5,  d4,  d5,  #1
 172         vrshrn.u16      d16, q8,  #6
 173         vrshrn.u16      d17, q9,  #6
 174 .ifc \type,avg
 175         vld1.64         {d20}, [lr,:64], r2
 176         vld1.64         {d21}, [lr,:64], r2
 177         vrhadd.u8       q8,  q8,  q10
 178 .endif
 179         vld1.64         {d6, d7}, [r1], r2
 180         vext.8          d7,  d6,  d7,  #1
 181         vst1.64         {d16}, [r0,:64], r2
 182         vst1.64         {d17}, [r0,:64], r2
 183         bgt             5b
 184
 185         pop             {r4-r7, pc}
 186         .endfunc
 187         .endm
 188
 189 /* chroma_mc4(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
 190         .macro  h264_chroma_mc4 type
 191 function ff_\type\()_h264_chroma_mc4_neon, export=1
 192         push            {r4-r7, lr}
 193         ldrd            r4,  [sp, #20]
 194 .ifc \type,avg
 195         mov             lr,  r0
 196 .endif
 197         pld             [r1]
 198         pld             [r1, r2]
 199
 200         muls            r7,  r4,  r5
 201         rsb             r6,  r7,  r5,  lsl #3
 202         rsb             ip,  r7,  r4,  lsl #3
 203         sub             r4,  r7,  r4,  lsl #3
 204         sub             r4,  r4,  r5,  lsl #3
 205         add             r4,  r4,  #64
 206
 207         beq             2f
 208
 209         add             r5,  r1,  r2
 210
 211         vdup.8          d0,  r4
 212         lsl             r4,  r2,  #1
 213         vdup.8          d1,  ip
 214         vld1.64         {d4},     [r1], r4
 215         vdup.8          d2,  r6
 216         vld1.64         {d6},     [r5], r4
 217         vdup.8          d3,  r7
 218
 219         vext.8          d5,  d4,  d5,  #1
 220         vext.8          d7,  d6,  d7,  #1
 221         vtrn.32         d4,  d5
 222         vtrn.32         d6,  d7
 223
 224         vtrn.32         d0,  d1
 225         vtrn.32         d2,  d3
 226
 227 1:      pld             [r5]
 228         vmull.u8        q8,  d4,  d0
 229         vmlal.u8        q8,  d6,  d2
 230         vld1.64         {d4},     [r1], r4
 231         vext.8          d5,  d4,  d5,  #1
 232         vtrn.32         d4,  d5
 233         vmull.u8        q9,  d6,  d0
 234         vmlal.u8        q9,  d4,  d2
 235         vld1.64         {d6},     [r5], r4
 236         vadd.i16        d16, d16, d17
 237         vadd.i16        d17, d18, d19
 238         vrshrn.u16      d16, q8,  #6
 239         subs            r3,  r3,  #2
 240         pld             [r1]
 241 .ifc \type,avg
 242         vld1.32         {d20[0]}, [lr,:32], r2
 243         vld1.32         {d20[1]}, [lr,:32], r2
 244         vrhadd.u8       d16, d16, d20
 245 .endif
 246         vext.8          d7,  d6,  d7,  #1
 247         vtrn.32         d6,  d7
 248         vst1.32         {d16[0]}, [r0,:32], r2
 249         vst1.32         {d16[1]}, [r0,:32], r2
 250         bgt             1b
 251
 252         pop             {r4-r7, pc}
 253
 254 2:      tst             r6,  r6
 255         add             ip,  ip,  r6
 256         vdup.8          d0,  r4
 257         vdup.8          d1,  ip
 258         vtrn.32         d0,  d1
 259
 260         beq             4f
 261
 262         vext.32         d1,  d0,  d1,  #1
 263         add             r5,  r1,  r2
 264         lsl             r4,  r2,  #1
 265         vld1.32         {d4[0]},  [r1], r4
 266         vld1.32         {d4[1]},  [r5], r4
 267
 268 3:      pld             [r5]
 269         vmull.u8        q8,  d4,  d0
 270         vld1.32         {d4[0]},  [r1], r4
 271         vmull.u8        q9,  d4,  d1
 272         vld1.32         {d4[1]},  [r5], r4
 273         vadd.i16        d16, d16, d17
 274         vadd.i16        d17, d18, d19
 275         vrshrn.u16      d16, q8,  #6
 276 .ifc \type,avg
 277         vld1.32         {d20[0]}, [lr,:32], r2
 278         vld1.32         {d20[1]}, [lr,:32], r2
 279         vrhadd.u8       d16, d16, d20
 280 .endif
 281         subs            r3,  r3,  #2
 282         pld             [r1]
 283         vst1.32         {d16[0]}, [r0,:32], r2
 284         vst1.32         {d16[1]}, [r0,:32], r2
 285         bgt             3b
 286
 287         pop             {r4-r7, pc}
 288
 289 4:      vld1.64         {d4},     [r1], r2
 290         vld1.64         {d6},     [r1], r2
 291         vext.8          d5,  d4,  d5,  #1
 292         vext.8          d7,  d6,  d7,  #1
 293         vtrn.32         d4,  d5
 294         vtrn.32         d6,  d7
 295
 296 5:      vmull.u8        q8,  d4,  d0
 297         vmull.u8        q9,  d6,  d0
 298         subs            r3,  r3,  #2
 299         vld1.64         {d4},     [r1], r2
 300         vext.8          d5,  d4,  d5,  #1
 301         vtrn.32         d4,  d5
 302         vadd.i16        d16, d16, d17
 303         vadd.i16        d17, d18, d19
 304         pld             [r1]
 305         vrshrn.u16      d16, q8,  #6
 306 .ifc \type,avg
 307         vld1.32         {d20[0]}, [lr,:32], r2
 308         vld1.32         {d20[1]}, [lr,:32], r2
 309         vrhadd.u8       d16, d16, d20
 310 .endif
 311         vld1.64         {d6},     [r1], r2
 312         vext.8          d7,  d6,  d7,  #1
 313         vtrn.32         d6,  d7
 314         pld             [r1]
 315         vst1.32         {d16[0]}, [r0,:32], r2
 316         vst1.32         {d16[1]}, [r0,:32], r2
 317         bgt             5b
 318
 319         pop             {r4-r7, pc}
 320         .endfunc
 321         .endm
 322
 323         .text
 324         .align
 325
 326         h264_chroma_mc8 put
 327         h264_chroma_mc8 avg
 328         h264_chroma_mc4 put
 329         h264_chroma_mc4 avg
 330
 331         /* H.264 loop filter */
 332
 333         .macro h264_loop_filter_start
 334         ldr             ip,  [sp]
 335         tst             r2,  r2
 336         ldr             ip,  [ip]
 337         tstne           r3,  r3
 338         vmov.32         d24[0], ip
 339         and             ip,  ip,  ip, lsl #16
 340         bxeq            lr
 341         ands            ip,  ip,  ip, lsl #8
 342         bxlt            lr
 343         .endm
 344
 345         .macro align_push_regs
 346         and             ip,  sp,  #15
 347         add             ip,  ip,  #32
 348         sub             sp,  sp,  ip
 349         vst1.64         {d12-d15}, [sp,:128]
 350         sub             sp,  sp,  #32
 351         vst1.64         {d8-d11},  [sp,:128]
 352         .endm
 353
 354         .macro align_pop_regs
 355         vld1.64         {d8-d11},  [sp,:128]!
 356         vld1.64         {d12-d15}, [sp,:128], ip
 357         .endm
 358
 359         .macro h264_loop_filter_luma
 360         vdup.8          q11, r2         @ alpha
 361         vmovl.u8        q12, d24
 362         vabd.u8         q6,  q8,  q0    @ abs(p0 - q0)
 363         vmovl.u16       q12, d24
 364         vabd.u8         q14, q9,  q8    @ abs(p1 - p0)
 365         vsli.16         q12, q12, #8
 366         vabd.u8         q15, q1,  q0    @ abs(q1 - q0)
 367         vsli.32         q12, q12, #16
 368         vclt.u8         q6,  q6,  q11   @ < alpha
 369         vdup.8          q11, r3         @ beta
 370         vclt.s8         q7,  q12, #0
 371         vclt.u8         q14, q14, q11   @ < beta
 372         vclt.u8         q15, q15, q11   @ < beta
 373         vbic            q6,  q6,  q7
 374         vabd.u8         q4,  q10, q8    @ abs(p2 - p0)
 375         vand            q6,  q6,  q14
 376         vabd.u8         q5,  q2,  q0    @ abs(q2 - q0)
 377         vclt.u8         q4,  q4,  q11   @ < beta
 378         vand            q6,  q6,  q15
 379         vclt.u8         q5,  q5,  q11   @ < beta
 380         vand            q4,  q4,  q6
 381         vand            q5,  q5,  q6
 382         vand            q12, q12, q6
 383         vrhadd.u8       q14, q8,  q0
 384         vsub.i8         q6,  q12, q4
 385         vqadd.u8        q7,  q9,  q12
 386         vhadd.u8        q10, q10, q14
 387         vsub.i8         q6,  q6,  q5
 388         vhadd.u8        q14, q2,  q14
 389         vmin.u8         q7,  q7,  q10
 390         vqsub.u8        q11, q9,  q12
 391         vqadd.u8        q2,  q1,  q12
 392         vmax.u8         q7,  q7,  q11
 393         vqsub.u8        q11, q1,  q12
 394         vmin.u8         q14, q2,  q14
 395         vmovl.u8        q2,  d0
 396         vmax.u8         q14, q14, q11
 397         vmovl.u8        q10, d1
 398         vsubw.u8        q2,  q2,  d16
 399         vsubw.u8        q10, q10, d17
 400         vshl.i16        q2,  q2,  #2
 401         vshl.i16        q10, q10, #2
 402         vaddw.u8        q2,  q2,  d18
 403         vaddw.u8        q10, q10, d19
 404         vsubw.u8        q2,  q2,  d2
 405         vsubw.u8        q10, q10, d3
 406         vrshrn.i16      d4,  q2,  #3
 407         vrshrn.i16      d5,  q10, #3
 408         vbsl            q4,  q7,  q9
 409         vbsl            q5,  q14, q1
 410         vneg.s8         q7,  q6
 411         vmovl.u8        q14, d16
 412         vmin.s8         q2,  q2,  q6
 413         vmovl.u8        q6,  d17
 414         vmax.s8         q2,  q2,  q7
 415         vmovl.u8        q11, d0
 416         vmovl.u8        q12, d1
 417         vaddw.s8        q14, q14, d4
 418         vaddw.s8        q6,  q6,  d5
 419         vsubw.s8        q11, q11, d4
 420         vsubw.s8        q12, q12, d5
 421         vqmovun.s16     d16, q14
 422         vqmovun.s16     d17, q6
 423         vqmovun.s16     d0,  q11
 424         vqmovun.s16     d1,  q12
 425         .endm
 426
 427 function ff_h264_v_loop_filter_luma_neon, export=1
 428         h264_loop_filter_start
 429
 430         vld1.64         {d0, d1},  [r0,:128], r1
 431         vld1.64         {d2, d3},  [r0,:128], r1
 432         vld1.64         {d4, d5},  [r0,:128], r1
 433         sub             r0,  r0,  r1, lsl #2
 434         sub             r0,  r0,  r1, lsl #1
 435         vld1.64         {d20,d21}, [r0,:128], r1
 436         vld1.64         {d18,d19}, [r0,:128], r1
 437         vld1.64         {d16,d17}, [r0,:128], r1
 438
 439         align_push_regs
 440
 441         h264_loop_filter_luma
 442
 443         sub             r0,  r0,  r1, lsl #1
 444         vst1.64         {d8, d9},  [r0,:128], r1
 445         vst1.64         {d16,d17}, [r0,:128], r1
 446         vst1.64         {d0, d1},  [r0,:128], r1
 447         vst1.64         {d10,d11}, [r0,:128]
 448
 449         align_pop_regs
 450         bx              lr
 451         .endfunc
 452
 453 function ff_h264_h_loop_filter_luma_neon, export=1
 454         h264_loop_filter_start
 455
 456         sub             r0,  r0,  #4
 457         vld1.64         {d6},  [r0], r1
 458         vld1.64         {d20}, [r0], r1
 459         vld1.64         {d18}, [r0], r1
 460         vld1.64         {d16}, [r0], r1
 461         vld1.64         {d0},  [r0], r1
 462         vld1.64         {d2},  [r0], r1
 463         vld1.64         {d4},  [r0], r1
 464         vld1.64         {d26}, [r0], r1
 465         vld1.64         {d7},  [r0], r1
 466         vld1.64         {d21}, [r0], r1
 467         vld1.64         {d19}, [r0], r1
 468         vld1.64         {d17}, [r0], r1
 469         vld1.64         {d1},  [r0], r1
 470         vld1.64         {d3},  [r0], r1
 471         vld1.64         {d5},  [r0], r1
 472         vld1.64         {d27}, [r0], r1
 473
 474         transpose_8x8   q3, q10, q9, q8, q0, q1, q2, q13
 475
 476         align_push_regs
 477
 478         h264_loop_filter_luma
 479
 480         transpose_4x4   q4, q8, q0, q5
 481
 482         sub             r0,  r0,  r1, lsl #4
 483         add             r0,  r0,  #2
 484         vst1.32         {d8[0]},  [r0], r1
 485         vst1.32         {d16[0]}, [r0], r1
 486         vst1.32         {d0[0]},  [r0], r1
 487         vst1.32         {d10[0]}, [r0], r1
 488         vst1.32         {d8[1]},  [r0], r1
 489         vst1.32         {d16[1]}, [r0], r1
 490         vst1.32         {d0[1]},  [r0], r1
 491         vst1.32         {d10[1]}, [r0], r1
 492         vst1.32         {d9[0]},  [r0], r1
 493         vst1.32         {d17[0]}, [r0], r1
 494         vst1.32         {d1[0]},  [r0], r1
 495         vst1.32         {d11[0]}, [r0], r1
 496         vst1.32         {d9[1]},  [r0], r1
 497         vst1.32         {d17[1]}, [r0], r1
 498         vst1.32         {d1[1]},  [r0], r1
 499         vst1.32         {d11[1]}, [r0], r1
 500
 501         align_pop_regs
 502         bx              lr
 503         .endfunc
 504
 505         .macro h264_loop_filter_chroma
 506         vdup.8          d22, r2         @ alpha
 507         vmovl.u8        q12, d24
 508         vabd.u8         d26, d16, d0    @ abs(p0 - q0)
 509         vmovl.u8        q2,  d0
 510         vabd.u8         d28, d18, d16   @ abs(p1 - p0)
 511         vsubw.u8        q2,  q2,  d16
 512         vsli.16         d24, d24, #8
 513         vshl.i16        q2,  q2,  #2
 514         vabd.u8         d30, d2,  d0    @ abs(q1 - q0)
 515         vaddw.u8        q2,  q2,  d18
 516         vclt.u8         d26, d26, d22   @ < alpha
 517         vsubw.u8        q2,  q2,  d2
 518         vdup.8          d22, r3         @ beta
 519         vclt.s8         d25, d24, #0
 520         vrshrn.i16      d4,  q2,  #3
 521         vclt.u8         d28, d28, d22   @ < beta
 522         vbic            d26, d26, d25
 523         vclt.u8         d30, d30, d22   @ < beta
 524         vand            d26, d26, d28
 525         vneg.s8         d25, d24
 526         vand            d26, d26, d30
 527         vmin.s8         d4,  d4,  d24
 528         vmovl.u8        q14, d16
 529         vand            d4,  d4,  d26
 530         vmax.s8         d4,  d4,  d25
 531         vmovl.u8        q11, d0
 532         vaddw.s8        q14, q14, d4
 533         vsubw.s8        q11, q11, d4
 534         vqmovun.s16     d16, q14
 535         vqmovun.s16     d0,  q11
 536         .endm
 537
 538 function ff_h264_v_loop_filter_chroma_neon, export=1
 539         h264_loop_filter_start
 540
 541         sub             r0,  r0,  r1, lsl #1
 542         vld1.64         {d18}, [r0,:64], r1
 543         vld1.64         {d16}, [r0,:64], r1
 544         vld1.64         {d0},  [r0,:64], r1
 545         vld1.64         {d2},  [r0,:64]
 546
 547         h264_loop_filter_chroma
 548
 549         sub             r0,  r0,  r1, lsl #1
 550         vst1.64         {d16}, [r0,:64], r1
 551         vst1.64         {d0},  [r0,:64], r1
 552
 553         bx              lr
 554         .endfunc
 555
 556 function ff_h264_h_loop_filter_chroma_neon, export=1
 557         h264_loop_filter_start
 558
 559         sub             r0,  r0,  #2
 560         vld1.32         {d18[0]}, [r0], r1
 561         vld1.32         {d16[0]}, [r0], r1
 562         vld1.32         {d0[0]},  [r0], r1
 563         vld1.32         {d2[0]},  [r0], r1
 564         vld1.32         {d18[1]}, [r0], r1
 565         vld1.32         {d16[1]}, [r0], r1
 566         vld1.32         {d0[1]},  [r0], r1
 567         vld1.32         {d2[1]},  [r0], r1
 568
 569         vtrn.16         d18, d0
 570         vtrn.16         d16, d2
 571         vtrn.8          d18, d16
 572         vtrn.8          d0,  d2
 573
 574         h264_loop_filter_chroma
 575
 576         vtrn.16         d18, d0
 577         vtrn.16         d16, d2
 578         vtrn.8          d18, d16
 579         vtrn.8          d0,  d2
 580
 581         sub             r0,  r0,  r1, lsl #3
 582         vst1.32         {d18[0]}, [r0], r1
 583         vst1.32         {d16[0]}, [r0], r1
 584         vst1.32         {d0[0]},  [r0], r1
 585         vst1.32         {d2[0]},  [r0], r1
 586         vst1.32         {d18[1]}, [r0], r1
 587         vst1.32         {d16[1]}, [r0], r1
 588         vst1.32         {d0[1]},  [r0], r1
 589         vst1.32         {d2[1]},  [r0], r1
 590
 591         bx              lr
 592         .endfunc
 593
 594         /* H.264 qpel MC */
 595
 596         .macro  lowpass_const r
 597         movw            \r,  #5
 598         movt            \r,  #20
 599         vmov.32         d6[0], \r
 600         .endm
 601
 602         .macro  lowpass_8 r0, r1, r2, r3, d0, d1, narrow=1
 603 .if \narrow
 604         t0 .req q0
 605         t1 .req q8
 606 .else
 607         t0 .req \d0
 608         t1 .req \d1
 609 .endif
 610         vext.8          d2,  \r0, \r1, #2
 611         vext.8          d3,  \r0, \r1, #3
 612         vaddl.u8        q1,  d2,  d3
 613         vext.8          d4,  \r0, \r1, #1
 614         vext.8          d5,  \r0, \r1, #4
 615         vaddl.u8        q2,  d4,  d5
 616         vext.8          d30, \r0, \r1, #5
 617         vaddl.u8        t0,  \r0, d30
 618         vext.8          d18, \r2, \r3, #2
 619         vmla.i16        t0,  q1,  d6[1]
 620         vext.8          d19, \r2, \r3, #3
 621         vaddl.u8        q9,  d18, d19
 622         vext.8          d20, \r2, \r3, #1
 623         vmls.i16        t0,  q2,  d6[0]
 624         vext.8          d21, \r2, \r3, #4
 625         vaddl.u8        q10, d20, d21
 626         vext.8          d31, \r2, \r3, #5
 627         vaddl.u8        t1,  \r2, d31
 628         vmla.i16        t1,  q9,  d6[1]
 629         vmls.i16        t1,  q10, d6[0]
 630 .if \narrow
 631         vqrshrun.s16    \d0, t0,  #5
 632         vqrshrun.s16    \d1, t1,  #5
 633 .endif
 634         .unreq  t0
 635         .unreq  t1
 636         .endm
 637
 638         .macro  lowpass_8_1 r0, r1, d0, narrow=1
 639 .if \narrow
 640         t0 .req q0
 641 .else
 642         t0 .req \d0
 643 .endif
 644         vext.8          d2,  \r0, \r1, #2
 645         vext.8          d3,  \r0, \r1, #3
 646         vaddl.u8        q1,  d2,  d3
 647         vext.8          d4,  \r0, \r1, #1
 648         vext.8          d5,  \r0, \r1, #4
 649         vaddl.u8        q2,  d4,  d5
 650         vext.8          d30, \r0, \r1, #5
 651         vaddl.u8        t0,  \r0, d30
 652         vmla.i16        t0,  q1,  d6[1]
 653         vmls.i16        t0,  q2,  d6[0]
 654 .if \narrow
 655         vqrshrun.s16    \d0, t0,  #5
 656 .endif
 657         .unreq  t0
 658         .endm
 659
 660         .macro  lowpass_8.16 r0, r1, l0, h0, l1, h1, d
 661         vext.16         q1,  \r0, \r1, #2
 662         vext.16         q0,  \r0, \r1, #3
 663         vaddl.s16       q9,  d2,  d0
 664         vext.16         q2,  \r0, \r1, #1
 665         vaddl.s16       q1,  d3,  d1
 666         vext.16         q3,  \r0, \r1, #4
 667         vaddl.s16       q10, d4,  d6
 668         vext.16         \r1, \r0, \r1, #5
 669         vaddl.s16       q2,  d5,  d7
 670         vaddl.s16       q0,  \h0, \h1
 671         vaddl.s16       q8,  \l0, \l1
 672
 673         vshl.i32        q3,  q9,  #4
 674         vshl.i32        q9,  q9,  #2
 675         vshl.i32        q15, q10, #2
 676         vadd.i32        q9,  q9,  q3
 677         vadd.i32        q10, q10, q15
 678
 679         vshl.i32        q3,  q1,  #4
 680         vshl.i32        q1,  q1,  #2
 681         vshl.i32        q15, q2,  #2
 682         vadd.i32        q1,  q1,  q3
 683         vadd.i32        q2,  q2,  q15
 684
 685         vadd.i32        q9,  q9,  q8
 686         vsub.i32        q9,  q9,  q10
 687
 688         vadd.i32        q1,  q1,  q0
 689         vsub.i32        q1,  q1,  q2
 690
 691         vrshrn.s32      d18, q9,  #10
 692         vrshrn.s32      d19, q1,  #10
 693
 694         vqmovun.s16     \d,  q9
 695         .endm
 696
 697 function put_h264_qpel16_h_lowpass_neon_packed
 698         mov             r4,  lr
 699         mov             ip,  #16
 700         mov             r3,  #8
 701         bl              put_h264_qpel8_h_lowpass_neon
 702         sub             r1,  r1,  r2, lsl #4
 703         add             r1,  r1,  #8
 704         mov             ip,  #16
 705         mov             lr,  r4
 706         b               put_h264_qpel8_h_lowpass_neon
 707         .endfunc
 708
 709 function put_h264_qpel16_h_lowpass_neon
 710         push            {lr}
 711         mov             ip,  #16
 712         bl              put_h264_qpel8_h_lowpass_neon
 713         sub             r0,  r0,  r3, lsl #4
 714         sub             r1,  r1,  r2, lsl #4
 715         add             r0,  r0,  #8
 716         add             r1,  r1,  #8
 717         mov             ip,  #16
 718         pop             {lr}
 719         .endfunc
 720
 721 function put_h264_qpel8_h_lowpass_neon
 722 1:      vld1.64         {d0, d1},  [r1], r2
 723         vld1.64         {d16,d17}, [r1], r2
 724         subs            ip,  ip,  #2
 725         lowpass_8       d0,  d1,  d16, d17, d0,  d16
 726         vst1.64         {d0},     [r0,:64], r3
 727         vst1.64         {d16},    [r0,:64], r3
 728         bne             1b
 729         bx              lr
 730         .endfunc
 731
 732 function put_h264_qpel16_h_lowpass_l2_neon
 733         push            {lr}
 734         mov             ip,  #16
 735         bl              put_h264_qpel8_h_lowpass_l2_neon
 736         sub             r0,  r0,  r2, lsl #4
 737         sub             r1,  r1,  r2, lsl #4
 738         sub             r3,  r3,  r2, lsl #4
 739         add             r0,  r0,  #8
 740         add             r1,  r1,  #8
 741         add             r3,  r3,  #8
 742         mov             ip,  #16
 743         pop             {lr}
 744         .endfunc
 745
 746 function put_h264_qpel8_h_lowpass_l2_neon
 747 1:      vld1.64         {d0, d1},  [r1], r2
 748         vld1.64         {d16,d17}, [r1], r2
 749         vld1.64         {d28},     [r3], r2
 750         vld1.64         {d29},     [r3], r2
 751         subs            ip,  ip,  #2
 752         lowpass_8       d0,  d1,  d16, d17, d0,  d1
 753         vrhadd.u8       q0,  q0,  q14
 754         vst1.64         {d0},      [r0,:64], r2
 755         vst1.64         {d1},      [r0,:64], r2
 756         bne             1b
 757         bx              lr
 758         .endfunc
 759
 760 function put_h264_qpel16_v_lowpass_neon_packed
 761         mov             r4,  lr
 762         mov             r2,  #8
 763         bl              put_h264_qpel8_v_lowpass_neon
 764         sub             r1,  r1,  r3, lsl #2
 765         bl              put_h264_qpel8_v_lowpass_neon
 766         sub             r1,  r1,  r3, lsl #4
 767         sub             r1,  r1,  r3, lsl #2
 768         add             r1,  r1,  #8
 769         bl              put_h264_qpel8_v_lowpass_neon
 770         sub             r1,  r1,  r3, lsl #2
 771         mov             lr,  r4
 772         b               put_h264_qpel8_v_lowpass_neon
 773         .endfunc
 774
 775 function put_h264_qpel16_v_lowpass_neon
 776         mov             r4,  lr
 777         bl              put_h264_qpel8_v_lowpass_neon
 778         sub             r1,  r1,  r3, lsl #2
 779         bl              put_h264_qpel8_v_lowpass_neon
 780         sub             r0,  r0,  r2, lsl #4
 781         add             r0,  r0,  #8
 782         sub             r1,  r1,  r3, lsl #4
 783         sub             r1,  r1,  r3, lsl #2
 784         add             r1,  r1,  #8
 785         bl              put_h264_qpel8_v_lowpass_neon
 786         sub             r1,  r1,  r3, lsl #2
 787         mov             lr,  r4
 788         .endfunc
 789
 790 function put_h264_qpel8_v_lowpass_neon
 791         vld1.64         {d8},  [r1], r3
 792         vld1.64         {d10}, [r1], r3
 793         vld1.64         {d12}, [r1], r3
 794         vld1.64         {d14}, [r1], r3
 795         vld1.64         {d22}, [r1], r3
 796         vld1.64         {d24}, [r1], r3
 797         vld1.64         {d26}, [r1], r3
 798         vld1.64         {d28}, [r1], r3
 799         vld1.64         {d9},  [r1], r3
 800         vld1.64         {d11}, [r1], r3
 801         vld1.64         {d13}, [r1], r3
 802         vld1.64         {d15}, [r1], r3
 803         vld1.64         {d23}, [r1]
 804
 805         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
 806         lowpass_8       d8,  d9,  d10, d11, d8,  d10
 807         lowpass_8       d12, d13, d14, d15, d12, d14
 808         lowpass_8       d22, d23, d24, d25, d22, d24
 809         lowpass_8       d26, d27, d28, d29, d26, d28
 810         transpose_8x8   d8,  d10, d12, d14, d22, d24, d26, d28
 811
 812         vst1.64         {d8},  [r0,:64], r2
 813         vst1.64         {d10}, [r0,:64], r2
 814         vst1.64         {d12}, [r0,:64], r2
 815         vst1.64         {d14}, [r0,:64], r2
 816         vst1.64         {d22}, [r0,:64], r2
 817         vst1.64         {d24}, [r0,:64], r2
 818         vst1.64         {d26}, [r0,:64], r2
 819         vst1.64         {d28}, [r0,:64], r2
 820
 821         bx              lr
 822         .endfunc
 823
 824 function put_h264_qpel16_v_lowpass_l2_neon
 825         mov             r4,  lr
 826         bl              put_h264_qpel8_v_lowpass_l2_neon
 827         sub             r1,  r1,  r3, lsl #2
 828         bl              put_h264_qpel8_v_lowpass_l2_neon
 829         sub             r0,  r0,  r3, lsl #4
 830         sub             ip,  ip,  r2, lsl #4
 831         add             r0,  r0,  #8
 832         add             ip,  ip,  #8
 833         sub             r1,  r1,  r3, lsl #4
 834         sub             r1,  r1,  r3, lsl #2
 835         add             r1,  r1,  #8
 836         bl              put_h264_qpel8_v_lowpass_l2_neon
 837         sub             r1,  r1,  r3, lsl #2
 838         mov             lr,  r4
 839         .endfunc
 840
 841 function put_h264_qpel8_v_lowpass_l2_neon
 842         vld1.64         {d8},  [r1], r3
 843         vld1.64         {d10}, [r1], r3
 844         vld1.64         {d12}, [r1], r3
 845         vld1.64         {d14}, [r1], r3
 846         vld1.64         {d22}, [r1], r3
 847         vld1.64         {d24}, [r1], r3
 848         vld1.64         {d26}, [r1], r3
 849         vld1.64         {d28}, [r1], r3
 850         vld1.64         {d9},  [r1], r3
 851         vld1.64         {d11}, [r1], r3
 852         vld1.64         {d13}, [r1], r3
 853         vld1.64         {d15}, [r1], r3
 854         vld1.64         {d23}, [r1]
 855
 856         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
 857         lowpass_8       d8,  d9,  d10, d11, d8,  d9
 858         lowpass_8       d12, d13, d14, d15, d12, d13
 859         lowpass_8       d22, d23, d24, d25, d22, d23
 860         lowpass_8       d26, d27, d28, d29, d26, d27
 861         transpose_8x8   d8,  d9,  d12, d13, d22, d23, d26, d27
 862
 863         vld1.64         {d0},  [ip], r2
 864         vld1.64         {d1},  [ip], r2
 865         vld1.64         {d2},  [ip], r2
 866         vld1.64         {d3},  [ip], r2
 867         vld1.64         {d4},  [ip], r2
 868         vrhadd.u8       q0,  q0,  q4
 869         vld1.64         {d5},  [ip], r2
 870         vrhadd.u8       q1,  q1,  q6
 871         vld1.64         {d10}, [ip], r2
 872         vrhadd.u8       q2,  q2,  q11
 873         vld1.64         {d11}, [ip], r2
 874
 875         vst1.64         {d0},  [r0,:64], r3
 876         vst1.64         {d1},  [r0,:64], r3
 877         vrhadd.u8       q5,  q5,  q13
 878         vst1.64         {d2},  [r0,:64], r3
 879         vst1.64         {d3},  [r0,:64], r3
 880         vst1.64         {d4},  [r0,:64], r3
 881         vst1.64         {d5},  [r0,:64], r3
 882         vst1.64         {d10}, [r0,:64], r3
 883         vst1.64         {d11}, [r0,:64], r3
 884
 885         bx              lr
 886         .endfunc
 887
 888 function put_h264_qpel8_hv_lowpass_neon_top
 889         lowpass_const   ip
 890         mov             ip,  #12
 891 1:      vld1.64         {d0, d1},  [r1], r3
 892         vld1.64         {d16,d17}, [r1], r3
 893         subs            ip,  ip,  #2
 894         lowpass_8       d0,  d1,  d16, d17, q11, q12, narrow=0
 895         vst1.64         {d22-d25}, [r4,:128]!
 896         bne             1b
 897
 898         vld1.64         {d0, d1},  [r1]
 899         lowpass_8_1     d0,  d1,  q12, narrow=0
 900
 901         mov             ip,  #-16
 902         add             r4,  r4,  ip
 903         vld1.64         {d30,d31}, [r4,:128], ip
 904         vld1.64         {d20,d21}, [r4,:128], ip
 905         vld1.64         {d18,d19}, [r4,:128], ip
 906         vld1.64         {d16,d17}, [r4,:128], ip
 907         vld1.64         {d14,d15}, [r4,:128], ip
 908         vld1.64         {d12,d13}, [r4,:128], ip
 909         vld1.64         {d10,d11}, [r4,:128], ip
 910         vld1.64         {d8, d9},  [r4,:128], ip
 911         vld1.64         {d6, d7},  [r4,:128], ip
 912         vld1.64         {d4, d5},  [r4,:128], ip
 913         vld1.64         {d2, d3},  [r4,:128], ip
 914         vld1.64         {d0, d1},  [r4,:128]
 915
 916         swap4           d1,  d3,  d5,  d7,  d8,  d10, d12, d14
 917         transpose16_4x4 q0,  q1,  q2,  q3,  q4,  q5,  q6,  q7
 918
 919         swap4           d17, d19, d21, d31, d24, d26, d28, d22
 920         transpose16_4x4 q8,  q9,  q10, q15, q12, q13, q14, q11
 921
 922         vst1.64         {d30,d31}, [r4,:128]!
 923         vst1.64         {d6, d7},  [r4,:128]!
 924         vst1.64         {d20,d21}, [r4,:128]!
 925         vst1.64         {d4, d5},  [r4,:128]!
 926         vst1.64         {d18,d19}, [r4,:128]!
 927         vst1.64         {d2, d3},  [r4,:128]!
 928         vst1.64         {d16,d17}, [r4,:128]!
 929         vst1.64         {d0, d1},  [r4,:128]
 930
 931         lowpass_8.16    q4,  q12, d8,  d9,  d24, d25, d8
 932         lowpass_8.16    q5,  q13, d10, d11, d26, d27, d9
 933         lowpass_8.16    q6,  q14, d12, d13, d28, d29, d10
 934         lowpass_8.16    q7,  q11, d14, d15, d22, d23, d11
 935
 936         vld1.64         {d16,d17}, [r4,:128], ip
 937         vld1.64         {d30,d31}, [r4,:128], ip
 938         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d12
 939         vld1.64         {d16,d17}, [r4,:128], ip
 940         vld1.64         {d30,d31}, [r4,:128], ip
 941         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d13
 942         vld1.64         {d16,d17}, [r4,:128], ip
 943         vld1.64         {d30,d31}, [r4,:128], ip
 944         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d14
 945         vld1.64         {d16,d17}, [r4,:128], ip
 946         vld1.64         {d30,d31}, [r4,:128]
 947         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d15
 948
 949         transpose_8x8   d12, d13, d14, d15, d8,  d9,  d10, d11
 950
 951         bx              lr
 952         .endfunc
 953
 954 function put_h264_qpel8_hv_lowpass_neon
 955         mov             r10, lr
 956         bl              put_h264_qpel8_hv_lowpass_neon_top
 957         vst1.64         {d12},     [r0,:64], r2
 958         vst1.64         {d13},     [r0,:64], r2
 959         vst1.64         {d14},     [r0,:64], r2
 960         vst1.64         {d15},     [r0,:64], r2
 961         vst1.64         {d8},      [r0,:64], r2
 962         vst1.64         {d9},      [r0,:64], r2
 963         vst1.64         {d10},     [r0,:64], r2
 964         vst1.64         {d11},     [r0,:64], r2
 965
 966         mov             lr,  r10
 967         bx              lr
 968         .endfunc
 969
 970 function put_h264_qpel8_hv_lowpass_l2_neon
 971         mov             r10, lr
 972         bl              put_h264_qpel8_hv_lowpass_neon_top
 973
 974         vld1.64         {d0, d1},  [r2,:128]!
 975         vld1.64         {d2, d3},  [r2,:128]!
 976         vrhadd.u8       q0,  q0,  q6
 977         vld1.64         {d4, d5},  [r2,:128]!
 978         vrhadd.u8       q1,  q1,  q7
 979         vld1.64         {d6, d7},  [r2,:128]!
 980         vrhadd.u8       q2,  q2,  q4
 981
 982         vst1.64         {d0},      [r0,:64], r3
 983         vrhadd.u8       q3,  q3,  q5
 984         vst1.64         {d1},      [r0,:64], r3
 985         vst1.64         {d2},      [r0,:64], r3
 986         vst1.64         {d3},      [r0,:64], r3
 987         vst1.64         {d4},      [r0,:64], r3
 988         vst1.64         {d5},      [r0,:64], r3
 989         vst1.64         {d6},      [r0,:64], r3
 990         vst1.64         {d7},      [r0,:64], r3
 991
 992         mov             lr,  r10
 993         bx              lr
 994         .endfunc
 995
 996 function put_h264_qpel16_hv_lowpass_neon
 997         mov             r9,  lr
 998         bl              put_h264_qpel8_hv_lowpass_neon
 999         sub             r1,  r1,  r3, lsl #2
1000         bl              put_h264_qpel8_hv_lowpass_neon
1001         sub             r1,  r1,  r3, lsl #4
1002         sub             r1,  r1,  r3, lsl #2
1003         add             r1,  r1,  #8
1004         sub             r0,  r0,  r2, lsl #4
1005         add             r0,  r0,  #8
1006         bl              put_h264_qpel8_hv_lowpass_neon
1007         sub             r1,  r1,  r3, lsl #2
1008         mov             lr,  r9
1009         b               put_h264_qpel8_hv_lowpass_neon
1010         .endfunc
1011
1012 function put_h264_qpel16_hv_lowpass_l2_neon
1013         mov             r9,  lr
1014         sub             r2,  r4,  #256
1015         bl              put_h264_qpel8_hv_lowpass_l2_neon
1016         sub             r1,  r1,  r3, lsl #2
1017         bl              put_h264_qpel8_hv_lowpass_l2_neon
1018         sub             r1,  r1,  r3, lsl #4
1019         sub             r1,  r1,  r3, lsl #2
1020         add             r1,  r1,  #8
1021         sub             r0,  r0,  r3, lsl #4
1022         add             r0,  r0,  #8
1023         bl              put_h264_qpel8_hv_lowpass_l2_neon
1024         sub             r1,  r1,  r3, lsl #2
1025         mov             lr,  r9
1026         b               put_h264_qpel8_hv_lowpass_l2_neon
1027         .endfunc
1028
1029 function ff_put_h264_qpel8_mc10_neon, export=1
1030         lowpass_const   r3
1031         mov             r3,  r1
1032         sub             r1,  r1,  #2
1033         mov             ip,  #8
1034         b               put_h264_qpel8_h_lowpass_l2_neon
1035         .endfunc
1036
1037 function ff_put_h264_qpel8_mc20_neon, export=1
1038         lowpass_const   r3
1039         sub             r1,  r1,  #2
1040         mov             r3,  r2
1041         mov             ip,  #8
1042         b               put_h264_qpel8_h_lowpass_neon
1043         .endfunc
1044
1045 function ff_put_h264_qpel8_mc30_neon, export=1
1046         lowpass_const   r3
1047         add             r3,  r1,  #1
1048         sub             r1,  r1,  #2
1049         mov             ip,  #8
1050         b               put_h264_qpel8_h_lowpass_l2_neon
1051         .endfunc
1052
1053 function ff_put_h264_qpel8_mc01_neon, export=1
1054         push            {lr}
1055         mov             ip,  r1
1056 put_h264_qpel8_mc01:
1057         lowpass_const   r3
1058         mov             r3,  r2
1059         sub             r1,  r1,  r2, lsl #1
1060         vpush           {d8-d15}
1061         bl              put_h264_qpel8_v_lowpass_l2_neon
1062         vpop            {d8-d15}
1063         pop             {pc}
1064         .endfunc
1065
1066 function ff_put_h264_qpel8_mc11_neon, export=1
1067         push            {r0, r1, r2, lr}
1068 put_h264_qpel8_mc11:
1069         lowpass_const   r3
1070         sub             sp,  sp,  #64
1071         mov             r0,  sp
1072         sub             r1,  r1,  #2
1073         mov             r3,  #8
1074         mov             ip,  #8
1075         vpush           {d8-d15}
1076         bl              put_h264_qpel8_h_lowpass_neon
1077         ldrd            r0,  [sp, #128]
1078         mov             r3,  r2
1079         add             ip,  sp,  #64
1080         sub             r1,  r1,  r2, lsl #1
1081         mov             r2,  #8
1082         bl              put_h264_qpel8_v_lowpass_l2_neon
1083         vpop            {d8-d15}
1084         add             sp,  sp,  #76
1085         pop             {pc}
1086         .endfunc
1087
1088 function ff_put_h264_qpel8_mc21_neon, export=1
1089         push            {r0, r1, r4, r10, r11, lr}
1090 put_h264_qpel8_mc21:
1091         lowpass_const   r3
1092         mov             r11, sp
1093         bic             sp,  sp,  #15
1094         sub             sp,  sp,  #(8*8+16*12)
1095         sub             r1,  r1,  #2
1096         mov             r3,  #8
1097         mov             r0,  sp
1098         mov             ip,  #8
1099         vpush           {d8-d15}
1100         bl              put_h264_qpel8_h_lowpass_neon
1101         mov             r4,  r0
1102         ldrd            r0,  [r11]
1103         sub             r1,  r1,  r2, lsl #1
1104         sub             r1,  r1,  #2
1105         mov             r3,  r2
1106         sub             r2,  r4,  #64
1107         bl              put_h264_qpel8_hv_lowpass_l2_neon
1108         vpop            {d8-d15}
1109         add             sp,  r11,  #8
1110         pop             {r4, r10, r11, pc}
1111         .endfunc
1112
1113 function ff_put_h264_qpel8_mc31_neon, export=1
1114         add             r1,  r1,  #1
1115         push            {r0, r1, r2, lr}
1116         sub             r1,  r1,  #1
1117         b               put_h264_qpel8_mc11
1118         .endfunc
1119
1120 function ff_put_h264_qpel8_mc02_neon, export=1
1121         push            {lr}
1122         lowpass_const   r3
1123         sub             r1,  r1,  r2, lsl #1
1124         mov             r3,  r2
1125         vpush           {d8-d15}
1126         bl              put_h264_qpel8_v_lowpass_neon
1127         vpop            {d8-d15}
1128         pop             {pc}
1129         .endfunc
1130
1131 function ff_put_h264_qpel8_mc12_neon, export=1
1132         push            {r0, r1, r4, r10, r11, lr}
1133 put_h264_qpel8_mc12:
1134         lowpass_const   r3
1135         mov             r11, sp
1136         bic             sp,  sp,  #15
1137         sub             sp,  sp,  #(8*8+16*12)
1138         sub             r1,  r1,  r2, lsl #1
1139         mov             r3,  r2
1140         mov             r2,  #8
1141         mov             r0,  sp
1142         vpush           {d8-d15}
1143         bl              put_h264_qpel8_v_lowpass_neon
1144         mov             r4,  r0
1145         ldrd            r0,  [r11]
1146         sub             r1,  r1,  r3, lsl #1
1147         sub             r1,  r1,  #2
1148         sub             r2,  r4,  #64
1149         bl              put_h264_qpel8_hv_lowpass_l2_neon
1150         vpop            {d8-d15}
1151         add             sp,  r11,  #8
1152         pop             {r4, r10, r11, pc}
1153         .endfunc
1154
1155 function ff_put_h264_qpel8_mc22_neon, export=1
1156         push            {r4, r10, r11, lr}
1157         mov             r11, sp
1158         bic             sp,  sp,  #15
1159         sub             r1,  r1,  r2, lsl #1
1160         sub             r1,  r1,  #2
1161         mov             r3,  r2
1162         sub             sp,  sp,  #(16*12)
1163         mov             r4,  sp
1164         vpush           {d8-d15}
1165         bl              put_h264_qpel8_hv_lowpass_neon
1166         vpop            {d8-d15}
1167         mov             sp,  r11
1168         pop             {r4, r10, r11, pc}
1169         .endfunc
1170
1171 function ff_put_h264_qpel8_mc32_neon, export=1
1172         push            {r0, r1, r4, r10, r11, lr}
1173         add             r1,  r1,  #1
1174         b               put_h264_qpel8_mc12
1175         .endfunc
1176
1177 function ff_put_h264_qpel8_mc03_neon, export=1
1178         push            {lr}
1179         add             ip,  r1,  r2
1180         b               put_h264_qpel8_mc01
1181         .endfunc
1182
1183 function ff_put_h264_qpel8_mc13_neon, export=1
1184         push            {r0, r1, r2, lr}
1185         add             r1,  r1,  r2
1186         b               put_h264_qpel8_mc11
1187         .endfunc
1188
1189 function ff_put_h264_qpel8_mc23_neon, export=1
1190         push            {r0, r1, r4, r10, r11, lr}
1191         add             r1,  r1,  r2
1192         b               put_h264_qpel8_mc21
1193         .endfunc
1194
1195 function ff_put_h264_qpel8_mc33_neon, export=1
1196         add             r1,  r1,  #1
1197         push            {r0, r1, r2, lr}
1198         add             r1,  r1,  r2
1199         sub             r1,  r1,  #1
1200         b               put_h264_qpel8_mc11
1201         .endfunc
1202
1203 function ff_put_h264_qpel16_mc10_neon, export=1
1204         lowpass_const   r3
1205         mov             r3,  r1
1206         sub             r1,  r1,  #2
1207         b               put_h264_qpel16_h_lowpass_l2_neon
1208         .endfunc
1209
1210 function ff_put_h264_qpel16_mc20_neon, export=1
1211         lowpass_const   r3
1212         sub             r1,  r1,  #2
1213         mov             r3,  r2
1214         b               put_h264_qpel16_h_lowpass_neon
1215         .endfunc
1216
1217 function ff_put_h264_qpel16_mc30_neon, export=1
1218         lowpass_const   r3
1219         add             r3,  r1,  #1
1220         sub             r1,  r1,  #2
1221         b               put_h264_qpel16_h_lowpass_l2_neon
1222         .endfunc
1223
1224 function ff_put_h264_qpel16_mc01_neon, export=1
1225         push            {r4, lr}
1226         mov             ip,  r1
1227 put_h264_qpel16_mc01:
1228         lowpass_const   r3
1229         mov             r3,  r2
1230         sub             r1,  r1,  r2, lsl #1
1231         vpush           {d8-d15}
1232         bl              put_h264_qpel16_v_lowpass_l2_neon
1233         vpop            {d8-d15}
1234         pop             {r4, pc}
1235         .endfunc
1236
1237 function ff_put_h264_qpel16_mc11_neon, export=1
1238         push            {r0, r1, r4, lr}
1239 put_h264_qpel16_mc11:
1240         lowpass_const   r3
1241         sub             sp,  sp,  #256
1242         mov             r0,  sp
1243         sub             r1,  r1,  #2
1244         mov             r3,  #16
1245         vpush           {d8-d15}
1246         bl              put_h264_qpel16_h_lowpass_neon
1247         add             r0,  sp,  #256
1248         ldrd            r0,  [r0, #64]
1249         mov             r3,  r2
1250         add             ip,  sp,  #64
1251         sub             r1,  r1,  r2, lsl #1
1252         mov             r2,  #16
1253         bl              put_h264_qpel16_v_lowpass_l2_neon
1254         vpop            {d8-d15}
1255         add             sp,  sp,  #(256+8)
1256         pop             {r4, pc}
1257         .endfunc
1258
1259 function ff_put_h264_qpel16_mc21_neon, export=1
1260         push            {r0, r1, r4-r5, r9-r11, lr}
1261 put_h264_qpel16_mc21:
1262         lowpass_const   r3
1263         mov             r11, sp
1264         bic             sp,  sp,  #15
1265         sub             sp,  sp,  #(16*16+16*12)
1266         sub             r1,  r1,  #2
1267         mov             r0,  sp
1268         vpush           {d8-d15}
1269         bl              put_h264_qpel16_h_lowpass_neon_packed
1270         mov             r4,  r0
1271         ldrd            r0,  [r11]
1272         sub             r1,  r1,  r2, lsl #1
1273         sub             r1,  r1,  #2
1274         mov             r3,  r2
1275         bl              put_h264_qpel16_hv_lowpass_l2_neon
1276         vpop            {d8-d15}
1277         add             sp,  r11,  #8
1278         pop             {r4-r5, r9-r11, pc}
1279         .endfunc
1280
1281 function ff_put_h264_qpel16_mc31_neon, export=1
1282         add             r1,  r1,  #1
1283         push            {r0, r1, r4, lr}
1284         sub             r1,  r1,  #1
1285         b               put_h264_qpel16_mc11
1286         .endfunc
1287
1288 function ff_put_h264_qpel16_mc02_neon, export=1
1289         push            {r4, lr}
1290         lowpass_const   r3
1291         sub             r1,  r1,  r2, lsl #1
1292         mov             r3,  r2
1293         vpush           {d8-d15}
1294         bl              put_h264_qpel16_v_lowpass_neon
1295         vpop            {d8-d15}
1296         pop             {r4, pc}
1297         .endfunc
1298
1299 function ff_put_h264_qpel16_mc12_neon, export=1
1300         push            {r0, r1, r4-r5, r9-r11, lr}
1301 put_h264_qpel16_mc12:
1302         lowpass_const   r3
1303         mov             r11, sp
1304         bic             sp,  sp,  #15
1305         sub             sp,  sp,  #(16*16+16*12)
1306         sub             r1,  r1,  r2, lsl #1
1307         mov             r0,  sp
1308         mov             r3,  r2
1309         vpush           {d8-d15}
1310         bl              put_h264_qpel16_v_lowpass_neon_packed
1311         mov             r4,  r0
1312         ldrd            r0,  [r11]
1313         sub             r1,  r1,  r3, lsl #1
1314         sub             r1,  r1,  #2
1315         mov             r2,  r3
1316         bl              put_h264_qpel16_hv_lowpass_l2_neon
1317         vpop            {d8-d15}
1318         add             sp,  r11,  #8
1319         pop             {r4-r5, r9-r11, pc}
1320         .endfunc
1321
1322 function ff_put_h264_qpel16_mc22_neon, export=1
1323         push            {r4, r9-r11, lr}
1324         lowpass_const   r3
1325         mov             r11, sp
1326         bic             sp,  sp,  #15
1327         sub             r1,  r1,  r2, lsl #1
1328         sub             r1,  r1,  #2
1329         mov             r3,  r2
1330         sub             sp,  sp,  #(16*12)
1331         mov             r4,  sp
1332         vpush           {d8-d15}
1333         bl              put_h264_qpel16_hv_lowpass_neon
1334         vpop            {d8-d15}
1335         mov             sp,  r11
1336         pop             {r4, r9-r11, pc}
1337         .endfunc
1338
1339 function ff_put_h264_qpel16_mc32_neon, export=1
1340         push            {r0, r1, r4-r5, r9-r11, lr}
1341         add             r1,  r1,  #1
1342         b               put_h264_qpel16_mc12
1343         .endfunc
1344
1345 function ff_put_h264_qpel16_mc03_neon, export=1
1346         push            {r4, lr}
1347         add             ip,  r1,  r2
1348         b               put_h264_qpel16_mc01
1349         .endfunc
1350
1351 function ff_put_h264_qpel16_mc13_neon, export=1
1352         push            {r0, r1, r4, lr}
1353         add             r1,  r1,  r2
1354         b               put_h264_qpel16_mc11
1355         .endfunc
1356
1357 function ff_put_h264_qpel16_mc23_neon, export=1
1358         push            {r0, r1, r4-r5, r9-r11, lr}
1359         add             r1,  r1,  r2
1360         b               put_h264_qpel16_mc21
1361         .endfunc
1362
1363 function ff_put_h264_qpel16_mc33_neon, export=1
1364         add             r1,  r1,  #1
1365         push            {r0, r1, r4, lr}
1366         add             r1,  r1,  r2
1367         sub             r1,  r1,  #1
1368         b               put_h264_qpel16_mc11
1369         .endfunc
1370
1371 @ Biweighted prediction
1372
1373         .macro  biweight_16 macs, macd
1374         vdup.8          d0,  r4
1375         vdup.8          d1,  r5
1376         vmov            q2,  q8
1377         vmov            q3,  q8
1378 1:      subs            ip,  ip,  #2
1379         vld1.8          {d20-d21},[r0,:128], r2
1380         \macd           q2,  d0,  d20
1381         pld             [r0]
1382         \macd           q3,  d0,  d21
1383         vld1.8          {d22-d23},[r1,:128], r2
1384         \macs           q2,  d1,  d22
1385         pld             [r1]
1386         \macs           q3,  d1,  d23
1387         vmov            q12, q8
1388         vld1.8          {d28-d29},[r0,:128], r2
1389         vmov            q13, q8
1390         \macd           q12, d0,  d28
1391         pld             [r0]
1392         \macd           q13, d0,  d29
1393         vld1.8          {d30-d31},[r1,:128], r2
1394         \macs           q12, d1,  d30
1395         pld             [r1]
1396         \macs           q13, d1,  d31
1397         vshl.s16        q2,  q2,  q9
1398         vshl.s16        q3,  q3,  q9
1399         vqmovun.s16     d4,  q2
1400         vqmovun.s16     d5,  q3
1401         vshl.s16        q12, q12, q9
1402         vshl.s16        q13, q13, q9
1403         vqmovun.s16     d24, q12
1404         vqmovun.s16     d25, q13
1405         vmov            q3,  q8
1406         vst1.8          {d4- d5}, [r6,:128], r2
1407         vmov            q2,  q8
1408         vst1.8          {d24-d25},[r6,:128], r2
1409         bne             1b
1410         pop             {r4-r6, pc}
1411         .endm
1412
1413         .macro  biweight_8 macs, macd
1414         vdup.8          d0,  r4
1415         vdup.8          d1,  r5
1416         vmov            q1,  q8
1417         vmov            q10, q8
1418 1:      subs            ip,  ip,  #2
1419         vld1.8          {d4},[r0,:64], r2
1420         \macd           q1,  d0,  d4
1421         pld             [r0]
1422         vld1.8          {d5},[r1,:64], r2
1423         \macs           q1,  d1,  d5
1424         pld             [r1]
1425         vld1.8          {d6},[r0,:64], r2
1426         \macd           q10, d0,  d6
1427         pld             [r0]
1428         vld1.8          {d7},[r1,:64], r2
1429         \macs           q10, d1,  d7
1430         pld             [r1]
1431         vshl.s16        q1,  q1,  q9
1432         vqmovun.s16     d2,  q1
1433         vshl.s16        q10, q10, q9
1434         vqmovun.s16     d4,  q10
1435         vmov            q10, q8
1436         vst1.8          {d2},[r6,:64], r2
1437         vmov            q1,  q8
1438         vst1.8          {d4},[r6,:64], r2
1439         bne             1b
1440         pop             {r4-r6, pc}
1441         .endm
1442
1443         .macro  biweight_4 macs, macd
1444         vdup.8          d0,  r4
1445         vdup.8          d1,  r5
1446         vmov            q1,  q8
1447         vmov            q10, q8
1448 1:      subs            ip,  ip,  #4
1449         vld1.32         {d4[0]},[r0,:32], r2
1450         vld1.32         {d4[1]},[r0,:32], r2
1451         \macd           q1,  d0,  d4
1452         pld             [r0]
1453         vld1.32         {d5[0]},[r1,:32], r2
1454         vld1.32         {d5[1]},[r1,:32], r2
1455         \macs           q1,  d1,  d5
1456         pld             [r1]
1457         blt             2f
1458         vld1.32         {d6[0]},[r0,:32], r2
1459         vld1.32         {d6[1]},[r0,:32], r2
1460         \macd           q10, d0,  d6
1461         pld             [r0]
1462         vld1.32         {d7[0]},[r1,:32], r2
1463         vld1.32         {d7[1]},[r1,:32], r2
1464         \macs           q10, d1,  d7
1465         pld             [r1]
1466         vshl.s16        q1,  q1,  q9
1467         vqmovun.s16     d2,  q1
1468         vshl.s16        q10, q10, q9
1469         vqmovun.s16     d4,  q10
1470         vmov            q10, q8
1471         vst1.32         {d2[0]},[r6,:32], r2
1472         vst1.32         {d2[1]},[r6,:32], r2
1473         vmov            q1,  q8
1474         vst1.32         {d4[0]},[r6,:32], r2
1475         vst1.32         {d4[1]},[r6,:32], r2
1476         bne             1b
1477         pop             {r4-r6, pc}
1478 2:      vshl.s16        q1,  q1,  q9
1479         vqmovun.s16     d2,  q1
1480         vst1.32         {d2[0]},[r6,:32], r2
1481         vst1.32         {d2[1]},[r6,:32], r2
1482         pop             {r4-r6, pc}
1483         .endm
1484
1485         .macro  biweight_func w
1486 function biweight_h264_pixels_\w\()_neon
1487         push            {r4-r6, lr}
1488         add             r4,  sp,  #16
1489         ldm             r4,  {r4-r6}
1490         lsr             lr,  r4,  #31
1491         add             r6,  r6,  #1
1492         eors            lr,  lr,  r5,  lsr #30
1493         orr             r6,  r6,  #1
1494         vdup.16         q9,  r3
1495         lsl             r6,  r6,  r3
1496         vmvn            q9,  q9
1497         vdup.16         q8,  r6
1498         mov             r6,  r0
1499         beq             10f
1500         subs            lr,  lr,  #1
1501         beq             20f
1502         subs            lr,  lr,  #1
1503         beq             30f
1504         b               40f
1505 10:     biweight_\w     vmlal.u8, vmlal.u8
1506 20:     rsb             r4,  r4,  #0
1507         biweight_\w     vmlal.u8, vmlsl.u8
1508 30:     rsb             r4,  r4,  #0
1509         rsb             r5,  r5,  #0
1510         biweight_\w     vmlsl.u8, vmlsl.u8
1511 40:     rsb             r5,  r5,  #0
1512         biweight_\w     vmlsl.u8, vmlal.u8
1513         .endfunc
1514         .endm
1515
1516         .macro  biweight_entry w, h, b=1
1517 function ff_biweight_h264_pixels_\w\()x\h\()_neon, export=1
1518         mov             ip,  #\h
1519 .if \b
1520         b               biweight_h264_pixels_\w\()_neon
1521 .endif
1522         .endfunc
1523         .endm
1524
1525         biweight_entry  16, 8
1526         biweight_entry  16, 16, b=0
1527         biweight_func   16
1528
1529         biweight_entry  8,  16
1530         biweight_entry  8,  4
1531         biweight_entry  8,  8,  b=0
1532         biweight_func   8
1533
1534         biweight_entry  4,  8
1535         biweight_entry  4,  2
1536         biweight_entry  4,  4,  b=0
1537         biweight_func   4
1538
1539 @ Weighted prediction
1540
1541         .macro  weight_16 add
1542         vdup.8          d0,  r3
1543 1:      subs            ip,  ip,  #2
1544         vld1.8          {d20-d21},[r0,:128], r1
1545         vmull.u8        q2,  d0,  d20
1546         pld             [r0]
1547         vmull.u8        q3,  d0,  d21
1548         vld1.8          {d28-d29},[r0,:128], r1
1549         vmull.u8        q12, d0,  d28
1550         pld             [r0]
1551         vmull.u8        q13, d0,  d29
1552         \add            q2,  q8,  q2
1553         vrshl.s16       q2,  q2,  q9
1554         \add            q3,  q8,  q3
1555         vrshl.s16       q3,  q3,  q9
1556         vqmovun.s16     d4,  q2
1557         vqmovun.s16     d5,  q3
1558         \add            q12, q8,  q12
1559         vrshl.s16       q12, q12, q9
1560         \add            q13, q8,  q13
1561         vrshl.s16       q13, q13, q9
1562         vqmovun.s16     d24, q12
1563         vqmovun.s16     d25, q13
1564         vst1.8          {d4- d5}, [r4,:128], r1
1565         vst1.8          {d24-d25},[r4,:128], r1
1566         bne             1b
1567         pop             {r4, pc}
1568         .endm
1569
1570         .macro  weight_8 add
1571         vdup.8          d0,  r3
1572 1:      subs            ip,  ip,  #2
1573         vld1.8          {d4},[r0,:64], r1
1574         vmull.u8        q1,  d0,  d4
1575         pld             [r0]
1576         vld1.8          {d6},[r0,:64], r1
1577         vmull.u8        q10, d0,  d6
1578         \add            q1,  q8,  q1
1579         pld             [r0]
1580         vrshl.s16       q1,  q1,  q9
1581         vqmovun.s16     d2,  q1
1582         \add            q10, q8,  q10
1583         vrshl.s16       q10, q10, q9
1584         vqmovun.s16     d4,  q10
1585         vst1.8          {d2},[r4,:64], r1
1586         vst1.8          {d4},[r4,:64], r1
1587         bne             1b
1588         pop             {r4, pc}
1589         .endm
1590
1591         .macro  weight_4 add
1592         vdup.8          d0,  r3
1593         vmov            q1,  q8
1594         vmov            q10, q8
1595 1:      subs            ip,  ip,  #4
1596         vld1.32         {d4[0]},[r0,:32], r1
1597         vld1.32         {d4[1]},[r0,:32], r1
1598         vmull.u8        q1,  d0,  d4
1599         pld             [r0]
1600         blt             2f
1601         vld1.32         {d6[0]},[r0,:32], r1
1602         vld1.32         {d6[1]},[r0,:32], r1
1603         vmull.u8        q10, d0,  d6
1604         pld             [r0]
1605         \add            q1,  q8,  q1
1606         vrshl.s16       q1,  q1,  q9
1607         vqmovun.s16     d2,  q1
1608         \add            q10, q8,  q10
1609         vrshl.s16       q10, q10, q9
1610         vqmovun.s16     d4,  q10
1611         vmov            q10, q8
1612         vst1.32         {d2[0]},[r4,:32], r1
1613         vst1.32         {d2[1]},[r4,:32], r1
1614         vmov            q1,  q8
1615         vst1.32         {d4[0]},[r4,:32], r1
1616         vst1.32         {d4[1]},[r4,:32], r1
1617         bne             1b
1618         pop             {r4, pc}
1619 2:      \add            q1,  q8,  q1
1620         vrshl.s16       q1,  q1,  q9
1621         vqmovun.s16     d2,  q1
1622         vst1.32         {d2[0]},[r4,:32], r1
1623         vst1.32         {d2[1]},[r4,:32], r1
1624         pop             {r4, pc}
1625         .endm
1626
1627         .macro  weight_func w
1628 function weight_h264_pixels_\w\()_neon
1629         push            {r4, lr}
1630         ldr             r4,  [sp, #8]
1631         cmp             r2,  #1
1632         lsl             r4,  r4,  r2
1633         vdup.16         q8,  r4
1634         mov             r4,  r0
1635         ble             20f
1636         rsb             lr,  r2,  #1
1637         vdup.16         q9,  lr
1638         cmp             r3,  #0
1639         blt             10f
1640         weight_\w       vhadd.s16
1641 10:     rsb             r3,  r3,  #0
1642         weight_\w       vhsub.s16
1643 20:     rsb             lr,  r2,  #0
1644         vdup.16         q9,  lr
1645         cmp             r3,  #0
1646         blt             10f
1647         weight_\w       vadd.s16
1648 10:     rsb             r3,  r3,  #0
1649         weight_\w       vsub.s16
1650         .endfunc
1651         .endm
1652
1653         .macro  weight_entry w, h, b=1
1654 function ff_weight_h264_pixels_\w\()x\h\()_neon, export=1
1655         mov             ip,  #\h
1656 .if \b
1657         b               weight_h264_pixels_\w\()_neon
1658 .endif
1659         .endfunc
1660         .endm
1661
1662         weight_entry    16, 8
1663         weight_entry    16, 16, b=0
1664         weight_func     16
1665
1666         weight_entry    8,  16
1667         weight_entry    8,  4
1668         weight_entry    8,  8,  b=0
1669         weight_func     8
1670
1671         weight_entry    4,  8
1672         weight_entry    4,  2
1673         weight_entry    4,  4,  b=0
1674         weight_func     4