git.sesse.net Git - ffmpeg/blob - libavcodec/aarch64/h264cmc_neon.S

   1 /*
   2  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
   3  * Copyright (c) 2013 Janne Grunau <janne-libav@jannau.net>
   4  *
   5  * This file is part of Libav.
   6  *
   7  * Libav is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2.1 of the License, or (at your option) any later version.
  11  *
  12  * Libav is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with Libav; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include "libavutil/aarch64/asm.S"
  23
  24 /* chroma_mc8(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
  25 .macro  h264_chroma_mc8 type, codec=h264
  26 function ff_\type\()_\codec\()_chroma_mc8_neon, export=1
  27         sxtw            x2,  w2
  28   .ifc \type,avg
  29         mov             x8,  x0
  30   .endif
  31         prfm            pldl1strm, [x1]
  32         prfm            pldl1strm, [x1, x2]
  33   .ifc \codec,rv40
  34         movrel          x6,  rv40bias
  35         lsr             w9,  w5,  #1
  36         lsr             w10, w4,  #1
  37         lsl             w9,  w9,  #3
  38         lsl             w10, w10, #1
  39         add             w9,  w9,  w10
  40         add             x6,  x6,  w9, UXTW
  41         ld1r            {v22.8H}, [x6]
  42   .endif
  43   .ifc \codec,vc1
  44         movi            v22.8H,   #28
  45   .endif
  46         mul             w7,  w4,  w5
  47         lsl             w14, w5,  #3
  48         lsl             w13, w4,  #3
  49         cmp             w7,  #0
  50         sub             w6,  w14, w7
  51         sub             w12, w13, w7
  52         sub             w4,  w7,  w13
  53         sub             w4,  w4,  w14
  54         add             w4,  w4,  #64
  55         b.eq            2f
  56
  57         dup             v0.8B,  w4
  58         dup             v1.8B,  w12
  59         ld1             {v4.8B, v5.8B}, [x1], x2
  60         dup             v2.8B,  w6
  61         dup             v3.8B,  w7
  62         ext             v5.8B,  v4.8B,  v5.8B,  #1
  63 1:      ld1             {v6.8B, v7.8B}, [x1], x2
  64         umull           v16.8H, v4.8B,  v0.8B
  65         umlal           v16.8H, v5.8B,  v1.8B
  66         ext             v7.8B,  v6.8B,  v7.8B,  #1
  67         ld1             {v4.8B, v5.8B}, [x1], x2
  68         umlal           v16.8H, v6.8B,  v2.8B
  69         prfm            pldl1strm, [x1]
  70         ext             v5.8B,  v4.8B,  v5.8B,  #1
  71         umlal           v16.8H, v7.8B,  v3.8B
  72         umull           v17.8H, v6.8B,  v0.8B
  73         subs            w3,  w3,  #2
  74         umlal           v17.8H, v7.8B, v1.8B
  75         umlal           v17.8H, v4.8B, v2.8B
  76         umlal           v17.8H, v5.8B, v3.8B
  77         prfm            pldl1strm, [x1, x2]
  78   .ifc \codec,h264
  79         rshrn           v16.8B, v16.8H, #6
  80         rshrn           v17.8B, v17.8H, #6
  81   .else
  82         add             v16.8H, v16.8H, v22.8H
  83         add             v17.8H, v17.8H, v22.8H
  84         shrn            v16.8B, v16.8H, #6
  85         shrn            v17.8B, v17.8H, #6
  86   .endif
  87   .ifc \type,avg
  88         ld1             {v20.8B}, [x8], x2
  89         ld1             {v21.8B}, [x8], x2
  90         urhadd          v16.8B, v16.8B, v20.8B
  91         urhadd          v17.8B, v17.8B, v21.8B
  92   .endif
  93         st1             {v16.8B}, [x0], x2
  94         st1             {v17.8B}, [x0], x2
  95         b.gt            1b
  96         ret
  97
  98 2:      tst             w6,  w6
  99         add             w12, w12, w6
 100         dup             v0.8B, w4
 101         dup             v1.8B, w12
 102         b.eq            4f
 103
 104         ld1             {v4.8B}, [x1], x2
 105 3:      ld1             {v6.8B}, [x1], x2
 106         umull           v16.8H, v4.8B,  v0.8B
 107         umlal           v16.8H, v6.8B,  v1.8B
 108         ld1             {v4.8B}, [x1], x2
 109         umull           v17.8H, v6.8B,  v0.8B
 110         umlal           v17.8H, v4.8B,  v1.8B
 111         prfm            pldl1strm, [x1]
 112   .ifc \codec,h264
 113         rshrn           v16.8B, v16.8H, #6
 114         rshrn           v17.8B, v17.8H, #6
 115   .else
 116         add             v16.8H, v16.8H, v22.8H
 117         add             v17.8H, v17.8H, v22.8H
 118         shrn            v16.8B, v16.8H, #6
 119         shrn            v17.8B, v17.8H, #6
 120   .endif
 121         prfm            pldl1strm, [x1, x2]
 122   .ifc \type,avg
 123         ld1             {v20.8B}, [x8], x2
 124         ld1             {v21.8B}, [x8], x2
 125         urhadd          v16.8B, v16.8B, v20.8B
 126         urhadd          v17.8B, v17.8B, v21.8B
 127   .endif
 128         subs            w3,  w3,  #2
 129         st1             {v16.8B}, [x0], x2
 130         st1             {v17.8B}, [x0], x2
 131         b.gt            3b
 132         ret
 133
 134 4:      ld1             {v4.8B, v5.8B}, [x1], x2
 135         ld1             {v6.8B, v7.8B}, [x1], x2
 136         ext             v5.8B,  v4.8B,  v5.8B,  #1
 137         ext             v7.8B,  v6.8B,  v7.8B,  #1
 138         prfm            pldl1strm, [x1]
 139         subs            w3,  w3,  #2
 140         umull           v16.8H, v4.8B, v0.8B
 141         umlal           v16.8H, v5.8B, v1.8B
 142         umull           v17.8H, v6.8B, v0.8B
 143         umlal           v17.8H, v7.8B, v1.8B
 144         prfm            pldl1strm, [x1, x2]
 145   .ifc \codec,h264
 146         rshrn           v16.8B, v16.8H, #6
 147         rshrn           v17.8B, v17.8H, #6
 148   .else
 149         add             v16.8H, v16.8H, v22.8H
 150         add             v17.8H, v17.8H, v22.8H
 151         shrn            v16.8B, v16.8H, #6
 152         shrn            v17.8B, v17.8H, #6
 153   .endif
 154   .ifc \type,avg
 155         ld1             {v20.8B}, [x8], x2
 156         ld1             {v21.8B}, [x8], x2
 157         urhadd          v16.8B, v16.8B, v20.8B
 158         urhadd          v17.8B, v17.8B, v21.8B
 159   .endif
 160         st1             {v16.8B}, [x0], x2
 161         st1             {v17.8B}, [x0], x2
 162         b.gt            4b
 163         ret
 164 endfunc
 165 .endm
 166
 167 /* chroma_mc4(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
 168 .macro  h264_chroma_mc4 type, codec=h264
 169 function ff_\type\()_\codec\()_chroma_mc4_neon, export=1
 170         sxtw            x2,  w2
 171   .ifc \type,avg
 172         mov             x8,  x0
 173   .endif
 174         prfm            pldl1strm, [x1]
 175         prfm            pldl1strm, [x1, x2]
 176   .ifc \codec,rv40
 177         movrel          x6,  rv40bias
 178         lsr             w9,  w5,  #1
 179         lsr             w10, w4,  #1
 180         lsl             w9,  w9,  #3
 181         lsl             w10, w10, #1
 182         add             w9,  w9,  w10
 183         add             x6,  x6,  w9, UXTW
 184         ld1r            {v22.8H}, [x6]
 185   .endif
 186   .ifc \codec,vc1
 187         movi            v22.8H,   #28
 188   .endif
 189         mul             w7,  w4,  w5
 190         lsl             w14, w5,  #3
 191         lsl             w13, w4,  #3
 192         cmp             w7,  #0
 193         sub             w6,  w14, w7
 194         sub             w12, w13, w7
 195         sub             w4,  w7,  w13
 196         sub             w4,  w4,  w14
 197         add             w4,  w4,  #64
 198         b.eq            2f
 199
 200         dup             v24.8B,  w4
 201         dup             v25.8B,  w12
 202         ld1             {v4.8B}, [x1], x2
 203         dup             v26.8B,  w6
 204         dup             v27.8B,  w7
 205         ext             v5.8B,  v4.8B,  v5.8B, #1
 206         trn1            v0.2S,  v24.2S, v25.2S
 207         trn1            v2.2S,  v26.2S, v27.2S
 208         trn1            v4.2S,  v4.2S,  v5.2S
 209 1:      ld1             {v6.8B}, [x1], x2
 210         ext             v7.8B,  v6.8B,  v7.8B, #1
 211         trn1            v6.2S,  v6.2S,  v7.2S
 212         umull           v18.8H, v4.8B,  v0.8B
 213         umlal           v18.8H, v6.8B,  v2.8B
 214         ld1             {v4.8B}, [x1], x2
 215         ext             v5.8B,  v4.8B,  v5.8B, #1
 216         trn1            v4.2S,  v4.2S,  v5.2S
 217         prfm            pldl1strm, [x1]
 218         umull           v19.8H, v6.8B,  v0.8B
 219         umlal           v19.8H, v4.8B,  v2.8B
 220         trn1            v30.2D, v18.2D, v19.2D
 221         trn2            v31.2D, v18.2D, v19.2D
 222         add             v18.8H, v30.8H, v31.8H
 223   .ifc \codec,h264
 224         rshrn           v16.8B, v18.8H, #6
 225   .else
 226         add             v18.8H, v18.8H, v22.8H
 227         shrn            v16.8B, v18.8H, #6
 228   .endif
 229         subs            w3,  w3,  #2
 230         prfm            pldl1strm, [x1, x2]
 231   .ifc \type,avg
 232         ld1             {v20.S}[0], [x8], x2
 233         ld1             {v20.S}[1], [x8], x2
 234         urhadd          v16.8B, v16.8B, v20.8B
 235   .endif
 236         st1             {v16.S}[0], [x0], x2
 237         st1             {v16.S}[1], [x0], x2
 238         b.gt            1b
 239         ret
 240
 241 2:      tst             w6,  w6
 242         add             w12, w12, w6
 243         dup             v30.8B, w4
 244         dup             v31.8B, w12
 245         trn1            v0.2S,  v30.2S, v31.2S
 246         trn2            v1.2S,  v30.2S, v31.2S
 247         b.eq            4f
 248
 249         ext             v1.8B,  v0.8B,  v1.8B, #4
 250         ld1             {v4.S}[0], [x1], x2
 251 3:      ld1             {v4.S}[1], [x1], x2
 252         umull           v18.8H, v4.8B,  v0.8B
 253         ld1             {v4.S}[0], [x1], x2
 254         umull           v19.8H, v4.8B,  v1.8B
 255         trn1            v30.2D, v18.2D, v19.2D
 256         trn2            v31.2D, v18.2D, v19.2D
 257         add             v18.8H, v30.8H, v31.8H
 258         prfm            pldl1strm, [x1]
 259   .ifc \codec,h264
 260         rshrn           v16.8B, v18.8H, #6
 261   .else
 262         add             v18.8H, v18.8H, v22.8H
 263         shrn            v16.8B, v18.8H, #6
 264   .endif
 265   .ifc \type,avg
 266         ld1             {v20.S}[0], [x8], x2
 267         ld1             {v20.S}[1], [x8], x2
 268         urhadd          v16.8B, v16.8B, v20.8B
 269   .endif
 270         subs            w3,  w3,  #2
 271         prfm            pldl1strm, [x1, x2]
 272         st1             {v16.S}[0], [x0], x2
 273         st1             {v16.S}[1], [x0], x2
 274         b.gt            3b
 275         ret
 276
 277 4:      ld1             {v4.8B}, [x1], x2
 278         ld1             {v6.8B}, [x1], x2
 279         ext             v5.8B,  v4.8B,  v5.8B, #1
 280         ext             v7.8B,  v6.8B,  v7.8B, #1
 281         trn1            v4.2S,  v4.2S,  v5.2S
 282         trn1            v6.2S,  v6.2S,  v7.2S
 283         umull           v18.8H, v4.8B,  v0.8B
 284         umull           v19.8H, v6.8B,  v0.8B
 285         subs            w3,  w3,  #2
 286         trn1            v30.2D, v18.2D, v19.2D
 287         trn2            v31.2D, v18.2D, v19.2D
 288         add             v18.8H, v30.8H, v31.8H
 289         prfm            pldl1strm, [x1]
 290   .ifc \codec,h264
 291         rshrn           v16.8B, v18.8H, #6
 292   .else
 293         add             v18.8H, v18.8H, v22.8H
 294         shrn            v16.8B, v18.8H, #6
 295   .endif
 296   .ifc \type,avg
 297         ld1             {v20.S}[0], [x8], x2
 298         ld1             {v20.S}[1], [x8], x2
 299         urhadd          v16.8B, v16.8B, v20.8B
 300   .endif
 301         prfm            pldl1strm, [x1]
 302         st1             {v16.S}[0], [x0], x2
 303         st1             {v16.S}[1], [x0], x2
 304         b.gt            4b
 305         ret
 306 endfunc
 307 .endm
 308
 309 .macro  h264_chroma_mc2 type
 310 function ff_\type\()_h264_chroma_mc2_neon, export=1
 311         sxtw            x2,  w2
 312         prfm            pldl1strm, [x1]
 313         prfm            pldl1strm, [x1, x2]
 314         orr             w7,  w4,  w5
 315         cbz             w7,  2f
 316
 317         mul             w7,  w4,  w5
 318         lsl             w14, w5,  #3
 319         lsl             w13, w4,  #3
 320         sub             w6,  w14, w7
 321         sub             w12, w13, w7
 322         sub             w4,  w7,  w13
 323         sub             w4,  w4,  w14
 324         add             w4,  w4,  #64
 325         dup             v0.8B,  w4
 326         dup             v2.8B,  w12
 327         dup             v1.8B,  w6
 328         dup             v3.8B,  w7
 329         trn1            v0.4H,  v0.4H,  v2.4H
 330         trn1            v1.4H,  v1.4H,  v3.4H
 331 1:
 332         ld1             {v4.S}[0],  [x1], x2
 333         ld1             {v4.S}[1],  [x1], x2
 334         rev64           v5.2S,  v4.2S
 335         ld1             {v5.S}[1],  [x1]
 336         ext             v6.8B,  v4.8B,  v5.8B,  #1
 337         ext             v7.8B,  v5.8B,  v4.8B,  #1
 338         trn1            v4.4H,  v4.4H,  v6.4H
 339         trn1            v5.4H,  v5.4H,  v7.4H
 340         umull           v16.8H, v4.8B,  v0.8B
 341         umlal           v16.8H, v5.8B,  v1.8B
 342   .ifc \type,avg
 343         ld1             {v18.H}[0], [x0], x2
 344         ld1             {v18.H}[2], [x0]
 345         sub             x0,  x0,  x2
 346   .endif
 347         rev64           v17.4S, v16.4S
 348         add             v16.8H, v16.8H, v17.8H
 349         rshrn           v16.8B, v16.8H, #6
 350   .ifc \type,avg
 351         urhadd          v16.8B, v16.8B, v18.8B
 352   .endif
 353         st1             {v16.H}[0], [x0], x2
 354         st1             {v16.H}[2], [x0], x2
 355         subs            w3,  w3,  #2
 356         b.gt            1b
 357         ret
 358
 359 2:
 360         ld1             {v16.H}[0], [x1], x2
 361         ld1             {v16.H}[1], [x1], x2
 362   .ifc \type,avg
 363         ld1             {v18.H}[0], [x0], x2
 364         ld1             {v18.H}[1], [x0]
 365         sub             x0,  x0,  x2
 366         urhadd          v16.8B, v16.8B, v18.8B
 367   .endif
 368         st1             {v16.H}[0], [x0], x2
 369         st1             {v16.H}[1], [x0], x2
 370         subs            w3,  w3,  #2
 371         b.gt            2b
 372         ret
 373 endfunc
 374 .endm
 375
 376         h264_chroma_mc8 put
 377         h264_chroma_mc8 avg
 378         h264_chroma_mc4 put
 379         h264_chroma_mc4 avg
 380         h264_chroma_mc2 put
 381         h264_chroma_mc2 avg
 382
 383 #if CONFIG_RV40_DECODER
 384 const   rv40bias
 385         .short           0, 16, 32, 16
 386         .short          32, 28, 32, 28
 387         .short           0, 32, 16, 32
 388         .short          32, 28, 32, 28
 389 endconst
 390
 391         h264_chroma_mc8 put, rv40
 392         h264_chroma_mc8 avg, rv40
 393         h264_chroma_mc4 put, rv40
 394         h264_chroma_mc4 avg, rv40
 395 #endif
 396
 397 #if CONFIG_VC1_DECODER
 398         h264_chroma_mc8 put, vc1
 399         h264_chroma_mc8 avg, vc1
 400         h264_chroma_mc4 put, vc1
 401         h264_chroma_mc4 avg, vc1
 402 #endif