git.sesse.net Git - ffmpeg/blob - libavcodec/x86/mpegvideoenc_template.c

   1 /*
   2  * MPEG video MMX templates
   3  *
   4  * Copyright (c) 2002 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * This file is part of Libav.
   7  *
   8  * Libav is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * Libav is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with Libav; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include <stdint.h>
  24
  25 #include "libavutil/internal.h"
  26 #include "libavutil/x86/asm.h"
  27 #include "libavcodec/mpegvideo.h"
  28
  29 #undef MMREG_WIDTH
  30 #undef MM
  31 #undef MOVQ
  32 #undef SPREADW
  33 #undef PMAXW
  34 #undef PMAX
  35 #undef SAVE_SIGN
  36 #undef RESTORE_SIGN
  37
  38 #if COMPILE_TEMPLATE_SSE2
  39 #define MMREG_WIDTH "16"
  40 #define MM "%%xmm"
  41 #define MOVQ "movdqa"
  42 #define SPREADW(a) \
  43             "pshuflw $0, "a", "a"       \n\t"\
  44             "punpcklwd "a", "a"         \n\t"
  45 #define PMAXW(a,b) "pmaxsw "a", "b"     \n\t"
  46 #define PMAX(a,b) \
  47             "movhlps "a", "b"           \n\t"\
  48             PMAXW(b, a)\
  49             "pshuflw $0x0E, "a", "b"    \n\t"\
  50             PMAXW(b, a)\
  51             "pshuflw $0x01, "a", "b"    \n\t"\
  52             PMAXW(b, a)
  53 #else
  54 #define MMREG_WIDTH "8"
  55 #define MM "%%mm"
  56 #define MOVQ "movq"
  57 #if COMPILE_TEMPLATE_MMXEXT
  58 #define SPREADW(a) "pshufw $0, "a", "a" \n\t"
  59 #define PMAXW(a,b) "pmaxsw "a", "b"     \n\t"
  60 #define PMAX(a,b) \
  61             "pshufw $0x0E, "a", "b"     \n\t"\
  62             PMAXW(b, a)\
  63             "pshufw $0x01, "a", "b"     \n\t"\
  64             PMAXW(b, a)
  65 #else
  66 #define SPREADW(a) \
  67             "punpcklwd "a", "a"         \n\t"\
  68             "punpcklwd "a", "a"         \n\t"
  69 #define PMAXW(a,b) \
  70             "psubusw "a", "b"           \n\t"\
  71             "paddw "a", "b"             \n\t"
  72 #define PMAX(a,b)  \
  73             "movq "a", "b"              \n\t"\
  74             "psrlq $32, "a"             \n\t"\
  75             PMAXW(b, a)\
  76             "movq "a", "b"              \n\t"\
  77             "psrlq $16, "a"             \n\t"\
  78             PMAXW(b, a)
  79
  80 #endif
  81 #endif
  82
  83 #if COMPILE_TEMPLATE_SSSE3
  84 #define SAVE_SIGN(a,b) \
  85             "movdqa "b", "a"            \n\t"\
  86             "pabsw  "b", "b"            \n\t"
  87 #define RESTORE_SIGN(a,b) \
  88             "psignw "a", "b"            \n\t"
  89 #else
  90 #define SAVE_SIGN(a,b) \
  91             "pxor "a", "a"              \n\t"\
  92             "pcmpgtw "b", "a"           \n\t" /* block[i] <= 0 ? 0xFF : 0x00 */\
  93             "pxor "a", "b"              \n\t"\
  94             "psubw "a", "b"             \n\t" /* ABS(block[i]) */
  95 #define RESTORE_SIGN(a,b) \
  96             "pxor "a", "b"              \n\t"\
  97             "psubw "a", "b"             \n\t" // out=((ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16)*sign(block[i])
  98 #endif
  99
 100 static int RENAME(dct_quantize)(MpegEncContext *s,
 101                             int16_t *block, int n,
 102                             int qscale, int *overflow)
 103 {
 104     x86_reg last_non_zero_p1;
 105     int level=0, q; //=0 is because gcc says uninitialized ...
 106     const uint16_t *qmat, *bias;
 107     LOCAL_ALIGNED_16(int16_t, temp_block, [64]);
 108
 109     assert((7&(int)(&temp_block[0])) == 0); //did gcc align it correctly?
 110
 111     //s->fdct (block);
 112     RENAMEl(ff_fdct) (block); //cannot be anything else ...
 113
 114     if(s->dct_error_sum)
 115         s->denoise_dct(s, block);
 116
 117     if (s->mb_intra) {
 118         int dummy;
 119         if (n < 4)
 120             q = s->y_dc_scale;
 121         else
 122             q = s->c_dc_scale;
 123         /* note: block[0] is assumed to be positive */
 124         if (!s->h263_aic) {
 125         __asm__ volatile (
 126                 "mul %%ecx                \n\t"
 127                 : "=d" (level), "=a"(dummy)
 128                 : "a" ((block[0]>>2) + q), "c" (ff_inverse[q<<1])
 129         );
 130         } else
 131             /* For AIC we skip quant/dequant of INTRADC */
 132             level = (block[0] + 4)>>3;
 133
 134         block[0]=0; //avoid fake overflow
 135 //        temp_block[0] = (block[0] + (q >> 1)) / q;
 136         last_non_zero_p1 = 1;
 137         bias = s->q_intra_matrix16[qscale][1];
 138         qmat = s->q_intra_matrix16[qscale][0];
 139     } else {
 140         last_non_zero_p1 = 0;
 141         bias = s->q_inter_matrix16[qscale][1];
 142         qmat = s->q_inter_matrix16[qscale][0];
 143     }
 144
 145     if((s->out_format == FMT_H263 || s->out_format == FMT_H261) && s->mpeg_quant==0){
 146
 147         __asm__ volatile(
 148             "movd %%"REG_a", "MM"3              \n\t" // last_non_zero_p1
 149             SPREADW(MM"3")
 150             "pxor "MM"7, "MM"7                  \n\t" // 0
 151             "pxor "MM"4, "MM"4                  \n\t" // 0
 152             MOVQ" (%2), "MM"5                   \n\t" // qmat[0]
 153             "pxor "MM"6, "MM"6                  \n\t"
 154             "psubw (%3), "MM"6                  \n\t" // -bias[0]
 155             "mov $-128, %%"REG_a"               \n\t"
 156             ".p2align 4                         \n\t"
 157             "1:                                 \n\t"
 158             MOVQ" (%1, %%"REG_a"), "MM"0        \n\t" // block[i]
 159             SAVE_SIGN(MM"1", MM"0")                   // ABS(block[i])
 160             "psubusw "MM"6, "MM"0               \n\t" // ABS(block[i]) + bias[0]
 161             "pmulhw "MM"5, "MM"0                \n\t" // (ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16
 162             "por "MM"0, "MM"4                   \n\t"
 163             RESTORE_SIGN(MM"1", MM"0")                // out=((ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16)*sign(block[i])
 164             MOVQ" "MM"0, (%5, %%"REG_a")        \n\t"
 165             "pcmpeqw "MM"7, "MM"0               \n\t" // out==0 ? 0xFF : 0x00
 166             MOVQ" (%4, %%"REG_a"), "MM"1        \n\t"
 167             MOVQ" "MM"7, (%1, %%"REG_a")        \n\t" // 0
 168             "pandn "MM"1, "MM"0                 \n\t"
 169             PMAXW(MM"0", MM"3")
 170             "add $"MMREG_WIDTH", %%"REG_a"      \n\t"
 171             " js 1b                             \n\t"
 172             PMAX(MM"3", MM"0")
 173             "movd "MM"3, %%"REG_a"              \n\t"
 174             "movzb %%al, %%"REG_a"              \n\t" // last_non_zero_p1
 175             : "+a" (last_non_zero_p1)
 176             : "r" (block+64), "r" (qmat), "r" (bias),
 177               "r" (inv_zigzag_direct16 + 64), "r" (temp_block + 64)
 178               XMM_CLOBBERS_ONLY("%xmm0", "%xmm1", "%xmm2", "%xmm3",
 179                                 "%xmm4", "%xmm5", "%xmm6", "%xmm7")
 180         );
 181     }else{ // FMT_H263
 182         __asm__ volatile(
 183             "movd %%"REG_a", "MM"3              \n\t" // last_non_zero_p1
 184             SPREADW(MM"3")
 185             "pxor "MM"7, "MM"7                  \n\t" // 0
 186             "pxor "MM"4, "MM"4                  \n\t" // 0
 187             "mov $-128, %%"REG_a"               \n\t"
 188             ".p2align 4                         \n\t"
 189             "1:                                 \n\t"
 190             MOVQ" (%1, %%"REG_a"), "MM"0        \n\t" // block[i]
 191             SAVE_SIGN(MM"1", MM"0")                   // ABS(block[i])
 192             MOVQ" (%3, %%"REG_a"), "MM"6        \n\t" // bias[0]
 193             "paddusw "MM"6, "MM"0               \n\t" // ABS(block[i]) + bias[0]
 194             MOVQ" (%2, %%"REG_a"), "MM"5        \n\t" // qmat[i]
 195             "pmulhw "MM"5, "MM"0                \n\t" // (ABS(block[i])*qmat[0] + bias[0]*qmat[0])>>16
 196             "por "MM"0, "MM"4                   \n\t"
 197             RESTORE_SIGN(MM"1", MM"0")                // out=((ABS(block[i])*qmat[0] - bias[0]*qmat[0])>>16)*sign(block[i])
 198             MOVQ" "MM"0, (%5, %%"REG_a")        \n\t"
 199             "pcmpeqw "MM"7, "MM"0               \n\t" // out==0 ? 0xFF : 0x00
 200             MOVQ" (%4, %%"REG_a"), "MM"1        \n\t"
 201             MOVQ" "MM"7, (%1, %%"REG_a")        \n\t" // 0
 202             "pandn "MM"1, "MM"0                 \n\t"
 203             PMAXW(MM"0", MM"3")
 204             "add $"MMREG_WIDTH", %%"REG_a"      \n\t"
 205             " js 1b                             \n\t"
 206             PMAX(MM"3", MM"0")
 207             "movd "MM"3, %%"REG_a"              \n\t"
 208             "movzb %%al, %%"REG_a"              \n\t" // last_non_zero_p1
 209             : "+a" (last_non_zero_p1)
 210             : "r" (block+64), "r" (qmat+64), "r" (bias+64),
 211               "r" (inv_zigzag_direct16 + 64), "r" (temp_block + 64)
 212               XMM_CLOBBERS_ONLY("%xmm0", "%xmm1", "%xmm2", "%xmm3",
 213                                 "%xmm4", "%xmm5", "%xmm6", "%xmm7")
 214         );
 215     }
 216     __asm__ volatile(
 217         "movd %1, "MM"1                     \n\t" // max_qcoeff
 218         SPREADW(MM"1")
 219         "psubusw "MM"1, "MM"4               \n\t"
 220         "packuswb "MM"4, "MM"4              \n\t"
 221 #if COMPILE_TEMPLATE_SSE2
 222         "packuswb "MM"4, "MM"4              \n\t"
 223 #endif
 224         "movd "MM"4, %0                     \n\t" // *overflow
 225         : "=g" (*overflow)
 226         : "g" (s->max_qcoeff)
 227     );
 228
 229     if(s->mb_intra) block[0]= level;
 230     else            block[0]= temp_block[0];
 231
 232     if (s->idsp.idct_permutation_type == FF_SIMPLE_IDCT_PERM) {
 233         if(last_non_zero_p1 <= 1) goto end;
 234         block[0x08] = temp_block[0x01]; block[0x10] = temp_block[0x08];
 235         block[0x20] = temp_block[0x10];
 236         if(last_non_zero_p1 <= 4) goto end;
 237         block[0x18] = temp_block[0x09]; block[0x04] = temp_block[0x02];
 238         block[0x09] = temp_block[0x03];
 239         if(last_non_zero_p1 <= 7) goto end;
 240         block[0x14] = temp_block[0x0A]; block[0x28] = temp_block[0x11];
 241         block[0x12] = temp_block[0x18]; block[0x02] = temp_block[0x20];
 242         if(last_non_zero_p1 <= 11) goto end;
 243         block[0x1A] = temp_block[0x19]; block[0x24] = temp_block[0x12];
 244         block[0x19] = temp_block[0x0B]; block[0x01] = temp_block[0x04];
 245         block[0x0C] = temp_block[0x05];
 246         if(last_non_zero_p1 <= 16) goto end;
 247         block[0x11] = temp_block[0x0C]; block[0x29] = temp_block[0x13];
 248         block[0x16] = temp_block[0x1A]; block[0x0A] = temp_block[0x21];
 249         block[0x30] = temp_block[0x28]; block[0x22] = temp_block[0x30];
 250         block[0x38] = temp_block[0x29]; block[0x06] = temp_block[0x22];
 251         if(last_non_zero_p1 <= 24) goto end;
 252         block[0x1B] = temp_block[0x1B]; block[0x21] = temp_block[0x14];
 253         block[0x1C] = temp_block[0x0D]; block[0x05] = temp_block[0x06];
 254         block[0x0D] = temp_block[0x07]; block[0x15] = temp_block[0x0E];
 255         block[0x2C] = temp_block[0x15]; block[0x13] = temp_block[0x1C];
 256         if(last_non_zero_p1 <= 32) goto end;
 257         block[0x0B] = temp_block[0x23]; block[0x34] = temp_block[0x2A];
 258         block[0x2A] = temp_block[0x31]; block[0x32] = temp_block[0x38];
 259         block[0x3A] = temp_block[0x39]; block[0x26] = temp_block[0x32];
 260         block[0x39] = temp_block[0x2B]; block[0x03] = temp_block[0x24];
 261         if(last_non_zero_p1 <= 40) goto end;
 262         block[0x1E] = temp_block[0x1D]; block[0x25] = temp_block[0x16];
 263         block[0x1D] = temp_block[0x0F]; block[0x2D] = temp_block[0x17];
 264         block[0x17] = temp_block[0x1E]; block[0x0E] = temp_block[0x25];
 265         block[0x31] = temp_block[0x2C]; block[0x2B] = temp_block[0x33];
 266         if(last_non_zero_p1 <= 48) goto end;
 267         block[0x36] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
 268         block[0x23] = temp_block[0x34]; block[0x3C] = temp_block[0x2D];
 269         block[0x07] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
 270         block[0x0F] = temp_block[0x27]; block[0x35] = temp_block[0x2E];
 271         if(last_non_zero_p1 <= 56) goto end;
 272         block[0x2E] = temp_block[0x35]; block[0x33] = temp_block[0x3C];
 273         block[0x3E] = temp_block[0x3D]; block[0x27] = temp_block[0x36];
 274         block[0x3D] = temp_block[0x2F]; block[0x2F] = temp_block[0x37];
 275         block[0x37] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
 276     }else{
 277         if(last_non_zero_p1 <= 1) goto end;
 278         block[0x01] = temp_block[0x01];
 279         block[0x08] = temp_block[0x08]; block[0x10] = temp_block[0x10];
 280         if(last_non_zero_p1 <= 4) goto end;
 281         block[0x09] = temp_block[0x09]; block[0x02] = temp_block[0x02];
 282         block[0x03] = temp_block[0x03];
 283         if(last_non_zero_p1 <= 7) goto end;
 284         block[0x0A] = temp_block[0x0A]; block[0x11] = temp_block[0x11];
 285         block[0x18] = temp_block[0x18]; block[0x20] = temp_block[0x20];
 286         if(last_non_zero_p1 <= 11) goto end;
 287         block[0x19] = temp_block[0x19];
 288         block[0x12] = temp_block[0x12]; block[0x0B] = temp_block[0x0B];
 289         block[0x04] = temp_block[0x04]; block[0x05] = temp_block[0x05];
 290         if(last_non_zero_p1 <= 16) goto end;
 291         block[0x0C] = temp_block[0x0C]; block[0x13] = temp_block[0x13];
 292         block[0x1A] = temp_block[0x1A]; block[0x21] = temp_block[0x21];
 293         block[0x28] = temp_block[0x28]; block[0x30] = temp_block[0x30];
 294         block[0x29] = temp_block[0x29]; block[0x22] = temp_block[0x22];
 295         if(last_non_zero_p1 <= 24) goto end;
 296         block[0x1B] = temp_block[0x1B]; block[0x14] = temp_block[0x14];
 297         block[0x0D] = temp_block[0x0D]; block[0x06] = temp_block[0x06];
 298         block[0x07] = temp_block[0x07]; block[0x0E] = temp_block[0x0E];
 299         block[0x15] = temp_block[0x15]; block[0x1C] = temp_block[0x1C];
 300         if(last_non_zero_p1 <= 32) goto end;
 301         block[0x23] = temp_block[0x23]; block[0x2A] = temp_block[0x2A];
 302         block[0x31] = temp_block[0x31]; block[0x38] = temp_block[0x38];
 303         block[0x39] = temp_block[0x39]; block[0x32] = temp_block[0x32];
 304         block[0x2B] = temp_block[0x2B]; block[0x24] = temp_block[0x24];
 305         if(last_non_zero_p1 <= 40) goto end;
 306         block[0x1D] = temp_block[0x1D]; block[0x16] = temp_block[0x16];
 307         block[0x0F] = temp_block[0x0F]; block[0x17] = temp_block[0x17];
 308         block[0x1E] = temp_block[0x1E]; block[0x25] = temp_block[0x25];
 309         block[0x2C] = temp_block[0x2C]; block[0x33] = temp_block[0x33];
 310         if(last_non_zero_p1 <= 48) goto end;
 311         block[0x3A] = temp_block[0x3A]; block[0x3B] = temp_block[0x3B];
 312         block[0x34] = temp_block[0x34]; block[0x2D] = temp_block[0x2D];
 313         block[0x26] = temp_block[0x26]; block[0x1F] = temp_block[0x1F];
 314         block[0x27] = temp_block[0x27]; block[0x2E] = temp_block[0x2E];
 315         if(last_non_zero_p1 <= 56) goto end;
 316         block[0x35] = temp_block[0x35]; block[0x3C] = temp_block[0x3C];
 317         block[0x3D] = temp_block[0x3D]; block[0x36] = temp_block[0x36];
 318         block[0x2F] = temp_block[0x2F]; block[0x37] = temp_block[0x37];
 319         block[0x3E] = temp_block[0x3E]; block[0x3F] = temp_block[0x3F];
 320     }
 321     end:
 322     return last_non_zero_p1 - 1;
 323 }