git.sesse.net Git - x264/blob - common/quant.c

   1 /*****************************************************************************
   2  * quant.c: quantization and level-run
   3  *****************************************************************************
   4  * Copyright (C) 2005-2013 x264 project
   5  *
   6  * Authors: Loren Merritt <lorenm@u.washington.edu>
   7  *          Fiona Glaser <fiona@x264.com>
   8  *          Christian Heine <sennindemokrit@gmx.net>
   9  *          Henrik Gramner <hengar-6@student.ltu.se>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
  24  *
  25  * This program is also available under a commercial proprietary license.
  26  * For more information, contact us at licensing@x264.com.
  27  *****************************************************************************/
  28
  29 #include "common.h"
  30
  31 #if HAVE_MMX
  32 #include "x86/quant.h"
  33 #endif
  34 #if ARCH_PPC
  35 #   include "ppc/quant.h"
  36 #endif
  37 #if ARCH_ARM
  38 #   include "arm/quant.h"
  39 #endif
  40
  41 #define QUANT_ONE( coef, mf, f ) \
  42 { \
  43     if( (coef) > 0 ) \
  44         (coef) = (f + (coef)) * (mf) >> 16; \
  45     else \
  46         (coef) = - ((f - (coef)) * (mf) >> 16); \
  47     nz |= (coef); \
  48 }
  49
  50 static int quant_8x8( dctcoef dct[64], udctcoef mf[64], udctcoef bias[64] )
  51 {
  52     int nz = 0;
  53     for( int i = 0; i < 64; i++ )
  54         QUANT_ONE( dct[i], mf[i], bias[i] );
  55     return !!nz;
  56 }
  57
  58 static int quant_4x4( dctcoef dct[16], udctcoef mf[16], udctcoef bias[16] )
  59 {
  60     int nz = 0;
  61     for( int i = 0; i < 16; i++ )
  62         QUANT_ONE( dct[i], mf[i], bias[i] );
  63     return !!nz;
  64 }
  65
  66 static int quant_4x4x4( dctcoef dct[4][16], udctcoef mf[16], udctcoef bias[16] )
  67 {
  68     int nza = 0;
  69     for( int j = 0; j < 4; j++ )
  70     {
  71         int nz = 0;
  72         for( int i = 0; i < 16; i++ )
  73             QUANT_ONE( dct[j][i], mf[i], bias[i] );
  74         nza |= (!!nz)<<j;
  75     }
  76     return nza;
  77 }
  78
  79 static int quant_4x4_dc( dctcoef dct[16], int mf, int bias )
  80 {
  81     int nz = 0;
  82     for( int i = 0; i < 16; i++ )
  83         QUANT_ONE( dct[i], mf, bias );
  84     return !!nz;
  85 }
  86
  87 static int quant_2x2_dc( dctcoef dct[4], int mf, int bias )
  88 {
  89     int nz = 0;
  90     QUANT_ONE( dct[0], mf, bias );
  91     QUANT_ONE( dct[1], mf, bias );
  92     QUANT_ONE( dct[2], mf, bias );
  93     QUANT_ONE( dct[3], mf, bias );
  94     return !!nz;
  95 }
  96
  97 #define DEQUANT_SHL( x ) \
  98     dct[x] = ( dct[x] * dequant_mf[i_mf][x] ) << i_qbits
  99
 100 #define DEQUANT_SHR( x ) \
 101     dct[x] = ( dct[x] * dequant_mf[i_mf][x] + f ) >> (-i_qbits)
 102
 103 static void dequant_4x4( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
 104 {
 105     const int i_mf = i_qp%6;
 106     const int i_qbits = i_qp/6 - 4;
 107
 108     if( i_qbits >= 0 )
 109     {
 110         for( int i = 0; i < 16; i++ )
 111             DEQUANT_SHL( i );
 112     }
 113     else
 114     {
 115         const int f = 1 << (-i_qbits-1);
 116         for( int i = 0; i < 16; i++ )
 117             DEQUANT_SHR( i );
 118     }
 119 }
 120
 121 static void dequant_8x8( dctcoef dct[64], int dequant_mf[6][64], int i_qp )
 122 {
 123     const int i_mf = i_qp%6;
 124     const int i_qbits = i_qp/6 - 6;
 125
 126     if( i_qbits >= 0 )
 127     {
 128         for( int i = 0; i < 64; i++ )
 129             DEQUANT_SHL( i );
 130     }
 131     else
 132     {
 133         const int f = 1 << (-i_qbits-1);
 134         for( int i = 0; i < 64; i++ )
 135             DEQUANT_SHR( i );
 136     }
 137 }
 138
 139 static void dequant_4x4_dc( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
 140 {
 141     const int i_qbits = i_qp/6 - 6;
 142
 143     if( i_qbits >= 0 )
 144     {
 145         const int i_dmf = dequant_mf[i_qp%6][0] << i_qbits;
 146         for( int i = 0; i < 16; i++ )
 147             dct[i] *= i_dmf;
 148     }
 149     else
 150     {
 151         const int i_dmf = dequant_mf[i_qp%6][0];
 152         const int f = 1 << (-i_qbits-1);
 153         for( int i = 0; i < 16; i++ )
 154             dct[i] = ( dct[i] * i_dmf + f ) >> (-i_qbits);
 155     }
 156 }
 157
 158 #define IDCT_DEQUANT_2X4_START \
 159     int a0 = dct[0] + dct[1]; \
 160     int a1 = dct[2] + dct[3]; \
 161     int a2 = dct[4] + dct[5]; \
 162     int a3 = dct[6] + dct[7]; \
 163     int a4 = dct[0] - dct[1]; \
 164     int a5 = dct[2] - dct[3]; \
 165     int a6 = dct[4] - dct[5]; \
 166     int a7 = dct[6] - dct[7]; \
 167     int b0 = a0 + a1; \
 168     int b1 = a2 + a3; \
 169     int b2 = a4 + a5; \
 170     int b3 = a6 + a7; \
 171     int b4 = a0 - a1; \
 172     int b5 = a2 - a3; \
 173     int b6 = a4 - a5; \
 174     int b7 = a6 - a7;
 175
 176 static void idct_dequant_2x4_dc( dctcoef dct[8], dctcoef dct4x4[8][16], int dequant_mf[6][16], int i_qp )
 177 {
 178     IDCT_DEQUANT_2X4_START
 179     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
 180     dct4x4[0][0] = ((b0 + b1) * dmf + 32) >> 6;
 181     dct4x4[1][0] = ((b2 + b3) * dmf + 32) >> 6;
 182     dct4x4[2][0] = ((b0 - b1) * dmf + 32) >> 6;
 183     dct4x4[3][0] = ((b2 - b3) * dmf + 32) >> 6;
 184     dct4x4[4][0] = ((b4 - b5) * dmf + 32) >> 6;
 185     dct4x4[5][0] = ((b6 - b7) * dmf + 32) >> 6;
 186     dct4x4[6][0] = ((b4 + b5) * dmf + 32) >> 6;
 187     dct4x4[7][0] = ((b6 + b7) * dmf + 32) >> 6;
 188 }
 189
 190 static void idct_dequant_2x4_dconly( dctcoef dct[8], int dequant_mf[6][16], int i_qp )
 191 {
 192     IDCT_DEQUANT_2X4_START
 193     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
 194     dct[0] = ((b0 + b1) * dmf + 32) >> 6;
 195     dct[1] = ((b2 + b3) * dmf + 32) >> 6;
 196     dct[2] = ((b0 - b1) * dmf + 32) >> 6;
 197     dct[3] = ((b2 - b3) * dmf + 32) >> 6;
 198     dct[4] = ((b4 - b5) * dmf + 32) >> 6;
 199     dct[5] = ((b6 - b7) * dmf + 32) >> 6;
 200     dct[6] = ((b4 + b5) * dmf + 32) >> 6;
 201     dct[7] = ((b6 + b7) * dmf + 32) >> 6;
 202 }
 203
 204 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x4( dctcoef out[8], dctcoef dct[8], int dmf )
 205 {
 206     IDCT_DEQUANT_2X4_START
 207     out[0] = ((b0 + b1) * dmf + 2080) >> 6; /* 2080 = 32 + (32<<6) */
 208     out[1] = ((b2 + b3) * dmf + 2080) >> 6;
 209     out[2] = ((b0 - b1) * dmf + 2080) >> 6;
 210     out[3] = ((b2 - b3) * dmf + 2080) >> 6;
 211     out[4] = ((b4 - b5) * dmf + 2080) >> 6;
 212     out[5] = ((b6 - b7) * dmf + 2080) >> 6;
 213     out[6] = ((b4 + b5) * dmf + 2080) >> 6;
 214     out[7] = ((b6 + b7) * dmf + 2080) >> 6;
 215 }
 216 #undef IDCT_DEQUANT_2X4_START
 217
 218 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x2( dctcoef out[4], dctcoef dct[4], int dmf )
 219 {
 220     int d0 = dct[0] + dct[1];
 221     int d1 = dct[2] + dct[3];
 222     int d2 = dct[0] - dct[1];
 223     int d3 = dct[2] - dct[3];
 224     out[0] = ((d0 + d1) * dmf >> 5) + 32;
 225     out[1] = ((d0 - d1) * dmf >> 5) + 32;
 226     out[2] = ((d2 + d3) * dmf >> 5) + 32;
 227     out[3] = ((d2 - d3) * dmf >> 5) + 32;
 228 }
 229
 230 static ALWAYS_INLINE int optimize_chroma_round( dctcoef *ref, dctcoef *dct, int dequant_mf, int chroma422 )
 231 {
 232     dctcoef out[8];
 233
 234     if( chroma422 )
 235         optimize_chroma_idct_dequant_2x4( out, dct, dequant_mf );
 236     else
 237         optimize_chroma_idct_dequant_2x2( out, dct, dequant_mf );
 238
 239     int sum = 0;
 240     for( int i = 0; i < (chroma422?8:4); i++ )
 241         sum |= ref[i] ^ out[i];
 242     return sum >> 6;
 243 }
 244
 245 static ALWAYS_INLINE int optimize_chroma_dc_internal( dctcoef *dct, int dequant_mf, int chroma422 )
 246 {
 247     /* dequant_mf = h->dequant4_mf[CQM_4IC + b_inter][i_qp%6][0] << i_qp/6, max 32*64 */
 248     dctcoef dct_orig[8];
 249     int coeff, nz;
 250
 251     if( chroma422 )
 252         optimize_chroma_idct_dequant_2x4( dct_orig, dct, dequant_mf );
 253     else
 254         optimize_chroma_idct_dequant_2x2( dct_orig, dct, dequant_mf );
 255
 256     /* If the DC coefficients already round to zero, terminate early. */
 257     int sum = 0;
 258     for( int i = 0; i < (chroma422?8:4); i++ )
 259         sum |= dct_orig[i];
 260     if( !(sum >> 6) )
 261         return 0;
 262
 263     /* Start with the highest frequency coefficient... is this the best option? */
 264     for( nz = 0, coeff = (chroma422?7:3); coeff >= 0; coeff-- )
 265     {
 266         int level = dct[coeff];
 267         int sign = level>>31 | 1; /* dct[coeff] < 0 ? -1 : 1 */
 268
 269         while( level )
 270         {
 271             dct[coeff] = level - sign;
 272             if( optimize_chroma_round( dct_orig, dct, dequant_mf, chroma422 ) )
 273             {
 274                 nz = 1;
 275                 dct[coeff] = level;
 276                 break;
 277             }
 278             level -= sign;
 279         }
 280     }
 281
 282     return nz;
 283 }
 284
 285 static int optimize_chroma_2x2_dc( dctcoef dct[4], int dequant_mf )
 286 {
 287     return optimize_chroma_dc_internal( dct, dequant_mf, 0 );
 288 }
 289
 290 static int optimize_chroma_2x4_dc( dctcoef dct[8], int dequant_mf )
 291 {
 292     return optimize_chroma_dc_internal( dct, dequant_mf, 1 );
 293 }
 294
 295 static void x264_denoise_dct( dctcoef *dct, uint32_t *sum, udctcoef *offset, int size )
 296 {
 297     for( int i = 0; i < size; i++ )
 298     {
 299         int level = dct[i];
 300         int sign = level>>31;
 301         level = (level+sign)^sign;
 302         sum[i] += level;
 303         level -= offset[i];
 304         dct[i] = level<0 ? 0 : (level^sign)-sign;
 305     }
 306 }
 307
 308 /* (ref: JVT-B118)
 309  * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
 310  * to 0 (low score means set it to null)
 311  * Used in inter macroblock (luma and chroma)
 312  *  luma: for a 8x8 block: if score < 4 -> null
 313  *        for the complete mb: if score < 6 -> null
 314  *  chroma: for the complete mb: if score < 7 -> null
 315  */
 316
 317 const uint8_t x264_decimate_table4[16] =
 318 {
 319     3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0
 320 };
 321 const uint8_t x264_decimate_table8[64] =
 322 {
 323     3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
 324     1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
 325     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 326     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 327 };
 328
 329 static int ALWAYS_INLINE x264_decimate_score_internal( dctcoef *dct, int i_max )
 330 {
 331     const uint8_t *ds_table = (i_max == 64) ? x264_decimate_table8 : x264_decimate_table4;
 332     int i_score = 0;
 333     int idx = i_max - 1;
 334
 335     while( idx >= 0 && dct[idx] == 0 )
 336         idx--;
 337     while( idx >= 0 )
 338     {
 339         int i_run;
 340
 341         if( (unsigned)(dct[idx--] + 1) > 2 )
 342             return 9;
 343
 344         i_run = 0;
 345         while( idx >= 0 && dct[idx] == 0 )
 346         {
 347             idx--;
 348             i_run++;
 349         }
 350         i_score += ds_table[i_run];
 351     }
 352
 353     return i_score;
 354 }
 355
 356 static int x264_decimate_score15( dctcoef *dct )
 357 {
 358     return x264_decimate_score_internal( dct+1, 15 );
 359 }
 360 static int x264_decimate_score16( dctcoef *dct )
 361 {
 362     return x264_decimate_score_internal( dct, 16 );
 363 }
 364 static int x264_decimate_score64( dctcoef *dct )
 365 {
 366     return x264_decimate_score_internal( dct, 64 );
 367 }
 368
 369 #define last(num)\
 370 static int x264_coeff_last##num( dctcoef *l )\
 371 {\
 372     int i_last = num-1;\
 373     while( i_last >= 0 && l[i_last] == 0 )\
 374         i_last--;\
 375     return i_last;\
 376 }
 377
 378 last(4)
 379 last(8)
 380 last(15)
 381 last(16)
 382 last(64)
 383
 384 #define level_run(num)\
 385 static int x264_coeff_level_run##num( dctcoef *dct, x264_run_level_t *runlevel )\
 386 {\
 387     int i_last = runlevel->last = x264_coeff_last##num(dct);\
 388     int i_total = 0;\
 389     int mask = 0;\
 390     do\
 391     {\
 392         runlevel->level[i_total++] = dct[i_last];\
 393         mask |= 1 << (i_last);\
 394         while( --i_last >= 0 && dct[i_last] == 0 );\
 395     } while( i_last >= 0 );\
 396     runlevel->mask = mask;\
 397     return i_total;\
 398 }
 399
 400 level_run(4)
 401 level_run(8)
 402 level_run(15)
 403 level_run(16)
 404
 405 #if ARCH_X86_64
 406 #define INIT_TRELLIS(cpu)\
 407     pf->trellis_cabac_4x4 = x264_trellis_cabac_4x4_##cpu;\
 408     pf->trellis_cabac_8x8 = x264_trellis_cabac_8x8_##cpu;\
 409     pf->trellis_cabac_4x4_psy = x264_trellis_cabac_4x4_psy_##cpu;\
 410     pf->trellis_cabac_8x8_psy = x264_trellis_cabac_8x8_psy_##cpu;\
 411     pf->trellis_cabac_dc = x264_trellis_cabac_dc_##cpu;\
 412     pf->trellis_cabac_chroma_422_dc = x264_trellis_cabac_chroma_422_dc_##cpu;
 413 #else
 414 #define INIT_TRELLIS(...)
 415 #endif
 416
 417 void x264_quant_init( x264_t *h, int cpu, x264_quant_function_t *pf )
 418 {
 419     pf->quant_8x8 = quant_8x8;
 420     pf->quant_4x4 = quant_4x4;
 421     pf->quant_4x4x4 = quant_4x4x4;
 422     pf->quant_4x4_dc = quant_4x4_dc;
 423     pf->quant_2x2_dc = quant_2x2_dc;
 424
 425     pf->dequant_4x4 = dequant_4x4;
 426     pf->dequant_4x4_dc = dequant_4x4_dc;
 427     pf->dequant_8x8 = dequant_8x8;
 428
 429     pf->idct_dequant_2x4_dc = idct_dequant_2x4_dc;
 430     pf->idct_dequant_2x4_dconly = idct_dequant_2x4_dconly;
 431
 432     pf->optimize_chroma_2x2_dc = optimize_chroma_2x2_dc;
 433     pf->optimize_chroma_2x4_dc = optimize_chroma_2x4_dc;
 434
 435     pf->denoise_dct = x264_denoise_dct;
 436     pf->decimate_score15 = x264_decimate_score15;
 437     pf->decimate_score16 = x264_decimate_score16;
 438     pf->decimate_score64 = x264_decimate_score64;
 439
 440     pf->coeff_last4 = x264_coeff_last4;
 441     pf->coeff_last8 = x264_coeff_last8;
 442     pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15;
 443     pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16;
 444     pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64;
 445     pf->coeff_level_run4 = x264_coeff_level_run4;
 446     pf->coeff_level_run8 = x264_coeff_level_run8;
 447     pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15;
 448     pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16;
 449
 450 #if HIGH_BIT_DEPTH
 451 #if HAVE_MMX
 452     INIT_TRELLIS( sse2 );
 453     if( cpu&X264_CPU_MMX2 )
 454     {
 455 #if ARCH_X86
 456         pf->denoise_dct = x264_denoise_dct_mmx;
 457         pf->decimate_score15 = x264_decimate_score15_mmx2;
 458         pf->decimate_score16 = x264_decimate_score16_mmx2;
 459         pf->decimate_score64 = x264_decimate_score64_mmx2;
 460         pf->coeff_last8 = x264_coeff_last8_mmx2;
 461         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
 462         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
 463         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
 464         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
 465         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
 466         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
 467 #endif
 468         pf->coeff_last4 = x264_coeff_last4_mmx2;
 469         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
 470         if( cpu&X264_CPU_LZCNT )
 471             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
 472     }
 473     if( cpu&X264_CPU_SSE2 )
 474     {
 475         pf->quant_4x4 = x264_quant_4x4_sse2;
 476         pf->quant_4x4x4 = x264_quant_4x4x4_sse2;
 477         pf->quant_8x8 = x264_quant_8x8_sse2;
 478         pf->quant_2x2_dc = x264_quant_2x2_dc_sse2;
 479         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
 480         pf->dequant_4x4 = x264_dequant_4x4_sse2;
 481         pf->dequant_8x8 = x264_dequant_8x8_sse2;
 482         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
 483         pf->denoise_dct = x264_denoise_dct_sse2;
 484         pf->decimate_score15 = x264_decimate_score15_sse2;
 485         pf->decimate_score16 = x264_decimate_score16_sse2;
 486         pf->decimate_score64 = x264_decimate_score64_sse2;
 487         pf->coeff_last8 = x264_coeff_last8_sse2;
 488         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
 489         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
 490         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
 491         pf->coeff_level_run8 = x264_coeff_level_run8_sse2;
 492         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
 493         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
 494         if( cpu&X264_CPU_LZCNT )
 495         {
 496             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
 497             pf->coeff_last8 = x264_coeff_last8_sse2_lzcnt;
 498             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
 499             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
 500             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
 501             pf->coeff_level_run8 = x264_coeff_level_run8_sse2_lzcnt;
 502             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
 503             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
 504         }
 505     }
 506     if( cpu&X264_CPU_SSSE3 )
 507     {
 508         pf->quant_4x4 = x264_quant_4x4_ssse3;
 509         pf->quant_4x4x4 = x264_quant_4x4x4_ssse3;
 510         pf->quant_8x8 = x264_quant_8x8_ssse3;
 511         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
 512         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
 513         pf->denoise_dct = x264_denoise_dct_ssse3;
 514         pf->decimate_score15 = x264_decimate_score15_ssse3;
 515         pf->decimate_score16 = x264_decimate_score16_ssse3;
 516         pf->decimate_score64 = x264_decimate_score64_ssse3;
 517         INIT_TRELLIS( ssse3 );
 518     }
 519     if( cpu&X264_CPU_SSE4 )
 520     {
 521         pf->quant_2x2_dc = x264_quant_2x2_dc_sse4;
 522         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
 523         pf->quant_4x4 = x264_quant_4x4_sse4;
 524         pf->quant_4x4x4 = x264_quant_4x4x4_sse4;
 525         pf->quant_8x8 = x264_quant_8x8_sse4;
 526     }
 527     if( cpu&X264_CPU_AVX )
 528     {
 529         pf->denoise_dct = x264_denoise_dct_avx;
 530     }
 531     if( cpu&X264_CPU_XOP )
 532     {
 533         pf->dequant_4x4_dc = x264_dequant_4x4dc_xop;
 534         if( h->param.i_cqm_preset != X264_CQM_FLAT )
 535         {
 536             pf->dequant_4x4 = x264_dequant_4x4_xop;
 537             pf->dequant_8x8 = x264_dequant_8x8_xop;
 538         }
 539     }
 540     if( cpu&X264_CPU_AVX2 )
 541     {
 542         pf->quant_4x4 = x264_quant_4x4_avx2;
 543         pf->quant_4x4_dc = x264_quant_4x4_dc_avx2;
 544         pf->quant_8x8 = x264_quant_8x8_avx2;
 545         pf->quant_4x4x4 = x264_quant_4x4x4_avx2;
 546         pf->dequant_4x4 = x264_dequant_4x4_avx2;
 547         pf->dequant_8x8 = x264_dequant_8x8_avx2;
 548         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx2;
 549         pf->denoise_dct = x264_denoise_dct_avx2;
 550     }
 551 #endif // HAVE_MMX
 552 #else // !HIGH_BIT_DEPTH
 553 #if HAVE_MMX
 554     INIT_TRELLIS( sse2 );
 555     if( cpu&X264_CPU_MMX )
 556     {
 557 #if ARCH_X86
 558         pf->quant_4x4 = x264_quant_4x4_mmx;
 559         pf->quant_4x4x4 = x264_quant_4x4x4_mmx;
 560         pf->quant_8x8 = x264_quant_8x8_mmx;
 561         pf->dequant_4x4 = x264_dequant_4x4_mmx;
 562         pf->dequant_4x4_dc = x264_dequant_4x4dc_mmx2;
 563         pf->dequant_8x8 = x264_dequant_8x8_mmx;
 564         if( h->param.i_cqm_preset == X264_CQM_FLAT )
 565         {
 566             pf->dequant_4x4 = x264_dequant_4x4_flat16_mmx;
 567             pf->dequant_8x8 = x264_dequant_8x8_flat16_mmx;
 568         }
 569         pf->denoise_dct = x264_denoise_dct_mmx;
 570 #endif
 571     }
 572
 573     if( cpu&X264_CPU_MMX2 )
 574     {
 575         pf->quant_2x2_dc = x264_quant_2x2_dc_mmx2;
 576 #if ARCH_X86
 577         pf->quant_4x4_dc = x264_quant_4x4_dc_mmx2;
 578         pf->decimate_score15 = x264_decimate_score15_mmx2;
 579         pf->decimate_score16 = x264_decimate_score16_mmx2;
 580         pf->decimate_score64 = x264_decimate_score64_mmx2;
 581         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
 582         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
 583         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
 584         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
 585         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
 586 #endif
 587         pf->coeff_last4 = x264_coeff_last4_mmx2;
 588         pf->coeff_last8 = x264_coeff_last8_mmx2;
 589         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
 590         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
 591         if( cpu&X264_CPU_LZCNT )
 592         {
 593             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
 594             pf->coeff_last8 = x264_coeff_last8_mmx2_lzcnt;
 595             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
 596             pf->coeff_level_run8 = x264_coeff_level_run8_mmx2_lzcnt;
 597         }
 598     }
 599
 600     if( cpu&X264_CPU_SSE2 )
 601     {
 602         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
 603         pf->quant_4x4 = x264_quant_4x4_sse2;
 604         pf->quant_4x4x4 = x264_quant_4x4x4_sse2;
 605         pf->quant_8x8 = x264_quant_8x8_sse2;
 606         pf->dequant_4x4 = x264_dequant_4x4_sse2;
 607         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
 608         pf->dequant_8x8 = x264_dequant_8x8_sse2;
 609         if( h->param.i_cqm_preset == X264_CQM_FLAT )
 610         {
 611             pf->dequant_4x4 = x264_dequant_4x4_flat16_sse2;
 612             pf->dequant_8x8 = x264_dequant_8x8_flat16_sse2;
 613         }
 614         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse2;
 615         pf->denoise_dct = x264_denoise_dct_sse2;
 616         pf->decimate_score15 = x264_decimate_score15_sse2;
 617         pf->decimate_score16 = x264_decimate_score16_sse2;
 618         pf->decimate_score64 = x264_decimate_score64_sse2;
 619         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
 620         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
 621         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
 622         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
 623         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
 624         if( cpu&X264_CPU_LZCNT )
 625         {
 626             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
 627             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
 628             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
 629             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
 630             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
 631         }
 632     }
 633
 634     if( cpu&X264_CPU_SSSE3 )
 635     {
 636         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
 637         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
 638         pf->quant_4x4 = x264_quant_4x4_ssse3;
 639         pf->quant_4x4x4 = x264_quant_4x4x4_ssse3;
 640         pf->quant_8x8 = x264_quant_8x8_ssse3;
 641         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_ssse3;
 642         pf->denoise_dct = x264_denoise_dct_ssse3;
 643         pf->decimate_score15 = x264_decimate_score15_ssse3;
 644         pf->decimate_score16 = x264_decimate_score16_ssse3;
 645         pf->decimate_score64 = x264_decimate_score64_ssse3;
 646         INIT_TRELLIS( ssse3 );
 647         pf->coeff_level_run4 = x264_coeff_level_run4_ssse3;
 648         pf->coeff_level_run8 = x264_coeff_level_run8_ssse3;
 649         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_ssse3;
 650         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_ssse3;
 651         if( cpu&X264_CPU_LZCNT )
 652         {
 653             pf->coeff_level_run4 = x264_coeff_level_run4_ssse3;
 654             pf->coeff_level_run8 = x264_coeff_level_run8_ssse3;
 655             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_ssse3_lzcnt;
 656             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_ssse3_lzcnt;
 657         }
 658     }
 659
 660     if( cpu&X264_CPU_SSE4 )
 661     {
 662         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
 663         pf->quant_4x4 = x264_quant_4x4_sse4;
 664         pf->quant_8x8 = x264_quant_8x8_sse4;
 665         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse4;
 666     }
 667
 668     if( cpu&X264_CPU_AVX )
 669     {
 670         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx;
 671         if( h->param.i_cqm_preset != X264_CQM_FLAT )
 672         {
 673             pf->dequant_4x4 = x264_dequant_4x4_avx;
 674             pf->dequant_8x8 = x264_dequant_8x8_avx;
 675         }
 676         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_avx;
 677         pf->denoise_dct = x264_denoise_dct_avx;
 678     }
 679
 680     if( cpu&X264_CPU_XOP )
 681     {
 682         if( h->param.i_cqm_preset != X264_CQM_FLAT )
 683         {
 684             pf->dequant_4x4 = x264_dequant_4x4_xop;
 685             pf->dequant_8x8 = x264_dequant_8x8_xop;
 686         }
 687     }
 688
 689     if( cpu&X264_CPU_AVX2 )
 690     {
 691         pf->quant_4x4 = x264_quant_4x4_avx2;
 692         pf->quant_4x4_dc = x264_quant_4x4_dc_avx2;
 693         pf->quant_8x8 = x264_quant_8x8_avx2;
 694         pf->quant_4x4x4 = x264_quant_4x4x4_avx2;
 695         pf->dequant_4x4 = x264_dequant_4x4_avx2;
 696         pf->dequant_8x8 = x264_dequant_8x8_avx2;
 697         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx2;
 698         if( h->param.i_cqm_preset == X264_CQM_FLAT )
 699         {
 700             pf->dequant_4x4 = x264_dequant_4x4_flat16_avx2;
 701             pf->dequant_8x8 = x264_dequant_8x8_flat16_avx2;
 702         }
 703         pf->decimate_score64 = x264_decimate_score64_avx2;
 704         pf->denoise_dct = x264_denoise_dct_avx2;
 705         if( cpu&X264_CPU_LZCNT )
 706         {
 707             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_avx2_lzcnt;
 708             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_avx2_lzcnt;
 709             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_avx2_lzcnt;
 710         }
 711     }
 712 #endif // HAVE_MMX
 713
 714 #if HAVE_ALTIVEC
 715     if( cpu&X264_CPU_ALTIVEC ) {
 716         pf->quant_2x2_dc = x264_quant_2x2_dc_altivec;
 717         pf->quant_4x4_dc = x264_quant_4x4_dc_altivec;
 718         pf->quant_4x4 = x264_quant_4x4_altivec;
 719         pf->quant_8x8 = x264_quant_8x8_altivec;
 720
 721         pf->dequant_4x4 = x264_dequant_4x4_altivec;
 722         pf->dequant_8x8 = x264_dequant_8x8_altivec;
 723     }
 724 #endif
 725
 726 #if HAVE_ARMV6
 727     if( cpu&X264_CPU_ARMV6 )
 728         pf->coeff_last4 = x264_coeff_last4_arm;
 729
 730     if( cpu&X264_CPU_NEON )
 731     {
 732         pf->quant_2x2_dc   = x264_quant_2x2_dc_neon;
 733         pf->quant_4x4      = x264_quant_4x4_neon;
 734         pf->quant_4x4_dc   = x264_quant_4x4_dc_neon;
 735         pf->quant_4x4x4    = x264_quant_4x4x4_neon;
 736         pf->quant_8x8      = x264_quant_8x8_neon;
 737         pf->dequant_4x4    = x264_dequant_4x4_neon;
 738         pf->dequant_4x4_dc = x264_dequant_4x4_dc_neon;
 739         pf->dequant_8x8    = x264_dequant_8x8_neon;
 740         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_neon;
 741         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_neon;
 742         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_neon;
 743     }
 744 #endif
 745 #endif // HIGH_BIT_DEPTH
 746     pf->coeff_last[DCT_LUMA_DC]     = pf->coeff_last[DCT_CHROMAU_DC]  = pf->coeff_last[DCT_CHROMAV_DC] =
 747     pf->coeff_last[DCT_CHROMAU_4x4] = pf->coeff_last[DCT_CHROMAV_4x4] = pf->coeff_last[DCT_LUMA_4x4];
 748     pf->coeff_last[DCT_CHROMA_AC]   = pf->coeff_last[DCT_CHROMAU_AC]  =
 749     pf->coeff_last[DCT_CHROMAV_AC]  = pf->coeff_last[DCT_LUMA_AC];
 750     pf->coeff_last[DCT_CHROMAU_8x8] = pf->coeff_last[DCT_CHROMAV_8x8] = pf->coeff_last[DCT_LUMA_8x8];
 751
 752     pf->coeff_level_run[DCT_LUMA_DC]     = pf->coeff_level_run[DCT_CHROMAU_DC]  = pf->coeff_level_run[DCT_CHROMAV_DC] =
 753     pf->coeff_level_run[DCT_CHROMAU_4x4] = pf->coeff_level_run[DCT_CHROMAV_4x4] = pf->coeff_level_run[DCT_LUMA_4x4];
 754     pf->coeff_level_run[DCT_CHROMA_AC]   = pf->coeff_level_run[DCT_CHROMAU_AC]  =
 755     pf->coeff_level_run[DCT_CHROMAV_AC]  = pf->coeff_level_run[DCT_LUMA_AC];
 756 }