git.sesse.net Git - x264/blob - common/quant.c

   1 /*****************************************************************************
   2  * quant.c: quantization and level-run
   3  *****************************************************************************
   4  * Copyright (C) 2005-2010 x264 project
   5  *
   6  * Authors: Loren Merritt <lorenm@u.washington.edu>
   7  *          Fiona Glaser <fiona@x264.com>
   8  *          Christian Heine <sennindemokrit@gmx.net>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
  23  *
  24  * This program is also available under a commercial proprietary license.
  25  * For more information, contact us at licensing@x264.com.
  26  *****************************************************************************/
  27
  28 #include "common.h"
  29
  30 #if HAVE_MMX
  31 #include "x86/quant.h"
  32 #endif
  33 #if ARCH_PPC
  34 #   include "ppc/quant.h"
  35 #endif
  36 #if ARCH_ARM
  37 #   include "arm/quant.h"
  38 #endif
  39
  40 #define QUANT_ONE( coef, mf, f ) \
  41 { \
  42     if( (coef) > 0 ) \
  43         (coef) = (f + (coef)) * (mf) >> 16; \
  44     else \
  45         (coef) = - ((f - (coef)) * (mf) >> 16); \
  46     nz |= (coef); \
  47 }
  48
  49 static int quant_8x8( dctcoef dct[64], udctcoef mf[64], udctcoef bias[64] )
  50 {
  51     int nz = 0;
  52     for( int i = 0; i < 64; i++ )
  53         QUANT_ONE( dct[i], mf[i], bias[i] );
  54     return !!nz;
  55 }
  56
  57 static int quant_4x4( dctcoef dct[16], udctcoef mf[16], udctcoef bias[16] )
  58 {
  59     int nz = 0;
  60     for( int i = 0; i < 16; i++ )
  61         QUANT_ONE( dct[i], mf[i], bias[i] );
  62     return !!nz;
  63 }
  64
  65 static int quant_4x4_dc( dctcoef dct[16], int mf, int bias )
  66 {
  67     int nz = 0;
  68     for( int i = 0; i < 16; i++ )
  69         QUANT_ONE( dct[i], mf, bias );
  70     return !!nz;
  71 }
  72
  73 static int quant_2x2_dc( dctcoef dct[4], int mf, int bias )
  74 {
  75     int nz = 0;
  76     QUANT_ONE( dct[0], mf, bias );
  77     QUANT_ONE( dct[1], mf, bias );
  78     QUANT_ONE( dct[2], mf, bias );
  79     QUANT_ONE( dct[3], mf, bias );
  80     return !!nz;
  81 }
  82
  83 #define DEQUANT_SHL( x ) \
  84     dct[x] = ( dct[x] * dequant_mf[i_mf][x] ) << i_qbits
  85
  86 #define DEQUANT_SHR( x ) \
  87     dct[x] = ( dct[x] * dequant_mf[i_mf][x] + f ) >> (-i_qbits)
  88
  89 static void dequant_4x4( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
  90 {
  91     const int i_mf = i_qp%6;
  92     const int i_qbits = i_qp/6 - 4;
  93
  94     if( i_qbits >= 0 )
  95     {
  96         for( int i = 0; i < 16; i++ )
  97             DEQUANT_SHL( i );
  98     }
  99     else
 100     {
 101         const int f = 1 << (-i_qbits-1);
 102         for( int i = 0; i < 16; i++ )
 103             DEQUANT_SHR( i );
 104     }
 105 }
 106
 107 static void dequant_8x8( dctcoef dct[64], int dequant_mf[6][64], int i_qp )
 108 {
 109     const int i_mf = i_qp%6;
 110     const int i_qbits = i_qp/6 - 6;
 111
 112     if( i_qbits >= 0 )
 113     {
 114         for( int i = 0; i < 64; i++ )
 115             DEQUANT_SHL( i );
 116     }
 117     else
 118     {
 119         const int f = 1 << (-i_qbits-1);
 120         for( int i = 0; i < 64; i++ )
 121             DEQUANT_SHR( i );
 122     }
 123 }
 124
 125 static void dequant_4x4_dc( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
 126 {
 127     const int i_qbits = i_qp/6 - 6;
 128
 129     if( i_qbits >= 0 )
 130     {
 131         const int i_dmf = dequant_mf[i_qp%6][0] << i_qbits;
 132         for( int i = 0; i < 16; i++ )
 133             dct[i] *= i_dmf;
 134     }
 135     else
 136     {
 137         const int i_dmf = dequant_mf[i_qp%6][0];
 138         const int f = 1 << (-i_qbits-1);
 139         for( int i = 0; i < 16; i++ )
 140             dct[i] = ( dct[i] * i_dmf + f ) >> (-i_qbits);
 141     }
 142 }
 143
 144 static void x264_denoise_dct( dctcoef *dct, uint32_t *sum, udctcoef *offset, int size )
 145 {
 146     for( int i = 1; i < size; i++ )
 147     {
 148         int level = dct[i];
 149         int sign = level>>31;
 150         level = (level+sign)^sign;
 151         sum[i] += level;
 152         level -= offset[i];
 153         dct[i] = level<0 ? 0 : (level^sign)-sign;
 154     }
 155 }
 156
 157 /* (ref: JVT-B118)
 158  * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
 159  * to 0 (low score means set it to null)
 160  * Used in inter macroblock (luma and chroma)
 161  *  luma: for a 8x8 block: if score < 4 -> null
 162  *        for the complete mb: if score < 6 -> null
 163  *  chroma: for the complete mb: if score < 7 -> null
 164  */
 165
 166 const uint8_t x264_decimate_table4[16] =
 167 {
 168     3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0
 169 };
 170 const uint8_t x264_decimate_table8[64] =
 171 {
 172     3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
 173     1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
 174     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
 175     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 176 };
 177
 178 static int ALWAYS_INLINE x264_decimate_score_internal( dctcoef *dct, int i_max )
 179 {
 180     const uint8_t *ds_table = (i_max == 64) ? x264_decimate_table8 : x264_decimate_table4;
 181     int i_score = 0;
 182     int idx = i_max - 1;
 183
 184     while( idx >= 0 && dct[idx] == 0 )
 185         idx--;
 186     while( idx >= 0 )
 187     {
 188         int i_run;
 189
 190         if( (unsigned)(dct[idx--] + 1) > 2 )
 191             return 9;
 192
 193         i_run = 0;
 194         while( idx >= 0 && dct[idx] == 0 )
 195         {
 196             idx--;
 197             i_run++;
 198         }
 199         i_score += ds_table[i_run];
 200     }
 201
 202     return i_score;
 203 }
 204
 205 static int x264_decimate_score15( dctcoef *dct )
 206 {
 207     return x264_decimate_score_internal( dct+1, 15 );
 208 }
 209 static int x264_decimate_score16( dctcoef *dct )
 210 {
 211     return x264_decimate_score_internal( dct, 16 );
 212 }
 213 static int x264_decimate_score64( dctcoef *dct )
 214 {
 215     return x264_decimate_score_internal( dct, 64 );
 216 }
 217
 218 static int ALWAYS_INLINE x264_coeff_last_internal( dctcoef *l, int i_count )
 219 {
 220     int i_last = i_count-1;
 221     while( i_last >= 0 && l[i_last] == 0 )
 222         i_last--;
 223     return i_last;
 224 }
 225
 226 static int x264_coeff_last4( dctcoef *l )
 227 {
 228     return x264_coeff_last_internal( l, 4 );
 229 }
 230 static int x264_coeff_last15( dctcoef *l )
 231 {
 232     return x264_coeff_last_internal( l, 15 );
 233 }
 234 static int x264_coeff_last16( dctcoef *l )
 235 {
 236     return x264_coeff_last_internal( l, 16 );
 237 }
 238 static int x264_coeff_last64( dctcoef *l )
 239 {
 240     return x264_coeff_last_internal( l, 64 );
 241 }
 242
 243 #define level_run(num)\
 244 static int x264_coeff_level_run##num( dctcoef *dct, x264_run_level_t *runlevel )\
 245 {\
 246     int i_last = runlevel->last = x264_coeff_last##num(dct);\
 247     int i_total = 0;\
 248     do\
 249     {\
 250         int r = 0;\
 251         runlevel->level[i_total] = dct[i_last];\
 252         while( --i_last >= 0 && dct[i_last] == 0 )\
 253             r++;\
 254         runlevel->run[i_total++] = r;\
 255     } while( i_last >= 0 );\
 256     return i_total;\
 257 }
 258
 259 level_run(4)
 260 level_run(15)
 261 level_run(16)
 262
 263
 264 void x264_quant_init( x264_t *h, int cpu, x264_quant_function_t *pf )
 265 {
 266     pf->quant_8x8 = quant_8x8;
 267     pf->quant_4x4 = quant_4x4;
 268     pf->quant_4x4_dc = quant_4x4_dc;
 269     pf->quant_2x2_dc = quant_2x2_dc;
 270
 271     pf->dequant_4x4 = dequant_4x4;
 272     pf->dequant_4x4_dc = dequant_4x4_dc;
 273     pf->dequant_8x8 = dequant_8x8;
 274
 275     pf->denoise_dct = x264_denoise_dct;
 276     pf->decimate_score15 = x264_decimate_score15;
 277     pf->decimate_score16 = x264_decimate_score16;
 278     pf->decimate_score64 = x264_decimate_score64;
 279
 280     pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4;
 281     pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15;
 282     pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16;
 283     pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64;
 284     pf->coeff_level_run[DCT_CHROMA_DC] = x264_coeff_level_run4;
 285     pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15;
 286     pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16;
 287
 288 #if HIGH_BIT_DEPTH
 289 #if HAVE_MMX
 290     if( cpu&X264_CPU_MMXEXT )
 291     {
 292 #if ARCH_X86
 293         pf->denoise_dct = x264_denoise_dct_mmx;
 294         pf->decimate_score15 = x264_decimate_score15_mmxext;
 295         pf->decimate_score16 = x264_decimate_score16_mmxext;
 296         if( cpu&X264_CPU_SLOW_CTZ )
 297         {
 298             pf->decimate_score15 = x264_decimate_score15_mmxext_slowctz;
 299             pf->decimate_score16 = x264_decimate_score16_mmxext_slowctz;
 300         }
 301         pf->decimate_score64 = x264_decimate_score64_mmxext;
 302         pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4_mmxext;
 303         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmxext;
 304         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmxext;
 305         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmxext;
 306         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmxext;
 307         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmxext;
 308 #endif
 309         pf->coeff_level_run[DCT_CHROMA_DC] = x264_coeff_level_run4_mmxext;
 310         if( cpu&X264_CPU_LZCNT )
 311             pf->coeff_level_run[DCT_CHROMA_DC] = x264_coeff_level_run4_mmxext_lzcnt;
 312     }
 313     if( cpu&X264_CPU_SSE2 )
 314     {
 315         pf->quant_4x4 = x264_quant_4x4_sse2;
 316         pf->quant_8x8 = x264_quant_8x8_sse2;
 317         pf->quant_2x2_dc = x264_quant_2x2_dc_sse2;
 318         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
 319         pf->dequant_4x4 = x264_dequant_4x4_sse2;
 320         pf->dequant_8x8 = x264_dequant_8x8_sse2;
 321         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
 322         pf->denoise_dct = x264_denoise_dct_sse2;
 323         pf->decimate_score15 = x264_decimate_score15_sse2;
 324         pf->decimate_score16 = x264_decimate_score16_sse2;
 325         pf->decimate_score64 = x264_decimate_score64_sse2;
 326         if( cpu&X264_CPU_SLOW_CTZ )
 327         {
 328             pf->decimate_score15 = x264_decimate_score15_sse2_slowctz;
 329             pf->decimate_score16 = x264_decimate_score16_sse2_slowctz;
 330         }
 331         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
 332         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
 333         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
 334         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
 335         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
 336         if( cpu&X264_CPU_LZCNT )
 337         {
 338             pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4_mmxext_lzcnt;
 339             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
 340             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
 341             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
 342             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
 343             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
 344         }
 345     }
 346     if( cpu&X264_CPU_SSSE3 )
 347     {
 348         pf->quant_4x4 = x264_quant_4x4_ssse3;
 349         pf->quant_8x8 = x264_quant_8x8_ssse3;
 350         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
 351         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
 352         pf->denoise_dct = x264_denoise_dct_ssse3;
 353         pf->decimate_score15 = x264_decimate_score15_ssse3;
 354         pf->decimate_score16 = x264_decimate_score16_ssse3;
 355         if( cpu&X264_CPU_SLOW_CTZ )
 356         {
 357             pf->decimate_score15 = x264_decimate_score15_ssse3_slowctz;
 358             pf->decimate_score16 = x264_decimate_score16_ssse3_slowctz;
 359         }
 360         pf->decimate_score64 = x264_decimate_score64_ssse3;
 361     }
 362     if( cpu&X264_CPU_SSE4 )
 363     {
 364         pf->quant_2x2_dc = x264_quant_2x2_dc_sse4;
 365         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
 366         pf->quant_4x4 = x264_quant_4x4_sse4;
 367         pf->quant_8x8 = x264_quant_8x8_sse4;
 368     }
 369 #endif // HAVE_MMX
 370 #else // !HIGH_BIT_DEPTH
 371 #if HAVE_MMX
 372     if( cpu&X264_CPU_MMX )
 373     {
 374 #if ARCH_X86
 375         pf->quant_4x4 = x264_quant_4x4_mmx;
 376         pf->quant_8x8 = x264_quant_8x8_mmx;
 377         pf->dequant_4x4 = x264_dequant_4x4_mmx;
 378         pf->dequant_4x4_dc = x264_dequant_4x4dc_mmxext;
 379         pf->dequant_8x8 = x264_dequant_8x8_mmx;
 380         if( h->param.i_cqm_preset == X264_CQM_FLAT )
 381         {
 382             pf->dequant_4x4 = x264_dequant_4x4_flat16_mmx;
 383             pf->dequant_8x8 = x264_dequant_8x8_flat16_mmx;
 384         }
 385         pf->denoise_dct = x264_denoise_dct_mmx;
 386 #endif
 387     }
 388
 389     if( cpu&X264_CPU_MMXEXT )
 390     {
 391         pf->quant_2x2_dc = x264_quant_2x2_dc_mmxext;
 392 #if ARCH_X86
 393         pf->quant_4x4_dc = x264_quant_4x4_dc_mmxext;
 394         pf->decimate_score15 = x264_decimate_score15_mmxext;
 395         pf->decimate_score16 = x264_decimate_score16_mmxext;
 396         if( cpu&X264_CPU_SLOW_CTZ )
 397         {
 398             pf->decimate_score15 = x264_decimate_score15_mmxext_slowctz;
 399             pf->decimate_score16 = x264_decimate_score16_mmxext_slowctz;
 400         }
 401         pf->decimate_score64 = x264_decimate_score64_mmxext;
 402         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmxext;
 403         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmxext;
 404         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmxext;
 405         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmxext;
 406         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmxext;
 407 #endif
 408         pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4_mmxext;
 409         pf->coeff_level_run[DCT_CHROMA_DC] = x264_coeff_level_run4_mmxext;
 410         if( cpu&X264_CPU_LZCNT )
 411         {
 412             pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4_mmxext_lzcnt;
 413             pf->coeff_level_run[DCT_CHROMA_DC] = x264_coeff_level_run4_mmxext_lzcnt;
 414         }
 415     }
 416
 417     if( cpu&X264_CPU_SSE2 )
 418     {
 419         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
 420         pf->quant_4x4 = x264_quant_4x4_sse2;
 421         pf->quant_8x8 = x264_quant_8x8_sse2;
 422         pf->dequant_4x4 = x264_dequant_4x4_sse2;
 423         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
 424         pf->dequant_8x8 = x264_dequant_8x8_sse2;
 425         if( h->param.i_cqm_preset == X264_CQM_FLAT )
 426         {
 427             pf->dequant_4x4 = x264_dequant_4x4_flat16_sse2;
 428             pf->dequant_8x8 = x264_dequant_8x8_flat16_sse2;
 429         }
 430         pf->denoise_dct = x264_denoise_dct_sse2;
 431         pf->decimate_score15 = x264_decimate_score15_sse2;
 432         pf->decimate_score16 = x264_decimate_score16_sse2;
 433         pf->decimate_score64 = x264_decimate_score64_sse2;
 434         if( cpu&X264_CPU_SLOW_CTZ )
 435         {
 436             pf->decimate_score15 = x264_decimate_score15_sse2_slowctz;
 437             pf->decimate_score16 = x264_decimate_score16_sse2_slowctz;
 438         }
 439         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
 440         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
 441         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
 442         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
 443         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
 444         if( cpu&X264_CPU_LZCNT )
 445         {
 446             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
 447             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
 448             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
 449             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
 450             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
 451         }
 452     }
 453
 454     if( cpu&X264_CPU_SSSE3 )
 455     {
 456         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
 457         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
 458         pf->quant_4x4 = x264_quant_4x4_ssse3;
 459         pf->quant_8x8 = x264_quant_8x8_ssse3;
 460         pf->denoise_dct = x264_denoise_dct_ssse3;
 461         pf->decimate_score15 = x264_decimate_score15_ssse3;
 462         pf->decimate_score16 = x264_decimate_score16_ssse3;
 463         if( cpu&X264_CPU_SLOW_CTZ )
 464         {
 465             pf->decimate_score15 = x264_decimate_score15_ssse3_slowctz;
 466             pf->decimate_score16 = x264_decimate_score16_ssse3_slowctz;
 467         }
 468         pf->decimate_score64 = x264_decimate_score64_ssse3;
 469     }
 470
 471     if( cpu&X264_CPU_SSE4 )
 472     {
 473         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
 474         pf->quant_4x4 = x264_quant_4x4_sse4;
 475         pf->quant_8x8 = x264_quant_8x8_sse4;
 476     }
 477 #endif // HAVE_MMX
 478
 479 #if HAVE_ALTIVEC
 480     if( cpu&X264_CPU_ALTIVEC ) {
 481         pf->quant_2x2_dc = x264_quant_2x2_dc_altivec;
 482         pf->quant_4x4_dc = x264_quant_4x4_dc_altivec;
 483         pf->quant_4x4 = x264_quant_4x4_altivec;
 484         pf->quant_8x8 = x264_quant_8x8_altivec;
 485
 486         pf->dequant_4x4 = x264_dequant_4x4_altivec;
 487         pf->dequant_8x8 = x264_dequant_8x8_altivec;
 488     }
 489 #endif
 490
 491 #if HAVE_ARMV6
 492     if( cpu&X264_CPU_ARMV6 )
 493         pf->coeff_last[DCT_CHROMA_DC] = x264_coeff_last4_arm;
 494
 495     if( cpu&X264_CPU_NEON )
 496     {
 497         pf->quant_2x2_dc   = x264_quant_2x2_dc_neon;
 498         pf->quant_4x4      = x264_quant_4x4_neon;
 499         pf->quant_4x4_dc   = x264_quant_4x4_dc_neon;
 500         pf->quant_8x8      = x264_quant_8x8_neon;
 501         pf->dequant_4x4    = x264_dequant_4x4_neon;
 502         pf->dequant_4x4_dc = x264_dequant_4x4_dc_neon;
 503         pf->dequant_8x8    = x264_dequant_8x8_neon;
 504         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_neon;
 505         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_neon;
 506         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_neon;
 507     }
 508 #endif
 509 #endif // HIGH_BIT_DEPTH
 510     pf->coeff_last[  DCT_LUMA_DC] = pf->coeff_last[DCT_LUMA_4x4];
 511     pf->coeff_last[DCT_CHROMA_AC] = pf->coeff_last[ DCT_LUMA_AC];
 512     pf->coeff_level_run[  DCT_LUMA_DC] = pf->coeff_level_run[DCT_LUMA_4x4];
 513     pf->coeff_level_run[DCT_CHROMA_AC] = pf->coeff_level_run[ DCT_LUMA_AC];
 514 }