]> git.sesse.net Git - x264/blobdiff - encoder/encoder.c
Eliminate the possibility of CAVLC level code overflow
[x264] / encoder / encoder.c
index dbe4b0e799d7e6aa88ebc4072cc2e9f09614a5f3..502ae636d4b473dd795f04d4341c0af6935a5fc3 100644 (file)
@@ -1,7 +1,7 @@
 /*****************************************************************************
  * encoder.c: top-level encoder functions
  *****************************************************************************
- * Copyright (C) 2003-2010 x264 project
+ * Copyright (C) 2003-2011 x264 project
  *
  * Authors: Laurent Aimar <fenrir@via.ecp.fr>
  *          Loren Merritt <lorenm@u.washington.edu>
@@ -25,8 +25,6 @@
  * For more information, contact us at licensing@x264.com.
  *****************************************************************************/
 
-#include <math.h>
-
 #include "common/common.h"
 
 #include "set.h"
@@ -164,8 +162,8 @@ static void x264_slice_header_init( x264_t *h, x264_slice_header_t *sh,
 
     sh->i_cabac_init_idc = param->i_cabac_init_idc;
 
-    sh->i_qp = i_qp;
-    sh->i_qp_delta = i_qp - pps->i_pic_init_qp;
+    sh->i_qp = SPEC_QP(i_qp);
+    sh->i_qp_delta = sh->i_qp - pps->i_pic_init_qp;
     sh->b_sp_for_swidth = 0;
     sh->i_qs_delta = 0;
 
@@ -774,7 +772,7 @@ static int x264_validate_parameters( x264_t *h )
     }
 
     h->param.analyse.i_weighted_pred = x264_clip3( h->param.analyse.i_weighted_pred, X264_WEIGHTP_NONE, X264_WEIGHTP_SMART );
-    if( !h->param.analyse.i_weighted_pred && h->param.rc.b_mb_tree && h->param.analyse.b_psy && !h->param.b_interlaced )
+    if( !h->param.analyse.i_weighted_pred && h->param.rc.b_mb_tree && h->param.analyse.b_psy )
         h->param.analyse.i_weighted_pred = X264_WEIGHTP_FAKE;
 
     if( h->i_thread_frames > 1 )
@@ -909,6 +907,7 @@ static void x264_set_aspect_ratio( x264_t *h, x264_param_t *param, int initial )
                 h->param.vui.i_sar_width = i_w;
                 h->param.vui.i_sar_height = i_h;
             }
+            x264_sps_init( h->sps, h->param.i_sps_id, &h->param );
         }
     }
 }
@@ -948,8 +947,6 @@ x264_t *x264_encoder_open( x264_param_t *param )
     if( h->param.rc.psz_stat_in )
         h->param.rc.psz_stat_in = strdup( h->param.rc.psz_stat_in );
 
-    x264_set_aspect_ratio( h, &h->param, 1 );
-
     x264_reduce_fraction( &h->param.i_fps_num, &h->param.i_fps_den );
     x264_reduce_fraction( &h->param.i_timebase_num, &h->param.i_timebase_den );
 
@@ -970,6 +967,8 @@ x264_t *x264_encoder_open( x264_param_t *param )
     h->pps = &h->pps_array[0];
     x264_pps_init( h->pps, h->param.i_sps_id, &h->param, h->sps );
 
+    x264_set_aspect_ratio( h, &h->param, 1 );
+
     x264_validate_levels( h, 1 );
 
     h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;
@@ -991,8 +990,7 @@ x264_t *x264_encoder_open( x264_param_t *param )
     i_slicetype_length = h->frames.i_delay;
     h->frames.i_delay += h->i_thread_frames - 1;
     h->frames.i_delay += h->param.i_sync_lookahead;
-    h->frames.i_delay += h->param.b_vfr_input && (h->param.rc.i_rc_method == X264_RC_ABR || h->param.rc.b_stat_write
-                                                 || h->param.rc.i_vbv_buffer_size);
+    h->frames.i_delay += h->param.b_vfr_input;
     h->frames.i_bframe_delay = h->param.i_bframe ? (h->param.i_bframe_pyramid ? 2 : 1) : 0;
 
     h->frames.i_max_ref0 = h->param.i_frame_reference;
@@ -1065,7 +1063,7 @@ x264_t *x264_encoder_open( x264_param_t *param )
         p += sprintf( p, " none!" );
     x264_log( h, X264_LOG_INFO, "%s\n", buf );
 
-    for( qp = h->param.rc.i_qp_min; qp <= h->param.rc.i_qp_max; qp++ )
+    for( qp = X264_MIN( h->param.rc.i_qp_min, QP_MAX_SPEC ); qp <= h->param.rc.i_qp_max; qp++ )
         if( x264_analyse_init_costs( h, qp ) )
             goto fail;
     if( x264_analyse_init_costs( h, X264_LOOKAHEAD_QP ) )
@@ -1073,7 +1071,7 @@ x264_t *x264_encoder_open( x264_param_t *param )
 
     static const uint16_t cost_mv_correct[7] = { 24, 47, 95, 189, 379, 757, 1515 };
     /* Checks for known miscompilation issues. */
-    if( h->cost_mv[x264_lambda_tab[X264_LOOKAHEAD_QP]][2013] != cost_mv_correct[BIT_DEPTH-8] )
+    if( h->cost_mv[X264_LOOKAHEAD_QP][2013] != cost_mv_correct[BIT_DEPTH-8] )
     {
         x264_log( h, X264_LOG_ERROR, "MV cost test failed: x264 has been miscompiled!\n" );
         goto fail;
@@ -1123,6 +1121,9 @@ x264_t *x264_encoder_open( x264_param_t *param )
         else
             h->thread[i]->fdec = h->thread[0]->fdec;
 
+        h->thread[i]->sps = &h->thread[i]->sps_array[0];
+        h->thread[i]->pps = &h->thread[i]->pps_array[0];
+
         CHECKED_MALLOC( h->thread[i]->out.p_bitstream, h->out.i_bitstream );
         /* Start each thread with room for init_nal_count NAL units; it'll realloc later if needed. */
         CHECKED_MALLOC( h->thread[i]->out.nal, init_nal_count*sizeof(x264_nal_t) );
@@ -1456,6 +1457,8 @@ int x264_weighted_reference_duplicate( x264_t *h, int i_ref, const x264_weight_t
         return -1;
 
     newframe = x264_frame_pop_blank_unused( h );
+    if( !newframe )
+        return -1;
 
     //FIXME: probably don't need to copy everything
     *newframe = *h->fref[0][i_ref];
@@ -1845,11 +1848,12 @@ static inline void x264_slice_init( x264_t *h, int i_nal_type, int i_global_qp )
         if( h->param.b_interlaced )
         {
             h->sh.i_delta_poc_bottom = h->param.b_tff ? 1 : -1;
-            if( h->sh.i_delta_poc_bottom == -1 )
-                h->sh.i_poc = h->fdec->i_poc + 1;
+            h->sh.i_poc += h->sh.i_delta_poc_bottom == -1;
         }
         else
             h->sh.i_delta_poc_bottom = 0;
+        h->fdec->i_delta_poc[0] = h->sh.i_delta_poc_bottom == -1;
+        h->fdec->i_delta_poc[1] = h->sh.i_delta_poc_bottom ==  1;
     }
     else if( h->sps->i_poc_type == 1 )
     {
@@ -1879,9 +1883,11 @@ static int x264_slice_write( x264_t *h )
      * other inaccuracies. */
     int overhead_guess = (NALU_OVERHEAD - (h->param.b_annexb && h->out.i_nal)) + 1 + h->param.b_cabac + 5;
     int slice_max_size = h->param.i_slice_max_size > 0 ? (h->param.i_slice_max_size-overhead_guess)*8 : 0;
+    int back_up_bitstream = slice_max_size || (!h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH);
     int starting_bits = bs_pos(&h->out.bs);
     int b_deblock = h->sh.i_disable_deblocking_filter_idc != 1;
     int b_hpel = h->fdec->b_kept_as_ref;
+    uint8_t *last_emu_check;
     b_deblock &= b_hpel || h->param.psz_dump_yuv;
     bs_realign( &h->out.bs );
 
@@ -1910,7 +1916,10 @@ static int x264_slice_write( x264_t *h )
         /* init cabac */
         x264_cabac_context_init( &h->cabac, h->sh.i_type, x264_clip3( h->sh.i_qp-QP_BD_OFFSET, 0, 51 ), h->sh.i_cabac_init_idc );
         x264_cabac_encode_init ( &h->cabac, h->out.bs.p, h->out.bs.p_end );
+        last_emu_check = h->cabac.p;
     }
+    else
+        last_emu_check = h->out.bs.p;
     h->mb.i_last_qp = h->sh.i_qp;
     h->mb.i_last_dqp = 0;
 
@@ -1925,7 +1934,7 @@ static int x264_slice_write( x264_t *h )
         if( x264_bitstream_check_buffer( h ) )
             return -1;
 
-        if( slice_max_size )
+        if( back_up_bitstream )
         {
             mv_bits_bak = h->stat.frame.i_mv_bits;
             tex_bits_bak = h->stat.frame.i_tex_bits;
@@ -1955,6 +1964,7 @@ static int x264_slice_write( x264_t *h )
         x264_macroblock_analyse( h );
 
         /* encode this macroblock -> be careful it can change the mb type to P_SKIP if needed */
+reencode:
         x264_macroblock_encode( h );
 
         if( h->param.b_cabac )
@@ -1983,6 +1993,19 @@ static int x264_slice_write( x264_t *h )
                     i_skip = 0;
                 }
                 x264_macroblock_write_cavlc( h );
+                /* If there was a CAVLC level code overflow, try again at a higher QP. */
+                if( h->mb.b_overflow )
+                {
+                    h->mb.i_chroma_qp = h->chroma_qp_table[++h->mb.i_qp];
+                    h->mb.i_skip_intra = 0;
+                    h->mb.b_skip_mc = 0;
+                    h->mb.b_overflow = 0;
+                    h->out.bs = bs_bak;
+                    i_skip = i_skip_bak;
+                    h->stat.frame.i_mv_bits = mv_bits_bak;
+                    h->stat.frame.i_tex_bits = tex_bits_bak;
+                    goto reencode;
+                }
             }
         }
 
@@ -1994,15 +2017,14 @@ static int x264_slice_write( x264_t *h )
             /* Count the skip run, just in case. */
             if( !h->param.b_cabac )
                 total_bits += bs_size_ue_big( i_skip );
-            /* HACK: we assume no more than 3 bytes of NALU escaping, but
-             * this can fail in CABAC streams with an extremely large number of identical
-             * blocks in sequence (e.g. all-black intra blocks).
-             * Thus, every 64 blocks, pretend we've used a byte.
-             * For reference, a seqeuence of identical empty-CBP i16x16 blocks will use
-             * one byte after 26 macroblocks, assuming a perfectly adapted CABAC.
-             * That's 78 macroblocks to generate the 3-byte sequence to trigger an escape. */
-            else if( ((mb_xy - h->sh.i_first_mb) & 63) == 63 )
-                slice_max_size -= 8;
+            /* Check for escape bytes. */
+            uint8_t *end = h->param.b_cabac ? h->cabac.p : h->out.bs.p;
+            for( ; last_emu_check < end - 2; last_emu_check++ )
+                if( last_emu_check[0] == 0 && last_emu_check[1] == 0 && last_emu_check[2] <= 3 )
+                {
+                    slice_max_size -= 8;
+                    last_emu_check++;
+                }
             /* We'll just re-encode this last macroblock if we go over the max slice size. */
             if( total_bits - starting_bits > slice_max_size && !h->mb.b_reencode_mb )
             {
@@ -2355,7 +2377,7 @@ int     x264_encoder_encode( x264_t *h,
         thread_oldest  = h;
     }
 #if HAVE_MMX
-    if( h->i_thread_frames == 1 && h->param.cpu&X264_CPU_SSE_MISALIGN )
+    if( h->param.cpu&X264_CPU_SSE_MISALIGN )
         x264_cpu_mask_misalign_sse();
 #endif
 
@@ -2734,15 +2756,15 @@ int     x264_encoder_encode( x264_t *h,
     if( h->i_ref[0] )
         h->fdec->i_poc_l0ref0 = h->fref[0][0]->i_poc;
 
+    /* ------------------------ Create slice header  ----------------------- */
+    x264_slice_init( h, i_nal_type, i_global_qp );
+
+    /*------------------------- Weights -------------------------------------*/
     if( h->sh.i_type == SLICE_TYPE_B )
         x264_macroblock_bipred_init( h );
 
-    /*------------------------- Weights -------------------------------------*/
     x264_weighted_pred_init( h );
 
-    /* ------------------------ Create slice header  ----------------------- */
-    x264_slice_init( h, i_nal_type, i_global_qp );
-
     if( i_nal_ref_idc != NAL_PRIORITY_DISPOSABLE )
         h->i_frame_num++;
 
@@ -2810,6 +2832,7 @@ static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
     pic_out->i_type = h->fenc->i_type;
 
     pic_out->b_keyframe = h->fenc->b_keyframe;
+    pic_out->i_pic_struct = h->fenc->i_pic_struct;
 
     pic_out->i_pts = h->fdec->i_pts;
     pic_out->i_dts = h->fdec->i_dts;
@@ -3310,6 +3333,12 @@ void    x264_encoder_close  ( x264_t *h )
 
     h = h->thread[0];
 
+    for( int i = 0; i < h->i_thread_frames; i++ )
+        if( h->thread[i]->b_thread_active )
+            for( int j = 0; j < h->thread[i]->i_ref[0]; j++ )
+                if( h->thread[i]->fref[0][j] && h->thread[i]->fref[0][j]->b_duplicate )
+                    x264_frame_delete( h->thread[i]->fref[0][j] );
+
     for( int i = h->param.i_threads - 1; i >= 0; i-- )
     {
         x264_frame_t **frame;
@@ -3324,10 +3353,13 @@ void    x264_encoder_close  ( x264_t *h )
                     x264_frame_delete( *frame );
             }
             frame = &h->thread[i]->fdec;
-            assert( (*frame)->i_reference_count > 0 );
-            (*frame)->i_reference_count--;
-            if( (*frame)->i_reference_count == 0 )
-                x264_frame_delete( *frame );
+            if( *frame )
+            {
+                assert( (*frame)->i_reference_count > 0 );
+                (*frame)->i_reference_count--;
+                if( (*frame)->i_reference_count == 0 )
+                    x264_frame_delete( *frame );
+            }
             x264_macroblock_cache_free( h->thread[i] );
         }
         x264_macroblock_thread_free( h->thread[i], 0 );