]> git.sesse.net Git - x264/commitdiff
Interactive encoder control: error resilience
authorFiona Glaser <fiona@x264.com>
Thu, 24 Jun 2010 00:29:34 +0000 (17:29 -0700)
committerFiona Glaser <fiona@x264.com>
Fri, 25 Jun 2010 07:43:39 +0000 (00:43 -0700)
In low-latency streaming with few clients, it is often feasible to modify encoder behavior in some fashion based on feedback from clients.
One possible application of this is error resilience: if a packet is lost, mark the associated frame (and any referenced from it) as lost.
This allows quick recovery from errors with minimal expense bit-wise.

The new i_dpb_size parameter allows a calling application to tell x264 to use a larger DPB size than required by the number of reference frames.
This lets x264 and the client keep a large buffer of old references to fall back to in case of lost frames.
If no recovery is possible even with the available buffer, x264 will force a keyframe.

This initial version does not support B-frames or intra refresh.
Recommended usage is to set keyint to a very large value, so that keyframes do not occur except as necessary for extreme error recovery.

Full documentation is in x264.h.

Move DTS/PTS calculation to before encoding each frame instead of after.
Improve documentation of x264_encoder_intra_refresh.

common/common.c
common/common.h
common/frame.c
common/frame.h
common/mvpred.c
encoder/encoder.c
encoder/set.c
x264.h

index 1ed983fb18471df186e69789d676387452ac75e3..8c7cf3c3ff48dc8c4d3c6a4335b1e24910be57fe 100644 (file)
@@ -634,6 +634,8 @@ int x264_param_parse( x264_param_t *p, const char *name, const char *value )
     }
     OPT2("ref", "frameref")
         p->i_frame_reference = atoi(value);
+    OPT("dpb-size")
+        p->i_dpb_size = atoi(value);
     OPT("keyint")
     {
         p->i_keyint_max = atoi(value);
index dfa11215640a0834efb838ca1769c381224b2414..7b60811fac14091f5a8073518dde75f7fd2c1b21 100644 (file)
@@ -421,6 +421,8 @@ struct x264_t
     int             i_cpb_delay_lookahead;
 
     int             b_queued_intra_refresh;
+    int64_t         i_reference_invalidate_pts;
+    int64_t         i_last_idr_pts;
 
     /* We use only one SPS and one PPS */
     x264_sps_t      sps_array[1];
index 3d007cca5154856f4e21c1b3b54999578e453dcd..9a8bbaee797aa0240b793f36d525ddce169c55ba 100644 (file)
@@ -443,6 +443,7 @@ x264_frame_t *x264_frame_pop_unused( x264_t *h, int b_fdec )
     frame->b_intra_calculated = 0;
     frame->b_scenecut = 1;
     frame->b_keyframe = 0;
+    frame->b_corrupt = 0;
 
     memset( frame->weight, 0, sizeof(frame->weight) );
     memset( frame->f_weighted_cost_delta, 0, sizeof(frame->f_weighted_cost_delta) );
index 26529ce8b095dfd15fbc76fad3ee7f10dbb62b7c..904ba4e40a5233a14672aa61ba9e2c8e7da8d58a 100644 (file)
@@ -35,6 +35,7 @@ typedef struct x264_frame
     int     i_type;
     int     i_qpplus1;
     int64_t i_pts;
+    int64_t i_dts;
     int64_t i_reordered_pts;
     int     i_duration;  /* in SPS time_scale units (i.e 2 * timebase units) used for vfr */
     int     i_cpb_duration;
@@ -143,6 +144,9 @@ typedef struct x264_frame
     int     i_pir_start_col;
     int     i_pir_end_col;
     int     i_frames_since_pir;
+
+    /* interactive encoder control */
+    int     b_corrupt;
 } x264_frame_t;
 
 /* synchronized frame list */
index 0056a2d983462242dca90bbd6fbce76c48623a84..7a8c93c93c8d8d55e36d05d93d86f9a17e36adf7 100644 (file)
@@ -409,12 +409,16 @@ void x264_mb_predict_mv_ref16x16( x264_t *h, int i_list, int i_ref, int16_t mvc[
 
     if( i_ref == 0 && h->frames.b_have_lowres )
     {
-        int16_t (*lowres_mv)[2] = i_list ? h->fenc->lowres_mvs[1][h->fref1[0]->i_frame-h->fenc->i_frame-1]
-                                         : h->fenc->lowres_mvs[0][h->fenc->i_frame-h->fref0[0]->i_frame-1];
-        if( lowres_mv[0][0] != 0x7fff )
+        int idx = i_list ? h->fref1[0]->i_frame-h->fenc->i_frame-1
+                         : h->fenc->i_frame-h->fref0[0]->i_frame-1;
+        if( idx <= h->param.i_bframe )
         {
-            M32( mvc[i] ) = (M32( lowres_mv[h->mb.i_mb_xy] )*2)&0xfffeffff;
-            i++;
+            int16_t (*lowres_mv)[2] = h->fenc->lowres_mvs[i_list][idx];
+            if( lowres_mv[0][0] != 0x7fff )
+            {
+                M32( mvc[i] ) = (M32( lowres_mv[h->mb.i_mb_xy] )*2)&0xfffeffff;
+                i++;
+            }
         }
     }
 
index 8ca45b171f4e73ba7af5e5c5c655e1afa95fdfbd..00b22d0ddd83dc391ca2687a3d1299c5fc6290e6 100644 (file)
@@ -564,6 +564,7 @@ static int x264_validate_parameters( x264_t *h )
     }
 
     h->param.i_frame_reference = x264_clip3( h->param.i_frame_reference, 1, 16 );
+    h->param.i_dpb_size = x264_clip3( h->param.i_dpb_size, 1, 16 );
     if( h->param.i_keyint_max <= 0 )
         h->param.i_keyint_max = 1;
     if( h->param.i_scenecut_threshold < 0 )
@@ -593,10 +594,11 @@ static int x264_validate_parameters( x264_t *h )
         x264_log( h, X264_LOG_WARNING, "b-pyramid normal + intra-refresh is not supported\n" );
         h->param.i_bframe_pyramid = X264_B_PYRAMID_STRICT;
     }
-    if( h->param.b_intra_refresh && h->param.i_frame_reference > 1 )
+    if( h->param.b_intra_refresh && (h->param.i_frame_reference > 1 || h->param.i_dpb_size > 1) )
     {
         x264_log( h, X264_LOG_WARNING, "ref > 1 + intra-refresh is not supported\n" );
         h->param.i_frame_reference = 1;
+        h->param.i_dpb_size = 1;
     }
     if( h->param.b_intra_refresh && h->param.i_open_gop )
     {
@@ -1481,6 +1483,8 @@ static inline void x264_reference_build_list( x264_t *h, int i_poc )
 
     for( int i = 0; h->frames.reference[i]; i++ )
     {
+        if( h->frames.reference[i]->b_corrupt )
+            continue;
         if( h->frames.reference[i]->i_poc < i_poc )
             h->fref0[h->i_ref0++] = h->frames.reference[i];
         else if( h->frames.reference[i]->i_poc > i_poc )
@@ -2185,6 +2189,23 @@ void x264_encoder_intra_refresh( x264_t *h )
     h->b_queued_intra_refresh = 1;
 }
 
+int x264_encoder_invalidate_reference( x264_t *h, int64_t pts )
+{
+    if( h->param.i_bframe )
+    {
+        x264_log( h, X264_LOG_ERROR, "x264_encoder_invalidate_reference is not supported with B-frames enabled\n" );
+        return -1;
+    }
+    if( h->param.b_intra_refresh )
+    {
+        x264_log( h, X264_LOG_ERROR, "x264_encoder_invalidate_reference is not supported with intra refresh enabled\n" );
+        return -1;
+    }
+    h = h->thread[h->i_thread_phase];
+    h->i_reference_invalidate_pts = pts;
+    return 0;
+}
+
 /****************************************************************************
  * x264_encoder_encode:
  *  XXX: i_poc   : is the poc of the current given picture
@@ -2330,6 +2351,29 @@ int     x264_encoder_encode( x264_t *h,
             h->fenc->param->param_free( h->fenc->param );
     }
 
+    if( h->i_reference_invalidate_pts )
+    {
+        if( h->i_reference_invalidate_pts >= h->i_last_idr_pts )
+            for( int i = 0; h->frames.reference[i]; i++ )
+                if( h->i_reference_invalidate_pts <= h->frames.reference[i]->i_pts )
+                    h->frames.reference[i]->b_corrupt = 1;
+        h->i_reference_invalidate_pts = 0;
+    }
+
+    if( !IS_X264_TYPE_I( h->fenc->i_type ) )
+    {
+        int valid_refs_left = 0;
+        for( int i = 0; h->frames.reference[i]; i++ )
+            if( !h->frames.reference[i]->b_corrupt )
+                valid_refs_left++;
+        /* No valid reference frames left: force an IDR. */
+        if( !valid_refs_left )
+        {
+            h->fenc->b_keyframe = 1;
+            h->fenc->i_type = X264_TYPE_IDR;
+        }
+    }
+
     if( h->fenc->b_keyframe )
     {
         h->frames.i_last_keyframe = h->fenc->i_frame;
@@ -2393,7 +2437,30 @@ int     x264_encoder_encode( x264_t *h,
     h->fenc->b_kept_as_ref =
     h->fdec->b_kept_as_ref = i_nal_ref_idc != NAL_PRIORITY_DISPOSABLE && h->param.i_keyint_max > 1;
 
-
+    h->fdec->i_pts = h->fenc->i_pts *= h->i_dts_compress_multiplier;
+    if( h->frames.i_bframe_delay )
+    {
+        int64_t *prev_reordered_pts = thread_current->frames.i_prev_reordered_pts;
+        if( h->i_frame <= h->frames.i_bframe_delay )
+        {
+            if( h->i_dts_compress_multiplier == 1 )
+                h->fdec->i_dts = h->fenc->i_reordered_pts - h->frames.i_bframe_delay_time;
+            else
+            {
+                /* DTS compression */
+                if( h->i_frame == 1 )
+                    thread_current->frames.i_init_delta = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
+                h->fdec->i_dts = h->i_frame * thread_current->frames.i_init_delta / h->i_dts_compress_multiplier;
+            }
+        }
+        else
+            h->fdec->i_dts = prev_reordered_pts[ (h->i_frame - h->frames.i_bframe_delay) % h->frames.i_bframe_delay ];
+        prev_reordered_pts[ h->i_frame % h->frames.i_bframe_delay ] = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
+    }
+    else
+        h->fdec->i_dts = h->fenc->i_reordered_pts;
+    if( h->fenc->i_type == X264_TYPE_IDR )
+        h->i_last_idr_pts = h->fdec->i_pts;
 
     /* ------------------- Init                ----------------------------- */
     /* build ref list 0/1 */
@@ -2616,28 +2683,9 @@ static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current,
 
     pic_out->b_keyframe = h->fenc->b_keyframe;
 
-    pic_out->i_pts = h->fenc->i_pts *= h->i_dts_compress_multiplier;
-    if( h->frames.i_bframe_delay )
-    {
-        int64_t *prev_reordered_pts = thread_current->frames.i_prev_reordered_pts;
-        if( h->i_frame <= h->frames.i_bframe_delay )
-        {
-            if( h->i_dts_compress_multiplier == 1 )
-                pic_out->i_dts = h->fenc->i_reordered_pts - h->frames.i_bframe_delay_time;
-            else
-            {
-                /* DTS compression */
-                if( h->i_frame == 1 )
-                    thread_current->frames.i_init_delta = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
-                pic_out->i_dts = h->i_frame * thread_current->frames.i_init_delta / h->i_dts_compress_multiplier;
-            }
-        }
-        else
-            pic_out->i_dts = prev_reordered_pts[ (h->i_frame - h->frames.i_bframe_delay) % h->frames.i_bframe_delay ];
-        prev_reordered_pts[ h->i_frame % h->frames.i_bframe_delay ] = h->fenc->i_reordered_pts * h->i_dts_compress_multiplier;
-    }
-    else
-        pic_out->i_dts = h->fenc->i_reordered_pts;
+    pic_out->i_pts = h->fdec->i_pts;
+    pic_out->i_dts = h->fdec->i_dts;
+
     if( pic_out->i_pts < pic_out->i_dts )
         x264_log( h, X264_LOG_WARNING, "invalid DTS: PTS is less than DTS\n" );
 
index 86b4a305ecc0555892655914608fdc7a6fb9a4e3..8d007aae4a0c1a1abddd582de6fd6798eb7d367c 100644 (file)
@@ -223,8 +223,8 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param )
     /* extra slot with pyramid so that we don't have to override the
      * order of forgetting old pictures */
     sps->vui.i_max_dec_frame_buffering =
-    sps->i_num_ref_frames = X264_MIN(16, X264_MAX3(param->i_frame_reference, 1 + sps->vui.i_num_reorder_frames,
-                                                   param->i_bframe_pyramid ? 4 : 1 ));
+    sps->i_num_ref_frames = X264_MIN(16, X264_MAX4(param->i_frame_reference, 1 + sps->vui.i_num_reorder_frames,
+                            param->i_bframe_pyramid ? 4 : 1, param->i_dpb_size));
     sps->i_num_ref_frames -= param->i_bframe_pyramid == X264_B_PYRAMID_STRICT;
 
     sps->vui.b_bitstream_restriction = 1;
diff --git a/x264.h b/x264.h
index 09183fde726b198209565aa9223b34d7dbe8a7ee..1138a8bafde49efc77e8adcbe4fc4256f6d1f82e 100644 (file)
--- a/x264.h
+++ b/x264.h
@@ -35,7 +35,7 @@
 
 #include <stdarg.h>
 
-#define X264_BUILD 99
+#define X264_BUILD 100
 
 /* x264_t:
  *      opaque handler for encoder */
@@ -217,6 +217,8 @@ typedef struct x264_param_t
 
     /* Bitstream parameters */
     int         i_frame_reference;  /* Maximum number of reference frames */
+    int         i_dpb_size;         /* Force a DPB size larger than that implied by B-frames and reference frames.
+                                     * Useful in combination with interactive error resilience. */
     int         i_keyint_max;       /* Force an IDR keyframe at this interval */
     int         i_keyint_min;       /* Scenecuts closer together than this are coded as I, not IDR. */
     int         i_scenecut_threshold; /* how aggressively to insert extra I frames */
@@ -682,9 +684,38 @@ int     x264_encoder_delayed_frames( x264_t * );
  *      If an intra refresh is not in progress, begin one with the next P-frame.
  *      If an intra refresh is in progress, begin one as soon as the current one finishes.
  *      Requires that b_intra_refresh be set.
+ *
  *      Useful for interactive streaming where the client can tell the server that packet loss has
  *      occurred.  In this case, keyint can be set to an extremely high value so that intra refreshes
- *      only occur when calling x264_encoder_intra_refresh. */
+ *      only occur when calling x264_encoder_intra_refresh.
+ *
+ *      In multi-pass encoding, if x264_encoder_intra_refresh is called differently in each pass,
+ *      behavior is undefined.
+ *
+ *      Should not be called during an x264_encoder_encode. */
 void    x264_encoder_intra_refresh( x264_t * );
+/* x264_encoder_invalidate_reference:
+ *      An interactive error resilience tool, designed for use in a low-latency one-encoder-few-clients
+ *      system.  When the client has packet loss or otherwise incorrectly decodes a frame, the encoder
+ *      can be told with this command to "forget" the frame and all frames that depend on it, referencing
+ *      only frames that occurred before the loss.  This will force a keyframe if no frames are left to
+ *      reference after the aforementioned "forgetting".
+ *
+ *      It is strongly recommended to use a large i_dpb_size in this case, which allows the encoder to
+ *      keep around extra, older frames to fall back on in case more recent frames are all invalidated.
+ *      Unlike increasing i_frame_reference, this does not increase the number of frames used for motion
+ *      estimation and thus has no speed impact.  It is also recommended to set a very large keyframe
+ *      interval, so that keyframes are not used except as necessary for error recovery.
+ *
+ *      x264_encoder_invalidate_reference is not currently compatible with the use of B-frames or intra
+ *      refresh.
+ *
+ *      In multi-pass encoding, if x264_encoder_invalidate_reference is called differently in each pass,
+ *      behavior is undefined.
+ *
+ *      Should not be called during an x264_encoder_encode.
+ *
+ *      Returns 0 on success, negative on failure. */
+int x264_encoder_invalidate_reference( x264_t *, int64_t pts );
 
 #endif