]> git.sesse.net Git - vlc/blobdiff - modules/stream_out/transcode/video.c
transcode: keep in track of input pts drift for video
[vlc] / modules / stream_out / transcode / video.c
index e6734b0921e47bc53f3243048a03456bb6582c89..60b66bbbac14620fc0da0fefdd344cd61ffd83c2 100644 (file)
@@ -100,7 +100,7 @@ static void transcode_video_filter_allocation_clear( filter_t *p_filter )
 static void* EncoderThread( void *obj )
 {
     sout_stream_sys_t *p_sys = (sout_stream_sys_t*)obj;
-    sout_stream_id_t *id = p_sys->id_video;
+    sout_stream_id_sys_t *id = p_sys->id_video;
     picture_t *p_pic = NULL;
     int canc = vlc_savecancel ();
     block_t *p_block = NULL;
@@ -137,7 +137,7 @@ static void* EncoderThread( void *obj )
             vlc_mutex_unlock( &p_sys->lock_out );
             break;
         }
-            vlc_mutex_unlock( &p_sys->lock_out );
+        vlc_mutex_unlock( &p_sys->lock_out );
     }
 
     /*Encode what we have in the buffer on closing*/
@@ -164,7 +164,7 @@ static void* EncoderThread( void *obj )
     return NULL;
 }
 
-int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
+int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
@@ -232,8 +232,6 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
           ? id->p_encoder->fmt_out.video.i_visible_height
           : id->p_decoder->fmt_in.video.i_visible_height
             ? id->p_decoder->fmt_in.video.i_visible_height : id->p_encoder->fmt_in.video.i_height;
-    id->p_encoder->fmt_in.video.i_frame_rate = id->p_decoder->fmt_out.video.i_frame_rate;
-    id->p_encoder->fmt_in.video.i_frame_rate_base = id->p_decoder->fmt_out.video.i_frame_rate_base;
 
     id->p_encoder->i_threads = p_sys->i_threads;
     id->p_encoder->p_cfg = p_sys->p_video_cfg;
@@ -298,7 +296,7 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
 }
 
 static void transcode_video_filter_init( sout_stream_t *p_stream,
-                                         sout_stream_id_t *id )
+                                         sout_stream_id_sys_t *id )
 {
     es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
     id->p_encoder->fmt_in.video.i_chroma = id->p_encoder->fmt_in.i_codec;
@@ -337,6 +335,13 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
                            p_stream->p_sys );
         filter_chain_Reset( id->p_uf_chain, p_fmt_out,
                             &id->p_encoder->fmt_in );
+        if( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma )
+        {
+            filter_chain_AppendFilter( id->p_uf_chain,
+                                   NULL, NULL,
+                                   p_fmt_out,
+                                   &id->p_encoder->fmt_in );
+        }
         filter_chain_AppendFromString( id->p_uf_chain, p_stream->p_sys->psz_vf2 );
         p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
         es_format_Copy( &id->p_encoder->fmt_in, p_fmt_out );
@@ -353,11 +358,12 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
 }
 
 /* Take care of the scaling and chroma conversions. */
-static void conversion_video_filter_append( sout_stream_id_t *id )
+static void conversion_video_filter_append( sout_stream_id_sys_t *id )
 {
     const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
     if( id->p_f_chain )
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
+
     if( id->p_uf_chain )
         p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
 
@@ -365,7 +371,7 @@ static void conversion_video_filter_append( sout_stream_id_t *id )
         ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
         ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
+        filter_chain_AppendFilter( id->p_uf_chain ? id->p_uf_chain : id->p_f_chain,
                                    NULL, NULL,
                                    p_fmt_out,
                                    &id->p_encoder->fmt_in );
@@ -373,7 +379,7 @@ static void conversion_video_filter_append( sout_stream_id_t *id )
 }
 
 static void transcode_video_encoder_init( sout_stream_t *p_stream,
-                                          sout_stream_id_t *id )
+                                          sout_stream_id_sys_t *id )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
@@ -413,8 +419,8 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     msg_Dbg( p_stream, "source pixel aspect is %f:1", f_aspect );
 
     /* Calculate scaling factor for specified parameters */
-    if( id->p_encoder->fmt_out.video.i_width <= 0 &&
-        id->p_encoder->fmt_out.video.i_height <= 0 && p_sys->f_scale )
+    if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
+        id->p_encoder->fmt_out.video.i_visible_height <= 0 && p_sys->f_scale )
     {
         /* Global scaling. Make sure width will remain a factor of 16 */
         float f_real_scale;
@@ -433,26 +439,26 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
         f_scale_width = f_real_scale;
         f_scale_height = (float) i_new_height / (float) i_src_visible_height;
     }
-    else if( id->p_encoder->fmt_out.video.i_width > 0 &&
-             id->p_encoder->fmt_out.video.i_height <= 0 )
+    else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
+             id->p_encoder->fmt_out.video.i_visible_height <= 0 )
     {
         /* Only width specified */
-        f_scale_width = (float)id->p_encoder->fmt_out.video.i_width/i_src_visible_width;
+        f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
         f_scale_height = f_scale_width;
     }
-    else if( id->p_encoder->fmt_out.video.i_width <= 0 &&
-             id->p_encoder->fmt_out.video.i_height > 0 )
+    else if( id->p_encoder->fmt_out.video.i_visible_width <= 0 &&
+             id->p_encoder->fmt_out.video.i_visible_height > 0 )
     {
          /* Only height specified */
-         f_scale_height = (float)id->p_encoder->fmt_out.video.i_height/i_src_visible_height;
+         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
          f_scale_width = f_scale_height;
      }
-     else if( id->p_encoder->fmt_out.video.i_width > 0 &&
-              id->p_encoder->fmt_out.video.i_height > 0 )
+     else if( id->p_encoder->fmt_out.video.i_visible_width > 0 &&
+              id->p_encoder->fmt_out.video.i_visible_height > 0 )
      {
          /* Width and height specified */
-         f_scale_width = (float)id->p_encoder->fmt_out.video.i_width/i_src_visible_width;
-         f_scale_height = (float)id->p_encoder->fmt_out.video.i_height/i_src_visible_height;
+         f_scale_width = (float)id->p_encoder->fmt_out.video.i_visible_width/i_src_visible_width;
+         f_scale_height = (float)id->p_encoder->fmt_out.video.i_visible_height/i_src_visible_height;
      }
 
      /* check maxwidth and maxheight */
@@ -522,6 +528,10 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
         }
     }
 
+    id->p_encoder->fmt_in.video.orientation =
+        id->p_encoder->fmt_out.video.orientation =
+        id->p_decoder->fmt_in.video.orientation;
+
     id->p_encoder->fmt_in.video.i_frame_rate =
         id->p_encoder->fmt_out.video.i_frame_rate;
     id->p_encoder->fmt_in.video.i_frame_rate_base =
@@ -538,14 +548,18 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
         id->p_encoder->fmt_in.video.i_frame_rate,
         id->p_encoder->fmt_in.video.i_frame_rate_base );
 
-    id->i_output_frame_interval = id->p_encoder->fmt_out.video.i_frame_rate_base * CLOCK_FREQ / id->p_encoder->fmt_out.video.i_frame_rate;
-    id->i_input_frame_interval = id->p_decoder->fmt_out.video.i_frame_rate_base * CLOCK_FREQ / id->p_decoder->fmt_out.video.i_frame_rate;
-    msg_Info( p_stream, "input interval %d (base %d)  output interval %d (base %d)", id->i_input_frame_interval, id->p_decoder->fmt_out.video.i_frame_rate_base,
+    id->i_input_frame_interval  = id->p_decoder->fmt_out.video.i_frame_rate_base * CLOCK_FREQ / id->p_decoder->fmt_out.video.i_frame_rate;
+    msg_Info( p_stream, "input interval %d (base %d)",
+                        id->i_input_frame_interval, id->p_decoder->fmt_out.video.i_frame_rate_base );
+
+    id->i_output_frame_interval = id->p_encoder->fmt_in.video.i_frame_rate_base * CLOCK_FREQ / id->p_encoder->fmt_in.video.i_frame_rate;
+    msg_Info( p_stream, "output interval %d (base %d)",
                         id->i_output_frame_interval, id->p_encoder->fmt_in.video.i_frame_rate_base );
 
-    date_Init( &id->interpolated_pts,
+    date_Init( &id->next_input_pts,
                id->p_decoder->fmt_out.video.i_frame_rate,
                1 );
+
     date_Init( &id->next_output_pts,
                id->p_encoder->fmt_in.video.i_frame_rate,
                1 );
@@ -581,7 +595,7 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
 }
 
 static int transcode_video_encoder_open( sout_stream_t *p_stream,
-                                         sout_stream_id_t *id )
+                                         sout_stream_id_sys_t *id )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
@@ -617,7 +631,7 @@ static int transcode_video_encoder_open( sout_stream_t *p_stream,
 }
 
 void transcode_video_close( sout_stream_t *p_stream,
-                                   sout_stream_id_t *id )
+                                   sout_stream_id_sys_t *id )
 {
     if( p_stream->p_sys->i_threads >= 1 )
     {
@@ -654,17 +668,21 @@ void transcode_video_close( sout_stream_t *p_stream,
         filter_chain_Delete( id->p_uf_chain );
 }
 
-static void OutputFrame( sout_stream_sys_t *p_sys, picture_t *p_pic, sout_stream_t *p_stream, sout_stream_id_t *id, block_t **out )
+static void OutputFrame( sout_stream_t *p_stream, picture_t *p_pic, sout_stream_id_sys_t *id, block_t **out )
 {
-
+    sout_stream_sys_t *p_sys = p_stream->p_sys;
     picture_t *p_pic2 = NULL;
+    const mtime_t original_date = p_pic->date;
     bool b_need_duplicate=false;
-    /* If input pts + input_frame_interval is lower than next_output_pts - output_frame_interval
+    /* If input pts is lower than next_output_pts - output_frame_interval
      * Then the future input frame should fit better and we can drop this one 
      *
-     * Duplication need is checked in OutputFrame */
-    if( ( p_pic->date + (mtime_t)id->i_input_frame_interval ) <
-        ( date_Get( &id->next_output_pts ) ) )
+     * We check it here also because we can have case that video filters outputs multiple
+     * pictures but we don't need to use them all, for example yadif2x and outputting to some
+     * different fps value
+     */
+    if( ( original_date ) <
+        ( date_Get( &id->next_output_pts ) - (mtime_t)id->i_output_frame_interval ) )
     {
 #if 0
         msg_Dbg( p_stream, "dropping frame (%"PRId64" + %"PRId64" vs %"PRId64")",
@@ -714,6 +732,9 @@ static void OutputFrame( sout_stream_sys_t *p_sys, picture_t *p_pic, sout_stream
             subpicture_Delete( p_subpic );
         }
     }
+
+    /* set output pts*/
+    p_pic->date = date_Get( &id->next_output_pts );
     /*This pts is handled, increase clock to next one*/
     date_Increment( &id->next_output_pts, id->p_encoder->fmt_in.video.i_frame_rate_base );
 
@@ -727,7 +748,7 @@ static void OutputFrame( sout_stream_sys_t *p_sys, picture_t *p_pic, sout_stream
 
     /* we need to duplicate while next_output_pts + output_frame_interval < input_pts (next input pts)*/
     b_need_duplicate = ( date_Get( &id->next_output_pts ) + id->i_output_frame_interval ) <
-                       ( date_Get( &id->interpolated_pts ) );
+                       ( original_date );
 
     if( p_sys->i_threads )
     {
@@ -772,7 +793,7 @@ static void OutputFrame( sout_stream_sys_t *p_sys, picture_t *p_pic, sout_stream
 #endif
         date_Increment( &id->next_output_pts, id->p_encoder->fmt_in.video.i_frame_rate_base );
         b_need_duplicate = ( date_Get( &id->next_output_pts ) + id->i_output_frame_interval ) <
-                           ( date_Get( &id->interpolated_pts ) );
+                           ( original_date );
     }
 
     if( p_sys->i_threads && p_pic2 )
@@ -781,7 +802,7 @@ static void OutputFrame( sout_stream_sys_t *p_sys, picture_t *p_pic, sout_stream
         picture_Release( p_pic );
 }
 
-int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
+int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_sys_t *id,
                                     block_t *in, block_t **out )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
@@ -821,27 +842,15 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
     while( (p_pic = id->p_decoder->pf_decode_video( id->p_decoder, &in )) )
     {
 
-        if( p_stream->p_sout->i_out_pace_nocontrol && p_sys->b_hurry_up )
-        {
-            mtime_t current_date = mdate();
-            if( unlikely( (current_date - 50000) > p_pic->date ) )
-            {
-                msg_Dbg( p_stream, "late picture skipped (%"PRId64")",
-                         current_date - 50000 - p_pic->date );
-                picture_Release( p_pic );
-                continue;
-            }
-        }
-
         if( unlikely (
              id->p_encoder->p_module &&
-             !video_format_IsSimilar( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video )
+             !video_format_IsSimilar( &id->fmt_input_video, &id->p_decoder->fmt_out.video )
             )
           )
         {
             msg_Info( p_stream, "aspect-ratio changed, reiniting. %i -> %i : %i -> %i.",
-                        p_sys->fmt_input_video.i_sar_num, id->p_decoder->fmt_out.video.i_sar_num,
-                        p_sys->fmt_input_video.i_sar_den, id->p_decoder->fmt_out.video.i_sar_den
+                        id->fmt_input_video.i_sar_num, id->p_decoder->fmt_out.video.i_sar_num,
+                        id->fmt_input_video.i_sar_den, id->p_decoder->fmt_out.video.i_sar_den
                     );
             /* Close filters */
             if( id->p_f_chain )
@@ -852,14 +861,14 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
             id->p_uf_chain = NULL;
 
             /* Reinitialize filters */
-            id->p_encoder->fmt_out.video.i_width  = p_sys->i_width & ~1;
-            id->p_encoder->fmt_out.video.i_height = p_sys->i_height & ~1;
+            id->p_encoder->fmt_out.video.i_visible_width  = p_sys->i_width & ~1;
+            id->p_encoder->fmt_out.video.i_visible_height = p_sys->i_height & ~1;
             id->p_encoder->fmt_out.video.i_sar_num = id->p_encoder->fmt_out.video.i_sar_den = 0;
 
             transcode_video_filter_init( p_stream, id );
             transcode_video_encoder_init( p_stream, id );
             conversion_video_filter_append( id );
-            memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
+            memcpy( &id->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
         }
 
 
@@ -874,7 +883,7 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
             transcode_video_filter_init( p_stream, id );
             transcode_video_encoder_init( p_stream, id );
             conversion_video_filter_append( id );
-            memcpy( &p_sys->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
+            memcpy( &id->fmt_input_video, &id->p_decoder->fmt_out.video, sizeof(video_format_t));
 
             if( transcode_video_encoder_open( p_stream, id ) != VLC_SUCCESS )
             {
@@ -883,47 +892,22 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
                 id->b_transcode = false;
                 return VLC_EGENERIC;
             }
+            date_Set( &id->next_output_pts, p_pic->date );
+            date_Set( &id->next_input_pts, p_pic->date );
         }
 
         /*Input lipsync and drop check */
         if( p_sys->b_master_sync )
         {
-            /* How much audio has drifted */
-            mtime_t i_master_drift = p_sys->i_master_drift;
-
-            /* This is the pts input should have now with constant frame rate */
-            mtime_t i_pts = date_Get( &id->interpolated_pts );
-
-            /* How much video pts is ahead of calculated pts */
-            mtime_t i_video_drift = p_pic->date - i_pts;
-
-            /* Check that we are having lipsync with input here */
-            if( unlikely ( ( (i_video_drift - i_master_drift ) > MASTER_SYNC_MAX_DRIFT
-                          || (i_video_drift + i_master_drift ) < -MASTER_SYNC_MAX_DRIFT ) ) )
-            {
-                msg_Warn( p_stream,
-                    "video drift too big, resetting sync %"PRId64" to %"PRId64,
-                    (i_video_drift + i_master_drift),
-                    p_pic->date
-                    );
-                date_Set( &id->interpolated_pts, p_pic->date );
-                date_Set( &id->next_output_pts, p_pic->date );
-                i_pts = date_Get( &id->interpolated_pts );
-            }
-
-            /* Set the pts of the frame being encoded */
-            p_pic->date = i_pts;
-
-            /* now take next input pts, pts dates are only enabled if p_module is set*/
-            date_Increment( &id->interpolated_pts, id->p_decoder->fmt_out.video.i_frame_rate_base );
-
-
-            /* If input pts + input_frame_interval is lower than next_output_pts - output_frame_interval
+            /* If input pts lower than next_output_pts - output_frame_interval
              * Then the future input frame should fit better and we can drop this one 
              *
+             * We check this here as we don't need to run video filter at all for pictures
+             * we are going to drop anyway
+             *
              * Duplication need is checked in OutputFrame */
-            if( ( p_pic->date + (mtime_t)id->i_input_frame_interval ) <
-                ( date_Get( &id->next_output_pts ) ) )
+            if( ( p_pic->date ) <
+                ( date_Get( &id->next_output_pts ) - (mtime_t)id->i_output_frame_interval ) )
             {
 #if 0
                 msg_Dbg( p_stream, "dropping frame (%"PRId64" + %"PRId64" vs %"PRId64")",
@@ -936,11 +920,22 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
             msg_Dbg( p_stream, "not dropping frame");
 #endif
 
-            /* input calculated pts isn't necessary what pts output should be, so use output pts*/
-            p_pic->date = date_Get( &id->next_output_pts );
-
-
         }
+        /* Check input drift regardless, if it's more than 100ms from our approximation, we most likely have lost pictures
+         * and are in danger to become out of sync, so better reset timestamps then */
+        if( likely( p_pic->date != VLC_TS_INVALID ) )
+        {
+            mtime_t input_drift = p_pic->date - date_Get( &id->next_input_pts );
+            if( unlikely( (input_drift > (CLOCK_FREQ/10)) ||
+                          (input_drift < -(CLOCK_FREQ/10))
+               ) )
+            {
+                msg_Warn( p_stream, "Reseting video sync" );
+                date_Set( &id->next_output_pts, p_pic->date );
+                date_Set( &id->next_input_pts, p_pic->date );
+            }
+        }
+        date_Increment( &id->next_input_pts, id->p_decoder->fmt_out.video.i_frame_rate_base );
 
         /* Run the filter and output chains; first with the picture,
          * and then with NULL as many times as we need until they
@@ -964,7 +959,7 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
                 if( !p_user_filtered_pic )
                     break;
 
-                OutputFrame( p_sys, p_user_filtered_pic, p_stream, id, out );
+                OutputFrame( p_stream, p_user_filtered_pic, id, out );
 
                 p_filtered_pic = NULL;
             }
@@ -986,7 +981,7 @@ int transcode_video_process( sout_stream_t *p_stream, sout_stream_id_t *id,
 }
 
 bool transcode_video_add( sout_stream_t *p_stream, es_format_t *p_fmt,
-                                sout_stream_id_t *id )
+                                sout_stream_id_sys_t *id )
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
@@ -996,8 +991,8 @@ bool transcode_video_add( sout_stream_t *p_stream, es_format_t *p_fmt,
 
     /* Complete destination format */
     id->p_encoder->fmt_out.i_codec = p_sys->i_vcodec;
-    id->p_encoder->fmt_out.video.i_width  = p_sys->i_width & ~1;
-    id->p_encoder->fmt_out.video.i_height = p_sys->i_height & ~1;
+    id->p_encoder->fmt_out.video.i_visible_width  = p_sys->i_width & ~1;
+    id->p_encoder->fmt_out.video.i_visible_height = p_sys->i_height & ~1;
     id->p_encoder->fmt_out.i_bitrate = p_sys->i_vbitrate;
 
     /* Build decoder -> filter -> encoder chain */
@@ -1011,10 +1006,10 @@ bool transcode_video_add( sout_stream_t *p_stream, es_format_t *p_fmt,
      * all the characteristics of the decoded stream yet */
     id->b_transcode = true;
 
-    if( p_sys->f_fps > 0 )
+    if( p_sys->fps_num )
     {
-        id->p_encoder->fmt_out.video.i_frame_rate = (p_sys->f_fps * ENC_FRAMERATE_BASE);
-        id->p_encoder->fmt_out.video.i_frame_rate_base = ENC_FRAMERATE_BASE;
+        id->p_encoder->fmt_out.video.i_frame_rate = (p_sys->fps_num );
+        id->p_encoder->fmt_out.video.i_frame_rate_base = (p_sys->fps_den ? p_sys->fps_den : 1);
     }
 
     return true;