In video transcode, use the correct format for calculations

The current code doesn't properly take into account that filters
can change the format (including the frame rate) from that of the
decoder's output, and thus uncritically use the decoder's output
format where it should use the output of the decoder -> filter
-> user_filter chain.

Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 7807cf813c7461bdba24ef47b6f7efa9e4c9ffc1..537177bcbec91e3d4e15801db456db4610442b91 100644 (file)
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -270,25 +270,24 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-       filter_chain_AppendFilter( id->p_f_chain,
-                                  p_stream->p_sys->psz_deinterlace,
-                                  p_stream->p_sys->p_deinterlace_cfg,
-                                  &id->p_decoder->fmt_out,
-                                  &id->p_decoder->fmt_out );
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   p_stream->p_sys->psz_deinterlace,
+                                   p_stream->p_sys->p_deinterlace_cfg,
+                                   &id->p_decoder->fmt_out,
+                                   &id->p_decoder->fmt_out );
+
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }
 
     /* Take care of the scaling and chroma conversions */
-    if( ( id->p_decoder->fmt_out.video.i_chroma !=
-          id->p_encoder->fmt_in.video.i_chroma ) ||
-        ( id->p_decoder->fmt_out.video.i_width !=
-          id->p_encoder->fmt_in.video.i_width ) ||
-        ( id->p_decoder->fmt_out.video.i_height !=
-          id->p_encoder->fmt_in.video.i_height ) )
+    if( ( p_fmt_out->video.i_chroma != id->p_encoder->fmt_in.video.i_chroma ) ||
+        ( p_fmt_out->video.i_width != id->p_encoder->fmt_in.video.i_width ) ||
+        ( p_fmt_out->video.i_height != id->p_encoder->fmt_in.video.i_height ) )
     {
-       filter_chain_AppendFilter( id->p_f_chain,
-                                  NULL, NULL,
-                                  &id->p_decoder->fmt_out,
-                                  &id->p_encoder->fmt_in );
+        filter_chain_AppendFilter( id->p_f_chain,
+                                   NULL, NULL,
+                                   p_fmt_out,
+                                   &id->p_encoder->fmt_in );
     }
 
     if( p_stream->p_sys->psz_vf2 )
@@ -321,10 +320,18 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
 {
     sout_stream_sys_t *p_sys = p_stream->p_sys;
 
+    const es_format_t *p_fmt_out = &id->p_decoder->fmt_out;
+    if( id->p_f_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
+    }
+    if( id->p_uf_chain ) {
+        p_fmt_out = filter_chain_GetFmtOut( id->p_uf_chain );
+    }
+
     /* Calculate scaling
      * width/height of source */
-    int i_src_width = id->p_decoder->fmt_out.video.i_width;
-    int i_src_height = id->p_decoder->fmt_out.video.i_height;
+    int i_src_width = p_fmt_out->video.i_width;
+    int i_src_height = p_fmt_out->video.i_height;
 
     /* with/height scaling */
     float f_scale_width = 1;
@@ -335,10 +342,10 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     int i_dst_height;
 
     /* aspect ratio */
-    float f_aspect = (double)id->p_decoder->fmt_out.video.i_sar_num *
-                     id->p_decoder->fmt_out.video.i_width /
-                     id->p_decoder->fmt_out.video.i_sar_den /
-                     id->p_decoder->fmt_out.video.i_height;
+    float f_aspect = (double)p_fmt_out->video.i_sar_num *
+                     p_fmt_out->video.i_width /
+                     p_fmt_out->video.i_sar_den /
+                     p_fmt_out->video.i_height;
 
     msg_Dbg( p_stream, "decoder aspect is %f:1", f_aspect );
 
@@ -437,13 +444,13 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     if( !id->p_encoder->fmt_out.video.i_frame_rate ||
         !id->p_encoder->fmt_out.video.i_frame_rate_base )
     {
-        if( id->p_decoder->fmt_out.video.i_frame_rate &&
-            id->p_decoder->fmt_out.video.i_frame_rate_base )
+        if( p_fmt_out->video.i_frame_rate &&
+            p_fmt_out->video.i_frame_rate_base )
         {
             id->p_encoder->fmt_out.video.i_frame_rate =
-                id->p_decoder->fmt_out.video.i_frame_rate;
+                p_fmt_out->video.i_frame_rate;
             id->p_encoder->fmt_out.video.i_frame_rate_base =
-                id->p_decoder->fmt_out.video.i_frame_rate_base;
+                p_fmt_out->video.i_frame_rate_base;
         }
         else
         {
@@ -468,8 +475,8 @@ static void transcode_video_encoder_init( sout_stream_t *p_stream,
     {
         vlc_ureduce( &id->p_encoder->fmt_out.video.i_sar_num,
                      &id->p_encoder->fmt_out.video.i_sar_den,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_num * i_src_width  * i_dst_height,
-                     (uint64_t)id->p_decoder->fmt_out.video.i_sar_den * i_src_height * i_dst_width,
+                     (uint64_t)p_fmt_out->video.i_sar_num * i_src_width  * i_dst_height,
+                     (uint64_t)p_fmt_out->video.i_sar_den * i_src_height * i_dst_width,
                      0 );
     }
     else