Fix deinterlacing of packed YUV formats.

[vlc] / modules / stream_out / transcode / video.c

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 5e07ca2ef92b86caa9e94efe0d085fd3140434f7..7196a40c24287100b43c67f3aeab70db5fcf2775 100644 (file)
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -258,6 +258,61 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
     return VLC_SUCCESS;
 }
 
+static filter_t *AppendDeinterlaceFilter( sout_stream_t *p_stream,
+                                          filter_chain_t *p_chain,
+                                          const char *psz_name,
+                                          config_chain_t *p_cfg,
+                                          const es_format_t *p_fmt_in,
+                                          const es_format_t *p_fmt_out )
+{
+    filter_t *p_filter =
+        filter_chain_AppendFilter( p_chain, psz_name, p_cfg, p_fmt_in, p_fmt_out );
+    if( p_filter )
+        return p_filter;
+
+    msg_Dbg( p_stream, "deinterlace init failed, trying colorspace conversion" );
+
+    /* Most likely the filter failed due to wrong (e.g. non-planar) input format.
+     * Try converting to a different color space and try again.
+     *
+     * We use the recommended fallback formats in order, to get a conversion that's
+     * as cheap (both CPU-wise and quality-wise) as possible.
+     */
+    for( const vlc_fourcc_t *fourcc = vlc_fourcc_GetYUVFallback( p_fmt_out->video.i_chroma );
+         *fourcc != 0;
+         ++fourcc )
+    {
+        msg_Dbg( p_stream, "trying deinterlace through colorspace conversion to %4.4s", (char *)fourcc );
+
+        es_format_t p_new_fmt;
+        es_format_Copy( &p_new_fmt, p_fmt_in );
+        p_new_fmt.video.i_chroma = *fourcc;  // XXX: is this enough?
+
+        filter_chain_Reset( p_chain, NULL, NULL ); 
+
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              NULL, NULL,
+                                              p_fmt_in,
+                                              &p_new_fmt );
+        if( !p_filter )
+            continue;
+                   
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              psz_name,
+                                              p_cfg,
+                                              &p_new_fmt,
+                                              &p_new_fmt );
+        if( p_filter )
+        {
+            msg_Dbg( p_stream, "deinterlace through colorspace conversion to %4.4s succeeded", (char *)fourcc );
+            return p_filter;
+        }
+    }
+
+    msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+    return NULL;
+}
+ 
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_t *id )
 {
@@ -273,11 +328,18 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   p_stream->p_sys->psz_deinterlace,
-                                   p_stream->p_sys->p_deinterlace_cfg,
-                                   &id->p_decoder->fmt_out,
-                                   &id->p_decoder->fmt_out );
+        filter_t *p_filter =
+            AppendDeinterlaceFilter( p_stream,
+                                     id->p_f_chain,
+                                     p_stream->p_sys->psz_deinterlace,
+                                     p_stream->p_sys->p_deinterlace_cfg,
+                                     p_fmt_out,
+                                     p_fmt_out );
+        if( !p_filter )
+        {
+            msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+            return;
+        } 
 
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }