From: Steinar H. Gunderson <steinar+vlc@gunderson.no>
Date: Fri, 16 Aug 2013 14:42:44 +0000 (+0200)
Subject: Fix deinterlacing of packed YUV formats.
X-Git-Url: https://git.sesse.net/?p=vlc;a=commitdiff_plain;h=20ce6b1fbbbb735b6274d0d5f21d1819719bbe30

Fix deinterlacing of packed YUV formats.

If adding the deinterlacer filter fails, try conversion filters
until we try something the deinterlacer wants, and then try again.
This fixes the problem where the input from the decklink module
(at least with --no-decklink-tenbits) is UYVY, while the deinterlacer only
wants planar formats.

Another possibility would be going straight for the format the
encoder wants, but often, this would be a 4:2:0 format, and 4:2:0
is pretty bad to deinterlace in, since the vertical chroma resolution
is gone already (the chroma interlacing in 4:2:0 is rather odd).
---

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index c0fc480010..19591e14fe 100644
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -245,6 +245,61 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
     return VLC_SUCCESS;
 }
 
+static filter_t *AppendDeinterlaceFilter( sout_stream_t *p_stream,
+                                          filter_chain_t *p_chain,
+                                          const char *psz_name,
+                                          config_chain_t *p_cfg,
+                                          const es_format_t *p_fmt_in,
+                                          const es_format_t *p_fmt_out )
+{
+    filter_t *p_filter =
+        filter_chain_AppendFilter( p_chain, psz_name, p_cfg, p_fmt_in, p_fmt_out );
+    if( p_filter )
+        return p_filter;
+
+    msg_Dbg( p_stream, "deinterlace init failed, trying colorspace conversion" );
+
+    /* Most likely the filter failed due to wrong (e.g. non-planar) input format.
+     * Try converting to a different color space and try again.
+     *
+     * We use the recommended fallback formats in order, to get a conversion that's
+     * as cheap (both CPU-wise and quality-wise) as possible.
+     */
+    for( const vlc_fourcc_t *fourcc = vlc_fourcc_GetYUVFallback( p_fmt_out->video.i_chroma );
+         *fourcc != 0;
+         ++fourcc )
+    {
+        msg_Dbg( p_stream, "trying deinterlace through colorspace conversion to %4.4s", (char *)fourcc );
+
+        es_format_t p_new_fmt;
+        es_format_Copy( &p_new_fmt, p_fmt_in );
+        p_new_fmt.video.i_chroma = *fourcc;  // XXX: is this enough?
+
+        filter_chain_Reset( p_chain, NULL, NULL ); 
+
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              NULL, NULL,
+                                              p_fmt_in,
+                                              &p_new_fmt );
+        if( !p_filter )
+            continue;
+                   
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              psz_name,
+                                              p_cfg,
+                                              &p_new_fmt,
+                                              &p_new_fmt );
+        if( p_filter )
+        {
+            msg_Dbg( p_stream, "deinterlace through colorspace conversion to %4.4s succeeded", (char *)fourcc );
+            return p_filter;
+        }
+    }
+
+    msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+    return NULL;
+}
+ 
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_sys_t *id )
 {
@@ -263,11 +318,18 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   p_stream->p_sys->psz_deinterlace,
-                                   p_stream->p_sys->p_deinterlace_cfg,
-                                   &id->p_decoder->fmt_out,
-                                   &id->p_decoder->fmt_out );
+        filter_t *p_filter =
+            AppendDeinterlaceFilter( p_stream,
+                                     id->p_f_chain,
+                                     p_stream->p_sys->psz_deinterlace,
+                                     p_stream->p_sys->p_deinterlace_cfg,
+                                     p_fmt_out,
+                                     p_fmt_out );
+        if( !p_filter )
+        {
+            msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+            return;
+        } 
 
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }