Fix deinterlacing of packed YUV formats.

If adding the deinterlacer filter fails, try conversion filters
until we try something the deinterlacer wants, and then try again.
This fixes the problem where the input from the decklink module
(at least with --no-decklink-tenbits) is UYVY, while the deinterlacer only
wants planar formats.

Another possibility would be going straight for the format the
encoder wants, but often, this would be a 4:2:0 format, and 4:2:0
is pretty bad to deinterlace in, since the vertical chroma resolution
is gone already (the chroma interlacing in 4:2:0 is rather odd).

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index c0fc4800104435dd558a537cada77de830f65375..19591e14fe510dabbf2b3c1a1dad6d292d393d1c 100644 (file)
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -245,6 +245,61 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_sys_t *id )
     return VLC_SUCCESS;
 }
 
+static filter_t *AppendDeinterlaceFilter( sout_stream_t *p_stream,
+                                          filter_chain_t *p_chain,
+                                          const char *psz_name,
+                                          config_chain_t *p_cfg,
+                                          const es_format_t *p_fmt_in,
+                                          const es_format_t *p_fmt_out )
+{
+    filter_t *p_filter =
+        filter_chain_AppendFilter( p_chain, psz_name, p_cfg, p_fmt_in, p_fmt_out );
+    if( p_filter )
+        return p_filter;
+
+    msg_Dbg( p_stream, "deinterlace init failed, trying colorspace conversion" );
+
+    /* Most likely the filter failed due to wrong (e.g. non-planar) input format.
+     * Try converting to a different color space and try again.
+     *
+     * We use the recommended fallback formats in order, to get a conversion that's
+     * as cheap (both CPU-wise and quality-wise) as possible.
+     */
+    for( const vlc_fourcc_t *fourcc = vlc_fourcc_GetYUVFallback( p_fmt_out->video.i_chroma );
+         *fourcc != 0;
+         ++fourcc )
+    {
+        msg_Dbg( p_stream, "trying deinterlace through colorspace conversion to %4.4s", (char *)fourcc );
+
+        es_format_t p_new_fmt;
+        es_format_Copy( &p_new_fmt, p_fmt_in );
+        p_new_fmt.video.i_chroma = *fourcc;  // XXX: is this enough?
+
+        filter_chain_Reset( p_chain, NULL, NULL ); 
+
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              NULL, NULL,
+                                              p_fmt_in,
+                                              &p_new_fmt );
+        if( !p_filter )
+            continue;
+                   
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              psz_name,
+                                              p_cfg,
+                                              &p_new_fmt,
+                                              &p_new_fmt );
+        if( p_filter )
+        {
+            msg_Dbg( p_stream, "deinterlace through colorspace conversion to %4.4s succeeded", (char *)fourcc );
+            return p_filter;
+        }
+    }
+
+    msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+    return NULL;
+}
+ 
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_sys_t *id )
 {
@@ -263,11 +318,18 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   p_stream->p_sys->psz_deinterlace,
-                                   p_stream->p_sys->p_deinterlace_cfg,
-                                   &id->p_decoder->fmt_out,
-                                   &id->p_decoder->fmt_out );
+        filter_t *p_filter =
+            AppendDeinterlaceFilter( p_stream,
+                                     id->p_f_chain,
+                                     p_stream->p_sys->psz_deinterlace,
+                                     p_stream->p_sys->p_deinterlace_cfg,
+                                     p_fmt_out,
+                                     p_fmt_out );
+        if( !p_filter )
+        {
+            msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+            return;
+        } 
 
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }