Fix deinterlacing of packed YUV formats.

If adding the deinterlacer filter fails, try conversion filters
until we try something the deinterlacer wants, and then try again.
This fixes the problem where the input from the decklink module
(at least with --no-decklink-tenbits) is UYVY, while the deinterlacer only
wants planar formats.

Another possibility would be going straight for the format the
encoder wants, but often, this would be a 4:2:0 format, and 4:2:0
is pretty bad to deinterlace in, since the vertical chroma resolution
is gone already (the chroma interlacing in 4:2:0 is rather odd).

diff --git a/modules/stream_out/transcode/video.c b/modules/stream_out/transcode/video.c
index 5e07ca2ef92b86caa9e94efe0d085fd3140434f7..7196a40c24287100b43c67f3aeab70db5fcf2775 100644 (file)
--- a/modules/stream_out/transcode/video.c
+++ b/modules/stream_out/transcode/video.c
@@ -258,6 +258,61 @@ int transcode_video_new( sout_stream_t *p_stream, sout_stream_id_t *id )
     return VLC_SUCCESS;
 }
 
+static filter_t *AppendDeinterlaceFilter( sout_stream_t *p_stream,
+                                          filter_chain_t *p_chain,
+                                          const char *psz_name,
+                                          config_chain_t *p_cfg,
+                                          const es_format_t *p_fmt_in,
+                                          const es_format_t *p_fmt_out )
+{
+    filter_t *p_filter =
+        filter_chain_AppendFilter( p_chain, psz_name, p_cfg, p_fmt_in, p_fmt_out );
+    if( p_filter )
+        return p_filter;
+
+    msg_Dbg( p_stream, "deinterlace init failed, trying colorspace conversion" );
+
+    /* Most likely the filter failed due to wrong (e.g. non-planar) input format.
+     * Try converting to a different color space and try again.
+     *
+     * We use the recommended fallback formats in order, to get a conversion that's
+     * as cheap (both CPU-wise and quality-wise) as possible.
+     */
+    for( const vlc_fourcc_t *fourcc = vlc_fourcc_GetYUVFallback( p_fmt_out->video.i_chroma );
+         *fourcc != 0;
+         ++fourcc )
+    {
+        msg_Dbg( p_stream, "trying deinterlace through colorspace conversion to %4.4s", (char *)fourcc );
+
+        es_format_t p_new_fmt;
+        es_format_Copy( &p_new_fmt, p_fmt_in );
+        p_new_fmt.video.i_chroma = *fourcc;  // XXX: is this enough?
+
+        filter_chain_Reset( p_chain, NULL, NULL ); 
+
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              NULL, NULL,
+                                              p_fmt_in,
+                                              &p_new_fmt );
+        if( !p_filter )
+            continue;
+                   
+        p_filter = filter_chain_AppendFilter( p_chain,
+                                              psz_name,
+                                              p_cfg,
+                                              &p_new_fmt,
+                                              &p_new_fmt );
+        if( p_filter )
+        {
+            msg_Dbg( p_stream, "deinterlace through colorspace conversion to %4.4s succeeded", (char *)fourcc );
+            return p_filter;
+        }
+    }
+
+    msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+    return NULL;
+}
+ 
 static void transcode_video_filter_init( sout_stream_t *p_stream,
                                          sout_stream_id_t *id )
 {
@@ -273,11 +328,18 @@ static void transcode_video_filter_init( sout_stream_t *p_stream,
     /* Deinterlace */
     if( p_stream->p_sys->b_deinterlace )
     {
-        filter_chain_AppendFilter( id->p_f_chain,
-                                   p_stream->p_sys->psz_deinterlace,
-                                   p_stream->p_sys->p_deinterlace_cfg,
-                                   &id->p_decoder->fmt_out,
-                                   &id->p_decoder->fmt_out );
+        filter_t *p_filter =
+            AppendDeinterlaceFilter( p_stream,
+                                     id->p_f_chain,
+                                     p_stream->p_sys->psz_deinterlace,
+                                     p_stream->p_sys->p_deinterlace_cfg,
+                                     p_fmt_out,
+                                     p_fmt_out );
+        if( !p_filter )
+        {
+            msg_Err( p_stream, "deinterlace init failed, even with colorspace conversion" );
+            return;
+        } 
 
         p_fmt_out = filter_chain_GetFmtOut( id->p_f_chain );
     }