Win32: fix NULL-deref in non embedded video mode (fixes #2426)

[vlc] / modules / video_filter / swscale.c

diff --git a/modules/video_filter/swscale.c b/modules/video_filter/swscale.c
index ae6964d1a40465527849675b9a4dd75f54f5a433..67a4eb90d95f135243bde9dd1397b0f162165d42 100644 (file)
--- a/modules/video_filter/swscale.c
+++ b/modules/video_filter/swscale.c
@@ -36,12 +36,17 @@
 
 #ifdef HAVE_LIBSWSCALE_SWSCALE_H
 #   include <libswscale/swscale.h>
+#   include <libavcodec/avcodec.h>
 #elif defined(HAVE_FFMPEG_SWSCALE_H)
 #   include <ffmpeg/swscale.h>
+#   include <ffmpeg/avcodec.h>
 #endif
 
 /* Gruikkkkkkkkkk!!!!! */
-#include "../codec/avcodec/chroma.h"
+#include "../codec/avcodec/avcodec.h"
+#undef AVPALETTE_SIZE
+
+#define AVPALETTE_SIZE (256 * sizeof(uint32_t))
 
 /*****************************************************************************
  * Module descriptor
@@ -59,15 +64,16 @@ const char *const ppsz_mode_descriptions[] =
   N_("Area"), N_("Luma bicubic / chroma bilinear"), N_("Gauss"),
   N_("SincR"), N_("Lanczos"), N_("Bicubic spline") };
 
-vlc_module_begin();
-    set_description( N_("Video scaling filter") );
-    set_capability( "video filter2", 150 );
-    set_category( CAT_VIDEO );
-    set_subcategory( SUBCAT_VIDEO_VFILTER );
-    set_callbacks( OpenScaler, CloseScaler );
-    add_integer( "swscale-mode", 2, NULL, SCALEMODE_TEXT, SCALEMODE_LONGTEXT, true );
-        change_integer_list( pi_mode_values, ppsz_mode_descriptions, NULL );
-vlc_module_end();
+vlc_module_begin ()
+    set_description( N_("Video scaling filter") )
+    set_shortname( N_("Swscale" ) )
+    set_capability( "video filter2", 150 )
+    set_category( CAT_VIDEO )
+    set_subcategory( SUBCAT_VIDEO_VFILTER )
+    set_callbacks( OpenScaler, CloseScaler )
+    add_integer( "swscale-mode", 2, NULL, SCALEMODE_TEXT, SCALEMODE_LONGTEXT, true )
+        change_integer_list( pi_mode_values, ppsz_mode_descriptions, NULL )
+vlc_module_end ()
 
 /* Version checking */
 #if LIBSWSCALE_VERSION_INT >= ((0<<16)+(5<<8)+0)
@@ -125,7 +131,7 @@ static int GetSwsCpuMask(void);
  * (change it to true to try) */
 #define ALLOW_YUVP (false)
 /* SwScaler does not like too small picture */
-#define MINIMUM_WIDTH (16)
+#define MINIMUM_WIDTH (32)
 
 /* XXX is it always 3 even for BIG_ENDIAN (blend.c seems to think so) ? */
 #define OFFSET_A (3)
@@ -140,10 +146,6 @@ static int OpenScaler( vlc_object_t *p_this )
 
     int i_sws_mode;
 
-    float sws_lum_gblur = 0.0, sws_chr_gblur = 0.0;
-    int sws_chr_vshift = 0, sws_chr_hshift = 0;
-    float sws_chr_sharpen = 0.0, sws_lum_sharpen = 0.0;
-
     if( GetParameters( NULL,
                        &p_filter->fmt_in.video,
                        &p_filter->fmt_out.video, 0 ) )
@@ -179,10 +181,7 @@ static int OpenScaler( vlc_object_t *p_this )
     default: p_sys->i_sws_flags = SWS_BICUBIC; i_sws_mode = 2; break;
     }
 
-    p_sys->p_src_filter =
-        sws_getDefaultFilter( sws_lum_gblur, sws_chr_gblur,
-                              sws_lum_sharpen, sws_chr_sharpen,
-                              sws_chr_hshift, sws_chr_vshift, 0 );
+    p_sys->p_src_filter = NULL;
     p_sys->p_dst_filter = NULL;
 
     /* Misc init */
@@ -308,7 +307,7 @@ static int GetParameters( ScalerConfiguration *p_cfg,
         p_cfg->i_fmto = i_fmto;
         p_cfg->b_has_a = b_has_ai && b_has_ao;
         p_cfg->b_add_a = (!b_has_ai) && b_has_ao;
-        p_cfg->b_copy = i_fmti == i_fmto && p_fmti->i_width == p_fmto->i_width && p_fmti->i_width && p_fmto->i_height;
+        p_cfg->b_copy = i_fmti == i_fmto && p_fmti->i_width == p_fmto->i_width && p_fmti->i_height == p_fmto->i_height;
         p_cfg->i_sws_flags = i_sws_flags;
     }
 
@@ -490,13 +489,25 @@ static void CopyPad( picture_t *p_dst, const picture_t *p_src )
     }
 }
 
-static void Convert( struct SwsContext *ctx,
+static void Convert( filter_t *p_filter, struct SwsContext *ctx,
                      picture_t *p_dst, picture_t *p_src, int i_height, int i_plane_start, int i_plane_count )
 {
+    uint8_t palette[AVPALETTE_SIZE];
+
     uint8_t *src[3]; int src_stride[3];
     uint8_t *dst[3]; int dst_stride[3];
 
     GetPixels( src, src_stride, p_src, i_plane_start, i_plane_count );
+    if( p_filter->fmt_in.video.i_chroma == VLC_FOURCC( 'R', 'G', 'B', 'P' ) )
+    {
+        memset( palette, 0, sizeof(palette) );
+        if( p_filter->fmt_in.video.p_palette )
+            memcpy( palette, p_filter->fmt_in.video.p_palette->palette,
+                    __MIN( sizeof(video_palette_t), AVPALETTE_SIZE ) );
+        src[1] = palette;
+        src_stride[1] = 4;
+    }
+
     GetPixels( dst, dst_stride, p_dst, i_plane_start, i_plane_count );
 
 #if LIBSWSCALE_VERSION_INT  >= ((0<<16)+(5<<8)+0)
@@ -549,7 +560,7 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
     if( p_sys->b_copy )
         picture_CopyPixels( p_dst, p_src );
     else
-        Convert( p_sys->ctx, p_dst, p_src, p_fmti->i_height, 0, 3 );
+        Convert( p_filter, p_sys->ctx, p_dst, p_src, p_fmti->i_height, 0, 3 );
     if( p_sys->ctxA )
     {
         /* We extract the A plane to rescale it, and then we reinject it. */
@@ -558,7 +569,7 @@ static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
         else
             plane_CopyPixels( p_sys->p_src_a->p, p_src->p+A_PLANE );
 
-        Convert( p_sys->ctxA, p_sys->p_dst_a, p_sys->p_src_a, p_fmti->i_height, 0, 1 );
+        Convert( p_filter, p_sys->ctxA, p_sys->p_dst_a, p_sys->p_src_a, p_fmti->i_height, 0, 1 );
         if( p_fmto->i_chroma == VLC_FOURCC( 'R', 'G', 'B', 'A' ) )
             InjectA( p_dst, p_sys->p_dst_a, p_fmto->i_width * p_sys->i_extend_factor, p_fmto->i_height );
         else