Fixed a potential integer overflow in block_Alloc().

[vlc] / src / misc / picture.c

diff --git a/src/misc/picture.c b/src/misc/picture.c
index b63f843af09df15c6a5197d6c7713bfefee47417..cb106a70d144b4bb5dc220fed9961774636ba2bc 100644 (file)
--- a/src/misc/picture.c
+++ b/src/misc/picture.c
@@ -1,7 +1,7 @@
 /*****************************************************************************
  * picture.c : picture management functions
  *****************************************************************************
- * Copyright (C) 2000-2010 the VideoLAN team
+ * Copyright (C) 2000-2010 VLC authors and VideoLAN
  * Copyright (C) 2009-2010 Laurent Aimar
  * $Id$
  *
@@ -9,19 +9,19 @@
  *          Samuel Hocevar <sam@zoy.org>
  *          Laurent Aimar <fenrir _AT_ videolan _DOT_ org>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  *****************************************************************************/
 
 /*****************************************************************************
@@ -62,7 +62,7 @@ static int vout_AllocatePicture( picture_t *p_pic,
         const plane_t *p = &p_pic->p[i];
 
         if( p->i_pitch <= 0 || p->i_lines <= 0 ||
-            p->i_pitch > (SIZE_MAX - i_bytes)/p->i_lines )
+            (size_t)p->i_pitch > (SIZE_MAX - i_bytes)/p->i_lines )
         {
             p_pic->i_planes = 0;
             return VLC_ENOMEM;
@@ -70,12 +70,13 @@ static int vout_AllocatePicture( picture_t *p_pic,
         i_bytes += p->i_pitch * p->i_lines;
     }
 
-    uint8_t *p_data = vlc_memalign( &p_pic->p_data_orig, 16, i_bytes );
+    uint8_t *p_data = vlc_memalign( 16, i_bytes );
     if( !p_data )
     {
         p_pic->i_planes = 0;
         return VLC_EGENERIC;
     }
+    p_pic->p_data_orig = p_data; /* TODO: get rid of this */
 
     /* Fill the p_pixels field for each plane */
     p_pic->p[0].p_pixels = p_data;
@@ -166,6 +167,7 @@ int picture_Setup( picture_t *p_picture, vlc_fourcc_t i_chroma,
         if( i_ratio_h < p_dsc->p[i].h.den )
             i_ratio_h = p_dsc->p[i].h.den;
     }
+    i_modulo_h = LCM( i_modulo_h, 32 );
 
     const int i_width_aligned  = ( i_width  + i_modulo_w - 1 ) / i_modulo_w * i_modulo_w;
     const int i_height_aligned = ( i_height + i_modulo_h - 1 ) / i_modulo_h * i_modulo_h;
@@ -267,7 +269,7 @@ void picture_Delete( picture_t *p_picture )
     assert( p_picture->p_release_sys == NULL );
 
     free( p_picture->p_q );
-    free( p_picture->p_data_orig );
+    vlc_free( p_picture->p_data_orig );
     free( p_picture->p_sys );
     free( p_picture );
 }
@@ -361,9 +363,9 @@ int picture_Export( vlc_object_t *p_obj,
 
     /* */
     fmt_out.i_width  = ( i_override_width < 0 ) ?
-                       i_original_width : i_override_width;
+                       i_original_width : (unsigned)i_override_width;
     fmt_out.i_height = ( i_override_height < 0 ) ?
-                       i_original_height : i_override_height;
+                       i_original_height : (unsigned)i_override_height;
 
     /* scale if only one direction is provided */
     if( fmt_out.i_height == 0 && fmt_out.i_width > 0 )
@@ -399,7 +401,6 @@ int picture_Export( vlc_object_t *p_obj,
 void picture_BlendSubpicture(picture_t *dst,
                              filter_t *blend, subpicture_t *src)
 {
-    assert(blend && dst && blend->fmt_out.video.i_chroma == dst->format.i_chroma);
     assert(src && !src->b_fade && src->b_absolute);
 
     for (subpicture_region_t *r = src->p_region; r != NULL; r = r->p_next) {