demux: mp4: fix heap buffer overflow (fix #12266)

[vlc] / modules / demux / real.c

diff --git a/modules/demux/real.c b/modules/demux/real.c
index a295f2bc18e1089636f38593a49e2448d8e75abf..fa3219867c8905bb55893ef2ae3b201f0bbba2fc 100644 (file)
--- a/modules/demux/real.c
+++ b/modules/demux/real.c
@@ -68,7 +68,7 @@ static void Close  ( vlc_object_t * );
 
 vlc_module_begin ()
     set_description( N_("Real demuxer" ) )
-    set_capability( "demux", 50 )
+    set_capability( "demux", 0 )
     set_category( CAT_INPUT )
     set_subcategory( SUBCAT_INPUT_DEMUX )
     set_callbacks( Open, Close )
@@ -547,7 +547,7 @@ static void DemuxVideo( demux_t *p_demux, real_track_t *tk, mtime_t i_dts, unsig
             if( tk->p_frame )
                 block_Release( tk->p_frame );
 
-            tk->p_frame = block_New( p_demux, tk->i_frame_size );
+            tk->p_frame = block_Alloc( tk->i_frame_size );
             if( !tk->p_frame )
             {
                 tk->i_frame_size = 0;
@@ -638,7 +638,7 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
             if( i_index >= tk->i_subpackets )
                 return;
 
-            block_t *p_block = block_New( p_demux, tk->i_subpacket_size );
+            block_t *p_block = block_Alloc( tk->i_subpacket_size );
             if( !p_block )
                 return;
             if( &p_buf[tk->i_subpacket_size] > &p_sys->buffer[p_sys->i_buffer] )
@@ -673,7 +673,7 @@ static void DemuxAudioMethod1( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
             if( i_index >= tk->i_subpackets )
                 return;
 
-            block_t *p_block = block_New( p_demux, tk->i_coded_frame_size);
+            block_t *p_block = block_Alloc( tk->i_coded_frame_size);
             if( !p_block )
                 return;
             if( &p_buf[tk->i_coded_frame_size] > &p_sys->buffer[p_sys->i_buffer] )
@@ -745,7 +745,7 @@ static void DemuxAudioMethod2( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
     for( unsigned i = 0; i < i_sub; i++ )
     {
         const int i_sub_size = GetWBE( &p_sys->buffer[2+i*2] );
-        block_t *p_block = block_New( p_demux, i_sub_size );
+        block_t *p_block = block_Alloc( i_sub_size );
         if( !p_block )
             break;
 
@@ -769,7 +769,7 @@ static void DemuxAudioMethod3( demux_t *p_demux, real_track_t *tk, mtime_t i_pts
     if( p_sys->i_buffer <= 0 )
         return;
 
-    block_t *p_block = block_New( p_demux, p_sys->i_buffer );
+    block_t *p_block = block_Alloc( p_sys->i_buffer );
     if( !p_block )
         return;
 
@@ -840,17 +840,17 @@ static void DemuxAudioSipr( demux_t *p_demux, real_track_t *tk, mtime_t i_pts )
     demux_sys_t *p_sys = p_demux->p_sys;
     block_t *p_block = tk->p_sipr_packet;
 
-    if( p_sys->i_buffer < tk->i_frame_size )
+    if( p_sys->i_buffer < tk->i_frame_size
+     || tk->i_sipr_subpacket_count >= tk->i_subpacket_h )
         return;
 
     if( !p_block )
     {
-        p_block = block_New( p_demux, tk->i_frame_size * tk->i_subpacket_h );
+        p_block = block_Alloc( tk->i_frame_size * tk->i_subpacket_h );
         if( !p_block )
             return;
         tk->p_sipr_packet = p_block;
     }
-
     memcpy( p_block->p_buffer + tk->i_sipr_subpacket_count * tk->i_frame_size,
             p_sys->buffer, tk->i_frame_size );
     if (!tk->i_sipr_subpacket_count)
@@ -1377,12 +1377,12 @@ static int CodecVideoParse( demux_t *p_demux, int i_tk_id, const uint8_t *p_data
     fmt.video.i_frame_rate = (GetWBE( &p_data[22] ) << 16) | GetWBE( &p_data[24] );
     fmt.video.i_frame_rate_base = 1 << 16;
 
-    fmt.i_extra = 8;
-    fmt.p_extra = malloc( 8 );
+    fmt.i_extra = i_data - 26;
+    fmt.p_extra = malloc( fmt.i_extra );
     if( !fmt.p_extra )
         return VLC_ENOMEM;
 
-    memcpy( fmt.p_extra, &p_data[26], 8 );
+    memcpy( fmt.p_extra, &p_data[26], fmt.i_extra );
 
     //msg_Dbg( p_demux, "    - video 0x%08x 0x%08x", dw0, dw1 );
 
@@ -1445,7 +1445,7 @@ static int CodecAudioParse( demux_t *p_demux, int i_tk_id, const uint8_t *p_data
     int i_subpacket_h = 0;
     int i_frame_size = 0;
     int i_subpacket_size = 0;
-    char p_genr[4];
+    char p_genr[4] = {0,0,0,0};
     int i_version = GetWBE( &p_data[4] );
     int i_extra_codec = 0;
 
@@ -1567,14 +1567,17 @@ static int CodecAudioParse( demux_t *p_demux, int i_tk_id, const uint8_t *p_data
         {
             fmt.p_extra = malloc( i_extra_codec );
             if( !fmt.p_extra || i_extra_codec > i_data )
+            {
+                free( fmt.p_extra );
                 return VLC_ENOMEM;
+            }
 
             fmt.i_extra = i_extra_codec;
             memcpy( fmt.p_extra, p_data, fmt.i_extra );
         }
         break;
 
-    case VLC_FOURCC( 's','i','p','r' ):
+    case VLC_CODEC_SIPR:
         fmt.i_codec = VLC_CODEC_SIPR;
         if( i_flavor > 3 )
         {
@@ -1590,8 +1593,8 @@ static int CodecAudioParse( demux_t *p_demux, int i_tk_id, const uint8_t *p_data
         fmt.i_bitrate = fmt.audio.i_rate;
         msg_Dbg( p_demux, "    - sipr flavor=%i", i_flavor );
 
-    case VLC_FOURCC( 'c','o','o','k' ):
-    case VLC_FOURCC( 'a','t','r','c' ):
+    case VLC_CODEC_COOK:
+    case VLC_CODEC_ATRAC3:
         if( i_subpacket_size <= 0 || i_frame_size / i_subpacket_size <= 0 )
         {
             es_format_Clean( &fmt );
@@ -1602,16 +1605,14 @@ static int CodecAudioParse( demux_t *p_demux, int i_tk_id, const uint8_t *p_data
         else
             fmt.audio.i_blockalign = i_coded_frame_size;
 
-        if( fmt.i_codec == VLC_FOURCC( 'c','o','o','k' ) )
-            fmt.i_codec = VLC_CODEC_COOK;
-        else if( fmt.i_codec == VLC_FOURCC( 'a','t','r','c' ) )
-            fmt.i_codec = VLC_CODEC_ATRAC3;
-
         if( i_extra_codec > 0 )
         {
             fmt.p_extra = malloc( i_extra_codec );
             if( !fmt.p_extra || i_extra_codec > i_data )
+            {
+                free( fmt.p_extra );
                 return VLC_ENOMEM;
+            }
 
             fmt.i_extra = i_extra_codec;
             memcpy( fmt.p_extra, p_data, fmt.i_extra );