Fix an exploit in indeo by checking we are not writing out of the strip array.

[ffmpeg] / libavcodec / vmdav.c

diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index 00b0f59af336012a58d2291c16eda0b49ca55a41..ed9ba28dacb558cd7b0ae6128418b2bd039cb3e2 100644 (file)
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -44,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "libavutil/intreadwrite.h"
 #include "avcodec.h"
 
 #define VMD_HEADER_SIZE 0x330
@@ -162,13 +163,13 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
         if (l & 0x80) {
             l = (l & 0x7F) * 2;
             if (pd + l > dest_end)
-                return (ps - src);
+                return ps - src;
             memcpy(pd, ps, l);
             ps += l;
             pd += l;
         } else {
             if (pd + i > dest_end)
-                return (ps - src);
+                return ps - src;
             for (i = 0; i < l; i++) {
                 *pd++ = ps[0];
                 *pd++ = ps[1];
@@ -178,7 +179,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
         i += l;
     } while (i < src_len);
 
-    return (ps - src);
+    return ps - src;
 }
 
 static void vmd_decode(VmdVideoContext *s)
@@ -422,7 +423,7 @@ typedef struct VmdAudioContext {
     int predictors[2];
 } VmdAudioContext;
 
-static uint16_t vmdaudio_table[128] = {
+static const uint16_t vmdaudio_table[128] = {
     0x000, 0x008, 0x010, 0x020, 0x030, 0x040, 0x050, 0x060, 0x070, 0x080,
     0x090, 0x0A0, 0x0B0, 0x0C0, 0x0D0, 0x0E0, 0x0F0, 0x100, 0x110, 0x120,
     0x130, 0x140, 0x150, 0x160, 0x170, 0x180, 0x190, 0x1A0, 0x1B0, 0x1C0,
@@ -444,8 +445,9 @@ static av_cold int vmdaudio_decode_init(AVCodecContext *avctx)
 
     s->avctx = avctx;
     s->channels = avctx->channels;
-    s->bits = avctx->bits_per_sample;
+    s->bits = avctx->bits_per_coded_sample;
     s->block_align = avctx->block_align;
+    avctx->sample_fmt = SAMPLE_FMT_S16;
 
     av_log(s->avctx, AV_LOG_DEBUG, "%d channels, %d bits/sample, block align = %d, sample rate = %d\n",
             s->channels, s->bits, s->block_align, avctx->sample_rate);
@@ -454,13 +456,13 @@ static av_cold int vmdaudio_decode_init(AVCodecContext *avctx)
 }
 
 static void vmdaudio_decode_audio(VmdAudioContext *s, unsigned char *data,
-    const uint8_t *buf, int stereo)
+    const uint8_t *buf, int buf_size, int stereo)
 {
     int i;
     int chan = 0;
     int16_t *out = (int16_t*)data;
 
-    for(i = 0; i < s->block_align; i++) {
+    for(i = 0; i < buf_size; i++) {
         if(buf[i] & 0x80)
             s->predictors[chan] -= vmdaudio_table[buf[i] & 0x7F];
         else
@@ -472,7 +474,7 @@ static void vmdaudio_decode_audio(VmdAudioContext *s, unsigned char *data,
 }
 
 static int vmdaudio_loadsound(VmdAudioContext *s, unsigned char *data,
-    const uint8_t *buf, int silence)
+    const uint8_t *buf, int silence, int data_size)
 {
     int bytes_decoded = 0;
     int i;
@@ -483,30 +485,30 @@ static int vmdaudio_loadsound(VmdAudioContext *s, unsigned char *data,
 
         /* stereo handling */
         if (silence) {
-            memset(data, 0, s->block_align * 2);
+            memset(data, 0, data_size * 2);
         } else {
             if (s->bits == 16)
-                vmdaudio_decode_audio(s, data, buf, 1);
+                vmdaudio_decode_audio(s, data, buf, data_size, 1);
             else {
                 /* copy the data but convert it to signed */
-                for (i = 0; i < s->block_align; i++){
+                for (i = 0; i < data_size; i++){
                     *data++ = buf[i] + 0x80;
                     *data++ = buf[i] + 0x80;
                 }
             }
         }
     } else {
-        bytes_decoded = s->block_align * 2;
+        bytes_decoded = data_size * 2;
 
         /* mono handling */
         if (silence) {
-            memset(data, 0, s->block_align * 2);
+            memset(data, 0, data_size * 2);
         } else {
             if (s->bits == 16) {
-                vmdaudio_decode_audio(s, data, buf, 0);
+                vmdaudio_decode_audio(s, data, buf, data_size, 0);
             } else {
                 /* copy the data but convert it to signed */
-                for (i = 0; i < s->block_align; i++){
+                for (i = 0; i < data_size; i++){
                     *data++ = buf[i] + 0x80;
                     *data++ = buf[i] + 0x80;
                 }
@@ -514,7 +516,7 @@ static int vmdaudio_loadsound(VmdAudioContext *s, unsigned char *data,
         }
     }
 
-    return s->block_align * 2;
+    return data_size * 2;
 }
 
 static int vmdaudio_decode_frame(AVCodecContext *avctx,
@@ -532,15 +534,26 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx,
 
     if (buf[6] == 1) {
         /* the chunk contains audio */
-        *data_size = vmdaudio_loadsound(s, output_samples, p, 0);
+        *data_size = vmdaudio_loadsound(s, output_samples, p, 0, buf_size - 16);
     } else if (buf[6] == 2) {
-        /* the chunk may contain audio */
-        p += 4;
-        *data_size = vmdaudio_loadsound(s, output_samples, p, (buf_size == 16));
-        output_samples += (s->block_align * s->bits / 8);
+        /* initial chunk, may contain audio and silence */
+        uint32_t flags = AV_RB32(p);
+        int raw_block_size = s->block_align * s->bits / 8;
+        int silent_chunks;
+        if(flags == 0xFFFFFFFF)
+            silent_chunks = 32;
+        else
+            silent_chunks = av_log2(flags + 1);
+        if(*data_size < (s->block_align*silent_chunks + buf_size - 20) * 2)
+            return -1;
+        *data_size = 0;
+        memset(output_samples, 0, raw_block_size * silent_chunks);
+        output_samples += raw_block_size * silent_chunks;
+        *data_size = raw_block_size * silent_chunks;
+        *data_size += vmdaudio_loadsound(s, output_samples, p + 4, 0, buf_size - 20);
     } else if (buf[6] == 3) {
         /* silent chunk */
-        *data_size = vmdaudio_loadsound(s, output_samples, p, 1);
+        *data_size = vmdaudio_loadsound(s, output_samples, p, 1, 0);
     }
 
     return buf_size;
@@ -561,7 +574,7 @@ AVCodec vmdvideo_decoder = {
     vmdvideo_decode_end,
     vmdvideo_decode_frame,
     CODEC_CAP_DR1,
-    .long_name = "Sierra VMD video",
+    .long_name = NULL_IF_CONFIG_SMALL("Sierra VMD video"),
 };
 
 AVCodec vmdaudio_decoder = {
@@ -573,5 +586,5 @@ AVCodec vmdaudio_decoder = {
     NULL,
     NULL,
     vmdaudio_decode_frame,
-    .long_name = "Sierra VMD audio",
+    .long_name = NULL_IF_CONFIG_SMALL("Sierra VMD audio"),
 };