wmv2enc: Check memory allocation

[ffmpeg] / libavcodec / mmvideo.c

diff --git a/libavcodec/mmvideo.c b/libavcodec/mmvideo.c
index 2187d7f585c2c2e9e4cce05d91f41187e30bc930..25124a3edf6fa9f3c883c7773da840a2ddaa36ee 100644 (file)
--- a/libavcodec/mmvideo.c
+++ b/libavcodec/mmvideo.c
@@ -34,6 +34,7 @@
 #include "libavutil/intreadwrite.h"
 #include "avcodec.h"
 #include "bytestream.h"
+#include "internal.h"
 
 #define MM_PREAMBLE_SIZE    6
 
@@ -47,7 +48,7 @@
 
 typedef struct MmContext {
     AVCodecContext *avctx;
-    AVFrame frame;
+    AVFrame *frame;
     int palette[AVPALETTE_COUNT];
     GetByteContext gb;
 } MmContext;
@@ -58,9 +59,18 @@ static av_cold int mm_decode_init(AVCodecContext *avctx)
 
     s->avctx = avctx;
 
-    avctx->pix_fmt = PIX_FMT_PAL8;
+    avctx->pix_fmt = AV_PIX_FMT_PAL8;
 
-    s->frame.reference = 1;
+    if (!avctx->width || !avctx->height ||
+        (avctx->width & 1) || (avctx->height & 1)) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid video dimensions: %dx%d\n",
+               avctx->width, avctx->height);
+        return AVERROR(EINVAL);
+    }
+
+    s->frame = av_frame_alloc();
+    if (!s->frame)
+        return AVERROR(ENOMEM);
 
     return 0;
 }
@@ -104,9 +114,9 @@ static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert)
             run_length *=2;
 
         if (color) {
-            memset(s->frame.data[0] + y*s->frame.linesize[0] + x, color, run_length);
+            memset(s->frame->data[0] + y*s->frame->linesize[0] + x, color, run_length);
             if (half_vert)
-                memset(s->frame.data[0] + (y+1)*s->frame.linesize[0] + x, color, run_length);
+                memset(s->frame->data[0] + (y+1)*s->frame->linesize[0] + x, color, run_length);
         }
         x+= run_length;
 
@@ -125,7 +135,8 @@ static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert)
  */
 static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
 {
-    int data_off = bytestream2_get_le16(&s->gb), y;
+    int data_off = bytestream2_get_le16(&s->gb);
+    int y = 0;
     GetByteContext data_ptr;
 
     if (bytestream2_get_bytes_left(&s->gb) < data_off)
@@ -150,15 +161,17 @@ static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
             int replace_array = bytestream2_get_byte(&s->gb);
             for(j=0; j<8; j++) {
                 int replace = (replace_array >> (7-j)) & 1;
+                if (x + half_horiz >= s->avctx->width)
+                    return AVERROR_INVALIDDATA;
                 if (replace) {
                     int color = bytestream2_get_byte(&data_ptr);
-                    s->frame.data[0][y*s->frame.linesize[0] + x] = color;
+                    s->frame->data[0][y*s->frame->linesize[0] + x] = color;
                     if (half_horiz)
-                        s->frame.data[0][y*s->frame.linesize[0] + x + 1] = color;
+                        s->frame->data[0][y*s->frame->linesize[0] + x + 1] = color;
                     if (half_vert) {
-                        s->frame.data[0][(y+1)*s->frame.linesize[0] + x] = color;
+                        s->frame->data[0][(y+1)*s->frame->linesize[0] + x] = color;
                         if (half_horiz)
-                            s->frame.data[0][(y+1)*s->frame.linesize[0] + x + 1] = color;
+                            s->frame->data[0][(y+1)*s->frame->linesize[0] + x + 1] = color;
                     }
                 }
                 x += 1 + half_horiz;
@@ -172,7 +185,7 @@ static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert)
 }
 
 static int mm_decode_frame(AVCodecContext *avctx,
-                            void *data, int *data_size,
+                            void *data, int *got_frame,
                             AVPacket *avpkt)
 {
     const uint8_t *buf = avpkt->data;
@@ -187,9 +200,9 @@ static int mm_decode_frame(AVCodecContext *avctx,
     buf_size -= MM_PREAMBLE_SIZE;
     bytestream2_init(&s->gb, buf, buf_size);
 
-    if (avctx->reget_buffer(avctx, &s->frame) < 0) {
+    if ((res = ff_reget_buffer(avctx, s->frame)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
-        return -1;
+        return res;
     }
 
     switch(type) {
@@ -207,10 +220,12 @@ static int mm_decode_frame(AVCodecContext *avctx,
     if (res < 0)
         return res;
 
-    memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE);
+    memcpy(s->frame->data[1], s->palette, AVPALETTE_SIZE);
 
-    *data_size = sizeof(AVFrame);
-    *(AVFrame*)data = s->frame;
+    if ((res = av_frame_ref(data, s->frame)) < 0)
+        return res;
+
+    *got_frame      = 1;
 
     return buf_size;
 }
@@ -219,14 +234,14 @@ static av_cold int mm_decode_end(AVCodecContext *avctx)
 {
     MmContext *s = avctx->priv_data;
 
-    if(s->frame.data[0])
-        avctx->release_buffer(avctx, &s->frame);
+    av_frame_free(&s->frame);
 
     return 0;
 }
 
 AVCodec ff_mmvideo_decoder = {
     .name           = "mmvideo",
+    .long_name      = NULL_IF_CONFIG_SMALL("American Laser Games MM Video"),
     .type           = AVMEDIA_TYPE_VIDEO,
     .id             = AV_CODEC_ID_MMVIDEO,
     .priv_data_size = sizeof(MmContext),
@@ -234,5 +249,4 @@ AVCodec ff_mmvideo_decoder = {
     .close          = mm_decode_end,
     .decode         = mm_decode_frame,
     .capabilities   = CODEC_CAP_DR1,
-    .long_name      = NULL_IF_CONFIG_SMALL("American Laser Games MM Video"),
 };