avformat/avidec: Avoid integer overflow in NI switch check

[ffmpeg] / libavformat / smacker.c

diff --git a/libavformat/smacker.c b/libavformat/smacker.c
index 8a21cc0767e400e4dd5f9cdf659f44310ee05d71..6de0e7a0f19615912ab41850a6a7b4c8af2851c8 100644 (file)
--- a/libavformat/smacker.c
+++ b/libavformat/smacker.c
@@ -92,7 +92,7 @@ static const uint8_t smk_pal[64] = {
 };
 
 
-static int smacker_probe(AVProbeData *p)
+static int smacker_probe(const AVProbeData *p)
 {
     if (   AV_RL32(p->buf) != MKTAG('S', 'M', 'K', '2')
         && AV_RL32(p->buf) != MKTAG('S', 'M', 'K', '4'))
@@ -172,8 +172,11 @@ static int smacker_read_header(AVFormatContext *s)
 
     /* init video codec */
     st = avformat_new_stream(s, NULL);
-    if (!st)
+    if (!st) {
+        av_freep(&smk->frm_size);
+        av_freep(&smk->frm_flags);
         return AVERROR(ENOMEM);
+    }
     smk->videoindex = st->index;
     st->codecpar->width = smk->width;
     st->codecpar->height = smk->height;
@@ -195,8 +198,11 @@ static int smacker_read_header(AVFormatContext *s)
         smk->indexes[i] = -1;
         if (smk->rates[i]) {
             ast[i] = avformat_new_stream(s, NULL);
-            if (!ast[i])
+            if (!ast[i]) {
+                av_freep(&smk->frm_size);
+                av_freep(&smk->frm_flags);
                 return AVERROR(ENOMEM);
+            }
             smk->indexes[i] = ast[i]->index;
             ast[i]->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
             if (smk->aflags[i] & SMK_AUD_BINKAUD) {
@@ -227,13 +233,13 @@ static int smacker_read_header(AVFormatContext *s)
 
 
     /* load trees to extradata, they will be unpacked by decoder */
-    if(ff_alloc_extradata(st->codecpar, smk->treesize + 16)){
+    if ((ret = ff_alloc_extradata(st->codecpar, smk->treesize + 16)) < 0) {
         av_log(s, AV_LOG_ERROR,
                "Cannot allocate %"PRIu32" bytes of extradata\n",
                smk->treesize + 16);
         av_freep(&smk->frm_size);
         av_freep(&smk->frm_flags);
-        return AVERROR(ENOMEM);
+        return ret;
     }
     ret = avio_read(pb, st->codecpar->extradata + 16, st->codecpar->extradata_size - 16);
     if(ret != st->codecpar->extradata_size - 16){
@@ -347,8 +353,8 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
         }
         if (frame_size < 0 || frame_size >= INT_MAX/2)
             return AVERROR_INVALIDDATA;
-        if (av_new_packet(pkt, frame_size + 769))
-            return AVERROR(ENOMEM);
+        if ((ret = av_new_packet(pkt, frame_size + 769)) < 0)
+            return ret;
         if(smk->frm_size[smk->cur_frame] & 1)
             palchange |= 2;
         pkt->data[0] = palchange;
@@ -364,8 +370,8 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
     } else {
         if (smk->stream_id[smk->curstream] < 0 || !smk->bufs[smk->curstream])
             return AVERROR_INVALIDDATA;
-        if (av_new_packet(pkt, smk->buf_sizes[smk->curstream]))
-            return AVERROR(ENOMEM);
+        if ((ret = av_new_packet(pkt, smk->buf_sizes[smk->curstream])) < 0)
+            return ret;
         memcpy(pkt->data, smk->bufs[smk->curstream], smk->buf_sizes[smk->curstream]);
         pkt->size = smk->buf_sizes[smk->curstream];
         pkt->stream_index = smk->stream_id[smk->curstream];