avcodec/atrac1: Cleanup generically after init failure

[ffmpeg] / libavcodec / smacker.c

diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index 61e316916bf20319f0a61be1f04561cd2f85bffe..9f054f071de477457b3fc068c882eb6e216e89f0 100644 (file)
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -319,6 +319,7 @@ error:
 static int decode_header_trees(SmackVContext *smk) {
     GetBitContext gb;
     int mmap_size, mclr_size, full_size, type_size, ret;
+    int skip = 0;
 
     mmap_size = AV_RL32(smk->avctx->extradata);
     mclr_size = AV_RL32(smk->avctx->extradata + 4);
@@ -330,6 +331,7 @@ static int decode_header_trees(SmackVContext *smk) {
         return ret;
 
     if(!get_bits1(&gb)) {
+        skip ++;
         av_log(smk->avctx, AV_LOG_INFO, "Skipping MMAP tree\n");
         smk->mmap_tbl = av_malloc(sizeof(int) * 2);
         if (!smk->mmap_tbl)
@@ -342,6 +344,7 @@ static int decode_header_trees(SmackVContext *smk) {
             return ret;
     }
     if(!get_bits1(&gb)) {
+        skip ++;
         av_log(smk->avctx, AV_LOG_INFO, "Skipping MCLR tree\n");
         smk->mclr_tbl = av_malloc(sizeof(int) * 2);
         if (!smk->mclr_tbl)
@@ -354,6 +357,7 @@ static int decode_header_trees(SmackVContext *smk) {
             return ret;
     }
     if(!get_bits1(&gb)) {
+        skip ++;
         av_log(smk->avctx, AV_LOG_INFO, "Skipping FULL tree\n");
         smk->full_tbl = av_malloc(sizeof(int) * 2);
         if (!smk->full_tbl)
@@ -366,6 +370,7 @@ static int decode_header_trees(SmackVContext *smk) {
             return ret;
     }
     if(!get_bits1(&gb)) {
+        skip ++;
         av_log(smk->avctx, AV_LOG_INFO, "Skipping TYPE tree\n");
         smk->type_tbl = av_malloc(sizeof(int) * 2);
         if (!smk->type_tbl)
@@ -377,6 +382,8 @@ static int decode_header_trees(SmackVContext *smk) {
         if (ret < 0)
             return ret;
     }
+    if (skip == 4)
+        return AVERROR_INVALIDDATA;
 
     return 0;
 }
@@ -391,6 +398,8 @@ static av_always_inline int smk_get_code(GetBitContext *gb, int *recode, int *la
     int v;
 
     while(*table & SMK_NODE) {
+        if (get_bits_left(gb) < 1)
+            return AVERROR_INVALIDDATA;
         if(get_bits1(gb))
             table += (*table) & (~SMK_NODE);
         table++;
@@ -421,7 +430,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if (avpkt->size <= 769)
         return AVERROR_INVALIDDATA;
 
-    if ((ret = ff_reget_buffer(avctx, smk->pic)) < 0)
+    if ((ret = ff_reget_buffer(avctx, smk->pic, 0)) < 0)
         return ret;
 
     /* make the palette available on the way out */
@@ -455,6 +464,8 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         uint16_t pix;
 
         type = smk_get_code(&gb, smk->type_tbl, smk->type_last);
+        if (type < 0)
+            return type;
         run = block_runs[(type >> 2) & 0x3F];
         switch(type & 3){
         case SMK_BLK_MONO:
@@ -536,7 +547,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             while(run-- && blk < blocks){
                 uint32_t col;
                 out = smk->pic->data[0] + (blk / bw) * (stride * 4) + (blk % bw) * 4;
-                col = mode * 0x01010101;
+                col = mode * 0x01010101U;
                 for(i = 0; i < 4; i++) {
                     *((uint32_t*)out) = col;
                     out += stride;
@@ -589,13 +600,11 @@ static av_cold int decode_init(AVCodecContext *avctx)
     /* decode huffman trees from extradata */
     if(avctx->extradata_size < 16){
         av_log(avctx, AV_LOG_ERROR, "Extradata missing!\n");
-        decode_end(avctx);
         return AVERROR(EINVAL);
     }
 
     ret = decode_header_trees(c);
     if (ret < 0) {
-        decode_end(avctx);
         return ret;
     }
 
@@ -714,8 +723,10 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
         for(i = 0; i <= stereo; i++)
             *samples++ = pred[i];
         for(; i < unp_size / 2; i++) {
-            if(get_bits_left(&gb)<0)
-                return AVERROR_INVALIDDATA;
+            if (get_bits_left(&gb) < 0) {
+                ret = AVERROR_INVALIDDATA;
+                goto error;
+            }
             if(i & stereo) {
                 if(vlc[2].table)
                     res = get_vlc2(&gb, vlc[2].table, SMKTREE_BITS, 3);
@@ -723,7 +734,8 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 val  = h[2].values[res];
                 if(vlc[3].table)
@@ -732,10 +744,11 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 val |= h[3].values[res] << 8;
-                pred[1] += sign_extend(val, 16);
+                pred[1] += (unsigned)sign_extend(val, 16);
                 *samples++ = pred[1];
             } else {
                 if(vlc[0].table)
@@ -744,7 +757,8 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 val  = h[0].values[res];
                 if(vlc[1].table)
@@ -753,10 +767,11 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 val |= h[1].values[res] << 8;
-                pred[0] += sign_extend(val, 16);
+                pred[0] += (unsigned)sign_extend(val, 16);
                 *samples++ = pred[0];
             }
         }
@@ -766,8 +781,10 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
         for(i = 0; i <= stereo; i++)
             *samples8++ = pred[i];
         for(; i < unp_size; i++) {
-            if(get_bits_left(&gb)<0)
-                return AVERROR_INVALIDDATA;
+            if (get_bits_left(&gb) < 0) {
+                ret = AVERROR_INVALIDDATA;
+                goto error;
+            }
             if(i & stereo){
                 if(vlc[1].table)
                     res = get_vlc2(&gb, vlc[1].table, SMKTREE_BITS, 3);
@@ -775,7 +792,8 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 pred[1] += sign_extend(h[1].values[res], 8);
                 *samples8++ = pred[1];
@@ -786,7 +804,8 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data,
                     res = 0;
                 if (res < 0) {
                     av_log(avctx, AV_LOG_ERROR, "invalid vlc\n");
-                    return AVERROR_INVALIDDATA;
+                    ret = AVERROR_INVALIDDATA;
+                    goto error;
                 }
                 pred[0] += sign_extend(h[0].values[res], 8);
                 *samples8++ = pred[0];
@@ -819,6 +838,7 @@ AVCodec ff_smacker_decoder = {
     .close          = decode_end,
     .decode         = decode_frame,
     .capabilities   = AV_CODEC_CAP_DR1,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
 };
 
 AVCodec ff_smackaud_decoder = {