avcodec/vp9: Fix stack-buffer overflow with VP9 VDPAU available

[ffmpeg] / libavcodec / mlpenc.c

diff --git a/libavcodec/mlpenc.c b/libavcodec/mlpenc.c
index a9501c694f50852f370265bee906c5846b3c81c8..f6159d39c89ef3a188fe98252d035403f4d13fc4 100644 (file)
--- a/libavcodec/mlpenc.c
+++ b/libavcodec/mlpenc.c
@@ -531,7 +531,7 @@ static av_cold int mlp_encode_init(AVCodecContext *avctx)
         av_log(avctx, AV_LOG_ERROR, "Unsupported sample rate %d. Supported "
                             "sample rates are 44100, 88200, 176400, 48000, "
                             "96000, and 192000.\n", avctx->sample_rate);
-        return -1;
+        return AVERROR(EINVAL);
     }
     ctx->coded_sample_rate[1] = -1 & 0xf;
 
@@ -564,7 +564,7 @@ static av_cold int mlp_encode_init(AVCodecContext *avctx)
     default:
         av_log(avctx, AV_LOG_ERROR, "Sample format not supported. "
                "Only 16- and 24-bit samples are supported.\n");
-        return -1;
+        return AVERROR(EINVAL);
     }
     ctx->coded_sample_fmt[1] = -1 & 0xf;
 
@@ -638,7 +638,7 @@ static av_cold int mlp_encode_init(AVCodecContext *avctx)
             ctx->channel_arrangement = 12; break;
         default:
             av_log(avctx, AV_LOG_ERROR, "Unsupported channel arrangement\n");
-            return -1;
+            return AVERROR(EINVAL);
         }
         ctx->flags = FLAGS_DVDA;
         ctx->channel_occupancy = ff_mlp_ch_info[ctx->channel_arrangement].channel_occupancy;
@@ -666,7 +666,7 @@ static av_cold int mlp_encode_init(AVCodecContext *avctx)
             break;
         default:
             av_log(avctx, AV_LOG_ERROR, "Unsupported channel arrangement\n");
-            return -1;
+            return AVERROR(EINVAL);
         }
         ctx->flags = 0;
         ctx->channel_occupancy = 0;
@@ -1190,7 +1190,7 @@ static unsigned int write_access_unit(MLPEncodeContext *ctx, uint8_t *buf,
     int total_length;
 
     if (buf_size < 4)
-        return -1;
+        return AVERROR(EINVAL);
 
     /* Frame header will be written at the end. */
     buf      += 4;
@@ -1198,7 +1198,7 @@ static unsigned int write_access_unit(MLPEncodeContext *ctx, uint8_t *buf,
 
     if (restart_frame) {
         if (buf_size < 28)
-            return -1;
+            return AVERROR(EINVAL);
         write_major_sync(ctx, buf, buf_size);
         buf      += 28;
         buf_size -= 28;
@@ -1820,7 +1820,8 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
         if (!filter_state_buffer[i]) {
             av_log(ctx->avctx, AV_LOG_ERROR,
                    "Not enough memory for applying filters.\n");
-            return -1;
+            ret = AVERROR(ENOMEM);
+            goto free_and_return;
         }
     }
 
@@ -1848,7 +1849,7 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel)
         residual = sample - (accum & mask);
 
         if (residual < SAMPLE_MIN(24) || residual > SAMPLE_MAX(24)) {
-            ret = -1;
+            ret = AVERROR_INVALIDDATA;
             goto free_and_return;
         }
 
@@ -2226,9 +2227,6 @@ static int mlp_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
     if ((ret = ff_alloc_packet2(avctx, avpkt, 87500 * avctx->channels, 0)) < 0)
         return ret;
 
-    if (!frame)
-        return 1;
-
     /* add current frame to queue */
     if ((ret = ff_af_queue_add(&ctx->afq, frame)) < 0)
         return ret;
@@ -2267,7 +2265,7 @@ static int mlp_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
     if (ctx->frame_size[ctx->frame_index] > MAX_BLOCKSIZE) {
         av_log(avctx, AV_LOG_ERROR, "Invalid frame size (%d > %d)\n",
                ctx->frame_size[ctx->frame_index], MAX_BLOCKSIZE);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     restart_frame = !ctx->frame_index;
@@ -2373,6 +2371,7 @@ static av_cold int mlp_encode_close(AVCodecContext *avctx)
     av_freep(&ctx->decoding_params);
     av_freep(&ctx->channel_params);
     av_freep(&ctx->frame_size);
+    av_freep(&ctx->max_output_bits);
     ff_af_queue_close(&ctx->afq);
 
     return 0;
@@ -2388,10 +2387,11 @@ AVCodec ff_mlp_encoder = {
     .init                   = mlp_encode_init,
     .encode2                = mlp_encode_frame,
     .close                  = mlp_encode_close,
-    .capabilities           = AV_CODEC_CAP_SMALL_LAST_FRAME | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_EXPERIMENTAL,
+    .capabilities           = AV_CODEC_CAP_SMALL_LAST_FRAME | AV_CODEC_CAP_EXPERIMENTAL,
     .sample_fmts            = (const enum AVSampleFormat[]) {AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_NONE},
     .supported_samplerates  = (const int[]) {44100, 48000, 88200, 96000, 176400, 192000, 0},
     .channel_layouts        = ff_mlp_channel_layouts,
+    .caps_internal          = FF_CODEC_CAP_INIT_CLEANUP,
 };
 #endif
 #if CONFIG_TRUEHD_ENCODER
@@ -2404,9 +2404,10 @@ AVCodec ff_truehd_encoder = {
     .init                   = mlp_encode_init,
     .encode2                = mlp_encode_frame,
     .close                  = mlp_encode_close,
-    .capabilities           = AV_CODEC_CAP_SMALL_LAST_FRAME | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_EXPERIMENTAL,
+    .capabilities           = AV_CODEC_CAP_SMALL_LAST_FRAME | AV_CODEC_CAP_EXPERIMENTAL,
     .sample_fmts            = (const enum AVSampleFormat[]) {AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_NONE},
     .supported_samplerates  = (const int[]) {44100, 48000, 88200, 96000, 176400, 192000, 0},
     .channel_layouts        = (const uint64_t[]) {AV_CH_LAYOUT_STEREO, AV_CH_LAYOUT_5POINT0_BACK, AV_CH_LAYOUT_5POINT1_BACK, 0},
+    .caps_internal          = FF_CODEC_CAP_INIT_CLEANUP,
 };
 #endif