lavc: Check CODEC_CAP_VARIABLE_FRAME_SIZE && !frame

This combination is quite odd and almost certainly a bug if
it happens.

Reviewed-by: Justin Ruggles <justin.ruggles@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 22a7ee3666f7125c4f262ba489c00161455264ad..657eb5b932d4c6c0c946462d6366da1ad1823d8d 100644 (file)
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -968,6 +968,8 @@ int attribute_align_arg avcodec_encode_audio2(AVCodecContext *avctx,
         if (!user_packet) {
             if (avctx->codec->capabilities & CODEC_CAP_VARIABLE_FRAME_SIZE) {
                 av_assert0(av_get_bits_per_sample(avctx->codec_id) != 0);
+                if (!frame)
+                    return AVERROR(EINVAL);
                 buf_size = nb_samples * avctx->channels *
                            av_get_bits_per_sample(avctx->codec_id) / 8;
             } else {