avcodec/qdm2: Check frame size

Fixes: index 2304 out of bounds for type 'float [2304]'
Fixes: 16332/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_QDM2_fuzzer-5679142481166336
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c
index eaffb36dcc08a879be32b950df19a9181e825d34..ac8ae8cbbb9b3dfedbdec1667809986daa36b2c6 100644 (file)
--- a/libavcodec/qdm2.c
+++ b/libavcodec/qdm2.c
@@ -1727,6 +1727,11 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx)
     s->sub_sampling = s->fft_order - 7;
     s->frequency_range = 255 / (1 << (2 - s->sub_sampling));
 
+    if (s->frame_size * 4 >> s->sub_sampling > MPA_FRAME_SIZE) {
+        avpriv_request_sample(avctx, "large frames");
+        return AVERROR_PATCHWELCOME;
+    }
+
     switch ((s->sub_sampling * 2 + s->channels - 1)) {
         case 0: tmp = 40; break;
         case 1: tmp = 48; break;