qdm2: Check data block size for bytes to bits overflow.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit dac56d9ce01eb9963f28f26b97a81db5cbd46c1c)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c
index 0eca7ade2136ea9d32f32c1c85bb4cdd867d31f1..5da21d757db536e3ae08cbe21f72a181bc7b61f3 100644 (file)
--- a/libavcodec/qdm2.c
+++ b/libavcodec/qdm2.c
@@ -1819,6 +1819,10 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx)
     extradata += 4;
 
     s->checksum_size = AV_RB32(extradata);
+    if (s->checksum_size >= 1U << 28) {
+        av_log(avctx, AV_LOG_ERROR, "data block size too large (%u)\n", s->checksum_size);
+        return AVERROR_INVALIDDATA;
+    }
 
     s->fft_order = av_log2(s->fft_size) + 1;
     s->fft_frame_size = 2 * s->fft_size; // complex has two floats