avcodec/cook: Move up and extend block_align check

Fixes: signed integer overflow: 2046820356 * 8 cannot be represented in type 'int'
Fixes: 18391/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_COOK_fuzzer-5631674666188800
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/cook.c b/libavcodec/cook.c
index eb6ff9f465d8beac4c2060c84ab45a37026cbbd6..248e59256f0343c3f49ac85ee8212f0cc24a8e4d 100644 (file)
--- a/libavcodec/cook.c
+++ b/libavcodec/cook.c
@@ -1075,6 +1075,9 @@ static av_cold int cook_decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
+    if (avctx->block_align >= INT_MAX / 8)
+        return AVERROR(EINVAL);
+
     /* Initialize RNG. */
     av_lfg_init(&q->random_state, 0);
 
@@ -1234,10 +1237,6 @@ static av_cold int cook_decode_init(AVCodecContext *avctx)
     if ((ret = init_cook_vlc_tables(q)))
         return ret;
 
-
-    if (avctx->block_align >= UINT_MAX / 2)
-        return AVERROR(EINVAL);
-
     /* Pad the databuffer with:
        DECODE_BYTES_PAD1 or DECODE_BYTES_PAD2 for decode_bytes(),
        AV_INPUT_BUFFER_PADDING_SIZE, for the bitstreamreader. */