avcodec/cook: Check samples_per_channel earlier

Fixes: division by zero
Fixes: 18362/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_COOK_fuzzer-5653727679086592
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/cook.c b/libavcodec/cook.c
index aa112051662d3c579519fd17d2fa1d801372db49..3a40ccaf2c2463ac574eb9a1b8f1d296f65ddbc5 100644 (file)
--- a/libavcodec/cook.c
+++ b/libavcodec/cook.c
@@ -1217,6 +1217,15 @@ static av_cold int cook_decode_init(AVCodecContext *avctx)
             return AVERROR_PATCHWELCOME;
         }
     }
+
+    /* Try to catch some obviously faulty streams, otherwise it might be exploitable */
+    if (q->samples_per_channel != 256 && q->samples_per_channel != 512 &&
+        q->samples_per_channel != 1024) {
+        avpriv_request_sample(avctx, "samples_per_channel = %d",
+                              q->samples_per_channel);
+        return AVERROR_PATCHWELCOME;
+    }
+
     /* Generate tables */
     init_pow2table();
     init_gain_table(q);
@@ -1252,14 +1261,6 @@ static av_cold int cook_decode_init(AVCodecContext *avctx)
         q->saturate_output = saturate_output_float;
     }
 
-    /* Try to catch some obviously faulty streams, otherwise it might be exploitable */
-    if (q->samples_per_channel != 256 && q->samples_per_channel != 512 &&
-        q->samples_per_channel != 1024) {
-        avpriv_request_sample(avctx, "samples_per_channel = %d",
-                              q->samples_per_channel);
-        return AVERROR_PATCHWELCOME;
-    }
-
     avctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
     if (channel_mask)
         avctx->channel_layout = channel_mask;