avcodec/rv10: Execute whole size check earlier for rv20

Fixes: Timeout
Fixes: 31380/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_RV20_fuzzer-5230899257016320
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c
index bd70689caba07be2135263e481dc6f785ca36a8a..9c3a48c251881ab1dfd6e36f6088b95b2f218cd5 100644 (file)
--- a/libavcodec/rv10.c
+++ b/libavcodec/rv10.c
@@ -154,7 +154,7 @@ static int rv10_decode_picture_header(MpegEncContext *s)
     return mb_count;
 }
 
-static int rv20_decode_picture_header(RVDecContext *rv)
+static int rv20_decode_picture_header(RVDecContext *rv, int whole_size)
 {
     MpegEncContext *s = &rv->m;
     int seq, mb_pos, i, ret;
@@ -232,6 +232,10 @@ static int rv20_decode_picture_header(RVDecContext *rv)
                    "attempting to change resolution to %dx%d\n", new_w, new_h);
             if (av_image_check_size(new_w, new_h, 0, s->avctx) < 0)
                 return AVERROR_INVALIDDATA;
+
+            if (whole_size < (new_w + 15)/16 * ((new_h + 15)/16) / 8)
+                return AVERROR_INVALIDDATA;
+
             ff_mpv_common_end(s);
 
             // attempt to keep aspect during typical resolution switches
@@ -447,7 +451,7 @@ static int rv10_decode_packet(AVCodecContext *avctx, const uint8_t *buf,
     if (s->codec_id == AV_CODEC_ID_RV10)
         mb_count = rv10_decode_picture_header(s);
     else
-        mb_count = rv20_decode_picture_header(rv);
+        mb_count = rv20_decode_picture_header(rv, whole_size);
     if (mb_count < 0) {
         if (mb_count != ERROR_SKIP_FRAME)
             av_log(s->avctx, AV_LOG_ERROR, "HEADER ERROR\n");