avcodec/gdv: Check input palette size before rescale()

Fixes: Timeout (22sec -> 11sec)
Fixes: 13576/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_GDV_fuzzer-5681024577568768
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/gdv.c b/libavcodec/gdv.c
index 183286b55f178f335fa9ff12e40bf97cc6db440e..9cf30c09e463fb0be149b8700ee0408c390305b9 100644 (file)
--- a/libavcodec/gdv.c
+++ b/libavcodec/gdv.c
@@ -475,6 +475,8 @@ static int gdv_decode_frame(AVCodecContext *avctx, void *data,
     if (pal && pal_size == AVPALETTE_SIZE)
         memcpy(gdv->pal, pal, AVPALETTE_SIZE);
 
+    if (compression < 2 && bytestream2_get_bytes_left(gb) < 256*3)
+        return AVERROR_INVALIDDATA;
     rescale(gdv, gdv->frame, avctx->width, avctx->height,
             !!(flags & 0x10), !!(flags & 0x20));
 
@@ -482,8 +484,6 @@ static int gdv_decode_frame(AVCodecContext *avctx, void *data,
     case 1:
         memset(gdv->frame + PREAMBLE_SIZE, 0, gdv->frame_size - PREAMBLE_SIZE);
     case 0:
-        if (bytestream2_get_bytes_left(gb) < 256*3)
-            return AVERROR_INVALIDDATA;
         for (i = 0; i < 256; i++) {
             unsigned r = bytestream2_get_byte(gb);
             unsigned g = bytestream2_get_byte(gb);