avcodec/brenderpix: Check input size before allocating image

An incomplete image is not supported prior to this and will
not produce any output. This commit moves the failure before
time consuming operations.

Fixes: Timeout (81sec -> 76ms)
Fixes: 15723/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BRENDER_PIX_fuzzer-5147265653538816
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/brenderpix.c b/libavcodec/brenderpix.c
index 0556858de17f6fc8588c390ae5939e21c99c64ec..46b7a59aa49562d168d285395d8fe8240f6ba10f 100644 (file)
--- a/libavcodec/brenderpix.c
+++ b/libavcodec/brenderpix.c
@@ -204,6 +204,10 @@ static int pix_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         avpriv_request_sample(avctx, "Format %d", hdr.format);
         return AVERROR_PATCHWELCOME;
     }
+    bytes_per_scanline = bytes_pp * hdr.width;
+
+    if (bytestream2_get_bytes_left(&gb) < hdr.height * bytes_per_scanline)
+        return AVERROR_INVALIDDATA;
 
     if ((ret = ff_set_dimensions(avctx, hdr.width, hdr.height)) < 0)
         return ret;
@@ -261,7 +265,6 @@ static int pix_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     bytestream2_skip(&gb, 8);
 
     // read the image data to the buffer
-    bytes_per_scanline = bytes_pp * hdr.width;
     bytes_left = bytestream2_get_bytes_left(&gb);
 
     if (chunk_type != IMAGE_DATA_CHUNK || data_len != bytes_left ||