avformat/mspdec: Check packet_size more completely

Fixes: OOM
Fixes: 28348/clusterfuzz-testcase-minimized-ffmpeg_dem_MSP_fuzzer-4612055872831488
Fixes: 28360/clusterfuzz-testcase-minimized-ffmpeg_dem_MSP_fuzzer-6245230626078720
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavformat/mspdec.c b/libavformat/mspdec.c
index b81d835a638e9b137e2b54d7aea204649743f8cd..4845eb3729282ed9bfd4fba00f471b362485f2c2 100644 (file)
--- a/libavformat/mspdec.c
+++ b/libavformat/mspdec.c
@@ -70,11 +70,12 @@ static int msp_read_header(AVFormatContext *s)
 
     if (st->codecpar->codec_id == AV_CODEC_ID_RAWVIDEO) {
         cntx->packet_size = av_image_get_buffer_size(st->codecpar->format, st->codecpar->width, st->codecpar->height, 1);
-        if (cntx->packet_size < 0)
-            return cntx->packet_size;
     } else
         cntx->packet_size = 2 * st->codecpar->height;
 
+    if (cntx->packet_size <= 0)
+        return cntx->packet_size < 0 ? cntx->packet_size : AVERROR_INVALIDDATA;
+
     return 0;
 }