avcodec/dvbsubdec: Check pixel buffer size constraint from ETSI EN 300 743 V1.3.1

Fixes: OOM
Fixes: 2143/clusterfuzz-testcase-minimized-5482288060039168
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/dvbsubdec.c b/libavcodec/dvbsubdec.c
index bc16eb735beb13ea633e4bea12a735a22240eb22..98619f9631a57efad61bc7b006ce2acc1ae0e918 100644 (file)
--- a/libavcodec/dvbsubdec.c
+++ b/libavcodec/dvbsubdec.c
@@ -1158,6 +1158,10 @@ static int dvbsub_parse_region_segment(AVCodecContext *avctx,
     buf += 2;
 
     ret = av_image_check_size2(region->width, region->height, avctx->max_pixels, AV_PIX_FMT_PAL8, 0, avctx);
+    if (ret >= 0 && region->width * region->height * 2 > 320 * 1024 * 8) {
+        ret = AVERROR_INVALIDDATA;
+        av_log(avctx, AV_LOG_ERROR, "Pixel buffer memory constraint violated\n");
+    }
     if (ret < 0) {
         region->width= region->height= 0;
         return ret;