avcodec/iff: finetune the palette size check in the mask case

Fixes: out of array access
Fixes: 15381/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_IFF_ILBM_fuzzer-5668057826983936
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Peter Ross <pross@xvid.org>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/iff.c b/libavcodec/iff.c
index b43bd507b368061a223ccf8aa53159c3a6c58dbd..4099d118e48e4592bfb522b3f6e7bff54bd52436 100644 (file)
--- a/libavcodec/iff.c
+++ b/libavcodec/iff.c
@@ -1514,7 +1514,7 @@ static int decode_frame(AVCodecContext *avctx,
     buf_size -= bytestream2_tell(gb);
     desc = av_pix_fmt_desc_get(avctx->pix_fmt);
 
-    if (!s->init && avctx->bits_per_coded_sample <= 8 &&
+    if (!s->init && avctx->bits_per_coded_sample <= 8 - (s->masking == MASK_HAS_MASK) &&
         avctx->pix_fmt == AV_PIX_FMT_PAL8) {
         if ((res = cmap_read_palette(avctx, (uint32_t *)frame->data[1])) < 0)
             return res;