avcodec/pngdec: Check trns more completely

Fixes out of array access
Fixes: 546/clusterfuzz-testcase-4809433909559296
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/pngdec.c b/libavcodec/pngdec.c
index 50a0a296b3775ffdfd4b4aa541396b71817ed43a..e87c2aa03bac6d61c9c0729c26fb9f3e0430de90 100644 (file)
--- a/libavcodec/pngdec.c
+++ b/libavcodec/pngdec.c
@@ -772,6 +772,16 @@ static int decode_trns_chunk(AVCodecContext *avctx, PNGDecContext *s,
 {
     int v, i;
 
+    if (!(s->state & PNG_IHDR)) {
+        av_log(avctx, AV_LOG_ERROR, "trns before IHDR\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    if (s->state & PNG_IDAT) {
+        av_log(avctx, AV_LOG_ERROR, "trns after IDAT\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     if (s->color_type == PNG_COLOR_TYPE_PALETTE) {
         if (length > 256 || !(s->state & PNG_PLTE))
             return AVERROR_INVALIDDATA;
@@ -782,7 +792,8 @@ static int decode_trns_chunk(AVCodecContext *avctx, PNGDecContext *s,
         }
     } else if (s->color_type == PNG_COLOR_TYPE_GRAY || s->color_type == PNG_COLOR_TYPE_RGB) {
         if ((s->color_type == PNG_COLOR_TYPE_GRAY && length != 2) ||
-            (s->color_type == PNG_COLOR_TYPE_RGB && length != 6))
+            (s->color_type == PNG_COLOR_TYPE_RGB && length != 6) ||
+            s->bit_depth == 1)
             return AVERROR_INVALIDDATA;
 
         for (i = 0; i < length / 2; i++) {
@@ -1242,6 +1253,8 @@ exit_loop:
         size_t raw_bpp = s->bpp - byte_depth;
         unsigned x, y;
 
+        av_assert0(s->bit_depth > 1);
+
         for (y = 0; y < s->height; ++y) {
             uint8_t *row = &s->image_buf[s->image_linesize * y];