Merge commit 'a97563c889fefd81ad6b3758471434d8c2e2e550'

[ffmpeg] / libavcodec / pixlet.c

diff --git a/libavcodec/pixlet.c b/libavcodec/pixlet.c
index d0313051550ca8b902aabbd8269e4809cd586524..a9cfe085c928397a380990c147382674951bd39f 100644 (file)
--- a/libavcodec/pixlet.c
+++ b/libavcodec/pixlet.c
@@ -206,8 +206,8 @@ static int read_high_coeffs(AVCodecContext *avctx, uint8_t *src, int16_t *dst, i
     if ((ret = init_get_bits8(b, src, bytestream2_get_bytes_left(&ctx->gb))) < 0)
       return ret;
 
-    if ((a >= 0) + (a ^ (a >> 31)) - (a >> 31) != 1) {
-        nbits = 33 - ff_clz((a >= 0) + (a ^ (a >> 31)) - (a >> 31) - 1);
+    if (a ^ (a >> 31)) {
+        nbits = 33 - ff_clz(a ^ (a >> 31));
         if (nbits > 16)
             return AVERROR_INVALIDDATA;
     } else {
@@ -229,6 +229,8 @@ static int read_high_coeffs(AVCodecContext *avctx, uint8_t *src, int16_t *dst, i
             cnt1 = get_bits(b, nbits);
         } else {
             pfx = 14 + ((((uint64_t)(value - 14)) >> 32) & (value - 14));
+            if (pfx < 1 || pfx > 25)
+                return AVERROR_INVALIDDATA;
             cnt1 *= (1 << pfx) - 1;
             shbits = show_bits(b, pfx);
             if (shbits <= 1) {
@@ -256,11 +258,11 @@ static int read_high_coeffs(AVCodecContext *avctx, uint8_t *src, int16_t *dst, i
             j = 0;
             dst += stride;
         }
-        state += (int64_t)d * yflag - (d * state >> 8);
+        state += (int64_t)d * (uint64_t)yflag - ((int64_t)(d * (uint64_t)state) >> 8);
 
         flag = 0;
 
-        if (state * 4ULL > 0xFF || i >= size)
+        if ((uint64_t)state > 0xFF / 4 || i >= size)
             continue;
 
         pfx = ((state + 8) >> 5) + (state ? ff_clz(state): 32) - 24;
@@ -329,6 +331,9 @@ static int read_highpass(AVCodecContext *avctx, uint8_t *ptr, int plane, AVFrame
             return AVERROR_INVALIDDATA;
         }
 
+        if (a == INT32_MIN)
+            return AVERROR_INVALIDDATA;
+
         ret = read_high_coeffs(avctx, ptr + bytestream2_tell(&ctx->gb), dest, size,
                                c, (b >= FFABS(a)) ? b : a, d,
                                ctx->band[plane][i + 1].width, stride);
@@ -592,6 +597,10 @@ static int pixlet_decode_frame(AVCodecContext *avctx, void *data,
     width  = bytestream2_get_be32(&ctx->gb);
     height = bytestream2_get_be32(&ctx->gb);
 
+    if (    width > INT_MAX - (1U << (NB_LEVELS + 1))
+        || height > INT_MAX - (1U << (NB_LEVELS + 1)))
+        return AVERROR_INVALIDDATA;
+
     w = FFALIGN(width,  1 << (NB_LEVELS + 1));
     h = FFALIGN(height, 1 << (NB_LEVELS + 1));