avcodec/pixlet: Fix reading invalid numbers of bits

[ffmpeg] / libavcodec / pixlet.c

diff --git a/libavcodec/pixlet.c b/libavcodec/pixlet.c
index c4f7597866c18c409204c830bbcb11ae315acf77..0dfd5edbf7e7a5a068ca6a043f52aaa3de8f3d22 100644 (file)
--- a/libavcodec/pixlet.c
+++ b/libavcodec/pixlet.c
@@ -154,7 +154,7 @@ static int read_low_coeffs(AVCodecContext *avctx, int16_t *dst, int size, int wi
         state = 120 * (escape + flag) + state - (120 * state >> 8);
         flag = 0;
 
-        if (state * 4 > 0xFF || i >= size)
+        if (state * 4ULL > 0xFF || i >= size)
             continue;
 
         nbits = ((state + 8) >> 5) + (state ? ff_clz(state) : 32) - 24;
@@ -229,6 +229,8 @@ static int read_high_coeffs(AVCodecContext *avctx, uint8_t *src, int16_t *dst, i
             cnt1 = get_bits(b, nbits);
         } else {
             pfx = 14 + ((((uint64_t)(value - 14)) >> 32) & (value - 14));
+            if (pfx < 1 || pfx > 25)
+                return AVERROR_INVALIDDATA;
             cnt1 *= (1 << pfx) - 1;
             shbits = show_bits(b, pfx);
             if (shbits <= 1) {
@@ -256,11 +258,11 @@ static int read_high_coeffs(AVCodecContext *avctx, uint8_t *src, int16_t *dst, i
             j = 0;
             dst += stride;
         }
-        state += (int64_t)d * yflag - (d * state >> 8);
+        state += (int64_t)d * (uint64_t)yflag - ((int64_t)(d * (uint64_t)state) >> 8);
 
         flag = 0;
 
-        if (state * 4 > 0xFF || i >= size)
+        if (state * 4ULL > 0xFF || i >= size)
             continue;
 
         pfx = ((state + 8) >> 5) + (state ? ff_clz(state): 32) - 24;