avcodec/mv30: Fix multiple integer overflows

Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int'
Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/mv30.c b/libavcodec/mv30.c
index ff60be881dc53e6134819ebad0e54049ff664f2b..9f28199478f3242939ab0c04290dfff4e03d8245 100644 (file)
--- a/libavcodec/mv30.c
+++ b/libavcodec/mv30.c
@@ -198,12 +198,12 @@ static void idct_add(uint8_t *dst, int stride,
 
 static inline void idct2_1d(int *blk, int step)
 {
-    const int t0 = blk[0 * step];
-    const int t1 = blk[1 * step];
-    const int t2 = (int)(t1 * 473U) >> 8;
-    const int t3 = t2 - t1;
-    const int t4 =  ((int)(t1 * 362U) >> 8) - t3;
-    const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
+    const unsigned int  t0 = blk[0 * step];
+    const unsigned int t1 = blk[1 * step];
+    const unsigned int t2 = (int)(t1 * 473U) >> 8;
+    const unsigned int t3 = t2 - t1;
+    const unsigned int t4 =  ((int)(t1 * 362U) >> 8) - t3;
+    const unsigned int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
 
     blk[0 * step] = t1 + t0;
     blk[1 * step] = t0 + t3;
@@ -305,14 +305,14 @@ static int decode_intra_block(AVCodecContext *avctx, int mode,
     case 1:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = ((pfill[0] * qtab[0]) >> 5) + 128;
+        block[0] = ((int)((unsigned)pfill[0] * qtab[0]) >> 5) + 128;
         s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8);
         break;
     case 2:
         memset(block, 0, sizeof(*block) * 64);
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
         block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
         block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -321,7 +321,7 @@ static int decode_intra_block(AVCodecContext *avctx, int mode,
     case 3:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         for (int i = 1; i < 64; i++)
             block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
         idct_put(dst, linesize, block);
@@ -346,14 +346,14 @@ static int decode_inter_block(AVCodecContext *avctx, int mode,
     case 1:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = (pfill[0] * qtab[0]) >> 5;
+        block[0] = (int)((unsigned)pfill[0] * qtab[0]) >> 5;
         update_inter_block(dst, linesize, src, in_linesize, block[0]);
         break;
     case 2:
         memset(block, 0, sizeof(*block) * 64);
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
         block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
         block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -362,7 +362,7 @@ static int decode_inter_block(AVCodecContext *avctx, int mode,
     case 3:
         fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
         pfill[0] += fill;
-        block[0] = pfill[0] * qtab[0];
+        block[0] = (unsigned)pfill[0] * qtab[0];
         for (int i = 1; i < 64; i++)
             block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
         idct_add(dst, linesize, src, in_linesize, block);