avcodec/mobiclip: Fix multiple integer overflows

Fixes: signed integer overflow: 872415232 * 7 cannot be represented in type 'int'
Fixes: signed integer overflow: -2013265888 + -1744830464 cannot be represented in type 'int'
Fixes: 25834/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MOBICLIP_fuzzer-5471406434025472
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/mobiclip.c b/libavcodec/mobiclip.c
index f890cb25990f9b3184d4a84d12fc94ed13d75f2f..0f150a551a89111ee8fff1a1549ef2088e50d5b6 100644 (file)
--- a/libavcodec/mobiclip.c
+++ b/libavcodec/mobiclip.c
@@ -407,12 +407,12 @@ static int setup_qtables(AVCodecContext *avctx, int quantizer)
     return 0;
 }
 
-static void inverse4(int *rs)
+static void inverse4(unsigned *rs)
 {
-    int a = rs[0] + rs[2];
-    int b = rs[0] - rs[2];
-    int c = rs[1] + (rs[3] >> 1);
-    int d = (rs[1] >> 1) - rs[3];
+    unsigned a = rs[0] + rs[2];
+    unsigned b = rs[0] - rs[2];
+    unsigned c = rs[1] + ((int)rs[3] >> 1);
+    unsigned d = ((int)rs[1] >> 1) - rs[3];
 
     rs[0] = a + c;
     rs[1] = b + d;
@@ -519,7 +519,7 @@ static int add_coefficients(AVCodecContext *avctx, AVFrame *frame,
         if (pos >= size * size)
             return AVERROR_INVALIDDATA;
         qval = qtab[pos];
-        mat[ztab[pos]] = qval * level;
+        mat[ztab[pos]] = qval *(unsigned)level;
 
         if (last)
             break;