avcodec/ralf: Fix integer overflows with the filter coefficient in decode_channel()

Fixes: signed integer overflow: 1145975808 - -1146173210 cannot be represented in type 'int'
Fixes: 18616/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_RALF_fuzzer-5121296757424128
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/ralf.c b/libavcodec/ralf.c
index ca8817aa21ecb70138f0b9af3b792942ab272bb3..d8f1803086f6204664a11fc11023b82e6b14febd 100644 (file)
--- a/libavcodec/ralf.c
+++ b/libavcodec/ralf.c
@@ -264,8 +264,8 @@ static int decode_channel(RALFContext *ctx, GetBitContext *gb, int ch,
             t = get_vlc2(gb, vlc[cmode].table, vlc[cmode].bits, 2);
             t = extend_code(gb, t, 21, add_bits);
             if (!cmode)
-                coeff -= 12 << add_bits;
-            coeff = t - coeff;
+                coeff -= 12U << add_bits;
+            coeff = (unsigned)t - coeff;
             ctx->filter[i] = coeff;
 
             cmode = coeff >> add_bits;