avcodec/apedec: Make coeffsA/B uint32_t, this avoids several cases of undefined behavior

Changing the type to an unsigned one to avoid many casts was suggested
This may be inadequate for fixing the UB on ILP64

Fixes: signed integer overflow: -1418162611 * 383 cannot be represented in type 'int'
Fixes: 15547/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_APE_fuzzer-5691384901664768
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/apedec.c b/libavcodec/apedec.c
index 7a7097e7a48620b03d6cc278adb67b70d92161c0..e9ffdfdcdfb952f1eddb963c8ed0f021d915a69f 100644 (file)
--- a/libavcodec/apedec.c
+++ b/libavcodec/apedec.c
@@ -125,8 +125,8 @@ typedef struct APEPredictor {
     int32_t filterA[2];
     int32_t filterB[2];
 
-    int32_t coeffsA[2][4];  ///< adaption coefficients
-    int32_t coeffsB[2][5];  ///< adaption coefficients
+    uint32_t coeffsA[2][4];  ///< adaption coefficients
+    uint32_t coeffsB[2][5];  ///< adaption coefficients
     int32_t historybuffer[HISTORY_SIZE + PREDICTOR_SIZE];
 
     unsigned int sample_pos;
@@ -829,7 +829,7 @@ static av_always_inline int filter_fast_3320(APEPredictor *p,
     }
 
     predictionA = p->buf[delayA] * 2 - p->buf[delayA - 1];
-    p->lastA[filter] = decoded + (predictionA  * p->coeffsA[filter][0] >> 9);
+    p->lastA[filter] = decoded + ((int32_t)(predictionA  * p->coeffsA[filter][0]) >> 9);
 
     if ((decoded ^ predictionA) > 0)
         p->coeffsA[filter][0]++;