Fixes: signed integer overflow: 44331634 * 65 cannot be represented in type 'int'
Fixes: 32120/clusterfuzz-testcase-minimized-ffmpeg_dem_RSD_fuzzer-5760221223583744
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
if (ba > 0) {
/* calc from frame_bytes, channels, and block_align */
int blocks = frame_bytes / ba;
+ int64_t tmp;
switch (id) {
case AV_CODEC_ID_ADPCM_IMA_WAV:
if (bps < 2 || bps > 5)
return 0;
- return blocks * (1 + (ba - 4 * ch) / (bps * ch) * 8);
+ tmp = blocks * (1LL + (ba - 4 * ch) / (bps * ch) * 8);
+ if (tmp != (int)tmp)
+ return 0;
+ return tmp;
case AV_CODEC_ID_ADPCM_IMA_DK3:
return blocks * (((ba - 16) * 2 / 3 * 4) / ch);
case AV_CODEC_ID_ADPCM_IMA_DK4: