avcodec/fitsdec: Prevent division by 0 with huge data_max

Fixes: division by 0
Fixes: 15657/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FITS_fuzzer-5738154838982656
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/fitsdec.c b/libavcodec/fitsdec.c
index 4f452422efd60783a59fe809c649dfb7a55b62c3..88b841a964453fd7ed5988becc7c3edbae1643ac 100644 (file)
--- a/libavcodec/fitsdec.c
+++ b/libavcodec/fitsdec.c
@@ -195,6 +195,7 @@ static int fits_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     uint8_t *dst8;
     uint16_t *dst16;
     uint64_t t;
+    double scale;
     FITSHeader header;
     FITSContext * fitsctx = avctx->priv_data;
 
@@ -204,6 +205,12 @@ static int fits_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if (ret < 0)
         return ret;
 
+    scale = header.data_max - header.data_min;
+    if (scale <= 0 || !isfinite(scale)) {
+        scale = 1;
+    }
+    scale = 1/scale;
+
     if (header.rgb) {
         if (header.bitpix == 8) {
             if (header.naxisn[2] == 3) {
@@ -272,7 +279,7 @@ static int fits_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             for (j = 0; j < avctx->width; j++) { \
                 t = rd; \
                 if (!header.blank_found || t != header.blank) { \
-                    *dst++ = ((t - header.data_min) * ((1 << (sizeof(type) * 8)) - 1)) / (header.data_max - header.data_min); \
+                    *dst++ = ((t - header.data_min) * ((1 << (sizeof(type) * 8)) - 1)) * scale; \
                 } else { \
                     *dst++ = fitsctx->blank_val; \
                 } \