avcodec/mjpegdec: quant_matrixes can be up to 65535, use uint16_t

Fixes invalid shift
Fixes: 870/clusterfuzz-testcase-5649105424482304
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index af12d5d71b2d6dd0b452848275713953f5b12451..f26e8a3f9ac053c6124d4e0126d157837bc8f835 100644 (file)
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -686,7 +686,7 @@ static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index)
 
 /* decode block and dequantize */
 static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
-                        int dc_index, int ac_index, int16_t *quant_matrix)
+                        int dc_index, int ac_index, uint16_t *quant_matrix)
 {
     int code, i, j, level, val;
 
@@ -736,7 +736,7 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
 
 static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
                                  int component, int dc_index,
-                                 int16_t *quant_matrix, int Al)
+                                 uint16_t *quant_matrix, int Al)
 {
     int val;
     s->bdsp.clear_block(block);
@@ -754,7 +754,7 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
 /* decode block and dequantize - progressive JPEG version */
 static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block,
                                     uint8_t *last_nnz, int ac_index,
-                                    int16_t *quant_matrix,
+                                    uint16_t *quant_matrix,
                                     int ss, int se, int Al, int *EOBRUN)
 {
     int code, i, j, level, val, run;
@@ -852,7 +852,7 @@ for (; ; i++) {                                                     \
 /* decode block and dequantize - progressive JPEG refinement pass */
 static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block,
                                    uint8_t *last_nnz,
-                                   int ac_index, int16_t *quant_matrix,
+                                   int ac_index, uint16_t *quant_matrix,
                                    int ss, int se, int Al, int *EOBRUN)
 {
     int code, i = ss, j, sign, val, run;
@@ -1383,7 +1383,7 @@ static int mjpeg_decode_scan_progressive_ac(MJpegDecodeContext *s, int ss,
     int mb_x, mb_y;
     int EOBRUN = 0;
     int c = s->comp_index[0];
-    int16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]];
+    uint16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]];
 
     av_assert0(ss>=0 && Ah>=0 && Al>=0);
     if (se < ss || se > 63) {

diff --git a/libavcodec/mjpegdec.h b/libavcodec/mjpegdec.h
index fb811294a146ff40d8d98f37f3fac4badd658e82..024cedcb5ad69915efae93b1c34d92ee07119a26 100644 (file)
--- a/libavcodec/mjpegdec.h
+++ b/libavcodec/mjpegdec.h
@@ -50,7 +50,7 @@ typedef struct MJpegDecodeContext {
     int buffer_size;
     uint8_t *buffer;
 
-    int16_t quant_matrixes[4][64];
+    uint16_t quant_matrixes[4][64];
     VLC vlcs[3][4];
     int qscale[4];      ///< quantizer scale calculated from quant_matrixes