libavcodec/decode: avoid UB when getting plane sizes

[ffmpeg] / libavcodec / mv30.c

diff --git a/libavcodec/mv30.c b/libavcodec/mv30.c
index ffa04ac493d4f9c1efec0afed18ed19077333e01..c83ba7ffbdb79794ae5be853491221b3b1e7b6dc 100644 (file)
--- a/libavcodec/mv30.c
+++ b/libavcodec/mv30.c
@@ -33,6 +33,7 @@
 #include "blockdsp.h"
 #include "get_bits.h"
 #include "internal.h"
+#include "aandcttab.h"
 
 typedef struct MV30Context {
     GetBitContext  gb;
@@ -58,25 +59,6 @@ typedef struct MV30Context {
 
 static VLC cbp_tab;
 
-static const int16_t scale_tab[] = {
-    16384,  22725,  21407,  19266,
-    16384,  12873,   8867,   4520,
-    22725,  31521,  29692,  26722,
-    22725,  17855,  12299,   6270,
-    21407,  29692,  27969,  25172,
-    21407,  16819,  11585,   5906,
-    19266,  26722,  25172,  22654,
-    19266,  15137,  10426,   5315,
-    16384,  22725,  21407,  19266,
-    16384,  12873,   8867,   4520,
-    12873,  17855,  16819,  15137,
-    12873,  10114,   6967,   3552,
-     8867,  12299,  11585,  10426,
-     8867,   6967,   4799,   2446,
-     4520,   6270,   5906,   5315,
-     4520,   3552,   2446,   1247,
-};
-
 static const uint8_t luma_tab[] = {
     12, 12, 15, 19, 25, 34, 40, 48,
     12, 12, 18, 22, 27, 44, 47, 46,
@@ -86,14 +68,6 @@ static const uint8_t luma_tab[] = {
     25, 31, 42, 48, 58, 72, 81, 75,
     38, 46, 54, 61, 71, 84, 88, 85,
     50, 61, 65, 68, 79, 78, 86, 91,
-    12, 12, 16, 18, 20, 30, 40, 45,
-    12, 12, 16, 18, 30, 40, 45, 50,
-    16, 16, 20, 30, 40, 45, 50, 55,
-    18, 18, 35, 40, 45, 50, 55, 60,
-    20, 30, 40, 45, 50, 55, 60, 65,
-    30, 40, 45, 50, 55, 60, 65, 70,
-    40, 45, 50, 55, 60, 65, 70, 75,
-    45, 50, 55, 60, 65, 70, 75, 80,
 };
 
 static const uint8_t chroma_tab[] = {
@@ -105,14 +79,6 @@ static const uint8_t chroma_tab[] = {
     99, 99, 99, 99, 99, 99, 99, 99,
     99, 99, 99, 99, 99, 99, 99, 99,
     99, 99, 99, 99, 99, 99, 99, 99,
-    12, 16, 20, 24, 28, 36, 40, 44,
-    16, 20, 24, 28, 36, 40, 44, 50,
-    20, 24, 28, 36, 40, 44, 50, 60,
-    24, 28, 36, 40, 44, 50, 60, 80,
-    28, 36, 40, 44, 50, 60, 80, 99,
-    36, 40, 44, 50, 60, 80, 99, 99,
-    40, 44, 50, 60, 80, 99, 99, 99,
-    44, 50, 60, 80, 99, 99, 99, 99,
 };
 
 static const uint8_t zigzag[] = {
@@ -132,7 +98,7 @@ static void get_qtable(int16_t *table, int quant, const uint8_t *quant_tab)
 
     for (int i = 0; i < 64; i++) {
         table[i] = av_clip((quant_tab[i] * factor + 0x32) / 100, 1, 0x7fff);
-        table[i] = ((int)scale_tab[i] * (int)table[i] + 0x800) >> 12;
+        table[i] = ((int)ff_aanscales[i] * (int)table[i] + 0x800) >> 12;
     }
 }
 
@@ -141,7 +107,7 @@ static inline void idct_1d(int *blk, int step)
     const int t0 = blk[0 * step] + blk[4 * step];
     const int t1 = blk[0 * step] - blk[4 * step];
     const int t2 = blk[2 * step] + blk[6 * step];
-    const int t3 = (((blk[2 * step] - blk[6 * step]) * 362) >> 8) - t2;
+    const int t3 = ((int)((blk[2 * step] - blk[6 * step]) * 362U) >> 8) - t2;
     const int t4 = t0 + t2;
     const int t5 = t0 - t2;
     const int t6 = t1 + t3;
@@ -151,10 +117,10 @@ static inline void idct_1d(int *blk, int step)
     const int tA = blk[1 * step] + blk[7 * step];
     const int tB = blk[1 * step] - blk[7 * step];
     const int tC = t8 + tA;
-    const int tD = (tB + t9) * 473 >> 8;
-    const int tE = ((t9 * -669 >> 8) - tC) + tD;
-    const int tF = ((tA - t8) * 362 >> 8) - tE;
-    const int t10 = ((tB * 277 >> 8) - tD) + tF;
+    const int tD = (int)((tB + t9) * 473U) >> 8;
+    const int tE = (((int)(t9 * -669U) >> 8) - tC) + tD;
+    const int tF = ((int)((tA - t8) * 362U) >> 8) - tE;
+    const int t10 = (((int)(tB * 277U) >> 8) - tD) + tF;
 
     blk[0 * step] = t4 + tC;
     blk[1 * step] = t6 + tE;
@@ -234,10 +200,10 @@ static inline void idct2_1d(int *blk, int step)
 {
     const int t0 = blk[0 * step];
     const int t1 = blk[1 * step];
-    const int t2 = t1 * 473 >> 8;
+    const int t2 = (int)(t1 * 473U) >> 8;
     const int t3 = t2 - t1;
-    const int t4 = (t1 * 362 >> 8) - t3;
-    const int t5 = ((t1 * 277 >> 8) - t2) + t4;
+    const int t4 =  ((int)(t1 * 362U) >> 8) - t3;
+    const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
 
     blk[0 * step] = t1 + t0;
     blk[1 * step] = t0 + t3;
@@ -444,6 +410,9 @@ static int decode_intra(AVCodecContext *avctx, GetBitContext *gb, AVFrame *frame
     int ret;
 
     mgb = *gb;
+    if (get_bits_left(gb) < s->mode_size * 8)
+        return AVERROR_INVALIDDATA;
+
     skip_bits_long(gb, s->mode_size * 8);
 
     linesize[0] = frame->linesize[0];
@@ -455,7 +424,7 @@ static int decode_intra(AVCodecContext *avctx, GetBitContext *gb, AVFrame *frame
 
     for (int y = 0; y < avctx->height; y += 16) {
         GetByteContext gbyte;
-        int pfill[3][1] = { 0 };
+        int pfill[3][1] = { {0} };
         int nb_codes = get_bits(gb, 16);
 
         av_fast_padded_malloc(&s->coeffs, &s->coeffs_size, nb_codes * sizeof(*s->coeffs));
@@ -538,7 +507,7 @@ static int decode_inter(AVCodecContext *avctx, GetBitContext *gb,
 
     for (int y = 0; y < avctx->height; y += 16) {
         GetByteContext gbyte;
-        int pfill[3][1] = { 0 };
+        int pfill[3][1] = { {0} };
         int nb_codes = get_bits(gb, 16);
 
         skip_bits(gb, 8);
@@ -579,8 +548,8 @@ static int decode_inter(AVCodecContext *avctx, GetBitContext *gb,
                 int px = x + mv_x;
                 int py = y + mv_y;
 
-                if (px < 0 || px >= avctx->width ||
-                    py < 0 || py >= avctx->height)
+                if (px < 0 || px > FFALIGN(avctx->width , 16) - 16 ||
+                    py < 0 || py > FFALIGN(avctx->height, 16) - 16)
                     return AVERROR_INVALIDDATA;
 
                 src[0] = prev->data[0] + in_linesize[0] * py + px;