avcodec/shorten: Fix multiple integer overflows

[ffmpeg] / libavcodec / mpeg4videodec.c

diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 9ee2f37c6947785c17c75dcf6838aa0ede514c50..51b58fac02f9ddda9573b0ea119a491caa7ca071 100644 (file)
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -46,6 +46,8 @@
 #define MB_TYPE_B_VLC_BITS 4
 #define STUDIO_INTRA_BITS 9
 
+static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb);
+
 static VLC dc_lum, dc_chrom;
 static VLC sprite_trajectory;
 static VLC mb_type_b_vlc;
@@ -2049,6 +2051,23 @@ static int decode_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     /* vol header */
     skip_bits(gb, 1);                   /* random access */
     s->vo_type = get_bits(gb, 8);
+
+    /* If we are in studio profile (per vo_type), check if its all consistent
+     * and if so continue pass control to decode_studio_vol_header().
+     * elIf something is inconsistent, error out
+     * else continue with (non studio) vol header decpoding.
+     */
+    if (s->vo_type == CORE_STUDIO_VO_TYPE ||
+        s->vo_type == SIMPLE_STUDIO_VO_TYPE) {
+        if (s->avctx->profile != FF_PROFILE_UNKNOWN && s->avctx->profile != FF_PROFILE_MPEG4_SIMPLE_STUDIO)
+            return AVERROR_INVALIDDATA;
+        s->studio_profile = 1;
+        s->avctx->profile = FF_PROFILE_MPEG4_SIMPLE_STUDIO;
+        return decode_studio_vol_header(ctx, gb);
+    } else if (s->studio_profile) {
+        return AVERROR_PATCHWELCOME;
+    }
+
     if (get_bits1(gb) != 0) {           /* is_ol_id */
         vo_ver_id = get_bits(gb, 4);    /* vo_ver_id */
         skip_bits(gb, 3);               /* vo_priority */
@@ -2931,9 +2950,6 @@ static int decode_studio_vop_header(Mpeg4DecContext *ctx, GetBitContext *gb)
     if (get_bits_left(gb) <= 32)
         return 0;
 
-    if (get_bits_long(gb, 32) != VOP_STARTCODE)
-        return AVERROR_INVALIDDATA;
-
     s->decode_mb = mpeg4_decode_studio_mb;
 
     decode_smpte_tc(ctx, gb);
@@ -3002,9 +3018,10 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
 {
     MpegEncContext *s = &ctx->m;
     int width, height;
+    int bits_per_raw_sample;
 
-            skip_bits1(gb); /* random_accessible_vol */
-            skip_bits(gb, 8); /* video_object_type_indication */
+            // random_accessible_vol and video_object_type_indication have already
+            // been read by the caller decode_vol_header()
             skip_bits(gb, 4); /* video_object_layer_verid */
             ctx->shape = get_bits(gb, 2); /* video_object_layer_shape */
             skip_bits(gb, 4); /* video_object_layer_shape_extension */
@@ -3017,8 +3034,8 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
                     return AVERROR_INVALIDDATA;
                 }
 
-                s->avctx->bits_per_raw_sample = get_bits(gb, 4); /* bit_depth */
-                if (s->avctx->bits_per_raw_sample == 10) {
+                bits_per_raw_sample = get_bits(gb, 4); /* bit_depth */
+                if (bits_per_raw_sample == 10) {
                     if (ctx->rgb) {
                         s->avctx->pix_fmt = AV_PIX_FMT_GBRP10;
                     }
@@ -3027,9 +3044,10 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
                     }
                 }
                 else {
-                    avpriv_request_sample(s->avctx, "MPEG-4 Studio profile bit-depth %u", s->avctx->bits_per_raw_sample);
+                    avpriv_request_sample(s->avctx, "MPEG-4 Studio profile bit-depth %u", bits_per_raw_sample);
                     return AVERROR_PATCHWELCOME;
                 }
+                s->avctx->bits_per_raw_sample = bits_per_raw_sample;
             }
             if (ctx->shape == RECT_SHAPE) {
                 check_marker(s->avctx, gb, "before video_object_layer_width");
@@ -3180,14 +3198,8 @@ int ff_mpeg4_decode_picture_header(Mpeg4DecContext *ctx, GetBitContext *gb)
                 continue;
             }
             vol++;
-            if (s->studio_profile) {
-                if ((ret = decode_studio_vol_header(ctx, gb)) < 0)
-                    return ret;
-                break;
-            } else {
-                if ((ret = decode_vol_header(ctx, gb)) < 0)
-                    return ret;
-            }
+            if ((ret = decode_vol_header(ctx, gb)) < 0)
+                return ret;
         } else if (startcode == USER_DATA_STARTCODE) {
             decode_user_data(ctx, gb);
         } else if (startcode == GOP_STARTCODE) {
@@ -3219,9 +3231,13 @@ end:
         s->low_delay = 1;
     s->avctx->has_b_frames = !s->low_delay;
 
-    if (s->studio_profile)
+    if (s->studio_profile) {
+        if (!s->avctx->bits_per_raw_sample) {
+            av_log(s->avctx, AV_LOG_ERROR, "Missing VOL header\n");
+            return AVERROR_INVALIDDATA;
+        }
         return decode_studio_vop_header(ctx, gb);
-    else
+    } else
         return decode_vop_header(ctx, gb);
 }