]> git.sesse.net Git - ffmpeg/blobdiff - libavcodec/vc1.c
libopusdec: fix out-of-bounds read
[ffmpeg] / libavcodec / vc1.c
index b467e4601e335d9d07abd6248ede853b3a288264..7a93e971bce6627fa1e42daa04bb1e3b2cbec3f0 100644 (file)
 /**
  * @file
  * VC-1 and WMV3 decoder common code
- *
  */
 
+#include "libavutil/attributes.h"
 #include "internal.h"
 #include "avcodec.h"
 #include "mpegvideo.h"
 #include "vc1.h"
 #include "vc1data.h"
-#include "msmpeg4data.h"
+#include "wmv2data.h"
 #include "unary.h"
 #include "simple_idct.h"
 
-#undef NDEBUG
-#include <assert.h>
-
 /***********************************************************************/
 /**
  * @name VC-1 Bitplane decoding
@@ -366,7 +363,7 @@ int ff_vc1_decode_sequence_header(AVCodecContext *avctx, VC1Context *v, GetBitCo
 
     v->overlap         = get_bits1(gb); //common
 
-    v->s.resync_marker = get_bits1(gb);
+    v->resync_marker   = get_bits1(gb);
     v->rangered        = get_bits1(gb);
     if (v->rangered && v->profile == PROFILE_SIMPLE) {
         av_log(avctx, AV_LOG_INFO,
@@ -404,10 +401,10 @@ int ff_vc1_decode_sequence_header(AVCodecContext *avctx, VC1Context *v, GetBitCo
            "Profile %i:\nfrmrtq_postproc=%i, bitrtq_postproc=%i\n"
            "LoopFilter=%i, MultiRes=%i, FastUVMC=%i, Extended MV=%i\n"
            "Rangered=%i, VSTransform=%i, Overlap=%i, SyncMarker=%i\n"
-           "DQuant=%i, Quantizer mode=%i, Max B frames=%i\n",
+           "DQuant=%i, Quantizer mode=%i, Max B-frames=%i\n",
            v->profile, v->frmrtq_postproc, v->bitrtq_postproc,
            v->s.loop_filter, v->multires, v->fastuvmc, v->extended_mv,
-           v->rangered, v->vstransform, v->overlap, v->s.resync_marker,
+           v->rangered, v->vstransform, v->overlap, v->resync_marker,
            v->dquant, v->quantizer_mode, avctx->max_b_frames);
     return 0;
 }
@@ -477,25 +474,26 @@ static int decode_sequence_header_adv(VC1Context *v, GetBitContext *gb)
                       v->s.avctx->width * h,
                       1 << 30);
         }
+        ff_set_sar(v->s.avctx, v->s.avctx->sample_aspect_ratio);
         av_log(v->s.avctx, AV_LOG_DEBUG, "Aspect: %i:%i\n",
                v->s.avctx->sample_aspect_ratio.num,
                v->s.avctx->sample_aspect_ratio.den);
 
         if (get_bits1(gb)) { //framerate stuff
             if (get_bits1(gb)) {
-                v->s.avctx->time_base.num = 32;
-                v->s.avctx->time_base.den = get_bits(gb, 16) + 1;
+                v->s.avctx->framerate.den = 32;
+                v->s.avctx->framerate.num = get_bits(gb, 16) + 1;
             } else {
                 int nr, dr;
                 nr = get_bits(gb, 8);
                 dr = get_bits(gb, 4);
                 if (nr > 0 && nr < 8 && dr > 0 && dr < 3) {
-                    v->s.avctx->time_base.num = ff_vc1_fps_dr[dr - 1];
-                    v->s.avctx->time_base.den = ff_vc1_fps_nr[nr - 1] * 1000;
+                    v->s.avctx->framerate.den = ff_vc1_fps_dr[dr - 1];
+                    v->s.avctx->framerate.num = ff_vc1_fps_nr[nr - 1] * 1000;
                 }
             }
             if (v->broadcast) { // Pulldown may be present
-                v->s.avctx->time_base.den  *= 2;
+                v->s.avctx->framerate.num  *= 2;
                 v->s.avctx->ticks_per_frame = 2;
             }
         }
@@ -569,6 +567,56 @@ int ff_vc1_decode_entry_point(AVCodecContext *avctx, VC1Context *v, GetBitContex
     return 0;
 }
 
+/* fill lookup tables for intensity compensation */
+#define INIT_LUT(lumscale, lumshift, luty, lutuv, chain) do {                 \
+        int scale, shift, i;                                                  \
+        if (!lumscale) {                                                      \
+            scale = -64;                                                      \
+            shift = (255 - lumshift * 2) << 6;                                \
+            if (lumshift > 31)                                                \
+                shift += 128 << 6;                                            \
+        } else {                                                              \
+            scale = lumscale + 32;                                            \
+            if (lumshift > 31)                                                \
+                shift = (lumshift - 64) << 6;                                 \
+            else                                                              \
+                shift = lumshift << 6;                                        \
+        }                                                                     \
+        for (i = 0; i < 256; i++) {                                           \
+            int iy = chain ? luty[i]  : i;                                    \
+            int iu = chain ? lutuv[i] : i;                                    \
+            luty[i]  = av_clip_uint8((scale * iy + shift + 32) >> 6);         \
+            lutuv[i] = av_clip_uint8((scale * (iu - 128) + 128*64 + 32) >> 6);\
+        }                                                                     \
+    } while(0)
+
+static void rotate_luts(VC1Context *v)
+{
+#define ROTATE(DEF, L, N, C, A) do {                          \
+        if (v->s.pict_type == AV_PICTURE_TYPE_BI || v->s.pict_type == AV_PICTURE_TYPE_B) { \
+            C = A;                                            \
+        } else {                                              \
+            DEF;                                              \
+            memcpy(&tmp, &L  , sizeof(tmp));                  \
+            memcpy(&L  , &N  , sizeof(tmp));                  \
+            memcpy(&N  , &tmp, sizeof(tmp));                  \
+            C = N;                                            \
+        }                                                     \
+    } while(0)
+
+    ROTATE(int tmp,             v->last_use_ic, v->next_use_ic, v->curr_use_ic, v->aux_use_ic);
+    ROTATE(uint8_t tmp[2][256], v->last_luty,   v->next_luty,   v->curr_luty,   v->aux_luty);
+    ROTATE(uint8_t tmp[2][256], v->last_lutuv,  v->next_lutuv,  v->curr_lutuv,  v->aux_lutuv);
+
+    INIT_LUT(32, 0, v->curr_luty[0], v->curr_lutuv[0], 0);
+    INIT_LUT(32, 0, v->curr_luty[1], v->curr_lutuv[1], 0);
+    v->curr_use_ic = 0;
+    if (v->curr_luty == v->next_luty) {
+        // If we just initialized next_lut, clear next_use_ic to match.
+        v->next_use_ic = 0;
+    }
+}
+
 int ff_vc1_parse_frame_header(VC1Context *v, GetBitContext* gb)
 {
     int pqindex, lowquant, status;
@@ -651,12 +699,12 @@ int ff_vc1_parse_frame_header(VC1Context *v, GetBitContext* gb)
         v->x8_type = get_bits1(gb);
     } else
         v->x8_type = 0;
-    av_dlog(v->s.avctx, "%c Frame: QP=[%i]%i (+%i/2) %i\n",
+    ff_dlog(v->s.avctx, "%c Frame: QP=[%i]%i (+%i/2) %i\n",
             (v->s.pict_type == AV_PICTURE_TYPE_P) ? 'P' : ((v->s.pict_type == AV_PICTURE_TYPE_I) ? 'I' : 'B'),
             pqindex, v->pq, v->halfpq, v->rangeredfrm);
 
-    if (v->s.pict_type == AV_PICTURE_TYPE_I || v->s.pict_type == AV_PICTURE_TYPE_P)
-        v->use_ic = 0;
+    if (v->first_pic_header_flag)
+        rotate_luts(v);
 
     switch (v->s.pict_type) {
     case AV_PICTURE_TYPE_P:
@@ -667,28 +715,13 @@ int ff_vc1_parse_frame_header(VC1Context *v, GetBitContext* gb)
         lowquant = (v->pq > 12) ? 0 : 1;
         v->mv_mode = ff_vc1_mv_pmode_table[lowquant][get_unary(gb, 1, 4)];
         if (v->mv_mode == MV_PMODE_INTENSITY_COMP) {
-            int scale, shift, i;
             v->mv_mode2 = ff_vc1_mv_pmode_table2[lowquant][get_unary(gb, 1, 3)];
             v->lumscale = get_bits(gb, 6);
             v->lumshift = get_bits(gb, 6);
-            v->use_ic   = 1;
+            v->last_use_ic = 1;
             /* fill lookup tables for intensity compensation */
-            if (!v->lumscale) {
-                scale = -64;
-                shift = (255 - v->lumshift * 2) << 6;
-                if (v->lumshift > 31)
-                    shift += 128 << 6;
-            } else {
-                scale = v->lumscale + 32;
-                if (v->lumshift > 31)
-                    shift = (v->lumshift - 64) << 6;
-                else
-                    shift = v->lumshift << 6;
-            }
-            for (i = 0; i < 256; i++) {
-                v->luty[i]  = av_clip_uint8((scale * i + shift + 32) >> 6);
-                v->lutuv[i] = av_clip_uint8((scale * (i - 128) + 128*64 + 32) >> 6);
-            }
+            INIT_LUT(v->lumscale, v->lumshift, v->last_luty[0], v->last_lutuv[0], 1);
+            INIT_LUT(v->lumscale, v->lumshift, v->last_luty[1], v->last_lutuv[1], 1);
         }
         v->qs_last = v->s.quarter_sample;
         if (v->mv_mode == MV_PMODE_1MV_HPEL || v->mv_mode == MV_PMODE_1MV_HPEL_BILIN)
@@ -720,7 +753,7 @@ int ff_vc1_parse_frame_header(VC1Context *v, GetBitContext* gb)
         av_log(v->s.avctx, AV_LOG_DEBUG, "MB Skip plane encoding: "
                "Imode: %i, Invert: %i\n", status>>1, status&1);
 
-        /* Hopefully this is correct for P frames */
+        /* Hopefully this is correct for P-frames */
         v->s.mv_table_index = get_bits(gb, 2); //but using ff_vc1_ tables
         v->cbpcy_vlc = &ff_vc1_cbpcy_p_vlc[get_bits(gb, 2)];
 
@@ -799,62 +832,47 @@ int ff_vc1_parse_frame_header(VC1Context *v, GetBitContext* gb)
     return 0;
 }
 
-/* fill lookup tables for intensity compensation */
-#define INIT_LUT(lumscale, lumshift, luty, lutuv)   \
-    if (!lumscale) {                                \
-        scale = -64;                                \
-        shift = (255 - lumshift * 2) << 6;          \
-        if (lumshift > 31)                          \
-            shift += 128 << 6;                      \
-    } else {                                        \
-        scale = lumscale + 32;                      \
-        if (lumshift > 31)                          \
-            shift = (lumshift - 64) << 6;           \
-        else                                        \
-            shift = lumshift << 6;                  \
-    }                                               \
-    for (i = 0; i < 256; i++) {                     \
-        luty[i]  = av_clip_uint8((scale * i + shift + 32) >> 6);           \
-        lutuv[i] = av_clip_uint8((scale * (i - 128) + 128*64 + 32) >> 6);  \
-    }
-
 int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
 {
     int pqindex, lowquant;
     int status;
     int mbmodetab, imvtab, icbptab, twomvbptab, fourmvbptab; /* useful only for debugging */
-    int scale, shift, i; /* for initializing LUT for intensity compensation */
+    int field_mode, fcm;
 
+    v->numref          = 0;
     v->p_frame_skipped = 0;
     if (v->second_field) {
         v->s.pict_type = (v->fptype & 1) ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_I;
         if (v->fptype & 4)
             v->s.pict_type = (v->fptype & 1) ? AV_PICTURE_TYPE_BI : AV_PICTURE_TYPE_B;
-        v->s.current_picture_ptr->f.pict_type = v->s.pict_type;
+        v->s.current_picture_ptr->f->pict_type = v->s.pict_type;
         if (!v->pic_header_flag)
             goto parse_common_info;
     }
 
-    v->field_mode = 0;
+    field_mode = 0;
     if (v->interlace) {
-        v->fcm = decode012(gb);
-        if (v->fcm) {
-            if (v->fcm == ILACE_FIELD)
-                v->field_mode = 1;
-            if (!v->warn_interlaced++)
-                av_log(v->s.avctx, AV_LOG_ERROR,
-                       "Interlaced frames/fields support is incomplete\n");
+        fcm = decode012(gb);
+        if (fcm) {
+            if (fcm == ILACE_FIELD)
+                field_mode = 1;
         }
     } else {
-        v->fcm = PROGRESSIVE;
+        fcm = PROGRESSIVE;
     }
+    if (!v->first_pic_header_flag && v->field_mode != field_mode)
+        return AVERROR_INVALIDDATA;
+    v->field_mode = field_mode;
+    v->fcm = fcm;
 
     if (v->field_mode) {
+        v->s.mb_height = FFALIGN(v->s.height + 15 >> 4, 2);
         v->fptype = get_bits(gb, 3);
         v->s.pict_type = (v->fptype & 2) ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_I;
         if (v->fptype & 4) // B-picture
             v->s.pict_type = (v->fptype & 2) ? AV_PICTURE_TYPE_BI : AV_PICTURE_TYPE_B;
     } else {
+        v->s.mb_height = v->s.height + 15 >> 4;
         switch (get_unary(gb, 0, 4)) {
         case 0:
             v->s.pict_type = AV_PICTURE_TYPE_P;
@@ -885,7 +903,7 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
         }
     }
     if (v->panscanflag) {
-        av_log_missing_feature(v->s.avctx, "Pan-scan", 0);
+        avpriv_report_missing_feature(v->s.avctx, "Pan-scan");
         //...
     }
     if (v->p_frame_skipped) {
@@ -951,12 +969,12 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
     if (v->postprocflag)
         v->postproc = get_bits(gb, 2);
 
-    if (v->s.pict_type == AV_PICTURE_TYPE_I || v->s.pict_type == AV_PICTURE_TYPE_P)
-        v->use_ic = 0;
-
     if (v->parse_only)
         return 0;
 
+    if (v->first_pic_header_flag)
+        rotate_luts(v);
+
     switch (v->s.pict_type) {
     case AV_PICTURE_TYPE_I:
     case AV_PICTURE_TYPE_BI:
@@ -1007,7 +1025,9 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
                 if (v->intcomp) {
                     v->lumscale = get_bits(gb, 6);
                     v->lumshift = get_bits(gb, 6);
-                    INIT_LUT(v->lumscale, v->lumshift, v->luty, v->lutuv);
+                    INIT_LUT(v->lumscale, v->lumshift, v->last_luty[0], v->last_lutuv[0], 1);
+                    INIT_LUT(v->lumscale, v->lumshift, v->last_luty[1], v->last_lutuv[1], 1);
+                    v->last_use_ic = 1;
                 }
                 status = bitplane_decoding(v->s.mbskip_table, &v->skip_is_raw, v);
                 av_log(v->s.avctx, AV_LOG_DEBUG, "SKIPMB plane encoding: "
@@ -1050,17 +1070,38 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
                 int mvmode2;
                 mvmode2 = get_unary(gb, 1, 3);
                 v->mv_mode2 = ff_vc1_mv_pmode_table2[lowquant][mvmode2];
-                if (v->field_mode)
-                    v->intcompfield = decode210(gb);
-                v->lumscale = get_bits(gb, 6);
-                v->lumshift = get_bits(gb, 6);
-                INIT_LUT(v->lumscale, v->lumshift, v->luty, v->lutuv);
-                if ((v->field_mode) && !v->intcompfield) {
+                if (v->field_mode) {
+                    v->intcompfield = decode210(gb) ^ 3;
+                } else
+                    v->intcompfield = 3;
+
+                v->lumscale2 = v->lumscale = 32;
+                v->lumshift2 = v->lumshift =  0;
+                if (v->intcompfield & 1) {
+                    v->lumscale = get_bits(gb, 6);
+                    v->lumshift = get_bits(gb, 6);
+                }
+                if ((v->intcompfield & 2) && v->field_mode) {
                     v->lumscale2 = get_bits(gb, 6);
                     v->lumshift2 = get_bits(gb, 6);
-                    INIT_LUT(v->lumscale2, v->lumshift2, v->luty2, v->lutuv2);
+                } else if(!v->field_mode) {
+                    v->lumscale2 = v->lumscale;
+                    v->lumshift2 = v->lumshift;
                 }
-                v->use_ic = 1;
+                if (v->field_mode && v->second_field) {
+                    if (v->cur_field_type) {
+                        INIT_LUT(v->lumscale , v->lumshift , v->curr_luty[v->cur_field_type^1], v->curr_lutuv[v->cur_field_type^1], 0);
+                        INIT_LUT(v->lumscale2, v->lumshift2, v->last_luty[v->cur_field_type  ], v->last_lutuv[v->cur_field_type  ], 1);
+                    } else {
+                        INIT_LUT(v->lumscale2, v->lumshift2, v->curr_luty[v->cur_field_type^1], v->curr_lutuv[v->cur_field_type^1], 0);
+                        INIT_LUT(v->lumscale , v->lumshift , v->last_luty[v->cur_field_type  ], v->last_lutuv[v->cur_field_type  ], 1);
+                    }
+                    v->next_use_ic = v->curr_use_ic = 1;
+                } else {
+                    INIT_LUT(v->lumscale , v->lumshift , v->last_luty[0], v->last_lutuv[0], 1);
+                    INIT_LUT(v->lumscale2, v->lumshift2, v->last_luty[1], v->last_lutuv[1], 1);
+                }
+                v->last_use_ic = 1;
             }
             v->qs_last = v->s.quarter_sample;
             if (v->mv_mode == MV_PMODE_1MV_HPEL || v->mv_mode == MV_PMODE_1MV_HPEL_BILIN)
@@ -1095,7 +1136,7 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
             av_log(v->s.avctx, AV_LOG_DEBUG, "MB Skip plane encoding: "
                    "Imode: %i, Invert: %i\n", status>>1, status&1);
 
-            /* Hopefully this is correct for P frames */
+            /* Hopefully this is correct for P-frames */
             v->s.mv_table_index = get_bits(gb, 2); //but using ff_vc1_ tables
             v->cbpcy_vlc        = &ff_vc1_cbpcy_p_vlc[get_bits(gb, 2)];
         } else if (v->fcm == ILACE_FRAME) { // frame interlaced
@@ -1137,9 +1178,13 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
         }
         break;
     case AV_PICTURE_TYPE_B:
-        // TODO: implement interlaced frame B picture decoding
-        if (v->fcm == ILACE_FRAME)
-            return -1;
+        if (v->fcm == ILACE_FRAME) {
+            v->bfraction_lut_index = get_vlc2(gb, ff_vc1_bfraction_vlc.table, VC1_BFRACTION_VLC_BITS, 1);
+            v->bfraction           = ff_vc1_bfraction_lut[v->bfraction_lut_index];
+            if (v->bfraction == 0) {
+                return -1;
+            }
+        }
         if (v->extended_mv)
             v->mvrange = get_unary(gb, 0, 3);
         else
@@ -1185,6 +1230,38 @@ int ff_vc1_parse_frame_header_adv(VC1Context *v, GetBitContext* gb)
                 v->fourmvbp_vlc = &ff_vc1_4mv_block_pattern_vlc[fourmvbptab];
             }
             v->numref = 1; // interlaced field B pictures are always 2-ref
+        } else if (v->fcm == ILACE_FRAME) {
+            if (v->extended_dmv)
+                v->dmvrange = get_unary(gb, 0, 3);
+            if (get_bits1(gb)) /* intcomp - present but shall always be 0 */
+                av_log(v->s.avctx, AV_LOG_WARNING, "Intensity compensation set for B picture\n");
+            v->intcomp          = 0;
+            v->mv_mode          = MV_PMODE_1MV;
+            v->fourmvswitch     = 0;
+            v->qs_last          = v->s.quarter_sample;
+            v->s.quarter_sample = 1;
+            v->s.mspel          = 1;
+            status              = bitplane_decoding(v->direct_mb_plane, &v->dmb_is_raw, v);
+            if (status < 0)
+                return -1;
+            av_log(v->s.avctx, AV_LOG_DEBUG, "MB Direct Type plane encoding: "
+                   "Imode: %i, Invert: %i\n", status>>1, status&1);
+            status = bitplane_decoding(v->s.mbskip_table, &v->skip_is_raw, v);
+            if (status < 0)
+                return -1;
+            av_log(v->s.avctx, AV_LOG_DEBUG, "MB Skip plane encoding: "
+                   "Imode: %i, Invert: %i\n", status>>1, status&1);
+            mbmodetab       = get_bits(gb, 2);
+            v->mbmode_vlc   = &ff_vc1_intfr_non4mv_mbmode_vlc[mbmodetab];
+            imvtab          = get_bits(gb, 2);
+            v->imv_vlc      = &ff_vc1_1ref_mvdata_vlc[imvtab];
+            // interlaced p/b-picture cbpcy range is [1, 63]
+            icbptab         = get_bits(gb, 3);
+            v->cbpcy_vlc    = &ff_vc1_icbpcy_vlc[icbptab];
+            twomvbptab      = get_bits(gb, 2);
+            v->twomvbp_vlc  = &ff_vc1_2mv_block_pattern_vlc[twomvbptab];
+            fourmvbptab     = get_bits(gb, 2);
+            v->fourmvbp_vlc = &ff_vc1_4mv_block_pattern_vlc[fourmvbptab];
         } else {
             v->mv_mode          = get_bits1(gb) ? MV_PMODE_1MV : MV_PMODE_1MV_HPEL_BILIN;
             v->qs_last          = v->s.quarter_sample;
@@ -1484,7 +1561,7 @@ static const uint16_t vlc_offs[] = {
  * @param v The VC1Context to initialize
  * @return Status
  */
-int ff_vc1_init_common(VC1Context *v)
+av_cold int ff_vc1_init_common(VC1Context *v)
 {
     static int done = 0;
     int i = 0;
@@ -1607,5 +1684,7 @@ int ff_vc1_init_common(VC1Context *v)
     v->pq      = -1;
     v->mvrange = 0; /* 7.1.1.18, p80 */
 
+    ff_vc1dsp_init(&v->vc1dsp);
+
     return 0;
 }