mjpegdec: Properly fail on malloc failure

[ffmpeg] / libavcodec / mpeg12dec.c

diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index 882fdf3d825643daba94364ac1873a8b84c68379..2d9c99d63fa6253855e5c2a1f7f0289abf913468 100644 (file)
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -138,92 +138,6 @@ static int mpeg_decode_motion(MpegEncContext *s, int fcode, int pred)
         }                                                                     \
     } while (0)
 
-static inline int mpeg1_decode_block_intra(MpegEncContext *s,
-                                           int16_t *block, int n)
-{
-    int level, dc, diff, i, j, run;
-    int component;
-    RLTable *rl                  = &ff_rl_mpeg1;
-    uint8_t *const scantable     = s->intra_scantable.permutated;
-    const uint16_t *quant_matrix = s->intra_matrix;
-    const int qscale             = s->qscale;
-
-    /* DC coefficient */
-    component = (n <= 3 ? 0 : n - 4 + 1);
-    diff = decode_dc(&s->gb, component);
-    if (diff >= 0xffff)
-        return AVERROR_INVALIDDATA;
-    dc  = s->last_dc[component];
-    dc += diff;
-    s->last_dc[component] = dc;
-    block[0] = dc * quant_matrix[0];
-    ff_dlog(s->avctx, "dc=%d diff=%d\n", dc, diff);
-    i = 0;
-    {
-        OPEN_READER(re, &s->gb);
-        /* now quantify & encode AC coefficients */
-        for (;;) {
-            UPDATE_CACHE(re, &s->gb);
-            GET_RL_VLC(level, run, re, &s->gb, rl->rl_vlc[0],
-                       TEX_VLC_BITS, 2, 0);
-
-            if (level == 127) {
-                break;
-            } else if (level != 0) {
-                i += run;
-                if (i > MAX_INDEX)
-                    break;
-                j = scantable[i];
-                level = (level * qscale * quant_matrix[j]) >> 4;
-                level = (level - 1) | 1;
-                level = (level ^ SHOW_SBITS(re, &s->gb, 1)) -
-                        SHOW_SBITS(re, &s->gb, 1);
-                LAST_SKIP_BITS(re, &s->gb, 1);
-            } else {
-                /* escape */
-                run = SHOW_UBITS(re, &s->gb, 6) + 1;
-                LAST_SKIP_BITS(re, &s->gb, 6);
-                UPDATE_CACHE(re, &s->gb);
-                level = SHOW_SBITS(re, &s->gb, 8);
-                SKIP_BITS(re, &s->gb, 8);
-                if (level == -128) {
-                    level = SHOW_UBITS(re, &s->gb, 8) - 256;
-                    LAST_SKIP_BITS(re, &s->gb, 8);
-                } else if (level == 0) {
-                    level = SHOW_UBITS(re, &s->gb, 8);
-                    LAST_SKIP_BITS(re, &s->gb, 8);
-                }
-                i += run;
-                if (i > MAX_INDEX)
-                    break;
-                j = scantable[i];
-                if (level < 0) {
-                    level = -level;
-                    level = (level * qscale * quant_matrix[j]) >> 4;
-                    level = (level - 1) | 1;
-                    level = -level;
-                } else {
-                    level = (level * qscale * quant_matrix[j]) >> 4;
-                    level = (level - 1) | 1;
-                }
-            }
-
-            block[j] = level;
-        }
-        CLOSE_READER(re, &s->gb);
-    }
-
-    check_scantable_index(s, i);
-
-    s->block_last_index[n] = i;
-    return 0;
-}
-
-int ff_mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, int n)
-{
-    return mpeg1_decode_block_intra(s, block, n);
-}
-
 static inline int mpeg1_decode_block_inter(MpegEncContext *s,
                                            int16_t *block, int n)
 {
@@ -784,7 +698,7 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
         if (get_bits1(&s->gb) == 0) {
             if (get_bits1(&s->gb) == 0) {
                 av_log(s->avctx, AV_LOG_ERROR,
-                       "invalid mb type in I Frame at %d %d\n",
+                       "Invalid mb type in I-frame at %d %d\n",
                        s->mb_x, s->mb_y);
                 return AVERROR_INVALIDDATA;
             }
@@ -797,7 +711,7 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
         mb_type = get_vlc2(&s->gb, ff_mb_ptype_vlc.table, MB_PTYPE_VLC_BITS, 1);
         if (mb_type < 0) {
             av_log(s->avctx, AV_LOG_ERROR,
-                   "invalid mb type in P Frame at %d %d\n", s->mb_x, s->mb_y);
+                   "Invalid mb type in P-frame at %d %d\n", s->mb_x, s->mb_y);
             return AVERROR_INVALIDDATA;
         }
         mb_type = ptype2mb_type[mb_type];
@@ -806,7 +720,7 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
         mb_type = get_vlc2(&s->gb, ff_mb_btype_vlc.table, MB_BTYPE_VLC_BITS, 1);
         if (mb_type < 0) {
             av_log(s->avctx, AV_LOG_ERROR,
-                   "invalid mb type in B Frame at %d %d\n", s->mb_x, s->mb_y);
+                   "Invalid mb type in B-frame at %d %d\n", s->mb_x, s->mb_y);
             return AVERROR_INVALIDDATA;
         }
         mb_type = btype2mb_type[mb_type];
@@ -867,9 +781,20 @@ FF_ENABLE_DEPRECATION_WARNINGS
                         return ret;
             }
         } else {
-            for (i = 0; i < 6; i++)
-                if ((ret = mpeg1_decode_block_intra(s, *s->pblocks[i], i)) < 0)
+            for (i = 0; i < 6; i++) {
+                ret = ff_mpeg1_decode_block_intra(&s->gb,
+                                                  s->intra_matrix,
+                                                  s->intra_scantable.permutated,
+                                                  s->last_dc, *s->pblocks[i],
+                                                  i, s->qscale);
+                if (ret < 0) {
+                    av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n",
+                           s->mb_x, s->mb_y);
                     return ret;
+                }
+
+                s->block_last_index[i] = ret;
+            }
         }
     } else {
         if (mb_type & MB_TYPE_ZERO_MV) {
@@ -1212,7 +1137,7 @@ static const enum AVPixelFormat mpeg12_hwaccel_pixfmt_list_420[] = {
     AV_PIX_FMT_D3D11VA_VLD,
 #endif
 #if CONFIG_MPEG2_VAAPI_HWACCEL
-    AV_PIX_FMT_VAAPI_VLD,
+    AV_PIX_FMT_VAAPI,
 #endif
 #if CONFIG_MPEG1_VDPAU_HWACCEL | CONFIG_MPEG2_VDPAU_HWACCEL
     AV_PIX_FMT_VDPAU,
@@ -1957,7 +1882,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
                 int i;
                 if (s->pict_type == AV_PICTURE_TYPE_I) {
                     av_log(s->avctx, AV_LOG_ERROR,
-                           "skipped MB in I frame at %d %d\n", s->mb_x, s->mb_y);
+                           "skipped MB in I-frame at %d %d\n", s->mb_x, s->mb_y);
                     return AVERROR_INVALIDDATA;
                 }
 
@@ -2258,7 +2183,7 @@ static int mpeg_decode_a53_cc(AVCodecContext *avctx,
         int cc_count = 0;
         int i;
         // There is a caption count field in the data, but it is often
-        // incorect.  So count the number of captions present.
+        // incorrect.  So count the number of captions present.
         for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
             cc_count++;
         // Transform the DVD format into A53 Part 4 format