aea: Return proper error code on invalid header

[ffmpeg] / libavcodec / svq3.c

diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index 1643f80fdc4328cb0db49a5a589b004647e0cd79..fc2120b2cb3cb8cb0e487922b1b5c1b6e54b51f6 100644 (file)
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -40,10 +40,12 @@
  *  http://samples.libav.org/V-codecs/SVQ3/Vertical400kbit.sorenson3.mov
  */
 
+#include <inttypes.h>
+
 #include "libavutil/attributes.h"
 #include "internal.h"
 #include "avcodec.h"
-#include "mpegvideo.h"
+#include "mpegutils.h"
 #include "h264.h"
 
 #include "h264data.h" // FIXME FIXME FIXME
@@ -52,6 +54,7 @@
 #include "golomb.h"
 #include "hpeldsp.h"
 #include "rectangle.h"
+#include "tpeldsp.h"
 
 #if CONFIG_ZLIB
 #include <zlib.h>
@@ -68,9 +71,10 @@
 typedef struct {
     H264Context h;
     HpelDSPContext hdsp;
-    Picture *cur_pic;
-    Picture *next_pic;
-    Picture *last_pic;
+    TpelDSPContext tdsp;
+    H264Picture *cur_pic;
+    H264Picture *next_pic;
+    H264Picture *last_pic;
     int halfpel_flag;
     int thirdpel_flag;
     int unknown_flag;
@@ -104,6 +108,13 @@ static const uint8_t svq3_scan[16] = {
     0 + 3 * 4, 1 + 3 * 4, 2 + 3 * 4, 3 + 3 * 4,
 };
 
+static const uint8_t luma_dc_zigzag_scan[16] = {
+    0 * 16 + 0 * 64, 1 * 16 + 0 * 64, 2 * 16 + 0 * 64, 0 * 16 + 2 * 64,
+    3 * 16 + 0 * 64, 0 * 16 + 1 * 64, 1 * 16 + 1 * 64, 2 * 16 + 1 * 64,
+    1 * 16 + 2 * 64, 2 * 16 + 2 * 64, 3 * 16 + 2 * 64, 0 * 16 + 3 * 64,
+    3 * 16 + 1 * 64, 1 * 16 + 3 * 64, 2 * 16 + 3 * 64, 3 * 16 + 3 * 64,
+};
+
 static const uint8_t svq3_pred_0[25][2] = {
     { 0, 0 },
     { 1, 0 }, { 0, 1 },
@@ -284,8 +295,8 @@ static inline void svq3_mc_dir_part(SVQ3Context *s,
                                     int mx, int my, int dxy,
                                     int thirdpel, int dir, int avg)
 {
-    H264Context *h     = &s->h;
-    const Picture *pic = (dir == 0) ? s->last_pic : s->next_pic;
+    H264Context *h = &s->h;
+    const H264Picture *pic = (dir == 0) ? s->last_pic : s->next_pic;
     uint8_t *src, *dest;
     int i, emu = 0;
     int blocksize = 2 - (width >> 3); // 16->0, 8->1, 4->2
@@ -305,15 +316,16 @@ static inline void svq3_mc_dir_part(SVQ3Context *s,
     src  = pic->f.data[0] + mx + my * h->linesize;
 
     if (emu) {
-        h->vdsp.emulated_edge_mc(h->edge_emu_buffer, src, h->linesize,
+        h->vdsp.emulated_edge_mc(h->edge_emu_buffer, src,
+                                 h->linesize, h->linesize,
                                  width + 1, height + 1,
                                  mx, my, s->h_edge_pos, s->v_edge_pos);
         src = h->edge_emu_buffer;
     }
     if (thirdpel)
-        (avg ? h->dsp.avg_tpel_pixels_tab
-             : h->dsp.put_tpel_pixels_tab)[dxy](dest, src, h->linesize,
-                                                width, height);
+        (avg ? s->tdsp.avg_tpel_pixels_tab
+             : s->tdsp.put_tpel_pixels_tab)[dxy](dest, src, h->linesize,
+                                                 width, height);
     else
         (avg ? s->hdsp.avg_pixels_tab
              : s->hdsp.put_pixels_tab)[blocksize][dxy](dest, src, h->linesize,
@@ -331,17 +343,18 @@ static inline void svq3_mc_dir_part(SVQ3Context *s,
             src  = pic->f.data[i] + mx + my * h->uvlinesize;
 
             if (emu) {
-                h->vdsp.emulated_edge_mc(h->edge_emu_buffer, src, h->uvlinesize,
+                h->vdsp.emulated_edge_mc(h->edge_emu_buffer, src,
+                                         h->uvlinesize, h->uvlinesize,
                                          width + 1, height + 1,
                                          mx, my, (s->h_edge_pos >> 1),
                                          s->v_edge_pos >> 1);
                 src = h->edge_emu_buffer;
             }
             if (thirdpel)
-                (avg ? h->dsp.avg_tpel_pixels_tab
-                     : h->dsp.put_tpel_pixels_tab)[dxy](dest, src,
-                                                        h->uvlinesize,
-                                                        width, height);
+                (avg ? s->tdsp.avg_tpel_pixels_tab
+                     : s->tdsp.put_tpel_pixels_tab)[dxy](dest, src,
+                                                         h->uvlinesize,
+                                                         width, height);
             else
                 (avg ? s->hdsp.avg_pixels_tab
                      : s->hdsp.put_pixels_tab)[blocksize][dxy](dest, src,
@@ -605,7 +618,8 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
                 vlc = svq3_get_ue_golomb(&h->gb);
 
                 if (vlc >= 25) {
-                    av_log(h->avctx, AV_LOG_ERROR, "luma prediction:%d\n", vlc);
+                    av_log(h->avctx, AV_LOG_ERROR,
+                           "luma prediction:%"PRIu32"\n", vlc);
                     return -1;
                 }
 
@@ -645,9 +659,9 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
         dir = i_mb_type_info[mb_type - 8].pred_mode;
         dir = (dir >> 1) ^ 3 * (dir & 1) ^ 1;
 
-        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1) {
-            av_log(h->avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n");
-            return -1;
+        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) < 0) {
+            av_log(h->avctx, AV_LOG_ERROR, "ff_h264_check_intra_pred_mode < 0\n");
+            return h->intra16x16_pred_mode;
         }
 
         cbp     = i_mb_type_info[mb_type - 8].cbp;
@@ -674,7 +688,7 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type)
     if (!IS_INTRA16x16(mb_type) &&
         (!IS_SKIP(mb_type) || h->pict_type == AV_PICTURE_TYPE_B)) {
         if ((vlc = svq3_get_ue_golomb(&h->gb)) >= 48) {
-            av_log(h->avctx, AV_LOG_ERROR, "cbp_vlc=%d\n", vlc);
+            av_log(h->avctx, AV_LOG_ERROR, "cbp_vlc=%"PRIu32"\n", vlc);
             return -1;
         }
 
@@ -796,7 +810,7 @@ static int svq3_decode_slice_header(AVCodecContext *avctx)
     }
 
     if ((slice_id = svq3_get_ue_golomb(&h->gb)) >= 3) {
-        av_log(h->avctx, AV_LOG_ERROR, "illegal slice type %d \n", slice_id);
+        av_log(h->avctx, AV_LOG_ERROR, "illegal slice type %u \n", slice_id);
         return -1;
     }
 
@@ -869,10 +883,13 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx)
         return -1;
 
     ff_hpeldsp_init(&s->hdsp, avctx->flags);
+    ff_tpeldsp_init(&s->tdsp);
+
     h->flags           = avctx->flags;
     h->is_complex      = 1;
     h->picture_structure = PICT_FRAME;
-    avctx->pix_fmt     = avctx->codec->pix_fmts[0];
+    avctx->pix_fmt     = AV_PIX_FMT_YUVJ420P;
+    avctx->color_range = AVCOL_RANGE_JPEG;
 
     h->chroma_qp[0] = h->chroma_qp[1] = 4;
     h->chroma_x_shift = h->chroma_y_shift = 1;
@@ -973,11 +990,12 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx)
             int offset                = get_bits_count(&gb) + 7 >> 3;
             uint8_t *buf;
 
-            if ((uint64_t)watermark_width * 4 > UINT_MAX / watermark_height)
+            if (watermark_height > 0 &&
+                (uint64_t)watermark_width * 4 > UINT_MAX / watermark_height)
                 return -1;
 
             buf = av_malloc(buf_len);
-            av_log(avctx, AV_LOG_DEBUG, "watermark size: %dx%d\n",
+            av_log(avctx, AV_LOG_DEBUG, "watermark size: %ux%u\n",
                    watermark_width, watermark_height);
             av_log(avctx, AV_LOG_DEBUG,
                    "u1: %x u2: %x u3: %x compressed data size: %d offset: %d\n",
@@ -992,7 +1010,7 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx)
             s->watermark_key = ff_svq1_packet_checksum(buf, buf_len, 0);
             s->watermark_key = s->watermark_key << 16 | s->watermark_key;
             av_log(avctx, AV_LOG_DEBUG,
-                   "watermark key %#x\n", s->watermark_key);
+                   "watermark key %#"PRIx32"\n", s->watermark_key);
             av_free(buf);
 #else
             av_log(avctx, AV_LOG_ERROR,
@@ -1020,7 +1038,7 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx)
     return 0;
 }
 
-static void free_picture(AVCodecContext *avctx, Picture *pic)
+static void free_picture(AVCodecContext *avctx, H264Picture *pic)
 {
     int i;
     for (i = 0; i < 2; i++) {
@@ -1032,7 +1050,7 @@ static void free_picture(AVCodecContext *avctx, Picture *pic)
     av_frame_unref(&pic->f);
 }
 
-static int get_buffer(AVCodecContext *avctx, Picture *pic)
+static int get_buffer(AVCodecContext *avctx, H264Picture *pic)
 {
     SVQ3Context *s = avctx->priv_data;
     H264Context *h = &s->h;
@@ -1115,7 +1133,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
     h->pict_type = h->slice_type;
 
     if (h->pict_type != AV_PICTURE_TYPE_B)
-        FFSWAP(Picture*, s->next_pic, s->last_pic);
+        FFSWAP(H264Picture*, s->next_pic, s->last_pic);
 
     av_frame_unref(&s->cur_pic->f);
 
@@ -1257,9 +1275,10 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
                     (h->pict_type == AV_PICTURE_TYPE_P && mb_type < 8) ? (mb_type - 1) : -1;
         }
 
-        ff_draw_horiz_band(avctx, NULL, s->cur_pic, s->last_pic->f.data[0] ? s->last_pic : NULL,
-                           16 * h->mb_y, 16, h->picture_structure, 0, 0,
-                           h->low_delay, h->mb_height * 16, h->mb_width * 16);
+        ff_draw_horiz_band(avctx, &s->cur_pic->f,
+                           s->last_pic->f.data[0] ? &s->last_pic->f : NULL,
+                           16 * h->mb_y, 16, h->picture_structure, 0,
+                           h->low_delay);
     }
 
     if (h->pict_type == AV_PICTURE_TYPE_B || h->low_delay)
@@ -1274,7 +1293,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
         *got_frame = 1;
 
     if (h->pict_type != AV_PICTURE_TYPE_B) {
-        FFSWAP(Picture*, s->cur_pic, s->next_pic);
+        FFSWAP(H264Picture*, s->cur_pic, s->next_pic);
     } else {
         av_frame_unref(&s->cur_pic->f);
     }
@@ -1303,6 +1322,7 @@ static av_cold int svq3_decode_end(AVCodecContext *avctx)
 
 AVCodec ff_svq3_decoder = {
     .name           = "svq3",
+    .long_name      = NULL_IF_CONFIG_SMALL("Sorenson Vector Quantizer 3 / Sorenson Video 3 / SVQ3"),
     .type           = AVMEDIA_TYPE_VIDEO,
     .id             = AV_CODEC_ID_SVQ3,
     .priv_data_size = sizeof(SVQ3Context),
@@ -1312,7 +1332,6 @@ AVCodec ff_svq3_decoder = {
     .capabilities   = CODEC_CAP_DRAW_HORIZ_BAND |
                       CODEC_CAP_DR1             |
                       CODEC_CAP_DELAY,
-    .long_name      = NULL_IF_CONFIG_SMALL("Sorenson Vector Quantizer 3 / Sorenson Video 3 / SVQ3"),
     .pix_fmts       = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUVJ420P,
                                                      AV_PIX_FMT_NONE},
 };