fraps: Make the input buffer size checks more strict

[ffmpeg] / libavcodec / xxan.c

diff --git a/libavcodec/xxan.c b/libavcodec/xxan.c
index 575cc34bd1d7362d9261a48762b290a7c8815fb6..2bc9ff697b6905da57cb22a91349d4f27460e043 100644 (file)
--- a/libavcodec/xxan.c
+++ b/libavcodec/xxan.c
@@ -22,11 +22,11 @@
 
 #include "avcodec.h"
 #include "libavutil/intreadwrite.h"
+#include "libavutil/mem.h"
 #include "bytestream.h"
 #define BITSTREAM_READER_LE
 #include "get_bits.h"
-// for av_memcpy_backptr
-#include "libavutil/lzo.h"
+#include "internal.h"
 
 typedef struct XanContext {
     AVCodecContext *avctx;
@@ -44,7 +44,12 @@ static av_cold int xan_decode_init(AVCodecContext *avctx)
 
     s->avctx = avctx;
 
-    avctx->pix_fmt = PIX_FMT_YUV420P;
+    avctx->pix_fmt = AV_PIX_FMT_YUV420P;
+
+    if (avctx->height < 8) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid frame height: %d.\n", avctx->height);
+        return AVERROR(EINVAL);
+    }
 
     s->buffer_size = avctx->width * avctx->height;
     s->y_buffer = av_malloc(s->buffer_size);
@@ -134,7 +139,7 @@ static int xan_unpack(XanContext *s,
             }
             if (dest + size + size2 > dest_end ||
                 dest - orig_dest + size < back)
-                return -1;
+                return AVERROR_INVALIDDATA;
             bytestream2_get_buffer(&s->gb, dest, size);
             dest += size;
             av_memcpy_backptr(dest, back, size2);
@@ -144,7 +149,7 @@ static int xan_unpack(XanContext *s,
 
             size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4;
             if (dest_end - dest < size)
-                return -1;
+                return AVERROR_INVALIDDATA;
             bytestream2_get_buffer(&s->gb, dest, size);
             dest += size;
             if (finish)
@@ -168,7 +173,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
         return 0;
     if (chroma_off + 4 >= bytestream2_get_bytes_left(&s->gb)) {
         av_log(avctx, AV_LOG_ERROR, "Invalid chroma block position\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     bytestream2_seek(&s->gb, chroma_off + 4, SEEK_SET);
     mode        = bytestream2_get_le16(&s->gb);
@@ -179,7 +184,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
 
     if (offset >= bytestream2_get_bytes_left(&s->gb)) {
         av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     bytestream2_skip(&s->gb, offset);
@@ -187,7 +192,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
     dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
     if (dec_size < 0) {
         av_log(avctx, AV_LOG_ERROR, "Chroma unpacking failed\n");
-        return -1;
+        return dec_size;
     }
 
     U = s->pic.data[1];
@@ -211,6 +216,10 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
             U += s->pic.linesize[1];
             V += s->pic.linesize[2];
         }
+        if (avctx->height & 1) {
+            memcpy(U, U - s->pic.linesize[1], avctx->width >> 1);
+            memcpy(V, V - s->pic.linesize[2], avctx->width >> 1);
+        }
     } else {
         uint8_t *U2 = U + s->pic.linesize[1];
         uint8_t *V2 = V + s->pic.linesize[2];
@@ -231,6 +240,12 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off)
             U2 += s->pic.linesize[1] * 2;
             V2 += s->pic.linesize[2] * 2;
         }
+        if (avctx->height & 3) {
+            int lines = ((avctx->height + 1) >> 1) - (avctx->height >> 2) * 2;
+
+            memcpy(U, U - lines * s->pic.linesize[1], lines * s->pic.linesize[1]);
+            memcpy(V, V - lines * s->pic.linesize[2], lines * s->pic.linesize[2]);
+        }
     }
 
     return 0;
@@ -293,7 +308,7 @@ static int xan_decode_frame_type0(AVCodecContext *avctx)
         int dec_size;
 
         bytestream2_seek(&s->gb, 8 + corr_off, SEEK_SET);
-        dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size);
+        dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size / 2);
         if (dec_size < 0)
             dec_size = 0;
         for (i = 0; i < dec_size; i++)
@@ -358,18 +373,14 @@ static int xan_decode_frame_type1(AVCodecContext *avctx)
 }
 
 static int xan_decode_frame(AVCodecContext *avctx,
-                            void *data, int *data_size,
+                            void *data, int *got_frame,
                             AVPacket *avpkt)
 {
     XanContext *s = avctx->priv_data;
     int ftype;
     int ret;
 
-    s->pic.reference = 1;
-    s->pic.buffer_hints = FF_BUFFER_HINTS_VALID |
-                          FF_BUFFER_HINTS_PRESERVE |
-                          FF_BUFFER_HINTS_REUSABLE;
-    if ((ret = avctx->reget_buffer(avctx, &s->pic))) {
+    if ((ret = ff_reget_buffer(avctx, &s->pic))) {
         av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
         return ret;
     }
@@ -385,13 +396,15 @@ static int xan_decode_frame(AVCodecContext *avctx,
         break;
     default:
         av_log(avctx, AV_LOG_ERROR, "Unknown frame type %d\n", ftype);
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     if (ret)
         return ret;
 
-    *data_size = sizeof(AVFrame);
-    *(AVFrame*)data = s->pic;
+    if ((ret = av_frame_ref(data, &s->pic)) < 0)
+        return ret;
+
+    *got_frame = 1;
 
     return avpkt->size;
 }
@@ -400,8 +413,7 @@ static av_cold int xan_decode_end(AVCodecContext *avctx)
 {
     XanContext *s = avctx->priv_data;
 
-    if (s->pic.data[0])
-        avctx->release_buffer(avctx, &s->pic);
+    av_frame_unref(&s->pic);
 
     av_freep(&s->y_buffer);
     av_freep(&s->scratch_buffer);
@@ -412,7 +424,7 @@ static av_cold int xan_decode_end(AVCodecContext *avctx)
 AVCodec ff_xan_wc4_decoder = {
     .name           = "xan_wc4",
     .type           = AVMEDIA_TYPE_VIDEO,
-    .id             = CODEC_ID_XAN_WC4,
+    .id             = AV_CODEC_ID_XAN_WC4,
     .priv_data_size = sizeof(XanContext),
     .init           = xan_decode_init,
     .close          = xan_decode_end,