Fixes avpicture_layout to not write past buffer end.

[ffmpeg] / libavcodec / imgconvert.c

diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index 66ce9de339b6e0b26b0b3a8272693e33562594aa..64b083885795d1c0ec03fcffdf0affa42fd4a124 100644 (file)
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -143,6 +143,14 @@ static const PixFmtInfo pix_fmt_info[PIX_FMT_NB] = {
     [PIX_FMT_RGB48LE] = {
         .color_type = FF_COLOR_RGB,
     },
+    [PIX_FMT_RGBA64BE] = {
+        .is_alpha = 1,
+        .color_type = FF_COLOR_RGB,
+    },
+    [PIX_FMT_RGBA64LE] = {
+        .is_alpha = 1,
+        .color_type = FF_COLOR_RGB,
+    },
     [PIX_FMT_RGB565BE] = {
         .color_type = FF_COLOR_RGB,
     },
@@ -195,6 +203,20 @@ static const PixFmtInfo pix_fmt_info[PIX_FMT_NB] = {
         .is_alpha = 1,
         .color_type = FF_COLOR_RGB,
     },
+    [PIX_FMT_BGR48BE] = {
+        .color_type = FF_COLOR_RGB,
+    },
+    [PIX_FMT_BGR48LE] = {
+        .color_type = FF_COLOR_RGB,
+    },
+    [PIX_FMT_BGRA64BE] = {
+        .is_alpha = 1,
+        .color_type = FF_COLOR_RGB,
+    },
+    [PIX_FMT_BGRA64LE] = {
+        .is_alpha = 1,
+        .color_type = FF_COLOR_RGB,
+    },
     [PIX_FMT_BGR565BE] = {
         .color_type = FF_COLOR_RGB,
         .padded_size = 16,
@@ -315,6 +337,16 @@ int avpicture_layout(const AVPicture* src, enum PixelFormat pix_fmt, int width,
         }
     }
 
+    switch (pix_fmt) {
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
+        // do not include palette for these pseudo-paletted formats
+        return size;
+    }
+
     if (desc->flags & PIX_FMT_PAL)
         memcpy((unsigned char *)(((size_t)dest + 3) & ~3), src->data[1], 256 * 4);
 
@@ -359,15 +391,17 @@ static int get_pix_fmt_depth(int *min, int *max, enum PixelFormat pix_fmt)
 int avcodec_get_pix_fmt_loss(enum PixelFormat dst_pix_fmt, enum PixelFormat src_pix_fmt,
                              int has_alpha)
 {
-    if (dst_pix_fmt>=PIX_FMT_NB || dst_pix_fmt<=PIX_FMT_NONE)
-        return ~0;
-
     const PixFmtInfo *pf, *ps;
-    const AVPixFmtDescriptor *src_desc = &av_pix_fmt_descriptors[src_pix_fmt];
-    const AVPixFmtDescriptor *dst_desc = &av_pix_fmt_descriptors[dst_pix_fmt];
+    const AVPixFmtDescriptor *src_desc;
+    const AVPixFmtDescriptor *dst_desc;
     int src_min_depth, src_max_depth, dst_min_depth, dst_max_depth;
     int ret, loss;
 
+    if (dst_pix_fmt >= PIX_FMT_NB || dst_pix_fmt <= PIX_FMT_NONE)
+        return ~0;
+
+    src_desc = &av_pix_fmt_descriptors[src_pix_fmt];
+    dst_desc = &av_pix_fmt_descriptors[dst_pix_fmt];
     ps = &pix_fmt_info[src_pix_fmt];
 
     /* compute loss */