Fixes avpicture_layout to not write past buffer end.

[ffmpeg] / libavcodec / imgconvert.c

diff --git a/libavcodec/imgconvert.c b/libavcodec/imgconvert.c
index 5f5a50ea192c8185cbc0f7d658eeb2d1b0ddcfb3..64b083885795d1c0ec03fcffdf0affa42fd4a124 100644 (file)
--- a/libavcodec/imgconvert.c
+++ b/libavcodec/imgconvert.c
@@ -337,6 +337,16 @@ int avpicture_layout(const AVPicture* src, enum PixelFormat pix_fmt, int width,
         }
     }
 
+    switch (pix_fmt) {
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
+        // do not include palette for these pseudo-paletted formats
+        return size;
+    }
+
     if (desc->flags & PIX_FMT_PAL)
         memcpy((unsigned char *)(((size_t)dest + 3) & ~3), src->data[1], 256 * 4);