avcodec/dvenc: Support adjusting the quantizer deadzone

[ffmpeg] / libavcodec / cdxl.c

diff --git a/libavcodec/cdxl.c b/libavcodec/cdxl.c
index c8d66b5845edfca81c4060aa9a7fcc90773bedfd..78f5d50102f1849c8cbd04b857247e4826c9936c 100644 (file)
--- a/libavcodec/cdxl.c
+++ b/libavcodec/cdxl.c
@@ -275,11 +275,11 @@ static int cdxl_decode_frame(AVCodecContext *avctx, void *data,
     else
         aligned_width = FFALIGN(c->avctx->width, 16);
     c->padded_bits  = aligned_width - c->avctx->width;
-    if (c->video_size < aligned_width * avctx->height * c->bpp / 8)
+    if (c->video_size < aligned_width * avctx->height * (int64_t)c->bpp / 8)
         return AVERROR_INVALIDDATA;
-    if (!encoding && c->palette_size && c->bpp <= 8) {
+    if (!encoding && c->palette_size && c->bpp <= 8 && c->format != CHUNKY) {
         avctx->pix_fmt = AV_PIX_FMT_PAL8;
-    } else if (encoding == 1 && (c->bpp == 6 || c->bpp == 8)) {
+    } else if (encoding == 1 && (c->bpp == 6 || c->bpp == 8) && c->format != CHUNKY) {
         if (c->palette_size != (1 << (c->bpp - 1)))
             return AVERROR_INVALIDDATA;
         avctx->pix_fmt = AV_PIX_FMT_BGR24;