avcodec/dnxhddec: fix decoding of DNxHR HQX 10-bit

[ffmpeg] / libavcodec / qdrw.c

diff --git a/libavcodec/qdrw.c b/libavcodec/qdrw.c
index 828cfea3fd59550c25b8762ded8557e48144120d..e3d69c226d8fdcccec27919e18c4198365c29e67 100644 (file)
--- a/libavcodec/qdrw.c
+++ b/libavcodec/qdrw.c
@@ -37,6 +37,8 @@ enum QuickdrawOpcodes {
     PACKBITSRGN,
     DIRECTBITSRECT,
     DIRECTBITSRGN,
+    SHORTCOMMENT = 0x00A0,
+    LONGCOMMENT,
 
     EOP = 0x00FF,
 };
@@ -66,6 +68,165 @@ static int parse_palette(AVCodecContext *avctx, GetByteContext *gbc,
     return 0;
 }
 
+static int decode_rle_bpp2(AVCodecContext *avctx, AVFrame *p, GetByteContext *gbc)
+{
+    int offset = avctx->width;
+    uint8_t *outdata = p->data[0];
+    int i, j;
+
+    for (i = 0; i < avctx->height; i++) {
+        int size, left, code, pix;
+        uint8_t *out = outdata;
+        int pos = 0;
+
+        /* size of packed line */
+        size = left = bytestream2_get_be16(gbc);
+        if (bytestream2_get_bytes_left(gbc) < size)
+            return AVERROR_INVALIDDATA;
+
+        /* decode line */
+        while (left > 0) {
+            code = bytestream2_get_byte(gbc);
+            if (code & 0x80 ) { /* run */
+                pix = bytestream2_get_byte(gbc);
+                for (j = 0; j < 257 - code; j++) {
+                    out[pos++] = (pix & 0xC0) >> 6;
+                    out[pos++] = (pix & 0x30) >> 4;
+                    out[pos++] = (pix & 0x0C) >> 2;
+                    out[pos++] = (pix & 0x03);
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 2;
+            } else { /* copy */
+                for (j = 0; j < code + 1; j++) {
+                    pix = bytestream2_get_byte(gbc);
+                    out[pos++] = (pix & 0xC0) >> 6;
+                    out[pos++] = (pix & 0x30) >> 4;
+                    out[pos++] = (pix & 0x0C) >> 2;
+                    out[pos++] = (pix & 0x03);
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 1 + (code + 1);
+            }
+        }
+        outdata += p->linesize[0];
+    }
+    return 0;
+}
+
+static int decode_rle_bpp4(AVCodecContext *avctx, AVFrame *p, GetByteContext *gbc)
+{
+    int offset = avctx->width;
+    uint8_t *outdata = p->data[0];
+    int i, j;
+
+    for (i = 0; i < avctx->height; i++) {
+        int size, left, code, pix;
+        uint8_t *out = outdata;
+        int pos = 0;
+
+        /* size of packed line */
+        size = left = bytestream2_get_be16(gbc);
+        if (bytestream2_get_bytes_left(gbc) < size)
+            return AVERROR_INVALIDDATA;
+
+        /* decode line */
+        while (left > 0) {
+            code = bytestream2_get_byte(gbc);
+            if (code & 0x80 ) { /* run */
+                pix = bytestream2_get_byte(gbc);
+                for (j = 0; j < 257 - code; j++) {
+                    out[pos++] = (pix & 0xF0) >> 4;
+                    out[pos++] = pix & 0xF;
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 2;
+            } else { /* copy */
+                for (j = 0; j < code + 1; j++) {
+                    pix = bytestream2_get_byte(gbc);
+                    out[pos++] = (pix & 0xF0) >> 4;
+                    out[pos++] = pix & 0xF;
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 1 + (code + 1);
+            }
+        }
+        outdata += p->linesize[0];
+    }
+    return 0;
+}
+
+static int decode_rle16(AVCodecContext *avctx, AVFrame *p, GetByteContext *gbc)
+{
+    int offset = avctx->width;
+    uint8_t *outdata = p->data[0];
+    int i, j;
+
+    for (i = 0; i < avctx->height; i++) {
+        int size, left, code, pix;
+        uint16_t *out = (uint16_t *)outdata;
+        int pos = 0;
+
+        /* size of packed line */
+        size = left = bytestream2_get_be16(gbc);
+        if (bytestream2_get_bytes_left(gbc) < size)
+            return AVERROR_INVALIDDATA;
+
+        /* decode line */
+        while (left > 0) {
+            code = bytestream2_get_byte(gbc);
+            if (code & 0x80 ) { /* run */
+                pix = bytestream2_get_be16(gbc);
+                for (j = 0; j < 257 - code; j++) {
+                    out[pos] = pix;
+                    pos++;
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 3;
+            } else { /* copy */
+                for (j = 0; j < code + 1; j++) {
+                    out[pos] = bytestream2_get_be16(gbc);
+                    pos++;
+                    if (pos >= offset) {
+                        pos -= offset;
+                        pos++;
+                    }
+                    if (pos >= offset)
+                        return AVERROR_INVALIDDATA;
+                }
+                left  -= 1 + (code + 1) * 2;
+            }
+        }
+        outdata += p->linesize[0];
+    }
+    return 0;
+}
+
 static int decode_rle(AVCodecContext *avctx, AVFrame *p, GetByteContext *gbc,
                       int step)
 {
@@ -203,6 +364,10 @@ static int decode_frame(AVCodecContext *avctx,
             av_log(avctx, AV_LOG_DEBUG, "bppcount %d bpp %d\n", bppcnt, bpp);
             if (bppcnt == 1 && bpp == 8) {
                 avctx->pix_fmt = AV_PIX_FMT_PAL8;
+            } else if (bppcnt == 1 && (bpp == 4 || bpp == 2)) {
+                avctx->pix_fmt = AV_PIX_FMT_PAL8;
+            } else if (bppcnt == 3 && bpp == 5) {
+                avctx->pix_fmt = AV_PIX_FMT_RGB555;
             } else {
                 av_log(avctx, AV_LOG_ERROR,
                        "Invalid pixel format (bppcnt %d bpp %d) in Packbit\n",
@@ -238,7 +403,14 @@ static int decode_frame(AVCodecContext *avctx,
                 avpriv_report_missing_feature(avctx, "Packbit mask region");
             }
 
-            ret = decode_rle(avctx, p, &gbc, bppcnt);
+            if (avctx->pix_fmt == AV_PIX_FMT_RGB555)
+                ret = decode_rle16(avctx, p, &gbc);
+            else if (bpp == 2)
+                ret = decode_rle_bpp2(avctx, p, &gbc);
+            else if (bpp == 4)
+                ret = decode_rle_bpp4(avctx, p, &gbc);
+            else
+                ret = decode_rle(avctx, p, &gbc, bppcnt);
             if (ret < 0)
                 return ret;
             *got_frame = 1;
@@ -264,6 +436,8 @@ static int decode_frame(AVCodecContext *avctx,
             av_log(avctx, AV_LOG_DEBUG, "bppcount %d bpp %d\n", bppcnt, bpp);
             if (bppcnt == 3 && bpp == 8) {
                 avctx->pix_fmt = AV_PIX_FMT_RGB24;
+            } else if (bppcnt == 3 && bpp == 5) {
+                avctx->pix_fmt = AV_PIX_FMT_RGB555;
             } else if (bppcnt == 4 && bpp == 8) {
                 avctx->pix_fmt = AV_PIX_FMT_ARGB;
             } else {
@@ -292,11 +466,18 @@ static int decode_frame(AVCodecContext *avctx,
                 avpriv_report_missing_feature(avctx, "DirectBit mask region");
             }
 
-            ret = decode_rle(avctx, p, &gbc, bppcnt);
+            if (avctx->pix_fmt == AV_PIX_FMT_RGB555)
+                ret = decode_rle16(avctx, p, &gbc);
+            else
+                ret = decode_rle(avctx, p, &gbc, bppcnt);
             if (ret < 0)
                 return ret;
             *got_frame = 1;
             break;
+        case LONGCOMMENT:
+            bytestream2_get_be16(&gbc);
+            bytestream2_skip(&gbc, bytestream2_get_be16(&gbc));
+            break;
         default:
             av_log(avctx, AV_LOG_TRACE, "Unknown 0x%04X opcode\n", opcode);
             break;