Merge commit '792b9c9dfcf44b657d7854368d975b5ca3bc22ca'

[ffmpeg] / libavcodec / sgienc.c

diff --git a/libavcodec/sgienc.c b/libavcodec/sgienc.c
index 4477fe1d1a174e8cc616c9a887d13819fb0a19dc..d631e23656421b962e4426f7171ab75bc0622caf 100644 (file)
--- a/libavcodec/sgienc.c
+++ b/libavcodec/sgienc.c
@@ -40,6 +40,47 @@ static av_cold int encode_init(AVCodecContext *avctx)
     return 0;
 }
 
+static int sgi_rle_encode(PutByteContext *pbc, const uint8_t *src,
+                          int w, int bpp)
+{
+    int val, count, x, start = bytestream2_tell_p(pbc);
+    void (*bytestream2_put)(PutByteContext *, unsigned int);
+
+    if (bpp == 1)
+        bytestream2_put = bytestream2_put_byte;
+    else
+        bytestream2_put = bytestream2_put_be16;
+
+    for (x = 0; x < w; x += count) {
+        /* see if we can encode the next set of pixels with RLE */
+        count = ff_rle_count_pixels(src, w - x, bpp, 1);
+        if (count > 1) {
+            if (bytestream2_get_bytes_left_p(pbc) < bpp * 2)
+                return AVERROR_INVALIDDATA;
+
+            val = bpp == 1 ? *src : AV_RB16(src);
+            bytestream2_put(pbc, count);
+            bytestream2_put(pbc, val);
+        } else {
+            int i;
+            /* fall back on uncompressed */
+            count = ff_rle_count_pixels(src, w - x, bpp, 0);
+            if (bytestream2_get_bytes_left_p(pbc) < bpp * (count + 1))
+                return AVERROR_INVALIDDATA;
+
+            bytestream2_put(pbc, count + 0x80);
+            for (i = 0; i < count; i++) {
+                val = bpp == 1 ? src[i] : AV_RB16(src + i * bpp);
+                bytestream2_put(pbc, val);
+            }
+        }
+
+        src += count * bpp;
+    }
+
+    return bytestream2_tell_p(pbc) - start;
+}
+
 static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                         const AVFrame *frame, int *got_packet)
 {
@@ -79,7 +120,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
     case AV_PIX_FMT_GRAY16LE:
         put_be = !HAVE_BIGENDIAN;
     case AV_PIX_FMT_GRAY16BE:
-        avctx->coder_type = FF_CODER_TYPE_RAW;
         bytes_per_channel = 2;
         pixmax = 0xFFFF;
         dimension = SGI_SINGLE_CHAN;
@@ -88,7 +128,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
     case AV_PIX_FMT_RGB48LE:
         put_be = !HAVE_BIGENDIAN;
     case AV_PIX_FMT_RGB48BE:
-        avctx->coder_type = FF_CODER_TYPE_RAW;
         bytes_per_channel = 2;
         pixmax = 0xFFFF;
         dimension = SGI_MULTI_CHAN;
@@ -97,7 +136,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
     case AV_PIX_FMT_RGBA64LE:
         put_be = !HAVE_BIGENDIAN;
     case AV_PIX_FMT_RGBA64BE:
-        avctx->coder_type = FF_CODER_TYPE_RAW;
         bytes_per_channel = 2;
         pixmax = 0xFFFF;
         dimension = SGI_MULTI_CHAN;
@@ -111,7 +149,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
     length = SGI_HEADER_SIZE;
     if (avctx->coder_type == FF_CODER_TYPE_RAW)
         length += depth * height * width;
-    else // assume ff_rl_encode() produces at most 2x size of input
+    else // assume sgi_rle_encode() produces at most 2x size of input
         length += tablesize * 2 + depth * height * (2 * width + 1);
 
     if ((ret = ff_alloc_packet2(avctx, pkt, bytes_per_channel * length, 0)) < 0)
@@ -153,27 +191,25 @@ FF_ENABLE_DEPRECATION_WARNINGS
         bytestream2_skip_p(&pbc, tablesize);
 
         /* Make an intermediate consecutive buffer. */
-        if (!(encode_buf = av_malloc(width)))
+        if (!(encode_buf = av_malloc(width * bytes_per_channel)))
             return AVERROR(ENOMEM);
 
         for (z = 0; z < depth; z++) {
-            in_buf = p->data[0] + p->linesize[0] * (height - 1) + z;
+            in_buf = p->data[0] + p->linesize[0] * (height - 1) + z * bytes_per_channel;
 
             for (y = 0; y < height; y++) {
                 bytestream2_put_be32(&taboff_pcb, bytestream2_tell_p(&pbc));
 
-                for (x = 0; x < width; x++)
+                for (x = 0; x < width * bytes_per_channel; x += bytes_per_channel)
                     encode_buf[x] = in_buf[depth * x];
 
-                length = ff_rle_encode(pbc.buffer,
-                                       bytestream2_get_bytes_left_p(&pbc),
-                                       encode_buf, 1, width, 0, 0, 0x80, 0);
+                length = sgi_rle_encode(&pbc, encode_buf, width,
+                                        bytes_per_channel);
                 if (length < 1) {
                     av_free(encode_buf);
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 }
 
-                bytestream2_skip_p(&pbc, length);
                 bytestream2_put_be32(&tablen_pcb, length);
                 in_buf -= p->linesize[0];
             }