libavcodec/iff: Use unsigned to avoid undefined behaviour

[ffmpeg] / libavcodec / hnm4video.c

diff --git a/libavcodec/hnm4video.c b/libavcodec/hnm4video.c
index 9e1ac49ddc3c3989c8425b075271f08d0ca196b8..95a284065e5c12acf5b5837476f117f6ab45566a 100644 (file)
--- a/libavcodec/hnm4video.c
+++ b/libavcodec/hnm4video.c
@@ -143,7 +143,7 @@ static void copy_processed_frame(AVCodecContext *avctx, AVFrame *frame)
     }
 }
 
-static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t size)
+static int decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t size)
 {
     Hnm4VideoContext *hnm = avctx->priv_data;
     GetByteContext gb;
@@ -162,7 +162,7 @@ static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t s
             if (tag == 0) {
                 if (writeoffset + 2 > hnm->width * hnm->height) {
                     av_log(avctx, AV_LOG_ERROR, "writeoffset out of bounds\n");
-                    break;
+                    return AVERROR_INVALIDDATA;
                 }
                 hnm->current[writeoffset++] = bytestream2_get_byte(&gb);
                 hnm->current[writeoffset++] = bytestream2_get_byte(&gb);
@@ -176,7 +176,7 @@ static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t s
                 count = bytestream2_get_byte(&gb) * 2;
                 if (writeoffset + count > hnm->width * hnm->height) {
                     av_log(avctx, AV_LOG_ERROR, "writeoffset out of bounds\n");
-                    break;
+                    return AVERROR_INVALIDDATA;
                 }
                 while (count > 0) {
                     hnm->current[writeoffset++] = bytestream2_peek_byte(&gb);
@@ -188,7 +188,7 @@ static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t s
             }
             if (writeoffset > hnm->width * hnm->height) {
                 av_log(avctx, AV_LOG_ERROR, "writeoffset out of bounds\n");
-                break;
+                return AVERROR_INVALIDDATA;
             }
         } else {
             previous = bytestream2_peek_byte(&gb) & 0x20;
@@ -204,24 +204,25 @@ static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t s
 
             if (!backward && offset + 2*count > hnm->width * hnm->height) {
                 av_log(avctx, AV_LOG_ERROR, "Attempting to read out of bounds\n");
-                break;
+                return AVERROR_INVALIDDATA;
             } else if (backward && offset + 1 >= hnm->width * hnm->height) {
                 av_log(avctx, AV_LOG_ERROR, "Attempting to read out of bounds\n");
-                break;
+                return AVERROR_INVALIDDATA;
             } else if (writeoffset + 2*count > hnm->width * hnm->height) {
                 av_log(avctx, AV_LOG_ERROR,
                        "Attempting to write out of bounds\n");
-                break;
+                return AVERROR_INVALIDDATA;
+
             }
             if(backward) {
                 if (offset < (!!backline)*(2 * hnm->width - 1) + 2*(left-1)) {
                     av_log(avctx, AV_LOG_ERROR, "Attempting to read out of bounds\n");
-                    break;
+                    return AVERROR_INVALIDDATA;
                 }
             } else {
                 if (offset < (!!backline)*(2 * hnm->width - 1)) {
                     av_log(avctx, AV_LOG_ERROR, "Attempting to read out of bounds\n");
-                    break;
+                    return AVERROR_INVALIDDATA;
                 }
             }
 
@@ -268,6 +269,7 @@ static void decode_interframe_v4(AVCodecContext *avctx, uint8_t *src, uint32_t s
             }
         }
     }
+    return 0;
 }
 
 static void decode_interframe_v4a(AVCodecContext *avctx, uint8_t *src,
@@ -435,7 +437,9 @@ static int hnm_decode_frame(AVCodecContext *avctx, void *data,
             decode_interframe_v4a(avctx, avpkt->data + 8, avpkt->size - 8);
             memcpy(hnm->processed, hnm->current, hnm->width * hnm->height);
         } else {
-            decode_interframe_v4(avctx, avpkt->data + 8, avpkt->size - 8);
+            int ret = decode_interframe_v4(avctx, avpkt->data + 8, avpkt->size - 8);
+            if (ret < 0)
+                return ret;
             postprocess_current_frame(avctx);
         }
         copy_processed_frame(avctx, frame);