dvbsubdec: check memory allocations and propagate errors

[ffmpeg] / libavcodec / dvbsubdec.c

diff --git a/libavcodec/dvbsubdec.c b/libavcodec/dvbsubdec.c
index 1dcefae4e37d7200c97ee86233e35b1d61cfc01e..1d62b8ad4235443b357df387c8dca5b3da5e1c45 100644 (file)
--- a/libavcodec/dvbsubdec.c
+++ b/libavcodec/dvbsubdec.c
@@ -693,26 +693,19 @@ static int dvbsub_read_8bit_string(uint8_t *destbuf, int dbuf_len,
                 if (run_length == 0) {
                     return pixels_read;
                 }
-
-                if (map_table)
-                    bits = map_table[0];
-                else
-                    bits = 0;
-                while (run_length-- > 0 && pixels_read < dbuf_len) {
-                    *destbuf++ = bits;
-                    pixels_read++;
-                }
             } else {
                 bits = *(*srcbuf)++;
 
                 if (non_mod == 1 && bits == 1)
                     pixels_read += run_length;
-                if (map_table)
-                    bits = map_table[bits];
-                else while (run_length-- > 0 && pixels_read < dbuf_len) {
-                    *destbuf++ = bits;
-                    pixels_read++;
-                }
+            }
+            if (map_table)
+                bits = map_table[0];
+            else
+                bits = 0;
+            while (run_length-- > 0 && pixels_read < dbuf_len) {
+                *destbuf++ = bits;
+                pixels_read++;
             }
         }
     }
@@ -839,8 +832,8 @@ static void dvbsub_parse_pixel_data_block(AVCodecContext *avctx, DVBSubObjectDis
 
 }
 
-static void dvbsub_parse_object_segment(AVCodecContext *avctx,
-                                        const uint8_t *buf, int buf_size)
+static int dvbsub_parse_object_segment(AVCodecContext *avctx,
+                                       const uint8_t *buf, int buf_size)
 {
     DVBSubContext *ctx = avctx->priv_data;
 
@@ -859,7 +852,7 @@ static void dvbsub_parse_object_segment(AVCodecContext *avctx,
     object = get_object(ctx, object_id);
 
     if (!object)
-        return;
+        return AVERROR_INVALIDDATA;
 
     coding_method = ((*buf) >> 2) & 3;
     non_modifying_color = ((*buf++) >> 1) & 1;
@@ -872,7 +865,7 @@ static void dvbsub_parse_object_segment(AVCodecContext *avctx,
 
         if (buf + top_field_len + bottom_field_len > buf_end) {
             av_log(avctx, AV_LOG_ERROR, "Field data size too large\n");
-            return;
+            return AVERROR_INVALIDDATA;
         }
 
         for (display = object->display_list; display; display = display->object_list_next) {
@@ -896,10 +889,11 @@ static void dvbsub_parse_object_segment(AVCodecContext *avctx,
         av_log(avctx, AV_LOG_ERROR, "Unknown object coding %d\n", coding_method);
     }
 
+    return 0;
 }
 
-static void dvbsub_parse_clut_segment(AVCodecContext *avctx,
-                                        const uint8_t *buf, int buf_size)
+static int dvbsub_parse_clut_segment(AVCodecContext *avctx,
+                                     const uint8_t *buf, int buf_size)
 {
     DVBSubContext *ctx = avctx->priv_data;
 
@@ -928,6 +922,8 @@ static void dvbsub_parse_clut_segment(AVCodecContext *avctx,
 
     if (!clut) {
         clut = av_malloc(sizeof(DVBSubCLUT));
+        if (!clut)
+            return AVERROR(ENOMEM);
 
         memcpy(clut, &default_clut, sizeof(DVBSubCLUT));
 
@@ -944,7 +940,7 @@ static void dvbsub_parse_clut_segment(AVCodecContext *avctx,
 
         if (depth == 0) {
             av_log(avctx, AV_LOG_ERROR, "Invalid clut depth 0x%x!\n", *buf);
-            return;
+            return AVERROR_INVALIDDATA;
         }
 
         full_range = (*buf++) & 1;
@@ -978,11 +974,13 @@ static void dvbsub_parse_clut_segment(AVCodecContext *avctx,
         if (depth & 0x20)
             clut->clut256[entry_id] = RGBA(r,g,b,255 - alpha);
     }
+
+    return 0;
 }
 
 
-static void dvbsub_parse_region_segment(AVCodecContext *avctx,
-                                        const uint8_t *buf, int buf_size)
+static int dvbsub_parse_region_segment(AVCodecContext *avctx,
+                                       const uint8_t *buf, int buf_size)
 {
     DVBSubContext *ctx = avctx->priv_data;
 
@@ -994,7 +992,7 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
     int fill;
 
     if (buf_size < 10)
-        return;
+        return AVERROR_INVALIDDATA;
 
     region_id = *buf++;
 
@@ -1002,6 +1000,8 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
 
     if (!region) {
         region = av_mallocz(sizeof(DVBSubRegion));
+        if (!region)
+            return AVERROR(ENOMEM);
 
         region->id = region_id;
 
@@ -1022,6 +1022,8 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
         region->buf_size = region->width * region->height;
 
         region->pbuf = av_malloc(region->buf_size);
+        if (!region->pbuf)
+            return AVERROR(ENOMEM);
 
         fill = 1;
     }
@@ -1061,6 +1063,8 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
 
         if (!object) {
             object = av_mallocz(sizeof(DVBSubObject));
+            if (!object)
+                return AVERROR(ENOMEM);
 
             object->id = object_id;
             object->next = ctx->object_list;
@@ -1070,6 +1074,8 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
         object->type = (*buf) >> 6;
 
         display = av_mallocz(sizeof(DVBSubObjectDisplay));
+        if (!display)
+            return AVERROR(ENOMEM);
 
         display->object_id = object_id;
         display->region_id = region_id;
@@ -1090,10 +1096,12 @@ static void dvbsub_parse_region_segment(AVCodecContext *avctx,
         display->object_list_next = object->display_list;
         object->display_list = display;
     }
+
+    return 0;
 }
 
-static void dvbsub_parse_page_segment(AVCodecContext *avctx,
-                                        const uint8_t *buf, int buf_size)
+static int dvbsub_parse_page_segment(AVCodecContext *avctx,
+                                     const uint8_t *buf, int buf_size)
 {
     DVBSubContext *ctx = avctx->priv_data;
     DVBSubRegionDisplay *display;
@@ -1104,7 +1112,7 @@ static void dvbsub_parse_page_segment(AVCodecContext *avctx,
     int page_state;
 
     if (buf_size < 1)
-        return;
+        return AVERROR_INVALIDDATA;
 
     ctx->time_out = *buf++;
     page_state = ((*buf++) >> 2) & 3;
@@ -1131,8 +1139,11 @@ static void dvbsub_parse_page_segment(AVCodecContext *avctx,
             display = display->next;
         }
 
-        if (!display)
+        if (!display) {
             display = av_mallocz(sizeof(DVBSubRegionDisplay));
+            if (!display)
+                return AVERROR(ENOMEM);
+        }
 
         display->region_id = region_id;
 
@@ -1158,6 +1169,7 @@ static void dvbsub_parse_page_segment(AVCodecContext *avctx,
         av_free(display);
     }
 
+    return 0;
 }
 
 
@@ -1211,6 +1223,8 @@ static void save_display_set(DVBSubContext *ctx)
     if (x_pos >= 0) {
 
         pbuf = av_malloc(width * height * 4);
+        if (!pbuf)
+            return AVERROR(ENOMEM);
 
         for (display = ctx->display_list; display; display = display->next) {
             region = get_region(ctx, display->region_id);
@@ -1256,28 +1270,28 @@ static void save_display_set(DVBSubContext *ctx)
 }
 #endif
 
-static void dvbsub_parse_display_definition_segment(AVCodecContext *avctx,
-                                                    const uint8_t *buf,
-                                                    int buf_size)
+static int dvbsub_parse_display_definition_segment(AVCodecContext *avctx,
+                                                   const uint8_t *buf,
+                                                   int buf_size)
 {
     DVBSubContext *ctx = avctx->priv_data;
     DVBSubDisplayDefinition *display_def = ctx->display_definition;
     int dds_version, info_byte;
 
     if (buf_size < 5)
-        return;
+        return AVERROR_INVALIDDATA;
 
     info_byte   = bytestream_get_byte(&buf);
     dds_version = info_byte >> 4;
     if (display_def && display_def->version == dds_version)
-        return; // already have this display definition version
+        return 0; // already have this display definition version
 
     if (!display_def) {
         display_def             = av_mallocz(sizeof(*display_def));
+        if (!display_def)
+            return AVERROR(ENOMEM);
         ctx->display_definition = display_def;
     }
-    if (!display_def)
-        return;
 
     display_def->version = dds_version;
     display_def->x       = 0;
@@ -1286,7 +1300,7 @@ static void dvbsub_parse_display_definition_segment(AVCodecContext *avctx,
     display_def->height  = bytestream_get_be16(&buf) + 1;
 
     if (buf_size < 13)
-        return;
+        return AVERROR_INVALIDDATA;
 
     if (info_byte & 1<<3) { // display_window_flag
         display_def->x = bytestream_get_be16(&buf);
@@ -1294,10 +1308,12 @@ static void dvbsub_parse_display_definition_segment(AVCodecContext *avctx,
         display_def->width  = bytestream_get_be16(&buf) - display_def->x + 1;
         display_def->height = bytestream_get_be16(&buf) - display_def->y + 1;
     }
+
+    return 0;
 }
 
 static int dvbsub_display_end_segment(AVCodecContext *avctx, const uint8_t *buf,
-                                        int buf_size, AVSubtitle *sub)
+                                      int buf_size, AVSubtitle *sub)
 {
     DVBSubContext *ctx = avctx->priv_data;
     DVBSubDisplayDefinition *display_def = ctx->display_definition;
@@ -1321,12 +1337,13 @@ static int dvbsub_display_end_segment(AVCodecContext *avctx, const uint8_t *buf,
     }
 
     sub->num_rects = ctx->display_list_size;
+    if (sub->num_rects <= 0)
+        return AVERROR_INVALIDDATA;
 
-    if (sub->num_rects > 0){
-        sub->rects = av_mallocz(sizeof(*sub->rects) * sub->num_rects);
-        for(i=0; i<sub->num_rects; i++)
-            sub->rects[i] = av_mallocz(sizeof(*sub->rects[i]));
-    }
+    sub->rects = av_mallocz_array(sub->num_rects * sub->num_rects,
+                                  sizeof(*sub->rects));
+    if (!sub->rects)
+        return AVERROR(ENOMEM);
 
     i = 0;
 
@@ -1364,9 +1381,18 @@ static int dvbsub_display_end_segment(AVCodecContext *avctx, const uint8_t *buf,
         }
 
         rect->pict.data[1] = av_mallocz(AVPALETTE_SIZE);
+        if (!rect->pict.data[1]) {
+            av_free(sub->rects);
+            return AVERROR(ENOMEM);
+        }
         memcpy(rect->pict.data[1], clut_table, (1 << region->depth) * sizeof(uint32_t));
 
         rect->pict.data[0] = av_malloc(region->buf_size);
+        if (!rect->pict.data[0]) {
+            av_free(rect->pict.data[1]);
+            av_free(sub->rects);
+            return AVERROR(ENOMEM);
+        }
         memcpy(rect->pict.data[0], region->pbuf, region->buf_size);
 
         i++;
@@ -1408,7 +1434,7 @@ static int dvbsub_decode(AVCodecContext *avctx,
 
     if (buf_size <= 6 || *buf != 0x0f) {
         av_dlog(avctx, "incomplete or broken packet");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     p = buf;
@@ -1429,29 +1455,35 @@ static int dvbsub_decode(AVCodecContext *avctx,
 
         if (page_id == ctx->composition_id || page_id == ctx->ancillary_id ||
             ctx->composition_id == -1 || ctx->ancillary_id == -1) {
+            int ret = 0;
             switch (segment_type) {
             case DVBSUB_PAGE_SEGMENT:
-                dvbsub_parse_page_segment(avctx, p, segment_length);
+                ret = dvbsub_parse_page_segment(avctx, p, segment_length);
                 break;
             case DVBSUB_REGION_SEGMENT:
-                dvbsub_parse_region_segment(avctx, p, segment_length);
+                ret = dvbsub_parse_region_segment(avctx, p, segment_length);
                 break;
             case DVBSUB_CLUT_SEGMENT:
-                dvbsub_parse_clut_segment(avctx, p, segment_length);
+                ret = dvbsub_parse_clut_segment(avctx, p, segment_length);
                 break;
             case DVBSUB_OBJECT_SEGMENT:
-                dvbsub_parse_object_segment(avctx, p, segment_length);
+                ret = dvbsub_parse_object_segment(avctx, p, segment_length);
                 break;
             case DVBSUB_DISPLAYDEFINITION_SEGMENT:
-                dvbsub_parse_display_definition_segment(avctx, p, segment_length);
+                ret = dvbsub_parse_display_definition_segment(avctx, p,
+                                                              segment_length);
+                break;
             case DVBSUB_DISPLAY_SEGMENT:
-                *data_size = dvbsub_display_end_segment(avctx, p, segment_length, sub);
+                ret = dvbsub_display_end_segment(avctx, p, segment_length, sub);
+                *data_size = ret;
                 break;
             default:
                 av_dlog(avctx, "Subtitling segment type 0x%x, page id %d, length %d\n",
                         segment_type, page_id, segment_length);
                 break;
             }
+            if (ret < 0)
+                return ret;
         }
 
         p += segment_length;