intrax8: Check and propagate errors from ff_intrax8_common_init

[ffmpeg] / libavcodec / intrax8.c

diff --git a/libavcodec/intrax8.c b/libavcodec/intrax8.c
index cbcf9aa159f2fcbd9e1d07b6358e093292f7185e..b961395cbb48616c0b0b9f3d48a0b4b8a54034bb 100644 (file)
--- a/libavcodec/intrax8.c
+++ b/libavcodec/intrax8.c
@@ -45,7 +45,7 @@ static VLC j_ac_vlc[2][2][8];  // [quant < 13], [intra / inter], [select]
 static VLC j_dc_vlc[2][8];     // [quant], [select]
 static VLC j_orient_vlc[2][4]; // [quant], [select]
 
-static av_cold void x8_vlc_init(void)
+static av_cold int x8_vlc_init(void)
 {
     int i;
     int offset = 0;
@@ -114,9 +114,13 @@ static av_cold void x8_vlc_init(void)
         init_or_vlc(j_orient_vlc[1][i], x8_orient_lowquant_table[i][0]);
 #undef init_or_vlc
 
-    if (offset != sizeof(table)/sizeof(VLC_TYPE)/2)
+    if (offset != sizeof(table) / sizeof(VLC_TYPE) / 2) {
         av_log(NULL, AV_LOG_ERROR, "table size %zd does not match needed %i\n",
                sizeof(table) / sizeof(VLC_TYPE) / 2, offset);
+        return AVERROR_INVALIDDATA;
+    }
+
+    return 0;
 }
 
 static void x8_reset_vlc_tables(IntraX8Context *w)
@@ -739,20 +743,18 @@ static void x8_init_block_index(MpegEncContext *s)
     s->dest[2] += (s->mb_y & (~1)) * uvlinesize << 2;
 }
 
-/**
- * Initialize IntraX8 frame decoder.
- * Requires valid MpegEncContext with valid s->mb_width before calling.
- * @param w pointer to IntraX8Context
- * @param s pointer to MpegEncContext of the parent codec
- */
-av_cold void ff_intrax8_common_init(IntraX8Context *w, MpegEncContext *const s)
+av_cold int ff_intrax8_common_init(IntraX8Context *w, MpegEncContext *const s)
 {
+    int ret = x8_vlc_init();
+    if (ret < 0)
+        return ret;
+
     w->s = s;
-    x8_vlc_init();
-    assert(s->mb_width > 0);
 
     // two rows, 2 blocks per cannon mb
     w->prediction_table = av_mallocz(s->mb_width * 2 * 2);
+    if (!w->prediction_table)
+        return AVERROR(ENOMEM);
 
     ff_init_scantable(s->idsp.idct_permutation, &w->scantable[0],
                       ff_wmv1_scantable[0]);
@@ -762,27 +764,15 @@ av_cold void ff_intrax8_common_init(IntraX8Context *w, MpegEncContext *const s)
                       ff_wmv1_scantable[3]);
 
     ff_intrax8dsp_init(&w->dsp);
+
+    return 0;
 }
 
-/**
- * Destroy IntraX8 frame structure.
- * @param w pointer to IntraX8Context
- */
 av_cold void ff_intrax8_common_end(IntraX8Context *w)
 {
     av_freep(&w->prediction_table);
 }
 
-/**
- * Decode single IntraX8 frame.
- * The parent codec must fill s->loopfilter and s->gb (bitstream).
- * The parent codec must call ff_mpv_frame_start() before calling this function.
- * The parent codec must call ff_mpv_frame_end() after calling this function.
- * This function does not use ff_mpv_decode_mb().
- * @param w pointer to IntraX8Context
- * @param dquant doubled quantizer, it would be odd in case of VC-1 halfpq==1.
- * @param quant_offset offset away from zero
- */
 int ff_intrax8_decode_picture(IntraX8Context *const w, int dquant,
                               int quant_offset)
 {