Merge remote-tracking branch 'qatar/master'

[ffmpeg] / libavcodec / cook.c

diff --git a/libavcodec/cook.c b/libavcodec/cook.c
index e161b1a1bcf782356732e857149f7a6e672a3ef7..fbd910e243bb2b8313de4c35f8b106d2f669f322 100644 (file)
--- a/libavcodec/cook.c
+++ b/libavcodec/cook.c
@@ -280,8 +280,8 @@ static av_cold void init_cplscales_table(COOKContext *q)
 static inline int decode_bytes(const uint8_t *inbuffer, uint8_t *out, int bytes)
 {
     static const uint32_t tab[4] = {
-        AV_BE2NE32C(0x37c511f2), AV_BE2NE32C(0xf237c511),
-        AV_BE2NE32C(0x11f237c5), AV_BE2NE32C(0xc511f237),
+        AV_BE2NE32C(0x37c511f2U), AV_BE2NE32C(0xf237c511U),
+        AV_BE2NE32C(0x11f237c5U), AV_BE2NE32C(0xc511f237U),
     };
     int i, off;
     uint32_t c;
@@ -321,11 +321,11 @@ static av_cold int cook_decode_close(AVCodecContext *avctx)
 
     /* Free the VLC tables. */
     for (i = 0; i < 13; i++)
-        free_vlc(&q->envelope_quant_index[i]);
+        ff_free_vlc(&q->envelope_quant_index[i]);
     for (i = 0; i < 7; i++)
-        free_vlc(&q->sqvh[i]);
+        ff_free_vlc(&q->sqvh[i]);
     for (i = 0; i < q->num_subpackets; i++)
-        free_vlc(&q->subpacket[i].ccpl);
+        ff_free_vlc(&q->subpacket[i].ccpl);
 
     av_log(avctx, AV_LOG_DEBUG, "Memory deallocated.\n");
 
@@ -366,7 +366,7 @@ static void decode_gain_info(GetBitContext *gb, int *gaininfo)
  * @param q                 pointer to the COOKContext
  * @param quant_index_table pointer to the array
  */
-static void decode_envelope(COOKContext *q, COOKSubpacket *p,
+static int decode_envelope(COOKContext *q, COOKSubpacket *p,
                             int *quant_index_table)
 {
     int i, j, vlc_index;
@@ -388,7 +388,14 @@ static void decode_envelope(COOKContext *q, COOKSubpacket *p,
         j = get_vlc2(&q->gb, q->envelope_quant_index[vlc_index - 1].table,
                      q->envelope_quant_index[vlc_index - 1].bits, 2);
         quant_index_table[i] = quant_index_table[i - 1] + j - 12; // differential encoding
+
+        if (quant_index_table[i] < -63 || quant_index_table[i] > 64) {
+            av_log(NULL, AV_LOG_ERROR, "quant_index_table value out of bounds\n");
+            return AVERROR_INVALIDDATA;
+        }
     }
+
+    return 0;
 }
 
 /**
@@ -399,7 +406,7 @@ static void decode_envelope(COOKContext *q, COOKSubpacket *p,
  * @param category              pointer to the category array
  * @param category_index        pointer to the category_index array
  */
-static void categorize(COOKContext *q, COOKSubpacket *p, int *quant_index_table,
+static void categorize(COOKContext *q, COOKSubpacket *p, const int *quant_index_table,
                        int *category, int *category_index)
 {
     int exp_idx, bias, tmpbias1, tmpbias2, bits_left, num_bits, index, v, i, j;
@@ -635,20 +642,28 @@ static void decode_vectors(COOKContext *q, COOKSubpacket *p, int *category,
  * @param q                 pointer to the COOKContext
  * @param mlt_buffer        pointer to mlt coefficients
  */
-static void mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
+static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
 {
     int category_index[128];
     int quant_index_table[102];
     int category[128];
+    int ret, i;
 
     memset(&category,       0, sizeof(category));
     memset(&category_index, 0, sizeof(category_index));
 
-    decode_envelope(q, p, quant_index_table);
+    if ((ret = decode_envelope(q, p, quant_index_table)) < 0)
+        return ret;
     q->num_vectors = get_bits(&q->gb, p->log2_numvector_size);
     categorize(q, p, quant_index_table, category, category_index);
     expand_category(q, category, category_index);
+    for (i=0; i<p->total_subbands; i++) {
+        if (category[i] > 7)
+            return AVERROR_INVALIDDATA;
+    }
     decode_vectors(q, p, category, quant_index_table, mlt_buffer);
+
+    return 0;
 }
 
 
@@ -746,7 +761,7 @@ static void imlt_gain(COOKContext *q, float *inbuffer,
  * @param decouple_tab      decoupling array
  *
  */
-static void decouple_info(COOKContext *q, COOKSubpacket *p, int *decouple_tab)
+static int decouple_info(COOKContext *q, COOKSubpacket *p, int *decouple_tab)
 {
     int i;
     int vlc    = get_bits1(&q->gb);
@@ -755,14 +770,21 @@ static void decouple_info(COOKContext *q, COOKSubpacket *p, int *decouple_tab)
     int length = end - start + 1;
 
     if (start > end)
-        return;
+        return 0;
 
     if (vlc)
         for (i = 0; i < length; i++)
             decouple_tab[start + i] = get_vlc2(&q->gb, p->ccpl.table, p->ccpl.bits, 2);
     else
-        for (i = 0; i < length; i++)
-            decouple_tab[start + i] = get_bits(&q->gb, p->js_vlc_bits);
+        for (i = 0; i < length; i++) {
+            int v = get_bits(&q->gb, p->js_vlc_bits);
+            if (v == (1<<p->js_vlc_bits)-1) {
+                av_log(q->avctx, AV_LOG_ERROR, "decouple value too large\n");
+                return AVERROR_INVALIDDATA;
+            }
+            decouple_tab[start + i] = v;
+        }
+    return 0;
 }
 
 /*
@@ -798,10 +820,10 @@ static void decouple_float(COOKContext *q,
  * @param mlt_buffer1       pointer to left channel mlt coefficients
  * @param mlt_buffer2       pointer to right channel mlt coefficients
  */
-static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1,
+static int joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1,
                          float *mlt_buffer2)
 {
-    int i, j;
+    int i, j, ret;
     int decouple_tab[SUBBAND_SIZE];
     float *decode_buffer = q->decode_buffer_0;
     int idx, cpl_tmp;
@@ -814,9 +836,10 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1,
     /* Make sure the buffers are zeroed out. */
     memset(mlt_buffer1, 0, 1024 * sizeof(*mlt_buffer1));
     memset(mlt_buffer2, 0, 1024 * sizeof(*mlt_buffer2));
-    decouple_info(q, p, decouple_tab);
-    mono_decode(q, p, decode_buffer);
-
+    if ((ret = decouple_info(q, p, decouple_tab)) < 0)
+        return ret;
+    if ((ret = mono_decode(q, p, decode_buffer)) < 0)
+        return ret;
     /* The two channels are stored interleaved in decode_buffer. */
     for (i = 0; i < p->js_subband_start; i++) {
         for (j = 0; j < SUBBAND_SIZE; j++) {
@@ -832,11 +855,12 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1,
         cpl_tmp = cplband[i];
         idx -= decouple_tab[cpl_tmp];
         cplscale = q->cplscales[p->js_vlc_bits - 2];  // choose decoupler table
-        f1 = cplscale[decouple_tab[cpl_tmp]];
-        f2 = cplscale[idx - 1];
+        f1 = cplscale[decouple_tab[cpl_tmp] + 1];
+        f2 = cplscale[idx];
         q->decouple(q, p, i, f1, f2, decode_buffer, mlt_buffer1, mlt_buffer2);
         idx = (1 << p->js_vlc_bits) - 1;
     }
+    return 0;
 }
 
 /**
@@ -909,10 +933,11 @@ static inline void mlt_compensate_output(COOKContext *q, float *decode_buffer,
  * @param inbuffer          pointer to the inbuffer
  * @param outbuffer         pointer to the outbuffer
  */
-static void decode_subpacket(COOKContext *q, COOKSubpacket *p,
+static int decode_subpacket(COOKContext *q, COOKSubpacket *p,
                              const uint8_t *inbuffer, float *outbuffer)
 {
     int sub_packet_size = p->size;
+    int ret;
     /* packet dump */
     // for (i = 0; i < sub_packet_size ; i++)
     //     av_log(q->avctx, AV_LOG_ERROR, "%02x", inbuffer[i]);
@@ -921,13 +946,16 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p,
     decode_bytes_and_gain(q, p, inbuffer, &p->gains1);
 
     if (p->joint_stereo) {
-        joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2);
+        if ((ret = joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2)) < 0)
+            return ret;
     } else {
-        mono_decode(q, p, q->decode_buffer_1);
+        if ((ret = mono_decode(q, p, q->decode_buffer_1)) < 0)
+            return ret;
 
         if (p->num_channels == 2) {
             decode_bytes_and_gain(q, p, inbuffer + sub_packet_size / 2, &p->gains2);
-            mono_decode(q, p, q->decode_buffer_2);
+            if ((ret = mono_decode(q, p, q->decode_buffer_2)) < 0)
+                return ret;
         }
     }
 
@@ -941,6 +969,7 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p,
         else
             mlt_compensate_output(q, q->decode_buffer_2, &p->gains2,
                                   p->mono_previous_buffer2, outbuffer, p->ch_idx + 1);
+    return 0;
 }
 
 
@@ -996,7 +1025,8 @@ static int cook_decode_frame(AVCodecContext *avctx, void *data,
                i, q->subpacket[i].size, q->subpacket[i].joint_stereo, offset,
                avctx->block_align);
 
-        decode_subpacket(q, &q->subpacket[i], buf + offset, samples);
+        if ((ret = decode_subpacket(q, &q->subpacket[i], buf + offset, samples)) < 0)
+            return ret;
         offset += q->subpacket[i].size;
         chidx += q->subpacket[i].num_channels;
         av_log(avctx, AV_LOG_DEBUG, "subpacket[%i] %i %i\n",
@@ -1078,6 +1108,10 @@ static av_cold int cook_decode_init(AVCodecContext *avctx)
     q->sample_rate = avctx->sample_rate;
     q->nb_channels = avctx->channels;
     q->bit_rate = avctx->bit_rate;
+    if (!q->nb_channels) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid number of channels\n");
+        return AVERROR_INVALIDDATA;
+    }
 
     /* Initialize RNG. */
     av_lfg_init(&q->random_state, 0);