avconv: remove a pointless check.

[ffmpeg] / libavcodec / g722enc.c

diff --git a/libavcodec/g722enc.c b/libavcodec/g722enc.c
index ceb18b46dbaa8bc516231a53ecefd4b00ee10a51..ba8ceeff86e356f5a9317fee690b1c5530675c92 100644 (file)
--- a/libavcodec/g722enc.c
+++ b/libavcodec/g722enc.c
@@ -36,9 +36,27 @@
    problems, so we limit it to a reasonable value */
 #define MAX_FRAME_SIZE 32768
 
+/* We clip the value of avctx->trellis to prevent data type overflows and
+   undefined behavior. Using larger values is insanely slow anyway. */
+#define MIN_TRELLIS 0
+#define MAX_TRELLIS 16
+
+static av_cold int g722_encode_close(AVCodecContext *avctx)
+{
+    G722Context *c = avctx->priv_data;
+    int i;
+    for (i = 0; i < 2; i++) {
+        av_freep(&c->paths[i]);
+        av_freep(&c->node_buf[i]);
+        av_freep(&c->nodep_buf[i]);
+    }
+    return 0;
+}
+
 static av_cold int g722_encode_init(AVCodecContext * avctx)
 {
     G722Context *c = avctx->priv_data;
+    int ret;
 
     if (avctx->channels != 1) {
         av_log(avctx, AV_LOG_ERROR, "Only mono tracks are allowed.\n");
@@ -57,6 +75,10 @@ static av_cold int g722_encode_init(AVCodecContext * avctx)
             c->paths[i] = av_mallocz(max_paths * sizeof(**c->paths));
             c->node_buf[i] = av_mallocz(2 * frontier * sizeof(**c->node_buf));
             c->nodep_buf[i] = av_mallocz(2 * frontier * sizeof(**c->nodep_buf));
+            if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i]) {
+                ret = AVERROR(ENOMEM);
+                goto error;
+            }
         }
     }
 
@@ -83,19 +105,21 @@ static av_cold int g722_encode_init(AVCodecContext * avctx)
         avctx->frame_size = 320;
     }
 
-    return 0;
-}
-
-static av_cold int g722_encode_close(AVCodecContext *avctx)
-{
-    G722Context *c = avctx->priv_data;
-    int i;
-    for (i = 0; i < 2; i++) {
-        av_freep(&c->paths[i]);
-        av_freep(&c->node_buf[i]);
-        av_freep(&c->nodep_buf[i]);
+    if (avctx->trellis) {
+        /* validate trellis */
+        if (avctx->trellis < MIN_TRELLIS || avctx->trellis > MAX_TRELLIS) {
+            int new_trellis = av_clip(avctx->trellis, MIN_TRELLIS, MAX_TRELLIS);
+            av_log(avctx, AV_LOG_WARNING, "Requested trellis value is not "
+                   "allowed. Using %d instead of %d\n", new_trellis,
+                   avctx->trellis);
+            avctx->trellis = new_trellis;
+        }
     }
+
     return 0;
+error:
+    g722_encode_close(avctx);
+    return ret;
 }
 
 static const int16_t low_quant[33] = {
@@ -112,8 +136,8 @@ static inline void filter_samples(G722Context *c, const int16_t *samples,
     c->prev_samples[c->prev_samples_pos++] = samples[0];
     c->prev_samples[c->prev_samples_pos++] = samples[1];
     ff_g722_apply_qmf(c->prev_samples + c->prev_samples_pos - 24, &xout1, &xout2);
-    *xlow  = xout1 + xout2 >> 13;
-    *xhigh = xout1 - xout2 >> 13;
+    *xlow  = xout1 + xout2 >> 14;
+    *xhigh = xout1 - xout2 >> 14;
     if (c->prev_samples_pos >= PREV_SAMPLES_BUF_SIZE) {
         memmove(c->prev_samples,
                 c->prev_samples + c->prev_samples_pos - 22,