avcodec/mpegpicture: Keep ff_mpeg_framesize_alloc() failure state consistent

Fixes: null pointer dereference
Fixes: ff_put_pixels16_sse2.mp4
Found-by: Rafael Dutra <rafael.dutra@cispa.de>
Regression-since: 4b2863ff01b1fe93d9a518523c9098d17a9d8c6f
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index e3f648895dbf724851e49106ce3268cecf4a1aea..0652b7c87946bd571bc1941067da2f0a5a3b1c24 100644 (file)
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -79,8 +79,11 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
     // linesize * interlaced * MBsize
     // we also use this buffer for encoding in encode_mb_internal() needig an additional 32 lines
     if (!FF_ALLOCZ_TYPED_ARRAY(sc->edge_emu_buffer, alloc_size * EMU_EDGE_HEIGHT) ||
-        !FF_ALLOCZ_TYPED_ARRAY(me->scratchpad,      alloc_size * 4 * 16 * 2))
+        !FF_ALLOCZ_TYPED_ARRAY(me->scratchpad,      alloc_size * 4 * 16 * 2)) {
+        av_freep(&sc->edge_emu_buffer);
         return AVERROR(ENOMEM);
+    }
+
     me->temp            = me->scratchpad;
     sc->rd_scratchpad   = me->scratchpad;
     sc->b_scratchpad    = me->scratchpad;