avcodec/decode: Check size before opening iconv

Avoids closing iconv when the size check fails.

Reviewed-by: James Almer <jamrial@gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>

diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index db6ee9cb042d4dca51cbca5499c5dc168e8b9d82..c97679531102d60af50ceeeb3c1c0e15c875cf39 100644 (file)
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -884,18 +884,17 @@ static int recode_subtitle(AVCodecContext *avctx,
         return 0;
 
 #if CONFIG_ICONV
-    cd = iconv_open("UTF-8", avctx->sub_charenc);
-    av_assert0(cd != (iconv_t)-1);
-
     inb = inpkt->data;
     inl = inpkt->size;
 
     if (inl >= INT_MAX / UTF8_MAX_BYTES - AV_INPUT_BUFFER_PADDING_SIZE) {
         av_log(avctx, AV_LOG_ERROR, "Subtitles packet is too big for recoding\n");
-        ret = AVERROR(ENOMEM);
-        goto end;
+        return AVERROR(ERANGE);
     }
 
+    cd = iconv_open("UTF-8", avctx->sub_charenc);
+    av_assert0(cd != (iconv_t)-1);
+
     ret = av_new_packet(&tmp, inl * UTF8_MAX_BYTES);
     if (ret < 0)
         goto end;