avcodec/j2kenc: Fix leaks on init failure

The JPEG2000 encoder did not clean up after itself on error.
This commit fixes this by modifying the cleanup function to be able to
handle only partially allocated structures and by setting the
FF_CODEC_CAP_INIT_CLEANUP flag.

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>

diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 4cefe6d7fbefe0e6a656b9c4660620d8cfe43bc8..e3c5a32188c3230e5f4c98dc7f8581e769ffc46b 100644 (file)
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -459,7 +459,7 @@ static int init_tiles(Jpeg2000EncoderContext *s)
     s->numXtiles = ff_jpeg2000_ceildiv(s->width, s->tile_width);
     s->numYtiles = ff_jpeg2000_ceildiv(s->height, s->tile_height);
 
-    s->tile = av_malloc_array(s->numXtiles, s->numYtiles * sizeof(Jpeg2000Tile));
+    s->tile = av_calloc(s->numXtiles, s->numYtiles * sizeof(Jpeg2000Tile));
     if (!s->tile)
         return AVERROR(ENOMEM);
     for (tileno = 0, tiley = 0; tiley < s->numYtiles; tiley++)
@@ -1495,12 +1495,16 @@ static void cleanup(Jpeg2000EncoderContext *s)
     int tileno, compno;
     Jpeg2000CodingStyle *codsty = &s->codsty;
 
+    if (!s->tile)
+        return;
     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
-        for (compno = 0; compno < s->ncomponents; compno++){
-            Jpeg2000Component *comp = s->tile[tileno].comp + compno;
-            ff_jpeg2000_cleanup(comp, codsty);
+        if (s->tile[tileno].comp) {
+            for (compno = 0; compno < s->ncomponents; compno++){
+                Jpeg2000Component *comp = s->tile[tileno].comp + compno;
+                ff_jpeg2000_cleanup(comp, codsty);
+            }
+            av_freep(&s->tile[tileno].comp);
         }
-        av_freep(&s->tile[tileno].comp);
         av_freep(&s->tile[tileno].layer_rates);
     }
     av_freep(&s->tile);
@@ -1853,4 +1857,5 @@ AVCodec ff_jpeg2000_encoder = {
         AV_PIX_FMT_NONE
     },
     .priv_class     = &j2k_class,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
 };