lavc/vaapi_encode: add FF_CODEC_CAP_INIT_CLEANUP caps for encoders

ff_vaapi_encode_close() is not enough to free the resources like cbs
if initialization failure happens after codec->configure (except for
vp8/vp9).

We need to call avctx->codec->close() to deallocate, otherwise memory
leak happens.

Add FF_CODEC_CAP_INIT_CLEANUP for vaapi encoders and deallocate the
resources at free_and_end inside avcodec_open2().

Reviewed-by: Timo Rothenpieler <timo@rothenpieler.org>
Signed-off-by: Linjie Fu <linjie.fu@intel.com>

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index f551967ad663f11b196b437fa783520b9ee09583..cb05ebd774a1a259b4f2659c0eeba992f4d53957 100644 (file)
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -2366,7 +2366,6 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     return 0;
 
 fail:
-    ff_vaapi_encode_close(avctx);
     return err;
 }
 

diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index 67b1ecae1b631e68dbe4c8e7d1d9c962a1733d3b..e195650ef76edfbf5fd98b88581f0574f3ad5436 100644 (file)
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -1356,6 +1356,7 @@ AVCodec ff_h264_vaapi_encoder = {
     .close          = &vaapi_encode_h264_close,
     .priv_class     = &vaapi_encode_h264_class,
     .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_h264_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,

diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index e20e8cbdd037e35b16bdb717f43a5b78a8774871..92e051091126718b0da640dd60d474abaf32226c 100644 (file)
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -1292,6 +1292,7 @@ AVCodec ff_hevc_vaapi_encoder = {
     .close          = &vaapi_encode_h265_close,
     .priv_class     = &vaapi_encode_h265_class,
     .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_h265_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,

diff --git a/libavcodec/vaapi_encode_mjpeg.c b/libavcodec/vaapi_encode_mjpeg.c
index 2b5d3bf4f9aefda65dea98e70f5f828206208c31..8b441d36c337864cb211f18daabf9a4e872df6d8 100644 (file)
--- a/libavcodec/vaapi_encode_mjpeg.c
+++ b/libavcodec/vaapi_encode_mjpeg.c
@@ -565,6 +565,7 @@ AVCodec ff_mjpeg_vaapi_encoder = {
     .priv_class     = &vaapi_encode_mjpeg_class,
     .capabilities   = AV_CODEC_CAP_HARDWARE |
                       AV_CODEC_CAP_INTRA_ONLY,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_mjpeg_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,

diff --git a/libavcodec/vaapi_encode_mpeg2.c b/libavcodec/vaapi_encode_mpeg2.c
index 0398c4cd8c2ab170579c4f555d99dce7c9661850..02c76552eff473996153f3d3e24da1b953cd9981 100644 (file)
--- a/libavcodec/vaapi_encode_mpeg2.c
+++ b/libavcodec/vaapi_encode_mpeg2.c
@@ -702,6 +702,7 @@ AVCodec ff_mpeg2_vaapi_encoder = {
     .close          = &vaapi_encode_mpeg2_close,
     .priv_class     = &vaapi_encode_mpeg2_class,
     .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_mpeg2_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,

diff --git a/libavcodec/vaapi_encode_vp8.c b/libavcodec/vaapi_encode_vp8.c
index 16cbcbd81ca80bb7bd518caa22d1a513c94904e2..cff926baae89088b6ca70f3b06f5396675ecd883 100644 (file)
--- a/libavcodec/vaapi_encode_vp8.c
+++ b/libavcodec/vaapi_encode_vp8.c
@@ -257,6 +257,7 @@ AVCodec ff_vp8_vaapi_encoder = {
     .close          = &ff_vaapi_encode_close,
     .priv_class     = &vaapi_encode_vp8_class,
     .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_vp8_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,

diff --git a/libavcodec/vaapi_encode_vp9.c b/libavcodec/vaapi_encode_vp9.c
index d05319ef39617d8daf3156447284755ad8b4e41d..8514b859917e103b85fcaa2452dac9b51fb6dd8e 100644 (file)
--- a/libavcodec/vaapi_encode_vp9.c
+++ b/libavcodec/vaapi_encode_vp9.c
@@ -291,6 +291,7 @@ AVCodec ff_vp9_vaapi_encoder = {
     .close          = &ff_vaapi_encode_close,
     .priv_class     = &vaapi_encode_vp9_class,
     .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
+    .caps_internal  = FF_CODEC_CAP_INIT_CLEANUP,
     .defaults       = vaapi_encode_vp9_defaults,
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,