pthread_frame: do not run hwaccel decoding asynchronously unless it's safe

Certain hardware decoding APIs are not guaranteed to be thread-safe, so
having the user access decoded hardware surfaces while the decoder is
running in another thread can cause failures (this is mainly known to
happen with DXVA2).

For such hwaccels, only allow the decoding thread to run while the user
is inside a lavc decode call (avcodec_send_packet/receive_frame).

Merges Libav commit d4a91e65.

Signed-off-by: wm4 <nfxjfg@googlemail.com>
Tested-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index fbdf89216d6bbc704dbb8a528f0e7416f04f3ed6..f9ebd14a7eaa933db831c7698306e73b267a278e 100644 (file)
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -3905,6 +3905,11 @@ typedef struct AVHWAccel {
      * AVCodecInternal.hwaccel_priv_data.
      */
     int priv_data_size;
+
+    /**
+     * Internal hwaccel capabilities.
+     */
+    int caps_internal;
 } AVHWAccel;
 
 /**

diff --git a/libavcodec/hwaccel.h b/libavcodec/hwaccel.h
new file mode 100644 (file)
index 0000000..17af437
--- /dev/null
+++ b/libavcodec/hwaccel.h
@@ -0,0 +1,24 @@
+/*
+ * This file is part of FFmpeg and was stolen from Libav.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_HWACCEL_H
+#define AVCODEC_HWACCEL_H
+
+#define HWACCEL_CAP_ASYNC_SAFE      (1 << 0)
+
+#endif /* AVCODEC_HWACCEL_H */

diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c
index e3bfbab872d9e0bd536c3252dbd37051305adcf4..f959175db32f794afc79a65be16547fdd4496673 100644 (file)
--- a/libavcodec/pthread_frame.c
+++ b/libavcodec/pthread_frame.c
@@ -28,6 +28,7 @@
 #include <stdint.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "internal.h"
 #include "pthread_internal.h"
 #include "thread.h"
@@ -105,6 +106,7 @@ typedef struct PerThreadContext {
     int die;                        ///< Set when the thread should exit.
 
     int hwaccel_serializing;
+    int async_serializing;
 } PerThreadContext;
 
 /**
@@ -120,6 +122,7 @@ typedef struct FrameThreadContext {
      * is used.
      */
     pthread_mutex_t hwaccel_mutex;
+    pthread_mutex_t async_mutex;
 
     int next_decoding;             ///< The next context to submit a packet to.
     int next_finished;             ///< The next context to return output from.
@@ -190,6 +193,11 @@ static attribute_align_arg void *frame_worker_thread(void *arg)
             pthread_mutex_unlock(&p->parent->hwaccel_mutex);
         }
 
+        if (p->async_serializing) {
+            p->async_serializing = 0;
+            pthread_mutex_unlock(&p->parent->async_mutex);
+        }
+
         pthread_mutex_lock(&p->progress_mutex);
 #if 0 //BUFREF-FIXME
         for (i = 0; i < MAX_BUFFERS; i++)
@@ -443,7 +451,11 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
     FrameThreadContext *fctx = avctx->internal->thread_ctx;
     int finished = fctx->next_finished;
     PerThreadContext *p;
-    int err;
+    int err, ret;
+
+    /* release the async lock, permitting blocked hwaccel threads to
+     * go forward while we are in this function */
+    pthread_mutex_unlock(&fctx->async_mutex);
 
     /*
      * Submit a packet to the next decoding thread.
@@ -451,9 +463,11 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
 
     p = &fctx->threads[fctx->next_decoding];
     err = update_context_from_user(p->avctx, avctx);
-    if (err) return err;
+    if (err)
+        goto finish;
     err = submit_packet(p, avpkt);
-    if (err) return err;
+    if (err)
+        goto finish;
 
     /*
      * If we're still receiving the initial packets, don't return a frame.
@@ -464,8 +478,10 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
 
     if (fctx->delaying) {
         *got_picture_ptr=0;
-        if (avpkt->size)
-            return avpkt->size;
+        if (avpkt->size) {
+            ret = avpkt->size;
+            goto finish;
+        }
     }
 
     /*
@@ -515,10 +531,15 @@ int ff_thread_decode_frame(AVCodecContext *avctx,
      * Otherwise the error can get lost.
      */
     if (!avpkt->size && !*got_picture_ptr)
-        return err;
+        goto finish;
 
     /* return the size of the consumed packet if no error occurred */
-    return (p->result >= 0) ? avpkt->size : p->result;
+    ret = (p->result >= 0) ? avpkt->size : p->result;
+finish:
+    pthread_mutex_lock(&fctx->async_mutex);
+    if (err < 0)
+        return err;
+    return ret;
 }
 
 void ff_thread_report_progress(ThreadFrame *f, int n, int field)
@@ -573,6 +594,13 @@ void ff_thread_finish_setup(AVCodecContext *avctx) {
         p->hwaccel_serializing = 1;
     }
 
+    /* this assumes that no hwaccel calls happen before ff_thread_finish_setup() */
+    if (avctx->hwaccel &&
+        !(avctx->hwaccel->caps_internal & HWACCEL_CAP_ASYNC_SAFE)) {
+        p->async_serializing = 1;
+        pthread_mutex_lock(&p->parent->async_mutex);
+    }
+
     pthread_mutex_lock(&p->progress_mutex);
     if(atomic_load(&p->state) == STATE_SETUP_FINISHED){
         av_log(avctx, AV_LOG_WARNING, "Multiple ff_thread_finish_setup() calls\n");
@@ -589,6 +617,8 @@ static void park_frame_worker_threads(FrameThreadContext *fctx, int thread_count
 {
     int i;
 
+    pthread_mutex_unlock(&fctx->async_mutex);
+
     for (i = 0; i < thread_count; i++) {
         PerThreadContext *p = &fctx->threads[i];
 
@@ -600,6 +630,8 @@ static void park_frame_worker_threads(FrameThreadContext *fctx, int thread_count
         }
         p->got_frame = 0;
     }
+
+    pthread_mutex_lock(&fctx->async_mutex);
 }
 
 void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
@@ -663,6 +695,10 @@ void ff_frame_thread_free(AVCodecContext *avctx, int thread_count)
     av_freep(&fctx->threads);
     pthread_mutex_destroy(&fctx->buffer_mutex);
     pthread_mutex_destroy(&fctx->hwaccel_mutex);
+
+    pthread_mutex_unlock(&fctx->async_mutex);
+    pthread_mutex_destroy(&fctx->async_mutex);
+
     av_freep(&avctx->internal->thread_ctx);
 
     if (avctx->priv_data && avctx->codec && avctx->codec->priv_class)
@@ -710,6 +746,10 @@ int ff_frame_thread_init(AVCodecContext *avctx)
 
     pthread_mutex_init(&fctx->buffer_mutex, NULL);
     pthread_mutex_init(&fctx->hwaccel_mutex, NULL);
+
+    pthread_mutex_init(&fctx->async_mutex, NULL);
+    pthread_mutex_lock(&fctx->async_mutex);
+
     fctx->delaying = 1;
 
     for (i = 0; i < thread_count; i++) {

diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c
index 44e84625220a378ec56e5453b3c417334c16938d..30e7026ccf0490575ea36ce6f76abf2be369244f 100644 (file)
--- a/libavcodec/vaapi_h264.c
+++ b/libavcodec/vaapi_h264.c
@@ -22,6 +22,7 @@
 
 #include "h264dec.h"
 #include "h264_ps.h"
+#include "hwaccel.h"
 #include "vaapi_decode.h"
 
 /**
@@ -399,4 +400,5 @@ AVHWAccel ff_h264_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vaapi_mpeg2.c b/libavcodec/vaapi_mpeg2.c
index b2417ee83046f9127b4228af5298bf3224f1dbc6..0d197c969288364081b487bd92813eb65cee76ae 100644 (file)
--- a/libavcodec/vaapi_mpeg2.c
+++ b/libavcodec/vaapi_mpeg2.c
@@ -20,6 +20,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "hwaccel.h"
 #include "mpegutils.h"
 #include "mpegvideo.h"
 #include "internal.h"
@@ -183,4 +184,5 @@ AVHWAccel ff_mpeg2_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vaapi_mpeg4.c b/libavcodec/vaapi_mpeg4.c
index b00f73dce1ce87d94f7603780cd8a10f4e43836a..f8c5ddf20953795bf2067aac75b19d63c1f89809 100644 (file)
--- a/libavcodec/vaapi_mpeg4.c
+++ b/libavcodec/vaapi_mpeg4.c
@@ -21,6 +21,7 @@
  */
 
 #include "h263.h"
+#include "hwaccel.h"
 #include "internal.h"
 #include "mpeg4video.h"
 #include "mpegvideo.h"
@@ -189,6 +190,7 @@ AVHWAccel ff_mpeg4_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -205,5 +207,6 @@ AVHWAccel ff_h263_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif

diff --git a/libavcodec/vaapi_vc1.c b/libavcodec/vaapi_vc1.c
index a456149e6e4ce96f6b10012d20c9a023cec86f78..30c9ed3c8b530d9ee3eef5c1f001157c09bf5d27 100644 (file)
--- a/libavcodec/vaapi_vc1.c
+++ b/libavcodec/vaapi_vc1.c
@@ -20,6 +20,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "hwaccel.h"
 #include "internal.h"
 #include "vaapi_decode.h"
 #include "vc1.h"
@@ -399,6 +400,7 @@ AVHWAccel ff_wmv3_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -414,4 +416,5 @@ AVHWAccel ff_vc1_vaapi_hwaccel = {
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
+    .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vdpau_h264.c b/libavcodec/vdpau_h264.c
index 7265af2890e53b4b404022e414e0fcb9ac86e7c5..be6ba7143334883d4270a56025f615b3fbb31ac5 100644 (file)
--- a/libavcodec/vdpau_h264.c
+++ b/libavcodec/vdpau_h264.c
@@ -27,6 +27,7 @@
 #include "internal.h"
 #include "h264dec.h"
 #include "h264_ps.h"
+#include "hwaccel.h"
 #include "mpegutils.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -273,4 +274,5 @@ AVHWAccel ff_h264_vdpau_hwaccel = {
     .init           = vdpau_h264_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vdpau_hevc.c b/libavcodec/vdpau_hevc.c
index ce2610f67f20e8f4120bcf4181dad26af2b0f1f8..ee93b3a5e8bd6904d1bfa1626964434463f986c3 100644 (file)
--- a/libavcodec/vdpau_hevc.c
+++ b/libavcodec/vdpau_hevc.c
@@ -25,6 +25,7 @@
 #include "avcodec.h"
 #include "internal.h"
 #include "hevc.h"
+#include "hwaccel.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
 
@@ -423,4 +424,5 @@ AVHWAccel ff_hevc_vdpau_hwaccel = {
     .init           = vdpau_hevc_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vdpau_mpeg12.c b/libavcodec/vdpau_mpeg12.c
index 3ac2cb827defc702a1b31423d1d744346875436e..b657007ee771022600243edfd51888be18783cfc 100644 (file)
--- a/libavcodec/vdpau_mpeg12.c
+++ b/libavcodec/vdpau_mpeg12.c
@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "mpegvideo.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -114,6 +115,7 @@ AVHWAccel ff_mpeg1_vdpau_hwaccel = {
     .init           = vdpau_mpeg1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -148,5 +150,6 @@ AVHWAccel ff_mpeg2_vdpau_hwaccel = {
     .init           = vdpau_mpeg2_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif

diff --git a/libavcodec/vdpau_mpeg4.c b/libavcodec/vdpau_mpeg4.c
index 46a00cb27cb998bc5e03d42b3877e380912e7b94..bbdd843a449c7ca963620b203b38ecd5d59cdd51 100644 (file)
--- a/libavcodec/vdpau_mpeg4.c
+++ b/libavcodec/vdpau_mpeg4.c
@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "mpeg4video.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -121,4 +122,5 @@ AVHWAccel ff_mpeg4_vdpau_hwaccel = {
     .init           = vdpau_mpeg4_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };

diff --git a/libavcodec/vdpau_vc1.c b/libavcodec/vdpau_vc1.c
index ffd6505d13f476f0bfc230999dbf1fdf2800e02f..665a2333f45a375e839357843cc626dde2938007 100644 (file)
--- a/libavcodec/vdpau_vc1.c
+++ b/libavcodec/vdpau_vc1.c
@@ -24,6 +24,7 @@
 #include <vdpau/vdpau.h>
 
 #include "avcodec.h"
+#include "hwaccel.h"
 #include "vc1.h"
 #include "vdpau.h"
 #include "vdpau_internal.h"
@@ -147,6 +148,7 @@ AVHWAccel ff_wmv3_vdpau_hwaccel = {
     .init           = vdpau_vc1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };
 #endif
 
@@ -162,4 +164,5 @@ AVHWAccel ff_vc1_vdpau_hwaccel = {
     .init           = vdpau_vc1_init,
     .uninit         = ff_vdpau_common_uninit,
     .priv_data_size = sizeof(VDPAUContext),
+    .caps_internal  = HWACCEL_CAP_ASYNC_SAFE,
 };