From 7092d222bb356549ef453f9c6f0b21123fd8a2ce Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 16 Apr 2023 00:33:28 +0200 Subject: [PATCH] Support decoding FFmpeg via VDPAU. This fixes a bug where we hit the VAAPI-by-VDPAU emulation, which is seemingly bad and just hangs. Note that this means that if you have a VDPAU card with worse format support than your VA-API card, you'll now lose some formats. --- nageru/ffmpeg_capture.cpp | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index d1b7e74..0dea559 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -454,7 +454,8 @@ void FFmpegCapture::send_disconnected_frame() } } -AVPixelFormat get_vaapi_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) +template +AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) { for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { for (int i = 0;; ++i) { // Termination condition inside loop. @@ -464,7 +465,7 @@ AVPixelFormat get_vaapi_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) break; } if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && - config->device_type == AV_HWDEVICE_TYPE_VAAPI && + config->device_type == type && config->pix_fmt == *fmt_ptr) { return config->pix_fmt; } @@ -544,14 +545,24 @@ bool FFmpegCapture::play_video(const string &pathname) } // Seemingly, it's not too easy to make something that just initializes - // “whatever goes”, so we don't get VDPAU or CUDA here without enumerating - // through several different types. VA-API will do for now. + // “whatever goes”, so we don't get CUDA or VULKAN or whatever here + // without enumerating through several different types. + // VA-API and VDPAU will do for now. We prioritize VDPAU for the + // simple reason that there's a VA-API-via-VDPAU emulation for NVidia + // cards that seems to work, but just hangs when trying to transfer the frame. + // + // Note that we don't actually check codec support beforehand, + // so if you have a low-end VDPAU device but a high-end VA-API device, + // you lose out on the extra codec support from the latter. AVBufferRef *hw_device_ctx = nullptr; - if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) { - fprintf(stderr, "Failed to initialize VA-API for FFmpeg acceleration. Decoding video in software.\n"); - } else { + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) { + video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + video_codec_ctx->get_format = get_hw_format; + } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) { video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); - video_codec_ctx->get_format = get_vaapi_hw_format; + video_codec_ctx->get_format = get_hw_format; + } else { + fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n"); } if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) { @@ -899,7 +910,8 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo // Decode video, if we have a frame. int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get()); if (err == 0) { - if (video_avframe->format == AV_PIX_FMT_VAAPI) { + if (video_avframe->format == AV_PIX_FMT_VAAPI || + video_avframe->format == AV_PIX_FMT_VDPAU) { // Get the frame down to the CPU. (TODO: See if we can keep it // on the GPU all the way, since it will be going up again later. // However, this only works if the OpenGL GPU is the same one.) -- 2.39.2