Support decoding FFmpeg videos via VA-API.

[nageru] / nageru / ffmpeg_capture.cpp

diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp
index c6824069aedc8786954531e3fb1f8f62deb52391..a667d4f5b033aca8f5393d90f0f73bee31a87e4d 100644 (file)
--- a/nageru/ffmpeg_capture.cpp
+++ b/nageru/ffmpeg_capture.cpp
@@ -452,6 +452,27 @@ void FFmpegCapture::send_disconnected_frame()
        }
 }
 
+AVPixelFormat get_vaapi_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
+{
+       for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+               for (int i = 0;; ++i) {  // Termination condition inside loop.
+                       const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
+                       if (config == nullptr) {  // End of list.
+                               fprintf(stderr, "Decoder %s does not support device.\n", ctx->codec->name);
+                               break;
+                       }
+                       if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
+                           config->device_type == AV_HWDEVICE_TYPE_VAAPI &&
+                           config->pix_fmt == *fmt_ptr) {
+                               return config->pix_fmt;
+                       }
+               }
+       }
+
+       // We found no VA-API formats, so take the best software format.
+       return fmt[0];
+}
+
 bool FFmpegCapture::play_video(const string &pathname)
 {
        // Note: Call before open, not after; otherwise, there's a race.
@@ -508,6 +529,7 @@ bool FFmpegCapture::play_video(const string &pathname)
        // Open video decoder.
        const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
        AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
+
        video_timebase = format_ctx->streams[video_stream_index]->time_base;
        AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
        if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
@@ -518,6 +540,18 @@ bool FFmpegCapture::play_video(const string &pathname)
                fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
                return false;
        }
+
+       // Seemingly, it's not too easy to make something that just initializes
+       // “whatever goes”, so we don't get VDPAU or CUDA here without enumerating
+       // through several different types. VA-API will do for now.
+       AVBufferRef *hw_device_ctx = nullptr;
+       if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) {
+               fprintf(stderr, "Failed to initialize VA-API for FFmpeg acceleration. Decoding video in software.\n");
+       } else {
+               video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+               video_codec_ctx->get_format = get_vaapi_hw_format;
+       }
+
        if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
                fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
                return false;
@@ -850,6 +884,19 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo
                // Decode video, if we have a frame.
                int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
                if (err == 0) {
+                       if (video_avframe->format == AV_PIX_FMT_VAAPI) {
+                               // Get the frame down to the CPU. (TODO: See if we can keep it
+                               // on the GPU all the way, since it will be going up again later.
+                               // However, this only works if the OpenGL GPU is the same one.)
+                               AVFrameWithDeleter sw_frame = av_frame_alloc_unique();
+                               int err = av_hwframe_transfer_data(sw_frame.get(), video_avframe.get(), 0);
+                               if (err != 0) {
+                                       fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
+                                       *error = true;
+                                       return AVFrameWithDeleter(nullptr);
+                               }
+                               video_avframe = move(sw_frame);
+                       }
                        frame_finished = true;
                        break;
                } else if (err != AVERROR(EAGAIN)) {