From 3fb867fa45d24d5f67ee1b070533f46c1f1f5462 Mon Sep 17 00:00:00 2001
From: "Steinar H. Gunderson" <sgunderson@bigfoot.com>
Date: Sun, 16 Apr 2023 01:31:44 +0200
Subject: [PATCH] Improve error messaging when VAAPI/VDPAU pixel format
 selection fails.

---
 nageru/ffmpeg_capture.cpp | 46 ++++++++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp
index 0dea559..863f4fb 100644
--- a/nageru/ffmpeg_capture.cpp
+++ b/nageru/ffmpeg_capture.cpp
@@ -26,6 +26,7 @@ extern "C" {
 #include <cstdint>
 #include <utility>
 #include <vector>
+#include <unordered_set>
 
 #include <Eigen/Core>
 #include <Eigen/LU>
@@ -457,19 +458,44 @@ void FFmpegCapture::send_disconnected_frame()
 template<AVHWDeviceType type>
 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
 {
-	for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
-		for (int i = 0;; ++i) {  // Termination condition inside loop.
-			const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
-			if (config == nullptr) {  // End of list.
-				fprintf(stderr, "Decoder %s does not support device.\n", ctx->codec->name);
-				break;
-			}
-			if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
-			    config->device_type == type &&
-			    config->pix_fmt == *fmt_ptr) {
+	bool found_config_of_right_type = false;
+	for (int i = 0;; ++i) {  // Termination condition inside loop.
+		const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
+		if (config == nullptr) {  // End of list.
+			break;
+		}
+		if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
+		    config->device_type != type) {
+			// Not interesting for us.
+			continue;
+		}
+
+		// We have a config of the right type, but does it actually support
+		// the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
+		// is to just replace the pixel format with a software-decoded one,
+		// such as yuv420p.)
+		found_config_of_right_type = true;
+		for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+			if (config->pix_fmt == *fmt_ptr) {
+				fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
+					av_hwdevice_get_type_name(type), ctx->codec->name);
 				return config->pix_fmt;
 			}
 		}
+		fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
+		unordered_set<AVPixelFormat> seen;
+		for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+			if (!seen.count(*fmt_ptr)) {
+				fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
+				seen.insert(*fmt_ptr);
+			}
+		}
+		fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
+
+	}
+
+	if (!found_config_of_right_type) {
+		fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
 	}
 
 	// We found no VA-API formats, so take the best software format.
-- 
2.39.2