]> git.sesse.net Git - nageru/commitdiff
Unify VA-API initialization.
authorSteinar H. Gunderson <sgunderson@bigfoot.com>
Thu, 21 May 2020 13:49:05 +0000 (15:49 +0200)
committerSteinar H. Gunderson <sgunderson@bigfoot.com>
Thu, 21 May 2020 13:50:53 +0000 (15:50 +0200)
We initialized VA-API and enumerated configs etc. in three different
ways for H.264 encoding (Nageru), MJPEG encoding (Nageru) and MJPEG
decoding (Futatabi). Unify them into one shared function, to reduce
the amount of duplication.

futatabi/vaapi_jpeg_decoder.cpp
futatabi/vaapi_jpeg_decoder.h
nageru/mjpeg_encoder.cpp
nageru/mjpeg_encoder.h
nageru/quicksync_encoder.cpp
shared/va_display.cpp
shared/va_display.h

index 452e61c3d13647b3c8f3204f75d9d882aa715bdd..0540db7da82061ff314ad26bdba2c76cb2cdca5a 100644 (file)
@@ -5,6 +5,7 @@
 #include "jpeglib_error_wrapper.h"
 #include "pbo_pool.h"
 #include "shared/memcpy_interleaved.h"
+#include "shared/va_display.h"
 
 #include <X11/Xlib.h>
 #include <assert.h>
@@ -138,77 +139,12 @@ private:
        bool committed = false;
 };
 
-unique_ptr<VADisplayWithCleanup> va_open_display(const string &va_display)
+static unique_ptr<VADisplayWithCleanup> try_open_va_mjpeg(const string &va_display)
 {
-       if (va_display.empty() || va_display[0] != '/') {  // An X display.
-               Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str());
-               if (x11_display == nullptr) {
-                       fprintf(stderr, "error: can't connect to X server!\n");
-                       return nullptr;
-               }
-
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->x11_display = x11_display;
-               ret->va_dpy = vaGetDisplay(x11_display);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       } else {  // A DRM node on the filesystem (e.g. /dev/dri/renderD128).
-               int drm_fd = open(va_display.c_str(), O_RDWR);
-               if (drm_fd == -1) {
-                       perror(va_display.c_str());
-                       return nullptr;
-               }
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->drm_fd = drm_fd;
-               ret->va_dpy = vaGetDisplayDRM(drm_fd);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       }
-}
-
-unique_ptr<VADisplayWithCleanup> try_open_va(const string &va_display, string *error)
-{
-       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
-       if (va_dpy == nullptr) {
-               if (error)
-                       *error = "Opening VA display failed";
-               return nullptr;
-       }
-       int major_ver, minor_ver;
-       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
-               if (error != nullptr)
-                       *error = buf;
-               return nullptr;
-       }
-
-       int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy);
-       unique_ptr<VAEntrypoint[]> entrypoints(new VAEntrypoint[num_entrypoints]);
-       if (entrypoints == nullptr) {
-               if (error != nullptr)
-                       *error = "Failed to allocate memory for VA entry points";
-               return nullptr;
-       }
-
-       vaQueryConfigEntrypoints(va_dpy->va_dpy, VAProfileJPEGBaseline, entrypoints.get(), &num_entrypoints);
-       for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) {
-               if (entrypoints[slice_entrypoint] != VAEntrypointVLD) {
-                       continue;
-               }
-
-               // We found a usable decode, so return it.
-               return va_dpy;
-       }
-
-       if (error != nullptr)
-               *error = "Can't find VAEntrypointVLD for the JPEG profile";
-       return nullptr;
+       // Seemingly VA_FOURCC_422H is no good for vaGetImage(). :-/
+       return try_open_va(va_display, { VAProfileJPEGBaseline }, VAEntrypointVLD,
+               { { "4:2:2", VA_RT_FORMAT_YUV422, VA_FOURCC_UYVY, &config_id, &uyvy_format } },
+               /*chosen_profile=*/nullptr, /*error=*/nullptr);
 }
 
 string get_usable_va_display()
@@ -222,7 +158,7 @@ string get_usable_va_display()
        }
 
        // First try the default (ie., whatever $DISPLAY is set to).
-       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va("", nullptr);
+       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va_mjpeg("");
        if (va_dpy != nullptr) {
                if (need_env_reset) {
                        unsetenv("LIBVA_MESSAGING_LEVEL");
@@ -240,7 +176,7 @@ string get_usable_va_display()
        } else {
                for (size_t i = 0; i < g.gl_pathc; ++i) {
                        string path = g.gl_pathv[i];
-                       va_dpy = try_open_va(path, nullptr);
+                       va_dpy = try_open_va_mjpeg(path);
                        if (va_dpy != nullptr) {
                                fprintf(stderr, "Autodetected %s as a suitable replacement; using it.\n",
                                        path.c_str());
@@ -267,37 +203,11 @@ void init_jpeg_vaapi()
                return;
        }
 
-       va_dpy = try_open_va(dpy, nullptr);
+       va_dpy = try_open_va_mjpeg(dpy);
        if (va_dpy == nullptr) {
                return;
        }
 
-       VAConfigAttrib attr = { VAConfigAttribRTFormat, VA_RT_FORMAT_YUV422 };
-
-       VAStatus va_status = vaCreateConfig(va_dpy->va_dpy, VAProfileJPEGBaseline, VAEntrypointVLD,
-                                           &attr, 1, &config_id);
-       CHECK_VASTATUS(va_status, "vaCreateConfig");
-
-       int num_formats = vaMaxNumImageFormats(va_dpy->va_dpy);
-       assert(num_formats > 0);
-
-       unique_ptr<VAImageFormat[]> formats(new VAImageFormat[num_formats]);
-       va_status = vaQueryImageFormats(va_dpy->va_dpy, formats.get(), &num_formats);
-       CHECK_VASTATUS(va_status, "vaQueryImageFormats");
-
-       bool found = false;
-       for (int i = 0; i < num_formats; ++i) {
-               // Seemingly VA_FOURCC_422H is no good for vaGetImage(). :-/
-               if (formats[i].fourcc == VA_FOURCC_UYVY) {
-                       memcpy(&uyvy_format, &formats[i], sizeof(VAImageFormat));
-                       found = true;
-                       break;
-               }
-       }
-       if (!found) {
-               return;
-       }
-
        fprintf(stderr, "VA-API JPEG decoding initialized.\n");
        vaapi_jpeg_decoding_usable = true;
 }
index 4182cfc3d0405c66208093d0de540d360861f83e..7ac799253ec05024a6da2bbfa6ca223341402e7a 100644 (file)
@@ -8,14 +8,6 @@
 
 struct Frame;
 
-struct VADisplayWithCleanup {
-       ~VADisplayWithCleanup();
-
-       VADisplay va_dpy;
-       Display *x11_display = nullptr;
-       int drm_fd = -1;
-};
-std::unique_ptr<VADisplayWithCleanup> va_open_display(const std::string &va_display);  // Can return nullptr on failure.
 std::string get_usable_va_display();
 
 void init_jpeg_vaapi();
index 857346db7592d67a79305cef7a1c643b167912bf..2cd1607bac6e1f6055439e2f62a3407135341a10 100644 (file)
@@ -222,7 +222,13 @@ MJPEGEncoder::MJPEGEncoder(HTTPD *httpd, const string &va_display)
 
        // Initialize VA-API.
        string error;
-       va_dpy = try_open_va(va_display, &error, &config_id_422, &config_id_420);
+       va_dpy = try_open_va(va_display, { VAProfileJPEGBaseline }, VAEntrypointEncPicture,
+               {
+                       { "4:2:2", VA_RT_FORMAT_YUV422, VA_FOURCC_UYVY, &config_id_422, &uyvy_format },
+                       // We'd prefer VA_FOURCC_I420, but it's not supported by Intel's driver.
+                       { "4:2:0", VA_RT_FORMAT_YUV420, VA_FOURCC_NV12, &config_id_420, &nv12_format }
+               },
+               /*chosen_profile=*/nullptr, &error);
        if (va_dpy == nullptr) {
                fprintf(stderr, "Could not initialize VA-API for MJPEG encoding: %s. JPEGs will be encoded in software if needed.\n", error.c_str());
        }
@@ -271,87 +277,6 @@ void MJPEGEncoder::stop()
        }
 }
 
-unique_ptr<VADisplayWithCleanup> MJPEGEncoder::try_open_va(const string &va_display, string *error, VAConfigID *config_id_422, VAConfigID *config_id_420)
-{
-       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
-       if (va_dpy == nullptr) {
-               if (error) *error = "Opening VA display failed";
-               return nullptr;
-       }
-       int major_ver, minor_ver;
-       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
-               if (error != nullptr) *error = buf;
-               return nullptr;
-       }
-
-       {
-               VAConfigAttrib attr = { VAConfigAttribRTFormat, VA_RT_FORMAT_YUV422 };
-               va_status = vaCreateConfig(va_dpy->va_dpy, VAProfileJPEGBaseline, VAEntrypointEncPicture,
-                       &attr, 1, config_id_422);
-               if (va_status == VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT) {
-                       if (error != nullptr) *error = "No 4:2:2 hardware support";
-                       return nullptr;
-               } else if (va_status != VA_STATUS_SUCCESS) {
-                       char buf[256];
-                       snprintf(buf, sizeof(buf), "vaCreateConfig() for 4:2:2 failed with status %d\n", va_status);
-                       if (error != nullptr) *error = buf;
-                       return nullptr;
-               }
-       }
-       {
-               VAConfigAttrib attr = { VAConfigAttribRTFormat, VA_RT_FORMAT_YUV420 };
-               va_status = vaCreateConfig(va_dpy->va_dpy, VAProfileJPEGBaseline, VAEntrypointEncPicture,
-                       &attr, 1, config_id_420);
-               if (va_status == VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT) {
-                       if (error != nullptr) *error = "No 4:2:0 hardware support";
-                       return nullptr;
-               } else if (va_status != VA_STATUS_SUCCESS) {
-                       char buf[256];
-                       snprintf(buf, sizeof(buf), "vaCreateConfig() for 4:2:0 failed with status %d\n", va_status);
-                       if (error != nullptr) *error = buf;
-                       return nullptr;
-               }
-       }
-
-       // TODO: Unify with the code in Futatabi.
-       int num_formats = vaMaxNumImageFormats(va_dpy->va_dpy);
-       assert(num_formats > 0);
-
-       unique_ptr<VAImageFormat[]> formats(new VAImageFormat[num_formats]);
-       va_status = vaQueryImageFormats(va_dpy->va_dpy, formats.get(), &num_formats);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaQueryImageFormats() failed with status %d\n", va_status);
-               if (error != nullptr) *error = buf;
-               return nullptr;
-       }
-
-       bool uyvy_found = false, nv12_found = false;
-       for (int i = 0; i < num_formats; ++i) {
-               if (formats[i].fourcc == VA_FOURCC_UYVY) {
-                       memcpy(&uyvy_format, &formats[i], sizeof(VAImageFormat));
-                       uyvy_found = true;
-               }
-               if (formats[i].fourcc == VA_FOURCC_NV12) {
-                       memcpy(&nv12_format, &formats[i], sizeof(VAImageFormat));
-                       nv12_found = true;
-               }
-       }
-       if (!uyvy_found) {
-               if (error != nullptr) *error = "UYVY format not found";
-               return nullptr;
-       }
-       if (!nv12_found) {
-               if (error != nullptr) *error = "NV12 format not found";
-               return nullptr;
-       }
-
-       return va_dpy;
-}
-
 namespace {
 
 bool is_uyvy(RefCountedFrame frame)
@@ -874,7 +799,6 @@ void MJPEGEncoder::encode_jpeg_va(QueuedFrame &&qf)
                        resources = get_va_resources(width, height, VA_FOURCC_UYVY);
                } else {
                        assert(is_i420(qf.frame));
-                       // We'd prefer VA_FOURCC_I420, but it's not supported by Intel's driver.
                        resources = get_va_resources(width, height, VA_FOURCC_NV12);
                }
                release = ReleaseVAResources(this, resources);
index 93394db3a5cc476b12c0935a8e4bd0e93d9641a8..330784bd94a30121cf30f9b3c7a5bfb1d48d2883 100644 (file)
@@ -190,8 +190,6 @@ private:
        VAResources get_va_resources(unsigned width, unsigned height, uint32_t fourcc);
        void release_va_resources(VAResources resources);
 
-       static std::unique_ptr<VADisplayWithCleanup> try_open_va(const std::string &va_display, std::string *error, VAConfigID *config_id_422, VAConfigID *config_id_420);
-
        uint8_t *tmp_y, *tmp_cbcr, *tmp_cb, *tmp_cr;  // Private to the encoder thread. Used by the libjpeg backend only.
 
        std::atomic<int64_t> metric_mjpeg_frames_zero_size_dropped{0};
index 7ec9f71c5eced50096af99bd4baed2a433cd8a2b..e22bab39aa9827283d72df23cb2c4338b68c551f 100644 (file)
@@ -728,88 +728,16 @@ void QuickSyncEncoderImpl::enable_zerocopy_if_possible()
        global_flags.use_zerocopy = use_zerocopy;
 }
 
-unique_ptr<VADisplayWithCleanup> va_open_display(const string &va_display)
+static unique_ptr<VADisplayWithCleanup> try_open_va_h264(const string &va_display, VAProfile *h264_profile, string *error)
 {
-       if (va_display.empty() || va_display[0] != '/') {  // An X display.
-               Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str());
-               if (x11_display == nullptr) {
-                       fprintf(stderr, "error: can't connect to X server!\n");
-                       return nullptr;
-               }
-
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->x11_display = x11_display;
-               ret->can_use_zerocopy = true;
-               ret->va_dpy = vaGetDisplay(x11_display);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       } else {  // A DRM node on the filesystem (e.g. /dev/dri/renderD128).
-               int drm_fd = open(va_display.c_str(), O_RDWR);
-               if (drm_fd == -1) {
-                       perror(va_display.c_str());
-                       return NULL;
-               }
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->drm_fd = drm_fd;
-               ret->can_use_zerocopy = false;
-               ret->va_dpy = vaGetDisplayDRM(drm_fd);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       }
-}
-
-unique_ptr<VADisplayWithCleanup> try_open_va(const string &va_display, VAProfile *h264_profile, string *error)
-{
-       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
-       if (va_dpy == nullptr) {
-               if (error) *error = "Opening VA display failed";
-               return nullptr;
-       }
-       int major_ver, minor_ver;
-       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
-               if (error != nullptr) *error = buf;
-               return nullptr;
-       }
-
-       int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy);
-       unique_ptr<VAEntrypoint[]> entrypoints(new VAEntrypoint[num_entrypoints]);
-       if (entrypoints == nullptr) {
-               if (error != nullptr) *error = "Failed to allocate memory for VA entry points";
-               return nullptr;
-       }
-
-       // Try the profiles from highest to lowest until we find one that can be encoded.
-       constexpr VAProfile profile_list[] = { VAProfileH264High, VAProfileH264Main, VAProfileH264ConstrainedBaseline };
-       for (unsigned i = 0; i < sizeof(profile_list) / sizeof(profile_list[0]); ++i) {
-               vaQueryConfigEntrypoints(va_dpy->va_dpy, profile_list[i], entrypoints.get(), &num_entrypoints);
-               for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) {
-                       if (entrypoints[slice_entrypoint] != VAEntrypointEncSlice) {
-                               continue;
-                       }
-
-                       // We found a usable encoder, so return it.
-                       if (h264_profile != nullptr) {
-                               *h264_profile = profile_list[i];
-                       }
-                       return va_dpy;
-               }
-       }
-
-       if (error != nullptr) *error = "Can't find VAEntrypointEncSlice for H264 profiles";
-       return nullptr;
+       return try_open_va(va_display, { VAProfileH264High, VAProfileH264Main, VAProfileH264ConstrainedBaseline },
+               VAEntrypointEncSlice, /*desired_configs=*/{}, h264_profile, error);
 }
 
 int QuickSyncEncoderImpl::init_va(const string &va_display)
 {
     string error;
-    va_dpy = try_open_va(va_display, &h264_profile, &error);
+    va_dpy = try_open_va_h264(va_display, &h264_profile, &error);
     if (va_dpy == nullptr) {
        fprintf(stderr, "error: %s\n", error.c_str());
         abort();
@@ -2140,7 +2068,7 @@ string QuickSyncEncoder::get_usable_va_display()
        }
 
        // First try the default (ie., whatever $DISPLAY is set to).
-       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va("", nullptr, nullptr);
+       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va_h264("", nullptr, nullptr);
        if (va_dpy != nullptr) {
                if (need_env_reset) {
                        unsetenv("LIBVA_MESSAGING_LEVEL");
@@ -2158,7 +2086,7 @@ string QuickSyncEncoder::get_usable_va_display()
        } else {
                for (size_t i = 0; i < g.gl_pathc; ++i) {
                        string path = g.gl_pathv[i];
-                       va_dpy = try_open_va(path, nullptr, nullptr);
+                       va_dpy = try_open_va_h264(path, nullptr, nullptr);
                        if (va_dpy != nullptr) {
                                fprintf(stderr, "Autodetected %s as a suitable replacement; using it.\n",
                                        path.c_str());
index 36c8f174acf3072c0927df46a0571cd7650856be..c74d75940dea1c1e4036e4c93403162510571d46 100644 (file)
@@ -1,5 +1,16 @@
-#include "shared/va_display.h"
+#include "va_display.h"
+#include <assert.h>
+#include <fcntl.h>
+#include <string.h>
 #include <unistd.h>
+#include <va/va.h>
+#include <va/va_drm.h>
+#include <va/va_x11.h>
+
+#include <string>
+#include <vector>
+
+using namespace std;
 
 VADisplayWithCleanup::~VADisplayWithCleanup()
 {
@@ -14,3 +25,131 @@ VADisplayWithCleanup::~VADisplayWithCleanup()
        }
 }
 
+unique_ptr<VADisplayWithCleanup> va_open_display(const string &va_display)
+{
+       if (va_display.empty() || va_display[0] != '/') {  // An X display.
+               Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str());
+               if (x11_display == nullptr) {
+                       fprintf(stderr, "error: can't connect to X server!\n");
+                       return nullptr;
+               }
+
+               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
+               ret->x11_display = x11_display;
+               ret->can_use_zerocopy = true;
+               ret->va_dpy = vaGetDisplay(x11_display);
+               if (ret->va_dpy == nullptr) {
+                       return nullptr;
+               }
+               return ret;
+       } else {  // A DRM node on the filesystem (e.g. /dev/dri/renderD128).
+               int drm_fd = open(va_display.c_str(), O_RDWR);
+               if (drm_fd == -1) {
+                       perror(va_display.c_str());
+                       return NULL;
+               }
+               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
+               ret->drm_fd = drm_fd;
+               ret->can_use_zerocopy = false;
+               ret->va_dpy = vaGetDisplayDRM(drm_fd);
+               if (ret->va_dpy == nullptr) {
+                       return nullptr;
+               }
+               return ret;
+       }
+}
+
+unique_ptr<VADisplayWithCleanup> try_open_va(
+       const string &va_display, const vector<VAProfile> &desired_profiles, VAEntrypoint entrypoint,
+       const vector<ConfigRequest> &desired_configs, VAProfile *chosen_profile, string *error)
+{
+       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
+       if (va_dpy == nullptr) {
+               if (error) *error = "Opening VA display failed";
+               return nullptr;
+       }
+       int major_ver, minor_ver;
+       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
+       if (va_status != VA_STATUS_SUCCESS) {
+               char buf[256];
+               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
+               if (error != nullptr) *error = buf;
+               return nullptr;
+       }
+
+       int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy);
+       unique_ptr<VAEntrypoint[]> entrypoints(new VAEntrypoint[num_entrypoints]);
+       if (entrypoints == nullptr) {
+               if (error != nullptr) *error = "Failed to allocate memory for VA entry points";
+               return nullptr;
+       }
+
+       // Try the profiles from highest to lowest until we find one that can be used.
+       VAProfile found_profile = VAProfileNone;
+       for (VAProfile profile : desired_profiles) {
+               vaQueryConfigEntrypoints(va_dpy->va_dpy, profile, entrypoints.get(), &num_entrypoints);
+               for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) {
+                       if (entrypoints[slice_entrypoint] != entrypoint) {
+                               continue;
+                       }
+
+                       // We found a usable encoder/decoder, so return it.
+                       if (chosen_profile != nullptr) {
+                               *chosen_profile = profile;
+                       }
+                       found_profile = profile;
+                       break;
+               }
+               if (found_profile != VAProfileNone) {
+                       break;
+               }
+       }
+       if (!found_profile) {
+               if (error != nullptr) *error = "Can't find entry points for suitable codec profile";
+               return nullptr;
+       }
+
+       int num_formats = vaMaxNumImageFormats(va_dpy->va_dpy);
+       assert(num_formats > 0);
+
+       unique_ptr<VAImageFormat[]> formats(new VAImageFormat[num_formats]);
+       va_status = vaQueryImageFormats(va_dpy->va_dpy, formats.get(), &num_formats);
+       if (va_status != VA_STATUS_SUCCESS) {
+               char buf[256];
+               snprintf(buf, sizeof(buf), "vaQueryImageFormats() failed with status %d\n", va_status);
+               if (error != nullptr) *error = buf;
+               return nullptr;
+       }
+
+       for (const ConfigRequest &request : desired_configs) {
+               // Create the config.
+               VAConfigAttrib attr = { VAConfigAttribRTFormat, request.rt_format };
+               va_status = vaCreateConfig(va_dpy->va_dpy, found_profile, entrypoint,
+                       &attr, 1, request.config_id);
+               if (va_status == VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT) {
+                       if (error != nullptr) *error = "No " + request.name + " hardware support";
+                       return nullptr;
+               } else if (va_status != VA_STATUS_SUCCESS) {
+                       char buf[256];
+                       snprintf(buf, sizeof(buf), "vaCreateConfig() for %s failed with status %d\n", request.name.c_str(), va_status);
+                       if (error != nullptr) *error = buf;
+                       return nullptr;
+               }
+
+               // Find out which image format we're going to be using.
+               bool format_found = false;
+               for (int i = 0; i < num_formats; ++i) {
+                       if (formats[i].fourcc == request.fourcc) {
+                               memcpy(request.image_format, &formats[i], sizeof(VAImageFormat));
+                               format_found = true;
+                               break;
+                       }
+               }
+               if (!format_found) {
+                       if (error != nullptr) *error = "Format for " + request.name + " not found";
+                       return nullptr;
+               }
+       }
+
+       return va_dpy;
+}
index 395ecbb5f1f007412e2605440bb1a59c88502d83..5809b65c3328596df2a04018cb85d5ad3a4817cb 100644 (file)
@@ -5,7 +5,8 @@
 #include <X11/Xlib.h>
 
 #include <memory>
-#include <string.h>
+#include <string>
+#include <vector>
 
 struct VADisplayWithCleanup {
        ~VADisplayWithCleanup();
@@ -15,6 +16,17 @@ struct VADisplayWithCleanup {
        bool can_use_zerocopy = true;  // For H.264 encoding in Nageru.
        int drm_fd = -1;
 };
-std::unique_ptr<VADisplayWithCleanup> va_open_display(const std::string &va_display);  // Can return nullptr on failure.
+
+struct ConfigRequest {
+        std::string name;  // For error texts only.
+        uint32_t rt_format, fourcc;
+
+        // Output.
+        VAConfigID *config_id;
+        VAImageFormat *image_format;
+};
+std::unique_ptr<VADisplayWithCleanup> try_open_va(
+        const std::string &va_display, const std::vector<VAProfile> &desired_profiles, VAEntrypoint entrypoint,
+        const std::vector<ConfigRequest> &desired_configs, VAProfile *chosen_profile, std::string *error);  // Can return nullptr on failure.
 
 #endif  // !defined(_VA_DISPLAY_H)