Unify VA-API initialization.

[nageru] / futatabi / vaapi_jpeg_decoder.cpp

diff --git a/futatabi/vaapi_jpeg_decoder.cpp b/futatabi/vaapi_jpeg_decoder.cpp
index 452e61c3d13647b3c8f3204f75d9d882aa715bdd..0540db7da82061ff314ad26bdba2c76cb2cdca5a 100644 (file)
--- a/futatabi/vaapi_jpeg_decoder.cpp
+++ b/futatabi/vaapi_jpeg_decoder.cpp
@@ -5,6 +5,7 @@
 #include "jpeglib_error_wrapper.h"
 #include "pbo_pool.h"
 #include "shared/memcpy_interleaved.h"
+#include "shared/va_display.h"
 
 #include <X11/Xlib.h>
 #include <assert.h>
@@ -138,77 +139,12 @@ private:
        bool committed = false;
 };
 
-unique_ptr<VADisplayWithCleanup> va_open_display(const string &va_display)
+static unique_ptr<VADisplayWithCleanup> try_open_va_mjpeg(const string &va_display)
 {
-       if (va_display.empty() || va_display[0] != '/') {  // An X display.
-               Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str());
-               if (x11_display == nullptr) {
-                       fprintf(stderr, "error: can't connect to X server!\n");
-                       return nullptr;
-               }
-
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->x11_display = x11_display;
-               ret->va_dpy = vaGetDisplay(x11_display);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       } else {  // A DRM node on the filesystem (e.g. /dev/dri/renderD128).
-               int drm_fd = open(va_display.c_str(), O_RDWR);
-               if (drm_fd == -1) {
-                       perror(va_display.c_str());
-                       return nullptr;
-               }
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->drm_fd = drm_fd;
-               ret->va_dpy = vaGetDisplayDRM(drm_fd);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       }
-}
-
-unique_ptr<VADisplayWithCleanup> try_open_va(const string &va_display, string *error)
-{
-       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
-       if (va_dpy == nullptr) {
-               if (error)
-                       *error = "Opening VA display failed";
-               return nullptr;
-       }
-       int major_ver, minor_ver;
-       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
-               if (error != nullptr)
-                       *error = buf;
-               return nullptr;
-       }
-
-       int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy);
-       unique_ptr<VAEntrypoint[]> entrypoints(new VAEntrypoint[num_entrypoints]);
-       if (entrypoints == nullptr) {
-               if (error != nullptr)
-                       *error = "Failed to allocate memory for VA entry points";
-               return nullptr;
-       }
-
-       vaQueryConfigEntrypoints(va_dpy->va_dpy, VAProfileJPEGBaseline, entrypoints.get(), &num_entrypoints);
-       for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) {
-               if (entrypoints[slice_entrypoint] != VAEntrypointVLD) {
-                       continue;
-               }
-
-               // We found a usable decode, so return it.
-               return va_dpy;
-       }
-
-       if (error != nullptr)
-               *error = "Can't find VAEntrypointVLD for the JPEG profile";
-       return nullptr;
+       // Seemingly VA_FOURCC_422H is no good for vaGetImage(). :-/
+       return try_open_va(va_display, { VAProfileJPEGBaseline }, VAEntrypointVLD,
+               { { "4:2:2", VA_RT_FORMAT_YUV422, VA_FOURCC_UYVY, &config_id, &uyvy_format } },
+               /*chosen_profile=*/nullptr, /*error=*/nullptr);
 }
 
 string get_usable_va_display()
@@ -222,7 +158,7 @@ string get_usable_va_display()
        }
 
        // First try the default (ie., whatever $DISPLAY is set to).
-       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va("", nullptr);
+       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va_mjpeg("");
        if (va_dpy != nullptr) {
                if (need_env_reset) {
                        unsetenv("LIBVA_MESSAGING_LEVEL");
@@ -240,7 +176,7 @@ string get_usable_va_display()
        } else {
                for (size_t i = 0; i < g.gl_pathc; ++i) {
                        string path = g.gl_pathv[i];
-                       va_dpy = try_open_va(path, nullptr);
+                       va_dpy = try_open_va_mjpeg(path);
                        if (va_dpy != nullptr) {
                                fprintf(stderr, "Autodetected %s as a suitable replacement; using it.\n",
                                        path.c_str());
@@ -267,37 +203,11 @@ void init_jpeg_vaapi()
                return;
        }
 
-       va_dpy = try_open_va(dpy, nullptr);
+       va_dpy = try_open_va_mjpeg(dpy);
        if (va_dpy == nullptr) {
                return;
        }
 
-       VAConfigAttrib attr = { VAConfigAttribRTFormat, VA_RT_FORMAT_YUV422 };
-
-       VAStatus va_status = vaCreateConfig(va_dpy->va_dpy, VAProfileJPEGBaseline, VAEntrypointVLD,
-                                           &attr, 1, &config_id);
-       CHECK_VASTATUS(va_status, "vaCreateConfig");
-
-       int num_formats = vaMaxNumImageFormats(va_dpy->va_dpy);
-       assert(num_formats > 0);
-
-       unique_ptr<VAImageFormat[]> formats(new VAImageFormat[num_formats]);
-       va_status = vaQueryImageFormats(va_dpy->va_dpy, formats.get(), &num_formats);
-       CHECK_VASTATUS(va_status, "vaQueryImageFormats");
-
-       bool found = false;
-       for (int i = 0; i < num_formats; ++i) {
-               // Seemingly VA_FOURCC_422H is no good for vaGetImage(). :-/
-               if (formats[i].fourcc == VA_FOURCC_UYVY) {
-                       memcpy(&uyvy_format, &formats[i], sizeof(VAImageFormat));
-                       found = true;
-                       break;
-               }
-       }
-       if (!found) {
-               return;
-       }
-
        fprintf(stderr, "VA-API JPEG decoding initialized.\n");
        vaapi_jpeg_decoding_usable = true;
 }