Unify VA-API initialization.

[nageru] / nageru / quicksync_encoder.cpp

diff --git a/nageru/quicksync_encoder.cpp b/nageru/quicksync_encoder.cpp
index 7ec9f71c5eced50096af99bd4baed2a433cd8a2b..e22bab39aa9827283d72df23cb2c4338b68c551f 100644 (file)
--- a/nageru/quicksync_encoder.cpp
+++ b/nageru/quicksync_encoder.cpp
@@ -728,88 +728,16 @@ void QuickSyncEncoderImpl::enable_zerocopy_if_possible()
        global_flags.use_zerocopy = use_zerocopy;
 }
 
-unique_ptr<VADisplayWithCleanup> va_open_display(const string &va_display)
+static unique_ptr<VADisplayWithCleanup> try_open_va_h264(const string &va_display, VAProfile *h264_profile, string *error)
 {
-       if (va_display.empty() || va_display[0] != '/') {  // An X display.
-               Display *x11_display = XOpenDisplay(va_display.empty() ? nullptr : va_display.c_str());
-               if (x11_display == nullptr) {
-                       fprintf(stderr, "error: can't connect to X server!\n");
-                       return nullptr;
-               }
-
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->x11_display = x11_display;
-               ret->can_use_zerocopy = true;
-               ret->va_dpy = vaGetDisplay(x11_display);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       } else {  // A DRM node on the filesystem (e.g. /dev/dri/renderD128).
-               int drm_fd = open(va_display.c_str(), O_RDWR);
-               if (drm_fd == -1) {
-                       perror(va_display.c_str());
-                       return NULL;
-               }
-               unique_ptr<VADisplayWithCleanup> ret(new VADisplayWithCleanup);
-               ret->drm_fd = drm_fd;
-               ret->can_use_zerocopy = false;
-               ret->va_dpy = vaGetDisplayDRM(drm_fd);
-               if (ret->va_dpy == nullptr) {
-                       return nullptr;
-               }
-               return ret;
-       }
-}
-
-unique_ptr<VADisplayWithCleanup> try_open_va(const string &va_display, VAProfile *h264_profile, string *error)
-{
-       unique_ptr<VADisplayWithCleanup> va_dpy = va_open_display(va_display);
-       if (va_dpy == nullptr) {
-               if (error) *error = "Opening VA display failed";
-               return nullptr;
-       }
-       int major_ver, minor_ver;
-       VAStatus va_status = vaInitialize(va_dpy->va_dpy, &major_ver, &minor_ver);
-       if (va_status != VA_STATUS_SUCCESS) {
-               char buf[256];
-               snprintf(buf, sizeof(buf), "vaInitialize() failed with status %d\n", va_status);
-               if (error != nullptr) *error = buf;
-               return nullptr;
-       }
-
-       int num_entrypoints = vaMaxNumEntrypoints(va_dpy->va_dpy);
-       unique_ptr<VAEntrypoint[]> entrypoints(new VAEntrypoint[num_entrypoints]);
-       if (entrypoints == nullptr) {
-               if (error != nullptr) *error = "Failed to allocate memory for VA entry points";
-               return nullptr;
-       }
-
-       // Try the profiles from highest to lowest until we find one that can be encoded.
-       constexpr VAProfile profile_list[] = { VAProfileH264High, VAProfileH264Main, VAProfileH264ConstrainedBaseline };
-       for (unsigned i = 0; i < sizeof(profile_list) / sizeof(profile_list[0]); ++i) {
-               vaQueryConfigEntrypoints(va_dpy->va_dpy, profile_list[i], entrypoints.get(), &num_entrypoints);
-               for (int slice_entrypoint = 0; slice_entrypoint < num_entrypoints; slice_entrypoint++) {
-                       if (entrypoints[slice_entrypoint] != VAEntrypointEncSlice) {
-                               continue;
-                       }
-
-                       // We found a usable encoder, so return it.
-                       if (h264_profile != nullptr) {
-                               *h264_profile = profile_list[i];
-                       }
-                       return va_dpy;
-               }
-       }
-
-       if (error != nullptr) *error = "Can't find VAEntrypointEncSlice for H264 profiles";
-       return nullptr;
+       return try_open_va(va_display, { VAProfileH264High, VAProfileH264Main, VAProfileH264ConstrainedBaseline },
+               VAEntrypointEncSlice, /*desired_configs=*/{}, h264_profile, error);
 }
 
 int QuickSyncEncoderImpl::init_va(const string &va_display)
 {
     string error;
-    va_dpy = try_open_va(va_display, &h264_profile, &error);
+    va_dpy = try_open_va_h264(va_display, &h264_profile, &error);
     if (va_dpy == nullptr) {
        fprintf(stderr, "error: %s\n", error.c_str());
         abort();
@@ -2140,7 +2068,7 @@ string QuickSyncEncoder::get_usable_va_display()
        }
 
        // First try the default (ie., whatever $DISPLAY is set to).
-       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va("", nullptr, nullptr);
+       unique_ptr<VADisplayWithCleanup> va_dpy = try_open_va_h264("", nullptr, nullptr);
        if (va_dpy != nullptr) {
                if (need_env_reset) {
                        unsetenv("LIBVA_MESSAGING_LEVEL");
@@ -2158,7 +2086,7 @@ string QuickSyncEncoder::get_usable_va_display()
        } else {
                for (size_t i = 0; i < g.gl_pathc; ++i) {
                        string path = g.gl_pathv[i];
-                       va_dpy = try_open_va(path, nullptr, nullptr);
+                       va_dpy = try_open_va_h264(path, nullptr, nullptr);
                        if (va_dpy != nullptr) {
                                fprintf(stderr, "Autodetected %s as a suitable replacement; using it.\n",
                                        path.c_str());