libavutil/hwcontext_dxva2: Add check for possible errors from GetAdapterDisplayModeEx

This prevents a possible crash in CreateDeviceEx when using faulty
response from GetAdapterDisplayModeEx and allows ffmpeg to fallback to
classic d3d9.

Signed-off-by: wm4 <nfxjfg@googlemail.com>

diff --git a/libavutil/hwcontext_dxva2.c b/libavutil/hwcontext_dxva2.c
index 2ddd4be7b1df8cd1c2362dd8f053c4cc649a83a9..44ebdbca9586a7f59ce2efa0a3aea9a3e066df5b 100644 (file)
--- a/libavutil/hwcontext_dxva2.c
+++ b/libavutil/hwcontext_dxva2.c
@@ -485,7 +485,11 @@ static int dxva2_device_create9ex(AVHWDeviceContext *ctx, UINT adapter)
     if (FAILED(hr))
         return AVERROR_UNKNOWN;
 
-    IDirect3D9Ex_GetAdapterDisplayModeEx(d3d9ex, adapter, &modeex, NULL);
+    hr = IDirect3D9Ex_GetAdapterDisplayModeEx(d3d9ex, adapter, &modeex, NULL);
+    if (FAILED(hr)) {
+        IDirect3D9Ex_Release(d3d9ex);
+        return AVERROR_UNKNOWN;
+    }
 
     d3dpp.BackBufferFormat = modeex.Format;