]> git.sesse.net Git - ffmpeg/blob - libavutil/hwcontext_dxva2.c
avutil/hwcontext_dxva2: Don't improperly free IDirect3DSurface9 objects
[ffmpeg] / libavutil / hwcontext_dxva2.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include <windows.h>
20
21 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < 0x0600
22 #undef _WIN32_WINNT
23 #define _WIN32_WINNT 0x0600
24 #endif
25 #define DXVA2API_USE_BITFIELDS
26 #define COBJMACROS
27
28 #include <d3d9.h>
29 #include <dxva2api.h>
30 #include <initguid.h>
31
32 #include "common.h"
33 #include "hwcontext.h"
34 #include "hwcontext_dxva2.h"
35 #include "hwcontext_internal.h"
36 #include "imgutils.h"
37 #include "pixdesc.h"
38 #include "pixfmt.h"
39
40 typedef IDirect3D9* WINAPI pDirect3DCreate9(UINT);
41 typedef HRESULT WINAPI pDirect3DCreate9Ex(UINT, IDirect3D9Ex **);
42 typedef HRESULT WINAPI pCreateDeviceManager9(UINT *, IDirect3DDeviceManager9 **);
43
44 #define FF_D3DCREATE_FLAGS (D3DCREATE_SOFTWARE_VERTEXPROCESSING | \
45                             D3DCREATE_MULTITHREADED | \
46                             D3DCREATE_FPU_PRESERVE)
47
48 static const D3DPRESENT_PARAMETERS dxva2_present_params = {
49     .Windowed         = TRUE,
50     .BackBufferWidth  = 640,
51     .BackBufferHeight = 480,
52     .BackBufferCount  = 0,
53     .SwapEffect       = D3DSWAPEFFECT_DISCARD,
54     .Flags            = D3DPRESENTFLAG_VIDEO,
55 };
56
57 typedef struct DXVA2Mapping {
58     uint32_t palette_dummy[256];
59 } DXVA2Mapping;
60
61 typedef struct DXVA2FramesContext {
62     IDirect3DSurface9 **surfaces_internal;
63     int              nb_surfaces_used;
64
65     HANDLE  device_handle;
66     IDirectXVideoAccelerationService *service;
67
68     D3DFORMAT format;
69 } DXVA2FramesContext;
70
71 typedef struct DXVA2DevicePriv {
72     HMODULE d3dlib;
73     HMODULE dxva2lib;
74
75     HANDLE device_handle;
76
77     IDirect3D9       *d3d9;
78     IDirect3DDevice9 *d3d9device;
79 } DXVA2DevicePriv;
80
81 static const struct {
82     D3DFORMAT d3d_format;
83     enum AVPixelFormat pix_fmt;
84 } supported_formats[] = {
85     { MKTAG('N', 'V', '1', '2'), AV_PIX_FMT_NV12 },
86     { MKTAG('P', '0', '1', '0'), AV_PIX_FMT_P010 },
87     { D3DFMT_P8,                 AV_PIX_FMT_PAL8 },
88 };
89
90 DEFINE_GUID(video_decoder_service,   0xfc51a551, 0xd5e7, 0x11d9, 0xaf, 0x55, 0x00, 0x05, 0x4e, 0x43, 0xff, 0x02);
91 DEFINE_GUID(video_processor_service, 0xfc51a552, 0xd5e7, 0x11d9, 0xaf, 0x55, 0x00, 0x05, 0x4e, 0x43, 0xff, 0x02);
92
93 static void dxva2_frames_uninit(AVHWFramesContext *ctx)
94 {
95     AVDXVA2DeviceContext *device_hwctx = ctx->device_ctx->hwctx;
96     AVDXVA2FramesContext *frames_hwctx = ctx->hwctx;
97     DXVA2FramesContext *s = ctx->internal->priv;
98     int i;
99
100     if (frames_hwctx->decoder_to_release)
101         IDirectXVideoDecoder_Release(frames_hwctx->decoder_to_release);
102
103     if (s->surfaces_internal) {
104         for (i = 0; i < frames_hwctx->nb_surfaces; i++) {
105             if (s->surfaces_internal[i])
106                 IDirect3DSurface9_Release(s->surfaces_internal[i]);
107         }
108     }
109     av_freep(&s->surfaces_internal);
110
111     if (s->service) {
112         IDirectXVideoAccelerationService_Release(s->service);
113         s->service = NULL;
114     }
115
116     if (s->device_handle != INVALID_HANDLE_VALUE) {
117         IDirect3DDeviceManager9_CloseDeviceHandle(device_hwctx->devmgr, s->device_handle);
118         s->device_handle = INVALID_HANDLE_VALUE;
119     }
120 }
121
122 static void dxva2_pool_release_dummy(void *opaque, uint8_t *data)
123 {
124     // important not to free anything here--data is a surface object
125     // associated with the call to CreateSurface(), and these surfaces are
126     // released in dxva2_frames_uninit()
127 }
128
129 static AVBufferRef *dxva2_pool_alloc(void *opaque, int size)
130 {
131     AVHWFramesContext      *ctx = (AVHWFramesContext*)opaque;
132     DXVA2FramesContext       *s = ctx->internal->priv;
133     AVDXVA2FramesContext *hwctx = ctx->hwctx;
134
135     if (s->nb_surfaces_used < hwctx->nb_surfaces) {
136         s->nb_surfaces_used++;
137         return av_buffer_create((uint8_t*)s->surfaces_internal[s->nb_surfaces_used - 1],
138                                 sizeof(*hwctx->surfaces), dxva2_pool_release_dummy, 0, 0);
139     }
140
141     return NULL;
142 }
143
144 static int dxva2_init_pool(AVHWFramesContext *ctx)
145 {
146     AVDXVA2FramesContext *frames_hwctx = ctx->hwctx;
147     AVDXVA2DeviceContext *device_hwctx = ctx->device_ctx->hwctx;
148     DXVA2FramesContext              *s = ctx->internal->priv;
149     int decode = (frames_hwctx->surface_type == DXVA2_VideoDecoderRenderTarget);
150
151     int i;
152     HRESULT hr;
153
154     if (ctx->initial_pool_size <= 0)
155         return 0;
156
157     hr = IDirect3DDeviceManager9_OpenDeviceHandle(device_hwctx->devmgr, &s->device_handle);
158     if (FAILED(hr)) {
159         av_log(ctx, AV_LOG_ERROR, "Failed to open device handle\n");
160         return AVERROR_UNKNOWN;
161     }
162
163     hr = IDirect3DDeviceManager9_GetVideoService(device_hwctx->devmgr,
164                                                  s->device_handle,
165                                                  decode ? &video_decoder_service : &video_processor_service,
166                                                  (void **)&s->service);
167     if (FAILED(hr)) {
168         av_log(ctx, AV_LOG_ERROR, "Failed to create the video service\n");
169         return AVERROR_UNKNOWN;
170     }
171
172     for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
173         if (ctx->sw_format == supported_formats[i].pix_fmt) {
174             s->format = supported_formats[i].d3d_format;
175             break;
176         }
177     }
178     if (i == FF_ARRAY_ELEMS(supported_formats)) {
179         av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n",
180                av_get_pix_fmt_name(ctx->sw_format));
181         return AVERROR(EINVAL);
182     }
183
184     s->surfaces_internal = av_mallocz_array(ctx->initial_pool_size,
185                                             sizeof(*s->surfaces_internal));
186     if (!s->surfaces_internal)
187         return AVERROR(ENOMEM);
188
189     hr = IDirectXVideoAccelerationService_CreateSurface(s->service,
190                                                         ctx->width, ctx->height,
191                                                         ctx->initial_pool_size - 1,
192                                                         s->format, D3DPOOL_DEFAULT, 0,
193                                                         frames_hwctx->surface_type,
194                                                         s->surfaces_internal, NULL);
195     if (FAILED(hr)) {
196         av_log(ctx, AV_LOG_ERROR, "Could not create the surfaces\n");
197         return AVERROR_UNKNOWN;
198     }
199
200     ctx->internal->pool_internal = av_buffer_pool_init2(sizeof(*s->surfaces_internal),
201                                                         ctx, dxva2_pool_alloc, NULL);
202     if (!ctx->internal->pool_internal)
203         return AVERROR(ENOMEM);
204
205     frames_hwctx->surfaces    = s->surfaces_internal;
206     frames_hwctx->nb_surfaces = ctx->initial_pool_size;
207
208     return 0;
209 }
210
211 static int dxva2_frames_init(AVHWFramesContext *ctx)
212 {
213     AVDXVA2FramesContext *hwctx = ctx->hwctx;
214     DXVA2FramesContext       *s = ctx->internal->priv;
215     int ret;
216
217     if (hwctx->surface_type != DXVA2_VideoDecoderRenderTarget &&
218         hwctx->surface_type != DXVA2_VideoProcessorRenderTarget) {
219         av_log(ctx, AV_LOG_ERROR, "Unknown surface type: %lu\n",
220                hwctx->surface_type);
221         return AVERROR(EINVAL);
222     }
223
224     s->device_handle = INVALID_HANDLE_VALUE;
225
226     /* init the frame pool if the caller didn't provide one */
227     if (!ctx->pool) {
228         ret = dxva2_init_pool(ctx);
229         if (ret < 0) {
230             av_log(ctx, AV_LOG_ERROR, "Error creating an internal frame pool\n");
231             return ret;
232         }
233     }
234
235     return 0;
236 }
237
238 static int dxva2_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
239 {
240     frame->buf[0] = av_buffer_pool_get(ctx->pool);
241     if (!frame->buf[0])
242         return AVERROR(ENOMEM);
243
244     frame->data[3] = frame->buf[0]->data;
245     frame->format  = AV_PIX_FMT_DXVA2_VLD;
246     frame->width   = ctx->width;
247     frame->height  = ctx->height;
248
249     return 0;
250 }
251
252 static int dxva2_transfer_get_formats(AVHWFramesContext *ctx,
253                                       enum AVHWFrameTransferDirection dir,
254                                       enum AVPixelFormat **formats)
255 {
256     enum AVPixelFormat *fmts;
257
258     fmts = av_malloc_array(2, sizeof(*fmts));
259     if (!fmts)
260         return AVERROR(ENOMEM);
261
262     fmts[0] = ctx->sw_format;
263     fmts[1] = AV_PIX_FMT_NONE;
264
265     *formats = fmts;
266
267     return 0;
268 }
269
270 static void dxva2_unmap_frame(AVHWFramesContext *ctx, HWMapDescriptor *hwmap)
271 {
272     IDirect3DSurface9 *surface = (IDirect3DSurface9*)hwmap->source->data[3];
273     IDirect3DSurface9_UnlockRect(surface);
274     av_freep(&hwmap->priv);
275 }
276
277 static int dxva2_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src,
278                            int flags)
279 {
280     IDirect3DSurface9 *surface = (IDirect3DSurface9*)src->data[3];
281     DXVA2Mapping      *map;
282     D3DSURFACE_DESC    surfaceDesc;
283     D3DLOCKED_RECT     LockedRect;
284     HRESULT            hr;
285     int i, err, nb_planes;
286     int lock_flags = 0;
287
288     nb_planes = av_pix_fmt_count_planes(dst->format);
289
290     hr = IDirect3DSurface9_GetDesc(surface, &surfaceDesc);
291     if (FAILED(hr)) {
292         av_log(ctx, AV_LOG_ERROR, "Error getting a surface description\n");
293         return AVERROR_UNKNOWN;
294     }
295
296     if (!(flags & AV_HWFRAME_MAP_WRITE))
297         lock_flags |= D3DLOCK_READONLY;
298     if (flags & AV_HWFRAME_MAP_OVERWRITE)
299         lock_flags |= D3DLOCK_DISCARD;
300
301     hr = IDirect3DSurface9_LockRect(surface, &LockedRect, NULL, lock_flags);
302     if (FAILED(hr)) {
303         av_log(ctx, AV_LOG_ERROR, "Unable to lock DXVA2 surface\n");
304         return AVERROR_UNKNOWN;
305     }
306
307     map = av_mallocz(sizeof(*map));
308     if (!map)
309         goto fail;
310
311     err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
312                                 dxva2_unmap_frame, map);
313     if (err < 0) {
314         av_freep(&map);
315         goto fail;
316     }
317
318     for (i = 0; i < nb_planes; i++)
319         dst->linesize[i] = LockedRect.Pitch;
320
321     av_image_fill_pointers(dst->data, dst->format, surfaceDesc.Height,
322                            (uint8_t*)LockedRect.pBits, dst->linesize);
323
324     if (dst->format == AV_PIX_FMT_PAL8)
325         dst->data[1] = (uint8_t*)map->palette_dummy;
326
327     return 0;
328 fail:
329     IDirect3DSurface9_UnlockRect(surface);
330     return err;
331 }
332
333 static int dxva2_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
334                                   const AVFrame *src)
335 {
336     AVFrame *map;
337     int ret;
338
339     if (src->format != ctx->sw_format)
340         return AVERROR(ENOSYS);
341
342     map = av_frame_alloc();
343     if (!map)
344         return AVERROR(ENOMEM);
345     map->format = dst->format;
346
347     ret = dxva2_map_frame(ctx, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE);
348     if (ret < 0)
349         goto fail;
350
351     av_image_copy(map->data, map->linesize, src->data, src->linesize,
352                   ctx->sw_format, src->width, src->height);
353
354 fail:
355     av_frame_free(&map);
356     return ret;
357 }
358
359 static int dxva2_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
360                                     const AVFrame *src)
361 {
362     AVFrame *map;
363     ptrdiff_t src_linesize[4], dst_linesize[4];
364     int ret, i;
365
366     if (dst->format != ctx->sw_format)
367         return AVERROR(ENOSYS);
368
369     map = av_frame_alloc();
370     if (!map)
371         return AVERROR(ENOMEM);
372     map->format = dst->format;
373
374     ret = dxva2_map_frame(ctx, map, src, AV_HWFRAME_MAP_READ);
375     if (ret < 0)
376         goto fail;
377
378     for (i = 0; i < 4; i++) {
379         dst_linesize[i] = dst->linesize[i];
380         src_linesize[i] = map->linesize[i];
381     }
382     av_image_copy_uc_from(dst->data, dst_linesize, map->data, src_linesize,
383                           ctx->sw_format, src->width, src->height);
384 fail:
385     av_frame_free(&map);
386     return ret;
387 }
388
389 static int dxva2_map_from(AVHWFramesContext *ctx,
390                           AVFrame *dst, const AVFrame *src, int flags)
391 {
392     int err;
393
394     if (dst->format != AV_PIX_FMT_NONE && dst->format != ctx->sw_format)
395         return AVERROR(ENOSYS);
396     dst->format = ctx->sw_format;
397
398     err = dxva2_map_frame(ctx, dst, src, flags);
399     if (err < 0)
400         return err;
401
402     err = av_frame_copy_props(dst, src);
403     if (err < 0)
404         return err;
405
406     return 0;
407 }
408
409 static void dxva2_device_free(AVHWDeviceContext *ctx)
410 {
411     AVDXVA2DeviceContext *hwctx = ctx->hwctx;
412     DXVA2DevicePriv       *priv = ctx->user_opaque;
413
414     if (hwctx->devmgr && priv->device_handle != INVALID_HANDLE_VALUE)
415         IDirect3DDeviceManager9_CloseDeviceHandle(hwctx->devmgr, priv->device_handle);
416
417     if (hwctx->devmgr)
418         IDirect3DDeviceManager9_Release(hwctx->devmgr);
419
420     if (priv->d3d9device)
421         IDirect3DDevice9_Release(priv->d3d9device);
422
423     if (priv->d3d9)
424         IDirect3D9_Release(priv->d3d9);
425
426     if (priv->d3dlib)
427         FreeLibrary(priv->d3dlib);
428
429     if (priv->dxva2lib)
430         FreeLibrary(priv->dxva2lib);
431
432     av_freep(&ctx->user_opaque);
433 }
434
435 static int dxva2_device_create9(AVHWDeviceContext *ctx, UINT adapter)
436 {
437     DXVA2DevicePriv *priv = ctx->user_opaque;
438     D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
439     D3DDISPLAYMODE d3ddm;
440     HRESULT hr;
441     pDirect3DCreate9 *createD3D = (pDirect3DCreate9 *)GetProcAddress(priv->d3dlib, "Direct3DCreate9");
442     if (!createD3D) {
443         av_log(ctx, AV_LOG_ERROR, "Failed to locate Direct3DCreate9\n");
444         return AVERROR_UNKNOWN;
445     }
446
447     priv->d3d9 = createD3D(D3D_SDK_VERSION);
448     if (!priv->d3d9) {
449         av_log(ctx, AV_LOG_ERROR, "Failed to create IDirect3D object\n");
450         return AVERROR_UNKNOWN;
451     }
452
453     IDirect3D9_GetAdapterDisplayMode(priv->d3d9, adapter, &d3ddm);
454
455     d3dpp.BackBufferFormat = d3ddm.Format;
456
457     hr = IDirect3D9_CreateDevice(priv->d3d9, adapter, D3DDEVTYPE_HAL, GetShellWindow(),
458                                 FF_D3DCREATE_FLAGS,
459                                 &d3dpp, &priv->d3d9device);
460     if (FAILED(hr)) {
461         av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device\n");
462         return AVERROR_UNKNOWN;
463     }
464
465     return 0;
466 }
467
468 static int dxva2_device_create9ex(AVHWDeviceContext *ctx, UINT adapter)
469 {
470     DXVA2DevicePriv *priv = ctx->user_opaque;
471     D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
472     D3DDISPLAYMODEEX modeex = {0};
473     IDirect3D9Ex *d3d9ex = NULL;
474     IDirect3DDevice9Ex *exdev = NULL;
475     HRESULT hr;
476     pDirect3DCreate9Ex *createD3DEx = (pDirect3DCreate9Ex *)GetProcAddress(priv->d3dlib, "Direct3DCreate9Ex");
477     if (!createD3DEx)
478         return AVERROR(ENOSYS);
479
480     hr = createD3DEx(D3D_SDK_VERSION, &d3d9ex);
481     if (FAILED(hr))
482         return AVERROR_UNKNOWN;
483
484     IDirect3D9Ex_GetAdapterDisplayModeEx(d3d9ex, adapter, &modeex, NULL);
485
486     d3dpp.BackBufferFormat = modeex.Format;
487
488     hr = IDirect3D9Ex_CreateDeviceEx(d3d9ex, adapter, D3DDEVTYPE_HAL, GetShellWindow(),
489                                      FF_D3DCREATE_FLAGS,
490                                      &d3dpp, NULL, &exdev);
491     if (FAILED(hr)) {
492         IDirect3D9Ex_Release(d3d9ex);
493         return AVERROR_UNKNOWN;
494     }
495
496     av_log(ctx, AV_LOG_VERBOSE, "Using D3D9Ex device.\n");
497     priv->d3d9 = (IDirect3D9 *)d3d9ex;
498     priv->d3d9device = (IDirect3DDevice9 *)exdev;
499     return 0;
500 }
501
502 static int dxva2_device_create(AVHWDeviceContext *ctx, const char *device,
503                                AVDictionary *opts, int flags)
504 {
505     AVDXVA2DeviceContext *hwctx = ctx->hwctx;
506     DXVA2DevicePriv *priv;
507     pCreateDeviceManager9 *createDeviceManager = NULL;
508     unsigned resetToken = 0;
509     UINT adapter = D3DADAPTER_DEFAULT;
510     HRESULT hr;
511     int err;
512
513     if (device)
514         adapter = atoi(device);
515
516     priv = av_mallocz(sizeof(*priv));
517     if (!priv)
518         return AVERROR(ENOMEM);
519
520     ctx->user_opaque = priv;
521     ctx->free        = dxva2_device_free;
522
523     priv->device_handle = INVALID_HANDLE_VALUE;
524
525     priv->d3dlib = LoadLibrary("d3d9.dll");
526     if (!priv->d3dlib) {
527         av_log(ctx, AV_LOG_ERROR, "Failed to load D3D9 library\n");
528         return AVERROR_UNKNOWN;
529     }
530     priv->dxva2lib = LoadLibrary("dxva2.dll");
531     if (!priv->dxva2lib) {
532         av_log(ctx, AV_LOG_ERROR, "Failed to load DXVA2 library\n");
533         return AVERROR_UNKNOWN;
534     }
535
536     createDeviceManager = (pCreateDeviceManager9 *)GetProcAddress(priv->dxva2lib,
537                                                                   "DXVA2CreateDirect3DDeviceManager9");
538     if (!createDeviceManager) {
539         av_log(ctx, AV_LOG_ERROR, "Failed to locate DXVA2CreateDirect3DDeviceManager9\n");
540         return AVERROR_UNKNOWN;
541     }
542
543     if (dxva2_device_create9ex(ctx, adapter) < 0) {
544         // Retry with "classic" d3d9
545         err = dxva2_device_create9(ctx, adapter);
546         if (err < 0)
547             return err;
548     }
549
550     hr = createDeviceManager(&resetToken, &hwctx->devmgr);
551     if (FAILED(hr)) {
552         av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device manager\n");
553         return AVERROR_UNKNOWN;
554     }
555
556     hr = IDirect3DDeviceManager9_ResetDevice(hwctx->devmgr, priv->d3d9device, resetToken);
557     if (FAILED(hr)) {
558         av_log(ctx, AV_LOG_ERROR, "Failed to bind Direct3D device to device manager\n");
559         return AVERROR_UNKNOWN;
560     }
561
562     hr = IDirect3DDeviceManager9_OpenDeviceHandle(hwctx->devmgr, &priv->device_handle);
563     if (FAILED(hr)) {
564         av_log(ctx, AV_LOG_ERROR, "Failed to open device handle\n");
565         return AVERROR_UNKNOWN;
566     }
567
568     return 0;
569 }
570
571 const HWContextType ff_hwcontext_type_dxva2 = {
572     .type                 = AV_HWDEVICE_TYPE_DXVA2,
573     .name                 = "DXVA2",
574
575     .device_hwctx_size    = sizeof(AVDXVA2DeviceContext),
576     .frames_hwctx_size    = sizeof(AVDXVA2FramesContext),
577     .frames_priv_size     = sizeof(DXVA2FramesContext),
578
579     .device_create        = dxva2_device_create,
580     .frames_init          = dxva2_frames_init,
581     .frames_uninit        = dxva2_frames_uninit,
582     .frames_get_buffer    = dxva2_get_buffer,
583     .transfer_get_formats = dxva2_transfer_get_formats,
584     .transfer_data_to     = dxva2_transfer_data_to,
585     .transfer_data_from   = dxva2_transfer_data_from,
586     .map_from             = dxva2_map_from,
587
588     .pix_fmts             = (const enum AVPixelFormat[]){ AV_PIX_FMT_DXVA2_VLD, AV_PIX_FMT_NONE },
589 };