]> git.sesse.net Git - vlc/blob - modules/codec/avcodec/dxva2.c
avcodec: fix double free on error
[vlc] / modules / codec / avcodec / dxva2.c
1 /*****************************************************************************
2  * dxva2.c: Video Acceleration helpers
3  *****************************************************************************
4  * Copyright (C) 2009 Geoffroy Couprie
5  * Copyright (C) 2009 Laurent Aimar
6  * $Id$
7  *
8  * Authors: Geoffroy Couprie <geal@videolan.org>
9  *          Laurent Aimar <fenrir _AT_ videolan _DOT_ org>
10  *
11  * This program is free software; you can redistribute it and/or modify it
12  * under the terms of the GNU Lesser General Public License as published by
13  * the Free Software Foundation; either version 2.1 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * GNU Lesser General Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser General Public License
22  * along with this program; if not, write to the Free Software Foundation,
23  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
24  *****************************************************************************/
25
26 #ifdef HAVE_CONFIG_H
27 # include "config.h"
28 #endif
29
30
31 # if _WIN32_WINNT < 0x600
32 /* dxva2 needs Vista support */
33 #  undef _WIN32_WINNT
34 #  define _WIN32_WINNT 0x600
35 # endif
36
37 #include <assert.h>
38
39 #include <vlc_common.h>
40 #include <vlc_picture.h>
41 #include <vlc_fourcc.h>
42 #include <vlc_cpu.h>
43 #include <vlc_plugin.h>
44
45 #include <libavcodec/avcodec.h>
46 #    define DXVA2API_USE_BITFIELDS
47 #    define COBJMACROS
48 #    include <libavcodec/dxva2.h>
49
50 #include "avcodec.h"
51 #include "va.h"
52 #include "../../video_chroma/copy.h"
53
54 static int Open(vlc_va_t *, AVCodecContext *, const es_format_t *);
55 static void Close(vlc_va_t *, AVCodecContext *);
56
57 vlc_module_begin()
58     set_description(N_("DirectX Video Acceleration (DXVA) 2.0"))
59     set_capability("hw decoder", 0)
60     set_category(CAT_INPUT)
61     set_subcategory(SUBCAT_INPUT_VCODEC)
62     set_callbacks(Open, Close)
63 vlc_module_end()
64
65 #include <windows.h>
66 #include <windowsx.h>
67 #include <ole2.h>
68 #include <commctrl.h>
69 #include <shlwapi.h>
70 #include <d3d9.h>
71 #include <dxva2api.h>
72
73 #include <initguid.h> /* must be last included to not redefine existing GUIDs */
74
75 /* dxva2api.h GUIDs: http://msdn.microsoft.com/en-us/library/windows/desktop/ms697067(v=vs100).aspx
76  * assume that they are declared in dxva2api.h */
77 #define MS_GUID(name, l, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8)
78
79 #ifdef __MINGW32__
80 # include <_mingw.h>
81
82 # if !defined(__MINGW64_VERSION_MAJOR)
83 #  undef MS_GUID
84 #  define MS_GUID DEFINE_GUID /* dxva2api.h fails to declare those, redefine as static */
85 #  define DXVA2_E_NEW_VIDEO_DEVICE MAKE_HRESULT(1, 4, 4097)
86 # else
87 #  include <dxva.h>
88 # endif
89
90 #endif /* __MINGW32__ */
91
92 MS_GUID(IID_IDirectXVideoDecoderService, 0xfc51a551, 0xd5e7, 0x11d9, 0xaf,0x55,0x00,0x05,0x4e,0x43,0xff,0x02);
93 MS_GUID(IID_IDirectXVideoAccelerationService, 0xfc51a550, 0xd5e7, 0x11d9, 0xaf,0x55,0x00,0x05,0x4e,0x43,0xff,0x02);
94
95 MS_GUID    (DXVA_NoEncrypt,                         0x1b81bed0, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
96
97 /* Codec capabilities GUID, sorted by codec */
98 MS_GUID    (DXVA2_ModeMPEG2_MoComp,                 0xe6a9f44b, 0x61b0, 0x4563, 0x9e, 0xa4, 0x63, 0xd2, 0xa3, 0xc6, 0xfe, 0x66);
99 MS_GUID    (DXVA2_ModeMPEG2_IDCT,                   0xbf22ad00, 0x03ea, 0x4690, 0x80, 0x77, 0x47, 0x33, 0x46, 0x20, 0x9b, 0x7e);
100 MS_GUID    (DXVA2_ModeMPEG2_VLD,                    0xee27417f, 0x5e28, 0x4e65, 0xbe, 0xea, 0x1d, 0x26, 0xb5, 0x08, 0xad, 0xc9);
101 DEFINE_GUID(DXVA2_ModeMPEG2and1_VLD,                0x86695f12, 0x340e, 0x4f04, 0x9f, 0xd3, 0x92, 0x53, 0xdd, 0x32, 0x74, 0x60);
102 DEFINE_GUID(DXVA2_ModeMPEG1_VLD,                    0x6f3ec719, 0x3735, 0x42cc, 0x80, 0x63, 0x65, 0xcc, 0x3c, 0xb3, 0x66, 0x16);
103
104 MS_GUID    (DXVA2_ModeH264_A,                       0x1b81be64, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
105 MS_GUID    (DXVA2_ModeH264_B,                       0x1b81be65, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
106 MS_GUID    (DXVA2_ModeH264_C,                       0x1b81be66, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
107 MS_GUID    (DXVA2_ModeH264_D,                       0x1b81be67, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
108 MS_GUID    (DXVA2_ModeH264_E,                       0x1b81be68, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
109 MS_GUID    (DXVA2_ModeH264_F,                       0x1b81be69, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
110 DEFINE_GUID(DXVA_ModeH264_VLD_Multiview,            0x9901CCD3, 0xca12, 0x4b7e, 0x86, 0x7a, 0xe2, 0x22, 0x3d, 0x92, 0x55, 0xc3); // MVC
111 DEFINE_GUID(DXVA_ModeH264_VLD_WithFMOASO_NoFGT,     0xd5f04ff9, 0x3418, 0x45d8, 0x95, 0x61, 0x32, 0xa7, 0x6a, 0xae, 0x2d, 0xdd);
112 DEFINE_GUID(DXVADDI_Intel_ModeH264_A,               0x604F8E64, 0x4951, 0x4c54, 0x88, 0xFE, 0xAB, 0xD2, 0x5C, 0x15, 0xB3, 0xD6);
113 DEFINE_GUID(DXVADDI_Intel_ModeH264_C,               0x604F8E66, 0x4951, 0x4c54, 0x88, 0xFE, 0xAB, 0xD2, 0x5C, 0x15, 0xB3, 0xD6);
114 DEFINE_GUID(DXVADDI_Intel_ModeH264_E,               0x604F8E68, 0x4951, 0x4c54, 0x88, 0xFE, 0xAB, 0xD2, 0x5C, 0x15, 0xB3, 0xD6); // DXVA_Intel_H264_NoFGT_ClearVideo
115 DEFINE_GUID(DXVA_ModeH264_VLD_NoFGT_Flash,          0x4245F676, 0x2BBC, 0x4166, 0xa0, 0xBB, 0x54, 0xE7, 0xB8, 0x49, 0xC3, 0x80);
116
117 MS_GUID    (DXVA2_ModeWMV8_A,                       0x1b81be80, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
118 MS_GUID    (DXVA2_ModeWMV8_B,                       0x1b81be81, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
119
120 MS_GUID    (DXVA2_ModeWMV9_A,                       0x1b81be90, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
121 MS_GUID    (DXVA2_ModeWMV9_B,                       0x1b81be91, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
122 MS_GUID    (DXVA2_ModeWMV9_C,                       0x1b81be94, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
123
124 MS_GUID    (DXVA2_ModeVC1_A,                        0x1b81beA0, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
125 MS_GUID    (DXVA2_ModeVC1_B,                        0x1b81beA1, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
126 MS_GUID    (DXVA2_ModeVC1_C,                        0x1b81beA2, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
127 MS_GUID    (DXVA2_ModeVC1_D,                        0x1b81beA3, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5);
128 DEFINE_GUID(DXVA2_ModeVC1_D2010,                    0x1b81beA4, 0xa0c7, 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5); // August 2010 update
129 DEFINE_GUID(DXVA_Intel_VC1_ClearVideo,              0xBCC5DB6D, 0xA2B6, 0x4AF0, 0xAC, 0xE4, 0xAD, 0xB1, 0xF7, 0x87, 0xBC, 0x89);
130 DEFINE_GUID(DXVA_Intel_VC1_ClearVideo_2,            0xE07EC519, 0xE651, 0x4CD6, 0xAC, 0x84, 0x13, 0x70, 0xCC, 0xEE, 0xC8, 0x51);
131
132 DEFINE_GUID(DXVA_nVidia_MPEG4_ASP,                  0x9947EC6F, 0x689B, 0x11DC, 0xA3, 0x20, 0x00, 0x19, 0xDB, 0xBC, 0x41, 0x84);
133 DEFINE_GUID(DXVA_ModeMPEG4pt2_VLD_Simple,           0xefd64d74, 0xc9e8, 0x41d7, 0xa5, 0xe9, 0xe9, 0xb0, 0xe3, 0x9f, 0xa3, 0x19);
134 DEFINE_GUID(DXVA_ModeMPEG4pt2_VLD_AdvSimple_NoGMC,  0xed418a9f, 0x010d, 0x4eda, 0x9a, 0xe3, 0x9a, 0x65, 0x35, 0x8d, 0x8d, 0x2e);
135 DEFINE_GUID(DXVA_ModeMPEG4pt2_VLD_AdvSimple_GMC,    0xab998b5b, 0x4258, 0x44a9, 0x9f, 0xeb, 0x94, 0xe5, 0x97, 0xa6, 0xba, 0xae);
136 DEFINE_GUID(DXVA_ModeMPEG4pt2_VLD_AdvSimple_Avivo,  0x7C74ADC6, 0xe2ba, 0x4ade, 0x86, 0xde, 0x30, 0xbe, 0xab, 0xb4, 0x0c, 0xc1);
137
138
139 /* */
140 typedef struct {
141     const char   *name;
142     const GUID   *guid;
143     int          codec;
144 } dxva2_mode_t;
145 /* XXX Prefered modes must come first */
146 static const dxva2_mode_t dxva2_modes[] = {
147     /* MPEG-1/2 */
148     { "MPEG-2 variable-length decoder",                                               &DXVA2_ModeMPEG2_VLD,                   AV_CODEC_ID_MPEG2VIDEO },
149     { "MPEG-2 & MPEG-1 variable-length decoder",                                      &DXVA2_ModeMPEG2and1_VLD,               AV_CODEC_ID_MPEG2VIDEO },
150     { "MPEG-2 motion compensation",                                                   &DXVA2_ModeMPEG2_MoComp,                0 },
151     { "MPEG-2 inverse discrete cosine transform",                                     &DXVA2_ModeMPEG2_IDCT,                  0 },
152
153     { "MPEG-1 variable-length decoder",                                               &DXVA2_ModeMPEG1_VLD,                   0 },
154
155     /* H.264 */
156     { "H.264 variable-length decoder, film grain technology",                         &DXVA2_ModeH264_F,                      AV_CODEC_ID_H264 },
157     { "H.264 variable-length decoder, no film grain technology (Intel ClearVideo)",   &DXVADDI_Intel_ModeH264_E,              AV_CODEC_ID_H264 },
158     { "H.264 variable-length decoder, no film grain technology",                      &DXVA2_ModeH264_E,                      AV_CODEC_ID_H264 },
159     { "H.264 variable-length decoder, no film grain technology, FMO/ASO",             &DXVA_ModeH264_VLD_WithFMOASO_NoFGT,    AV_CODEC_ID_H264 },
160     { "H.264 variable-length decoder, no film grain technology, Flash",               &DXVA_ModeH264_VLD_NoFGT_Flash,         AV_CODEC_ID_H264 },
161
162     { "H.264 inverse discrete cosine transform, film grain technology",               &DXVA2_ModeH264_D,                      0 },
163     { "H.264 inverse discrete cosine transform, no film grain technology",            &DXVA2_ModeH264_C,                      0 },
164     { "H.264 inverse discrete cosine transform, no film grain technology (Intel)",    &DXVADDI_Intel_ModeH264_C,              0 },
165
166     { "H.264 motion compensation, film grain technology",                             &DXVA2_ModeH264_B,                      0 },
167     { "H.264 motion compensation, no film grain technology",                          &DXVA2_ModeH264_A,                      0 },
168     { "H.264 motion compensation, no film grain technology (Intel)",                  &DXVADDI_Intel_ModeH264_A,              0 },
169
170     /* WMV */
171     { "Windows Media Video 8 motion compensation",                                    &DXVA2_ModeWMV8_B,                      0 },
172     { "Windows Media Video 8 post processing",                                        &DXVA2_ModeWMV8_A,                      0 },
173
174     { "Windows Media Video 9 IDCT",                                                   &DXVA2_ModeWMV9_C,                      0 },
175     { "Windows Media Video 9 motion compensation",                                    &DXVA2_ModeWMV9_B,                      0 },
176     { "Windows Media Video 9 post processing",                                        &DXVA2_ModeWMV9_A,                      0 },
177
178     /* VC-1 */
179     { "VC-1 variable-length decoder",                                                 &DXVA2_ModeVC1_D,                       AV_CODEC_ID_VC1 },
180     { "VC-1 variable-length decoder",                                                 &DXVA2_ModeVC1_D,                       AV_CODEC_ID_WMV3 },
181     { "VC-1 variable-length decoder",                                                 &DXVA2_ModeVC1_D2010,                   AV_CODEC_ID_VC1 },
182     { "VC-1 variable-length decoder",                                                 &DXVA2_ModeVC1_D2010,                   AV_CODEC_ID_WMV3 },
183     { "VC-1 variable-length decoder 2 (Intel)",                                       &DXVA_Intel_VC1_ClearVideo_2,           0 },
184     { "VC-1 variable-length decoder (Intel)",                                         &DXVA_Intel_VC1_ClearVideo,             0 },
185
186     { "VC-1 inverse discrete cosine transform",                                       &DXVA2_ModeVC1_C,                       0 },
187     { "VC-1 motion compensation",                                                     &DXVA2_ModeVC1_B,                       0 },
188     { "VC-1 post processing",                                                         &DXVA2_ModeVC1_A,                       0 },
189
190     /* Xvid/Divx: TODO */
191     { "MPEG-4 Part 2 nVidia bitstream decoder",                                       &DXVA_nVidia_MPEG4_ASP,                 0 },
192     { "MPEG-4 Part 2 variable-length decoder, Simple Profile",                        &DXVA_ModeMPEG4pt2_VLD_Simple,          0 },
193     { "MPEG-4 Part 2 variable-length decoder, Simple&Advanced Profile, no GMC",       &DXVA_ModeMPEG4pt2_VLD_AdvSimple_NoGMC, 0 },
194     { "MPEG-4 Part 2 variable-length decoder, Simple&Advanced Profile, GMC",          &DXVA_ModeMPEG4pt2_VLD_AdvSimple_GMC,   0 },
195     { "MPEG-4 Part 2 variable-length decoder, Simple&Advanced Profile, Avivo",        &DXVA_ModeMPEG4pt2_VLD_AdvSimple_Avivo, 0 },
196
197     { NULL, NULL, 0 }
198 };
199
200 static const dxva2_mode_t *Dxva2FindMode(const GUID *guid)
201 {
202     for (unsigned i = 0; dxva2_modes[i].name; i++) {
203         if (IsEqualGUID(dxva2_modes[i].guid, guid))
204             return &dxva2_modes[i];
205     }
206     return NULL;
207 }
208
209 /* */
210 typedef struct {
211     const char   *name;
212     D3DFORMAT    format;
213     vlc_fourcc_t codec;
214 } d3d_format_t;
215 /* XXX Prefered format must come first */
216 static const d3d_format_t d3d_formats[] = {
217     { "YV12",   MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_YV12 },
218     { "NV12",   MAKEFOURCC('N','V','1','2'),    VLC_CODEC_NV12 },
219     { "IMC3",   MAKEFOURCC('I','M','C','3'),    VLC_CODEC_YV12 },
220
221     { NULL, 0, 0 }
222 };
223
224 static const d3d_format_t *D3dFindFormat(D3DFORMAT format)
225 {
226     for (unsigned i = 0; d3d_formats[i].name; i++) {
227         if (d3d_formats[i].format == format)
228             return &d3d_formats[i];
229     }
230     return NULL;
231 }
232
233 /* */
234 typedef struct {
235     LPDIRECT3DSURFACE9 d3d;
236     int                refcount;
237     unsigned int       order;
238 } vlc_va_surface_t;
239
240 #define VA_DXVA2_MAX_SURFACE_COUNT (64)
241 struct vlc_va_sys_t
242 {
243     int          codec_id;
244     int          width;
245     int          height;
246
247     /* DLL */
248     HINSTANCE             hd3d9_dll;
249     HINSTANCE             hdxva2_dll;
250
251     /* Direct3D */
252     D3DPRESENT_PARAMETERS  d3dpp;
253     LPDIRECT3D9            d3dobj;
254     D3DADAPTER_IDENTIFIER9 d3dai;
255     LPDIRECT3DDEVICE9      d3ddev;
256
257     /* Device manager */
258     UINT                     token;
259     IDirect3DDeviceManager9  *devmng;
260     HANDLE                   device;
261
262     /* Video service */
263     IDirectXVideoDecoderService  *vs;
264     GUID                         input;
265     D3DFORMAT                    render;
266
267     /* Video decoder */
268     DXVA2_ConfigPictureDecode    cfg;
269     IDirectXVideoDecoder         *decoder;
270
271     /* Option conversion */
272     D3DFORMAT                    output;
273     copy_cache_t                 surface_cache;
274
275     /* */
276     struct dxva_context hw;
277
278     /* */
279     unsigned     surface_count;
280     unsigned     surface_order;
281     int          surface_width;
282     int          surface_height;
283     vlc_fourcc_t surface_chroma;
284
285     int          thread_count;
286
287     vlc_va_surface_t surface[VA_DXVA2_MAX_SURFACE_COUNT];
288     LPDIRECT3DSURFACE9 hw_surface[VA_DXVA2_MAX_SURFACE_COUNT];
289 };
290
291 /* */
292 static int D3dCreateDevice(vlc_va_t *);
293 static void D3dDestroyDevice(vlc_va_sys_t *);
294 static char *DxDescribe(vlc_va_sys_t *);
295
296 static int D3dCreateDeviceManager(vlc_va_t *);
297 static void D3dDestroyDeviceManager(vlc_va_sys_t *);
298
299 static int DxCreateVideoService(vlc_va_t *);
300 static void DxDestroyVideoService(vlc_va_sys_t *);
301 static int DxFindVideoServiceConversion(vlc_va_t *, GUID *input, D3DFORMAT *output);
302
303 static int DxCreateVideoDecoder(vlc_va_t *,
304                                 int codec_id, const video_format_t *);
305 static void DxDestroyVideoDecoder(vlc_va_sys_t *);
306 static int DxResetVideoDecoder(vlc_va_t *);
307
308 static void DxCreateVideoConversion(vlc_va_sys_t *);
309 static void DxDestroyVideoConversion(vlc_va_sys_t *);
310
311 /* */
312 static int Setup(vlc_va_t *va, AVCodecContext *avctx, vlc_fourcc_t *chroma)
313 {
314     vlc_va_sys_t *sys = va->sys;
315
316     if (sys->width == avctx->coded_width && sys->height == avctx->coded_height
317      && sys->decoder != NULL)
318         goto ok;
319
320     /* */
321     DxDestroyVideoConversion(sys);
322     DxDestroyVideoDecoder(sys);
323
324     avctx->hwaccel_context = NULL;
325     *chroma = 0;
326     if (avctx->coded_width <= 0 || avctx->coded_height <= 0)
327         return VLC_EGENERIC;
328
329     /* FIXME transmit a video_format_t by VaSetup directly */
330     video_format_t fmt;
331     memset(&fmt, 0, sizeof(fmt));
332     fmt.i_width = avctx->coded_width;
333     fmt.i_height = avctx->coded_height;
334
335     if (DxCreateVideoDecoder(va, sys->codec_id, &fmt))
336         return VLC_EGENERIC;
337     /* */
338     sys->hw.decoder = sys->decoder;
339     sys->hw.cfg = &sys->cfg;
340     sys->hw.surface_count = sys->surface_count;
341     sys->hw.surface = sys->hw_surface;
342
343     /* */
344     DxCreateVideoConversion(sys);
345
346     /* */
347 ok:
348     avctx->hwaccel_context = &sys->hw;
349     const d3d_format_t *output = D3dFindFormat(sys->output);
350     *chroma = output->codec;
351
352     return VLC_SUCCESS;
353 }
354
355 static int Extract(vlc_va_t *va, picture_t *picture, void *opaque,
356                    uint8_t *data)
357 {
358     vlc_va_sys_t *sys = va->sys;
359     LPDIRECT3DSURFACE9 d3d = (LPDIRECT3DSURFACE9)(uintptr_t)data;
360
361     if (!sys->surface_cache.buffer)
362         return VLC_EGENERIC;
363
364     /* */
365     assert(sys->output == MAKEFOURCC('Y','V','1','2'));
366
367     /* */
368     D3DLOCKED_RECT lock;
369     if (FAILED(IDirect3DSurface9_LockRect(d3d, &lock, NULL, D3DLOCK_READONLY))) {
370         msg_Err(va, "Failed to lock surface");
371         return VLC_EGENERIC;
372     }
373
374     if (sys->render == MAKEFOURCC('Y','V','1','2') ||
375         sys->render == MAKEFOURCC('I','M','C','3')) {
376         bool imc3 = sys->render == MAKEFOURCC('I','M','C','3');
377         size_t chroma_pitch = imc3 ? lock.Pitch : (lock.Pitch / 2);
378
379         size_t pitch[3] = {
380             lock.Pitch,
381             chroma_pitch,
382             chroma_pitch,
383         };
384
385         uint8_t *plane[3] = {
386             (uint8_t*)lock.pBits,
387             (uint8_t*)lock.pBits + pitch[0] * sys->surface_height,
388             (uint8_t*)lock.pBits + pitch[0] * sys->surface_height
389                                  + pitch[1] * sys->surface_height / 2,
390         };
391
392         if (imc3) {
393             uint8_t *V = plane[1];
394             plane[1] = plane[2];
395             plane[2] = V;
396         }
397         CopyFromYv12(picture, plane, pitch, sys->width, sys->height,
398                      &sys->surface_cache);
399     } else {
400         assert(sys->render == MAKEFOURCC('N','V','1','2'));
401         uint8_t *plane[2] = {
402             lock.pBits,
403             (uint8_t*)lock.pBits + lock.Pitch * sys->surface_height
404         };
405         size_t  pitch[2] = {
406             lock.Pitch,
407             lock.Pitch,
408         };
409         CopyFromNv12(picture, plane, pitch, sys->width, sys->height,
410                      &sys->surface_cache);
411     }
412
413     /* */
414     IDirect3DSurface9_UnlockRect(d3d);
415     (void) opaque;
416     return VLC_SUCCESS;
417 }
418
419 /* FIXME it is nearly common with VAAPI */
420 static int Get(vlc_va_t *va, void **opaque, uint8_t **data)
421 {
422     vlc_va_sys_t *sys = va->sys;
423
424     /* Check the device */
425     HRESULT hr = IDirect3DDeviceManager9_TestDevice(sys->devmng, sys->device);
426     if (hr == DXVA2_E_NEW_VIDEO_DEVICE) {
427         if (DxResetVideoDecoder(va))
428             return VLC_EGENERIC;
429     } else if (FAILED(hr)) {
430         msg_Err(va, "IDirect3DDeviceManager9_TestDevice %u", (unsigned)hr);
431         return VLC_EGENERIC;
432     }
433
434     /* Grab an unused surface, in case none are, try the oldest
435      * XXX using the oldest is a workaround in case a problem happens with libavcodec */
436     unsigned i, old;
437     for (i = 0, old = 0; i < sys->surface_count; i++) {
438         vlc_va_surface_t *surface = &sys->surface[i];
439
440         if (!surface->refcount)
441             break;
442
443         if (surface->order < sys->surface[old].order)
444             old = i;
445     }
446     if (i >= sys->surface_count)
447         i = old;
448
449     vlc_va_surface_t *surface = &sys->surface[i];
450
451     surface->refcount = 1;
452     surface->order = sys->surface_order++;
453     *data = (void *)surface->d3d;
454     *opaque = surface;
455     return VLC_SUCCESS;
456 }
457
458 static void Release(void *opaque, uint8_t *data)
459 {
460     vlc_va_surface_t *surface = opaque;
461
462     surface->refcount--;
463     (void) data;
464 }
465
466 static void Close(vlc_va_t *va, AVCodecContext *ctx)
467 {
468     vlc_va_sys_t *sys = va->sys;
469
470     (void) ctx;
471     DxDestroyVideoConversion(sys);
472     DxDestroyVideoDecoder(sys);
473     DxDestroyVideoService(sys);
474     D3dDestroyDeviceManager(sys);
475     D3dDestroyDevice(sys);
476
477     if (sys->hdxva2_dll)
478         FreeLibrary(sys->hdxva2_dll);
479     if (sys->hd3d9_dll)
480         FreeLibrary(sys->hd3d9_dll);
481
482     free((char *)va->description);
483     free(sys);
484 }
485
486 static int Open(vlc_va_t *va, AVCodecContext *ctx, const es_format_t *fmt)
487 {
488     vlc_va_sys_t *sys = calloc(1, sizeof (*sys));
489     if (unlikely(sys == NULL))
490         return VLC_ENOMEM;
491
492     va->sys = sys;
493     sys->codec_id = ctx->codec_id;
494     (void) fmt;
495
496     /* Load dll*/
497     sys->hd3d9_dll = LoadLibrary(TEXT("D3D9.DLL"));
498     if (!sys->hd3d9_dll) {
499         msg_Warn(va, "cannot load d3d9.dll");
500         goto error;
501     }
502     sys->hdxva2_dll = LoadLibrary(TEXT("DXVA2.DLL"));
503     if (!sys->hdxva2_dll) {
504         msg_Warn(va, "cannot load dxva2.dll");
505         goto error;
506     }
507     msg_Dbg(va, "DLLs loaded");
508
509     /* */
510     if (D3dCreateDevice(va)) {
511         msg_Err(va, "Failed to create Direct3D device");
512         goto error;
513     }
514     msg_Dbg(va, "D3dCreateDevice succeed");
515
516     if (D3dCreateDeviceManager(va)) {
517         msg_Err(va, "D3dCreateDeviceManager failed");
518         goto error;
519     }
520
521     if (DxCreateVideoService(va)) {
522         msg_Err(va, "DxCreateVideoService failed");
523         goto error;
524     }
525
526     /* */
527     if (DxFindVideoServiceConversion(va, &sys->input, &sys->render)) {
528         msg_Err(va, "DxFindVideoServiceConversion failed");
529         goto error;
530     }
531
532     sys->thread_count = ctx->thread_count;
533
534     /* TODO print the hardware name/vendor for debugging purposes */
535     va->description = DxDescribe(sys);
536     va->pix_fmt = PIX_FMT_DXVA2_VLD;
537     va->setup   = Setup;
538     va->get     = Get;
539     va->release = Release;
540     va->extract = Extract;
541     return VLC_SUCCESS;
542
543 error:
544     Close(va, ctx);
545     return VLC_EGENERIC;
546 }
547 /* */
548
549 /**
550  * It creates a Direct3D device usable for DXVA 2
551  */
552 static int D3dCreateDevice(vlc_va_t *va)
553 {
554     vlc_va_sys_t *sys = va->sys;
555
556     /* */
557     LPDIRECT3D9 (WINAPI *Create9)(UINT SDKVersion);
558     Create9 = (void *)GetProcAddress(sys->hd3d9_dll, "Direct3DCreate9");
559     if (!Create9) {
560         msg_Err(va, "Cannot locate reference to Direct3DCreate9 ABI in DLL");
561         return VLC_EGENERIC;
562     }
563
564     /* */
565     LPDIRECT3D9 d3dobj;
566     d3dobj = Create9(D3D_SDK_VERSION);
567     if (!d3dobj) {
568         msg_Err(va, "Direct3DCreate9 failed");
569         return VLC_EGENERIC;
570     }
571     sys->d3dobj = d3dobj;
572
573     /* */
574     D3DADAPTER_IDENTIFIER9 *d3dai = &sys->d3dai;
575     if (FAILED(IDirect3D9_GetAdapterIdentifier(sys->d3dobj,
576                                                D3DADAPTER_DEFAULT, 0, d3dai))) {
577         msg_Warn(va, "IDirect3D9_GetAdapterIdentifier failed");
578         ZeroMemory(d3dai, sizeof(*d3dai));
579     }
580
581     /* */
582     D3DPRESENT_PARAMETERS *d3dpp = &sys->d3dpp;
583     ZeroMemory(d3dpp, sizeof(*d3dpp));
584     d3dpp->Flags                  = D3DPRESENTFLAG_VIDEO;
585     d3dpp->Windowed               = TRUE;
586     d3dpp->hDeviceWindow          = NULL;
587     d3dpp->SwapEffect             = D3DSWAPEFFECT_DISCARD;
588     d3dpp->MultiSampleType        = D3DMULTISAMPLE_NONE;
589     d3dpp->PresentationInterval   = D3DPRESENT_INTERVAL_DEFAULT;
590     d3dpp->BackBufferCount        = 0;                  /* FIXME what to put here */
591     d3dpp->BackBufferFormat       = D3DFMT_X8R8G8B8;    /* FIXME what to put here */
592     d3dpp->BackBufferWidth        = 0;
593     d3dpp->BackBufferHeight       = 0;
594     d3dpp->EnableAutoDepthStencil = FALSE;
595
596     /* Direct3D needs a HWND to create a device, even without using ::Present
597     this HWND is used to alert Direct3D when there's a change of focus window.
598     For now, use GetDesktopWindow, as it looks harmless */
599     LPDIRECT3DDEVICE9 d3ddev;
600     if (FAILED(IDirect3D9_CreateDevice(d3dobj, D3DADAPTER_DEFAULT,
601                                        D3DDEVTYPE_HAL, GetDesktopWindow(),
602                                        D3DCREATE_SOFTWARE_VERTEXPROCESSING |
603                                        D3DCREATE_MULTITHREADED,
604                                        d3dpp, &d3ddev))) {
605         msg_Err(va, "IDirect3D9_CreateDevice failed");
606         return VLC_EGENERIC;
607     }
608     sys->d3ddev = d3ddev;
609
610     return VLC_SUCCESS;
611 }
612
613 /**
614  * It releases a Direct3D device and its resources.
615  */
616 static void D3dDestroyDevice(vlc_va_sys_t *va)
617 {
618     if (va->d3ddev)
619         IDirect3DDevice9_Release(va->d3ddev);
620     if (va->d3dobj)
621         IDirect3D9_Release(va->d3dobj);
622 }
623 /**
624  * It describes our Direct3D object
625  */
626 static char *DxDescribe(vlc_va_sys_t *va)
627 {
628     static const struct {
629         unsigned id;
630         char     name[32];
631     } vendors [] = {
632         { 0x1002, "ATI" },
633         { 0x10DE, "NVIDIA" },
634         { 0x1106, "VIA" },
635         { 0x8086, "Intel" },
636         { 0x5333, "S3 Graphics" },
637         { 0, "" }
638     };
639     D3DADAPTER_IDENTIFIER9 *id = &va->d3dai;
640
641     const char *vendor = "Unknown";
642     for (int i = 0; vendors[i].id != 0; i++) {
643         if (vendors[i].id == id->VendorId) {
644             vendor = vendors[i].name;
645             break;
646         }
647     }
648
649     char *description;
650     if (asprintf(&description, "DXVA2 (%.*s, vendor %lu(%s), device %lu, revision %lu)",
651                  sizeof(id->Description), id->Description,
652                  id->VendorId, vendor, id->DeviceId, id->Revision) < 0)
653         return NULL;
654     return description;
655 }
656
657 /**
658  * It creates a Direct3D device manager
659  */
660 static int D3dCreateDeviceManager(vlc_va_t *va)
661 {
662     vlc_va_sys_t *sys = va->sys;
663
664     HRESULT (WINAPI *CreateDeviceManager9)(UINT *pResetToken,
665                                            IDirect3DDeviceManager9 **);
666     CreateDeviceManager9 =
667       (void *)GetProcAddress(sys->hdxva2_dll,
668                              "DXVA2CreateDirect3DDeviceManager9");
669
670     if (!CreateDeviceManager9) {
671         msg_Err(va, "cannot load function");
672         return VLC_EGENERIC;
673     }
674     msg_Dbg(va, "OurDirect3DCreateDeviceManager9 Success!");
675
676     UINT token;
677     IDirect3DDeviceManager9 *devmng;
678     if (FAILED(CreateDeviceManager9(&token, &devmng))) {
679         msg_Err(va, " OurDirect3DCreateDeviceManager9 failed");
680         return VLC_EGENERIC;
681     }
682     sys->token  = token;
683     sys->devmng = devmng;
684     msg_Info(va, "obtained IDirect3DDeviceManager9");
685
686     HRESULT hr = IDirect3DDeviceManager9_ResetDevice(devmng, sys->d3ddev, token);
687     if (FAILED(hr)) {
688         msg_Err(va, "IDirect3DDeviceManager9_ResetDevice failed: %08x", (unsigned)hr);
689         return VLC_EGENERIC;
690     }
691     return VLC_SUCCESS;
692 }
693 /**
694  * It destroys a Direct3D device manager
695  */
696 static void D3dDestroyDeviceManager(vlc_va_sys_t *va)
697 {
698     if (va->devmng)
699         IDirect3DDeviceManager9_Release(va->devmng);
700 }
701
702 /**
703  * It creates a DirectX video service
704  */
705 static int DxCreateVideoService(vlc_va_t *va)
706 {
707     vlc_va_sys_t *sys = va->sys;
708
709     HRESULT (WINAPI *CreateVideoService)(IDirect3DDevice9 *,
710                                          REFIID riid,
711                                          void **ppService);
712     CreateVideoService =
713       (void *)GetProcAddress(sys->hdxva2_dll, "DXVA2CreateVideoService");
714
715     if (!CreateVideoService) {
716         msg_Err(va, "cannot load function");
717         return 4;
718     }
719     msg_Info(va, "DXVA2CreateVideoService Success!");
720
721     HRESULT hr;
722
723     HANDLE device;
724     hr = IDirect3DDeviceManager9_OpenDeviceHandle(sys->devmng, &device);
725     if (FAILED(hr)) {
726         msg_Err(va, "OpenDeviceHandle failed");
727         return VLC_EGENERIC;
728     }
729     sys->device = device;
730
731     void *pv;
732     hr = IDirect3DDeviceManager9_GetVideoService(sys->devmng, device,
733                                         &IID_IDirectXVideoDecoderService, &pv);
734     if (FAILED(hr)) {
735         msg_Err(va, "GetVideoService failed");
736         return VLC_EGENERIC;
737     }
738     sys->vs = pv;
739
740     return VLC_SUCCESS;
741 }
742
743 /**
744  * It destroys a DirectX video service
745  */
746 static void DxDestroyVideoService(vlc_va_sys_t *va)
747 {
748     if (va->device)
749         IDirect3DDeviceManager9_CloseDeviceHandle(va->devmng, va->device);
750     if (va->vs)
751         IDirectXVideoDecoderService_Release(va->vs);
752 }
753 /**
754  * Find the best suited decoder mode GUID and render format.
755  */
756 static int DxFindVideoServiceConversion(vlc_va_t *va, GUID *input, D3DFORMAT *output)
757 {
758     vlc_va_sys_t *sys = va->sys;
759
760     /* Retreive supported modes from the decoder service */
761     UINT input_count = 0;
762     GUID *input_list = NULL;
763     if (FAILED(IDirectXVideoDecoderService_GetDecoderDeviceGuids(sys->vs,
764                                                                  &input_count,
765                                                                  &input_list))) {
766         msg_Err(va, "IDirectXVideoDecoderService_GetDecoderDeviceGuids failed");
767         return VLC_EGENERIC;
768     }
769     for (unsigned i = 0; i < input_count; i++) {
770         const GUID *g = &input_list[i];
771         const dxva2_mode_t *mode = Dxva2FindMode(g);
772         if (mode) {
773             msg_Dbg(va, "- '%s' is supported by hardware", mode->name);
774         } else {
775             msg_Warn(va, "- Unknown GUID = %08X-%04x-%04x-XXXX",
776                      (unsigned)g->Data1, g->Data2, g->Data3);
777         }
778     }
779
780     /* Try all supported mode by our priority */
781     for (unsigned i = 0; dxva2_modes[i].name; i++) {
782         const dxva2_mode_t *mode = &dxva2_modes[i];
783         if (!mode->codec || mode->codec != sys->codec_id)
784             continue;
785
786         /* */
787         bool is_supported = false;
788         for (const GUID *g = &input_list[0]; !is_supported && g < &input_list[input_count]; g++) {
789             is_supported = IsEqualGUID(mode->guid, g);
790         }
791         if (!is_supported)
792             continue;
793
794         /* */
795         msg_Dbg(va, "Trying to use '%s' as input", mode->name);
796         UINT      output_count = 0;
797         D3DFORMAT *output_list = NULL;
798         if (FAILED(IDirectXVideoDecoderService_GetDecoderRenderTargets(sys->vs, mode->guid,
799                                                                        &output_count,
800                                                                        &output_list))) {
801             msg_Err(va, "IDirectXVideoDecoderService_GetDecoderRenderTargets failed");
802             continue;
803         }
804         for (unsigned j = 0; j < output_count; j++) {
805             const D3DFORMAT f = output_list[j];
806             const d3d_format_t *format = D3dFindFormat(f);
807             if (format) {
808                 msg_Dbg(va, "%s is supported for output", format->name);
809             } else {
810                 msg_Dbg(va, "%d is supported for output (%4.4s)", f, (const char*)&f);
811             }
812         }
813
814         /* */
815         for (unsigned j = 0; d3d_formats[j].name; j++) {
816             const d3d_format_t *format = &d3d_formats[j];
817
818             /* */
819             bool is_supported = false;
820             for (unsigned k = 0; !is_supported && k < output_count; k++) {
821                 is_supported = format->format == output_list[k];
822             }
823             if (!is_supported)
824                 continue;
825
826             /* We have our solution */
827             msg_Dbg(va, "Using '%s' to decode to '%s'", mode->name, format->name);
828             *input  = *mode->guid;
829             *output = format->format;
830             CoTaskMemFree(output_list);
831             CoTaskMemFree(input_list);
832             return VLC_SUCCESS;
833         }
834         CoTaskMemFree(output_list);
835     }
836     CoTaskMemFree(input_list);
837     return VLC_EGENERIC;
838 }
839
840 /**
841  * It creates a DXVA2 decoder using the given video format
842  */
843 static int DxCreateVideoDecoder(vlc_va_t *va,
844                                 int codec_id, const video_format_t *fmt)
845 {
846     vlc_va_sys_t *sys = va->sys;
847
848     /* */
849     msg_Dbg(va, "DxCreateVideoDecoder id %d %dx%d",
850             codec_id, fmt->i_width, fmt->i_height);
851
852     sys->width  = fmt->i_width;
853     sys->height = fmt->i_height;
854
855     /* Allocates all surfaces needed for the decoder */
856     sys->surface_width  = (fmt->i_width  + 15) & ~15;
857     sys->surface_height = (fmt->i_height + 15) & ~15;
858     int surface_count;
859     switch (codec_id) {
860     case AV_CODEC_ID_H264:
861         surface_count = 16 + sys->thread_count + 2;
862         break;
863     case AV_CODEC_ID_MPEG1VIDEO:
864     case AV_CODEC_ID_MPEG2VIDEO:
865         surface_count = 2 + 2;
866         break;
867     default:
868         surface_count = 2 + 1;
869         break;
870     }
871     if (surface_count > VA_DXVA2_MAX_SURFACE_COUNT)
872         return VLC_EGENERIC;
873     sys->surface_count = surface_count;
874     if (FAILED(IDirectXVideoDecoderService_CreateSurface(sys->vs,
875                                                          sys->surface_width,
876                                                          sys->surface_height,
877                                                          sys->surface_count - 1,
878                                                          sys->render,
879                                                          D3DPOOL_DEFAULT,
880                                                          0,
881                                                          DXVA2_VideoDecoderRenderTarget,
882                                                          sys->hw_surface,
883                                                          NULL))) {
884         msg_Err(va, "IDirectXVideoAccelerationService_CreateSurface failed");
885         sys->surface_count = 0;
886         return VLC_EGENERIC;
887     }
888     for (unsigned i = 0; i < sys->surface_count; i++) {
889         vlc_va_surface_t *surface = &sys->surface[i];
890         surface->d3d = sys->hw_surface[i];
891         surface->refcount = 0;
892         surface->order = 0;
893     }
894     msg_Dbg(va, "IDirectXVideoAccelerationService_CreateSurface succeed with %d surfaces (%dx%d)",
895             sys->surface_count, fmt->i_width, fmt->i_height);
896
897     /* */
898     DXVA2_VideoDesc dsc;
899     ZeroMemory(&dsc, sizeof(dsc));
900     dsc.SampleWidth     = fmt->i_width;
901     dsc.SampleHeight    = fmt->i_height;
902     dsc.Format          = sys->render;
903     if (fmt->i_frame_rate > 0 && fmt->i_frame_rate_base > 0) {
904         dsc.InputSampleFreq.Numerator   = fmt->i_frame_rate;
905         dsc.InputSampleFreq.Denominator = fmt->i_frame_rate_base;
906     } else {
907         dsc.InputSampleFreq.Numerator   = 0;
908         dsc.InputSampleFreq.Denominator = 0;
909     }
910     dsc.OutputFrameFreq = dsc.InputSampleFreq;
911     dsc.UABProtectionLevel = FALSE;
912     dsc.Reserved = 0;
913
914     /* FIXME I am unsure we can let unknown everywhere */
915     DXVA2_ExtendedFormat *ext = &dsc.SampleFormat;
916     ext->SampleFormat = 0;//DXVA2_SampleUnknown;
917     ext->VideoChromaSubsampling = 0;//DXVA2_VideoChromaSubsampling_Unknown;
918     ext->NominalRange = 0;//DXVA2_NominalRange_Unknown;
919     ext->VideoTransferMatrix = 0;//DXVA2_VideoTransferMatrix_Unknown;
920     ext->VideoLighting = 0;//DXVA2_VideoLighting_Unknown;
921     ext->VideoPrimaries = 0;//DXVA2_VideoPrimaries_Unknown;
922     ext->VideoTransferFunction = 0;//DXVA2_VideoTransFunc_Unknown;
923
924     /* List all configurations available for the decoder */
925     UINT                      cfg_count = 0;
926     DXVA2_ConfigPictureDecode *cfg_list = NULL;
927     if (FAILED(IDirectXVideoDecoderService_GetDecoderConfigurations(sys->vs,
928                                                                     &sys->input,
929                                                                     &dsc,
930                                                                     NULL,
931                                                                     &cfg_count,
932                                                                     &cfg_list))) {
933         msg_Err(va, "IDirectXVideoDecoderService_GetDecoderConfigurations failed");
934         return VLC_EGENERIC;
935     }
936     msg_Dbg(va, "we got %d decoder configurations", cfg_count);
937
938     /* Select the best decoder configuration */
939     int cfg_score = 0;
940     for (unsigned i = 0; i < cfg_count; i++) {
941         const DXVA2_ConfigPictureDecode *cfg = &cfg_list[i];
942
943         /* */
944         msg_Dbg(va, "configuration[%d] ConfigBitstreamRaw %d",
945                 i, cfg->ConfigBitstreamRaw);
946
947         /* */
948         int score;
949         if (cfg->ConfigBitstreamRaw == 1)
950             score = 1;
951         else if (codec_id == AV_CODEC_ID_H264 && cfg->ConfigBitstreamRaw == 2)
952             score = 2;
953         else
954             continue;
955         if (IsEqualGUID(&cfg->guidConfigBitstreamEncryption, &DXVA_NoEncrypt))
956             score += 16;
957
958         if (cfg_score < score) {
959             sys->cfg = *cfg;
960             cfg_score = score;
961         }
962     }
963     CoTaskMemFree(cfg_list);
964     if (cfg_score <= 0) {
965         msg_Err(va, "Failed to find a supported decoder configuration");
966         return VLC_EGENERIC;
967     }
968
969     /* Create the decoder */
970     IDirectXVideoDecoder *decoder;
971     if (FAILED(IDirectXVideoDecoderService_CreateVideoDecoder(sys->vs,
972                                                               &sys->input,
973                                                               &dsc,
974                                                               &sys->cfg,
975                                                               sys->hw_surface,
976                                                               sys->surface_count,
977                                                               &decoder))) {
978         msg_Err(va, "IDirectXVideoDecoderService_CreateVideoDecoder failed");
979         return VLC_EGENERIC;
980     }
981     sys->decoder = decoder;
982     msg_Dbg(va, "IDirectXVideoDecoderService_CreateVideoDecoder succeed");
983     return VLC_SUCCESS;
984 }
985
986 static void DxDestroyVideoDecoder(vlc_va_sys_t *va)
987 {
988     if (va->decoder)
989         IDirectXVideoDecoder_Release(va->decoder);
990     va->decoder = NULL;
991
992     for (unsigned i = 0; i < va->surface_count; i++)
993         IDirect3DSurface9_Release(va->surface[i].d3d);
994     va->surface_count = 0;
995 }
996
997 static int DxResetVideoDecoder(vlc_va_t *va)
998 {
999     msg_Err(va, "DxResetVideoDecoder unimplemented");
1000     return VLC_EGENERIC;
1001 }
1002
1003 static void DxCreateVideoConversion(vlc_va_sys_t *va)
1004 {
1005     switch (va->render) {
1006     case MAKEFOURCC('N','V','1','2'):
1007     case MAKEFOURCC('I','M','C','3'):
1008         va->output = MAKEFOURCC('Y','V','1','2');
1009         break;
1010     default:
1011         va->output = va->render;
1012         break;
1013     }
1014     CopyInitCache(&va->surface_cache, va->surface_width);
1015 }
1016
1017 static void DxDestroyVideoConversion(vlc_va_sys_t *va)
1018 {
1019     CopyCleanCache(&va->surface_cache);
1020 }