git.sesse.net Git - vlc/blob - modules/video_output/msw/direct3d.c

   1 /*****************************************************************************
   2  * direct3d.c: Windows Direct3D video output module
   3  *****************************************************************************
   4  * Copyright (C) 2006-2009 the VideoLAN team
   5  *$Id$
   6  *
   7  * Authors: Damien Fouilleul <damienf@videolan.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * Preamble:
  26  *
  27  * This plugin will use YUV surface if supported, using YUV will result in
  28  * the best video quality (hardware filering when rescaling the picture)
  29  * and the fastest display as it requires less processing.
  30  *
  31  * If YUV overlay is not supported this plugin will use RGB offscreen video
  32  * surfaces that will be blitted onto the primary surface (display) to
  33  * effectively display the pictures.
  34  *
  35  *****************************************************************************/
  36 #ifdef HAVE_CONFIG_H
  37 # include "config.h"
  38 #endif
  39
  40 #include <vlc_common.h>
  41 #include <vlc_plugin.h>
  42 #include <vlc_playlist.h>
  43 #include <vlc_vout_display.h>
  44
  45 #include <windows.h>
  46 #include <d3d9.h>
  47
  48 #include "common.h"
  49
  50 /*****************************************************************************
  51  * Module descriptor
  52  *****************************************************************************/
  53 static int  OpenVideoXP(vlc_object_t *);
  54 static int  OpenVideoVista(vlc_object_t *);
  55 static void Close(vlc_object_t *);
  56
  57 #define DESKTOP_TEXT N_("Enable desktop mode ")
  58 #define DESKTOP_LONGTEXT N_(\
  59     "The desktop mode allows you to display the video on the desktop.")
  60
  61 #define HW_BLENDING_TEXT N_("Use hardware blending support")
  62 #define HW_BLENDING_LONGTEXT N_(\
  63     "Try to use hardware acceleration for subtitles/OSD blending.")
  64
  65 #define D3D_HELP N_("Recommended video output for Windows Vista and later versions")
  66
  67 vlc_module_begin ()
  68     set_shortname("Direct3D")
  69     set_description(N_("Direct3D video output"))
  70     set_help(D3D_HELP)
  71     set_category(CAT_VIDEO)
  72     set_subcategory(SUBCAT_VIDEO_VOUT)
  73
  74     add_bool("direct3d-desktop", false, DESKTOP_TEXT, DESKTOP_LONGTEXT, true)
  75     add_bool("direct3d-hw-blending", true, HW_BLENDING_TEXT, HW_BLENDING_LONGTEXT, true)
  76
  77     set_capability("vout display", 240)
  78     add_shortcut("direct3d")
  79     set_callbacks(OpenVideoVista, Close)
  80
  81     add_submodule()
  82         set_description(N_("Direct3D video output (XP)"))
  83         set_capability("vout display", 220)
  84         add_shortcut("direct3d_xp")
  85         set_callbacks(OpenVideoXP, Close)
  86
  87 vlc_module_end ()
  88
  89 /*****************************************************************************
  90  * Local prototypes.
  91  *****************************************************************************/
  92 static const vlc_fourcc_t d3d_subpicture_chromas[] = {
  93     VLC_CODEC_RGBA,
  94     0
  95 };
  96
  97 struct picture_sys_t
  98 {
  99     LPDIRECT3DSURFACE9 surface;
 100     picture_t          *fallback;
 101 };
 102
 103 static int  Open(vlc_object_t *);
 104
 105 static picture_pool_t *Pool  (vout_display_t *, unsigned);
 106 static void           Prepare(vout_display_t *, picture_t *, subpicture_t *subpicture);
 107 static void           Display(vout_display_t *, picture_t *, subpicture_t *subpicture);
 108 static int            Control(vout_display_t *, int, va_list);
 109 static void           Manage (vout_display_t *);
 110
 111 static int  Direct3DCreate (vout_display_t *);
 112 static int  Direct3DReset  (vout_display_t *);
 113 static void Direct3DDestroy(vout_display_t *);
 114
 115 static int  Direct3DOpen (vout_display_t *, video_format_t *);
 116 static void Direct3DClose(vout_display_t *);
 117
 118 /* */
 119 typedef struct
 120 {
 121     FLOAT       x,y,z;      // vertex untransformed position
 122     FLOAT       rhw;        // eye distance
 123     D3DCOLOR    diffuse;    // diffuse color
 124     FLOAT       tu, tv;     // texture relative coordinates
 125 } CUSTOMVERTEX;
 126 #define D3DFVF_CUSTOMVERTEX (D3DFVF_XYZRHW|D3DFVF_DIFFUSE|D3DFVF_TEX1)
 127
 128 typedef struct d3d_region_t {
 129     D3DFORMAT          format;
 130     unsigned           width;
 131     unsigned           height;
 132     CUSTOMVERTEX       vertex[4];
 133     LPDIRECT3DTEXTURE9 texture;
 134 } d3d_region_t;
 135
 136 static void Direct3DDeleteRegions(int, d3d_region_t *);
 137
 138 static int  Direct3DImportPicture(vout_display_t *vd, d3d_region_t *, LPDIRECT3DSURFACE9 surface);
 139 static void Direct3DImportSubpicture(vout_display_t *vd, int *, d3d_region_t **, subpicture_t *);
 140
 141 static void Direct3DRenderScene(vout_display_t *vd, d3d_region_t *, int, d3d_region_t *);
 142
 143 /* */
 144 static int DesktopCallback(vlc_object_t *, char const *, vlc_value_t, vlc_value_t, void *);
 145
 146 /**
 147  * It creates a Direct3D vout display.
 148  */
 149 static int Open(vlc_object_t *object)
 150 {
 151     vout_display_t *vd = (vout_display_t *)object;
 152     vout_display_sys_t *sys;
 153
 154     /* Allocate structure */
 155     vd->sys = sys = calloc(1, sizeof(vout_display_sys_t));
 156     if (!sys)
 157         return VLC_ENOMEM;
 158
 159     if (Direct3DCreate(vd)) {
 160         msg_Err(vd, "Direct3D could not be initialized");
 161         Direct3DDestroy(vd);
 162         free(sys);
 163         return VLC_EGENERIC;
 164     }
 165
 166     sys->use_desktop = var_CreateGetBool(vd, "direct3d-desktop");
 167     sys->reset_device = false;
 168     sys->reset_device = false;
 169     sys->allow_hw_yuv = var_CreateGetBool(vd, "directx-hw-yuv");
 170     sys->desktop_save.is_fullscreen = vd->cfg->is_fullscreen;
 171     sys->desktop_save.is_on_top     = false;
 172     sys->desktop_save.win.left      = var_InheritInteger(vd, "video-x");
 173     sys->desktop_save.win.right     = vd->cfg->display.width;
 174     sys->desktop_save.win.top       = var_InheritInteger(vd, "video-y");
 175     sys->desktop_save.win.bottom    = vd->cfg->display.height;
 176
 177     if (CommonInit(vd))
 178         goto error;
 179
 180     /* */
 181     video_format_t fmt;
 182     if (Direct3DOpen(vd, &fmt)) {
 183         msg_Err(vd, "Direct3D could not be opened");
 184         goto error;
 185     }
 186
 187     /* */
 188     vout_display_info_t info = vd->info;
 189     info.is_slow = true;
 190     info.has_double_click = true;
 191     info.has_hide_mouse = false;
 192     info.has_pictures_invalid = true;
 193     info.has_event_thread = true;
 194     if (var_InheritBool(vd, "direct3d-hw-blending") &&
 195         (sys->d3dcaps.SrcBlendCaps  & D3DPBLENDCAPS_SRCALPHA) &&
 196         (sys->d3dcaps.DestBlendCaps & D3DPBLENDCAPS_INVSRCALPHA) &&
 197         (sys->d3dcaps.TextureCaps   & D3DPTEXTURECAPS_ALPHA) &&
 198         (sys->d3dcaps.TextureOpCaps & D3DTEXOPCAPS_SELECTARG1) &&
 199         (sys->d3dcaps.TextureOpCaps & D3DTEXOPCAPS_MODULATE))
 200         info.subpicture_chromas = d3d_subpicture_chromas;
 201     else
 202         info.subpicture_chromas = NULL;
 203
 204     /* Interaction */
 205     vlc_mutex_init(&sys->lock);
 206     sys->ch_desktop = false;
 207     sys->desktop_requested = sys->use_desktop;
 208
 209     vlc_value_t val;
 210     val.psz_string = _("Desktop");
 211     var_Change(vd, "direct3d-desktop", VLC_VAR_SETTEXT, &val, NULL);
 212     var_AddCallback(vd, "direct3d-desktop", DesktopCallback, NULL);
 213
 214     /* Setup vout_display now that everything is fine */
 215     vd->fmt  = fmt;
 216     vd->info = info;
 217
 218     vd->pool    = Pool;
 219     vd->prepare = Prepare;
 220     vd->display = Display;
 221     vd->control = Control;
 222     vd->manage  = Manage;
 223
 224     /* Fix state in case of desktop mode */
 225     if (sys->use_desktop && vd->cfg->is_fullscreen)
 226         vout_display_SendEventFullscreen(vd, false);
 227
 228     return VLC_SUCCESS;
 229 error:
 230     Direct3DClose(vd);
 231     CommonClean(vd);
 232     Direct3DDestroy(vd);
 233     free(vd->sys);
 234     return VLC_EGENERIC;
 235 }
 236
 237 static bool IsVistaOrAbove(void)
 238 {
 239     OSVERSIONINFO winVer;
 240     winVer.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
 241     return GetVersionEx(&winVer) && winVer.dwMajorVersion > 5;
 242 }
 243
 244 static int OpenVideoXP(vlc_object_t *obj)
 245 {
 246     /* Windows XP or lower, make sure this module isn't the default */
 247     return IsVistaOrAbove() ? VLC_EGENERIC : Open(obj);
 248 }
 249
 250 static int OpenVideoVista(vlc_object_t *obj)
 251 {
 252     /* Windows Vista or above, make this module the default */
 253     return IsVistaOrAbove() ? Open(obj) : VLC_EGENERIC;
 254 }
 255
 256 /**
 257  * It destroyes a Direct3D vout display.
 258  */
 259 static void Close(vlc_object_t *object)
 260 {
 261     vout_display_t * vd = (vout_display_t *)object;
 262
 263     var_DelCallback(vd, "direct3d-desktop", DesktopCallback, NULL);
 264     vlc_mutex_destroy(&vd->sys->lock);
 265
 266     Direct3DClose(vd);
 267
 268     CommonClean(vd);
 269
 270     Direct3DDestroy(vd);
 271
 272     free(vd->sys);
 273 }
 274
 275 /* */
 276 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
 277 {
 278     VLC_UNUSED(count);
 279     return vd->sys->pool;
 280 }
 281
 282 static int  Direct3DLockSurface(picture_t *);
 283 static void Direct3DUnlockSurface(picture_t *);
 284
 285 static void Prepare(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
 286 {
 287     vout_display_sys_t *sys = vd->sys;
 288     LPDIRECT3DSURFACE9 surface = picture->p_sys->surface;
 289 #if 0
 290     picture_Release(picture);
 291     VLC_UNUSED(subpicture);
 292 #else
 293     /* FIXME it is a bit ugly, we need the surface to be unlocked for
 294      * rendering.
 295      *  The clean way would be to release the picture (and ensure that
 296      * the vout doesn't keep a reference). But because of the vout
 297      * wrapper, we can't */
 298
 299     Direct3DUnlockSurface(picture);
 300     VLC_UNUSED(subpicture);
 301 #endif
 302
 303     /* check if device is still available */
 304     HRESULT hr = IDirect3DDevice9_TestCooperativeLevel(sys->d3ddev);
 305     if (FAILED(hr)) {
 306         if (hr == D3DERR_DEVICENOTRESET && !sys->reset_device) {
 307             vout_display_SendEventPicturesInvalid(vd);
 308             sys->reset_device = true;
 309         }
 310         return;
 311     }
 312
 313     d3d_region_t picture_region;
 314     if (!Direct3DImportPicture(vd, &picture_region, surface)) {
 315         int subpicture_region_count     = 0;
 316         d3d_region_t *subpicture_region = NULL;
 317         if (subpicture)
 318             Direct3DImportSubpicture(vd, &subpicture_region_count, &subpicture_region,
 319                                      subpicture);
 320
 321         Direct3DRenderScene(vd, &picture_region,
 322                             subpicture_region_count, subpicture_region);
 323
 324         Direct3DDeleteRegions(sys->d3dregion_count, sys->d3dregion);
 325         sys->d3dregion_count = subpicture_region_count;
 326         sys->d3dregion       = subpicture_region;
 327     }
 328 }
 329
 330 static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
 331 {
 332     vout_display_sys_t *sys = vd->sys;
 333     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
 334
 335     // Present the back buffer contents to the display
 336     // No stretching should happen here !
 337     const RECT src = sys->rect_dest_clipped;
 338     const RECT dst = sys->rect_dest_clipped;
 339     HRESULT hr = IDirect3DDevice9_Present(d3ddev, &src, &dst, NULL, NULL);
 340     if (FAILED(hr)) {
 341         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
 342     }
 343
 344 #if 0
 345     VLC_UNUSED(picture);
 346     VLC_UNUSED(subpicture);
 347 #else
 348     /* XXX See Prepare() */
 349     Direct3DLockSurface(picture);
 350     picture_Release(picture);
 351 #endif
 352     if (subpicture)
 353         subpicture_Delete(subpicture);
 354
 355     CommonDisplay(vd);
 356 }
 357 static int ControlResetDevice(vout_display_t *vd)
 358 {
 359     return Direct3DReset(vd);
 360 }
 361 static int ControlReopenDevice(vout_display_t *vd)
 362 {
 363     vout_display_sys_t *sys = vd->sys;
 364
 365     if (!sys->use_desktop) {
 366         /* Save non-desktop state */
 367         sys->desktop_save.is_fullscreen = vd->cfg->is_fullscreen;
 368         sys->desktop_save.is_on_top     = sys->is_on_top;
 369
 370         WINDOWPLACEMENT wp = { .length = sizeof(wp), };
 371         GetWindowPlacement(sys->hparent ? sys->hparent : sys->hwnd, &wp);
 372         sys->desktop_save.win = wp.rcNormalPosition;
 373     }
 374
 375     /* */
 376     Direct3DClose(vd);
 377     EventThreadStop(sys->event);
 378
 379     /* */
 380     vlc_mutex_lock(&sys->lock);
 381     sys->use_desktop = sys->desktop_requested;
 382     sys->ch_desktop = false;
 383     vlc_mutex_unlock(&sys->lock);
 384
 385     /* */
 386     event_cfg_t cfg;
 387     memset(&cfg, 0, sizeof(cfg));
 388     cfg.use_desktop = sys->use_desktop;
 389     if (!sys->use_desktop) {
 390         cfg.win.type   = VOUT_WINDOW_TYPE_HWND;
 391         cfg.win.x      = sys->desktop_save.win.left;
 392         cfg.win.y      = sys->desktop_save.win.top;
 393         cfg.win.width  = sys->desktop_save.win.right  - sys->desktop_save.win.left;
 394         cfg.win.height = sys->desktop_save.win.bottom - sys->desktop_save.win.top;
 395     }
 396
 397     event_hwnd_t hwnd;
 398     if (EventThreadStart(sys->event, &hwnd, &cfg)) {
 399         msg_Err(vd, "Failed to restart event thread");
 400         return VLC_EGENERIC;
 401     }
 402     sys->parent_window = hwnd.parent_window;
 403     sys->hparent       = hwnd.hparent;
 404     sys->hwnd          = hwnd.hwnd;
 405     sys->hvideownd     = hwnd.hvideownd;
 406     sys->hfswnd        = hwnd.hfswnd;
 407     SetRectEmpty(&sys->rect_parent);
 408
 409     /* */
 410     video_format_t fmt;
 411     if (Direct3DOpen(vd, &fmt)) {
 412         CommonClean(vd);
 413         msg_Err(vd, "Failed to reopen device");
 414         return VLC_EGENERIC;
 415     }
 416     vd->fmt = fmt;
 417     sys->is_first_display = true;
 418
 419     if (sys->use_desktop) {
 420         /* Disable fullscreen/on_top while using desktop */
 421         if (sys->desktop_save.is_fullscreen)
 422             vout_display_SendEventFullscreen(vd, false);
 423         if (sys->desktop_save.is_on_top)
 424             vout_display_SendWindowState(vd, VOUT_WINDOW_STATE_NORMAL);
 425     } else {
 426         /* Restore fullscreen/on_top */
 427         if (sys->desktop_save.is_fullscreen)
 428             vout_display_SendEventFullscreen(vd, true);
 429         if (sys->desktop_save.is_on_top)
 430             vout_display_SendWindowState(vd, VOUT_WINDOW_STATE_ABOVE);
 431     }
 432     return VLC_SUCCESS;
 433 }
 434 static int Control(vout_display_t *vd, int query, va_list args)
 435 {
 436     vout_display_sys_t *sys = vd->sys;
 437
 438     switch (query) {
 439     case VOUT_DISPLAY_RESET_PICTURES:
 440         /* FIXME what to do here in case of failure */
 441         if (sys->reset_device) {
 442             if (ControlResetDevice(vd)) {
 443                 msg_Err(vd, "Failed to reset device");
 444                 return VLC_EGENERIC;
 445             }
 446             sys->reset_device = false;
 447         } else if(sys->reopen_device) {
 448             if (ControlReopenDevice(vd)) {
 449                 msg_Err(vd, "Failed to reopen device");
 450                 return VLC_EGENERIC;
 451             }
 452             sys->reopen_device = false;
 453         }
 454         return VLC_SUCCESS;
 455     default:
 456         return CommonControl(vd, query, args);
 457     }
 458 }
 459 static void Manage (vout_display_t *vd)
 460 {
 461     vout_display_sys_t *sys = vd->sys;
 462
 463     CommonManage(vd);
 464
 465     /* Desktop mode change */
 466     vlc_mutex_lock(&sys->lock);
 467     const bool ch_desktop = sys->ch_desktop;
 468     sys->ch_desktop = false;
 469     vlc_mutex_unlock(&sys->lock);
 470
 471     if (ch_desktop) {
 472         sys->reopen_device = true;
 473         vout_display_SendEventPicturesInvalid(vd);
 474     }
 475
 476 #if 0
 477     /*
 478      * Position Change
 479      */
 480     if (sys->changes & DX_POSITION_CHANGE) {
 481 #if 0 /* need that when bicubic filter is available */
 482         RECT rect;
 483         UINT width, height;
 484
 485         GetClientRect(p_sys->hvideownd, &rect);
 486         width  = rect.right-rect.left;
 487         height = rect.bottom-rect.top;
 488
 489         if (width != p_sys->d3dpp.BackBufferWidth || height != p_sys->d3dpp.BackBufferHeight)
 490         {
 491             msg_Dbg(vd, "resizing device back buffers to (%lux%lu)", width, height);
 492             // need to reset D3D device to resize back buffer
 493             if (VLC_SUCCESS != Direct3DResetDevice(vd, width, height))
 494                 return VLC_EGENERIC;
 495         }
 496 #endif
 497         sys->changes &= ~DX_POSITION_CHANGE;
 498     }
 499 #endif
 500 }
 501
 502 /**
 503  * It initializes an instance of Direct3D9
 504  */
 505 static int Direct3DCreate(vout_display_t *vd)
 506 {
 507     vout_display_sys_t *sys = vd->sys;
 508
 509     sys->hd3d9_dll = LoadLibrary(TEXT("D3D9.DLL"));
 510     if (!sys->hd3d9_dll) {
 511         msg_Warn(vd, "cannot load d3d9.dll, aborting");
 512         return VLC_EGENERIC;
 513     }
 514
 515     LPDIRECT3D9 (WINAPI *OurDirect3DCreate9)(UINT SDKVersion);
 516     OurDirect3DCreate9 =
 517         (void *)GetProcAddress(sys->hd3d9_dll, TEXT("Direct3DCreate9"));
 518     if (!OurDirect3DCreate9) {
 519         msg_Err(vd, "Cannot locate reference to Direct3DCreate9 ABI in DLL");
 520         return VLC_EGENERIC;
 521     }
 522
 523     /* Create the D3D object. */
 524     LPDIRECT3D9 d3dobj = OurDirect3DCreate9(D3D_SDK_VERSION);
 525     if (!d3dobj) {
 526        msg_Err(vd, "Could not create Direct3D9 instance.");
 527        return VLC_EGENERIC;
 528     }
 529     sys->d3dobj = d3dobj;
 530
 531     /*
 532     ** Get device capabilities
 533     */
 534     ZeroMemory(&sys->d3dcaps, sizeof(sys->d3dcaps));
 535     HRESULT hr = IDirect3D9_GetDeviceCaps(d3dobj, D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, &sys->d3dcaps);
 536     if (FAILED(hr)) {
 537        msg_Err(vd, "Could not read adapter capabilities. (hr=0x%lX)", hr);
 538        return VLC_EGENERIC;
 539     }
 540
 541     /* TODO: need to test device capabilities and select the right render function */
 542
 543     return VLC_SUCCESS;
 544 }
 545
 546 /**
 547  * It releases an instance of Direct3D9
 548  */
 549 static void Direct3DDestroy(vout_display_t *vd)
 550 {
 551     vout_display_sys_t *sys = vd->sys;
 552
 553     if (sys->d3dobj)
 554        IDirect3D9_Release(sys->d3dobj);
 555     if (sys->hd3d9_dll)
 556         FreeLibrary(sys->hd3d9_dll);
 557
 558     sys->d3dobj = NULL;
 559     sys->hd3d9_dll = NULL;
 560 }
 561
 562
 563 /**
 564  * It setup vout_display_sys_t::d3dpp and vout_display_sys_t::rect_display
 565  * from the default adapter.
 566  */
 567 static int Direct3DFillPresentationParameters(vout_display_t *vd)
 568 {
 569     vout_display_sys_t *sys = vd->sys;
 570
 571     /*
 572     ** Get the current desktop display mode, so we can set up a back
 573     ** buffer of the same format
 574     */
 575     D3DDISPLAYMODE d3ddm;
 576     HRESULT hr = IDirect3D9_GetAdapterDisplayMode(sys->d3dobj,
 577                                                   D3DADAPTER_DEFAULT, &d3ddm);
 578     if (FAILED(hr)) {
 579        msg_Err(vd, "Could not read adapter display mode. (hr=0x%lX)", hr);
 580        return VLC_EGENERIC;
 581     }
 582
 583     /* Set up the structure used to create the D3DDevice. */
 584     D3DPRESENT_PARAMETERS *d3dpp = &vd->sys->d3dpp;
 585     ZeroMemory(d3dpp, sizeof(D3DPRESENT_PARAMETERS));
 586     d3dpp->Flags                  = D3DPRESENTFLAG_VIDEO;
 587     d3dpp->Windowed               = TRUE;
 588     d3dpp->hDeviceWindow          = vd->sys->hvideownd;
 589     d3dpp->BackBufferWidth        = __MAX((unsigned int)GetSystemMetrics(SM_CXVIRTUALSCREEN),
 590                                           d3ddm.Width);
 591     d3dpp->BackBufferHeight       = __MAX((unsigned int)GetSystemMetrics(SM_CYVIRTUALSCREEN),
 592                                           d3ddm.Height);
 593     d3dpp->SwapEffect             = D3DSWAPEFFECT_COPY;
 594     d3dpp->MultiSampleType        = D3DMULTISAMPLE_NONE;
 595     d3dpp->PresentationInterval   = D3DPRESENT_INTERVAL_DEFAULT;
 596     d3dpp->BackBufferFormat       = d3ddm.Format;
 597     d3dpp->BackBufferCount        = 1;
 598     d3dpp->EnableAutoDepthStencil = FALSE;
 599
 600     /* */
 601     RECT *display = &vd->sys->rect_display;
 602     display->left   = 0;
 603     display->top    = 0;
 604     display->right  = d3dpp->BackBufferWidth;
 605     display->bottom = d3dpp->BackBufferHeight;
 606
 607     return VLC_SUCCESS;
 608 }
 609
 610 /* */
 611 static int  Direct3DCreateResources (vout_display_t *, video_format_t *);
 612 static void Direct3DDestroyResources(vout_display_t *);
 613
 614 /**
 615  * It creates a Direct3D device and the associated resources.
 616  */
 617 static int Direct3DOpen(vout_display_t *vd, video_format_t *fmt)
 618 {
 619     vout_display_sys_t *sys = vd->sys;
 620     LPDIRECT3D9 d3dobj = sys->d3dobj;
 621
 622     if (Direct3DFillPresentationParameters(vd))
 623         return VLC_EGENERIC;
 624
 625     // Create the D3DDevice
 626     LPDIRECT3DDEVICE9 d3ddev;
 627
 628     UINT AdapterToUse = D3DADAPTER_DEFAULT;
 629     D3DDEVTYPE DeviceType = D3DDEVTYPE_HAL;
 630
 631 #ifndef NDEBUG
 632     // Look for 'NVIDIA PerfHUD' adapter
 633     // If it is present, override default settings
 634     for (UINT Adapter=0; Adapter< IDirect3D9_GetAdapterCount(d3dobj); ++Adapter) {
 635         D3DADAPTER_IDENTIFIER9 Identifier;
 636         HRESULT Res;
 637         Res = IDirect3D9_GetAdapterIdentifier(d3dobj,Adapter,0,&Identifier);
 638         if (strstr(Identifier.Description,"PerfHUD") != 0) {
 639             AdapterToUse = Adapter;
 640             DeviceType = D3DDEVTYPE_REF;
 641             break;
 642         }
 643     }
 644 #endif
 645
 646     HRESULT hr = IDirect3D9_CreateDevice(d3dobj, AdapterToUse,
 647                                          DeviceType, sys->hvideownd,
 648                                          D3DCREATE_SOFTWARE_VERTEXPROCESSING|
 649                                          D3DCREATE_MULTITHREADED,
 650                                          &sys->d3dpp, &d3ddev);
 651     if (FAILED(hr)) {
 652        msg_Err(vd, "Could not create the D3D device! (hr=0x%lX)", hr);
 653        return VLC_EGENERIC;
 654     }
 655     sys->d3ddev = d3ddev;
 656
 657     UpdateRects(vd, NULL, NULL, true);
 658
 659     if (Direct3DCreateResources(vd, fmt)) {
 660         msg_Err(vd, "Failed to allocate resources");
 661         return VLC_EGENERIC;
 662     }
 663
 664     /* Change the window title bar text */
 665     EventThreadUpdateTitle(sys->event, VOUT_TITLE " (Direct3D output)");
 666
 667     msg_Dbg(vd, "Direct3D device adapter successfully initialized");
 668     return VLC_SUCCESS;
 669 }
 670
 671 /**
 672  * It releases the Direct3D9 device and its resources.
 673  */
 674 static void Direct3DClose(vout_display_t *vd)
 675 {
 676     vout_display_sys_t *sys = vd->sys;
 677
 678     Direct3DDestroyResources(vd);
 679
 680     if (sys->d3ddev)
 681        IDirect3DDevice9_Release(sys->d3ddev);
 682
 683     sys->d3ddev = NULL;
 684 }
 685
 686 /**
 687  * It reset the Direct3D9 device and its resources.
 688  */
 689 static int Direct3DReset(vout_display_t *vd)
 690 {
 691     vout_display_sys_t *sys = vd->sys;
 692     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
 693
 694     if (Direct3DFillPresentationParameters(vd))
 695         return VLC_EGENERIC;
 696
 697     /* release all D3D objects */
 698     Direct3DDestroyResources(vd);
 699
 700     /* */
 701     HRESULT hr = IDirect3DDevice9_Reset(d3ddev, &sys->d3dpp);
 702     if (FAILED(hr)) {
 703         msg_Err(vd, "%s failed ! (hr=%08lX)", __FUNCTION__, hr);
 704         return VLC_EGENERIC;
 705     }
 706
 707     UpdateRects(vd, NULL, NULL, true);
 708
 709     /* re-create them */
 710     if (Direct3DCreateResources(vd, &vd->fmt)) {
 711         msg_Dbg(vd, "%s failed !", __FUNCTION__);
 712         return VLC_EGENERIC;
 713     }
 714     return VLC_SUCCESS;
 715 }
 716
 717 /* */
 718 static int  Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt);
 719 static void Direct3DDestroyPool(vout_display_t *vd);
 720
 721 static int  Direct3DCreateScene(vout_display_t *vd, const video_format_t *fmt);
 722 static void Direct3DDestroyScene(vout_display_t *vd);
 723
 724 /**
 725  * It creates the picture and scene resources.
 726  */
 727 static int Direct3DCreateResources(vout_display_t *vd, video_format_t *fmt)
 728 {
 729     if (Direct3DCreatePool(vd, fmt)) {
 730         msg_Err(vd, "Direct3D picture pool initialization failed");
 731         return VLC_EGENERIC;
 732     }
 733     if (Direct3DCreateScene(vd, fmt)) {
 734         msg_Err(vd, "Direct3D scene initialization failed !");
 735         return VLC_EGENERIC;
 736     }
 737     return VLC_SUCCESS;
 738 }
 739 /**
 740  * It destroys the picture and scene resources.
 741  */
 742 static void Direct3DDestroyResources(vout_display_t *vd)
 743 {
 744     Direct3DDestroyScene(vd);
 745     Direct3DDestroyPool(vd);
 746 }
 747
 748 /**
 749  * It tests if the conversion from src to dst is supported.
 750  */
 751 static int Direct3DCheckConversion(vout_display_t *vd,
 752                                    D3DFORMAT src, D3DFORMAT dst)
 753 {
 754     vout_display_sys_t *sys = vd->sys;
 755     LPDIRECT3D9 d3dobj = sys->d3dobj;
 756     HRESULT hr;
 757
 758     /* test whether device can create a surface of that format */
 759     hr = IDirect3D9_CheckDeviceFormat(d3dobj, D3DADAPTER_DEFAULT,
 760                                       D3DDEVTYPE_HAL, dst, 0,
 761                                       D3DRTYPE_SURFACE, src);
 762     if (SUCCEEDED(hr)) {
 763         /* test whether device can perform color-conversion
 764         ** from that format to target format
 765         */
 766         hr = IDirect3D9_CheckDeviceFormatConversion(d3dobj,
 767                                                     D3DADAPTER_DEFAULT,
 768                                                     D3DDEVTYPE_HAL,
 769                                                     src, dst);
 770     }
 771     if (!SUCCEEDED(hr)) {
 772         if (D3DERR_NOTAVAILABLE != hr)
 773             msg_Err(vd, "Could not query adapter supported formats. (hr=0x%lX)", hr);
 774         return VLC_EGENERIC;
 775     }
 776     return VLC_SUCCESS;
 777 }
 778
 779 typedef struct
 780 {
 781     const char   *name;
 782     D3DFORMAT    format;    /* D3D format */
 783     vlc_fourcc_t fourcc;    /* VLC fourcc */
 784     uint32_t     rmask;
 785     uint32_t     gmask;
 786     uint32_t     bmask;
 787 } d3d_format_t;
 788
 789 static const d3d_format_t d3d_formats[] = {
 790     /* YV12 is always used for planar 420, the planes are then swapped in Lock() */
 791     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_YV12,  0,0,0 },
 792     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_I420,  0,0,0 },
 793     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_J420,  0,0,0 },
 794     { "UYVY",       D3DFMT_UYVY,    VLC_CODEC_UYVY,  0,0,0 },
 795     { "YUY2",       D3DFMT_YUY2,    VLC_CODEC_YUYV,  0,0,0 },
 796     { "X8R8G8B8",   D3DFMT_X8R8G8B8,VLC_CODEC_RGB32, 0xff0000, 0x00ff00, 0x0000ff },
 797     { "A8R8G8B8",   D3DFMT_A8R8G8B8,VLC_CODEC_RGB32, 0xff0000, 0x00ff00, 0x0000ff },
 798     { "8G8B8",      D3DFMT_R8G8B8,  VLC_CODEC_RGB24, 0xff0000, 0x00ff00, 0x0000ff },
 799     { "R5G6B5",     D3DFMT_R5G6B5,  VLC_CODEC_RGB16, 0x1f<<11, 0x3f<<5,  0x1f<<0 },
 800     { "X1R5G5B5",   D3DFMT_X1R5G5B5,VLC_CODEC_RGB15, 0x1f<<10, 0x1f<<5,  0x1f<<0 },
 801
 802     { NULL, 0, 0, 0,0,0}
 803 };
 804
 805 /**
 806  * It returns the format (closest to chroma) that can be converted to target */
 807 static const d3d_format_t *Direct3DFindFormat(vout_display_t *vd, vlc_fourcc_t chroma, D3DFORMAT target)
 808 {
 809     vout_display_sys_t *sys = vd->sys;
 810
 811     for (unsigned pass = 0; pass < 2; pass++) {
 812         const vlc_fourcc_t *list;
 813
 814         if (pass == 0 && sys->allow_hw_yuv && vlc_fourcc_IsYUV(chroma))
 815             list = vlc_fourcc_GetYUVFallback(chroma);
 816         else if (pass == 1)
 817             list = vlc_fourcc_GetRGBFallback(chroma);
 818         else
 819             continue;
 820
 821         for (unsigned i = 0; list[i] != 0; i++) {
 822             for (unsigned j = 0; d3d_formats[j].name; j++) {
 823                 const d3d_format_t *format = &d3d_formats[j];
 824
 825                 if (format->fourcc != list[i])
 826                     continue;
 827
 828                 msg_Warn(vd, "trying surface pixel format: %s",
 829                          format->name);
 830                 if (!Direct3DCheckConversion(vd, format->format, target)) {
 831                     msg_Dbg(vd, "selected surface pixel format is %s",
 832                             format->name);
 833                     return format;
 834                 }
 835             }
 836         }
 837     }
 838     return NULL;
 839 }
 840
 841 /**
 842  * It locks the surface associated to the picture and get the surface
 843  * descriptor which amongst other things has the pointer to the picture
 844  * data and its pitch.
 845  */
 846 static int Direct3DLockSurface(picture_t *picture)
 847 {
 848     /* Lock the surface to get a valid pointer to the picture buffer */
 849     D3DLOCKED_RECT d3drect;
 850     HRESULT hr = IDirect3DSurface9_LockRect(picture->p_sys->surface, &d3drect, NULL, 0);
 851     if (FAILED(hr)) {
 852         //msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
 853         return CommonUpdatePicture(picture, &picture->p_sys->fallback, NULL, 0);
 854     }
 855
 856     CommonUpdatePicture(picture, NULL, d3drect.pBits, d3drect.Pitch);
 857     return VLC_SUCCESS;
 858 }
 859 /**
 860  * It unlocks the surface associated to the picture.
 861  */
 862 static void Direct3DUnlockSurface(picture_t *picture)
 863 {
 864     /* Unlock the Surface */
 865     HRESULT hr = IDirect3DSurface9_UnlockRect(picture->p_sys->surface);
 866     if (FAILED(hr)) {
 867         //msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
 868     }
 869 }
 870
 871 /**
 872  * It creates the pool of picture (only 1).
 873  *
 874  * Each picture has an associated offscreen surface in video memory
 875  * depending on hardware capabilities the picture chroma will be as close
 876  * as possible to the orginal render chroma to reduce CPU conversion overhead
 877  * and delegate this work to video card GPU
 878  */
 879 static int Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt)
 880 {
 881     vout_display_sys_t *sys = vd->sys;
 882     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
 883
 884     /* */
 885     *fmt = vd->source;
 886
 887     /* Find the appropriate D3DFORMAT for the render chroma, the format will be the closest to
 888      * the requested chroma which is usable by the hardware in an offscreen surface, as they
 889      * typically support more formats than textures */
 890     const d3d_format_t *d3dfmt = Direct3DFindFormat(vd, fmt->i_chroma, sys->d3dpp.BackBufferFormat);
 891     if (!d3dfmt) {
 892         msg_Err(vd, "surface pixel format is not supported.");
 893         return VLC_EGENERIC;
 894     }
 895     fmt->i_chroma = d3dfmt->fourcc;
 896     fmt->i_rmask  = d3dfmt->rmask;
 897     fmt->i_gmask  = d3dfmt->gmask;
 898     fmt->i_bmask  = d3dfmt->bmask;
 899
 900     /* We create one picture.
 901      * It is useless to create more as we can't be used for direct rendering */
 902
 903     /* Create a surface */
 904     LPDIRECT3DSURFACE9 surface;
 905     HRESULT hr = IDirect3DDevice9_CreateOffscreenPlainSurface(d3ddev,
 906                                                               fmt->i_width,
 907                                                               fmt->i_height,
 908                                                               d3dfmt->format,
 909                                                               D3DPOOL_DEFAULT,
 910                                                               &surface,
 911                                                               NULL);
 912     if (FAILED(hr)) {
 913         msg_Err(vd, "Failed to create picture surface. (hr=0x%lx)", hr);
 914         return VLC_EGENERIC;
 915     }
 916     /* fill surface with black color */
 917     IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0));
 918
 919     /* Create the associated picture */
 920     picture_resource_t *rsc = &sys->resource;
 921     rsc->p_sys = malloc(sizeof(*rsc->p_sys));
 922     if (!rsc->p_sys) {
 923         IDirect3DSurface9_Release(surface);
 924         return VLC_ENOMEM;
 925     }
 926     rsc->p_sys->surface = surface;
 927     rsc->p_sys->fallback = NULL;
 928     for (int i = 0; i < PICTURE_PLANE_MAX; i++) {
 929         rsc->p[i].p_pixels = NULL;
 930         rsc->p[i].i_pitch = 0;
 931         rsc->p[i].i_lines = fmt->i_height / (i > 0 ? 2 : 1);
 932     }
 933     picture_t *picture = picture_NewFromResource(fmt, rsc);
 934     if (!picture) {
 935         IDirect3DSurface9_Release(surface);
 936         free(rsc->p_sys);
 937         return VLC_ENOMEM;
 938     }
 939
 940     /* Wrap it into a picture pool */
 941     picture_pool_configuration_t pool_cfg;
 942     memset(&pool_cfg, 0, sizeof(pool_cfg));
 943     pool_cfg.picture_count = 1;
 944     pool_cfg.picture       = &picture;
 945     pool_cfg.lock          = Direct3DLockSurface;
 946     pool_cfg.unlock        = Direct3DUnlockSurface;
 947
 948     sys->pool = picture_pool_NewExtended(&pool_cfg);
 949     if (!sys->pool) {
 950         picture_Release(picture);
 951         IDirect3DSurface9_Release(surface);
 952         return VLC_ENOMEM;
 953     }
 954     return VLC_SUCCESS;
 955 }
 956 /**
 957  * It destroys the pool of picture and its resources.
 958  */
 959 static void Direct3DDestroyPool(vout_display_t *vd)
 960 {
 961     vout_display_sys_t *sys = vd->sys;
 962
 963     if (sys->pool) {
 964         picture_resource_t *rsc = &sys->resource;
 965         IDirect3DSurface9_Release(rsc->p_sys->surface);
 966         if (rsc->p_sys->fallback)
 967             picture_Release(rsc->p_sys->fallback);
 968         picture_pool_Delete(sys->pool);
 969     }
 970     sys->pool = NULL;
 971 }
 972
 973 /**
 974  * It allocates and initializes the resources needed to render the scene.
 975  */
 976 static int Direct3DCreateScene(vout_display_t *vd, const video_format_t *fmt)
 977 {
 978     vout_display_sys_t *sys = vd->sys;
 979     LPDIRECT3DDEVICE9       d3ddev = sys->d3ddev;
 980     HRESULT hr;
 981
 982     /*
 983      * Create a texture for use when rendering a scene
 984      * for performance reason, texture format is identical to backbuffer
 985      * which would usually be a RGB format
 986      */
 987     LPDIRECT3DTEXTURE9 d3dtex;
 988     hr = IDirect3DDevice9_CreateTexture(d3ddev,
 989                                         fmt->i_width,
 990                                         fmt->i_height,
 991                                         1,
 992                                         D3DUSAGE_RENDERTARGET,
 993                                         sys->d3dpp.BackBufferFormat,
 994                                         D3DPOOL_DEFAULT,
 995                                         &d3dtex,
 996                                         NULL);
 997     if (FAILED(hr)) {
 998         msg_Err(vd, "Failed to create texture. (hr=0x%lx)", hr);
 999         return VLC_EGENERIC;
1000     }
1001
1002     /*
1003     ** Create a vertex buffer for use when rendering scene
1004     */
1005     LPDIRECT3DVERTEXBUFFER9 d3dvtc;
1006     hr = IDirect3DDevice9_CreateVertexBuffer(d3ddev,
1007                                              sizeof(CUSTOMVERTEX)*4,
1008                                              D3DUSAGE_DYNAMIC|D3DUSAGE_WRITEONLY,
1009                                              D3DFVF_CUSTOMVERTEX,
1010                                              D3DPOOL_DEFAULT,
1011                                              &d3dvtc,
1012                                              NULL);
1013     if (FAILED(hr)) {
1014         msg_Err(vd, "Failed to create vertex buffer. (hr=0x%lx)", hr);
1015         IDirect3DTexture9_Release(d3dtex);
1016         return VLC_EGENERIC;
1017     }
1018
1019     /* */
1020     sys->d3dtex = d3dtex;
1021     sys->d3dvtc = d3dvtc;
1022
1023     sys->d3dregion_count = 0;
1024     sys->d3dregion       = NULL;
1025
1026     // Texture coordinates outside the range [0.0, 1.0] are set
1027     // to the texture color at 0.0 or 1.0, respectively.
1028     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_ADDRESSU, D3DTADDRESS_CLAMP);
1029     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_ADDRESSV, D3DTADDRESS_CLAMP);
1030
1031     // Set linear filtering quality
1032     if (sys->d3dcaps.TextureFilterCaps & D3DPTFILTERCAPS_MINFLINEAR) {
1033         msg_Dbg(vd, "Using D3DTEXF_LINEAR for minification");
1034         IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR);
1035     } else {
1036         msg_Dbg(vd, "Using D3DTEXF_POINT for minification");
1037         IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MINFILTER, D3DTEXF_POINT);
1038     }
1039     if (sys->d3dcaps.TextureFilterCaps & D3DPTFILTERCAPS_MAGFLINEAR) {
1040         msg_Dbg(vd, "Using D3DTEXF_LINEAR for magnification");
1041         IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR);
1042     } else {
1043         msg_Dbg(vd, "Using D3DTEXF_POINT for magnification");
1044         IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MAGFILTER, D3DTEXF_POINT);
1045     }
1046
1047     // set maximum ambient light
1048     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_AMBIENT, D3DCOLOR_XRGB(255,255,255));
1049
1050     // Turn off culling
1051     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_CULLMODE, D3DCULL_NONE);
1052
1053     // Turn off the zbuffer
1054     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ZENABLE, D3DZB_FALSE);
1055
1056     // Turn off lights
1057     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_LIGHTING, FALSE);
1058
1059     // Enable dithering
1060     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_DITHERENABLE, TRUE);
1061
1062     // disable stencil
1063     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_STENCILENABLE, FALSE);
1064
1065     // manage blending
1066     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHABLENDENABLE, FALSE);
1067     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_SRCBLEND,D3DBLEND_SRCALPHA);
1068     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_DESTBLEND,D3DBLEND_INVSRCALPHA);
1069
1070     if (sys->d3dcaps.AlphaCmpCaps & D3DPCMPCAPS_GREATER) {
1071         IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHATESTENABLE,TRUE);
1072         IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHAREF, 0x00);
1073         IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHAFUNC,D3DCMP_GREATER);
1074     }
1075
1076     // Set texture states
1077     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_COLOROP,D3DTOP_SELECTARG1);
1078     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_COLORARG1,D3DTA_TEXTURE);
1079
1080     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_ALPHAOP, D3DTOP_MODULATE);
1081     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_ALPHAARG1,D3DTA_TEXTURE);
1082     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_ALPHAARG2,D3DTA_DIFFUSE);
1083
1084     msg_Dbg(vd, "Direct3D scene created successfully");
1085
1086     return VLC_SUCCESS;
1087 }
1088
1089 /**
1090  * It releases the scene resources.
1091  */
1092 static void Direct3DDestroyScene(vout_display_t *vd)
1093 {
1094     vout_display_sys_t *sys = vd->sys;
1095
1096     Direct3DDeleteRegions(sys->d3dregion_count, sys->d3dregion);
1097
1098     LPDIRECT3DVERTEXBUFFER9 d3dvtc = sys->d3dvtc;
1099     if (d3dvtc)
1100         IDirect3DVertexBuffer9_Release(d3dvtc);
1101
1102     LPDIRECT3DTEXTURE9 d3dtex = sys->d3dtex;
1103     if (d3dtex)
1104         IDirect3DTexture9_Release(d3dtex);
1105
1106     sys->d3dvtc = NULL;
1107     sys->d3dtex = NULL;
1108
1109     sys->d3dregion_count = 0;
1110     sys->d3dregion       = NULL;
1111
1112     msg_Dbg(vd, "Direct3D scene released successfully");
1113 }
1114
1115 static void Direct3DSetupVertices(CUSTOMVERTEX *vertices,
1116                                   const RECT src_full,
1117                                   const RECT src_crop,
1118                                   const RECT dst,
1119                                   int alpha)
1120 {
1121     const float src_full_width  = src_full.right  - src_full.left;
1122     const float src_full_height = src_full.bottom - src_full.top;
1123     vertices[0].x  = dst.left;
1124     vertices[0].y  = dst.top;
1125     vertices[0].tu = src_crop.left / src_full_width;
1126     vertices[0].tv = src_crop.top  / src_full_height;
1127
1128     vertices[1].x  = dst.right;
1129     vertices[1].y  = dst.top;
1130     vertices[1].tu = src_crop.right / src_full_width;
1131     vertices[1].tv = src_crop.top   / src_full_height;
1132
1133     vertices[2].x  = dst.right;
1134     vertices[2].y  = dst.bottom;
1135     vertices[2].tu = src_crop.right  / src_full_width;
1136     vertices[2].tv = src_crop.bottom / src_full_height;
1137
1138     vertices[3].x  = dst.left;
1139     vertices[3].y  = dst.bottom;
1140     vertices[3].tu = src_crop.left   / src_full_width;
1141     vertices[3].tv = src_crop.bottom / src_full_height;
1142
1143     for (int i = 0; i < 4; i++) {
1144         /* -0.5f is a "feature" of DirectX and it seems to apply to Direct3d also */
1145         /* http://www.sjbrown.co.uk/2003/05/01/fix-directx-rasterisation/ */
1146         vertices[i].x -= 0.5;
1147         vertices[i].y -= 0.5;
1148
1149         vertices[i].z       = 0.0f;
1150         vertices[i].rhw     = 1.0f;
1151         vertices[i].diffuse = D3DCOLOR_ARGB(alpha, 255, 255, 255);
1152     }
1153 }
1154
1155 /**
1156  * It copies picture surface into a texture and setup the associated d3d_region_t.
1157  */
1158 static int Direct3DImportPicture(vout_display_t *vd,
1159                                  d3d_region_t *region,
1160                                  LPDIRECT3DSURFACE9 source)
1161 {
1162     vout_display_sys_t *sys = vd->sys;
1163     HRESULT hr;
1164
1165     if (!source) {
1166         msg_Dbg(vd, "no surface to render ?");
1167         return VLC_EGENERIC;
1168     }
1169
1170     /* retrieve texture top-level surface */
1171     LPDIRECT3DSURFACE9 destination;
1172     hr = IDirect3DTexture9_GetSurfaceLevel(sys->d3dtex, 0, &destination);
1173     if (FAILED(hr)) {
1174         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1175         return VLC_EGENERIC;
1176     }
1177
1178     /* Copy picture surface into texture surface
1179      * color space conversion happen here */
1180     hr = IDirect3DDevice9_StretchRect(sys->d3ddev, source, NULL, destination, NULL, D3DTEXF_NONE);
1181     IDirect3DSurface9_Release(destination);
1182     if (FAILED(hr)) {
1183         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1184         return VLC_EGENERIC;
1185     }
1186
1187     /* */
1188     region->texture = sys->d3dtex;
1189     Direct3DSetupVertices(region->vertex,
1190                           vd->sys->rect_src,
1191                           vd->sys->rect_src_clipped,
1192                           vd->sys->rect_dest_clipped, 255);
1193     return VLC_SUCCESS;
1194 }
1195
1196 static void Direct3DDeleteRegions(int count, d3d_region_t *region)
1197 {
1198     for (int i = 0; i < count; i++) {
1199         if (region[i].texture)
1200             IDirect3DTexture9_Release(region[i].texture);
1201     }
1202     free(region);
1203 }
1204
1205 static void Direct3DImportSubpicture(vout_display_t *vd,
1206                                      int *count_ptr, d3d_region_t **region,
1207                                      subpicture_t *subpicture)
1208 {
1209     vout_display_sys_t *sys = vd->sys;
1210
1211     int count = 0;
1212     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
1213         count++;
1214
1215     *count_ptr = count;
1216     *region    = calloc(count, sizeof(**region));
1217     if (*region == NULL) {
1218         *count_ptr = 0;
1219         return;
1220     }
1221
1222     int i = 0;
1223     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) {
1224         d3d_region_t *d3dr = &(*region)[i];
1225         HRESULT hr;
1226
1227         d3dr->texture = NULL;
1228         for (int j = 0; j < sys->d3dregion_count; j++) {
1229             d3d_region_t *cache = &sys->d3dregion[j];
1230             if (cache->texture &&
1231                 cache->format == D3DFMT_A8R8G8B8 &&
1232                 cache->width  == r->fmt.i_visible_width &&
1233                 cache->height == r->fmt.i_visible_height) {
1234                 msg_Dbg(vd, "Reusing %dx%d texture for OSD",
1235                         cache->width, cache->height);
1236                 *d3dr = *cache;
1237                 memset(cache, 0, sizeof(*cache));
1238             }
1239         }
1240         if (!d3dr->texture) {
1241             d3dr->format = D3DFMT_A8R8G8B8;
1242             d3dr->width  = r->fmt.i_visible_width;
1243             d3dr->height = r->fmt.i_visible_height;
1244             hr = IDirect3DDevice9_CreateTexture(sys->d3ddev,
1245                                                 d3dr->width, d3dr->height,
1246                                                 1,
1247                                                 D3DUSAGE_DYNAMIC,
1248                                                 d3dr->format,
1249                                                 D3DPOOL_DEFAULT,
1250                                                 &d3dr->texture,
1251                                                 NULL);
1252             if (FAILED(hr)) {
1253                 d3dr->texture = NULL;
1254                 msg_Err(vd, "Failed to create %dx%d texture for OSD",
1255                         d3dr->width, d3dr->height);
1256                 continue;
1257             }
1258             msg_Dbg(vd, "Created %dx%d texture for OSD",
1259                     r->fmt.i_visible_width, r->fmt.i_visible_height);
1260         }
1261
1262         D3DLOCKED_RECT lock;
1263         hr = IDirect3DTexture9_LockRect(d3dr->texture, 0, &lock, NULL, 0);
1264         if (SUCCEEDED(hr)) {
1265             uint8_t *dst_data  = lock.pBits;
1266             int      dst_pitch = lock.Pitch;
1267             for (unsigned y = 0; y < r->fmt.i_visible_height; y++) {
1268                 int copy_pitch = __MIN(dst_pitch, r->p_picture->p->i_visible_pitch);
1269                 memcpy(&dst_data[y * dst_pitch],
1270                        &r->p_picture->p->p_pixels[y * r->p_picture->p->i_pitch],
1271                        copy_pitch);
1272             }
1273             hr = IDirect3DTexture9_UnlockRect(d3dr->texture, 0);
1274             if (FAILED(hr))
1275                 msg_Err(vd, "Failed to unlock the texture");
1276         } else {
1277             msg_Err(vd, "Failed to lock the texture");
1278         }
1279
1280         /* Map the subpicture to sys->rect_dest */
1281         RECT src;
1282         src.left   = 0;
1283         src.right  = src.left + r->fmt.i_visible_width;
1284         src.top    = 0;
1285         src.bottom = src.top  + r->fmt.i_visible_height;
1286
1287         const RECT video = sys->rect_dest;
1288         const float scale_w = (float)(video.right  - video.left) / subpicture->i_original_picture_width;
1289         const float scale_h = (float)(video.bottom - video.top)  / subpicture->i_original_picture_height;
1290
1291         RECT dst;
1292         dst.left   = video.left + scale_w * r->i_x,
1293         dst.right  = dst.left + scale_w * r->fmt.i_visible_width,
1294         dst.top    = video.top  + scale_h * r->i_y,
1295         dst.bottom = dst.top  + scale_h * r->fmt.i_visible_height,
1296         Direct3DSetupVertices(d3dr->vertex,
1297                               src, src, dst,
1298                               subpicture->i_alpha * r->i_alpha / 255);
1299     }
1300 }
1301
1302 static int Direct3DRenderRegion(vout_display_t *vd,
1303                                 d3d_region_t *region)
1304 {
1305     vout_display_sys_t *sys = vd->sys;
1306
1307     LPDIRECT3DDEVICE9 d3ddev = vd->sys->d3ddev;
1308
1309     LPDIRECT3DVERTEXBUFFER9 d3dvtc = sys->d3dvtc;
1310     LPDIRECT3DTEXTURE9      d3dtex = region->texture;
1311
1312     HRESULT hr;
1313
1314     /* Import vertices */
1315     void *vertex;
1316     hr = IDirect3DVertexBuffer9_Lock(d3dvtc, 0, 0, &vertex, D3DLOCK_DISCARD);
1317     if (FAILED(hr)) {
1318         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1319         return -1;
1320     }
1321     memcpy(vertex, region->vertex, sizeof(region->vertex));
1322     hr = IDirect3DVertexBuffer9_Unlock(d3dvtc);
1323     if (FAILED(hr)) {
1324         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1325         return -1;
1326     }
1327
1328     // Setup our texture. Using textures introduces the texture stage states,
1329     // which govern how textures get blended together (in the case of multiple
1330     // textures) and lighting information. In this case, we are modulating
1331     // (blending) our texture with the diffuse color of the vertices.
1332     hr = IDirect3DDevice9_SetTexture(d3ddev, 0, (LPDIRECT3DBASETEXTURE9)d3dtex);
1333     if (FAILED(hr)) {
1334         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1335         return -1;
1336     }
1337
1338     // Render the vertex buffer contents
1339     hr = IDirect3DDevice9_SetStreamSource(d3ddev, 0, d3dvtc, 0, sizeof(CUSTOMVERTEX));
1340     if (FAILED(hr)) {
1341         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1342         return -1;
1343     }
1344
1345     // we use FVF instead of vertex shader
1346     hr = IDirect3DDevice9_SetFVF(d3ddev, D3DFVF_CUSTOMVERTEX);
1347     if (FAILED(hr)) {
1348         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1349         return -1;
1350     }
1351
1352     // draw rectangle
1353     hr = IDirect3DDevice9_DrawPrimitive(d3ddev, D3DPT_TRIANGLEFAN, 0, 2);
1354     if (FAILED(hr)) {
1355         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1356         return -1;
1357     }
1358     return 0;
1359 }
1360
1361 /**
1362  * It renders the scene.
1363  *
1364  * This function is intented for higher end 3D cards, with pixel shader support
1365  * and at least 64 MiB of video RAM.
1366  */
1367 static void Direct3DRenderScene(vout_display_t *vd,
1368                                 d3d_region_t *picture,
1369                                 int subpicture_count,
1370                                 d3d_region_t *subpicture)
1371 {
1372     vout_display_sys_t *sys = vd->sys;
1373     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
1374     HRESULT hr;
1375
1376     /* Clear the backbuffer and the zbuffer */
1377     hr = IDirect3DDevice9_Clear(d3ddev, 0, NULL, D3DCLEAR_TARGET,
1378                               D3DCOLOR_XRGB(0, 0, 0), 1.0f, 0);
1379     if (FAILED(hr)) {
1380         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1381         return;
1382     }
1383
1384     // Begin the scene
1385     hr = IDirect3DDevice9_BeginScene(d3ddev);
1386     if (FAILED(hr)) {
1387         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1388         return;
1389     }
1390
1391     Direct3DRenderRegion(vd, picture);
1392
1393     if (subpicture_count > 0)
1394         IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHABLENDENABLE, TRUE);
1395     for (int i = 0; i < subpicture_count; i++) {
1396         d3d_region_t *r = &subpicture[i];
1397         if (r->texture)
1398             Direct3DRenderRegion(vd, r);
1399     }
1400     if (subpicture_count > 0)
1401         IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHABLENDENABLE, FALSE);
1402
1403     // End the scene
1404     hr = IDirect3DDevice9_EndScene(d3ddev);
1405     if (FAILED(hr)) {
1406         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1407         return;
1408     }
1409 }
1410
1411 /*****************************************************************************
1412  * DesktopCallback: desktop mode variable callback
1413  *****************************************************************************/
1414 static int DesktopCallback(vlc_object_t *object, char const *psz_cmd,
1415                             vlc_value_t oldval, vlc_value_t newval,
1416                             void *p_data)
1417 {
1418     vout_display_t *vd = (vout_display_t *)object;
1419     vout_display_sys_t *sys = vd->sys;
1420     VLC_UNUSED(psz_cmd);
1421     VLC_UNUSED(oldval);
1422     VLC_UNUSED(p_data);
1423
1424     vlc_mutex_lock(&sys->lock);
1425     const bool ch_desktop = !sys->desktop_requested != !newval.b_bool;
1426     sys->ch_desktop |= ch_desktop;
1427     sys->desktop_requested = newval.b_bool;
1428     vlc_mutex_unlock(&sys->lock);
1429
1430     /* FIXME we should have a way to export variable to be saved */
1431     if (ch_desktop) {
1432         playlist_t *p_playlist = pl_Get(vd);
1433         /* Modify playlist as well because the vout might have to be
1434          * restarted */
1435         var_Create(p_playlist, "direct3d-desktop", VLC_VAR_BOOL);
1436         var_SetBool(p_playlist, "direct3d-desktop", newval.b_bool);
1437     }
1438     return VLC_SUCCESS;
1439 }