2 * This file is part of FFmpeg.
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 #define CL_USE_DEPRECATED_OPENCL_1_2_APIS
28 #include "hwcontext.h"
29 #include "hwcontext_internal.h"
30 #include "hwcontext_opencl.h"
34 #if HAVE_OPENCL_VAAPI_BEIGNET
37 #include <va/va_drmcommon.h>
38 #include <CL/cl_intel.h>
39 #include "hwcontext_vaapi.h"
42 #if HAVE_OPENCL_DRM_BEIGNET
44 #include <CL/cl_intel.h>
45 #include "hwcontext_drm.h"
48 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
50 #include <mfx/mfxstructures.h>
53 #include <CL/va_ext.h>
54 #include "hwcontext_vaapi.h"
59 #include <CL/cl_dx9_media_sharing.h>
61 #include "hwcontext_dxva2.h"
65 #include <CL/cl_d3d11.h>
66 #include "hwcontext_d3d11va.h"
69 #if HAVE_OPENCL_DRM_ARM
70 #include <CL/cl_ext.h>
71 #include <drm_fourcc.h>
72 #include "hwcontext_drm.h"
76 typedef struct OpenCLDeviceContext {
77 // Default command queue to use for transfer/mapping operations on
78 // the device. If the user supplies one, this is a reference to it.
79 // Otherwise, it is newly-created.
80 cl_command_queue command_queue;
82 // The platform the context exists on. This is needed to query and
83 // retrieve extension functions.
84 cl_platform_id platform_id;
86 // Platform/device-specific functions.
87 #if HAVE_OPENCL_DRM_BEIGNET
88 int beignet_drm_mapping_usable;
89 clCreateImageFromFdINTEL_fn clCreateImageFromFdINTEL;
92 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
93 int qsv_mapping_usable;
94 clCreateFromVA_APIMediaSurfaceINTEL_fn
95 clCreateFromVA_APIMediaSurfaceINTEL;
96 clEnqueueAcquireVA_APIMediaSurfacesINTEL_fn
97 clEnqueueAcquireVA_APIMediaSurfacesINTEL;
98 clEnqueueReleaseVA_APIMediaSurfacesINTEL_fn
99 clEnqueueReleaseVA_APIMediaSurfacesINTEL;
102 #if HAVE_OPENCL_DXVA2
103 int dxva2_mapping_usable;
104 cl_dx9_media_adapter_type_khr dx9_media_adapter_type;
106 clCreateFromDX9MediaSurfaceKHR_fn
107 clCreateFromDX9MediaSurfaceKHR;
108 clEnqueueAcquireDX9MediaSurfacesKHR_fn
109 clEnqueueAcquireDX9MediaSurfacesKHR;
110 clEnqueueReleaseDX9MediaSurfacesKHR_fn
111 clEnqueueReleaseDX9MediaSurfacesKHR;
114 #if HAVE_OPENCL_D3D11
115 int d3d11_mapping_usable;
116 clCreateFromD3D11Texture2DKHR_fn
117 clCreateFromD3D11Texture2DKHR;
118 clEnqueueAcquireD3D11ObjectsKHR_fn
119 clEnqueueAcquireD3D11ObjectsKHR;
120 clEnqueueReleaseD3D11ObjectsKHR_fn
121 clEnqueueReleaseD3D11ObjectsKHR;
124 #if HAVE_OPENCL_DRM_ARM
125 int drm_arm_mapping_usable;
127 } OpenCLDeviceContext;
129 typedef struct OpenCLFramesContext {
130 // Command queue used for transfer/mapping operations on this frames
131 // context. If the user supplies one, this is a reference to it.
132 // Otherwise, it is a reference to the default command queue for the
134 cl_command_queue command_queue;
136 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
137 // For mapping APIs which have separate creation and acquire/release
138 // steps, this stores the OpenCL memory objects corresponding to each
140 int nb_mapped_frames;
141 AVOpenCLFrameDescriptor *mapped_frames;
143 } OpenCLFramesContext;
146 static void CL_CALLBACK opencl_error_callback(const char *errinfo,
147 const void *private_info,
151 AVHWDeviceContext *ctx = user_data;
152 av_log(ctx, AV_LOG_ERROR, "OpenCL error: %s\n", errinfo);
155 static void opencl_device_free(AVHWDeviceContext *hwdev)
157 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
160 cle = clReleaseContext(hwctx->context);
161 if (cle != CL_SUCCESS) {
162 av_log(hwdev, AV_LOG_ERROR, "Failed to release OpenCL "
163 "context: %d.\n", cle);
169 cl_platform_info name;
170 } opencl_platform_params[] = {
171 { "platform_profile", CL_PLATFORM_PROFILE },
172 { "platform_version", CL_PLATFORM_VERSION },
173 { "platform_name", CL_PLATFORM_NAME },
174 { "platform_vendor", CL_PLATFORM_VENDOR },
175 { "platform_extensions", CL_PLATFORM_EXTENSIONS },
181 } opencl_device_params[] = {
182 { "device_name", CL_DEVICE_NAME },
183 { "device_vendor", CL_DEVICE_VENDOR },
184 { "driver_version", CL_DRIVER_VERSION },
185 { "device_version", CL_DEVICE_VERSION },
186 { "device_profile", CL_DEVICE_PROFILE },
187 { "device_extensions", CL_DEVICE_EXTENSIONS },
193 } opencl_device_types[] = {
194 { "cpu", CL_DEVICE_TYPE_CPU },
195 { "gpu", CL_DEVICE_TYPE_GPU },
196 { "accelerator", CL_DEVICE_TYPE_ACCELERATOR },
197 { "custom", CL_DEVICE_TYPE_CUSTOM },
198 { "default", CL_DEVICE_TYPE_DEFAULT },
199 { "all", CL_DEVICE_TYPE_ALL },
202 static char *opencl_get_platform_string(cl_platform_id platform_id,
203 cl_platform_info key)
208 cle = clGetPlatformInfo(platform_id, key, 0, NULL, &size);
209 if (cle != CL_SUCCESS)
211 str = av_malloc(size);
214 cle = clGetPlatformInfo(platform_id, key, size, str, &size);
215 if (cle != CL_SUCCESS) {
219 av_assert0(strlen(str) + 1 == size);
223 static char *opencl_get_device_string(cl_device_id device_id,
229 cle = clGetDeviceInfo(device_id, key, 0, NULL, &size);
230 if (cle != CL_SUCCESS)
232 str = av_malloc(size);
235 cle = clGetDeviceInfo(device_id, key, size, str, &size);
236 if (cle != CL_SUCCESS) {
240 av_assert0(strlen(str) + 1== size);
244 static int opencl_check_platform_extension(cl_platform_id platform_id,
249 str = opencl_get_platform_string(platform_id,
250 CL_PLATFORM_EXTENSIONS);
251 if (str && strstr(str, name))
257 static int opencl_check_device_extension(cl_device_id device_id,
262 str = opencl_get_device_string(device_id,
263 CL_DEVICE_EXTENSIONS);
264 if (str && strstr(str, name))
270 static av_unused int opencl_check_extension(AVHWDeviceContext *hwdev,
273 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
274 OpenCLDeviceContext *priv = hwdev->internal->priv;
276 if (opencl_check_platform_extension(priv->platform_id, name)) {
277 av_log(hwdev, AV_LOG_DEBUG,
278 "%s found as platform extension.\n", name);
282 if (opencl_check_device_extension(hwctx->device_id, name)) {
283 av_log(hwdev, AV_LOG_DEBUG,
284 "%s found as device extension.\n", name);
291 static int opencl_enumerate_platforms(AVHWDeviceContext *hwdev,
292 cl_uint *nb_platforms,
293 cl_platform_id **platforms,
298 cle = clGetPlatformIDs(0, NULL, nb_platforms);
299 if (cle != CL_SUCCESS) {
300 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of "
301 "OpenCL platforms: %d.\n", cle);
302 return AVERROR(ENODEV);
304 av_log(hwdev, AV_LOG_DEBUG, "%u OpenCL platforms found.\n",
307 *platforms = av_malloc_array(*nb_platforms, sizeof(**platforms));
309 return AVERROR(ENOMEM);
311 cle = clGetPlatformIDs(*nb_platforms, *platforms, NULL);
312 if (cle != CL_SUCCESS) {
313 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of OpenCL "
314 "platforms: %d.\n", cle);
316 return AVERROR(ENODEV);
322 static int opencl_filter_platform(AVHWDeviceContext *hwdev,
323 cl_platform_id platform_id,
324 const char *platform_name,
327 AVDictionary *opts = context;
328 const AVDictionaryEntry *param;
332 for (i = 0; i < FF_ARRAY_ELEMS(opencl_platform_params); i++) {
333 param = av_dict_get(opts, opencl_platform_params[i].key,
338 str = opencl_get_platform_string(platform_id,
339 opencl_platform_params[i].name);
341 av_log(hwdev, AV_LOG_ERROR, "Failed to query %s "
342 "of platform \"%s\".\n",
343 opencl_platform_params[i].key, platform_name);
344 return AVERROR_UNKNOWN;
346 if (!av_stristr(str, param->value)) {
347 av_log(hwdev, AV_LOG_DEBUG, "%s does not match (\"%s\").\n",
357 static int opencl_enumerate_devices(AVHWDeviceContext *hwdev,
358 cl_platform_id platform_id,
359 const char *platform_name,
361 cl_device_id **devices,
366 cle = clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_ALL,
367 0, NULL, nb_devices);
368 if (cle == CL_DEVICE_NOT_FOUND) {
369 av_log(hwdev, AV_LOG_DEBUG, "No devices found "
370 "on platform \"%s\".\n", platform_name);
373 } else if (cle != CL_SUCCESS) {
374 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
375 "on platform \"%s\": %d.\n", platform_name, cle);
376 return AVERROR(ENODEV);
378 av_log(hwdev, AV_LOG_DEBUG, "%u OpenCL devices found on "
379 "platform \"%s\".\n", *nb_devices, platform_name);
381 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
383 return AVERROR(ENOMEM);
385 cle = clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_ALL,
386 *nb_devices, *devices, NULL);
387 if (cle != CL_SUCCESS) {
388 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of devices "
389 "on platform \"%s\": %d.\n", platform_name, cle);
391 return AVERROR(ENODEV);
397 static int opencl_filter_device(AVHWDeviceContext *hwdev,
398 cl_device_id device_id,
399 const char *device_name,
402 AVDictionary *opts = context;
403 const AVDictionaryEntry *param;
407 param = av_dict_get(opts, "device_type", NULL, 0);
409 cl_device_type match_type = 0, device_type;
412 for (i = 0; i < FF_ARRAY_ELEMS(opencl_device_types); i++) {
413 if (!strcmp(opencl_device_types[i].key, param->value)) {
414 match_type = opencl_device_types[i].type;
419 av_log(hwdev, AV_LOG_ERROR, "Unknown device type %s.\n",
421 return AVERROR(EINVAL);
424 cle = clGetDeviceInfo(device_id, CL_DEVICE_TYPE,
425 sizeof(device_type), &device_type, NULL);
426 if (cle != CL_SUCCESS) {
427 av_log(hwdev, AV_LOG_ERROR, "Failed to query device type "
428 "of device \"%s\".\n", device_name);
429 return AVERROR_UNKNOWN;
432 if (!(device_type & match_type)) {
433 av_log(hwdev, AV_LOG_DEBUG, "device_type does not match.\n");
438 for (i = 0; i < FF_ARRAY_ELEMS(opencl_device_params); i++) {
439 param = av_dict_get(opts, opencl_device_params[i].key,
444 str = opencl_get_device_string(device_id,
445 opencl_device_params[i].name);
447 av_log(hwdev, AV_LOG_ERROR, "Failed to query %s "
448 "of device \"%s\".\n",
449 opencl_device_params[i].key, device_name);
450 return AVERROR_UNKNOWN;
452 if (!av_stristr(str, param->value)) {
453 av_log(hwdev, AV_LOG_DEBUG, "%s does not match (\"%s\").\n",
463 typedef struct OpenCLDeviceSelector {
467 int (*enumerate_platforms)(AVHWDeviceContext *hwdev,
468 cl_uint *nb_platforms,
469 cl_platform_id **platforms,
471 int (*filter_platform) (AVHWDeviceContext *hwdev,
472 cl_platform_id platform_id,
473 const char *platform_name,
475 int (*enumerate_devices) (AVHWDeviceContext *hwdev,
476 cl_platform_id platform_id,
477 const char *platform_name,
479 cl_device_id **devices,
481 int (*filter_device) (AVHWDeviceContext *hwdev,
482 cl_device_id device_id,
483 const char *device_name,
485 } OpenCLDeviceSelector;
487 static int opencl_device_create_internal(AVHWDeviceContext *hwdev,
488 const OpenCLDeviceSelector *selector,
489 cl_context_properties *props)
491 cl_uint nb_platforms;
492 cl_platform_id *platforms = NULL;
493 cl_platform_id platform_id;
495 cl_device_id *devices = NULL;
496 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
498 cl_context_properties default_props[3];
499 char *platform_name_src = NULL,
500 *device_name_src = NULL;
501 int err, found, p, d;
503 av_assert0(selector->enumerate_platforms &&
504 selector->enumerate_devices);
506 err = selector->enumerate_platforms(hwdev, &nb_platforms, &platforms,
512 for (p = 0; p < nb_platforms; p++) {
513 const char *platform_name;
515 if (selector->platform_index >= 0 &&
516 selector->platform_index != p)
519 av_freep(&platform_name_src);
520 platform_name_src = opencl_get_platform_string(platforms[p],
522 if (platform_name_src)
523 platform_name = platform_name_src;
525 platform_name = "Unknown Platform";
527 if (selector->filter_platform) {
528 err = selector->filter_platform(hwdev, platforms[p],
537 err = selector->enumerate_devices(hwdev, platforms[p], platform_name,
538 &nb_devices, &devices,
543 for (d = 0; d < nb_devices; d++) {
544 const char *device_name;
546 if (selector->device_index >= 0 &&
547 selector->device_index != d)
550 av_freep(&device_name_src);
551 device_name_src = opencl_get_device_string(devices[d],
554 device_name = device_name_src;
556 device_name = "Unknown Device";
558 if (selector->filter_device) {
559 err = selector->filter_device(hwdev, devices[d],
568 av_log(hwdev, AV_LOG_VERBOSE, "%d.%d: %s / %s\n", p, d,
569 platform_name, device_name);
572 platform_id = platforms[p];
573 hwctx->device_id = devices[d];
580 av_log(hwdev, AV_LOG_ERROR, "No matching devices found.\n");
581 err = AVERROR(ENODEV);
585 av_log(hwdev, AV_LOG_ERROR, "More than one matching device found.\n");
586 err = AVERROR(ENODEV);
591 props = default_props;
592 default_props[0] = CL_CONTEXT_PLATFORM;
593 default_props[1] = (intptr_t)platform_id;
594 default_props[2] = 0;
596 if (props[0] == CL_CONTEXT_PLATFORM && props[1] == 0)
597 props[1] = (intptr_t)platform_id;
600 hwctx->context = clCreateContext(props, 1, &hwctx->device_id,
601 &opencl_error_callback, hwdev, &cle);
602 if (!hwctx->context) {
603 av_log(hwdev, AV_LOG_ERROR, "Failed to create OpenCL context: "
605 err = AVERROR(ENODEV);
609 hwdev->free = &opencl_device_free;
613 av_freep(&platform_name_src);
614 av_freep(&device_name_src);
615 av_freep(&platforms);
620 static int opencl_device_create(AVHWDeviceContext *hwdev, const char *device,
621 AVDictionary *opts, int flags)
623 OpenCLDeviceSelector selector = {
625 .enumerate_platforms = &opencl_enumerate_platforms,
626 .filter_platform = &opencl_filter_platform,
627 .enumerate_devices = &opencl_enumerate_devices,
628 .filter_device = &opencl_filter_device,
631 if (device && device[0]) {
632 // Match one or both indices for platform and device.
633 int d = -1, p = -1, ret;
634 if (device[0] == '.')
635 ret = sscanf(device, ".%d", &d);
637 ret = sscanf(device, "%d.%d", &p, &d);
639 av_log(hwdev, AV_LOG_ERROR, "Invalid OpenCL platform/device "
640 "index specification \"%s\".\n", device);
641 return AVERROR(EINVAL);
643 selector.platform_index = p;
644 selector.device_index = d;
646 selector.platform_index = -1;
647 selector.device_index = -1;
650 return opencl_device_create_internal(hwdev, &selector, NULL);
653 static int opencl_device_init(AVHWDeviceContext *hwdev)
655 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
656 OpenCLDeviceContext *priv = hwdev->internal->priv;
659 if (hwctx->command_queue) {
660 cle = clRetainCommandQueue(hwctx->command_queue);
661 if (cle != CL_SUCCESS) {
662 av_log(hwdev, AV_LOG_ERROR, "Failed to retain external "
663 "command queue: %d.\n", cle);
666 priv->command_queue = hwctx->command_queue;
668 priv->command_queue = clCreateCommandQueue(hwctx->context,
671 if (!priv->command_queue) {
672 av_log(hwdev, AV_LOG_ERROR, "Failed to create internal "
673 "command queue: %d.\n", cle);
678 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_PLATFORM,
679 sizeof(priv->platform_id), &priv->platform_id,
681 if (cle != CL_SUCCESS) {
682 av_log(hwdev, AV_LOG_ERROR, "Failed to determine the OpenCL "
683 "platform containing the device.\n");
687 #define CL_FUNC(name, desc) do { \
690 priv->name = clGetExtensionFunctionAddressForPlatform( \
691 priv->platform_id, #name); \
693 av_log(hwdev, AV_LOG_VERBOSE, \
694 desc " function not found (%s).\n", #name); \
697 av_log(hwdev, AV_LOG_VERBOSE, \
698 desc " function found (%s).\n", #name); \
702 #if HAVE_OPENCL_DRM_BEIGNET
706 CL_FUNC(clCreateImageFromFdINTEL,
707 "Beignet DRM to OpenCL image mapping");
710 av_log(hwdev, AV_LOG_WARNING, "Beignet DRM to OpenCL "
711 "mapping not usable.\n");
712 priv->beignet_drm_mapping_usable = 0;
714 priv->beignet_drm_mapping_usable = 1;
719 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
722 cl_context_properties *props = NULL;
723 VADisplay va_display;
724 const char *va_ext = "cl_intel_va_api_media_sharing";
727 if (!opencl_check_extension(hwdev, va_ext)) {
728 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
729 "required for QSV to OpenCL mapping.\n", va_ext);
733 cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
734 0, NULL, &props_size);
735 if (cle != CL_SUCCESS) {
736 av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
737 "properties: %d.\n", cle);
740 if (props_size == 0) {
741 av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
742 "enabled on context creation to use QSV to "
743 "OpenCL mapping.\n");
747 props = av_malloc(props_size);
749 return AVERROR(ENOMEM);
751 cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
752 props_size, props, NULL);
753 if (cle != CL_SUCCESS) {
754 av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
755 "properties: %d.\n", cle);
760 for (i = 0; i < (props_size / sizeof(*props) - 1); i++) {
761 if (props[i] == CL_CONTEXT_VA_API_DISPLAY_INTEL) {
762 va_display = (VADisplay)(intptr_t)props[i+1];
767 av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
768 "enabled on context creation to use QSV to "
769 "OpenCL mapping.\n");
772 if (!vaDisplayIsValid(va_display)) {
773 av_log(hwdev, AV_LOG_VERBOSE, "A valid VADisplay is "
774 "required on context creation to use QSV to "
775 "OpenCL mapping.\n");
779 CL_FUNC(clCreateFromVA_APIMediaSurfaceINTEL,
780 "Intel QSV to OpenCL mapping");
781 CL_FUNC(clEnqueueAcquireVA_APIMediaSurfacesINTEL,
782 "Intel QSV in OpenCL acquire");
783 CL_FUNC(clEnqueueReleaseVA_APIMediaSurfacesINTEL,
784 "Intel QSV in OpenCL release");
788 av_log(hwdev, AV_LOG_WARNING, "QSV to OpenCL mapping "
790 priv->qsv_mapping_usable = 0;
792 priv->qsv_mapping_usable = 1;
798 #if HAVE_OPENCL_DXVA2
802 CL_FUNC(clCreateFromDX9MediaSurfaceKHR,
803 "DXVA2 to OpenCL mapping");
804 CL_FUNC(clEnqueueAcquireDX9MediaSurfacesKHR,
805 "DXVA2 in OpenCL acquire");
806 CL_FUNC(clEnqueueReleaseDX9MediaSurfacesKHR,
807 "DXVA2 in OpenCL release");
810 av_log(hwdev, AV_LOG_WARNING, "DXVA2 to OpenCL mapping "
812 priv->dxva2_mapping_usable = 0;
814 priv->dx9_media_adapter_type = CL_ADAPTER_D3D9EX_KHR;
815 priv->dxva2_mapping_usable = 1;
820 #if HAVE_OPENCL_D3D11
822 const char *d3d11_ext = "cl_khr_d3d11_sharing";
823 const char *nv12_ext = "cl_intel_d3d11_nv12_media_sharing";
826 if (!opencl_check_extension(hwdev, d3d11_ext)) {
827 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
828 "required for D3D11 to OpenCL mapping.\n", d3d11_ext);
830 } else if (!opencl_check_extension(hwdev, nv12_ext)) {
831 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension may be "
832 "required for D3D11 to OpenCL mapping.\n", nv12_ext);
836 CL_FUNC(clCreateFromD3D11Texture2DKHR,
837 "D3D11 to OpenCL mapping");
838 CL_FUNC(clEnqueueAcquireD3D11ObjectsKHR,
839 "D3D11 in OpenCL acquire");
840 CL_FUNC(clEnqueueReleaseD3D11ObjectsKHR,
841 "D3D11 in OpenCL release");
844 av_log(hwdev, AV_LOG_WARNING, "D3D11 to OpenCL mapping "
846 priv->d3d11_mapping_usable = 0;
848 priv->d3d11_mapping_usable = 1;
853 #if HAVE_OPENCL_DRM_ARM
855 const char *drm_arm_ext = "cl_arm_import_memory";
856 const char *image_ext = "cl_khr_image2d_from_buffer";
859 if (!opencl_check_extension(hwdev, drm_arm_ext)) {
860 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
861 "required for DRM to OpenCL mapping on ARM.\n",
865 if (!opencl_check_extension(hwdev, image_ext)) {
866 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
867 "required for DRM to OpenCL mapping on ARM.\n",
872 // clImportMemoryARM() is linked statically.
875 av_log(hwdev, AV_LOG_WARNING, "DRM to OpenCL mapping on ARM "
877 priv->drm_arm_mapping_usable = 0;
879 priv->drm_arm_mapping_usable = 1;
889 static void opencl_device_uninit(AVHWDeviceContext *hwdev)
891 OpenCLDeviceContext *priv = hwdev->internal->priv;
894 if (priv->command_queue) {
895 cle = clReleaseCommandQueue(priv->command_queue);
896 if (cle != CL_SUCCESS) {
897 av_log(hwdev, AV_LOG_ERROR, "Failed to release internal "
898 "command queue reference: %d.\n", cle);
900 priv->command_queue = NULL;
904 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
905 static int opencl_filter_intel_media_vaapi_platform(AVHWDeviceContext *hwdev,
906 cl_platform_id platform_id,
907 const char *platform_name,
910 // This doesn't exist as a platform extension, so just test whether
911 // the function we will use for device enumeration exists.
913 if (!clGetExtensionFunctionAddressForPlatform(platform_id,
914 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL")) {
915 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not export the "
916 "VAAPI device enumeration function.\n", platform_name);
923 static int opencl_enumerate_intel_media_vaapi_devices(AVHWDeviceContext *hwdev,
924 cl_platform_id platform_id,
925 const char *platform_name,
927 cl_device_id **devices,
930 VADisplay va_display = context;
931 clGetDeviceIDsFromVA_APIMediaAdapterINTEL_fn
932 clGetDeviceIDsFromVA_APIMediaAdapterINTEL;
935 clGetDeviceIDsFromVA_APIMediaAdapterINTEL =
936 clGetExtensionFunctionAddressForPlatform(platform_id,
937 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL");
938 if (!clGetDeviceIDsFromVA_APIMediaAdapterINTEL) {
939 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
940 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL().\n");
941 return AVERROR_UNKNOWN;
944 cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
945 platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
946 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, 0, NULL, nb_devices);
947 if (cle == CL_DEVICE_NOT_FOUND) {
948 av_log(hwdev, AV_LOG_DEBUG, "No VAAPI-supporting devices found "
949 "on platform \"%s\".\n", platform_name);
952 } else if (cle != CL_SUCCESS) {
953 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
954 "on platform \"%s\": %d.\n", platform_name, cle);
955 return AVERROR_UNKNOWN;
958 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
960 return AVERROR(ENOMEM);
962 cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
963 platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
964 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, *nb_devices, *devices, NULL);
965 if (cle != CL_SUCCESS) {
966 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of VAAPI-supporting "
967 "devices on platform \"%s\": %d.\n", platform_name, cle);
969 return AVERROR_UNKNOWN;
975 static int opencl_filter_intel_media_vaapi_device(AVHWDeviceContext *hwdev,
976 cl_device_id device_id,
977 const char *device_name,
980 const char *va_ext = "cl_intel_va_api_media_sharing";
982 if (opencl_check_device_extension(device_id, va_ext)) {
985 av_log(hwdev, AV_LOG_DEBUG, "Device %s does not support the "
986 "%s extension.\n", device_name, va_ext);
992 #if HAVE_OPENCL_DXVA2
993 static int opencl_filter_dxva2_platform(AVHWDeviceContext *hwdev,
994 cl_platform_id platform_id,
995 const char *platform_name,
998 const char *dx9_ext = "cl_khr_dx9_media_sharing";
1000 if (opencl_check_platform_extension(platform_id, dx9_ext)) {
1003 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1004 "%s extension.\n", platform_name, dx9_ext);
1009 static int opencl_enumerate_dxva2_devices(AVHWDeviceContext *hwdev,
1010 cl_platform_id platform_id,
1011 const char *platform_name,
1012 cl_uint *nb_devices,
1013 cl_device_id **devices,
1016 IDirect3DDevice9 *device = context;
1017 clGetDeviceIDsFromDX9MediaAdapterKHR_fn
1018 clGetDeviceIDsFromDX9MediaAdapterKHR;
1019 cl_dx9_media_adapter_type_khr media_adapter_type = CL_ADAPTER_D3D9EX_KHR;
1022 clGetDeviceIDsFromDX9MediaAdapterKHR =
1023 clGetExtensionFunctionAddressForPlatform(platform_id,
1024 "clGetDeviceIDsFromDX9MediaAdapterKHR");
1025 if (!clGetDeviceIDsFromDX9MediaAdapterKHR) {
1026 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
1027 "clGetDeviceIDsFromDX9MediaAdapterKHR().\n");
1028 return AVERROR_UNKNOWN;
1031 cle = clGetDeviceIDsFromDX9MediaAdapterKHR(
1032 platform_id, 1, &media_adapter_type, (void**)&device,
1033 CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR,
1034 0, NULL, nb_devices);
1035 if (cle == CL_DEVICE_NOT_FOUND) {
1036 av_log(hwdev, AV_LOG_DEBUG, "No DXVA2-supporting devices found "
1037 "on platform \"%s\".\n", platform_name);
1040 } else if (cle != CL_SUCCESS) {
1041 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
1042 "on platform \"%s\": %d.\n", platform_name, cle);
1043 return AVERROR_UNKNOWN;
1046 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
1048 return AVERROR(ENOMEM);
1050 cle = clGetDeviceIDsFromDX9MediaAdapterKHR(
1051 platform_id, 1, &media_adapter_type, (void**)&device,
1052 CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR,
1053 *nb_devices, *devices, NULL);
1054 if (cle != CL_SUCCESS) {
1055 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of DXVA2-supporting "
1056 "devices on platform \"%s\": %d.\n", platform_name, cle);
1058 return AVERROR_UNKNOWN;
1065 #if HAVE_OPENCL_D3D11
1066 static int opencl_filter_d3d11_platform(AVHWDeviceContext *hwdev,
1067 cl_platform_id platform_id,
1068 const char *platform_name,
1071 const char *d3d11_ext = "cl_khr_d3d11_sharing";
1073 if (opencl_check_platform_extension(platform_id, d3d11_ext)) {
1076 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1077 "%s extension.\n", platform_name, d3d11_ext);
1082 static int opencl_enumerate_d3d11_devices(AVHWDeviceContext *hwdev,
1083 cl_platform_id platform_id,
1084 const char *platform_name,
1085 cl_uint *nb_devices,
1086 cl_device_id **devices,
1089 ID3D11Device *device = context;
1090 clGetDeviceIDsFromD3D11KHR_fn clGetDeviceIDsFromD3D11KHR;
1093 clGetDeviceIDsFromD3D11KHR =
1094 clGetExtensionFunctionAddressForPlatform(platform_id,
1095 "clGetDeviceIDsFromD3D11KHR");
1096 if (!clGetDeviceIDsFromD3D11KHR) {
1097 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
1098 "clGetDeviceIDsFromD3D11KHR().\n");
1099 return AVERROR_UNKNOWN;
1102 cle = clGetDeviceIDsFromD3D11KHR(platform_id,
1103 CL_D3D11_DEVICE_KHR, device,
1104 CL_PREFERRED_DEVICES_FOR_D3D11_KHR,
1105 0, NULL, nb_devices);
1106 if (cle == CL_DEVICE_NOT_FOUND) {
1107 av_log(hwdev, AV_LOG_DEBUG, "No D3D11-supporting devices found "
1108 "on platform \"%s\".\n", platform_name);
1111 } else if (cle != CL_SUCCESS) {
1112 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
1113 "on platform \"%s\": %d.\n", platform_name, cle);
1114 return AVERROR_UNKNOWN;
1117 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
1119 return AVERROR(ENOMEM);
1121 cle = clGetDeviceIDsFromD3D11KHR(platform_id,
1122 CL_D3D11_DEVICE_KHR, device,
1123 CL_PREFERRED_DEVICES_FOR_D3D11_KHR,
1124 *nb_devices, *devices, NULL);
1125 if (cle != CL_SUCCESS) {
1126 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of D3D11-supporting "
1127 "devices on platform \"%s\": %d.\n", platform_name, cle);
1129 return AVERROR_UNKNOWN;
1136 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
1137 static int opencl_filter_gpu_device(AVHWDeviceContext *hwdev,
1138 cl_device_id device_id,
1139 const char *device_name,
1142 cl_device_type device_type;
1145 cle = clGetDeviceInfo(device_id, CL_DEVICE_TYPE,
1146 sizeof(device_type), &device_type, NULL);
1147 if (cle != CL_SUCCESS) {
1148 av_log(hwdev, AV_LOG_ERROR, "Failed to query device type "
1149 "of device \"%s\".\n", device_name);
1150 return AVERROR_UNKNOWN;
1152 if (!(device_type & CL_DEVICE_TYPE_GPU)) {
1153 av_log(hwdev, AV_LOG_DEBUG, "Device %s skipped (not GPU).\n",
1162 #if HAVE_OPENCL_DRM_ARM
1163 static int opencl_filter_drm_arm_platform(AVHWDeviceContext *hwdev,
1164 cl_platform_id platform_id,
1165 const char *platform_name,
1168 const char *drm_arm_ext = "cl_arm_import_memory";
1170 if (opencl_check_platform_extension(platform_id, drm_arm_ext)) {
1173 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1174 "%s extension.\n", platform_name, drm_arm_ext);
1179 static int opencl_filter_drm_arm_device(AVHWDeviceContext *hwdev,
1180 cl_device_id device_id,
1181 const char *device_name,
1184 const char *drm_arm_ext = "cl_arm_import_memory";
1186 if (opencl_check_device_extension(device_id, drm_arm_ext)) {
1189 av_log(hwdev, AV_LOG_DEBUG, "Device %s does not support the "
1190 "%s extension.\n", device_name, drm_arm_ext);
1196 static int opencl_device_derive(AVHWDeviceContext *hwdev,
1197 AVHWDeviceContext *src_ctx,
1201 switch (src_ctx->type) {
1203 #if HAVE_OPENCL_DRM_BEIGNET
1204 case AV_HWDEVICE_TYPE_DRM:
1205 case AV_HWDEVICE_TYPE_VAAPI:
1207 // Surface mapping works via DRM PRIME fds with no special
1208 // initialisation required in advance. This just finds the
1209 // Beignet ICD by name.
1210 AVDictionary *opts = NULL;
1212 err = av_dict_set(&opts, "platform_vendor", "Intel", 0);
1214 err = av_dict_set(&opts, "platform_version", "beignet", 0);
1216 OpenCLDeviceSelector selector = {
1217 .platform_index = -1,
1220 .enumerate_platforms = &opencl_enumerate_platforms,
1221 .filter_platform = &opencl_filter_platform,
1222 .enumerate_devices = &opencl_enumerate_devices,
1223 .filter_device = NULL,
1225 err = opencl_device_create_internal(hwdev, &selector, NULL);
1227 av_dict_free(&opts);
1232 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
1233 // The generic code automatically attempts to derive from all
1234 // ancestors of the given device, so we can ignore QSV devices here
1235 // and just consider the inner VAAPI device it was derived from.
1236 case AV_HWDEVICE_TYPE_VAAPI:
1238 AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
1239 cl_context_properties props[7] = {
1240 CL_CONTEXT_PLATFORM,
1242 CL_CONTEXT_VA_API_DISPLAY_INTEL,
1243 (intptr_t)src_hwctx->display,
1244 CL_CONTEXT_INTEROP_USER_SYNC,
1248 OpenCLDeviceSelector selector = {
1249 .platform_index = -1,
1251 .context = src_hwctx->display,
1252 .enumerate_platforms = &opencl_enumerate_platforms,
1253 .filter_platform = &opencl_filter_intel_media_vaapi_platform,
1254 .enumerate_devices = &opencl_enumerate_intel_media_vaapi_devices,
1255 .filter_device = &opencl_filter_intel_media_vaapi_device,
1258 err = opencl_device_create_internal(hwdev, &selector, props);
1263 #if HAVE_OPENCL_DXVA2
1264 case AV_HWDEVICE_TYPE_DXVA2:
1266 AVDXVA2DeviceContext *src_hwctx = src_ctx->hwctx;
1267 IDirect3DDevice9 *device;
1268 HANDLE device_handle;
1271 hr = IDirect3DDeviceManager9_OpenDeviceHandle(src_hwctx->devmgr,
1274 av_log(hwdev, AV_LOG_ERROR, "Failed to open device handle "
1275 "for Direct3D9 device: %lx.\n", (unsigned long)hr);
1276 err = AVERROR_UNKNOWN;
1280 hr = IDirect3DDeviceManager9_LockDevice(src_hwctx->devmgr,
1283 if (SUCCEEDED(hr)) {
1284 cl_context_properties props[5] = {
1285 CL_CONTEXT_PLATFORM,
1287 CL_CONTEXT_ADAPTER_D3D9EX_KHR,
1291 OpenCLDeviceSelector selector = {
1292 .platform_index = -1,
1295 .enumerate_platforms = &opencl_enumerate_platforms,
1296 .filter_platform = &opencl_filter_dxva2_platform,
1297 .enumerate_devices = &opencl_enumerate_dxva2_devices,
1298 .filter_device = &opencl_filter_gpu_device,
1301 err = opencl_device_create_internal(hwdev, &selector, props);
1303 IDirect3DDeviceManager9_UnlockDevice(src_hwctx->devmgr,
1304 device_handle, FALSE);
1306 av_log(hwdev, AV_LOG_ERROR, "Failed to lock device handle "
1307 "for Direct3D9 device: %lx.\n", (unsigned long)hr);
1308 err = AVERROR_UNKNOWN;
1311 IDirect3DDeviceManager9_CloseDeviceHandle(src_hwctx->devmgr,
1317 #if HAVE_OPENCL_D3D11
1318 case AV_HWDEVICE_TYPE_D3D11VA:
1320 AVD3D11VADeviceContext *src_hwctx = src_ctx->hwctx;
1321 cl_context_properties props[5] = {
1322 CL_CONTEXT_PLATFORM,
1324 CL_CONTEXT_D3D11_DEVICE_KHR,
1325 (intptr_t)src_hwctx->device,
1328 OpenCLDeviceSelector selector = {
1329 .platform_index = -1,
1331 .context = src_hwctx->device,
1332 .enumerate_platforms = &opencl_enumerate_platforms,
1333 .filter_platform = &opencl_filter_d3d11_platform,
1334 .enumerate_devices = &opencl_enumerate_d3d11_devices,
1335 .filter_device = &opencl_filter_gpu_device,
1338 err = opencl_device_create_internal(hwdev, &selector, props);
1343 #if HAVE_OPENCL_DRM_ARM
1344 case AV_HWDEVICE_TYPE_DRM:
1346 OpenCLDeviceSelector selector = {
1347 .platform_index = -1,
1350 .enumerate_platforms = &opencl_enumerate_platforms,
1351 .filter_platform = &opencl_filter_drm_arm_platform,
1352 .enumerate_devices = &opencl_enumerate_devices,
1353 .filter_device = &opencl_filter_drm_arm_device,
1356 err = opencl_device_create_internal(hwdev, &selector, NULL);
1362 err = AVERROR(ENOSYS);
1369 static int opencl_get_plane_format(enum AVPixelFormat pixfmt,
1370 int plane, int width, int height,
1371 cl_image_format *image_format,
1372 cl_image_desc *image_desc)
1374 const AVPixFmtDescriptor *desc;
1375 const AVComponentDescriptor *comp;
1376 int channels = 0, order = 0, depth = 0, step = 0;
1377 int wsub, hsub, alpha;
1380 if (plane >= AV_NUM_DATA_POINTERS)
1381 return AVERROR(ENOENT);
1383 desc = av_pix_fmt_desc_get(pixfmt);
1385 // Only normal images are allowed.
1386 if (desc->flags & (AV_PIX_FMT_FLAG_BITSTREAM |
1387 AV_PIX_FMT_FLAG_HWACCEL |
1388 AV_PIX_FMT_FLAG_PAL))
1389 return AVERROR(EINVAL);
1391 wsub = 1 << desc->log2_chroma_w;
1392 hsub = 1 << desc->log2_chroma_h;
1393 // Subsampled components must be exact.
1394 if (width & wsub - 1 || height & hsub - 1)
1395 return AVERROR(EINVAL);
1397 for (c = 0; c < desc->nb_components; c++) {
1398 comp = &desc->comp[c];
1399 if (comp->plane != plane)
1401 // The step size must be a power of two.
1402 if (comp->step != 1 && comp->step != 2 &&
1403 comp->step != 4 && comp->step != 8)
1404 return AVERROR(EINVAL);
1405 // The bits in each component must be packed in the
1406 // most-significant-bits of the relevant bytes.
1407 if (comp->shift + comp->depth != 8 &&
1408 comp->shift + comp->depth != 16)
1409 return AVERROR(EINVAL);
1410 // The depth must not vary between components.
1411 if (depth && comp->depth != depth)
1412 return AVERROR(EINVAL);
1413 // If a single data element crosses multiple bytes then
1414 // it must match the native endianness.
1415 if (comp->depth > 8 &&
1416 HAVE_BIGENDIAN == !(desc->flags & AV_PIX_FMT_FLAG_BE))
1417 return AVERROR(EINVAL);
1418 // A single data element must not contain multiple samples
1419 // from the same component.
1420 if (step && comp->step != step)
1421 return AVERROR(EINVAL);
1422 order = order * 10 + c + 1;
1423 depth = comp->depth;
1425 alpha = (desc->flags & AV_PIX_FMT_FLAG_ALPHA &&
1426 c == desc->nb_components - 1);
1430 return AVERROR(ENOENT);
1432 memset(image_format, 0, sizeof(*image_format));
1433 memset(image_desc, 0, sizeof(*image_desc));
1434 image_desc->image_type = CL_MEM_OBJECT_IMAGE2D;
1436 if (plane == 0 || alpha) {
1437 image_desc->image_width = width;
1438 image_desc->image_height = height;
1439 image_desc->image_row_pitch = step * width;
1441 image_desc->image_width = width / wsub;
1442 image_desc->image_height = height / hsub;
1443 image_desc->image_row_pitch = step * width / wsub;
1447 image_format->image_channel_data_type = CL_UNORM_INT8;
1450 image_format->image_channel_data_type = CL_UNORM_INT16;
1452 return AVERROR(EINVAL);
1455 #define CHANNEL_ORDER(order, type) \
1456 case order: image_format->image_channel_order = type; break;
1458 CHANNEL_ORDER(1, CL_R);
1459 CHANNEL_ORDER(2, CL_R);
1460 CHANNEL_ORDER(3, CL_R);
1461 CHANNEL_ORDER(4, CL_R);
1462 CHANNEL_ORDER(12, CL_RG);
1463 CHANNEL_ORDER(23, CL_RG);
1464 CHANNEL_ORDER(1234, CL_RGBA);
1465 CHANNEL_ORDER(3214, CL_BGRA);
1466 CHANNEL_ORDER(4123, CL_ARGB);
1468 CHANNEL_ORDER(4321, CL_ABGR);
1471 return AVERROR(EINVAL);
1473 #undef CHANNEL_ORDER
1478 static int opencl_frames_get_constraints(AVHWDeviceContext *hwdev,
1479 const void *hwconfig,
1480 AVHWFramesConstraints *constraints)
1482 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
1483 cl_uint nb_image_formats;
1484 cl_image_format *image_formats = NULL;
1486 enum AVPixelFormat pix_fmt;
1487 int err, pix_fmts_found;
1488 size_t max_width, max_height;
1490 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_IMAGE2D_MAX_WIDTH,
1491 sizeof(max_width), &max_width, NULL);
1492 if (cle != CL_SUCCESS) {
1493 av_log(hwdev, AV_LOG_ERROR, "Failed to query maximum "
1494 "supported image width: %d.\n", cle);
1496 constraints->max_width = max_width;
1498 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_IMAGE2D_MAX_HEIGHT,
1499 sizeof(max_height), &max_height, NULL);
1500 if (cle != CL_SUCCESS) {
1501 av_log(hwdev, AV_LOG_ERROR, "Failed to query maximum "
1502 "supported image height: %d.\n", cle);
1504 constraints->max_height = max_height;
1506 av_log(hwdev, AV_LOG_DEBUG, "Maximum supported image size %dx%d.\n",
1507 constraints->max_width, constraints->max_height);
1509 cle = clGetSupportedImageFormats(hwctx->context,
1511 CL_MEM_OBJECT_IMAGE2D,
1512 0, NULL, &nb_image_formats);
1513 if (cle != CL_SUCCESS) {
1514 av_log(hwdev, AV_LOG_ERROR, "Failed to query supported "
1515 "image formats: %d.\n", cle);
1516 err = AVERROR(ENOSYS);
1519 if (nb_image_formats == 0) {
1520 av_log(hwdev, AV_LOG_ERROR, "No image support in OpenCL "
1521 "driver (zero supported image formats).\n");
1522 err = AVERROR(ENOSYS);
1527 av_malloc_array(nb_image_formats, sizeof(*image_formats));
1528 if (!image_formats) {
1529 err = AVERROR(ENOMEM);
1533 cle = clGetSupportedImageFormats(hwctx->context,
1535 CL_MEM_OBJECT_IMAGE2D,
1537 image_formats, NULL);
1538 if (cle != CL_SUCCESS) {
1539 av_log(hwdev, AV_LOG_ERROR, "Failed to query supported "
1540 "image formats: %d.\n", cle);
1541 err = AVERROR(ENOSYS);
1546 for (pix_fmt = 0; pix_fmt < AV_PIX_FMT_NB; pix_fmt++) {
1547 cl_image_format image_format;
1548 cl_image_desc image_desc;
1551 for (plane = 0;; plane++) {
1552 err = opencl_get_plane_format(pix_fmt, plane, 0, 0,
1558 for (i = 0; i < nb_image_formats; i++) {
1559 if (image_formats[i].image_channel_order ==
1560 image_format.image_channel_order &&
1561 image_formats[i].image_channel_data_type ==
1562 image_format.image_channel_data_type)
1565 if (i == nb_image_formats) {
1566 err = AVERROR(EINVAL);
1570 if (err != AVERROR(ENOENT))
1573 av_log(hwdev, AV_LOG_DEBUG, "Format %s supported.\n",
1574 av_get_pix_fmt_name(pix_fmt));
1576 err = av_reallocp_array(&constraints->valid_sw_formats,
1578 sizeof(*constraints->valid_sw_formats));
1581 constraints->valid_sw_formats[pix_fmts_found] = pix_fmt;
1582 constraints->valid_sw_formats[pix_fmts_found + 1] =
1587 av_freep(&image_formats);
1589 constraints->valid_hw_formats =
1590 av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
1591 if (!constraints->valid_hw_formats) {
1592 err = AVERROR(ENOMEM);
1595 constraints->valid_hw_formats[0] = AV_PIX_FMT_OPENCL;
1596 constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
1601 av_freep(&image_formats);
1605 static void opencl_pool_free(void *opaque, uint8_t *data)
1607 AVHWFramesContext *hwfc = opaque;
1608 AVOpenCLFrameDescriptor *desc = (AVOpenCLFrameDescriptor*)data;
1612 for (p = 0; p < desc->nb_planes; p++) {
1613 cle = clReleaseMemObject(desc->planes[p]);
1614 if (cle != CL_SUCCESS) {
1615 av_log(hwfc, AV_LOG_ERROR, "Failed to release plane %d: "
1623 static AVBufferRef *opencl_pool_alloc(void *opaque, int size)
1625 AVHWFramesContext *hwfc = opaque;
1626 AVOpenCLDeviceContext *hwctx = hwfc->device_ctx->hwctx;
1627 AVOpenCLFrameDescriptor *desc;
1630 cl_image_format image_format;
1631 cl_image_desc image_desc;
1635 desc = av_mallocz(sizeof(*desc));
1640 err = opencl_get_plane_format(hwfc->sw_format, p,
1641 hwfc->width, hwfc->height,
1642 &image_format, &image_desc);
1643 if (err == AVERROR(ENOENT))
1648 // For generic image objects, the pitch is determined by the
1650 image_desc.image_row_pitch = 0;
1652 image = clCreateImage(hwctx->context, CL_MEM_READ_WRITE,
1653 &image_format, &image_desc, NULL, &cle);
1655 av_log(hwfc, AV_LOG_ERROR, "Failed to create image for "
1656 "plane %d: %d.\n", p, cle);
1660 desc->planes[p] = image;
1663 desc->nb_planes = p;
1665 ref = av_buffer_create((uint8_t*)desc, sizeof(*desc),
1666 &opencl_pool_free, hwfc, 0);
1673 for (p = 0; desc->planes[p]; p++)
1674 clReleaseMemObject(desc->planes[p]);
1679 static int opencl_frames_init_command_queue(AVHWFramesContext *hwfc)
1681 AVOpenCLFramesContext *hwctx = hwfc->hwctx;
1682 OpenCLDeviceContext *devpriv = hwfc->device_ctx->internal->priv;
1683 OpenCLFramesContext *priv = hwfc->internal->priv;
1686 priv->command_queue = hwctx->command_queue ? hwctx->command_queue
1687 : devpriv->command_queue;
1688 cle = clRetainCommandQueue(priv->command_queue);
1689 if (cle != CL_SUCCESS) {
1690 av_log(hwfc, AV_LOG_ERROR, "Failed to retain frame "
1691 "command queue: %d.\n", cle);
1692 return AVERROR(EIO);
1698 static int opencl_frames_init(AVHWFramesContext *hwfc)
1701 hwfc->internal->pool_internal =
1702 av_buffer_pool_init2(sizeof(cl_mem), hwfc,
1703 &opencl_pool_alloc, NULL);
1704 if (!hwfc->internal->pool_internal)
1705 return AVERROR(ENOMEM);
1708 return opencl_frames_init_command_queue(hwfc);
1711 static void opencl_frames_uninit(AVHWFramesContext *hwfc)
1713 OpenCLFramesContext *priv = hwfc->internal->priv;
1716 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
1718 for (i = 0; i < priv->nb_mapped_frames; i++) {
1719 AVOpenCLFrameDescriptor *desc = &priv->mapped_frames[i];
1720 for (p = 0; p < desc->nb_planes; p++) {
1721 cle = clReleaseMemObject(desc->planes[p]);
1722 if (cle != CL_SUCCESS) {
1723 av_log(hwfc, AV_LOG_ERROR, "Failed to release mapped "
1724 "frame object (frame %d plane %d): %d.\n",
1729 av_freep(&priv->mapped_frames);
1732 if (priv->command_queue) {
1733 cle = clReleaseCommandQueue(priv->command_queue);
1734 if (cle != CL_SUCCESS) {
1735 av_log(hwfc, AV_LOG_ERROR, "Failed to release frame "
1736 "command queue: %d.\n", cle);
1738 priv->command_queue = NULL;
1742 static int opencl_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
1744 AVOpenCLFrameDescriptor *desc;
1747 frame->buf[0] = av_buffer_pool_get(hwfc->pool);
1749 return AVERROR(ENOMEM);
1751 desc = (AVOpenCLFrameDescriptor*)frame->buf[0]->data;
1753 for (p = 0; p < desc->nb_planes; p++)
1754 frame->data[p] = (uint8_t*)desc->planes[p];
1756 frame->format = AV_PIX_FMT_OPENCL;
1757 frame->width = hwfc->width;
1758 frame->height = hwfc->height;
1763 static int opencl_transfer_get_formats(AVHWFramesContext *hwfc,
1764 enum AVHWFrameTransferDirection dir,
1765 enum AVPixelFormat **formats)
1767 enum AVPixelFormat *fmts;
1769 fmts = av_malloc_array(2, sizeof(*fmts));
1771 return AVERROR(ENOMEM);
1773 fmts[0] = hwfc->sw_format;
1774 fmts[1] = AV_PIX_FMT_NONE;
1780 static int opencl_wait_events(AVHWFramesContext *hwfc,
1781 cl_event *events, int nb_events)
1786 cle = clWaitForEvents(nb_events, events);
1787 if (cle != CL_SUCCESS) {
1788 av_log(hwfc, AV_LOG_ERROR, "Failed to wait for event "
1789 "completion: %d.\n", cle);
1790 return AVERROR(EIO);
1793 for (i = 0; i < nb_events; i++) {
1794 cle = clReleaseEvent(events[i]);
1795 if (cle != CL_SUCCESS) {
1796 av_log(hwfc, AV_LOG_ERROR, "Failed to release "
1797 "event: %d.\n", cle);
1804 static int opencl_transfer_data_from(AVHWFramesContext *hwfc,
1805 AVFrame *dst, const AVFrame *src)
1807 OpenCLFramesContext *priv = hwfc->internal->priv;
1808 cl_image_format image_format;
1809 cl_image_desc image_desc;
1811 size_t origin[3] = { 0, 0, 0 };
1813 cl_event events[AV_NUM_DATA_POINTERS];
1816 if (dst->format != hwfc->sw_format)
1817 return AVERROR(EINVAL);
1820 err = opencl_get_plane_format(hwfc->sw_format, p,
1821 src->width, src->height,
1822 &image_format, &image_desc);
1824 if (err == AVERROR(ENOENT))
1829 if (!dst->data[p]) {
1830 av_log(hwfc, AV_LOG_ERROR, "Plane %d missing on "
1831 "destination frame for transfer.\n", p);
1832 err = AVERROR(EINVAL);
1836 region[0] = image_desc.image_width;
1837 region[1] = image_desc.image_height;
1840 cle = clEnqueueReadImage(priv->command_queue,
1841 (cl_mem)src->data[p],
1842 CL_FALSE, origin, region,
1843 dst->linesize[p], 0,
1845 0, NULL, &events[p]);
1846 if (cle != CL_SUCCESS) {
1847 av_log(hwfc, AV_LOG_ERROR, "Failed to enqueue read of "
1848 "OpenCL image plane %d: %d.\n", p, cle);
1854 opencl_wait_events(hwfc, events, p);
1859 static int opencl_transfer_data_to(AVHWFramesContext *hwfc,
1860 AVFrame *dst, const AVFrame *src)
1862 OpenCLFramesContext *priv = hwfc->internal->priv;
1863 cl_image_format image_format;
1864 cl_image_desc image_desc;
1866 size_t origin[3] = { 0, 0, 0 };
1868 cl_event events[AV_NUM_DATA_POINTERS];
1871 if (src->format != hwfc->sw_format)
1872 return AVERROR(EINVAL);
1875 err = opencl_get_plane_format(hwfc->sw_format, p,
1876 src->width, src->height,
1877 &image_format, &image_desc);
1879 if (err == AVERROR(ENOENT))
1884 if (!src->data[p]) {
1885 av_log(hwfc, AV_LOG_ERROR, "Plane %d missing on "
1886 "source frame for transfer.\n", p);
1887 err = AVERROR(EINVAL);
1891 region[0] = image_desc.image_width;
1892 region[1] = image_desc.image_height;
1895 cle = clEnqueueWriteImage(priv->command_queue,
1896 (cl_mem)dst->data[p],
1897 CL_FALSE, origin, region,
1898 src->linesize[p], 0,
1900 0, NULL, &events[p]);
1901 if (cle != CL_SUCCESS) {
1902 av_log(hwfc, AV_LOG_ERROR, "Failed to enqueue write of "
1903 "OpenCL image plane %d: %d.\n", p, cle);
1909 opencl_wait_events(hwfc, events, p);
1914 typedef struct OpenCLMapping {
1915 // The mapped addresses for each plane.
1916 // The destination frame is not available when we unmap, so these
1917 // need to be stored separately.
1918 void *address[AV_NUM_DATA_POINTERS];
1921 static void opencl_unmap_frame(AVHWFramesContext *hwfc,
1922 HWMapDescriptor *hwmap)
1924 OpenCLFramesContext *priv = hwfc->internal->priv;
1925 OpenCLMapping *map = hwmap->priv;
1926 cl_event events[AV_NUM_DATA_POINTERS];
1930 for (p = e = 0; p < FF_ARRAY_ELEMS(map->address); p++) {
1931 if (!map->address[p])
1934 cle = clEnqueueUnmapMemObject(priv->command_queue,
1935 (cl_mem)hwmap->source->data[p],
1937 0, NULL, &events[e]);
1938 if (cle != CL_SUCCESS) {
1939 av_log(hwfc, AV_LOG_ERROR, "Failed to unmap OpenCL "
1940 "image plane %d: %d.\n", p, cle);
1945 opencl_wait_events(hwfc, events, e);
1950 static int opencl_map_frame(AVHWFramesContext *hwfc, AVFrame *dst,
1951 const AVFrame *src, int flags)
1953 OpenCLFramesContext *priv = hwfc->internal->priv;
1954 cl_map_flags map_flags;
1955 cl_image_format image_format;
1956 cl_image_desc image_desc;
1959 size_t origin[3] = { 0, 0, 0 };
1962 cl_event events[AV_NUM_DATA_POINTERS];
1965 av_assert0(hwfc->sw_format == dst->format);
1967 if (flags & AV_HWFRAME_MAP_OVERWRITE &&
1968 !(flags & AV_HWFRAME_MAP_READ)) {
1969 // This is mutually exclusive with the read/write flags, so
1970 // there is no way to map with read here.
1971 map_flags = CL_MAP_WRITE_INVALIDATE_REGION;
1974 if (flags & AV_HWFRAME_MAP_READ)
1975 map_flags |= CL_MAP_READ;
1976 if (flags & AV_HWFRAME_MAP_WRITE)
1977 map_flags |= CL_MAP_WRITE;
1980 map = av_mallocz(sizeof(*map));
1982 return AVERROR(ENOMEM);
1985 err = opencl_get_plane_format(hwfc->sw_format, p,
1986 src->width, src->height,
1987 &image_format, &image_desc);
1988 if (err == AVERROR(ENOENT))
1993 region[0] = image_desc.image_width;
1994 region[1] = image_desc.image_height;
1998 clEnqueueMapImage(priv->command_queue,
1999 (cl_mem)src->data[p],
2000 CL_FALSE, map_flags, origin, region,
2001 &row_pitch, NULL, 0, NULL,
2003 if (!map->address[p]) {
2004 av_log(hwfc, AV_LOG_ERROR, "Failed to map OpenCL "
2005 "image plane %d: %d.\n", p, cle);
2010 dst->data[p] = map->address[p];
2012 av_log(hwfc, AV_LOG_DEBUG, "Map plane %d (%p -> %p).\n",
2013 p, src->data[p], dst->data[p]);
2016 err = opencl_wait_events(hwfc, events, p);
2020 err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
2021 &opencl_unmap_frame, map);
2025 dst->width = src->width;
2026 dst->height = src->height;
2031 for (p = 0; p < AV_NUM_DATA_POINTERS; p++) {
2032 if (!map->address[p])
2034 clEnqueueUnmapMemObject(priv->command_queue,
2035 (cl_mem)src->data[p],
2037 0, NULL, &events[p]);
2040 opencl_wait_events(hwfc, events, p);
2045 #if HAVE_OPENCL_DRM_BEIGNET
2047 typedef struct DRMBeignetToOpenCLMapping {
2049 AVDRMFrameDescriptor *drm_desc;
2051 AVOpenCLFrameDescriptor frame;
2052 } DRMBeignetToOpenCLMapping;
2054 static void opencl_unmap_from_drm_beignet(AVHWFramesContext *dst_fc,
2055 HWMapDescriptor *hwmap)
2057 DRMBeignetToOpenCLMapping *mapping = hwmap->priv;
2061 for (i = 0; i < mapping->frame.nb_planes; i++) {
2062 cle = clReleaseMemObject(mapping->frame.planes[i]);
2063 if (cle != CL_SUCCESS) {
2064 av_log(dst_fc, AV_LOG_ERROR, "Failed to release CL image "
2065 "of plane %d of DRM frame: %d.\n", i, cle);
2072 static int opencl_map_from_drm_beignet(AVHWFramesContext *dst_fc,
2073 AVFrame *dst, const AVFrame *src,
2076 AVOpenCLDeviceContext *hwctx = dst_fc->device_ctx->hwctx;
2077 OpenCLDeviceContext *priv = dst_fc->device_ctx->internal->priv;
2078 DRMBeignetToOpenCLMapping *mapping;
2079 const AVDRMFrameDescriptor *desc;
2083 desc = (const AVDRMFrameDescriptor*)src->data[0];
2085 mapping = av_mallocz(sizeof(*mapping));
2087 return AVERROR(ENOMEM);
2090 for (i = 0; i < desc->nb_layers; i++) {
2091 const AVDRMLayerDescriptor *layer = &desc->layers[i];
2092 for (j = 0; j < layer->nb_planes; j++) {
2093 const AVDRMPlaneDescriptor *plane = &layer->planes[j];
2094 const AVDRMObjectDescriptor *object =
2095 &desc->objects[plane->object_index];
2097 cl_import_image_info_intel image_info = {
2099 .size = object->size,
2100 .type = CL_MEM_OBJECT_IMAGE2D,
2101 .offset = plane->offset,
2102 .row_pitch = plane->pitch,
2104 cl_image_desc image_desc;
2106 err = opencl_get_plane_format(dst_fc->sw_format, p,
2107 src->width, src->height,
2111 av_log(dst_fc, AV_LOG_ERROR, "DRM frame layer %d "
2112 "plane %d is not representable in OpenCL: %d.\n",
2116 image_info.width = image_desc.image_width;
2117 image_info.height = image_desc.image_height;
2119 mapping->frame.planes[p] =
2120 priv->clCreateImageFromFdINTEL(hwctx->context,
2122 if (!mapping->frame.planes[p]) {
2123 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL image "
2124 "from layer %d plane %d of DRM frame: %d.\n",
2130 dst->data[p] = (uint8_t*)mapping->frame.planes[p];
2131 mapping->frame.nb_planes = ++p;
2135 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2136 &opencl_unmap_from_drm_beignet,
2141 dst->width = src->width;
2142 dst->height = src->height;
2147 for (p = 0; p < mapping->frame.nb_planes; p++) {
2148 if (mapping->frame.planes[p])
2149 clReleaseMemObject(mapping->frame.planes[p]);
2155 #if HAVE_OPENCL_VAAPI_BEIGNET
2157 static int opencl_map_from_vaapi(AVHWFramesContext *dst_fc,
2158 AVFrame *dst, const AVFrame *src,
2164 tmp = av_frame_alloc();
2166 return AVERROR(ENOMEM);
2168 tmp->format = AV_PIX_FMT_DRM_PRIME;
2170 err = av_hwframe_map(tmp, src, flags);
2174 err = opencl_map_from_drm_beignet(dst_fc, dst, tmp, flags);
2178 err = ff_hwframe_map_replace(dst, src);
2181 av_frame_free(&tmp);
2185 #endif /* HAVE_OPENCL_VAAPI_BEIGNET */
2186 #endif /* HAVE_OPENCL_DRM_BEIGNET */
2188 static inline cl_mem_flags opencl_mem_flags_for_mapping(int map_flags)
2190 if ((map_flags & AV_HWFRAME_MAP_READ) &&
2191 (map_flags & AV_HWFRAME_MAP_WRITE))
2192 return CL_MEM_READ_WRITE;
2193 else if (map_flags & AV_HWFRAME_MAP_READ)
2194 return CL_MEM_READ_ONLY;
2195 else if (map_flags & AV_HWFRAME_MAP_WRITE)
2196 return CL_MEM_WRITE_ONLY;
2201 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2203 static void opencl_unmap_from_qsv(AVHWFramesContext *dst_fc,
2204 HWMapDescriptor *hwmap)
2206 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2207 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2208 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2213 av_log(dst_fc, AV_LOG_DEBUG, "Unmap QSV/VAAPI surface from OpenCL.\n");
2215 cle = device_priv->clEnqueueReleaseVA_APIMediaSurfacesINTEL(
2216 frames_priv->command_queue, desc->nb_planes, desc->planes,
2218 if (cle != CL_SUCCESS) {
2219 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2220 "handles: %d.\n", cle);
2223 opencl_wait_events(dst_fc, &event, 1);
2225 for (p = 0; p < desc->nb_planes; p++) {
2226 cle = clReleaseMemObject(desc->planes[p]);
2227 if (cle != CL_SUCCESS) {
2228 av_log(dst_fc, AV_LOG_ERROR, "Failed to release CL "
2229 "image of plane %d of QSV/VAAPI surface: %d\n",
2237 static int opencl_map_from_qsv(AVHWFramesContext *dst_fc, AVFrame *dst,
2238 const AVFrame *src, int flags)
2240 AVHWFramesContext *src_fc =
2241 (AVHWFramesContext*)src->hw_frames_ctx->data;
2242 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2243 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2244 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2245 AVOpenCLFrameDescriptor *desc;
2246 VASurfaceID va_surface;
2247 cl_mem_flags cl_flags;
2253 if (src->format == AV_PIX_FMT_QSV) {
2254 mfxFrameSurface1 *mfx_surface = (mfxFrameSurface1*)src->data[3];
2255 va_surface = *(VASurfaceID*)mfx_surface->Data.MemId;
2258 if (src->format == AV_PIX_FMT_VAAPI) {
2259 va_surface = (VASurfaceID)(uintptr_t)src->data[3];
2261 return AVERROR(ENOSYS);
2264 cl_flags = opencl_mem_flags_for_mapping(flags);
2266 return AVERROR(EINVAL);
2268 av_log(src_fc, AV_LOG_DEBUG, "Map QSV/VAAPI surface %#x to "
2269 "OpenCL.\n", va_surface);
2271 desc = av_mallocz(sizeof(*desc));
2273 return AVERROR(ENOMEM);
2275 // The cl_intel_va_api_media_sharing extension only supports NV12
2276 // surfaces, so for now there are always exactly two planes.
2277 desc->nb_planes = 2;
2279 for (p = 0; p < desc->nb_planes; p++) {
2281 device_priv->clCreateFromVA_APIMediaSurfaceINTEL(
2282 dst_dev->context, cl_flags, &va_surface, p, &cle);
2283 if (!desc->planes[p]) {
2284 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2285 "image from plane %d of QSV/VAAPI surface "
2286 "%#x: %d.\n", p, va_surface, cle);
2291 dst->data[p] = (uint8_t*)desc->planes[p];
2294 cle = device_priv->clEnqueueAcquireVA_APIMediaSurfacesINTEL(
2295 frames_priv->command_queue, desc->nb_planes, desc->planes,
2297 if (cle != CL_SUCCESS) {
2298 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2299 "handles: %d.\n", cle);
2304 err = opencl_wait_events(dst_fc, &event, 1);
2308 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2309 &opencl_unmap_from_qsv, desc);
2313 dst->width = src->width;
2314 dst->height = src->height;
2319 for (p = 0; p < desc->nb_planes; p++)
2320 if (desc->planes[p])
2321 clReleaseMemObject(desc->planes[p]);
2328 #if HAVE_OPENCL_DXVA2
2330 static void opencl_unmap_from_dxva2(AVHWFramesContext *dst_fc,
2331 HWMapDescriptor *hwmap)
2333 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2334 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2335 OpenCLFramesContext *frames_priv = dst_fc->device_ctx->internal->priv;
2339 av_log(dst_fc, AV_LOG_DEBUG, "Unmap DXVA2 surface from OpenCL.\n");
2341 cle = device_priv->clEnqueueReleaseDX9MediaSurfacesKHR(
2342 frames_priv->command_queue, desc->nb_planes, desc->planes,
2344 if (cle != CL_SUCCESS) {
2345 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2346 "handle: %d.\n", cle);
2350 opencl_wait_events(dst_fc, &event, 1);
2353 static int opencl_map_from_dxva2(AVHWFramesContext *dst_fc, AVFrame *dst,
2354 const AVFrame *src, int flags)
2356 AVHWFramesContext *src_fc =
2357 (AVHWFramesContext*)src->hw_frames_ctx->data;
2358 AVDXVA2FramesContext *src_hwctx = src_fc->hwctx;
2359 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2360 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2361 AVOpenCLFrameDescriptor *desc;
2366 av_log(dst_fc, AV_LOG_DEBUG, "Map DXVA2 surface %p to "
2367 "OpenCL.\n", src->data[3]);
2369 for (i = 0; i < src_hwctx->nb_surfaces; i++) {
2370 if (src_hwctx->surfaces[i] == (IDirect3DSurface9*)src->data[3])
2373 if (i >= src_hwctx->nb_surfaces) {
2374 av_log(dst_fc, AV_LOG_ERROR, "Trying to map from a surface which "
2375 "is not in the mapped frames context.\n");
2376 return AVERROR(EINVAL);
2379 desc = &frames_priv->mapped_frames[i];
2381 cle = device_priv->clEnqueueAcquireDX9MediaSurfacesKHR(
2382 frames_priv->command_queue, desc->nb_planes, desc->planes,
2384 if (cle != CL_SUCCESS) {
2385 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2386 "handle: %d.\n", cle);
2387 return AVERROR(EIO);
2390 err = opencl_wait_events(dst_fc, &event, 1);
2394 for (i = 0; i < desc->nb_planes; i++)
2395 dst->data[i] = (uint8_t*)desc->planes[i];
2397 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2398 &opencl_unmap_from_dxva2, desc);
2402 dst->width = src->width;
2403 dst->height = src->height;
2408 cle = device_priv->clEnqueueReleaseDX9MediaSurfacesKHR(
2409 frames_priv->command_queue, desc->nb_planes, desc->planes,
2411 if (cle == CL_SUCCESS)
2412 opencl_wait_events(dst_fc, &event, 1);
2416 static int opencl_frames_derive_from_dxva2(AVHWFramesContext *dst_fc,
2417 AVHWFramesContext *src_fc, int flags)
2419 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2420 AVDXVA2FramesContext *src_hwctx = src_fc->hwctx;
2421 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2422 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2423 cl_mem_flags cl_flags;
2425 int err, i, p, nb_planes;
2427 if (src_fc->sw_format != AV_PIX_FMT_NV12) {
2428 av_log(dst_fc, AV_LOG_ERROR, "Only NV12 textures are supported "
2429 "for DXVA2 to OpenCL mapping.\n");
2430 return AVERROR(EINVAL);
2434 if (src_fc->initial_pool_size == 0) {
2435 av_log(dst_fc, AV_LOG_ERROR, "Only fixed-size pools are supported "
2436 "for DXVA2 to OpenCL mapping.\n");
2437 return AVERROR(EINVAL);
2440 cl_flags = opencl_mem_flags_for_mapping(flags);
2442 return AVERROR(EINVAL);
2444 frames_priv->nb_mapped_frames = src_hwctx->nb_surfaces;
2446 frames_priv->mapped_frames =
2447 av_mallocz_array(frames_priv->nb_mapped_frames,
2448 sizeof(*frames_priv->mapped_frames));
2449 if (!frames_priv->mapped_frames)
2450 return AVERROR(ENOMEM);
2452 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2453 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2454 cl_dx9_surface_info_khr surface_info = {
2455 .resource = src_hwctx->surfaces[i],
2456 .shared_handle = NULL,
2458 desc->nb_planes = nb_planes;
2459 for (p = 0; p < nb_planes; p++) {
2461 device_priv->clCreateFromDX9MediaSurfaceKHR(
2462 dst_dev->context, cl_flags,
2463 device_priv->dx9_media_adapter_type,
2464 &surface_info, p, &cle);
2465 if (!desc->planes[p]) {
2466 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2467 "image from plane %d of DXVA2 surface %d: %d.\n",
2478 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2479 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2480 for (p = 0; p < desc->nb_planes; p++) {
2481 if (desc->planes[p])
2482 clReleaseMemObject(desc->planes[p]);
2485 av_freep(&frames_priv->mapped_frames);
2486 frames_priv->nb_mapped_frames = 0;
2492 #if HAVE_OPENCL_D3D11
2494 static void opencl_unmap_from_d3d11(AVHWFramesContext *dst_fc,
2495 HWMapDescriptor *hwmap)
2497 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2498 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2499 OpenCLFramesContext *frames_priv = dst_fc->device_ctx->internal->priv;
2503 cle = device_priv->clEnqueueReleaseD3D11ObjectsKHR(
2504 frames_priv->command_queue, desc->nb_planes, desc->planes,
2506 if (cle != CL_SUCCESS) {
2507 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2508 "handle: %d.\n", cle);
2511 opencl_wait_events(dst_fc, &event, 1);
2514 static int opencl_map_from_d3d11(AVHWFramesContext *dst_fc, AVFrame *dst,
2515 const AVFrame *src, int flags)
2517 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2518 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2519 AVOpenCLFrameDescriptor *desc;
2524 index = (intptr_t)src->data[1];
2525 if (index >= frames_priv->nb_mapped_frames) {
2526 av_log(dst_fc, AV_LOG_ERROR, "Texture array index out of range for "
2527 "mapping: %d >= %d.\n", index, frames_priv->nb_mapped_frames);
2528 return AVERROR(EINVAL);
2531 av_log(dst_fc, AV_LOG_DEBUG, "Map D3D11 texture %d to OpenCL.\n",
2534 desc = &frames_priv->mapped_frames[index];
2536 cle = device_priv->clEnqueueAcquireD3D11ObjectsKHR(
2537 frames_priv->command_queue, desc->nb_planes, desc->planes,
2539 if (cle != CL_SUCCESS) {
2540 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2541 "handle: %d.\n", cle);
2542 return AVERROR(EIO);
2545 err = opencl_wait_events(dst_fc, &event, 1);
2549 for (i = 0; i < desc->nb_planes; i++)
2550 dst->data[i] = (uint8_t*)desc->planes[i];
2552 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2553 &opencl_unmap_from_d3d11, desc);
2557 dst->width = src->width;
2558 dst->height = src->height;
2563 cle = device_priv->clEnqueueReleaseD3D11ObjectsKHR(
2564 frames_priv->command_queue, desc->nb_planes, desc->planes,
2566 if (cle == CL_SUCCESS)
2567 opencl_wait_events(dst_fc, &event, 1);
2571 static int opencl_frames_derive_from_d3d11(AVHWFramesContext *dst_fc,
2572 AVHWFramesContext *src_fc, int flags)
2574 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2575 AVD3D11VAFramesContext *src_hwctx = src_fc->hwctx;
2576 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2577 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2578 cl_mem_flags cl_flags;
2580 int err, i, p, nb_planes;
2582 if (src_fc->sw_format != AV_PIX_FMT_NV12) {
2583 av_log(dst_fc, AV_LOG_ERROR, "Only NV12 textures are supported "
2584 "for D3D11 to OpenCL mapping.\n");
2585 return AVERROR(EINVAL);
2589 if (src_fc->initial_pool_size == 0) {
2590 av_log(dst_fc, AV_LOG_ERROR, "Only fixed-size pools are supported "
2591 "for D3D11 to OpenCL mapping.\n");
2592 return AVERROR(EINVAL);
2595 cl_flags = opencl_mem_flags_for_mapping(flags);
2597 return AVERROR(EINVAL);
2599 frames_priv->nb_mapped_frames = src_fc->initial_pool_size;
2601 frames_priv->mapped_frames =
2602 av_mallocz_array(frames_priv->nb_mapped_frames,
2603 sizeof(*frames_priv->mapped_frames));
2604 if (!frames_priv->mapped_frames)
2605 return AVERROR(ENOMEM);
2607 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2608 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2609 desc->nb_planes = nb_planes;
2610 for (p = 0; p < nb_planes; p++) {
2611 UINT subresource = 2 * i + p;
2614 device_priv->clCreateFromD3D11Texture2DKHR(
2615 dst_dev->context, cl_flags, src_hwctx->texture,
2617 if (!desc->planes[p]) {
2618 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2619 "image from plane %d of D3D texture "
2620 "index %d (subresource %u): %d.\n",
2621 p, i, (unsigned int)subresource, cle);
2631 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2632 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2633 for (p = 0; p < desc->nb_planes; p++) {
2634 if (desc->planes[p])
2635 clReleaseMemObject(desc->planes[p]);
2638 av_freep(&frames_priv->mapped_frames);
2639 frames_priv->nb_mapped_frames = 0;
2645 #if HAVE_OPENCL_DRM_ARM
2647 typedef struct DRMARMtoOpenCLMapping {
2649 cl_mem object_buffers[AV_DRM_MAX_PLANES];
2651 cl_mem plane_images[AV_DRM_MAX_PLANES];
2652 } DRMARMtoOpenCLMapping;
2654 static void opencl_unmap_from_drm_arm(AVHWFramesContext *dst_fc,
2655 HWMapDescriptor *hwmap)
2657 DRMARMtoOpenCLMapping *mapping = hwmap->priv;
2660 for (i = 0; i < mapping->nb_planes; i++)
2661 clReleaseMemObject(mapping->plane_images[i]);
2663 for (i = 0; i < mapping->nb_objects; i++)
2664 clReleaseMemObject(mapping->object_buffers[i]);
2669 static int opencl_map_from_drm_arm(AVHWFramesContext *dst_fc, AVFrame *dst,
2670 const AVFrame *src, int flags)
2672 AVHWFramesContext *src_fc =
2673 (AVHWFramesContext*)src->hw_frames_ctx->data;
2674 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2675 const AVDRMFrameDescriptor *desc;
2676 DRMARMtoOpenCLMapping *mapping = NULL;
2677 cl_mem_flags cl_flags;
2678 const cl_import_properties_arm props[3] = {
2679 CL_IMPORT_TYPE_ARM, CL_IMPORT_TYPE_DMA_BUF_ARM, 0,
2684 desc = (const AVDRMFrameDescriptor*)src->data[0];
2686 cl_flags = opencl_mem_flags_for_mapping(flags);
2688 return AVERROR(EINVAL);
2690 mapping = av_mallocz(sizeof(*mapping));
2692 return AVERROR(ENOMEM);
2694 mapping->nb_objects = desc->nb_objects;
2695 for (i = 0; i < desc->nb_objects; i++) {
2696 int fd = desc->objects[i].fd;
2698 av_log(dst_fc, AV_LOG_DEBUG, "Map DRM PRIME fd %d to OpenCL.\n", fd);
2700 if (desc->objects[i].format_modifier) {
2701 av_log(dst_fc, AV_LOG_DEBUG, "Warning: object %d fd %d has "
2702 "nonzero format modifier %"PRId64", result may not "
2703 "be as expected.\n", i, fd,
2704 desc->objects[i].format_modifier);
2707 mapping->object_buffers[i] =
2708 clImportMemoryARM(dst_dev->context, cl_flags, props,
2709 &fd, desc->objects[i].size, &cle);
2710 if (!mapping->object_buffers[i]) {
2711 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL buffer "
2712 "from object %d (fd %d, size %"SIZE_SPECIFIER") of DRM frame: %d.\n",
2713 i, fd, desc->objects[i].size, cle);
2719 mapping->nb_planes = 0;
2720 for (i = 0; i < desc->nb_layers; i++) {
2721 const AVDRMLayerDescriptor *layer = &desc->layers[i];
2723 for (j = 0; j < layer->nb_planes; j++) {
2724 const AVDRMPlaneDescriptor *plane = &layer->planes[j];
2725 cl_mem plane_buffer;
2726 cl_image_format image_format;
2727 cl_image_desc image_desc;
2728 cl_buffer_region region;
2729 int p = mapping->nb_planes;
2731 err = opencl_get_plane_format(src_fc->sw_format, p,
2732 src_fc->width, src_fc->height,
2733 &image_format, &image_desc);
2735 av_log(dst_fc, AV_LOG_ERROR, "Invalid plane %d (DRM "
2736 "layer %d plane %d): %d.\n", p, i, j, err);
2740 region.origin = plane->offset;
2741 region.size = image_desc.image_row_pitch *
2742 image_desc.image_height;
2745 clCreateSubBuffer(mapping->object_buffers[plane->object_index],
2747 CL_BUFFER_CREATE_TYPE_REGION,
2749 if (!plane_buffer) {
2750 av_log(dst_fc, AV_LOG_ERROR, "Failed to create sub-buffer "
2751 "for plane %d: %d.\n", p, cle);
2756 image_desc.buffer = plane_buffer;
2758 mapping->plane_images[p] =
2759 clCreateImage(dst_dev->context, cl_flags,
2760 &image_format, &image_desc, NULL, &cle);
2762 // Unreference the sub-buffer immediately - we don't need it
2763 // directly and a reference is held by the image.
2764 clReleaseMemObject(plane_buffer);
2766 if (!mapping->plane_images[p]) {
2767 av_log(dst_fc, AV_LOG_ERROR, "Failed to create image "
2768 "for plane %d: %d.\n", p, cle);
2773 ++mapping->nb_planes;
2777 for (i = 0; i < mapping->nb_planes; i++)
2778 dst->data[i] = (uint8_t*)mapping->plane_images[i];
2780 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2781 &opencl_unmap_from_drm_arm, mapping);
2785 dst->width = src->width;
2786 dst->height = src->height;
2791 for (i = 0; i < mapping->nb_planes; i++) {
2792 clReleaseMemObject(mapping->plane_images[i]);
2794 for (i = 0; i < mapping->nb_objects; i++) {
2795 if (mapping->object_buffers[i])
2796 clReleaseMemObject(mapping->object_buffers[i]);
2804 static int opencl_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
2805 const AVFrame *src, int flags)
2807 av_assert0(src->format == AV_PIX_FMT_OPENCL);
2808 if (hwfc->sw_format != dst->format)
2809 return AVERROR(ENOSYS);
2810 return opencl_map_frame(hwfc, dst, src, flags);
2813 static int opencl_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
2814 const AVFrame *src, int flags)
2816 av_unused OpenCLDeviceContext *priv = hwfc->device_ctx->internal->priv;
2817 av_assert0(dst->format == AV_PIX_FMT_OPENCL);
2818 switch (src->format) {
2819 #if HAVE_OPENCL_DRM_BEIGNET
2820 case AV_PIX_FMT_DRM_PRIME:
2821 if (priv->beignet_drm_mapping_usable)
2822 return opencl_map_from_drm_beignet(hwfc, dst, src, flags);
2824 #if HAVE_OPENCL_VAAPI_BEIGNET
2825 case AV_PIX_FMT_VAAPI:
2826 if (priv->beignet_drm_mapping_usable)
2827 return opencl_map_from_vaapi(hwfc, dst, src, flags);
2829 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2830 case AV_PIX_FMT_QSV:
2831 case AV_PIX_FMT_VAAPI:
2832 if (priv->qsv_mapping_usable)
2833 return opencl_map_from_qsv(hwfc, dst, src, flags);
2835 #if HAVE_OPENCL_DXVA2
2836 case AV_PIX_FMT_DXVA2_VLD:
2837 if (priv->dxva2_mapping_usable)
2838 return opencl_map_from_dxva2(hwfc, dst, src, flags);
2840 #if HAVE_OPENCL_D3D11
2841 case AV_PIX_FMT_D3D11:
2842 if (priv->d3d11_mapping_usable)
2843 return opencl_map_from_d3d11(hwfc, dst, src, flags);
2845 #if HAVE_OPENCL_DRM_ARM
2846 case AV_PIX_FMT_DRM_PRIME:
2847 if (priv->drm_arm_mapping_usable)
2848 return opencl_map_from_drm_arm(hwfc, dst, src, flags);
2851 return AVERROR(ENOSYS);
2854 static int opencl_frames_derive_to(AVHWFramesContext *dst_fc,
2855 AVHWFramesContext *src_fc, int flags)
2857 av_unused OpenCLDeviceContext *priv = dst_fc->device_ctx->internal->priv;
2858 switch (src_fc->device_ctx->type) {
2859 #if HAVE_OPENCL_DRM_BEIGNET
2860 case AV_HWDEVICE_TYPE_DRM:
2861 if (!priv->beignet_drm_mapping_usable)
2862 return AVERROR(ENOSYS);
2865 #if HAVE_OPENCL_VAAPI_BEIGNET
2866 case AV_HWDEVICE_TYPE_VAAPI:
2867 if (!priv->beignet_drm_mapping_usable)
2868 return AVERROR(ENOSYS);
2871 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2872 case AV_HWDEVICE_TYPE_QSV:
2873 case AV_HWDEVICE_TYPE_VAAPI:
2874 if (!priv->qsv_mapping_usable)
2875 return AVERROR(ENOSYS);
2878 #if HAVE_OPENCL_DXVA2
2879 case AV_HWDEVICE_TYPE_DXVA2:
2880 if (!priv->dxva2_mapping_usable)
2881 return AVERROR(ENOSYS);
2884 err = opencl_frames_derive_from_dxva2(dst_fc, src_fc, flags);
2890 #if HAVE_OPENCL_D3D11
2891 case AV_HWDEVICE_TYPE_D3D11VA:
2892 if (!priv->d3d11_mapping_usable)
2893 return AVERROR(ENOSYS);
2896 err = opencl_frames_derive_from_d3d11(dst_fc, src_fc, flags);
2902 #if HAVE_OPENCL_DRM_ARM
2903 case AV_HWDEVICE_TYPE_DRM:
2904 if (!priv->drm_arm_mapping_usable)
2905 return AVERROR(ENOSYS);
2909 return AVERROR(ENOSYS);
2911 return opencl_frames_init_command_queue(dst_fc);
2914 const HWContextType ff_hwcontext_type_opencl = {
2915 .type = AV_HWDEVICE_TYPE_OPENCL,
2918 .device_hwctx_size = sizeof(AVOpenCLDeviceContext),
2919 .device_priv_size = sizeof(OpenCLDeviceContext),
2920 .frames_hwctx_size = sizeof(AVOpenCLFramesContext),
2921 .frames_priv_size = sizeof(OpenCLFramesContext),
2923 .device_create = &opencl_device_create,
2924 .device_derive = &opencl_device_derive,
2925 .device_init = &opencl_device_init,
2926 .device_uninit = &opencl_device_uninit,
2928 .frames_get_constraints = &opencl_frames_get_constraints,
2929 .frames_init = &opencl_frames_init,
2930 .frames_uninit = &opencl_frames_uninit,
2931 .frames_get_buffer = &opencl_get_buffer,
2933 .transfer_get_formats = &opencl_transfer_get_formats,
2934 .transfer_data_to = &opencl_transfer_data_to,
2935 .transfer_data_from = &opencl_transfer_data_from,
2937 .map_from = &opencl_map_from,
2938 .map_to = &opencl_map_to,
2939 .frames_derive_to = &opencl_frames_derive_to,
2941 .pix_fmts = (const enum AVPixelFormat[]) {