2 * This file is part of FFmpeg.
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 #define CL_USE_DEPRECATED_OPENCL_1_2_APIS
28 #include "hwcontext.h"
29 #include "hwcontext_internal.h"
30 #include "hwcontext_opencl.h"
34 #if HAVE_OPENCL_VAAPI_BEIGNET
37 #include <va/va_drmcommon.h>
38 #include <CL/cl_intel.h>
39 #include "hwcontext_vaapi.h"
42 #if HAVE_OPENCL_DRM_BEIGNET
44 #include <CL/cl_intel.h>
45 #include "hwcontext_drm.h"
48 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
50 #include <mfx/mfxstructures.h>
53 #include <CL/va_ext.h>
54 #include "hwcontext_vaapi.h"
59 #include <CL/cl_dx9_media_sharing.h>
61 #include "hwcontext_dxva2.h"
65 #include <CL/cl_d3d11.h>
66 #include "hwcontext_d3d11va.h"
69 #if HAVE_OPENCL_DRM_ARM
70 #include <CL/cl_ext.h>
71 #include <drm_fourcc.h>
72 #include "hwcontext_drm.h"
76 typedef struct OpenCLDeviceContext {
77 // Default command queue to use for transfer/mapping operations on
78 // the device. If the user supplies one, this is a reference to it.
79 // Otherwise, it is newly-created.
80 cl_command_queue command_queue;
82 // The platform the context exists on. This is needed to query and
83 // retrieve extension functions.
84 cl_platform_id platform_id;
86 // Platform/device-specific functions.
87 #if HAVE_OPENCL_DRM_BEIGNET
88 int beignet_drm_mapping_usable;
89 clCreateImageFromFdINTEL_fn clCreateImageFromFdINTEL;
92 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
93 int qsv_mapping_usable;
94 clCreateFromVA_APIMediaSurfaceINTEL_fn
95 clCreateFromVA_APIMediaSurfaceINTEL;
96 clEnqueueAcquireVA_APIMediaSurfacesINTEL_fn
97 clEnqueueAcquireVA_APIMediaSurfacesINTEL;
98 clEnqueueReleaseVA_APIMediaSurfacesINTEL_fn
99 clEnqueueReleaseVA_APIMediaSurfacesINTEL;
102 #if HAVE_OPENCL_DXVA2
103 int dxva2_mapping_usable;
104 cl_dx9_media_adapter_type_khr dx9_media_adapter_type;
106 clCreateFromDX9MediaSurfaceKHR_fn
107 clCreateFromDX9MediaSurfaceKHR;
108 clEnqueueAcquireDX9MediaSurfacesKHR_fn
109 clEnqueueAcquireDX9MediaSurfacesKHR;
110 clEnqueueReleaseDX9MediaSurfacesKHR_fn
111 clEnqueueReleaseDX9MediaSurfacesKHR;
114 #if HAVE_OPENCL_D3D11
115 int d3d11_mapping_usable;
116 clCreateFromD3D11Texture2DKHR_fn
117 clCreateFromD3D11Texture2DKHR;
118 clEnqueueAcquireD3D11ObjectsKHR_fn
119 clEnqueueAcquireD3D11ObjectsKHR;
120 clEnqueueReleaseD3D11ObjectsKHR_fn
121 clEnqueueReleaseD3D11ObjectsKHR;
124 #if HAVE_OPENCL_DRM_ARM
125 int drm_arm_mapping_usable;
127 } OpenCLDeviceContext;
129 typedef struct OpenCLFramesContext {
130 // Command queue used for transfer/mapping operations on this frames
131 // context. If the user supplies one, this is a reference to it.
132 // Otherwise, it is a reference to the default command queue for the
134 cl_command_queue command_queue;
136 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
137 // For mapping APIs which have separate creation and acquire/release
138 // steps, this stores the OpenCL memory objects corresponding to each
140 int nb_mapped_frames;
141 AVOpenCLFrameDescriptor *mapped_frames;
143 } OpenCLFramesContext;
146 static void CL_CALLBACK opencl_error_callback(const char *errinfo,
147 const void *private_info,
151 AVHWDeviceContext *ctx = user_data;
152 av_log(ctx, AV_LOG_ERROR, "OpenCL error: %s\n", errinfo);
155 static void opencl_device_free(AVHWDeviceContext *hwdev)
157 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
160 cle = clReleaseContext(hwctx->context);
161 if (cle != CL_SUCCESS) {
162 av_log(hwdev, AV_LOG_ERROR, "Failed to release OpenCL "
163 "context: %d.\n", cle);
169 cl_platform_info name;
170 } opencl_platform_params[] = {
171 { "platform_profile", CL_PLATFORM_PROFILE },
172 { "platform_version", CL_PLATFORM_VERSION },
173 { "platform_name", CL_PLATFORM_NAME },
174 { "platform_vendor", CL_PLATFORM_VENDOR },
175 { "platform_extensions", CL_PLATFORM_EXTENSIONS },
181 } opencl_device_params[] = {
182 { "device_name", CL_DEVICE_NAME },
183 { "device_vendor", CL_DEVICE_VENDOR },
184 { "driver_version", CL_DRIVER_VERSION },
185 { "device_version", CL_DEVICE_VERSION },
186 { "device_profile", CL_DEVICE_PROFILE },
187 { "device_extensions", CL_DEVICE_EXTENSIONS },
193 } opencl_device_types[] = {
194 { "cpu", CL_DEVICE_TYPE_CPU },
195 { "gpu", CL_DEVICE_TYPE_GPU },
196 { "accelerator", CL_DEVICE_TYPE_ACCELERATOR },
197 { "custom", CL_DEVICE_TYPE_CUSTOM },
198 { "default", CL_DEVICE_TYPE_DEFAULT },
199 { "all", CL_DEVICE_TYPE_ALL },
202 static char *opencl_get_platform_string(cl_platform_id platform_id,
203 cl_platform_info key)
208 cle = clGetPlatformInfo(platform_id, key, 0, NULL, &size);
209 if (cle != CL_SUCCESS)
211 str = av_malloc(size);
214 cle = clGetPlatformInfo(platform_id, key, size, str, &size);
215 if (cle != CL_SUCCESS) {
219 av_assert0(strlen(str) + 1 == size);
223 static char *opencl_get_device_string(cl_device_id device_id,
229 cle = clGetDeviceInfo(device_id, key, 0, NULL, &size);
230 if (cle != CL_SUCCESS)
232 str = av_malloc(size);
235 cle = clGetDeviceInfo(device_id, key, size, str, &size);
236 if (cle != CL_SUCCESS) {
240 av_assert0(strlen(str) + 1== size);
244 static int opencl_check_platform_extension(cl_platform_id platform_id,
249 str = opencl_get_platform_string(platform_id,
250 CL_PLATFORM_EXTENSIONS);
251 if (str && strstr(str, name))
257 static int opencl_check_device_extension(cl_device_id device_id,
262 str = opencl_get_device_string(device_id,
263 CL_DEVICE_EXTENSIONS);
264 if (str && strstr(str, name))
270 static av_unused int opencl_check_extension(AVHWDeviceContext *hwdev,
273 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
274 OpenCLDeviceContext *priv = hwdev->internal->priv;
276 if (opencl_check_platform_extension(priv->platform_id, name)) {
277 av_log(hwdev, AV_LOG_DEBUG,
278 "%s found as platform extension.\n", name);
282 if (opencl_check_device_extension(hwctx->device_id, name)) {
283 av_log(hwdev, AV_LOG_DEBUG,
284 "%s found as device extension.\n", name);
291 static int opencl_enumerate_platforms(AVHWDeviceContext *hwdev,
292 cl_uint *nb_platforms,
293 cl_platform_id **platforms,
298 cle = clGetPlatformIDs(0, NULL, nb_platforms);
299 if (cle != CL_SUCCESS) {
300 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of "
301 "OpenCL platforms: %d.\n", cle);
302 return AVERROR(ENODEV);
304 av_log(hwdev, AV_LOG_DEBUG, "%u OpenCL platforms found.\n",
307 *platforms = av_malloc_array(*nb_platforms, sizeof(**platforms));
309 return AVERROR(ENOMEM);
311 cle = clGetPlatformIDs(*nb_platforms, *platforms, NULL);
312 if (cle != CL_SUCCESS) {
313 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of OpenCL "
314 "platforms: %d.\n", cle);
316 return AVERROR(ENODEV);
322 static int opencl_filter_platform(AVHWDeviceContext *hwdev,
323 cl_platform_id platform_id,
324 const char *platform_name,
327 AVDictionary *opts = context;
328 const AVDictionaryEntry *param;
332 for (i = 0; i < FF_ARRAY_ELEMS(opencl_platform_params); i++) {
333 param = av_dict_get(opts, opencl_platform_params[i].key,
338 str = opencl_get_platform_string(platform_id,
339 opencl_platform_params[i].name);
341 av_log(hwdev, AV_LOG_ERROR, "Failed to query %s "
342 "of platform \"%s\".\n",
343 opencl_platform_params[i].key, platform_name);
344 return AVERROR_UNKNOWN;
346 if (!av_stristr(str, param->value)) {
347 av_log(hwdev, AV_LOG_DEBUG, "%s does not match (\"%s\").\n",
357 static int opencl_enumerate_devices(AVHWDeviceContext *hwdev,
358 cl_platform_id platform_id,
359 const char *platform_name,
361 cl_device_id **devices,
366 cle = clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_ALL,
367 0, NULL, nb_devices);
368 if (cle == CL_DEVICE_NOT_FOUND) {
369 av_log(hwdev, AV_LOG_DEBUG, "No devices found "
370 "on platform \"%s\".\n", platform_name);
373 } else if (cle != CL_SUCCESS) {
374 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
375 "on platform \"%s\": %d.\n", platform_name, cle);
376 return AVERROR(ENODEV);
378 av_log(hwdev, AV_LOG_DEBUG, "%u OpenCL devices found on "
379 "platform \"%s\".\n", *nb_devices, platform_name);
381 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
383 return AVERROR(ENOMEM);
385 cle = clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_ALL,
386 *nb_devices, *devices, NULL);
387 if (cle != CL_SUCCESS) {
388 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of devices "
389 "on platform \"%s\": %d.\n", platform_name, cle);
391 return AVERROR(ENODEV);
397 static int opencl_filter_device(AVHWDeviceContext *hwdev,
398 cl_device_id device_id,
399 const char *device_name,
402 AVDictionary *opts = context;
403 const AVDictionaryEntry *param;
407 param = av_dict_get(opts, "device_type", NULL, 0);
409 cl_device_type match_type = 0, device_type;
412 for (i = 0; i < FF_ARRAY_ELEMS(opencl_device_types); i++) {
413 if (!strcmp(opencl_device_types[i].key, param->value)) {
414 match_type = opencl_device_types[i].type;
419 av_log(hwdev, AV_LOG_ERROR, "Unknown device type %s.\n",
421 return AVERROR(EINVAL);
424 cle = clGetDeviceInfo(device_id, CL_DEVICE_TYPE,
425 sizeof(device_type), &device_type, NULL);
426 if (cle != CL_SUCCESS) {
427 av_log(hwdev, AV_LOG_ERROR, "Failed to query device type "
428 "of device \"%s\".\n", device_name);
429 return AVERROR_UNKNOWN;
432 if (!(device_type & match_type)) {
433 av_log(hwdev, AV_LOG_DEBUG, "device_type does not match.\n");
438 for (i = 0; i < FF_ARRAY_ELEMS(opencl_device_params); i++) {
439 param = av_dict_get(opts, opencl_device_params[i].key,
444 str = opencl_get_device_string(device_id,
445 opencl_device_params[i].name);
447 av_log(hwdev, AV_LOG_ERROR, "Failed to query %s "
448 "of device \"%s\".\n",
449 opencl_device_params[i].key, device_name);
450 return AVERROR_UNKNOWN;
452 if (!av_stristr(str, param->value)) {
453 av_log(hwdev, AV_LOG_DEBUG, "%s does not match (\"%s\").\n",
463 typedef struct OpenCLDeviceSelector {
467 int (*enumerate_platforms)(AVHWDeviceContext *hwdev,
468 cl_uint *nb_platforms,
469 cl_platform_id **platforms,
471 int (*filter_platform) (AVHWDeviceContext *hwdev,
472 cl_platform_id platform_id,
473 const char *platform_name,
475 int (*enumerate_devices) (AVHWDeviceContext *hwdev,
476 cl_platform_id platform_id,
477 const char *platform_name,
479 cl_device_id **devices,
481 int (*filter_device) (AVHWDeviceContext *hwdev,
482 cl_device_id device_id,
483 const char *device_name,
485 } OpenCLDeviceSelector;
487 static int opencl_device_create_internal(AVHWDeviceContext *hwdev,
488 const OpenCLDeviceSelector *selector,
489 cl_context_properties *props)
491 cl_uint nb_platforms;
492 cl_platform_id *platforms = NULL;
493 cl_platform_id platform_id;
495 cl_device_id *devices = NULL;
496 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
498 cl_context_properties default_props[3];
499 char *platform_name_src = NULL,
500 *device_name_src = NULL;
501 int err, found, p, d;
503 err = selector->enumerate_platforms(hwdev, &nb_platforms, &platforms,
509 for (p = 0; p < nb_platforms; p++) {
510 const char *platform_name;
512 if (selector->platform_index >= 0 &&
513 selector->platform_index != p)
516 av_freep(&platform_name_src);
517 platform_name_src = opencl_get_platform_string(platforms[p],
519 if (platform_name_src)
520 platform_name = platform_name_src;
522 platform_name = "Unknown Platform";
524 if (selector->filter_platform) {
525 err = selector->filter_platform(hwdev, platforms[p],
534 err = opencl_enumerate_devices(hwdev, platforms[p], platform_name,
535 &nb_devices, &devices,
540 for (d = 0; d < nb_devices; d++) {
541 const char *device_name;
543 if (selector->device_index >= 0 &&
544 selector->device_index != d)
547 av_freep(&device_name_src);
548 device_name_src = opencl_get_device_string(devices[d],
551 device_name = device_name_src;
553 device_name = "Unknown Device";
555 if (selector->filter_device) {
556 err = selector->filter_device(hwdev, devices[d],
565 av_log(hwdev, AV_LOG_VERBOSE, "%d.%d: %s / %s\n", p, d,
566 platform_name, device_name);
569 platform_id = platforms[p];
570 hwctx->device_id = devices[d];
577 av_log(hwdev, AV_LOG_ERROR, "No matching devices found.\n");
578 err = AVERROR(ENODEV);
582 av_log(hwdev, AV_LOG_ERROR, "More than one matching device found.\n");
583 err = AVERROR(ENODEV);
588 props = default_props;
589 default_props[0] = CL_CONTEXT_PLATFORM;
590 default_props[1] = (intptr_t)platform_id;
591 default_props[2] = 0;
593 if (props[0] == CL_CONTEXT_PLATFORM && props[1] == 0)
594 props[1] = (intptr_t)platform_id;
597 hwctx->context = clCreateContext(props, 1, &hwctx->device_id,
598 &opencl_error_callback, hwdev, &cle);
599 if (!hwctx->context) {
600 av_log(hwdev, AV_LOG_ERROR, "Failed to create OpenCL context: "
602 err = AVERROR(ENODEV);
606 hwdev->free = &opencl_device_free;
610 av_freep(&platform_name_src);
611 av_freep(&device_name_src);
612 av_freep(&platforms);
617 static int opencl_device_create(AVHWDeviceContext *hwdev, const char *device,
618 AVDictionary *opts, int flags)
620 OpenCLDeviceSelector selector = {
622 .enumerate_platforms = &opencl_enumerate_platforms,
623 .filter_platform = &opencl_filter_platform,
624 .enumerate_devices = &opencl_enumerate_devices,
625 .filter_device = &opencl_filter_device,
628 if (device && device[0]) {
629 // Match one or both indices for platform and device.
630 int d = -1, p = -1, ret;
631 if (device[0] == '.')
632 ret = sscanf(device, ".%d", &d);
634 ret = sscanf(device, "%d.%d", &p, &d);
636 av_log(hwdev, AV_LOG_ERROR, "Invalid OpenCL platform/device "
637 "index specification \"%s\".\n", device);
638 return AVERROR(EINVAL);
640 selector.platform_index = p;
641 selector.device_index = d;
643 selector.platform_index = -1;
644 selector.device_index = -1;
647 return opencl_device_create_internal(hwdev, &selector, NULL);
650 static int opencl_device_init(AVHWDeviceContext *hwdev)
652 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
653 OpenCLDeviceContext *priv = hwdev->internal->priv;
656 if (hwctx->command_queue) {
657 cle = clRetainCommandQueue(hwctx->command_queue);
658 if (cle != CL_SUCCESS) {
659 av_log(hwdev, AV_LOG_ERROR, "Failed to retain external "
660 "command queue: %d.\n", cle);
663 priv->command_queue = hwctx->command_queue;
665 priv->command_queue = clCreateCommandQueue(hwctx->context,
668 if (!priv->command_queue) {
669 av_log(hwdev, AV_LOG_ERROR, "Failed to create internal "
670 "command queue: %d.\n", cle);
675 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_PLATFORM,
676 sizeof(priv->platform_id), &priv->platform_id,
678 if (cle != CL_SUCCESS) {
679 av_log(hwdev, AV_LOG_ERROR, "Failed to determine the OpenCL "
680 "platform containing the device.\n");
684 #define CL_FUNC(name, desc) do { \
687 priv->name = clGetExtensionFunctionAddressForPlatform( \
688 priv->platform_id, #name); \
690 av_log(hwdev, AV_LOG_VERBOSE, \
691 desc " function not found (%s).\n", #name); \
694 av_log(hwdev, AV_LOG_VERBOSE, \
695 desc " function found (%s).\n", #name); \
699 #if HAVE_OPENCL_DRM_BEIGNET
703 CL_FUNC(clCreateImageFromFdINTEL,
704 "Beignet DRM to OpenCL image mapping");
707 av_log(hwdev, AV_LOG_WARNING, "Beignet DRM to OpenCL "
708 "mapping not usable.\n");
709 priv->beignet_drm_mapping_usable = 0;
711 priv->beignet_drm_mapping_usable = 1;
716 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
719 cl_context_properties *props = NULL;
720 VADisplay va_display;
721 const char *va_ext = "cl_intel_va_api_media_sharing";
724 if (!opencl_check_extension(hwdev, va_ext)) {
725 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
726 "required for QSV to OpenCL mapping.\n", va_ext);
730 cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
731 0, NULL, &props_size);
732 if (cle != CL_SUCCESS) {
733 av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
734 "properties: %d.\n", cle);
737 if (props_size == 0) {
738 av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
739 "enabled on context creation to use QSV to "
740 "OpenCL mapping.\n");
744 props = av_malloc(props_size);
746 return AVERROR(ENOMEM);
748 cle = clGetContextInfo(hwctx->context, CL_CONTEXT_PROPERTIES,
749 props_size, props, NULL);
750 if (cle != CL_SUCCESS) {
751 av_log(hwdev, AV_LOG_VERBOSE, "Failed to get context "
752 "properties: %d.\n", cle);
757 for (i = 0; i < (props_size / sizeof(*props) - 1); i++) {
758 if (props[i] == CL_CONTEXT_VA_API_DISPLAY_INTEL) {
759 va_display = (VADisplay)(intptr_t)props[i+1];
764 av_log(hwdev, AV_LOG_VERBOSE, "Media sharing must be "
765 "enabled on context creation to use QSV to "
766 "OpenCL mapping.\n");
769 if (!vaDisplayIsValid(va_display)) {
770 av_log(hwdev, AV_LOG_VERBOSE, "A valid VADisplay is "
771 "required on context creation to use QSV to "
772 "OpenCL mapping.\n");
776 CL_FUNC(clCreateFromVA_APIMediaSurfaceINTEL,
777 "Intel QSV to OpenCL mapping");
778 CL_FUNC(clEnqueueAcquireVA_APIMediaSurfacesINTEL,
779 "Intel QSV in OpenCL acquire");
780 CL_FUNC(clEnqueueReleaseVA_APIMediaSurfacesINTEL,
781 "Intel QSV in OpenCL release");
785 av_log(hwdev, AV_LOG_WARNING, "QSV to OpenCL mapping "
787 priv->qsv_mapping_usable = 0;
789 priv->qsv_mapping_usable = 1;
795 #if HAVE_OPENCL_DXVA2
799 CL_FUNC(clCreateFromDX9MediaSurfaceKHR,
800 "DXVA2 to OpenCL mapping");
801 CL_FUNC(clEnqueueAcquireDX9MediaSurfacesKHR,
802 "DXVA2 in OpenCL acquire");
803 CL_FUNC(clEnqueueReleaseDX9MediaSurfacesKHR,
804 "DXVA2 in OpenCL release");
807 av_log(hwdev, AV_LOG_WARNING, "DXVA2 to OpenCL mapping "
809 priv->dxva2_mapping_usable = 0;
811 priv->dx9_media_adapter_type = CL_ADAPTER_D3D9EX_KHR;
812 priv->dxva2_mapping_usable = 1;
817 #if HAVE_OPENCL_D3D11
819 const char *d3d11_ext = "cl_khr_d3d11_sharing";
820 const char *nv12_ext = "cl_intel_d3d11_nv12_media_sharing";
823 if (!opencl_check_extension(hwdev, d3d11_ext)) {
824 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
825 "required for D3D11 to OpenCL mapping.\n", d3d11_ext);
827 } else if (!opencl_check_extension(hwdev, nv12_ext)) {
828 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension may be "
829 "required for D3D11 to OpenCL mapping.\n", nv12_ext);
833 CL_FUNC(clCreateFromD3D11Texture2DKHR,
834 "D3D11 to OpenCL mapping");
835 CL_FUNC(clEnqueueAcquireD3D11ObjectsKHR,
836 "D3D11 in OpenCL acquire");
837 CL_FUNC(clEnqueueReleaseD3D11ObjectsKHR,
838 "D3D11 in OpenCL release");
841 av_log(hwdev, AV_LOG_WARNING, "D3D11 to OpenCL mapping "
843 priv->d3d11_mapping_usable = 0;
845 priv->d3d11_mapping_usable = 1;
850 #if HAVE_OPENCL_DRM_ARM
852 const char *drm_arm_ext = "cl_arm_import_memory";
853 const char *image_ext = "cl_khr_image2d_from_buffer";
856 if (!opencl_check_extension(hwdev, drm_arm_ext)) {
857 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
858 "required for DRM to OpenCL mapping on ARM.\n",
862 if (!opencl_check_extension(hwdev, image_ext)) {
863 av_log(hwdev, AV_LOG_VERBOSE, "The %s extension is "
864 "required for DRM to OpenCL mapping on ARM.\n",
869 // clImportMemoryARM() is linked statically.
872 av_log(hwdev, AV_LOG_WARNING, "DRM to OpenCL mapping on ARM "
874 priv->drm_arm_mapping_usable = 0;
876 priv->drm_arm_mapping_usable = 1;
886 static void opencl_device_uninit(AVHWDeviceContext *hwdev)
888 OpenCLDeviceContext *priv = hwdev->internal->priv;
891 if (priv->command_queue) {
892 cle = clReleaseCommandQueue(priv->command_queue);
893 if (cle != CL_SUCCESS) {
894 av_log(hwdev, AV_LOG_ERROR, "Failed to release internal "
895 "command queue reference: %d.\n", cle);
897 priv->command_queue = NULL;
901 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
902 static int opencl_filter_intel_media_vaapi_platform(AVHWDeviceContext *hwdev,
903 cl_platform_id platform_id,
904 const char *platform_name,
907 // This doesn't exist as a platform extension, so just test whether
908 // the function we will use for device enumeration exists.
910 if (!clGetExtensionFunctionAddressForPlatform(platform_id,
911 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL")) {
912 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not export the "
913 "VAAPI device enumeration function.\n", platform_name);
920 static int opencl_enumerate_intel_media_vaapi_devices(AVHWDeviceContext *hwdev,
921 cl_platform_id platform_id,
922 const char *platform_name,
924 cl_device_id **devices,
927 VADisplay va_display = context;
928 clGetDeviceIDsFromVA_APIMediaAdapterINTEL_fn
929 clGetDeviceIDsFromVA_APIMediaAdapterINTEL;
932 clGetDeviceIDsFromVA_APIMediaAdapterINTEL =
933 clGetExtensionFunctionAddressForPlatform(platform_id,
934 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL");
935 if (!clGetDeviceIDsFromVA_APIMediaAdapterINTEL) {
936 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
937 "clGetDeviceIDsFromVA_APIMediaAdapterINTEL().\n");
938 return AVERROR_UNKNOWN;
941 cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
942 platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
943 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, 0, NULL, nb_devices);
944 if (cle == CL_DEVICE_NOT_FOUND) {
945 av_log(hwdev, AV_LOG_DEBUG, "No VAAPI-supporting devices found "
946 "on platform \"%s\".\n", platform_name);
949 } else if (cle != CL_SUCCESS) {
950 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
951 "on platform \"%s\": %d.\n", platform_name, cle);
952 return AVERROR_UNKNOWN;
955 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
957 return AVERROR(ENOMEM);
959 cle = clGetDeviceIDsFromVA_APIMediaAdapterINTEL(
960 platform_id, CL_VA_API_DISPLAY_INTEL, va_display,
961 CL_PREFERRED_DEVICES_FOR_VA_API_INTEL, *nb_devices, *devices, NULL);
962 if (cle != CL_SUCCESS) {
963 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of VAAPI-supporting "
964 "devices on platform \"%s\": %d.\n", platform_name, cle);
966 return AVERROR_UNKNOWN;
972 static int opencl_filter_intel_media_vaapi_device(AVHWDeviceContext *hwdev,
973 cl_device_id device_id,
974 const char *device_name,
977 const char *va_ext = "cl_intel_va_api_media_sharing";
979 if (opencl_check_device_extension(device_id, va_ext)) {
982 av_log(hwdev, AV_LOG_DEBUG, "Device %s does not support the "
983 "%s extension.\n", device_name, va_ext);
989 #if HAVE_OPENCL_DXVA2
990 static int opencl_filter_dxva2_platform(AVHWDeviceContext *hwdev,
991 cl_platform_id platform_id,
992 const char *platform_name,
995 const char *dx9_ext = "cl_khr_dx9_media_sharing";
997 if (opencl_check_platform_extension(platform_id, dx9_ext)) {
1000 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1001 "%s extension.\n", platform_name, dx9_ext);
1006 static int opencl_enumerate_dxva2_devices(AVHWDeviceContext *hwdev,
1007 cl_platform_id platform_id,
1008 const char *platform_name,
1009 cl_uint *nb_devices,
1010 cl_device_id **devices,
1013 IDirect3DDevice9 *device = context;
1014 clGetDeviceIDsFromDX9MediaAdapterKHR_fn
1015 clGetDeviceIDsFromDX9MediaAdapterKHR;
1016 cl_dx9_media_adapter_type_khr media_adapter_type = CL_ADAPTER_D3D9EX_KHR;
1019 clGetDeviceIDsFromDX9MediaAdapterKHR =
1020 clGetExtensionFunctionAddressForPlatform(platform_id,
1021 "clGetDeviceIDsFromDX9MediaAdapterKHR");
1022 if (!clGetDeviceIDsFromDX9MediaAdapterKHR) {
1023 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
1024 "clGetDeviceIDsFromDX9MediaAdapterKHR().\n");
1025 return AVERROR_UNKNOWN;
1028 cle = clGetDeviceIDsFromDX9MediaAdapterKHR(
1029 platform_id, 1, &media_adapter_type, (void**)&device,
1030 CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR,
1031 0, NULL, nb_devices);
1032 if (cle == CL_DEVICE_NOT_FOUND) {
1033 av_log(hwdev, AV_LOG_DEBUG, "No DXVA2-supporting devices found "
1034 "on platform \"%s\".\n", platform_name);
1037 } else if (cle != CL_SUCCESS) {
1038 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
1039 "on platform \"%s\": %d.\n", platform_name, cle);
1040 return AVERROR_UNKNOWN;
1043 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
1045 return AVERROR(ENOMEM);
1047 cle = clGetDeviceIDsFromDX9MediaAdapterKHR(
1048 platform_id, 1, &media_adapter_type, (void**)&device,
1049 CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR,
1050 *nb_devices, *devices, NULL);
1051 if (cle != CL_SUCCESS) {
1052 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of DXVA2-supporting "
1053 "devices on platform \"%s\": %d.\n", platform_name, cle);
1055 return AVERROR_UNKNOWN;
1062 #if HAVE_OPENCL_D3D11
1063 static int opencl_filter_d3d11_platform(AVHWDeviceContext *hwdev,
1064 cl_platform_id platform_id,
1065 const char *platform_name,
1068 const char *d3d11_ext = "cl_khr_d3d11_sharing";
1070 if (opencl_check_platform_extension(platform_id, d3d11_ext)) {
1073 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1074 "%s extension.\n", platform_name, d3d11_ext);
1079 static int opencl_enumerate_d3d11_devices(AVHWDeviceContext *hwdev,
1080 cl_platform_id platform_id,
1081 const char *platform_name,
1082 cl_uint *nb_devices,
1083 cl_device_id **devices,
1086 ID3D11Device *device = context;
1087 clGetDeviceIDsFromD3D11KHR_fn clGetDeviceIDsFromD3D11KHR;
1090 clGetDeviceIDsFromD3D11KHR =
1091 clGetExtensionFunctionAddressForPlatform(platform_id,
1092 "clGetDeviceIDsFromD3D11KHR");
1093 if (!clGetDeviceIDsFromD3D11KHR) {
1094 av_log(hwdev, AV_LOG_ERROR, "Failed to get address of "
1095 "clGetDeviceIDsFromD3D11KHR().\n");
1096 return AVERROR_UNKNOWN;
1099 cle = clGetDeviceIDsFromD3D11KHR(platform_id,
1100 CL_D3D11_DEVICE_KHR, device,
1101 CL_PREFERRED_DEVICES_FOR_D3D11_KHR,
1102 0, NULL, nb_devices);
1103 if (cle == CL_DEVICE_NOT_FOUND) {
1104 av_log(hwdev, AV_LOG_DEBUG, "No D3D11-supporting devices found "
1105 "on platform \"%s\".\n", platform_name);
1108 } else if (cle != CL_SUCCESS) {
1109 av_log(hwdev, AV_LOG_ERROR, "Failed to get number of devices "
1110 "on platform \"%s\": %d.\n", platform_name, cle);
1111 return AVERROR_UNKNOWN;
1114 *devices = av_malloc_array(*nb_devices, sizeof(**devices));
1116 return AVERROR(ENOMEM);
1118 cle = clGetDeviceIDsFromD3D11KHR(platform_id,
1119 CL_D3D11_DEVICE_KHR, device,
1120 CL_PREFERRED_DEVICES_FOR_D3D11_KHR,
1121 *nb_devices, *devices, NULL);
1122 if (cle != CL_SUCCESS) {
1123 av_log(hwdev, AV_LOG_ERROR, "Failed to get list of D3D11-supporting "
1124 "devices on platform \"%s\": %d.\n", platform_name, cle);
1126 return AVERROR_UNKNOWN;
1133 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
1134 static int opencl_filter_gpu_device(AVHWDeviceContext *hwdev,
1135 cl_device_id device_id,
1136 const char *device_name,
1139 cl_device_type device_type;
1142 cle = clGetDeviceInfo(device_id, CL_DEVICE_TYPE,
1143 sizeof(device_type), &device_type, NULL);
1144 if (cle != CL_SUCCESS) {
1145 av_log(hwdev, AV_LOG_ERROR, "Failed to query device type "
1146 "of device \"%s\".\n", device_name);
1147 return AVERROR_UNKNOWN;
1149 if (!(device_type & CL_DEVICE_TYPE_GPU)) {
1150 av_log(hwdev, AV_LOG_DEBUG, "Device %s skipped (not GPU).\n",
1159 #if HAVE_OPENCL_DRM_ARM
1160 static int opencl_filter_drm_arm_platform(AVHWDeviceContext *hwdev,
1161 cl_platform_id platform_id,
1162 const char *platform_name,
1165 const char *drm_arm_ext = "cl_arm_import_memory";
1167 if (opencl_check_platform_extension(platform_id, drm_arm_ext)) {
1170 av_log(hwdev, AV_LOG_DEBUG, "Platform %s does not support the "
1171 "%s extension.\n", platform_name, drm_arm_ext);
1176 static int opencl_filter_drm_arm_device(AVHWDeviceContext *hwdev,
1177 cl_device_id device_id,
1178 const char *device_name,
1181 const char *drm_arm_ext = "cl_arm_import_memory";
1183 if (opencl_check_device_extension(device_id, drm_arm_ext)) {
1186 av_log(hwdev, AV_LOG_DEBUG, "Device %s does not support the "
1187 "%s extension.\n", device_name, drm_arm_ext);
1193 static int opencl_device_derive(AVHWDeviceContext *hwdev,
1194 AVHWDeviceContext *src_ctx,
1198 switch (src_ctx->type) {
1200 #if HAVE_OPENCL_DRM_BEIGNET
1201 case AV_HWDEVICE_TYPE_DRM:
1202 case AV_HWDEVICE_TYPE_VAAPI:
1204 // Surface mapping works via DRM PRIME fds with no special
1205 // initialisation required in advance. This just finds the
1206 // Beignet ICD by name.
1207 AVDictionary *opts = NULL;
1209 err = av_dict_set(&opts, "platform_vendor", "Intel", 0);
1211 err = av_dict_set(&opts, "platform_version", "beignet", 0);
1213 OpenCLDeviceSelector selector = {
1214 .platform_index = -1,
1217 .enumerate_platforms = &opencl_enumerate_platforms,
1218 .filter_platform = &opencl_filter_platform,
1219 .enumerate_devices = &opencl_enumerate_devices,
1220 .filter_device = NULL,
1222 err = opencl_device_create_internal(hwdev, &selector, NULL);
1224 av_dict_free(&opts);
1229 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
1230 // The generic code automatically attempts to derive from all
1231 // ancestors of the given device, so we can ignore QSV devices here
1232 // and just consider the inner VAAPI device it was derived from.
1233 case AV_HWDEVICE_TYPE_VAAPI:
1235 AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
1236 cl_context_properties props[7] = {
1237 CL_CONTEXT_PLATFORM,
1239 CL_CONTEXT_VA_API_DISPLAY_INTEL,
1240 (intptr_t)src_hwctx->display,
1241 CL_CONTEXT_INTEROP_USER_SYNC,
1245 OpenCLDeviceSelector selector = {
1246 .platform_index = -1,
1248 .context = src_hwctx->display,
1249 .enumerate_platforms = &opencl_enumerate_platforms,
1250 .filter_platform = &opencl_filter_intel_media_vaapi_platform,
1251 .enumerate_devices = &opencl_enumerate_intel_media_vaapi_devices,
1252 .filter_device = &opencl_filter_intel_media_vaapi_device,
1255 err = opencl_device_create_internal(hwdev, &selector, props);
1260 #if HAVE_OPENCL_DXVA2
1261 case AV_HWDEVICE_TYPE_DXVA2:
1263 AVDXVA2DeviceContext *src_hwctx = src_ctx->hwctx;
1264 IDirect3DDevice9 *device;
1265 HANDLE device_handle;
1268 hr = IDirect3DDeviceManager9_OpenDeviceHandle(src_hwctx->devmgr,
1271 av_log(hwdev, AV_LOG_ERROR, "Failed to open device handle "
1272 "for Direct3D9 device: %lx.\n", (unsigned long)hr);
1273 err = AVERROR_UNKNOWN;
1277 hr = IDirect3DDeviceManager9_LockDevice(src_hwctx->devmgr,
1280 if (SUCCEEDED(hr)) {
1281 cl_context_properties props[5] = {
1282 CL_CONTEXT_PLATFORM,
1284 CL_CONTEXT_ADAPTER_D3D9EX_KHR,
1288 OpenCLDeviceSelector selector = {
1289 .platform_index = -1,
1292 .enumerate_platforms = &opencl_enumerate_platforms,
1293 .filter_platform = &opencl_filter_dxva2_platform,
1294 .enumerate_devices = &opencl_enumerate_dxva2_devices,
1295 .filter_device = &opencl_filter_gpu_device,
1298 err = opencl_device_create_internal(hwdev, &selector, props);
1300 IDirect3DDeviceManager9_UnlockDevice(src_hwctx->devmgr,
1301 device_handle, FALSE);
1303 av_log(hwdev, AV_LOG_ERROR, "Failed to lock device handle "
1304 "for Direct3D9 device: %lx.\n", (unsigned long)hr);
1305 err = AVERROR_UNKNOWN;
1308 IDirect3DDeviceManager9_CloseDeviceHandle(src_hwctx->devmgr,
1314 #if HAVE_OPENCL_D3D11
1315 case AV_HWDEVICE_TYPE_D3D11VA:
1317 AVD3D11VADeviceContext *src_hwctx = src_ctx->hwctx;
1318 cl_context_properties props[5] = {
1319 CL_CONTEXT_PLATFORM,
1321 CL_CONTEXT_D3D11_DEVICE_KHR,
1322 (intptr_t)src_hwctx->device,
1325 OpenCLDeviceSelector selector = {
1326 .platform_index = -1,
1328 .context = src_hwctx->device,
1329 .enumerate_platforms = &opencl_enumerate_platforms,
1330 .filter_platform = &opencl_filter_d3d11_platform,
1331 .enumerate_devices = &opencl_enumerate_d3d11_devices,
1332 .filter_device = &opencl_filter_gpu_device,
1335 err = opencl_device_create_internal(hwdev, &selector, props);
1340 #if HAVE_OPENCL_DRM_ARM
1341 case AV_HWDEVICE_TYPE_DRM:
1343 OpenCLDeviceSelector selector = {
1344 .platform_index = -1,
1347 .enumerate_platforms = &opencl_enumerate_platforms,
1348 .filter_platform = &opencl_filter_drm_arm_platform,
1349 .enumerate_devices = &opencl_enumerate_devices,
1350 .filter_device = &opencl_filter_drm_arm_device,
1353 err = opencl_device_create_internal(hwdev, &selector, NULL);
1359 err = AVERROR(ENOSYS);
1366 static int opencl_get_plane_format(enum AVPixelFormat pixfmt,
1367 int plane, int width, int height,
1368 cl_image_format *image_format,
1369 cl_image_desc *image_desc)
1371 const AVPixFmtDescriptor *desc;
1372 const AVComponentDescriptor *comp;
1373 int channels = 0, order = 0, depth = 0, step = 0;
1374 int wsub, hsub, alpha;
1377 if (plane >= AV_NUM_DATA_POINTERS)
1378 return AVERROR(ENOENT);
1380 desc = av_pix_fmt_desc_get(pixfmt);
1382 // Only normal images are allowed.
1383 if (desc->flags & (AV_PIX_FMT_FLAG_BITSTREAM |
1384 AV_PIX_FMT_FLAG_HWACCEL |
1385 AV_PIX_FMT_FLAG_PAL))
1386 return AVERROR(EINVAL);
1388 wsub = 1 << desc->log2_chroma_w;
1389 hsub = 1 << desc->log2_chroma_h;
1390 // Subsampled components must be exact.
1391 if (width & wsub - 1 || height & hsub - 1)
1392 return AVERROR(EINVAL);
1394 for (c = 0; c < desc->nb_components; c++) {
1395 comp = &desc->comp[c];
1396 if (comp->plane != plane)
1398 // The step size must be a power of two.
1399 if (comp->step != 1 && comp->step != 2 &&
1400 comp->step != 4 && comp->step != 8)
1401 return AVERROR(EINVAL);
1402 // The bits in each component must be packed in the
1403 // most-significant-bits of the relevant bytes.
1404 if (comp->shift + comp->depth != 8 &&
1405 comp->shift + comp->depth != 16)
1406 return AVERROR(EINVAL);
1407 // The depth must not vary between components.
1408 if (depth && comp->depth != depth)
1409 return AVERROR(EINVAL);
1410 // If a single data element crosses multiple bytes then
1411 // it must match the native endianness.
1412 if (comp->depth > 8 &&
1413 HAVE_BIGENDIAN == !(desc->flags & AV_PIX_FMT_FLAG_BE))
1414 return AVERROR(EINVAL);
1415 // A single data element must not contain multiple samples
1416 // from the same component.
1417 if (step && comp->step != step)
1418 return AVERROR(EINVAL);
1419 order = order * 10 + c + 1;
1420 depth = comp->depth;
1422 alpha = (desc->flags & AV_PIX_FMT_FLAG_ALPHA &&
1423 c == desc->nb_components - 1);
1427 return AVERROR(ENOENT);
1429 memset(image_format, 0, sizeof(*image_format));
1430 memset(image_desc, 0, sizeof(*image_desc));
1431 image_desc->image_type = CL_MEM_OBJECT_IMAGE2D;
1433 if (plane == 0 || alpha) {
1434 image_desc->image_width = width;
1435 image_desc->image_height = height;
1436 image_desc->image_row_pitch = step * width;
1438 image_desc->image_width = width / wsub;
1439 image_desc->image_height = height / hsub;
1440 image_desc->image_row_pitch = step * width / wsub;
1444 image_format->image_channel_data_type = CL_UNORM_INT8;
1447 image_format->image_channel_data_type = CL_UNORM_INT16;
1449 return AVERROR(EINVAL);
1452 #define CHANNEL_ORDER(order, type) \
1453 case order: image_format->image_channel_order = type; break;
1455 CHANNEL_ORDER(1, CL_R);
1456 CHANNEL_ORDER(2, CL_R);
1457 CHANNEL_ORDER(3, CL_R);
1458 CHANNEL_ORDER(4, CL_R);
1459 CHANNEL_ORDER(12, CL_RG);
1460 CHANNEL_ORDER(23, CL_RG);
1461 CHANNEL_ORDER(1234, CL_RGBA);
1462 CHANNEL_ORDER(3214, CL_BGRA);
1463 CHANNEL_ORDER(4123, CL_ARGB);
1465 CHANNEL_ORDER(4321, CL_ABGR);
1468 return AVERROR(EINVAL);
1470 #undef CHANNEL_ORDER
1475 static int opencl_frames_get_constraints(AVHWDeviceContext *hwdev,
1476 const void *hwconfig,
1477 AVHWFramesConstraints *constraints)
1479 AVOpenCLDeviceContext *hwctx = hwdev->hwctx;
1480 cl_uint nb_image_formats;
1481 cl_image_format *image_formats = NULL;
1483 enum AVPixelFormat pix_fmt;
1484 int err, pix_fmts_found;
1485 size_t max_width, max_height;
1487 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_IMAGE2D_MAX_WIDTH,
1488 sizeof(max_width), &max_width, NULL);
1489 if (cle != CL_SUCCESS) {
1490 av_log(hwdev, AV_LOG_ERROR, "Failed to query maximum "
1491 "supported image width: %d.\n", cle);
1493 constraints->max_width = max_width;
1495 cle = clGetDeviceInfo(hwctx->device_id, CL_DEVICE_IMAGE2D_MAX_HEIGHT,
1496 sizeof(max_height), &max_height, NULL);
1497 if (cle != CL_SUCCESS) {
1498 av_log(hwdev, AV_LOG_ERROR, "Failed to query maximum "
1499 "supported image height: %d.\n", cle);
1501 constraints->max_height = max_height;
1503 av_log(hwdev, AV_LOG_DEBUG, "Maximum supported image size %dx%d.\n",
1504 constraints->max_width, constraints->max_height);
1506 cle = clGetSupportedImageFormats(hwctx->context,
1508 CL_MEM_OBJECT_IMAGE2D,
1509 0, NULL, &nb_image_formats);
1510 if (cle != CL_SUCCESS) {
1511 av_log(hwdev, AV_LOG_ERROR, "Failed to query supported "
1512 "image formats: %d.\n", cle);
1513 err = AVERROR(ENOSYS);
1516 if (nb_image_formats == 0) {
1517 av_log(hwdev, AV_LOG_ERROR, "No image support in OpenCL "
1518 "driver (zero supported image formats).\n");
1519 err = AVERROR(ENOSYS);
1524 av_malloc_array(nb_image_formats, sizeof(*image_formats));
1525 if (!image_formats) {
1526 err = AVERROR(ENOMEM);
1530 cle = clGetSupportedImageFormats(hwctx->context,
1532 CL_MEM_OBJECT_IMAGE2D,
1534 image_formats, NULL);
1535 if (cle != CL_SUCCESS) {
1536 av_log(hwdev, AV_LOG_ERROR, "Failed to query supported "
1537 "image formats: %d.\n", cle);
1538 err = AVERROR(ENOSYS);
1543 for (pix_fmt = 0; pix_fmt < AV_PIX_FMT_NB; pix_fmt++) {
1544 cl_image_format image_format;
1545 cl_image_desc image_desc;
1548 for (plane = 0;; plane++) {
1549 err = opencl_get_plane_format(pix_fmt, plane, 0, 0,
1555 for (i = 0; i < nb_image_formats; i++) {
1556 if (image_formats[i].image_channel_order ==
1557 image_format.image_channel_order &&
1558 image_formats[i].image_channel_data_type ==
1559 image_format.image_channel_data_type)
1562 if (i == nb_image_formats) {
1563 err = AVERROR(EINVAL);
1567 if (err != AVERROR(ENOENT))
1570 av_log(hwdev, AV_LOG_DEBUG, "Format %s supported.\n",
1571 av_get_pix_fmt_name(pix_fmt));
1573 err = av_reallocp_array(&constraints->valid_sw_formats,
1575 sizeof(*constraints->valid_sw_formats));
1578 constraints->valid_sw_formats[pix_fmts_found] = pix_fmt;
1579 constraints->valid_sw_formats[pix_fmts_found + 1] =
1584 av_freep(&image_formats);
1586 constraints->valid_hw_formats =
1587 av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
1588 if (!constraints->valid_hw_formats) {
1589 err = AVERROR(ENOMEM);
1592 constraints->valid_hw_formats[0] = AV_PIX_FMT_OPENCL;
1593 constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
1598 av_freep(&image_formats);
1602 static void opencl_pool_free(void *opaque, uint8_t *data)
1604 AVHWFramesContext *hwfc = opaque;
1605 AVOpenCLFrameDescriptor *desc = (AVOpenCLFrameDescriptor*)data;
1609 for (p = 0; p < desc->nb_planes; p++) {
1610 cle = clReleaseMemObject(desc->planes[p]);
1611 if (cle != CL_SUCCESS) {
1612 av_log(hwfc, AV_LOG_ERROR, "Failed to release plane %d: "
1620 static AVBufferRef *opencl_pool_alloc(void *opaque, int size)
1622 AVHWFramesContext *hwfc = opaque;
1623 AVOpenCLDeviceContext *hwctx = hwfc->device_ctx->hwctx;
1624 AVOpenCLFrameDescriptor *desc;
1627 cl_image_format image_format;
1628 cl_image_desc image_desc;
1632 desc = av_mallocz(sizeof(*desc));
1637 err = opencl_get_plane_format(hwfc->sw_format, p,
1638 hwfc->width, hwfc->height,
1639 &image_format, &image_desc);
1640 if (err == AVERROR(ENOENT))
1645 // For generic image objects, the pitch is determined by the
1647 image_desc.image_row_pitch = 0;
1649 image = clCreateImage(hwctx->context, CL_MEM_READ_WRITE,
1650 &image_format, &image_desc, NULL, &cle);
1652 av_log(hwfc, AV_LOG_ERROR, "Failed to create image for "
1653 "plane %d: %d.\n", p, cle);
1657 desc->planes[p] = image;
1660 desc->nb_planes = p;
1662 ref = av_buffer_create((uint8_t*)desc, sizeof(*desc),
1663 &opencl_pool_free, hwfc, 0);
1670 for (p = 0; desc->planes[p]; p++)
1671 clReleaseMemObject(desc->planes[p]);
1676 static int opencl_frames_init_command_queue(AVHWFramesContext *hwfc)
1678 AVOpenCLFramesContext *hwctx = hwfc->hwctx;
1679 OpenCLDeviceContext *devpriv = hwfc->device_ctx->internal->priv;
1680 OpenCLFramesContext *priv = hwfc->internal->priv;
1683 priv->command_queue = hwctx->command_queue ? hwctx->command_queue
1684 : devpriv->command_queue;
1685 cle = clRetainCommandQueue(priv->command_queue);
1686 if (cle != CL_SUCCESS) {
1687 av_log(hwfc, AV_LOG_ERROR, "Failed to retain frame "
1688 "command queue: %d.\n", cle);
1689 return AVERROR(EIO);
1695 static int opencl_frames_init(AVHWFramesContext *hwfc)
1698 hwfc->internal->pool_internal =
1699 av_buffer_pool_init2(sizeof(cl_mem), hwfc,
1700 &opencl_pool_alloc, NULL);
1701 if (!hwfc->internal->pool_internal)
1702 return AVERROR(ENOMEM);
1705 return opencl_frames_init_command_queue(hwfc);
1708 static void opencl_frames_uninit(AVHWFramesContext *hwfc)
1710 OpenCLFramesContext *priv = hwfc->internal->priv;
1713 #if HAVE_OPENCL_DXVA2 || HAVE_OPENCL_D3D11
1715 for (i = 0; i < priv->nb_mapped_frames; i++) {
1716 AVOpenCLFrameDescriptor *desc = &priv->mapped_frames[i];
1717 for (p = 0; p < desc->nb_planes; p++) {
1718 cle = clReleaseMemObject(desc->planes[p]);
1719 if (cle != CL_SUCCESS) {
1720 av_log(hwfc, AV_LOG_ERROR, "Failed to release mapped "
1721 "frame object (frame %d plane %d): %d.\n",
1726 av_freep(&priv->mapped_frames);
1729 cle = clReleaseCommandQueue(priv->command_queue);
1730 if (cle != CL_SUCCESS) {
1731 av_log(hwfc, AV_LOG_ERROR, "Failed to release frame "
1732 "command queue: %d.\n", cle);
1736 static int opencl_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
1738 AVOpenCLFrameDescriptor *desc;
1741 frame->buf[0] = av_buffer_pool_get(hwfc->pool);
1743 return AVERROR(ENOMEM);
1745 desc = (AVOpenCLFrameDescriptor*)frame->buf[0]->data;
1747 for (p = 0; p < desc->nb_planes; p++)
1748 frame->data[p] = (uint8_t*)desc->planes[p];
1750 frame->format = AV_PIX_FMT_OPENCL;
1751 frame->width = hwfc->width;
1752 frame->height = hwfc->height;
1757 static int opencl_transfer_get_formats(AVHWFramesContext *hwfc,
1758 enum AVHWFrameTransferDirection dir,
1759 enum AVPixelFormat **formats)
1761 enum AVPixelFormat *fmts;
1763 fmts = av_malloc_array(2, sizeof(*fmts));
1765 return AVERROR(ENOMEM);
1767 fmts[0] = hwfc->sw_format;
1768 fmts[1] = AV_PIX_FMT_NONE;
1774 static int opencl_wait_events(AVHWFramesContext *hwfc,
1775 cl_event *events, int nb_events)
1780 cle = clWaitForEvents(nb_events, events);
1781 if (cle != CL_SUCCESS) {
1782 av_log(hwfc, AV_LOG_ERROR, "Failed to wait for event "
1783 "completion: %d.\n", cle);
1784 return AVERROR(EIO);
1787 for (i = 0; i < nb_events; i++) {
1788 cle = clReleaseEvent(events[i]);
1789 if (cle != CL_SUCCESS) {
1790 av_log(hwfc, AV_LOG_ERROR, "Failed to release "
1791 "event: %d.\n", cle);
1798 static int opencl_transfer_data_from(AVHWFramesContext *hwfc,
1799 AVFrame *dst, const AVFrame *src)
1801 OpenCLFramesContext *priv = hwfc->internal->priv;
1802 cl_image_format image_format;
1803 cl_image_desc image_desc;
1805 size_t origin[3] = { 0, 0, 0 };
1807 cl_event events[AV_NUM_DATA_POINTERS];
1810 if (dst->format != hwfc->sw_format)
1811 return AVERROR(EINVAL);
1814 err = opencl_get_plane_format(hwfc->sw_format, p,
1815 src->width, src->height,
1816 &image_format, &image_desc);
1818 if (err == AVERROR(ENOENT))
1823 if (!dst->data[p]) {
1824 av_log(hwfc, AV_LOG_ERROR, "Plane %d missing on "
1825 "destination frame for transfer.\n", p);
1826 err = AVERROR(EINVAL);
1830 region[0] = image_desc.image_width;
1831 region[1] = image_desc.image_height;
1834 cle = clEnqueueReadImage(priv->command_queue,
1835 (cl_mem)src->data[p],
1836 CL_FALSE, origin, region,
1837 dst->linesize[p], 0,
1839 0, NULL, &events[p]);
1840 if (cle != CL_SUCCESS) {
1841 av_log(hwfc, AV_LOG_ERROR, "Failed to enqueue read of "
1842 "OpenCL image plane %d: %d.\n", p, cle);
1848 opencl_wait_events(hwfc, events, p);
1853 static int opencl_transfer_data_to(AVHWFramesContext *hwfc,
1854 AVFrame *dst, const AVFrame *src)
1856 OpenCLFramesContext *priv = hwfc->internal->priv;
1857 cl_image_format image_format;
1858 cl_image_desc image_desc;
1860 size_t origin[3] = { 0, 0, 0 };
1862 cl_event events[AV_NUM_DATA_POINTERS];
1865 if (src->format != hwfc->sw_format)
1866 return AVERROR(EINVAL);
1869 err = opencl_get_plane_format(hwfc->sw_format, p,
1870 src->width, src->height,
1871 &image_format, &image_desc);
1873 if (err == AVERROR(ENOENT))
1878 if (!src->data[p]) {
1879 av_log(hwfc, AV_LOG_ERROR, "Plane %d missing on "
1880 "source frame for transfer.\n", p);
1881 err = AVERROR(EINVAL);
1885 region[0] = image_desc.image_width;
1886 region[1] = image_desc.image_height;
1889 cle = clEnqueueWriteImage(priv->command_queue,
1890 (cl_mem)dst->data[p],
1891 CL_FALSE, origin, region,
1892 src->linesize[p], 0,
1894 0, NULL, &events[p]);
1895 if (cle != CL_SUCCESS) {
1896 av_log(hwfc, AV_LOG_ERROR, "Failed to enqueue write of "
1897 "OpenCL image plane %d: %d.\n", p, cle);
1903 opencl_wait_events(hwfc, events, p);
1908 typedef struct OpenCLMapping {
1909 // The mapped addresses for each plane.
1910 // The destination frame is not available when we unmap, so these
1911 // need to be stored separately.
1912 void *address[AV_NUM_DATA_POINTERS];
1915 static void opencl_unmap_frame(AVHWFramesContext *hwfc,
1916 HWMapDescriptor *hwmap)
1918 OpenCLFramesContext *priv = hwfc->internal->priv;
1919 OpenCLMapping *map = hwmap->priv;
1920 cl_event events[AV_NUM_DATA_POINTERS];
1924 for (p = e = 0; p < FF_ARRAY_ELEMS(map->address); p++) {
1925 if (!map->address[p])
1928 cle = clEnqueueUnmapMemObject(priv->command_queue,
1929 (cl_mem)hwmap->source->data[p],
1931 0, NULL, &events[e]);
1932 if (cle != CL_SUCCESS) {
1933 av_log(hwfc, AV_LOG_ERROR, "Failed to unmap OpenCL "
1934 "image plane %d: %d.\n", p, cle);
1939 opencl_wait_events(hwfc, events, e);
1944 static int opencl_map_frame(AVHWFramesContext *hwfc, AVFrame *dst,
1945 const AVFrame *src, int flags)
1947 OpenCLFramesContext *priv = hwfc->internal->priv;
1948 cl_map_flags map_flags;
1949 cl_image_format image_format;
1950 cl_image_desc image_desc;
1953 size_t origin[3] = { 0, 0, 0 };
1956 cl_event events[AV_NUM_DATA_POINTERS];
1959 av_assert0(hwfc->sw_format == dst->format);
1961 if (flags & AV_HWFRAME_MAP_OVERWRITE &&
1962 !(flags & AV_HWFRAME_MAP_READ)) {
1963 // This is mutually exclusive with the read/write flags, so
1964 // there is no way to map with read here.
1965 map_flags = CL_MAP_WRITE_INVALIDATE_REGION;
1968 if (flags & AV_HWFRAME_MAP_READ)
1969 map_flags |= CL_MAP_READ;
1970 if (flags & AV_HWFRAME_MAP_WRITE)
1971 map_flags |= CL_MAP_WRITE;
1974 map = av_mallocz(sizeof(*map));
1976 return AVERROR(ENOMEM);
1979 err = opencl_get_plane_format(hwfc->sw_format, p,
1980 src->width, src->height,
1981 &image_format, &image_desc);
1982 if (err == AVERROR(ENOENT))
1987 region[0] = image_desc.image_width;
1988 region[1] = image_desc.image_height;
1992 clEnqueueMapImage(priv->command_queue,
1993 (cl_mem)src->data[p],
1994 CL_FALSE, map_flags, origin, region,
1995 &row_pitch, NULL, 0, NULL,
1997 if (!map->address[p]) {
1998 av_log(hwfc, AV_LOG_ERROR, "Failed to map OpenCL "
1999 "image plane %d: %d.\n", p, cle);
2004 dst->data[p] = map->address[p];
2006 av_log(hwfc, AV_LOG_DEBUG, "Map plane %d (%p -> %p).\n",
2007 p, src->data[p], dst->data[p]);
2010 err = opencl_wait_events(hwfc, events, p);
2014 err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
2015 &opencl_unmap_frame, map);
2019 dst->width = src->width;
2020 dst->height = src->height;
2025 for (p = 0; p < AV_NUM_DATA_POINTERS; p++) {
2026 if (!map->address[p])
2028 clEnqueueUnmapMemObject(priv->command_queue,
2029 (cl_mem)src->data[p],
2031 0, NULL, &events[p]);
2034 opencl_wait_events(hwfc, events, p);
2039 #if HAVE_OPENCL_DRM_BEIGNET
2041 typedef struct DRMBeignetToOpenCLMapping {
2043 AVDRMFrameDescriptor *drm_desc;
2045 AVOpenCLFrameDescriptor frame;
2046 } DRMBeignetToOpenCLMapping;
2048 static void opencl_unmap_from_drm_beignet(AVHWFramesContext *dst_fc,
2049 HWMapDescriptor *hwmap)
2051 DRMBeignetToOpenCLMapping *mapping = hwmap->priv;
2055 for (i = 0; i < mapping->frame.nb_planes; i++) {
2056 cle = clReleaseMemObject(mapping->frame.planes[i]);
2057 if (cle != CL_SUCCESS) {
2058 av_log(dst_fc, AV_LOG_ERROR, "Failed to release CL image "
2059 "of plane %d of DRM frame: %d.\n", i, cle);
2066 static int opencl_map_from_drm_beignet(AVHWFramesContext *dst_fc,
2067 AVFrame *dst, const AVFrame *src,
2070 AVOpenCLDeviceContext *hwctx = dst_fc->device_ctx->hwctx;
2071 OpenCLDeviceContext *priv = dst_fc->device_ctx->internal->priv;
2072 DRMBeignetToOpenCLMapping *mapping;
2073 const AVDRMFrameDescriptor *desc;
2077 desc = (const AVDRMFrameDescriptor*)src->data[0];
2079 mapping = av_mallocz(sizeof(*mapping));
2081 return AVERROR(ENOMEM);
2084 for (i = 0; i < desc->nb_layers; i++) {
2085 const AVDRMLayerDescriptor *layer = &desc->layers[i];
2086 for (j = 0; j < layer->nb_planes; j++) {
2087 const AVDRMPlaneDescriptor *plane = &layer->planes[j];
2088 const AVDRMObjectDescriptor *object =
2089 &desc->objects[plane->object_index];
2091 cl_import_image_info_intel image_info = {
2093 .size = object->size,
2094 .type = CL_MEM_OBJECT_IMAGE2D,
2095 .offset = plane->offset,
2096 .row_pitch = plane->pitch,
2098 cl_image_desc image_desc;
2100 err = opencl_get_plane_format(dst_fc->sw_format, p,
2101 src->width, src->height,
2105 av_log(dst_fc, AV_LOG_ERROR, "DRM frame layer %d "
2106 "plane %d is not representable in OpenCL: %d.\n",
2110 image_info.width = image_desc.image_width;
2111 image_info.height = image_desc.image_height;
2113 mapping->frame.planes[p] =
2114 priv->clCreateImageFromFdINTEL(hwctx->context,
2116 if (!mapping->frame.planes[p]) {
2117 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL image "
2118 "from layer %d plane %d of DRM frame: %d.\n",
2124 dst->data[p] = (uint8_t*)mapping->frame.planes[p];
2125 mapping->frame.nb_planes = ++p;
2129 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2130 &opencl_unmap_from_drm_beignet,
2135 dst->width = src->width;
2136 dst->height = src->height;
2141 for (p = 0; p < mapping->frame.nb_planes; p++) {
2142 if (mapping->frame.planes[p])
2143 clReleaseMemObject(mapping->frame.planes[p]);
2149 #if HAVE_OPENCL_VAAPI_BEIGNET
2151 static int opencl_map_from_vaapi(AVHWFramesContext *dst_fc,
2152 AVFrame *dst, const AVFrame *src,
2158 tmp = av_frame_alloc();
2160 return AVERROR(ENOMEM);
2162 tmp->format = AV_PIX_FMT_DRM_PRIME;
2164 err = av_hwframe_map(tmp, src, flags);
2168 err = opencl_map_from_drm_beignet(dst_fc, dst, tmp, flags);
2172 err = ff_hwframe_map_replace(dst, src);
2175 av_frame_free(&tmp);
2179 #endif /* HAVE_OPENCL_VAAPI_BEIGNET */
2180 #endif /* HAVE_OPENCL_DRM_BEIGNET */
2182 static inline cl_mem_flags opencl_mem_flags_for_mapping(int map_flags)
2184 if ((map_flags & AV_HWFRAME_MAP_READ) &&
2185 (map_flags & AV_HWFRAME_MAP_WRITE))
2186 return CL_MEM_READ_WRITE;
2187 else if (map_flags & AV_HWFRAME_MAP_READ)
2188 return CL_MEM_READ_ONLY;
2189 else if (map_flags & AV_HWFRAME_MAP_WRITE)
2190 return CL_MEM_WRITE_ONLY;
2195 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2197 static void opencl_unmap_from_qsv(AVHWFramesContext *dst_fc,
2198 HWMapDescriptor *hwmap)
2200 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2201 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2202 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2207 av_log(dst_fc, AV_LOG_DEBUG, "Unmap QSV/VAAPI surface from OpenCL.\n");
2209 cle = device_priv->clEnqueueReleaseVA_APIMediaSurfacesINTEL(
2210 frames_priv->command_queue, desc->nb_planes, desc->planes,
2212 if (cle != CL_SUCCESS) {
2213 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2214 "handles: %d.\n", cle);
2217 opencl_wait_events(dst_fc, &event, 1);
2219 for (p = 0; p < desc->nb_planes; p++) {
2220 cle = clReleaseMemObject(desc->planes[p]);
2221 if (cle != CL_SUCCESS) {
2222 av_log(dst_fc, AV_LOG_ERROR, "Failed to release CL "
2223 "image of plane %d of QSV/VAAPI surface: %d\n",
2231 static int opencl_map_from_qsv(AVHWFramesContext *dst_fc, AVFrame *dst,
2232 const AVFrame *src, int flags)
2234 AVHWFramesContext *src_fc =
2235 (AVHWFramesContext*)src->hw_frames_ctx->data;
2236 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2237 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2238 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2239 AVOpenCLFrameDescriptor *desc;
2240 VASurfaceID va_surface;
2241 cl_mem_flags cl_flags;
2247 if (src->format == AV_PIX_FMT_QSV) {
2248 mfxFrameSurface1 *mfx_surface = (mfxFrameSurface1*)src->data[3];
2249 va_surface = *(VASurfaceID*)mfx_surface->Data.MemId;
2252 if (src->format == AV_PIX_FMT_VAAPI) {
2253 va_surface = (VASurfaceID)(uintptr_t)src->data[3];
2255 return AVERROR(ENOSYS);
2258 cl_flags = opencl_mem_flags_for_mapping(flags);
2260 return AVERROR(EINVAL);
2262 av_log(src_fc, AV_LOG_DEBUG, "Map QSV/VAAPI surface %#x to "
2263 "OpenCL.\n", va_surface);
2265 desc = av_mallocz(sizeof(*desc));
2267 return AVERROR(ENOMEM);
2269 // The cl_intel_va_api_media_sharing extension only supports NV12
2270 // surfaces, so for now there are always exactly two planes.
2271 desc->nb_planes = 2;
2273 for (p = 0; p < desc->nb_planes; p++) {
2275 device_priv->clCreateFromVA_APIMediaSurfaceINTEL(
2276 dst_dev->context, cl_flags, &va_surface, p, &cle);
2277 if (!desc->planes[p]) {
2278 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2279 "image from plane %d of QSV/VAAPI surface "
2280 "%#x: %d.\n", p, va_surface, cle);
2285 dst->data[p] = (uint8_t*)desc->planes[p];
2288 cle = device_priv->clEnqueueAcquireVA_APIMediaSurfacesINTEL(
2289 frames_priv->command_queue, desc->nb_planes, desc->planes,
2291 if (cle != CL_SUCCESS) {
2292 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2293 "handles: %d.\n", cle);
2298 err = opencl_wait_events(dst_fc, &event, 1);
2302 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2303 &opencl_unmap_from_qsv, desc);
2307 dst->width = src->width;
2308 dst->height = src->height;
2313 for (p = 0; p < desc->nb_planes; p++)
2314 if (desc->planes[p])
2315 clReleaseMemObject(desc->planes[p]);
2322 #if HAVE_OPENCL_DXVA2
2324 static void opencl_unmap_from_dxva2(AVHWFramesContext *dst_fc,
2325 HWMapDescriptor *hwmap)
2327 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2328 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2329 OpenCLFramesContext *frames_priv = dst_fc->device_ctx->internal->priv;
2333 av_log(dst_fc, AV_LOG_DEBUG, "Unmap DXVA2 surface from OpenCL.\n");
2335 cle = device_priv->clEnqueueReleaseDX9MediaSurfacesKHR(
2336 frames_priv->command_queue, desc->nb_planes, desc->planes,
2338 if (cle != CL_SUCCESS) {
2339 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2340 "handle: %d.\n", cle);
2344 opencl_wait_events(dst_fc, &event, 1);
2347 static int opencl_map_from_dxva2(AVHWFramesContext *dst_fc, AVFrame *dst,
2348 const AVFrame *src, int flags)
2350 AVHWFramesContext *src_fc =
2351 (AVHWFramesContext*)src->hw_frames_ctx->data;
2352 AVDXVA2FramesContext *src_hwctx = src_fc->hwctx;
2353 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2354 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2355 AVOpenCLFrameDescriptor *desc;
2360 av_log(dst_fc, AV_LOG_DEBUG, "Map DXVA2 surface %p to "
2361 "OpenCL.\n", src->data[3]);
2363 for (i = 0; i < src_hwctx->nb_surfaces; i++) {
2364 if (src_hwctx->surfaces[i] == (IDirect3DSurface9*)src->data[3])
2367 if (i >= src_hwctx->nb_surfaces) {
2368 av_log(dst_fc, AV_LOG_ERROR, "Trying to map from a surface which "
2369 "is not in the mapped frames context.\n");
2370 return AVERROR(EINVAL);
2373 desc = &frames_priv->mapped_frames[i];
2375 cle = device_priv->clEnqueueAcquireDX9MediaSurfacesKHR(
2376 frames_priv->command_queue, desc->nb_planes, desc->planes,
2378 if (cle != CL_SUCCESS) {
2379 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2380 "handle: %d.\n", cle);
2381 return AVERROR(EIO);
2384 err = opencl_wait_events(dst_fc, &event, 1);
2388 for (i = 0; i < desc->nb_planes; i++)
2389 dst->data[i] = (uint8_t*)desc->planes[i];
2391 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2392 &opencl_unmap_from_dxva2, desc);
2396 dst->width = src->width;
2397 dst->height = src->height;
2402 cle = device_priv->clEnqueueReleaseDX9MediaSurfacesKHR(
2403 frames_priv->command_queue, desc->nb_planes, desc->planes,
2405 if (cle == CL_SUCCESS)
2406 opencl_wait_events(dst_fc, &event, 1);
2410 static int opencl_frames_derive_from_dxva2(AVHWFramesContext *dst_fc,
2411 AVHWFramesContext *src_fc, int flags)
2413 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2414 AVDXVA2FramesContext *src_hwctx = src_fc->hwctx;
2415 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2416 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2417 cl_mem_flags cl_flags;
2419 int err, i, p, nb_planes;
2421 if (src_fc->sw_format != AV_PIX_FMT_NV12) {
2422 av_log(dst_fc, AV_LOG_ERROR, "Only NV12 textures are supported "
2423 "for DXVA2 to OpenCL mapping.\n");
2424 return AVERROR(EINVAL);
2428 if (src_fc->initial_pool_size == 0) {
2429 av_log(dst_fc, AV_LOG_ERROR, "Only fixed-size pools are supported "
2430 "for DXVA2 to OpenCL mapping.\n");
2431 return AVERROR(EINVAL);
2434 cl_flags = opencl_mem_flags_for_mapping(flags);
2436 return AVERROR(EINVAL);
2438 frames_priv->nb_mapped_frames = src_hwctx->nb_surfaces;
2440 frames_priv->mapped_frames =
2441 av_mallocz_array(frames_priv->nb_mapped_frames,
2442 sizeof(*frames_priv->mapped_frames));
2443 if (!frames_priv->mapped_frames)
2444 return AVERROR(ENOMEM);
2446 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2447 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2448 cl_dx9_surface_info_khr surface_info = {
2449 .resource = src_hwctx->surfaces[i],
2450 .shared_handle = NULL,
2452 desc->nb_planes = nb_planes;
2453 for (p = 0; p < nb_planes; p++) {
2455 device_priv->clCreateFromDX9MediaSurfaceKHR(
2456 dst_dev->context, cl_flags,
2457 device_priv->dx9_media_adapter_type,
2458 &surface_info, p, &cle);
2459 if (!desc->planes[p]) {
2460 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2461 "image from plane %d of DXVA2 surface %d: %d.\n",
2472 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2473 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2474 for (p = 0; p < desc->nb_planes; p++) {
2475 if (desc->planes[p])
2476 clReleaseMemObject(desc->planes[p]);
2479 av_freep(&frames_priv->mapped_frames);
2480 frames_priv->nb_mapped_frames = 0;
2486 #if HAVE_OPENCL_D3D11
2488 static void opencl_unmap_from_d3d11(AVHWFramesContext *dst_fc,
2489 HWMapDescriptor *hwmap)
2491 AVOpenCLFrameDescriptor *desc = hwmap->priv;
2492 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2493 OpenCLFramesContext *frames_priv = dst_fc->device_ctx->internal->priv;
2497 cle = device_priv->clEnqueueReleaseD3D11ObjectsKHR(
2498 frames_priv->command_queue, desc->nb_planes, desc->planes,
2500 if (cle != CL_SUCCESS) {
2501 av_log(dst_fc, AV_LOG_ERROR, "Failed to release surface "
2502 "handle: %d.\n", cle);
2505 opencl_wait_events(dst_fc, &event, 1);
2508 static int opencl_map_from_d3d11(AVHWFramesContext *dst_fc, AVFrame *dst,
2509 const AVFrame *src, int flags)
2511 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2512 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2513 AVOpenCLFrameDescriptor *desc;
2518 index = (intptr_t)src->data[1];
2519 if (index >= frames_priv->nb_mapped_frames) {
2520 av_log(dst_fc, AV_LOG_ERROR, "Texture array index out of range for "
2521 "mapping: %d >= %d.\n", index, frames_priv->nb_mapped_frames);
2522 return AVERROR(EINVAL);
2525 av_log(dst_fc, AV_LOG_DEBUG, "Map D3D11 texture %d to OpenCL.\n",
2528 desc = &frames_priv->mapped_frames[index];
2530 cle = device_priv->clEnqueueAcquireD3D11ObjectsKHR(
2531 frames_priv->command_queue, desc->nb_planes, desc->planes,
2533 if (cle != CL_SUCCESS) {
2534 av_log(dst_fc, AV_LOG_ERROR, "Failed to acquire surface "
2535 "handle: %d.\n", cle);
2536 return AVERROR(EIO);
2539 err = opencl_wait_events(dst_fc, &event, 1);
2543 for (i = 0; i < desc->nb_planes; i++)
2544 dst->data[i] = (uint8_t*)desc->planes[i];
2546 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2547 &opencl_unmap_from_d3d11, desc);
2551 dst->width = src->width;
2552 dst->height = src->height;
2557 cle = device_priv->clEnqueueReleaseD3D11ObjectsKHR(
2558 frames_priv->command_queue, desc->nb_planes, desc->planes,
2560 if (cle == CL_SUCCESS)
2561 opencl_wait_events(dst_fc, &event, 1);
2565 static int opencl_frames_derive_from_d3d11(AVHWFramesContext *dst_fc,
2566 AVHWFramesContext *src_fc, int flags)
2568 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2569 AVD3D11VAFramesContext *src_hwctx = src_fc->hwctx;
2570 OpenCLDeviceContext *device_priv = dst_fc->device_ctx->internal->priv;
2571 OpenCLFramesContext *frames_priv = dst_fc->internal->priv;
2572 cl_mem_flags cl_flags;
2574 int err, i, p, nb_planes;
2576 if (src_fc->sw_format != AV_PIX_FMT_NV12) {
2577 av_log(dst_fc, AV_LOG_ERROR, "Only NV12 textures are supported "
2578 "for D3D11 to OpenCL mapping.\n");
2579 return AVERROR(EINVAL);
2583 if (src_fc->initial_pool_size == 0) {
2584 av_log(dst_fc, AV_LOG_ERROR, "Only fixed-size pools are supported "
2585 "for D3D11 to OpenCL mapping.\n");
2586 return AVERROR(EINVAL);
2589 cl_flags = opencl_mem_flags_for_mapping(flags);
2591 return AVERROR(EINVAL);
2593 frames_priv->nb_mapped_frames = src_fc->initial_pool_size;
2595 frames_priv->mapped_frames =
2596 av_mallocz_array(frames_priv->nb_mapped_frames,
2597 sizeof(*frames_priv->mapped_frames));
2598 if (!frames_priv->mapped_frames)
2599 return AVERROR(ENOMEM);
2601 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2602 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2603 desc->nb_planes = nb_planes;
2604 for (p = 0; p < nb_planes; p++) {
2605 UINT subresource = 2 * i + p;
2608 device_priv->clCreateFromD3D11Texture2DKHR(
2609 dst_dev->context, cl_flags, src_hwctx->texture,
2611 if (!desc->planes[p]) {
2612 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL "
2613 "image from plane %d of D3D texture "
2614 "index %d (subresource %u): %d.\n",
2615 p, i, (unsigned int)subresource, cle);
2625 for (i = 0; i < frames_priv->nb_mapped_frames; i++) {
2626 AVOpenCLFrameDescriptor *desc = &frames_priv->mapped_frames[i];
2627 for (p = 0; p < desc->nb_planes; p++) {
2628 if (desc->planes[p])
2629 clReleaseMemObject(desc->planes[p]);
2632 av_freep(&frames_priv->mapped_frames);
2633 frames_priv->nb_mapped_frames = 0;
2639 #if HAVE_OPENCL_DRM_ARM
2641 typedef struct DRMARMtoOpenCLMapping {
2643 cl_mem object_buffers[AV_DRM_MAX_PLANES];
2645 cl_mem plane_images[AV_DRM_MAX_PLANES];
2646 } DRMARMtoOpenCLMapping;
2648 static void opencl_unmap_from_drm_arm(AVHWFramesContext *dst_fc,
2649 HWMapDescriptor *hwmap)
2651 DRMARMtoOpenCLMapping *mapping = hwmap->priv;
2654 for (i = 0; i < mapping->nb_planes; i++)
2655 clReleaseMemObject(mapping->plane_images[i]);
2657 for (i = 0; i < mapping->nb_objects; i++)
2658 clReleaseMemObject(mapping->object_buffers[i]);
2663 static int opencl_map_from_drm_arm(AVHWFramesContext *dst_fc, AVFrame *dst,
2664 const AVFrame *src, int flags)
2666 AVHWFramesContext *src_fc =
2667 (AVHWFramesContext*)src->hw_frames_ctx->data;
2668 AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
2669 const AVDRMFrameDescriptor *desc;
2670 DRMARMtoOpenCLMapping *mapping = NULL;
2671 cl_mem_flags cl_flags;
2672 const cl_import_properties_arm props[3] = {
2673 CL_IMPORT_TYPE_ARM, CL_IMPORT_TYPE_DMA_BUF_ARM, 0,
2678 desc = (const AVDRMFrameDescriptor*)src->data[0];
2680 cl_flags = opencl_mem_flags_for_mapping(flags);
2682 return AVERROR(EINVAL);
2684 mapping = av_mallocz(sizeof(*mapping));
2686 return AVERROR(ENOMEM);
2688 mapping->nb_objects = desc->nb_objects;
2689 for (i = 0; i < desc->nb_objects; i++) {
2690 int fd = desc->objects[i].fd;
2692 av_log(dst_fc, AV_LOG_DEBUG, "Map DRM PRIME fd %d to OpenCL.\n", fd);
2694 if (desc->objects[i].format_modifier) {
2695 av_log(dst_fc, AV_LOG_DEBUG, "Warning: object %d fd %d has "
2696 "nonzero format modifier %"PRId64", result may not "
2697 "be as expected.\n", i, fd,
2698 desc->objects[i].format_modifier);
2701 mapping->object_buffers[i] =
2702 clImportMemoryARM(dst_dev->context, cl_flags, props,
2703 &fd, desc->objects[i].size, &cle);
2704 if (!mapping->object_buffers[i]) {
2705 av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL buffer "
2706 "from object %d (fd %d, size %"SIZE_SPECIFIER") of DRM frame: %d.\n",
2707 i, fd, desc->objects[i].size, cle);
2713 mapping->nb_planes = 0;
2714 for (i = 0; i < desc->nb_layers; i++) {
2715 const AVDRMLayerDescriptor *layer = &desc->layers[i];
2717 for (j = 0; j < layer->nb_planes; j++) {
2718 const AVDRMPlaneDescriptor *plane = &layer->planes[j];
2719 cl_mem plane_buffer;
2720 cl_image_format image_format;
2721 cl_image_desc image_desc;
2722 cl_buffer_region region;
2723 int p = mapping->nb_planes;
2725 err = opencl_get_plane_format(src_fc->sw_format, p,
2726 src_fc->width, src_fc->height,
2727 &image_format, &image_desc);
2729 av_log(dst_fc, AV_LOG_ERROR, "Invalid plane %d (DRM "
2730 "layer %d plane %d): %d.\n", p, i, j, err);
2734 region.origin = plane->offset;
2735 region.size = image_desc.image_row_pitch *
2736 image_desc.image_height;
2739 clCreateSubBuffer(mapping->object_buffers[plane->object_index],
2741 CL_BUFFER_CREATE_TYPE_REGION,
2743 if (!plane_buffer) {
2744 av_log(dst_fc, AV_LOG_ERROR, "Failed to create sub-buffer "
2745 "for plane %d: %d.\n", p, cle);
2750 image_desc.buffer = plane_buffer;
2752 mapping->plane_images[p] =
2753 clCreateImage(dst_dev->context, cl_flags,
2754 &image_format, &image_desc, NULL, &cle);
2756 // Unreference the sub-buffer immediately - we don't need it
2757 // directly and a reference is held by the image.
2758 clReleaseMemObject(plane_buffer);
2760 if (!mapping->plane_images[p]) {
2761 av_log(dst_fc, AV_LOG_ERROR, "Failed to create image "
2762 "for plane %d: %d.\n", p, cle);
2767 ++mapping->nb_planes;
2771 for (i = 0; i < mapping->nb_planes; i++)
2772 dst->data[i] = (uint8_t*)mapping->plane_images[i];
2774 err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
2775 &opencl_unmap_from_drm_arm, mapping);
2779 dst->width = src->width;
2780 dst->height = src->height;
2785 for (i = 0; i < mapping->nb_planes; i++) {
2786 clReleaseMemObject(mapping->plane_images[i]);
2788 for (i = 0; i < mapping->nb_objects; i++) {
2789 if (mapping->object_buffers[i])
2790 clReleaseMemObject(mapping->object_buffers[i]);
2798 static int opencl_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
2799 const AVFrame *src, int flags)
2801 av_assert0(src->format == AV_PIX_FMT_OPENCL);
2802 if (hwfc->sw_format != dst->format)
2803 return AVERROR(ENOSYS);
2804 return opencl_map_frame(hwfc, dst, src, flags);
2807 static int opencl_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
2808 const AVFrame *src, int flags)
2810 av_unused OpenCLDeviceContext *priv = hwfc->device_ctx->internal->priv;
2811 av_assert0(dst->format == AV_PIX_FMT_OPENCL);
2812 switch (src->format) {
2813 #if HAVE_OPENCL_DRM_BEIGNET
2814 case AV_PIX_FMT_DRM_PRIME:
2815 if (priv->beignet_drm_mapping_usable)
2816 return opencl_map_from_drm_beignet(hwfc, dst, src, flags);
2818 #if HAVE_OPENCL_VAAPI_BEIGNET
2819 case AV_PIX_FMT_VAAPI:
2820 if (priv->beignet_drm_mapping_usable)
2821 return opencl_map_from_vaapi(hwfc, dst, src, flags);
2823 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2824 case AV_PIX_FMT_QSV:
2825 case AV_PIX_FMT_VAAPI:
2826 if (priv->qsv_mapping_usable)
2827 return opencl_map_from_qsv(hwfc, dst, src, flags);
2829 #if HAVE_OPENCL_DXVA2
2830 case AV_PIX_FMT_DXVA2_VLD:
2831 if (priv->dxva2_mapping_usable)
2832 return opencl_map_from_dxva2(hwfc, dst, src, flags);
2834 #if HAVE_OPENCL_D3D11
2835 case AV_PIX_FMT_D3D11:
2836 if (priv->d3d11_mapping_usable)
2837 return opencl_map_from_d3d11(hwfc, dst, src, flags);
2839 #if HAVE_OPENCL_DRM_ARM
2840 case AV_PIX_FMT_DRM_PRIME:
2841 if (priv->drm_arm_mapping_usable)
2842 return opencl_map_from_drm_arm(hwfc, dst, src, flags);
2845 return AVERROR(ENOSYS);
2848 static int opencl_frames_derive_to(AVHWFramesContext *dst_fc,
2849 AVHWFramesContext *src_fc, int flags)
2851 av_unused OpenCLDeviceContext *priv = dst_fc->device_ctx->internal->priv;
2852 switch (src_fc->device_ctx->type) {
2853 #if HAVE_OPENCL_DRM_BEIGNET
2854 case AV_HWDEVICE_TYPE_DRM:
2855 if (!priv->beignet_drm_mapping_usable)
2856 return AVERROR(ENOSYS);
2859 #if HAVE_OPENCL_VAAPI_BEIGNET
2860 case AV_HWDEVICE_TYPE_VAAPI:
2861 if (!priv->beignet_drm_mapping_usable)
2862 return AVERROR(ENOSYS);
2865 #if HAVE_OPENCL_VAAPI_INTEL_MEDIA
2866 case AV_HWDEVICE_TYPE_QSV:
2867 case AV_HWDEVICE_TYPE_VAAPI:
2868 if (!priv->qsv_mapping_usable)
2869 return AVERROR(ENOSYS);
2872 #if HAVE_OPENCL_DXVA2
2873 case AV_HWDEVICE_TYPE_DXVA2:
2874 if (!priv->dxva2_mapping_usable)
2875 return AVERROR(ENOSYS);
2878 err = opencl_frames_derive_from_dxva2(dst_fc, src_fc, flags);
2884 #if HAVE_OPENCL_D3D11
2885 case AV_HWDEVICE_TYPE_D3D11VA:
2886 if (!priv->d3d11_mapping_usable)
2887 return AVERROR(ENOSYS);
2890 err = opencl_frames_derive_from_d3d11(dst_fc, src_fc, flags);
2896 #if HAVE_OPENCL_DRM_ARM
2897 case AV_HWDEVICE_TYPE_DRM:
2898 if (!priv->drm_arm_mapping_usable)
2899 return AVERROR(ENOSYS);
2903 return AVERROR(ENOSYS);
2905 return opencl_frames_init_command_queue(dst_fc);
2908 const HWContextType ff_hwcontext_type_opencl = {
2909 .type = AV_HWDEVICE_TYPE_OPENCL,
2912 .device_hwctx_size = sizeof(AVOpenCLDeviceContext),
2913 .device_priv_size = sizeof(OpenCLDeviceContext),
2914 .frames_hwctx_size = sizeof(AVOpenCLFramesContext),
2915 .frames_priv_size = sizeof(OpenCLFramesContext),
2917 .device_create = &opencl_device_create,
2918 .device_derive = &opencl_device_derive,
2919 .device_init = &opencl_device_init,
2920 .device_uninit = &opencl_device_uninit,
2922 .frames_get_constraints = &opencl_frames_get_constraints,
2923 .frames_init = &opencl_frames_init,
2924 .frames_uninit = &opencl_frames_uninit,
2925 .frames_get_buffer = &opencl_get_buffer,
2927 .transfer_get_formats = &opencl_transfer_get_formats,
2928 .transfer_data_to = &opencl_transfer_data_to,
2929 .transfer_data_from = &opencl_transfer_data_from,
2931 .map_from = &opencl_map_from,
2932 .map_to = &opencl_map_to,
2933 .frames_derive_to = &opencl_frames_derive_to,
2935 .pix_fmts = (const enum AVPixelFormat[]) {