2 * This file is part of FFmpeg.
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "hwcontext.h"
22 #include "hwcontext_internal.h"
23 #include "hwcontext_cuda_internal.h"
25 #include "hwcontext_vulkan.h"
27 #include "cuda_check.h"
33 typedef struct CUDAFramesContext {
34 int shift_width, shift_height;
38 static const enum AVPixelFormat supported_formats[] = {
53 #define CHECK_CU(x) FF_CUDA_CHECK_DL(device_ctx, cu, x)
55 static int cuda_frames_get_constraints(AVHWDeviceContext *ctx,
57 AVHWFramesConstraints *constraints)
61 constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
62 sizeof(*constraints->valid_sw_formats));
63 if (!constraints->valid_sw_formats)
64 return AVERROR(ENOMEM);
66 for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
67 constraints->valid_sw_formats[i] = supported_formats[i];
68 constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE;
70 constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
71 if (!constraints->valid_hw_formats)
72 return AVERROR(ENOMEM);
74 constraints->valid_hw_formats[0] = AV_PIX_FMT_CUDA;
75 constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
80 static void cuda_buffer_free(void *opaque, uint8_t *data)
82 AVHWFramesContext *ctx = opaque;
83 AVHWDeviceContext *device_ctx = ctx->device_ctx;
84 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
85 CudaFunctions *cu = hwctx->internal->cuda_dl;
89 CHECK_CU(cu->cuCtxPushCurrent(hwctx->cuda_ctx));
91 CHECK_CU(cu->cuMemFree((CUdeviceptr)data));
93 CHECK_CU(cu->cuCtxPopCurrent(&dummy));
96 static AVBufferRef *cuda_pool_alloc(void *opaque, buffer_size_t size)
98 AVHWFramesContext *ctx = opaque;
99 AVHWDeviceContext *device_ctx = ctx->device_ctx;
100 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
101 CudaFunctions *cu = hwctx->internal->cuda_dl;
103 AVBufferRef *ret = NULL;
104 CUcontext dummy = NULL;
108 err = CHECK_CU(cu->cuCtxPushCurrent(hwctx->cuda_ctx));
112 err = CHECK_CU(cu->cuMemAlloc(&data, size));
116 ret = av_buffer_create((uint8_t*)data, size, cuda_buffer_free, ctx, 0);
118 CHECK_CU(cu->cuMemFree(data));
123 CHECK_CU(cu->cuCtxPopCurrent(&dummy));
127 static int cuda_frames_init(AVHWFramesContext *ctx)
129 AVHWDeviceContext *device_ctx = ctx->device_ctx;
130 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
131 CUDAFramesContext *priv = ctx->internal->priv;
132 CudaFunctions *cu = hwctx->internal->cuda_dl;
135 for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
136 if (ctx->sw_format == supported_formats[i])
139 if (i == FF_ARRAY_ELEMS(supported_formats)) {
140 av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
141 av_get_pix_fmt_name(ctx->sw_format));
142 return AVERROR(ENOSYS);
145 err = CHECK_CU(cu->cuDeviceGetAttribute(&priv->tex_alignment,
146 14 /* CU_DEVICE_ATTRIBUTE_TEXTURE_ALIGNMENT */,
147 hwctx->internal->cuda_device));
151 av_log(ctx, AV_LOG_DEBUG, "CUDA texture alignment: %d\n", priv->tex_alignment);
153 // YUV420P is a special case.
154 // Since nvenc expects the U/V planes to have half the linesize of the Y plane
155 // alignment has to be doubled to ensure the U/V planes still end up aligned.
156 if (ctx->sw_format == AV_PIX_FMT_YUV420P)
157 priv->tex_alignment *= 2;
159 av_pix_fmt_get_chroma_sub_sample(ctx->sw_format, &priv->shift_width, &priv->shift_height);
162 int size = av_image_get_buffer_size(ctx->sw_format, ctx->width, ctx->height, priv->tex_alignment);
166 ctx->internal->pool_internal = av_buffer_pool_init2(size, ctx, cuda_pool_alloc, NULL);
167 if (!ctx->internal->pool_internal)
168 return AVERROR(ENOMEM);
174 static int cuda_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
176 CUDAFramesContext *priv = ctx->internal->priv;
179 frame->buf[0] = av_buffer_pool_get(ctx->pool);
181 return AVERROR(ENOMEM);
183 res = av_image_fill_arrays(frame->data, frame->linesize, frame->buf[0]->data,
184 ctx->sw_format, ctx->width, ctx->height, priv->tex_alignment);
188 // YUV420P is a special case.
189 // Nvenc expects the U/V planes in swapped order from how ffmpeg expects them, also chroma is half-aligned
190 if (ctx->sw_format == AV_PIX_FMT_YUV420P) {
191 frame->linesize[1] = frame->linesize[2] = frame->linesize[0] / 2;
192 frame->data[2] = frame->data[1];
193 frame->data[1] = frame->data[2] + frame->linesize[2] * (ctx->height / 2);
196 frame->format = AV_PIX_FMT_CUDA;
197 frame->width = ctx->width;
198 frame->height = ctx->height;
203 static int cuda_transfer_get_formats(AVHWFramesContext *ctx,
204 enum AVHWFrameTransferDirection dir,
205 enum AVPixelFormat **formats)
207 enum AVPixelFormat *fmts;
209 fmts = av_malloc_array(2, sizeof(*fmts));
211 return AVERROR(ENOMEM);
213 fmts[0] = ctx->sw_format;
214 fmts[1] = AV_PIX_FMT_NONE;
221 static int cuda_transfer_data(AVHWFramesContext *ctx, AVFrame *dst,
224 CUDAFramesContext *priv = ctx->internal->priv;
225 AVHWDeviceContext *device_ctx = ctx->device_ctx;
226 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
227 CudaFunctions *cu = hwctx->internal->cuda_dl;
232 if ((src->hw_frames_ctx && ((AVHWFramesContext*)src->hw_frames_ctx->data)->format != AV_PIX_FMT_CUDA) ||
233 (dst->hw_frames_ctx && ((AVHWFramesContext*)dst->hw_frames_ctx->data)->format != AV_PIX_FMT_CUDA))
234 return AVERROR(ENOSYS);
236 ret = CHECK_CU(cu->cuCtxPushCurrent(hwctx->cuda_ctx));
240 for (i = 0; i < FF_ARRAY_ELEMS(src->data) && src->data[i]; i++) {
241 CUDA_MEMCPY2D cpy = {
242 .srcPitch = src->linesize[i],
243 .dstPitch = dst->linesize[i],
244 .WidthInBytes = FFMIN(src->linesize[i], dst->linesize[i]),
245 .Height = src->height >> ((i == 0 || i == 3) ? 0 : priv->shift_height),
248 if (src->hw_frames_ctx) {
249 cpy.srcMemoryType = CU_MEMORYTYPE_DEVICE;
250 cpy.srcDevice = (CUdeviceptr)src->data[i];
252 cpy.srcMemoryType = CU_MEMORYTYPE_HOST;
253 cpy.srcHost = src->data[i];
256 if (dst->hw_frames_ctx) {
257 cpy.dstMemoryType = CU_MEMORYTYPE_DEVICE;
258 cpy.dstDevice = (CUdeviceptr)dst->data[i];
260 cpy.dstMemoryType = CU_MEMORYTYPE_HOST;
261 cpy.dstHost = dst->data[i];
264 ret = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, hwctx->stream));
269 if (!dst->hw_frames_ctx) {
270 ret = CHECK_CU(cu->cuStreamSynchronize(hwctx->stream));
276 CHECK_CU(cu->cuCtxPopCurrent(&dummy));
281 static void cuda_device_uninit(AVHWDeviceContext *device_ctx)
283 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
285 if (hwctx->internal) {
286 CudaFunctions *cu = hwctx->internal->cuda_dl;
288 if (hwctx->internal->is_allocated && hwctx->cuda_ctx) {
289 if (hwctx->internal->flags & AV_CUDA_USE_PRIMARY_CONTEXT)
290 CHECK_CU(cu->cuDevicePrimaryCtxRelease(hwctx->internal->cuda_device));
292 CHECK_CU(cu->cuCtxDestroy(hwctx->cuda_ctx));
294 hwctx->cuda_ctx = NULL;
297 cuda_free_functions(&hwctx->internal->cuda_dl);
300 av_freep(&hwctx->internal);
303 static int cuda_device_init(AVHWDeviceContext *ctx)
305 AVCUDADeviceContext *hwctx = ctx->hwctx;
308 if (!hwctx->internal) {
309 hwctx->internal = av_mallocz(sizeof(*hwctx->internal));
310 if (!hwctx->internal)
311 return AVERROR(ENOMEM);
314 if (!hwctx->internal->cuda_dl) {
315 ret = cuda_load_functions(&hwctx->internal->cuda_dl, ctx);
317 av_log(ctx, AV_LOG_ERROR, "Could not dynamically load CUDA\n");
325 cuda_device_uninit(ctx);
329 static int cuda_context_init(AVHWDeviceContext *device_ctx, int flags) {
330 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
333 int ret, dev_active = 0;
334 unsigned int dev_flags = 0;
336 const unsigned int desired_flags = CU_CTX_SCHED_BLOCKING_SYNC;
338 cu = hwctx->internal->cuda_dl;
340 hwctx->internal->flags = flags;
342 if (flags & AV_CUDA_USE_PRIMARY_CONTEXT) {
343 ret = CHECK_CU(cu->cuDevicePrimaryCtxGetState(hwctx->internal->cuda_device,
344 &dev_flags, &dev_active));
348 if (dev_active && dev_flags != desired_flags) {
349 av_log(device_ctx, AV_LOG_ERROR, "Primary context already active with incompatible flags.\n");
350 return AVERROR(ENOTSUP);
351 } else if (dev_flags != desired_flags) {
352 ret = CHECK_CU(cu->cuDevicePrimaryCtxSetFlags(hwctx->internal->cuda_device,
358 ret = CHECK_CU(cu->cuDevicePrimaryCtxRetain(&hwctx->cuda_ctx,
359 hwctx->internal->cuda_device));
363 ret = CHECK_CU(cu->cuCtxCreate(&hwctx->cuda_ctx, desired_flags,
364 hwctx->internal->cuda_device));
368 CHECK_CU(cu->cuCtxPopCurrent(&dummy));
371 hwctx->internal->is_allocated = 1;
373 // Setting stream to NULL will make functions automatically use the default CUstream
374 hwctx->stream = NULL;
379 static int cuda_device_create(AVHWDeviceContext *device_ctx,
381 AVDictionary *opts, int flags)
383 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
385 int ret, device_idx = 0;
388 device_idx = strtol(device, NULL, 0);
390 if (cuda_device_init(device_ctx) < 0)
393 cu = hwctx->internal->cuda_dl;
395 ret = CHECK_CU(cu->cuInit(0));
399 ret = CHECK_CU(cu->cuDeviceGet(&hwctx->internal->cuda_device, device_idx));
403 ret = cuda_context_init(device_ctx, flags);
410 cuda_device_uninit(device_ctx);
411 return AVERROR_UNKNOWN;
414 static int cuda_device_derive(AVHWDeviceContext *device_ctx,
415 AVHWDeviceContext *src_ctx, AVDictionary *opts,
417 AVCUDADeviceContext *hwctx = device_ctx->hwctx;
419 const char *src_uuid = NULL;
420 int ret, i, device_count;
423 VkPhysicalDeviceIDProperties vk_idp = {
424 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES,
428 switch (src_ctx->type) {
430 case AV_HWDEVICE_TYPE_VULKAN: {
431 AVVulkanDeviceContext *vkctx = src_ctx->hwctx;
432 VkPhysicalDeviceProperties2 vk_dev_props = {
433 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
436 vkGetPhysicalDeviceProperties2(vkctx->phys_dev, &vk_dev_props);
437 src_uuid = vk_idp.deviceUUID;
442 return AVERROR(ENOSYS);
446 av_log(device_ctx, AV_LOG_ERROR,
447 "Failed to get UUID of source device.\n");
451 if (cuda_device_init(device_ctx) < 0)
454 cu = hwctx->internal->cuda_dl;
456 ret = CHECK_CU(cu->cuInit(0));
460 ret = CHECK_CU(cu->cuDeviceGetCount(&device_count));
464 hwctx->internal->cuda_device = -1;
465 for (i = 0; i < device_count; i++) {
469 ret = CHECK_CU(cu->cuDeviceGet(&dev, i));
473 ret = CHECK_CU(cu->cuDeviceGetUuid(&uuid, dev));
477 if (memcmp(src_uuid, uuid.bytes, sizeof (uuid.bytes)) == 0) {
478 hwctx->internal->cuda_device = dev;
483 if (hwctx->internal->cuda_device == -1) {
484 av_log(device_ctx, AV_LOG_ERROR, "Could not derive CUDA device.\n");
488 ret = cuda_context_init(device_ctx, flags);
495 cuda_device_uninit(device_ctx);
496 return AVERROR_UNKNOWN;
499 const HWContextType ff_hwcontext_type_cuda = {
500 .type = AV_HWDEVICE_TYPE_CUDA,
503 .device_hwctx_size = sizeof(AVCUDADeviceContext),
504 .frames_priv_size = sizeof(CUDAFramesContext),
506 .device_create = cuda_device_create,
507 .device_derive = cuda_device_derive,
508 .device_init = cuda_device_init,
509 .device_uninit = cuda_device_uninit,
510 .frames_get_constraints = cuda_frames_get_constraints,
511 .frames_init = cuda_frames_init,
512 .frames_get_buffer = cuda_get_buffer,
513 .transfer_get_formats = cuda_transfer_get_formats,
514 .transfer_data_to = cuda_transfer_data,
515 .transfer_data_from = cuda_transfer_data,
517 .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_CUDA, AV_PIX_FMT_NONE },