2 * Videotoolbox hardware acceleration
4 * copyright (c) 2012 Sebastien Zwickert
6 * This file is part of FFmpeg.
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 #if CONFIG_VIDEOTOOLBOX
25 # include "videotoolbox.h"
29 #include "vda_vt_internal.h"
30 #include "libavutil/avutil.h"
31 #include "bytestream.h"
33 #include "mpegvideo.h"
35 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
36 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
39 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
41 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
43 CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
44 CVPixelBufferRelease(cv_buffer);
47 static int videotoolbox_buffer_copy(VTContext *vtctx,
48 const uint8_t *buffer,
53 tmp = av_fast_realloc(vtctx->bitstream,
54 &vtctx->allocated_size,
58 return AVERROR(ENOMEM);
60 vtctx->bitstream = tmp;
61 memcpy(vtctx->bitstream, buffer, size);
62 vtctx->bitstream_size = size;
67 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
69 frame->width = avctx->width;
70 frame->height = avctx->height;
71 frame->format = avctx->pix_fmt;
72 frame->buf[0] = av_buffer_alloc(1);
75 return AVERROR(ENOMEM);
80 CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
82 CFDataRef data = NULL;
84 /* Each VCL NAL in the bitstream sent to the decoder
85 * is preceded by a 4 bytes length header.
86 * Change the avcC atom header if needed, to signal headers of 4 bytes. */
87 if (avctx->extradata_size >= 4 && (avctx->extradata[4] & 0x03) != 0x03) {
88 uint8_t *rw_extradata = av_memdup(avctx->extradata, avctx->extradata_size);
93 rw_extradata[4] |= 0x03;
95 data = CFDataCreate(kCFAllocatorDefault, rw_extradata, avctx->extradata_size);
97 av_freep(&rw_extradata);
99 data = CFDataCreate(kCFAllocatorDefault, avctx->extradata, avctx->extradata_size);
105 int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
107 av_buffer_unref(&frame->buf[0]);
109 frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
110 sizeof(vtctx->frame),
111 videotoolbox_buffer_release,
113 AV_BUFFER_FLAG_READONLY);
114 if (!frame->buf[0]) {
115 return AVERROR(ENOMEM);
118 frame->data[3] = (uint8_t*)vtctx->frame;
124 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
125 const uint8_t *buffer,
128 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
129 H264Context *h = avctx->priv_data;
131 vtctx->bitstream_size = 0;
133 if (h->is_avc == 1) {
134 return videotoolbox_buffer_copy(vtctx, buffer, size);
140 int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
141 const uint8_t *buffer,
144 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
145 H264Context *h = avctx->priv_data;
151 tmp = av_fast_realloc(vtctx->bitstream,
152 &vtctx->allocated_size,
153 vtctx->bitstream_size+size+4);
155 return AVERROR(ENOMEM);
157 vtctx->bitstream = tmp;
159 AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
160 memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
162 vtctx->bitstream_size += size + 4;
167 int ff_videotoolbox_uninit(AVCodecContext *avctx)
169 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
171 av_freep(&vtctx->bitstream);
173 CVPixelBufferRelease(vtctx->frame);
179 #if CONFIG_VIDEOTOOLBOX
180 static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
185 for (i = 3; i >= 0; i--) {
186 b = (length >> (i * 7)) & 0x7F;
190 bytestream2_put_byteu(pb, b);
194 static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
197 uint8_t *rw_extradata;
199 int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
200 // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
201 int config_size = 13 + 5 + avctx->extradata_size;
204 if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
207 bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
208 bytestream2_put_byteu(&pb, 0); // version
209 bytestream2_put_ne24(&pb, 0); // flags
211 // elementary stream descriptor
212 bytestream2_put_byteu(&pb, 0x03); // ES_DescrTag
213 videotoolbox_write_mp4_descr_length(&pb, full_size);
214 bytestream2_put_ne16(&pb, 0); // esid
215 bytestream2_put_byteu(&pb, 0); // stream priority (0-32)
217 // decoder configuration descriptor
218 bytestream2_put_byteu(&pb, 0x04); // DecoderConfigDescrTag
219 videotoolbox_write_mp4_descr_length(&pb, config_size);
220 bytestream2_put_byteu(&pb, 32); // object type indication. 32 = AV_CODEC_ID_MPEG4
221 bytestream2_put_byteu(&pb, 0x11); // stream type
222 bytestream2_put_ne24(&pb, 0); // buffer size
223 bytestream2_put_ne32(&pb, 0); // max bitrate
224 bytestream2_put_ne32(&pb, 0); // avg bitrate
226 // decoder specific descriptor
227 bytestream2_put_byteu(&pb, 0x05); ///< DecSpecificInfoTag
228 videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
230 bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
232 // SLConfigDescriptor
233 bytestream2_put_byteu(&pb, 0x06); // SLConfigDescrTag
234 bytestream2_put_byteu(&pb, 0x01); // length
235 bytestream2_put_byteu(&pb, 0x02); //
237 s = bytestream2_size_p(&pb);
239 data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
241 av_freep(&rw_extradata);
245 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
250 CMBlockBufferRef block_buf;
251 CMSampleBufferRef sample_buf;
256 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
257 buffer, // memoryBlock
259 kCFAllocatorNull, // blockAllocator
260 NULL, // customBlockSource
267 status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
268 block_buf, // dataBuffer
270 0, // makeDataReadyCallback
271 0, // makeDataReadyRefcon
272 fmt_desc, // formatDescription
274 0, // numSampleTimingEntries
275 NULL, // sampleTimingArray
276 0, // numSampleSizeEntries
277 NULL, // sampleSizeArray
282 CFRelease(block_buf);
287 static void videotoolbox_decoder_callback(void *opaque,
288 void *sourceFrameRefCon,
290 VTDecodeInfoFlags flags,
291 CVImageBufferRef image_buffer,
295 AVCodecContext *avctx = opaque;
296 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
299 CVPixelBufferRelease(vtctx->frame);
304 av_log(NULL, AV_LOG_DEBUG, "vt decoder cb: output image buffer is null\n");
308 vtctx->frame = CVPixelBufferRetain(image_buffer);
311 static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
314 CMSampleBufferRef sample_buf;
315 AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
316 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
318 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
320 vtctx->bitstream_size);
325 status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
328 NULL, // sourceFrameRefCon
331 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
333 CFRelease(sample_buf);
338 static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
341 AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
342 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
344 if (!videotoolbox->session || !vtctx->bitstream)
345 return AVERROR_INVALIDDATA;
347 status = videotoolbox_session_decode_frame(avctx);
350 av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
351 return AVERROR_UNKNOWN;
355 return AVERROR_UNKNOWN;
357 return ff_videotoolbox_buffer_create(vtctx, frame);
360 static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
362 H264Context *h = avctx->priv_data;
363 AVFrame *frame = h->cur_pic_ptr->f;
365 return videotoolbox_common_end_frame(avctx, frame);
368 static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
369 const uint8_t *buffer,
372 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
374 return videotoolbox_buffer_copy(vtctx, buffer, size);
377 static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
378 const uint8_t *buffer,
384 static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
386 MpegEncContext *s = avctx->priv_data;
387 AVFrame *frame = s->current_picture_ptr->f;
389 return videotoolbox_common_end_frame(avctx, frame);
392 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
393 AVCodecContext *avctx)
395 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
397 &kCFTypeDictionaryKeyCallBacks,
398 &kCFTypeDictionaryValueCallBacks);
400 CFDictionarySetValue(config_info,
401 kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
404 if (avctx->extradata_size) {
405 CFMutableDictionaryRef avc_info;
406 CFDataRef data = NULL;
408 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
410 &kCFTypeDictionaryKeyCallBacks,
411 &kCFTypeDictionaryValueCallBacks);
413 switch (codec_type) {
414 case kCMVideoCodecType_MPEG4Video :
415 data = videotoolbox_esds_extradata_create(avctx);
417 CFDictionarySetValue(avc_info, CFSTR("esds"), data);
419 case kCMVideoCodecType_H264 :
420 data = ff_videotoolbox_avcc_extradata_create(avctx);
422 CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
428 CFDictionarySetValue(config_info,
429 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
440 static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
444 CFMutableDictionaryRef buffer_attributes;
445 CFMutableDictionaryRef io_surface_properties;
446 CFNumberRef cv_pix_fmt;
450 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
451 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
452 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
454 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
456 &kCFTypeDictionaryKeyCallBacks,
457 &kCFTypeDictionaryValueCallBacks);
458 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
460 &kCFTypeDictionaryKeyCallBacks,
461 &kCFTypeDictionaryValueCallBacks);
463 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
464 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
465 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
466 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
468 CFRelease(io_surface_properties);
469 CFRelease(cv_pix_fmt);
473 return buffer_attributes;
476 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
477 CFDictionaryRef decoder_spec,
481 CMFormatDescriptionRef cm_fmt_desc;
484 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
488 decoder_spec, // Dictionary of extension
497 static int videotoolbox_default_init(AVCodecContext *avctx)
499 AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
501 VTDecompressionOutputCallbackRecord decoder_cb;
502 CFDictionaryRef decoder_spec;
503 CFDictionaryRef buf_attr;
506 av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
510 switch( avctx->codec_id ) {
511 case AV_CODEC_ID_H263 :
512 videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
514 case AV_CODEC_ID_H264 :
515 videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
517 case AV_CODEC_ID_MPEG1VIDEO :
518 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
520 case AV_CODEC_ID_MPEG2VIDEO :
521 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
523 case AV_CODEC_ID_MPEG4 :
524 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
530 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
532 videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
536 if (!videotoolbox->cm_fmt_desc) {
538 CFRelease(decoder_spec);
540 av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
544 buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
546 videotoolbox->cv_pix_fmt_type);
548 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
549 decoder_cb.decompressionOutputRefCon = avctx;
551 status = VTDecompressionSessionCreate(NULL, // allocator
552 videotoolbox->cm_fmt_desc, // videoFormatDescription
553 decoder_spec, // videoDecoderSpecification
554 buf_attr, // destinationImageBufferAttributes
555 &decoder_cb, // outputCallback
556 &videotoolbox->session); // decompressionSessionOut
559 CFRelease(decoder_spec);
564 case kVTVideoDecoderNotAvailableNowErr:
565 case kVTVideoDecoderUnsupportedDataFormatErr:
566 return AVERROR(ENOSYS);
567 case kVTVideoDecoderMalfunctionErr:
568 return AVERROR(EINVAL);
569 case kVTVideoDecoderBadDataErr :
570 return AVERROR_INVALIDDATA;
574 return AVERROR_UNKNOWN;
578 static void videotoolbox_default_free(AVCodecContext *avctx)
580 AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
583 if (videotoolbox->cm_fmt_desc)
584 CFRelease(videotoolbox->cm_fmt_desc);
586 if (videotoolbox->session)
587 VTDecompressionSessionInvalidate(videotoolbox->session);
591 AVHWAccel ff_h263_videotoolbox_hwaccel = {
592 .name = "h263_videotoolbox",
593 .type = AVMEDIA_TYPE_VIDEO,
594 .id = AV_CODEC_ID_H263,
595 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
596 .alloc_frame = ff_videotoolbox_alloc_frame,
597 .start_frame = videotoolbox_mpeg_start_frame,
598 .decode_slice = videotoolbox_mpeg_decode_slice,
599 .end_frame = videotoolbox_mpeg_end_frame,
600 .uninit = ff_videotoolbox_uninit,
601 .priv_data_size = sizeof(VTContext),
604 AVHWAccel ff_h264_videotoolbox_hwaccel = {
605 .name = "h264_videotoolbox",
606 .type = AVMEDIA_TYPE_VIDEO,
607 .id = AV_CODEC_ID_H264,
608 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
609 .alloc_frame = ff_videotoolbox_alloc_frame,
610 .start_frame = ff_videotoolbox_h264_start_frame,
611 .decode_slice = ff_videotoolbox_h264_decode_slice,
612 .end_frame = videotoolbox_h264_end_frame,
613 .uninit = ff_videotoolbox_uninit,
614 .priv_data_size = sizeof(VTContext),
617 AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
618 .name = "mpeg1_videotoolbox",
619 .type = AVMEDIA_TYPE_VIDEO,
620 .id = AV_CODEC_ID_MPEG1VIDEO,
621 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
622 .alloc_frame = ff_videotoolbox_alloc_frame,
623 .start_frame = videotoolbox_mpeg_start_frame,
624 .decode_slice = videotoolbox_mpeg_decode_slice,
625 .end_frame = videotoolbox_mpeg_end_frame,
626 .uninit = ff_videotoolbox_uninit,
627 .priv_data_size = sizeof(VTContext),
630 AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
631 .name = "mpeg2_videotoolbox",
632 .type = AVMEDIA_TYPE_VIDEO,
633 .id = AV_CODEC_ID_MPEG2VIDEO,
634 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
635 .alloc_frame = ff_videotoolbox_alloc_frame,
636 .start_frame = videotoolbox_mpeg_start_frame,
637 .decode_slice = videotoolbox_mpeg_decode_slice,
638 .end_frame = videotoolbox_mpeg_end_frame,
639 .uninit = ff_videotoolbox_uninit,
640 .priv_data_size = sizeof(VTContext),
643 AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
644 .name = "mpeg4_videotoolbox",
645 .type = AVMEDIA_TYPE_VIDEO,
646 .id = AV_CODEC_ID_MPEG4,
647 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
648 .alloc_frame = ff_videotoolbox_alloc_frame,
649 .start_frame = videotoolbox_mpeg_start_frame,
650 .decode_slice = videotoolbox_mpeg_decode_slice,
651 .end_frame = videotoolbox_mpeg_end_frame,
652 .uninit = ff_videotoolbox_uninit,
653 .priv_data_size = sizeof(VTContext),
656 AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
658 AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
661 ret->output_callback = videotoolbox_decoder_callback;
662 ret->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
668 int av_videotoolbox_default_init(AVCodecContext *avctx)
670 return av_videotoolbox_default_init2(avctx, NULL);
673 int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
675 avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context();
676 if (!avctx->hwaccel_context)
677 return AVERROR(ENOMEM);
678 return videotoolbox_default_init(avctx);
681 void av_videotoolbox_default_free(AVCodecContext *avctx)
684 videotoolbox_default_free(avctx);
685 av_freep(&avctx->hwaccel_context);
687 #endif /* CONFIG_VIDEOTOOLBOX */