2 * Videotoolbox hardware acceleration
4 * copyright (c) 2012 Sebastien Zwickert
6 * This file is part of FFmpeg.
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 #include "videotoolbox.h"
25 #include "libavutil/hwcontext_videotoolbox.h"
26 #include "vt_internal.h"
27 #include "libavutil/avutil.h"
28 #include "libavutil/hwcontext.h"
29 #include "bytestream.h"
32 #include "mpegvideo.h"
33 #include <TargetConditionals.h>
35 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
36 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
43 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
45 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
47 CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
48 CVPixelBufferRelease(cv_buffer);
51 static int videotoolbox_buffer_copy(VTContext *vtctx,
52 const uint8_t *buffer,
57 tmp = av_fast_realloc(vtctx->bitstream,
58 &vtctx->allocated_size,
62 return AVERROR(ENOMEM);
64 vtctx->bitstream = tmp;
65 memcpy(vtctx->bitstream, buffer, size);
66 vtctx->bitstream_size = size;
71 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
73 frame->width = avctx->width;
74 frame->height = avctx->height;
75 frame->format = avctx->pix_fmt;
76 frame->buf[0] = av_buffer_alloc(1);
79 return AVERROR(ENOMEM);
84 #define AV_W8(p, v) *(p) = (v)
86 CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
88 H264Context *h = avctx->priv_data;
89 CFDataRef data = NULL;
91 int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size;
92 uint8_t *vt_extradata = av_malloc(vt_extradata_size);
98 AV_W8(p + 0, 1); /* version */
99 AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
100 AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
101 AV_W8(p + 3, h->ps.sps->data[3]); /* level */
102 AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
103 AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
104 AV_WB16(p + 6, h->ps.sps->data_size);
105 memcpy(p + 8, h->ps.sps->data, h->ps.sps->data_size);
106 p += 8 + h->ps.sps->data_size;
107 AV_W8(p + 0, 1); /* number of pps */
108 AV_WB16(p + 1, h->ps.pps->data_size);
109 memcpy(p + 3, h->ps.pps->data, h->ps.pps->data_size);
111 p += 3 + h->ps.pps->data_size;
112 av_assert0(p - vt_extradata == vt_extradata_size);
114 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
115 av_free(vt_extradata);
119 CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
121 HEVCContext *h = avctx->priv_data;
122 const HEVCVPS *vps = (const HEVCVPS *)h->ps.vps_list[0]->data;
123 const HEVCSPS *sps = (const HEVCSPS *)h->ps.sps_list[0]->data;
125 const HEVCPPS *pps = h->ps.pps;
126 PTLCommon ptlc = vps->ptl.general_ptl;
128 uint8_t parallelismType;
129 CFDataRef data = NULL;
131 int vt_extradata_size = 23 + 5 + vps->data_size + 5 + sps->data_size + 3;
132 uint8_t *vt_extradata;
134 for (i = 0; i < MAX_PPS_COUNT; i++) {
135 if (h->ps.pps_list[i]) {
136 const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
137 vt_extradata_size += 2 + pps->data_size;
142 vt_extradata = av_malloc(vt_extradata_size);
147 /* unsigned int(8) configurationVersion = 1; */
151 * unsigned int(2) general_profile_space;
152 * unsigned int(1) general_tier_flag;
153 * unsigned int(5) general_profile_idc;
155 AV_W8(p + 1, ptlc.profile_space << 6 |
156 ptlc.tier_flag << 5 |
159 /* unsigned int(32) general_profile_compatibility_flags; */
160 memcpy(p + 2, ptlc.profile_compatibility_flag, 4);
162 /* unsigned int(48) general_constraint_indicator_flags; */
163 AV_W8(p + 6, ptlc.progressive_source_flag << 7 |
164 ptlc.interlaced_source_flag << 6 |
165 ptlc.non_packed_constraint_flag << 5 |
166 ptlc.frame_only_constraint_flag << 4);
170 /* unsigned int(8) general_level_idc; */
171 AV_W8(p + 12, ptlc.level_idc);
174 * bit(4) reserved = ‘1111’b;
175 * unsigned int(12) min_spatial_segmentation_idc;
177 AV_W8(p + 13, 0xf0 | (vui.min_spatial_segmentation_idc >> 4));
178 AV_W8(p + 14, vui.min_spatial_segmentation_idc & 0xff);
181 * bit(6) reserved = ‘111111’b;
182 * unsigned int(2) parallelismType;
184 if (!vui.min_spatial_segmentation_idc)
186 else if (pps->entropy_coding_sync_enabled_flag && pps->tiles_enabled_flag)
188 else if (pps->entropy_coding_sync_enabled_flag)
190 else if (pps->tiles_enabled_flag)
194 AV_W8(p + 15, 0xfc | parallelismType);
197 * bit(6) reserved = ‘111111’b;
198 * unsigned int(2) chromaFormat;
200 AV_W8(p + 16, sps->chroma_format_idc | 0xfc);
203 * bit(5) reserved = ‘11111’b;
204 * unsigned int(3) bitDepthLumaMinus8;
206 AV_W8(p + 17, (sps->bit_depth - 8) | 0xfc);
209 * bit(5) reserved = ‘11111’b;
210 * unsigned int(3) bitDepthChromaMinus8;
212 AV_W8(p + 18, (sps->bit_depth_chroma - 8) | 0xfc);
214 /* bit(16) avgFrameRate; */
218 * bit(2) constantFrameRate;
219 * bit(3) numTemporalLayers;
220 * bit(1) temporalIdNested;
221 * unsigned int(2) lengthSizeMinusOne;
223 AV_W8(p + 21, 0 << 6 |
224 sps->max_sub_layers << 3 |
225 sps->temporal_id_nesting_flag << 2 |
228 /* unsigned int(8) numOfArrays; */
234 * bit(1) array_completeness;
235 * unsigned int(1) reserved = 0;
236 * unsigned int(6) NAL_unit_type;
239 HEVC_NAL_VPS & 0x3f);
240 /* unsigned int(16) numNalus; */
242 /* unsigned int(16) nalUnitLength; */
243 AV_WB16(p + 3, vps->data_size);
244 /* bit(8*nalUnitLength) nalUnit; */
245 memcpy(p + 5, vps->data, vps->data_size);
246 p += 5 + vps->data_size;
250 HEVC_NAL_SPS & 0x3f);
252 AV_WB16(p + 3, sps->data_size);
253 memcpy(p + 5, sps->data, sps->data_size);
254 p += 5 + sps->data_size;
258 HEVC_NAL_PPS & 0x3f);
259 AV_WB16(p + 1, num_pps);
261 for (i = 0; i < MAX_PPS_COUNT; i++) {
262 if (h->ps.pps_list[i]) {
263 const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
264 AV_WB16(p, pps->data_size);
265 memcpy(p + 2, pps->data, pps->data_size);
266 p += 2 + pps->data_size;
270 av_assert0(p - vt_extradata == vt_extradata_size);
272 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
273 av_free(vt_extradata);
277 int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
279 av_buffer_unref(&frame->buf[0]);
281 frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
282 sizeof(vtctx->frame),
283 videotoolbox_buffer_release,
285 AV_BUFFER_FLAG_READONLY);
286 if (!frame->buf[0]) {
287 return AVERROR(ENOMEM);
290 frame->data[3] = (uint8_t*)vtctx->frame;
296 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
297 const uint8_t *buffer,
300 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
301 H264Context *h = avctx->priv_data;
303 vtctx->bitstream_size = 0;
305 if (h->is_avc == 1) {
306 return videotoolbox_buffer_copy(vtctx, buffer, size);
312 int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
313 const uint8_t *buffer,
316 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
317 H264Context *h = avctx->priv_data;
323 tmp = av_fast_realloc(vtctx->bitstream,
324 &vtctx->allocated_size,
325 vtctx->bitstream_size+size+4);
327 return AVERROR(ENOMEM);
329 vtctx->bitstream = tmp;
331 AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
332 memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
334 vtctx->bitstream_size += size + 4;
339 int ff_videotoolbox_uninit(AVCodecContext *avctx)
341 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
343 av_freep(&vtctx->bitstream);
345 CVPixelBufferRelease(vtctx->frame);
351 #if CONFIG_VIDEOTOOLBOX
352 // Return the AVVideotoolboxContext that matters currently. Where it comes from
353 // depends on the API used.
354 static AVVideotoolboxContext *videotoolbox_get_context(AVCodecContext *avctx)
356 // Somewhat tricky because the user can call av_videotoolbox_default_free()
357 // at any time, even when the codec is closed.
358 if (avctx->internal && avctx->internal->hwaccel_priv_data) {
359 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
361 return vtctx->vt_ctx;
363 return avctx->hwaccel_context;
366 static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
368 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
369 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->frame;
370 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
371 enum AVPixelFormat sw_format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
372 int width = CVPixelBufferGetWidth(pixbuf);
373 int height = CVPixelBufferGetHeight(pixbuf);
374 AVHWFramesContext *cached_frames;
377 ret = ff_videotoolbox_buffer_create(vtctx, frame);
381 // Old API code path.
382 if (!vtctx->cached_hw_frames_ctx)
385 cached_frames = (AVHWFramesContext*)vtctx->cached_hw_frames_ctx->data;
387 if (cached_frames->sw_format != sw_format ||
388 cached_frames->width != width ||
389 cached_frames->height != height) {
390 AVBufferRef *hw_frames_ctx = av_hwframe_ctx_alloc(cached_frames->device_ref);
391 AVHWFramesContext *hw_frames;
393 return AVERROR(ENOMEM);
395 hw_frames = (AVHWFramesContext*)hw_frames_ctx->data;
396 hw_frames->format = cached_frames->format;
397 hw_frames->sw_format = sw_format;
398 hw_frames->width = width;
399 hw_frames->height = height;
401 ret = av_hwframe_ctx_init(hw_frames_ctx);
403 av_buffer_unref(&hw_frames_ctx);
407 av_buffer_unref(&vtctx->cached_hw_frames_ctx);
408 vtctx->cached_hw_frames_ctx = hw_frames_ctx;
411 av_buffer_unref(&frame->hw_frames_ctx);
412 frame->hw_frames_ctx = av_buffer_ref(vtctx->cached_hw_frames_ctx);
413 if (!frame->hw_frames_ctx)
414 return AVERROR(ENOMEM);
419 static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
424 for (i = 3; i >= 0; i--) {
425 b = (length >> (i * 7)) & 0x7F;
429 bytestream2_put_byteu(pb, b);
433 static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
436 uint8_t *rw_extradata;
438 int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
439 // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
440 int config_size = 13 + 5 + avctx->extradata_size;
443 if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
446 bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
447 bytestream2_put_byteu(&pb, 0); // version
448 bytestream2_put_ne24(&pb, 0); // flags
450 // elementary stream descriptor
451 bytestream2_put_byteu(&pb, 0x03); // ES_DescrTag
452 videotoolbox_write_mp4_descr_length(&pb, full_size);
453 bytestream2_put_ne16(&pb, 0); // esid
454 bytestream2_put_byteu(&pb, 0); // stream priority (0-32)
456 // decoder configuration descriptor
457 bytestream2_put_byteu(&pb, 0x04); // DecoderConfigDescrTag
458 videotoolbox_write_mp4_descr_length(&pb, config_size);
459 bytestream2_put_byteu(&pb, 32); // object type indication. 32 = AV_CODEC_ID_MPEG4
460 bytestream2_put_byteu(&pb, 0x11); // stream type
461 bytestream2_put_ne24(&pb, 0); // buffer size
462 bytestream2_put_ne32(&pb, 0); // max bitrate
463 bytestream2_put_ne32(&pb, 0); // avg bitrate
465 // decoder specific descriptor
466 bytestream2_put_byteu(&pb, 0x05); ///< DecSpecificInfoTag
467 videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
469 bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
471 // SLConfigDescriptor
472 bytestream2_put_byteu(&pb, 0x06); // SLConfigDescrTag
473 bytestream2_put_byteu(&pb, 0x01); // length
474 bytestream2_put_byteu(&pb, 0x02); //
476 s = bytestream2_size_p(&pb);
478 data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
480 av_freep(&rw_extradata);
484 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
489 CMBlockBufferRef block_buf;
490 CMSampleBufferRef sample_buf;
495 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
496 buffer, // memoryBlock
498 kCFAllocatorNull, // blockAllocator
499 NULL, // customBlockSource
506 status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
507 block_buf, // dataBuffer
509 0, // makeDataReadyCallback
510 0, // makeDataReadyRefcon
511 fmt_desc, // formatDescription
513 0, // numSampleTimingEntries
514 NULL, // sampleTimingArray
515 0, // numSampleSizeEntries
516 NULL, // sampleSizeArray
521 CFRelease(block_buf);
526 static void videotoolbox_decoder_callback(void *opaque,
527 void *sourceFrameRefCon,
529 VTDecodeInfoFlags flags,
530 CVImageBufferRef image_buffer,
534 AVCodecContext *avctx = opaque;
535 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
538 CVPixelBufferRelease(vtctx->frame);
543 av_log(NULL, AV_LOG_DEBUG, "vt decoder cb: output image buffer is null\n");
547 vtctx->frame = CVPixelBufferRetain(image_buffer);
550 static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
553 CMSampleBufferRef sample_buf;
554 AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
555 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
557 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
559 vtctx->bitstream_size);
564 status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
567 NULL, // sourceFrameRefCon
570 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
572 CFRelease(sample_buf);
577 static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
580 AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
581 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
583 if (!videotoolbox->session || !vtctx->bitstream)
584 return AVERROR_INVALIDDATA;
586 status = videotoolbox_session_decode_frame(avctx);
589 av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
590 return AVERROR_UNKNOWN;
594 return AVERROR_UNKNOWN;
596 return videotoolbox_buffer_create(avctx, frame);
599 static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
601 H264Context *h = avctx->priv_data;
602 AVFrame *frame = h->cur_pic_ptr->f;
604 return videotoolbox_common_end_frame(avctx, frame);
607 static int videotoolbox_hevc_end_frame(AVCodecContext *avctx)
609 HEVCContext *h = avctx->priv_data;
610 AVFrame *frame = h->ref->frame;
611 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
614 ret = videotoolbox_common_end_frame(avctx, frame);
615 vtctx->bitstream_size = 0;
619 static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
620 const uint8_t *buffer,
623 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
625 return videotoolbox_buffer_copy(vtctx, buffer, size);
628 static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
629 const uint8_t *buffer,
635 static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
637 MpegEncContext *s = avctx->priv_data;
638 AVFrame *frame = s->current_picture_ptr->f;
640 return videotoolbox_common_end_frame(avctx, frame);
643 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
644 AVCodecContext *avctx)
646 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
648 &kCFTypeDictionaryKeyCallBacks,
649 &kCFTypeDictionaryValueCallBacks);
651 CFDictionarySetValue(config_info,
652 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
655 if (avctx->extradata_size) {
656 CFMutableDictionaryRef avc_info;
657 CFDataRef data = NULL;
659 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
661 &kCFTypeDictionaryKeyCallBacks,
662 &kCFTypeDictionaryValueCallBacks);
664 switch (codec_type) {
665 case kCMVideoCodecType_MPEG4Video :
666 data = videotoolbox_esds_extradata_create(avctx);
668 CFDictionarySetValue(avc_info, CFSTR("esds"), data);
670 case kCMVideoCodecType_H264 :
671 data = ff_videotoolbox_avcc_extradata_create(avctx);
673 CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
675 case kCMVideoCodecType_HEVC :
676 data = ff_videotoolbox_hvcc_extradata_create(avctx);
678 CFDictionarySetValue(avc_info, CFSTR("hvcC"), data);
684 CFDictionarySetValue(config_info,
685 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
696 static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
700 CFMutableDictionaryRef buffer_attributes;
701 CFMutableDictionaryRef io_surface_properties;
702 CFNumberRef cv_pix_fmt;
706 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
707 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
708 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
710 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
712 &kCFTypeDictionaryKeyCallBacks,
713 &kCFTypeDictionaryValueCallBacks);
714 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
716 &kCFTypeDictionaryKeyCallBacks,
717 &kCFTypeDictionaryValueCallBacks);
720 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
721 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
722 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
723 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
725 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
727 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
730 CFRelease(io_surface_properties);
731 CFRelease(cv_pix_fmt);
735 return buffer_attributes;
738 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
739 CFDictionaryRef decoder_spec,
743 CMFormatDescriptionRef cm_fmt_desc;
746 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
750 decoder_spec, // Dictionary of extension
759 static int videotoolbox_default_init(AVCodecContext *avctx)
761 AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
763 VTDecompressionOutputCallbackRecord decoder_cb;
764 CFDictionaryRef decoder_spec;
765 CFDictionaryRef buf_attr;
768 av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
772 switch( avctx->codec_id ) {
773 case AV_CODEC_ID_H263 :
774 videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
776 case AV_CODEC_ID_H264 :
777 videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
779 case AV_CODEC_ID_HEVC :
780 videotoolbox->cm_codec_type = kCMVideoCodecType_HEVC;
782 case AV_CODEC_ID_MPEG1VIDEO :
783 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
785 case AV_CODEC_ID_MPEG2VIDEO :
786 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
788 case AV_CODEC_ID_MPEG4 :
789 videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
795 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
797 videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
801 if (!videotoolbox->cm_fmt_desc) {
803 CFRelease(decoder_spec);
805 av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
809 buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
811 videotoolbox->cv_pix_fmt_type);
813 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
814 decoder_cb.decompressionOutputRefCon = avctx;
816 status = VTDecompressionSessionCreate(NULL, // allocator
817 videotoolbox->cm_fmt_desc, // videoFormatDescription
818 decoder_spec, // videoDecoderSpecification
819 buf_attr, // destinationImageBufferAttributes
820 &decoder_cb, // outputCallback
821 &videotoolbox->session); // decompressionSessionOut
824 CFRelease(decoder_spec);
829 case kVTVideoDecoderNotAvailableNowErr:
830 av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox session not available.\n");
831 return AVERROR(ENOSYS);
832 case kVTVideoDecoderUnsupportedDataFormatErr:
833 av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox does not support this format.\n");
834 return AVERROR(ENOSYS);
835 case kVTVideoDecoderMalfunctionErr:
836 av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n");
837 return AVERROR(EINVAL);
838 case kVTVideoDecoderBadDataErr :
839 av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n");
840 return AVERROR_INVALIDDATA;
844 av_log(avctx, AV_LOG_VERBOSE, "Unknown VideoToolbox session creation error %u\n", (unsigned)status);
845 return AVERROR_UNKNOWN;
849 static void videotoolbox_default_free(AVCodecContext *avctx)
851 AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
854 if (videotoolbox->cm_fmt_desc)
855 CFRelease(videotoolbox->cm_fmt_desc);
857 if (videotoolbox->session) {
858 VTDecompressionSessionInvalidate(videotoolbox->session);
859 CFRelease(videotoolbox->session);
864 static int videotoolbox_uninit(AVCodecContext *avctx)
866 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
870 ff_videotoolbox_uninit(avctx);
873 videotoolbox_default_free(avctx);
875 av_buffer_unref(&vtctx->cached_hw_frames_ctx);
876 av_freep(&vtctx->vt_ctx);
881 static int videotoolbox_common_init(AVCodecContext *avctx)
883 VTContext *vtctx = avctx->internal->hwaccel_priv_data;
884 AVHWFramesContext *hw_frames;
887 // Old API - do nothing.
888 if (avctx->hwaccel_context)
891 if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
892 av_log(avctx, AV_LOG_ERROR,
893 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
894 return AVERROR(EINVAL);
897 vtctx->vt_ctx = av_videotoolbox_alloc_context();
898 if (!vtctx->vt_ctx) {
899 err = AVERROR(ENOMEM);
903 if (avctx->hw_frames_ctx) {
904 hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
906 avctx->hw_frames_ctx = av_hwframe_ctx_alloc(avctx->hw_device_ctx);
907 if (!avctx->hw_frames_ctx) {
908 err = AVERROR(ENOMEM);
912 hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
913 hw_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
914 hw_frames->sw_format = AV_PIX_FMT_NV12; // same as av_videotoolbox_alloc_context()
915 hw_frames->width = avctx->width;
916 hw_frames->height = avctx->height;
918 err = av_hwframe_ctx_init(avctx->hw_frames_ctx);
920 av_buffer_unref(&avctx->hw_frames_ctx);
925 vtctx->cached_hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
926 if (!vtctx->cached_hw_frames_ctx) {
927 err = AVERROR(ENOMEM);
931 vtctx->vt_ctx->cv_pix_fmt_type =
932 av_map_videotoolbox_format_from_pixfmt(hw_frames->sw_format);
933 if (!vtctx->vt_ctx->cv_pix_fmt_type) {
934 av_log(avctx, AV_LOG_ERROR, "Unknown sw_format.\n");
935 err = AVERROR(EINVAL);
939 err = videotoolbox_default_init(avctx);
946 videotoolbox_uninit(avctx);
950 AVHWAccel ff_h263_videotoolbox_hwaccel = {
951 .name = "h263_videotoolbox",
952 .type = AVMEDIA_TYPE_VIDEO,
953 .id = AV_CODEC_ID_H263,
954 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
955 .alloc_frame = ff_videotoolbox_alloc_frame,
956 .start_frame = videotoolbox_mpeg_start_frame,
957 .decode_slice = videotoolbox_mpeg_decode_slice,
958 .end_frame = videotoolbox_mpeg_end_frame,
959 .init = videotoolbox_common_init,
960 .uninit = videotoolbox_uninit,
961 .priv_data_size = sizeof(VTContext),
964 AVHWAccel ff_hevc_videotoolbox_hwaccel = {
965 .name = "hevc_videotoolbox",
966 .type = AVMEDIA_TYPE_VIDEO,
967 .id = AV_CODEC_ID_HEVC,
968 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
969 .alloc_frame = ff_videotoolbox_alloc_frame,
970 .start_frame = ff_videotoolbox_h264_start_frame,
971 .decode_slice = ff_videotoolbox_h264_decode_slice,
972 .end_frame = videotoolbox_hevc_end_frame,
973 .init = videotoolbox_common_init,
974 .uninit = ff_videotoolbox_uninit,
975 .priv_data_size = sizeof(VTContext),
978 AVHWAccel ff_h264_videotoolbox_hwaccel = {
979 .name = "h264_videotoolbox",
980 .type = AVMEDIA_TYPE_VIDEO,
981 .id = AV_CODEC_ID_H264,
982 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
983 .alloc_frame = ff_videotoolbox_alloc_frame,
984 .start_frame = ff_videotoolbox_h264_start_frame,
985 .decode_slice = ff_videotoolbox_h264_decode_slice,
986 .end_frame = videotoolbox_h264_end_frame,
987 .init = videotoolbox_common_init,
988 .uninit = videotoolbox_uninit,
989 .priv_data_size = sizeof(VTContext),
992 AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
993 .name = "mpeg1_videotoolbox",
994 .type = AVMEDIA_TYPE_VIDEO,
995 .id = AV_CODEC_ID_MPEG1VIDEO,
996 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
997 .alloc_frame = ff_videotoolbox_alloc_frame,
998 .start_frame = videotoolbox_mpeg_start_frame,
999 .decode_slice = videotoolbox_mpeg_decode_slice,
1000 .end_frame = videotoolbox_mpeg_end_frame,
1001 .init = videotoolbox_common_init,
1002 .uninit = videotoolbox_uninit,
1003 .priv_data_size = sizeof(VTContext),
1006 AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
1007 .name = "mpeg2_videotoolbox",
1008 .type = AVMEDIA_TYPE_VIDEO,
1009 .id = AV_CODEC_ID_MPEG2VIDEO,
1010 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
1011 .alloc_frame = ff_videotoolbox_alloc_frame,
1012 .start_frame = videotoolbox_mpeg_start_frame,
1013 .decode_slice = videotoolbox_mpeg_decode_slice,
1014 .end_frame = videotoolbox_mpeg_end_frame,
1015 .init = videotoolbox_common_init,
1016 .uninit = videotoolbox_uninit,
1017 .priv_data_size = sizeof(VTContext),
1020 AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
1021 .name = "mpeg4_videotoolbox",
1022 .type = AVMEDIA_TYPE_VIDEO,
1023 .id = AV_CODEC_ID_MPEG4,
1024 .pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX,
1025 .alloc_frame = ff_videotoolbox_alloc_frame,
1026 .start_frame = videotoolbox_mpeg_start_frame,
1027 .decode_slice = videotoolbox_mpeg_decode_slice,
1028 .end_frame = videotoolbox_mpeg_end_frame,
1029 .init = videotoolbox_common_init,
1030 .uninit = videotoolbox_uninit,
1031 .priv_data_size = sizeof(VTContext),
1034 AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
1036 AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
1039 ret->output_callback = videotoolbox_decoder_callback;
1040 ret->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1046 int av_videotoolbox_default_init(AVCodecContext *avctx)
1048 return av_videotoolbox_default_init2(avctx, NULL);
1051 int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
1053 avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context();
1054 if (!avctx->hwaccel_context)
1055 return AVERROR(ENOMEM);
1056 return videotoolbox_default_init(avctx);
1059 void av_videotoolbox_default_free(AVCodecContext *avctx)
1062 videotoolbox_default_free(avctx);
1063 av_freep(&avctx->hwaccel_context);
1065 #endif /* CONFIG_VIDEOTOOLBOX */