X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libavcodec%2Fvideotoolbox.c;h=d29607363c0988edacab0492787d02e5aa2104b7;hb=aea0f06db74c1f924429d3cf9ff9f803d00feaf5;hp=ecb2502c1f3b8213d15c799ad0e47f97bcafa8cc;hpb=b125822d1d07e305c0012ef67b8be0cb9261ee9f;p=ffmpeg diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c index ecb2502c1f3..d29607363c0 100644 --- a/libavcodec/videotoolbox.c +++ b/libavcodec/videotoolbox.c @@ -27,6 +27,7 @@ #include "libavutil/avutil.h" #include "libavutil/hwcontext.h" #include "bytestream.h" +#include "decode.h" #include "h264dec.h" #include "hevcdec.h" #include "mpegvideo.h" @@ -70,6 +71,10 @@ static int videotoolbox_buffer_copy(VTContext *vtctx, int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame) { + int ret = ff_attach_decode_data(frame); + if (ret < 0) + return ret; + frame->width = avctx->width; frame->height = avctx->height; frame->format = avctx->pix_fmt; @@ -85,7 +90,8 @@ int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame) CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx) { - H264Context *h = avctx->priv_data; + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + H264Context *h = avctx->priv_data; CFDataRef data = NULL; uint8_t *p; int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size; @@ -111,6 +117,10 @@ CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx) p += 3 + h->ps.pps->data_size; av_assert0(p - vt_extradata == vt_extradata_size); + // save sps header (profile/level) used to create decoder session, + // so we can detect changes and recreate it. + memcpy(vtctx->sps, h->ps.sps->data + 1, 3); + data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size); av_free(vt_extradata); return data; @@ -300,8 +310,6 @@ int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx, VTContext *vtctx = avctx->internal->hwaccel_priv_data; H264Context *h = avctx->priv_data; - vtctx->bitstream_size = 0; - if (h->is_avc == 1) { return videotoolbox_buffer_copy(vtctx, buffer, size); } @@ -309,6 +317,24 @@ int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx, return 0; } +static int videotoolbox_h264_decode_params(AVCodecContext *avctx, + int type, + const uint8_t *buffer, + uint32_t size) +{ + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + + if (type == H264_NAL_SPS) { + if (size > 4 && memcmp(vtctx->sps, buffer + 1, 3) != 0) { + vtctx->reconfig_needed = true; + memcpy(vtctx->sps, buffer + 1, 3); + } + } + + // pass-through SPS/PPS changes to the decoder + return ff_videotoolbox_h264_decode_slice(avctx, buffer, size); +} + int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) @@ -574,123 +600,25 @@ static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx) return status; } -static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame) -{ - int status; - AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx); - VTContext *vtctx = avctx->internal->hwaccel_priv_data; - - if (!videotoolbox->session || !vtctx->bitstream) - return AVERROR_INVALIDDATA; - - status = videotoolbox_session_decode_frame(avctx); - - if (status) { - av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status); - return AVERROR_UNKNOWN; - } - - if (!vtctx->frame) - return AVERROR_UNKNOWN; - - return videotoolbox_buffer_create(avctx, frame); -} - -static int videotoolbox_h264_end_frame(AVCodecContext *avctx) -{ - H264Context *h = avctx->priv_data; - AVFrame *frame = h->cur_pic_ptr->f; - - return videotoolbox_common_end_frame(avctx, frame); -} - -static int videotoolbox_hevc_end_frame(AVCodecContext *avctx) -{ - HEVCContext *h = avctx->priv_data; - AVFrame *frame = h->ref->frame; - VTContext *vtctx = avctx->internal->hwaccel_priv_data; - int ret; - - ret = videotoolbox_common_end_frame(avctx, frame); - vtctx->bitstream_size = 0; - return ret; -} - -static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx, - const uint8_t *buffer, - uint32_t size) -{ - VTContext *vtctx = avctx->internal->hwaccel_priv_data; - - return videotoolbox_buffer_copy(vtctx, buffer, size); -} - -static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx, - const uint8_t *buffer, - uint32_t size) -{ - return 0; -} - -static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx) -{ - MpegEncContext *s = avctx->priv_data; - AVFrame *frame = s->current_picture_ptr->f; - - return videotoolbox_common_end_frame(avctx, frame); -} - -static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type, - AVCodecContext *avctx) +static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type, + CFDictionaryRef decoder_spec, + int width, + int height) { - CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault, - 0, - &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); - - CFDictionarySetValue(config_info, - kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, - kCFBooleanTrue); - - if (avctx->extradata_size) { - CFMutableDictionaryRef avc_info; - CFDataRef data = NULL; - - avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault, - 1, - &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); - - switch (codec_type) { - case kCMVideoCodecType_MPEG4Video : - data = videotoolbox_esds_extradata_create(avctx); - if (data) - CFDictionarySetValue(avc_info, CFSTR("esds"), data); - break; - case kCMVideoCodecType_H264 : - data = ff_videotoolbox_avcc_extradata_create(avctx); - if (data) - CFDictionarySetValue(avc_info, CFSTR("avcC"), data); - break; - case kCMVideoCodecType_HEVC : - data = ff_videotoolbox_hvcc_extradata_create(avctx); - if (data) - CFDictionarySetValue(avc_info, CFSTR("hvcC"), data); - break; - default: - break; - } + CMFormatDescriptionRef cm_fmt_desc; + OSStatus status; - CFDictionarySetValue(config_info, - kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, - avc_info); + status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault, + codec_type, + width, + height, + decoder_spec, // Dictionary of extension + &cm_fmt_desc); - if (data) - CFRelease(data); + if (status) + return NULL; - CFRelease(avc_info); - } - return config_info; + return cm_fmt_desc; } static CFDictionaryRef videotoolbox_buffer_attributes_create(int width, @@ -735,28 +663,59 @@ static CFDictionaryRef videotoolbox_buffer_attributes_create(int width, return buffer_attributes; } -static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type, - CFDictionaryRef decoder_spec, - int width, - int height) +static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type, + AVCodecContext *avctx) { - CMFormatDescriptionRef cm_fmt_desc; - OSStatus status; + CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault, + 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); - status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault, - codec_type, - width, - height, - decoder_spec, // Dictionary of extension - &cm_fmt_desc); + CFDictionarySetValue(config_info, + kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder, + kCFBooleanTrue); - if (status) - return NULL; + CFMutableDictionaryRef avc_info; + CFDataRef data = NULL; - return cm_fmt_desc; + avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault, + 1, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + switch (codec_type) { + case kCMVideoCodecType_MPEG4Video : + if (avctx->extradata_size) + data = videotoolbox_esds_extradata_create(avctx); + if (data) + CFDictionarySetValue(avc_info, CFSTR("esds"), data); + break; + case kCMVideoCodecType_H264 : + data = ff_videotoolbox_avcc_extradata_create(avctx); + if (data) + CFDictionarySetValue(avc_info, CFSTR("avcC"), data); + break; + case kCMVideoCodecType_HEVC : + data = ff_videotoolbox_hvcc_extradata_create(avctx); + if (data) + CFDictionarySetValue(avc_info, CFSTR("hvcC"), data); + break; + default: + break; + } + + CFDictionarySetValue(config_info, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, + avc_info); + + if (data) + CFRelease(data); + + CFRelease(avc_info); + return config_info; } -static int videotoolbox_default_init(AVCodecContext *avctx) +static int videotoolbox_start(AVCodecContext *avctx) { AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx); OSStatus status; @@ -794,6 +753,11 @@ static int videotoolbox_default_init(AVCodecContext *avctx) decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx); + if (!decoder_spec) { + av_log(avctx, AV_LOG_ERROR, "decoder specification creation failed\n"); + return -1; + } + videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type, decoder_spec, avctx->width, @@ -835,7 +799,7 @@ static int videotoolbox_default_init(AVCodecContext *avctx) case kVTVideoDecoderMalfunctionErr: av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n"); return AVERROR(EINVAL); - case kVTVideoDecoderBadDataErr : + case kVTVideoDecoderBadDataErr: av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n"); return AVERROR_INVALIDDATA; case 0: @@ -846,19 +810,123 @@ static int videotoolbox_default_init(AVCodecContext *avctx) } } -static void videotoolbox_default_free(AVCodecContext *avctx) +static void videotoolbox_stop(AVCodecContext *avctx) { AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx); + if (!videotoolbox) + return; - if (videotoolbox) { - if (videotoolbox->cm_fmt_desc) - CFRelease(videotoolbox->cm_fmt_desc); + if (videotoolbox->cm_fmt_desc) { + CFRelease(videotoolbox->cm_fmt_desc); + videotoolbox->cm_fmt_desc = NULL; + } - if (videotoolbox->session) { - VTDecompressionSessionInvalidate(videotoolbox->session); - CFRelease(videotoolbox->session); + if (videotoolbox->session) { + VTDecompressionSessionInvalidate(videotoolbox->session); + CFRelease(videotoolbox->session); + videotoolbox->session = NULL; + } +} + +static const char *videotoolbox_error_string(OSStatus status) +{ + switch (status) { + case kVTVideoDecoderBadDataErr: + return "bad data"; + case kVTVideoDecoderMalfunctionErr: + return "decoder malfunction"; + case kVTInvalidSessionErr: + return "invalid session"; + } + return "unknown"; +} + +static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame) +{ + OSStatus status; + AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx); + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + + if (vtctx->reconfig_needed == true) { + vtctx->reconfig_needed = false; + av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox decoder needs reconfig, restarting..\n"); + videotoolbox_stop(avctx); + if (videotoolbox_start(avctx) != 0) { + return AVERROR_EXTERNAL; } } + + if (!videotoolbox->session || !vtctx->bitstream || !vtctx->bitstream_size) + return AVERROR_INVALIDDATA; + + status = videotoolbox_session_decode_frame(avctx); + if (status != noErr) { + if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr) + vtctx->reconfig_needed = true; + av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (int)status); + return AVERROR_UNKNOWN; + } + + if (!vtctx->frame) { + vtctx->reconfig_needed = true; + return AVERROR_UNKNOWN; + } + + return videotoolbox_buffer_create(avctx, frame); +} + +static int videotoolbox_h264_end_frame(AVCodecContext *avctx) +{ + H264Context *h = avctx->priv_data; + AVFrame *frame = h->cur_pic_ptr->f; + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + int ret = videotoolbox_common_end_frame(avctx, frame); + vtctx->bitstream_size = 0; + return ret; +} + +static int videotoolbox_hevc_decode_params(AVCodecContext *avctx, + int type, + const uint8_t *buffer, + uint32_t size) +{ + return ff_videotoolbox_h264_decode_slice(avctx, buffer, size); +} + +static int videotoolbox_hevc_end_frame(AVCodecContext *avctx) +{ + HEVCContext *h = avctx->priv_data; + AVFrame *frame = h->ref->frame; + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + int ret; + + ret = videotoolbox_common_end_frame(avctx, frame); + vtctx->bitstream_size = 0; + return ret; +} + +static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx, + const uint8_t *buffer, + uint32_t size) +{ + VTContext *vtctx = avctx->internal->hwaccel_priv_data; + + return videotoolbox_buffer_copy(vtctx, buffer, size); +} + +static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx, + const uint8_t *buffer, + uint32_t size) +{ + return 0; +} + +static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx) +{ + MpegEncContext *s = avctx->priv_data; + AVFrame *frame = s->current_picture_ptr->f; + + return videotoolbox_common_end_frame(avctx, frame); } static int videotoolbox_uninit(AVCodecContext *avctx) @@ -870,7 +938,7 @@ static int videotoolbox_uninit(AVCodecContext *avctx) ff_videotoolbox_uninit(avctx); if (vtctx->vt_ctx) - videotoolbox_default_free(avctx); + videotoolbox_stop(avctx); av_buffer_unref(&vtctx->cached_hw_frames_ctx); av_freep(&vtctx->vt_ctx); @@ -936,7 +1004,7 @@ static int videotoolbox_common_init(AVCodecContext *avctx) goto fail; } - err = videotoolbox_default_init(avctx); + err = videotoolbox_start(avctx); if (err < 0) goto fail; @@ -947,6 +1015,19 @@ fail: return err; } +static int videotoolbox_frame_params(AVCodecContext *avctx, + AVBufferRef *hw_frames_ctx) +{ + AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data; + + frames_ctx->format = AV_PIX_FMT_VIDEOTOOLBOX; + frames_ctx->width = avctx->coded_width; + frames_ctx->height = avctx->coded_height; + frames_ctx->sw_format = AV_PIX_FMT_NV12; + + return 0; +} + AVHWAccel ff_h263_videotoolbox_hwaccel = { .name = "h263_videotoolbox", .type = AVMEDIA_TYPE_VIDEO, @@ -956,6 +1037,7 @@ AVHWAccel ff_h263_videotoolbox_hwaccel = { .start_frame = videotoolbox_mpeg_start_frame, .decode_slice = videotoolbox_mpeg_decode_slice, .end_frame = videotoolbox_mpeg_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -969,7 +1051,9 @@ AVHWAccel ff_hevc_videotoolbox_hwaccel = { .alloc_frame = ff_videotoolbox_alloc_frame, .start_frame = ff_videotoolbox_h264_start_frame, .decode_slice = ff_videotoolbox_h264_decode_slice, + .decode_params = videotoolbox_hevc_decode_params, .end_frame = videotoolbox_hevc_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = ff_videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -983,7 +1067,9 @@ AVHWAccel ff_h264_videotoolbox_hwaccel = { .alloc_frame = ff_videotoolbox_alloc_frame, .start_frame = ff_videotoolbox_h264_start_frame, .decode_slice = ff_videotoolbox_h264_decode_slice, + .decode_params = videotoolbox_h264_decode_params, .end_frame = videotoolbox_h264_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -998,6 +1084,7 @@ AVHWAccel ff_mpeg1_videotoolbox_hwaccel = { .start_frame = videotoolbox_mpeg_start_frame, .decode_slice = videotoolbox_mpeg_decode_slice, .end_frame = videotoolbox_mpeg_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -1012,6 +1099,7 @@ AVHWAccel ff_mpeg2_videotoolbox_hwaccel = { .start_frame = videotoolbox_mpeg_start_frame, .decode_slice = videotoolbox_mpeg_decode_slice, .end_frame = videotoolbox_mpeg_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -1026,6 +1114,7 @@ AVHWAccel ff_mpeg4_videotoolbox_hwaccel = { .start_frame = videotoolbox_mpeg_start_frame, .decode_slice = videotoolbox_mpeg_decode_slice, .end_frame = videotoolbox_mpeg_end_frame, + .frame_params = videotoolbox_frame_params, .init = videotoolbox_common_init, .uninit = videotoolbox_uninit, .priv_data_size = sizeof(VTContext), @@ -1053,13 +1142,13 @@ int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext * avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context(); if (!avctx->hwaccel_context) return AVERROR(ENOMEM); - return videotoolbox_default_init(avctx); + return videotoolbox_start(avctx); } void av_videotoolbox_default_free(AVCodecContext *avctx) { - videotoolbox_default_free(avctx); + videotoolbox_stop(avctx); av_freep(&avctx->hwaccel_context); } #endif /* CONFIG_VIDEOTOOLBOX */