2 * This file is part of FFmpeg.
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < 0x0602
22 #define _WIN32_WINNT 0x0602
26 #include "libavutil/imgutils.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/time.h"
31 typedef struct MFContext {
33 int is_video, is_audio;
36 IMFMediaEventGenerator *async_events;
37 DWORD in_stream_id, out_stream_id;
38 MFT_INPUT_STREAM_INFO in_info;
39 MFT_OUTPUT_STREAM_INFO out_info;
40 int out_stream_provides_samples;
41 int draining, draining_done;
43 int async_need_input, async_have_output, async_marker;
44 int64_t reorder_delay;
53 static int mf_choose_output_type(AVCodecContext *avctx);
54 static int mf_setup_context(AVCodecContext *avctx);
56 #define MF_TIMEBASE (AVRational){1, 10000000}
57 // Sentinel value only used by us.
58 #define MF_INVALID_TIME AV_NOPTS_VALUE
60 static int mf_wait_events(AVCodecContext *avctx)
62 MFContext *c = avctx->priv_data;
67 while (!(c->async_need_input || c->async_have_output || c->draining_done || c->async_marker)) {
68 IMFMediaEvent *ev = NULL;
69 MediaEventType ev_id = 0;
70 HRESULT hr = IMFMediaEventGenerator_GetEvent(c->async_events, 0, &ev);
72 av_log(avctx, AV_LOG_ERROR, "IMFMediaEventGenerator_GetEvent() failed: %s\n",
74 return AVERROR_EXTERNAL;
76 IMFMediaEvent_GetType(ev, &ev_id);
78 case ff_METransformNeedInput:
80 c->async_need_input = 1;
82 case ff_METransformHaveOutput:
83 c->async_have_output = 1;
85 case ff_METransformDrainComplete:
88 case ff_METransformMarker:
93 IMFMediaEvent_Release(ev);
99 static AVRational mf_get_tb(AVCodecContext *avctx)
101 if (avctx->pkt_timebase.num > 0 && avctx->pkt_timebase.den > 0)
102 return avctx->pkt_timebase;
103 if (avctx->time_base.num > 0 && avctx->time_base.den > 0)
104 return avctx->time_base;
108 static LONGLONG mf_to_mf_time(AVCodecContext *avctx, int64_t av_pts)
110 if (av_pts == AV_NOPTS_VALUE)
111 return MF_INVALID_TIME;
112 return av_rescale_q(av_pts, mf_get_tb(avctx), MF_TIMEBASE);
115 static void mf_sample_set_pts(AVCodecContext *avctx, IMFSample *sample, int64_t av_pts)
117 LONGLONG stime = mf_to_mf_time(avctx, av_pts);
118 if (stime != MF_INVALID_TIME)
119 IMFSample_SetSampleTime(sample, stime);
122 static int64_t mf_from_mf_time(AVCodecContext *avctx, LONGLONG stime)
124 return av_rescale_q(stime, MF_TIMEBASE, mf_get_tb(avctx));
127 static int64_t mf_sample_get_pts(AVCodecContext *avctx, IMFSample *sample)
130 HRESULT hr = IMFSample_GetSampleTime(sample, &pts);
132 return AV_NOPTS_VALUE;
133 return mf_from_mf_time(avctx, pts);
136 static int mf_enca_output_type_get(AVCodecContext *avctx, IMFMediaType *type)
138 MFContext *c = avctx->priv_data;
142 if (avctx->codec_id != AV_CODEC_ID_MP3 && avctx->codec_id != AV_CODEC_ID_AC3) {
143 hr = IMFAttributes_GetBlobSize(type, &MF_MT_USER_DATA, &sz);
144 if (!FAILED(hr) && sz > 0) {
145 avctx->extradata = av_mallocz(sz + AV_INPUT_BUFFER_PADDING_SIZE);
146 if (!avctx->extradata)
147 return AVERROR(ENOMEM);
148 avctx->extradata_size = sz;
149 hr = IMFAttributes_GetBlob(type, &MF_MT_USER_DATA, avctx->extradata, sz, NULL);
151 return AVERROR_EXTERNAL;
153 if (avctx->codec_id == AV_CODEC_ID_AAC && avctx->extradata_size >= 12) {
154 // Get rid of HEAACWAVEINFO (after wfx field, 12 bytes).
155 avctx->extradata_size = avctx->extradata_size - 12;
156 memmove(avctx->extradata, avctx->extradata + 12, avctx->extradata_size);
161 // I don't know where it's documented that we need this. It happens with the
162 // MS mp3 encoder MFT. The idea for the workaround is taken from NAudio.
163 // (Certainly any lossy codec will have frames much smaller than 1 second.)
164 if (!c->out_info.cbSize && !c->out_stream_provides_samples) {
165 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_AVG_BYTES_PER_SECOND, &sz);
167 av_log(avctx, AV_LOG_VERBOSE, "MFT_OUTPUT_STREAM_INFO.cbSize set to 0, "
168 "assuming %d bytes instead.\n", (int)sz);
169 c->out_info.cbSize = sz;
176 static int mf_encv_output_type_get(AVCodecContext *avctx, IMFMediaType *type)
181 hr = IMFAttributes_GetBlobSize(type, &MF_MT_MPEG_SEQUENCE_HEADER, &sz);
182 if (!FAILED(hr) && sz > 0) {
183 uint8_t *extradata = av_mallocz(sz + AV_INPUT_BUFFER_PADDING_SIZE);
185 return AVERROR(ENOMEM);
186 hr = IMFAttributes_GetBlob(type, &MF_MT_MPEG_SEQUENCE_HEADER, extradata, sz, NULL);
189 return AVERROR_EXTERNAL;
191 av_freep(&avctx->extradata);
192 avctx->extradata = extradata;
193 avctx->extradata_size = sz;
199 static int mf_output_type_get(AVCodecContext *avctx)
201 MFContext *c = avctx->priv_data;
206 hr = IMFTransform_GetOutputCurrentType(c->mft, c->out_stream_id, &type);
208 av_log(avctx, AV_LOG_ERROR, "could not get output type\n");
209 return AVERROR_EXTERNAL;
212 av_log(avctx, AV_LOG_VERBOSE, "final output type:\n");
213 ff_media_type_dump(avctx, type);
217 ret = mf_encv_output_type_get(avctx, type);
218 } else if (c->is_audio) {
219 ret = mf_enca_output_type_get(avctx, type);
223 av_log(avctx, AV_LOG_ERROR, "output type not supported\n");
225 IMFMediaType_Release(type);
229 static int mf_sample_to_avpacket(AVCodecContext *avctx, IMFSample *sample, AVPacket *avpkt)
231 MFContext *c = avctx->priv_data;
235 IMFMediaBuffer *buffer;
240 hr = IMFSample_GetTotalLength(sample, &len);
242 return AVERROR_EXTERNAL;
244 if ((ret = av_new_packet(avpkt, len)) < 0)
247 IMFSample_ConvertToContiguousBuffer(sample, &buffer);
249 return AVERROR_EXTERNAL;
251 hr = IMFMediaBuffer_Lock(buffer, &data, NULL, NULL);
253 IMFMediaBuffer_Release(buffer);
254 return AVERROR_EXTERNAL;
257 memcpy(avpkt->data, data, len);
259 IMFMediaBuffer_Unlock(buffer);
260 IMFMediaBuffer_Release(buffer);
262 avpkt->pts = avpkt->dts = mf_sample_get_pts(avctx, sample);
264 hr = IMFAttributes_GetUINT32(sample, &MFSampleExtension_CleanPoint, &t32);
265 if (c->is_audio || (!FAILED(hr) && t32 != 0))
266 avpkt->flags |= AV_PKT_FLAG_KEY;
268 hr = IMFAttributes_GetUINT64(sample, &MFSampleExtension_DecodeTimestamp, &t);
270 avpkt->dts = mf_from_mf_time(avctx, t);
271 // At least on Qualcomm's HEVC encoder on SD 835, the output dts
272 // starts from the input pts of the first frame, while the output pts
273 // is shifted forward. Therefore, shift the output values back so that
274 // the output pts matches the input.
275 if (c->reorder_delay == AV_NOPTS_VALUE)
276 c->reorder_delay = avpkt->pts - avpkt->dts;
277 avpkt->dts -= c->reorder_delay;
278 avpkt->pts -= c->reorder_delay;
284 static IMFSample *mf_a_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
286 MFContext *c = avctx->priv_data;
291 bps = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->channels;
292 len = frame->nb_samples * bps;
294 sample = ff_create_memory_sample(frame->data[0], len, c->in_info.cbAlignment);
296 IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->nb_samples));
300 static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
302 MFContext *c = avctx->priv_data;
304 IMFMediaBuffer *buffer;
310 size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, avctx->height, 1);
314 sample = ff_create_memory_sample(NULL, size, c->in_info.cbAlignment);
318 hr = IMFSample_GetBufferByIndex(sample, 0, &buffer);
320 IMFSample_Release(sample);
324 hr = IMFMediaBuffer_Lock(buffer, &data, NULL, NULL);
326 IMFMediaBuffer_Release(buffer);
327 IMFSample_Release(sample);
331 ret = av_image_copy_to_buffer((uint8_t *)data, size, (void *)frame->data, frame->linesize,
332 avctx->pix_fmt, avctx->width, avctx->height, 1);
333 IMFMediaBuffer_SetCurrentLength(buffer, size);
334 IMFMediaBuffer_Unlock(buffer);
335 IMFMediaBuffer_Release(buffer);
337 IMFSample_Release(sample);
341 IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->pkt_duration));
346 static IMFSample *mf_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
348 MFContext *c = avctx->priv_data;
352 sample = mf_a_avframe_to_sample(avctx, frame);
354 sample = mf_v_avframe_to_sample(avctx, frame);
358 mf_sample_set_pts(avctx, sample, frame->pts);
363 static int mf_send_sample(AVCodecContext *avctx, IMFSample *sample)
365 MFContext *c = avctx->priv_data;
370 if (c->async_events) {
371 if ((ret = mf_wait_events(avctx)) < 0)
373 if (!c->async_need_input)
374 return AVERROR(EAGAIN);
377 IMFSample_SetUINT32(sample, &MFSampleExtension_Discontinuity, TRUE);
379 hr = IMFTransform_ProcessInput(c->mft, c->in_stream_id, sample, 0);
380 if (hr == MF_E_NOTACCEPTING) {
381 return AVERROR(EAGAIN);
382 } else if (FAILED(hr)) {
383 av_log(avctx, AV_LOG_ERROR, "failed processing input: %s\n", ff_hr_str(hr));
384 return AVERROR_EXTERNAL;
386 c->async_need_input = 0;
387 } else if (!c->draining) {
388 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_COMMAND_DRAIN, 0);
390 av_log(avctx, AV_LOG_ERROR, "failed draining: %s\n", ff_hr_str(hr));
391 // Some MFTs (AC3) will send a frame after each drain command (???), so
392 // this is required to make draining actually terminate.
394 c->async_need_input = 0;
401 static int mf_send_frame(AVCodecContext *avctx, const AVFrame *frame)
403 MFContext *c = avctx->priv_data;
405 IMFSample *sample = NULL;
407 sample = mf_avframe_to_sample(avctx, frame);
409 return AVERROR(ENOMEM);
410 if (c->is_video && c->codec_api) {
411 if (frame->pict_type == AV_PICTURE_TYPE_I || !c->sample_sent)
412 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncVideoForceKeyFrame, FF_VAL_VT_UI4(1));
415 ret = mf_send_sample(avctx, sample);
417 IMFSample_Release(sample);
421 static int mf_receive_sample(AVCodecContext *avctx, IMFSample **out_sample)
423 MFContext *c = avctx->priv_data;
426 MFT_OUTPUT_DATA_BUFFER out_buffers;
434 if (c->async_events) {
435 if ((ret = mf_wait_events(avctx)) < 0)
437 if (!c->async_have_output || c->draining_done) {
443 if (!c->out_stream_provides_samples) {
444 sample = ff_create_memory_sample(NULL, c->out_info.cbSize, c->out_info.cbAlignment);
446 return AVERROR(ENOMEM);
449 out_buffers = (MFT_OUTPUT_DATA_BUFFER) {
450 .dwStreamID = c->out_stream_id,
455 hr = IMFTransform_ProcessOutput(c->mft, 0, 1, &out_buffers, &st);
457 if (out_buffers.pEvents)
458 IMFCollection_Release(out_buffers.pEvents);
461 *out_sample = out_buffers.pSample;
466 if (out_buffers.pSample)
467 IMFSample_Release(out_buffers.pSample);
469 if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) {
471 c->draining_done = 1;
473 } else if (hr == MF_E_TRANSFORM_STREAM_CHANGE) {
474 av_log(avctx, AV_LOG_WARNING, "stream format change\n");
475 ret = mf_choose_output_type(avctx);
476 if (ret == 0) // we don't expect renegotiating the input type
477 ret = AVERROR_EXTERNAL;
479 ret = mf_setup_context(avctx);
481 c->async_have_output = 0;
486 av_log(avctx, AV_LOG_ERROR, "failed processing output: %s\n", ff_hr_str(hr));
487 ret = AVERROR_EXTERNAL;
493 c->async_have_output = 0;
495 if (ret >= 0 && !*out_sample)
496 ret = c->draining_done ? AVERROR_EOF : AVERROR(EAGAIN);
501 static int mf_receive_packet(AVCodecContext *avctx, AVPacket *avpkt)
506 ret = mf_receive_sample(avctx, &sample);
510 ret = mf_sample_to_avpacket(avctx, sample, avpkt);
511 IMFSample_Release(sample);
516 // Most encoders seem to enumerate supported audio formats on the output types,
517 // at least as far as channel configuration and sample rate is concerned. Pick
518 // the one which seems to match best.
519 static int64_t mf_enca_output_score(AVCodecContext *avctx, IMFMediaType *type)
521 MFContext *c = avctx->priv_data;
527 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_SAMPLES_PER_SECOND, &t);
528 if (!FAILED(hr) && t == avctx->sample_rate)
531 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_NUM_CHANNELS, &t);
532 if (!FAILED(hr) && t == avctx->channels)
535 hr = IMFAttributes_GetGUID(type, &MF_MT_SUBTYPE, &tg);
537 if (IsEqualGUID(&c->main_subtype, &tg))
541 // Select the bitrate (lowest priority).
542 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_AVG_BYTES_PER_SECOND, &t);
544 int diff = (int)t - avctx->bit_rate / 8;
546 score |= (1LL << 31) - diff; // prefer lower bitrate
548 score |= (1LL << 30) + diff; // prefer higher bitrate
552 hr = IMFAttributes_GetUINT32(type, &MF_MT_AAC_PAYLOAD_TYPE, &t);
553 if (!FAILED(hr) && t != 0)
559 static int mf_enca_output_adjust(AVCodecContext *avctx, IMFMediaType *type)
561 // (some decoders allow adjusting this freely, but it can also cause failure
562 // to set the output type - so it's commented for being too fragile)
563 //IMFAttributes_SetUINT32(type, &MF_MT_AUDIO_AVG_BYTES_PER_SECOND, avctx->bit_rate / 8);
564 //IMFAttributes_SetUINT32(type, &MF_MT_AVG_BITRATE, avctx->bit_rate);
569 static int64_t mf_enca_input_score(AVCodecContext *avctx, IMFMediaType *type)
575 enum AVSampleFormat sformat = ff_media_type_to_sample_fmt((IMFAttributes *)type);
576 if (sformat == AV_SAMPLE_FMT_NONE)
577 return -1; // can not use
579 if (sformat == avctx->sample_fmt)
582 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_SAMPLES_PER_SECOND, &t);
583 if (!FAILED(hr) && t == avctx->sample_rate)
586 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_NUM_CHANNELS, &t);
587 if (!FAILED(hr) && t == avctx->channels)
593 static int mf_enca_input_adjust(AVCodecContext *avctx, IMFMediaType *type)
598 enum AVSampleFormat sformat = ff_media_type_to_sample_fmt((IMFAttributes *)type);
599 if (sformat != avctx->sample_fmt) {
600 av_log(avctx, AV_LOG_ERROR, "unsupported input sample format set\n");
601 return AVERROR(EINVAL);
604 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_SAMPLES_PER_SECOND, &t);
605 if (FAILED(hr) || t != avctx->sample_rate) {
606 av_log(avctx, AV_LOG_ERROR, "unsupported input sample rate set\n");
607 return AVERROR(EINVAL);
610 hr = IMFAttributes_GetUINT32(type, &MF_MT_AUDIO_NUM_CHANNELS, &t);
611 if (FAILED(hr) || t != avctx->channels) {
612 av_log(avctx, AV_LOG_ERROR, "unsupported input channel number set\n");
613 return AVERROR(EINVAL);
619 static int64_t mf_encv_output_score(AVCodecContext *avctx, IMFMediaType *type)
621 MFContext *c = avctx->priv_data;
626 hr = IMFAttributes_GetGUID(type, &MF_MT_SUBTYPE, &tg);
628 if (IsEqualGUID(&c->main_subtype, &tg))
635 static int mf_encv_output_adjust(AVCodecContext *avctx, IMFMediaType *type)
637 MFContext *c = avctx->priv_data;
638 AVRational framerate;
640 ff_MFSetAttributeSize((IMFAttributes *)type, &MF_MT_FRAME_SIZE, avctx->width, avctx->height);
641 IMFAttributes_SetUINT32(type, &MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
643 if (avctx->framerate.num > 0 && avctx->framerate.den > 0) {
644 framerate = avctx->framerate;
646 framerate = av_inv_q(avctx->time_base);
647 framerate.den *= avctx->ticks_per_frame;
650 ff_MFSetAttributeRatio((IMFAttributes *)type, &MF_MT_FRAME_RATE, framerate.num, framerate.den);
652 // (MS HEVC supports eAVEncH265VProfile_Main_420_8 only.)
653 if (avctx->codec_id == AV_CODEC_ID_H264) {
654 UINT32 profile = eAVEncH264VProfile_Base;
655 switch (avctx->profile) {
656 case FF_PROFILE_H264_MAIN:
657 profile = eAVEncH264VProfile_Main;
659 case FF_PROFILE_H264_HIGH:
660 profile = eAVEncH264VProfile_High;
663 IMFAttributes_SetUINT32(type, &MF_MT_MPEG2_PROFILE, profile);
666 IMFAttributes_SetUINT32(type, &MF_MT_AVG_BITRATE, avctx->bit_rate);
668 // Note that some of the ICodecAPI options must be set before SetOutputType.
671 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncCommonMeanBitRate, FF_VAL_VT_UI4(avctx->bit_rate));
673 if (c->opt_enc_rc >= 0)
674 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncCommonRateControlMode, FF_VAL_VT_UI4(c->opt_enc_rc));
676 if (c->opt_enc_quality >= 0)
677 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncCommonQuality, FF_VAL_VT_UI4(c->opt_enc_quality));
679 // Always set the number of b-frames. Qualcomm's HEVC encoder on SD835
680 // defaults this to 1, and that setting is buggy with many of the
681 // rate control modes. (0 or 2 b-frames works fine with most rate
682 // control modes, but 2 seems buggy with the u_vbr mode.) Setting
683 // "scenario" to "camera_record" sets it in CFR mode (where the default
684 // is VFR), which makes the encoder avoid dropping frames.
685 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncMPVDefaultBPictureCount, FF_VAL_VT_UI4(avctx->max_b_frames));
686 avctx->has_b_frames = avctx->max_b_frames > 0;
688 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVEncH264CABACEnable, FF_VAL_VT_BOOL(1));
690 if (c->opt_enc_scenario >= 0)
691 ICodecAPI_SetValue(c->codec_api, &ff_CODECAPI_AVScenarioInfo, FF_VAL_VT_UI4(c->opt_enc_scenario));
697 static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type)
699 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
700 if (pix_fmt != avctx->pix_fmt)
701 return -1; // can not use
706 static int mf_encv_input_adjust(AVCodecContext *avctx, IMFMediaType *type)
708 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
709 if (pix_fmt != avctx->pix_fmt) {
710 av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
711 return AVERROR(EINVAL);
714 //ff_MFSetAttributeSize((IMFAttributes *)type, &MF_MT_FRAME_SIZE, avctx->width, avctx->height);
719 static int mf_choose_output_type(AVCodecContext *avctx)
721 MFContext *c = avctx->priv_data;
724 IMFMediaType *out_type = NULL;
725 int64_t out_type_score = -1;
726 int out_type_index = -1;
729 av_log(avctx, AV_LOG_VERBOSE, "output types:\n");
734 hr = IMFTransform_GetOutputAvailableType(c->mft, c->out_stream_id, n, &type);
735 if (hr == MF_E_NO_MORE_TYPES || hr == E_NOTIMPL)
737 if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) {
738 av_log(avctx, AV_LOG_VERBOSE, "(need to set input type)\n");
743 av_log(avctx, AV_LOG_ERROR, "error getting output type: %s\n", ff_hr_str(hr));
744 ret = AVERROR_EXTERNAL;
748 av_log(avctx, AV_LOG_VERBOSE, "output type %d:\n", n);
749 ff_media_type_dump(avctx, type);
752 score = mf_encv_output_score(avctx, type);
753 } else if (c->is_audio) {
754 score = mf_enca_output_score(avctx, type);
757 if (score > out_type_score) {
759 IMFMediaType_Release(out_type);
761 out_type_score = score;
763 IMFMediaType_AddRef(out_type);
766 IMFMediaType_Release(type);
770 av_log(avctx, AV_LOG_VERBOSE, "picking output type %d.\n", out_type_index);
772 hr = MFCreateMediaType(&out_type);
774 ret = AVERROR(ENOMEM);
781 ret = mf_encv_output_adjust(avctx, out_type);
782 } else if (c->is_audio) {
783 ret = mf_enca_output_adjust(avctx, out_type);
787 av_log(avctx, AV_LOG_VERBOSE, "setting output type:\n");
788 ff_media_type_dump(avctx, out_type);
790 hr = IMFTransform_SetOutputType(c->mft, c->out_stream_id, out_type, 0);
793 } else if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) {
794 av_log(avctx, AV_LOG_VERBOSE, "rejected - need to set input type\n");
797 av_log(avctx, AV_LOG_ERROR, "could not set output type (%s)\n", ff_hr_str(hr));
798 ret = AVERROR_EXTERNAL;
804 IMFMediaType_Release(out_type);
808 static int mf_choose_input_type(AVCodecContext *avctx)
810 MFContext *c = avctx->priv_data;
813 IMFMediaType *in_type = NULL;
814 int64_t in_type_score = -1;
815 int in_type_index = -1;
818 av_log(avctx, AV_LOG_VERBOSE, "input types:\n");
820 IMFMediaType *type = NULL;
823 hr = IMFTransform_GetInputAvailableType(c->mft, c->in_stream_id, n, &type);
824 if (hr == MF_E_NO_MORE_TYPES || hr == E_NOTIMPL)
826 if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) {
827 av_log(avctx, AV_LOG_VERBOSE, "(need to set output type 1)\n");
832 av_log(avctx, AV_LOG_ERROR, "error getting input type: %s\n", ff_hr_str(hr));
833 ret = AVERROR_EXTERNAL;
837 av_log(avctx, AV_LOG_VERBOSE, "input type %d:\n", n);
838 ff_media_type_dump(avctx, type);
841 score = mf_encv_input_score(avctx, type);
842 } else if (c->is_audio) {
843 score = mf_enca_input_score(avctx, type);
846 if (score > in_type_score) {
848 IMFMediaType_Release(in_type);
850 in_type_score = score;
852 IMFMediaType_AddRef(in_type);
855 IMFMediaType_Release(type);
859 av_log(avctx, AV_LOG_VERBOSE, "picking input type %d.\n", in_type_index);
861 // Some buggy MFTs (WMA encoder) fail to return MF_E_TRANSFORM_TYPE_NOT_SET.
862 av_log(avctx, AV_LOG_VERBOSE, "(need to set output type 2)\n");
869 ret = mf_encv_input_adjust(avctx, in_type);
870 } else if (c->is_audio) {
871 ret = mf_enca_input_adjust(avctx, in_type);
875 av_log(avctx, AV_LOG_VERBOSE, "setting input type:\n");
876 ff_media_type_dump(avctx, in_type);
878 hr = IMFTransform_SetInputType(c->mft, c->in_stream_id, in_type, 0);
881 } else if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) {
882 av_log(avctx, AV_LOG_VERBOSE, "rejected - need to set output type\n");
885 av_log(avctx, AV_LOG_ERROR, "could not set input type (%s)\n", ff_hr_str(hr));
886 ret = AVERROR_EXTERNAL;
892 IMFMediaType_Release(in_type);
896 static int mf_negotiate_types(AVCodecContext *avctx)
898 // This follows steps 1-5 on:
899 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa965264(v=vs.85).aspx
900 // If every MFT implementer does this correctly, this loop should at worst
902 int need_input = 1, need_output = 1;
904 for (n = 0; n < 2 && (need_input || need_output); n++) {
906 ret = mf_choose_input_type(avctx);
909 need_input = ret < 1;
910 ret = mf_choose_output_type(avctx);
913 need_output = ret < 1;
915 if (need_input || need_output) {
916 av_log(avctx, AV_LOG_ERROR, "format negotiation failed (%d/%d)\n",
917 need_input, need_output);
918 return AVERROR_EXTERNAL;
923 static int mf_setup_context(AVCodecContext *avctx)
925 MFContext *c = avctx->priv_data;
929 hr = IMFTransform_GetInputStreamInfo(c->mft, c->in_stream_id, &c->in_info);
931 return AVERROR_EXTERNAL;
932 av_log(avctx, AV_LOG_VERBOSE, "in_info: size=%d, align=%d\n",
933 (int)c->in_info.cbSize, (int)c->in_info.cbAlignment);
935 hr = IMFTransform_GetOutputStreamInfo(c->mft, c->out_stream_id, &c->out_info);
937 return AVERROR_EXTERNAL;
938 c->out_stream_provides_samples =
939 (c->out_info.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES) ||
940 (c->out_info.dwFlags & MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES);
941 av_log(avctx, AV_LOG_VERBOSE, "out_info: size=%d, align=%d%s\n",
942 (int)c->out_info.cbSize, (int)c->out_info.cbAlignment,
943 c->out_stream_provides_samples ? " (provides samples)" : "");
945 if ((ret = mf_output_type_get(avctx)) < 0)
951 static int mf_unlock_async(AVCodecContext *avctx)
953 MFContext *c = avctx->priv_data;
955 IMFAttributes *attrs;
957 int res = AVERROR_EXTERNAL;
959 // For hw encoding we unfortunately need to use async mode, otherwise
960 // play it safe and avoid it.
961 if (!(c->is_video && c->opt_enc_hw))
964 hr = IMFTransform_GetAttributes(c->mft, &attrs);
966 av_log(avctx, AV_LOG_ERROR, "error retrieving MFT attributes: %s\n", ff_hr_str(hr));
970 hr = IMFAttributes_GetUINT32(attrs, &MF_TRANSFORM_ASYNC, &v);
972 av_log(avctx, AV_LOG_ERROR, "error querying async: %s\n", ff_hr_str(hr));
977 av_log(avctx, AV_LOG_ERROR, "hardware MFT is not async\n");
981 hr = IMFAttributes_SetUINT32(attrs, &MF_TRANSFORM_ASYNC_UNLOCK, TRUE);
983 av_log(avctx, AV_LOG_ERROR, "could not set async unlock: %s\n", ff_hr_str(hr));
987 hr = IMFTransform_QueryInterface(c->mft, &IID_IMFMediaEventGenerator, (void **)&c->async_events);
989 av_log(avctx, AV_LOG_ERROR, "could not get async interface\n");
996 IMFAttributes_Release(attrs);
1000 static int mf_create(void *log, IMFTransform **mft, const AVCodec *codec, int use_hw)
1002 int is_audio = codec->type == AVMEDIA_TYPE_AUDIO;
1003 const CLSID *subtype = ff_codec_to_mf_subtype(codec->id);
1004 MFT_REGISTER_TYPE_INFO reg = {0};
1011 return AVERROR(ENOSYS);
1013 reg.guidSubtype = *subtype;
1016 reg.guidMajorType = MFMediaType_Audio;
1017 category = MFT_CATEGORY_AUDIO_ENCODER;
1019 reg.guidMajorType = MFMediaType_Video;
1020 category = MFT_CATEGORY_VIDEO_ENCODER;
1023 if ((ret = ff_instantiate_mf(log, category, NULL, ®, use_hw, mft)) < 0)
1029 static int mf_init(AVCodecContext *avctx)
1031 MFContext *c = avctx->priv_data;
1034 const CLSID *subtype = ff_codec_to_mf_subtype(avctx->codec_id);
1037 c->is_audio = avctx->codec_type == AVMEDIA_TYPE_AUDIO;
1038 c->is_video = !c->is_audio;
1039 c->reorder_delay = AV_NOPTS_VALUE;
1041 if (c->is_video && c->opt_enc_hw)
1045 return AVERROR(ENOSYS);
1047 c->main_subtype = *subtype;
1049 if ((ret = mf_create(avctx, &c->mft, avctx->codec, use_hw)) < 0)
1052 if ((ret = mf_unlock_async(avctx)) < 0)
1055 hr = IMFTransform_QueryInterface(c->mft, &IID_ICodecAPI, (void **)&c->codec_api);
1057 av_log(avctx, AV_LOG_VERBOSE, "MFT supports ICodecAPI.\n");
1060 hr = IMFTransform_GetStreamIDs(c->mft, 1, &c->in_stream_id, 1, &c->out_stream_id);
1061 if (hr == E_NOTIMPL) {
1062 c->in_stream_id = c->out_stream_id = 0;
1063 } else if (FAILED(hr)) {
1064 av_log(avctx, AV_LOG_ERROR, "could not get stream IDs (%s)\n", ff_hr_str(hr));
1065 return AVERROR_EXTERNAL;
1068 if ((ret = mf_negotiate_types(avctx)) < 0)
1071 if ((ret = mf_setup_context(avctx)) < 0)
1074 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
1076 av_log(avctx, AV_LOG_ERROR, "could not start streaming (%s)\n", ff_hr_str(hr));
1077 return AVERROR_EXTERNAL;
1080 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
1082 av_log(avctx, AV_LOG_ERROR, "could not start stream (%s)\n", ff_hr_str(hr));
1083 return AVERROR_EXTERNAL;
1086 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER && c->async_events &&
1087 c->is_video && !avctx->extradata) {
1088 int sleep = 10000, total = 0;
1089 av_log(avctx, AV_LOG_VERBOSE, "Awaiting extradata\n");
1090 while (total < 70*1000) {
1091 // The Qualcomm H264 encoder on SD835 doesn't provide extradata
1092 // immediately, but it becomes available soon after init (without
1093 // any waitable event). In practice, it's available after less
1094 // than 10 ms, but wait for up to 70 ms before giving up.
1095 // Some encoders (Qualcomm's HEVC encoder on SD835, some versions
1096 // of the QSV H264 encoder at least) don't provide extradata this
1097 // way at all, not even after encoding a frame - it's only
1098 // available prepended to frames.
1101 mf_output_type_get(avctx);
1102 if (avctx->extradata)
1106 av_log(avctx, AV_LOG_VERBOSE, "%s extradata in %d ms\n",
1107 avctx->extradata ? "Got" : "Didn't get", total / 1000);
1113 static int mf_close(AVCodecContext *avctx)
1115 MFContext *c = avctx->priv_data;
1118 ICodecAPI_Release(c->codec_api);
1120 if (c->async_events)
1121 IMFMediaEventGenerator_Release(c->async_events);
1123 ff_free_mf(&c->mft);
1125 av_freep(&avctx->extradata);
1126 avctx->extradata_size = 0;
1131 #define OFFSET(x) offsetof(MFContext, x)
1133 #define MF_ENCODER(MEDIATYPE, NAME, ID, OPTS, EXTRA) \
1134 static const AVClass ff_ ## NAME ## _mf_encoder_class = { \
1135 .class_name = #NAME "_mf", \
1136 .item_name = av_default_item_name, \
1138 .version = LIBAVUTIL_VERSION_INT, \
1140 AVCodec ff_ ## NAME ## _mf_encoder = { \
1141 .priv_class = &ff_ ## NAME ## _mf_encoder_class, \
1142 .name = #NAME "_mf", \
1143 .long_name = NULL_IF_CONFIG_SMALL(#ID " via MediaFoundation"), \
1144 .type = AVMEDIA_TYPE_ ## MEDIATYPE, \
1145 .id = AV_CODEC_ID_ ## ID, \
1146 .priv_data_size = sizeof(MFContext), \
1148 .close = mf_close, \
1149 .send_frame = mf_send_frame, \
1150 .receive_packet = mf_receive_packet, \
1152 .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HYBRID, \
1153 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | \
1154 FF_CODEC_CAP_INIT_CLEANUP, \
1158 .sample_fmts = (const enum AVSampleFormat[]){ AV_SAMPLE_FMT_S16, \
1159 AV_SAMPLE_FMT_NONE },
1161 MF_ENCODER(AUDIO, aac, AAC, NULL, AFMTS);
1162 MF_ENCODER(AUDIO, ac3, AC3, NULL, AFMTS);
1163 MF_ENCODER(AUDIO, mp3, MP3, NULL, AFMTS);
1165 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
1166 static const AVOption venc_opts[] = {
1167 {"rate_control", "Select rate control mode", OFFSET(opt_enc_rc), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VE, "rate_control"},
1168 { "default", "Default mode", 0, AV_OPT_TYPE_CONST, {.i64 = -1}, 0, 0, VE, "rate_control"},
1169 { "cbr", "CBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_CBR}, 0, 0, VE, "rate_control"},
1170 { "pc_vbr", "Peak constrained VBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_PeakConstrainedVBR}, 0, 0, VE, "rate_control"},
1171 { "u_vbr", "Unconstrained VBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_UnconstrainedVBR}, 0, 0, VE, "rate_control"},
1172 { "quality", "Quality mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_Quality}, 0, 0, VE, "rate_control" },
1173 // The following rate_control modes require Windows 8.
1174 { "ld_vbr", "Low delay VBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_LowDelayVBR}, 0, 0, VE, "rate_control"},
1175 { "g_vbr", "Global VBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_GlobalVBR}, 0, 0, VE, "rate_control" },
1176 { "gld_vbr", "Global low delay VBR mode", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVEncCommonRateControlMode_GlobalLowDelayVBR}, 0, 0, VE, "rate_control"},
1178 {"scenario", "Select usage scenario", OFFSET(opt_enc_scenario), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VE, "scenario"},
1179 { "default", "Default scenario", 0, AV_OPT_TYPE_CONST, {.i64 = -1}, 0, 0, VE, "scenario"},
1180 { "display_remoting", "Display remoting", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_DisplayRemoting}, 0, 0, VE, "scenario"},
1181 { "video_conference", "Video conference", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_VideoConference}, 0, 0, VE, "scenario"},
1182 { "archive", "Archive", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_Archive}, 0, 0, VE, "scenario"},
1183 { "live_streaming", "Live streaming", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_LiveStreaming}, 0, 0, VE, "scenario"},
1184 { "camera_record", "Camera record", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_CameraRecord}, 0, 0, VE, "scenario"},
1185 { "display_remoting_with_feature_map", "Display remoting with feature map", 0, AV_OPT_TYPE_CONST, {.i64 = ff_eAVScenarioInfo_DisplayRemotingWithFeatureMap}, 0, 0, VE, "scenario"},
1187 {"quality", "Quality", OFFSET(opt_enc_quality), AV_OPT_TYPE_INT, {.i64 = -1}, -1, 100, VE},
1188 {"hw_encoding", "Force hardware encoding", OFFSET(opt_enc_hw), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, VE},
1193 .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_NV12, \
1194 AV_PIX_FMT_YUV420P, \
1197 MF_ENCODER(VIDEO, h264, H264, venc_opts, VFMTS);
1198 MF_ENCODER(VIDEO, hevc, HEVC, venc_opts, VFMTS);