2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
33 #include "filter/audio_filter.h"
35 #include <common/param.h>
36 #include <common/diagnostics/graph.h>
37 #include <common/future.h>
39 #include <core/frame/draw_frame.h>
40 #include <core/help/help_repository.h>
41 #include <core/help/help_sink.h>
42 #include <core/producer/media_info/media_info.h>
43 #include <core/producer/framerate/framerate_producer.h>
44 #include <core/frame/frame_factory.h>
49 namespace caspar { namespace ffmpeg {
50 struct seek_out_of_range : virtual user_error {};
52 std::wstring get_relative_or_original(
53 const std::wstring& filename,
54 const boost::filesystem::path& relative_to)
56 boost::filesystem::path file(filename);
57 auto result = file.filename().wstring();
59 boost::filesystem::path current_path = file;
63 current_path = current_path.parent_path();
65 if (boost::filesystem::equivalent(current_path, relative_to))
68 if (current_path.empty())
71 result = current_path.filename().wstring() + L"/" + result;
77 struct ffmpeg_producer : public core::frame_producer_base
79 spl::shared_ptr<core::monitor::subject> monitor_subject_;
80 const std::wstring filename_;
81 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
83 const spl::shared_ptr<diagnostics::graph> graph_;
86 const spl::shared_ptr<core::frame_factory> frame_factory_;
88 std::shared_ptr<void> initial_logger_disabler_;
90 core::constraints constraints_;
93 std::unique_ptr<video_decoder> video_decoder_;
94 std::vector<std::unique_ptr<audio_decoder>> audio_decoders_;
95 std::unique_ptr<frame_muxer> muxer_;
97 const boost::rational<int> framerate_;
98 const uint32_t start_;
99 const uint32_t length_;
100 const bool thumbnail_mode_;
102 core::draw_frame last_frame_;
104 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
106 int64_t frame_number_ = 0;
107 uint32_t file_frame_number_ = 0;
109 explicit ffmpeg_producer(
110 const spl::shared_ptr<core::frame_factory>& frame_factory,
111 const core::video_format_desc& format_desc,
112 const std::wstring& url_or_file,
113 const std::wstring& filter,
118 const std::wstring& custom_channel_order,
119 const ffmpeg_options& vid_params)
120 : filename_(url_or_file)
121 , frame_factory_(frame_factory)
122 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
123 , input_(graph_, url_or_file, loop, start, length, thumbnail_mode, vid_params)
124 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
127 , thumbnail_mode_(thumbnail_mode)
128 , last_frame_(core::draw_frame::empty())
131 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
132 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
133 diagnostics::register_graph(graph_);
137 video_decoder_.reset(new video_decoder(input_.context()));
138 if (!thumbnail_mode_)
139 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
141 constraints_.width.set(video_decoder_->width());
142 constraints_.height.set(video_decoder_->height());
144 catch (averror_stream_not_found&)
146 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
150 if (!thumbnail_mode_)
152 CASPAR_LOG_CURRENT_EXCEPTION();
153 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
157 auto channel_layout = core::audio_channel_layout::invalid();
158 std::vector<audio_input_pad> audio_input_pads;
160 if (!thumbnail_mode_)
162 for (unsigned stream_index = 0; stream_index < input_.context()->nb_streams; ++stream_index)
164 auto stream = input_.context()->streams[stream_index];
166 if (stream->codec->codec_type != AVMediaType::AVMEDIA_TYPE_AUDIO)
171 audio_decoders_.push_back(std::unique_ptr<audio_decoder>(new audio_decoder(stream_index, input_.context(), format_desc.audio_sample_rate)));
172 audio_input_pads.emplace_back(
173 boost::rational<int>(1, format_desc.audio_sample_rate),
174 format_desc.audio_sample_rate,
175 AVSampleFormat::AV_SAMPLE_FMT_S32,
176 audio_decoders_.back()->ffmpeg_channel_layout());
177 CASPAR_LOG(info) << print() << L" " << audio_decoders_.back()->print();
179 catch (averror_stream_not_found&)
181 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
185 CASPAR_LOG_CURRENT_EXCEPTION();
186 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
190 if (audio_decoders_.size() == 1)
192 channel_layout = get_audio_channel_layout(
193 audio_decoders_.at(0)->num_channels(),
194 audio_decoders_.at(0)->ffmpeg_channel_layout(),
195 custom_channel_order);
197 else if (audio_decoders_.size() > 1)
199 auto num_channels = cpplinq::from(audio_decoders_)
200 .select(std::mem_fn(&audio_decoder::num_channels))
201 .aggregate(0, std::plus<int>());
202 auto ffmpeg_channel_layout = av_get_default_channel_layout(num_channels);
204 channel_layout = get_audio_channel_layout(
206 ffmpeg_channel_layout,
207 custom_channel_order);
211 if (!video_decoder_ && audio_decoders_.empty())
212 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
214 muxer_.reset(new frame_muxer(framerate_, std::move(audio_input_pads), frame_factory, format_desc, channel_layout, filter, true));
219 core::draw_frame receive_impl() override
221 return render_frame().first;
224 core::draw_frame last_frame() override
226 return core::draw_frame::still(last_frame_);
229 core::constraints& pixel_constraints() override
234 double out_fps() const
236 auto out_framerate = muxer_->out_framerate();
237 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
242 std::pair<core::draw_frame, uint32_t> render_frame()
244 frame_timer_.restart();
245 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
247 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
250 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
252 if (frame_buffer_.empty())
257 return std::make_pair(last_frame(), -1);
261 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
263 return std::make_pair(last_frame_, -1);
268 return std::make_pair(last_frame_, -1);
272 auto frame = frame_buffer_.front();
276 file_frame_number_ = frame.second;
278 graph_->set_text(print());
280 last_frame_ = frame.first;
289 return boost::contains(filename_, L"://");
294 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
296 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
298 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
299 % (file_nb_frames()/fps)
300 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
301 % static_cast<int32_t>(file_nb_frames())
302 << core::monitor::message("/file/fps") % fps
303 << core::monitor::message("/file/path") % path_relative_to_media_
304 << core::monitor::message("/loop") % input_.loop();
307 core::draw_frame render_specific_frame(uint32_t file_position)
309 // Some trial and error and undeterministic stuff here
310 static const int NUM_RETRIES = 32;
312 if (file_position > 0) // Assume frames are requested in sequential order,
313 // therefore no seeking should be necessary for the first frame.
315 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
316 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
319 for (int i = 0; i < NUM_RETRIES; ++i)
321 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
323 auto frame = render_frame();
325 if (frame.second == std::numeric_limits<uint32_t>::max())
330 else if (frame.second == file_position + 1 || frame.second == file_position)
332 else if (frame.second > file_position + 1)
334 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
335 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
337 if (adjusted_seek > 1 && file_position > 0)
339 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
340 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
341 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
348 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
349 return core::draw_frame::empty();
352 core::draw_frame create_thumbnail_frame()
354 auto total_frames = nb_frames();
355 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
359 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
360 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
365 return render_specific_frame(total_frames / 2);
368 auto num_snapshots = grid * grid;
370 std::vector<core::draw_frame> frames;
372 for (int i = 0; i < num_snapshots; ++i)
379 desired_frame = 0; // first
380 else if (i == num_snapshots - 1)
381 desired_frame = total_frames - 1; // last
383 // evenly distributed across the file.
384 desired_frame = total_frames * i / (num_snapshots - 1);
386 auto frame = render_specific_frame(desired_frame);
387 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
388 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
389 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
390 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
392 frames.push_back(frame);
395 return core::draw_frame(frames);
398 uint32_t file_frame_number() const
400 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
403 uint32_t nb_frames() const override
405 if (is_url() || input_.loop())
406 return std::numeric_limits<uint32_t>::max();
408 uint32_t nb_frames = file_nb_frames();
410 nb_frames = std::min(length_, nb_frames - start_);
411 nb_frames = muxer_->calc_nb_frames(nb_frames);
416 uint32_t file_nb_frames() const
418 uint32_t file_nb_frames = 0;
419 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
420 return file_nb_frames;
423 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
425 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
426 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>(\+|-)?\d+)(\s+(?<WHENCE>REL|END))?)", boost::regex::icase);
427 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
428 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
430 auto param = boost::algorithm::join(params, L" ");
435 if(boost::regex_match(param, what, loop_exp))
437 auto value = what["VALUE"].str();
439 input_.loop(boost::lexical_cast<bool>(value));
440 result = boost::lexical_cast<std::wstring>(input_.loop());
442 else if(boost::regex_match(param, what, seek_exp))
444 auto value = boost::lexical_cast<int64_t>(what["VALUE"].str());
445 auto whence = what["WHENCE"].str();
446 auto total = file_nb_frames();
448 if(boost::iequals(whence, L"REL"))
449 value = file_frame_number() + value;
450 else if(boost::iequals(whence, L"END"))
451 value = total - value;
455 else if(value >= total)
458 input_.seek(static_cast<uint32_t>(value));
460 else if(boost::regex_match(param, what, length_exp))
462 auto value = what["VALUE"].str();
464 input_.length(boost::lexical_cast<uint32_t>(value));
465 result = boost::lexical_cast<std::wstring>(input_.length());
467 else if(boost::regex_match(param, what, start_exp))
469 auto value = what["VALUE"].str();
471 input_.start(boost::lexical_cast<uint32_t>(value));
472 result = boost::lexical_cast<std::wstring>(input_.start());
475 CASPAR_THROW_EXCEPTION(invalid_argument());
477 return make_ready_future(std::move(result));
480 std::wstring print() const override
482 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
483 + print_mode() + L"|"
484 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
487 std::wstring name() const override
492 boost::property_tree::wptree info() const override
494 boost::property_tree::wptree info;
495 info.add(L"type", L"ffmpeg-producer");
496 info.add(L"filename", filename_);
497 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
498 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
499 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
500 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
501 info.add(L"loop", input_.loop());
502 info.add(L"frame-number", frame_number_);
503 auto nb_frames2 = nb_frames();
504 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
505 info.add(L"file-frame-number", file_frame_number_);
506 info.add(L"file-nb-frames", file_nb_frames());
510 core::monitor::subject& monitor_output()
512 return *monitor_subject_;
517 std::wstring print_mode() const
519 return video_decoder_ ? ffmpeg::print_mode(
520 video_decoder_->width(),
521 video_decoder_->height(),
522 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
523 !video_decoder_->is_progressive()) : L"";
526 bool all_audio_decoders_ready() const
528 for (auto& audio_decoder : audio_decoders_)
529 if (!audio_decoder->ready())
535 void try_decode_frame()
537 std::shared_ptr<AVPacket> pkt;
539 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || !all_audio_decoders_ready()) && input_.try_pop(pkt); ++n)
542 video_decoder_->push(pkt);
544 for (auto& audio_decoder : audio_decoders_)
545 audio_decoder->push(pkt);
548 std::shared_ptr<AVFrame> video;
549 std::vector<std::shared_ptr<core::mutable_audio_buffer>> audio;
551 tbb::parallel_invoke(
554 if (!muxer_->video_ready() && video_decoder_)
555 video = video_decoder_->poll();
559 if (!muxer_->audio_ready())
561 for (auto& audio_decoder : audio_decoders_)
563 auto audio_for_stream = audio_decoder->poll();
565 if (audio_for_stream)
566 audio.push_back(audio_for_stream);
574 if (audio_decoders_.empty())
576 if (video == flush_video())
577 muxer_->push({ flush_audio() });
578 else if (!muxer_->audio_ready())
579 muxer_->push({ empty_audio() });
584 if (boost::count_if(audio, [](std::shared_ptr<core::mutable_audio_buffer> a) { return a == flush_audio(); }) > 0)
585 muxer_->push(flush_video());
586 else if (!muxer_->video_ready())
587 muxer_->push(empty_video());
590 uint32_t file_frame_number = 0;
591 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
593 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
594 frame_buffer_.push(std::make_pair(frame, file_frame_number));
597 bool audio_only() const
599 return !video_decoder_;
602 boost::rational<int> get_out_framerate() const
604 return muxer_->out_framerate();
608 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
610 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
611 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
613 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
614 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
615 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
617 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
618 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg, dshow://video={webcam_name} or v4l2://{video device}.")
619 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
620 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
621 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
622 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
623 ->item(L"channel_layout",
624 L"Optionally override the automatically deduced audio channel layout. "
625 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
626 sink.para()->text(L"Examples:");
627 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
628 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
629 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
630 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
631 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
632 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
633 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
634 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
635 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
636 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input on Windows.");
637 sink.example(L">> PLAY 1-10 v4l2:///dev/video0", L"to use a web camera as video input on Linux.");
638 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
639 sink.example(L">> CALL 1-10 LOOP 1");
640 sink.example(L">> CALL 1-10 START 10");
641 sink.example(L">> CALL 1-10 LENGTH 50");
642 sink.example(L">> CALL 1-10 SEEK 30");
643 core::describe_framerate_producer(sink);
646 spl::shared_ptr<core::frame_producer> create_producer(
647 const core::frame_producer_dependencies& dependencies,
648 const std::vector<std::wstring>& params,
649 const spl::shared_ptr<core::media_info_repository>& info_repo)
651 auto file_or_url = params.at(0);
653 if (!boost::contains(file_or_url, L"://"))
656 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
659 if (file_or_url.empty())
660 return core::frame_producer::empty();
662 auto loop = contains_param(L"LOOP", params);
663 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
664 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
665 auto filter_str = get_param(L"FILTER", params, L"");
666 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
668 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
669 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
670 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
672 ffmpeg_options vid_params;
673 bool haveFFMPEGStartIndicator = false;
674 for (size_t i = 0; i < params.size() - 1; ++i)
676 if (!haveFFMPEGStartIndicator && params[i] == L"--")
678 haveFFMPEGStartIndicator = true;
681 if (haveFFMPEGStartIndicator)
683 auto name = u8(params.at(i++)).substr(1);
684 auto value = u8(params.at(i));
685 vid_params.push_back(std::make_pair(name, value));
689 auto producer = spl::make_shared<ffmpeg_producer>(
690 dependencies.frame_factory,
691 dependencies.format_desc,
698 custom_channel_order,
701 if (producer->audio_only())
702 return core::create_destroy_proxy(producer);
704 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
705 auto target_framerate = dependencies.format_desc.framerate;
707 return core::create_destroy_proxy(core::create_framerate_producer(
709 get_source_framerate,
711 dependencies.format_desc.field_mode,
712 dependencies.format_desc.audio_cadence));
715 core::draw_frame create_thumbnail_frame(
716 const core::frame_producer_dependencies& dependencies,
717 const std::wstring& media_file,
718 const spl::shared_ptr<core::media_info_repository>& info_repo)
720 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
721 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
723 if (filename.empty())
724 return core::draw_frame::empty();
728 auto length = std::numeric_limits<uint32_t>::max();
729 auto filter_str = L"";
731 ffmpeg_options vid_params;
732 auto producer = spl::make_shared<ffmpeg_producer>(
733 dependencies.frame_factory,
734 dependencies.format_desc,
744 return producer->create_thumbnail_frame();