2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
33 #include "filter/audio_filter.h"
35 #include <common/param.h>
36 #include <common/diagnostics/graph.h>
37 #include <common/future.h>
39 #include <core/frame/draw_frame.h>
40 #include <core/help/help_repository.h>
41 #include <core/help/help_sink.h>
42 #include <core/producer/media_info/media_info.h>
43 #include <core/producer/framerate/framerate_producer.h>
44 #include <core/frame/frame_factory.h>
49 namespace caspar { namespace ffmpeg {
50 struct seek_out_of_range : virtual user_error {};
52 std::wstring get_relative_or_original(
53 const std::wstring& filename,
54 const boost::filesystem::path& relative_to)
56 boost::filesystem::path file(filename);
57 auto result = file.filename().wstring();
59 boost::filesystem::path current_path = file;
63 current_path = current_path.parent_path();
65 if (boost::filesystem::equivalent(current_path, relative_to))
68 if (current_path.empty())
71 result = current_path.filename().wstring() + L"/" + result;
77 struct ffmpeg_producer : public core::frame_producer_base
79 spl::shared_ptr<core::monitor::subject> monitor_subject_;
80 const std::wstring filename_;
81 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
83 const spl::shared_ptr<diagnostics::graph> graph_;
86 const spl::shared_ptr<core::frame_factory> frame_factory_;
88 std::shared_ptr<void> initial_logger_disabler_;
90 core::constraints constraints_;
93 std::unique_ptr<video_decoder> video_decoder_;
94 std::vector<std::unique_ptr<audio_decoder>> audio_decoders_;
95 std::unique_ptr<frame_muxer> muxer_;
97 const boost::rational<int> framerate_;
98 const uint32_t start_;
99 const uint32_t length_;
100 const bool thumbnail_mode_;
102 core::draw_frame last_frame_;
104 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
106 int64_t frame_number_ = 0;
107 uint32_t file_frame_number_ = 0;
109 explicit ffmpeg_producer(
110 const spl::shared_ptr<core::frame_factory>& frame_factory,
111 const core::video_format_desc& format_desc,
112 const std::wstring& url_or_file,
113 const std::wstring& filter,
118 const std::wstring& custom_channel_order,
119 const ffmpeg_options& vid_params)
120 : filename_(url_or_file)
121 , frame_factory_(frame_factory)
122 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
123 , input_(graph_, url_or_file, loop, start, length, thumbnail_mode, vid_params)
124 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
127 , thumbnail_mode_(thumbnail_mode)
128 , last_frame_(core::draw_frame::empty())
131 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
132 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
133 diagnostics::register_graph(graph_);
137 video_decoder_.reset(new video_decoder(input_.context()));
138 if (!thumbnail_mode_)
139 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
141 constraints_.width.set(video_decoder_->width());
142 constraints_.height.set(video_decoder_->height());
144 catch (averror_stream_not_found&)
146 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
150 if (!thumbnail_mode_)
152 CASPAR_LOG_CURRENT_EXCEPTION();
153 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
157 auto channel_layout = core::audio_channel_layout::invalid();
158 std::vector<audio_input_pad> audio_input_pads;
160 if (!thumbnail_mode_)
162 for (unsigned stream_index = 0; stream_index < input_.context()->nb_streams; ++stream_index)
164 auto stream = input_.context()->streams[stream_index];
166 if (stream->codec->codec_type != AVMediaType::AVMEDIA_TYPE_AUDIO)
171 audio_decoders_.push_back(std::unique_ptr<audio_decoder>(new audio_decoder(stream_index, input_.context(), format_desc.audio_sample_rate)));
172 audio_input_pads.emplace_back(
173 boost::rational<int>(1, format_desc.audio_sample_rate),
174 format_desc.audio_sample_rate,
175 AVSampleFormat::AV_SAMPLE_FMT_S32,
176 audio_decoders_.back()->ffmpeg_channel_layout());
177 CASPAR_LOG(info) << print() << L" " << audio_decoders_.back()->print();
179 catch (averror_stream_not_found&)
181 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
185 CASPAR_LOG_CURRENT_EXCEPTION();
186 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
190 if (audio_decoders_.size() == 1)
192 channel_layout = get_audio_channel_layout(
193 audio_decoders_.at(0)->num_channels(),
194 audio_decoders_.at(0)->ffmpeg_channel_layout(),
195 custom_channel_order);
197 else if (audio_decoders_.size() > 1)
199 auto num_channels = cpplinq::from(audio_decoders_)
200 .select(std::mem_fn(&audio_decoder::num_channels))
201 .aggregate(0, std::plus<int>());
202 auto ffmpeg_channel_layout = av_get_default_channel_layout(num_channels);
204 channel_layout = get_audio_channel_layout(
206 ffmpeg_channel_layout,
207 custom_channel_order);
211 if (!video_decoder_ && audio_decoders_.empty())
212 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
214 muxer_.reset(new frame_muxer(framerate_, std::move(audio_input_pads), frame_factory, format_desc, channel_layout, filter, true));
219 core::draw_frame receive_impl() override
221 return render_frame().first;
224 core::draw_frame last_frame() override
226 return core::draw_frame::still(last_frame_);
229 core::constraints& pixel_constraints() override
234 double out_fps() const
236 auto out_framerate = muxer_->out_framerate();
237 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
242 std::pair<core::draw_frame, uint32_t> render_frame()
244 frame_timer_.restart();
245 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
247 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
250 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
252 if (frame_buffer_.empty())
257 return std::make_pair(last_frame(), -1);
261 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
263 return std::make_pair(last_frame_, -1);
268 return std::make_pair(last_frame_, -1);
272 auto frame = frame_buffer_.front();
276 file_frame_number_ = frame.second;
278 graph_->set_text(print());
280 last_frame_ = frame.first;
289 return boost::contains(filename_, L"://");
294 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
296 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
298 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
299 % (file_nb_frames()/fps)
300 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
301 % static_cast<int32_t>(file_nb_frames())
302 << core::monitor::message("/file/fps") % fps
303 << core::monitor::message("/file/path") % path_relative_to_media_
304 << core::monitor::message("/loop") % input_.loop();
307 core::draw_frame render_specific_frame(uint32_t file_position)
309 // Some trial and error and undeterministic stuff here
310 static const int NUM_RETRIES = 32;
312 if (file_position > 0) // Assume frames are requested in sequential order,
313 // therefore no seeking should be necessary for the first frame.
315 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
316 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
319 for (int i = 0; i < NUM_RETRIES; ++i)
321 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
323 auto frame = render_frame();
325 if (frame.second == std::numeric_limits<uint32_t>::max())
330 else if (frame.second == file_position + 1 || frame.second == file_position)
332 else if (frame.second > file_position + 1)
334 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
335 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
337 if (adjusted_seek > 1 && file_position > 0)
339 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
340 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
341 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
348 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
349 return core::draw_frame::empty();
352 core::draw_frame create_thumbnail_frame()
354 auto total_frames = nb_frames();
355 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
359 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
360 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
365 return render_specific_frame(total_frames / 2);
368 auto num_snapshots = grid * grid;
370 std::vector<core::draw_frame> frames;
372 for (int i = 0; i < num_snapshots; ++i)
379 desired_frame = 0; // first
380 else if (i == num_snapshots - 1)
381 desired_frame = total_frames - 1; // last
383 // evenly distributed across the file.
384 desired_frame = total_frames * i / (num_snapshots - 1);
386 auto frame = render_specific_frame(desired_frame);
387 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
388 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
389 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
390 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
392 frames.push_back(frame);
395 return core::draw_frame(frames);
398 uint32_t file_frame_number() const
400 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
403 uint32_t nb_frames() const override
405 if (is_url() || input_.loop())
406 return std::numeric_limits<uint32_t>::max();
408 uint32_t nb_frames = file_nb_frames();
410 nb_frames = std::min(length_, nb_frames - start_);
411 nb_frames = muxer_->calc_nb_frames(nb_frames);
416 uint32_t file_nb_frames() const
418 uint32_t file_nb_frames = 0;
419 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
420 return file_nb_frames;
423 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
425 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
426 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>(\+|-)?\d+)(\s+(?<WHENCE>REL|END))?)", boost::regex::icase);
427 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
428 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
430 auto param = boost::algorithm::join(params, L" ");
435 if(boost::regex_match(param, what, loop_exp))
437 auto value = what["VALUE"].str();
439 input_.loop(boost::lexical_cast<bool>(value));
440 result = boost::lexical_cast<std::wstring>(input_.loop());
442 else if(boost::regex_match(param, what, seek_exp))
444 auto value = boost::lexical_cast<uint32_t>(what["VALUE"].str());
445 auto whence = what["WHENCE"].str();
447 if(boost::iequals(whence, L"REL"))
449 value = file_frame_number() + value;
451 else if(boost::iequals(whence, L"END"))
453 value = file_nb_frames() - value;
458 else if(boost::regex_match(param, what, length_exp))
460 auto value = what["VALUE"].str();
462 input_.length(boost::lexical_cast<uint32_t>(value));
463 result = boost::lexical_cast<std::wstring>(input_.length());
465 else if(boost::regex_match(param, what, start_exp))
467 auto value = what["VALUE"].str();
469 input_.start(boost::lexical_cast<uint32_t>(value));
470 result = boost::lexical_cast<std::wstring>(input_.start());
473 CASPAR_THROW_EXCEPTION(invalid_argument());
475 return make_ready_future(std::move(result));
478 std::wstring print() const override
480 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
481 + print_mode() + L"|"
482 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
485 std::wstring name() const override
490 boost::property_tree::wptree info() const override
492 boost::property_tree::wptree info;
493 info.add(L"type", L"ffmpeg-producer");
494 info.add(L"filename", filename_);
495 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
496 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
497 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
498 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
499 info.add(L"loop", input_.loop());
500 info.add(L"frame-number", frame_number_);
501 auto nb_frames2 = nb_frames();
502 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
503 info.add(L"file-frame-number", file_frame_number_);
504 info.add(L"file-nb-frames", file_nb_frames());
508 core::monitor::subject& monitor_output()
510 return *monitor_subject_;
515 std::wstring print_mode() const
517 return video_decoder_ ? ffmpeg::print_mode(
518 video_decoder_->width(),
519 video_decoder_->height(),
520 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
521 !video_decoder_->is_progressive()) : L"";
524 bool not_all_audio_decoders_ready() const
526 for (auto& audio_decoder : audio_decoders_)
527 if (!audio_decoder->ready())
533 void try_decode_frame()
535 std::shared_ptr<AVPacket> pkt;
537 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || not_all_audio_decoders_ready()) && input_.try_pop(pkt); ++n)
540 video_decoder_->push(pkt);
542 for (auto& audio_decoder : audio_decoders_)
543 audio_decoder->push(pkt);
546 std::shared_ptr<AVFrame> video;
547 std::vector<std::shared_ptr<core::mutable_audio_buffer>> audio;
549 tbb::parallel_invoke(
552 if (!muxer_->video_ready() && video_decoder_)
553 video = video_decoder_->poll();
557 if (!muxer_->audio_ready())
559 for (auto& audio_decoder : audio_decoders_)
561 auto audio_for_stream = audio_decoder->poll();
563 if (audio_for_stream)
564 audio.push_back(audio_for_stream);
572 if (audio_decoders_.empty())
574 if (video == flush_video())
575 muxer_->push({ flush_audio() });
576 else if (!muxer_->audio_ready())
577 muxer_->push({ empty_audio() });
582 if (boost::count_if(audio, [](std::shared_ptr<core::mutable_audio_buffer> a) { return a == flush_audio(); }) > 0)
583 muxer_->push(flush_video());
584 else if (!muxer_->video_ready())
585 muxer_->push(empty_video());
588 uint32_t file_frame_number = 0;
589 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
591 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
592 frame_buffer_.push(std::make_pair(frame, file_frame_number));
595 bool audio_only() const
597 return !video_decoder_;
600 boost::rational<int> get_out_framerate() const
602 return muxer_->out_framerate();
606 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
608 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
609 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
611 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
612 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
613 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
615 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
616 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg, dshow://video={webcam_name} or v4l2://{video device}.")
617 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
618 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
619 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
620 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
621 ->item(L"channel_layout",
622 L"Optionally override the automatically deduced audio channel layout. "
623 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
624 sink.para()->text(L"Examples:");
625 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
626 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
627 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
628 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
629 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
630 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
631 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
632 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
633 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
634 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input on Windows.");
635 sink.example(L">> PLAY 1-10 v4l2:///dev/video0", L"to use a web camera as video input on Linux.");
636 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
637 sink.example(L">> CALL 1-10 LOOP 1");
638 sink.example(L">> CALL 1-10 START 10");
639 sink.example(L">> CALL 1-10 LENGTH 50");
640 sink.example(L">> CALL 1-10 SEEK 30");
641 core::describe_framerate_producer(sink);
644 spl::shared_ptr<core::frame_producer> create_producer(
645 const core::frame_producer_dependencies& dependencies,
646 const std::vector<std::wstring>& params,
647 const spl::shared_ptr<core::media_info_repository>& info_repo)
649 auto file_or_url = params.at(0);
651 if (!boost::contains(file_or_url, L"://"))
654 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
657 if (file_or_url.empty())
658 return core::frame_producer::empty();
660 auto loop = contains_param(L"LOOP", params);
661 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
662 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
663 auto filter_str = get_param(L"FILTER", params, L"");
664 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
666 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
667 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
668 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
670 ffmpeg_options vid_params;
671 bool haveFFMPEGStartIndicator = false;
672 for (size_t i = 0; i < params.size() - 1; ++i)
674 if (!haveFFMPEGStartIndicator && params[i] == L"--")
676 haveFFMPEGStartIndicator = true;
679 if (haveFFMPEGStartIndicator)
681 auto name = u8(params.at(i++)).substr(1);
682 auto value = u8(params.at(i));
683 vid_params.push_back(std::make_pair(name, value));
687 auto producer = spl::make_shared<ffmpeg_producer>(
688 dependencies.frame_factory,
689 dependencies.format_desc,
696 custom_channel_order,
699 if (producer->audio_only())
700 return core::create_destroy_proxy(producer);
702 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
703 auto target_framerate = dependencies.format_desc.framerate;
705 return core::create_destroy_proxy(core::create_framerate_producer(
707 get_source_framerate,
709 dependencies.format_desc.field_mode,
710 dependencies.format_desc.audio_cadence));
713 core::draw_frame create_thumbnail_frame(
714 const core::frame_producer_dependencies& dependencies,
715 const std::wstring& media_file,
716 const spl::shared_ptr<core::media_info_repository>& info_repo)
718 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
719 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
721 if (filename.empty())
722 return core::draw_frame::empty();
726 auto length = std::numeric_limits<uint32_t>::max();
727 auto filter_str = L"";
729 ffmpeg_options vid_params;
730 auto producer = spl::make_shared<ffmpeg_producer>(
731 dependencies.frame_factory,
732 dependencies.format_desc,
742 return producer->create_thumbnail_frame();