2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
33 #include "filter/audio_filter.h"
35 #include <common/param.h>
36 #include <common/diagnostics/graph.h>
37 #include <common/future.h>
39 #include <core/frame/draw_frame.h>
40 #include <core/help/help_repository.h>
41 #include <core/help/help_sink.h>
42 #include <core/producer/media_info/media_info.h>
43 #include <core/producer/framerate/framerate_producer.h>
44 #include <core/frame/frame_factory.h>
49 namespace caspar { namespace ffmpeg {
50 struct seek_out_of_range : virtual user_error {};
52 std::wstring get_relative_or_original(
53 const std::wstring& filename,
54 const boost::filesystem::path& relative_to)
56 boost::filesystem::path file(filename);
57 auto result = file.filename().wstring();
59 boost::filesystem::path current_path = file;
63 current_path = current_path.parent_path();
65 if (boost::filesystem::equivalent(current_path, relative_to))
68 if (current_path.empty())
71 result = current_path.filename().wstring() + L"/" + result;
77 struct ffmpeg_producer : public core::frame_producer_base
79 spl::shared_ptr<core::monitor::subject> monitor_subject_;
80 const std::wstring filename_;
81 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
83 const spl::shared_ptr<diagnostics::graph> graph_;
86 const spl::shared_ptr<core::frame_factory> frame_factory_;
88 std::shared_ptr<void> initial_logger_disabler_;
90 core::constraints constraints_;
93 std::unique_ptr<video_decoder> video_decoder_;
94 std::vector<std::unique_ptr<audio_decoder>> audio_decoders_;
95 std::unique_ptr<frame_muxer> muxer_;
97 const boost::rational<int> framerate_;
98 const uint32_t start_;
99 const uint32_t length_;
100 const bool thumbnail_mode_;
102 core::draw_frame last_frame_;
104 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
106 int64_t frame_number_ = 0;
107 uint32_t file_frame_number_ = 0;
109 explicit ffmpeg_producer(
110 const spl::shared_ptr<core::frame_factory>& frame_factory,
111 const core::video_format_desc& format_desc,
112 const std::wstring& url_or_file,
113 const std::wstring& filter,
118 const std::wstring& custom_channel_order,
119 const ffmpeg_options& vid_params)
120 : filename_(url_or_file)
121 , frame_factory_(frame_factory)
122 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
123 , input_(graph_, url_or_file, loop, start, length, thumbnail_mode, vid_params)
124 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
127 , thumbnail_mode_(thumbnail_mode)
128 , last_frame_(core::draw_frame::empty())
131 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
132 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
133 diagnostics::register_graph(graph_);
137 video_decoder_.reset(new video_decoder(input_.context()));
138 if (!thumbnail_mode_)
139 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
141 constraints_.width.set(video_decoder_->width());
142 constraints_.height.set(video_decoder_->height());
144 catch (averror_stream_not_found&)
146 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
150 if (!thumbnail_mode_)
152 CASPAR_LOG_CURRENT_EXCEPTION();
153 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
157 auto channel_layout = core::audio_channel_layout::invalid();
158 std::vector<audio_input_pad> audio_input_pads;
160 if (!thumbnail_mode_)
162 for (unsigned stream_index = 0; stream_index < input_.context()->nb_streams; ++stream_index)
164 auto stream = input_.context()->streams[stream_index];
166 if (stream->codec->codec_type != AVMediaType::AVMEDIA_TYPE_AUDIO)
171 audio_decoders_.push_back(std::unique_ptr<audio_decoder>(new audio_decoder(stream_index, input_.context(), format_desc.audio_sample_rate)));
172 audio_input_pads.emplace_back(
173 boost::rational<int>(1, format_desc.audio_sample_rate),
174 format_desc.audio_sample_rate,
175 AVSampleFormat::AV_SAMPLE_FMT_S32,
176 audio_decoders_.back()->ffmpeg_channel_layout());
177 CASPAR_LOG(info) << print() << L" " << audio_decoders_.back()->print();
179 catch (averror_stream_not_found&)
181 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
185 CASPAR_LOG_CURRENT_EXCEPTION();
186 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
190 if (audio_decoders_.size() == 1)
192 channel_layout = get_audio_channel_layout(
193 audio_decoders_.at(0)->num_channels(),
194 audio_decoders_.at(0)->ffmpeg_channel_layout(),
195 custom_channel_order);
197 else if (audio_decoders_.size() > 1)
199 auto num_channels = cpplinq::from(audio_decoders_)
200 .select(std::mem_fn(&audio_decoder::num_channels))
201 .aggregate(0, std::plus<int>());
202 auto ffmpeg_channel_layout = av_get_default_channel_layout(num_channels);
204 channel_layout = get_audio_channel_layout(
206 ffmpeg_channel_layout,
207 custom_channel_order);
211 if (!video_decoder_ && audio_decoders_.empty())
212 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
214 muxer_.reset(new frame_muxer(framerate_, std::move(audio_input_pads), frame_factory, format_desc, channel_layout, filter, true));
219 core::draw_frame receive_impl() override
221 return render_frame().first;
224 core::draw_frame last_frame() override
226 return core::draw_frame::still(last_frame_);
229 core::constraints& pixel_constraints() override
234 double out_fps() const
236 auto out_framerate = muxer_->out_framerate();
237 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
242 std::pair<core::draw_frame, uint32_t> render_frame()
244 frame_timer_.restart();
245 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
247 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
250 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
252 if (frame_buffer_.empty())
257 return std::make_pair(last_frame(), -1);
261 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
263 return std::make_pair(last_frame_, -1);
268 return std::make_pair(last_frame_, -1);
272 auto frame = frame_buffer_.front();
276 file_frame_number_ = frame.second;
278 graph_->set_text(print());
280 last_frame_ = frame.first;
289 return boost::contains(filename_, L"://");
294 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
296 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
298 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
299 % (file_nb_frames()/fps)
300 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
301 % static_cast<int32_t>(file_nb_frames())
302 << core::monitor::message("/file/fps") % fps
303 << core::monitor::message("/file/path") % path_relative_to_media_
304 << core::monitor::message("/loop") % input_.loop();
307 core::draw_frame render_specific_frame(uint32_t file_position)
309 // Some trial and error and undeterministic stuff here
310 static const int NUM_RETRIES = 32;
312 if (file_position > 0) // Assume frames are requested in sequential order,
313 // therefore no seeking should be necessary for the first frame.
315 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
316 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
319 for (int i = 0; i < NUM_RETRIES; ++i)
321 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
323 auto frame = render_frame();
325 if (frame.second == std::numeric_limits<uint32_t>::max())
330 else if (frame.second == file_position + 1 || frame.second == file_position)
332 else if (frame.second > file_position + 1)
334 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
335 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
337 if (adjusted_seek > 1 && file_position > 0)
339 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
340 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
341 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
348 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
349 return core::draw_frame::empty();
352 core::draw_frame create_thumbnail_frame()
354 auto total_frames = nb_frames();
355 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
359 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
360 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
365 return render_specific_frame(total_frames / 2);
368 auto num_snapshots = grid * grid;
370 std::vector<core::draw_frame> frames;
372 for (int i = 0; i < num_snapshots; ++i)
379 desired_frame = 0; // first
380 else if (i == num_snapshots - 1)
381 desired_frame = total_frames - 1; // last
383 // evenly distributed across the file.
384 desired_frame = total_frames * i / (num_snapshots - 1);
386 auto frame = render_specific_frame(desired_frame);
387 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
388 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
389 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
390 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
392 frames.push_back(frame);
395 return core::draw_frame(frames);
398 uint32_t file_frame_number() const
400 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
403 uint32_t nb_frames() const override
405 if (is_url() || input_.loop())
406 return std::numeric_limits<uint32_t>::max();
408 uint32_t nb_frames = file_nb_frames();
410 nb_frames = std::min(length_, nb_frames - start_);
411 nb_frames = muxer_->calc_nb_frames(nb_frames);
416 uint32_t file_nb_frames() const
418 uint32_t file_nb_frames = 0;
419 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
420 return file_nb_frames;
423 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
425 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
426 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>(\+|-)?\d+)(\s+(?<WHENCE>REL|END))?)", boost::regex::icase);
427 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
428 static const boost::wregex start_exp(LR"(START\s+(?<VALUE>\d+)?)", boost::regex::icase);
430 auto param = boost::algorithm::join(params, L" ");
435 if(boost::regex_match(param, what, loop_exp))
437 auto value = what["VALUE"].str();
439 input_.loop(boost::lexical_cast<bool>(value));
440 result = boost::lexical_cast<std::wstring>(input_.loop());
442 else if(boost::regex_match(param, what, seek_exp))
444 auto value = boost::lexical_cast<int64_t>(what["VALUE"].str());
445 auto whence = what["WHENCE"].str();
446 auto total = file_nb_frames();
448 if(boost::iequals(whence, L"REL"))
449 value = file_frame_number() + value;
450 else if(boost::iequals(whence, L"END"))
451 value = total - value;
455 else if(value >= total)
458 input_.seek(static_cast<uint32_t>(value));
460 else if(boost::regex_match(param, what, length_exp))
462 auto value = what["VALUE"].str();
464 input_.length(boost::lexical_cast<uint32_t>(value));
465 result = boost::lexical_cast<std::wstring>(input_.length());
467 else if(boost::regex_match(param, what, start_exp))
469 auto value = what["VALUE"].str();
471 input_.start(boost::lexical_cast<uint32_t>(value));
472 result = boost::lexical_cast<std::wstring>(input_.start());
475 CASPAR_THROW_EXCEPTION(invalid_argument());
477 return make_ready_future(std::move(result));
480 std::wstring print() const override
482 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
483 + print_mode() + L"|"
484 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
487 std::wstring name() const override
492 boost::property_tree::wptree info() const override
494 boost::property_tree::wptree info;
495 info.add(L"type", L"ffmpeg-producer");
496 info.add(L"filename", filename_);
497 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
498 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
499 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
500 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
501 info.add(L"loop", input_.loop());
502 info.add(L"frame-number", frame_number_);
503 auto nb_frames2 = nb_frames();
504 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
505 info.add(L"file-frame-number", file_frame_number_);
506 info.add(L"file-nb-frames", file_nb_frames());
510 core::monitor::subject& monitor_output()
512 return *monitor_subject_;
517 std::wstring print_mode() const
519 return video_decoder_ ? ffmpeg::print_mode(
520 video_decoder_->width(),
521 video_decoder_->height(),
522 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
523 !video_decoder_->is_progressive()) : L"";
526 bool all_audio_decoders_ready() const
528 for (auto& audio_decoder : audio_decoders_)
529 if (!audio_decoder->ready())
535 void try_decode_frame()
537 std::shared_ptr<AVPacket> pkt;
539 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || !all_audio_decoders_ready()) && input_.try_pop(pkt); ++n)
542 video_decoder_->push(pkt);
544 for (auto& audio_decoder : audio_decoders_)
545 audio_decoder->push(pkt);
548 std::shared_ptr<AVFrame> video;
549 std::vector<std::shared_ptr<core::mutable_audio_buffer>> audio;
551 tbb::parallel_invoke(
556 if (!muxer_->video_ready() && video_decoder_)
558 video = video_decoder_->poll();
564 } while (!video_decoder_->empty());
568 if (!muxer_->audio_ready())
570 for (auto& audio_decoder : audio_decoders_)
572 auto audio_for_stream = audio_decoder->poll();
574 if (audio_for_stream)
575 audio.push_back(audio_for_stream);
583 if (audio_decoders_.empty())
585 if (video == flush_video())
586 muxer_->push({ flush_audio() });
587 else if (!muxer_->audio_ready())
588 muxer_->push({ empty_audio() });
593 if (boost::count_if(audio, [](std::shared_ptr<core::mutable_audio_buffer> a) { return a == flush_audio(); }) > 0)
594 muxer_->push(flush_video());
595 else if (!muxer_->video_ready())
596 muxer_->push(empty_video());
599 uint32_t file_frame_number = 0;
600 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
602 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
603 if (frame != core::draw_frame::empty())
604 frame_buffer_.push(std::make_pair(frame, file_frame_number));
607 bool audio_only() const
609 return !video_decoder_;
612 boost::rational<int> get_out_framerate() const
614 return muxer_->out_framerate();
618 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
620 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
621 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
623 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
624 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
625 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
627 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
628 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg, dshow://video={webcam_name} or v4l2://{video device}.")
629 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
630 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
631 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
632 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
633 ->item(L"channel_layout",
634 L"Optionally override the automatically deduced audio channel layout. "
635 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
636 sink.para()->text(L"Examples:");
637 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
638 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
639 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
640 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
641 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
642 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
643 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
644 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
645 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
646 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input on Windows.");
647 sink.example(L">> PLAY 1-10 v4l2:///dev/video0", L"to use a web camera as video input on Linux.");
648 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
649 sink.example(L">> CALL 1-10 LOOP 1");
650 sink.example(L">> CALL 1-10 START 10");
651 sink.example(L">> CALL 1-10 LENGTH 50");
652 sink.example(L">> CALL 1-10 SEEK 30");
653 core::describe_framerate_producer(sink);
656 spl::shared_ptr<core::frame_producer> create_producer(
657 const core::frame_producer_dependencies& dependencies,
658 const std::vector<std::wstring>& params,
659 const spl::shared_ptr<core::media_info_repository>& info_repo)
661 auto file_or_url = params.at(0);
663 if (!boost::contains(file_or_url, L"://"))
666 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
669 if (file_or_url.empty())
670 return core::frame_producer::empty();
672 auto loop = contains_param(L"LOOP", params);
673 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
674 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
675 auto filter_str = get_param(L"FILTER", params, L"");
676 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
678 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
679 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
680 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
682 ffmpeg_options vid_params;
683 bool haveFFMPEGStartIndicator = false;
684 for (size_t i = 0; i < params.size() - 1; ++i)
686 if (!haveFFMPEGStartIndicator && params[i] == L"--")
688 haveFFMPEGStartIndicator = true;
691 if (haveFFMPEGStartIndicator)
693 auto name = u8(params.at(i++)).substr(1);
694 auto value = u8(params.at(i));
695 vid_params.push_back(std::make_pair(name, value));
699 auto producer = spl::make_shared<ffmpeg_producer>(
700 dependencies.frame_factory,
701 dependencies.format_desc,
708 custom_channel_order,
711 if (producer->audio_only())
712 return core::create_destroy_proxy(producer);
714 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
715 auto target_framerate = dependencies.format_desc.framerate;
717 return core::create_destroy_proxy(core::create_framerate_producer(
719 get_source_framerate,
721 dependencies.format_desc.field_mode,
722 dependencies.format_desc.audio_cadence));
725 core::draw_frame create_thumbnail_frame(
726 const core::frame_producer_dependencies& dependencies,
727 const std::wstring& media_file,
728 const spl::shared_ptr<core::media_info_repository>& info_repo)
730 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
731 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
733 if (filename.empty())
734 return core::draw_frame::empty();
738 auto length = std::numeric_limits<uint32_t>::max();
739 auto filter_str = L"";
741 ffmpeg_options vid_params;
742 auto producer = spl::make_shared<ffmpeg_producer>(
743 dependencies.frame_factory,
744 dependencies.format_desc,
754 return producer->create_thumbnail_frame();