2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
33 #include "filter/audio_filter.h"
35 #include <common/param.h>
36 #include <common/diagnostics/graph.h>
37 #include <common/future.h>
39 #include <core/frame/draw_frame.h>
40 #include <core/help/help_repository.h>
41 #include <core/help/help_sink.h>
42 #include <core/producer/media_info/media_info.h>
43 #include <core/producer/framerate/framerate_producer.h>
44 #include <core/frame/frame_factory.h>
49 namespace caspar { namespace ffmpeg {
50 struct seek_out_of_range : virtual user_error {};
52 std::wstring get_relative_or_original(
53 const std::wstring& filename,
54 const boost::filesystem::path& relative_to)
56 boost::filesystem::path file(filename);
57 auto result = file.filename().wstring();
59 boost::filesystem::path current_path = file;
63 current_path = current_path.parent_path();
65 if (boost::filesystem::equivalent(current_path, relative_to))
68 if (current_path.empty())
71 result = current_path.filename().wstring() + L"/" + result;
77 struct ffmpeg_producer : public core::frame_producer_base
79 spl::shared_ptr<core::monitor::subject> monitor_subject_;
80 const std::wstring filename_;
81 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
83 const spl::shared_ptr<diagnostics::graph> graph_;
86 const spl::shared_ptr<core::frame_factory> frame_factory_;
88 std::shared_ptr<void> initial_logger_disabler_;
90 core::constraints constraints_;
93 std::unique_ptr<video_decoder> video_decoder_;
94 std::vector<std::unique_ptr<audio_decoder>> audio_decoders_;
95 std::unique_ptr<frame_muxer> muxer_;
97 const boost::rational<int> framerate_;
98 const bool thumbnail_mode_;
100 core::draw_frame last_frame_;
102 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
104 int64_t frame_number_ = 0;
105 uint32_t file_frame_number_ = 0;
107 explicit ffmpeg_producer(
108 const spl::shared_ptr<core::frame_factory>& frame_factory,
109 const core::video_format_desc& format_desc,
110 const std::wstring& url_or_file,
111 const std::wstring& filter,
116 const std::wstring& custom_channel_order,
117 const ffmpeg_options& vid_params)
118 : filename_(url_or_file)
119 , frame_factory_(frame_factory)
120 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
121 , input_(graph_, url_or_file, loop, in, out, thumbnail_mode, vid_params)
122 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
123 , thumbnail_mode_(thumbnail_mode)
124 , last_frame_(core::draw_frame::empty())
126 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
127 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
128 diagnostics::register_graph(graph_);
132 video_decoder_.reset(new video_decoder(input_.context()));
133 if (!thumbnail_mode_)
134 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
136 constraints_.width.set(video_decoder_->width());
137 constraints_.height.set(video_decoder_->height());
139 catch (averror_stream_not_found&)
141 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
145 if (!thumbnail_mode_)
147 CASPAR_LOG_CURRENT_EXCEPTION();
148 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
152 auto channel_layout = core::audio_channel_layout::invalid();
153 std::vector<audio_input_pad> audio_input_pads;
155 if (!thumbnail_mode_)
157 for (unsigned stream_index = 0; stream_index < input_.context()->nb_streams; ++stream_index)
159 auto stream = input_.context()->streams[stream_index];
161 if (stream->codec->codec_type != AVMediaType::AVMEDIA_TYPE_AUDIO)
166 audio_decoders_.push_back(std::unique_ptr<audio_decoder>(new audio_decoder(stream_index, input_.context(), format_desc.audio_sample_rate)));
167 audio_input_pads.emplace_back(
168 boost::rational<int>(1, format_desc.audio_sample_rate),
169 format_desc.audio_sample_rate,
170 AVSampleFormat::AV_SAMPLE_FMT_S32,
171 audio_decoders_.back()->ffmpeg_channel_layout());
172 CASPAR_LOG(info) << print() << L" " << audio_decoders_.back()->print();
174 catch (averror_stream_not_found&)
176 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
180 CASPAR_LOG_CURRENT_EXCEPTION();
181 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
185 if (audio_decoders_.size() == 1)
187 channel_layout = get_audio_channel_layout(
188 audio_decoders_.at(0)->num_channels(),
189 audio_decoders_.at(0)->ffmpeg_channel_layout(),
190 custom_channel_order);
192 else if (audio_decoders_.size() > 1)
194 auto num_channels = cpplinq::from(audio_decoders_)
195 .select(std::mem_fn(&audio_decoder::num_channels))
196 .aggregate(0, std::plus<int>());
197 auto ffmpeg_channel_layout = av_get_default_channel_layout(num_channels);
199 channel_layout = get_audio_channel_layout(
201 ffmpeg_channel_layout,
202 custom_channel_order);
206 if (!video_decoder_ && audio_decoders_.empty())
207 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
209 muxer_.reset(new frame_muxer(framerate_, std::move(audio_input_pads), frame_factory, format_desc, channel_layout, filter, true));
211 if (auto nb_frames = file_nb_frames())
213 out = std::min(out, nb_frames);
220 core::draw_frame receive_impl() override
222 return render_frame().first;
225 core::draw_frame last_frame() override
227 return core::draw_frame::still(last_frame_);
230 core::constraints& pixel_constraints() override
235 double out_fps() const
237 auto out_framerate = muxer_->out_framerate();
238 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
243 std::pair<core::draw_frame, uint32_t> render_frame()
245 frame_timer_.restart();
246 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
248 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
251 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
253 if (frame_buffer_.empty())
258 return std::make_pair(last_frame(), -1);
262 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
264 return std::make_pair(last_frame_, -1);
269 return std::make_pair(last_frame_, -1);
273 auto frame = frame_buffer_.front();
277 file_frame_number_ = frame.second;
279 graph_->set_text(print());
281 last_frame_ = frame.first;
290 return boost::contains(filename_, L"://");
295 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
297 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
299 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
300 % (file_nb_frames()/fps)
301 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
302 % static_cast<int32_t>(file_nb_frames())
303 << core::monitor::message("/file/fps") % fps
304 << core::monitor::message("/file/path") % path_relative_to_media_
305 << core::monitor::message("/loop") % input_.loop();
308 core::draw_frame render_specific_frame(uint32_t file_position)
310 // Some trial and error and undeterministic stuff here
311 static const int NUM_RETRIES = 32;
313 if (file_position > 0) // Assume frames are requested in sequential order,
314 // therefore no seeking should be necessary for the first frame.
316 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
317 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
320 for (int i = 0; i < NUM_RETRIES; ++i)
322 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
324 auto frame = render_frame();
326 if (frame.second == std::numeric_limits<uint32_t>::max())
331 else if (frame.second == file_position + 1 || frame.second == file_position)
333 else if (frame.second > file_position + 1)
335 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
336 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
338 if (adjusted_seek > 1 && file_position > 0)
340 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
341 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
342 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
349 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
350 return core::draw_frame::empty();
353 core::draw_frame create_thumbnail_frame()
355 auto total_frames = nb_frames();
356 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
360 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
361 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
366 return render_specific_frame(total_frames / 2);
369 auto num_snapshots = grid * grid;
371 std::vector<core::draw_frame> frames;
373 for (int i = 0; i < num_snapshots; ++i)
380 desired_frame = 0; // first
381 else if (i == num_snapshots - 1)
382 desired_frame = total_frames - 1; // last
384 // evenly distributed across the file.
385 desired_frame = total_frames * i / (num_snapshots - 1);
387 auto frame = render_specific_frame(desired_frame);
388 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
389 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
390 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
391 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
393 frames.push_back(frame);
396 return core::draw_frame(frames);
399 uint32_t file_frame_number() const
401 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
404 uint32_t nb_frames() const override
406 if (is_url() || input_.loop())
407 return std::numeric_limits<uint32_t>::max();
409 auto nb_frames = std::min(input_.out(), file_nb_frames());
410 if (nb_frames >= input_.in())
411 nb_frames -= input_.in();
415 return muxer_->calc_nb_frames(nb_frames);
418 uint32_t file_nb_frames() const
420 return video_decoder_ ? video_decoder_->nb_frames() : 0;
423 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
427 std::wstring cmd = params.at(0);
429 if (params.size() > 1)
430 value = params.at(1);
432 if (boost::iequals(cmd, L"loop"))
435 input_.loop(boost::lexical_cast<bool>(value));
436 result = boost::lexical_cast<std::wstring>(input_.loop());
438 else if (boost::iequals(cmd, L"in") || boost::iequals(cmd, L"start"))
441 input_.in(boost::lexical_cast<uint32_t>(value));
442 result = boost::lexical_cast<std::wstring>(input_.in());
444 else if (boost::iequals(cmd, L"out"))
447 input_.out(boost::lexical_cast<uint32_t>(value));
448 result = boost::lexical_cast<std::wstring>(input_.out());
450 else if (boost::iequals(cmd, L"length"))
453 input_.length(boost::lexical_cast<uint32_t>(value));
454 result = boost::lexical_cast<std::wstring>(input_.length());
456 else if (boost::iequals(cmd, L"seek") && !value.empty())
458 auto nb_frames = file_nb_frames();
461 if (boost::iequals(value, L"rel"))
462 seek = file_frame_number();
463 else if (boost::iequals(value, L"in"))
465 else if (boost::iequals(value, L"out"))
467 else if (boost::iequals(value, L"end"))
470 seek = boost::lexical_cast<int64_t>(value);
472 if (params.size() > 2)
473 seek += boost::lexical_cast<int64_t>(params.at(2));
477 else if (seek >= nb_frames)
478 seek = nb_frames - 1;
480 input_.seek(static_cast<uint32_t>(seek));
483 CASPAR_THROW_EXCEPTION(invalid_argument());
485 return make_ready_future(std::move(result));
488 std::wstring print() const override
490 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
491 + print_mode() + L"|"
492 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
495 std::wstring name() const override
500 boost::property_tree::wptree info() const override
502 boost::property_tree::wptree info;
503 info.add(L"type", L"ffmpeg-producer");
504 info.add(L"filename", filename_);
505 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
506 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
507 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
508 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
509 info.add(L"loop", input_.loop());
510 info.add(L"frame-number", frame_number_);
511 auto nb_frames2 = nb_frames();
512 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
513 info.add(L"file-frame-number", file_frame_number_);
514 info.add(L"file-nb-frames", file_nb_frames());
518 core::monitor::subject& monitor_output()
520 return *monitor_subject_;
525 std::wstring print_mode() const
527 return video_decoder_ ? ffmpeg::print_mode(
528 video_decoder_->width(),
529 video_decoder_->height(),
530 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
531 !video_decoder_->is_progressive()) : L"";
534 bool all_audio_decoders_ready() const
536 for (auto& audio_decoder : audio_decoders_)
537 if (!audio_decoder->ready())
543 void try_decode_frame()
545 std::shared_ptr<AVPacket> pkt;
547 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || !all_audio_decoders_ready()) && input_.try_pop(pkt); ++n)
550 video_decoder_->push(pkt);
552 for (auto& audio_decoder : audio_decoders_)
553 audio_decoder->push(pkt);
556 std::shared_ptr<AVFrame> video;
557 std::vector<std::shared_ptr<core::mutable_audio_buffer>> audio;
559 tbb::parallel_invoke(
564 if (!muxer_->video_ready() && video_decoder_)
566 video = video_decoder_->poll();
572 } while (!video_decoder_->empty());
576 if (!muxer_->audio_ready())
578 for (auto& audio_decoder : audio_decoders_)
580 auto audio_for_stream = audio_decoder->poll();
582 if (audio_for_stream)
583 audio.push_back(audio_for_stream);
591 if (audio_decoders_.empty())
593 if (video == flush_video())
594 muxer_->push({ flush_audio() });
595 else if (!muxer_->audio_ready())
596 muxer_->push({ empty_audio() });
601 if (boost::count_if(audio, [](std::shared_ptr<core::mutable_audio_buffer> a) { return a == flush_audio(); }) > 0)
602 muxer_->push(flush_video());
603 else if (!muxer_->video_ready())
604 muxer_->push(empty_video());
607 uint32_t file_frame_number = 0;
608 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
610 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
611 if (frame != core::draw_frame::empty())
612 frame_buffer_.push(std::make_pair(frame, file_frame_number));
615 bool audio_only() const
617 return !video_decoder_;
620 boost::rational<int> get_out_framerate() const
622 return muxer_->out_framerate();
626 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
628 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
629 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {IN,SEEK [in:int]} {OUT [out:int] | LENGTH [length:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
631 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
632 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
633 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
635 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
636 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg, dshow://video={webcam_name} or v4l2://{video device}.")
637 ->item(L"loop", L"Will cause the media file to loop between in and out.")
638 ->item(L"in", L"Optionally sets the first frame. 0 by default. If loop is specified, this will be the frame where it starts over again.")
639 ->item(L"out", L"Optionally sets the last frame. If not specified the clip will be played to the end. If loop is specified, the file will jump to start position once it reaches the last frame.")
640 ->item(L"length", L"Optionally sets the length of the clip. Equivalent to OUT in + length.")
641 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
642 ->item(L"channel_layout",
643 L"Optionally override the automatically deduced audio channel layout."
644 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
645 sink.para()->text(L"Examples:");
646 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
647 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
648 sink.example(L">> PLAY 1-10 folder/clip LOOP IN 10", L"to loop a clip between frame 10 and the last frame.");
649 sink.example(L">> PLAY 1-10 folder/clip LOOP IN 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
650 sink.example(L">> PLAY 1-10 folder/clip IN 10 OUT 60", L"to play frames 10-60 in a clip and stop.");
651 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
652 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
653 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
654 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
655 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input on Windows.");
656 sink.example(L">> PLAY 1-10 v4l2:///dev/video0", L"to use a web camera as video input on Linux.");
657 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
658 sink.example(L">> CALL 1-10 LOOP 1");
659 sink.example(L">> CALL 1-10 IN 10");
660 sink.example(L">> CALL 1-10 OUT 60");
661 sink.example(L">> CALL 1-10 LENGTH 50");
662 sink.example(L">> CALL 1-10 SEEK 30");
663 core::describe_framerate_producer(sink);
666 spl::shared_ptr<core::frame_producer> create_producer(
667 const core::frame_producer_dependencies& dependencies,
668 const std::vector<std::wstring>& params,
669 const spl::shared_ptr<core::media_info_repository>& info_repo)
671 auto file_or_url = params.at(0);
673 if (!boost::contains(file_or_url, L"://"))
676 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
679 if (file_or_url.empty())
680 return core::frame_producer::empty();
682 constexpr auto uint32_max = std::numeric_limits<uint32_t>::max();
684 auto loop = contains_param(L"LOOP", params);
686 auto in = get_param(L"SEEK", params, static_cast<uint32_t>(0)); // compatibility
687 in = get_param(L"IN", params, in);
689 auto out = get_param(L"LENGTH", params, uint32_max);
690 if (out < uint32_max - in)
694 out = get_param(L"OUT", params, out);
696 auto filter_str = get_param(L"FILTER", params, L"");
697 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
699 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
700 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
701 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
703 ffmpeg_options vid_params;
704 bool haveFFMPEGStartIndicator = false;
705 for (size_t i = 0; i < params.size() - 1; ++i)
707 if (!haveFFMPEGStartIndicator && params[i] == L"--")
709 haveFFMPEGStartIndicator = true;
712 if (haveFFMPEGStartIndicator)
714 auto name = u8(params.at(i++)).substr(1);
715 auto value = u8(params.at(i));
716 vid_params.push_back(std::make_pair(name, value));
720 auto producer = spl::make_shared<ffmpeg_producer>(
721 dependencies.frame_factory,
722 dependencies.format_desc,
729 custom_channel_order,
732 if (producer->audio_only())
733 return core::create_destroy_proxy(producer);
735 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
736 auto target_framerate = dependencies.format_desc.framerate;
738 return core::create_destroy_proxy(core::create_framerate_producer(
740 get_source_framerate,
742 dependencies.format_desc.field_mode,
743 dependencies.format_desc.audio_cadence));
746 core::draw_frame create_thumbnail_frame(
747 const core::frame_producer_dependencies& dependencies,
748 const std::wstring& media_file,
749 const spl::shared_ptr<core::media_info_repository>& info_repo)
751 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
752 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
754 if (filename.empty())
755 return core::draw_frame::empty();
759 auto out = std::numeric_limits<uint32_t>::max();
760 auto filter_str = L"";
762 ffmpeg_options vid_params;
763 auto producer = spl::make_shared<ffmpeg_producer>(
764 dependencies.frame_factory,
765 dependencies.format_desc,
775 return producer->create_thumbnail_frame();