2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
34 #include <common/param.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/future.h>
38 #include <core/frame/draw_frame.h>
39 #include <core/help/help_repository.h>
40 #include <core/help/help_sink.h>
41 #include <core/producer/media_info/media_info.h>
42 #include <core/producer/framerate/framerate_producer.h>
43 #include <core/frame/frame_factory.h>
48 namespace caspar { namespace ffmpeg {
49 struct seek_out_of_range : virtual user_error {};
51 std::wstring get_relative_or_original(
52 const std::wstring& filename,
53 const boost::filesystem::path& relative_to)
55 boost::filesystem::path file(filename);
56 auto result = file.filename().wstring();
58 boost::filesystem::path current_path = file;
62 current_path = current_path.parent_path();
64 if (boost::filesystem::equivalent(current_path, relative_to))
67 if (current_path.empty())
70 result = current_path.filename().wstring() + L"/" + result;
76 struct ffmpeg_producer : public core::frame_producer_base
78 spl::shared_ptr<core::monitor::subject> monitor_subject_;
79 const std::wstring filename_;
80 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
82 FFMPEG_Resource resource_type_;
84 const spl::shared_ptr<diagnostics::graph> graph_;
87 const spl::shared_ptr<core::frame_factory> frame_factory_;
89 std::shared_ptr<void> initial_logger_disabler_;
91 core::constraints constraints_;
94 std::unique_ptr<video_decoder> video_decoder_;
95 std::unique_ptr<audio_decoder> audio_decoder_;
96 std::unique_ptr<frame_muxer> muxer_;
98 const boost::rational<int> framerate_;
99 const uint32_t start_;
100 const uint32_t length_;
101 const bool thumbnail_mode_;
103 core::draw_frame last_frame_;
105 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
107 int64_t frame_number_ = 0;
108 uint32_t file_frame_number_ = 0;
110 explicit ffmpeg_producer(
111 const spl::shared_ptr<core::frame_factory>& frame_factory,
112 const core::video_format_desc& format_desc,
113 const std::wstring& filename,
114 FFMPEG_Resource resource_type,
115 const std::wstring& filter,
120 const std::wstring& custom_channel_order,
121 const ffmpeg_options& vid_params)
122 : filename_(filename)
123 , resource_type_(resource_type)
124 , frame_factory_(frame_factory)
125 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
126 , input_(graph_, filename_, resource_type, loop, start, length, thumbnail_mode, vid_params)
127 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
130 , thumbnail_mode_(thumbnail_mode)
131 , last_frame_(core::draw_frame::empty())
134 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
135 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
136 diagnostics::register_graph(graph_);
140 video_decoder_.reset(new video_decoder(input_.context()));
141 if (!thumbnail_mode_)
142 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
144 constraints_.width.set(video_decoder_->width());
145 constraints_.height.set(video_decoder_->height());
147 catch (averror_stream_not_found&)
149 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
153 if (!thumbnail_mode_)
155 CASPAR_LOG_CURRENT_EXCEPTION();
156 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
160 auto channel_layout = core::audio_channel_layout::invalid();
162 if (!thumbnail_mode_)
166 audio_decoder_.reset(new audio_decoder(input_.context(), format_desc.audio_sample_rate));
167 channel_layout = get_audio_channel_layout(
168 audio_decoder_->num_channels(),
169 audio_decoder_->ffmpeg_channel_layout(),
170 custom_channel_order);
171 CASPAR_LOG(info) << print() << L" " << audio_decoder_->print();
173 catch (averror_stream_not_found&)
175 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
179 CASPAR_LOG_CURRENT_EXCEPTION();
180 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
184 if (!video_decoder_ && !audio_decoder_)
185 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
187 muxer_.reset(new frame_muxer(framerate_, frame_factory, format_desc, channel_layout, filter, true));
192 core::draw_frame receive_impl() override
194 return render_frame().first;
197 core::draw_frame last_frame() override
199 return core::draw_frame::still(last_frame_);
202 core::constraints& pixel_constraints() override
207 double out_fps() const
209 auto out_framerate = muxer_->out_framerate();
210 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
215 std::pair<core::draw_frame, uint32_t> render_frame()
217 frame_timer_.restart();
218 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
220 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
223 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
225 if (frame_buffer_.empty())
230 return std::make_pair(last_frame(), -1);
232 else if (resource_type_ == FFMPEG_Resource::FFMPEG_FILE)
234 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
236 return std::make_pair(core::draw_frame::late(), -1);
241 return std::make_pair(last_frame(), -1);
245 auto frame = frame_buffer_.front();
249 file_frame_number_ = frame.second;
251 graph_->set_text(print());
253 last_frame_ = frame.first;
262 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
264 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
266 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
267 % (file_nb_frames()/fps)
268 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
269 % static_cast<int32_t>(file_nb_frames())
270 << core::monitor::message("/file/fps") % fps
271 << core::monitor::message("/file/path") % path_relative_to_media_
272 << core::monitor::message("/loop") % input_.loop();
275 core::draw_frame render_specific_frame(uint32_t file_position)
277 // Some trial and error and undeterministic stuff here
278 static const int NUM_RETRIES = 32;
280 if (file_position > 0) // Assume frames are requested in sequential order,
281 // therefore no seeking should be necessary for the first frame.
283 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
284 boost::this_thread::sleep(boost::posix_time::milliseconds(40));
287 for (int i = 0; i < NUM_RETRIES; ++i)
289 boost::this_thread::sleep(boost::posix_time::milliseconds(40));
291 auto frame = render_frame();
293 if (frame.second == std::numeric_limits<uint32_t>::max())
298 else if (frame.second == file_position + 1 || frame.second == file_position)
300 else if (frame.second > file_position + 1)
302 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
303 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
305 if (adjusted_seek > 1 && file_position > 0)
307 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
308 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
309 boost::this_thread::sleep(boost::posix_time::milliseconds(40));
316 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
317 return core::draw_frame::empty();
320 core::draw_frame create_thumbnail_frame()
322 auto total_frames = nb_frames();
323 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
327 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
328 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
333 return render_specific_frame(total_frames / 2);
336 auto num_snapshots = grid * grid;
338 std::vector<core::draw_frame> frames;
340 for (int i = 0; i < num_snapshots; ++i)
347 desired_frame = 0; // first
348 else if (i == num_snapshots - 1)
349 desired_frame = total_frames - 1; // last
351 // evenly distributed across the file.
352 desired_frame = total_frames * i / (num_snapshots - 1);
354 auto frame = render_specific_frame(desired_frame);
355 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
356 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
357 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
358 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
360 frames.push_back(frame);
363 return core::draw_frame(frames);
366 uint32_t file_frame_number() const
368 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
371 uint32_t nb_frames() const override
373 if (resource_type_ == FFMPEG_Resource::FFMPEG_DEVICE || resource_type_ == FFMPEG_Resource::FFMPEG_STREAM || input_.loop())
374 return std::numeric_limits<uint32_t>::max();
376 uint32_t nb_frames = file_nb_frames();
378 nb_frames = std::min(length_, nb_frames - start_);
379 nb_frames = muxer_->calc_nb_frames(nb_frames);
384 uint32_t file_nb_frames() const
386 uint32_t file_nb_frames = 0;
387 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
388 return file_nb_frames;
391 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
393 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
394 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>\d+))", boost::regex::icase);
395 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
396 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
398 auto param = boost::algorithm::join(params, L" ");
403 if(boost::regex_match(param, what, loop_exp))
405 auto value = what["VALUE"].str();
407 input_.loop(boost::lexical_cast<bool>(value));
408 result = boost::lexical_cast<std::wstring>(input_.loop());
410 else if(boost::regex_match(param, what, seek_exp))
412 auto value = what["VALUE"].str();
413 input_.seek(boost::lexical_cast<uint32_t>(value));
415 else if(boost::regex_match(param, what, length_exp))
417 auto value = what["VALUE"].str();
419 input_.length(boost::lexical_cast<uint32_t>(value));
420 result = boost::lexical_cast<std::wstring>(input_.length());
422 else if(boost::regex_match(param, what, start_exp))
424 auto value = what["VALUE"].str();
426 input_.start(boost::lexical_cast<uint32_t>(value));
427 result = boost::lexical_cast<std::wstring>(input_.start());
430 CASPAR_THROW_EXCEPTION(invalid_argument());
432 return make_ready_future(std::move(result));
435 std::wstring print() const override
437 return L"ffmpeg[" + boost::filesystem::path(filename_).filename().wstring() + L"|"
438 + print_mode() + L"|"
439 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
442 std::wstring name() const override
447 boost::property_tree::wptree info() const override
449 boost::property_tree::wptree info;
450 info.add(L"type", L"ffmpeg-producer");
451 info.add(L"filename", filename_);
452 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
453 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
454 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
455 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
456 info.add(L"loop", input_.loop());
457 info.add(L"frame-number", frame_number_);
458 auto nb_frames2 = nb_frames();
459 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
460 info.add(L"file-frame-number", file_frame_number_);
461 info.add(L"file-nb-frames", file_nb_frames());
465 core::monitor::subject& monitor_output()
467 return *monitor_subject_;
472 std::wstring print_mode() const
474 return video_decoder_ ? ffmpeg::print_mode(
475 video_decoder_->width(),
476 video_decoder_->height(),
477 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
478 !video_decoder_->is_progressive()) : L"";
481 void try_decode_frame()
483 std::shared_ptr<AVPacket> pkt;
485 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || (audio_decoder_ && !audio_decoder_->ready())) && input_.try_pop(pkt); ++n)
488 video_decoder_->push(pkt);
490 audio_decoder_->push(pkt);
493 std::shared_ptr<AVFrame> video;
494 std::shared_ptr<core::mutable_audio_buffer> audio;
496 tbb::parallel_invoke(
499 if (!muxer_->video_ready() && video_decoder_)
500 video = video_decoder_->poll();
504 if (!muxer_->audio_ready() && audio_decoder_)
505 audio = audio_decoder_->poll();
513 if(video == flush_video())
514 muxer_->push(flush_audio());
515 else if(!muxer_->audio_ready())
516 muxer_->push(empty_audio());
521 if(audio == flush_audio())
522 muxer_->push(flush_video());
523 else if(!muxer_->video_ready())
524 muxer_->push(empty_video());
527 uint32_t file_frame_number = 0;
528 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
529 //file_frame_number = std::max(file_frame_number, audio_decoder_ ? audio_decoder_->file_frame_number() : 0);
531 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
532 frame_buffer_.push(std::make_pair(frame, file_frame_number));
535 bool audio_only() const
537 return !video_decoder_;
540 boost::rational<int> get_out_framerate() const
542 return muxer_->out_framerate();
546 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
548 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
549 sink.syntax(L"[clip:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
551 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
552 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
553 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
555 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
556 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
557 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
558 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
559 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
560 ->item(L"channel_layout",
561 L"Optionally override the automatically deduced audio channel layout. "
562 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
563 sink.para()->text(L"Examples:");
564 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
565 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
566 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
567 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
568 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
569 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
570 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
571 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
572 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
573 sink.example(L">> CALL 1-10 LOOP 1");
574 sink.example(L">> CALL 1-10 START 10");
575 sink.example(L">> CALL 1-10 LENGTH 50");
576 sink.example(L">> CALL 1-10 SEEK 30");
577 core::describe_framerate_producer(sink);
580 spl::shared_ptr<core::frame_producer> create_producer(
581 const core::frame_producer_dependencies& dependencies,
582 const std::vector<std::wstring>& params,
583 const spl::shared_ptr<core::media_info_repository>& info_repo)
585 // Infer the resource type from the resource_name
586 auto resource_type = FFMPEG_Resource::FFMPEG_FILE;
587 auto tokens = protocol_split(params.at(0));
588 auto filename = params.at(0);
590 if (!tokens[0].empty())
592 if (tokens[0] == L"dshow")
595 resource_type = FFMPEG_Resource::FFMPEG_DEVICE;
596 filename = tokens[1];
601 resource_type = FFMPEG_Resource::FFMPEG_STREAM;
602 filename = params.at(0);
608 resource_type = FFMPEG_Resource::FFMPEG_FILE;
609 filename = probe_stem(env::media_folder() + L"/" + params.at(0), false);
612 if (filename.empty())
613 return core::frame_producer::empty();
615 auto loop = contains_param(L"LOOP", params);
616 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
617 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
618 auto filter_str = get_param(L"FILTER", params, L"");
619 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
621 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
622 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
623 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
625 ffmpeg_options vid_params;
626 bool haveFFMPEGStartIndicator = false;
627 for (size_t i = 0; i < params.size() - 1; ++i)
629 if (!haveFFMPEGStartIndicator && params[i] == L"--")
631 haveFFMPEGStartIndicator = true;
634 if (haveFFMPEGStartIndicator)
636 auto name = u8(params.at(i++)).substr(1);
637 auto value = u8(params.at(i));
638 vid_params.push_back(std::make_pair(name, value));
642 auto producer = spl::make_shared<ffmpeg_producer>(
643 dependencies.frame_factory,
644 dependencies.format_desc,
652 custom_channel_order,
655 if (producer->audio_only())
656 return core::create_destroy_proxy(producer);
658 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
659 auto target_framerate = dependencies.format_desc.framerate;
661 return core::create_destroy_proxy(core::create_framerate_producer(
663 get_source_framerate,
665 dependencies.format_desc.field_mode,
666 dependencies.format_desc.audio_cadence));
669 core::draw_frame create_thumbnail_frame(
670 const core::frame_producer_dependencies& dependencies,
671 const std::wstring& media_file,
672 const spl::shared_ptr<core::media_info_repository>& info_repo)
674 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
675 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
677 if (filename.empty())
678 return core::draw_frame::empty();
682 auto length = std::numeric_limits<uint32_t>::max();
683 auto filter_str = L"";
685 ffmpeg_options vid_params;
686 auto producer = spl::make_shared<ffmpeg_producer>(
687 dependencies.frame_factory,
688 dependencies.format_desc,
690 FFMPEG_Resource::FFMPEG_FILE,
699 return producer->create_thumbnail_frame();