2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
34 #include <common/param.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/future.h>
38 #include <core/frame/draw_frame.h>
39 #include <core/help/help_repository.h>
40 #include <core/help/help_sink.h>
41 #include <core/producer/media_info/media_info.h>
42 #include <core/producer/framerate/framerate_producer.h>
43 #include <core/frame/frame_factory.h>
48 namespace caspar { namespace ffmpeg {
49 struct seek_out_of_range : virtual user_error {};
51 std::wstring get_relative_or_original(
52 const std::wstring& filename,
53 const boost::filesystem::path& relative_to)
55 boost::filesystem::path file(filename);
56 auto result = file.filename().wstring();
58 boost::filesystem::path current_path = file;
62 current_path = current_path.parent_path();
64 if (boost::filesystem::equivalent(current_path, relative_to))
67 if (current_path.empty())
70 result = current_path.filename().wstring() + L"/" + result;
76 struct ffmpeg_producer : public core::frame_producer_base
78 spl::shared_ptr<core::monitor::subject> monitor_subject_;
79 const std::wstring filename_;
80 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
82 const spl::shared_ptr<diagnostics::graph> graph_;
85 const spl::shared_ptr<core::frame_factory> frame_factory_;
87 std::shared_ptr<void> initial_logger_disabler_;
89 core::constraints constraints_;
92 std::unique_ptr<video_decoder> video_decoder_;
93 std::unique_ptr<audio_decoder> audio_decoder_;
94 std::unique_ptr<frame_muxer> muxer_;
96 const boost::rational<int> framerate_;
97 const uint32_t start_;
98 const uint32_t length_;
99 const bool thumbnail_mode_;
101 core::draw_frame last_frame_;
103 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
105 int64_t frame_number_ = 0;
106 uint32_t file_frame_number_ = 0;
108 explicit ffmpeg_producer(
109 const spl::shared_ptr<core::frame_factory>& frame_factory,
110 const core::video_format_desc& format_desc,
111 const std::wstring& url_or_file,
112 const std::wstring& filter,
117 const std::wstring& custom_channel_order,
118 const ffmpeg_options& vid_params)
119 : filename_(url_or_file)
120 , frame_factory_(frame_factory)
121 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
122 , input_(graph_, url_or_file, loop, start, length, thumbnail_mode, vid_params)
123 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
126 , thumbnail_mode_(thumbnail_mode)
127 , last_frame_(core::draw_frame::empty())
130 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
131 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
132 diagnostics::register_graph(graph_);
136 video_decoder_.reset(new video_decoder(input_.context()));
137 if (!thumbnail_mode_)
138 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
140 constraints_.width.set(video_decoder_->width());
141 constraints_.height.set(video_decoder_->height());
143 catch (averror_stream_not_found&)
145 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
149 if (!thumbnail_mode_)
151 CASPAR_LOG_CURRENT_EXCEPTION();
152 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
156 auto channel_layout = core::audio_channel_layout::invalid();
158 if (!thumbnail_mode_)
162 audio_decoder_.reset(new audio_decoder(input_.context(), format_desc.audio_sample_rate));
163 channel_layout = get_audio_channel_layout(
164 audio_decoder_->num_channels(),
165 audio_decoder_->ffmpeg_channel_layout(),
166 custom_channel_order);
167 CASPAR_LOG(info) << print() << L" " << audio_decoder_->print();
169 catch (averror_stream_not_found&)
171 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
175 CASPAR_LOG_CURRENT_EXCEPTION();
176 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
180 if (!video_decoder_ && !audio_decoder_)
181 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
183 muxer_.reset(new frame_muxer(framerate_, frame_factory, format_desc, channel_layout, filter, true));
188 core::draw_frame receive_impl() override
190 return render_frame().first;
193 core::draw_frame last_frame() override
195 return core::draw_frame::still(last_frame_);
198 core::constraints& pixel_constraints() override
203 double out_fps() const
205 auto out_framerate = muxer_->out_framerate();
206 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
211 std::pair<core::draw_frame, uint32_t> render_frame()
213 frame_timer_.restart();
214 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
216 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
219 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
221 if (frame_buffer_.empty())
226 return std::make_pair(last_frame(), -1);
230 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
232 return std::make_pair(last_frame_, -1);
237 return std::make_pair(last_frame_, -1);
241 auto frame = frame_buffer_.front();
245 file_frame_number_ = frame.second;
247 graph_->set_text(print());
249 last_frame_ = frame.first;
258 return boost::contains(filename_, L"://");
263 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
265 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
267 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
268 % (file_nb_frames()/fps)
269 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
270 % static_cast<int32_t>(file_nb_frames())
271 << core::monitor::message("/file/fps") % fps
272 << core::monitor::message("/file/path") % path_relative_to_media_
273 << core::monitor::message("/loop") % input_.loop();
276 core::draw_frame render_specific_frame(uint32_t file_position)
278 // Some trial and error and undeterministic stuff here
279 static const int NUM_RETRIES = 32;
281 if (file_position > 0) // Assume frames are requested in sequential order,
282 // therefore no seeking should be necessary for the first frame.
284 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
285 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
288 for (int i = 0; i < NUM_RETRIES; ++i)
290 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
292 auto frame = render_frame();
294 if (frame.second == std::numeric_limits<uint32_t>::max())
299 else if (frame.second == file_position + 1 || frame.second == file_position)
301 else if (frame.second > file_position + 1)
303 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
304 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
306 if (adjusted_seek > 1 && file_position > 0)
308 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
309 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
310 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
317 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
318 return core::draw_frame::empty();
321 core::draw_frame create_thumbnail_frame()
323 auto total_frames = nb_frames();
324 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
328 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
329 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
334 return render_specific_frame(total_frames / 2);
337 auto num_snapshots = grid * grid;
339 std::vector<core::draw_frame> frames;
341 for (int i = 0; i < num_snapshots; ++i)
348 desired_frame = 0; // first
349 else if (i == num_snapshots - 1)
350 desired_frame = total_frames - 1; // last
352 // evenly distributed across the file.
353 desired_frame = total_frames * i / (num_snapshots - 1);
355 auto frame = render_specific_frame(desired_frame);
356 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
357 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
358 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
359 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
361 frames.push_back(frame);
364 return core::draw_frame(frames);
367 uint32_t file_frame_number() const
369 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
372 uint32_t nb_frames() const override
374 if (is_url() || input_.loop())
375 return std::numeric_limits<uint32_t>::max();
377 uint32_t nb_frames = file_nb_frames();
379 nb_frames = std::min(length_, nb_frames - start_);
380 nb_frames = muxer_->calc_nb_frames(nb_frames);
385 uint32_t file_nb_frames() const
387 uint32_t file_nb_frames = 0;
388 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
389 return file_nb_frames;
392 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
394 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
395 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>\d+))", boost::regex::icase);
396 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
397 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
399 auto param = boost::algorithm::join(params, L" ");
404 if(boost::regex_match(param, what, loop_exp))
406 auto value = what["VALUE"].str();
408 input_.loop(boost::lexical_cast<bool>(value));
409 result = boost::lexical_cast<std::wstring>(input_.loop());
411 else if(boost::regex_match(param, what, seek_exp))
413 auto value = what["VALUE"].str();
414 input_.seek(boost::lexical_cast<uint32_t>(value));
416 else if(boost::regex_match(param, what, length_exp))
418 auto value = what["VALUE"].str();
420 input_.length(boost::lexical_cast<uint32_t>(value));
421 result = boost::lexical_cast<std::wstring>(input_.length());
423 else if(boost::regex_match(param, what, start_exp))
425 auto value = what["VALUE"].str();
427 input_.start(boost::lexical_cast<uint32_t>(value));
428 result = boost::lexical_cast<std::wstring>(input_.start());
431 CASPAR_THROW_EXCEPTION(invalid_argument());
433 return make_ready_future(std::move(result));
436 std::wstring print() const override
438 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
439 + print_mode() + L"|"
440 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
443 std::wstring name() const override
448 boost::property_tree::wptree info() const override
450 boost::property_tree::wptree info;
451 info.add(L"type", L"ffmpeg-producer");
452 info.add(L"filename", filename_);
453 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
454 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
455 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
456 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
457 info.add(L"loop", input_.loop());
458 info.add(L"frame-number", frame_number_);
459 auto nb_frames2 = nb_frames();
460 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
461 info.add(L"file-frame-number", file_frame_number_);
462 info.add(L"file-nb-frames", file_nb_frames());
466 core::monitor::subject& monitor_output()
468 return *monitor_subject_;
473 std::wstring print_mode() const
475 return video_decoder_ ? ffmpeg::print_mode(
476 video_decoder_->width(),
477 video_decoder_->height(),
478 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
479 !video_decoder_->is_progressive()) : L"";
482 void try_decode_frame()
484 std::shared_ptr<AVPacket> pkt;
486 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || (audio_decoder_ && !audio_decoder_->ready())) && input_.try_pop(pkt); ++n)
489 video_decoder_->push(pkt);
491 audio_decoder_->push(pkt);
494 std::shared_ptr<AVFrame> video;
495 std::shared_ptr<core::mutable_audio_buffer> audio;
497 tbb::parallel_invoke(
500 if (!muxer_->video_ready() && video_decoder_)
501 video = video_decoder_->poll();
505 if (!muxer_->audio_ready() && audio_decoder_)
506 audio = audio_decoder_->poll();
514 if(video == flush_video())
515 muxer_->push(flush_audio());
516 else if(!muxer_->audio_ready())
517 muxer_->push(empty_audio());
522 if(audio == flush_audio())
523 muxer_->push(flush_video());
524 else if(!muxer_->video_ready())
525 muxer_->push(empty_video());
528 uint32_t file_frame_number = 0;
529 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
530 //file_frame_number = std::max(file_frame_number, audio_decoder_ ? audio_decoder_->file_frame_number() : 0);
532 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
533 frame_buffer_.push(std::make_pair(frame, file_frame_number));
536 bool audio_only() const
538 return !video_decoder_;
541 boost::rational<int> get_out_framerate() const
543 return muxer_->out_framerate();
547 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
549 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
550 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
552 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
553 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
554 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
556 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
557 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg or dshow://video={webcam_name}.")
558 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
559 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
560 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
561 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
562 ->item(L"channel_layout",
563 L"Optionally override the automatically deduced audio channel layout. "
564 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
565 sink.para()->text(L"Examples:");
566 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
567 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
568 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
569 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
570 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
571 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
572 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
573 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
574 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
575 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input.");
576 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
577 sink.example(L">> CALL 1-10 LOOP 1");
578 sink.example(L">> CALL 1-10 START 10");
579 sink.example(L">> CALL 1-10 LENGTH 50");
580 sink.example(L">> CALL 1-10 SEEK 30");
581 core::describe_framerate_producer(sink);
584 spl::shared_ptr<core::frame_producer> create_producer(
585 const core::frame_producer_dependencies& dependencies,
586 const std::vector<std::wstring>& params,
587 const spl::shared_ptr<core::media_info_repository>& info_repo)
589 auto file_or_url = params.at(0);
591 if (!boost::contains(file_or_url, L"://"))
594 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
597 if (file_or_url.empty())
598 return core::frame_producer::empty();
600 auto loop = contains_param(L"LOOP", params);
601 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
602 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
603 auto filter_str = get_param(L"FILTER", params, L"");
604 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
606 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
607 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
608 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
610 ffmpeg_options vid_params;
611 bool haveFFMPEGStartIndicator = false;
612 for (size_t i = 0; i < params.size() - 1; ++i)
614 if (!haveFFMPEGStartIndicator && params[i] == L"--")
616 haveFFMPEGStartIndicator = true;
619 if (haveFFMPEGStartIndicator)
621 auto name = u8(params.at(i++)).substr(1);
622 auto value = u8(params.at(i));
623 vid_params.push_back(std::make_pair(name, value));
627 auto producer = spl::make_shared<ffmpeg_producer>(
628 dependencies.frame_factory,
629 dependencies.format_desc,
636 custom_channel_order,
639 if (producer->audio_only())
640 return core::create_destroy_proxy(producer);
642 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
643 auto target_framerate = dependencies.format_desc.framerate;
645 return core::create_destroy_proxy(core::create_framerate_producer(
647 get_source_framerate,
649 dependencies.format_desc.field_mode,
650 dependencies.format_desc.audio_cadence));
653 core::draw_frame create_thumbnail_frame(
654 const core::frame_producer_dependencies& dependencies,
655 const std::wstring& media_file,
656 const spl::shared_ptr<core::media_info_repository>& info_repo)
658 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
659 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
661 if (filename.empty())
662 return core::draw_frame::empty();
666 auto length = std::numeric_limits<uint32_t>::max();
667 auto filter_str = L"";
669 ffmpeg_options vid_params;
670 auto producer = spl::make_shared<ffmpeg_producer>(
671 dependencies.frame_factory,
672 dependencies.format_desc,
682 return producer->create_thumbnail_frame();