2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../StdAfx.h"
24 #include "ffmpeg_producer.h"
26 #include "../ffmpeg.h"
27 #include "../ffmpeg_error.h"
28 #include "util/util.h"
29 #include "input/input.h"
30 #include "audio/audio_decoder.h"
31 #include "video/video_decoder.h"
32 #include "muxer/frame_muxer.h"
34 #include <common/param.h>
35 #include <common/diagnostics/graph.h>
36 #include <common/future.h>
38 #include <core/frame/draw_frame.h>
39 #include <core/help/help_repository.h>
40 #include <core/help/help_sink.h>
41 #include <core/producer/media_info/media_info.h>
42 #include <core/producer/framerate/framerate_producer.h>
43 #include <core/frame/frame_factory.h>
48 namespace caspar { namespace ffmpeg {
49 struct seek_out_of_range : virtual user_error {};
51 std::wstring get_relative_or_original(
52 const std::wstring& filename,
53 const boost::filesystem::path& relative_to)
55 boost::filesystem::path file(filename);
56 auto result = file.filename().wstring();
58 boost::filesystem::path current_path = file;
62 current_path = current_path.parent_path();
64 if (boost::filesystem::equivalent(current_path, relative_to))
67 if (current_path.empty())
70 result = current_path.filename().wstring() + L"/" + result;
76 struct ffmpeg_producer : public core::frame_producer_base
78 spl::shared_ptr<core::monitor::subject> monitor_subject_;
79 const std::wstring filename_;
80 const std::wstring path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
82 const spl::shared_ptr<diagnostics::graph> graph_;
85 const spl::shared_ptr<core::frame_factory> frame_factory_;
87 std::shared_ptr<void> initial_logger_disabler_;
89 core::constraints constraints_;
92 std::unique_ptr<video_decoder> video_decoder_;
93 std::unique_ptr<audio_decoder> audio_decoder_;
94 std::unique_ptr<frame_muxer> muxer_;
96 const boost::rational<int> framerate_;
97 const uint32_t start_;
98 const uint32_t length_;
99 const bool thumbnail_mode_;
101 core::draw_frame last_frame_;
103 std::queue<std::pair<core::draw_frame, uint32_t>> frame_buffer_;
105 int64_t frame_number_ = 0;
106 uint32_t file_frame_number_ = 0;
108 explicit ffmpeg_producer(
109 const spl::shared_ptr<core::frame_factory>& frame_factory,
110 const core::video_format_desc& format_desc,
111 const std::wstring& url_or_file,
112 const std::wstring& filter,
117 const std::wstring& custom_channel_order,
118 const ffmpeg_options& vid_params)
119 : filename_(url_or_file)
120 , frame_factory_(frame_factory)
121 , initial_logger_disabler_(temporary_enable_quiet_logging_for_thread(thumbnail_mode))
122 , input_(graph_, url_or_file, loop, start, length, thumbnail_mode, vid_params)
123 , framerate_(read_framerate(*input_.context(), format_desc.framerate))
126 , thumbnail_mode_(thumbnail_mode)
127 , last_frame_(core::draw_frame::empty())
130 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
131 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
132 diagnostics::register_graph(graph_);
136 video_decoder_.reset(new video_decoder(input_.context()));
137 if (!thumbnail_mode_)
138 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
140 constraints_.width.set(video_decoder_->width());
141 constraints_.height.set(video_decoder_->height());
143 catch (averror_stream_not_found&)
145 //CASPAR_LOG(warning) << print() << " No video-stream found. Running without video.";
149 if (!thumbnail_mode_)
151 CASPAR_LOG_CURRENT_EXCEPTION();
152 CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
156 auto channel_layout = core::audio_channel_layout::invalid();
158 if (!thumbnail_mode_)
162 audio_decoder_.reset(new audio_decoder(input_.context(), format_desc.audio_sample_rate));
163 channel_layout = get_audio_channel_layout(
164 audio_decoder_->num_channels(),
165 audio_decoder_->ffmpeg_channel_layout(),
166 custom_channel_order);
167 CASPAR_LOG(info) << print() << L" " << audio_decoder_->print();
169 catch (averror_stream_not_found&)
171 //CASPAR_LOG(warning) << print() << " No audio-stream found. Running without audio.";
175 CASPAR_LOG_CURRENT_EXCEPTION();
176 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
180 if (!video_decoder_ && !audio_decoder_)
181 CASPAR_THROW_EXCEPTION(averror_stream_not_found() << msg_info("No streams found"));
183 muxer_.reset(new frame_muxer(framerate_, frame_factory, format_desc, channel_layout, filter, true));
188 core::draw_frame receive_impl() override
190 return render_frame().first;
193 core::draw_frame last_frame() override
195 return core::draw_frame::still(last_frame_);
198 core::constraints& pixel_constraints() override
203 double out_fps() const
205 auto out_framerate = muxer_->out_framerate();
206 auto fps = static_cast<double>(out_framerate.numerator()) / static_cast<double>(out_framerate.denominator());
211 std::pair<core::draw_frame, uint32_t> render_frame()
213 frame_timer_.restart();
214 auto disable_logging = temporary_enable_quiet_logging_for_thread(thumbnail_mode_);
216 for (int n = 0; n < 16 && frame_buffer_.size() < 2; ++n)
219 graph_->set_value("frame-time", frame_timer_.elapsed() * out_fps() *0.5);
221 if (frame_buffer_.empty())
226 return std::make_pair(last_frame(), -1);
230 graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
232 return std::make_pair(last_frame_, -1);
237 return std::make_pair(last_frame_, -1);
241 auto frame = frame_buffer_.front();
245 file_frame_number_ = frame.second;
247 graph_->set_text(print());
249 last_frame_ = frame.first;
258 return boost::contains(filename_, L"://");
263 double fps = static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator());
265 *monitor_subject_ << core::monitor::message("/profiler/time") % frame_timer_.elapsed() % (1.0/out_fps());
267 *monitor_subject_ << core::monitor::message("/file/time") % (file_frame_number()/fps)
268 % (file_nb_frames()/fps)
269 << core::monitor::message("/file/frame") % static_cast<int32_t>(file_frame_number())
270 % static_cast<int32_t>(file_nb_frames())
271 << core::monitor::message("/file/fps") % fps
272 << core::monitor::message("/file/path") % path_relative_to_media_
273 << core::monitor::message("/loop") % input_.loop();
276 core::draw_frame render_specific_frame(uint32_t file_position)
278 // Some trial and error and undeterministic stuff here
279 static const int NUM_RETRIES = 32;
281 if (file_position > 0) // Assume frames are requested in sequential order,
282 // therefore no seeking should be necessary for the first frame.
284 input_.seek(file_position > 1 ? file_position - 2: file_position).get();
285 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
288 for (int i = 0; i < NUM_RETRIES; ++i)
290 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
292 auto frame = render_frame();
294 if (frame.second == std::numeric_limits<uint32_t>::max())
299 else if (frame.second == file_position + 1 || frame.second == file_position)
301 else if (frame.second > file_position + 1)
303 CASPAR_LOG(trace) << print() << L" " << frame.second << L" received, wanted " << file_position + 1;
304 int64_t adjusted_seek = file_position - (frame.second - file_position + 1);
306 if (adjusted_seek > 1 && file_position > 0)
308 CASPAR_LOG(trace) << print() << L" adjusting to " << adjusted_seek;
309 input_.seek(static_cast<uint32_t>(adjusted_seek) - 1).get();
310 boost::this_thread::sleep_for(boost::chrono::milliseconds(40));
317 CASPAR_LOG(trace) << print() << " Giving up finding frame at " << file_position;
318 return core::draw_frame::empty();
321 core::draw_frame create_thumbnail_frame()
323 auto total_frames = nb_frames();
324 auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
328 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
329 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
334 return render_specific_frame(total_frames / 2);
337 auto num_snapshots = grid * grid;
339 std::vector<core::draw_frame> frames;
341 for (int i = 0; i < num_snapshots; ++i)
348 desired_frame = 0; // first
349 else if (i == num_snapshots - 1)
350 desired_frame = total_frames - 1; // last
352 // evenly distributed across the file.
353 desired_frame = total_frames * i / (num_snapshots - 1);
355 auto frame = render_specific_frame(desired_frame);
356 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
357 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
358 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
359 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
361 frames.push_back(frame);
364 return core::draw_frame(frames);
367 uint32_t file_frame_number() const
369 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
372 uint32_t nb_frames() const override
374 if (is_url() || input_.loop())
375 return std::numeric_limits<uint32_t>::max();
377 uint32_t nb_frames = file_nb_frames();
379 nb_frames = std::min(length_, nb_frames - start_);
380 nb_frames = muxer_->calc_nb_frames(nb_frames);
385 uint32_t file_nb_frames() const
387 uint32_t file_nb_frames = 0;
388 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
389 return file_nb_frames;
392 std::future<std::wstring> call(const std::vector<std::wstring>& params) override
394 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
395 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>(\+|-)?\d+)(\s+(?<WHENCE>REL|END))?)", boost::regex::icase);
396 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
397 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
399 auto param = boost::algorithm::join(params, L" ");
404 if(boost::regex_match(param, what, loop_exp))
406 auto value = what["VALUE"].str();
408 input_.loop(boost::lexical_cast<bool>(value));
409 result = boost::lexical_cast<std::wstring>(input_.loop());
411 else if(boost::regex_match(param, what, seek_exp))
413 auto value = boost::lexical_cast<uint32_t>(what["VALUE"].str());
414 auto whence = what["WHENCE"].str();
416 if(boost::iequals(whence, L"REL"))
418 value = file_frame_number() + value;
420 else if(boost::iequals(whence, L"END"))
422 value = file_nb_frames() - value;
427 else if(boost::regex_match(param, what, length_exp))
429 auto value = what["VALUE"].str();
431 input_.length(boost::lexical_cast<uint32_t>(value));
432 result = boost::lexical_cast<std::wstring>(input_.length());
434 else if(boost::regex_match(param, what, start_exp))
436 auto value = what["VALUE"].str();
438 input_.start(boost::lexical_cast<uint32_t>(value));
439 result = boost::lexical_cast<std::wstring>(input_.start());
442 CASPAR_THROW_EXCEPTION(invalid_argument());
444 return make_ready_future(std::move(result));
447 std::wstring print() const override
449 return L"ffmpeg[" + (is_url() ? filename_ : boost::filesystem::path(filename_).filename().wstring()) + L"|"
450 + print_mode() + L"|"
451 + boost::lexical_cast<std::wstring>(file_frame_number_) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
454 std::wstring name() const override
459 boost::property_tree::wptree info() const override
461 boost::property_tree::wptree info;
462 info.add(L"type", L"ffmpeg-producer");
463 info.add(L"filename", filename_);
464 info.add(L"width", video_decoder_ ? video_decoder_->width() : 0);
465 info.add(L"height", video_decoder_ ? video_decoder_->height() : 0);
466 info.add(L"progressive", video_decoder_ ? video_decoder_->is_progressive() : false);
467 info.add(L"fps", static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()));
468 info.add(L"loop", input_.loop());
469 info.add(L"frame-number", frame_number_);
470 auto nb_frames2 = nb_frames();
471 info.add(L"nb-frames", nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
472 info.add(L"file-frame-number", file_frame_number_);
473 info.add(L"file-nb-frames", file_nb_frames());
477 core::monitor::subject& monitor_output()
479 return *monitor_subject_;
484 std::wstring print_mode() const
486 return video_decoder_ ? ffmpeg::print_mode(
487 video_decoder_->width(),
488 video_decoder_->height(),
489 static_cast<double>(framerate_.numerator()) / static_cast<double>(framerate_.denominator()),
490 !video_decoder_->is_progressive()) : L"";
493 void try_decode_frame()
495 std::shared_ptr<AVPacket> pkt;
497 for (int n = 0; n < 32 && ((video_decoder_ && !video_decoder_->ready()) || (audio_decoder_ && !audio_decoder_->ready())) && input_.try_pop(pkt); ++n)
500 video_decoder_->push(pkt);
502 audio_decoder_->push(pkt);
505 std::shared_ptr<AVFrame> video;
506 std::shared_ptr<core::mutable_audio_buffer> audio;
508 tbb::parallel_invoke(
511 if (!muxer_->video_ready() && video_decoder_)
512 video = video_decoder_->poll();
516 if (!muxer_->audio_ready() && audio_decoder_)
517 audio = audio_decoder_->poll();
525 if(video == flush_video())
526 muxer_->push(flush_audio());
527 else if(!muxer_->audio_ready())
528 muxer_->push(empty_audio());
533 if(audio == flush_audio())
534 muxer_->push(flush_video());
535 else if(!muxer_->video_ready())
536 muxer_->push(empty_video());
539 uint32_t file_frame_number = 0;
540 file_frame_number = std::max(file_frame_number, video_decoder_ ? video_decoder_->file_frame_number() : 0);
541 //file_frame_number = std::max(file_frame_number, audio_decoder_ ? audio_decoder_->file_frame_number() : 0);
543 for (auto frame = muxer_->poll(); frame != core::draw_frame::empty(); frame = muxer_->poll())
544 frame_buffer_.push(std::make_pair(frame, file_frame_number));
547 bool audio_only() const
549 return !video_decoder_;
552 boost::rational<int> get_out_framerate() const
554 return muxer_->out_framerate();
558 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
560 sink.short_description(L"A producer for playing media files supported by FFmpeg.");
561 sink.syntax(L"[clip,url:string] {[loop:LOOP]} {SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
563 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
564 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
565 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
567 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
568 ->item(L"url", L"If clip contains :// it is instead treated as the URL parameter. The URL can either be any streaming protocol supported by FFmpeg or dshow://video={webcam_name}.")
569 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
570 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
571 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
572 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
573 ->item(L"channel_layout",
574 L"Optionally override the automatically deduced audio channel layout. "
575 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
576 sink.para()->text(L"Examples:");
577 sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
578 sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
579 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10", L"to loop a clip between frame 10 and the last frame.");
580 sink.example(L">> PLAY 1-10 folder/clip LOOP SEEK 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
581 sink.example(L">> PLAY 1-10 folder/clip SEEK 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
582 sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
583 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
584 sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
585 sink.example(L">> PLAY 1-10 rtmp://example.com/live/stream", L"to play an RTMP stream.");
586 sink.example(L">> PLAY 1-10 \"dshow://video=Live! Cam Chat HD VF0790\"", L"to use a web camera as video input.");
587 sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via ")->code(L"CALL")->text(L":");
588 sink.example(L">> CALL 1-10 LOOP 1");
589 sink.example(L">> CALL 1-10 START 10");
590 sink.example(L">> CALL 1-10 LENGTH 50");
591 sink.example(L">> CALL 1-10 SEEK 30");
592 core::describe_framerate_producer(sink);
595 spl::shared_ptr<core::frame_producer> create_producer(
596 const core::frame_producer_dependencies& dependencies,
597 const std::vector<std::wstring>& params,
598 const spl::shared_ptr<core::media_info_repository>& info_repo)
600 auto file_or_url = params.at(0);
602 if (!boost::contains(file_or_url, L"://"))
605 file_or_url = probe_stem(env::media_folder() + L"/" + file_or_url, false);
608 if (file_or_url.empty())
609 return core::frame_producer::empty();
611 auto loop = contains_param(L"LOOP", params);
612 auto start = get_param(L"SEEK", params, static_cast<uint32_t>(0));
613 auto length = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
614 auto filter_str = get_param(L"FILTER", params, L"");
615 auto custom_channel_order = get_param(L"CHANNEL_LAYOUT", params, L"");
617 boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1");
618 boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS");
619 boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1");
621 ffmpeg_options vid_params;
622 bool haveFFMPEGStartIndicator = false;
623 for (size_t i = 0; i < params.size() - 1; ++i)
625 if (!haveFFMPEGStartIndicator && params[i] == L"--")
627 haveFFMPEGStartIndicator = true;
630 if (haveFFMPEGStartIndicator)
632 auto name = u8(params.at(i++)).substr(1);
633 auto value = u8(params.at(i));
634 vid_params.push_back(std::make_pair(name, value));
638 auto producer = spl::make_shared<ffmpeg_producer>(
639 dependencies.frame_factory,
640 dependencies.format_desc,
647 custom_channel_order,
650 if (producer->audio_only())
651 return core::create_destroy_proxy(producer);
653 auto get_source_framerate = [=] { return producer->get_out_framerate(); };
654 auto target_framerate = dependencies.format_desc.framerate;
656 return core::create_destroy_proxy(core::create_framerate_producer(
658 get_source_framerate,
660 dependencies.format_desc.field_mode,
661 dependencies.format_desc.audio_cadence));
664 core::draw_frame create_thumbnail_frame(
665 const core::frame_producer_dependencies& dependencies,
666 const std::wstring& media_file,
667 const spl::shared_ptr<core::media_info_repository>& info_repo)
669 auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
670 auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
672 if (filename.empty())
673 return core::draw_frame::empty();
677 auto length = std::numeric_limits<uint32_t>::max();
678 auto filter_str = L"";
680 ffmpeg_options vid_params;
681 auto producer = spl::make_shared<ffmpeg_producer>(
682 dependencies.frame_factory,
683 dependencies.format_desc,
693 return producer->create_thumbnail_frame();