git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/ffmpeg_producer.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../StdAfx.h"
  23
  24 #include "ffmpeg_producer.h"
  25
  26 #include "../ffmpeg_error.h"
  27 #include "../ffmpeg.h"
  28
  29 #include "muxer/frame_muxer.h"
  30 #include "input/input.h"
  31 #include "util/util.h"
  32 #include "audio/audio_decoder.h"
  33 #include "video/video_decoder.h"
  34
  35 #include <common/env.h>
  36 #include <common/log.h>
  37 #include <common/param.h>
  38 #include <common/diagnostics/graph.h>
  39 #include <common/future.h>
  40 #include <common/timer.h>
  41 #include <common/assert.h>
  42
  43 #include <core/video_format.h>
  44 #include <core/producer/frame_producer.h>
  45 #include <core/frame/audio_channel_layout.h>
  46 #include <core/frame/frame_factory.h>
  47 #include <core/frame/draw_frame.h>
  48 #include <core/frame/frame_transform.h>
  49 #include <core/monitor/monitor.h>
  50 #include <core/help/help_repository.h>
  51 #include <core/help/help_sink.h>
  52 #include <core/producer/media_info/media_info_repository.h>
  53 #include <core/producer/media_info/media_info.h>
  54
  55 #include <boost/algorithm/string.hpp>
  56 #include <boost/filesystem.hpp>
  57 #include <boost/property_tree/ptree.hpp>
  58 #include <boost/regex.hpp>
  59 #include <boost/thread/future.hpp>
  60
  61 #include <tbb/parallel_invoke.h>
  62
  63 #include <limits>
  64 #include <memory>
  65 #include <queue>
  66
  67 namespace caspar { namespace ffmpeg {
  68
  69 struct seek_out_of_range : virtual user_error {};
  70
  71 std::wstring get_relative_or_original(
  72                 const std::wstring& filename,
  73                 const boost::filesystem::path& relative_to)
  74 {
  75         boost::filesystem::path file(filename);
  76         auto result = file.filename().wstring();
  77
  78         boost::filesystem::path current_path = file;
  79
  80         while (true)
  81         {
  82                 current_path = current_path.parent_path();
  83
  84                 if (boost::filesystem::equivalent(current_path, relative_to))
  85                         break;
  86
  87                 if (current_path.empty())
  88                         return filename;
  89
  90                 result = current_path.filename().wstring() + L"/" + result;
  91         }
  92
  93         return result;
  94 }
  95
  96 struct ffmpeg_producer : public core::frame_producer_base
  97 {
  98         spl::shared_ptr<core::monitor::subject>                 monitor_subject_;
  99         const std::wstring                                                              filename_;
 100         const std::wstring                                                              path_relative_to_media_ = get_relative_or_original(filename_, env::media_folder());
 101
 102         const spl::shared_ptr<diagnostics::graph>               graph_;
 103
 104         const spl::shared_ptr<core::frame_factory>              frame_factory_;
 105         const core::video_format_desc                                   format_desc_;
 106
 107         input                                                                                   input_;
 108
 109         const double                                                                    fps_                                    = read_fps(input_.context(), format_desc_.fps);
 110         const uint32_t                                                                  start_;
 111         const bool                                                                              thumbnail_mode_;
 112         const boost::optional<core::media_info>                 info_;
 113
 114         std::unique_ptr<video_decoder>                                  video_decoder_;
 115         std::unique_ptr<audio_decoder>                                  audio_decoder_;
 116         std::unique_ptr<frame_muxer>                                    muxer_;
 117         core::constraints                                                               constraints_;
 118
 119         core::draw_frame                                                                last_frame_                             = core::draw_frame::empty();
 120
 121         boost::optional<uint32_t>                                               seek_target_;
 122
 123 public:
 124         explicit ffmpeg_producer(
 125                         const spl::shared_ptr<core::frame_factory>& frame_factory,
 126                         const core::video_format_desc& format_desc,
 127                         const std::wstring& channel_layout_spec,
 128                         const std::wstring& filename,
 129                         const std::wstring& filter,
 130                         bool loop,
 131                         uint32_t start,
 132                         uint32_t length,
 133                         bool thumbnail_mode,
 134                         boost::optional<core::media_info> info)
 135                 : filename_(filename)
 136                 , frame_factory_(frame_factory)
 137                 , format_desc_(format_desc)
 138                 , input_(graph_, filename_, loop, start, length, thumbnail_mode)
 139                 , start_(start)
 140                 , thumbnail_mode_(thumbnail_mode)
 141                 , info_(info)
 142         {
 143                 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
 144                 graph_->set_color("underflow", diagnostics::color(0.6f, 0.3f, 0.9f));
 145                 diagnostics::register_graph(graph_);
 146
 147
 148                 try
 149                 {
 150                         video_decoder_.reset(new video_decoder(input_, thumbnail_mode));
 151                         video_decoder_->monitor_output().attach_parent(monitor_subject_);
 152                         constraints_.width.set(video_decoder_->width());
 153                         constraints_.height.set(video_decoder_->height());
 154
 155                         if (is_logging_quiet_for_thread())
 156                                 CASPAR_LOG(debug) << print() << L" " << video_decoder_->print();
 157                         else
 158                                 CASPAR_LOG(info) << print() << L" " << video_decoder_->print();
 159                 }
 160                 catch(averror_stream_not_found&)
 161                 {
 162                         CASPAR_LOG(debug) << print() << " No video-stream found. Running without video.";
 163                 }
 164                 catch(...)
 165                 {
 166                         CASPAR_LOG_CURRENT_EXCEPTION();
 167                         CASPAR_LOG(warning) << print() << "Failed to open video-stream. Running without video.";
 168                 }
 169
 170                 auto channel_layout = core::audio_channel_layout::invalid();
 171
 172                 if (!thumbnail_mode)
 173                 {
 174                         try
 175                         {
 176                                 audio_decoder_.reset(new audio_decoder(input_, format_desc_, channel_layout_spec));
 177                                 audio_decoder_->monitor_output().attach_parent(monitor_subject_);
 178
 179                                 channel_layout = audio_decoder_->channel_layout();
 180
 181                                 CASPAR_LOG(info) << print() << L" " << audio_decoder_->print();
 182                         }
 183                         catch (averror_stream_not_found&)
 184                         {
 185                                 CASPAR_LOG(debug) << print() << " No audio-stream found. Running without audio.";
 186                         }
 187                         catch (...)
 188                         {
 189                                 CASPAR_LOG_CURRENT_EXCEPTION();
 190                                 CASPAR_LOG(warning) << print() << " Failed to open audio-stream. Running without audio.";
 191                         }
 192                 }
 193
 194                 if (start_ > file_nb_frames())
 195                         CASPAR_THROW_EXCEPTION(seek_out_of_range() << msg_info("SEEK out of range"));
 196
 197                 muxer_.reset(new frame_muxer(fps_, frame_factory, format_desc_, channel_layout, filter));
 198
 199                 decode_next_frame();
 200
 201                 if (is_logging_quiet_for_thread())
 202                         CASPAR_LOG(debug) << print() << L" Initialized";
 203                 else
 204                         CASPAR_LOG(info) << print() << L" Initialized";
 205         }
 206
 207         // frame_producer
 208
 209         core::draw_frame receive_impl() override
 210         {
 211                 auto frame = core::draw_frame::late();
 212
 213                 caspar::timer frame_timer;
 214
 215                 end_seek();
 216
 217                 decode_next_frame();
 218
 219                 if(!muxer_->empty())
 220                 {
 221                         last_frame_ = frame = std::move(muxer_->front());
 222                         muxer_->pop();
 223                 }
 224                 else if (!input_.eof())
 225                         graph_->set_tag(diagnostics::tag_severity::WARNING, "underflow");
 226
 227                 graph_->set_value("frame-time", frame_timer.elapsed()*format_desc_.fps*0.5);
 228                 *monitor_subject_
 229                                 << core::monitor::message("/profiler/time")     % frame_timer.elapsed() % (1.0/format_desc_.fps);
 230                 *monitor_subject_
 231                                 << core::monitor::message("/file/frame")        % static_cast<int32_t>(file_frame_number())
 232                                                                                                                         % static_cast<int32_t>(file_nb_frames())
 233                                 << core::monitor::message("/file/fps")          % fps_
 234                                 << core::monitor::message("/file/path")         % path_relative_to_media_
 235                                 << core::monitor::message("/loop")                      % input_.loop();
 236
 237                 return frame;
 238         }
 239
 240         core::draw_frame last_frame() override
 241         {
 242                 end_seek();
 243                 return core::draw_frame::still(last_frame_);
 244         }
 245
 246         core::constraints& pixel_constraints() override
 247         {
 248                 return constraints_;
 249         }
 250
 251         uint32_t nb_frames() const override
 252         {
 253                 if(input_.loop())
 254                         return std::numeric_limits<uint32_t>::max();
 255
 256                 uint32_t nb_frames = file_nb_frames();
 257
 258                 nb_frames = std::min(input_.length(), nb_frames);
 259                 nb_frames = muxer_->calc_nb_frames(nb_frames);
 260
 261                 return nb_frames > start_ ? nb_frames - start_ : 0;
 262         }
 263
 264         uint32_t file_nb_frames() const
 265         {
 266                 uint32_t file_nb_frames = 0;
 267
 268                 if (info_)
 269                         file_nb_frames = static_cast<uint32_t>(info_->duration);
 270
 271                 file_nb_frames = std::max(file_nb_frames, video_decoder_ ? video_decoder_->nb_frames() : 0);
 272                 file_nb_frames = std::max(file_nb_frames, audio_decoder_ ? audio_decoder_->nb_frames() : 0);
 273                 return file_nb_frames;
 274         }
 275
 276         uint32_t file_frame_number() const
 277         {
 278                 return video_decoder_ ? video_decoder_->file_frame_number() : 0;
 279         }
 280
 281         std::future<std::wstring> call(const std::vector<std::wstring>& params) override
 282         {
 283                 static const boost::wregex loop_exp(LR"(LOOP\s*(?<VALUE>\d?)?)", boost::regex::icase);
 284                 static const boost::wregex seek_exp(LR"(SEEK\s+(?<VALUE>\d+))", boost::regex::icase);
 285                 static const boost::wregex length_exp(LR"(LENGTH\s+(?<VALUE>\d+)?)", boost::regex::icase);
 286                 static const boost::wregex start_exp(LR"(START\\s+(?<VALUE>\\d+)?)", boost::regex::icase);
 287
 288                 auto param = boost::algorithm::join(params, L" ");
 289
 290                 std::wstring result;
 291
 292                 boost::wsmatch what;
 293                 if(boost::regex_match(param, what, loop_exp))
 294                 {
 295                         auto value = what["VALUE"].str();
 296                         if(!value.empty())
 297                                 input_.loop(boost::lexical_cast<bool>(value));
 298                         result = boost::lexical_cast<std::wstring>(loop());
 299                 }
 300                 else if(boost::regex_match(param, what, seek_exp))
 301                 {
 302                         auto value = what["VALUE"].str();
 303                         seek(boost::lexical_cast<uint32_t>(value));
 304                 }
 305                 else if(boost::regex_match(param, what, length_exp))
 306                 {
 307                         auto value = what["VALUE"].str();
 308                         if(!value.empty())
 309                                 length(boost::lexical_cast<uint32_t>(value));
 310                         result = boost::lexical_cast<std::wstring>(length());
 311                 }
 312                 else if(boost::regex_match(param, what, start_exp))
 313                 {
 314                         auto value = what["VALUE"].str();
 315                         if(!value.empty())
 316                                 start(boost::lexical_cast<uint32_t>(value));
 317                         result = boost::lexical_cast<std::wstring>(start());
 318                 }
 319                 else
 320                         CASPAR_THROW_EXCEPTION(invalid_argument());
 321
 322                 return make_ready_future(std::move(result));
 323         }
 324
 325         std::wstring print() const override
 326         {
 327                 return L"ffmpeg[" + boost::filesystem::path(filename_).filename().wstring() + L"|"
 328                                                   + print_mode() + L"|"
 329                                                   + boost::lexical_cast<std::wstring>(file_frame_number()) + L"/" + boost::lexical_cast<std::wstring>(file_nb_frames()) + L"]";
 330         }
 331
 332         std::wstring name() const override
 333         {
 334                 return L"ffmpeg";
 335         }
 336
 337         boost::property_tree::wptree info() const override
 338         {
 339                 boost::property_tree::wptree info;
 340                 info.add(L"type",                               L"ffmpeg");
 341                 info.add(L"filename",                   filename_);
 342                 info.add(L"width",                              video_decoder_ ? video_decoder_->width() : 0);
 343                 info.add(L"height",                             video_decoder_ ? video_decoder_->height() : 0);
 344                 info.add(L"progressive",                video_decoder_ ? video_decoder_->is_progressive() : 0);
 345                 info.add(L"fps",                                fps_);
 346                 info.add(L"loop",                               input_.loop());
 347                 info.add(L"frame-number",               frame_number());
 348                 auto nb_frames2 = nb_frames();
 349                 info.add(L"nb-frames",                  nb_frames2 == std::numeric_limits<int64_t>::max() ? -1 : nb_frames2);
 350                 info.add(L"file-frame-number",  file_frame_number());
 351                 info.add(L"file-nb-frames",             file_nb_frames());
 352                 return info;
 353         }
 354
 355         core::monitor::subject& monitor_output()
 356         {
 357                 return *monitor_subject_;
 358         }
 359
 360         // ffmpeg_producer
 361
 362         void end_seek()
 363         {
 364                 for(int n = 0; n < 8 && (last_frame_ == core::draw_frame::empty() || (seek_target_ && file_frame_number() != *seek_target_+2)); ++n)
 365                 {
 366                         decode_next_frame();
 367                         if(!muxer_->empty())
 368                         {
 369                                 last_frame_ = muxer_->front();
 370                                 seek_target_.reset();
 371                         }
 372                 }
 373         }
 374
 375         void loop(bool value)
 376         {
 377                 input_.loop(value);
 378         }
 379
 380         bool loop() const
 381         {
 382                 return input_.loop();
 383         }
 384
 385         void length(uint32_t value)
 386         {
 387                 input_.length(value);
 388         }
 389
 390         uint32_t length()
 391         {
 392                 return input_.length();
 393         }
 394
 395         void start(uint32_t value)
 396         {
 397                 input_.start(value);
 398         }
 399
 400         uint32_t start()
 401         {
 402                 return input_.start();
 403         }
 404
 405         void seek(uint32_t target)
 406         {
 407                 if (target > file_nb_frames())
 408                         CASPAR_THROW_EXCEPTION(seek_out_of_range() << msg_info("SEEK out of range"));
 409
 410                 seek_target_ = target;
 411
 412                 input_.seek(*seek_target_);
 413                 muxer_->clear();
 414         }
 415
 416         std::wstring print_mode() const
 417         {
 418                 return ffmpeg::print_mode(video_decoder_ ? video_decoder_->width() : 0,
 419                         video_decoder_ ? video_decoder_->height() : 0,
 420                         fps_,
 421                         video_decoder_ ? !video_decoder_->is_progressive() : false);
 422         }
 423
 424         void decode_next_frame()
 425         {
 426                 for(int n = 0; n < 32 && muxer_->empty(); ++n)
 427                 {
 428                         std::shared_ptr<AVFrame> video;
 429                         std::shared_ptr<AVFrame> audio;
 430                         bool needs_video = !muxer_->video_ready();
 431                         bool needs_audio = !muxer_->audio_ready();
 432
 433                         tbb::parallel_invoke(
 434                                         [&]
 435                                         {
 436                                                 if (needs_video)
 437                                                         video = video_decoder_ ? (*video_decoder_)() : create_frame();
 438                                         },
 439                                         [&]
 440                                         {
 441                                                 if (needs_audio)
 442                                                         audio = audio_decoder_ ? (*audio_decoder_)() : create_frame();
 443                                         });
 444
 445                         muxer_->push_video(video);
 446                         muxer_->push_audio(audio);
 447                 }
 448
 449                 graph_->set_text(print());
 450         }
 451 };
 452
 453 void describe_producer(core::help_sink& sink, const core::help_repository& repo)
 454 {
 455         sink.short_description(L"A producer for playing media files supported by FFmpeg.");
 456         sink.syntax(L"[clip:string] {[loop:LOOP]} {START,SEEK [start:int]} {LENGTH [start:int]} {FILTER [filter:string]} {CHANNEL_LAYOUT [channel_layout:string]}");
 457         sink.para()
 458                 ->text(L"The FFmpeg Producer can play all media that FFmpeg can play, which includes many ")
 459                 ->text(L"QuickTime video codec such as Animation, PNG, PhotoJPEG, MotionJPEG, as well as ")
 460                 ->text(L"H.264, FLV, WMV and several audio codecs as well as uncompressed audio.");
 461         sink.definitions()
 462                 ->item(L"clip", L"The file without the file extension to play. It should reside under the media folder.")
 463                 ->item(L"loop", L"Will cause the media file to loop between start and start + length")
 464                 ->item(L"start", L"Optionally sets the start frame. 0 by default. If loop is specified this will be the frame where it starts over again.")
 465                 ->item(L"length", L"Optionally sets the length of the clip. If not specified the clip will be played to the end. If loop is specified the file will jump to start position once this number of frames has been played.")
 466                 ->item(L"filter", L"If specified, will be used as an FFmpeg video filter.")
 467                 ->item(L"channel_layout",
 468                                 L"Optionally override the automatically deduced audio channel layout. "
 469                                 L"Either a named layout as specified in casparcg.config or in the format [type:string]:[channel_order:string] for a custom layout.");
 470         sink.para()->text(L"Examples:");
 471         sink.example(L">> PLAY 1-10 folder/clip", L"to play all frames in a clip and stop at the last frame.");
 472         sink.example(L">> PLAY 1-10 folder/clip LOOP", L"to loop a clip between the first frame and the last frame.");
 473         sink.example(L">> PLAY 1-10 folder/clip LOOP START 10", L"to loop a clip between frame 10 and the last frame.");
 474         sink.example(L">> PLAY 1-10 folder/clip LOOP START 10 LENGTH 50", L"to loop a clip between frame 10 and frame 60.");
 475         sink.example(L">> PLAY 1-10 folder/clip START 10 LENGTH 50", L"to play frames 10-60 in a clip and stop.");
 476         sink.example(L">> PLAY 1-10 folder/clip FILTER yadif=1,-1", L"to deinterlace the video.");
 477         sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT film", L"given the defaults in casparcg.config this will specifies that the clip has 6 audio channels of the type 5.1 and that they are in the order FL FC FR BL BR LFE regardless of what ffmpeg says.");
 478         sink.example(L">> PLAY 1-10 folder/clip CHANNEL_LAYOUT \"5.1:LFE FL FC FR BL BR\"", L"specifies that the clip has 6 audio channels of the type 5.1 and that they are in the specified order regardless of what ffmpeg says.");
 479         sink.para()->text(L"The FFmpeg producer also supports changing some of the settings via CALL:");
 480         sink.example(L">> CALL 1-10 LOOP 1");
 481         sink.example(L">> CALL 1-10 START 10");
 482         sink.example(L">> CALL 1-10 LENGTH 50");
 483 }
 484
 485 spl::shared_ptr<core::frame_producer> create_producer(
 486                 const core::frame_producer_dependencies& dependencies,
 487                 const std::vector<std::wstring>& params,
 488                 const spl::shared_ptr<core::media_info_repository>& info_repo)
 489 {
 490         auto filename = probe_stem(env::media_folder() + L"/" + params.at(0), false);
 491
 492         if(filename.empty())
 493                 return core::frame_producer::empty();
 494
 495         bool loop                       = contains_param(L"LOOP", params);
 496         auto start                      = get_param(L"START", params, get_param(L"SEEK", params, static_cast<uint32_t>(0)));
 497         auto length                     = get_param(L"LENGTH", params, std::numeric_limits<uint32_t>::max());
 498         auto filter_str         = get_param(L"FILTER", params, L"");
 499         auto channel_layout     = get_param(L"CHANNEL_LAYOUT", params, L"");
 500         bool thumbnail_mode     = false;
 501         auto info                       = info_repo->get(filename);
 502
 503         return create_destroy_proxy(spl::make_shared_ptr(std::make_shared<ffmpeg_producer>(
 504                         dependencies.frame_factory,
 505                         dependencies.format_desc,
 506                         channel_layout,
 507                         filename,
 508                         filter_str,
 509                         loop,
 510                         start,
 511                         length,
 512                         thumbnail_mode,
 513                         info)));
 514 }
 515
 516 core::draw_frame create_thumbnail_frame(
 517         const core::frame_producer_dependencies& dependencies,
 518         const std::wstring& media_file,
 519         const spl::shared_ptr<core::media_info_repository>& info_repo)
 520 {
 521         auto quiet_logging = temporary_enable_quiet_logging_for_thread(true);
 522         auto filename = probe_stem(env::media_folder() + L"/" + media_file, true);
 523
 524         if (filename.empty())
 525                 return core::draw_frame::empty();
 526
 527         auto render_specific_frame = [&](std::int64_t frame_num)
 528         {
 529                 spl::shared_ptr<core::frame_producer> producer = spl::make_shared<ffmpeg_producer>(
 530                                 dependencies.frame_factory,
 531                                 dependencies.format_desc,
 532                                 L"",
 533                                 filename,
 534                                 L"",
 535                                 false,
 536                                 static_cast<uint32_t>(frame_num),
 537                                 std::numeric_limits<uint32_t>::max(),
 538                                 true,
 539                                 info_repo->get(filename));
 540                 return producer->receive();
 541         };
 542
 543         auto info = info_repo->get(filename);
 544
 545         if (!info)
 546                 return core::draw_frame::empty();
 547
 548         auto total_frames = info->duration;
 549         auto grid = env::properties().get(L"configuration.thumbnails.video-grid", 2);
 550
 551         if (grid < 1)
 552         {
 553                 CASPAR_LOG(error) << L"configuration/thumbnails/video-grid cannot be less than 1";
 554                 BOOST_THROW_EXCEPTION(caspar_exception() << msg_info("configuration/thumbnails/video-grid cannot be less than 1"));
 555         }
 556
 557         if (grid == 1)
 558         {
 559                 return render_specific_frame(total_frames / 2);
 560         }
 561
 562         auto num_snapshots = grid * grid;
 563
 564         std::vector<core::draw_frame> frames;
 565
 566         for (int i = 0; i < num_snapshots; ++i)
 567         {
 568                 int x = i % grid;
 569                 int y = i / grid;
 570                 std::int64_t desired_frame;
 571
 572                 if (i == 0)
 573                         desired_frame = 0; // first
 574                 else if (i == num_snapshots - 1)
 575                         desired_frame = total_frames - 30; // last
 576                 else
 577                         // evenly distributed across the file.
 578                         desired_frame = total_frames * i / (num_snapshots - 1);
 579
 580                 auto frame = render_specific_frame(desired_frame);
 581                 frame.transform().image_transform.fill_scale[0] = 1.0 / static_cast<double>(grid);
 582                 frame.transform().image_transform.fill_scale[1] = 1.0 / static_cast<double>(grid);
 583                 frame.transform().image_transform.fill_translation[0] = 1.0 / static_cast<double>(grid) * x;
 584                 frame.transform().image_transform.fill_translation[1] = 1.0 / static_cast<double>(grid) * y;
 585
 586                 frames.push_back(frame);
 587         }
 588
 589         return core::draw_frame(frames);
 590 }
 591
 592 }}