git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/frame_muxer.cpp

   1 #include "../StdAfx.h"\r
   2 \r
   3 #include "frame_muxer.h"\r
   4 \r
   5 #include "filter/filter.h"\r
   6 \r
   7 #include "util.h"\r
   8 \r
   9 #include <core/producer/frame_producer.h>\r
  10 #include <core/producer/frame/basic_frame.h>\r
  11 #include <core/producer/frame/frame_transform.h>\r
  12 #include <core/producer/frame/pixel_format.h>\r
  13 #include <core/producer/frame/frame_factory.h>\r
  14 #include <core/mixer/write_frame.h>\r
  15 \r
  16 #include <common/env.h>\r
  17 #include <common/exception/exceptions.h>\r
  18 #include <common/log/log.h>\r
  19 \r
  20 #if defined(_MSC_VER)\r
  21 #pragma warning (push)\r
  22 #pragma warning (disable : 4244)\r
  23 #endif\r
  24 extern "C" \r
  25 {\r
  26         #define __STDC_CONSTANT_MACROS\r
  27         #define __STDC_LIMIT_MACROS\r
  28         #include <libavcodec/avcodec.h>\r
  29         #include <libavformat/avformat.h>\r
  30 }\r
  31 #if defined(_MSC_VER)\r
  32 #pragma warning (pop)\r
  33 #endif\r
  34 \r
  35 #include <boost/foreach.hpp>\r
  36 #include <boost/range/algorithm_ext/push_back.hpp>\r
  37 #include <boost/assign.hpp>\r
  38 \r
  39 #include <agents_extras.h>\r
  40 \r
  41 #include <deque>\r
  42 #include <tuple>\r
  43 #include <queue>\r
  44 #include <vector>\r
  45 \r
  46 using namespace caspar::core;\r
  47 \r
  48 using namespace Concurrency;\r
  49 \r
  50 namespace caspar { namespace ffmpeg {\r
  51 \r
  52 struct display_mode\r
  53 {\r
  54         enum type\r
  55         {\r
  56                 simple,\r
  57                 duplicate,\r
  58                 half,\r
  59                 interlace,\r
  60                 deinterlace_bob,\r
  61                 deinterlace_bob_reinterlace,\r
  62                 deinterlace,\r
  63                 count,\r
  64                 invalid\r
  65         };\r
  66 \r
  67         static std::wstring print(display_mode::type value)\r
  68         {\r
  69                 switch(value)\r
  70                 {\r
  71                         case simple:                                            return L"simple";\r
  72                         case duplicate:                                         return L"duplicate";\r
  73                         case half:                                                      return L"half";\r
  74                         case interlace:                                         return L"interlace";\r
  75                         case deinterlace_bob:                           return L"deinterlace_bob";\r
  76                         case deinterlace_bob_reinterlace:       return L"deinterlace_bob_reinterlace";\r
  77                         case deinterlace:                                       return L"deinterlace";\r
  78                         default:                                                        return L"invalid";\r
  79                 }\r
  80         }\r
  81 };\r
  82 \r
  83 display_mode::type get_display_mode(const core::field_mode::type in_mode, double in_fps, const core::field_mode::type out_mode, double out_fps)\r
  84 {               \r
  85         static const auto epsilon = 2.0;\r
  86 \r
  87         if(in_fps < 20.0 || in_fps > 80.0)\r
  88         {\r
  89                 //if(out_mode != core::field_mode::progressive && in_mode == core::field_mode::progressive)\r
  90                 //      return display_mode::interlace;\r
  91                 \r
  92                 if(out_mode == core::field_mode::progressive && in_mode != core::field_mode::progressive)\r
  93                 {\r
  94                         if(in_fps < 35.0)\r
  95                                 return display_mode::deinterlace;\r
  96                         else\r
  97                                 return display_mode::deinterlace_bob;\r
  98                 }\r
  99         }\r
 100 \r
 101         if(std::abs(in_fps - out_fps) < epsilon)\r
 102         {\r
 103                 if(in_mode != core::field_mode::progressive && out_mode == core::field_mode::progressive)\r
 104                         return display_mode::deinterlace;\r
 105                 //else if(in_mode == core::field_mode::progressive && out_mode != core::field_mode::progressive)\r
 106                 //      simple(); // interlace_duplicate();\r
 107                 else\r
 108                         return display_mode::simple;\r
 109         }\r
 110         else if(std::abs(in_fps/2.0 - out_fps) < epsilon)\r
 111         {\r
 112                 if(in_mode != core::field_mode::progressive)\r
 113                         return display_mode::invalid;\r
 114 \r
 115                 if(out_mode != core::field_mode::progressive)\r
 116                         return display_mode::interlace;\r
 117                 else\r
 118                         return display_mode::half;\r
 119         }\r
 120         else if(std::abs(in_fps - out_fps/2.0) < epsilon)\r
 121         {\r
 122                 if(out_mode != core::field_mode::progressive)\r
 123                         return display_mode::invalid;\r
 124 \r
 125                 if(in_mode != core::field_mode::progressive)\r
 126                         return display_mode::deinterlace_bob;\r
 127                 else\r
 128                         return display_mode::duplicate;\r
 129         }\r
 130 \r
 131         return display_mode::invalid;\r
 132 }\r
 133 \r
 134 \r
 135 struct frame_muxer2::implementation : public Concurrency::agent, boost::noncopyable\r
 136 {               \r
 137         ITarget<frame_message_t>&                                                                                                               target_;\r
 138         display_mode::type                                                                                                                              display_mode_;\r
 139         const double                                                                                                                                    in_fps_;\r
 140         const video_format_desc                                                                                                                 format_desc_;\r
 141         bool                                                                                                                                                    auto_transcode_;\r
 142         \r
 143         filter                                                                                                                                                  filter_;\r
 144         const safe_ptr<core::frame_factory>                                                                                             frame_factory_;\r
 145         \r
 146         call<video_message_t>                                                                                                                   push_video_;\r
 147         call<audio_message_t>                                                                                                                   push_audio_;\r
 148         \r
 149         transformer<video_message_t, std::shared_ptr<message<std::shared_ptr<write_frame>>>>    video_;\r
 150         unbounded_buffer<audio_message_t>                                                                                               audio_;\r
 151         \r
 152         core::audio_buffer                                                                                                                              audio_data_;\r
 153 \r
 154         Concurrency::overwrite_buffer<bool>                                                                                             is_running_;\r
 155                                                         \r
 156         implementation(frame_muxer2::video_source_t* video_source,\r
 157                                    frame_muxer2::audio_source_t* audio_source,\r
 158                                    frame_muxer2::target_t& target,\r
 159                                    double in_fps, \r
 160                                    const safe_ptr<core::frame_factory>& frame_factory)\r
 161                 : target_(target)\r
 162                 , display_mode_(display_mode::invalid)\r
 163                 , in_fps_(in_fps)\r
 164                 , format_desc_(frame_factory->get_video_format_desc())\r
 165                 , auto_transcode_(env::properties().get("configuration.producers.auto-transcode", false))\r
 166                 , frame_factory_(make_safe<core::concrt_frame_factory>(frame_factory))\r
 167                 , push_video_(std::bind(&implementation::push_video, this, std::placeholders::_1))\r
 168                 , push_audio_(std::bind(&implementation::push_audio, this, std::placeholders::_1))\r
 169                 , video_(std::bind(&implementation::make_write_frame, this, std::placeholders::_1))\r
 170         {\r
 171                 if(video_source)\r
 172                         video_source->link_target(&push_video_);\r
 173                 if(audio_source)\r
 174                         audio_source->link_target(&push_audio_);\r
 175                 \r
 176                 start();\r
 177         }\r
 178 \r
 179         ~implementation()\r
 180         {\r
 181                 send(is_running_, false);\r
 182                 agent::wait(this);\r
 183         }\r
 184 \r
 185         std::shared_ptr<message<std::shared_ptr<core::write_frame>>> make_write_frame(const video_message_t& message)\r
 186         {\r
 187                 if(message->payload == eof_video())\r
 188                         return make_message<std::shared_ptr<core::write_frame>>(nullptr);\r
 189 \r
 190                 if(message->payload == empty_video())\r
 191                         return make_message<std::shared_ptr<core::write_frame>>(std::make_shared<core::write_frame>(this));\r
 192 \r
 193                 return make_message<std::shared_ptr<core::write_frame>>(ffmpeg::make_write_frame(this, make_safe_ptr(message->payload), frame_factory_, 0), message->token);\r
 194         }       \r
 195 \r
 196         virtual void run()\r
 197         {\r
 198                 try\r
 199                 {\r
 200                         send(is_running_, true);\r
 201                         while(is_running_.value())\r
 202                         {\r
 203                                 auto video = receive(video_);\r
 204                                 auto audio = receive(audio_);                                                           \r
 205 \r
 206                                 if(!audio->payload)\r
 207                                 {\r
 208                                         send(is_running_ , false);\r
 209                                         break;\r
 210                                 }\r
 211 \r
 212                                 if(!video->payload)\r
 213                                 {\r
 214                                         send(is_running_ , false);\r
 215                                         break;\r
 216                                 }\r
 217 \r
 218                                 video->payload->audio_data() = std::move(*audio->payload);\r
 219 \r
 220                                 switch(display_mode_)\r
 221                                 {\r
 222                                 case display_mode::simple:                      \r
 223                                 case display_mode::deinterlace:\r
 224                                 case display_mode::deinterlace_bob:\r
 225                                         {\r
 226                                                 auto message = make_message(safe_ptr<core::basic_frame>(video->payload), video->token ? video->token : audio->token);\r
 227                                                 send(target_, message);\r
 228 \r
 229                                                 break;\r
 230                                         }\r
 231                                 case display_mode::duplicate:                                   \r
 232                                         {                                                                               \r
 233                                                 auto message = make_message(safe_ptr<core::basic_frame>(video->payload), video->token ? video->token : audio->token);\r
 234                                                 send(target_, message);\r
 235                                                 send(target_, message);\r
 236 \r
 237                                                 break;\r
 238                                         }\r
 239                                 case display_mode::half:                                                \r
 240                                         {                                       \r
 241                                                 receive(video_);\r
 242 \r
 243                                                 auto message = make_message(safe_ptr<core::basic_frame>(video->payload), video->token ? video->token : audio->token);\r
 244                                                 send(target_, message);\r
 245 \r
 246                                                 break;\r
 247                                         }\r
 248                                 case display_mode::deinterlace_bob_reinterlace:\r
 249                                 case display_mode::interlace:                                   \r
 250                                         {                                       \r
 251                                                 auto frame = safe_ptr<core::basic_frame>(video->payload);\r
 252                                                 auto video2 = receive(video_);  \r
 253                                                 if(video->payload)\r
 254                                                         frame = core::basic_frame::interlace(make_safe_ptr(video->payload), make_safe_ptr(video2->payload), format_desc_.field_mode);\r
 255                                                 else\r
 256                                                         send(is_running_, false);\r
 257 \r
 258                                                 auto message = make_message<safe_ptr<core::basic_frame>>(frame, video2->token ? video2->token : audio->token);\r
 259                                                 send(target_, message);\r
 260 \r
 261                                                 break;\r
 262                                         }\r
 263                                 default:        \r
 264                                         BOOST_THROW_EXCEPTION(invalid_operation() << msg_info("invalid display-mode"));\r
 265                                 }\r
 266                         }\r
 267                 }\r
 268                 catch(...)\r
 269                 {\r
 270                         CASPAR_LOG_CURRENT_EXCEPTION();\r
 271                 }\r
 272                 \r
 273                 send(is_running_ , false);\r
 274                 send(target_, make_message(core::basic_frame::eof()));\r
 275 \r
 276                 done();\r
 277         }\r
 278                         \r
 279         void push_video(const video_message_t& message)\r
 280         {               \r
 281                 auto video_frame = message->payload;\r
 282 \r
 283                 if(!video_frame)\r
 284                         return;\r
 285                 \r
 286                 if(video_frame == eof_video())\r
 287                 {\r
 288                         send(video_, make_message(eof_video()));\r
 289                         return;\r
 290                 }\r
 291 \r
 292                 if(video_frame == loop_video())         \r
 293                         return; \r
 294                                 \r
 295                 if(video_frame == empty_video())\r
 296                 {\r
 297                         send(video_, make_message(empty_video()));\r
 298                         return;\r
 299                 }\r
 300                 \r
 301                 if(display_mode_ == display_mode::invalid)\r
 302                 {\r
 303                         if(auto_transcode_)\r
 304                         {\r
 305                                 auto in_mode = get_mode(*video_frame);\r
 306                                 display_mode_ = get_display_mode(in_mode, in_fps_, format_desc_.field_mode, format_desc_.fps);\r
 307                         \r
 308                                 if(display_mode_ == display_mode::simple && in_mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && video_frame->height != static_cast<int>(format_desc_.height))\r
 309                                         display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace \r
 310                                 \r
 311                                 if(display_mode_ == display_mode::deinterlace)\r
 312                                         filter_ = filter(L"YADIF=0:-1");\r
 313                                 else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace)\r
 314                                         filter_ = filter(L"YADIF=1:-1");\r
 315                         }\r
 316                         else\r
 317                                 display_mode_ = display_mode::simple;\r
 318 \r
 319                         if(display_mode_ == display_mode::invalid)\r
 320                         {\r
 321                                 CASPAR_LOG(warning) << L"[frame_muxer] Failed to detect display-mode.";\r
 322                                 display_mode_ = display_mode::simple;\r
 323                         }\r
 324 \r
 325                         // copy <= We need to release frames\r
 326                         if(display_mode_ != display_mode::simple && filter_.filter_str().empty())\r
 327                                 filter_ = filter(L"copy"); \r
 328 \r
 329                         CASPAR_LOG(info) << "[frame_muxer] " << display_mode::print(display_mode_);\r
 330                 }\r
 331                                                 \r
 332                 //if(hints & core::frame_producer::ALPHA_HINT)\r
 333                 //      video_frame->format = make_alpha_format(video_frame->format);\r
 334                 \r
 335                 auto format = video_frame->format;\r
 336                 if(video_frame->format == CASPAR_PIX_FMT_LUMA) // CASPAR_PIX_FMT_LUMA is not valid for filter, change it to GRAY8\r
 337                         video_frame->format = PIX_FMT_GRAY8;\r
 338 \r
 339                 filter_.push(video_frame);\r
 340 \r
 341                 BOOST_FOREACH(auto av_frame, filter_.poll_all())\r
 342                 {               \r
 343                         av_frame->format = format;                      \r
 344                         send(video_, make_message(std::shared_ptr<AVFrame>(av_frame), message->token));\r
 345                 }\r
 346         }\r
 347 \r
 348         void push_audio(const audio_message_t& message)\r
 349         {\r
 350                 auto audio_samples = message->payload;\r
 351 \r
 352                 if(!audio_samples)\r
 353                         return;\r
 354 \r
 355                 if(audio_samples == eof_audio())\r
 356                 {\r
 357                         send(audio_, make_message(std::shared_ptr<core::audio_buffer>()));\r
 358                         return;\r
 359                 }\r
 360 \r
 361                 if(audio_samples == loop_audio())                       \r
 362                         return;         \r
 363 \r
 364                 if(audio_samples == empty_audio())              \r
 365                         send(audio_, make_message(std::make_shared<core::audio_buffer>(format_desc_.audio_samples_per_frame, 0)));              \r
 366 \r
 367                 audio_data_.insert(audio_data_.end(), audio_samples->begin(), audio_samples->end());\r
 368                 \r
 369                 while(audio_data_.size() >= format_desc_.audio_samples_per_frame)\r
 370                 {\r
 371                         auto begin = audio_data_.begin(); \r
 372                         auto end   = begin + format_desc_.audio_samples_per_frame;\r
 373                                         \r
 374                         send(audio_, make_message(std::make_shared<core::audio_buffer>(begin, end), message->token));\r
 375 \r
 376                         audio_data_.erase(begin, end);\r
 377                 }\r
 378         }\r
 379                                         \r
 380         int64_t calc_nb_frames(int64_t nb_frames) const\r
 381         {\r
 382                 switch(display_mode_)\r
 383                 {\r
 384                 case display_mode::interlace:\r
 385                 case display_mode::half:\r
 386                         return nb_frames/2;\r
 387                 case display_mode::duplicate:\r
 388                 case display_mode::deinterlace_bob:\r
 389                         return nb_frames*2;\r
 390                 default:\r
 391                         return nb_frames;\r
 392                 }\r
 393         }\r
 394 };\r
 395 \r
 396 frame_muxer2::frame_muxer2(video_source_t* video_source, \r
 397                                                    audio_source_t* audio_source,\r
 398                                                    target_t& target,\r
 399                                                    double in_fps, \r
 400                                                    const safe_ptr<core::frame_factory>& frame_factory)\r
 401         : impl_(new implementation(video_source, audio_source, target, in_fps, frame_factory))\r
 402 {\r
 403 }\r
 404 \r
 405 int64_t frame_muxer2::calc_nb_frames(int64_t nb_frames) const\r
 406 {\r
 407         return impl_->calc_nb_frames(nb_frames);\r
 408 }\r
 409 \r
 410 }}