git.sesse.net Git - casparcg/blob - modules/ffmpeg/producer/frame_muxer.cpp

   1 #include "../StdAfx.h"\r
   2 \r
   3 #include "frame_muxer.h"\r
   4 \r
   5 #include "filter/filter.h"\r
   6 \r
   7 #include "util.h"\r
   8 \r
   9 #include <core/producer/frame_producer.h>\r
  10 #include <core/producer/frame/basic_frame.h>\r
  11 #include <core/producer/frame/frame_transform.h>\r
  12 #include <core/producer/frame/pixel_format.h>\r
  13 #include <core/producer/frame/frame_factory.h>\r
  14 #include <core/mixer/write_frame.h>\r
  15 \r
  16 #include <common/env.h>\r
  17 #include <common/exception/exceptions.h>\r
  18 #include <common/log/log.h>\r
  19 \r
  20 #if defined(_MSC_VER)\r
  21 #pragma warning (push)\r
  22 #pragma warning (disable : 4244)\r
  23 #endif\r
  24 extern "C" \r
  25 {\r
  26         #define __STDC_CONSTANT_MACROS\r
  27         #define __STDC_LIMIT_MACROS\r
  28         #include <libavcodec/avcodec.h>\r
  29         #include <libavformat/avformat.h>\r
  30 }\r
  31 #if defined(_MSC_VER)\r
  32 #pragma warning (pop)\r
  33 #endif\r
  34 \r
  35 #include <boost/foreach.hpp>\r
  36 #include <boost/range/algorithm_ext/push_back.hpp>\r
  37 \r
  38 #include <agents.h>\r
  39 #include <ppl.h>\r
  40 \r
  41 #include <deque>\r
  42 #include <queue>\r
  43 #include <vector>\r
  44 \r
  45 using namespace caspar::core;\r
  46 \r
  47 namespace caspar { namespace ffmpeg {\r
  48 \r
  49 struct display_mode\r
  50 {\r
  51         enum type\r
  52         {\r
  53                 simple,\r
  54                 duplicate,\r
  55                 half,\r
  56                 interlace,\r
  57                 deinterlace_bob,\r
  58                 deinterlace_bob_reinterlace,\r
  59                 deinterlace,\r
  60                 count,\r
  61                 invalid\r
  62         };\r
  63 \r
  64         static std::wstring print(display_mode::type value)\r
  65         {\r
  66                 switch(value)\r
  67                 {\r
  68                         case simple:\r
  69                                 return L"simple";\r
  70                         case duplicate:\r
  71                                 return L"duplicate";\r
  72                         case half:\r
  73                                 return L"half";\r
  74                         case interlace:\r
  75                                 return L"interlace";\r
  76                         case deinterlace_bob:\r
  77                                 return L"deinterlace_bob";\r
  78                         case deinterlace_bob_reinterlace:\r
  79                                 return L"deinterlace_bob_reinterlace";\r
  80                         case deinterlace:\r
  81                                 return L"deinterlace";\r
  82                         default:\r
  83                                 return L"invalid";\r
  84                 }\r
  85         }\r
  86 };\r
  87 \r
  88 display_mode::type get_display_mode(const core::field_mode::type in_mode, double in_fps, const core::field_mode::type out_mode, double out_fps)\r
  89 {               \r
  90         static const auto epsilon = 2.0;\r
  91 \r
  92         if(in_fps < 20.0 || in_fps > 80.0)\r
  93         {\r
  94                 //if(out_mode != core::field_mode::progressive && in_mode == core::field_mode::progressive)\r
  95                 //      return display_mode::interlace;\r
  96                 \r
  97                 if(out_mode == core::field_mode::progressive && in_mode != core::field_mode::progressive)\r
  98                 {\r
  99                         if(in_fps < 35.0)\r
 100                                 return display_mode::deinterlace;\r
 101                         else\r
 102                                 return display_mode::deinterlace_bob;\r
 103                 }\r
 104         }\r
 105 \r
 106         if(std::abs(in_fps - out_fps) < epsilon)\r
 107         {\r
 108                 if(in_mode != core::field_mode::progressive && out_mode == core::field_mode::progressive)\r
 109                         return display_mode::deinterlace;\r
 110                 //else if(in_mode == core::field_mode::progressive && out_mode != core::field_mode::progressive)\r
 111                 //      simple(); // interlace_duplicate();\r
 112                 else\r
 113                         return display_mode::simple;\r
 114         }\r
 115         else if(std::abs(in_fps/2.0 - out_fps) < epsilon)\r
 116         {\r
 117                 if(in_mode != core::field_mode::progressive)\r
 118                         return display_mode::invalid;\r
 119 \r
 120                 if(out_mode != core::field_mode::progressive)\r
 121                         return display_mode::interlace;\r
 122                 else\r
 123                         return display_mode::half;\r
 124         }\r
 125         else if(std::abs(in_fps - out_fps/2.0) < epsilon)\r
 126         {\r
 127                 if(out_mode != core::field_mode::progressive)\r
 128                         return display_mode::invalid;\r
 129 \r
 130                 if(in_mode != core::field_mode::progressive)\r
 131                         return display_mode::deinterlace_bob;\r
 132                 else\r
 133                         return display_mode::duplicate;\r
 134         }\r
 135 \r
 136         return display_mode::invalid;\r
 137 }\r
 138 \r
 139 struct frame_muxer::implementation : boost::noncopyable\r
 140 {       \r
 141         std::deque<std::queue<safe_ptr<write_frame>>>   video_streams_;\r
 142         std::deque<core::audio_buffer>                                  audio_streams_;\r
 143         std::deque<safe_ptr<basic_frame>>                               frame_buffer_;\r
 144         display_mode::type                                                              display_mode_;\r
 145         const double                                                                    in_fps_;\r
 146         const video_format_desc                                                 format_desc_;\r
 147         bool                                                                                    auto_transcode_;\r
 148 \r
 149         size_t                                                                                  audio_sample_count_;\r
 150         size_t                                                                                  video_frame_count_;\r
 151                 \r
 152         size_t                                                                                  processed_audio_sample_count_;\r
 153         size_t                                                                                  processed_video_frame_count_;\r
 154 \r
 155         filter                                                                                  filter_;\r
 156         safe_ptr<core::frame_factory>                                   frame_factory_;\r
 157                 \r
 158         implementation(double in_fps, const safe_ptr<core::frame_factory>& frame_factory)\r
 159                 : video_streams_(1)\r
 160                 , audio_streams_(1)\r
 161                 , display_mode_(display_mode::invalid)\r
 162                 , in_fps_(in_fps)\r
 163                 , format_desc_(frame_factory->get_video_format_desc())\r
 164                 , auto_transcode_(env::properties().get("configuration.producers.auto-transcode", false))\r
 165                 , audio_sample_count_(0)\r
 166                 , video_frame_count_(0)\r
 167                 , frame_factory_(frame_factory)\r
 168         {\r
 169         }\r
 170 \r
 171         void push(const std::shared_ptr<AVFrame>& video_frame, int hints)\r
 172         {               \r
 173                 if(!video_frame)\r
 174                 {       \r
 175                         CASPAR_LOG(debug) << L"video-frame-count: " << static_cast<float>(video_frame_count_);\r
 176                         video_frame_count_ = 0;\r
 177                         video_streams_.push_back(std::queue<safe_ptr<write_frame>>());\r
 178                         return;\r
 179                 }\r
 180 \r
 181                 if(video_frame->data[0] == nullptr)\r
 182                 {\r
 183                         video_streams_.back().push(make_safe<core::write_frame>(this));\r
 184                         ++video_frame_count_;\r
 185                         display_mode_ = display_mode::simple;\r
 186                         return;\r
 187                 }\r
 188 \r
 189                 if(display_mode_ == display_mode::invalid)\r
 190                 {\r
 191                         if(auto_transcode_)\r
 192                         {\r
 193                                 auto in_mode = get_mode(*video_frame);\r
 194                                 display_mode_ = get_display_mode(in_mode, in_fps_, format_desc_.field_mode, format_desc_.fps);\r
 195                         \r
 196                                 if(display_mode_ == display_mode::simple && in_mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && video_frame->height != static_cast<int>(format_desc_.height))\r
 197                                         display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace \r
 198                                 \r
 199                                 if(display_mode_ == display_mode::deinterlace)\r
 200                                         filter_ = filter(L"YADIF=0:-1");\r
 201                                 else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace)\r
 202                                         filter_ = filter(L"YADIF=1:-1");\r
 203                         }\r
 204                         else\r
 205                                 display_mode_ = display_mode::simple;\r
 206 \r
 207                         if(display_mode_ == display_mode::invalid)\r
 208                         {\r
 209                                 CASPAR_LOG(warning) << L"[frame_muxer] Failed to detect display-mode.";\r
 210                                 display_mode_ = display_mode::simple;\r
 211                         }\r
 212 \r
 213                         CASPAR_LOG(info) << "[frame_muxer] " << display_mode::print(display_mode_);\r
 214                 }\r
 215 \r
 216                                 \r
 217                 if(hints & core::frame_producer::ALPHA_HINT)\r
 218                         video_frame->format = make_alpha_format(video_frame->format);\r
 219                 \r
 220                 auto format = video_frame->format;\r
 221                 if(video_frame->format == CASPAR_PIX_FMT_LUMA) // CASPAR_PIX_FMT_LUMA is not valid for filter, change it to GRAY8\r
 222                         video_frame->format = PIX_FMT_GRAY8;\r
 223 \r
 224                 BOOST_FOREACH(auto& av_frame, filter_.execute(video_frame))\r
 225                 {\r
 226                         av_frame->format = format;\r
 227 \r
 228                         auto frame = make_write_frame(this, av_frame, frame_factory_, hints);\r
 229 \r
 230                         // Fix field-order if needed\r
 231                         if(frame->get_type() == core::field_mode::lower && format_desc_.field_mode == core::field_mode::upper)\r
 232                                 frame->get_frame_transform().fill_translation[1] += 1.0/static_cast<double>(format_desc_.height);\r
 233                         else if(frame->get_type() == core::field_mode::upper && format_desc_.field_mode == core::field_mode::lower)\r
 234                                 frame->get_frame_transform().fill_translation[1] -= 1.0/static_cast<double>(format_desc_.height);\r
 235 \r
 236                         video_streams_.back().push(frame);\r
 237                         ++video_frame_count_;\r
 238                 }\r
 239 \r
 240                 if(video_streams_.back().size() > 8)\r
 241                         BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));\r
 242         }\r
 243 \r
 244         void push(const std::shared_ptr<core::audio_buffer>& audio_samples)\r
 245         {\r
 246                 if(!audio_samples)      \r
 247                 {\r
 248                         CASPAR_LOG(debug) << L"audio-chunk-count: " << audio_sample_count_/format_desc_.audio_samples_per_frame;\r
 249                         audio_streams_.push_back(core::audio_buffer());\r
 250                         audio_sample_count_ = 0;\r
 251                         return;\r
 252                 }\r
 253 \r
 254                 audio_sample_count_ += audio_samples->size();\r
 255 \r
 256                 boost::range::push_back(audio_streams_.back(), *audio_samples);\r
 257 \r
 258                 if(audio_streams_.back().size() > 8*format_desc_.audio_samples_per_frame)\r
 259                         BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));\r
 260         }\r
 261 \r
 262         safe_ptr<basic_frame> pop()\r
 263         {               \r
 264                 auto frame = frame_buffer_.front();\r
 265                 frame_buffer_.pop_front();              \r
 266                 return frame;\r
 267         }\r
 268 \r
 269         size_t size() const\r
 270         {\r
 271                 return frame_buffer_.size();\r
 272         }\r
 273 \r
 274         safe_ptr<core::write_frame> pop_video()\r
 275         {\r
 276                 auto frame = video_streams_.front().front();\r
 277                 video_streams_.front().pop();\r
 278                 \r
 279                 return frame;\r
 280         }\r
 281 \r
 282         core::audio_buffer pop_audio()\r
 283         {\r
 284                 CASPAR_VERIFY(audio_streams_.front().size() >= format_desc_.audio_samples_per_frame);\r
 285 \r
 286                 auto begin = audio_streams_.front().begin();\r
 287                 auto end   = begin + format_desc_.audio_samples_per_frame;\r
 288 \r
 289                 auto samples = core::audio_buffer(begin, end);\r
 290                 audio_streams_.front().erase(begin, end);\r
 291 \r
 292                 return samples;\r
 293         }\r
 294         \r
 295         bool video_ready() const\r
 296         {               \r
 297                 return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2());\r
 298         }\r
 299         \r
 300         bool audio_ready() const\r
 301         {\r
 302                 return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2());\r
 303         }\r
 304 \r
 305         bool video_ready2() const\r
 306         {               \r
 307                 switch(display_mode_)\r
 308                 {\r
 309                 case display_mode::deinterlace_bob_reinterlace:                                 \r
 310                 case display_mode::interlace:                                   \r
 311                         return video_streams_.front().size() >= 2;\r
 312                 default:                                                                                \r
 313                         return !video_streams_.front().empty();\r
 314                 }\r
 315         }\r
 316         \r
 317         bool audio_ready2() const\r
 318         {\r
 319                 switch(display_mode_)\r
 320                 {\r
 321                 case display_mode::duplicate:                                   \r
 322                         return audio_streams_.front().size()/2 >= format_desc_.audio_samples_per_frame;\r
 323                 default:                                                                                \r
 324                         return audio_streams_.front().size() >= format_desc_.audio_samples_per_frame;\r
 325                 }\r
 326         }\r
 327                 \r
 328         void commit()\r
 329         {\r
 330                 if(video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2()))\r
 331                 {\r
 332                         if(!video_streams_.front().empty() || !audio_streams_.front().empty())\r
 333                                 CASPAR_LOG(debug) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples.";\r
 334 \r
 335                         video_streams_.pop_front();\r
 336                         audio_streams_.pop_front();\r
 337                 }\r
 338 \r
 339                 if(!video_ready2() || !audio_ready2())\r
 340                         return;\r
 341                 \r
 342                 switch(display_mode_)\r
 343                 {\r
 344                 case display_mode::simple:                                              return simple(frame_buffer_);\r
 345                 case display_mode::duplicate:                                   return duplicate(frame_buffer_);\r
 346                 case display_mode::half:                                                return half(frame_buffer_);\r
 347                 case display_mode::interlace:                                   return interlace(frame_buffer_);\r
 348                 case display_mode::deinterlace_bob:                             return simple(frame_buffer_);\r
 349                 case display_mode::deinterlace_bob_reinterlace: return interlace(frame_buffer_);\r
 350                 case display_mode::deinterlace:                                 return simple(frame_buffer_);\r
 351                 default:                                                                                BOOST_THROW_EXCEPTION(invalid_operation() << msg_info("invalid display-mode"));\r
 352                 }\r
 353         }\r
 354         \r
 355         void simple(std::deque<safe_ptr<basic_frame>>& dest)\r
 356         {               \r
 357                 auto frame1 = pop_video();\r
 358                 frame1->audio_data() = pop_audio();\r
 359 \r
 360                 dest.push_back(frame1);         \r
 361         }\r
 362 \r
 363         void duplicate(std::deque<safe_ptr<basic_frame>>& dest)\r
 364         {               \r
 365                 auto frame = pop_video();\r
 366 \r
 367                 auto frame1 = make_safe<core::write_frame>(*frame); // make a copy\r
 368                 frame1->audio_data() = pop_audio();\r
 369 \r
 370                 auto frame2 = frame;\r
 371                 frame2->audio_data() = pop_audio();\r
 372 \r
 373                 dest.push_back(frame1);\r
 374                 dest.push_back(frame2);\r
 375         }\r
 376 \r
 377         void half(std::deque<safe_ptr<basic_frame>>& dest)\r
 378         {                                                       \r
 379                 auto frame1 = pop_video();\r
 380                 frame1->audio_data() = pop_audio();\r
 381                                 \r
 382                 video_streams_.front().pop(); // Throw away\r
 383 \r
 384                 dest.push_back(frame1);\r
 385         }\r
 386         \r
 387         void interlace(std::deque<safe_ptr<basic_frame>>& dest)\r
 388         {                               \r
 389                 auto frame1 = pop_video();\r
 390                 frame1->audio_data() = pop_audio();\r
 391                                 \r
 392                 auto frame2 = pop_video();\r
 393 \r
 394                 dest.push_back(core::basic_frame::interlace(frame1, frame2, format_desc_.field_mode));          \r
 395         }\r
 396         \r
 397         int64_t calc_nb_frames(int64_t nb_frames) const\r
 398         {\r
 399                 switch(display_mode_)\r
 400                 {\r
 401                 case display_mode::interlace:\r
 402                 case display_mode::half:\r
 403                         return nb_frames/2;\r
 404                 case display_mode::duplicate:\r
 405                 case display_mode::deinterlace_bob:\r
 406                         return nb_frames*2;\r
 407                 default:\r
 408                         return nb_frames;\r
 409                 }\r
 410         }\r
 411 };\r
 412 \r
 413 frame_muxer::frame_muxer(double in_fps, const safe_ptr<core::frame_factory>& frame_factory)\r
 414         : impl_(new implementation(in_fps, frame_factory)){}\r
 415 void frame_muxer::push(const std::shared_ptr<AVFrame>& video_frame, int hints){impl_->push(video_frame, hints);}\r
 416 void frame_muxer::push(const std::shared_ptr<core::audio_buffer>& audio_samples){return impl_->push(audio_samples);}\r
 417 void frame_muxer::commit(){impl_->commit();}\r
 418 safe_ptr<basic_frame> frame_muxer::pop(){return impl_->pop();}\r
 419 size_t frame_muxer::size() const {return impl_->size();}\r
 420 bool frame_muxer::empty() const {return impl_->size() == 0;}\r
 421 bool frame_muxer::video_ready() const{return impl_->video_ready();}\r
 422 bool frame_muxer::audio_ready() const{return impl_->audio_ready();}\r
 423 int64_t frame_muxer::calc_nb_frames(int64_t nb_frames) const {return impl_->calc_nb_frames(nb_frames);}\r
 424 \r
 425 \r
 426 struct frame_muxer2::implementation : public Concurrency::agent, boost::noncopyable\r
 427 {       \r
 428         frame_muxer2::token_t&                  active_token_;\r
 429         frame_muxer2::video_source_t&   video_source_;\r
 430         frame_muxer2::audio_source_t&   audio_source_;\r
 431         frame_muxer2::target_t&                 target_;\r
 432 \r
 433         std::deque<std::queue<safe_ptr<write_frame>>>   video_streams_;\r
 434         std::deque<core::audio_buffer>                                  audio_streams_;\r
 435         std::deque<safe_ptr<basic_frame>>                               frame_buffer_;\r
 436         display_mode::type                                                              display_mode_;\r
 437         const double                                                                    in_fps_;\r
 438         const video_format_desc                                                 format_desc_;\r
 439         bool                                                                                    auto_transcode_;\r
 440 \r
 441         size_t                                                                                  audio_sample_count_;\r
 442         size_t                                                                                  video_frame_count_;\r
 443                 \r
 444         size_t                                                                                  processed_audio_sample_count_;\r
 445         size_t                                                                                  processed_video_frame_count_;\r
 446 \r
 447         filter                                                                                  filter_;\r
 448         safe_ptr<core::frame_factory>                                   frame_factory_;\r
 449                                 \r
 450         implementation(frame_muxer2::token_t& active_token,\r
 451                                    frame_muxer2::video_source_t& video_source,\r
 452                                    frame_muxer2::audio_source_t& audio_source,\r
 453                                    frame_muxer2::target_t& target,\r
 454                                    double in_fps, \r
 455                                    const safe_ptr<core::frame_factory>& frame_factory)\r
 456                 : active_token_(active_token)\r
 457                 , video_source_(video_source)\r
 458                 , audio_source_(audio_source)\r
 459                 , target_(target)\r
 460                 , video_streams_(1)\r
 461                 , audio_streams_(1)\r
 462                 , display_mode_(display_mode::invalid)\r
 463                 , in_fps_(in_fps)\r
 464                 , format_desc_(frame_factory->get_video_format_desc())\r
 465                 , auto_transcode_(env::properties().get("configuration.producers.auto-transcode", false))\r
 466                 , audio_sample_count_(0)\r
 467                 , video_frame_count_(0)\r
 468                 , frame_factory_(make_safe<core::concrt_frame_factory>(frame_factory))\r
 469         {\r
 470                 start();\r
 471         }\r
 472 \r
 473         ~implementation()\r
 474         {\r
 475                 agent::wait(this);\r
 476         }\r
 477 \r
 478         virtual void run()\r
 479         {\r
 480                 try\r
 481                 {\r
 482                         while(Concurrency::receive(active_token_))\r
 483                         {\r
 484                                 Concurrency::parallel_invoke(\r
 485                                 [&]\r
 486                                 {\r
 487                                         while(!video_ready())\r
 488                                         {\r
 489                                                 auto video = Concurrency::receive(video_source_);\r
 490                                                 if(video == eof_video())\r
 491                                                         break;\r
 492                                                 push(video, 0); \r
 493                                         }\r
 494                                 },\r
 495                                 [&]\r
 496                                 {\r
 497                                         while(!audio_ready())\r
 498                                         {\r
 499                                                 auto audio = Concurrency::receive(audio_source_);\r
 500                                                 if(audio == eof_audio())\r
 501                                                         break;\r
 502                                                 push(audio);    \r
 503                                         }                                       \r
 504                                 });\r
 505 \r
 506                                 if(!video_ready() || !audio_ready())\r
 507                                 {\r
 508                                         Concurrency::send(target_, std::shared_ptr<core::basic_frame>(core::basic_frame::eof()));\r
 509                                         break;\r
 510                                 }\r
 511 \r
 512                                 commit();\r
 513                         \r
 514                                 if(!frame_buffer_.empty())\r
 515                                 {\r
 516                                         Concurrency::send(target_, std::shared_ptr<core::basic_frame>(frame_buffer_.front()));\r
 517                                         frame_buffer_.pop_front();      \r
 518                                 }\r
 519                         }\r
 520                 }\r
 521                 catch(...)\r
 522                 {\r
 523                         CASPAR_LOG_CURRENT_EXCEPTION();\r
 524                 }\r
 525 \r
 526                 std::shared_ptr<AVFrame> video;\r
 527                 Concurrency::try_receive(video_source_, video);\r
 528                 std::shared_ptr<core::audio_buffer> audio;\r
 529                 Concurrency::try_receive(audio_source_, audio);\r
 530                                         \r
 531                 done();\r
 532         }\r
 533 \r
 534         void push(const std::shared_ptr<AVFrame>& video_frame, int hints)\r
 535         {               \r
 536                 if(video_frame == loop_video())\r
 537                 {       \r
 538                         CASPAR_LOG(debug) << L"video-frame-count: " << static_cast<float>(video_frame_count_);\r
 539                         video_frame_count_ = 0;\r
 540                         video_streams_.push_back(std::queue<safe_ptr<write_frame>>());\r
 541                         return;\r
 542                 }\r
 543 \r
 544                 if(video_frame == empty_video())\r
 545                 {\r
 546                         video_streams_.back().push(make_safe<core::write_frame>(this));\r
 547                         ++video_frame_count_;\r
 548                         display_mode_ = display_mode::simple;\r
 549                         return;\r
 550                 }\r
 551 \r
 552                 if(display_mode_ == display_mode::invalid)\r
 553                 {\r
 554                         if(auto_transcode_)\r
 555                         {\r
 556                                 auto in_mode = get_mode(*video_frame);\r
 557                                 display_mode_ = get_display_mode(in_mode, in_fps_, format_desc_.field_mode, format_desc_.fps);\r
 558                         \r
 559                                 if(display_mode_ == display_mode::simple && in_mode != core::field_mode::progressive && format_desc_.field_mode != core::field_mode::progressive && video_frame->height != static_cast<int>(format_desc_.height))\r
 560                                         display_mode_ = display_mode::deinterlace_bob_reinterlace; // The frame will most likely be scaled, we need to deinterlace->reinterlace \r
 561                                 \r
 562                                 if(display_mode_ == display_mode::deinterlace)\r
 563                                         filter_ = filter(L"YADIF=0:-1");\r
 564                                 else if(display_mode_ == display_mode::deinterlace_bob || display_mode_ == display_mode::deinterlace_bob_reinterlace)\r
 565                                         filter_ = filter(L"YADIF=1:-1");\r
 566                         }\r
 567                         else\r
 568                                 display_mode_ = display_mode::simple;\r
 569 \r
 570                         if(display_mode_ == display_mode::invalid)\r
 571                         {\r
 572                                 CASPAR_LOG(warning) << L"[frame_muxer] Failed to detect display-mode.";\r
 573                                 display_mode_ = display_mode::simple;\r
 574                         }\r
 575 \r
 576                         CASPAR_LOG(info) << "[frame_muxer] " << display_mode::print(display_mode_);\r
 577                 }\r
 578 \r
 579                                 \r
 580                 if(hints & core::frame_producer::ALPHA_HINT)\r
 581                         video_frame->format = make_alpha_format(video_frame->format);\r
 582                 \r
 583                 auto format = video_frame->format;\r
 584                 if(video_frame->format == CASPAR_PIX_FMT_LUMA) // CASPAR_PIX_FMT_LUMA is not valid for filter, change it to GRAY8\r
 585                         video_frame->format = PIX_FMT_GRAY8;\r
 586 \r
 587                 BOOST_FOREACH(auto& av_frame, filter_.execute(video_frame))\r
 588                 {\r
 589                         av_frame->format = format;\r
 590 \r
 591                         auto frame = make_write_frame(this, av_frame, frame_factory_, hints);\r
 592 \r
 593                         // Fix field-order if needed\r
 594                         if(frame->get_type() == core::field_mode::lower && format_desc_.field_mode == core::field_mode::upper)\r
 595                                 frame->get_frame_transform().fill_translation[1] += 1.0/static_cast<double>(format_desc_.height);\r
 596                         else if(frame->get_type() == core::field_mode::upper && format_desc_.field_mode == core::field_mode::lower)\r
 597                                 frame->get_frame_transform().fill_translation[1] -= 1.0/static_cast<double>(format_desc_.height);\r
 598 \r
 599                         video_streams_.back().push(frame);\r
 600                         ++video_frame_count_;\r
 601                 }\r
 602 \r
 603                 if(video_streams_.back().size() > 8)\r
 604                         BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("video-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));\r
 605         }\r
 606 \r
 607         void push(std::shared_ptr<core::audio_buffer> audio_samples)\r
 608         {\r
 609                 if(audio_samples == loop_audio())       \r
 610                 {\r
 611                         CASPAR_LOG(debug) << L"audio-chunk-count: " << audio_sample_count_/format_desc_.audio_samples_per_frame;\r
 612                         audio_streams_.push_back(core::audio_buffer());\r
 613                         audio_sample_count_ = 0;\r
 614                         return;\r
 615                 }\r
 616 \r
 617                 if(audio_samples == empty_audio())              \r
 618                         audio_samples = std::make_shared<core::audio_buffer>(format_desc_.audio_samples_per_frame);             \r
 619 \r
 620                 audio_sample_count_ += audio_samples->size();\r
 621 \r
 622                 boost::range::push_back(audio_streams_.back(), *audio_samples);\r
 623 \r
 624                 if(audio_streams_.back().size() > 8*format_desc_.audio_samples_per_frame)\r
 625                         BOOST_THROW_EXCEPTION(invalid_operation() << source_info("frame_muxer") << msg_info("audio-stream overflow. This can be caused by incorrect frame-rate. Check clip meta-data."));\r
 626         }\r
 627         \r
 628         size_t size() const\r
 629         {\r
 630                 return frame_buffer_.size();\r
 631         }\r
 632 \r
 633         safe_ptr<core::write_frame> pop_video()\r
 634         {\r
 635                 auto frame = video_streams_.front().front();\r
 636                 video_streams_.front().pop();\r
 637                 \r
 638                 return frame;\r
 639         }\r
 640 \r
 641         core::audio_buffer pop_audio()\r
 642         {\r
 643                 CASPAR_VERIFY(audio_streams_.front().size() >= format_desc_.audio_samples_per_frame);\r
 644 \r
 645                 auto begin = audio_streams_.front().begin();\r
 646                 auto end   = begin + format_desc_.audio_samples_per_frame;\r
 647 \r
 648                 auto samples = core::audio_buffer(begin, end);\r
 649                 audio_streams_.front().erase(begin, end);\r
 650 \r
 651                 return samples;\r
 652         }\r
 653         \r
 654         bool video_ready() const\r
 655         {               \r
 656                 return video_streams_.size() > 1 || (video_streams_.size() >= audio_streams_.size() && video_ready2());\r
 657         }\r
 658         \r
 659         bool audio_ready() const\r
 660         {\r
 661                 return audio_streams_.size() > 1 || (audio_streams_.size() >= video_streams_.size() && audio_ready2());\r
 662         }\r
 663 \r
 664         bool video_ready2() const\r
 665         {               \r
 666                 switch(display_mode_)\r
 667                 {\r
 668                 case display_mode::deinterlace_bob_reinterlace:                                 \r
 669                 case display_mode::interlace:                                   \r
 670                         return video_streams_.front().size() >= 2;\r
 671                 default:                                                                                \r
 672                         return !video_streams_.front().empty();\r
 673                 }\r
 674         }\r
 675         \r
 676         bool audio_ready2() const\r
 677         {\r
 678                 switch(display_mode_)\r
 679                 {\r
 680                 case display_mode::duplicate:                                   \r
 681                         return audio_streams_.front().size()/2 >= format_desc_.audio_samples_per_frame;\r
 682                 default:                                                                                \r
 683                         return audio_streams_.front().size() >= format_desc_.audio_samples_per_frame;\r
 684                 }\r
 685         }\r
 686                 \r
 687         void commit()\r
 688         {\r
 689                 if(video_streams_.size() > 1 && audio_streams_.size() > 1 && (!video_ready2() || !audio_ready2()))\r
 690                 {\r
 691                         if(!video_streams_.front().empty() || !audio_streams_.front().empty())\r
 692                                 CASPAR_LOG(debug) << "Truncating: " << video_streams_.front().size() << L" video-frames, " << audio_streams_.front().size() << L" audio-samples.";\r
 693 \r
 694                         video_streams_.pop_front();\r
 695                         audio_streams_.pop_front();\r
 696                 }\r
 697 \r
 698                 if(!video_ready2() || !audio_ready2())\r
 699                         return;\r
 700                 \r
 701                 switch(display_mode_)\r
 702                 {\r
 703                 case display_mode::simple:                                              return simple(frame_buffer_);\r
 704                 case display_mode::duplicate:                                   return duplicate(frame_buffer_);\r
 705                 case display_mode::half:                                                return half(frame_buffer_);\r
 706                 case display_mode::interlace:                                   return interlace(frame_buffer_);\r
 707                 case display_mode::deinterlace_bob:                             return simple(frame_buffer_);\r
 708                 case display_mode::deinterlace_bob_reinterlace: return interlace(frame_buffer_);\r
 709                 case display_mode::deinterlace:                                 return simple(frame_buffer_);\r
 710                 default:                                                                                BOOST_THROW_EXCEPTION(invalid_operation() << msg_info("invalid display-mode"));\r
 711                 }\r
 712         }\r
 713         \r
 714         void simple(std::deque<safe_ptr<basic_frame>>& dest)\r
 715         {               \r
 716                 auto frame1 = pop_video();\r
 717                 frame1->audio_data() = pop_audio();\r
 718 \r
 719                 dest.push_back(frame1);         \r
 720         }\r
 721 \r
 722         void duplicate(std::deque<safe_ptr<basic_frame>>& dest)\r
 723         {               \r
 724                 auto frame = pop_video();\r
 725 \r
 726                 auto frame1 = make_safe<core::write_frame>(*frame); // make a copy\r
 727                 frame1->audio_data() = pop_audio();\r
 728 \r
 729                 auto frame2 = frame;\r
 730                 frame2->audio_data() = pop_audio();\r
 731 \r
 732                 dest.push_back(frame1);\r
 733                 dest.push_back(frame2);\r
 734         }\r
 735 \r
 736         void half(std::deque<safe_ptr<basic_frame>>& dest)\r
 737         {                                                       \r
 738                 auto frame1 = pop_video();\r
 739                 frame1->audio_data() = pop_audio();\r
 740                                 \r
 741                 video_streams_.front().pop(); // Throw away\r
 742 \r
 743                 dest.push_back(frame1);\r
 744         }\r
 745         \r
 746         void interlace(std::deque<safe_ptr<basic_frame>>& dest)\r
 747         {                               \r
 748                 auto frame1 = pop_video();\r
 749                 frame1->audio_data() = pop_audio();\r
 750                                 \r
 751                 auto frame2 = pop_video();\r
 752 \r
 753                 dest.push_back(core::basic_frame::interlace(frame1, frame2, format_desc_.field_mode));          \r
 754         }\r
 755         \r
 756         int64_t calc_nb_frames(int64_t nb_frames) const\r
 757         {\r
 758                 switch(display_mode_)\r
 759                 {\r
 760                 case display_mode::interlace:\r
 761                 case display_mode::half:\r
 762                         return nb_frames/2;\r
 763                 case display_mode::duplicate:\r
 764                 case display_mode::deinterlace_bob:\r
 765                         return nb_frames*2;\r
 766                 default:\r
 767                         return nb_frames;\r
 768                 }\r
 769         }\r
 770 };\r
 771 \r
 772 frame_muxer2::frame_muxer2(token_t& active_token,\r
 773                                                    video_source_t& video_source, \r
 774                                                    audio_source_t& audio_source,\r
 775                                                    target_t& target,\r
 776                                                    double in_fps, \r
 777                                                    const safe_ptr<core::frame_factory>& frame_factory)\r
 778         : impl_(new implementation(active_token, video_source, audio_source, target, in_fps, frame_factory))\r
 779 {\r
 780 }\r
 781 int64_t frame_muxer2::calc_nb_frames(int64_t nb_frames) const\r
 782 {\r
 783         return impl_->calc_nb_frames(nb_frames);\r
 784 }\r
 785 \r
 786 }}