]> git.sesse.net Git - casparcg/blob - modules/flash/producer/flash_producer.cpp
8cefe4d4cb633bef78e53d4813b1ea9874cddd21
[casparcg] / modules / flash / producer / flash_producer.cpp
1 /*
2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
3 *
4 * This file is part of CasparCG (www.casparcg.com).
5 *
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
18 *
19 * Author: Robert Nagy, ronag89@gmail.com
20 */
21
22 #include "../stdafx.h"
23
24 #if defined(_MSC_VER)
25 #pragma warning (disable : 4146)
26 #pragma warning (disable : 4244)
27 #endif
28
29 #include "flash_producer.h"
30 #include "FlashAxContainer.h"
31
32 #include "../util/swf.h"
33
34 #include <core/video_format.h>
35
36 #include <core/frame/frame.h>
37 #include <core/frame/draw_frame.h>
38 #include <core/frame/frame_factory.h>
39 #include <core/frame/pixel_format.h>
40 #include <core/frame/audio_channel_layout.h>
41 #include <core/producer/frame_producer.h>
42 #include <core/monitor/monitor.h>
43 #include <core/help/help_repository.h>
44 #include <core/help/help_sink.h>
45
46 #include <common/env.h>
47 #include <common/executor.h>
48 #include <common/lock.h>
49 #include <common/diagnostics/graph.h>
50 #include <common/prec_timer.h>
51 #include <common/array.h>
52 #include <common/memset.h>
53 #include <common/memcpy.h>
54
55 #include <boost/filesystem.hpp>
56 #include <boost/property_tree/ptree.hpp>
57 #include <boost/thread.hpp>
58 #include <boost/timer.hpp>
59 #include <boost/algorithm/string.hpp>
60
61 #include <tbb/spin_mutex.h>
62
63 #include <functional>
64
65 namespace caspar { namespace flash {
66                 
67 class bitmap
68 {
69 public:
70         bitmap(int width, int height)
71                 : bmp_data_(nullptr)
72                 , hdc_(CreateCompatibleDC(0), DeleteDC)
73         {       
74                 BITMAPINFO info;
75                 memset(&info, 0, sizeof(BITMAPINFO));
76                 info.bmiHeader.biBitCount = 32;
77                 info.bmiHeader.biCompression = BI_RGB;
78                 info.bmiHeader.biHeight = static_cast<LONG>(-height);
79                 info.bmiHeader.biPlanes = 1;
80                 info.bmiHeader.biSize = sizeof(BITMAPINFO);
81                 info.bmiHeader.biWidth = static_cast<LONG>(width);
82
83                 bmp_.reset(CreateDIBSection(static_cast<HDC>(hdc_.get()), &info, DIB_RGB_COLORS, reinterpret_cast<void**>(&bmp_data_), 0, 0), DeleteObject);
84                 SelectObject(static_cast<HDC>(hdc_.get()), bmp_.get()); 
85
86                 if(!bmp_data_)
87                         CASPAR_THROW_EXCEPTION(bad_alloc());
88         }
89
90         operator HDC() {return static_cast<HDC>(hdc_.get());}
91
92         BYTE* data() { return bmp_data_;}
93         const BYTE* data() const { return bmp_data_;}
94
95 private:
96         BYTE* bmp_data_;        
97         std::shared_ptr<void> hdc_;
98         std::shared_ptr<void> bmp_;
99 };
100
101 struct template_host
102 {
103         std::wstring  video_mode;
104         std::wstring  filename;
105         int                       width;
106         int                       height;
107 };
108
109 template_host get_template_host(const core::video_format_desc& desc)
110 {
111         try
112         {
113                 std::vector<template_host> template_hosts;
114                 auto template_hosts_element = env::properties().get_child_optional(
115                         L"configuration.template-hosts");
116
117                 if (template_hosts_element)
118                 {
119                         for (auto& xml_mapping : *template_hosts_element)
120                         {
121                                 try
122                                 {
123                                         template_host template_host;
124                                         template_host.video_mode                = xml_mapping.second.get(L"video-mode", L"");
125                                         template_host.filename                  = xml_mapping.second.get(L"filename",   L"cg.fth");
126                                         template_host.width                             = xml_mapping.second.get(L"width",              desc.width);
127                                         template_host.height                    = xml_mapping.second.get(L"height",             desc.height);
128                                         template_hosts.push_back(template_host);
129                                 }
130                                 catch(...){}
131                         }
132                 }
133
134                 auto template_host_it = boost::find_if(template_hosts, [&](template_host template_host){return template_host.video_mode == desc.name;});
135                 if(template_host_it == template_hosts.end())
136                         template_host_it = boost::find_if(template_hosts, [&](template_host template_host){return template_host.video_mode == L"";});
137
138                 if(template_host_it != template_hosts.end())
139                         return *template_host_it;
140         }
141         catch(...){}
142                 
143         template_host template_host;
144         template_host.filename = L"cg.fth";
145
146         for(auto it = boost::filesystem::directory_iterator(env::template_folder()); it != boost::filesystem::directory_iterator(); ++it)
147         {
148                 if(boost::iequals(it->path().extension().wstring(), L"." + desc.name))
149                 {
150                         template_host.filename = it->path().filename().wstring();
151                         break;
152                 }
153         }
154
155         template_host.width =  desc.square_width;
156         template_host.height = desc.square_height;
157         return template_host;
158 }
159
160 boost::mutex& get_global_init_destruct_mutex()
161 {
162         static boost::mutex m;
163
164         return m;
165 }
166
167 class flash_renderer
168 {       
169         struct com_init
170         {
171                 HRESULT result_ = CoInitialize(nullptr);
172
173                 com_init()
174                 {
175                         if(FAILED(result_))
176                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info("Failed to initialize com-context for flash-player"));
177                 }
178
179                 ~com_init()
180                 {
181                         if(SUCCEEDED(result_))
182                                 ::CoUninitialize();
183                 }
184         } com_init_;
185
186         core::monitor::subject&                                                 monitor_subject_;
187
188         const std::wstring                                                              filename_;
189         const int                                                                               width_;
190         const int                                                                               height_;
191
192         const std::shared_ptr<core::frame_factory>              frame_factory_;
193         
194         CComObject<caspar::flash::FlashAxContainer>*    ax_                                     = nullptr;
195         core::draw_frame                                                                head_                           = core::draw_frame::late();
196         bitmap                                                                                  bmp_                            { width_, height_ };
197         prec_timer                                                                              timer_;
198         caspar::timer                                                                   tick_timer_;
199         
200         spl::shared_ptr<diagnostics::graph>                             graph_;
201         
202 public:
203         flash_renderer(core::monitor::subject& monitor_subject, const spl::shared_ptr<diagnostics::graph>& graph, const std::shared_ptr<core::frame_factory>& frame_factory, const std::wstring& filename, int width, int height) 
204                 : monitor_subject_(monitor_subject)
205                 , graph_(graph)
206                 , filename_(filename)
207                 , width_(width)
208                 , height_(height)
209                 , frame_factory_(frame_factory)
210                 , bmp_(width, height)
211         {               
212                 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
213                 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
214                 graph_->set_color("param", diagnostics::color(1.0f, 0.5f, 0.0f));
215                 
216                 if(FAILED(CComObject<caspar::flash::FlashAxContainer>::CreateInstance(&ax_)))
217                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to create FlashAxContainer"));
218                 
219                 if(FAILED(ax_->CreateAxControl()))
220                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to Create FlashAxControl"));
221
222                 ax_->set_print([this]{return print(); });
223
224                 CComPtr<IShockwaveFlash> spFlash;
225                 if(FAILED(ax_->QueryControl(&spFlash)))
226                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to Query FlashAxControl"));
227                                                                                                 
228                 if(FAILED(spFlash->put_Playing(true)) )
229                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to start playing Flash"));
230
231                 // Concurrent initialization of two template hosts causes a
232                 // SecurityException later when CG ADD is performed. Initialization is
233                 // therefore serialized via a global mutex.
234                 lock(get_global_init_destruct_mutex(), [&]
235                 {
236                         if (FAILED(spFlash->put_Movie(CComBSTR(filename.c_str()))))
237                                 CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to Load Template Host"));
238                 });
239                                                                                 
240                 if(FAILED(spFlash->put_ScaleMode(2)))  //Exact fit. Scale without respect to the aspect ratio.
241                         CASPAR_THROW_EXCEPTION(caspar_exception() << msg_info(print() + L" Failed to Set Scale Mode"));
242                                                 
243                 ax_->SetSize(width_, height_);
244                 render_frame(0.0);
245
246                 CASPAR_LOG(info) << print() << L" Initialized.";
247         }
248
249         ~flash_renderer()
250         {               
251                 if(ax_)
252                 {
253                         ax_->DestroyAxControl();
254                         ax_->Release();
255                 }
256                 graph_->set_value("tick-time", 0.0f);
257                 graph_->set_value("frame-time", 0.0f);
258                 CASPAR_LOG(info) << print() << L" Uninitialized.";
259         }
260         
261         std::wstring call(const std::wstring& param)
262         {               
263                 std::wstring result;
264
265                 CASPAR_LOG(debug) << print() << " Call: " << param;
266
267                 if(!ax_->FlashCall(param, result))
268                         CASPAR_LOG(warning) << print() << L" Flash call failed:" << param;//CASPAR_THROW_EXCEPTION(invalid_operation() << msg_info("Flash function call failed.") << arg_name_info("param") << arg_value_info(narrow(param)));
269
270                 if (boost::starts_with(result, L"<exception>"))
271                         CASPAR_LOG(warning) << print() << L" Flash call failed:" << result;
272
273                 graph_->set_tag(diagnostics::tag_severity::INFO, "param");
274
275                 return result;
276         }
277         
278         core::draw_frame render_frame(double sync)
279         {                       
280                 const float frame_time = 1.0f/fps();
281
282                 if (!ax_->IsReadyToRender())
283                         return head_;
284
285                 if (is_empty())
286                         return core::draw_frame::empty();
287
288                 if (sync > 0.00001)
289                         timer_.tick(frame_time*sync); // This will block the thread.
290
291                 graph_->set_value("tick-time", static_cast<float>(tick_timer_.elapsed() / frame_time) * 0.5f);
292                 tick_timer_.restart();
293                 boost::timer frame_timer;
294                 ax_->Tick();
295
296                 if (ax_->InvalidRect())
297                 {                       
298                         core::pixel_format_desc desc = core::pixel_format::bgra;
299                         desc.planes.push_back(core::pixel_format_desc::plane(width_, height_, 4));
300                         auto frame = frame_factory_->create_frame(this, desc, core::audio_channel_layout::invalid());
301
302                         fast_memset(bmp_.data(), 0, width_ * height_ * 4);
303                         ax_->DrawControl(bmp_);
304                 
305                         fast_memcpy(frame.image_data(0).begin(), bmp_.data(), width_*height_*4);
306                         head_ = core::draw_frame(std::move(frame));     
307                 }               
308
309                 MSG msg;
310                 while (PeekMessage(&msg, NULL, NULL, NULL, PM_REMOVE)) // DO NOT REMOVE THE MESSAGE DISPATCH LOOP. Without this some stuff doesn't work!  
311                 {
312                         if (msg.message == WM_TIMER && msg.wParam == 3 && msg.lParam == 0) // We tick this inside FlashAxContainer
313                                 continue;
314
315                         TranslateMessage(&msg);
316                         DispatchMessage(&msg);
317                 }
318
319                 graph_->set_value("frame-time", static_cast<float>(frame_timer.elapsed()/frame_time)*0.5f);
320                 monitor_subject_ << core::monitor::message("/renderer/profiler/time") % frame_timer.elapsed() % frame_time;
321                 return head_;
322         }
323         
324         bool is_empty() const
325         {
326                 return ax_->IsEmpty();
327         }
328
329         double fps() const
330         {
331                 return ax_->GetFPS();   
332         }
333         
334         std::wstring print()
335         {
336                 return L"flash-player[" + boost::filesystem::path(filename_).filename().wstring()
337                                   + L"|" + boost::lexical_cast<std::wstring>(width_)
338                                   + L"x" + boost::lexical_cast<std::wstring>(height_)
339                                   + L"]";               
340         }
341 };
342
343 struct flash_producer : public core::frame_producer_base
344 {       
345         core::monitor::subject                                                  monitor_subject_;
346         const std::wstring                                                              filename_;      
347         const spl::shared_ptr<core::frame_factory>              frame_factory_;
348         const core::video_format_desc                                   format_desc_;
349         const int                                                                               width_;
350         const int                                                                               height_;
351         core::constraints                                                               constraints_            { static_cast<double>(width_), static_cast<double>(height_) };
352         const int                                                                               buffer_size_            = env::properties().get(L"configuration.flash.buffer-depth", format_desc_.fps > 30.0 ? 4 : 2);
353
354         tbb::atomic<int>                                                                fps_;
355
356         spl::shared_ptr<diagnostics::graph>                             graph_;
357
358         std::queue<core::draw_frame>                                    frame_buffer_;
359         tbb::concurrent_bounded_queue<core::draw_frame> output_buffer_;
360
361         core::draw_frame                                                                last_frame_                     = core::draw_frame::empty();
362                                 
363         std::unique_ptr<flash_renderer>                                 renderer_;
364         tbb::atomic<bool>                                                               has_renderer_;
365
366         executor                                                                                executor_                       = L"flash_producer";
367 public:
368         flash_producer(const spl::shared_ptr<core::frame_factory>& frame_factory, const core::video_format_desc& format_desc, const std::wstring& filename, int width, int height) 
369                 : filename_(filename)           
370                 , frame_factory_(frame_factory)
371                 , format_desc_(format_desc)
372                 , width_(width > 0 ? width : format_desc.width)
373                 , height_(height > 0 ? height : format_desc.height)
374         {       
375                 fps_ = 0;
376          
377                 graph_->set_color("buffered", diagnostics::color(1.0f, 1.0f, 0.0f));
378                 graph_->set_color("late-frame", diagnostics::color(0.6f, 0.3f, 0.9f));
379                 graph_->set_text(print());
380                 diagnostics::register_graph(graph_);
381
382                 has_renderer_ = false;
383
384                 CASPAR_LOG(info) << print() << L" Initialized";
385         }
386
387         ~flash_producer()
388         {
389                 executor_.invoke([this]
390                 {
391                         renderer_.reset();
392                 }, task_priority::high_priority);
393         }
394
395         // frame_producer
396         
397         void log_buffered()
398         {
399                 double buffered = output_buffer_.size();
400                 auto ratio = buffered / buffer_size_;
401                 graph_->set_value("buffered", ratio);
402         }
403
404         core::draw_frame receive_impl() override
405         {                                       
406                 auto frame = last_frame_;
407
408                 double buffered = output_buffer_.size();
409                 auto ratio = buffered / buffer_size_;
410                 graph_->set_value("buffered", ratio);
411                 
412                 if (output_buffer_.try_pop(frame))
413                         last_frame_ = frame;
414                 else            
415                         graph_->set_tag(diagnostics::tag_severity::WARNING, "late-frame");
416
417                 fill_buffer();
418                                 
419                 monitor_subject_ << core::monitor::message("/host/path")        % filename_
420                                                 << core::monitor::message("/host/width")        % width_
421                                                 << core::monitor::message("/host/height")       % height_
422                                                 << core::monitor::message("/host/fps")          % fps_
423                                                 << core::monitor::message("/buffer")            % output_buffer_.size() % buffer_size_;
424
425                 return frame;
426         }
427
428         core::constraints& pixel_constraints() override
429         {
430                 return constraints_;
431         }
432                 
433         std::future<std::wstring> call(const std::vector<std::wstring>& params) override
434         {
435                 auto param = boost::algorithm::join(params, L" ");
436
437                 if (param == L"?")
438                         return make_ready_future(std::wstring(has_renderer_ ? L"1" : L"0"));
439
440                 return executor_.begin_invoke([this, param]() -> std::wstring
441                 {                       
442                         try
443                         {
444                                 bool initialize_renderer = !renderer_;
445
446                                 if (initialize_renderer)
447                                 {
448                                         renderer_.reset(new flash_renderer(monitor_subject_, graph_, frame_factory_, filename_, width_, height_));
449
450                                         has_renderer_ = true;
451                                 }
452
453                                 std::wstring result = param == L"start_rendering"
454                                                 ? L"" : renderer_->call(param);
455
456                                 if (initialize_renderer)
457                                 {
458                                         do_fill_buffer(true);
459                                 }
460
461                                 return result;  
462                         }
463                         catch(...)
464                         {
465                                 CASPAR_LOG_CURRENT_EXCEPTION();
466                                 renderer_.reset(nullptr);
467                                 has_renderer_ = false;
468                         }
469
470                         return L"";
471                 }, task_priority::high_priority);
472         }
473                 
474         std::wstring print() const override
475         { 
476                 return L"flash[" + boost::filesystem::path(filename_).wstring() + L"|" + boost::lexical_cast<std::wstring>(fps_) + L"]";                
477         }       
478
479         std::wstring name() const override
480         {
481                 return L"flash";
482         }
483
484         boost::property_tree::wptree info() const override
485         {
486                 boost::property_tree::wptree info;
487                 info.add(L"type", L"flash");
488                 return info;
489         }
490
491         core::monitor::subject& monitor_output()
492         {
493                 return monitor_subject_;
494         }
495
496         // flash_producer
497         
498         void fill_buffer()
499         {
500                 if (executor_.size() > 0)
501                         return;
502
503                 executor_.begin_invoke([this]
504                 {
505                         do_fill_buffer(false);
506                 });
507         }
508
509         void do_fill_buffer(bool initial_buffer_fill)
510         {
511                 int nothing_rendered = 0;
512                 const int MAX_NOTHING_RENDERED_RETRIES = 4;
513
514                 auto to_render = buffer_size_ - output_buffer_.size();
515                 bool allow_faster_rendering = !initial_buffer_fill;
516                 int rendered = 0;
517
518                 while (rendered < to_render)
519                 {
520                         bool was_rendered = next(allow_faster_rendering);
521                         log_buffered();
522
523                         if (was_rendered)
524                         {
525                                 ++rendered;
526                         }
527                         else
528                         {
529                                 if (nothing_rendered++ < MAX_NOTHING_RENDERED_RETRIES)
530                                 {
531                                         // Flash player not ready with first frame, sleep to not busy-loop;
532                                         boost::this_thread::sleep(boost::posix_time::milliseconds(10));
533                                         boost::this_thread::yield();
534                                 }
535                                 else
536                                         return;
537                         }
538
539                         executor_.yield(task_priority::high_priority);
540                 }
541         }
542
543         bool next(bool allow_faster_rendering)
544         {
545                 if (!renderer_)
546                         frame_buffer_.push(core::draw_frame::empty());
547
548                 if (frame_buffer_.empty())
549                 {
550                         if (abs(renderer_->fps() / 2.0 - format_desc_.fps) < 2.0) // flash == 2 * format -> interlace
551                         {
552                                 auto frame1 = render_frame(allow_faster_rendering);
553
554                                 if (frame1 != core::draw_frame::late())
555                                 {
556                                         auto frame2 = render_frame(allow_faster_rendering);
557                                         frame_buffer_.push(core::draw_frame::interlace(frame1, frame2, format_desc_.field_mode));
558                                 }
559                         }
560                         else if (abs(renderer_->fps() - format_desc_.fps / 2.0) < 2.0) // format == 2 * flash -> duplicate
561                         {
562                                 auto frame = render_frame(allow_faster_rendering);
563
564                                 if (frame != core::draw_frame::late())
565                                 {
566                                         frame_buffer_.push(frame);
567                                         frame_buffer_.push(frame);
568                                 }
569                         }
570                         else //if(abs(renderer_->fps() - format_desc_.fps) < 0.1) // format == flash -> simple
571                         {
572                                 auto frame = render_frame(allow_faster_rendering);
573
574                                 if (frame != core::draw_frame::late())
575                                         frame_buffer_.push(frame);
576                         }
577
578                         fps_.fetch_and_store(static_cast<int>(renderer_->fps()*100.0));
579                         graph_->set_text(print());
580
581                         if (renderer_->is_empty())
582                         {
583                                 renderer_.reset();
584                                 has_renderer_ = false;
585                         }
586                 }
587
588                 if (frame_buffer_.empty())
589                 {
590                         return false;
591                 }
592                 else
593                 {
594                         output_buffer_.push(std::move(frame_buffer_.front()));
595                         frame_buffer_.pop();
596                         return true;
597                 }
598         }
599
600         core::draw_frame render_frame(bool allow_faster_rendering)
601         {
602                 double sync;
603
604                 if (allow_faster_rendering)
605                 {
606                         double ratio = std::min(
607                                         1.0,
608                                         static_cast<double>(output_buffer_.size())
609                                                         / static_cast<double>(std::max(1, buffer_size_ - 1)));
610                         sync = 2 * ratio - ratio * ratio;
611                 }
612                 else
613                 {
614                         sync = 1.0;
615                 }
616
617                 return renderer_->render_frame(sync);
618         }
619 };
620
621 spl::shared_ptr<core::frame_producer> create_producer(const core::frame_producer_dependencies& dependencies, const std::vector<std::wstring>& params)
622 {
623         auto template_host = get_template_host(dependencies.format_desc);
624         
625         auto filename = env::template_folder() + L"\\" + template_host.filename;
626         
627         if(!boost::filesystem::exists(filename))
628                 CASPAR_THROW_EXCEPTION(file_not_found() << boost::errinfo_file_name(u8(filename)));     
629
630         return create_destroy_proxy(spl::make_shared<flash_producer>(dependencies.frame_factory, dependencies.format_desc, filename, template_host.width, template_host.height));
631 }
632
633 void describe_swf_producer(core::help_sink& sink, const core::help_repository& repo)
634 {
635         sink.short_description(L"Plays flash files (.swf files).");
636         sink.syntax(L"[swf_file:string]");
637         sink.para()->text(L"Plays flash files (.swf files). The file should reside under the media folder.");
638         sink.para()->text(L"Examples:");
639         sink.example(L">> PLAY 1-10 folder/swf_file");
640 }
641
642 spl::shared_ptr<core::frame_producer> create_swf_producer(const core::frame_producer_dependencies& dependencies, const std::vector<std::wstring>& params)
643 {
644         auto filename = env::media_folder() + L"\\" + params.at(0) + L".swf";
645
646         if (!boost::filesystem::exists(filename))
647                 return core::frame_producer::empty();
648
649         swf_t::header_t header(filename);
650
651         auto producer = spl::make_shared<flash_producer>(
652                         dependencies.frame_factory, dependencies.format_desc, filename, header.frame_width, header.frame_height);
653
654         producer->call({ L"start_rendering" }).get();
655
656         return create_destroy_proxy(producer);
657 }
658
659 std::wstring find_template(const std::wstring& template_name)
660 {
661         if(boost::filesystem::exists(template_name + L".ft")) 
662                 return template_name + L".ft";
663         
664         if(boost::filesystem::exists(template_name + L".ct"))
665                 return template_name + L".ct";
666         
667         if(boost::filesystem::exists(template_name + L".swf"))
668                 return template_name + L".swf";
669
670         return L"";
671 }
672
673 }}