2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "screen_consumer.h"
25 #include <SFML/Window.hpp>
27 #include <common/diagnostics/graph.h>
28 #include <common/gl/gl_check.h>
29 #include <common/log.h>
30 #include <common/memory.h>
31 #include <common/array.h>
32 #include <common/memshfl.h>
33 #include <common/utf.h>
34 #include <common/prec_timer.h>
35 #include <common/future.h>
36 #include <common/timer.h>
37 #include <common/param.h>
38 #include <common/os/general_protection_fault.h>
39 #include <common/scope_exit.h>
41 //#include <windows.h>
43 #include <ffmpeg/producer/filter/filter.h>
45 #include <core/video_format.h>
46 #include <core/frame/frame.h>
47 #include <core/consumer/frame_consumer.h>
48 #include <core/interaction/interaction_sink.h>
49 #include <core/help/help_sink.h>
50 #include <core/help/help_repository.h>
52 #include <boost/circular_buffer.hpp>
53 #include <boost/lexical_cast.hpp>
54 #include <boost/property_tree/ptree.hpp>
55 #include <boost/thread.hpp>
56 #include <boost/algorithm/string.hpp>
58 #include <tbb/atomic.h>
59 #include <tbb/concurrent_queue.h>
60 #include <tbb/parallel_for.h>
68 #pragma warning (push)
69 #pragma warning (disable : 4244)
73 #define __STDC_CONSTANT_MACROS
74 #define __STDC_LIMIT_MACROS
75 #include <libavcodec/avcodec.h>
76 #include <libavutil/imgutils.h>
82 namespace caspar { namespace screen {
94 enum class aspect_ratio
101 std::wstring name = L"Screen consumer";
102 int screen_index = 0;
103 screen::stretch stretch = screen::stretch::fill;
104 bool windowed = true;
105 bool auto_deinterlace = true;
106 bool key_only = false;
107 aspect_ratio aspect = aspect_ratio::aspect_invalid;
109 bool interactive = true;
110 bool borderless = false;
113 struct screen_consumer : boost::noncopyable
115 const configuration config_;
116 core::video_format_desc format_desc_;
120 std::vector<GLuint> pbos_ = std::vector<GLuint> { 0, 0 };
126 int screen_width_ = format_desc_.width;
127 int screen_height_ = format_desc_.height;
128 int square_width_ = format_desc_.square_width;
129 int square_height_ = format_desc_.square_height;
132 tbb::atomic<bool> polling_event_;
135 spl::shared_ptr<diagnostics::graph> graph_;
136 caspar::timer perf_timer_;
137 caspar::timer tick_timer_;
139 caspar::prec_timer wait_timer_;
141 tbb::concurrent_bounded_queue<core::const_frame> frame_buffer_;
142 core::interaction_sink* sink_;
144 boost::thread thread_;
145 tbb::atomic<bool> is_running_;
146 tbb::atomic<int64_t> current_presentation_age_;
148 ffmpeg::filter filter_;
151 const configuration& config,
152 const core::video_format_desc& format_desc,
154 core::interaction_sink* sink)
156 , format_desc_(format_desc)
157 , channel_index_(channel_index)
160 , filter_([&]() -> ffmpeg::filter
162 const auto sample_aspect_ratio =
163 boost::rational<int>(
164 format_desc.square_width,
165 format_desc.square_height) /
166 boost::rational<int>(
170 return ffmpeg::filter(
173 boost::rational<int>(format_desc.duration, format_desc.time_scale),
174 boost::rational<int>(format_desc.time_scale, format_desc.duration),
178 format_desc.field_mode == core::field_mode::progressive || !config.auto_deinterlace ? "" : "format=pix_fmts=gbrp,YADIF=1:-1");
181 if (format_desc_.format == core::video_format::ntsc && config_.aspect == configuration::aspect_ratio::aspect_4_3)
183 // Use default values which are 4:3.
187 if (config_.aspect == configuration::aspect_ratio::aspect_16_9)
188 square_width_ = (format_desc.height*16)/9;
189 else if (config_.aspect == configuration::aspect_ratio::aspect_4_3)
190 square_width_ = (format_desc.height*4)/3;
193 frame_buffer_.set_capacity(1);
195 graph_->set_color("tick-time", diagnostics::color(0.0f, 0.6f, 0.9f));
196 graph_->set_color("frame-time", diagnostics::color(0.1f, 1.0f, 0.1f));
197 graph_->set_color("dropped-frame", diagnostics::color(0.3f, 0.6f, 0.3f));
198 graph_->set_text(print());
199 diagnostics::register_graph(graph_);
201 /*DISPLAY_DEVICE d_device = {sizeof(d_device), 0};
202 std::vector<DISPLAY_DEVICE> displayDevices;
203 for(int n = 0; EnumDisplayDevices(NULL, n, &d_device, NULL); ++n)
204 displayDevices.push_back(d_device);
206 if(config_.screen_index >= displayDevices.size())
207 CASPAR_LOG(warning) << print() << L" Invalid screen-index: " << config_.screen_index;
209 DEVMODE devmode = {};
210 if(!EnumDisplaySettings(displayDevices[config_.screen_index].DeviceName, ENUM_CURRENT_SETTINGS, &devmode))
211 CASPAR_LOG(warning) << print() << L" Could not find display settings for screen-index: " << config_.screen_index;
213 screen_x_ = devmode.dmPosition.x;
214 screen_y_ = devmode.dmPosition.y;
215 screen_width_ = config_.windowed ? square_width_ : devmode.dmPelsWidth;
216 screen_height_ = config_.windowed ? square_height_ : devmode.dmPelsHeight;*/
219 screen_width_ = square_width_;
220 screen_height_ = square_height_;
222 polling_event_ = false;
224 current_presentation_age_ = 0;
225 thread_ = boost::thread([this]{run();});
231 frame_buffer_.try_push(core::const_frame::empty());
237 auto window_style = config_.borderless
240 ? sf::Style::Resize | sf::Style::Close
241 : sf::Style::Fullscreen);
242 window_.create(sf::VideoMode(screen_width_, screen_height_, 32), u8(print()), window_style);
243 window_.setMouseCursorVisible(config_.interactive);
244 window_.setPosition(sf::Vector2i(screen_x_, screen_y_));
245 window_.setSize(sf::Vector2u(screen_width_, screen_height_));
248 if(!GLEW_VERSION_2_1 && glewInit() != GLEW_OK)
249 CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to initialize GLEW."));
251 if(!GLEW_VERSION_2_1)
252 CASPAR_THROW_EXCEPTION(not_supported() << msg_info("Missing OpenGL 2.1 support."));
254 GL(glEnable(GL_TEXTURE_2D));
255 GL(glDisable(GL_DEPTH_TEST));
256 GL(glClearColor(0.0, 0.0, 0.0, 0.0));
257 GL(glViewport(0, 0, format_desc_.width, format_desc_.height));
258 GL(glLoadIdentity());
262 GL(glGenTextures(1, &texture_));
263 GL(glBindTexture(GL_TEXTURE_2D, texture_));
264 GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR));
265 GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR));
266 GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP));
267 GL(glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP));
268 GL(glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, format_desc_.width, format_desc_.height, 0, GL_BGRA, GL_UNSIGNED_BYTE, 0));
269 GL(glBindTexture(GL_TEXTURE_2D, 0));
271 GL(glGenBuffers(2, pbos_.data()));
273 glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pbos_[0]);
274 glBufferDataARB(GL_PIXEL_UNPACK_BUFFER_ARB, format_desc_.size, 0, GL_STREAM_DRAW_ARB);
275 glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, pbos_[1]);
276 glBufferDataARB(GL_PIXEL_UNPACK_BUFFER_ARB, format_desc_.size, 0, GL_STREAM_DRAW_ARB);
277 glBindBufferARB(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
279 window_.setVerticalSyncEnabled(config_.vsync);
283 CASPAR_LOG(info) << print() << " Enabled vsync.";
285 /*auto wglSwapIntervalEXT = reinterpret_cast<void(APIENTRY*)(int)>(wglGetProcAddress("wglSwapIntervalEXT"));
286 if(wglSwapIntervalEXT)
290 wglSwapIntervalEXT(1);
291 CASPAR_LOG(info) << print() << " Enabled vsync.";
294 wglSwapIntervalEXT(0);
301 glDeleteTextures(1, &texture_);
303 for (auto& pbo : pbos_)
306 glDeleteBuffers(1, &pbo);
312 ensure_gpf_handler_installed_for_thread("screen-consumer-thread");
322 auto poll_event = [this](sf::Event& e)
324 polling_event_ = true;
327 polling_event_ = false;
329 return window_.pollEvent(e);
335 if (e.type == sf::Event::Resized)
337 else if (e.type == sf::Event::Closed)
339 else if (config_.interactive && sink_)
343 case sf::Event::MouseMoved:
345 auto& mouse_move = e.mouseMove;
346 sink_->on_interaction(spl::make_shared<core::mouse_move_event>(
348 static_cast<double>(mouse_move.x) / screen_width_,
349 static_cast<double>(mouse_move.y) / screen_height_));
352 case sf::Event::MouseButtonPressed:
353 case sf::Event::MouseButtonReleased:
355 auto& mouse_button = e.mouseButton;
356 sink_->on_interaction(spl::make_shared<core::mouse_button_event>(
358 static_cast<double>(mouse_button.x) / screen_width_,
359 static_cast<double>(mouse_button.y) / screen_height_,
360 static_cast<int>(mouse_button.button),
361 e.type == sf::Event::MouseButtonPressed));
364 case sf::Event::MouseWheelMoved:
366 auto& wheel_moved = e.mouseWheel;
367 sink_->on_interaction(spl::make_shared<core::mouse_wheel_event>(
369 static_cast<double>(wheel_moved.x) / screen_width_,
370 static_cast<double>(wheel_moved.y) / screen_height_,
378 core::const_frame frame;
379 frame_buffer_.pop(frame);
381 render_and_draw_frame(frame);
383 /*perf_timer_.restart();
385 graph_->set_value("frame-time", perf_timer_.elapsed()*format_desc_.fps*0.5);
389 current_presentation_age_ = frame.get_age_millis();
390 graph_->set_value("tick-time", tick_timer_.elapsed()*format_desc_.fps*0.5);
391 tick_timer_.restart();
395 CASPAR_LOG_CURRENT_EXCEPTION();
404 CASPAR_LOG_CURRENT_EXCEPTION();
408 void try_sleep_almost_until_vblank()
410 static const double THRESHOLD = 0.003;
411 double threshold = config_.vsync ? THRESHOLD : 0.0;
413 auto frame_time = 1.0 / (format_desc_.fps * format_desc_.field_count);
415 wait_timer_.tick(frame_time - threshold);
418 void wait_for_vblank_and_display()
420 try_sleep_almost_until_vblank();
422 // Make sure that the next tick measures the duration from this point in time.
423 wait_timer_.tick(0.0);
426 spl::shared_ptr<AVFrame> get_av_frame()
428 spl::shared_ptr<AVFrame> av_frame(av_frame_alloc(), [](AVFrame* p) { av_frame_free(&p); });
429 avcodec_get_frame_defaults(av_frame.get());
431 av_frame->linesize[0] = format_desc_.width*4;
432 av_frame->format = PIX_FMT_BGRA;
433 av_frame->width = format_desc_.width;
434 av_frame->height = format_desc_.height;
435 av_frame->interlaced_frame = format_desc_.field_mode != core::field_mode::progressive;
436 av_frame->top_field_first = format_desc_.field_mode == core::field_mode::upper ? 1 : 0;
437 av_frame->pts = pts_++;
442 void render_and_draw_frame(core::const_frame input_frame)
444 if(static_cast<size_t>(input_frame.image_data().size()) != format_desc_.size)
447 if(screen_width_ == 0 && screen_height_ == 0)
450 perf_timer_.restart();
451 auto av_frame = get_av_frame();
452 av_frame->data[0] = const_cast<uint8_t*>(input_frame.image_data().begin());
454 filter_.push(av_frame);
455 auto frame = filter_.poll();
460 if (!filter_.is_double_rate())
462 render(spl::make_shared_ptr(frame));
463 graph_->set_value("frame-time", perf_timer_.elapsed() * format_desc_.fps * 0.5);
465 wait_for_vblank_and_display(); // progressive frame
469 render(spl::make_shared_ptr(frame));
470 double perf_elapsed = perf_timer_.elapsed();
472 wait_for_vblank_and_display(); // field1
474 perf_timer_.restart();
475 frame = filter_.poll();
476 render(spl::make_shared_ptr(frame));
477 perf_elapsed += perf_timer_.elapsed();
478 graph_->set_value("frame-time", perf_elapsed * format_desc_.fps * 0.5);
480 wait_for_vblank_and_display(); // field2
484 void render(spl::shared_ptr<AVFrame> av_frame)
486 GL(glBindTexture(GL_TEXTURE_2D, texture_));
488 GL(glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pbos_[0]));
489 GL(glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, format_desc_.width, format_desc_.height, GL_BGRA, GL_UNSIGNED_BYTE, 0));
491 GL(glBindBuffer(GL_PIXEL_UNPACK_BUFFER, pbos_[1]));
492 GL(glBufferData(GL_PIXEL_UNPACK_BUFFER, format_desc_.size, 0, GL_STREAM_DRAW));
494 auto ptr = reinterpret_cast<char*>(GL2(glMapBuffer(GL_PIXEL_UNPACK_BUFFER, GL_WRITE_ONLY)));
499 tbb::parallel_for(tbb::blocked_range<int>(0, format_desc_.height), [&](const tbb::blocked_range<int>& r)
501 for(int n = r.begin(); n != r.end(); ++n)
502 aligned_memshfl(ptr+n*format_desc_.width*4, av_frame->data[0]+n*av_frame->linesize[0], format_desc_.width*4, 0x0F0F0F0F, 0x0B0B0B0B, 0x07070707, 0x03030303);
507 tbb::parallel_for(tbb::blocked_range<int>(0, format_desc_.height), [&](const tbb::blocked_range<int>& r)
509 for(int n = r.begin(); n != r.end(); ++n)
510 A_memcpy(ptr+n*format_desc_.width*4, av_frame->data[0]+n*av_frame->linesize[0], format_desc_.width*4);
514 GL(glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER)); // release the mapped buffer
517 GL(glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0));
519 GL(glClear(GL_COLOR_BUFFER_BIT));
521 glTexCoord2f(0.0f, 1.0f); glVertex2f(-width_, -height_);
522 glTexCoord2f(1.0f, 1.0f); glVertex2f( width_, -height_);
523 glTexCoord2f(1.0f, 0.0f); glVertex2f( width_, height_);
524 glTexCoord2f(0.0f, 0.0f); glVertex2f(-width_, height_);
527 GL(glBindTexture(GL_TEXTURE_2D, 0));
529 std::rotate(pbos_.begin(), pbos_.begin() + 1, pbos_.end());
533 std::future<bool> send(core::const_frame frame)
535 if(!frame_buffer_.try_push(frame) && !polling_event_)
536 graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame");
538 return make_ready_future(is_running_.load());
541 std::wstring channel_and_format() const
543 return L"[" + boost::lexical_cast<std::wstring>(channel_index_) + L"|" + format_desc_.name + L"]";
546 std::wstring print() const
548 return config_.name + L" " + channel_and_format();
551 void calculate_aspect()
555 screen_height_ = window_.getSize().y;
556 screen_width_ = window_.getSize().x;
559 GL(glViewport(0, 0, screen_width_, screen_height_));
561 std::pair<float, float> target_ratio = None();
562 if (config_.stretch == screen::stretch::fill)
563 target_ratio = Fill();
564 else if (config_.stretch == screen::stretch::uniform)
565 target_ratio = Uniform();
566 else if (config_.stretch == screen::stretch::uniform_to_fill)
567 target_ratio = UniformToFill();
569 width_ = target_ratio.first;
570 height_ = target_ratio.second;
573 std::pair<float, float> None()
575 float width = static_cast<float>(square_width_)/static_cast<float>(screen_width_);
576 float height = static_cast<float>(square_height_)/static_cast<float>(screen_height_);
578 return std::make_pair(width, height);
581 std::pair<float, float> Uniform()
583 float aspect = static_cast<float>(square_width_)/static_cast<float>(square_height_);
584 float width = std::min(1.0f, static_cast<float>(screen_height_)*aspect/static_cast<float>(screen_width_));
585 float height = static_cast<float>(screen_width_*width)/static_cast<float>(screen_height_*aspect);
587 return std::make_pair(width, height);
590 std::pair<float, float> Fill()
592 return std::make_pair(1.0f, 1.0f);
595 std::pair<float, float> UniformToFill()
597 float wr = static_cast<float>(square_width_)/static_cast<float>(screen_width_);
598 float hr = static_cast<float>(square_height_)/static_cast<float>(screen_height_);
599 float r_inv = 1.0f/std::min(wr, hr);
601 float width = wr*r_inv;
602 float height = hr*r_inv;
604 return std::make_pair(width, height);
609 struct screen_consumer_proxy : public core::frame_consumer
611 core::monitor::subject monitor_subject_;
612 const configuration config_;
613 std::unique_ptr<screen_consumer> consumer_;
614 core::interaction_sink* sink_;
618 screen_consumer_proxy(const configuration& config, core::interaction_sink* sink)
626 void initialize(const core::video_format_desc& format_desc, const core::audio_channel_layout&, int channel_index) override
629 consumer_.reset(new screen_consumer(config_, format_desc, channel_index, sink_));
632 int64_t presentation_frame_age_millis() const override
634 return consumer_ ? static_cast<int64_t>(consumer_->current_presentation_age_) : 0;
637 std::future<bool> send(core::const_frame frame) override
639 return consumer_->send(frame);
642 std::wstring print() const override
644 return consumer_ ? consumer_->print() : L"[screen_consumer]";
647 std::wstring name() const override
652 boost::property_tree::wptree info() const override
654 boost::property_tree::wptree info;
655 info.add(L"type", L"screen");
656 info.add(L"key-only", config_.key_only);
657 info.add(L"windowed", config_.windowed);
658 info.add(L"auto-deinterlace", config_.auto_deinterlace);
659 info.add(L"vsync", config_.vsync);
663 bool has_synchronization_clock() const override
668 int buffer_depth() const override
673 int index() const override
675 return 600 + (config_.key_only ? 10 : 0) + config_.screen_index;
678 core::monitor::subject& monitor_output()
680 return monitor_subject_;
684 void describe_consumer(core::help_sink& sink, const core::help_repository& repo)
686 sink.short_description(L"Displays the contents of a channel on screen using OpenGL.");
689 L"{[screen_index:int]|1} "
690 L"{[fullscreen:FULLSCREEN]} "
691 L"{[borderless:BORDERLESS]} "
692 L"{[key_only:KEY_ONLY]} "
693 L"{[non_interactive:NON_INTERACTIVE]} "
694 L"{[no_auto_deinterlace:NO_AUTO_DEINTERLACE]} "
695 L"{NAME [name:string]}");
696 sink.para()->text(L"Displays the contents of a channel on screen using OpenGL.");
698 ->item(L"screen_index", L"Determines which screen the channel should be displayed on. Defaults to 1.")
699 ->item(L"fullscreen", L"If specified opens the window in fullscreen.")
700 ->item(L"borderless", L"Makes the window appear without any window decorations.")
701 ->item(L"key_only", L"Only displays the alpha channel of the video channel if specified.")
702 ->item(L"non_interactive", L"If specified does not send mouse input to producers on the video channel.")
703 ->item(L"no_auto_deinterlace", L"If the video mode of the channel is an interlaced mode, specifying this will turn of deinterlacing.")
704 ->item(L"name", L"Optionally specifies a name of the window to show.");
705 sink.para()->text(L"Examples:");
706 sink.example(L">> ADD 1 SCREEN", L"opens a screen consumer on the default screen.");
707 sink.example(L">> ADD 1 SCREEN 2", L"opens a screen consumer on the screen 2.");
708 sink.example(L">> ADD 1 SCREEN 1 FULLSCREEN", L"opens a screen consumer in fullscreen on screen 1.");
709 sink.example(L">> ADD 1 SCREEN 1 BORDERLESS", L"opens a screen consumer without borders/window decorations on screen 1.");
712 spl::shared_ptr<core::frame_consumer> create_consumer(const std::vector<std::wstring>& params, core::interaction_sink* sink)
714 if (params.size() < 1 || !boost::iequals(params.at(0), L"SCREEN"))
715 return core::frame_consumer::empty();
717 configuration config;
719 if (params.size() > 1)
720 config.screen_index = boost::lexical_cast<int>(params.at(1));
722 config.windowed = !contains_param(L"FULLSCREEN", params);
723 config.key_only = contains_param(L"KEY_ONLY", params);
724 config.interactive = !contains_param(L"NON_INTERACTIVE", params);
725 config.auto_deinterlace = !contains_param(L"NO_AUTO_DEINTERLACE", params);
726 config.borderless = contains_param(L"BORDERLESS", params);
728 if (contains_param(L"NAME", params))
729 config.name = get_param(L"NAME", params);
731 return spl::make_shared<screen_consumer_proxy>(config, sink);
734 spl::shared_ptr<core::frame_consumer> create_preconfigured_consumer(const boost::property_tree::wptree& ptree, core::interaction_sink* sink)
736 configuration config;
737 config.name = ptree.get(L"name", config.name);
738 config.screen_index = ptree.get(L"device", config.screen_index + 1) - 1;
739 config.windowed = ptree.get(L"windowed", config.windowed);
740 config.key_only = ptree.get(L"key-only", config.key_only);
741 config.auto_deinterlace = ptree.get(L"auto-deinterlace", config.auto_deinterlace);
742 config.vsync = ptree.get(L"vsync", config.vsync);
743 config.interactive = ptree.get(L"interactive", config.interactive);
744 config.borderless = ptree.get(L"borderless", config.borderless);
746 auto stretch_str = ptree.get(L"stretch", L"default");
747 if(stretch_str == L"uniform")
748 config.stretch = screen::stretch::uniform;
749 else if(stretch_str == L"uniform_to_fill")
750 config.stretch = screen::stretch::uniform_to_fill;
752 auto aspect_str = ptree.get(L"aspect-ratio", L"default");
753 if(aspect_str == L"16:9")
754 config.aspect = configuration::aspect_ratio::aspect_16_9;
755 else if(aspect_str == L"4:3")
756 config.aspect = configuration::aspect_ratio::aspect_4_3;
758 return spl::make_shared<screen_consumer_proxy>(config, sink);