From 533f00a9b992d06767737f9db236b4cf76b9c124 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 18 May 2019 11:26:33 +0200 Subject: [PATCH] Make the ImageInput cache store textures, not images. This saves on a lot of texture memory when the same image is used in multiple chains, but perhaps more importantly, will allow us to decouple ImageInputs from which images they show later. --- nageru/image_input.cpp | 87 +++++++++++++++++++++++++----------- nageru/image_input.h | 22 +++++---- nageru/main.cpp | 1 - nageru/mixer.cpp | 8 +++- nageru/mixer.h | 2 +- nageru/ref_counted_texture.h | 20 +++++++++ 6 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 nageru/ref_counted_texture.h diff --git a/nageru/image_input.cpp b/nageru/image_input.cpp index 12789dc..d2d6611 100644 --- a/nageru/image_input.cpp +++ b/nageru/image_input.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ extern "C" { #include } +#include #include #include #include @@ -30,6 +32,7 @@ extern "C" { #include #include +#include "shared/context.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" #include "flags.h" @@ -38,6 +41,7 @@ struct SwsContext; using namespace std; +// FIXME gl context ImageInput::ImageInput(const string &filename) : movit::FlatInput({movit::COLORSPACE_sRGB, movit::GAMMA_sRGB}, movit::FORMAT_RGBA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, 1280, 720), // Resolution will be overwritten. @@ -50,7 +54,7 @@ ImageInput::ImageInput(const string &filename) } set_width(current_image->width); set_height(current_image->height); - set_pixel_data(current_image->pixels.get()); + set_texture_num(*current_image->tex); } void ImageInput::set_gl_state(GLuint glsl_program_num, const string& prefix, unsigned *sampler_num) @@ -65,7 +69,7 @@ void ImageInput::set_gl_state(GLuint glsl_program_num, const string& prefix, uns lock_guard lock(all_images_lock); if (all_images[pathname] != current_image) { current_image = all_images[pathname]; - set_pixel_data(current_image->pixels.get()); + set_texture_num(*current_image->tex); } } movit::FlatInput::set_gl_state(glsl_program_num, prefix, sampler_num); @@ -79,12 +83,6 @@ shared_ptr ImageInput::load_image(const string &filenam } all_images[pathname] = load_image_raw(pathname); - - if (!update_thread_started) { - update_thread = thread(update_thread_func); - update_thread_started = true; - } - return all_images[pathname]; } @@ -196,23 +194,59 @@ shared_ptr ImageInput::load_image_raw(const string &pat unique_ptr image_data(new uint8_t[len]); av_image_copy_to_buffer(image_data.get(), len, pic_data, linesizes, AV_PIX_FMT_RGBA, frame->width, frame->height, 1); - shared_ptr image(new Image{unsigned(frame->width), unsigned(frame->height), move(image_data), last_modified}); + // Create and upload the texture. We always make mipmaps, since we have + // generally no idea of all the different chains that might crop up. + GLuint tex; + glGenTextures(1, &tex); + check_error(); + glBindTexture(GL_TEXTURE_2D, tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + // Actual upload. + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + check_error(); + glPixelStorei(GL_UNPACK_ROW_LENGTH, linesizes[0] / 4); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, frame->width, frame->height, 0, GL_RGBA, GL_UNSIGNED_BYTE, image_data.get()); + check_error(); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + check_error(); + + glGenerateMipmap(GL_TEXTURE_2D); + check_error(); + glBindTexture(GL_TEXTURE_2D, 0); + check_error(); + + shared_ptr image(new Image{unsigned(frame->width), unsigned(frame->height), RefCountedTexture(new GLuint(tex)), last_modified}); return image; } // Fire up a thread to update all images every second. // We could do inotify, but this is good enough for now. -void ImageInput::update_thread_func() +void ImageInput::update_thread_func(QSurface *surface) { pthread_setname_np(pthread_self(), "Update_Images"); + eglBindAPI(EGL_OPENGL_API); + QOpenGLContext *context = create_context(surface); + if (!make_current(context, surface)) { + printf("Couldn't initialize OpenGL context!\n"); + abort(); + } + struct stat buf; for ( ;; ) { { - unique_lock lock(threads_should_quit_mu); - threads_should_quit_modified.wait_for(lock, chrono::seconds(1), []() { return threads_should_quit; }); + unique_lock lock(update_thread_should_quit_mu); + update_thread_should_quit_modified.wait_for(lock, chrono::seconds(1), [] { return update_thread_should_quit; }); } - if (threads_should_quit) { + if (update_thread_should_quit) { return; } @@ -255,24 +289,25 @@ void ImageInput::update_thread_func() } } -void ImageInput::shutdown_updaters() +void ImageInput::start_update_thread(QSurface *surface) { - { - lock_guard lock(threads_should_quit_mu); - threads_should_quit = true; - threads_should_quit_modified.notify_all(); - } + update_thread = thread(update_thread_func, surface); +} - lock_guard lock(all_images_lock); - if (update_thread_started) { - update_thread.join(); +void ImageInput::end_update_thread() + +{ + { + lock_guard lock(update_thread_should_quit_mu); + update_thread_should_quit = true; + update_thread_should_quit_modified.notify_all(); } + update_thread.join(); } mutex ImageInput::all_images_lock; map> ImageInput::all_images; -bool ImageInput::update_thread_started = false; thread ImageInput::update_thread; -mutex ImageInput::threads_should_quit_mu; -bool ImageInput::threads_should_quit = false; -condition_variable ImageInput::threads_should_quit_modified; +mutex ImageInput::update_thread_should_quit_mu; +bool ImageInput::update_thread_should_quit = false; +condition_variable ImageInput::update_thread_should_quit_modified; diff --git a/nageru/image_input.h b/nageru/image_input.h index ac3c519..f7289f3 100644 --- a/nageru/image_input.h +++ b/nageru/image_input.h @@ -13,21 +13,28 @@ #include #include +#include "ref_counted_texture.h" + +class QSurface; + // An output that takes its input from a static image, loaded with ffmpeg. // comes from a single 2D array with chunky pixels. The image is refreshed // from disk about every second. class ImageInput : public movit::FlatInput { public: + // NOTE: You will need to call start_update_thread() yourself, once per program. ImageInput(const std::string &filename); std::string effect_type_id() const override { return "ImageInput"; } void set_gl_state(GLuint glsl_program_num, const std::string& prefix, unsigned *sampler_num) override; - static void shutdown_updaters(); + + static void start_update_thread(QSurface *surface); + static void end_update_thread(); private: struct Image { unsigned width, height; - std::unique_ptr pixels; + RefCountedTexture tex; timespec last_modified; }; @@ -36,15 +43,14 @@ private: static std::shared_ptr load_image(const std::string &filename, const std::string &pathname); static std::shared_ptr load_image_raw(const std::string &pathname); - static void update_thread_func(); + static void update_thread_func(QSurface *surface); static std::mutex all_images_lock; static std::map> all_images; // Under all_images_lock. - static bool update_thread_started; // Under all_images_lock. - static std::thread update_thread; // Under all_images_lock. - static std::mutex threads_should_quit_mu; - static bool threads_should_quit; // Under threads_should_quit_mu. - static std::condition_variable threads_should_quit_modified; // Signals when threads_should_quit is set. + static std::thread update_thread; + static std::mutex update_thread_should_quit_mu; + static bool update_thread_should_quit; // Under thread_should_quit_mu. + static std::condition_variable update_thread_should_quit_modified; // Signals when threads_should_quit is set. }; #endif // !defined(_IMAGE_INPUT_H) diff --git a/nageru/main.cpp b/nageru/main.cpp index b546375..3516620 100644 --- a/nageru/main.cpp +++ b/nageru/main.cpp @@ -120,6 +120,5 @@ int main(int argc, char *argv[]) int rc = app.exec(); delete global_mixer; - ImageInput::shutdown_updaters(); return rc; } diff --git a/nageru/mixer.cpp b/nageru/mixer.cpp index ef73b61..83d6ead 100644 --- a/nageru/mixer.cpp +++ b/nageru/mixer.cpp @@ -44,6 +44,7 @@ #include "shared/disk_space_estimator.h" #include "ffmpeg_capture.h" #include "flags.h" +#include "image_input.h" #include "input_mapping.h" #include "shared/metrics.h" #include "mjpeg_encoder.h" @@ -306,7 +307,8 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) num_cards(num_cards), mixer_surface(create_surface(format)), h264_encoder_surface(create_surface(format)), - decklink_output_surface(create_surface(format)) + decklink_output_surface(create_surface(format)), + image_update_surface(create_surface(format)) { memcpy(ycbcr_interpretation, global_flags.ycbcr_interpretation, sizeof(ycbcr_interpretation)); CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF)); @@ -500,10 +502,14 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) } output_jitter_history.register_metrics({{ "card", "output" }}); + + ImageInput::start_update_thread(image_update_surface); } Mixer::~Mixer() { + ImageInput::end_update_thread(); + if (mjpeg_encoder != nullptr) { mjpeg_encoder->stop(); } diff --git a/nageru/mixer.h b/nageru/mixer.h index de8b457..5af2016 100644 --- a/nageru/mixer.h +++ b/nageru/mixer.h @@ -456,7 +456,7 @@ private: HTTPD httpd; unsigned num_cards, num_video_inputs, num_html_inputs = 0; - QSurface *mixer_surface, *h264_encoder_surface, *decklink_output_surface; + QSurface *mixer_surface, *h264_encoder_surface, *decklink_output_surface, *image_update_surface; std::unique_ptr resource_pool; std::unique_ptr theme; std::atomic audio_source_channel{0}; diff --git a/nageru/ref_counted_texture.h b/nageru/ref_counted_texture.h new file mode 100644 index 0000000..1a94476 --- /dev/null +++ b/nageru/ref_counted_texture.h @@ -0,0 +1,20 @@ +#ifndef _UNIQUE_TEXTURE_H +#define _UNIQUE_TEXTURE_H 1 + +// A wrapper around an OpenGL texture that is automatically deleted. +// Used only by ImageInput. + +#include +#include + +struct TextureDeleter { + void operator() (GLuint *tex) + { + glDeleteTextures(1, tex); + delete tex; + } +}; + +typedef std::unique_ptr RefCountedTexture; + +#endif // !defined(_REF_COUNTED_TEXTURE) -- 2.39.2