From 6cf87ad853439f8565c575fb29dc539a15fdba87 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Tue, 11 Feb 2020 00:18:38 +0100 Subject: [PATCH] Heed the Exif white point when playing back (MJPEG) video. --- meson.build | 3 ++- nageru/ffmpeg_capture.cpp | 32 ++++++++++++++++++++++++++++++++ nageru/ffmpeg_capture.h | 9 +++++++++ nageru/mixer.cpp | 27 +++++++++++++++++++++++++-- nageru/mixer.h | 3 +++ nageru/pbo_frame_allocator.h | 1 + nageru/theme.cpp | 18 +++++++++++++++++- nageru/theme.h | 2 ++ 8 files changed, 91 insertions(+), 4 deletions(-) diff --git a/meson.build b/meson.build index 9880137..b23f6d9 100644 --- a/meson.build +++ b/meson.build @@ -9,6 +9,7 @@ embedded_bmusb = get_option('embedded_bmusb') alsadep = dependency('alsa') bmusbdep = dependency('bmusb', required: not embedded_bmusb) dldep = cxx.find_library('dl') +eigendep = dependency('eigen3') epoxydep = dependency('epoxy') libavcodecdep = dependency('libavcodec') libavformatdep = dependency('libavformat') @@ -65,7 +66,7 @@ nageru_srcs = [] nageru_deps = [shareddep, qt5deps, libjpegdep, movitdep, protobufdep, vax11dep, vadrmdep, x11dep, libavformatdep, libswresampledep, libavcodecdep, libavutildep, libswscaledep, libusbdep, luajitdep, dldep, x264dep, alsadep, zitaresamplerdep, - qcustomplotdep, threaddep] + qcustomplotdep, threaddep, eigendep] nageru_include_dirs = [include_directories('nageru')] nageru_link_with = [] nageru_build_rpath = '' diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index b9eb982..2d1b338 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -27,6 +27,10 @@ extern "C" { #include #include +#include +#include +#include + #include "bmusb/bmusb.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" @@ -41,6 +45,7 @@ using namespace std; using namespace std::chrono; using namespace bmusb; using namespace movit; +using namespace Eigen; namespace { @@ -214,6 +219,32 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f return format; } +RGBTriplet get_neutral_color(AVDictionary *metadata) +{ + if (metadata == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0); + if (entry == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + unsigned x_nom, x_den, y_nom, y_den; + if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) { + fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value); + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + double x = double(x_nom) / x_den; + double y = double(y_nom) / y_den; + double z = 1.0 - x - y; + + Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB); + Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z); + + return RGBTriplet(rgb[0], rgb[1], rgb[2]); +} + } // namespace FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) @@ -575,6 +606,7 @@ bool FFmpegCapture::play_video(const string &pathname) // audio discontinuity.) timecode += MAX_FPS * 2 + 1; } + last_neutral_color = get_neutral_color(frame->metadata); frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, video_frame.get_and_release(), 0, video_format, audio_frame.get_and_release(), 0, audio_format); diff --git a/nageru/ffmpeg_capture.h b/nageru/ffmpeg_capture.h index c7b8a61..6084b68 100644 --- a/nageru/ffmpeg_capture.h +++ b/nageru/ffmpeg_capture.h @@ -33,6 +33,7 @@ #include #include +#include #include extern "C" { @@ -181,6 +182,12 @@ public: return has_last_subtitle; } + // Same. + movit::RGBTriplet get_last_neutral_color() const + { + return last_neutral_color; + } + void set_dequeue_thread_callbacks(std::function init, std::function cleanup) override { dequeue_init_callback = init; @@ -297,6 +304,8 @@ private: // Subtitles (no decoding done, really). bool has_last_subtitle = false; std::string last_subtitle; + + movit::RGBTriplet last_neutral_color; }; #endif // !defined(_FFMPEG_CAPTURE_H) diff --git a/nageru/mixer.cpp b/nageru/mixer.cpp index 1457aa9..9a86682 100644 --- a/nageru/mixer.cpp +++ b/nageru/mixer.cpp @@ -348,6 +348,11 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) ycbcr_format.cb_y_position = 0.5f; ycbcr_format.cr_y_position = 0.5f; + // Initialize the neutral colors to sane values. + for (unsigned i = 0; i < MAX_VIDEO_CARDS; ++i) { + last_received_neutral_color[i] = RGBTriplet(1.0f, 1.0f, 1.0f); + } + // Display chain; shows the live output produced by the main chain (or rather, a copy of it). display_chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool.get())); check_error(); @@ -1004,6 +1009,10 @@ void Mixer::bm_frame(unsigned card_index, uint16_t timecode, new_frame.video_format = video_format; new_frame.y_offset = y_offset; new_frame.cbcr_offset = cbcr_offset; + if (card->type == CardType::FFMPEG_INPUT) { + FFmpegCapture *ffmpeg_capture = static_cast(card->capture.get()); + new_frame.neutral_color = ffmpeg_capture->get_last_neutral_color(); + } card->new_frames.push_back(move(new_frame)); card->jitter_history.frame_arrived(video_frame.received_timestamp, frame_length, dropped_frames); card->may_have_dropped_last_frame = false; @@ -1115,13 +1124,27 @@ void Mixer::thread_func() new_frame->upload_func = nullptr; } + // Only set the white balance if it actually changed. This means that the user + // is free to override the white balance in a video with no white balance information + // actually set (ie. r=g=b=1 all the time), or one where the white point is wrong, + // but frame-to-frame decisions will be heeded. We do this pretty much as late + // as possible (ie., after picking out the frame from the buffer), so that we are sure + // that the change takes effect on exactly the right frame. + if (fabs(new_frame->neutral_color.r - last_received_neutral_color[card_index].r) > 1e-3 || + fabs(new_frame->neutral_color.g - last_received_neutral_color[card_index].g) > 1e-3 || + fabs(new_frame->neutral_color.b - last_received_neutral_color[card_index].b) > 1e-3) { + theme->set_wb_for_signal(card_index, new_frame->neutral_color.r, new_frame->neutral_color.g, new_frame->neutral_color.b); + last_received_neutral_color[card_index] = new_frame->neutral_color; + } + if (new_frame->frame->data_copy != nullptr) { int mjpeg_card_index = mjpeg_encoder->get_mjpeg_stream_for_card(card_index); if (mjpeg_card_index != -1) { - RGBTriplet white_balance = theme->get_white_balance_for_signal(card_index); - mjpeg_encoder->upload_frame(pts_int, mjpeg_card_index, new_frame->frame, new_frame->video_format, new_frame->y_offset, new_frame->cbcr_offset, move(raw_audio[card_index]), white_balance); + RGBTriplet neutral_color = theme->get_white_balance_for_signal(card_index); + mjpeg_encoder->upload_frame(pts_int, mjpeg_card_index, new_frame->frame, new_frame->video_format, new_frame->y_offset, new_frame->cbcr_offset, move(raw_audio[card_index]), neutral_color); } } + } int64_t frame_duration = output_frame_info.frame_duration; diff --git a/nageru/mixer.h b/nageru/mixer.h index 9aa3756..49afa7f 100644 --- a/nageru/mixer.h +++ b/nageru/mixer.h @@ -23,6 +23,7 @@ #include #include +#include #include #include "audio_mixer.h" @@ -537,6 +538,7 @@ private: std::function upload_func; // Needs to be called to actually upload the texture to OpenGL. unsigned dropped_frames = 0; // Number of dropped frames before this one. std::chrono::steady_clock::time_point received_timestamp = std::chrono::steady_clock::time_point::min(); + movit::RGBTriplet neutral_color{1.0f, 1.0f, 1.0f}; // Used for MJPEG encoding. (upload_func packs everything it needs // into the functor, but would otherwise also use these.) @@ -575,6 +577,7 @@ private: JitterHistory output_jitter_history; CaptureCard cards[MAX_VIDEO_CARDS]; // Protected by . YCbCrInterpretation ycbcr_interpretation[MAX_VIDEO_CARDS]; // Protected by . + movit::RGBTriplet last_received_neutral_color[MAX_VIDEO_CARDS]; // Used by the mixer thread only. Constructor-initialiezd. std::unique_ptr audio_mixer; // Same as global_audio_mixer (see audio_mixer.h). bool input_card_is_master_clock(unsigned card_index, unsigned master_card_index) const; struct OutputFrameInfo { diff --git a/nageru/pbo_frame_allocator.h b/nageru/pbo_frame_allocator.h index d7559dc..0080dd0 100644 --- a/nageru/pbo_frame_allocator.h +++ b/nageru/pbo_frame_allocator.h @@ -60,6 +60,7 @@ public: unsigned last_frame_rate_nom, last_frame_rate_den; bool has_last_subtitle = false; std::string last_subtitle; + movit::RGBTriplet white_balance{1.0f, 1.0f, 1.0f}; // These are the source of the “data_copy” member in Frame, // used for MJPEG encoding. There are three possibilities: diff --git a/nageru/theme.cpp b/nageru/theme.cpp index 51aacd5..f3e1aa1 100644 --- a/nageru/theme.cpp +++ b/nageru/theme.cpp @@ -1687,11 +1687,27 @@ bool Theme::get_supports_set_wb(unsigned channel) void Theme::set_wb(unsigned channel, float r, float g, float b) { lock_guard lock(m); - if (channel_signals.count(channel)) { white_balance_for_signal[channel_signals[channel]] = RGBTriplet{ r, g, b }; } + call_lua_wb_callback(channel, r, g, b); +} + +void Theme::set_wb_for_signal(int signal, float r, float g, float b) +{ + lock_guard lock(m); + white_balance_for_signal[signal] = RGBTriplet{ r, g, b }; + + for (const auto &channel_and_signal : channel_signals) { + if (channel_and_signal.second == signal) { + call_lua_wb_callback(channel_and_signal.first, r, g, b); + } + } +} + +void Theme::call_lua_wb_callback(unsigned channel, float r, float g, float b) +{ lua_getglobal(L, "set_wb"); if (lua_isnil(L, -1)) { // The function doesn't exist, to just ignore. We've stored the white balance, diff --git a/nageru/theme.h b/nageru/theme.h index 46f2489..8e45373 100644 --- a/nageru/theme.h +++ b/nageru/theme.h @@ -105,6 +105,7 @@ public: int get_channel_signal(unsigned channel); bool get_supports_set_wb(unsigned channel); void set_wb(unsigned channel, float r, float g, float b); + void set_wb_for_signal(int signal, float r, float g, float b); movit::RGBTriplet get_white_balance_for_signal(int signal); std::string get_channel_color(unsigned channel); @@ -195,6 +196,7 @@ private: void register_class(const char *class_name, const luaL_Reg *funcs, EffectType effect_type = NO_EFFECT_TYPE); int set_theme_menu(lua_State *L); Chain get_chain_from_effect_chain(movit::EffectChain *effect_chain, unsigned num, const InputState &input_state); + void call_lua_wb_callback(unsigned channel, float r, float g, float b); std::string theme_path; -- 2.39.2