From: Steinar H. Gunderson Date: Tue, 11 Feb 2020 21:42:53 +0000 (+0100) Subject: Make Futatabi fades apply white balance. X-Git-Tag: 1.9.2~43 X-Git-Url: https://git.sesse.net/?p=nageru;a=commitdiff_plain;h=e15251d2787cb8e6b677af801de6180e55171763 Make Futatabi fades apply white balance. When there are two different video streams involved, they will often have different white points, so we can't just echo the Exif data back; we need to apply it when converting, and then get back a result in standard sRGB. It's not entirely correct since we still run with crushed whites/black, but it's good enough. This completes Nageru/Futatabi white balance round trip support. --- diff --git a/futatabi/exif_parser.cpp b/futatabi/exif_parser.cpp new file mode 100644 index 0000000..316d3c4 --- /dev/null +++ b/futatabi/exif_parser.cpp @@ -0,0 +1,99 @@ +#include "exif_parser.h" + +#include +#include +#include +#include + +using namespace Eigen; +using namespace movit; +using namespace std; + +uint32_t read32be(const uint8_t *data) +{ + return (uint32_t(data[0]) << 24) | + (uint32_t(data[1]) << 16) | + (uint32_t(data[2]) << 8) | + uint32_t(data[3]); +} + +uint16_t read16be(const uint8_t *data) +{ + return (uint16_t(data[0]) << 8) | uint16_t(data[1]); +} + +RGBTriplet get_neutral_color(const string &exif) +{ + if (exif.empty()) { + return {1.0f, 1.0f, 1.0f}; + } + + const uint8_t *data = reinterpret_cast(exif.data()); + + // Very rudimentary Exif parser (and probably integer-overflowable); + // we really only care about what Nageru sends us (MJPEGEncoder::init_jpeg_422()), + // but it would be nice to have a little bit of future-proofing, just in case. + if (exif.size() < 14 || memcmp(data, "Exif\0\0MM\0\x2a", 10) != 0) { + fprintf(stderr, "WARNING: Truncated or malformed Exif header, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + // We only care about the first IFD. + uint32_t ifd_offset = read32be(data + 10); + ifd_offset += 6; // Relative to the MM. + + if (ifd_offset < 14 || ifd_offset >= exif.size()) { + fprintf(stderr, "WARNING: Truncated or malformed Exif IFD, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + // Skip over number of tags (16 bits); if the white point is not the first one, + // we're bailing anyway. + if (ifd_offset + 2 > exif.size() || ifd_offset + 2 < ifd_offset) { + fprintf(stderr, "WARNING: Exif IFD has no rom for number of tags, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + if (ifd_offset + 4 > exif.size() || ifd_offset + 4 < ifd_offset) { + fprintf(stderr, "WARNING: Exif IFD has no rom for tag, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + uint16_t tag = read16be(data + ifd_offset + 2); + if (tag != 0x13e) { // WhitePoint. + fprintf(stderr, "WARNING: Unexpected first Exif tag, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + if (ifd_offset + 14 > exif.size() || ifd_offset + 14 < ifd_offset) { + fprintf(stderr, "WARNING: WhitePoint Exif tag was truncated, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + // Just assume we're rational type and two values... + uint32_t white_point_offset = read32be(data + ifd_offset + 10); + white_point_offset += 6; // Relative to the MM. + + if (white_point_offset >= exif.size()) { + fprintf(stderr, "WARNING: WhitePoint Exif tag was out of bounds, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + if (white_point_offset + 16 > exif.size()) { + fprintf(stderr, "WARNING: WhitePoint Exif tag was truncated, ignoring.\n"); + return {1.0f, 1.0f, 1.0f}; + } + + uint32_t x_nom = read32be(data + white_point_offset); + uint32_t x_den = read32be(data + white_point_offset + 4); + uint32_t y_nom = read32be(data + white_point_offset + 8); + uint32_t y_den = read32be(data + white_point_offset + 12); + + double x = double(x_nom) / x_den; + double y = double(y_nom) / y_den; + double z = 1.0 - x - y; + + Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB); + Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z); + + return RGBTriplet(rgb[0], rgb[1], rgb[2]); +} + diff --git a/futatabi/exif_parser.h b/futatabi/exif_parser.h new file mode 100644 index 0000000..09700b1 --- /dev/null +++ b/futatabi/exif_parser.h @@ -0,0 +1,16 @@ +#ifndef _EXIF_PARSER_H +#define _EXIF_PARSER_H + +#include +#include + +class Frame; + +// Try to parse the WhitePoint tag in the given Exif data. +// If the string is empty, or the tag is corrupted, or if it was +// just more complicated than our makeshift parser could deal with, +// returns (1.0, 1.0, 1.0), giving a regular D65 white point. +movit::RGBTriplet get_neutral_color(const std::string &exif); + +#endif // !defined(_EXIF_PARSER_H) + diff --git a/futatabi/video_stream.cpp b/futatabi/video_stream.cpp index 9a120b5..d7d9fd0 100644 --- a/futatabi/video_stream.cpp +++ b/futatabi/video_stream.cpp @@ -6,6 +6,7 @@ extern "C" { } #include "chroma_subsampler.h" +#include "exif_parser.h" #include "flags.h" #include "flow.h" #include "jpeg_frame_view.h" @@ -22,6 +23,7 @@ extern "C" { #include #include +using namespace movit; using namespace std; using namespace std::chrono; @@ -526,12 +528,16 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts shared_ptr frame2 = decode_jpeg_with_cache(secondary_frame, DECODE_IF_NOT_IN_CACHE, &frame_reader, &did_decode); // Then fade against it, putting it into the fade Y' and CbCr textures. - ycbcr_semiplanar_converter->prepare_chain_for_fade_from_texture(qf.output_tex, global_flags.width, global_flags.height, frame2, fade_alpha)->render_to_fbo(resources->fade_fbo, global_flags.width, global_flags.height); + RGBTriplet neutral_color = get_neutral_color(qf.exif_data); + ycbcr_semiplanar_converter->prepare_chain_for_fade_from_texture(qf.output_tex, neutral_color, global_flags.width, global_flags.height, frame2, fade_alpha)->render_to_fbo(resources->fade_fbo, global_flags.width, global_flags.height); // Subsample and split Cb/Cr. chroma_subsampler->subsample_chroma(resources->fade_cbcr_output_tex, global_flags.width, global_flags.height, resources->cb_tex, resources->cr_tex); interpolate_no_split->release_texture(qf.output_tex); + + // We already applied the white balance, so don't have the client redo it. + qf.exif_data.clear(); } else { tie(qf.output_tex, qf.cbcr_tex) = interpolate->exec(resources->input_tex, resources->gray_tex, flow_tex, global_flags.width, global_flags.height, alpha); check_error(); @@ -710,9 +716,10 @@ void VideoStream::encode_thread_func() glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); shared_ptr frame = frame_from_pbo(qf.resources->pbo_contents, global_flags.width, global_flags.height); + assert(frame->exif_data.empty()); // Now JPEG encode it, and send it on to the stream. - string jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height, move(frame->exif_data)); + string jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height, /*exif_data=*/""); AVPacket pkt; av_init_packet(&pkt); diff --git a/futatabi/ycbcr_converter.cpp b/futatabi/ycbcr_converter.cpp index ef402a5..2d2f32f 100644 --- a/futatabi/ycbcr_converter.cpp +++ b/futatabi/ycbcr_converter.cpp @@ -1,13 +1,16 @@ #include "ycbcr_converter.h" +#include "exif_parser.h" #include "flags.h" #include "jpeg_frame.h" +#include #include +#include #include -using namespace std; using namespace movit; +using namespace std; namespace { @@ -53,12 +56,12 @@ YCbCrConverter::YCbCrConverter(YCbCrConverter::OutputMode output_mode, ResourceP // sources with no conversion, so we ought to have had false here. // However, in the off chance that we're actually getting real MJPEG, // we don't want to crush its blacks (or whites) by clamping. All of - // our processing is fades, so if we're in limited-range input, we'll - // stay in limited-range output. (Fading between limited-range and - // full-range sources will be broken, of course.) There will be some - // slight confusion in the parts of the algorithms dealing with RGB, - // but they're small and we'll manage. - ycbcr_format.full_range = true; + // our processing is fades or other linear operations, so if we're in + // limited-range input, we'll stay in limited-range output. (Fading + // between limited-range and full-range sources will be broken, + // of course.) There will be some slight confusion in the parts of the + // algorithms dealing with RGB, but they're small and we'll manage. + ycbcr_format.full_range = false; YCbCrFormat ycbcr_output_format = ycbcr_format; ycbcr_output_format.chroma_subsampling_x = 1; @@ -77,19 +80,23 @@ YCbCrConverter::YCbCrConverter(YCbCrConverter::OutputMode output_mode, ResourceP semiplanar_chain->set_dither_bits(8); semiplanar_chain->finalize(); - // Fade chains. + // Fade chains. These include white balance adjustments. for (bool first_input_is_semiplanar : { false, true }) { for (bool second_input_is_semiplanar : { false, true }) { FadeChain &fade_chain = fade_chains[first_input_is_semiplanar][second_input_is_semiplanar]; fade_chain.chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool)); - fade_chain.input[0] = (movit::YCbCrInput *)fade_chain.chain->add_input( + fade_chain.input[0] = (YCbCrInput *)fade_chain.chain->add_input( new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, first_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR)); - fade_chain.input[1] = (movit::YCbCrInput *)fade_chain.chain->add_input( + fade_chain.input[1] = (YCbCrInput *)fade_chain.chain->add_input( new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, second_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR)); - fade_chain.mix_effect = (movit::MixEffect *)fade_chain.chain->add_effect( - new MixEffect, fade_chain.input[0], fade_chain.input[1]); + fade_chain.wb_effect[0] = (WhiteBalanceEffect *)fade_chain.chain->add_effect( + new WhiteBalanceEffect, fade_chain.input[0]); + fade_chain.wb_effect[1] = (WhiteBalanceEffect *)fade_chain.chain->add_effect( + new WhiteBalanceEffect, fade_chain.input[1]); + fade_chain.mix_effect = (MixEffect *)fade_chain.chain->add_effect( + new MixEffect, fade_chain.wb_effect[0], fade_chain.wb_effect[1]); setup_outputs(output_mode, inout_format, ycbcr_output_format, fade_chain.chain.get()); fade_chain.chain->set_dither_bits(8); fade_chain.chain->finalize(); @@ -103,17 +110,21 @@ YCbCrConverter::YCbCrConverter(YCbCrConverter::OutputMode output_mode, ResourceP fade_chain.chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool)); ycbcr_format.chroma_subsampling_x = 1; - fade_chain.input[0] = (movit::YCbCrInput *)fade_chain.chain->add_input( + fade_chain.input[0] = (YCbCrInput *)fade_chain.chain->add_input( new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, YCBCR_INPUT_INTERLEAVED)); ycbcr_format.chroma_subsampling_x = 2; - fade_chain.input[1] = (movit::YCbCrInput *)fade_chain.chain->add_input( + fade_chain.input[1] = (YCbCrInput *)fade_chain.chain->add_input( new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, second_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR)); - fade_chain.mix_effect = (movit::MixEffect *)fade_chain.chain->add_effect( - new MixEffect, fade_chain.input[0], fade_chain.input[1]); + fade_chain.wb_effect[0] = (WhiteBalanceEffect *)fade_chain.chain->add_effect( + new WhiteBalanceEffect, fade_chain.input[0]); + fade_chain.wb_effect[1] = (WhiteBalanceEffect *)fade_chain.chain->add_effect( + new WhiteBalanceEffect, fade_chain.input[1]); + fade_chain.mix_effect = (MixEffect *)fade_chain.chain->add_effect( + new MixEffect, fade_chain.wb_effect[0], fade_chain.wb_effect[1]); setup_outputs(output_mode, inout_format, ycbcr_output_format, fade_chain.chain.get()); fade_chain.chain->set_dither_bits(8); fade_chain.chain->finalize(); @@ -138,11 +149,15 @@ EffectChain *YCbCrConverter::prepare_chain_for_fade(shared_ptr frame, sha setup_input_for_frame(secondary_frame, ycbcr_format, fade_chain.input[1]); bool ok = fade_chain.mix_effect->set_float("strength_first", 1.0f - fade_alpha); ok |= fade_chain.mix_effect->set_float("strength_second", fade_alpha); + RGBTriplet neutral_color0 = get_neutral_color(frame->exif_data); + RGBTriplet neutral_color1 = get_neutral_color(secondary_frame->exif_data); + ok |= fade_chain.wb_effect[0]->set_vec3("neutral_color", (float *)&neutral_color0); + ok |= fade_chain.wb_effect[1]->set_vec3("neutral_color", (float *)&neutral_color1); assert(ok); return fade_chain.chain.get(); } -EffectChain *YCbCrConverter::prepare_chain_for_fade_from_texture(GLuint tex, unsigned width, unsigned height, std::shared_ptr secondary_frame, float fade_alpha) +EffectChain *YCbCrConverter::prepare_chain_for_fade_from_texture(GLuint tex, RGBTriplet tex_neutral_color, unsigned width, unsigned height, std::shared_ptr secondary_frame, float fade_alpha) { const FadeChain &fade_chain = interleaved_fade_chains[secondary_frame->is_semiplanar]; { @@ -163,6 +178,9 @@ EffectChain *YCbCrConverter::prepare_chain_for_fade_from_texture(GLuint tex, uns setup_input_for_frame(secondary_frame, ycbcr_format, fade_chain.input[1]); bool ok = fade_chain.mix_effect->set_float("strength_first", 1.0f - fade_alpha); ok |= fade_chain.mix_effect->set_float("strength_second", fade_alpha); + RGBTriplet neutral_color1 = get_neutral_color(secondary_frame->exif_data); + ok |= fade_chain.wb_effect[0]->set_vec3("neutral_color", (float *)&tex_neutral_color); + ok |= fade_chain.wb_effect[1]->set_vec3("neutral_color", (float *)&neutral_color1); assert(ok); return fade_chain.chain.get(); } diff --git a/futatabi/ycbcr_converter.h b/futatabi/ycbcr_converter.h index 4618f1e..c2ad2fa 100644 --- a/futatabi/ycbcr_converter.h +++ b/futatabi/ycbcr_converter.h @@ -3,6 +3,7 @@ #include #include +#include #include namespace movit { @@ -10,6 +11,7 @@ namespace movit { class EffectChain; class MixEffect; class ResourcePool; +class WhiteBalanceEffect; struct YCbCrFormat; } // namespace movit @@ -25,12 +27,13 @@ public: }; YCbCrConverter(OutputMode output_mode, movit::ResourcePool *resource_pool); - // Returns the appropriate chain for rendering. + // Returns the appropriate chain for rendering. Fades apply white balance, + // straight-up conversion does not. movit::EffectChain *prepare_chain_for_conversion(std::shared_ptr frame); movit::EffectChain *prepare_chain_for_fade(std::shared_ptr frame, std::shared_ptr secondary_frame, float fade_alpha); // must be interleaved Y'CbCr. - movit::EffectChain *prepare_chain_for_fade_from_texture(GLuint tex, unsigned width, unsigned height, std::shared_ptr secondary_frame, float fade_alpha); + movit::EffectChain *prepare_chain_for_fade_from_texture(GLuint tex, movit::RGBTriplet neutral_color, unsigned width, unsigned height, std::shared_ptr secondary_frame, float fade_alpha); private: movit::YCbCrFormat ycbcr_format; @@ -45,6 +48,7 @@ private: struct FadeChain { std::unique_ptr chain; movit::YCbCrInput *input[2]; + movit::WhiteBalanceEffect *wb_effect[2]; movit::MixEffect *mix_effect; }; FadeChain fade_chains[2][2]; diff --git a/meson.build b/meson.build index b23f6d9..d705c7f 100644 --- a/meson.build +++ b/meson.build @@ -290,6 +290,7 @@ futatabi_srcs += ['futatabi/main.cpp', 'futatabi/player.cpp', 'futatabi/video_st futatabi_srcs += ['futatabi/vaapi_jpeg_decoder.cpp', 'futatabi/db.cpp', 'futatabi/ycbcr_converter.cpp', 'futatabi/flags.cpp'] futatabi_srcs += ['futatabi/mainwindow.cpp', 'futatabi/jpeg_frame_view.cpp', 'futatabi/clip_list.cpp', 'futatabi/frame_on_disk.cpp'] futatabi_srcs += ['futatabi/export.cpp', 'futatabi/midi_mapper.cpp', 'futatabi/midi_mapping_dialog.cpp'] +futatabi_srcs += ['futatabi/exif_parser.cpp'] futatabi_srcs += moc_files futatabi_srcs += proto_generated @@ -307,7 +308,7 @@ futatabi_shader_srcs = bin2h_gen.process(shaders) futatabi_srcs += futatabi_shader_srcs executable('futatabi', futatabi_srcs, - dependencies: [shareddep, qt5deps, libjpegdep, movitdep, libmicrohttpddep, protobufdep, sqlite3dep, vax11dep, vadrmdep, x11dep, threaddep, libavformatdep, libavcodecdep, libavutildep, libswscaledep], + dependencies: [shareddep, qt5deps, libjpegdep, movitdep, libmicrohttpddep, protobufdep, sqlite3dep, vax11dep, vadrmdep, x11dep, threaddep, libavformatdep, libavcodecdep, libavutildep, libswscaledep, eigendep], link_with: shared, include_directories: [include_directories('futatabi')], install: true)