From 80fc4a6e806e5638ae050c3020962137ca5fd76b Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Wed, 9 Sep 2015 23:51:48 +0200 Subject: [PATCH] Add support for Y'CbCr output. Currently only 8-bit and only 4:4:4 packed, but it should be a useful building block. --- .gitignore | 1 + Makefile.in | 1 + effect_chain.cpp | 39 ++++++- effect_chain.h | 13 +++ test_util.cpp | 25 ++++- test_util.h | 3 + ycbcr.h | 3 +- ycbcr_conversion_effect.cpp | 47 ++++++++ ycbcr_conversion_effect.frag | 24 ++++ ycbcr_conversion_effect.h | 35 ++++++ ycbcr_conversion_effect_test.cpp | 185 +++++++++++++++++++++++++++++++ 11 files changed, 367 insertions(+), 9 deletions(-) create mode 100644 ycbcr_conversion_effect.cpp create mode 100644 ycbcr_conversion_effect.frag create mode 100644 ycbcr_conversion_effect.h create mode 100644 ycbcr_conversion_effect_test.cpp diff --git a/.gitignore b/.gitignore index 366d38c..74045c5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ fp16_test luma_mix_effect_test slice_effect_test vignette_effect_test +ycbcr_conversion_effect_test chain-*.frag movit.info coverage/ diff --git a/Makefile.in b/Makefile.in index 35fc71e..53eb6ff 100644 --- a/Makefile.in +++ b/Makefile.in @@ -76,6 +76,7 @@ TESTED_EFFECTS += slice_effect TESTED_EFFECTS += complex_modulate_effect TESTED_EFFECTS += luma_mix_effect TESTED_EFFECTS += fft_convolution_effect +TESTED_EFFECTS += ycbcr_conversion_effect UNTESTED_EFFECTS = sandbox_effect UNTESTED_EFFECTS += mirror_effect diff --git a/effect_chain.cpp b/effect_chain.cpp index 519cb2c..e1ae832 100644 --- a/effect_chain.cpp +++ b/effect_chain.cpp @@ -25,6 +25,7 @@ #include "input.h" #include "resource_pool.h" #include "util.h" +#include "ycbcr_conversion_effect.h" using namespace std; @@ -74,6 +75,20 @@ void EffectChain::add_output(const ImageFormat &format, OutputAlphaFormat alpha_ assert(!finalized); output_format = format; output_alpha_format = alpha_format; + output_color_type = OUTPUT_COLOR_RGB; +} + +void EffectChain::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, + const YCbCrFormat &ycbcr_format) +{ + assert(!finalized); + output_format = format; + output_alpha_format = alpha_format; + output_color_type = OUTPUT_COLOR_YCBCR; + output_ycbcr_format = ycbcr_format; + + assert(ycbcr_format.chroma_subsampling_x == 1); + assert(ycbcr_format.chroma_subsampling_y == 1); } Node *EffectChain::add_node(Effect *effect) @@ -1363,6 +1378,22 @@ void EffectChain::fix_output_gamma() connect_nodes(output, conversion); } } + +// If the user has requested Y'CbCr output, we need to do this conversion +// _after_ GammaCompressionEffect etc., but before dither (see below). +// This is because Y'CbCr, with the exception of a special optional mode +// in Rec. 2020 (which we currently don't support), is defined to work on +// gamma-encoded data. +void EffectChain::add_ycbcr_conversion_if_needed() +{ + assert(output_color_type == OUTPUT_COLOR_RGB || output_color_type == OUTPUT_COLOR_YCBCR); + if (output_color_type != OUTPUT_COLOR_YCBCR) { + return; + } + Node *output = find_output_node(); + Node *ycbcr = add_node(new YCbCrConversionEffect(output_ycbcr_format)); + connect_nodes(output, ycbcr); +} // If the user has requested dither, add a DitherEffect right at the end // (after GammaCompressionEffect etc.). This needs to be done after everything else, @@ -1445,11 +1476,13 @@ void EffectChain::finalize() fix_internal_gamma_by_asking_inputs(15); fix_internal_gamma_by_inserting_nodes(16); - output_dot("step17-before-dither.dot"); + output_dot("step17-before-ycbcr.dot"); + add_ycbcr_conversion_if_needed(); + output_dot("step18-before-dither.dot"); add_dither_if_needed(); - output_dot("step18-final.dot"); + output_dot("step19-final.dot"); // Construct all needed GLSL programs, starting at the output. // We need to keep track of which effects have already been computed, @@ -1458,7 +1491,7 @@ void EffectChain::finalize() map completed_effects; construct_phase(find_output_node(), &completed_effects); - output_dot("step19-split-to-phases.dot"); + output_dot("step20-split-to-phases.dot"); assert(phases[0]->inputs.empty()); diff --git a/effect_chain.h b/effect_chain.h index 2f088b6..d1ab0a2 100644 --- a/effect_chain.h +++ b/effect_chain.h @@ -29,6 +29,7 @@ #include #include "image_format.h" +#include "ycbcr.h" namespace movit { @@ -162,8 +163,15 @@ public: } Effect *add_effect(Effect *effect, const std::vector &inputs); + // Adds an RGB output. Note that you can only have one output. void add_output(const ImageFormat &format, OutputAlphaFormat alpha_format); + // Adds an YCbCr output. Note that you can only have one output. + // Currently, only chunked packed output is supported, and only 4:4:4 + // (so chroma_subsampling_x and chroma_subsampling_y must both be 1). + void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, + const YCbCrFormat &ycbcr_format); + // Set number of output bits, to scale the dither. // 8 is the right value for most outputs. // The default, 0, is a special value that means no dither. @@ -303,12 +311,17 @@ private: void fix_internal_gamma_by_asking_inputs(unsigned step); void fix_internal_gamma_by_inserting_nodes(unsigned step); void fix_output_gamma(); + void add_ycbcr_conversion_if_needed(); void add_dither_if_needed(); float aspect_nom, aspect_denom; ImageFormat output_format; OutputAlphaFormat output_alpha_format; + enum OutputColorType { OUTPUT_COLOR_RGB, OUTPUT_COLOR_YCBCR }; + OutputColorType output_color_type; + YCbCrFormat output_ycbcr_format; // If output_color_type == OUTPUT_COLOR_YCBCR. + std::vector nodes; std::map node_map; Effect *dither_effect; diff --git a/test_util.cpp b/test_util.cpp index a3ccb5c..2073bc2 100644 --- a/test_util.cpp +++ b/test_util.cpp @@ -47,7 +47,7 @@ void vertical_flip(T *data, unsigned width, unsigned height) EffectChainTester::EffectChainTester(const float *data, unsigned width, unsigned height, MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve, GLenum framebuffer_format) - : chain(width, height, get_static_pool()), width(width), height(height), finalized(false) + : chain(width, height, get_static_pool()), width(width), height(height), output_added(false), finalized(false) { CHECK(init_movit(".", MOVIT_DEBUG_OFF)); @@ -215,13 +215,28 @@ void EffectChainTester::run(unsigned char *out_data, GLenum format, Colorspace c vertical_flip(out_data, width, height); } +void EffectChainTester::add_output(const ImageFormat &format, OutputAlphaFormat alpha_format) +{ + chain.add_output(format, alpha_format); + output_added = true; +} + +void EffectChainTester::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, const YCbCrFormat &ycbcr_format) +{ + chain.add_ycbcr_output(format, alpha_format, ycbcr_format); + output_added = true; +} + void EffectChainTester::finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format) { assert(!finalized); - ImageFormat image_format; - image_format.color_space = color_space; - image_format.gamma_curve = gamma_curve; - chain.add_output(image_format, alpha_format); + if (!output_added) { + ImageFormat image_format; + image_format.color_space = color_space; + image_format.gamma_curve = gamma_curve; + chain.add_output(image_format, alpha_format); + output_added = true; + } chain.finalize(); finalized = true; } diff --git a/test_util.h b/test_util.h index e5e6551..f413a39 100644 --- a/test_util.h +++ b/test_util.h @@ -23,6 +23,8 @@ public: Input *add_input(const unsigned char *data, MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve, int input_width = -1, int input_height = -1); void run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); void run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); + void add_output(const ImageFormat &format, OutputAlphaFormat alpha_format); + void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, const YCbCrFormat &ycbcr_format); private: void finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format); @@ -30,6 +32,7 @@ private: EffectChain chain; GLuint fbo, texnum; unsigned width, height; + bool output_added; bool finalized; }; diff --git a/ycbcr.h b/ycbcr.h index 9179b19..4eb9e73 100644 --- a/ycbcr.h +++ b/ycbcr.h @@ -1,7 +1,8 @@ #ifndef _MOVIT_YCBCR_H #define _MOVIT_YCBCR_H 1 -// Shared utility functions between YCbCrInput and YCbCr422InterleavedInput. +// Shared utility functions between YCbCrInput, YCbCr422InterleavedInput +// and YCbCrConversionEffect. // // Conversion from integer to floating-point representation in case of // Y'CbCr is seemingly tricky: diff --git a/ycbcr_conversion_effect.cpp b/ycbcr_conversion_effect.cpp new file mode 100644 index 0000000..74ca789 --- /dev/null +++ b/ycbcr_conversion_effect.cpp @@ -0,0 +1,47 @@ +#include +#include +#include +#include +#include +#include + +#include "ycbcr_conversion_effect.h" +#include "effect_util.h" +#include "util.h" +#include "ycbcr.h" + +using namespace std; +using namespace Eigen; + +namespace movit { + +YCbCrConversionEffect::YCbCrConversionEffect(const YCbCrFormat &ycbcr_format) + : ycbcr_format(ycbcr_format) +{ +} + +string YCbCrConversionEffect::output_fragment_shader() +{ + float offset[3]; + Matrix3d ycbcr_to_rgb; + compute_ycbcr_matrix(ycbcr_format, offset, &ycbcr_to_rgb); + + string frag_shader = output_glsl_mat3("PREFIX(ycbcr_matrix)", ycbcr_to_rgb.inverse()); + frag_shader += output_glsl_vec3("PREFIX(offset)", offset[0], offset[1], offset[2]); + + if (ycbcr_format.full_range) { + // The card will clamp for us later. + frag_shader += "#define YCBCR_CLAMP_RANGE 0\n"; + } else { + frag_shader += "#define YCBCR_CLAMP_RANGE 1\n"; + + // These limits come from BT.601 page 8, or BT.701, page 5. + // TODO: Use num_levels. Currently we support 8-bit levels only. + frag_shader += output_glsl_vec3("PREFIX(ycbcr_min)", 16.0 / 255.0, 16.0 / 255.0, 16.0 / 255.0); + frag_shader += output_glsl_vec3("PREFIX(ycbcr_max)", 235.0 / 255.0, 240.0 / 255.0, 240.0 / 255.0); + } + + return frag_shader + read_file("ycbcr_conversion_effect.frag"); +} + +} // namespace movit diff --git a/ycbcr_conversion_effect.frag b/ycbcr_conversion_effect.frag new file mode 100644 index 0000000..6bc29b1 --- /dev/null +++ b/ycbcr_conversion_effect.frag @@ -0,0 +1,24 @@ +uniform sampler2D PREFIX(tex_y); +uniform sampler2D PREFIX(tex_cb); +uniform sampler2D PREFIX(tex_cr); + +vec4 FUNCNAME(vec2 tc) { + vec4 rgba = INPUT(tc); + vec4 ycbcr_a; + + ycbcr_a.rgb = PREFIX(ycbcr_matrix) * rgba.rgb + PREFIX(offset); + +#if YCBCR_CLAMP_RANGE + // If we use limited-range Y'CbCr, the card's usual 0–255 clamping + // won't be enough, so we need to clamp ourselves here. + // + // We clamp before dither, which is a bit unfortunate, since + // it means dither can take us out of the clamped range again. + // However, since DitherEffect never adds enough dither to change + // the quantized levels, we will be fine in practice. + ycbcr_a.rgb = clamp(ycbcr_a.rgb, PREFIX(ycbcr_min), PREFIX(ycbcr_max)); +#endif + + ycbcr_a.a = rgba.a; + return ycbcr_a; +} diff --git a/ycbcr_conversion_effect.h b/ycbcr_conversion_effect.h new file mode 100644 index 0000000..46113bf --- /dev/null +++ b/ycbcr_conversion_effect.h @@ -0,0 +1,35 @@ +#ifndef _MOVIT_YCBCR_CONVERSION_EFFECT_H +#define _MOVIT_YCBCR_CONVERSION_EFFECT_H 1 + +// Converts from R'G'B' to Y'CbCr; that is, more or less the opposite of YCbCrInput, +// except that it keeps the data as 4:4:4 chunked Y'CbCr; you'll need to subsample +// and/or convert to planar somehow else. + +#include +#include + +#include "effect.h" +#include "ycbcr.h" + +namespace movit { + +class YCbCrConversionEffect : public Effect { +private: + // Should not be instantiated by end users; + // call EffectChain::add_ycbcr_output() instead. + YCbCrConversionEffect(const YCbCrFormat &ycbcr_format); + friend class EffectChain; + +public: + virtual std::string effect_type_id() const { return "YCbCrConversionEffect"; } + std::string output_fragment_shader(); + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } + virtual bool one_to_one_sampling() const { return true; } + +private: + YCbCrFormat ycbcr_format; +}; + +} // namespace movit + +#endif // !defined(_MOVIT_YCBCR_CONVERSION_EFFECT_H) diff --git a/ycbcr_conversion_effect_test.cpp b/ycbcr_conversion_effect_test.cpp new file mode 100644 index 0000000..c067baf --- /dev/null +++ b/ycbcr_conversion_effect_test.cpp @@ -0,0 +1,185 @@ +// Unit tests for YCbCrConversionEffect. Mostly done by leveraging +// YCbCrInput and seeing that the right thing comes out at the +// other end. + +#include +#include + +#include "effect_chain.h" +#include "gtest/gtest.h" +#include "image_format.h" +#include "test_util.h" +#include "util.h" +#include "ycbcr_input.h" + +namespace movit { + +TEST(YCbCrConversionEffectTest, BasicInOut) { + const int width = 1; + const int height = 5; + + // Pure-color test inputs, calculated with the formulas in Rec. 601 + // section 2.5.4. + unsigned char y[width * height] = { + 16, 235, 81, 145, 41, + }; + unsigned char cb[width * height] = { + 128, 128, 90, 54, 240, + }; + unsigned char cr[width * height] = { + 128, 128, 240, 34, 110, + }; + unsigned char expected_data[width * height * 4] = { + // The same data, just rearranged. + 16, 128, 128, 255, + 235, 128, 128, 255, + 81, 90, 240, 255, + 145, 54, 34, 255, + 41, 240, 110, 255 + }; + + unsigned char out_data[width * height * 4]; + + EffectChainTester tester(NULL, width, height); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat ycbcr_format; + ycbcr_format.luma_coefficients = YCBCR_REC_601; + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 1; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.5f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_x_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format); + + YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height); + input->set_pixel_data(0, y); + input->set_pixel_data(1, cb); + input->set_pixel_data(2, cr); + tester.get_chain()->add_input(input); + + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + expect_equal(expected_data, out_data, 4 * width, height); +} + +TEST(YCbCrConversionEffectTest, ClampToValidRange) { + const int width = 1; + const int height = 6; + + // Some out-of-range of at-range values. + // Y should be clamped to 16-235 and Cb/Cr to 16-240. + // (Alpha should still be 255.) + unsigned char y[width * height] = { + 0, 10, 16, 235, 240, 255 + }; + unsigned char cb[width * height] = { + 0, 10, 16, 235, 240, 255, + }; + unsigned char cr[width * height] = { + 255, 240, 235, 16, 10, 0, + }; + unsigned char expected_data[width * height * 4] = { + 16, 16, 240, 255, + 16, 16, 240, 255, + 16, 16, 235, 255, + 235, 235, 16, 255, + 235, 240, 16, 255, + 235, 240, 16, 255, + }; + + unsigned char out_data[width * height * 4]; + + EffectChainTester tester(NULL, width, height); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat ycbcr_format; + ycbcr_format.luma_coefficients = YCBCR_REC_601; + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 1; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.5f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_x_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format); + + YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height); + input->set_pixel_data(0, y); + input->set_pixel_data(1, cb); + input->set_pixel_data(2, cr); + tester.get_chain()->add_input(input); + + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + expect_equal(expected_data, out_data, 4 * width, height); +} + +TEST(YCbCrConversionEffectTest, LimitedRangeToFullRange) { + const int width = 1; + const int height = 5; + + // Pure-color test inputs, calculated with the formulas in Rec. 601 + // section 2.5.4. + unsigned char y[width * height] = { + 16, 235, 81, 145, 41, + }; + unsigned char cb[width * height] = { + 128, 128, 90, 54, 240, + }; + unsigned char cr[width * height] = { + 128, 128, 240, 34, 110, + }; + unsigned char expected_data[width * height * 4] = { + // Range now from 0-255 for all components, and values in-between + // also adjusted a bit. + 0, 128, 128, 255, + 255, 128, 128, 255, + 76, 85, 255, 255, + 150, 44, 21, 255, + 29, 255, 107, 255 + }; + + unsigned char out_data[width * height * 4]; + + EffectChainTester tester(NULL, width, height); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat ycbcr_format; + ycbcr_format.luma_coefficients = YCBCR_REC_601; + ycbcr_format.full_range = true; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 1; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.5f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_x_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format); + + ycbcr_format.full_range = false; + YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height); + input->set_pixel_data(0, y); + input->set_pixel_data(1, cb); + input->set_pixel_data(2, cr); + tester.get_chain()->add_input(input); + + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + expect_equal(expected_data, out_data, 4 * width, height); +} + +} // namespace movit -- 2.39.2