Add support for Y'CbCr output.
authorSteinar H. Gunderson <sgunderson@bigfoot.com>
Wed, 9 Sep 2015 21:51:48 +0000 (23:51 +0200)
committerSteinar H. Gunderson <sgunderson@bigfoot.com>
Wed, 9 Sep 2015 21:52:44 +0000 (23:52 +0200)
Currently only 8-bit and only 4:4:4 packed, but it should be a useful
building block.

.gitignore
Makefile.in
effect_chain.cpp
effect_chain.h
test_util.cpp
test_util.h
ycbcr.h
ycbcr_conversion_effect.cpp [new file with mode: 0644]
ycbcr_conversion_effect.frag [new file with mode: 0644]
ycbcr_conversion_effect.h [new file with mode: 0644]
ycbcr_conversion_effect_test.cpp [new file with mode: 0644]

index 366d38c..74045c5 100644 (file)
@@ -40,6 +40,7 @@ fp16_test
 luma_mix_effect_test
 slice_effect_test
 vignette_effect_test
+ycbcr_conversion_effect_test
 chain-*.frag
 movit.info
 coverage/
index 35fc71e..53eb6ff 100644 (file)
@@ -76,6 +76,7 @@ TESTED_EFFECTS += slice_effect
 TESTED_EFFECTS += complex_modulate_effect
 TESTED_EFFECTS += luma_mix_effect
 TESTED_EFFECTS += fft_convolution_effect
+TESTED_EFFECTS += ycbcr_conversion_effect
 
 UNTESTED_EFFECTS = sandbox_effect
 UNTESTED_EFFECTS += mirror_effect
index 519cb2c..e1ae832 100644 (file)
@@ -25,6 +25,7 @@
 #include "input.h"
 #include "resource_pool.h"
 #include "util.h"
+#include "ycbcr_conversion_effect.h"
 
 using namespace std;
 
@@ -74,6 +75,20 @@ void EffectChain::add_output(const ImageFormat &format, OutputAlphaFormat alpha_
        assert(!finalized);
        output_format = format;
        output_alpha_format = alpha_format;
+       output_color_type = OUTPUT_COLOR_RGB;
+}
+
+void EffectChain::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format,
+                                   const YCbCrFormat &ycbcr_format)
+{
+       assert(!finalized);
+       output_format = format;
+       output_alpha_format = alpha_format;
+       output_color_type = OUTPUT_COLOR_YCBCR;
+       output_ycbcr_format = ycbcr_format;
+
+       assert(ycbcr_format.chroma_subsampling_x == 1);
+       assert(ycbcr_format.chroma_subsampling_y == 1);
 }
 
 Node *EffectChain::add_node(Effect *effect)
@@ -1363,6 +1378,22 @@ void EffectChain::fix_output_gamma()
                connect_nodes(output, conversion);
        }
 }
+
+// If the user has requested Y'CbCr output, we need to do this conversion
+// _after_ GammaCompressionEffect etc., but before dither (see below).
+// This is because Y'CbCr, with the exception of a special optional mode
+// in Rec. 2020 (which we currently don't support), is defined to work on
+// gamma-encoded data.
+void EffectChain::add_ycbcr_conversion_if_needed()
+{
+       assert(output_color_type == OUTPUT_COLOR_RGB || output_color_type == OUTPUT_COLOR_YCBCR);
+       if (output_color_type != OUTPUT_COLOR_YCBCR) {
+               return;
+       }
+       Node *output = find_output_node();
+       Node *ycbcr = add_node(new YCbCrConversionEffect(output_ycbcr_format));
+       connect_nodes(output, ycbcr);
+}
        
 // If the user has requested dither, add a DitherEffect right at the end
 // (after GammaCompressionEffect etc.). This needs to be done after everything else,
@@ -1445,11 +1476,13 @@ void EffectChain::finalize()
        fix_internal_gamma_by_asking_inputs(15);
        fix_internal_gamma_by_inserting_nodes(16);
 
-       output_dot("step17-before-dither.dot");
+       output_dot("step17-before-ycbcr.dot");
+       add_ycbcr_conversion_if_needed();
 
+       output_dot("step18-before-dither.dot");
        add_dither_if_needed();
 
-       output_dot("step18-final.dot");
+       output_dot("step19-final.dot");
        
        // Construct all needed GLSL programs, starting at the output.
        // We need to keep track of which effects have already been computed,
@@ -1458,7 +1491,7 @@ void EffectChain::finalize()
        map<Node *, Phase *> completed_effects;
        construct_phase(find_output_node(), &completed_effects);
 
-       output_dot("step19-split-to-phases.dot");
+       output_dot("step20-split-to-phases.dot");
 
        assert(phases[0]->inputs.empty());
        
index 2f088b6..d1ab0a2 100644 (file)
@@ -29,6 +29,7 @@
 #include <vector>
 
 #include "image_format.h"
+#include "ycbcr.h"
 
 namespace movit {
 
@@ -162,8 +163,15 @@ public:
        }
        Effect *add_effect(Effect *effect, const std::vector<Effect *> &inputs);
 
+       // Adds an RGB output. Note that you can only have one output.
        void add_output(const ImageFormat &format, OutputAlphaFormat alpha_format);
 
+       // Adds an YCbCr output. Note that you can only have one output.
+       // Currently, only chunked packed output is supported, and only 4:4:4
+       // (so chroma_subsampling_x and chroma_subsampling_y must both be 1).
+       void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format,
+                             const YCbCrFormat &ycbcr_format);
+
        // Set number of output bits, to scale the dither.
        // 8 is the right value for most outputs.
        // The default, 0, is a special value that means no dither.
@@ -303,12 +311,17 @@ private:
        void fix_internal_gamma_by_asking_inputs(unsigned step);
        void fix_internal_gamma_by_inserting_nodes(unsigned step);
        void fix_output_gamma();
+       void add_ycbcr_conversion_if_needed();
        void add_dither_if_needed();
 
        float aspect_nom, aspect_denom;
        ImageFormat output_format;
        OutputAlphaFormat output_alpha_format;
 
+       enum OutputColorType { OUTPUT_COLOR_RGB, OUTPUT_COLOR_YCBCR };
+       OutputColorType output_color_type;
+       YCbCrFormat output_ycbcr_format;  // If output_color_type == OUTPUT_COLOR_YCBCR.
+
        std::vector<Node *> nodes;
        std::map<Effect *, Node *> node_map;
        Effect *dither_effect;
index a3ccb5c..2073bc2 100644 (file)
@@ -47,7 +47,7 @@ void vertical_flip(T *data, unsigned width, unsigned height)
 EffectChainTester::EffectChainTester(const float *data, unsigned width, unsigned height,
                                      MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve,
                                      GLenum framebuffer_format)
-       : chain(width, height, get_static_pool()), width(width), height(height), finalized(false)
+       : chain(width, height, get_static_pool()), width(width), height(height), output_added(false), finalized(false)
 {
        CHECK(init_movit(".", MOVIT_DEBUG_OFF));
 
@@ -215,13 +215,28 @@ void EffectChainTester::run(unsigned char *out_data, GLenum format, Colorspace c
        vertical_flip(out_data, width, height);
 }
 
+void EffectChainTester::add_output(const ImageFormat &format, OutputAlphaFormat alpha_format)
+{
+       chain.add_output(format, alpha_format);
+       output_added = true;
+}
+
+void EffectChainTester::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, const YCbCrFormat &ycbcr_format)
+{
+       chain.add_ycbcr_output(format, alpha_format, ycbcr_format);
+       output_added = true;
+}
+
 void EffectChainTester::finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format)
 {
        assert(!finalized);
-       ImageFormat image_format;
-       image_format.color_space = color_space;
-       image_format.gamma_curve = gamma_curve;
-       chain.add_output(image_format, alpha_format);
+       if (!output_added) {
+               ImageFormat image_format;
+               image_format.color_space = color_space;
+               image_format.gamma_curve = gamma_curve;
+               chain.add_output(image_format, alpha_format);
+               output_added = true;
+       }
        chain.finalize();
        finalized = true;
 }
index e5e6551..f413a39 100644 (file)
@@ -23,6 +23,8 @@ public:
        Input *add_input(const unsigned char *data, MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve, int input_width = -1, int input_height = -1);
        void run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
        void run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
+       void add_output(const ImageFormat &format, OutputAlphaFormat alpha_format);
+       void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, const YCbCrFormat &ycbcr_format);
 
 private:
        void finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format);
@@ -30,6 +32,7 @@ private:
        EffectChain chain;
        GLuint fbo, texnum;
        unsigned width, height;
+       bool output_added;
        bool finalized;
 };
 
diff --git a/ycbcr.h b/ycbcr.h
index 9179b19..4eb9e73 100644 (file)
--- a/ycbcr.h
+++ b/ycbcr.h
@@ -1,7 +1,8 @@
 #ifndef _MOVIT_YCBCR_H
 #define _MOVIT_YCBCR_H 1
 
-// Shared utility functions between YCbCrInput and YCbCr422InterleavedInput.
+// Shared utility functions between YCbCrInput, YCbCr422InterleavedInput
+// and YCbCrConversionEffect.
 //
 // Conversion from integer to floating-point representation in case of
 // Y'CbCr is seemingly tricky:
diff --git a/ycbcr_conversion_effect.cpp b/ycbcr_conversion_effect.cpp
new file mode 100644 (file)
index 0000000..74ca789
--- /dev/null
@@ -0,0 +1,47 @@
+#include <epoxy/gl.h>
+#include <assert.h>
+#include <stdio.h>
+#include <algorithm>
+#include <Eigen/Core>
+#include <Eigen/LU>
+
+#include "ycbcr_conversion_effect.h"
+#include "effect_util.h"
+#include "util.h"
+#include "ycbcr.h"
+
+using namespace std;
+using namespace Eigen;
+
+namespace movit {
+
+YCbCrConversionEffect::YCbCrConversionEffect(const YCbCrFormat &ycbcr_format)
+       : ycbcr_format(ycbcr_format)
+{
+}
+
+string YCbCrConversionEffect::output_fragment_shader()
+{
+       float offset[3];
+       Matrix3d ycbcr_to_rgb;
+       compute_ycbcr_matrix(ycbcr_format, offset, &ycbcr_to_rgb);
+
+        string frag_shader = output_glsl_mat3("PREFIX(ycbcr_matrix)", ycbcr_to_rgb.inverse());
+        frag_shader += output_glsl_vec3("PREFIX(offset)", offset[0], offset[1], offset[2]);
+
+       if (ycbcr_format.full_range) {
+               // The card will clamp for us later.
+               frag_shader += "#define YCBCR_CLAMP_RANGE 0\n";
+       } else {
+               frag_shader += "#define YCBCR_CLAMP_RANGE 1\n";
+
+               // These limits come from BT.601 page 8, or BT.701, page 5.
+               // TODO: Use num_levels. Currently we support 8-bit levels only.
+               frag_shader += output_glsl_vec3("PREFIX(ycbcr_min)", 16.0 / 255.0, 16.0 / 255.0, 16.0 / 255.0);
+               frag_shader += output_glsl_vec3("PREFIX(ycbcr_max)", 235.0 / 255.0, 240.0 / 255.0, 240.0 / 255.0);
+       }
+
+       return frag_shader + read_file("ycbcr_conversion_effect.frag");
+}
+
+}  // namespace movit
diff --git a/ycbcr_conversion_effect.frag b/ycbcr_conversion_effect.frag
new file mode 100644 (file)
index 0000000..6bc29b1
--- /dev/null
@@ -0,0 +1,24 @@
+uniform sampler2D PREFIX(tex_y);
+uniform sampler2D PREFIX(tex_cb);
+uniform sampler2D PREFIX(tex_cr);
+
+vec4 FUNCNAME(vec2 tc) {
+       vec4 rgba = INPUT(tc);
+       vec4 ycbcr_a;
+
+       ycbcr_a.rgb = PREFIX(ycbcr_matrix) * rgba.rgb + PREFIX(offset);
+
+#if YCBCR_CLAMP_RANGE
+       // If we use limited-range Y'CbCr, the card's usual 0–255 clamping
+       // won't be enough, so we need to clamp ourselves here.
+       //
+       // We clamp before dither, which is a bit unfortunate, since
+       // it means dither can take us out of the clamped range again.
+       // However, since DitherEffect never adds enough dither to change
+       // the quantized levels, we will be fine in practice.
+       ycbcr_a.rgb = clamp(ycbcr_a.rgb, PREFIX(ycbcr_min), PREFIX(ycbcr_max));
+#endif
+
+       ycbcr_a.a = rgba.a;
+       return ycbcr_a;
+}
diff --git a/ycbcr_conversion_effect.h b/ycbcr_conversion_effect.h
new file mode 100644 (file)
index 0000000..46113bf
--- /dev/null
@@ -0,0 +1,35 @@
+#ifndef _MOVIT_YCBCR_CONVERSION_EFFECT_H
+#define _MOVIT_YCBCR_CONVERSION_EFFECT_H 1
+
+// Converts from R'G'B' to Y'CbCr; that is, more or less the opposite of YCbCrInput,
+// except that it keeps the data as 4:4:4 chunked Y'CbCr; you'll need to subsample
+// and/or convert to planar somehow else.
+
+#include <epoxy/gl.h>
+#include <string>
+
+#include "effect.h"
+#include "ycbcr.h"
+
+namespace movit {
+
+class YCbCrConversionEffect : public Effect {
+private:
+       // Should not be instantiated by end users;
+       // call EffectChain::add_ycbcr_output() instead.
+       YCbCrConversionEffect(const YCbCrFormat &ycbcr_format);
+       friend class EffectChain;
+
+public:
+       virtual std::string effect_type_id() const { return "YCbCrConversionEffect"; }
+       std::string output_fragment_shader();
+       virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; }
+       virtual bool one_to_one_sampling() const { return true; }
+
+private:
+       YCbCrFormat ycbcr_format;
+};
+
+}  // namespace movit
+
+#endif // !defined(_MOVIT_YCBCR_CONVERSION_EFFECT_H)
diff --git a/ycbcr_conversion_effect_test.cpp b/ycbcr_conversion_effect_test.cpp
new file mode 100644 (file)
index 0000000..c067baf
--- /dev/null
@@ -0,0 +1,185 @@
+// Unit tests for YCbCrConversionEffect. Mostly done by leveraging
+// YCbCrInput and seeing that the right thing comes out at the
+// other end.
+
+#include <epoxy/gl.h>
+#include <math.h>
+
+#include "effect_chain.h"
+#include "gtest/gtest.h"
+#include "image_format.h"
+#include "test_util.h"
+#include "util.h"
+#include "ycbcr_input.h"
+
+namespace movit {
+
+TEST(YCbCrConversionEffectTest, BasicInOut) {
+       const int width = 1;
+       const int height = 5;
+
+       // Pure-color test inputs, calculated with the formulas in Rec. 601
+       // section 2.5.4.
+       unsigned char y[width * height] = {
+               16, 235, 81, 145, 41,
+       };
+       unsigned char cb[width * height] = {
+               128, 128, 90, 54, 240,
+       };
+       unsigned char cr[width * height] = {
+               128, 128, 240, 34, 110,
+       };
+       unsigned char expected_data[width * height * 4] = {
+               // The same data, just rearranged.
+                16, 128, 128, 255,
+               235, 128, 128, 255,
+                81,  90, 240, 255,
+               145,  54,  34, 255,
+                41, 240, 110, 255
+       };
+
+       unsigned char out_data[width * height * 4];
+
+       EffectChainTester tester(NULL, width, height);
+
+       ImageFormat format;
+       format.color_space = COLORSPACE_sRGB;
+       format.gamma_curve = GAMMA_sRGB;
+
+       YCbCrFormat ycbcr_format;
+       ycbcr_format.luma_coefficients = YCBCR_REC_601;
+       ycbcr_format.full_range = false;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 1;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.5f;
+       ycbcr_format.cb_y_position = 0.5f;
+       ycbcr_format.cr_x_position = 0.5f;
+       ycbcr_format.cr_y_position = 0.5f;
+
+       tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format);
+
+       YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height);
+       input->set_pixel_data(0, y);
+       input->set_pixel_data(1, cb);
+       input->set_pixel_data(2, cr);
+       tester.get_chain()->add_input(input);
+
+       tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB);
+       expect_equal(expected_data, out_data, 4 * width, height);
+}
+
+TEST(YCbCrConversionEffectTest, ClampToValidRange) {
+       const int width = 1;
+       const int height = 6;
+
+       // Some out-of-range of at-range values.
+       // Y should be clamped to 16-235 and Cb/Cr to 16-240.
+       // (Alpha should still be 255.)
+       unsigned char y[width * height] = {
+               0, 10, 16, 235, 240, 255
+       };
+       unsigned char cb[width * height] = {
+               0, 10, 16, 235, 240, 255,
+       };
+       unsigned char cr[width * height] = {
+               255, 240, 235, 16, 10, 0,
+       };
+       unsigned char expected_data[width * height * 4] = {
+               16, 16, 240, 255,
+               16, 16, 240, 255,
+               16, 16, 235, 255,
+               235, 235, 16, 255,
+               235, 240, 16, 255,
+               235, 240, 16, 255,
+       };
+
+       unsigned char out_data[width * height * 4];
+
+       EffectChainTester tester(NULL, width, height);
+
+       ImageFormat format;
+       format.color_space = COLORSPACE_sRGB;
+       format.gamma_curve = GAMMA_sRGB;
+
+       YCbCrFormat ycbcr_format;
+       ycbcr_format.luma_coefficients = YCBCR_REC_601;
+       ycbcr_format.full_range = false;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 1;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.5f;
+       ycbcr_format.cb_y_position = 0.5f;
+       ycbcr_format.cr_x_position = 0.5f;
+       ycbcr_format.cr_y_position = 0.5f;
+
+       tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format);
+
+       YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height);
+       input->set_pixel_data(0, y);
+       input->set_pixel_data(1, cb);
+       input->set_pixel_data(2, cr);
+       tester.get_chain()->add_input(input);
+
+       tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB);
+       expect_equal(expected_data, out_data, 4 * width, height);
+}
+
+TEST(YCbCrConversionEffectTest, LimitedRangeToFullRange) {
+       const int width = 1;
+       const int height = 5;
+
+       // Pure-color test inputs, calculated with the formulas in Rec. 601
+       // section 2.5.4.
+       unsigned char y[width * height] = {
+               16, 235, 81, 145, 41,
+       };
+       unsigned char cb[width * height] = {
+               128, 128, 90, 54, 240,
+       };
+       unsigned char cr[width * height] = {
+               128, 128, 240, 34, 110,
+       };
+       unsigned char expected_data[width * height * 4] = {
+               // Range now from 0-255 for all components, and values in-between
+               // also adjusted a bit.
+                 0, 128, 128, 255,
+               255, 128, 128, 255,
+                76,  85, 255, 255,
+               150,  44,  21, 255,
+                29, 255, 107, 255
+       };
+
+       unsigned char out_data[width * height * 4];
+
+       EffectChainTester tester(NULL, width, height);
+
+       ImageFormat format;
+       format.color_space = COLORSPACE_sRGB;
+       format.gamma_curve = GAMMA_sRGB;
+
+       YCbCrFormat ycbcr_format;
+       ycbcr_format.luma_coefficients = YCBCR_REC_601;
+       ycbcr_format.full_range = true;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 1;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.5f;
+       ycbcr_format.cb_y_position = 0.5f;
+       ycbcr_format.cr_x_position = 0.5f;
+       ycbcr_format.cr_y_position = 0.5f;
+
+       tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format);
+
+       ycbcr_format.full_range = false;
+       YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height);
+       input->set_pixel_data(0, y);
+       input->set_pixel_data(1, cb);
+       input->set_pixel_data(2, cr);
+       tester.get_chain()->add_input(input);
+
+       tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB);
+       expect_equal(expected_data, out_data, 4 * width, height);
+}
+
+}  // namespace movit