Add support for Y'CbCr output split between multiple textures.
authorSteinar H. Gunderson <sgunderson@bigfoot.com>
Wed, 16 Sep 2015 21:51:30 +0000 (23:51 +0200)
committerSteinar H. Gunderson <sgunderson@bigfoot.com>
Wed, 16 Sep 2015 21:54:04 +0000 (23:54 +0200)
This is useful primarily for avoiding copies in later stages;
e.g., when rendering directly into a video encoder buffer.
We support both full planar and NV12-style interleaved Cb+Cr.
You still have to subsample chroma yourself, though; we don't
really support chains that diverge except in the final output node
(and changing resolution would definitely need a bounce;
and even worse, one in a non-fp16 intermediate format).

effect_chain.cpp
effect_chain.h
footer.130.frag
footer.300es.frag
footer.frag
version.h
ycbcr_conversion_effect_test.cpp

index bf01716..4c2df4c 100644 (file)
@@ -82,13 +82,14 @@ void EffectChain::add_output(const ImageFormat &format, OutputAlphaFormat alpha_
 }
 
 void EffectChain::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format,
-                                   const YCbCrFormat &ycbcr_format)
+                                   const YCbCrFormat &ycbcr_format, YCbCrOutputSplitting output_splitting)
 {
        assert(!finalized);
        output_format = format;
        output_alpha_format = alpha_format;
        output_color_type = OUTPUT_COLOR_YCBCR;
        output_ycbcr_format = ycbcr_format;
+       output_ycbcr_splitting = output_splitting;
 
        assert(ycbcr_format.chroma_subsampling_x == 1);
        assert(ycbcr_format.chroma_subsampling_y == 1);
@@ -361,6 +362,23 @@ void EffectChain::compile_glsl_program(Phase *phase)
                frag_shader += "\n";
        }
        frag_shader += string("#define INPUT ") + phase->effect_ids[phase->effects.back()] + "\n";
+
+       // If we're the last phase, add the right #defines for Y'CbCr multi-output as needed.
+       if (phase->output_node->outgoing_links.empty() && output_color_type == OUTPUT_COLOR_YCBCR) {
+               switch (output_ycbcr_splitting) {
+               case YCBCR_OUTPUT_INTERLEAVED:
+                       // No #defines set.
+                       break;
+               case YCBCR_OUTPUT_SPLIT_Y_AND_CBCR:
+                       frag_shader += "#define YCBCR_OUTPUT_SPLIT_Y_AND_CBCR 1\n";
+                       break;
+               case YCBCR_OUTPUT_PLANAR:
+                       frag_shader += "#define YCBCR_OUTPUT_PLANAR 1\n";
+                       break;
+               default:
+                       assert(false);
+               }
+       }
        frag_shader.append(read_version_dependent_file("footer", "frag"));
 
        // Collect uniforms from all effects and output them. Note that this needs
index 1be9891..e07c2e9 100644 (file)
@@ -55,6 +55,34 @@ enum OutputAlphaFormat {
        OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED,
 };
 
+// RGBA output is nearly always packed; Y'CbCr, however, is often planar
+// due to chroma subsampling. This enum controls how add_ycbcr_output()
+// distributes the color channels between the fragment shader outputs.
+// Obviously, anything except YCBCR_OUTPUT_INTERLEAVED will be meaningless
+// unless you use render_to_fbo() and have an FBO with multiple render
+// targets attached (the other outputs will be discarded).
+enum YCbCrOutputSplitting {
+       // Only one output: Store Y'CbCr into the first three output channels,
+       // respectively, plus alpha. This is also called “chunked” or
+       // ”packed” mode.
+       YCBCR_OUTPUT_INTERLEAVED,
+
+       // Store Y' and alpha into the first output (in the red and alpha
+       // channels; effect to the others is undefined), and Cb and Cr into
+       // the first two channels of the second output. This is particularly
+       // useful if you want to end up in a format like NV12, where all the
+       // Y' samples come first and then Cb and Cr come interlevaed afterwards.
+       // You will still need to do the chroma subsampling yourself to actually
+       // get down to NV12, though.
+       YCBCR_OUTPUT_SPLIT_Y_AND_CBCR,
+
+       // Store Y' and alpha into the first output, Cb into the first channel
+       // of the second output and Cr into the first channel of the third output.
+       // (Effect on the other channels is undefined.) Essentially gives you
+       // 4:4:4 planar, or ”yuv444p”.
+       YCBCR_OUTPUT_PLANAR,
+};
+
 // A node in the graph; basically an effect and some associated information.
 class Node {
 public:
@@ -186,7 +214,8 @@ public:
        // Currently, only chunked packed output is supported, and only 4:4:4
        // (so chroma_subsampling_x and chroma_subsampling_y must both be 1).
        void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format,
-                             const YCbCrFormat &ycbcr_format);
+                             const YCbCrFormat &ycbcr_format,
+                             YCbCrOutputSplitting output_splitting = YCBCR_OUTPUT_INTERLEAVED);
 
        // Set number of output bits, to scale the dither.
        // 8 is the right value for most outputs.
@@ -338,7 +367,8 @@ private:
 
        enum OutputColorType { OUTPUT_COLOR_RGB, OUTPUT_COLOR_YCBCR };
        OutputColorType output_color_type;
-       YCbCrFormat output_ycbcr_format;  // If output_color_type == OUTPUT_COLOR_YCBCR.
+       YCbCrFormat output_ycbcr_format;              // If output_color_type == OUTPUT_COLOR_YCBCR.
+       YCbCrOutputSplitting output_ycbcr_splitting;  // If output_color_type == OUTPUT_COLOR_YCBCR.
 
        std::vector<Node *> nodes;
        std::map<Effect *, Node *> node_map;
index 83f615f..9921b34 100644 (file)
@@ -1,6 +1,25 @@
+#if YCBCR_OUTPUT_PLANAR
+out vec4 Y;
+out vec4 Cb;
+out vec4 Cr;
+#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR
+out vec4 Y;
+out vec4 Chroma;
+#else
 out vec4 FragColor;
+#endif
 
 void main()
 {
-       FragColor = INPUT(tc);
+       vec4 color = INPUT(tc);
+#if YCBCR_OUTPUT_PLANAR
+       Y = color.rrra;
+       Cb = color.ggga;
+       Cr = color.bbba;
+#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR
+       Y = color.rrra;
+       Chroma = color.gbba;
+#else
+       FragColor = color;
+#endif
 }
index 83f615f..9921b34 100644 (file)
@@ -1,6 +1,25 @@
+#if YCBCR_OUTPUT_PLANAR
+out vec4 Y;
+out vec4 Cb;
+out vec4 Cr;
+#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR
+out vec4 Y;
+out vec4 Chroma;
+#else
 out vec4 FragColor;
+#endif
 
 void main()
 {
-       FragColor = INPUT(tc);
+       vec4 color = INPUT(tc);
+#if YCBCR_OUTPUT_PLANAR
+       Y = color.rrra;
+       Cb = color.ggga;
+       Cr = color.bbba;
+#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR
+       Y = color.rrra;
+       Chroma = color.gbba;
+#else
+       FragColor = color;
+#endif
 }
index e41e83c..c140b46 100644 (file)
@@ -1,4 +1,14 @@
 void main()
 {
-       gl_FragColor = INPUT(tc);
+       vec4 color = INPUT(tc);
+#if YCBCR_OUTPUT_PLANAR
+       gl_FragData[0] = color.rrra;
+       gl_FragData[1] = color.ggga;
+       gl_FragData[2] = color.bbba;
+#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR
+       gl_FragData[0] = color.rrra;
+       gl_FragData[1] = color.gbba;
+#else
+       gl_FragColor = color;
+#endif
 }
index 4d645f4..d1a773a 100644 (file)
--- a/version.h
+++ b/version.h
@@ -5,6 +5,6 @@
 // changes, even within git versions. There is no specific version
 // documentation outside the regular changelogs, though.
 
-#define MOVIT_VERSION 3
+#define MOVIT_VERSION 4
 
 #endif // !defined(_MOVIT_VERSION_H)
index c067baf..d2b5d5c 100644 (file)
@@ -182,4 +182,122 @@ TEST(YCbCrConversionEffectTest, LimitedRangeToFullRange) {
        expect_equal(expected_data, out_data, 4 * width, height);
 }
 
+TEST(YCbCrConversionEffectTest, PlanarOutput) {
+       const int width = 1;
+       const int height = 5;
+
+       // Pure-color test inputs, calculated with the formulas in Rec. 601
+       // section 2.5.4.
+       unsigned char y[width * height] = {
+               16, 235, 81, 145, 41,
+       };
+       unsigned char cb[width * height] = {
+               128, 128, 90, 54, 240,
+       };
+       unsigned char cr[width * height] = {
+               128, 128, 240, 34, 110,
+       };
+
+       unsigned char out_y[width * height], out_cb[width * height], out_cr[width * height];
+
+       EffectChainTester tester(NULL, width, height);
+
+       ImageFormat format;
+       format.color_space = COLORSPACE_sRGB;
+       format.gamma_curve = GAMMA_sRGB;
+
+       YCbCrFormat ycbcr_format;
+       ycbcr_format.luma_coefficients = YCBCR_REC_601;
+       ycbcr_format.full_range = false;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 1;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.5f;
+       ycbcr_format.cb_y_position = 0.5f;
+       ycbcr_format.cr_x_position = 0.5f;
+       ycbcr_format.cr_y_position = 0.5f;
+
+       tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format, YCBCR_OUTPUT_PLANAR);
+
+       YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height);
+       input->set_pixel_data(0, y);
+       input->set_pixel_data(1, cb);
+       input->set_pixel_data(2, cr);
+       tester.get_chain()->add_input(input);
+
+       tester.run(out_y, out_cb, out_cr, GL_RED, COLORSPACE_sRGB, GAMMA_sRGB);
+       expect_equal(y, out_y, width, height);
+       expect_equal(cb, out_cb, width, height);
+       expect_equal(cr, out_cr, width, height);
+}
+
+TEST(YCbCrConversionEffectTest, SplitLumaAndChroma) {
+       const int width = 1;
+       const int height = 5;
+
+       // Pure-color test inputs, calculated with the formulas in Rec. 601
+       // section 2.5.4.
+       unsigned char y[width * height] = {
+               16, 235, 81, 145, 41,
+       };
+       unsigned char cb[width * height] = {
+               128, 128, 90, 54, 240,
+       };
+       unsigned char cr[width * height] = {
+               128, 128, 240, 34, 110,
+       };
+
+       // The R and A data, rearranged. Note: The G and B channels
+       // (the middle columns) are undefined. If we change the behavior,
+       // the test will need to be updated, but a failure is expected.
+       unsigned char expected_y[width * height * 4] = {
+                16, /*undefined:*/  16, /*undefined:*/  16, 255,
+               235, /*undefined:*/ 235, /*undefined:*/ 235, 255,
+                81, /*undefined:*/  81, /*undefined:*/  81, 255,
+               145, /*undefined:*/ 145, /*undefined:*/ 145, 255,
+                41, /*undefined:*/  41, /*undefined:*/  41, 255,
+       };
+
+       // Just the Cb and Cr data, rearranged. The B and A channels
+       // are undefined, as below.
+       unsigned char expected_cbcr[width * height * 4] = {
+               128, 128, /*undefined:*/ 128, /*undefined:*/ 255,
+               128, 128, /*undefined:*/ 128, /*undefined:*/ 255,
+                90, 240, /*undefined:*/ 240, /*undefined:*/ 255,
+                54,  34, /*undefined:*/  34, /*undefined:*/ 255,
+               240, 110, /*undefined:*/ 110, /*undefined:*/ 255,
+       };
+
+       unsigned char out_y[width * height], out_cbcr[width * height * 4];
+
+       EffectChainTester tester(NULL, width, height);
+
+       ImageFormat format;
+       format.color_space = COLORSPACE_sRGB;
+       format.gamma_curve = GAMMA_sRGB;
+
+       YCbCrFormat ycbcr_format;
+       ycbcr_format.luma_coefficients = YCBCR_REC_601;
+       ycbcr_format.full_range = false;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 1;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.5f;
+       ycbcr_format.cb_y_position = 0.5f;
+       ycbcr_format.cr_x_position = 0.5f;
+       ycbcr_format.cr_y_position = 0.5f;
+
+       tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR);
+
+       YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height);
+       input->set_pixel_data(0, y);
+       input->set_pixel_data(1, cb);
+       input->set_pixel_data(2, cr);
+       tester.get_chain()->add_input(input);
+
+       tester.run(out_y, out_cbcr, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB);
+       expect_equal(expected_y, out_y, width * 4, height);
+       expect_equal(expected_cbcr, out_cbcr, width * 4, height);
+}
+
 }  // namespace movit