From 06ba8d86c019208682d8883dc7187df37b027814 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Wed, 16 Sep 2015 23:51:30 +0200 Subject: [PATCH] Add support for Y'CbCr output split between multiple textures. This is useful primarily for avoiding copies in later stages; e.g., when rendering directly into a video encoder buffer. We support both full planar and NV12-style interleaved Cb+Cr. You still have to subsample chroma yourself, though; we don't really support chains that diverge except in the final output node (and changing resolution would definitely need a bounce; and even worse, one in a non-fp16 intermediate format). --- effect_chain.cpp | 20 +++++- effect_chain.h | 34 ++++++++- footer.130.frag | 21 +++++- footer.300es.frag | 21 +++++- footer.frag | 12 +++- version.h | 2 +- ycbcr_conversion_effect_test.cpp | 118 +++++++++++++++++++++++++++++++ 7 files changed, 221 insertions(+), 7 deletions(-) diff --git a/effect_chain.cpp b/effect_chain.cpp index bf01716..4c2df4c 100644 --- a/effect_chain.cpp +++ b/effect_chain.cpp @@ -82,13 +82,14 @@ void EffectChain::add_output(const ImageFormat &format, OutputAlphaFormat alpha_ } void EffectChain::add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, - const YCbCrFormat &ycbcr_format) + const YCbCrFormat &ycbcr_format, YCbCrOutputSplitting output_splitting) { assert(!finalized); output_format = format; output_alpha_format = alpha_format; output_color_type = OUTPUT_COLOR_YCBCR; output_ycbcr_format = ycbcr_format; + output_ycbcr_splitting = output_splitting; assert(ycbcr_format.chroma_subsampling_x == 1); assert(ycbcr_format.chroma_subsampling_y == 1); @@ -361,6 +362,23 @@ void EffectChain::compile_glsl_program(Phase *phase) frag_shader += "\n"; } frag_shader += string("#define INPUT ") + phase->effect_ids[phase->effects.back()] + "\n"; + + // If we're the last phase, add the right #defines for Y'CbCr multi-output as needed. + if (phase->output_node->outgoing_links.empty() && output_color_type == OUTPUT_COLOR_YCBCR) { + switch (output_ycbcr_splitting) { + case YCBCR_OUTPUT_INTERLEAVED: + // No #defines set. + break; + case YCBCR_OUTPUT_SPLIT_Y_AND_CBCR: + frag_shader += "#define YCBCR_OUTPUT_SPLIT_Y_AND_CBCR 1\n"; + break; + case YCBCR_OUTPUT_PLANAR: + frag_shader += "#define YCBCR_OUTPUT_PLANAR 1\n"; + break; + default: + assert(false); + } + } frag_shader.append(read_version_dependent_file("footer", "frag")); // Collect uniforms from all effects and output them. Note that this needs diff --git a/effect_chain.h b/effect_chain.h index 1be9891..e07c2e9 100644 --- a/effect_chain.h +++ b/effect_chain.h @@ -55,6 +55,34 @@ enum OutputAlphaFormat { OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, }; +// RGBA output is nearly always packed; Y'CbCr, however, is often planar +// due to chroma subsampling. This enum controls how add_ycbcr_output() +// distributes the color channels between the fragment shader outputs. +// Obviously, anything except YCBCR_OUTPUT_INTERLEAVED will be meaningless +// unless you use render_to_fbo() and have an FBO with multiple render +// targets attached (the other outputs will be discarded). +enum YCbCrOutputSplitting { + // Only one output: Store Y'CbCr into the first three output channels, + // respectively, plus alpha. This is also called “chunked” or + // ”packed” mode. + YCBCR_OUTPUT_INTERLEAVED, + + // Store Y' and alpha into the first output (in the red and alpha + // channels; effect to the others is undefined), and Cb and Cr into + // the first two channels of the second output. This is particularly + // useful if you want to end up in a format like NV12, where all the + // Y' samples come first and then Cb and Cr come interlevaed afterwards. + // You will still need to do the chroma subsampling yourself to actually + // get down to NV12, though. + YCBCR_OUTPUT_SPLIT_Y_AND_CBCR, + + // Store Y' and alpha into the first output, Cb into the first channel + // of the second output and Cr into the first channel of the third output. + // (Effect on the other channels is undefined.) Essentially gives you + // 4:4:4 planar, or ”yuv444p”. + YCBCR_OUTPUT_PLANAR, +}; + // A node in the graph; basically an effect and some associated information. class Node { public: @@ -186,7 +214,8 @@ public: // Currently, only chunked packed output is supported, and only 4:4:4 // (so chroma_subsampling_x and chroma_subsampling_y must both be 1). void add_ycbcr_output(const ImageFormat &format, OutputAlphaFormat alpha_format, - const YCbCrFormat &ycbcr_format); + const YCbCrFormat &ycbcr_format, + YCbCrOutputSplitting output_splitting = YCBCR_OUTPUT_INTERLEAVED); // Set number of output bits, to scale the dither. // 8 is the right value for most outputs. @@ -338,7 +367,8 @@ private: enum OutputColorType { OUTPUT_COLOR_RGB, OUTPUT_COLOR_YCBCR }; OutputColorType output_color_type; - YCbCrFormat output_ycbcr_format; // If output_color_type == OUTPUT_COLOR_YCBCR. + YCbCrFormat output_ycbcr_format; // If output_color_type == OUTPUT_COLOR_YCBCR. + YCbCrOutputSplitting output_ycbcr_splitting; // If output_color_type == OUTPUT_COLOR_YCBCR. std::vector nodes; std::map node_map; diff --git a/footer.130.frag b/footer.130.frag index 83f615f..9921b34 100644 --- a/footer.130.frag +++ b/footer.130.frag @@ -1,6 +1,25 @@ +#if YCBCR_OUTPUT_PLANAR +out vec4 Y; +out vec4 Cb; +out vec4 Cr; +#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR +out vec4 Y; +out vec4 Chroma; +#else out vec4 FragColor; +#endif void main() { - FragColor = INPUT(tc); + vec4 color = INPUT(tc); +#if YCBCR_OUTPUT_PLANAR + Y = color.rrra; + Cb = color.ggga; + Cr = color.bbba; +#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR + Y = color.rrra; + Chroma = color.gbba; +#else + FragColor = color; +#endif } diff --git a/footer.300es.frag b/footer.300es.frag index 83f615f..9921b34 100644 --- a/footer.300es.frag +++ b/footer.300es.frag @@ -1,6 +1,25 @@ +#if YCBCR_OUTPUT_PLANAR +out vec4 Y; +out vec4 Cb; +out vec4 Cr; +#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR +out vec4 Y; +out vec4 Chroma; +#else out vec4 FragColor; +#endif void main() { - FragColor = INPUT(tc); + vec4 color = INPUT(tc); +#if YCBCR_OUTPUT_PLANAR + Y = color.rrra; + Cb = color.ggga; + Cr = color.bbba; +#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR + Y = color.rrra; + Chroma = color.gbba; +#else + FragColor = color; +#endif } diff --git a/footer.frag b/footer.frag index e41e83c..c140b46 100644 --- a/footer.frag +++ b/footer.frag @@ -1,4 +1,14 @@ void main() { - gl_FragColor = INPUT(tc); + vec4 color = INPUT(tc); +#if YCBCR_OUTPUT_PLANAR + gl_FragData[0] = color.rrra; + gl_FragData[1] = color.ggga; + gl_FragData[2] = color.bbba; +#elif YCBCR_OUTPUT_SPLIT_Y_AND_CBCR + gl_FragData[0] = color.rrra; + gl_FragData[1] = color.gbba; +#else + gl_FragColor = color; +#endif } diff --git a/version.h b/version.h index 4d645f4..d1a773a 100644 --- a/version.h +++ b/version.h @@ -5,6 +5,6 @@ // changes, even within git versions. There is no specific version // documentation outside the regular changelogs, though. -#define MOVIT_VERSION 3 +#define MOVIT_VERSION 4 #endif // !defined(_MOVIT_VERSION_H) diff --git a/ycbcr_conversion_effect_test.cpp b/ycbcr_conversion_effect_test.cpp index c067baf..d2b5d5c 100644 --- a/ycbcr_conversion_effect_test.cpp +++ b/ycbcr_conversion_effect_test.cpp @@ -182,4 +182,122 @@ TEST(YCbCrConversionEffectTest, LimitedRangeToFullRange) { expect_equal(expected_data, out_data, 4 * width, height); } +TEST(YCbCrConversionEffectTest, PlanarOutput) { + const int width = 1; + const int height = 5; + + // Pure-color test inputs, calculated with the formulas in Rec. 601 + // section 2.5.4. + unsigned char y[width * height] = { + 16, 235, 81, 145, 41, + }; + unsigned char cb[width * height] = { + 128, 128, 90, 54, 240, + }; + unsigned char cr[width * height] = { + 128, 128, 240, 34, 110, + }; + + unsigned char out_y[width * height], out_cb[width * height], out_cr[width * height]; + + EffectChainTester tester(NULL, width, height); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat ycbcr_format; + ycbcr_format.luma_coefficients = YCBCR_REC_601; + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 1; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.5f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_x_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format, YCBCR_OUTPUT_PLANAR); + + YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height); + input->set_pixel_data(0, y); + input->set_pixel_data(1, cb); + input->set_pixel_data(2, cr); + tester.get_chain()->add_input(input); + + tester.run(out_y, out_cb, out_cr, GL_RED, COLORSPACE_sRGB, GAMMA_sRGB); + expect_equal(y, out_y, width, height); + expect_equal(cb, out_cb, width, height); + expect_equal(cr, out_cr, width, height); +} + +TEST(YCbCrConversionEffectTest, SplitLumaAndChroma) { + const int width = 1; + const int height = 5; + + // Pure-color test inputs, calculated with the formulas in Rec. 601 + // section 2.5.4. + unsigned char y[width * height] = { + 16, 235, 81, 145, 41, + }; + unsigned char cb[width * height] = { + 128, 128, 90, 54, 240, + }; + unsigned char cr[width * height] = { + 128, 128, 240, 34, 110, + }; + + // The R and A data, rearranged. Note: The G and B channels + // (the middle columns) are undefined. If we change the behavior, + // the test will need to be updated, but a failure is expected. + unsigned char expected_y[width * height * 4] = { + 16, /*undefined:*/ 16, /*undefined:*/ 16, 255, + 235, /*undefined:*/ 235, /*undefined:*/ 235, 255, + 81, /*undefined:*/ 81, /*undefined:*/ 81, 255, + 145, /*undefined:*/ 145, /*undefined:*/ 145, 255, + 41, /*undefined:*/ 41, /*undefined:*/ 41, 255, + }; + + // Just the Cb and Cr data, rearranged. The B and A channels + // are undefined, as below. + unsigned char expected_cbcr[width * height * 4] = { + 128, 128, /*undefined:*/ 128, /*undefined:*/ 255, + 128, 128, /*undefined:*/ 128, /*undefined:*/ 255, + 90, 240, /*undefined:*/ 240, /*undefined:*/ 255, + 54, 34, /*undefined:*/ 34, /*undefined:*/ 255, + 240, 110, /*undefined:*/ 110, /*undefined:*/ 255, + }; + + unsigned char out_y[width * height], out_cbcr[width * height * 4]; + + EffectChainTester tester(NULL, width, height); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_sRGB; + + YCbCrFormat ycbcr_format; + ycbcr_format.luma_coefficients = YCBCR_REC_601; + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 1; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.5f; + ycbcr_format.cb_y_position = 0.5f; + ycbcr_format.cr_x_position = 0.5f; + ycbcr_format.cr_y_position = 0.5f; + + tester.add_ycbcr_output(format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR); + + YCbCrInput *input = new YCbCrInput(format, ycbcr_format, width, height); + input->set_pixel_data(0, y); + input->set_pixel_data(1, cb); + input->set_pixel_data(2, cr); + tester.get_chain()->add_input(input); + + tester.run(out_y, out_cbcr, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + expect_equal(expected_y, out_y, width * 4, height); + expect_equal(expected_cbcr, out_cbcr, width * 4, height); +} + } // namespace movit -- 2.39.2