1 #include "ycbcr_converter.h"
3 #include "exif_parser.h"
5 #include "jpeg_frame.h"
8 #include <movit/mix_effect.h>
9 #include <movit/white_balance_effect.h>
10 #include <movit/ycbcr_input.h>
12 using namespace movit;
17 void setup_outputs(YCbCrConverter::OutputMode output_mode, const ImageFormat &output_format, const YCbCrFormat &ycbcr_output_format, EffectChain *chain)
19 if (output_mode == YCbCrConverter::OUTPUT_TO_RGBA) {
20 chain->add_output(output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
21 chain->set_output_origin(OUTPUT_ORIGIN_BOTTOM_LEFT);
22 } else if (output_mode == YCbCrConverter::OUTPUT_TO_SEMIPLANAR) {
23 chain->add_ycbcr_output(output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_output_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR);
24 chain->set_output_origin(OUTPUT_ORIGIN_TOP_LEFT);
26 assert(output_mode == YCbCrConverter::OUTPUT_TO_DUAL_YCBCR);
28 // One full Y'CbCr texture (for interpolation), one that's just Y (throwing away the
29 // Cb and Cr channels). The second copy is sort of redundant, but it's the easiest way
30 // of getting the gray data into a layered texture.
31 chain->add_ycbcr_output(output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_output_format);
32 chain->add_ycbcr_output(output_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_output_format);
33 chain->set_output_origin(OUTPUT_ORIGIN_TOP_LEFT);
39 YCbCrConverter::YCbCrConverter(YCbCrConverter::OutputMode output_mode, ResourcePool *resource_pool)
41 ImageFormat inout_format;
42 inout_format.color_space = COLORSPACE_sRGB;
43 inout_format.gamma_curve = GAMMA_sRGB;
45 ycbcr_format.luma_coefficients = YCBCR_REC_709;
46 ycbcr_format.num_levels = 256;
47 ycbcr_format.chroma_subsampling_x = 2;
48 ycbcr_format.chroma_subsampling_y = 1;
49 ycbcr_format.cb_x_position = 0.0f; // H.264 -- _not_ JPEG, even though our input is MJPEG-encoded
50 ycbcr_format.cb_y_position = 0.5f; // Irrelevant.
51 ycbcr_format.cr_x_position = 0.0f;
52 ycbcr_format.cr_y_position = 0.5f;
54 // This is a hack. Even though we're sending MJPEG around, which is
55 // full-range, it's mostly transporting signals from limited-range
56 // sources with no conversion, so we ought to have had false here.
57 // However, in the off chance that we're actually getting real MJPEG,
58 // we don't want to crush its blacks (or whites) by clamping. All of
59 // our processing is fades or other linear operations, so if we're in
60 // limited-range input, we'll stay in limited-range output. (Fading
61 // between limited-range and full-range sources will be broken,
62 // of course.) There will be some slight confusion in the parts of the
63 // algorithms dealing with RGB, but they're small and we'll manage.
64 ycbcr_format.full_range = false;
66 YCbCrFormat ycbcr_output_format = ycbcr_format;
67 ycbcr_output_format.chroma_subsampling_x = 1;
69 // Planar Y'CbCr decoding chain.
70 planar_chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool));
71 ycbcr_planar_input = (YCbCrInput *)planar_chain->add_input(new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, YCBCR_INPUT_PLANAR));
72 setup_outputs(output_mode, inout_format, ycbcr_output_format, planar_chain.get());
73 planar_chain->set_dither_bits(8);
74 planar_chain->finalize();
76 // Semiplanar Y'CbCr decoding chain (for images coming from VA-API).
77 semiplanar_chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool));
78 ycbcr_semiplanar_input = (YCbCrInput *)semiplanar_chain->add_input(new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height, YCBCR_INPUT_SPLIT_Y_AND_CBCR));
79 setup_outputs(output_mode, inout_format, ycbcr_output_format, semiplanar_chain.get());
80 semiplanar_chain->set_dither_bits(8);
81 semiplanar_chain->finalize();
83 // Fade chains. These include white balance adjustments.
84 for (bool first_input_is_semiplanar : { false, true }) {
85 for (bool second_input_is_semiplanar : { false, true }) {
86 FadeChain &fade_chain = fade_chains[first_input_is_semiplanar][second_input_is_semiplanar];
87 fade_chain.chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool));
88 fade_chain.input[0] = (YCbCrInput *)fade_chain.chain->add_input(
89 new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height,
90 first_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR));
91 fade_chain.input[1] = (YCbCrInput *)fade_chain.chain->add_input(
92 new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height,
93 second_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR));
94 fade_chain.wb_effect[0] = (WhiteBalanceEffect *)fade_chain.chain->add_effect(
95 new WhiteBalanceEffect, fade_chain.input[0]);
96 fade_chain.wb_effect[1] = (WhiteBalanceEffect *)fade_chain.chain->add_effect(
97 new WhiteBalanceEffect, fade_chain.input[1]);
98 fade_chain.mix_effect = (MixEffect *)fade_chain.chain->add_effect(
99 new MixEffect, fade_chain.wb_effect[0], fade_chain.wb_effect[1]);
100 setup_outputs(output_mode, inout_format, ycbcr_output_format, fade_chain.chain.get());
101 fade_chain.chain->set_dither_bits(8);
102 fade_chain.chain->finalize();
106 // Fade from interleaved chain (ie., first input is interleaved, since it comes
107 // directly from the GPU anyway).
108 for (bool second_input_is_semiplanar : { false, true }) {
109 FadeChain &fade_chain = interleaved_fade_chains[second_input_is_semiplanar];
110 fade_chain.chain.reset(new EffectChain(global_flags.width, global_flags.height, resource_pool));
112 ycbcr_format.chroma_subsampling_x = 1;
113 fade_chain.input[0] = (YCbCrInput *)fade_chain.chain->add_input(
114 new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height,
115 YCBCR_INPUT_INTERLEAVED));
117 ycbcr_format.chroma_subsampling_x = 2;
118 fade_chain.input[1] = (YCbCrInput *)fade_chain.chain->add_input(
119 new YCbCrInput(inout_format, ycbcr_format, global_flags.width, global_flags.height,
120 second_input_is_semiplanar ? YCBCR_INPUT_SPLIT_Y_AND_CBCR : YCBCR_INPUT_PLANAR));
122 fade_chain.wb_effect[0] = (WhiteBalanceEffect *)fade_chain.chain->add_effect(
123 new WhiteBalanceEffect, fade_chain.input[0]);
124 fade_chain.wb_effect[1] = (WhiteBalanceEffect *)fade_chain.chain->add_effect(
125 new WhiteBalanceEffect, fade_chain.input[1]);
126 fade_chain.mix_effect = (MixEffect *)fade_chain.chain->add_effect(
127 new MixEffect, fade_chain.wb_effect[0], fade_chain.wb_effect[1]);
128 setup_outputs(output_mode, inout_format, ycbcr_output_format, fade_chain.chain.get());
129 fade_chain.chain->set_dither_bits(8);
130 fade_chain.chain->finalize();
134 EffectChain *YCbCrConverter::prepare_chain_for_conversion(shared_ptr<Frame> frame)
136 if (frame->is_semiplanar) {
137 setup_input_for_frame(frame, ycbcr_format, ycbcr_semiplanar_input);
138 return semiplanar_chain.get();
140 setup_input_for_frame(frame, ycbcr_format, ycbcr_planar_input);
141 return planar_chain.get();
145 EffectChain *YCbCrConverter::prepare_chain_for_fade(shared_ptr<Frame> frame, shared_ptr<Frame> secondary_frame, float fade_alpha)
147 const FadeChain &fade_chain = fade_chains[frame->is_semiplanar][secondary_frame->is_semiplanar];
148 setup_input_for_frame(frame, ycbcr_format, fade_chain.input[0]);
149 setup_input_for_frame(secondary_frame, ycbcr_format, fade_chain.input[1]);
150 bool ok = fade_chain.mix_effect->set_float("strength_first", 1.0f - fade_alpha);
151 ok |= fade_chain.mix_effect->set_float("strength_second", fade_alpha);
152 RGBTriplet neutral_color0 = get_neutral_color(frame->exif_data);
153 RGBTriplet neutral_color1 = get_neutral_color(secondary_frame->exif_data);
154 ok |= fade_chain.wb_effect[0]->set_vec3("neutral_color", (float *)&neutral_color0);
155 ok |= fade_chain.wb_effect[1]->set_vec3("neutral_color", (float *)&neutral_color1);
157 return fade_chain.chain.get();
160 EffectChain *YCbCrConverter::prepare_chain_for_fade_from_texture(GLuint tex, RGBTriplet tex_neutral_color, unsigned width, unsigned height, std::shared_ptr<Frame> secondary_frame, float fade_alpha)
162 const FadeChain &fade_chain = interleaved_fade_chains[secondary_frame->is_semiplanar];
164 YCbCrFormat format_copy = ycbcr_format;
165 format_copy.chroma_subsampling_x = 1;
166 format_copy.chroma_subsampling_y = 1;
167 fade_chain.input[0]->change_ycbcr_format(format_copy);
169 fade_chain.input[0]->set_width(width); // Doesn't really matter.
170 fade_chain.input[0]->set_height(height);
171 fade_chain.input[0]->set_texture_num(0, tex);
173 glTextureParameteri(tex, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
174 glTextureParameteri(tex, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
175 glTextureParameteri(tex, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
176 glTextureParameteri(tex, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
178 setup_input_for_frame(secondary_frame, ycbcr_format, fade_chain.input[1]);
179 bool ok = fade_chain.mix_effect->set_float("strength_first", 1.0f - fade_alpha);
180 ok |= fade_chain.mix_effect->set_float("strength_second", fade_alpha);
181 RGBTriplet neutral_color1 = get_neutral_color(secondary_frame->exif_data);
182 ok |= fade_chain.wb_effect[0]->set_vec3("neutral_color", (float *)&tex_neutral_color);
183 ok |= fade_chain.wb_effect[1]->set_vec3("neutral_color", (float *)&neutral_color1);
185 return fade_chain.chain.get();
188 void setup_input_for_frame(shared_ptr<Frame> frame, const YCbCrFormat &ycbcr_format, YCbCrInput *input)
190 YCbCrFormat format_copy = ycbcr_format;
191 format_copy.chroma_subsampling_x = frame->chroma_subsampling_x;
192 format_copy.chroma_subsampling_y = frame->chroma_subsampling_y;
193 input->change_ycbcr_format(format_copy);
195 input->set_width(frame->width);
196 input->set_height(frame->height);
197 input->set_texture_num(0, *frame->y);
198 if (frame->is_semiplanar) {
199 input->set_texture_num(1, *frame->cbcr);
201 input->set_texture_num(1, *frame->cb);
202 input->set_texture_num(2, *frame->cr);