git.sesse.net Git - nageru/blob - chroma_subsampler.cpp

   1 #include "chroma_subsampler.h"
   2
   3 #include <vector>
   4
   5 #include <movit/effect_util.h>
   6 #include <movit/resource_pool.h>
   7 #include <movit/util.h>
   8
   9 using namespace movit;
  10 using namespace std;
  11
  12 ChromaSubsampler::ChromaSubsampler(ResourcePool *resource_pool)
  13         : resource_pool(resource_pool)
  14 {
  15         vector<string> frag_shader_outputs;
  16
  17         // Set up stuff for NV12 conversion.
  18         //
  19         // Note: Due to the horizontally co-sited chroma/luma samples in H.264
  20         // (chrome position is left for horizontal and center for vertical),
  21         // we need to be a bit careful in our subsampling. A diagram will make
  22         // this clearer, showing some luma and chroma samples:
  23         //
  24         //     a   b   c   d
  25         //   +---+---+---+---+
  26         //   |   |   |   |   |
  27         //   | Y | Y | Y | Y |
  28         //   |   |   |   |   |
  29         //   +---+---+---+---+
  30         //
  31         // +-------+-------+
  32         // |       |       |
  33         // |   C   |   C   |
  34         // |       |       |
  35         // +-------+-------+
  36         //
  37         // Clearly, the rightmost chroma sample here needs to be equivalent to
  38         // b/4 + c/2 + d/4. (We could also implement more sophisticated filters,
  39         // of course, but as long as the upsampling is not going to be equally
  40         // sophisticated, it's probably not worth it.) If we sample once with
  41         // no mipmapping, we get just c, ie., no actual filtering in the
  42         // horizontal direction. (For the vertical direction, we can just
  43         // sample in the middle to get the right filtering.) One could imagine
  44         // we could use mipmapping (assuming we can create mipmaps cheaply),
  45         // but then, what we'd get is this:
  46         //
  47         //    (a+b)/2 (c+d)/2
  48         //   +-------+-------+
  49         //   |       |       |
  50         //   |   Y   |   Y   |
  51         //   |       |       |
  52         //   +-------+-------+
  53         //
  54         // +-------+-------+
  55         // |       |       |
  56         // |   C   |   C   |
  57         // |       |       |
  58         // +-------+-------+
  59         //
  60         // which ends up sampling equally from a and b, which clearly isn't right. Instead,
  61         // we need to do two (non-mipmapped) chroma samples, both hitting exactly in-between
  62         // source pixels.
  63         //
  64         // Sampling in-between b and c gives us the sample (b+c)/2, and similarly for c and d.
  65         // Taking the average of these gives of (b+c)/4 + (c+d)/4 = b/4 + c/2 + d/4, which is
  66         // exactly what we want.
  67         //
  68         // See also http://www.poynton.com/PDFs/Merging_RGB_and_422.pdf, pages 6–7.
  69
  70         // Cb/Cr shader.
  71         string cbcr_vert_shader =
  72                 "#version 130 \n"
  73                 " \n"
  74                 "in vec2 position; \n"
  75                 "in vec2 texcoord; \n"
  76                 "out vec2 tc0, tc1; \n"
  77                 "uniform vec2 foo_chroma_offset_0; \n"
  78                 "uniform vec2 foo_chroma_offset_1; \n"
  79                 " \n"
  80                 "void main() \n"
  81                 "{ \n"
  82                 "    // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n"
  83                 "    // \n"
  84                 "    //   2.000  0.000  0.000 -1.000 \n"
  85                 "    //   0.000  2.000  0.000 -1.000 \n"
  86                 "    //   0.000  0.000 -2.000 -1.000 \n"
  87                 "    //   0.000  0.000  0.000  1.000 \n"
  88                 "    gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n"
  89                 "    vec2 flipped_tc = texcoord; \n"
  90                 "    tc0 = flipped_tc + foo_chroma_offset_0; \n"
  91                 "    tc1 = flipped_tc + foo_chroma_offset_1; \n"
  92                 "} \n";
  93         string cbcr_frag_shader =
  94                 "#version 130 \n"
  95                 "in vec2 tc0, tc1; \n"
  96                 "uniform sampler2D cbcr_tex; \n"
  97                 "out vec4 FragColor, FragColor2; \n"
  98                 "void main() { \n"
  99                 "    FragColor = 0.5 * (texture(cbcr_tex, tc0) + texture(cbcr_tex, tc1)); \n"
 100                 "    FragColor2 = FragColor; \n"
 101                 "} \n";
 102         cbcr_program_num = resource_pool->compile_glsl_program(cbcr_vert_shader, cbcr_frag_shader, frag_shader_outputs);
 103         check_error();
 104
 105         cbcr_texture_sampler_uniform = glGetUniformLocation(cbcr_program_num, "cbcr_tex");
 106         check_error();
 107         cbcr_position_attribute_index = glGetAttribLocation(cbcr_program_num, "position");
 108         check_error();
 109         cbcr_texcoord_attribute_index = glGetAttribLocation(cbcr_program_num, "texcoord");
 110         check_error();
 111
 112         // Same, for UYVY conversion.
 113         string uyvy_vert_shader =
 114                 "#version 130 \n"
 115                 " \n"
 116                 "in vec2 position; \n"
 117                 "in vec2 texcoord; \n"
 118                 "out vec2 y_tc0, y_tc1, cbcr_tc0, cbcr_tc1; \n"
 119                 "uniform vec2 foo_luma_offset_0; \n"
 120                 "uniform vec2 foo_luma_offset_1; \n"
 121                 "uniform vec2 foo_chroma_offset_0; \n"
 122                 "uniform vec2 foo_chroma_offset_1; \n"
 123                 " \n"
 124                 "void main() \n"
 125                 "{ \n"
 126                 "    // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n"
 127                 "    // \n"
 128                 "    //   2.000  0.000  0.000 -1.000 \n"
 129                 "    //   0.000  2.000  0.000 -1.000 \n"
 130                 "    //   0.000  0.000 -2.000 -1.000 \n"
 131                 "    //   0.000  0.000  0.000  1.000 \n"
 132                 "    gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n"
 133                 "    vec2 flipped_tc = texcoord; \n"
 134                 "    y_tc0 = flipped_tc + foo_luma_offset_0; \n"
 135                 "    y_tc1 = flipped_tc + foo_luma_offset_1; \n"
 136                 "    cbcr_tc0 = flipped_tc + foo_chroma_offset_0; \n"
 137                 "    cbcr_tc1 = flipped_tc + foo_chroma_offset_1; \n"
 138                 "} \n";
 139         string uyvy_frag_shader =
 140                 "#version 130 \n"
 141                 "in vec2 y_tc0, y_tc1, cbcr_tc0, cbcr_tc1; \n"
 142                 "uniform sampler2D y_tex, cbcr_tex; \n"
 143                 "out vec4 FragColor; \n"
 144                 "void main() { \n"
 145                 "    float y0 = texture(y_tex, y_tc0).r; \n"
 146                 "    float y1 = texture(y_tex, y_tc1).r; \n"
 147                 "    vec2 cbcr0 = texture(cbcr_tex, cbcr_tc0).rg; \n"
 148                 "    vec2 cbcr1 = texture(cbcr_tex, cbcr_tc1).rg; \n"
 149                 "    vec2 cbcr = 0.5 * (cbcr0 + cbcr1); \n"
 150                 "    FragColor = vec4(cbcr.g, y0, cbcr.r, y1); \n"
 151                 "} \n";
 152
 153         uyvy_program_num = resource_pool->compile_glsl_program(uyvy_vert_shader, uyvy_frag_shader, frag_shader_outputs);
 154         check_error();
 155
 156         uyvy_y_texture_sampler_uniform = glGetUniformLocation(uyvy_program_num, "y_tex");
 157         check_error();
 158         uyvy_cbcr_texture_sampler_uniform = glGetUniformLocation(uyvy_program_num, "cbcr_tex");
 159         check_error();
 160         uyvy_position_attribute_index = glGetAttribLocation(uyvy_program_num, "position");
 161         check_error();
 162         uyvy_texcoord_attribute_index = glGetAttribLocation(uyvy_program_num, "texcoord");
 163         check_error();
 164
 165         // Shared between the two.
 166         float vertices[] = {
 167                 0.0f, 2.0f,
 168                 0.0f, 0.0f,
 169                 2.0f, 0.0f
 170         };
 171         vbo = generate_vbo(2, GL_FLOAT, sizeof(vertices), vertices);
 172         check_error();
 173 }
 174
 175 ChromaSubsampler::~ChromaSubsampler()
 176 {
 177         resource_pool->release_glsl_program(cbcr_program_num);
 178         check_error();
 179         resource_pool->release_glsl_program(uyvy_program_num);
 180         check_error();
 181         glDeleteBuffers(1, &vbo);
 182         check_error();
 183 }
 184
 185 void ChromaSubsampler::subsample_chroma(GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex, GLuint dst2_tex)
 186 {
 187         GLuint vao;
 188         glGenVertexArrays(1, &vao);
 189         check_error();
 190
 191         glBindVertexArray(vao);
 192         check_error();
 193
 194         // Extract Cb/Cr.
 195         GLuint fbo;
 196         if (dst2_tex <= 0) {
 197                 fbo = resource_pool->create_fbo(dst_tex);
 198         } else {
 199                 fbo = resource_pool->create_fbo(dst_tex, dst2_tex);
 200         }
 201         glBindFramebuffer(GL_FRAMEBUFFER, fbo);
 202         glViewport(0, 0, width/2, height/2);
 203         check_error();
 204
 205         glUseProgram(cbcr_program_num);
 206         check_error();
 207
 208         glActiveTexture(GL_TEXTURE0);
 209         check_error();
 210         glBindTexture(GL_TEXTURE_2D, cbcr_tex);
 211         check_error();
 212         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 213         check_error();
 214         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 215         check_error();
 216         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 217         check_error();
 218
 219         float chroma_offset_0[] = { -1.0f / width, 0.0f };
 220         float chroma_offset_1[] = { -0.0f / width, 0.0f };
 221         set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_0", chroma_offset_0);
 222         set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_1", chroma_offset_1);
 223
 224         glUniform1i(cbcr_texture_sampler_uniform, 0);
 225
 226         glBindBuffer(GL_ARRAY_BUFFER, vbo);
 227         check_error();
 228
 229         for (GLint attr_index : { cbcr_position_attribute_index, cbcr_texcoord_attribute_index }) {
 230                 glEnableVertexAttribArray(attr_index);
 231                 check_error();
 232                 glVertexAttribPointer(attr_index, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0));
 233                 check_error();
 234         }
 235
 236         glDrawArrays(GL_TRIANGLES, 0, 3);
 237         check_error();
 238
 239         for (GLint attr_index : { cbcr_position_attribute_index, cbcr_texcoord_attribute_index }) {
 240                 glDisableVertexAttribArray(attr_index);
 241                 check_error();
 242         }
 243
 244         glUseProgram(0);
 245         check_error();
 246         glBindFramebuffer(GL_FRAMEBUFFER, 0);
 247         check_error();
 248
 249         resource_pool->release_fbo(fbo);
 250         glDeleteVertexArrays(1, &vao);
 251         check_error();
 252 }
 253
 254 void ChromaSubsampler::create_uyvy(GLuint y_tex, GLuint cbcr_tex, unsigned width, unsigned height, GLuint dst_tex)
 255 {
 256         GLuint vao;
 257         glGenVertexArrays(1, &vao);
 258         check_error();
 259
 260         glBindVertexArray(vao);
 261         check_error();
 262
 263         GLuint fbo = resource_pool->create_fbo(dst_tex);
 264         glBindFramebuffer(GL_FRAMEBUFFER, fbo);
 265         glViewport(0, 0, width/2, height);
 266         check_error();
 267
 268         glUseProgram(uyvy_program_num);
 269         check_error();
 270
 271         glUniform1i(uyvy_y_texture_sampler_uniform, 0);
 272         check_error();
 273         glUniform1i(uyvy_cbcr_texture_sampler_uniform, 1);
 274         check_error();
 275
 276         glActiveTexture(GL_TEXTURE0);
 277         check_error();
 278         glBindTexture(GL_TEXTURE_2D, y_tex);
 279         check_error();
 280         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 281         check_error();
 282         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 283         check_error();
 284         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 285         check_error();
 286
 287         glActiveTexture(GL_TEXTURE1);
 288         check_error();
 289         glBindTexture(GL_TEXTURE_2D, cbcr_tex);
 290         check_error();
 291         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 292         check_error();
 293         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 294         check_error();
 295         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 296         check_error();
 297
 298         float y_offset_0[] = { -0.5f / width, 0.0f };
 299         float y_offset_1[] = {  0.5f / width, 0.0f };
 300         float cbcr_offset0[] = { -1.0f / width, 0.0f };
 301         float cbcr_offset1[] = { -0.0f / width, 0.0f };
 302         set_uniform_vec2(uyvy_program_num, "foo", "luma_offset_0", y_offset_0);
 303         set_uniform_vec2(uyvy_program_num, "foo", "luma_offset_1", y_offset_1);
 304         set_uniform_vec2(uyvy_program_num, "foo", "chroma_offset_0", cbcr_offset0);
 305         set_uniform_vec2(uyvy_program_num, "foo", "chroma_offset_1", cbcr_offset1);
 306
 307         glBindBuffer(GL_ARRAY_BUFFER, vbo);
 308         check_error();
 309
 310         for (GLint attr_index : { uyvy_position_attribute_index, uyvy_texcoord_attribute_index }) {
 311                 if (attr_index == -1) continue;
 312                 glEnableVertexAttribArray(attr_index);
 313                 check_error();
 314                 glVertexAttribPointer(attr_index, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0));
 315                 check_error();
 316         }
 317
 318         glDrawArrays(GL_TRIANGLES, 0, 3);
 319         check_error();
 320
 321         for (GLint attr_index : { uyvy_position_attribute_index, uyvy_texcoord_attribute_index }) {
 322                 if (attr_index == -1) continue;
 323                 glDisableVertexAttribArray(attr_index);
 324                 check_error();
 325         }
 326
 327         glActiveTexture(GL_TEXTURE0);
 328         check_error();
 329         glUseProgram(0);
 330         check_error();
 331         glBindFramebuffer(GL_FRAMEBUFFER, 0);
 332         check_error();
 333
 334         resource_pool->release_fbo(fbo);
 335         glDeleteVertexArrays(1, &vao);
 336 }