git.sesse.net Git - nageru/blob - mixer.cpp

   1 #define GL_GLEXT_PROTOTYPES 1
   2 #define NO_SDL_GLEXT 1
   3 #define NUM_CARDS 2
   4
   5 #define WIDTH 1280
   6 #define HEIGHT 720
   7
   8 #include <epoxy/gl.h>
   9 #include <epoxy/egl.h>
  10
  11 #undef Success
  12
  13 #include <assert.h>
  14 #include <features.h>
  15 #include <math.h>
  16 #include <png.h>
  17 #include <pngconf.h>
  18 #include <setjmp.h>
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <sys/time.h>
  22 #include <time.h>
  23 #include <mutex>
  24 #include <queue>
  25 #include <condition_variable>
  26
  27 #include <diffusion_effect.h>
  28 #include <effect.h>
  29 #include <effect_chain.h>
  30 #include <flat_input.h>
  31 #include <image_format.h>
  32 #include <init.h>
  33 #include <lift_gamma_gain_effect.h>
  34 #include <saturation_effect.h>
  35 #include <util.h>
  36 #include <ycbcr_input.h>
  37 #include <vignette_effect.h>
  38 #include <resample_effect.h>
  39 #include <resize_effect.h>
  40 #include <overlay_effect.h>
  41 #include <padding_effect.h>
  42 #include <white_balance_effect.h>
  43 #include <ycbcr.h>
  44 #include <resource_pool.h>
  45 #include <effect_util.h>
  46
  47 #include <EGL/egl.h>
  48
  49 #include "h264encode.h"
  50 #include "context.h"
  51 #include "bmusb.h"
  52 #include "pbo_frame_allocator.h"
  53
  54 using namespace movit;
  55 using namespace std;
  56 using namespace std::placeholders;
  57
  58 // shared between all EGL contexts
  59 EGLDisplay egl_display;
  60 EGLSurface egl_surface;
  61 EGLConfig ecfg;
  62 EGLint ctxattr[] = {
  63         EGL_CONTEXT_CLIENT_VERSION, 2,
  64         EGL_CONTEXT_MAJOR_VERSION_KHR, 3,
  65         EGL_CONTEXT_MINOR_VERSION_KHR, 1,
  66         //EGL_CONTEXT_OPENGL_PROFILE_MASK_KHR, EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT_KHR,
  67         EGL_CONTEXT_OPENGL_PROFILE_MASK_KHR, EGL_CONTEXT_OPENGL_COMPATIBILITY_PROFILE_BIT,
  68         EGL_NONE
  69 };
  70
  71 EGLConfig pbuffer_ecfg;
  72
  73 std::mutex bmusb_mutex;  // protects <cards>
  74
  75 struct CaptureCard {
  76         BMUSBCapture *usb;
  77
  78         // Threading stuff
  79         bool thread_initialized;
  80         QSurface *surface;
  81         QOpenGLContext *context;
  82
  83         bool new_data_ready;  // Whether new_frame contains anything.
  84         PBOFrameAllocator::Frame new_frame;
  85         GLsync new_data_ready_fence;  // Whether new_frame is ready for rendering.
  86         std::condition_variable new_data_ready_changed;  // Set whenever new_data_ready is changed.
  87 };
  88 CaptureCard cards[NUM_CARDS];
  89
  90 void bm_frame(int card_index, uint16_t timecode,
  91               FrameAllocator::Frame video_frame, size_t video_offset, uint16_t video_format,
  92               FrameAllocator::Frame audio_frame, size_t audio_offset, uint16_t audio_format)
  93 {
  94         CaptureCard *card = &cards[card_index];
  95         if (!card->thread_initialized) {
  96                 printf("initializing context for bmusb thread %d\n", card_index);
  97                 eglBindAPI(EGL_OPENGL_API);
  98                 card->context = create_context();
  99                 if (!make_current(card->context, card->surface)) {
 100                         printf("failed to create bmusb context\n");
 101                         exit(1);
 102                 }
 103                 card->thread_initialized = true;
 104         }
 105
 106         if (video_frame.len - video_offset != 1280 * 750 * 2) {
 107                 printf("dropping frame with wrong length (%ld)\n", video_frame.len - video_offset);
 108                 FILE *fp = fopen("frame.raw", "wb");
 109                 fwrite(video_frame.data, video_frame.len, 1, fp);
 110                 fclose(fp);
 111                 //exit(1);
 112                 card->usb->get_video_frame_allocator()->release_frame(video_frame);
 113                 card->usb->get_audio_frame_allocator()->release_frame(audio_frame);
 114                 return;
 115         }
 116         {
 117                 // Wait until the previous frame was consumed.
 118                 std::unique_lock<std::mutex> lock(bmusb_mutex);
 119                 card->new_data_ready_changed.wait(lock, [card]{ return !card->new_data_ready; });
 120         }
 121         GLuint pbo = (GLint)(intptr_t)video_frame.userdata;
 122         check_error();
 123         glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
 124         check_error();
 125         glFlushMappedBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, video_frame.size);
 126         check_error();
 127         //glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
 128         //check_error();
 129         GLsync fence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
 130         check_error();
 131         assert(fence != nullptr);
 132         {
 133                 std::unique_lock<std::mutex> lock(bmusb_mutex);
 134                 card->new_data_ready = true;
 135                 card->new_frame = video_frame;
 136                 card->new_data_ready_fence = fence;
 137                 card->new_data_ready_changed.notify_all();
 138         }
 139
 140         // Video frame will be released later.
 141         card->usb->get_audio_frame_allocator()->release_frame(audio_frame);
 142 }
 143
 144 void place_rectangle(Effect *resample_effect, Effect *padding_effect, float x0, float y0, float x1, float y1)
 145 {
 146         float srcx0 = 0.0f;
 147         float srcx1 = 1.0f;
 148         float srcy0 = 0.0f;
 149         float srcy1 = 1.0f;
 150
 151         // Cull.
 152         if (x0 > 1280.0 || x1 < 0.0 || y0 > 720.0 || y1 < 0.0) {
 153                 CHECK(resample_effect->set_int("width", 1));
 154                 CHECK(resample_effect->set_int("height", 1));
 155                 CHECK(resample_effect->set_float("zoom_x", 1280.0));
 156                 CHECK(resample_effect->set_float("zoom_y", 720.0));
 157                 CHECK(padding_effect->set_int("left", 2000));
 158                 CHECK(padding_effect->set_int("top", 2000));
 159                 return;
 160         }
 161
 162         // Clip. (TODO: Clip on upper/left sides, too.)
 163         if (x1 > 1280.0) {
 164                 srcx1 = (1280.0 - x0) / (x1 - x0);
 165                 x1 = 1280.0;
 166         }
 167         if (y1 > 720.0) {
 168                 srcy1 = (720.0 - y0) / (y1 - y0);
 169                 y1 = 720.0;
 170         }
 171
 172         float x_subpixel_offset = x0 - floor(x0);
 173         float y_subpixel_offset = y0 - floor(y0);
 174
 175         // Resampling must be to an integral number of pixels. Round up,
 176         // and then add an extra pixel so we have some leeway for the border.
 177         int width = int(ceil(x1 - x0)) + 1;
 178         int height = int(ceil(y1 - y0)) + 1;
 179         CHECK(resample_effect->set_int("width", width));
 180         CHECK(resample_effect->set_int("height", height));
 181
 182         // Correct the discrepancy with zoom. (This will leave a small
 183         // excess edge of pixels and subpixels, which we'll correct for soon.)
 184         float zoom_x = (x1 - x0) / (width * (srcx1 - srcx0));
 185         float zoom_y = (y1 - y0) / (height * (srcy1 - srcy0));
 186         CHECK(resample_effect->set_float("zoom_x", zoom_x));
 187         CHECK(resample_effect->set_float("zoom_y", zoom_y));
 188         CHECK(resample_effect->set_float("zoom_center_x", 0.0f));
 189         CHECK(resample_effect->set_float("zoom_center_y", 0.0f));
 190
 191         // Padding must also be to a whole-pixel offset.
 192         CHECK(padding_effect->set_int("left", floor(x0)));
 193         CHECK(padding_effect->set_int("top", floor(y0)));
 194
 195         // Correct _that_ discrepancy by subpixel offset in the resampling.
 196         CHECK(resample_effect->set_float("left", -x_subpixel_offset / zoom_x));
 197         CHECK(resample_effect->set_float("top", -y_subpixel_offset / zoom_y));
 198
 199         // Finally, adjust the border so it is exactly where we want it.
 200         CHECK(padding_effect->set_float("border_offset_left", x_subpixel_offset));
 201         CHECK(padding_effect->set_float("border_offset_right", x1 - (floor(x0) + width)));
 202         CHECK(padding_effect->set_float("border_offset_top", y_subpixel_offset));
 203         CHECK(padding_effect->set_float("border_offset_bottom", y1 - (floor(y0) + height)));
 204 }
 205
 206 void mixer_thread(QSurface *surface, QSurface *surface2, QSurface *surface3, QSurface *surface4)
 207 {
 208         bool quit = false;
 209
 210         cards[0].surface = surface3;
 211 #if NUM_CARDS == 2
 212         cards[1].surface = surface4;
 213 #endif
 214
 215         eglBindAPI(EGL_OPENGL_API);
 216         //QSurface *surface = create_surface();
 217         QOpenGLContext *context = create_context();
 218         if (!make_current(context, surface)) {
 219                 printf("oops\n");
 220                 exit(1);
 221         }
 222         printf("egl=%p\n", eglGetCurrentContext());
 223
 224         CHECK(init_movit("/usr/share/movit", MOVIT_DEBUG_ON));
 225         printf("GPU texture subpixel precision: about %.1f bits\n",
 226                 log2(1.0f / movit_texel_subpixel_precision));
 227         printf("Wrongly rounded x+0.48 or x+0.52 values: %d/510\n",
 228                 movit_num_wrongly_rounded);
 229         if (movit_num_wrongly_rounded > 0) {
 230                 if (movit_shader_rounding_supported) {
 231                         printf("Rounding off in the shader to compensate.\n");
 232                 } else {
 233                         printf("No shader roundoff available; cannot compensate.\n");
 234                 }
 235         }
 236
 237         //printf("egl_api=%d EGL_OPENGL_API=%d\n", epoxy_egl_get_current_gl_context_api(), EGL_OPENGL_API);
 238         //exit(0);
 239
 240         check_error();
 241
 242         EffectChain chain(WIDTH, HEIGHT);
 243         check_error();
 244 #if 0
 245         glViewport(0, 0, WIDTH, HEIGHT);
 246         check_error();
 247
 248         glMatrixMode(GL_PROJECTION);
 249         check_error();
 250         glLoadIdentity();
 251         check_error();
 252         glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0);
 253         check_error();
 254
 255         glMatrixMode(GL_MODELVIEW);
 256         glLoadIdentity();
 257 #endif
 258
 259         ImageFormat inout_format;
 260         inout_format.color_space = COLORSPACE_sRGB;
 261         inout_format.gamma_curve = GAMMA_sRGB;
 262
 263         YCbCrFormat ycbcr_format;
 264         ycbcr_format.chroma_subsampling_x = 2;
 265         ycbcr_format.chroma_subsampling_y = 1;
 266         ycbcr_format.cb_x_position = 0.0;
 267         ycbcr_format.cr_x_position = 0.0;
 268         ycbcr_format.cb_y_position = 0.5;
 269         ycbcr_format.cr_y_position = 0.5;
 270         ycbcr_format.luma_coefficients = YCBCR_REC_601;
 271         ycbcr_format.full_range = false;
 272
 273         YCbCrInput *input[NUM_CARDS];
 274
 275         input[0] = new YCbCrInput(inout_format, ycbcr_format, WIDTH, HEIGHT, YCBCR_INPUT_SPLIT_Y_AND_CBCR);
 276         chain.add_input(input[0]);
 277         //if (NUM_CARDS == 2) {
 278                 input[1] = new YCbCrInput(inout_format, ycbcr_format, WIDTH, HEIGHT, YCBCR_INPUT_SPLIT_Y_AND_CBCR);
 279                 chain.add_input(input[1]);
 280         //}
 281         //YCbCr422InterleavedInput *input = new YCbCr422InterleavedInput(inout_format, ycbcr_format, WIDTH, HEIGHT);
 282         //YCbCr422InterleavedInput *input = new YCbCr422InterleavedInput(inout_format, ycbcr_format, 2, 1);
 283         Effect *resample_effect = chain.add_effect(new ResampleEffect(), input[0]);
 284         Effect *padding_effect = chain.add_effect(new IntegralPaddingEffect());
 285         float border_color[] = { 0.0f, 0.0f, 0.0f, 1.0f };
 286         CHECK(padding_effect->set_vec4("border_color", border_color));
 287
 288         //Effect *resample2_effect = chain.add_effect(new ResampleEffect(), input[1 % NUM_CARDS]);
 289         Effect *resample2_effect = chain.add_effect(new ResampleEffect(), input[1]);
 290         Effect *saturation_effect = chain.add_effect(new SaturationEffect());
 291         CHECK(saturation_effect->set_float("saturation", 0.3f));
 292         Effect *wb_effect = chain.add_effect(new WhiteBalanceEffect());
 293         CHECK(wb_effect->set_float("output_color_temperature", 3500.0));
 294         Effect *padding2_effect = chain.add_effect(new IntegralPaddingEffect());
 295
 296         chain.add_effect(new OverlayEffect(), padding_effect, padding2_effect);
 297
 298         ycbcr_format.chroma_subsampling_x = 1;
 299
 300         chain.add_ycbcr_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED, ycbcr_format, YCBCR_OUTPUT_SPLIT_Y_AND_CBCR);
 301         chain.set_dither_bits(8);
 302         chain.set_output_origin(OUTPUT_ORIGIN_TOP_LEFT);
 303         chain.finalize();
 304
 305 #if 0
 306         // generate a PBO to hold the data we read back with glReadPixels()
 307         // (Intel/DRI goes into a slow path if we don't read to PBO)
 308         GLuint pbo;
 309         glGenBuffers(1, &pbo);
 310         glBindBuffer(GL_PIXEL_PACK_BUFFER_ARB, pbo);
 311         glBufferData(GL_PIXEL_PACK_BUFFER_ARB, WIDTH * HEIGHT * 4, NULL, GL_STREAM_READ);
 312 #endif
 313
 314         //make_hsv_wheel_texture();
 315
 316 //      QSurface *create_surface(const QSurfaceFormat &format);
 317         H264Encoder h264_encoder(surface2, WIDTH, HEIGHT, "test.mp4");
 318
 319         printf("Configuring first card...\n");
 320         cards[0].usb = new BMUSBCapture(0x1edb, 0xbd3b);  // 0xbd4f
 321         //cards[0].usb = new BMUSBCapture(0x1edb, 0xbd4f);
 322         cards[0].usb->set_frame_callback(std::bind(bm_frame, 0, _1, _2, _3, _4, _5, _6, _7));
 323         std::unique_ptr<PBOFrameAllocator> pbo_allocator1(new PBOFrameAllocator(1280 * 750 * 2 + 44));
 324         cards[0].usb->set_video_frame_allocator(pbo_allocator1.get());
 325         cards[0].usb->configure_card();
 326
 327         std::unique_ptr<PBOFrameAllocator> pbo_allocator2(new PBOFrameAllocator(1280 * 750 * 2 + 44));
 328         if (NUM_CARDS == 2) {
 329                 printf("Configuring second card...\n");
 330                 cards[1].usb = new BMUSBCapture(0x1edb, 0xbd4f);
 331                 cards[1].usb->set_frame_callback(std::bind(bm_frame, 1, _1, _2, _3, _4, _5, _6, _7));
 332                 cards[1].usb->set_video_frame_allocator(pbo_allocator2.get());
 333                 cards[1].usb->configure_card();
 334         }
 335
 336         BMUSBCapture::start_bm_thread();
 337
 338         for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
 339                 cards[card_index].usb->start_bm_capture();
 340         }
 341
 342         int frame = 0;
 343 #if _POSIX_C_SOURCE >= 199309L
 344         struct timespec start, now;
 345         clock_gettime(CLOCK_MONOTONIC, &start);
 346 #else
 347         struct timeval start, now;
 348         gettimeofday(&start, NULL);
 349 #endif
 350
 351         PBOFrameAllocator::Frame bmusb_current_rendering_frame[NUM_CARDS];
 352         for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
 353                 bmusb_current_rendering_frame[card_index] =
 354                         cards[card_index].usb->get_video_frame_allocator()->alloc_frame();
 355                 GLint input_tex_pbo = (GLint)(intptr_t)bmusb_current_rendering_frame[card_index].userdata;
 356                 input[card_index]->set_pixel_data(0, nullptr, input_tex_pbo);
 357                 input[card_index]->set_pixel_data(1, nullptr, input_tex_pbo);
 358                 check_error();
 359         }
 360
 361         //chain.enable_phase_timing(true);
 362
 363         // Set up stuff for NV12 conversion.
 364 #if 0
 365         PBOFrameAllocator nv12_frame_pool(WIDTH * HEIGHT * 3 / 2, /*num_frames=*/24,
 366                 GL_PIXEL_PACK_BUFFER_ARB, GL_MAP_READ_BIT | GL_MAP_COHERENT_BIT, 0);
 367 #endif
 368         ResourcePool *resource_pool = chain.get_resource_pool();
 369         //GLuint ycbcr_tex = resource_pool->create_2d_texture(GL_RGBA8, WIDTH, HEIGHT);
 370         GLuint chroma_tex = resource_pool->create_2d_texture(GL_RG8, WIDTH, HEIGHT);
 371
 372 #if 0
 373         // Y shader.
 374         string y_vert_shader = read_version_dependent_file("vs-y", "vert");
 375         string y_frag_shader =
 376                 "#version 130 \n"
 377                 "in vec2 tc; \n"
 378                 "uniform sampler2D ycbcr_tex; \n"
 379                 "void main() { \n"
 380                 "    gl_FragColor = texture2D(ycbcr_tex, tc); \n"
 381                 "} \n";
 382         GLuint y_program_num = resource_pool->compile_glsl_program(y_vert_shader, y_frag_shader);
 383 #endif
 384
 385 #if 1
 386         // Cb/Cr shader.
 387         string cbcr_vert_shader = read_version_dependent_file("vs-cbcr", "vert");
 388         string cbcr_frag_shader =
 389                 "#version 130 \n"
 390                 "in vec2 tc0; \n"
 391 //              "in vec2 tc1; \n"
 392                 "uniform sampler2D cbcr_tex; \n"
 393                 "void main() { \n"
 394                 "    gl_FragColor = texture2D(cbcr_tex, tc0); \n"
 395 //              "    gl_FragColor.ba = texture2D(cbcr_tex, tc1).gb; \n"
 396                 "} \n";
 397         GLuint cbcr_program_num = resource_pool->compile_glsl_program(cbcr_vert_shader, cbcr_frag_shader);
 398 #endif
 399
 400         GLuint vao;
 401         glGenVertexArrays(1, &vao);
 402         check_error();
 403
 404         while (!quit) {
 405                 ++frame;
 406
 407                 //int width0 = lrintf(848 * (1.0 + 0.2 * sin(frame * 0.02)));
 408                 int width0 = 848;
 409                 int height0 = lrintf(width0 * 9.0 / 16.0);
 410
 411                 //float top0 = 96 + 48 * sin(frame * 0.005);
 412                 //float left0 = 96 + 48 * cos(frame * 0.006);
 413                 float top0 = 48;
 414                 float left0 = 16;
 415                 float bottom0 = top0 + height0;
 416                 float right0 = left0 + width0;
 417
 418                 int width1 = 384;
 419                 int height1 = 216;
 420
 421                 float bottom1 = 720 - 48;
 422                 float right1 = 1280 - 16;
 423                 float top1 = bottom1 - height1;
 424                 float left1 = right1 - width1;
 425
 426                 float t = 0.5 + 0.5 * cos(frame * 0.006);
 427                 //float t = 0.0;
 428                 float scale0 = 1.0 + t * (1280.0 / 848.0 - 1.0);
 429                 float tx0 = 0.0 + t * (-16.0 * scale0);
 430                 float ty0 = 0.0 + t * (-48.0 * scale0);
 431
 432                 top0 = top0 * scale0 + ty0;
 433                 bottom0 = bottom0 * scale0 + ty0;
 434                 left0 = left0 * scale0 + tx0;
 435                 right0 = right0 * scale0 + tx0;
 436
 437                 top1 = top1 * scale0 + ty0;
 438                 bottom1 = bottom1 * scale0 + ty0;
 439                 left1 = left1 * scale0 + tx0;
 440                 right1 = right1 * scale0 + tx0;
 441
 442                 place_rectangle(resample_effect, padding_effect, left0, top0, right0, bottom0);
 443                 place_rectangle(resample2_effect, padding2_effect, left1, top1, right1, bottom1);
 444
 445                 CaptureCard card_copy[NUM_CARDS];
 446
 447                 {
 448                         std::unique_lock<std::mutex> lock(bmusb_mutex);
 449
 450                         // The first card is the master timer, so wait for it to have a new frame.
 451                         // TODO: Make configurable, and with a timeout.
 452                         cards[0].new_data_ready_changed.wait(lock, []{ return cards[0].new_data_ready; });
 453
 454                         for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
 455                                 CaptureCard *card = &cards[card_index];
 456                                 card_copy[card_index].usb = card->usb;
 457                                 card_copy[card_index].new_data_ready = card->new_data_ready;
 458                                 card_copy[card_index].new_frame = card->new_frame;
 459                                 card_copy[card_index].new_data_ready_fence = card->new_data_ready_fence;
 460                                 card->new_data_ready = false;
 461                                 card->new_data_ready_changed.notify_all();
 462                         }
 463                 }
 464
 465                 for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
 466                         CaptureCard *card = &card_copy[card_index];
 467                         if (!card->new_data_ready)
 468                                 continue;
 469
 470                         // FIXME: We could still be rendering from it!
 471                         card->usb->get_video_frame_allocator()->release_frame(bmusb_current_rendering_frame[card_index]);
 472                         bmusb_current_rendering_frame[card_index] = card->new_frame;
 473
 474                         // The new texture might still be uploaded,
 475                         // tell the GPU to wait until it's there.
 476                         if (card->new_data_ready_fence)
 477                                 glWaitSync(card->new_data_ready_fence, /*flags=*/0, GL_TIMEOUT_IGNORED);
 478                         check_error();
 479                         glDeleteSync(card->new_data_ready_fence);
 480                         check_error();
 481                         GLint input_tex_pbo = (GLint)(intptr_t)bmusb_current_rendering_frame[card_index].userdata;
 482                         input[card_index]->set_pixel_data(0, (unsigned char *)BUFFER_OFFSET((1280 * 750 * 2 + 44) / 2 + 1280 * 25 + 22), input_tex_pbo);
 483                         input[card_index]->set_pixel_data(1, (unsigned char *)BUFFER_OFFSET(1280 * 25 + 22), input_tex_pbo);
 484
 485                         if (NUM_CARDS == 1) {
 486                                 // Set to the other one, too.
 487                                 input[1]->set_pixel_data(0, (unsigned char *)BUFFER_OFFSET((1280 * 750 * 2 + 44) / 2 + 1280 * 25 + 22), input_tex_pbo);
 488                                 input[1]->set_pixel_data(1, (unsigned char *)BUFFER_OFFSET(1280 * 25 + 22), input_tex_pbo);
 489                         }
 490                 }
 491
 492                 GLuint y_tex, cbcr_tex;
 493                 bool got_frame = h264_encoder.begin_frame(&y_tex, &cbcr_tex);
 494                 assert(got_frame);
 495
 496                 // Render chain.
 497                 {
 498                         GLuint ycbcr_fbo = resource_pool->create_fbo(y_tex, chroma_tex);
 499                         chain.render_to_fbo(ycbcr_fbo, WIDTH, HEIGHT);
 500                         resource_pool->release_fbo(ycbcr_fbo);
 501                 }
 502                 if (false) {
 503                         glViewport(0, 0, WIDTH, HEIGHT);
 504                         chain.render_to_screen();
 505                 }
 506
 507 #if 0
 508                 PBOFrameAllocator::Frame nv12_frame = nv12_frame_pool.alloc_frame();
 509                 assert(nv12_frame.data != nullptr);  // should never happen... maybe?
 510                 GLuint pbo = (GLuint)(intptr_t)nv12_frame.userdata;
 511                 glBindBuffer(GL_PIXEL_PACK_BUFFER_ARB, pbo);
 512                 check_error();
 513 #endif
 514
 515                 // Set up for extraction.
 516                 float vertices[] = {
 517                         0.0f, 2.0f,
 518                         0.0f, 0.0f,
 519                         2.0f, 0.0f
 520                 };
 521
 522                 glBindVertexArray(vao);
 523                 check_error();
 524
 525 #if 0
 526                 // Extract Y.
 527                 GLuint y_fbo = resource_pool->create_fbo(y_tex);
 528                 glBindFramebuffer(GL_FRAMEBUFFER, y_fbo);
 529                 glViewport(0, 0, WIDTH, HEIGHT);
 530                 check_error();
 531                 {
 532                         glUseProgram(y_program_num);
 533                         check_error();
 534                         glActiveTexture(GL_TEXTURE0);
 535                         check_error();
 536                         glBindTexture(GL_TEXTURE_2D, ycbcr_tex);
 537                         check_error();
 538                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 539                         check_error();
 540                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 541                         check_error();
 542                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 543                         check_error();
 544
 545                         GLuint position_vbo = fill_vertex_attribute(y_program_num, "position", 2, GL_FLOAT, sizeof(vertices), vertices);
 546                         GLuint texcoord_vbo = fill_vertex_attribute(y_program_num, "texcoord", 2, GL_FLOAT, sizeof(vertices), vertices);  // Same as vertices.
 547
 548                         glDrawArrays(GL_TRIANGLES, 0, 3);
 549                         check_error();
 550
 551                         cleanup_vertex_attribute(y_program_num, "position", position_vbo);
 552                         check_error();
 553                         cleanup_vertex_attribute(y_program_num, "texcoord", texcoord_vbo);
 554                         check_error();
 555
 556                         resource_pool->release_fbo(y_fbo);
 557                 }
 558 #endif
 559
 560                 // Extract Cb/Cr.
 561                 GLuint cbcr_fbo = resource_pool->create_fbo(cbcr_tex);
 562                 glBindFramebuffer(GL_FRAMEBUFFER, cbcr_fbo);
 563                 glViewport(0, 0, WIDTH/2, HEIGHT/2);
 564                 check_error();
 565                 GLsync fence;
 566                 {
 567                         glUseProgram(cbcr_program_num);
 568                         check_error();
 569
 570                         glActiveTexture(GL_TEXTURE0);
 571                         check_error();
 572                         glBindTexture(GL_TEXTURE_2D, chroma_tex);
 573                         check_error();
 574                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 575                         check_error();
 576                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 577                         check_error();
 578                         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 579                         check_error();
 580
 581                         float chroma_offset_0[] = { -0.5f / WIDTH, 0.0f };
 582 //                      float chroma_offset_1[] = { +0.5f / WIDTH, 0.0f };
 583                         set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_0", chroma_offset_0);
 584 //                      set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_1", chroma_offset_1);
 585
 586                         GLuint position_vbo = fill_vertex_attribute(cbcr_program_num, "position", 2, GL_FLOAT, sizeof(vertices), vertices);
 587                         GLuint texcoord_vbo = fill_vertex_attribute(cbcr_program_num, "texcoord", 2, GL_FLOAT, sizeof(vertices), vertices);  // Same as vertices.
 588
 589                         glDrawArrays(GL_TRIANGLES, 0, 3);
 590                         check_error();
 591
 592                         cleanup_vertex_attribute(cbcr_program_num, "position", position_vbo);
 593                         cleanup_vertex_attribute(cbcr_program_num, "texcoord", texcoord_vbo);
 594
 595                         glUseProgram(0);
 596                         check_error();
 597
 598 #if 0
 599                         glReadPixels(0, 0, WIDTH/4, HEIGHT/2, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, BUFFER_OFFSET(WIDTH * HEIGHT));
 600                         check_error();
 601 #endif
 602
 603                         fence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
 604                         check_error();
 605
 606                         resource_pool->release_fbo(cbcr_fbo);
 607                 }
 608
 609 #if 0
 610                 glBindBuffer(GL_PIXEL_PACK_BUFFER_ARB, 0);
 611 #endif
 612
 613 #if 1
 614                 h264_encoder.end_frame(fence);
 615 #else
 616                 nv12_frame_pool.release_frame(nv12_frame);
 617 #endif
 618
 619                 //eglSwapBuffers(egl_display, egl_surface);
 620
 621
 622 #if 1
 623 #if _POSIX_C_SOURCE >= 199309L
 624                 clock_gettime(CLOCK_MONOTONIC, &now);
 625                 double elapsed = now.tv_sec - start.tv_sec +
 626                         1e-9 * (now.tv_nsec - start.tv_nsec);
 627 #else
 628                 gettimeofday(&now, NULL);
 629                 double elapsed = now.tv_sec - start.tv_sec +
 630                         1e-6 * (now.tv_usec - start.tv_usec);
 631 #endif
 632                 if (frame % 100 == 0) {
 633                         printf("%d frames in %.3f seconds = %.1f fps (%.1f ms/frame)\n",
 634                                 frame, elapsed, frame / elapsed,
 635                                 1e3 * elapsed / frame);
 636                 //      chain.print_phase_timing();
 637                 }
 638
 639                 // Reset every 100 frames, so that local variations in frame times
 640                 // (especially for the first few frames, when the shaders are
 641                 // compiled etc.) don't make it hard to measure for the entire
 642                 // remaining duration of the program.
 643                 if (frame == 10000) {
 644                         frame = 0;
 645                         start = now;
 646                 }
 647 #endif
 648         }
 649         glDeleteVertexArrays(1, &vao);
 650         //resource_pool->release_glsl_program(y_program_num);
 651         resource_pool->release_glsl_program(cbcr_program_num);
 652         resource_pool->release_2d_texture(chroma_tex);
 653         BMUSBCapture::stop_bm_thread();
 654 }