10 #include <effect_chain.h>
11 #include <effect_util.h>
12 #include <epoxy/egl.h>
14 #include <image_format.h>
16 #include <overlay_effect.h>
17 #include <padding_effect.h>
18 #include <resample_effect.h>
19 #include <resource_pool.h>
20 #include <saturation_effect.h>
27 #include <white_balance_effect.h>
29 #include <ycbcr_input.h>
31 #include <condition_variable>
41 #include "h264encode.h"
42 #include "pbo_frame_allocator.h"
43 #include "ref_counted_gl_sync.h"
47 using namespace movit;
49 using namespace std::placeholders;
51 Mixer *global_mixer = nullptr;
53 Mixer::Mixer(const QSurfaceFormat &format)
54 : mixer_surface(create_surface(format)),
55 h264_encoder_surface(create_surface(format))
57 CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF));
60 resource_pool.reset(new ResourcePool);
61 theme.reset(new Theme("theme.lua", resource_pool.get()));
62 output_channel[OUTPUT_LIVE].parent = this;
63 output_channel[OUTPUT_PREVIEW].parent = this;
64 output_channel[OUTPUT_INPUT0].parent = this;
65 output_channel[OUTPUT_INPUT1].parent = this;
67 ImageFormat inout_format;
68 inout_format.color_space = COLORSPACE_sRGB;
69 inout_format.gamma_curve = GAMMA_sRGB;
71 // Display chain; shows the live output produced by the main chain (its RGBA version).
72 display_chain.reset(new EffectChain(WIDTH, HEIGHT, resource_pool.get()));
74 display_input = new FlatInput(inout_format, FORMAT_RGB, GL_UNSIGNED_BYTE, WIDTH, HEIGHT); // FIXME: GL_UNSIGNED_BYTE is really wrong.
75 display_chain->add_input(display_input);
76 display_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
77 display_chain->set_dither_bits(0); // Don't bother.
78 display_chain->finalize();
80 h264_encoder.reset(new H264Encoder(h264_encoder_surface, WIDTH, HEIGHT, "test.mp4"));
82 printf("Configuring first card...\n");
83 cards[0].usb = new BMUSBCapture(0x1edb, 0xbd3b); // 0xbd4f
84 cards[0].usb->set_frame_callback(std::bind(&Mixer::bm_frame, this, 0, _1, _2, _3, _4, _5, _6, _7));
85 cards[0].frame_allocator.reset(new PBOFrameAllocator(1280 * 750 * 2 + 44, 1280, 720));
86 cards[0].usb->set_video_frame_allocator(cards[0].frame_allocator.get());
87 cards[0].usb->configure_card();
88 cards[0].surface = create_surface(format);
90 cards[1].surface = create_surface(format);
94 printf("Configuring second card...\n");
95 cards[1].usb = new BMUSBCapture(0x1edb, 0xbd4f);
96 cards[1].usb->set_frame_callback(std::bind(&Mixer::bm_frame, this, 1, _1, _2, _3, _4, _5, _6, _7));
97 cards[1].frame_allocator.reset(new PBOFrameAllocator(1280 * 750 * 2 + 44, 1280, 720));
98 cards[1].usb->set_video_frame_allocator(cards[1].frame_allocator.get());
99 cards[1].usb->configure_card();
102 BMUSBCapture::start_bm_thread();
104 for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
105 cards[card_index].usb->start_bm_capture();
108 //chain->enable_phase_timing(true);
110 // Set up stuff for NV12 conversion.
113 string cbcr_vert_shader = read_file("vs-cbcr.130.vert");
114 string cbcr_frag_shader =
117 "uniform sampler2D cbcr_tex; \n"
119 " gl_FragColor = texture2D(cbcr_tex, tc0); \n"
121 cbcr_program_num = resource_pool->compile_glsl_program(cbcr_vert_shader, cbcr_frag_shader);
126 resource_pool->release_glsl_program(cbcr_program_num);
127 BMUSBCapture::stop_bm_thread();
130 void Mixer::bm_frame(int card_index, uint16_t timecode,
131 FrameAllocator::Frame video_frame, size_t video_offset, uint16_t video_format,
132 FrameAllocator::Frame audio_frame, size_t audio_offset, uint16_t audio_format)
134 CaptureCard *card = &cards[card_index];
135 if (!card->thread_initialized) {
136 printf("initializing context for bmusb thread %d\n", card_index);
137 eglBindAPI(EGL_OPENGL_API);
138 card->context = create_context();
139 if (!make_current(card->context, card->surface)) {
140 printf("failed to create bmusb context\n");
143 card->thread_initialized = true;
146 if (video_frame.len - video_offset != 1280 * 750 * 2) {
147 printf("dropping frame with wrong length (%ld)\n", video_frame.len - video_offset);
148 FILE *fp = fopen("frame.raw", "wb");
149 fwrite(video_frame.data, video_frame.len, 1, fp);
152 card->usb->get_video_frame_allocator()->release_frame(video_frame);
153 card->usb->get_audio_frame_allocator()->release_frame(audio_frame);
157 // Wait until the previous frame was consumed.
158 std::unique_lock<std::mutex> lock(bmusb_mutex);
159 card->new_data_ready_changed.wait(lock, [card]{ return !card->new_data_ready; });
161 const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)video_frame.userdata;
162 GLuint pbo = userdata->pbo;
164 glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
166 glFlushMappedBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, video_frame.size);
168 //glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
171 // Upload the textures.
172 glBindTexture(GL_TEXTURE_2D, userdata->tex_y);
174 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, 1280, 720, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET((1280 * 750 * 2 + 44) / 2 + 1280 * 25 + 22));
176 glBindTexture(GL_TEXTURE_2D, userdata->tex_cbcr);
178 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, 1280/2, 720, GL_RG, GL_UNSIGNED_BYTE, BUFFER_OFFSET(1280 * 25 + 22));
180 glBindTexture(GL_TEXTURE_2D, 0);
182 GLsync fence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
184 assert(fence != nullptr);
186 std::unique_lock<std::mutex> lock(bmusb_mutex);
187 card->new_data_ready = true;
188 card->new_frame = RefCountedFrame(video_frame);
189 card->new_data_ready_fence = fence;
190 card->new_data_ready_changed.notify_all();
193 // Video frame will be released when last user of card->new_frame goes out of scope.
194 card->usb->get_audio_frame_allocator()->release_frame(audio_frame);
197 void Mixer::place_rectangle(Effect *resample_effect, Effect *padding_effect, float x0, float y0, float x1, float y1)
205 if (x0 > 1280.0 || x1 < 0.0 || y0 > 720.0 || y1 < 0.0) {
206 CHECK(resample_effect->set_int("width", 1));
207 CHECK(resample_effect->set_int("height", 1));
208 CHECK(resample_effect->set_float("zoom_x", 1280.0));
209 CHECK(resample_effect->set_float("zoom_y", 720.0));
210 CHECK(padding_effect->set_int("left", 2000));
211 CHECK(padding_effect->set_int("top", 2000));
215 // Clip. (TODO: Clip on upper/left sides, too.)
217 srcx1 = (1280.0 - x0) / (x1 - x0);
221 srcy1 = (720.0 - y0) / (y1 - y0);
225 float x_subpixel_offset = x0 - floor(x0);
226 float y_subpixel_offset = y0 - floor(y0);
228 // Resampling must be to an integral number of pixels. Round up,
229 // and then add an extra pixel so we have some leeway for the border.
230 int width = int(ceil(x1 - x0)) + 1;
231 int height = int(ceil(y1 - y0)) + 1;
232 CHECK(resample_effect->set_int("width", width));
233 CHECK(resample_effect->set_int("height", height));
235 // Correct the discrepancy with zoom. (This will leave a small
236 // excess edge of pixels and subpixels, which we'll correct for soon.)
237 float zoom_x = (x1 - x0) / (width * (srcx1 - srcx0));
238 float zoom_y = (y1 - y0) / (height * (srcy1 - srcy0));
239 CHECK(resample_effect->set_float("zoom_x", zoom_x));
240 CHECK(resample_effect->set_float("zoom_y", zoom_y));
241 CHECK(resample_effect->set_float("zoom_center_x", 0.0f));
242 CHECK(resample_effect->set_float("zoom_center_y", 0.0f));
244 // Padding must also be to a whole-pixel offset.
245 CHECK(padding_effect->set_int("left", floor(x0)));
246 CHECK(padding_effect->set_int("top", floor(y0)));
248 // Correct _that_ discrepancy by subpixel offset in the resampling.
249 CHECK(resample_effect->set_float("left", -x_subpixel_offset / zoom_x));
250 CHECK(resample_effect->set_float("top", -y_subpixel_offset / zoom_y));
252 // Finally, adjust the border so it is exactly where we want it.
253 CHECK(padding_effect->set_float("border_offset_left", x_subpixel_offset));
254 CHECK(padding_effect->set_float("border_offset_right", x1 - (floor(x0) + width)));
255 CHECK(padding_effect->set_float("border_offset_top", y_subpixel_offset));
256 CHECK(padding_effect->set_float("border_offset_bottom", y1 - (floor(y0) + height)));
259 void Mixer::thread_func()
261 eglBindAPI(EGL_OPENGL_API);
262 QOpenGLContext *context = create_context();
263 if (!make_current(context, mixer_surface)) {
268 struct timespec start, now;
269 clock_gettime(CLOCK_MONOTONIC, &start);
271 while (!should_quit) {
274 //int width0 = lrintf(848 * (1.0 + 0.2 * sin(frame * 0.02)));
276 int height0 = lrintf(width0 * 9.0 / 16.0);
278 //float top0 = 96 + 48 * sin(frame * 0.005);
279 //float left0 = 96 + 48 * cos(frame * 0.006);
282 float bottom0 = top0 + height0;
283 float right0 = left0 + width0;
288 float bottom1 = 720 - 48;
289 float right1 = 1280 - 16;
290 float top1 = bottom1 - height1;
291 float left1 = right1 - width1;
293 if (current_source == SOURCE_INPUT1) {
300 bottom1 = HEIGHT + 20;
303 } else if (current_source == SOURCE_INPUT2) {
310 bottom0 = HEIGHT + 20;
314 float t = 0.5 + 0.5 * cos(frame * 0.006);
315 float scale0 = 1.0 + t * (1280.0 / 848.0 - 1.0);
316 float tx0 = 0.0 + t * (-16.0 * scale0);
317 float ty0 = 0.0 + t * (-48.0 * scale0);
319 top0 = top0 * scale0 + ty0;
320 bottom0 = bottom0 * scale0 + ty0;
321 left0 = left0 * scale0 + tx0;
322 right0 = right0 * scale0 + tx0;
324 top1 = top1 * scale0 + ty0;
325 bottom1 = bottom1 * scale0 + ty0;
326 left1 = left1 * scale0 + tx0;
327 right1 = right1 * scale0 + tx0;
331 place_rectangle(resample_effect, padding_effect, left0, top0, right0, bottom0);
332 place_rectangle(resample2_effect, padding2_effect, left1, top1, right1, bottom1);
335 CaptureCard card_copy[NUM_CARDS];
338 std::unique_lock<std::mutex> lock(bmusb_mutex);
340 // The first card is the master timer, so wait for it to have a new frame.
341 // TODO: Make configurable, and with a timeout.
342 cards[0].new_data_ready_changed.wait(lock, [this]{ return cards[0].new_data_ready; });
344 for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
345 CaptureCard *card = &cards[card_index];
346 card_copy[card_index].usb = card->usb;
347 card_copy[card_index].new_data_ready = card->new_data_ready;
348 card_copy[card_index].new_frame = card->new_frame;
349 card_copy[card_index].new_data_ready_fence = card->new_data_ready_fence;
350 card->new_data_ready = false;
351 card->new_data_ready_changed.notify_all();
355 for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
356 CaptureCard *card = &card_copy[card_index];
357 if (!card->new_data_ready)
360 assert(card->new_frame != nullptr);
361 bmusb_current_rendering_frame[card_index] = card->new_frame;
364 // The new texture might still be uploaded,
365 // tell the GPU to wait until it's there.
366 if (card->new_data_ready_fence)
367 glWaitSync(card->new_data_ready_fence, /*flags=*/0, GL_TIMEOUT_IGNORED);
369 glDeleteSync(card->new_data_ready_fence);
371 const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)card->new_frame->userdata;
372 theme->set_input_textures(card_index, userdata->tex_y, userdata->tex_cbcr);
375 // Get the main chain from the theme, and set its state immediately.
376 pair<EffectChain *, function<void()>> theme_main_chain = theme->get_chain(0, frame / 60.0f, WIDTH, HEIGHT);
377 EffectChain *chain = theme_main_chain.first;
378 theme_main_chain.second();
380 GLuint y_tex, cbcr_tex;
381 bool got_frame = h264_encoder->begin_frame(&y_tex, &cbcr_tex);
384 // Render main chain.
385 GLuint cbcr_full_tex = resource_pool->create_2d_texture(GL_RG8, WIDTH, HEIGHT);
386 GLuint rgba_tex = resource_pool->create_2d_texture(GL_RGB565, WIDTH, HEIGHT); // Saves texture bandwidth, although dithering gets messed up.
387 GLuint fbo = resource_pool->create_fbo(y_tex, cbcr_full_tex, rgba_tex);
388 chain->render_to_fbo(fbo, WIDTH, HEIGHT);
389 resource_pool->release_fbo(fbo);
391 subsample_chroma(cbcr_full_tex, cbcr_tex);
392 resource_pool->release_2d_texture(cbcr_full_tex);
394 // Set the right state for rgba_tex.
395 glBindFramebuffer(GL_FRAMEBUFFER, 0);
396 glBindTexture(GL_TEXTURE_2D, rgba_tex);
397 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
398 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
399 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
401 RefCountedGLsync fence(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
404 // Make sure the H.264 gets a reference to all the
405 // input frames needed, so that they are not released back
406 // until the rendering is done.
407 vector<RefCountedFrame> input_frames;
408 for (int card_index = 0; card_index < NUM_CARDS; ++card_index) {
409 input_frames.push_back(bmusb_current_rendering_frame[card_index]);
411 h264_encoder->end_frame(fence, input_frames);
413 // The live frame just shows the RGBA texture we just rendered.
414 // It owns rgba_tex now.
415 DisplayFrame live_frame;
416 live_frame.chain = display_chain.get();
417 live_frame.setup_chain = [this, rgba_tex]{
418 display_input->set_texture_num(rgba_tex);
420 live_frame.ready_fence = fence;
421 live_frame.input_frames = {};
422 live_frame.temp_textures = { rgba_tex };
423 output_channel[OUTPUT_LIVE].output_frame(live_frame);
425 // Set up non-live inputs.
426 for (unsigned i = 1; i < 4; ++i) { // FIXME: Don't lock to 4, ask Lua.
427 DisplayFrame display_frame;
428 pair<EffectChain *, function<void()>> chain = theme->get_chain(i, frame / 60.0f, WIDTH, HEIGHT); // FIXME: dimensions
429 display_frame.chain = chain.first;
430 display_frame.setup_chain = chain.second;
431 display_frame.ready_fence = fence;
432 display_frame.input_frames = { bmusb_current_rendering_frame[0], bmusb_current_rendering_frame[1] }; // FIXME: possible to do better?
433 display_frame.temp_textures = {};
434 output_channel[i].output_frame(display_frame);
437 clock_gettime(CLOCK_MONOTONIC, &now);
438 double elapsed = now.tv_sec - start.tv_sec +
439 1e-9 * (now.tv_nsec - start.tv_nsec);
440 if (frame % 100 == 0) {
441 printf("%d frames in %.3f seconds = %.1f fps (%.1f ms/frame)\n",
442 frame, elapsed, frame / elapsed,
443 1e3 * elapsed / frame);
444 // chain->print_phase_timing();
447 // Reset every 100 frames, so that local variations in frame times
448 // (especially for the first few frames, when the shaders are
449 // compiled etc.) don't make it hard to measure for the entire
450 // remaining duration of the program.
451 if (frame == 10000) {
459 void Mixer::subsample_chroma(GLuint src_tex, GLuint dst_tex)
462 glGenVertexArrays(1, &vao);
471 glBindVertexArray(vao);
475 GLuint fbo = resource_pool->create_fbo(dst_tex);
476 glBindFramebuffer(GL_FRAMEBUFFER, fbo);
477 glViewport(0, 0, WIDTH/2, HEIGHT/2);
480 glUseProgram(cbcr_program_num);
483 glActiveTexture(GL_TEXTURE0);
485 glBindTexture(GL_TEXTURE_2D, src_tex);
487 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
489 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
491 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
494 float chroma_offset_0[] = { -0.5f / WIDTH, 0.0f };
495 set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_0", chroma_offset_0);
497 GLuint position_vbo = fill_vertex_attribute(cbcr_program_num, "position", 2, GL_FLOAT, sizeof(vertices), vertices);
498 GLuint texcoord_vbo = fill_vertex_attribute(cbcr_program_num, "texcoord", 2, GL_FLOAT, sizeof(vertices), vertices); // Same as vertices.
500 glDrawArrays(GL_TRIANGLES, 0, 3);
503 cleanup_vertex_attribute(cbcr_program_num, "position", position_vbo);
504 cleanup_vertex_attribute(cbcr_program_num, "texcoord", texcoord_vbo);
509 resource_pool->release_fbo(fbo);
510 glDeleteVertexArrays(1, &vao);
513 void Mixer::release_display_frame(DisplayFrame *frame)
515 for (GLuint texnum : frame->temp_textures) {
516 resource_pool->release_2d_texture(texnum);
518 frame->temp_textures.clear();
519 frame->ready_fence.reset();
520 frame->input_frames.clear();
525 mixer_thread = std::thread(&Mixer::thread_func, this);
534 void Mixer::cut(Source source)
536 current_source = source;
539 void Mixer::OutputChannel::output_frame(DisplayFrame frame)
541 // Store this frame for display. Remove the ready frame if any
542 // (it was seemingly never used).
544 std::unique_lock<std::mutex> lock(frame_mutex);
545 if (has_ready_frame) {
546 parent->release_display_frame(&ready_frame);
549 has_ready_frame = true;
552 if (has_new_frame_ready_callback) {
553 new_frame_ready_callback();
557 bool Mixer::OutputChannel::get_display_frame(DisplayFrame *frame)
559 std::unique_lock<std::mutex> lock(frame_mutex);
560 if (!has_current_frame && !has_ready_frame) {
564 if (has_current_frame && has_ready_frame) {
565 // We have a new ready frame. Toss the current one.
566 parent->release_display_frame(¤t_frame);
567 has_current_frame = false;
569 if (has_ready_frame) {
570 assert(!has_current_frame);
571 current_frame = ready_frame;
572 ready_frame.ready_fence.reset(); // Drop the refcount.
573 ready_frame.input_frames.clear(); // Drop the refcounts.
574 has_current_frame = true;
575 has_ready_frame = false;
578 *frame = current_frame;
582 void Mixer::OutputChannel::set_frame_ready_callback(Mixer::new_frame_ready_callback_t callback)
584 new_frame_ready_callback = callback;
585 has_new_frame_ready_callback = true;