#include <movit/image_format.h>
#include <movit/init.h>
#include <movit/resource_pool.h>
-#include <movit/util.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "alsa_output.h"
#include "bmusb/bmusb.h"
#include "bmusb/fake_capture.h"
+#include "chroma_subsampler.h"
#include "context.h"
#include "decklink_capture.h"
#include "defs.h"
BMUSBCapture::set_card_connected_callback(bind(&Mixer::bm_hotplug_add, this, _1));
BMUSBCapture::start_bm_thread();
- for (card_index = 0; card_index < num_cards; ++card_index) {
+ for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
cards[card_index].queue_length_policy.reset(card_index);
- cards[card_index].capture->start_bm_capture();
}
- // Set up stuff for NV12 conversion.
- //
- // Note: Due to the horizontally co-sited chroma/luma samples in H.264
- // (chrome position is left for horizontal and center for vertical),
- // we need to be a bit careful in our subsampling. A diagram will make
- // this clearer, showing some luma and chroma samples:
- //
- // a b c d
- // +---+---+---+---+
- // | | | | |
- // | Y | Y | Y | Y |
- // | | | | |
- // +---+---+---+---+
- //
- // +-------+-------+
- // | | |
- // | C | C |
- // | | |
- // +-------+-------+
- //
- // Clearly, the rightmost chroma sample here needs to be equivalent to
- // b/4 + c/2 + d/4. (We could also implement more sophisticated filters,
- // of course, but as long as the upsampling is not going to be equally
- // sophisticated, it's probably not worth it.) If we sample once with
- // no mipmapping, we get just c, ie., no actual filtering in the
- // horizontal direction. (For the vertical direction, we can just
- // sample in the middle to get the right filtering.) One could imagine
- // we could use mipmapping (assuming we can create mipmaps cheaply),
- // but then, what we'd get is this:
- //
- // (a+b)/2 (c+d)/2
- // +-------+-------+
- // | | |
- // | Y | Y |
- // | | |
- // +-------+-------+
- //
- // +-------+-------+
- // | | |
- // | C | C |
- // | | |
- // +-------+-------+
- //
- // which ends up sampling equally from a and b, which clearly isn't right. Instead,
- // we need to do two (non-mipmapped) chroma samples, both hitting exactly in-between
- // source pixels.
- //
- // Sampling in-between b and c gives us the sample (b+c)/2, and similarly for c and d.
- // Taking the average of these gives of (b+c)/4 + (c+d)/4 = b/4 + c/2 + d/4, which is
- // exactly what we want.
- //
- // See also http://www.poynton.com/PDFs/Merging_RGB_and_422.pdf, pages 6–7.
-
- // Cb/Cr shader.
- string cbcr_vert_shader =
- "#version 130 \n"
- " \n"
- "in vec2 position; \n"
- "in vec2 texcoord; \n"
- "out vec2 tc0, tc1; \n"
- "uniform vec2 foo_chroma_offset_0; \n"
- "uniform vec2 foo_chroma_offset_1; \n"
- " \n"
- "void main() \n"
- "{ \n"
- " // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n"
- " // \n"
- " // 2.000 0.000 0.000 -1.000 \n"
- " // 0.000 2.000 0.000 -1.000 \n"
- " // 0.000 0.000 -2.000 -1.000 \n"
- " // 0.000 0.000 0.000 1.000 \n"
- " gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n"
- " vec2 flipped_tc = texcoord; \n"
- " tc0 = flipped_tc + foo_chroma_offset_0; \n"
- " tc1 = flipped_tc + foo_chroma_offset_1; \n"
- "} \n";
- string cbcr_frag_shader =
- "#version 130 \n"
- "in vec2 tc0, tc1; \n"
- "uniform sampler2D cbcr_tex; \n"
- "out vec4 FragColor; \n"
- "void main() { \n"
- " FragColor = 0.5 * (texture(cbcr_tex, tc0) + texture(cbcr_tex, tc1)); \n"
- "} \n";
- vector<string> frag_shader_outputs;
- cbcr_program_num = resource_pool->compile_glsl_program(cbcr_vert_shader, cbcr_frag_shader, frag_shader_outputs);
-
- float vertices[] = {
- 0.0f, 2.0f,
- 0.0f, 0.0f,
- 2.0f, 0.0f
- };
- cbcr_vbo = generate_vbo(2, GL_FLOAT, sizeof(vertices), vertices);
- cbcr_texture_sampler_uniform = glGetUniformLocation(cbcr_program_num, "cbcr_tex");
- cbcr_position_attribute_index = glGetAttribLocation(cbcr_program_num, "position");
- cbcr_texcoord_attribute_index = glGetAttribLocation(cbcr_program_num, "texcoord");
+ chroma_subsampler.reset(new ChromaSubsampler(resource_pool.get()));
if (global_flags.enable_alsa_output) {
alsa.reset(new ALSAOutput(OUTPUT_FREQUENCY, /*num_channels=*/2));
Mixer::~Mixer()
{
- resource_pool->release_glsl_program(cbcr_program_num);
- glDeleteBuffers(1, &cbcr_vbo);
BMUSBCapture::stop_bm_thread();
for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
{
- unique_lock<mutex> lock(bmusb_mutex);
+ unique_lock<mutex> lock(card_mutex);
cards[card_index].should_quit = true; // Unblock thread.
cards[card_index].new_frames_changed.notify_all();
}
card->surface = create_surface_with_same_format(mixer_surface);
}
while (!card->new_frames.empty()) card->new_frames.pop();
- card->fractional_samples = 0;
card->last_timecode = -1;
card->capture->configure_card();
// Found a stable signal, so stop scanning.
is_mode_scanning[card_index] = false;
} else {
- static constexpr double switch_time_s = 0.5; // Should be enough time for the signal to stabilize.
+ static constexpr double switch_time_s = 0.1; // Should be enough time for the signal to stabilize.
steady_clock::time_point now = steady_clock::now();
double sec_since_last_switch = duration<double>(steady_clock::now() - last_mode_scan_change[card_index]).count();
if (sec_since_last_switch > switch_time_s) {
// Still send on the information that we _had_ a frame, even though it's corrupted,
// so that pts can go up accordingly.
{
- unique_lock<mutex> lock(bmusb_mutex);
+ unique_lock<mutex> lock(card_mutex);
CaptureCard::NewFrame new_frame;
new_frame.frame = RefCountedFrame(FrameAllocator::Frame());
new_frame.length = frame_length;
unsigned num_fields = video_format.interlaced ? 2 : 1;
steady_clock::time_point frame_upload_start;
+ bool interlaced_stride = false;
if (video_format.interlaced) {
// Send the two fields along as separate frames; the other side will need to add
// a deinterlacer to actually get this right.
assert(frame_length % 2 == 0);
frame_length /= 2;
num_fields = 2;
+ if (video_format.second_field_start == 1) {
+ interlaced_stride = true;
+ }
frame_upload_start = steady_clock::now();
}
userdata->last_interlaced = video_format.interlaced;
// Note that this means we must hold on to the actual frame data in <userdata>
// until the upload command is run, but we hold on to <frame> much longer than that
// (in fact, all the way until we no longer use the texture in rendering).
- auto upload_func = [field, video_format, y_offset, cbcr_offset, cbcr_width, userdata]() {
- unsigned field_start_line = (field == 1) ? video_format.second_field_start : video_format.extra_lines_top + field * (video_format.height + 22);
+ auto upload_func = [field, video_format, y_offset, cbcr_offset, cbcr_width, interlaced_stride, userdata]() {
+ unsigned field_start_line;
+ if (field == 1) {
+ field_start_line = video_format.second_field_start;
+ } else {
+ field_start_line = video_format.extra_lines_top;
+ }
if (userdata->tex_y[field] == 0 ||
userdata->tex_cbcr[field] == 0 ||
glBindTexture(GL_TEXTURE_2D, userdata->tex_cbcr[field]);
check_error();
+ if (interlaced_stride) {
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, cbcr_width * 2);
+ check_error();
+ } else {
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+ check_error();
+ }
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, cbcr_width, video_format.height, GL_RG, GL_UNSIGNED_BYTE, BUFFER_OFFSET(field_cbcr_start));
check_error();
glBindTexture(GL_TEXTURE_2D, userdata->tex_y[field]);
check_error();
+ if (interlaced_stride) {
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, video_format.width * 2);
+ check_error();
+ } else {
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+ check_error();
+ }
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, video_format.width, video_format.height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(field_y_start));
check_error();
glBindTexture(GL_TEXTURE_2D, 0);
check_error();
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
check_error();
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+ check_error();
};
if (field == 1) {
}
{
- unique_lock<mutex> lock(bmusb_mutex);
+ unique_lock<mutex> lock(card_mutex);
CaptureCard::NewFrame new_frame;
new_frame.frame = frame;
new_frame.length = frame_length;
exit(1);
}
+ // Start the actual capture. (We don't want to do it before we're actually ready
+ // to process output frames.)
+ for (unsigned card_index = 0; card_index < num_cards; ++card_index) {
+ cards[card_index].capture->start_bm_capture();
+ }
+
steady_clock::time_point start, now;
start = steady_clock::now();
while (!should_quit) {
CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS];
bool has_new_frame[MAX_VIDEO_CARDS] = { false };
- int num_samples[MAX_VIDEO_CARDS] = { 0 };
unsigned master_card_index = theme->map_signal(master_clock_channel);
assert(master_card_index < num_cards);
- get_one_frame_from_each_card(master_card_index, new_frames, has_new_frame, num_samples);
- schedule_audio_resampling_tasks(new_frames[master_card_index].dropped_frames, num_samples[master_card_index], new_frames[master_card_index].length);
- stats_dropped_frames += new_frames[master_card_index].dropped_frames;
+ OutputFrameInfo output_frame_info = get_one_frame_from_each_card(master_card_index, new_frames, has_new_frame);
+ schedule_audio_resampling_tasks(output_frame_info.dropped_frames, output_frame_info.num_samples, output_frame_info.frame_duration);
+ stats_dropped_frames += output_frame_info.dropped_frames;
handle_hotplugged_cards();
}
}
- int64_t frame_duration = new_frames[master_card_index].length;
+ int64_t frame_duration = output_frame_info.frame_duration;
render_one_frame(frame_duration);
++frame;
pts_int += frame_duration;
assert(false);
}
- printf(", using %ld / %ld MB lockable memory (%.1f%%)",
- long(used.ru_maxrss / 1024),
- long(limit.rlim_cur / 1048576),
- float(100.0 * (used.ru_maxrss * 1024.0) / limit.rlim_cur));
+ if (limit.rlim_cur == 0) {
+ printf(", using %ld MB memory (locked)",
+ long(used.ru_maxrss / 1024));
+ } else {
+ printf(", using %ld / %ld MB lockable memory (%.1f%%)",
+ long(used.ru_maxrss / 1024),
+ long(limit.rlim_cur / 1048576),
+ float(100.0 * (used.ru_maxrss * 1024.0) / limit.rlim_cur));
+ }
} else {
printf(", using %ld MB memory (not locked)",
long(used.ru_maxrss / 1024));
resource_pool->clean_context();
}
-void Mixer::get_one_frame_from_each_card(unsigned master_card_index, CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS], bool has_new_frame[MAX_VIDEO_CARDS], int num_samples[MAX_VIDEO_CARDS])
+Mixer::OutputFrameInfo Mixer::get_one_frame_from_each_card(unsigned master_card_index, CaptureCard::NewFrame new_frames[MAX_VIDEO_CARDS], bool has_new_frame[MAX_VIDEO_CARDS])
{
+ OutputFrameInfo output_frame_info;
+
start:
// The first card is the master timer, so wait for it to have a new frame.
// TODO: Add a timeout.
- unique_lock<mutex> lock(bmusb_mutex);
+ unique_lock<mutex> lock(card_mutex);
cards[master_card_index].new_frames_changed.wait(lock, [this, master_card_index]{ return !cards[master_card_index].new_frames.empty() || cards[master_card_index].capture->get_disconnected(); });
if (cards[master_card_index].new_frames.empty()) {
card->new_frames.pop();
card->new_frames_changed.notify_all();
- int num_samples_times_timebase = OUTPUT_FREQUENCY * new_frames[card_index].length + card->fractional_samples;
- num_samples[card_index] = num_samples_times_timebase / TIMEBASE;
- card->fractional_samples = num_samples_times_timebase % TIMEBASE;
- assert(num_samples[card_index] >= 0);
-
if (card_index == master_card_index) {
// We don't use the queue length policy for the master card,
// but we will if it stops being the master. Thus, clear out
}
}
}
+
+ output_frame_info.dropped_frames = new_frames[master_card_index].dropped_frames;
+ output_frame_info.frame_duration = new_frames[master_card_index].length;
+
+ // This might get off by a fractional sample when changing master card
+ // between ones with different frame rates, but that's fine.
+ int num_samples_times_timebase = OUTPUT_FREQUENCY * output_frame_info.frame_duration + fractional_samples;
+ output_frame_info.num_samples = num_samples_times_timebase / TIMEBASE;
+ fractional_samples = num_samples_times_timebase % TIMEBASE;
+ assert(output_frame_info.num_samples >= 0);
+
+ return output_frame_info;
}
void Mixer::handle_hotplugged_cards()
chain->render_to_fbo(fbo, global_flags.width, global_flags.height);
resource_pool->release_fbo(fbo);
- subsample_chroma(cbcr_full_tex, cbcr_tex);
+ chroma_subsampler->subsample_chroma(cbcr_full_tex, global_flags.width, global_flags.height, cbcr_tex);
resource_pool->release_2d_texture(cbcr_full_tex);
// Set the right state for rgba_tex.
if (alsa) {
alsa->write(samples_out);
}
+ decklink_output->send_audio(task.pts_int, samples_out);
video_encoder->add_audio(task.pts_int, move(samples_out));
}
}
-void Mixer::subsample_chroma(GLuint src_tex, GLuint dst_tex)
-{
- GLuint vao;
- glGenVertexArrays(1, &vao);
- check_error();
-
- glBindVertexArray(vao);
- check_error();
-
- // Extract Cb/Cr.
- GLuint fbo = resource_pool->create_fbo(dst_tex);
- glBindFramebuffer(GL_FRAMEBUFFER, fbo);
- glViewport(0, 0, global_flags.width/2, global_flags.height/2);
- check_error();
-
- glUseProgram(cbcr_program_num);
- check_error();
-
- glActiveTexture(GL_TEXTURE0);
- check_error();
- glBindTexture(GL_TEXTURE_2D, src_tex);
- check_error();
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
- check_error();
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
- check_error();
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
- check_error();
-
- float chroma_offset_0[] = { -1.0f / global_flags.width, 0.0f };
- float chroma_offset_1[] = { -0.0f / global_flags.width, 0.0f };
- set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_0", chroma_offset_0);
- set_uniform_vec2(cbcr_program_num, "foo", "chroma_offset_1", chroma_offset_1);
-
- glUniform1i(cbcr_texture_sampler_uniform, 0);
-
- glBindBuffer(GL_ARRAY_BUFFER, cbcr_vbo);
- check_error();
-
- for (GLint attr_index : { cbcr_position_attribute_index, cbcr_texcoord_attribute_index }) {
- glEnableVertexAttribArray(attr_index);
- check_error();
- glVertexAttribPointer(attr_index, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0));
- check_error();
- }
-
- glDrawArrays(GL_TRIANGLES, 0, 3);
- check_error();
-
- for (GLint attr_index : { cbcr_position_attribute_index, cbcr_texcoord_attribute_index }) {
- glDisableVertexAttribArray(attr_index);
- check_error();
- }
-
- glUseProgram(0);
- check_error();
- glBindFramebuffer(GL_FRAMEBUFFER, 0);
- check_error();
-
- resource_pool->release_fbo(fbo);
- glDeleteVertexArrays(1, &vao);
-}
-
void Mixer::release_display_frame(DisplayFrame *frame)
{
for (GLuint texnum : frame->temp_textures) {
projects / nageru / blobdiff