From 25f6b207f157c081495d7abd4ea314b5ef760322 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 9 Apr 2017 00:52:33 +0200 Subject: [PATCH] Add support for RGBA frame types. Nothing produces such frames yet, but now, we would be capable of receiving them. --- bmusb | 2 +- mixer.cpp | 44 ++++++++++++++--- pbo_frame_allocator.cpp | 57 ++++++++++++++++++---- pbo_frame_allocator.h | 1 + theme.cpp | 103 +++++++++++++++++++++++++--------------- theme.h | 11 +++-- 6 files changed, 159 insertions(+), 59 deletions(-) diff --git a/bmusb b/bmusb index 0853ce1..824260a 160000 --- a/bmusb +++ b/bmusb @@ -1 +1 @@ -Subproject commit 0853ce1f4e8632d1190569d53c1f5825d2812eb1 +Subproject commit 824260a0732a49b5a18f2e16e04b0373c37a3bea diff --git a/mixer.cpp b/mixer.cpp index b00c4af..317c762 100644 --- a/mixer.cpp +++ b/mixer.cpp @@ -81,10 +81,18 @@ void insert_new_frame(RefCountedFrame frame, unsigned field_num, bool interlaced void ensure_texture_resolution(PBOFrameAllocator::Userdata *userdata, unsigned field, unsigned width, unsigned height, unsigned v210_width) { bool first; - if (userdata->pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (userdata->pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: first = userdata->tex_v210[field] == 0 || userdata->tex_444[field] == 0; - } else { + break; + case bmusb::PixelFormat_8BitYCbCr: first = userdata->tex_y[field] == 0 || userdata->tex_cbcr[field] == 0; + break; + case bmusb::PixelFormat_8BitRGBA: + first = userdata->tex_rgba[field] == 0; + break; + default: + assert(false); } if (first || @@ -93,12 +101,14 @@ void ensure_texture_resolution(PBOFrameAllocator::Userdata *userdata, unsigned f // We changed resolution since last use of this texture, so we need to create // a new object. Note that this each card has its own PBOFrameAllocator, // we don't need to worry about these flip-flopping between resolutions. - if (userdata->pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (userdata->pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: glBindTexture(GL_TEXTURE_2D, userdata->tex_444[field]); check_error(); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, nullptr); check_error(); - } else { + break; + case bmusb::PixelFormat_8BitYCbCr: { size_t cbcr_width = width / 2; glBindTexture(GL_TEXTURE_2D, userdata->tex_cbcr[field]); @@ -109,6 +119,14 @@ void ensure_texture_resolution(PBOFrameAllocator::Userdata *userdata, unsigned f check_error(); glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); check_error(); + break; + } + case bmusb::PixelFormat_8BitRGBA: + glBindTexture(GL_TEXTURE_2D, userdata->tex_rgba[field]); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + check_error(); + break; } userdata->last_width[field] = width; userdata->last_height[field] = height; @@ -591,24 +609,36 @@ void Mixer::bm_frame(unsigned card_index, uint16_t timecode, field_start_line = video_format.extra_lines_top; } - // For 8-bit input, v210_width will be nonsensical but not used. + // For anything not FRAME_FORMAT_YCBCR_10BIT, v210_width will be nonsensical but not used. size_t v210_width = video_format.stride / sizeof(uint32_t); ensure_texture_resolution(userdata, field, video_format.width, video_format.height, v210_width); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, userdata->pbo); check_error(); - if (userdata->pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (userdata->pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: { size_t field_start = video_offset + video_format.stride * field_start_line; upload_texture(userdata->tex_v210[field], v210_width, video_format.height, video_format.stride, interlaced_stride, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, field_start); v210_converter->convert(userdata->tex_v210[field], userdata->tex_444[field], video_format.width, video_format.height); - } else { + break; + } + case bmusb::PixelFormat_8BitYCbCr: { size_t field_y_start = y_offset + video_format.width * field_start_line; size_t field_cbcr_start = cbcr_offset + cbcr_width * field_start_line * sizeof(uint16_t); // Make up our own strides, since we are interleaving. upload_texture(userdata->tex_y[field], video_format.width, video_format.height, video_format.width, interlaced_stride, GL_RED, GL_UNSIGNED_BYTE, field_y_start); upload_texture(userdata->tex_cbcr[field], cbcr_width, video_format.height, cbcr_width * sizeof(uint16_t), interlaced_stride, GL_RG, GL_UNSIGNED_BYTE, field_cbcr_start); + break; + } + case bmusb::PixelFormat_8BitRGBA: { + size_t field_start = video_offset + video_format.stride * field_start_line; + upload_texture(userdata->tex_rgba[field], video_format.width, video_format.height, video_format.stride, interlaced_stride, GL_RGBA, GL_UNSIGNED_BYTE, field_start); + break; + } + default: + assert(false); } glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); diff --git a/pbo_frame_allocator.cpp b/pbo_frame_allocator.cpp index 87f11eb..e27aa69 100644 --- a/pbo_frame_allocator.cpp +++ b/pbo_frame_allocator.cpp @@ -37,22 +37,30 @@ PBOFrameAllocator::PBOFrameAllocator(bmusb::PixelFormat pixel_format, size_t fra // For 8-bit Y'CbCr, we ask the driver to split Y' and Cb/Cr // into separate textures. For 10-bit, the input format (v210) // is complicated enough that we need to interpolate up to 4:4:4, - // which we do in a compute shader ourselves. + // which we do in a compute shader ourselves. For RGBA, the data + // is already 4:4:4:4. frame.interleaved = (pixel_format == bmusb::PixelFormat_8BitYCbCr); // Create textures. We don't allocate any data for the second field at this point // (just create the texture state with the samplers), since our default assumed // resolution is progressive. - if (pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (pixel_format) { + case bmusb::PixelFormat_8BitYCbCr: + glGenTextures(2, userdata[i].tex_y); + check_error(); + glGenTextures(2, userdata[i].tex_cbcr); + check_error(); + break; + case bmusb::PixelFormat_10BitYCbCr: glGenTextures(2, userdata[i].tex_v210); check_error(); glGenTextures(2, userdata[i].tex_444); check_error(); - } else { - glGenTextures(2, userdata[i].tex_y); - check_error(); - glGenTextures(2, userdata[i].tex_cbcr); + break; + case bmusb::PixelFormat_8BitRGBA: + glGenTextures(2, userdata[i].tex_rgba); check_error(); + break; } userdata[i].last_width[0] = width; @@ -67,7 +75,8 @@ PBOFrameAllocator::PBOFrameAllocator(bmusb::PixelFormat pixel_format, size_t fra userdata[i].last_has_signal = false; userdata[i].last_is_connected = false; for (unsigned field = 0; field < 2; ++field) { - if (pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: { const size_t v210_width = v210Converter::get_minimum_v210_texture_width(width); // Seemingly we need to set the minification filter even though @@ -95,7 +104,9 @@ PBOFrameAllocator::PBOFrameAllocator(bmusb::PixelFormat pixel_format, size_t fra glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB10_A2, width, height, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, NULL); check_error(); } - } else { + break; + } + case bmusb::PixelFormat_8BitYCbCr: glBindTexture(GL_TEXTURE_2D, userdata[i].tex_y[field]); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); @@ -121,6 +132,23 @@ PBOFrameAllocator::PBOFrameAllocator(bmusb::PixelFormat pixel_format, size_t fra glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, width / 2, height, 0, GL_RG, GL_UNSIGNED_BYTE, NULL); check_error(); } + break; + case bmusb::PixelFormat_8BitRGBA: + glBindTexture(GL_TEXTURE_2D, userdata[i].tex_rgba[field]); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + if (field == 0) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); + check_error(); + } + break; + default: + assert(false); } } @@ -146,16 +174,25 @@ PBOFrameAllocator::~PBOFrameAllocator() check_error(); glDeleteBuffers(1, &pbo); check_error(); - if (pixel_format == bmusb::PixelFormat_10BitYCbCr) { + switch (pixel_format) { + case bmusb::PixelFormat_10BitYCbCr: glDeleteTextures(2, ((Userdata *)frame.userdata)->tex_v210); check_error(); glDeleteTextures(2, ((Userdata *)frame.userdata)->tex_444); check_error(); - } else { + break; + case bmusb::PixelFormat_8BitYCbCr: glDeleteTextures(2, ((Userdata *)frame.userdata)->tex_y); check_error(); glDeleteTextures(2, ((Userdata *)frame.userdata)->tex_cbcr); check_error(); + break; + case bmusb::PixelFormat_8BitRGBA: + glDeleteTextures(2, ((Userdata *)frame.userdata)->tex_rgba); + check_error(); + break; + default: + assert(false); } } } diff --git a/pbo_frame_allocator.h b/pbo_frame_allocator.h index 4ee3baa..118ecd6 100644 --- a/pbo_frame_allocator.h +++ b/pbo_frame_allocator.h @@ -40,6 +40,7 @@ public: // The second set is only used for the second field of interlaced inputs. GLuint tex_y[2], tex_cbcr[2]; // For FRAME_FORMAT_YCBCR_8BIT. GLuint tex_v210[2], tex_444[2]; // For FRAME_FORMAT_YCBCR_10BIT. + GLuint tex_rgba[2]; // For FRAME_FORMAT_RGBA_8BIT. GLuint last_width[2], last_height[2]; GLuint last_v210_width[2]; // FRAME_FORMAT_YCBCR_10BIT. bool last_interlaced, last_has_signal, last_is_connected; diff --git a/theme.cpp b/theme.cpp index 9e04740..b6a19ac 100644 --- a/theme.cpp +++ b/theme.cpp @@ -612,22 +612,6 @@ LiveInputWrapper::LiveInputWrapper(Theme *theme, EffectChain *chain, bmusb::Pixe // So we pick sRGB as the least evil here. inout_format.gamma_curve = GAMMA_sRGB; - // The Blackmagic driver docs claim that the device outputs Y'CbCr - // according to Rec. 601, but practical testing indicates it definitely - // is Rec. 709 (at least up to errors attributable to rounding errors). - // Perhaps 601 was only to indicate the subsampling positions, not the - // colorspace itself? Tested with a Lenovo X1 gen 3 as input. - YCbCrFormat input_ycbcr_format; - input_ycbcr_format.chroma_subsampling_x = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1 : 2; - input_ycbcr_format.chroma_subsampling_y = 1; - input_ycbcr_format.num_levels = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1024 : 256; - input_ycbcr_format.cb_x_position = 0.0; - input_ycbcr_format.cr_x_position = 0.0; - input_ycbcr_format.cb_y_position = 0.5; - input_ycbcr_format.cr_y_position = 0.5; - input_ycbcr_format.luma_coefficients = YCBCR_REC_709; - input_ycbcr_format.full_range = false; - unsigned num_inputs; if (deinterlace) { deinterlace_effect = new movit::DeinterlaceEffect(); @@ -643,20 +627,50 @@ LiveInputWrapper::LiveInputWrapper(Theme *theme, EffectChain *chain, bmusb::Pixe } else { num_inputs = 1; } - for (unsigned i = 0; i < num_inputs; ++i) { - // When using 10-bit input, we're converting to interleaved through v210Converter. - YCbCrInputSplitting splitting = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? YCBCR_INPUT_INTERLEAVED : YCBCR_INPUT_SPLIT_Y_AND_CBCR; - if (override_bounce) { - inputs.push_back(new NonBouncingYCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); - } else { - inputs.push_back(new YCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); + + if (pixel_format == bmusb::PixelFormat_8BitRGBA) { + for (unsigned i = 0; i < num_inputs; ++i) { + rgba_inputs.push_back(new FlatInput(inout_format, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, global_flags.width, global_flags.height)); + chain->add_input(rgba_inputs.back()); } - chain->add_input(inputs.back()); - } - if (deinterlace) { - vector reverse_inputs(inputs.rbegin(), inputs.rend()); - chain->add_effect(deinterlace_effect, reverse_inputs); + if (deinterlace) { + vector reverse_inputs(rgba_inputs.rbegin(), rgba_inputs.rend()); + chain->add_effect(deinterlace_effect, reverse_inputs); + } + } else { + assert(pixel_format == bmusb::PixelFormat_8BitYCbCr || pixel_format == bmusb::PixelFormat_10BitYCbCr); + // The Blackmagic driver docs claim that the device outputs Y'CbCr + // according to Rec. 601, but practical testing indicates it definitely + // is Rec. 709 (at least up to errors attributable to rounding errors). + // Perhaps 601 was only to indicate the subsampling positions, not the + // colorspace itself? Tested with a Lenovo X1 gen 3 as input. + YCbCrFormat input_ycbcr_format; + input_ycbcr_format.chroma_subsampling_x = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1 : 2; + input_ycbcr_format.chroma_subsampling_y = 1; + input_ycbcr_format.num_levels = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? 1024 : 256; + input_ycbcr_format.cb_x_position = 0.0; + input_ycbcr_format.cr_x_position = 0.0; + input_ycbcr_format.cb_y_position = 0.5; + input_ycbcr_format.cr_y_position = 0.5; + input_ycbcr_format.luma_coefficients = YCBCR_REC_709; + input_ycbcr_format.full_range = false; + + for (unsigned i = 0; i < num_inputs; ++i) { + // When using 10-bit input, we're converting to interleaved through v210Converter. + YCbCrInputSplitting splitting = (pixel_format == bmusb::PixelFormat_10BitYCbCr) ? YCBCR_INPUT_INTERLEAVED : YCBCR_INPUT_SPLIT_Y_AND_CBCR; + if (override_bounce) { + ycbcr_inputs.push_back(new NonBouncingYCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); + } else { + ycbcr_inputs.push_back(new YCbCrInput(inout_format, input_ycbcr_format, global_flags.width, global_flags.height, splitting)); + } + chain->add_input(ycbcr_inputs.back()); + } + + if (deinterlace) { + vector reverse_inputs(ycbcr_inputs.rbegin(), ycbcr_inputs.rend()); + chain->add_effect(deinterlace_effect, reverse_inputs); + } } } @@ -682,7 +696,7 @@ void LiveInputWrapper::connect_signal(int signal_num) } BufferedFrame last_good_frame = first_frame; - for (unsigned i = 0; i < inputs.size(); ++i) { + for (unsigned i = 0; i < max(ycbcr_inputs.size(), rgba_inputs.size()); ++i) { BufferedFrame frame = theme->input_state->buffered_frames[signal_num][i]; if (frame.frame == nullptr) { // Not enough data; reuse last frame (well, field). @@ -691,22 +705,35 @@ void LiveInputWrapper::connect_signal(int signal_num) } const PBOFrameAllocator::Userdata *userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; - if (userdata->last_width[frame.field_number] != width || - userdata->last_height[frame.field_number] != height) { + unsigned this_width = userdata->last_width[frame.field_number]; + unsigned this_height = userdata->last_height[frame.field_number]; + if (this_width != width || this_height != height) { // Resolution changed; reuse last frame/field. frame = last_good_frame; userdata = (const PBOFrameAllocator::Userdata *)frame.frame->userdata; } assert(userdata->pixel_format == pixel_format); - if (pixel_format == bmusb::PixelFormat_10BitYCbCr) { - inputs[i]->set_texture_num(0, userdata->tex_444[frame.field_number]); - } else { - inputs[i]->set_texture_num(0, userdata->tex_y[frame.field_number]); - inputs[i]->set_texture_num(1, userdata->tex_cbcr[frame.field_number]); + switch (pixel_format) { + case bmusb::PixelFormat_8BitYCbCr: + ycbcr_inputs[i]->set_texture_num(0, userdata->tex_y[frame.field_number]); + ycbcr_inputs[i]->set_texture_num(1, userdata->tex_cbcr[frame.field_number]); + ycbcr_inputs[i]->set_width(this_width); + ycbcr_inputs[i]->set_height(this_height); + break; + case bmusb::PixelFormat_10BitYCbCr: + ycbcr_inputs[i]->set_texture_num(0, userdata->tex_444[frame.field_number]); + ycbcr_inputs[i]->set_width(this_width); + ycbcr_inputs[i]->set_height(this_height); + break; + case bmusb::PixelFormat_8BitRGBA: + rgba_inputs[i]->set_texture_num(userdata->tex_rgba[frame.field_number]); + rgba_inputs[i]->set_width(this_width); + rgba_inputs[i]->set_height(this_height); + break; + default: + assert(false); } - inputs[i]->set_width(userdata->last_width[frame.field_number]); - inputs[i]->set_height(userdata->last_height[frame.field_number]); last_good_frame = frame; } diff --git a/theme.h b/theme.h index 4394513..2a8b9d6 100644 --- a/theme.h +++ b/theme.h @@ -2,6 +2,7 @@ #define _THEME_H 1 #include +#include #include #include #include @@ -16,9 +17,9 @@ struct InputState; namespace movit { -class ResourcePool; class Effect; class EffectChain; +class ResourcePool; struct ImageFormat; struct YCbCrFormat; } // namespace movit @@ -88,6 +89,7 @@ private: // the mixer, and communicates that state over to the actual YCbCrInput. class LiveInputWrapper { public: + // Note: is irrelevant for PixelFormat_8BitRGBA. LiveInputWrapper(Theme *theme, movit::EffectChain *chain, bmusb::PixelFormat pixel_format, bool override_bounce, bool deinterlace); void connect_signal(int signal_num); @@ -95,15 +97,18 @@ public: { if (deinterlace) { return deinterlace_effect; + } else if (pixel_format == bmusb::PixelFormat_8BitRGBA) { + return rgba_inputs[0]; } else { - return inputs[0]; + return ycbcr_inputs[0]; } } private: Theme *theme; // Not owned by us. bmusb::PixelFormat pixel_format; - std::vector inputs; // Multiple ones if deinterlacing. Owned by the chain. + std::vector ycbcr_inputs; // Multiple ones if deinterlacing. Owned by the chain. + std::vector rgba_inputs; // Multiple ones if deinterlacing. Owned by the chain. movit::Effect *deinterlace_effect = nullptr; // Owned by the chain. bool deinterlace; }; -- 2.39.2