From 7f726d57e80de3e18686f0e45482ca48db91e82f Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 12 Feb 2017 19:23:08 +0100 Subject: [PATCH] Add a switch for writing a timecode to the stream; useful for latency debugging. --- Makefile | 2 +- flags.cpp | 12 +++ flags.h | 2 + mainwindow.cpp | 15 ++++ mainwindow.h | 2 + mixer.cpp | 28 ++++-- mixer.h | 15 ++++ timecode_renderer.cpp | 204 ++++++++++++++++++++++++++++++++++++++++++ timecode_renderer.h | 48 ++++++++++ ui_mainwindow.ui | 30 ++++++- 10 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 timecode_renderer.cpp create mode 100644 timecode_renderer.h diff --git a/Makefile b/Makefile index dba6200..970450e 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ OBJS += midi_mapper.o midi_mapping.pb.o # Mixer objects AUDIO_MIXER_OBJS = audio_mixer.o alsa_input.o alsa_pool.o ebu_r128_proc.o stereocompressor.o resampling_queue.o flags.o correlation_measurer.o filter.o input_mapping.o state.pb.o -OBJS += chroma_subsampler.o mixer.o pbo_frame_allocator.o context.o ref_counted_frame.o theme.o httpd.o flags.o image_input.o alsa_output.o disk_space_estimator.o print_latency.o $(AUDIO_MIXER_OBJS) +OBJS += chroma_subsampler.o mixer.o pbo_frame_allocator.o context.o ref_counted_frame.o theme.o httpd.o flags.o image_input.o alsa_output.o disk_space_estimator.o print_latency.o timecode_renderer.o $(AUDIO_MIXER_OBJS) # Streaming and encoding objects OBJS += quicksync_encoder.o x264_encoder.o x264_speed_control.o video_encoder.o metacube2.o mux.o audio_encoder.o ffmpeg_raii.o diff --git a/flags.cpp b/flags.cpp index b926d14..9c91bf7 100644 --- a/flags.cpp +++ b/flags.cpp @@ -51,6 +51,8 @@ enum LongOption { OPTION_OUTPUT_YCBCR_COEFFICIENTS, OPTION_OUTPUT_BUFFER_FRAMES, OPTION_OUTPUT_SLOP_FRAMES, + OPTION_TIMECODE_STREAM, + OPTION_TIMECODE_STDOUT, }; void usage() @@ -119,6 +121,8 @@ void usage() fprintf(stderr, " --output-slop-frames=NUM if more less than this number of frames behind for\n"); fprintf(stderr, " --output-card, try to submit anyway instead of\n"); fprintf(stderr, " dropping the frame (default 0.5)\n"); + fprintf(stderr, " --timecode-stream show timestamp and timecode in stream\n"); + fprintf(stderr, " --timecode-stdout show timestamp and timecode on standard output\n"); } void parse_flags(int argc, char * const argv[]) @@ -171,6 +175,8 @@ void parse_flags(int argc, char * const argv[]) { "output-ycbcr-coefficients", required_argument, 0, OPTION_OUTPUT_YCBCR_COEFFICIENTS }, { "output-buffer-frames", required_argument, 0, OPTION_OUTPUT_BUFFER_FRAMES }, { "output-slop-frames", required_argument, 0, OPTION_OUTPUT_SLOP_FRAMES }, + { "timecode-stream", no_argument, 0, OPTION_TIMECODE_STREAM }, + { "timecode-stdout", no_argument, 0, OPTION_TIMECODE_STDOUT }, { 0, 0, 0, 0 } }; vector theme_dirs; @@ -342,6 +348,12 @@ void parse_flags(int argc, char * const argv[]) case OPTION_OUTPUT_SLOP_FRAMES: global_flags.output_slop_frames = atof(optarg); break; + case OPTION_TIMECODE_STREAM: + global_flags.display_timecode_in_stream = true; + break; + case OPTION_TIMECODE_STDOUT: + global_flags.display_timecode_on_stdout = true; + break; case OPTION_HELP: usage(); exit(0); diff --git a/flags.h b/flags.h index 7bfab5d..91b2221 100644 --- a/flags.h +++ b/flags.h @@ -47,6 +47,8 @@ struct Flags { double output_buffer_frames = 6.0; double output_slop_frames = 0.5; int max_input_queue_frames = 6; + bool display_timecode_in_stream = false; + bool display_timecode_on_stdout = false; }; extern Flags global_flags; diff --git a/mainwindow.cpp b/mainwindow.cpp index fc7b6ea..04a16a8 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -203,6 +203,11 @@ MainWindow::MainWindow() connect(ui->multichannel_audio_mode, &QAction::triggered, this, &MainWindow::multichannel_audio_mode_triggered); connect(ui->input_mapping_action, &QAction::triggered, this, &MainWindow::input_mapping_triggered); connect(ui->midi_mapping_action, &QAction::triggered, this, &MainWindow::midi_mapping_triggered); + connect(ui->timecode_stream_action, &QAction::triggered, this, &MainWindow::timecode_stream_triggered); + connect(ui->timecode_stdout_action, &QAction::triggered, this, &MainWindow::timecode_stdout_triggered); + + ui->timecode_stream_action->setChecked(global_flags.display_timecode_in_stream); + ui->timecode_stdout_action->setChecked(global_flags.display_timecode_on_stdout); if (global_flags.x264_video_to_http) { connect(ui->x264_bitrate_action, &QAction::triggered, this, &MainWindow::x264_bitrate_triggered); @@ -631,6 +636,16 @@ void MainWindow::midi_mapping_triggered() MIDIMappingDialog(&midi_mapper).exec(); } +void MainWindow::timecode_stream_triggered() +{ + global_mixer->set_display_timecode_in_stream(ui->timecode_stream_action->isChecked()); +} + +void MainWindow::timecode_stdout_triggered() +{ + global_mixer->set_display_timecode_on_stdout(ui->timecode_stdout_action->isChecked()); +} + void MainWindow::gain_staging_knob_changed(unsigned bus_index, int value) { if (bus_index == 0) { diff --git a/mainwindow.h b/mainwindow.h index 1c7f0e3..b5e43fd 100644 --- a/mainwindow.h +++ b/mainwindow.h @@ -51,6 +51,8 @@ public slots: void multichannel_audio_mode_triggered(); void input_mapping_triggered(); void midi_mapping_triggered(); + void timecode_stream_triggered(); + void timecode_stdout_triggered(); void transition_clicked(int transition_number); void channel_clicked(int channel_number); void wb_button_clicked(int channel_number); diff --git a/mixer.cpp b/mixer.cpp index 26e0095..26cccb4 100644 --- a/mixer.cpp +++ b/mixer.cpp @@ -45,6 +45,7 @@ #include "ref_counted_gl_sync.h" #include "resampling_queue.h" #include "timebase.h" +#include "timecode_renderer.h" #include "video_encoder.h" class IDeckLink; @@ -199,6 +200,10 @@ Mixer::Mixer(const QSurfaceFormat &format, unsigned num_cards) chroma_subsampler.reset(new ChromaSubsampler(resource_pool.get())); + timecode_renderer.reset(new TimecodeRenderer(resource_pool.get(), global_flags.width, global_flags.height)); + display_timecode_in_stream = global_flags.display_timecode_in_stream; + display_timecode_on_stdout = global_flags.display_timecode_on_stdout; + if (global_flags.enable_alsa_output) { alsa.reset(new ALSAOutput(OUTPUT_FREQUENCY, /*num_channels=*/2)); } @@ -592,7 +597,6 @@ void Mixer::thread_func() steady_clock::time_point start, now; start = steady_clock::now(); - int frame = 0; int stats_dropped_frames = 0; while (!should_quit) { @@ -666,15 +670,15 @@ void Mixer::thread_func() int64_t frame_duration = output_frame_info.frame_duration; render_one_frame(frame_duration); - ++frame; + ++frame_num; pts_int += frame_duration; now = steady_clock::now(); double elapsed = duration(now - start).count(); - if (frame % 100 == 0) { + if (frame_num % 100 == 0) { printf("%d frames (%d dropped) in %.3f seconds = %.1f fps (%.1f ms/frame)", - frame, stats_dropped_frames, elapsed, frame / elapsed, - 1e3 * elapsed / frame); + frame_num, stats_dropped_frames, elapsed, frame_num / elapsed, + 1e3 * elapsed / frame_num); // chain->print_phase_timing(); // Check our memory usage, to see if we are close to our mlockall() @@ -711,7 +715,7 @@ void Mixer::thread_func() if (should_cut.exchange(false)) { // Test and clear. - video_encoder->do_cut(frame); + video_encoder->do_cut(frame_num); } #if 0 @@ -930,6 +934,12 @@ void Mixer::schedule_audio_resampling_tasks(unsigned dropped_frames, int num_sam void Mixer::render_one_frame(int64_t duration) { + // Determine the time code for this frame before we start rendering. + string timecode_text = timecode_renderer->get_timecode_text(double(pts_int) / TIMEBASE, frame_num); + if (display_timecode_on_stdout) { + printf("Timecode: '%s'\n", timecode_text.c_str()); + } + // Get the main chain from the theme, and set its state immediately. Theme::Chain theme_main_chain = theme->get_chain(0, pts(), global_flags.width, global_flags.height, input_state); EffectChain *chain = theme_main_chain.chain; @@ -946,6 +956,12 @@ void Mixer::render_one_frame(int64_t duration) GLuint fbo = resource_pool->create_fbo(y_tex, cbcr_full_tex, rgba_tex); check_error(); chain->render_to_fbo(fbo, global_flags.width, global_flags.height); + + if (display_timecode_in_stream) { + // Render the timecode on top. + timecode_renderer->render_timecode(fbo, timecode_text); + } + resource_pool->release_fbo(fbo); chroma_subsampler->subsample_chroma(cbcr_full_tex, global_flags.width, global_flags.height, cbcr_tex); diff --git a/mixer.h b/mixer.h index 191ea3d..2543c24 100644 --- a/mixer.h +++ b/mixer.h @@ -39,6 +39,7 @@ class ALSAOutput; class ChromaSubsampler; class DeckLinkOutput; +class TimecodeRenderer; class QSurface; class QSurfaceFormat; @@ -321,6 +322,14 @@ public: desired_output_video_mode = mode; } + void set_display_timecode_in_stream(bool enable) { + display_timecode_in_stream = enable; + } + + void set_display_timecode_on_stdout(bool enable) { + display_timecode_on_stdout = enable; + } + private: struct CaptureCard; @@ -335,6 +344,7 @@ private: void thread_func(); void handle_hotplugged_cards(); void schedule_audio_resampling_tasks(unsigned dropped_frames, int num_samples_per_frame, int length_per_frame, bool is_preroll, std::chrono::steady_clock::time_point frame_timestamp); + std::string get_timecode_text() const; void render_one_frame(int64_t duration); void audio_thread_func(); void release_display_frame(DisplayFrame *frame); @@ -366,10 +376,15 @@ private: std::unique_ptr chroma_subsampler; std::unique_ptr video_encoder; + std::unique_ptr timecode_renderer; + std::atomic display_timecode_in_stream{false}; + std::atomic display_timecode_on_stdout{false}; + // Effects part of . Owned by . movit::FlatInput *display_input; int64_t pts_int = 0; // In TIMEBASE units. + unsigned frame_num = 0; // Accumulated errors in number of 1/TIMEBASE audio samples. If OUTPUT_FREQUENCY divided by // frame rate is integer, will always stay zero. diff --git a/timecode_renderer.cpp b/timecode_renderer.cpp new file mode 100644 index 0000000..218120e --- /dev/null +++ b/timecode_renderer.cpp @@ -0,0 +1,204 @@ +#include "timecode_renderer.h" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; +using namespace movit; + +TimecodeRenderer::TimecodeRenderer(movit::ResourcePool *resource_pool, unsigned display_width, unsigned display_height) + : resource_pool(resource_pool), display_width(display_width), display_height(display_height), height(28) +{ + string vert_shader = + "#version 130 \n" + " \n" + "in vec2 position; \n" + "in vec2 texcoord; \n" + "out vec2 tc0; \n" + " \n" + "void main() \n" + "{ \n" + " // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is: \n" + " // \n" + " // 2.000 0.000 0.000 -1.000 \n" + " // 0.000 2.000 0.000 -1.000 \n" + " // 0.000 0.000 -2.000 -1.000 \n" + " // 0.000 0.000 0.000 1.000 \n" + " gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0); \n" + " tc0 = texcoord; \n" + "} \n"; + string frag_shader = + "#version 130 \n" + "in vec2 tc0; \n" + "uniform sampler2D tex; \n" + "out vec4 Y, CbCr, RGBA; \n" + "void main() { \n" + " vec4 gray = texture(tex, tc0); \n" + " RGBA = gray.rrra; \n" + " gray.r = gray.r * ((235.0-16.0)/255.0) + 16.0/255.0; \n" // Limited-range Y'CbCr. + " Y = gray.rrra; \n" + " CbCr = vec4(128.0/255.0, 128.0/255.0, 0.0, 1.0); \n" + "} \n"; + + vector frag_shader_outputs; + program_num = resource_pool->compile_glsl_program(vert_shader, frag_shader, frag_shader_outputs); + check_error(); + + texture_sampler_uniform = glGetUniformLocation(program_num, "tex"); + check_error(); + position_attribute_index = glGetAttribLocation(program_num, "position"); + check_error(); + texcoord_attribute_index = glGetAttribLocation(program_num, "texcoord"); + check_error(); + + // Shared between the two. + float vertices[] = { + 0.0f, 2.0f, + 0.0f, 0.0f, + 2.0f, 0.0f + }; + vbo = generate_vbo(2, GL_FLOAT, sizeof(vertices), vertices); + check_error(); + + tex = resource_pool->create_2d_texture(GL_R8, display_width, height); + + image.reset(new QImage(display_width, height, QImage::Format_Grayscale8)); +} + +TimecodeRenderer::~TimecodeRenderer() +{ + resource_pool->release_2d_texture(tex); + check_error(); + resource_pool->release_glsl_program(program_num); + check_error(); + glDeleteBuffers(1, &vbo); + check_error(); +} + +string TimecodeRenderer::get_timecode_text(double pts, unsigned frame_num) +{ + // Find the wall time, and round it to the nearest millisecond. + timeval now; + gettimeofday(&now, nullptr); + time_t unixtime = now.tv_sec; + unsigned msecs = (now.tv_usec + 500) / 1000; + if (msecs >= 1000) { + msecs -= 1000; + ++unixtime; + } + + tm utc_tm; + gmtime_r(&unixtime, &utc_tm); + char clock_text[256]; + strftime(clock_text, sizeof(clock_text), "%Y-%m-%d %H:%M:%S", &utc_tm); + + // Make the stream timecode, rounded to the nearest millisecond. + long stream_time = lrint(pts * 1e3); + assert(stream_time >= 0); + unsigned stream_time_ms = stream_time % 1000; + stream_time /= 1000; + unsigned stream_time_sec = stream_time % 60; + stream_time /= 60; + unsigned stream_time_min = stream_time % 60; + unsigned stream_time_hour = stream_time / 60; + + char timecode_text[256]; + snprintf(timecode_text, sizeof(timecode_text), "Nageru - %s.%03u UTC - Stream time %02u:%02u:%02u.%03u (frame %u)", + clock_text, msecs, stream_time_hour, stream_time_min, stream_time_sec, stream_time_ms, frame_num); + return timecode_text; +} + +void TimecodeRenderer::render_timecode(GLuint fbo, const string &text) +{ + render_string_to_buffer(text); + render_buffer_to_fbo(fbo); +} + +void TimecodeRenderer::render_string_to_buffer(const string &text) +{ + image->fill(0); + QPainter painter(image.get()); + + painter.setPen(Qt::white); + QFont font = painter.font(); + font.setPointSize(16); + painter.setFont(font); + + painter.drawText(QRectF(0, 0, display_width, height), Qt::AlignCenter, QString::fromStdString(text)); +} + +void TimecodeRenderer::render_buffer_to_fbo(GLuint fbo) +{ + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + check_error(); + + GLuint vao; + glGenVertexArrays(1, &vao); + check_error(); + + glBindVertexArray(vao); + check_error(); + + glViewport(0, display_height - height, display_width, height); + check_error(); + + glActiveTexture(GL_TEXTURE0); + check_error(); + glBindTexture(GL_TEXTURE_2D, tex); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + check_error(); + + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, display_width, height, GL_RED, GL_UNSIGNED_BYTE, image->bits()); + check_error(); + + glUseProgram(program_num); + check_error(); + glUniform1i(texture_sampler_uniform, 0); + check_error(); + + glBindBuffer(GL_ARRAY_BUFFER, vbo); + check_error(); + + for (GLint attr_index : { position_attribute_index, texcoord_attribute_index }) { + if (attr_index == -1) continue; + glEnableVertexAttribArray(attr_index); + check_error(); + glVertexAttribPointer(attr_index, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0)); + check_error(); + } + + glDrawArrays(GL_TRIANGLES, 0, 3); + check_error(); + + for (GLint attr_index : { position_attribute_index, texcoord_attribute_index }) { + if (attr_index == -1) continue; + glDisableVertexAttribArray(attr_index); + check_error(); + } + + glActiveTexture(GL_TEXTURE0); + check_error(); + glUseProgram(0); + check_error(); + + glDeleteVertexArrays(1, &vao); + check_error(); + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + check_error(); +} diff --git a/timecode_renderer.h b/timecode_renderer.h new file mode 100644 index 0000000..809a829 --- /dev/null +++ b/timecode_renderer.h @@ -0,0 +1,48 @@ +#ifndef _TIMECODE_RENDERER_H +#define _TIMECODE_RENDERER_H 1 + +#include +#include + +#include + +// A class to render a simple text string onto the picture using Qt and OpenGL. + +namespace movit { + +class ResourcePool; + +} // namespace movit + +class QImage; + +class TimecodeRenderer { +public: + TimecodeRenderer(movit::ResourcePool *resource_pool, unsigned display_width, unsigned display_height); + ~TimecodeRenderer(); + + // Return a string with the current wall clock time and the + // logical stream time. + static std::string get_timecode_text(double pts, unsigned frame_num); + + // The FBO is assumed to contain three outputs (Y', Cb/Cr and RGBA). + void render_timecode(GLuint fbo, const std::string &text); + +private: + void render_string_to_buffer(const std::string &text); + void render_buffer_to_fbo(GLuint fbo); + + movit::ResourcePool *resource_pool; + unsigned display_width, display_height, height; + + GLuint vbo; // Holds position and texcoord data. + GLuint tex; + //std::unique_ptr pixel_buffer; + std::unique_ptr image; + + GLuint program_num; // Owned by . + GLuint texture_sampler_uniform; + GLuint position_attribute_index, texcoord_attribute_index; +}; + +#endif diff --git a/ui_mainwindow.ui b/ui_mainwindow.ui index 15e4361..b11141e 100644 --- a/ui_mainwindow.ui +++ b/ui_mainwindow.ui @@ -640,8 +640,8 @@ 0 0 - 514 - 238 + 505 + 236 @@ -1390,15 +1390,23 @@ 0 0 1089 - 19 + 23 &Video + + + Display &time code + + + + + @@ -1476,6 +1484,22 @@ Online &manual… + + + true + + + In &stream + + + + + true + + + On standard &output + + -- 2.39.2