From a6b07abd7ffa9583d7157ea9d8cff8deff06f7b4 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 10 Mar 2019 12:04:27 +0100 Subject: [PATCH] For FFmpeg inputs, add an option for playing as fast as possible. This is intended for live streams, where setting rate 2.0 or similar would cause it to spew errors and keep resetting the clock. This mode is automatically activated if rate >= 10.0. --- nageru/ffmpeg_capture.cpp | 106 +++++++++++++++++++++----------------- nageru/ffmpeg_capture.h | 1 + nageru/kaeru.cpp | 2 +- 3 files changed, 61 insertions(+), 48 deletions(-) diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 9beea85..b4fec06 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -503,56 +503,67 @@ bool FFmpegCapture::play_video(const string &pathname) if (last_pts == 0 && pts_origin == 0) { pts_origin = frame->pts; } - next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); - if (first_frame && last_frame_was_connected) { - // If reconnect took more than one second, this is probably a live feed, - // and we should reset the resampler. (Or the rate is really, really low, - // in which case a reset on the first frame is fine anyway.) - if (duration(next_frame_start - last_frame).count() >= 1.0) { - last_frame_was_connected = false; + steady_clock::time_point now = steady_clock::now(); + if (play_as_fast_as_possible) { + video_frame->received_timestamp = now; + audio_frame->received_timestamp = now; + next_frame_start = now; + } else { + next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); + if (first_frame && last_frame_was_connected) { + // If reconnect took more than one second, this is probably a live feed, + // and we should reset the resampler. (Or the rate is really, really low, + // in which case a reset on the first frame is fine anyway.) + if (duration(next_frame_start - last_frame).count() >= 1.0) { + last_frame_was_connected = false; + } + } + video_frame->received_timestamp = next_frame_start; + + // The easiest way to get all the rate conversions etc. right is to move the + // audio PTS into the video PTS timebase and go from there. (We'll get some + // rounding issues, but they should not be a big problem.) + int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); + audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); + + if (audio_frame->len != 0) { + // The received timestamps in Nageru are measured after we've just received the frame. + // However, pts (especially audio pts) is at the _beginning_ of the frame. + // If we have locked audio, the distinction doesn't really matter, as pts is + // on a relative scale and a fixed offset is fine. But if we don't, we will have + // a different number of samples each time, which will cause huge audio jitter + // and throw off the resampler. + // + // In a sense, we should have compensated by adding the frame and audio lengths + // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, + // but that would mean extra waiting in sleep_until(). All we need is that they + // are correct relative to each other, though (and to the other frames we send), + // so just align the end of the audio frame, and we're fine. + size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; + double offset = double(num_samples) / OUTPUT_FREQUENCY - + double(video_format.frame_rate_den) / video_format.frame_rate_nom; + audio_frame->received_timestamp += duration_cast(duration(offset)); } - } - video_frame->received_timestamp = next_frame_start; - - // The easiest way to get all the rate conversions etc. right is to move the - // audio PTS into the video PTS timebase and go from there. (We'll get some - // rounding issues, but they should not be a big problem.) - int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); - audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); - - if (audio_frame->len != 0) { - // The received timestamps in Nageru are measured after we've just received the frame. - // However, pts (especially audio pts) is at the _beginning_ of the frame. - // If we have locked audio, the distinction doesn't really matter, as pts is - // on a relative scale and a fixed offset is fine. But if we don't, we will have - // a different number of samples each time, which will cause huge audio jitter - // and throw off the resampler. - // - // In a sense, we should have compensated by adding the frame and audio lengths - // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, - // but that would mean extra waiting in sleep_until(). All we need is that they - // are correct relative to each other, though (and to the other frames we send), - // so just align the end of the audio frame, and we're fine. - size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; - double offset = double(num_samples) / OUTPUT_FREQUENCY - - double(video_format.frame_rate_den) / video_format.frame_rate_nom; - audio_frame->received_timestamp += duration_cast(duration(offset)); - } - steady_clock::time_point now = steady_clock::now(); - if (duration(now - next_frame_start).count() >= 0.1) { - // If we don't have enough CPU to keep up, or if we have a live stream - // where the initial origin was somehow wrong, we could be behind indefinitely. - // In particular, this will give the audio resampler problems as it tries - // to speed up to reduce the delay, hitting the low end of the buffer every time. - fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", - pathname.c_str(), - 1e3 * duration(now - next_frame_start).count()); - pts_origin = frame->pts; - start = next_frame_start = now; - timecode += MAX_FPS * 2 + 1; + if (duration(now - next_frame_start).count() >= 0.1) { + // If we don't have enough CPU to keep up, or if we have a live stream + // where the initial origin was somehow wrong, we could be behind indefinitely. + // In particular, this will give the audio resampler problems as it tries + // to speed up to reduce the delay, hitting the low end of the buffer every time. + fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", + pathname.c_str(), + 1e3 * duration(now - next_frame_start).count()); + pts_origin = frame->pts; + start = next_frame_start = now; + timecode += MAX_FPS * 2 + 1; + } + } + bool finished_wakeup; + if (play_as_fast_as_possible) { + finished_wakeup = !producer_thread_should_quit.should_quit(); + } else { + finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); } - bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); if (finished_wakeup) { if (audio_frame->len > 0) { assert(audio_pts != -1); @@ -636,6 +647,7 @@ bool FFmpegCapture::process_queued_commands(AVFormatContext *format_ctx, const s start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate); pts_origin = last_pts; rate = cmd.new_rate; + play_as_fast_as_possible = (rate >= 10.0); break; } } diff --git a/nageru/ffmpeg_capture.h b/nageru/ffmpeg_capture.h index a9a8371..c7b8a61 100644 --- a/nageru/ffmpeg_capture.h +++ b/nageru/ffmpeg_capture.h @@ -254,6 +254,7 @@ private: bool running = false; int card_index = -1; double rate = 1.0; + bool play_as_fast_as_possible = false; // Activated iff rate >= 10.0. std::atomic should_interrupt{false}; bool last_frame_was_connected = true; diff --git a/nageru/kaeru.cpp b/nageru/kaeru.cpp index 9d97383..843bd81 100644 --- a/nageru/kaeru.cpp +++ b/nageru/kaeru.cpp @@ -209,7 +209,7 @@ int main(int argc, char *argv[]) } video.configure_card(); video.start_bm_capture(); - video.change_rate(2.0); // Be sure never to really fall behind, but also don't dump huge amounts of stuff onto x264. + video.change_rate(10.0); // Play as fast as possible. BasicStats basic_stats(/*verbose=*/false, /*use_opengl=*/false); global_basic_stats = &basic_stats; -- 2.39.2