X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=b9eb982008f166fbc7cab97b0e3acc990ee19340;hb=HEAD;hp=9beea857dd3ed41bdb69df3fae26cfe5123bb305;hpb=d0e576aa0c63a260d4a27d57326c0d16fb7997a2;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 9beea85..14ad91f 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -1,24 +1,49 @@ #include "ffmpeg_capture.h" +#include "defs.h" +#include "shared/shared_defs.h" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include #include #include +#include #include -#include +#include +#include extern "C" { #include +#include +#include +#include #include #include +#include +#include +#include +#include #include #include -#include -#include +#include +#include +#include #include -#include +#include +#include +#include +#include #include } @@ -26,21 +51,34 @@ extern "C" { #include #include #include +#include + +#include +#include +#include #include "bmusb/bmusb.h" +#include "shared/context.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" #include "flags.h" -#include "image_input.h" #include "ref_counted_frame.h" #include "shared/timebase.h" -#define FRAME_SIZE (8 << 20) // 8 MB. +#ifdef HAVE_SRT +#include +#endif using namespace std; using namespace std::chrono; using namespace bmusb; using namespace movit; +using namespace Eigen; + +// Avoid deprecation warnings, but we don't want to drop FFmpeg 5.1 support just yet. +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 30, 100) +#define pkt_duration duration +#endif namespace { @@ -115,8 +153,8 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst if (desc->comp[0].depth != 8) continue; // Same or better chroma resolution only. - int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w; - int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h; + int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w; + int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h; if (chroma_w_diff < 0 || chroma_h_diff < 0) continue; @@ -134,7 +172,7 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst return av_get_pix_fmt(best_format); } -YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg) +YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg, AVColorSpace *last_colorspace, AVChromaLocation *last_chroma_location) { YCbCrFormat format; AVColorSpace colorspace = frame->colorspace; @@ -154,11 +192,14 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.luma_coefficients = (frame->height >= 720 ? YCBCR_REC_709 : YCBCR_REC_601); break; default: - fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", - colorspace); + if (colorspace != *last_colorspace) { + fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + colorspace); + } format.luma_coefficients = YCBCR_REC_709; break; } + *last_colorspace = colorspace; format.full_range = is_full_range(desc); format.num_levels = 1 << desc->comp[0].depth; @@ -191,12 +232,15 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.cb_y_position = 1.0; break; default: - fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing Rec. 709.\n", - frame->chroma_location); + if (frame->chroma_location != *last_chroma_location) { + fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", + frame->chroma_location); + } format.cb_x_position = 0.5; format.cb_y_position = 0.5; break; } + *last_chroma_location = frame->chroma_location; if (is_mjpeg && !format.full_range) { // Limited-range MJPEG is only detected by FFmpeg whenever a special @@ -214,10 +258,36 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f return format; } +RGBTriplet get_neutral_color(AVDictionary *metadata) +{ + if (metadata == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0); + if (entry == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + unsigned x_nom, x_den, y_nom, y_den; + if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) { + fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value); + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + double x = double(x_nom) / x_den; + double y = double(y_nom) / y_den; + double z = 1.0 - x - y; + + Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB); + Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z); + + return RGBTriplet(rgb[0], rgb[1], rgb[2]); +} + } // namespace -FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) - : filename(filename), width(width), height(height), video_timebase{1, 1} +FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height, QSurface *surface) + : filename(filename), width(width), height(height), video_timebase{1, 1}, surface(surface) { description = "Video: " + filename; @@ -226,12 +296,38 @@ FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned he avformat_network_init(); // In case someone wants this. } +#ifdef HAVE_SRT +FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id, QSurface *surface) + : srt_sock(srt_sock), + width(0), // Don't resize; SRT streams typically have stable resolution, and should behave much like regular cards in general. + height(0), + pixel_format(bmusb::PixelFormat_8BitYCbCrPlanar), + video_timebase{1, 1}, + surface(surface) +{ + if (stream_id.empty()) { + description = "SRT stream"; + } else { + description = stream_id; + } + play_as_fast_as_possible = true; + play_once = true; + last_frame = steady_clock::now(); +} +#endif + FFmpegCapture::~FFmpegCapture() { if (has_dequeue_callbacks) { dequeue_cleanup_callback(); } swr_free(&resampler); +#ifdef HAVE_SRT + if (srt_sock != -1) { + srt_close(srt_sock); + } +#endif + delete surface; } void FFmpegCapture::configure_card() @@ -273,12 +369,12 @@ std::map FFmpegCapture::get_available_video_modes() const VideoMode mode; char buf[256]; - snprintf(buf, sizeof(buf), "%ux%u", width, height); + snprintf(buf, sizeof(buf), "%ux%u", sws_last_width, sws_last_height); mode.name = buf; mode.autodetect = false; - mode.width = width; - mode.height = height; + mode.width = sws_last_width; + mode.height = sws_last_height; mode.frame_rate_num = 60; mode.frame_rate_den = 1; mode.interlaced = false; @@ -292,6 +388,21 @@ void FFmpegCapture::producer_thread_func() snprintf(thread_name, sizeof(thread_name), "FFmpeg_C_%d", card_index); pthread_setname_np(pthread_self(), thread_name); + // We need a context in case create_frame() needs to reallocate something. + // (If none is given, we are probably in Kaeru, which uses MallocFrameAllocator + // anyway, which doesn't reallocate currently and definitely doesn't need + // an active OpenGL context to do so.) + QOpenGLContext *context = nullptr; + if (surface != nullptr) { + context = create_context(this->surface); + eglBindAPI(EGL_OPENGL_API); + if (!make_current(context, this->surface)) { + printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(), + eglGetError()); + abort(); + } + } + while (!producer_thread_should_quit.should_quit()) { string filename_copy; { @@ -299,48 +410,68 @@ void FFmpegCapture::producer_thread_func() filename_copy = filename; } - string pathname = search_for_file(filename_copy); + string pathname; + if (srt_sock == -1) { + pathname = search_for_file(filename_copy); + } else { + pathname = description; + } if (pathname.empty()) { - fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str()); send_disconnected_frame(); + if (play_once) { + break; + } producer_thread_should_quit.sleep_for(seconds(1)); + fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str()); continue; } should_interrupt = false; if (!play_video(pathname)) { // Error. - fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str()); send_disconnected_frame(); + if (play_once) { + break; + } + fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str()); producer_thread_should_quit.sleep_for(seconds(1)); continue; } + if (play_once) { + send_disconnected_frame(); + break; + } + // Probably just EOF, will exit the loop above on next test. } if (has_dequeue_callbacks) { - dequeue_cleanup_callback(); + dequeue_cleanup_callback(); has_dequeue_callbacks = false; - } + } + + delete_context(context); } void FFmpegCapture::send_disconnected_frame() { // Send an empty frame to signal that we have no signal anymore. FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame(); + size_t frame_width = width == 0 ? global_flags.width : width; + size_t frame_height = height == 0 ? global_flags.height : height; if (video_frame.data) { VideoFormat video_format; - video_format.width = width; - video_format.height = height; + video_format.width = frame_width; + video_format.height = frame_height; video_format.frame_rate_nom = 60; video_format.frame_rate_den = 1; video_format.is_connected = false; if (pixel_format == bmusb::PixelFormat_8BitBGRA) { - video_format.stride = width * 4; - video_frame.len = width * height * 4; + video_format.stride = frame_width * 4; + video_frame.len = frame_width * frame_height * 4; memset(video_frame.data, 0, video_frame.len); } else { - video_format.stride = width; + video_format.stride = frame_width; current_frame_ycbcr_format.luma_coefficients = YCBCR_REC_709; current_frame_ycbcr_format.full_range = true; current_frame_ycbcr_format.num_levels = 256; @@ -350,16 +481,85 @@ void FFmpegCapture::send_disconnected_frame() current_frame_ycbcr_format.cb_y_position = 0.0f; current_frame_ycbcr_format.cr_x_position = 0.0f; current_frame_ycbcr_format.cr_y_position = 0.0f; - video_frame.len = width * height * 2; - memset(video_frame.data, 0, width * height); - memset(video_frame.data + width * height, 128, width * height); // Valid for both NV12 and planar. + video_frame.len = frame_width * frame_height * 2; + memset(video_frame.data, 0, frame_width * frame_height); + memset(video_frame.data + frame_width * frame_height, 128, frame_width * frame_height); // Valid for both NV12 and planar. } - frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, - video_frame, /*video_offset=*/0, video_format, - FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); + if (frame_callback != nullptr) { + frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, + video_frame, /*video_offset=*/0, video_format, + FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); + } last_frame_was_connected = false; } + + if (play_once) { + disconnected = true; + if (card_disconnected_callback != nullptr) { + card_disconnected_callback(); + } + } +} + +template +AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) +{ + bool found_config_of_right_type = false; + for (int i = 0;; ++i) { // Termination condition inside loop. + const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i); + if (config == nullptr) { // End of list. + break; + } + if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) || + config->device_type != type) { + // Not interesting for us. + continue; + } + + // We have a config of the right type, but does it actually support + // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors + // is to just replace the pixel format with a software-decoded one, + // such as yuv420p.) + found_config_of_right_type = true; + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if (config->pix_fmt == *fmt_ptr) { + fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n", + av_hwdevice_get_type_name(type), ctx->codec->name); + if (ctx->profile == FF_PROFILE_H264_BASELINE) { + fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n"); + fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n"); + fprintf(stderr, " Decoding might fail and fall back to software.\n"); + } + return config->pix_fmt; + } + } + fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name); + unordered_set seen; + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if (!seen.count(*fmt_ptr)) { + fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr)); + seen.insert(*fmt_ptr); + } + } + fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt)); + + } + + if (!found_config_of_right_type) { + fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type)); + } + + // We found no VA-API formats, so take the first software format. + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) { + fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr)); + return *fmt_ptr; + } + } + + // Fallback: Just return anything. (Should never really happen.) + return fmt[0]; } bool FFmpegCapture::play_video(const string &pathname) @@ -375,8 +575,29 @@ bool FFmpegCapture::play_video(const string &pathname) } else { last_modified = buf.st_mtim; } - - auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + last_colorspace = static_cast(-1); + last_chroma_location = static_cast(-1); + + AVFormatContextWithCloser format_ctx; + if (srt_sock == -1) { + // Regular file (or stream). + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; + format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr, + /*options=*/nullptr, + AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + frame_timeout_valid = false; + } else { +#ifdef HAVE_SRT + // SRT socket, already opened. + const AVInputFormat *mpegts_fmt = av_find_input_format("mpegts"); + format_ctx = avformat_open_input_unique(&FFmpegCapture::read_srt_thunk, this, + mpegts_fmt, /*options=*/nullptr, + AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); +#else + assert(false); +#endif + } if (format_ctx == nullptr) { fprintf(stderr, "%s: Error opening file\n", pathname.c_str()); return false; @@ -399,7 +620,8 @@ bool FFmpegCapture::play_video(const string &pathname) // Open video decoder. const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar; - AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id); + const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id); + video_timebase = format_ctx->streams[video_stream_index]->time_base; AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr); if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) { @@ -410,6 +632,28 @@ bool FFmpegCapture::play_video(const string &pathname) fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str()); return false; } + + // Seemingly, it's not too easy to make something that just initializes + // “whatever goes”, so we don't get CUDA or VULKAN or whatever here + // without enumerating through several different types. + // VA-API and VDPAU will do for now. We prioritize VDPAU for the + // simple reason that there's a VA-API-via-VDPAU emulation for NVidia + // cards that seems to work, but just hangs when trying to transfer the frame. + // + // Note that we don't actually check codec support beforehand, + // so if you have a low-end VDPAU device but a high-end VA-API device, + // you lose out on the extra codec support from the latter. + AVBufferRef *hw_device_ctx = nullptr; + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) { + video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + video_codec_ctx->get_format = get_hw_format; + } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) { + video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + video_codec_ctx->get_format = get_hw_format; + } else { + fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n"); + } + if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) { fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str()); return false; @@ -430,7 +674,7 @@ bool FFmpegCapture::play_video(const string &pathname) fprintf(stderr, "%s: Cannot fill audio codec parameters\n", pathname.c_str()); return false; } - AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id); + const AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id); if (audio_codec == nullptr) { fprintf(stderr, "%s: Cannot find audio decoder\n", pathname.c_str()); return false; @@ -447,6 +691,7 @@ bool FFmpegCapture::play_video(const string &pathname) // Main loop. bool first_frame = true; + int consecutive_errors = 0; while (!producer_thread_should_quit.should_quit()) { if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) { return true; @@ -460,11 +705,25 @@ bool FFmpegCapture::play_video(const string &pathname) int64_t audio_pts; bool error; + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); - if (error) { + frame_timeout_valid = false; + if (should_interrupt.load()) { + // Abort no matter whether we got a frame or not. return false; } + if (error) { + if (++consecutive_errors >= 100) { + fprintf(stderr, "More than 100 consecutive error video frames, aborting playback.\n"); + return false; + } else { + continue; + } + } else { + consecutive_errors = 0; + } if (frame == nullptr) { // EOF. Loop back to the start if we can. if (format_ctx->pb != nullptr && format_ctx->pb->seekable == 0) { @@ -494,6 +753,22 @@ bool FFmpegCapture::play_video(const string &pathname) } VideoFormat video_format = construct_video_format(frame.get(), video_timebase); + if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { + // Invalid frame rate; try constructing it from the previous frame length. + // (This is especially important if we are the master card, for SRT, + // since it affects audio. Not all senders have good timebases + // (e.g., Larix rounds first to timebase 1000 and then multiplies by + // 90 from there, it seems), but it's much better to have an oscillating + // value than just locking at 60. + if (last_pts != 0 && frame->pts > last_pts) { + int64_t pts_diff = frame->pts - last_pts; + video_format.frame_rate_nom = video_timebase.den; + video_format.frame_rate_den = video_timebase.num * pts_diff; + } else { + video_format.frame_rate_nom = 60; + video_format.frame_rate_den = 1; + } + } UniqueFrame video_frame = make_video_frame(frame.get(), pathname, &error); if (error) { return false; @@ -503,56 +778,67 @@ bool FFmpegCapture::play_video(const string &pathname) if (last_pts == 0 && pts_origin == 0) { pts_origin = frame->pts; } - next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); - if (first_frame && last_frame_was_connected) { - // If reconnect took more than one second, this is probably a live feed, - // and we should reset the resampler. (Or the rate is really, really low, - // in which case a reset on the first frame is fine anyway.) - if (duration(next_frame_start - last_frame).count() >= 1.0) { - last_frame_was_connected = false; + steady_clock::time_point now = steady_clock::now(); + if (play_as_fast_as_possible) { + video_frame->received_timestamp = now; + audio_frame->received_timestamp = now; + next_frame_start = now; + } else { + next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); + if (first_frame && last_frame_was_connected) { + // If reconnect took more than one second, this is probably a live feed, + // and we should reset the resampler. (Or the rate is really, really low, + // in which case a reset on the first frame is fine anyway.) + if (duration(next_frame_start - last_frame).count() >= 1.0) { + last_frame_was_connected = false; + } + } + video_frame->received_timestamp = next_frame_start; + + // The easiest way to get all the rate conversions etc. right is to move the + // audio PTS into the video PTS timebase and go from there. (We'll get some + // rounding issues, but they should not be a big problem.) + int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); + audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); + + if (audio_frame->len != 0) { + // The received timestamps in Nageru are measured after we've just received the frame. + // However, pts (especially audio pts) is at the _beginning_ of the frame. + // If we have locked audio, the distinction doesn't really matter, as pts is + // on a relative scale and a fixed offset is fine. But if we don't, we will have + // a different number of samples each time, which will cause huge audio jitter + // and throw off the resampler. + // + // In a sense, we should have compensated by adding the frame and audio lengths + // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, + // but that would mean extra waiting in sleep_until(). All we need is that they + // are correct relative to each other, though (and to the other frames we send), + // so just align the end of the audio frame, and we're fine. + size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; + double offset = double(num_samples) / OUTPUT_FREQUENCY - + double(video_format.frame_rate_den) / video_format.frame_rate_nom; + audio_frame->received_timestamp += duration_cast(duration(offset)); } - } - video_frame->received_timestamp = next_frame_start; - - // The easiest way to get all the rate conversions etc. right is to move the - // audio PTS into the video PTS timebase and go from there. (We'll get some - // rounding issues, but they should not be a big problem.) - int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); - audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); - - if (audio_frame->len != 0) { - // The received timestamps in Nageru are measured after we've just received the frame. - // However, pts (especially audio pts) is at the _beginning_ of the frame. - // If we have locked audio, the distinction doesn't really matter, as pts is - // on a relative scale and a fixed offset is fine. But if we don't, we will have - // a different number of samples each time, which will cause huge audio jitter - // and throw off the resampler. - // - // In a sense, we should have compensated by adding the frame and audio lengths - // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, - // but that would mean extra waiting in sleep_until(). All we need is that they - // are correct relative to each other, though (and to the other frames we send), - // so just align the end of the audio frame, and we're fine. - size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; - double offset = double(num_samples) / OUTPUT_FREQUENCY - - double(video_format.frame_rate_den) / video_format.frame_rate_nom; - audio_frame->received_timestamp += duration_cast(duration(offset)); - } - steady_clock::time_point now = steady_clock::now(); - if (duration(now - next_frame_start).count() >= 0.1) { - // If we don't have enough CPU to keep up, or if we have a live stream - // where the initial origin was somehow wrong, we could be behind indefinitely. - // In particular, this will give the audio resampler problems as it tries - // to speed up to reduce the delay, hitting the low end of the buffer every time. - fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", - pathname.c_str(), - 1e3 * duration(now - next_frame_start).count()); - pts_origin = frame->pts; - start = next_frame_start = now; - timecode += MAX_FPS * 2 + 1; + if (duration(now - next_frame_start).count() >= 0.1) { + // If we don't have enough CPU to keep up, or if we have a live stream + // where the initial origin was somehow wrong, we could be behind indefinitely. + // In particular, this will give the audio resampler problems as it tries + // to speed up to reduce the delay, hitting the low end of the buffer every time. + fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", + pathname.c_str(), + 1e3 * duration(now - next_frame_start).count()); + pts_origin = frame->pts; + start = next_frame_start = now; + timecode += TYPICAL_FPS * 2 + 1; + } + } + bool finished_wakeup; + if (play_as_fast_as_possible) { + finished_wakeup = !producer_thread_should_quit.should_quit(); + } else { + finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); } - bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); if (finished_wakeup) { if (audio_frame->len > 0) { assert(audio_pts != -1); @@ -562,11 +848,14 @@ bool FFmpegCapture::play_video(const string &pathname) // Make sure to get the audio resampler reset. (This is a hack; // ideally, the frame callback should just accept a way to signal // audio discontinuity.) - timecode += MAX_FPS * 2 + 1; + timecode += TYPICAL_FPS * 2 + 1; + } + last_neutral_color = get_neutral_color(frame->metadata); + if (frame_callback != nullptr) { + frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, + video_frame.get_and_release(), 0, video_format, + audio_frame.get_and_release(), 0, audio_format); } - frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, - video_frame.get_and_release(), 0, video_format, - audio_frame.get_and_release(), 0, audio_format); first_frame = false; last_frame = steady_clock::now(); last_frame_was_connected = true; @@ -636,6 +925,7 @@ bool FFmpegCapture::process_queued_commands(AVFormatContext *format_ctx, const s start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate); pts_origin = last_pts; rate = cmd.new_rate; + play_as_fast_as_possible = (rate >= 10.0); break; } } @@ -660,31 +950,31 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo *audio_pts = -1; bool has_audio = false; do { - AVPacket pkt; - unique_ptr pkt_cleanup( - &pkt, av_packet_unref); - av_init_packet(&pkt); - pkt.data = nullptr; - pkt.size = 0; - if (av_read_frame(format_ctx, &pkt) == 0) { - if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) { - audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base); + AVPacketWithDeleter pkt = av_packet_alloc_unique(); + pkt->data = nullptr; + pkt->size = 0; + if (av_read_frame(format_ctx, pkt.get()) == 0) { + if (pkt->stream_index == audio_stream_index && audio_callback != nullptr) { + audio_callback(pkt.get(), format_ctx->streams[audio_stream_index]->time_base); + } + if (pkt->stream_index == video_stream_index && video_callback != nullptr) { + video_callback(pkt.get(), format_ctx->streams[video_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index) { - if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) { + if (pkt->stream_index == video_stream_index && global_flags.transcode_video) { + if (avcodec_send_packet(video_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == audio_stream_index) { + } else if (pkt->stream_index == audio_stream_index && global_flags.transcode_audio) { has_audio = true; - if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) { + if (avcodec_send_packet(audio_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == subtitle_stream_index) { - last_subtitle = string(reinterpret_cast(pkt.data), pkt.size); + } else if (pkt->stream_index == subtitle_stream_index) { + last_subtitle = string(reinterpret_cast(pkt->data), pkt->size); has_last_subtitle = true; } } else { @@ -713,6 +1003,22 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo // Decode video, if we have a frame. int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get()); if (err == 0) { + if (video_avframe->format == AV_PIX_FMT_VAAPI || + video_avframe->format == AV_PIX_FMT_VDPAU) { + // Get the frame down to the CPU. (TODO: See if we can keep it + // on the GPU all the way, since it will be going up again later. + // However, this only works if the OpenGL GPU is the same one.) + AVFrameWithDeleter sw_frame = av_frame_alloc_unique(); + int err = av_hwframe_transfer_data(sw_frame.get(), video_avframe.get(), 0); + if (err != 0) { + fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + sw_frame->pts = video_avframe->pts; + sw_frame->pkt_duration = video_avframe->pkt_duration; + video_avframe = move(sw_frame); + } frame_finished = true; break; } else if (err != AVERROR(EAGAIN)) { @@ -758,35 +1064,43 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: } audio_format->num_channels = 2; - int64_t channel_layout = audio_avframe->channel_layout; - if (channel_layout == 0) { - channel_layout = av_get_default_channel_layout(audio_avframe->channels); + AVChannelLayout channel_layout = audio_avframe->ch_layout; + if (!av_channel_layout_check(&channel_layout) || + channel_layout.order == AV_CHANNEL_ORDER_UNSPEC) { + av_channel_layout_default(&channel_layout, audio_avframe->ch_layout.nb_channels); } if (resampler == nullptr || audio_avframe->format != last_src_format || dst_format != last_dst_format || - channel_layout != last_channel_layout || + av_channel_layout_compare(&channel_layout, &last_channel_layout) != 0|| audio_avframe->sample_rate != last_sample_rate) { + // TODO: When we get C++20, use AV_CHANNEL_LAYOUT_STEREO_DOWNMIX. + AVChannelLayout stereo_downmix; + stereo_downmix.order = AV_CHANNEL_ORDER_NATIVE; + stereo_downmix.nb_channels = 2; + stereo_downmix.u.mask = AV_CH_LAYOUT_STEREO_DOWNMIX; + swr_free(&resampler); - resampler = swr_alloc_set_opts(nullptr, - /*out_ch_layout=*/AV_CH_LAYOUT_STEREO_DOWNMIX, - /*out_sample_fmt=*/dst_format, - /*out_sample_rate=*/OUTPUT_FREQUENCY, - /*in_ch_layout=*/channel_layout, - /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format), - /*in_sample_rate=*/audio_avframe->sample_rate, - /*log_offset=*/0, - /*log_ctx=*/nullptr); - - if (resampler == nullptr) { + resampler = nullptr; + int err = swr_alloc_set_opts2(&resampler, + /*out_ch_layout=*/&stereo_downmix, + /*out_sample_fmt=*/dst_format, + /*out_sample_rate=*/OUTPUT_FREQUENCY, + /*in_ch_layout=*/&channel_layout, + /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format), + /*in_sample_rate=*/audio_avframe->sample_rate, + /*log_offset=*/0, + /*log_ctx=*/nullptr); + + if (err != 0 || resampler == nullptr) { fprintf(stderr, "Allocating resampler failed.\n"); - exit(1); + abort(); } if (swr_init(resampler) < 0) { fprintf(stderr, "Could not open resample context.\n"); - exit(1); + abort(); } last_src_format = AVSampleFormat(audio_avframe->format); @@ -802,9 +1116,9 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: int out_samples = swr_convert(resampler, &data, num_samples_room, const_cast(audio_avframe->data), audio_avframe->nb_samples); if (out_samples < 0) { - fprintf(stderr, "Audio conversion failed.\n"); - exit(1); - } + fprintf(stderr, "Audio conversion failed.\n"); + abort(); + } audio_frame->len += out_samples * bytes_per_sample; } @@ -812,23 +1126,18 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRational video_timebase) { VideoFormat video_format; - video_format.width = width; - video_format.height = height; + video_format.width = frame_width(frame); + video_format.height = frame_height(frame); if (pixel_format == bmusb::PixelFormat_8BitBGRA) { - video_format.stride = width * 4; + video_format.stride = frame_width(frame) * 4; } else if (pixel_format == FFmpegCapture::PixelFormat_NV12) { - video_format.stride = width; + video_format.stride = frame_width(frame); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); - video_format.stride = width; + video_format.stride = frame_width(frame); } video_format.frame_rate_nom = video_timebase.den; video_format.frame_rate_den = frame->pkt_duration * video_timebase.num; - if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { - // Invalid frame rate. - video_format.frame_rate_nom = 60; - video_format.frame_rate_den = 1; - } video_format.has_signal = true; video_format.is_connected = true; return video_format; @@ -838,7 +1147,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & { *error = false; - UniqueFrame video_frame(video_frame_allocator->alloc_frame()); + UniqueFrame video_frame(video_frame_allocator->create_frame(frame->width, frame->height, frame->width)); if (video_frame->data == nullptr) { return video_frame; } @@ -850,7 +1159,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & sws_dst_format = decide_dst_format(AVPixelFormat(frame->format), pixel_format); sws_ctx.reset( sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format), - width, height, sws_dst_format, + frame_width(frame), frame_height(frame), sws_dst_format, SWS_BICUBIC, nullptr, nullptr, nullptr)); sws_last_width = frame->width; sws_last_height = frame->height; @@ -866,50 +1175,106 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & int linesizes[4] = { 0, 0, 0, 0 }; if (pixel_format == bmusb::PixelFormat_8BitBGRA) { pic_data[0] = video_frame->data; - linesizes[0] = width * 4; - video_frame->len = (width * 4) * height; + linesizes[0] = frame_width(frame) * 4; + video_frame->len = (frame_width(frame) * 4) * frame_height(frame); } else if (pixel_format == PixelFormat_NV12) { pic_data[0] = video_frame->data; - linesizes[0] = width; + linesizes[0] = frame_width(frame); - pic_data[1] = pic_data[0] + width * height; - linesizes[1] = width; + pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame); + linesizes[1] = frame_width(frame); - video_frame->len = (width * 2) * height; + video_frame->len = (frame_width(frame) * 2) * frame_height(frame); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - int chroma_width = AV_CEIL_RSHIFT(int(width), desc->log2_chroma_w); - int chroma_height = AV_CEIL_RSHIFT(int(height), desc->log2_chroma_h); + int chroma_width = AV_CEIL_RSHIFT(int(frame_width(frame)), desc->log2_chroma_w); + int chroma_height = AV_CEIL_RSHIFT(int(frame_height(frame)), desc->log2_chroma_h); pic_data[0] = video_frame->data; - linesizes[0] = width; + linesizes[0] = frame_width(frame); - pic_data[1] = pic_data[0] + width * height; + pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame); linesizes[1] = chroma_width; pic_data[2] = pic_data[1] + chroma_width * chroma_height; linesizes[2] = chroma_width; - video_frame->len = width * height + 2 * chroma_width * chroma_height; + video_frame->len = frame_width(frame) * frame_height(frame) + 2 * chroma_width * chroma_height; - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } + + // FIXME: Currently, if the video is too high-res for one of the allocated + // frames, we simply refuse to scale it here to avoid crashes. It would be better + // if we could somehow signal getting larger frames, especially as 4K is a thing now. + if (video_frame->len > video_frame->size) { + fprintf(stderr, "%s: Decoded frame would be larger than supported frame size (%zu > %zu), not decoding.\n", pathname.c_str(), video_frame->len, video_frame->size); + *error = true; + return video_frame; + } + sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); return video_frame; } -int FFmpegCapture::interrupt_cb_thunk(void *unique) +int FFmpegCapture::interrupt_cb_thunk(void *opaque) { - return reinterpret_cast(unique)->interrupt_cb(); + return reinterpret_cast(opaque)->interrupt_cb(); } int FFmpegCapture::interrupt_cb() { + // If ten seconds is gone without anything happening, we assume that + // we are in a network stream that died and FFmpeg just didn't + // pick it up (or perhaps it just hung, keeping the connection open). + // Called back approximately every 100 ms if something is hanging, + // so we get more than enough accuracy for our purposes. + if (!should_interrupt && frame_timeout_valid && + duration(steady_clock::now() - frame_timeout_started).count() >= 10.0) { + string filename_copy; + { + lock_guard lock(filename_mu); + filename_copy = filename; + } + fprintf(stderr, "%s: No frame for more than 10 seconds, restarting stream.\n", filename.c_str()); + should_interrupt = true; + } return should_interrupt.load(); } + +unsigned FFmpegCapture::frame_width(const AVFrame *frame) const +{ + if (width == 0) { + return frame->width; + } else { + return width; + } +} + +unsigned FFmpegCapture::frame_height(const AVFrame *frame) const +{ + if (height == 0) { + return frame->height; + } else { + return height; + } +} + +#ifdef HAVE_SRT +int FFmpegCapture::read_srt_thunk(void *opaque, uint8_t *buf, int buf_size) +{ + return reinterpret_cast(opaque)->read_srt(buf, buf_size); +} + +int FFmpegCapture::read_srt(uint8_t *buf, int buf_size) +{ + SRT_MSGCTRL mc = srt_msgctrl_default; + return srt_recvmsg2(srt_sock, reinterpret_cast(buf), buf_size, &mc); +} +#endif