X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=6dbcbd602b79b544c8319ddd9446b9992c66ef10;hb=HEAD;hp=863f4fba4feaecc40b14d07b4662a04740c1894c;hpb=3fb867fa45d24d5f67ee1b070533f46c1f1f5462;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 863f4fb..14ad91f 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -1,24 +1,49 @@ #include "ffmpeg_capture.h" +#include "defs.h" +#include "shared/shared_defs.h" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include #include #include #include +#include #include -#include +#include +#include extern "C" { #include +#include +#include +#include #include #include +#include +#include +#include +#include #include #include -#include -#include +#include +#include +#include #include -#include +#include +#include +#include +#include #include } @@ -33,10 +58,10 @@ extern "C" { #include #include "bmusb/bmusb.h" +#include "shared/context.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" #include "flags.h" -#include "image_input.h" #include "ref_counted_frame.h" #include "shared/timebase.h" @@ -44,14 +69,17 @@ extern "C" { #include #endif -#define FRAME_SIZE (8 << 20) // 8 MB. - using namespace std; using namespace std::chrono; using namespace bmusb; using namespace movit; using namespace Eigen; +// Avoid deprecation warnings, but we don't want to drop FFmpeg 5.1 support just yet. +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 30, 100) +#define pkt_duration duration +#endif + namespace { steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate) @@ -125,8 +153,8 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst if (desc->comp[0].depth != 8) continue; // Same or better chroma resolution only. - int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w; - int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h; + int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w; + int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h; if (chroma_w_diff < 0 || chroma_h_diff < 0) continue; @@ -258,8 +286,8 @@ RGBTriplet get_neutral_color(AVDictionary *metadata) } // namespace -FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) - : filename(filename), width(width), height(height), video_timebase{1, 1} +FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height, QSurface *surface) + : filename(filename), width(width), height(height), video_timebase{1, 1}, surface(surface) { description = "Video: " + filename; @@ -269,12 +297,13 @@ FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned he } #ifdef HAVE_SRT -FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id) +FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id, QSurface *surface) : srt_sock(srt_sock), width(0), // Don't resize; SRT streams typically have stable resolution, and should behave much like regular cards in general. height(0), pixel_format(bmusb::PixelFormat_8BitYCbCrPlanar), - video_timebase{1, 1} + video_timebase{1, 1}, + surface(surface) { if (stream_id.empty()) { description = "SRT stream"; @@ -298,6 +327,7 @@ FFmpegCapture::~FFmpegCapture() srt_close(srt_sock); } #endif + delete surface; } void FFmpegCapture::configure_card() @@ -358,6 +388,21 @@ void FFmpegCapture::producer_thread_func() snprintf(thread_name, sizeof(thread_name), "FFmpeg_C_%d", card_index); pthread_setname_np(pthread_self(), thread_name); + // We need a context in case create_frame() needs to reallocate something. + // (If none is given, we are probably in Kaeru, which uses MallocFrameAllocator + // anyway, which doesn't reallocate currently and definitely doesn't need + // an active OpenGL context to do so.) + QOpenGLContext *context = nullptr; + if (surface != nullptr) { + context = create_context(this->surface); + eglBindAPI(EGL_OPENGL_API); + if (!make_current(context, this->surface)) { + printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(), + eglGetError()); + abort(); + } + } + while (!producer_thread_should_quit.should_quit()) { string filename_copy; { @@ -401,9 +446,11 @@ void FFmpegCapture::producer_thread_func() } if (has_dequeue_callbacks) { - dequeue_cleanup_callback(); + dequeue_cleanup_callback(); has_dequeue_callbacks = false; - } + } + + delete_context(context); } void FFmpegCapture::send_disconnected_frame() @@ -479,6 +526,11 @@ AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) if (config->pix_fmt == *fmt_ptr) { fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n", av_hwdevice_get_type_name(type), ctx->codec->name); + if (ctx->profile == FF_PROFILE_H264_BASELINE) { + fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n"); + fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n"); + fprintf(stderr, " Decoding might fail and fall back to software.\n"); + } return config->pix_fmt; } } @@ -498,7 +550,15 @@ AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type)); } - // We found no VA-API formats, so take the best software format. + // We found no VA-API formats, so take the first software format. + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) { + fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr)); + return *fmt_ptr; + } + } + + // Fallback: Just return anything. (Should never really happen.) return fmt[0]; } @@ -520,10 +580,13 @@ bool FFmpegCapture::play_video(const string &pathname) AVFormatContextWithCloser format_ctx; if (srt_sock == -1) { - // Regular file. + // Regular file (or stream). + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr, /*options=*/nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + frame_timeout_valid = false; } else { #ifdef HAVE_SRT // SRT socket, already opened. @@ -642,11 +705,18 @@ bool FFmpegCapture::play_video(const string &pathname) int64_t audio_pts; bool error; + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); + frame_timeout_valid = false; + if (should_interrupt.load()) { + // Abort no matter whether we got a frame or not. + return false; + } if (error) { if (++consecutive_errors >= 100) { - fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n"); + fprintf(stderr, "More than 100 consecutive error video frames, aborting playback.\n"); return false; } else { continue; @@ -880,34 +950,31 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo *audio_pts = -1; bool has_audio = false; do { - AVPacket pkt; - unique_ptr pkt_cleanup( - &pkt, av_packet_unref); - av_init_packet(&pkt); - pkt.data = nullptr; - pkt.size = 0; - if (av_read_frame(format_ctx, &pkt) == 0) { - if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) { - audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base); + AVPacketWithDeleter pkt = av_packet_alloc_unique(); + pkt->data = nullptr; + pkt->size = 0; + if (av_read_frame(format_ctx, pkt.get()) == 0) { + if (pkt->stream_index == audio_stream_index && audio_callback != nullptr) { + audio_callback(pkt.get(), format_ctx->streams[audio_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index && video_callback != nullptr) { - video_callback(&pkt, format_ctx->streams[video_stream_index]->time_base); + if (pkt->stream_index == video_stream_index && video_callback != nullptr) { + video_callback(pkt.get(), format_ctx->streams[video_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index && global_flags.transcode_video) { - if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) { + if (pkt->stream_index == video_stream_index && global_flags.transcode_video) { + if (avcodec_send_packet(video_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == audio_stream_index && global_flags.transcode_audio) { + } else if (pkt->stream_index == audio_stream_index && global_flags.transcode_audio) { has_audio = true; - if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) { + if (avcodec_send_packet(audio_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == subtitle_stream_index) { - last_subtitle = string(reinterpret_cast(pkt.data), pkt.size); + } else if (pkt->stream_index == subtitle_stream_index) { + last_subtitle = string(reinterpret_cast(pkt->data), pkt->size); has_last_subtitle = true; } } else { @@ -997,28 +1064,36 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: } audio_format->num_channels = 2; - int64_t channel_layout = audio_avframe->channel_layout; - if (channel_layout == 0) { - channel_layout = av_get_default_channel_layout(audio_avframe->channels); + AVChannelLayout channel_layout = audio_avframe->ch_layout; + if (!av_channel_layout_check(&channel_layout) || + channel_layout.order == AV_CHANNEL_ORDER_UNSPEC) { + av_channel_layout_default(&channel_layout, audio_avframe->ch_layout.nb_channels); } if (resampler == nullptr || audio_avframe->format != last_src_format || dst_format != last_dst_format || - channel_layout != last_channel_layout || + av_channel_layout_compare(&channel_layout, &last_channel_layout) != 0|| audio_avframe->sample_rate != last_sample_rate) { + // TODO: When we get C++20, use AV_CHANNEL_LAYOUT_STEREO_DOWNMIX. + AVChannelLayout stereo_downmix; + stereo_downmix.order = AV_CHANNEL_ORDER_NATIVE; + stereo_downmix.nb_channels = 2; + stereo_downmix.u.mask = AV_CH_LAYOUT_STEREO_DOWNMIX; + swr_free(&resampler); - resampler = swr_alloc_set_opts(nullptr, - /*out_ch_layout=*/AV_CH_LAYOUT_STEREO_DOWNMIX, - /*out_sample_fmt=*/dst_format, - /*out_sample_rate=*/OUTPUT_FREQUENCY, - /*in_ch_layout=*/channel_layout, - /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format), - /*in_sample_rate=*/audio_avframe->sample_rate, - /*log_offset=*/0, - /*log_ctx=*/nullptr); - - if (resampler == nullptr) { + resampler = nullptr; + int err = swr_alloc_set_opts2(&resampler, + /*out_ch_layout=*/&stereo_downmix, + /*out_sample_fmt=*/dst_format, + /*out_sample_rate=*/OUTPUT_FREQUENCY, + /*in_ch_layout=*/&channel_layout, + /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format), + /*in_sample_rate=*/audio_avframe->sample_rate, + /*log_offset=*/0, + /*log_ctx=*/nullptr); + + if (err != 0 || resampler == nullptr) { fprintf(stderr, "Allocating resampler failed.\n"); abort(); } @@ -1041,9 +1116,9 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: int out_samples = swr_convert(resampler, &data, num_samples_room, const_cast(audio_avframe->data), audio_avframe->nb_samples); if (out_samples < 0) { - fprintf(stderr, "Audio conversion failed.\n"); - abort(); - } + fprintf(stderr, "Audio conversion failed.\n"); + abort(); + } audio_frame->len += out_samples * bytes_per_sample; } @@ -1072,7 +1147,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & { *error = false; - UniqueFrame video_frame(video_frame_allocator->alloc_frame()); + UniqueFrame video_frame(video_frame_allocator->create_frame(frame->width, frame->height, frame->width)); if (video_frame->data == nullptr) { return video_frame; } @@ -1133,6 +1208,16 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } + + // FIXME: Currently, if the video is too high-res for one of the allocated + // frames, we simply refuse to scale it here to avoid crashes. It would be better + // if we could somehow signal getting larger frames, especially as 4K is a thing now. + if (video_frame->len > video_frame->size) { + fprintf(stderr, "%s: Decoded frame would be larger than supported frame size (%zu > %zu), not decoding.\n", pathname.c_str(), video_frame->len, video_frame->size); + *error = true; + return video_frame; + } + sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); return video_frame; @@ -1145,6 +1230,21 @@ int FFmpegCapture::interrupt_cb_thunk(void *opaque) int FFmpegCapture::interrupt_cb() { + // If ten seconds is gone without anything happening, we assume that + // we are in a network stream that died and FFmpeg just didn't + // pick it up (or perhaps it just hung, keeping the connection open). + // Called back approximately every 100 ms if something is hanging, + // so we get more than enough accuracy for our purposes. + if (!should_interrupt && frame_timeout_valid && + duration(steady_clock::now() - frame_timeout_started).count() >= 10.0) { + string filename_copy; + { + lock_guard lock(filename_mu); + filename_copy = filename; + } + fprintf(stderr, "%s: No frame for more than 10 seconds, restarting stream.\n", filename.c_str()); + should_interrupt = true; + } return should_interrupt.load(); }