X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=09c1d26e15b3642be4ea64eff4fa3e8da606c01c;hb=2f92c975a3cf9f4803a58267fd2a12765e34a69e;hp=504eff24f59d5bc7bfa5ceda9345f92ffce8785f;hpb=ba3ea588faa8e7908bce832820c08e43a09863fe;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 504eff2..09c1d26 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -26,6 +26,7 @@ extern "C" { #include #include #include +#include #include #include @@ -43,8 +44,6 @@ extern "C" { #include #endif -#define FRAME_SIZE (8 << 20) // 8 MB. - using namespace std; using namespace std::chrono; using namespace bmusb; @@ -124,8 +123,8 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst if (desc->comp[0].depth != 8) continue; // Same or better chroma resolution only. - int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w; - int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h; + int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w; + int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h; if (chroma_w_diff < 0 || chroma_h_diff < 0) continue; @@ -454,24 +453,63 @@ void FFmpegCapture::send_disconnected_frame() } } -AVPixelFormat get_vaapi_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) +template +AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) { - for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { - for (int i = 0;; ++i) { // Termination condition inside loop. - const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i); - if (config == nullptr) { // End of list. - fprintf(stderr, "Decoder %s does not support device.\n", ctx->codec->name); - break; - } - if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && - config->device_type == AV_HWDEVICE_TYPE_VAAPI && - config->pix_fmt == *fmt_ptr) { + bool found_config_of_right_type = false; + for (int i = 0;; ++i) { // Termination condition inside loop. + const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i); + if (config == nullptr) { // End of list. + break; + } + if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) || + config->device_type != type) { + // Not interesting for us. + continue; + } + + // We have a config of the right type, but does it actually support + // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors + // is to just replace the pixel format with a software-decoded one, + // such as yuv420p.) + found_config_of_right_type = true; + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if (config->pix_fmt == *fmt_ptr) { + fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n", + av_hwdevice_get_type_name(type), ctx->codec->name); + if (ctx->profile == FF_PROFILE_H264_BASELINE) { + fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n"); + fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n"); + fprintf(stderr, " Decoding might fail and fall back to software.\n"); + } return config->pix_fmt; } } + fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name); + unordered_set seen; + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if (!seen.count(*fmt_ptr)) { + fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr)); + seen.insert(*fmt_ptr); + } + } + fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt)); + + } + + if (!found_config_of_right_type) { + fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type)); } - // We found no VA-API formats, so take the best software format. + // We found no VA-API formats, so take the first software format. + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) { + fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr)); + return *fmt_ptr; + } + } + + // Fallback: Just return anything. (Should never really happen.) return fmt[0]; } @@ -493,10 +531,13 @@ bool FFmpegCapture::play_video(const string &pathname) AVFormatContextWithCloser format_ctx; if (srt_sock == -1) { - // Regular file. + // Regular file (or stream). + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr, /*options=*/nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + frame_timeout_valid = false; } else { #ifdef HAVE_SRT // SRT socket, already opened. @@ -544,14 +585,24 @@ bool FFmpegCapture::play_video(const string &pathname) } // Seemingly, it's not too easy to make something that just initializes - // “whatever goes”, so we don't get VDPAU or CUDA here without enumerating - // through several different types. VA-API will do for now. + // “whatever goes”, so we don't get CUDA or VULKAN or whatever here + // without enumerating through several different types. + // VA-API and VDPAU will do for now. We prioritize VDPAU for the + // simple reason that there's a VA-API-via-VDPAU emulation for NVidia + // cards that seems to work, but just hangs when trying to transfer the frame. + // + // Note that we don't actually check codec support beforehand, + // so if you have a low-end VDPAU device but a high-end VA-API device, + // you lose out on the extra codec support from the latter. AVBufferRef *hw_device_ctx = nullptr; - if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) { - fprintf(stderr, "Failed to initialize VA-API for FFmpeg acceleration. Decoding video in software.\n"); - } else { + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) { video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); - video_codec_ctx->get_format = get_vaapi_hw_format; + video_codec_ctx->get_format = get_hw_format; + } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) { + video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + video_codec_ctx->get_format = get_hw_format; + } else { + fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n"); } if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) { @@ -591,6 +642,7 @@ bool FFmpegCapture::play_video(const string &pathname) // Main loop. bool first_frame = true; + int consecutive_errors = 0; while (!producer_thread_should_quit.should_quit()) { if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) { return true; @@ -604,11 +656,25 @@ bool FFmpegCapture::play_video(const string &pathname) int64_t audio_pts; bool error; + frame_timeout_started = steady_clock::now(); + frame_timeout_valid = true; AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); - if (error) { + frame_timeout_valid = false; + if (should_interrupt.load()) { + // Abort no matter whether we got a frame or not. return false; } + if (error) { + if (++consecutive_errors >= 100) { + fprintf(stderr, "More than 100 consecutive error video frames, aborting playback.\n"); + return false; + } else { + continue; + } + } else { + consecutive_errors = 0; + } if (frame == nullptr) { // EOF. Loop back to the start if we can. if (format_ctx->pb != nullptr && format_ctx->pb->seekable == 0) { @@ -835,34 +901,31 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo *audio_pts = -1; bool has_audio = false; do { - AVPacket pkt; - unique_ptr pkt_cleanup( - &pkt, av_packet_unref); - av_init_packet(&pkt); - pkt.data = nullptr; - pkt.size = 0; - if (av_read_frame(format_ctx, &pkt) == 0) { - if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) { - audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base); + AVPacketWithDeleter pkt = av_packet_alloc_unique(); + pkt->data = nullptr; + pkt->size = 0; + if (av_read_frame(format_ctx, pkt.get()) == 0) { + if (pkt->stream_index == audio_stream_index && audio_callback != nullptr) { + audio_callback(pkt.get(), format_ctx->streams[audio_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index && video_callback != nullptr) { - video_callback(&pkt, format_ctx->streams[video_stream_index]->time_base); + if (pkt->stream_index == video_stream_index && video_callback != nullptr) { + video_callback(pkt.get(), format_ctx->streams[video_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index && global_flags.transcode_video) { - if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) { + if (pkt->stream_index == video_stream_index && global_flags.transcode_video) { + if (avcodec_send_packet(video_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == audio_stream_index && global_flags.transcode_audio) { + } else if (pkt->stream_index == audio_stream_index && global_flags.transcode_audio) { has_audio = true; - if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) { + if (avcodec_send_packet(audio_codec_ctx, pkt.get()) < 0) { fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == subtitle_stream_index) { - last_subtitle = string(reinterpret_cast(pkt.data), pkt.size); + } else if (pkt->stream_index == subtitle_stream_index) { + last_subtitle = string(reinterpret_cast(pkt->data), pkt->size); has_last_subtitle = true; } } else { @@ -891,7 +954,8 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo // Decode video, if we have a frame. int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get()); if (err == 0) { - if (video_avframe->format == AV_PIX_FMT_VAAPI) { + if (video_avframe->format == AV_PIX_FMT_VAAPI || + video_avframe->format == AV_PIX_FMT_VDPAU) { // Get the frame down to the CPU. (TODO: See if we can keep it // on the GPU all the way, since it will be going up again later. // However, this only works if the OpenGL GPU is the same one.) @@ -1087,6 +1151,16 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } + + // FIXME: Currently, if the video is too high-res for one of the allocated + // frames, we simply refuse to scale it here to avoid crashes. It would be better + // if we could somehow signal getting larger frames, especially as 4K is a thing now. + if (video_frame->len > FRAME_SIZE) { + fprintf(stderr, "%s: Decoded frame would be larger than supported FRAME_SIZE (%zu > %u), not decoding.\n", pathname.c_str(), video_frame->len, FRAME_SIZE); + *error = true; + return video_frame; + } + sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); return video_frame; @@ -1099,6 +1173,21 @@ int FFmpegCapture::interrupt_cb_thunk(void *opaque) int FFmpegCapture::interrupt_cb() { + // If ten seconds is gone without anything happening, we assume that + // we are in a network stream that died and FFmpeg just didn't + // pick it up (or perhaps it just hung, keeping the connection open). + // Called back approximately every 100 ms if something is hanging, + // so we get more than enough accuracy for our purposes. + if (!should_interrupt && frame_timeout_valid && + duration(steady_clock::now() - frame_timeout_started).count() >= 10.0) { + string filename_copy; + { + lock_guard lock(filename_mu); + filename_copy = filename; + } + fprintf(stderr, "%s: No frame for more than 10 seconds, restarting stream.\n", filename.c_str()); + should_interrupt = true; + } return should_interrupt.load(); }