X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=a9ad73d8d7ee0948744111bfaffc451159c25e9f;hb=bdc9f1ea04141e71906d486f9d254c3346835e72;hp=0c3aa57bcfde150a221ec86bcc72a253ee606e37;hpb=446d357f647112a5e164d6526cdba9ed9ade9fc6;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 0c3aa57..a9ad73d 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -143,7 +143,7 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst return av_get_pix_fmt(best_format); } -YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg) +YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg, AVColorSpace *last_colorspace, AVChromaLocation *last_chroma_location) { YCbCrFormat format; AVColorSpace colorspace = frame->colorspace; @@ -163,11 +163,14 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.luma_coefficients = (frame->height >= 720 ? YCBCR_REC_709 : YCBCR_REC_601); break; default: - fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", - colorspace); + if (colorspace != *last_colorspace) { + fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + colorspace); + } format.luma_coefficients = YCBCR_REC_709; break; } + *last_colorspace = colorspace; format.full_range = is_full_range(desc); format.num_levels = 1 << desc->comp[0].depth; @@ -200,12 +203,15 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.cb_y_position = 1.0; break; default: - fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", - frame->chroma_location); + if (frame->chroma_location != *last_chroma_location) { + fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", + frame->chroma_location); + } format.cb_x_position = 0.5; format.cb_y_position = 0.5; break; } + *last_chroma_location = frame->chroma_location; if (is_mjpeg && !format.full_range) { // Limited-range MJPEG is only detected by FFmpeg whenever a special @@ -286,6 +292,11 @@ FFmpegCapture::~FFmpegCapture() dequeue_cleanup_callback(); } swr_free(&resampler); +#ifdef HAVE_SRT + if (srt_sock != -1) { + srt_close(srt_sock); + } +#endif } void FFmpegCapture::configure_card() @@ -427,9 +438,11 @@ void FFmpegCapture::send_disconnected_frame() memset(video_frame.data + frame_width * frame_height, 128, frame_width * frame_height); // Valid for both NV12 and planar. } - frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, - video_frame, /*video_offset=*/0, video_format, - FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); + if (frame_callback != nullptr) { + frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, + video_frame, /*video_offset=*/0, video_format, + FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); + } last_frame_was_connected = false; } @@ -441,6 +454,27 @@ void FFmpegCapture::send_disconnected_frame() } } +AVPixelFormat get_vaapi_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt) +{ + for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) { + for (int i = 0;; ++i) { // Termination condition inside loop. + const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i); + if (config == nullptr) { // End of list. + fprintf(stderr, "Decoder %s does not support device.\n", ctx->codec->name); + break; + } + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && + config->device_type == AV_HWDEVICE_TYPE_VAAPI && + config->pix_fmt == *fmt_ptr) { + return config->pix_fmt; + } + } + } + + // We found no VA-API formats, so take the best software format. + return fmt[0]; +} + bool FFmpegCapture::play_video(const string &pathname) { // Note: Call before open, not after; otherwise, there's a race. @@ -454,6 +488,8 @@ bool FFmpegCapture::play_video(const string &pathname) } else { last_modified = buf.st_mtim; } + last_colorspace = static_cast(-1); + last_chroma_location = static_cast(-1); AVFormatContextWithCloser format_ctx; if (srt_sock == -1) { @@ -464,7 +500,7 @@ bool FFmpegCapture::play_video(const string &pathname) } else { #ifdef HAVE_SRT // SRT socket, already opened. - AVInputFormat *mpegts_fmt = av_find_input_format("mpegts"); + const AVInputFormat *mpegts_fmt = av_find_input_format("mpegts"); format_ctx = avformat_open_input_unique(&FFmpegCapture::read_srt_thunk, this, mpegts_fmt, /*options=*/nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); @@ -494,7 +530,8 @@ bool FFmpegCapture::play_video(const string &pathname) // Open video decoder. const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar; - AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id); + const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id); + video_timebase = format_ctx->streams[video_stream_index]->time_base; AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr); if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) { @@ -505,6 +542,18 @@ bool FFmpegCapture::play_video(const string &pathname) fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str()); return false; } + + // Seemingly, it's not too easy to make something that just initializes + // “whatever goes”, so we don't get VDPAU or CUDA here without enumerating + // through several different types. VA-API will do for now. + AVBufferRef *hw_device_ctx = nullptr; + if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) { + fprintf(stderr, "Failed to initialize VA-API for FFmpeg acceleration. Decoding video in software.\n"); + } else { + video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); + video_codec_ctx->get_format = get_vaapi_hw_format; + } + if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) { fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str()); return false; @@ -525,7 +574,7 @@ bool FFmpegCapture::play_video(const string &pathname) fprintf(stderr, "%s: Cannot fill audio codec parameters\n", pathname.c_str()); return false; } - AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id); + const AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id); if (audio_codec == nullptr) { fprintf(stderr, "%s: Cannot find audio decoder\n", pathname.c_str()); return false; @@ -589,6 +638,22 @@ bool FFmpegCapture::play_video(const string &pathname) } VideoFormat video_format = construct_video_format(frame.get(), video_timebase); + if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { + // Invalid frame rate; try constructing it from the previous frame length. + // (This is especially important if we are the master card, for SRT, + // since it affects audio. Not all senders have good timebases + // (e.g., Larix rounds first to timebase 1000 and then multiplies by + // 90 from there, it seems), but it's much better to have an oscillating + // value than just locking at 60. + if (last_pts != 0 && frame->pts > last_pts) { + int64_t pts_diff = frame->pts - last_pts; + video_format.frame_rate_nom = video_timebase.den; + video_format.frame_rate_den = video_timebase.num * pts_diff; + } else { + video_format.frame_rate_nom = 60; + video_format.frame_rate_den = 1; + } + } UniqueFrame video_frame = make_video_frame(frame.get(), pathname, &error); if (error) { return false; @@ -671,9 +736,11 @@ bool FFmpegCapture::play_video(const string &pathname) timecode += MAX_FPS * 2 + 1; } last_neutral_color = get_neutral_color(frame->metadata); - frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, - video_frame.get_and_release(), 0, video_format, - audio_frame.get_and_release(), 0, audio_format); + if (frame_callback != nullptr) { + frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, + video_frame.get_and_release(), 0, video_format, + audio_frame.get_and_release(), 0, audio_format); + } first_frame = false; last_frame = steady_clock::now(); last_frame_was_connected = true; @@ -778,13 +845,16 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) { audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base); } - if (pkt.stream_index == video_stream_index) { + if (pkt.stream_index == video_stream_index && video_callback != nullptr) { + video_callback(&pkt, format_ctx->streams[video_stream_index]->time_base); + } + if (pkt.stream_index == video_stream_index && global_flags.transcode_video) { if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) { fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str()); *error = true; return AVFrameWithDeleter(nullptr); } - } else if (pkt.stream_index == audio_stream_index) { + } else if (pkt.stream_index == audio_stream_index && global_flags.transcode_audio) { has_audio = true; if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) { fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str()); @@ -821,6 +891,21 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo // Decode video, if we have a frame. int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get()); if (err == 0) { + if (video_avframe->format == AV_PIX_FMT_VAAPI) { + // Get the frame down to the CPU. (TODO: See if we can keep it + // on the GPU all the way, since it will be going up again later. + // However, this only works if the OpenGL GPU is the same one.) + AVFrameWithDeleter sw_frame = av_frame_alloc_unique(); + int err = av_hwframe_transfer_data(sw_frame.get(), video_avframe.get(), 0); + if (err != 0) { + fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str()); + *error = true; + return AVFrameWithDeleter(nullptr); + } + sw_frame->pts = video_avframe->pts; + sw_frame->pkt_duration = video_avframe->pkt_duration; + video_avframe = move(sw_frame); + } frame_finished = true; break; } else if (err != AVERROR(EAGAIN)) { @@ -932,11 +1017,6 @@ VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRation } video_format.frame_rate_nom = video_timebase.den; video_format.frame_rate_den = frame->pkt_duration * video_timebase.num; - if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { - // Invalid frame rate. - video_format.frame_rate_nom = 60; - video_format.frame_rate_den = 1; - } video_format.has_signal = true; video_format.is_connected = true; return video_format; @@ -986,7 +1066,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & video_frame->len = (frame_width(frame) * 2) * frame_height(frame); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); @@ -1005,7 +1085,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & video_frame->len = frame_width(frame) * frame_height(frame) + 2 * chroma_width * chroma_height; - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); @@ -1036,7 +1116,7 @@ unsigned FFmpegCapture::frame_height(const AVFrame *frame) const if (height == 0) { return frame->height; } else { - return width; + return height; } }