X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=2d1b3385b4ac0d64c137bba4c3590819f06f05f6;hb=2cb648106d32b9968f2026536fbead096308c7d1;hp=7081da823d61f952b2140fe980136b6a021272ec;hpb=af8007aa958271ba16b15e7d11a891570488f6c9;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 7081da8..2d1b338 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -27,6 +27,10 @@ extern "C" { #include #include +#include +#include +#include + #include "bmusb/bmusb.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" @@ -41,6 +45,7 @@ using namespace std; using namespace std::chrono; using namespace bmusb; using namespace movit; +using namespace Eigen; namespace { @@ -134,10 +139,10 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst return av_get_pix_fmt(best_format); } -YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame) +YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg) { YCbCrFormat format; - AVColorSpace colorspace = av_frame_get_colorspace(frame); + AVColorSpace colorspace = frame->colorspace; switch (colorspace) { case AVCOL_SPC_BT709: format.luma_coefficients = YCBCR_REC_709; @@ -191,18 +196,55 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.cb_y_position = 1.0; break; default: - fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", frame->chroma_location); format.cb_x_position = 0.5; format.cb_y_position = 0.5; break; } + if (is_mjpeg && !format.full_range) { + // Limited-range MJPEG is only detected by FFmpeg whenever a special + // JPEG comment is set, which means that in practice, the stream is + // almost certainly generated by Futatabi. Override FFmpeg's forced + // MJPEG defaults (it disregards the values set in the mux) with what + // Futatabi sets. + format.luma_coefficients = YCBCR_REC_709; + format.cb_x_position = 0.0; + format.cb_y_position = 0.5; + } + format.cr_x_position = format.cb_x_position; format.cr_y_position = format.cb_y_position; return format; } +RGBTriplet get_neutral_color(AVDictionary *metadata) +{ + if (metadata == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0); + if (entry == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + unsigned x_nom, x_den, y_nom, y_den; + if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) { + fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value); + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + double x = double(x_nom) / x_den; + double y = double(y_nom) / y_den; + double z = 1.0 - x - y; + + Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB); + Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z); + + return RGBTriplet(rgb[0], rgb[1], rgb[2]); +} + } // namespace FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) @@ -220,7 +262,7 @@ FFmpegCapture::~FFmpegCapture() if (has_dequeue_callbacks) { dequeue_cleanup_callback(); } - avresample_free(&resampler); + swr_free(&resampler); } void FFmpegCapture::configure_card() @@ -365,10 +407,7 @@ bool FFmpegCapture::play_video(const string &pathname) last_modified = buf.st_mtim; } - AVDictionary *opts = nullptr; - av_dict_set(&opts, "fflags", "nobuffer", 0); - - auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, &opts, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); if (format_ctx == nullptr) { fprintf(stderr, "%s: Error opening file\n", pathname.c_str()); return false; @@ -386,6 +425,8 @@ bool FFmpegCapture::play_video(const string &pathname) } int audio_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_AUDIO); + int subtitle_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_SUBTITLE); + has_last_subtitle = false; // Open video decoder. const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar; @@ -407,6 +448,9 @@ bool FFmpegCapture::play_video(const string &pathname) unique_ptr video_codec_ctx_cleanup( video_codec_ctx.get(), avcodec_close); + // Used in decode_ycbcr_format(). + is_mjpeg = video_codecpar->codec_id == AV_CODEC_ID_MJPEG; + // Open audio decoder, if we have audio. AVCodecContextWithDeleter audio_codec_ctx; if (audio_stream_index != -1) { @@ -438,18 +482,27 @@ bool FFmpegCapture::play_video(const string &pathname) if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) { return true; } + if (should_interrupt.load()) { + // Check as a failsafe, so that we don't need to rely on avio if we don't have to. + return false; + } UniqueFrame audio_frame = audio_frame_allocator->alloc_frame(); AudioFormat audio_format; int64_t audio_pts; bool error; AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), - pathname, video_stream_index, audio_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); + pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); if (error) { return false; } if (frame == nullptr) { // EOF. Loop back to the start if we can. + if (format_ctx->pb != nullptr && format_ctx->pb->seekable == 0) { + // Not seekable (but seemingly, sometimes av_seek_frame() would return 0 anyway, + // so don't try). + return true; + } if (av_seek_frame(format_ctx.get(), /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) { fprintf(stderr, "%s: Rewind failed, not looping.\n", pathname.c_str()); return true; @@ -481,56 +534,67 @@ bool FFmpegCapture::play_video(const string &pathname) if (last_pts == 0 && pts_origin == 0) { pts_origin = frame->pts; } - next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); - if (first_frame && last_frame_was_connected) { - // If reconnect took more than one second, this is probably a live feed, - // and we should reset the resampler. (Or the rate is really, really low, - // in which case a reset on the first frame is fine anyway.) - if (duration(next_frame_start - last_frame).count() >= 1.0) { - last_frame_was_connected = false; + steady_clock::time_point now = steady_clock::now(); + if (play_as_fast_as_possible) { + video_frame->received_timestamp = now; + audio_frame->received_timestamp = now; + next_frame_start = now; + } else { + next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate); + if (first_frame && last_frame_was_connected) { + // If reconnect took more than one second, this is probably a live feed, + // and we should reset the resampler. (Or the rate is really, really low, + // in which case a reset on the first frame is fine anyway.) + if (duration(next_frame_start - last_frame).count() >= 1.0) { + last_frame_was_connected = false; + } + } + video_frame->received_timestamp = next_frame_start; + + // The easiest way to get all the rate conversions etc. right is to move the + // audio PTS into the video PTS timebase and go from there. (We'll get some + // rounding issues, but they should not be a big problem.) + int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); + audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); + + if (audio_frame->len != 0) { + // The received timestamps in Nageru are measured after we've just received the frame. + // However, pts (especially audio pts) is at the _beginning_ of the frame. + // If we have locked audio, the distinction doesn't really matter, as pts is + // on a relative scale and a fixed offset is fine. But if we don't, we will have + // a different number of samples each time, which will cause huge audio jitter + // and throw off the resampler. + // + // In a sense, we should have compensated by adding the frame and audio lengths + // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, + // but that would mean extra waiting in sleep_until(). All we need is that they + // are correct relative to each other, though (and to the other frames we send), + // so just align the end of the audio frame, and we're fine. + size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; + double offset = double(num_samples) / OUTPUT_FREQUENCY - + double(video_format.frame_rate_den) / video_format.frame_rate_nom; + audio_frame->received_timestamp += duration_cast(duration(offset)); } - } - video_frame->received_timestamp = next_frame_start; - - // The easiest way to get all the rate conversions etc. right is to move the - // audio PTS into the video PTS timebase and go from there. (We'll get some - // rounding issues, but they should not be a big problem.) - int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase); - audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate); - - if (audio_frame->len != 0) { - // The received timestamps in Nageru are measured after we've just received the frame. - // However, pts (especially audio pts) is at the _beginning_ of the frame. - // If we have locked audio, the distinction doesn't really matter, as pts is - // on a relative scale and a fixed offset is fine. But if we don't, we will have - // a different number of samples each time, which will cause huge audio jitter - // and throw off the resampler. - // - // In a sense, we should have compensated by adding the frame and audio lengths - // to video_frame->received_timestamp and audio_frame->received_timestamp respectively, - // but that would mean extra waiting in sleep_until(). All we need is that they - // are correct relative to each other, though (and to the other frames we send), - // so just align the end of the audio frame, and we're fine. - size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels; - double offset = double(num_samples) / OUTPUT_FREQUENCY - - double(video_format.frame_rate_den) / video_format.frame_rate_nom; - audio_frame->received_timestamp += duration_cast(duration(offset)); - } - steady_clock::time_point now = steady_clock::now(); - if (duration(now - next_frame_start).count() >= 0.1) { - // If we don't have enough CPU to keep up, or if we have a live stream - // where the initial origin was somehow wrong, we could be behind indefinitely. - // In particular, this will give the audio resampler problems as it tries - // to speed up to reduce the delay, hitting the low end of the buffer every time. - fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", - pathname.c_str(), - 1e3 * duration(now - next_frame_start).count()); - pts_origin = frame->pts; - start = next_frame_start = now; - timecode += MAX_FPS * 2 + 1; + if (duration(now - next_frame_start).count() >= 0.1) { + // If we don't have enough CPU to keep up, or if we have a live stream + // where the initial origin was somehow wrong, we could be behind indefinitely. + // In particular, this will give the audio resampler problems as it tries + // to speed up to reduce the delay, hitting the low end of the buffer every time. + fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n", + pathname.c_str(), + 1e3 * duration(now - next_frame_start).count()); + pts_origin = frame->pts; + start = next_frame_start = now; + timecode += MAX_FPS * 2 + 1; + } + } + bool finished_wakeup; + if (play_as_fast_as_possible) { + finished_wakeup = !producer_thread_should_quit.should_quit(); + } else { + finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); } - bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start); if (finished_wakeup) { if (audio_frame->len > 0) { assert(audio_pts != -1); @@ -542,6 +606,7 @@ bool FFmpegCapture::play_video(const string &pathname) // audio discontinuity.) timecode += MAX_FPS * 2 + 1; } + last_neutral_color = get_neutral_color(frame->metadata); frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, video_frame.get_and_release(), 0, video_format, audio_frame.get_and_release(), 0, audio_format); @@ -614,6 +679,7 @@ bool FFmpegCapture::process_queued_commands(AVFormatContext *format_ctx, const s start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate); pts_origin = last_pts; rate = cmd.new_rate; + play_as_fast_as_possible = (rate >= 10.0); break; } } @@ -625,7 +691,7 @@ namespace { } // namespace AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx, - const std::string &pathname, int video_stream_index, int audio_stream_index, + const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index, FrameAllocator::Frame *audio_frame, AudioFormat *audio_format, int64_t *audio_pts, bool *error) { *error = false; @@ -661,6 +727,9 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo *error = true; return AVFrameWithDeleter(nullptr); } + } else if (pkt.stream_index == subtitle_stream_index) { + last_subtitle = string(reinterpret_cast(pkt.data), pkt.size); + has_last_subtitle = true; } } else { eof = true; // Or error, but ignore that for the time being. @@ -742,41 +811,43 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: audio_avframe->format != last_src_format || dst_format != last_dst_format || channel_layout != last_channel_layout || - av_frame_get_sample_rate(audio_avframe) != last_sample_rate) { - avresample_free(&resampler); - resampler = avresample_alloc_context(); + audio_avframe->sample_rate != last_sample_rate) { + swr_free(&resampler); + resampler = swr_alloc_set_opts(nullptr, + /*out_ch_layout=*/AV_CH_LAYOUT_STEREO_DOWNMIX, + /*out_sample_fmt=*/dst_format, + /*out_sample_rate=*/OUTPUT_FREQUENCY, + /*in_ch_layout=*/channel_layout, + /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format), + /*in_sample_rate=*/audio_avframe->sample_rate, + /*log_offset=*/0, + /*log_ctx=*/nullptr); + if (resampler == nullptr) { fprintf(stderr, "Allocating resampler failed.\n"); - exit(1); + abort(); } - av_opt_set_int(resampler, "in_channel_layout", channel_layout, 0); - av_opt_set_int(resampler, "out_channel_layout", AV_CH_LAYOUT_STEREO_DOWNMIX, 0); - av_opt_set_int(resampler, "in_sample_rate", av_frame_get_sample_rate(audio_avframe), 0); - av_opt_set_int(resampler, "out_sample_rate", OUTPUT_FREQUENCY, 0); - av_opt_set_int(resampler, "in_sample_fmt", audio_avframe->format, 0); - av_opt_set_int(resampler, "out_sample_fmt", dst_format, 0); - - if (avresample_open(resampler) < 0) { + if (swr_init(resampler) < 0) { fprintf(stderr, "Could not open resample context.\n"); - exit(1); + abort(); } last_src_format = AVSampleFormat(audio_avframe->format); last_dst_format = dst_format; last_channel_layout = channel_layout; - last_sample_rate = av_frame_get_sample_rate(audio_avframe); + last_sample_rate = audio_avframe->sample_rate; } size_t bytes_per_sample = (audio_format->bits_per_sample / 8) * 2; size_t num_samples_room = (audio_frame->size - audio_frame->len) / bytes_per_sample; uint8_t *data = audio_frame->data + audio_frame->len; - int out_samples = avresample_convert(resampler, &data, 0, num_samples_room, - const_cast(audio_avframe->data), audio_avframe->linesize[0], audio_avframe->nb_samples); + int out_samples = swr_convert(resampler, &data, num_samples_room, + const_cast(audio_avframe->data), audio_avframe->nb_samples); if (out_samples < 0) { fprintf(stderr, "Audio conversion failed.\n"); - exit(1); + abort(); } audio_frame->len += out_samples * bytes_per_sample; @@ -796,7 +867,7 @@ VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRation video_format.stride = width; } video_format.frame_rate_nom = video_timebase.den; - video_format.frame_rate_den = av_frame_get_pkt_duration(frame) * video_timebase.num; + video_format.frame_rate_den = frame->pkt_duration * video_timebase.num; if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { // Invalid frame rate. video_format.frame_rate_nom = 60; @@ -851,7 +922,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & video_frame->len = (width * 2) * height; const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); @@ -870,7 +941,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & video_frame->len = width * height + 2 * chroma_width * chroma_height; - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); } sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);