X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=c6824069aedc8786954531e3fb1f8f62deb52391;hb=1be4f26ad8f13698671d63f203ec0433cf4cf4aa;hp=b9eb982008f166fbc7cab97b0e3acc990ee19340;hpb=5e9f3fe04e04a7c53a8e64106797e14e44fe9024;p=nageru diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index b9eb982..c682406 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -27,6 +27,10 @@ extern "C" { #include #include +#include +#include +#include + #include "bmusb/bmusb.h" #include "shared/ffmpeg_raii.h" #include "ffmpeg_util.h" @@ -35,12 +39,17 @@ extern "C" { #include "ref_counted_frame.h" #include "shared/timebase.h" +#ifdef HAVE_SRT +#include +#endif + #define FRAME_SIZE (8 << 20) // 8 MB. using namespace std; using namespace std::chrono; using namespace bmusb; using namespace movit; +using namespace Eigen; namespace { @@ -134,7 +143,7 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst return av_get_pix_fmt(best_format); } -YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg) +YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg, AVColorSpace *last_colorspace, AVChromaLocation *last_chroma_location) { YCbCrFormat format; AVColorSpace colorspace = frame->colorspace; @@ -154,11 +163,14 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.luma_coefficients = (frame->height >= 720 ? YCBCR_REC_709 : YCBCR_REC_601); break; default: - fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", - colorspace); + if (colorspace != *last_colorspace) { + fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n", + colorspace); + } format.luma_coefficients = YCBCR_REC_709; break; } + *last_colorspace = colorspace; format.full_range = is_full_range(desc); format.num_levels = 1 << desc->comp[0].depth; @@ -191,12 +203,15 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f format.cb_y_position = 1.0; break; default: - fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", - frame->chroma_location); + if (frame->chroma_location != *last_chroma_location) { + fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n", + frame->chroma_location); + } format.cb_x_position = 0.5; format.cb_y_position = 0.5; break; } + *last_chroma_location = frame->chroma_location; if (is_mjpeg && !format.full_range) { // Limited-range MJPEG is only detected by FFmpeg whenever a special @@ -214,6 +229,32 @@ YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *f return format; } +RGBTriplet get_neutral_color(AVDictionary *metadata) +{ + if (metadata == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0); + if (entry == nullptr) { + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + unsigned x_nom, x_den, y_nom, y_den; + if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) { + fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value); + return RGBTriplet(1.0f, 1.0f, 1.0f); + } + + double x = double(x_nom) / x_den; + double y = double(y_nom) / y_den; + double z = 1.0 - x - y; + + Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB); + Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z); + + return RGBTriplet(rgb[0], rgb[1], rgb[2]); +} + } // namespace FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height) @@ -226,12 +267,36 @@ FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned he avformat_network_init(); // In case someone wants this. } +#ifdef HAVE_SRT +FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id) + : srt_sock(srt_sock), + width(0), // Don't resize; SRT streams typically have stable resolution, and should behave much like regular cards in general. + height(0), + pixel_format(bmusb::PixelFormat_8BitYCbCrPlanar), + video_timebase{1, 1} +{ + if (stream_id.empty()) { + description = "SRT stream"; + } else { + description = stream_id; + } + play_as_fast_as_possible = true; + play_once = true; + last_frame = steady_clock::now(); +} +#endif + FFmpegCapture::~FFmpegCapture() { if (has_dequeue_callbacks) { dequeue_cleanup_callback(); } swr_free(&resampler); +#ifdef HAVE_SRT + if (srt_sock != -1) { + srt_close(srt_sock); + } +#endif } void FFmpegCapture::configure_card() @@ -273,12 +338,12 @@ std::map FFmpegCapture::get_available_video_modes() const VideoMode mode; char buf[256]; - snprintf(buf, sizeof(buf), "%ux%u", width, height); + snprintf(buf, sizeof(buf), "%ux%u", sws_last_width, sws_last_height); mode.name = buf; mode.autodetect = false; - mode.width = width; - mode.height = height; + mode.width = sws_last_width; + mode.height = sws_last_height; mode.frame_rate_num = 60; mode.frame_rate_den = 1; mode.interlaced = false; @@ -299,22 +364,38 @@ void FFmpegCapture::producer_thread_func() filename_copy = filename; } - string pathname = search_for_file(filename_copy); + string pathname; + if (srt_sock == -1) { + pathname = search_for_file(filename_copy); + } else { + pathname = description; + } if (pathname.empty()) { - fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str()); send_disconnected_frame(); + if (play_once) { + break; + } producer_thread_should_quit.sleep_for(seconds(1)); + fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str()); continue; } should_interrupt = false; if (!play_video(pathname)) { // Error. - fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str()); send_disconnected_frame(); + if (play_once) { + break; + } + fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str()); producer_thread_should_quit.sleep_for(seconds(1)); continue; } + if (play_once) { + send_disconnected_frame(); + break; + } + // Probably just EOF, will exit the loop above on next test. } @@ -328,19 +409,21 @@ void FFmpegCapture::send_disconnected_frame() { // Send an empty frame to signal that we have no signal anymore. FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame(); + size_t frame_width = width == 0 ? global_flags.width : width; + size_t frame_height = height == 0 ? global_flags.height : height; if (video_frame.data) { VideoFormat video_format; - video_format.width = width; - video_format.height = height; + video_format.width = frame_width; + video_format.height = frame_height; video_format.frame_rate_nom = 60; video_format.frame_rate_den = 1; video_format.is_connected = false; if (pixel_format == bmusb::PixelFormat_8BitBGRA) { - video_format.stride = width * 4; - video_frame.len = width * height * 4; + video_format.stride = frame_width * 4; + video_frame.len = frame_width * frame_height * 4; memset(video_frame.data, 0, video_frame.len); } else { - video_format.stride = width; + video_format.stride = frame_width; current_frame_ycbcr_format.luma_coefficients = YCBCR_REC_709; current_frame_ycbcr_format.full_range = true; current_frame_ycbcr_format.num_levels = 256; @@ -350,9 +433,9 @@ void FFmpegCapture::send_disconnected_frame() current_frame_ycbcr_format.cb_y_position = 0.0f; current_frame_ycbcr_format.cr_x_position = 0.0f; current_frame_ycbcr_format.cr_y_position = 0.0f; - video_frame.len = width * height * 2; - memset(video_frame.data, 0, width * height); - memset(video_frame.data + width * height, 128, width * height); // Valid for both NV12 and planar. + video_frame.len = frame_width * frame_height * 2; + memset(video_frame.data, 0, frame_width * frame_height); + memset(video_frame.data + frame_width * frame_height, 128, frame_width * frame_height); // Valid for both NV12 and planar. } frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++, @@ -360,6 +443,13 @@ void FFmpegCapture::send_disconnected_frame() FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat()); last_frame_was_connected = false; } + + if (play_once) { + disconnected = true; + if (card_disconnected_callback != nullptr) { + card_disconnected_callback(); + } + } } bool FFmpegCapture::play_video(const string &pathname) @@ -375,8 +465,26 @@ bool FFmpegCapture::play_video(const string &pathname) } else { last_modified = buf.st_mtim; } - - auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + last_colorspace = static_cast(-1); + last_chroma_location = static_cast(-1); + + AVFormatContextWithCloser format_ctx; + if (srt_sock == -1) { + // Regular file. + format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr, + /*options=*/nullptr, + AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); + } else { +#ifdef HAVE_SRT + // SRT socket, already opened. + AVInputFormat *mpegts_fmt = av_find_input_format("mpegts"); + format_ctx = avformat_open_input_unique(&FFmpegCapture::read_srt_thunk, this, + mpegts_fmt, /*options=*/nullptr, + AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this }); +#else + assert(false); +#endif + } if (format_ctx == nullptr) { fprintf(stderr, "%s: Error opening file\n", pathname.c_str()); return false; @@ -494,6 +602,22 @@ bool FFmpegCapture::play_video(const string &pathname) } VideoFormat video_format = construct_video_format(frame.get(), video_timebase); + if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { + // Invalid frame rate; try constructing it from the previous frame length. + // (This is especially important if we are the master card, for SRT, + // since it affects audio. Not all senders have good timebases + // (e.g., Larix rounds first to timebase 1000 and then multiplies by + // 90 from there, it seems), but it's much better to have an oscillating + // value than just locking at 60. + if (last_pts != 0 && frame->pts > last_pts) { + int64_t pts_diff = frame->pts - last_pts; + video_format.frame_rate_nom = video_timebase.den; + video_format.frame_rate_den = video_timebase.num * pts_diff; + } else { + video_format.frame_rate_nom = 60; + video_format.frame_rate_den = 1; + } + } UniqueFrame video_frame = make_video_frame(frame.get(), pathname, &error); if (error) { return false; @@ -575,6 +699,7 @@ bool FFmpegCapture::play_video(const string &pathname) // audio discontinuity.) timecode += MAX_FPS * 2 + 1; } + last_neutral_color = get_neutral_color(frame->metadata); frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++, video_frame.get_and_release(), 0, video_format, audio_frame.get_and_release(), 0, audio_format); @@ -824,23 +949,18 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator:: VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRational video_timebase) { VideoFormat video_format; - video_format.width = width; - video_format.height = height; + video_format.width = frame_width(frame); + video_format.height = frame_height(frame); if (pixel_format == bmusb::PixelFormat_8BitBGRA) { - video_format.stride = width * 4; + video_format.stride = frame_width(frame) * 4; } else if (pixel_format == FFmpegCapture::PixelFormat_NV12) { - video_format.stride = width; + video_format.stride = frame_width(frame); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); - video_format.stride = width; + video_format.stride = frame_width(frame); } video_format.frame_rate_nom = video_timebase.den; video_format.frame_rate_den = frame->pkt_duration * video_timebase.num; - if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) { - // Invalid frame rate. - video_format.frame_rate_nom = 60; - video_format.frame_rate_den = 1; - } video_format.has_signal = true; video_format.is_connected = true; return video_format; @@ -862,7 +982,7 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & sws_dst_format = decide_dst_format(AVPixelFormat(frame->format), pixel_format); sws_ctx.reset( sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format), - width, height, sws_dst_format, + frame_width(frame), frame_height(frame), sws_dst_format, SWS_BICUBIC, nullptr, nullptr, nullptr)); sws_last_width = frame->width; sws_last_height = frame->height; @@ -878,50 +998,81 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string & int linesizes[4] = { 0, 0, 0, 0 }; if (pixel_format == bmusb::PixelFormat_8BitBGRA) { pic_data[0] = video_frame->data; - linesizes[0] = width * 4; - video_frame->len = (width * 4) * height; + linesizes[0] = frame_width(frame) * 4; + video_frame->len = (frame_width(frame) * 4) * frame_height(frame); } else if (pixel_format == PixelFormat_NV12) { pic_data[0] = video_frame->data; - linesizes[0] = width; + linesizes[0] = frame_width(frame); - pic_data[1] = pic_data[0] + width * height; - linesizes[1] = width; + pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame); + linesizes[1] = frame_width(frame); - video_frame->len = (width * 2) * height; + video_frame->len = (frame_width(frame) * 2) * frame_height(frame); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } else { assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar); const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format); - int chroma_width = AV_CEIL_RSHIFT(int(width), desc->log2_chroma_w); - int chroma_height = AV_CEIL_RSHIFT(int(height), desc->log2_chroma_h); + int chroma_width = AV_CEIL_RSHIFT(int(frame_width(frame)), desc->log2_chroma_w); + int chroma_height = AV_CEIL_RSHIFT(int(frame_height(frame)), desc->log2_chroma_h); pic_data[0] = video_frame->data; - linesizes[0] = width; + linesizes[0] = frame_width(frame); - pic_data[1] = pic_data[0] + width * height; + pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame); linesizes[1] = chroma_width; pic_data[2] = pic_data[1] + chroma_width * chroma_height; linesizes[2] = chroma_width; - video_frame->len = width * height + 2 * chroma_width * chroma_height; + video_frame->len = frame_width(frame) * frame_height(frame) + 2 * chroma_width * chroma_height; - current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg); + current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location); } sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes); return video_frame; } -int FFmpegCapture::interrupt_cb_thunk(void *unique) +int FFmpegCapture::interrupt_cb_thunk(void *opaque) { - return reinterpret_cast(unique)->interrupt_cb(); + return reinterpret_cast(opaque)->interrupt_cb(); } int FFmpegCapture::interrupt_cb() { return should_interrupt.load(); } + +unsigned FFmpegCapture::frame_width(const AVFrame *frame) const +{ + if (width == 0) { + return frame->width; + } else { + return width; + } +} + +unsigned FFmpegCapture::frame_height(const AVFrame *frame) const +{ + if (height == 0) { + return frame->height; + } else { + return width; + } +} + +#ifdef HAVE_SRT +int FFmpegCapture::read_srt_thunk(void *opaque, uint8_t *buf, int buf_size) +{ + return reinterpret_cast(opaque)->read_srt(buf, buf_size); +} + +int FFmpegCapture::read_srt(uint8_t *buf, int buf_size) +{ + SRT_MSGCTRL mc = srt_msgctrl_default; + return srt_recvmsg2(srt_sock, reinterpret_cast(buf), buf_size, &mc); +} +#endif