#include "ffmpeg_capture.h"
+#include "defs.h"
+#include "shared/shared_defs.h"
#include <assert.h>
+#include <errno.h>
+#include <epoxy/egl.h>
+#include <limits>
+#include <map>
+#include <memory>
+#include <movit/effect.h>
+#include <movit/image_format.h>
+#include <movit/ycbcr.h>
+#include <mutex>
#include <pthread.h>
+#include <time.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <string>
#include <sys/stat.h>
-#include <unistd.h>
+#include <thread>
+#include <QSurface>
extern "C" {
#include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
+#include <libavcodec/codec_id.h>
+#include <libavcodec/codec_par.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
+#include <libavutil/buffer.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/common.h>
+#include <libavutil/dict.h>
#include <libavutil/error.h>
#include <libavutil/frame.h>
-#include <libavutil/imgutils.h>
-#include <libavutil/mem.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/pixdesc.h>
#include <libavutil/pixfmt.h>
-#include <libavutil/opt.h>
+#include <libavutil/rational.h>
+#include <libavutil/samplefmt.h>
+#include <libavutil/version.h>
+#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
}
#include <cstdint>
#include <utility>
#include <vector>
+#include <unordered_set>
+
+#include <Eigen/Core>
+#include <Eigen/LU>
+#include <movit/colorspace_conversion_effect.h>
#include "bmusb/bmusb.h"
+#include "shared/context.h"
#include "shared/ffmpeg_raii.h"
#include "ffmpeg_util.h"
#include "flags.h"
-#include "image_input.h"
#include "ref_counted_frame.h"
#include "shared/timebase.h"
-#define FRAME_SIZE (8 << 20) // 8 MB.
+#ifdef HAVE_SRT
+#include <srt/srt.h>
+#endif
using namespace std;
using namespace std::chrono;
using namespace bmusb;
using namespace movit;
+using namespace Eigen;
+
+// Avoid deprecation warnings, but we don't want to drop FFmpeg 5.1 support just yet.
+#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 30, 100)
+#define pkt_duration duration
+#endif
namespace {
if (desc->comp[0].depth != 8) continue;
// Same or better chroma resolution only.
- int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w;
- int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h;
+ int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
+ int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
if (chroma_w_diff < 0 || chroma_h_diff < 0)
continue;
return av_get_pix_fmt(best_format);
}
-YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg)
+YCbCrFormat decode_ycbcr_format(const AVPixFmtDescriptor *desc, const AVFrame *frame, bool is_mjpeg, AVColorSpace *last_colorspace, AVChromaLocation *last_chroma_location)
{
YCbCrFormat format;
AVColorSpace colorspace = frame->colorspace;
format.luma_coefficients = (frame->height >= 720 ? YCBCR_REC_709 : YCBCR_REC_601);
break;
default:
- fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n",
- colorspace);
+ if (colorspace != *last_colorspace) {
+ fprintf(stderr, "Unknown Y'CbCr coefficient enum %d from FFmpeg; choosing Rec. 709.\n",
+ colorspace);
+ }
format.luma_coefficients = YCBCR_REC_709;
break;
}
+ *last_colorspace = colorspace;
format.full_range = is_full_range(desc);
format.num_levels = 1 << desc->comp[0].depth;
format.cb_y_position = 1.0;
break;
default:
- fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing Rec. 709.\n",
- frame->chroma_location);
+ if (frame->chroma_location != *last_chroma_location) {
+ fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n",
+ frame->chroma_location);
+ }
format.cb_x_position = 0.5;
format.cb_y_position = 0.5;
break;
}
+ *last_chroma_location = frame->chroma_location;
if (is_mjpeg && !format.full_range) {
// Limited-range MJPEG is only detected by FFmpeg whenever a special
return format;
}
+RGBTriplet get_neutral_color(AVDictionary *metadata)
+{
+ if (metadata == nullptr) {
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+ AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0);
+ if (entry == nullptr) {
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+
+ unsigned x_nom, x_den, y_nom, y_den;
+ if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) {
+ fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value);
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+
+ double x = double(x_nom) / x_den;
+ double y = double(y_nom) / y_den;
+ double z = 1.0 - x - y;
+
+ Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB);
+ Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z);
+
+ return RGBTriplet(rgb[0], rgb[1], rgb[2]);
+}
+
} // namespace
-FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height)
- : filename(filename), width(width), height(height), video_timebase{1, 1}
+FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height, QSurface *surface)
+ : filename(filename), width(width), height(height), video_timebase{1, 1}, surface(surface)
{
description = "Video: " + filename;
avformat_network_init(); // In case someone wants this.
}
+#ifdef HAVE_SRT
+FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id, QSurface *surface)
+ : srt_sock(srt_sock),
+ width(0), // Don't resize; SRT streams typically have stable resolution, and should behave much like regular cards in general.
+ height(0),
+ pixel_format(bmusb::PixelFormat_8BitYCbCrPlanar),
+ video_timebase{1, 1},
+ surface(surface)
+{
+ if (stream_id.empty()) {
+ description = "SRT stream";
+ } else {
+ description = stream_id;
+ }
+ play_as_fast_as_possible = true;
+ play_once = true;
+ last_frame = steady_clock::now();
+}
+#endif
+
FFmpegCapture::~FFmpegCapture()
{
if (has_dequeue_callbacks) {
dequeue_cleanup_callback();
}
swr_free(&resampler);
+#ifdef HAVE_SRT
+ if (srt_sock != -1) {
+ srt_close(srt_sock);
+ }
+#endif
+ delete surface;
}
void FFmpegCapture::configure_card()
VideoMode mode;
char buf[256];
- snprintf(buf, sizeof(buf), "%ux%u", width, height);
+ snprintf(buf, sizeof(buf), "%ux%u", sws_last_width, sws_last_height);
mode.name = buf;
mode.autodetect = false;
- mode.width = width;
- mode.height = height;
+ mode.width = sws_last_width;
+ mode.height = sws_last_height;
mode.frame_rate_num = 60;
mode.frame_rate_den = 1;
mode.interlaced = false;
snprintf(thread_name, sizeof(thread_name), "FFmpeg_C_%d", card_index);
pthread_setname_np(pthread_self(), thread_name);
+ // We need a context in case create_frame() needs to reallocate something.
+ // (If none is given, we are probably in Kaeru, which uses MallocFrameAllocator
+ // anyway, which doesn't reallocate currently and definitely doesn't need
+ // an active OpenGL context to do so.)
+ QOpenGLContext *context = nullptr;
+ if (surface != nullptr) {
+ context = create_context(this->surface);
+ eglBindAPI(EGL_OPENGL_API);
+ if (!make_current(context, this->surface)) {
+ printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(),
+ eglGetError());
+ abort();
+ }
+ }
+
while (!producer_thread_should_quit.should_quit()) {
string filename_copy;
{
filename_copy = filename;
}
- string pathname = search_for_file(filename_copy);
+ string pathname;
+ if (srt_sock == -1) {
+ pathname = search_for_file(filename_copy);
+ } else {
+ pathname = description;
+ }
if (pathname.empty()) {
- fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str());
send_disconnected_frame();
+ if (play_once) {
+ break;
+ }
producer_thread_should_quit.sleep_for(seconds(1));
+ fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str());
continue;
}
should_interrupt = false;
if (!play_video(pathname)) {
// Error.
- fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str());
send_disconnected_frame();
+ if (play_once) {
+ break;
+ }
+ fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str());
producer_thread_should_quit.sleep_for(seconds(1));
continue;
}
+ if (play_once) {
+ send_disconnected_frame();
+ break;
+ }
+
// Probably just EOF, will exit the loop above on next test.
}
if (has_dequeue_callbacks) {
- dequeue_cleanup_callback();
+ dequeue_cleanup_callback();
has_dequeue_callbacks = false;
- }
+ }
+
+ delete_context(context);
}
void FFmpegCapture::send_disconnected_frame()
{
// Send an empty frame to signal that we have no signal anymore.
FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame();
+ size_t frame_width = width == 0 ? global_flags.width : width;
+ size_t frame_height = height == 0 ? global_flags.height : height;
if (video_frame.data) {
VideoFormat video_format;
- video_format.width = width;
- video_format.height = height;
+ video_format.width = frame_width;
+ video_format.height = frame_height;
video_format.frame_rate_nom = 60;
video_format.frame_rate_den = 1;
video_format.is_connected = false;
if (pixel_format == bmusb::PixelFormat_8BitBGRA) {
- video_format.stride = width * 4;
- video_frame.len = width * height * 4;
+ video_format.stride = frame_width * 4;
+ video_frame.len = frame_width * frame_height * 4;
memset(video_frame.data, 0, video_frame.len);
} else {
- video_format.stride = width;
+ video_format.stride = frame_width;
current_frame_ycbcr_format.luma_coefficients = YCBCR_REC_709;
current_frame_ycbcr_format.full_range = true;
current_frame_ycbcr_format.num_levels = 256;
current_frame_ycbcr_format.cb_y_position = 0.0f;
current_frame_ycbcr_format.cr_x_position = 0.0f;
current_frame_ycbcr_format.cr_y_position = 0.0f;
- video_frame.len = width * height * 2;
- memset(video_frame.data, 0, width * height);
- memset(video_frame.data + width * height, 128, width * height); // Valid for both NV12 and planar.
+ video_frame.len = frame_width * frame_height * 2;
+ memset(video_frame.data, 0, frame_width * frame_height);
+ memset(video_frame.data + frame_width * frame_height, 128, frame_width * frame_height); // Valid for both NV12 and planar.
}
- frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++,
- video_frame, /*video_offset=*/0, video_format,
- FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat());
+ if (frame_callback != nullptr) {
+ frame_callback(-1, AVRational{1, TIMEBASE}, -1, AVRational{1, TIMEBASE}, timecode++,
+ video_frame, /*video_offset=*/0, video_format,
+ FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat());
+ }
last_frame_was_connected = false;
}
+
+ if (play_once) {
+ disconnected = true;
+ if (card_disconnected_callback != nullptr) {
+ card_disconnected_callback();
+ }
+ }
+}
+
+template<AVHWDeviceType type>
+AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
+{
+ bool found_config_of_right_type = false;
+ for (int i = 0;; ++i) { // Termination condition inside loop.
+ const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
+ if (config == nullptr) { // End of list.
+ break;
+ }
+ if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
+ config->device_type != type) {
+ // Not interesting for us.
+ continue;
+ }
+
+ // We have a config of the right type, but does it actually support
+ // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
+ // is to just replace the pixel format with a software-decoded one,
+ // such as yuv420p.)
+ found_config_of_right_type = true;
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if (config->pix_fmt == *fmt_ptr) {
+ fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
+ av_hwdevice_get_type_name(type), ctx->codec->name);
+ if (ctx->profile == FF_PROFILE_H264_BASELINE) {
+ fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
+ fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n");
+ fprintf(stderr, " Decoding might fail and fall back to software.\n");
+ }
+ return config->pix_fmt;
+ }
+ }
+ fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
+ unordered_set<AVPixelFormat> seen;
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if (!seen.count(*fmt_ptr)) {
+ fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
+ seen.insert(*fmt_ptr);
+ }
+ }
+ fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
+
+ }
+
+ if (!found_config_of_right_type) {
+ fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
+ }
+
+ // We found no VA-API formats, so take the first software format.
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
+ fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
+ return *fmt_ptr;
+ }
+ }
+
+ // Fallback: Just return anything. (Should never really happen.)
+ return fmt[0];
}
bool FFmpegCapture::play_video(const string &pathname)
} else {
last_modified = buf.st_mtim;
}
-
- auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+ last_colorspace = static_cast<AVColorSpace>(-1);
+ last_chroma_location = static_cast<AVChromaLocation>(-1);
+
+ AVFormatContextWithCloser format_ctx;
+ if (srt_sock == -1) {
+ // Regular file (or stream).
+ frame_timeout_started = steady_clock::now();
+ frame_timeout_valid = true;
+ format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
+ /*options=*/nullptr,
+ AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+ frame_timeout_valid = false;
+ } else {
+#ifdef HAVE_SRT
+ // SRT socket, already opened.
+ const AVInputFormat *mpegts_fmt = av_find_input_format("mpegts");
+ format_ctx = avformat_open_input_unique(&FFmpegCapture::read_srt_thunk, this,
+ mpegts_fmt, /*options=*/nullptr,
+ AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+#else
+ assert(false);
+#endif
+ }
if (format_ctx == nullptr) {
fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
return false;
// Open video decoder.
const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
- AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
+ const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
+
video_timebase = format_ctx->streams[video_stream_index]->time_base;
AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
return false;
}
+
+ // Seemingly, it's not too easy to make something that just initializes
+ // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
+ // without enumerating through several different types.
+ // VA-API and VDPAU will do for now. We prioritize VDPAU for the
+ // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
+ // cards that seems to work, but just hangs when trying to transfer the frame.
+ //
+ // Note that we don't actually check codec support beforehand,
+ // so if you have a low-end VDPAU device but a high-end VA-API device,
+ // you lose out on the extra codec support from the latter.
+ AVBufferRef *hw_device_ctx = nullptr;
+ if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
+ video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+ video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
+ } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
+ video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+ video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
+ } else {
+ fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
+ }
+
if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
return false;
fprintf(stderr, "%s: Cannot fill audio codec parameters\n", pathname.c_str());
return false;
}
- AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id);
+ const AVCodec *audio_codec = avcodec_find_decoder(audio_codecpar->codec_id);
if (audio_codec == nullptr) {
fprintf(stderr, "%s: Cannot find audio decoder\n", pathname.c_str());
return false;
// Main loop.
bool first_frame = true;
+ int consecutive_errors = 0;
while (!producer_thread_should_quit.should_quit()) {
if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) {
return true;
int64_t audio_pts;
bool error;
+ frame_timeout_started = steady_clock::now();
+ frame_timeout_valid = true;
AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(),
pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
- if (error) {
+ frame_timeout_valid = false;
+ if (should_interrupt.load()) {
+ // Abort no matter whether we got a frame or not.
return false;
}
+ if (error) {
+ if (++consecutive_errors >= 100) {
+ fprintf(stderr, "More than 100 consecutive error video frames, aborting playback.\n");
+ return false;
+ } else {
+ continue;
+ }
+ } else {
+ consecutive_errors = 0;
+ }
if (frame == nullptr) {
// EOF. Loop back to the start if we can.
if (format_ctx->pb != nullptr && format_ctx->pb->seekable == 0) {
}
VideoFormat video_format = construct_video_format(frame.get(), video_timebase);
+ if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) {
+ // Invalid frame rate; try constructing it from the previous frame length.
+ // (This is especially important if we are the master card, for SRT,
+ // since it affects audio. Not all senders have good timebases
+ // (e.g., Larix rounds first to timebase 1000 and then multiplies by
+ // 90 from there, it seems), but it's much better to have an oscillating
+ // value than just locking at 60.
+ if (last_pts != 0 && frame->pts > last_pts) {
+ int64_t pts_diff = frame->pts - last_pts;
+ video_format.frame_rate_nom = video_timebase.den;
+ video_format.frame_rate_den = video_timebase.num * pts_diff;
+ } else {
+ video_format.frame_rate_nom = 60;
+ video_format.frame_rate_den = 1;
+ }
+ }
UniqueFrame video_frame = make_video_frame(frame.get(), pathname, &error);
if (error) {
return false;
if (last_pts == 0 && pts_origin == 0) {
pts_origin = frame->pts;
}
- next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
- if (first_frame && last_frame_was_connected) {
- // If reconnect took more than one second, this is probably a live feed,
- // and we should reset the resampler. (Or the rate is really, really low,
- // in which case a reset on the first frame is fine anyway.)
- if (duration<double>(next_frame_start - last_frame).count() >= 1.0) {
- last_frame_was_connected = false;
+ steady_clock::time_point now = steady_clock::now();
+ if (play_as_fast_as_possible) {
+ video_frame->received_timestamp = now;
+ audio_frame->received_timestamp = now;
+ next_frame_start = now;
+ } else {
+ next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
+ if (first_frame && last_frame_was_connected) {
+ // If reconnect took more than one second, this is probably a live feed,
+ // and we should reset the resampler. (Or the rate is really, really low,
+ // in which case a reset on the first frame is fine anyway.)
+ if (duration<double>(next_frame_start - last_frame).count() >= 1.0) {
+ last_frame_was_connected = false;
+ }
+ }
+ video_frame->received_timestamp = next_frame_start;
+
+ // The easiest way to get all the rate conversions etc. right is to move the
+ // audio PTS into the video PTS timebase and go from there. (We'll get some
+ // rounding issues, but they should not be a big problem.)
+ int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase);
+ audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate);
+
+ if (audio_frame->len != 0) {
+ // The received timestamps in Nageru are measured after we've just received the frame.
+ // However, pts (especially audio pts) is at the _beginning_ of the frame.
+ // If we have locked audio, the distinction doesn't really matter, as pts is
+ // on a relative scale and a fixed offset is fine. But if we don't, we will have
+ // a different number of samples each time, which will cause huge audio jitter
+ // and throw off the resampler.
+ //
+ // In a sense, we should have compensated by adding the frame and audio lengths
+ // to video_frame->received_timestamp and audio_frame->received_timestamp respectively,
+ // but that would mean extra waiting in sleep_until(). All we need is that they
+ // are correct relative to each other, though (and to the other frames we send),
+ // so just align the end of the audio frame, and we're fine.
+ size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels;
+ double offset = double(num_samples) / OUTPUT_FREQUENCY -
+ double(video_format.frame_rate_den) / video_format.frame_rate_nom;
+ audio_frame->received_timestamp += duration_cast<steady_clock::duration>(duration<double>(offset));
}
- }
- video_frame->received_timestamp = next_frame_start;
-
- // The easiest way to get all the rate conversions etc. right is to move the
- // audio PTS into the video PTS timebase and go from there. (We'll get some
- // rounding issues, but they should not be a big problem.)
- int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase);
- audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate);
-
- if (audio_frame->len != 0) {
- // The received timestamps in Nageru are measured after we've just received the frame.
- // However, pts (especially audio pts) is at the _beginning_ of the frame.
- // If we have locked audio, the distinction doesn't really matter, as pts is
- // on a relative scale and a fixed offset is fine. But if we don't, we will have
- // a different number of samples each time, which will cause huge audio jitter
- // and throw off the resampler.
- //
- // In a sense, we should have compensated by adding the frame and audio lengths
- // to video_frame->received_timestamp and audio_frame->received_timestamp respectively,
- // but that would mean extra waiting in sleep_until(). All we need is that they
- // are correct relative to each other, though (and to the other frames we send),
- // so just align the end of the audio frame, and we're fine.
- size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels;
- double offset = double(num_samples) / OUTPUT_FREQUENCY -
- double(video_format.frame_rate_den) / video_format.frame_rate_nom;
- audio_frame->received_timestamp += duration_cast<steady_clock::duration>(duration<double>(offset));
- }
- steady_clock::time_point now = steady_clock::now();
- if (duration<double>(now - next_frame_start).count() >= 0.1) {
- // If we don't have enough CPU to keep up, or if we have a live stream
- // where the initial origin was somehow wrong, we could be behind indefinitely.
- // In particular, this will give the audio resampler problems as it tries
- // to speed up to reduce the delay, hitting the low end of the buffer every time.
- fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
- pathname.c_str(),
- 1e3 * duration<double>(now - next_frame_start).count());
- pts_origin = frame->pts;
- start = next_frame_start = now;
- timecode += MAX_FPS * 2 + 1;
+ if (duration<double>(now - next_frame_start).count() >= 0.1) {
+ // If we don't have enough CPU to keep up, or if we have a live stream
+ // where the initial origin was somehow wrong, we could be behind indefinitely.
+ // In particular, this will give the audio resampler problems as it tries
+ // to speed up to reduce the delay, hitting the low end of the buffer every time.
+ fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
+ pathname.c_str(),
+ 1e3 * duration<double>(now - next_frame_start).count());
+ pts_origin = frame->pts;
+ start = next_frame_start = now;
+ timecode += TYPICAL_FPS * 2 + 1;
+ }
+ }
+ bool finished_wakeup;
+ if (play_as_fast_as_possible) {
+ finished_wakeup = !producer_thread_should_quit.should_quit();
+ } else {
+ finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
}
- bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
if (finished_wakeup) {
if (audio_frame->len > 0) {
assert(audio_pts != -1);
// Make sure to get the audio resampler reset. (This is a hack;
// ideally, the frame callback should just accept a way to signal
// audio discontinuity.)
- timecode += MAX_FPS * 2 + 1;
+ timecode += TYPICAL_FPS * 2 + 1;
+ }
+ last_neutral_color = get_neutral_color(frame->metadata);
+ if (frame_callback != nullptr) {
+ frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++,
+ video_frame.get_and_release(), 0, video_format,
+ audio_frame.get_and_release(), 0, audio_format);
}
- frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++,
- video_frame.get_and_release(), 0, video_format,
- audio_frame.get_and_release(), 0, audio_format);
first_frame = false;
last_frame = steady_clock::now();
last_frame_was_connected = true;
start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate);
pts_origin = last_pts;
rate = cmd.new_rate;
+ play_as_fast_as_possible = (rate >= 10.0);
break;
}
}
*audio_pts = -1;
bool has_audio = false;
do {
- AVPacket pkt;
- unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
- &pkt, av_packet_unref);
- av_init_packet(&pkt);
- pkt.data = nullptr;
- pkt.size = 0;
- if (av_read_frame(format_ctx, &pkt) == 0) {
- if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) {
- audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base);
+ AVPacketWithDeleter pkt = av_packet_alloc_unique();
+ pkt->data = nullptr;
+ pkt->size = 0;
+ if (av_read_frame(format_ctx, pkt.get()) == 0) {
+ if (pkt->stream_index == audio_stream_index && audio_callback != nullptr) {
+ audio_callback(pkt.get(), format_ctx->streams[audio_stream_index]->time_base);
+ }
+ if (pkt->stream_index == video_stream_index && video_callback != nullptr) {
+ video_callback(pkt.get(), format_ctx->streams[video_stream_index]->time_base);
}
- if (pkt.stream_index == video_stream_index) {
- if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) {
+ if (pkt->stream_index == video_stream_index && global_flags.transcode_video) {
+ if (avcodec_send_packet(video_codec_ctx, pkt.get()) < 0) {
fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
*error = true;
return AVFrameWithDeleter(nullptr);
}
- } else if (pkt.stream_index == audio_stream_index) {
+ } else if (pkt->stream_index == audio_stream_index && global_flags.transcode_audio) {
has_audio = true;
- if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) {
+ if (avcodec_send_packet(audio_codec_ctx, pkt.get()) < 0) {
fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str());
*error = true;
return AVFrameWithDeleter(nullptr);
}
- } else if (pkt.stream_index == subtitle_stream_index) {
- last_subtitle = string(reinterpret_cast<const char *>(pkt.data), pkt.size);
+ } else if (pkt->stream_index == subtitle_stream_index) {
+ last_subtitle = string(reinterpret_cast<const char *>(pkt->data), pkt->size);
has_last_subtitle = true;
}
} else {
// Decode video, if we have a frame.
int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
if (err == 0) {
+ if (video_avframe->format == AV_PIX_FMT_VAAPI ||
+ video_avframe->format == AV_PIX_FMT_VDPAU) {
+ // Get the frame down to the CPU. (TODO: See if we can keep it
+ // on the GPU all the way, since it will be going up again later.
+ // However, this only works if the OpenGL GPU is the same one.)
+ AVFrameWithDeleter sw_frame = av_frame_alloc_unique();
+ int err = av_hwframe_transfer_data(sw_frame.get(), video_avframe.get(), 0);
+ if (err != 0) {
+ fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
+ *error = true;
+ return AVFrameWithDeleter(nullptr);
+ }
+ sw_frame->pts = video_avframe->pts;
+ sw_frame->pkt_duration = video_avframe->pkt_duration;
+ video_avframe = move(sw_frame);
+ }
frame_finished = true;
break;
} else if (err != AVERROR(EAGAIN)) {
}
audio_format->num_channels = 2;
- int64_t channel_layout = audio_avframe->channel_layout;
- if (channel_layout == 0) {
- channel_layout = av_get_default_channel_layout(audio_avframe->channels);
+ AVChannelLayout channel_layout = audio_avframe->ch_layout;
+ if (!av_channel_layout_check(&channel_layout) ||
+ channel_layout.order == AV_CHANNEL_ORDER_UNSPEC) {
+ av_channel_layout_default(&channel_layout, audio_avframe->ch_layout.nb_channels);
}
if (resampler == nullptr ||
audio_avframe->format != last_src_format ||
dst_format != last_dst_format ||
- channel_layout != last_channel_layout ||
+ av_channel_layout_compare(&channel_layout, &last_channel_layout) != 0||
audio_avframe->sample_rate != last_sample_rate) {
+ // TODO: When we get C++20, use AV_CHANNEL_LAYOUT_STEREO_DOWNMIX.
+ AVChannelLayout stereo_downmix;
+ stereo_downmix.order = AV_CHANNEL_ORDER_NATIVE;
+ stereo_downmix.nb_channels = 2;
+ stereo_downmix.u.mask = AV_CH_LAYOUT_STEREO_DOWNMIX;
+
swr_free(&resampler);
- resampler = swr_alloc_set_opts(nullptr,
- /*out_ch_layout=*/AV_CH_LAYOUT_STEREO_DOWNMIX,
- /*out_sample_fmt=*/dst_format,
- /*out_sample_rate=*/OUTPUT_FREQUENCY,
- /*in_ch_layout=*/channel_layout,
- /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format),
- /*in_sample_rate=*/audio_avframe->sample_rate,
- /*log_offset=*/0,
- /*log_ctx=*/nullptr);
-
- if (resampler == nullptr) {
+ resampler = nullptr;
+ int err = swr_alloc_set_opts2(&resampler,
+ /*out_ch_layout=*/&stereo_downmix,
+ /*out_sample_fmt=*/dst_format,
+ /*out_sample_rate=*/OUTPUT_FREQUENCY,
+ /*in_ch_layout=*/&channel_layout,
+ /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format),
+ /*in_sample_rate=*/audio_avframe->sample_rate,
+ /*log_offset=*/0,
+ /*log_ctx=*/nullptr);
+
+ if (err != 0 || resampler == nullptr) {
fprintf(stderr, "Allocating resampler failed.\n");
- exit(1);
+ abort();
}
if (swr_init(resampler) < 0) {
fprintf(stderr, "Could not open resample context.\n");
- exit(1);
+ abort();
}
last_src_format = AVSampleFormat(audio_avframe->format);
int out_samples = swr_convert(resampler, &data, num_samples_room,
const_cast<const uint8_t **>(audio_avframe->data), audio_avframe->nb_samples);
if (out_samples < 0) {
- fprintf(stderr, "Audio conversion failed.\n");
- exit(1);
- }
+ fprintf(stderr, "Audio conversion failed.\n");
+ abort();
+ }
audio_frame->len += out_samples * bytes_per_sample;
}
VideoFormat FFmpegCapture::construct_video_format(const AVFrame *frame, AVRational video_timebase)
{
VideoFormat video_format;
- video_format.width = width;
- video_format.height = height;
+ video_format.width = frame_width(frame);
+ video_format.height = frame_height(frame);
if (pixel_format == bmusb::PixelFormat_8BitBGRA) {
- video_format.stride = width * 4;
+ video_format.stride = frame_width(frame) * 4;
} else if (pixel_format == FFmpegCapture::PixelFormat_NV12) {
- video_format.stride = width;
+ video_format.stride = frame_width(frame);
} else {
assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar);
- video_format.stride = width;
+ video_format.stride = frame_width(frame);
}
video_format.frame_rate_nom = video_timebase.den;
video_format.frame_rate_den = frame->pkt_duration * video_timebase.num;
- if (video_format.frame_rate_nom == 0 || video_format.frame_rate_den == 0) {
- // Invalid frame rate.
- video_format.frame_rate_nom = 60;
- video_format.frame_rate_den = 1;
- }
video_format.has_signal = true;
video_format.is_connected = true;
return video_format;
{
*error = false;
- UniqueFrame video_frame(video_frame_allocator->alloc_frame());
+ UniqueFrame video_frame(video_frame_allocator->create_frame(frame->width, frame->height, frame->width));
if (video_frame->data == nullptr) {
return video_frame;
}
sws_dst_format = decide_dst_format(AVPixelFormat(frame->format), pixel_format);
sws_ctx.reset(
sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
- width, height, sws_dst_format,
+ frame_width(frame), frame_height(frame), sws_dst_format,
SWS_BICUBIC, nullptr, nullptr, nullptr));
sws_last_width = frame->width;
sws_last_height = frame->height;
int linesizes[4] = { 0, 0, 0, 0 };
if (pixel_format == bmusb::PixelFormat_8BitBGRA) {
pic_data[0] = video_frame->data;
- linesizes[0] = width * 4;
- video_frame->len = (width * 4) * height;
+ linesizes[0] = frame_width(frame) * 4;
+ video_frame->len = (frame_width(frame) * 4) * frame_height(frame);
} else if (pixel_format == PixelFormat_NV12) {
pic_data[0] = video_frame->data;
- linesizes[0] = width;
+ linesizes[0] = frame_width(frame);
- pic_data[1] = pic_data[0] + width * height;
- linesizes[1] = width;
+ pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame);
+ linesizes[1] = frame_width(frame);
- video_frame->len = (width * 2) * height;
+ video_frame->len = (frame_width(frame) * 2) * frame_height(frame);
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
- current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg);
+ current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location);
} else {
assert(pixel_format == bmusb::PixelFormat_8BitYCbCrPlanar);
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
- int chroma_width = AV_CEIL_RSHIFT(int(width), desc->log2_chroma_w);
- int chroma_height = AV_CEIL_RSHIFT(int(height), desc->log2_chroma_h);
+ int chroma_width = AV_CEIL_RSHIFT(int(frame_width(frame)), desc->log2_chroma_w);
+ int chroma_height = AV_CEIL_RSHIFT(int(frame_height(frame)), desc->log2_chroma_h);
pic_data[0] = video_frame->data;
- linesizes[0] = width;
+ linesizes[0] = frame_width(frame);
- pic_data[1] = pic_data[0] + width * height;
+ pic_data[1] = pic_data[0] + frame_width(frame) * frame_height(frame);
linesizes[1] = chroma_width;
pic_data[2] = pic_data[1] + chroma_width * chroma_height;
linesizes[2] = chroma_width;
- video_frame->len = width * height + 2 * chroma_width * chroma_height;
+ video_frame->len = frame_width(frame) * frame_height(frame) + 2 * chroma_width * chroma_height;
- current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg);
+ current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location);
}
+
+ // FIXME: Currently, if the video is too high-res for one of the allocated
+ // frames, we simply refuse to scale it here to avoid crashes. It would be better
+ // if we could somehow signal getting larger frames, especially as 4K is a thing now.
+ if (video_frame->len > video_frame->size) {
+ fprintf(stderr, "%s: Decoded frame would be larger than supported frame size (%zu > %zu), not decoding.\n", pathname.c_str(), video_frame->len, video_frame->size);
+ *error = true;
+ return video_frame;
+ }
+
sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
return video_frame;
}
-int FFmpegCapture::interrupt_cb_thunk(void *unique)
+int FFmpegCapture::interrupt_cb_thunk(void *opaque)
{
- return reinterpret_cast<FFmpegCapture *>(unique)->interrupt_cb();
+ return reinterpret_cast<FFmpegCapture *>(opaque)->interrupt_cb();
}
int FFmpegCapture::interrupt_cb()
{
+ // If ten seconds is gone without anything happening, we assume that
+ // we are in a network stream that died and FFmpeg just didn't
+ // pick it up (or perhaps it just hung, keeping the connection open).
+ // Called back approximately every 100 ms if something is hanging,
+ // so we get more than enough accuracy for our purposes.
+ if (!should_interrupt && frame_timeout_valid &&
+ duration<double>(steady_clock::now() - frame_timeout_started).count() >= 10.0) {
+ string filename_copy;
+ {
+ lock_guard<mutex> lock(filename_mu);
+ filename_copy = filename;
+ }
+ fprintf(stderr, "%s: No frame for more than 10 seconds, restarting stream.\n", filename.c_str());
+ should_interrupt = true;
+ }
return should_interrupt.load();
}
+
+unsigned FFmpegCapture::frame_width(const AVFrame *frame) const
+{
+ if (width == 0) {
+ return frame->width;
+ } else {
+ return width;
+ }
+}
+
+unsigned FFmpegCapture::frame_height(const AVFrame *frame) const
+{
+ if (height == 0) {
+ return frame->height;
+ } else {
+ return height;
+ }
+}
+
+#ifdef HAVE_SRT
+int FFmpegCapture::read_srt_thunk(void *opaque, uint8_t *buf, int buf_size)
+{
+ return reinterpret_cast<FFmpegCapture *>(opaque)->read_srt(buf, buf_size);
+}
+
+int FFmpegCapture::read_srt(uint8_t *buf, int buf_size)
+{
+ SRT_MSGCTRL mc = srt_msgctrl_default;
+ return srt_recvmsg2(srt_sock, reinterpret_cast<char *>(buf), buf_size, &mc);
+}
+#endif
projects / nageru / blobdiff