#include <utility>
#include <vector>
+#include <Eigen/Core>
+#include <Eigen/LU>
+#include <movit/colorspace_conversion_effect.h>
+
#include "bmusb/bmusb.h"
#include "shared/ffmpeg_raii.h"
#include "ffmpeg_util.h"
#include "ref_counted_frame.h"
#include "shared/timebase.h"
+#ifdef HAVE_SRT
+#include <srt/srt.h>
+#endif
+
#define FRAME_SIZE (8 << 20) // 8 MB.
using namespace std;
using namespace std::chrono;
using namespace bmusb;
using namespace movit;
+using namespace Eigen;
namespace {
format.cb_y_position = 1.0;
break;
default:
- fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing Rec. 709.\n",
+ fprintf(stderr, "Unknown chroma location coefficient enum %d from FFmpeg; choosing center.\n",
frame->chroma_location);
format.cb_x_position = 0.5;
format.cb_y_position = 0.5;
return format;
}
+RGBTriplet get_neutral_color(AVDictionary *metadata)
+{
+ if (metadata == nullptr) {
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+ AVDictionaryEntry *entry = av_dict_get(metadata, "WhitePoint", nullptr, 0);
+ if (entry == nullptr) {
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+
+ unsigned x_nom, x_den, y_nom, y_den;
+ if (sscanf(entry->value, " %u:%u , %u:%u", &x_nom, &x_den, &y_nom, &y_den) != 4) {
+ fprintf(stderr, "WARNING: Unable to parse white point '%s', using default white point\n", entry->value);
+ return RGBTriplet(1.0f, 1.0f, 1.0f);
+ }
+
+ double x = double(x_nom) / x_den;
+ double y = double(y_nom) / y_den;
+ double z = 1.0 - x - y;
+
+ Matrix3d rgb_to_xyz_matrix = movit::ColorspaceConversionEffect::get_xyz_matrix(COLORSPACE_sRGB);
+ Vector3d rgb = rgb_to_xyz_matrix.inverse() * Vector3d(x, y, z);
+
+ return RGBTriplet(rgb[0], rgb[1], rgb[2]);
+}
+
} // namespace
FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height)
avformat_network_init(); // In case someone wants this.
}
+#ifdef HAVE_SRT
+FFmpegCapture::FFmpegCapture(int srt_sock, const string &stream_id)
+ : srt_sock(srt_sock),
+ width(global_flags.width),
+ height(global_flags.height),
+ pixel_format(bmusb::PixelFormat_8BitYCbCrPlanar),
+ video_timebase{1, 1}
+{
+ if (stream_id.empty()) {
+ description = "SRT stream";
+ } else {
+ description = stream_id;
+ }
+ play_as_fast_as_possible = true;
+ play_once = true;
+ last_frame = steady_clock::now();
+}
+#endif
+
FFmpegCapture::~FFmpegCapture()
{
if (has_dequeue_callbacks) {
filename_copy = filename;
}
- string pathname = search_for_file(filename_copy);
+ string pathname;
+ if (srt_sock == -1) {
+ pathname = search_for_file(filename_copy);
+ } else {
+ pathname = description;
+ }
if (pathname.empty()) {
- fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str());
send_disconnected_frame();
+ if (play_once) {
+ break;
+ }
producer_thread_should_quit.sleep_for(seconds(1));
+ fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename_copy.c_str());
continue;
}
should_interrupt = false;
if (!play_video(pathname)) {
// Error.
- fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str());
send_disconnected_frame();
+ if (play_once) {
+ break;
+ }
+ fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str());
producer_thread_should_quit.sleep_for(seconds(1));
continue;
}
+ if (play_once) {
+ send_disconnected_frame();
+ break;
+ }
+
// Probably just EOF, will exit the loop above on next test.
}
FrameAllocator::Frame(), /*audio_offset=*/0, AudioFormat());
last_frame_was_connected = false;
}
+
+ if (play_once) {
+ disconnected = true;
+ if (card_disconnected_callback != nullptr) {
+ card_disconnected_callback();
+ }
+ }
}
bool FFmpegCapture::play_video(const string &pathname)
last_modified = buf.st_mtim;
}
- auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr, AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+ AVFormatContextWithCloser format_ctx;
+ if (srt_sock == -1) {
+ // Regular file.
+ format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
+ /*options=*/nullptr,
+ AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+ } else {
+#ifdef HAVE_SRT
+ // SRT socket, already opened.
+ AVInputFormat *mpegts_fmt = av_find_input_format("mpegts");
+ format_ctx = avformat_open_input_unique(&FFmpegCapture::read_srt_thunk, this,
+ mpegts_fmt, /*options=*/nullptr,
+ AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+#else
+ assert(false);
+#endif
+ }
if (format_ctx == nullptr) {
fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
return false;
}
int audio_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_AUDIO);
+ int subtitle_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_SUBTITLE);
+ has_last_subtitle = false;
// Open video decoder.
const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
if (process_queued_commands(format_ctx.get(), pathname, last_modified, /*rewound=*/nullptr)) {
return true;
}
+ if (should_interrupt.load()) {
+ // Check as a failsafe, so that we don't need to rely on avio if we don't have to.
+ return false;
+ }
UniqueFrame audio_frame = audio_frame_allocator->alloc_frame();
AudioFormat audio_format;
int64_t audio_pts;
bool error;
AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(),
- pathname, video_stream_index, audio_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
+ pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
if (error) {
return false;
}
if (frame == nullptr) {
// EOF. Loop back to the start if we can.
+ if (format_ctx->pb != nullptr && format_ctx->pb->seekable == 0) {
+ // Not seekable (but seemingly, sometimes av_seek_frame() would return 0 anyway,
+ // so don't try).
+ return true;
+ }
if (av_seek_frame(format_ctx.get(), /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) {
fprintf(stderr, "%s: Rewind failed, not looping.\n", pathname.c_str());
return true;
if (last_pts == 0 && pts_origin == 0) {
pts_origin = frame->pts;
}
- next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
- if (first_frame && last_frame_was_connected) {
- // If reconnect took more than one second, this is probably a live feed,
- // and we should reset the resampler. (Or the rate is really, really low,
- // in which case a reset on the first frame is fine anyway.)
- if (duration<double>(next_frame_start - last_frame).count() >= 1.0) {
- last_frame_was_connected = false;
+ steady_clock::time_point now = steady_clock::now();
+ if (play_as_fast_as_possible) {
+ video_frame->received_timestamp = now;
+ audio_frame->received_timestamp = now;
+ next_frame_start = now;
+ } else {
+ next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
+ if (first_frame && last_frame_was_connected) {
+ // If reconnect took more than one second, this is probably a live feed,
+ // and we should reset the resampler. (Or the rate is really, really low,
+ // in which case a reset on the first frame is fine anyway.)
+ if (duration<double>(next_frame_start - last_frame).count() >= 1.0) {
+ last_frame_was_connected = false;
+ }
+ }
+ video_frame->received_timestamp = next_frame_start;
+
+ // The easiest way to get all the rate conversions etc. right is to move the
+ // audio PTS into the video PTS timebase and go from there. (We'll get some
+ // rounding issues, but they should not be a big problem.)
+ int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase);
+ audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate);
+
+ if (audio_frame->len != 0) {
+ // The received timestamps in Nageru are measured after we've just received the frame.
+ // However, pts (especially audio pts) is at the _beginning_ of the frame.
+ // If we have locked audio, the distinction doesn't really matter, as pts is
+ // on a relative scale and a fixed offset is fine. But if we don't, we will have
+ // a different number of samples each time, which will cause huge audio jitter
+ // and throw off the resampler.
+ //
+ // In a sense, we should have compensated by adding the frame and audio lengths
+ // to video_frame->received_timestamp and audio_frame->received_timestamp respectively,
+ // but that would mean extra waiting in sleep_until(). All we need is that they
+ // are correct relative to each other, though (and to the other frames we send),
+ // so just align the end of the audio frame, and we're fine.
+ size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels;
+ double offset = double(num_samples) / OUTPUT_FREQUENCY -
+ double(video_format.frame_rate_den) / video_format.frame_rate_nom;
+ audio_frame->received_timestamp += duration_cast<steady_clock::duration>(duration<double>(offset));
}
- }
- video_frame->received_timestamp = next_frame_start;
-
- // The easiest way to get all the rate conversions etc. right is to move the
- // audio PTS into the video PTS timebase and go from there. (We'll get some
- // rounding issues, but they should not be a big problem.)
- int64_t audio_pts_as_video_pts = av_rescale_q(audio_pts, audio_timebase, video_timebase);
- audio_frame->received_timestamp = compute_frame_start(audio_pts_as_video_pts, pts_origin, video_timebase, start, rate);
-
- if (audio_frame->len != 0) {
- // The received timestamps in Nageru are measured after we've just received the frame.
- // However, pts (especially audio pts) is at the _beginning_ of the frame.
- // If we have locked audio, the distinction doesn't really matter, as pts is
- // on a relative scale and a fixed offset is fine. But if we don't, we will have
- // a different number of samples each time, which will cause huge audio jitter
- // and throw off the resampler.
- //
- // In a sense, we should have compensated by adding the frame and audio lengths
- // to video_frame->received_timestamp and audio_frame->received_timestamp respectively,
- // but that would mean extra waiting in sleep_until(). All we need is that they
- // are correct relative to each other, though (and to the other frames we send),
- // so just align the end of the audio frame, and we're fine.
- size_t num_samples = (audio_frame->len * 8) / audio_format.bits_per_sample / audio_format.num_channels;
- double offset = double(num_samples) / OUTPUT_FREQUENCY -
- double(video_format.frame_rate_den) / video_format.frame_rate_nom;
- audio_frame->received_timestamp += duration_cast<steady_clock::duration>(duration<double>(offset));
- }
- steady_clock::time_point now = steady_clock::now();
- if (duration<double>(now - next_frame_start).count() >= 0.1) {
- // If we don't have enough CPU to keep up, or if we have a live stream
- // where the initial origin was somehow wrong, we could be behind indefinitely.
- // In particular, this will give the audio resampler problems as it tries
- // to speed up to reduce the delay, hitting the low end of the buffer every time.
- fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
- pathname.c_str(),
- 1e3 * duration<double>(now - next_frame_start).count());
- pts_origin = frame->pts;
- start = next_frame_start = now;
- timecode += MAX_FPS * 2 + 1;
+ if (duration<double>(now - next_frame_start).count() >= 0.1) {
+ // If we don't have enough CPU to keep up, or if we have a live stream
+ // where the initial origin was somehow wrong, we could be behind indefinitely.
+ // In particular, this will give the audio resampler problems as it tries
+ // to speed up to reduce the delay, hitting the low end of the buffer every time.
+ fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
+ pathname.c_str(),
+ 1e3 * duration<double>(now - next_frame_start).count());
+ pts_origin = frame->pts;
+ start = next_frame_start = now;
+ timecode += MAX_FPS * 2 + 1;
+ }
+ }
+ bool finished_wakeup;
+ if (play_as_fast_as_possible) {
+ finished_wakeup = !producer_thread_should_quit.should_quit();
+ } else {
+ finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
}
- bool finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
if (finished_wakeup) {
if (audio_frame->len > 0) {
assert(audio_pts != -1);
// audio discontinuity.)
timecode += MAX_FPS * 2 + 1;
}
+ last_neutral_color = get_neutral_color(frame->metadata);
frame_callback(frame->pts, video_timebase, audio_pts, audio_timebase, timecode++,
video_frame.get_and_release(), 0, video_format,
audio_frame.get_and_release(), 0, audio_format);
start = compute_frame_start(last_pts, pts_origin, video_timebase, start, rate);
pts_origin = last_pts;
rate = cmd.new_rate;
+ play_as_fast_as_possible = (rate >= 10.0);
break;
}
}
} // namespace
AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx,
- const std::string &pathname, int video_stream_index, int audio_stream_index,
+ const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index,
FrameAllocator::Frame *audio_frame, AudioFormat *audio_format, int64_t *audio_pts, bool *error)
{
*error = false;
*error = true;
return AVFrameWithDeleter(nullptr);
}
+ } else if (pkt.stream_index == subtitle_stream_index) {
+ last_subtitle = string(reinterpret_cast<const char *>(pkt.data), pkt.size);
+ has_last_subtitle = true;
}
} else {
eof = true; // Or error, but ignore that for the time being.
if (resampler == nullptr) {
fprintf(stderr, "Allocating resampler failed.\n");
- exit(1);
+ abort();
}
if (swr_init(resampler) < 0) {
fprintf(stderr, "Could not open resample context.\n");
- exit(1);
+ abort();
}
last_src_format = AVSampleFormat(audio_avframe->format);
const_cast<const uint8_t **>(audio_avframe->data), audio_avframe->nb_samples);
if (out_samples < 0) {
fprintf(stderr, "Audio conversion failed.\n");
- exit(1);
+ abort();
}
audio_frame->len += out_samples * bytes_per_sample;
return video_frame;
}
-int FFmpegCapture::interrupt_cb_thunk(void *unique)
+int FFmpegCapture::interrupt_cb_thunk(void *opaque)
{
- return reinterpret_cast<FFmpegCapture *>(unique)->interrupt_cb();
+ return reinterpret_cast<FFmpegCapture *>(opaque)->interrupt_cb();
}
int FFmpegCapture::interrupt_cb()
{
return should_interrupt.load();
}
+
+#ifdef HAVE_SRT
+int FFmpegCapture::read_srt_thunk(void *opaque, uint8_t *buf, int buf_size)
+{
+ return reinterpret_cast<FFmpegCapture *>(opaque)->read_srt(buf, buf_size);
+}
+
+int FFmpegCapture::read_srt(uint8_t *buf, int buf_size)
+{
+ SRT_MSGCTRL mc = srt_msgctrl_default;
+ return srt_recvmsg2(srt_sock, reinterpret_cast<char *>(buf), buf_size, &mc);
+}
+#endif
projects / nageru / blobdiff