X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=nageru%2Fffmpeg_capture.cpp;h=6dbcbd602b79b544c8319ddd9446b9992c66ef10;hb=f81ae3be1aae619fe4ad022f55d95a4a83ace076;hp=5cf1d8c9ec75f35a6932a6bd7a282914c3017d92;hpb=cb1dcf409cde897b011e9d9d3cd3cf2974956190;p=nageru

diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp
index 5cf1d8c..6dbcbd6 100644
--- a/nageru/ffmpeg_capture.cpp
+++ b/nageru/ffmpeg_capture.cpp
@@ -1,24 +1,47 @@
 #include "ffmpeg_capture.h"
+#include "defs.h"
+#include "shared/shared_defs.h"
 
 #include <assert.h>
+#include <cerrno>
+#include <ctime>
+#include <limits>
+#include <map>
+#include <memory>
+#include <movit/effect.h>
+#include <movit/image_format.h>
+#include <movit/ycbcr.h>
+#include <mutex>
 #include <pthread.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <string>
 #include <sys/stat.h>
-#include <unistd.h>
+#include <thread>
 
 extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libavcodec/codec.h>
+#include <libavcodec/codec_id.h>
+#include <libavcodec/codec_par.h>
 #include <libavformat/avformat.h>
 #include <libavutil/avutil.h>
+#include <libavutil/buffer.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/common.h>
+#include <libavutil/dict.h>
 #include <libavutil/error.h>
 #include <libavutil/frame.h>
-#include <libavutil/imgutils.h>
-#include <libavutil/mem.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/pixdesc.h>
 #include <libavutil/pixfmt.h>
-#include <libavutil/opt.h>
+#include <libavutil/rational.h>
+#include <libavutil/samplefmt.h>
+#include <libavutil/version.h>
+#include <libswresample/swresample.h>
 #include <libswscale/swscale.h>
 }
 
@@ -36,7 +59,6 @@ extern "C" {
 #include "shared/ffmpeg_raii.h"
 #include "ffmpeg_util.h"
 #include "flags.h"
-#include "image_input.h"
 #include "ref_counted_frame.h"
 #include "shared/timebase.h"
 
@@ -44,14 +66,17 @@ extern "C" {
 #include <srt/srt.h>
 #endif
 
-#define FRAME_SIZE (8 << 20)  // 8 MB.
-
 using namespace std;
 using namespace std::chrono;
 using namespace bmusb;
 using namespace movit;
 using namespace Eigen;
 
+// Avoid deprecation warnings, but we don't want to drop FFmpeg 5.1 support just yet.
+#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 30, 100)
+#define pkt_duration duration
+#endif
+
 namespace {
 
 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
@@ -125,8 +150,8 @@ AVPixelFormat decide_dst_format(AVPixelFormat src_format, bmusb::PixelFormat dst
 		if (desc->comp[0].depth != 8) continue;
 
 		// Same or better chroma resolution only.
-		int chroma_w_diff = desc->log2_chroma_w - src_desc->log2_chroma_w;
-		int chroma_h_diff = desc->log2_chroma_h - src_desc->log2_chroma_h;
+		int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
+		int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
 		if (chroma_w_diff < 0 || chroma_h_diff < 0)
 			continue;
 
@@ -533,10 +558,13 @@ bool FFmpegCapture::play_video(const string &pathname)
 
 	AVFormatContextWithCloser format_ctx;
 	if (srt_sock == -1) {
-		// Regular file.
+		// Regular file (or stream).
+		frame_timeout_started = steady_clock::now();
+		frame_timeout_valid = true;
 		format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
 			/*options=*/nullptr,
 			AVIOInterruptCB{ &FFmpegCapture::interrupt_cb_thunk, this });
+		frame_timeout_valid = false;
 	} else {
 #ifdef HAVE_SRT
 		// SRT socket, already opened.
@@ -655,11 +683,18 @@ bool FFmpegCapture::play_video(const string &pathname)
 
 		int64_t audio_pts;
 		bool error;
+		frame_timeout_started = steady_clock::now();
+		frame_timeout_valid = true;
 		AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(),
 			pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
+		frame_timeout_valid = false;
+		if (should_interrupt.load()) {
+			// Abort no matter whether we got a frame or not.
+			return false;
+		}
 		if (error) {
 			if (++consecutive_errors >= 100) {
-				fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
+				fprintf(stderr, "More than 100 consecutive error video frames, aborting playback.\n");
 				return false;
 			} else {
 				continue;
@@ -893,34 +928,31 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo
 	*audio_pts = -1;
 	bool has_audio = false;
 	do {
-		AVPacket pkt;
-		unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
-			&pkt, av_packet_unref);
-		av_init_packet(&pkt);
-		pkt.data = nullptr;
-		pkt.size = 0;
-		if (av_read_frame(format_ctx, &pkt) == 0) {
-			if (pkt.stream_index == audio_stream_index && audio_callback != nullptr) {
-				audio_callback(&pkt, format_ctx->streams[audio_stream_index]->time_base);
+		AVPacketWithDeleter pkt = av_packet_alloc_unique();
+		pkt->data = nullptr;
+		pkt->size = 0;
+		if (av_read_frame(format_ctx, pkt.get()) == 0) {
+			if (pkt->stream_index == audio_stream_index && audio_callback != nullptr) {
+				audio_callback(pkt.get(), format_ctx->streams[audio_stream_index]->time_base);
 			}
-			if (pkt.stream_index == video_stream_index && video_callback != nullptr) {
-				video_callback(&pkt, format_ctx->streams[video_stream_index]->time_base);
+			if (pkt->stream_index == video_stream_index && video_callback != nullptr) {
+				video_callback(pkt.get(), format_ctx->streams[video_stream_index]->time_base);
 			}
-			if (pkt.stream_index == video_stream_index && global_flags.transcode_video) {
-				if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) {
+			if (pkt->stream_index == video_stream_index && global_flags.transcode_video) {
+				if (avcodec_send_packet(video_codec_ctx, pkt.get()) < 0) {
 					fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
 					*error = true;
 					return AVFrameWithDeleter(nullptr);
 				}
-			} else if (pkt.stream_index == audio_stream_index && global_flags.transcode_audio) {
+			} else if (pkt->stream_index == audio_stream_index && global_flags.transcode_audio) {
 				has_audio = true;
-				if (avcodec_send_packet(audio_codec_ctx, &pkt) < 0) {
+				if (avcodec_send_packet(audio_codec_ctx, pkt.get()) < 0) {
 					fprintf(stderr, "%s: Cannot send packet to audio codec.\n", pathname.c_str());
 					*error = true;
 					return AVFrameWithDeleter(nullptr);
 				}
-			} else if (pkt.stream_index == subtitle_stream_index) {
-				last_subtitle = string(reinterpret_cast<const char *>(pkt.data), pkt.size);
+			} else if (pkt->stream_index == subtitle_stream_index) {
+				last_subtitle = string(reinterpret_cast<const char *>(pkt->data), pkt->size);
 				has_last_subtitle = true;
 			}
 		} else {
@@ -1010,28 +1042,36 @@ void FFmpegCapture::convert_audio(const AVFrame *audio_avframe, FrameAllocator::
 	}
 	audio_format->num_channels = 2;
 
-	int64_t channel_layout = audio_avframe->channel_layout;
-	if (channel_layout == 0) {
-		channel_layout = av_get_default_channel_layout(audio_avframe->channels);
+	AVChannelLayout channel_layout = audio_avframe->ch_layout;
+	if (!av_channel_layout_check(&channel_layout) ||
+	    channel_layout.order == AV_CHANNEL_ORDER_UNSPEC) {
+		av_channel_layout_default(&channel_layout, audio_avframe->ch_layout.nb_channels);
 	}
 
 	if (resampler == nullptr ||
 	    audio_avframe->format != last_src_format ||
 	    dst_format != last_dst_format ||
-	    channel_layout != last_channel_layout ||
+	    av_channel_layout_compare(&channel_layout, &last_channel_layout) !=  0||
 	    audio_avframe->sample_rate != last_sample_rate) {
+		// TODO: When we get C++20, use AV_CHANNEL_LAYOUT_STEREO_DOWNMIX.
+		AVChannelLayout stereo_downmix;
+		stereo_downmix.order = AV_CHANNEL_ORDER_NATIVE;
+		stereo_downmix.nb_channels = 2;
+		stereo_downmix.u.mask = AV_CH_LAYOUT_STEREO_DOWNMIX;
+
 		swr_free(&resampler);
-		resampler = swr_alloc_set_opts(nullptr,
-		                               /*out_ch_layout=*/AV_CH_LAYOUT_STEREO_DOWNMIX,
-		                               /*out_sample_fmt=*/dst_format,
-		                               /*out_sample_rate=*/OUTPUT_FREQUENCY,
-		                               /*in_ch_layout=*/channel_layout,
-		                               /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format),
-		                               /*in_sample_rate=*/audio_avframe->sample_rate,
-		                               /*log_offset=*/0,
-		                               /*log_ctx=*/nullptr);
-
-		if (resampler == nullptr) {
+		resampler = nullptr;
+		int err = swr_alloc_set_opts2(&resampler,
+		                              /*out_ch_layout=*/&stereo_downmix,
+		                              /*out_sample_fmt=*/dst_format,
+		                              /*out_sample_rate=*/OUTPUT_FREQUENCY,
+		                              /*in_ch_layout=*/&channel_layout,
+		                              /*in_sample_fmt=*/AVSampleFormat(audio_avframe->format),
+		                              /*in_sample_rate=*/audio_avframe->sample_rate,
+		                              /*log_offset=*/0,
+		                              /*log_ctx=*/nullptr);
+
+		if (err != 0 || resampler == nullptr) {
 			fprintf(stderr, "Allocating resampler failed.\n");
 			abort();
 		}
@@ -1146,6 +1186,16 @@ UniqueFrame FFmpegCapture::make_video_frame(const AVFrame *frame, const string &
 
 		current_frame_ycbcr_format = decode_ycbcr_format(desc, frame, is_mjpeg, &last_colorspace, &last_chroma_location);
 	}
+
+	// FIXME: Currently, if the video is too high-res for one of the allocated
+	// frames, we simply refuse to scale it here to avoid crashes. It would be better
+	// if we could somehow signal getting larger frames, especially as 4K is a thing now.
+	if (video_frame->len > FRAME_SIZE) {
+		fprintf(stderr, "%s: Decoded frame would be larger than supported FRAME_SIZE (%zu > %u), not decoding.\n", pathname.c_str(), video_frame->len, FRAME_SIZE);
+		*error = true;
+		return video_frame;
+	}
+
 	sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
 
 	return video_frame;
@@ -1158,6 +1208,21 @@ int FFmpegCapture::interrupt_cb_thunk(void *opaque)
 
 int FFmpegCapture::interrupt_cb()
 {
+	// If ten seconds is gone without anything happening, we assume that
+	// we are in a network stream that died and FFmpeg just didn't
+	// pick it up (or perhaps it just hung, keeping the connection open).
+	// Called back approximately every 100 ms if something is hanging,
+	// so we get more than enough accuracy for our purposes.
+	if (!should_interrupt && frame_timeout_valid &&
+	    duration<double>(steady_clock::now() - frame_timeout_started).count() >= 10.0) {
+		string filename_copy;
+		{
+			lock_guard<mutex> lock(filename_mu);
+			filename_copy = filename;
+		}
+		fprintf(stderr, "%s: No frame for more than 10 seconds, restarting stream.\n", filename.c_str());
+		should_interrupt = true;
+	}
 	return should_interrupt.load();
 }