1 #include "ffmpeg_capture.h"
12 #include <libavcodec/avcodec.h>
13 #include <libavformat/avformat.h>
14 #include <libavutil/avutil.h>
15 #include <libavutil/error.h>
16 #include <libavutil/frame.h>
17 #include <libavutil/imgutils.h>
18 #include <libavutil/mem.h>
19 #include <libavutil/pixfmt.h>
20 #include <libswscale/swscale.h>
28 #include "bmusb/bmusb.h"
29 #include "ffmpeg_raii.h"
31 #include "image_input.h"
33 #define FRAME_SIZE (8 << 20) // 8 MB.
36 using namespace std::chrono;
37 using namespace bmusb;
41 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
43 const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
44 return origin + duration_cast<steady_clock::duration>(pts / rate);
49 FFmpegCapture::FFmpegCapture(const string &filename, unsigned width, unsigned height)
50 : filename(filename), width(width), height(height)
52 // Not really used for anything.
53 description = "Video: " + filename;
56 FFmpegCapture::~FFmpegCapture()
58 if (has_dequeue_callbacks) {
59 dequeue_cleanup_callback();
63 void FFmpegCapture::configure_card()
65 if (video_frame_allocator == nullptr) {
66 owned_video_frame_allocator.reset(new MallocFrameAllocator(FRAME_SIZE, NUM_QUEUED_VIDEO_FRAMES));
67 set_video_frame_allocator(owned_video_frame_allocator.get());
69 if (audio_frame_allocator == nullptr) {
70 owned_audio_frame_allocator.reset(new MallocFrameAllocator(65536, NUM_QUEUED_AUDIO_FRAMES));
71 set_audio_frame_allocator(owned_audio_frame_allocator.get());
75 void FFmpegCapture::start_bm_capture()
81 producer_thread_should_quit = false;
82 producer_thread = thread(&FFmpegCapture::producer_thread_func, this);
85 void FFmpegCapture::stop_dequeue_thread()
91 producer_thread_should_quit = true;
92 producer_thread.join();
95 std::map<uint32_t, VideoMode> FFmpegCapture::get_available_video_modes() const
97 // Note: This will never really be shown in the UI.
101 snprintf(buf, sizeof(buf), "%ux%u", width, height);
104 mode.autodetect = false;
106 mode.height = height;
107 mode.frame_rate_num = 60;
108 mode.frame_rate_den = 1;
109 mode.interlaced = false;
111 return {{ 0, mode }};
114 void FFmpegCapture::producer_thread_func()
116 char thread_name[16];
117 snprintf(thread_name, sizeof(thread_name), "FFmpeg_C_%d", card_index);
118 pthread_setname_np(pthread_self(), thread_name);
120 while (!producer_thread_should_quit) {
121 string pathname = search_for_file(filename);
122 if (filename.empty()) {
123 fprintf(stderr, "%s not found, sleeping one second and trying again...\n", filename.c_str());
127 if (!play_video(pathname)) {
129 fprintf(stderr, "Error when playing %s, sleeping one second and trying again...\n", pathname.c_str());
134 // Probably just EOF, will exit the loop above on next test.
137 if (has_dequeue_callbacks) {
138 dequeue_cleanup_callback();
139 has_dequeue_callbacks = false;
143 bool FFmpegCapture::play_video(const string &pathname)
145 auto format_ctx = avformat_open_input_unique(pathname.c_str(), nullptr, nullptr);
146 if (format_ctx == nullptr) {
147 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
151 if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
152 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
156 int video_stream_index = -1, audio_stream_index = -1;
157 AVRational video_timebase{ 1, 1 };
158 for (unsigned i = 0; i < format_ctx->nb_streams; ++i) {
159 if (format_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
160 video_stream_index == -1) {
161 video_stream_index = i;
162 video_timebase = format_ctx->streams[i]->time_base;
164 if (format_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO &&
165 audio_stream_index == -1) {
166 audio_stream_index = i;
169 if (video_stream_index == -1) {
170 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
174 const AVCodecParameters *codecpar = format_ctx->streams[video_stream_index]->codecpar;
175 AVCodecContextWithDeleter codec_ctx = avcodec_alloc_context3_unique(nullptr);
176 if (avcodec_parameters_to_context(codec_ctx.get(), codecpar) < 0) {
177 fprintf(stderr, "%s: Cannot fill codec parameters\n", pathname.c_str());
180 AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
181 if (codec == nullptr) {
182 fprintf(stderr, "%s: Cannot find decoder\n", pathname.c_str());
185 if (avcodec_open2(codec_ctx.get(), codec, nullptr) < 0) {
186 fprintf(stderr, "%s: Cannot open decoder\n", pathname.c_str());
189 unique_ptr<AVCodecContext, decltype(avcodec_close)*> codec_ctx_cleanup(
190 codec_ctx.get(), avcodec_close);
192 int64_t pts_origin = 0, last_pts = 0;
193 steady_clock::time_point start = steady_clock::now();
194 steady_clock::time_point next_frame_start = start;
197 unique_ptr<SwsContext, decltype(sws_freeContext)*> sws_ctx(nullptr, sws_freeContext);
198 int sws_last_width = -1, sws_last_height = -1;
201 while (!producer_thread_should_quit) {
202 // Process any queued commands from other threads.
203 vector<QueuedCommand> commands;
205 lock_guard<mutex> lock(queue_mu);
206 swap(commands, command_queue);
208 for (const QueuedCommand &cmd : commands) {
209 switch (cmd.command) {
210 case QueuedCommand::REWIND:
211 if (av_seek_frame(format_ctx.get(), /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) {
212 fprintf(stderr, "%s: Rewind failed, stopping play.\n", pathname.c_str());
214 pts_origin = last_pts = 0;
215 start = next_frame_start = steady_clock::now();
218 case QueuedCommand::CHANGE_RATE:
219 start = next_frame_start;
220 pts_origin = last_pts;
226 // Read packets until we have a frame or there are none left.
227 int frame_finished = 0;
228 AVFrameWithDeleter frame = av_frame_alloc_unique();
232 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
233 &pkt, av_packet_unref);
234 av_init_packet(&pkt);
237 if (av_read_frame(format_ctx.get(), &pkt) == 0) {
238 if (pkt.stream_index != video_stream_index) {
239 // Ignore audio for now.
242 if (avcodec_send_packet(codec_ctx.get(), &pkt) < 0) {
243 fprintf(stderr, "%s: Cannot send packet to codec.\n", pathname.c_str());
247 eof = true; // Or error, but ignore that for the time being.
250 int err = avcodec_receive_frame(codec_ctx.get(), frame.get());
252 frame_finished = true;
254 } else if (err != AVERROR(EAGAIN)) {
255 fprintf(stderr, "%s: Cannot receive frame from codec.\n", pathname.c_str());
260 if (!frame_finished) {
261 // EOF. Loop back to the start if we can.
262 if (av_seek_frame(format_ctx.get(), /*stream_index=*/-1, /*timestamp=*/0, /*flags=*/0) < 0) {
263 fprintf(stderr, "%s: Rewind failed, not looping.\n", pathname.c_str());
266 pts_origin = last_pts = 0;
267 start = steady_clock::now();
271 if (sws_ctx == nullptr || sws_last_width != frame->width || sws_last_height != frame->height) {
273 sws_getContext(frame->width, frame->height, (AVPixelFormat)frame->format,
274 width, height, AV_PIX_FMT_RGBA,
275 SWS_BICUBIC, nullptr, nullptr, nullptr));
276 sws_last_width = frame->width;
277 sws_last_height = frame->height;
279 if (sws_ctx == nullptr) {
280 fprintf(stderr, "%s: Could not create scaler context\n", pathname.c_str());
284 VideoFormat video_format;
285 video_format.width = width;
286 video_format.height = height;
287 video_format.stride = width * 4;
288 video_format.frame_rate_nom = video_timebase.den;
289 video_format.frame_rate_den = av_frame_get_pkt_duration(frame.get()) * video_timebase.num;
290 video_format.has_signal = true;
291 video_format.is_connected = true;
293 next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
294 last_pts = frame->pts;
296 FrameAllocator::Frame video_frame = video_frame_allocator->alloc_frame();
297 if (video_frame.data != nullptr) {
298 uint8_t *pic_data[4] = { video_frame.data, nullptr, nullptr, nullptr };
299 int linesizes[4] = { int(video_format.stride), 0, 0, 0 };
300 sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
301 video_frame.len = video_format.stride * height;
302 video_frame.received_timestamp = next_frame_start;
305 FrameAllocator::Frame audio_frame;
306 AudioFormat audio_format;
307 audio_format.bits_per_sample = 32;
308 audio_format.num_channels = 8;
310 // TODO: Make it interruptable somehow.
311 this_thread::sleep_until(next_frame_start);
312 frame_callback(timecode++,
313 video_frame, 0, video_format,
314 audio_frame, 0, audio_format);