1 #define GL_GLEXT_PROTOTYPES
3 #include "video_widget.h"
15 #include <libavcodec/avcodec.h>
16 #include <libavformat/avformat.h>
17 #include <libavutil/avutil.h>
18 #include <libavutil/error.h>
19 #include <libavutil/frame.h>
20 #include <libavutil/imgutils.h>
21 #include <libavutil/mem.h>
22 #include <libavutil/pixfmt.h>
23 #include <libavutil/opt.h>
24 #include <libswscale/swscale.h>
31 #include <unordered_set>
33 #include <QOpenGLFunctions>
36 using namespace std::chrono;
40 bool is_full_range(const AVPixFmtDescriptor *desc)
42 // This is horrible, but there's no better way that I know of.
43 return (strchr(desc->name, 'j') != nullptr);
46 AVPixelFormat decide_dst_format(AVPixelFormat src_format)
48 // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
49 // and be done with it. It's too strange to spend a lot of time on.
50 // (Let's hope there's no alpha.)
51 const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
52 if (src_desc == nullptr ||
53 src_desc->nb_components != 3 ||
54 (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
55 return AV_PIX_FMT_YUV444P;
58 // The best for us would be Cb and Cr together if possible,
59 // but FFmpeg doesn't support that except in the special case of
60 // NV12, so we need to go to planar even for the case of NV12.
61 // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
62 // that matches in color range. (This will also include the case of
63 // the source format already being acceptable.)
64 bool src_full_range = is_full_range(src_desc);
65 const char *best_format = "yuv444p";
66 unsigned best_score = numeric_limits<unsigned>::max();
67 for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
69 desc = av_pix_fmt_desc_next(desc)) {
70 // Find planar Y'CbCr formats only.
71 if (desc->nb_components != 3) continue;
72 if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
73 if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
74 if (desc->comp[0].plane != 0 ||
75 desc->comp[1].plane != 1 ||
76 desc->comp[2].plane != 2) continue;
78 // 8-bit formats only.
79 if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
80 if (desc->comp[0].depth != 8) continue;
82 // Same or better chroma resolution only.
83 int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
84 int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
85 if (chroma_w_diff < 0 || chroma_h_diff < 0)
88 // Matching full/limited range only.
89 if (is_full_range(desc) != src_full_range)
92 // Pick something with as little excess chroma resolution as possible.
93 unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
94 if (score < best_score) {
96 best_format = desc->name;
99 return av_get_pix_fmt(best_format);
104 bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
106 // Process any queued commands from other threads.
107 vector<QueuedCommand> commands;
109 lock_guard<mutex> lock(queue_mu);
110 swap(commands, command_queue);
113 for (const QueuedCommand &cmd : commands) {
114 switch (cmd.command) {
115 case QueuedCommand::PAUSE:
118 case QueuedCommand::RESUME:
120 pts_origin = last_pts;
121 start = next_frame_start = steady_clock::now();
123 case QueuedCommand::SEEK:
124 case QueuedCommand::SEEK_ABSOLUTE:
130 // Combine all seeks into one big one. (There are edge cases where this is probably
131 // subtly wrong, but we'll live with it.)
132 int64_t base_pts = last_pts;
133 int64_t relative_seek_ms = 0;
134 int64_t relative_seek_frames = 0;
135 for (const QueuedCommand &cmd : commands) {
136 if (cmd.command == QueuedCommand::SEEK) {
137 relative_seek_ms += cmd.relative_seek_ms;
138 relative_seek_frames += cmd.relative_seek_frames;
139 } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
140 base_pts = cmd.seek_ms;
141 relative_seek_ms = 0;
142 relative_seek_frames = 0;
145 int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
146 if (relative_seek_ms != 0 && relative_seek_pts == 0) {
147 // Just to be sure rounding errors don't move us into nothingness.
148 relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
150 int64_t goal_pts = base_pts + relative_seek_pts;
151 if (goal_pts != last_pts || relative_seek_frames < 0) {
152 avcodec_flush_buffers(video_codec_ctx);
153 queued_frames.clear();
155 // Seek to the last keyframe before this point.
156 int64_t seek_pts = goal_pts;
157 if (relative_seek_frames < 0) {
158 // If we're frame-skipping backwards, add 100 ms of slop for each frame
159 // so we're fairly certain we are able to see the ones we want.
160 seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
162 av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
164 // Decode frames until EOF, or until we see something past our seek point.
165 std::deque<AVFrameWithDeleter> queue;
168 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
169 pathname, video_stream_index, &error);
170 if (frame == nullptr || error) {
174 int64_t frame_pts = frame->pts;
175 if (relative_seek_frames < 0) {
176 // Buffer this frame; don't display it unless we know it's the Nth-latest.
177 queue.push_back(std::move(frame));
178 if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
182 if (frame_pts >= goal_pts) {
183 if (relative_seek_frames > 0) {
184 --relative_seek_frames;
186 if (relative_seek_frames < 0) {
187 // Hope we have the right amount.
188 // The rest will remain in the queue for when we play forward again.
189 frame = std::move(queue.front());
191 queued_frames = std::move(queue);
193 current_frame.reset(new Frame(make_video_frame(frame.get())));
195 store_pts(frame->pts);
201 // NOTE: We keep pause status as-is.
203 pts_origin = last_pts;
204 start = next_frame_start = last_frame = steady_clock::now();
208 } else if (relative_seek_frames > 0) {
209 // The base PTS is fine, we only need to skip a few frames forwards.
210 while (relative_seek_frames > 1) {
211 // Eat a frame (ignore errors).
213 decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
214 --relative_seek_frames;
217 // Display the last one.
219 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
220 pathname, video_stream_index, &error);
221 if (frame == nullptr || error) {
224 current_frame.reset(new Frame(make_video_frame(frame.get())));
226 store_pts(frame->pts);
231 VideoWidget::VideoWidget(QWidget *parent)
232 : QOpenGLWidget(parent) {}
234 GLuint compile_shader(const string &shader_src, GLenum type)
236 GLuint obj = glCreateShader(type);
237 const GLchar* source[] = { shader_src.data() };
238 const GLint length[] = { (GLint)shader_src.size() };
239 glShaderSource(obj, 1, source, length);
240 glCompileShader(obj);
242 GLchar info_log[4096];
243 GLsizei log_length = sizeof(info_log) - 1;
244 glGetShaderInfoLog(obj, log_length, &log_length, info_log);
245 info_log[log_length] = 0;
246 if (strlen(info_log) > 0) {
247 fprintf(stderr, "Shader compile log: %s\n", info_log);
251 glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
252 if (status == GL_FALSE) {
253 // Add some line numbers to easier identify compile errors.
254 string src_with_lines = "/* 1 */ ";
256 for (char ch : shader_src) {
257 src_with_lines.push_back(ch);
260 snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
261 src_with_lines += buf;
265 fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
272 void VideoWidget::initializeGL()
275 glDisable(GL_DEPTH_TEST);
276 glDepthMask(GL_FALSE);
277 glCreateTextures(GL_TEXTURE_2D, 3, tex);
279 ycbcr_vertex_shader = compile_shader(R"(
282 layout(location = 0) in vec2 position;
283 layout(location = 1) in vec2 texcoord;
288 // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
290 // 2.000 0.000 0.000 -1.000
291 // 0.000 2.000 0.000 -1.000
292 // 0.000 0.000 -2.000 -1.000
293 // 0.000 0.000 0.000 1.000
294 gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
298 )", GL_VERTEX_SHADER);
299 ycbcr_fragment_shader = compile_shader(R"(
302 layout(location = 0) uniform sampler2D tex_y;
303 layout(location = 1) uniform sampler2D tex_cb;
304 layout(location = 2) uniform sampler2D tex_cr;
305 layout(location = 3) uniform vec2 cbcr_offset;
310 // Computed statically by Movit, for limited-range BT.709.
311 // (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
312 const mat3 inv_ycbcr_matrix = mat3(
313 1.16438f, 1.16438f, 1.16438f,
314 0.0f, -0.21325f, 2.11240f,
315 1.79274f, -0.53291f, 0.0f
320 if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
321 FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
326 ycbcr.r = texture(tex_y, tc).r;
327 ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
328 ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
329 ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
330 FragColor.rgb = inv_ycbcr_matrix * ycbcr;
333 )", GL_FRAGMENT_SHADER);
334 ycbcr_program = glCreateProgram();
335 glAttachShader(ycbcr_program, ycbcr_vertex_shader);
336 glAttachShader(ycbcr_program, ycbcr_fragment_shader);
337 glLinkProgram(ycbcr_program);
340 glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
341 if (success == GL_FALSE) {
342 GLchar error_log[1024] = {0};
343 glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
344 fprintf(stderr, "Error linking program: %s\n", error_log);
348 glCreateSamplers(1, &bilinear_sampler);
349 glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
350 glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
351 glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
352 glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
355 void VideoWidget::resizeGL(int w, int h)
357 glViewport(0, 0, w, h);
358 display_aspect = double(w) / h;
361 int num_levels(GLuint width, GLuint height)
364 while (width > 1 || height > 1) {
365 width = max(width / 2, 1u);
366 height = max(height / 2, 1u);
372 void VideoWidget::paintGL()
374 std::shared_ptr<Frame> frame = current_frame;
375 if (frame == nullptr) {
376 glClear(GL_COLOR_BUFFER_BIT);
380 glUseProgram(ycbcr_program);
381 if (frame->width != last_width || frame->height != last_height) {
382 glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
384 if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
385 for (GLuint num : { tex[1], tex[2] }) {
386 glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
390 glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get());
391 glGenerateTextureMipmap(tex[0]);
393 glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height);
394 glGenerateTextureMipmap(tex[1]);
396 glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height + frame->chroma_width * frame->chroma_height);
397 glGenerateTextureMipmap(tex[2]);
399 glBindTextureUnit(0, tex[0]);
400 glBindTextureUnit(1, tex[1]);
401 glBindTextureUnit(2, tex[2]);
402 glBindSampler(0, bilinear_sampler);
403 glBindSampler(1, bilinear_sampler);
404 glBindSampler(2, bilinear_sampler);
405 glProgramUniform1i(ycbcr_program, 0, 0);
406 glProgramUniform1i(ycbcr_program, 1, 1);
407 glProgramUniform1i(ycbcr_program, 2, 2);
408 glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
415 double video_aspect = double(frame->width) / frame->height;
416 if (display_aspect > video_aspect) {
417 double extra_width = frame->height * display_aspect - frame->width;
418 tx1 = -0.5 * extra_width / frame->width;
419 tx2 = 1.0 + 0.5 * extra_width / frame->width;
420 } else if (display_aspect < video_aspect) {
421 double extra_height = frame->width / display_aspect - frame->height;
422 ty1 = -0.5 * extra_height / frame->height;
423 ty2 = 1.0 + 0.5 * extra_height / frame->height;
428 glVertexAttrib2f(1, tx1, ty1);
429 glVertex2f(0.0f, 0.0f);
431 glVertexAttrib2f(1, tx1, ty2);
432 glVertex2f(0.0f, 1.0f);
434 glVertexAttrib2f(1, tx2, ty2);
435 glVertex2f(1.0f, 1.0f);
437 glVertexAttrib2f(1, tx2, ty1);
438 glVertex2f(1.0f, 0.0f);
443 void VideoWidget::open(const string &filename)
451 void VideoWidget::play()
454 std::lock_guard<std::mutex> lock(queue_mu);
455 command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
456 producer_thread_should_quit.wakeup();
460 producer_thread_should_quit.unquit();
461 producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
464 void VideoWidget::pause()
469 std::lock_guard<std::mutex> lock(queue_mu);
470 command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
471 producer_thread_should_quit.wakeup();
474 void VideoWidget::seek(int64_t relative_seek_ms)
479 std::lock_guard<std::mutex> lock(queue_mu);
480 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
481 producer_thread_should_quit.wakeup();
484 void VideoWidget::seek_frames(int64_t relative_seek_frames)
489 std::lock_guard<std::mutex> lock(queue_mu);
490 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
491 producer_thread_should_quit.wakeup();
494 void VideoWidget::seek_absolute(int64_t position_ms)
499 std::lock_guard<std::mutex> lock(queue_mu);
500 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
501 producer_thread_should_quit.wakeup();
504 void VideoWidget::stop()
510 producer_thread_should_quit.quit();
511 producer_thread.join();
514 void VideoWidget::producer_thread_func()
516 if (!producer_thread_should_quit.should_quit()) {
517 if (!play_video(pathname)) {
518 // TODO: Send the error back to the UI somehow.
523 void VideoWidget::internal_rewind()
525 pts_origin = last_pts = 0;
527 start = next_frame_start = steady_clock::now();
530 template<AVHWDeviceType type>
531 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
533 bool found_config_of_right_type = false;
534 for (int i = 0;; ++i) { // Termination condition inside loop.
535 const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
536 if (config == nullptr) { // End of list.
539 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
540 config->device_type != type) {
541 // Not interesting for us.
545 // We have a config of the right type, but does it actually support
546 // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
547 // is to just replace the pixel format with a software-decoded one,
549 found_config_of_right_type = true;
550 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
551 if (config->pix_fmt == *fmt_ptr) {
552 fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
553 av_hwdevice_get_type_name(type), ctx->codec->name);
554 if (ctx->profile == FF_PROFILE_H264_BASELINE) {
555 fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
556 fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n");
557 fprintf(stderr, " Decoding might fail and fall back to software.\n");
559 return config->pix_fmt;
562 fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
563 unordered_set<AVPixelFormat> seen;
564 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
565 if (!seen.count(*fmt_ptr)) {
566 fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
567 seen.insert(*fmt_ptr);
570 fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
574 if (!found_config_of_right_type) {
575 fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
578 // We found no VA-API formats, so take the first software format.
579 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
580 if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
581 fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
586 // Fallback: Just return anything. (Should never really happen.)
590 AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
591 const std::string &pathname, int video_stream_index,
596 if (!queued_frames.empty()) {
597 AVFrameWithDeleter frame = std::move(queued_frames.front());
598 queued_frames.pop_front();
602 // Read packets until we have a frame or there are none left.
603 bool frame_finished = false;
604 AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
608 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
609 &pkt, av_packet_unref);
610 av_init_packet(&pkt);
613 if (av_read_frame(format_ctx, &pkt) == 0) {
614 if (pkt.stream_index == video_stream_index) {
615 if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) {
616 fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
618 return AVFrameWithDeleter(nullptr);
622 eof = true; // Or error, but ignore that for the time being.
625 // Decode video, if we have a frame.
626 int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
628 frame_finished = true;
630 } else if (err != AVERROR(EAGAIN)) {
631 fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
633 return AVFrameWithDeleter(nullptr);
638 return video_avframe;
640 return AVFrameWithDeleter(nullptr);
643 int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
645 for (unsigned i = 0; i < ctx->nb_streams; ++i) {
646 if (ctx->streams[i]->codecpar->codec_type == media_type) {
653 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
655 const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
656 return origin + duration_cast<steady_clock::duration>(pts / rate);
659 bool VideoWidget::play_video(const string &pathname)
661 queued_frames.clear();
662 AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
663 /*options=*/nullptr);
664 if (format_ctx == nullptr) {
665 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
669 if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
670 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
674 int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
675 if (video_stream_index == -1) {
676 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
680 // Open video decoder.
681 const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
682 const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
684 video_timebase = format_ctx->streams[video_stream_index]->time_base;
685 AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
686 if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
687 fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
690 if (video_codec == nullptr) {
691 fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
695 // Seemingly, it's not too easy to make something that just initializes
696 // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
697 // without enumerating through several different types.
698 // VA-API and VDPAU will do for now. We prioritize VDPAU for the
699 // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
700 // cards that seems to work, but just hangs when trying to transfer the frame.
702 // Note that we don't actually check codec support beforehand,
703 // so if you have a low-end VDPAU device but a high-end VA-API device,
704 // you lose out on the extra codec support from the latter.
705 AVBufferRef *hw_device_ctx = nullptr;
706 if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
707 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
708 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
709 } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
710 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
711 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
713 fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
716 if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
717 fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
720 unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
721 video_codec_ctx.get(), avcodec_close);
726 int consecutive_errors = 0;
728 while (!producer_thread_should_quit.should_quit()) {
729 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
733 producer_thread_should_quit.sleep_for(hours(1));
738 AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
739 pathname, video_stream_index, &error);
741 if (++consecutive_errors >= 100) {
742 fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
748 consecutive_errors = 0;
750 if (frame == nullptr) {
755 // Sleep until it's time to present this frame.
757 if (last_pts == 0 && pts_origin == 0) {
758 pts_origin = frame->pts;
760 steady_clock::time_point now = steady_clock::now();
761 next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
763 if (duration<double>(now - next_frame_start).count() >= 0.1) {
764 // If we don't have enough CPU to keep up, or if we have a live stream
765 // where the initial origin was somehow wrong, we could be behind indefinitely.
766 fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
768 1e3 * duration<double>(now - next_frame_start).count());
769 pts_origin = frame->pts;
770 start = next_frame_start = now;
772 bool finished_wakeup;
773 finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
774 if (finished_wakeup) {
775 current_frame.reset(new Frame(make_video_frame(frame.get())));
776 last_frame = steady_clock::now();
780 if (producer_thread_should_quit.should_quit()) break;
783 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
788 // Just paused, so present the frame immediately and then go into deep sleep.
789 current_frame.reset(new Frame(make_video_frame(frame.get())));
790 last_frame = steady_clock::now();
795 // If we just seeked, drop this frame on the floor and be done.
801 store_pts(frame->pts);
806 void VideoWidget::store_pts(int64_t pts)
809 last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
810 emit position_changed(last_position);
813 // Taken from Movit (see the comment there for explanation)
814 float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
816 float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
817 if (fabs(local_chroma_pos - 0.5) < 1e-10) {
818 // x + (-0) can be optimized away freely, as opposed to x + 0.
821 return (0.5 - local_chroma_pos) / resolution;
825 VideoWidget::Frame VideoWidget::make_video_frame(const AVFrame *frame)
828 AVFrameWithDeleter sw_frame;
830 if (frame->format == AV_PIX_FMT_VAAPI ||
831 frame->format == AV_PIX_FMT_VDPAU) {
832 // Get the frame down to the CPU. (TODO: See if we can keep it
833 // on the GPU all the way, since it will be going up again later.
834 // However, this only works if the OpenGL GPU is the same one.)
835 sw_frame = av_frame_alloc_unique();
836 int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
838 fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
840 sw_frame->pts = frame->pts;
841 sw_frame->pkt_duration = frame->pkt_duration;
842 frame = sw_frame.get();
846 if (sws_ctx == nullptr ||
847 sws_last_width != frame->width ||
848 sws_last_height != frame->height ||
849 sws_last_src_format != frame->format) {
850 sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
852 sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
853 frame->width, frame->height, sws_dst_format,
854 SWS_BICUBIC, nullptr, nullptr, nullptr));
855 sws_last_width = frame->width;
856 sws_last_height = frame->height;
857 sws_last_src_format = frame->format;
859 if (sws_ctx == nullptr) {
860 fprintf(stderr, "Could not create scaler context\n");
864 uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
865 int linesizes[4] = { 0, 0, 0, 0 };
866 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
868 video_frame.width = frame->width;
869 video_frame.height = frame->height;
870 video_frame.chroma_width = AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w);
871 video_frame.chroma_height = AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h);
873 // We always assume left chroma placement for now.
874 cbcr_offset[0] = compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame.chroma_width);
875 cbcr_offset[1] = compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame.chroma_height);
877 size_t len = frame->width * frame->height + 2 * video_frame.chroma_width * video_frame.chroma_height;
878 video_frame.data.reset(new uint8_t[len]);
880 pic_data[0] = video_frame.data.get();
881 linesizes[0] = frame->width;
883 pic_data[1] = pic_data[0] + frame->width * frame->height;
884 linesizes[1] = video_frame.chroma_width;
886 pic_data[2] = pic_data[1] + video_frame.chroma_width * video_frame.chroma_height;
887 linesizes[2] = video_frame.chroma_width;
889 sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);