1 #include "video_widget.h"
13 #include <libavcodec/avcodec.h>
14 #include <libavformat/avformat.h>
15 #include <libavutil/avutil.h>
16 #include <libavutil/error.h>
17 #include <libavutil/frame.h>
18 #include <libavutil/imgutils.h>
19 #include <libavutil/mem.h>
20 #include <libavutil/pixfmt.h>
21 #include <libavutil/opt.h>
22 #include <libswscale/swscale.h>
29 #include <unordered_set>
31 #include "post_to_main_thread.h"
33 #include <QOpenGLFunctions>
34 #include <QWheelEvent>
35 #include <QMouseEvent>
36 #include <QMouseEvent>
37 #include <QHBoxLayout>
38 #include <QOpenGLFunctions_4_5_Compatibility>
39 #include <QOpenGLVersionFunctionsFactory>
41 #define BUFFER_OFFSET(i) ((char *)nullptr + (i))
44 using namespace std::chrono;
48 bool is_full_range(const AVPixFmtDescriptor *desc)
50 // This is horrible, but there's no better way that I know of.
51 return (strchr(desc->name, 'j') != nullptr);
54 AVPixelFormat decide_dst_format(AVPixelFormat src_format)
56 // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
57 // and be done with it. It's too strange to spend a lot of time on.
58 // (Let's hope there's no alpha.)
59 const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
60 if (src_desc == nullptr ||
61 src_desc->nb_components != 3 ||
62 (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
63 return AV_PIX_FMT_YUV444P;
66 // The best for us would be Cb and Cr together if possible,
67 // but FFmpeg doesn't support that except in the special case of
68 // NV12, so we need to go to planar even for the case of NV12.
69 // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
70 // that matches in color range. (This will also include the case of
71 // the source format already being acceptable.)
72 bool src_full_range = is_full_range(src_desc);
73 const char *best_format = "yuv444p";
74 unsigned best_score = numeric_limits<unsigned>::max();
75 for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
77 desc = av_pix_fmt_desc_next(desc)) {
78 // Find planar Y'CbCr formats only.
79 if (desc->nb_components != 3) continue;
80 if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
81 if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
82 if (desc->comp[0].plane != 0 ||
83 desc->comp[1].plane != 1 ||
84 desc->comp[2].plane != 2) continue;
86 // 8-bit formats only.
87 if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
88 if (desc->comp[0].depth != 8) continue;
90 // Same or better chroma resolution only.
91 int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
92 int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
93 if (chroma_w_diff < 0 || chroma_h_diff < 0)
96 // Matching full/limited range only.
97 if (is_full_range(desc) != src_full_range)
100 // Pick something with as little excess chroma resolution as possible.
101 unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
102 if (score < best_score) {
104 best_format = desc->name;
107 return av_get_pix_fmt(best_format);
112 bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
114 // Process any queued commands from other threads.
115 vector<QueuedCommand> commands;
117 lock_guard<mutex> lock(queue_mu);
118 swap(commands, command_queue);
121 for (const QueuedCommand &cmd : commands) {
122 switch (cmd.command) {
123 case QueuedCommand::PAUSE:
126 case QueuedCommand::RESUME:
128 pts_origin = last_pts;
129 start = next_frame_start = steady_clock::now();
131 case QueuedCommand::SEEK:
132 case QueuedCommand::SEEK_ABSOLUTE:
138 // Combine all seeks into one big one. (There are edge cases where this is probably
139 // subtly wrong, but we'll live with it.)
140 int64_t base_pts = last_pts;
141 int64_t relative_seek_ms = 0;
142 int64_t relative_seek_frames = 0;
143 for (const QueuedCommand &cmd : commands) {
144 if (cmd.command == QueuedCommand::SEEK) {
145 relative_seek_ms += cmd.relative_seek_ms;
146 relative_seek_frames += cmd.relative_seek_frames;
147 } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
148 base_pts = av_rescale_q(cmd.seek_ms, AVRational{ 1, 1000 }, video_timebase);
149 relative_seek_ms = 0;
150 relative_seek_frames = 0;
153 int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
154 if (relative_seek_ms != 0 && relative_seek_pts == 0) {
155 // Just to be sure rounding errors don't move us into nothingness.
156 relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
158 int64_t goal_pts = base_pts + relative_seek_pts;
159 if (goal_pts != last_pts || relative_seek_frames < 0) {
160 avcodec_flush_buffers(video_codec_ctx);
161 queued_frames.clear();
163 // Seek to the last keyframe before this point.
164 int64_t seek_pts = goal_pts;
165 if (relative_seek_frames < 0) {
166 // If we're frame-skipping backwards, add 100 ms of slop for each frame
167 // so we're fairly certain we are able to see the ones we want.
168 seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
170 av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
172 // Decode frames until EOF, or until we see something past our seek point.
173 std::deque<AVFrameWithDeleter> queue;
176 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
177 pathname, video_stream_index, &error);
178 if (frame == nullptr || error) {
182 int64_t frame_pts = frame->pts;
183 if (relative_seek_frames < 0) {
184 // Buffer this frame; don't display it unless we know it's the Nth-latest.
185 queue.push_back(std::move(frame));
186 if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
190 if (frame_pts >= goal_pts) {
191 if (relative_seek_frames > 0) {
192 --relative_seek_frames;
194 if (relative_seek_frames < 0) {
195 // Hope we have the right amount.
196 // The rest will remain in the queue for when we play forward again.
197 frame = std::move(queue.front());
199 queued_frames = std::move(queue);
201 video_window->set_current_frame(make_video_frame(frame.get()));
203 store_pts(frame->pts);
209 // NOTE: We keep pause status as-is.
211 pts_origin = last_pts;
212 start = next_frame_start = last_frame = steady_clock::now();
216 } else if (relative_seek_frames > 0) {
217 // The base PTS is fine, we only need to skip a few frames forwards.
218 while (relative_seek_frames > 1) {
219 // Eat a frame (ignore errors).
221 decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
222 --relative_seek_frames;
225 // Display the last one.
227 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
228 pathname, video_stream_index, &error);
229 if (frame == nullptr || error) {
232 video_window->set_current_frame(make_video_frame(frame.get()));
234 store_pts(frame->pts);
239 VideoWidget::VideoWidget(QWidget *parent)
241 video_window(new VideoWindow(this)) {
242 setLayout(new QHBoxLayout);
243 layout()->setContentsMargins(QMargins());
244 layout()->addWidget(QWidget::createWindowContainer(video_window));
247 connect(video_window, &VideoWindow::mouse_wheel, this, &VideoWidget::wheelEvent);
248 connect(video_window, &VideoWindow::mouse_pressed, this, &VideoWidget::mousePressEvent);
249 connect(video_window, &VideoWindow::mouse_released, this, &VideoWidget::mouseReleaseEvent);
250 connect(video_window, &VideoWindow::mouse_moved, this, &VideoWidget::mouseMoveEvent);
253 VideoWidget::~VideoWidget()
257 // Qt will delete video_window for us after we're gone,
258 // so make sure its destructor does not try to mess with
259 // our freelist. The actual freelist frames will leak.
260 video_window->set_current_frame(nullptr);
263 GLuint compile_shader(QOpenGLFunctions_4_5_Compatibility *gl, const string &shader_src, GLenum type)
265 GLuint obj = gl->glCreateShader(type);
266 const GLchar* source[] = { shader_src.data() };
267 const GLint length[] = { (GLint)shader_src.size() };
268 gl->glShaderSource(obj, 1, source, length);
269 gl->glCompileShader(obj);
271 GLchar info_log[4096];
272 GLsizei log_length = sizeof(info_log) - 1;
273 gl->glGetShaderInfoLog(obj, log_length, &log_length, info_log);
274 info_log[log_length] = 0;
275 if (strlen(info_log) > 0) {
276 fprintf(stderr, "Shader compile log: %s\n", info_log);
280 gl->glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
281 if (status == GL_FALSE) {
282 // Add some line numbers to easier identify compile errors.
283 string src_with_lines = "/* 1 */ ";
285 for (char ch : shader_src) {
286 src_with_lines.push_back(ch);
289 snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
290 src_with_lines += buf;
294 fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
301 void VideoWindow::initializeGL()
303 gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(context());
306 glDisable(GL_DEPTH_TEST);
307 glDepthMask(GL_FALSE);
308 gl->glCreateTextures(GL_TEXTURE_2D, 3, tex);
310 ycbcr_vertex_shader = compile_shader(gl, R"(
313 layout(location = 0) in vec2 position;
314 layout(location = 1) in vec2 texcoord;
319 // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
321 // 2.000 0.000 0.000 -1.000
322 // 0.000 2.000 0.000 -1.000
323 // 0.000 0.000 -2.000 -1.000
324 // 0.000 0.000 0.000 1.000
325 gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
329 )", GL_VERTEX_SHADER);
330 ycbcr_fragment_shader = compile_shader(gl, R"(
333 layout(location = 0) uniform sampler2D tex_y;
334 layout(location = 1) uniform sampler2D tex_cb;
335 layout(location = 2) uniform sampler2D tex_cr;
336 layout(location = 3) uniform vec2 cbcr_offset;
341 // Computed statically by Movit, for limited-range BT.709.
342 // (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
343 const mat3 inv_ycbcr_matrix = mat3(
344 1.16438f, 1.16438f, 1.16438f,
345 0.0f, -0.21325f, 2.11240f,
346 1.79274f, -0.53291f, 0.0f
351 if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
352 FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
357 ycbcr.r = texture(tex_y, tc).r;
358 ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
359 ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
360 ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
361 FragColor.rgb = inv_ycbcr_matrix * ycbcr;
364 )", GL_FRAGMENT_SHADER);
365 ycbcr_program = gl->glCreateProgram();
366 gl->glAttachShader(ycbcr_program, ycbcr_vertex_shader);
367 gl->glAttachShader(ycbcr_program, ycbcr_fragment_shader);
368 gl->glLinkProgram(ycbcr_program);
371 gl->glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
372 if (success == GL_FALSE) {
373 GLchar error_log[1024] = {0};
374 gl->glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
375 fprintf(stderr, "Error linking program: %s\n", error_log);
379 gl->glCreateSamplers(1, &bilinear_sampler);
380 gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
381 gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
382 gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
383 gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
386 void VideoWindow::resizeGL(int w, int h)
388 glViewport(0, 0, w, h);
389 display_aspect = double(w) / h;
392 int num_levels(GLuint width, GLuint height)
395 while (width > 1 || height > 1) {
396 width = max(width / 2, 1u);
397 height = max(height / 2, 1u);
403 void VideoWindow::paintGL()
405 std::shared_ptr<VideoWidget::Frame> frame;
407 lock_guard lock(current_frame_mu);
408 frame = current_frame;
410 if (frame == nullptr) {
411 glClear(GL_COLOR_BUFFER_BIT);
415 gl->glUseProgram(ycbcr_program);
416 if (frame->width != last_width || frame->height != last_height) {
417 gl->glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
419 if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
420 for (GLuint num : { tex[1], tex[2] }) {
421 gl->glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
425 gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, frame->pbo);
427 if (frame->need_flush_len > 0) {
428 gl->glFlushMappedNamedBufferRange(frame->pbo, 0, frame->need_flush_len);
429 frame->need_flush_len = 0;
432 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
433 gl->glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(0));
434 gl->glGenerateTextureMipmap(tex[0]);
436 gl->glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(frame->width * frame->height));
437 gl->glGenerateTextureMipmap(tex[1]);
439 gl->glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(frame->width * frame->height + frame->chroma_width * frame->chroma_height));
440 gl->glGenerateTextureMipmap(tex[2]);
442 gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
444 gl->glBindTextureUnit(0, tex[0]);
445 gl->glBindTextureUnit(1, tex[1]);
446 gl->glBindTextureUnit(2, tex[2]);
447 gl->glBindSampler(0, bilinear_sampler);
448 gl->glBindSampler(1, bilinear_sampler);
449 gl->glBindSampler(2, bilinear_sampler);
450 gl->glProgramUniform1i(ycbcr_program, 0, 0);
451 gl->glProgramUniform1i(ycbcr_program, 1, 1);
452 gl->glProgramUniform1i(ycbcr_program, 2, 2);
453 gl->glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
460 double video_aspect = double(frame->width) / frame->height;
461 if (display_aspect > video_aspect) {
462 double extra_width = frame->height * display_aspect - frame->width;
463 tx1 = -0.5 * extra_width / frame->width;
464 tx2 = 1.0 + 0.5 * extra_width / frame->width;
465 } else if (display_aspect < video_aspect) {
466 double extra_height = frame->width / display_aspect - frame->height;
467 ty1 = -0.5 * extra_height / frame->height;
468 ty2 = 1.0 + 0.5 * extra_height / frame->height;
474 gl->glVertexAttrib2f(1, tx1, ty1);
475 glVertex2f(zoom_matrix[2 * 3 + 0], zoom_matrix[2 * 3 + 1]);
478 gl->glVertexAttrib2f(1, tx1, ty2);
479 glVertex2f(zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0], zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
482 gl->glVertexAttrib2f(1, tx2, ty2);
483 glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0],
484 zoom_matrix[1 * 3 + 0] + zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
487 gl->glVertexAttrib2f(1, tx2, ty1);
488 glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[2 * 3 + 0],
489 zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 1]);
494 void VideoWindow::set_current_frame(shared_ptr<VideoWidget::Frame> new_frame)
497 lock_guard lock(current_frame_mu);
498 current_frame = std::move(new_frame);
503 void matmul3x3(const double a[9], const double b[9], double res[9])
505 for (int i = 0; i < 3; ++i) {
506 for (int j = 0; j < 3; ++j) {
508 for (int k = 0; k < 3; ++k) {
509 sum += a[i * 3 + k] * b[k * 3 + j];
511 res[i * 3 + j] = sum;
516 void VideoWidget::wheelEvent(QWheelEvent *event)
518 int delta = event->angleDelta().y();
522 double x = event->position().x() / width();
523 double y = 1.0 - event->position().y() / height();
524 double zoom = delta > 0 ? pow(1.005, delta) : pow(1/1.005, -delta);
526 const double inv_translation_matrix[9] = {
531 const double scale_matrix[9] = {
536 const double translation_matrix[9] = {
541 double tmp1[9], tmp2[9];
542 matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
543 matmul3x3(tmp1, scale_matrix, tmp2);
544 matmul3x3(tmp2, translation_matrix, zoom_matrix);
547 video_window->set_zoom_matrix(zoom_matrix);
551 void VideoWidget::mousePressEvent(QMouseEvent *e)
553 if (e->button() == Qt::BackButton) {
554 emit mouse_back_clicked();
555 } else if (e->button() == Qt::ForwardButton) {
556 emit mouse_forward_clicked();
557 } else if (e->button() == Qt::LeftButton) {
559 last_drag_x = e->position().x();
560 last_drag_y = e->position().y();
564 void VideoWidget::mouseReleaseEvent(QMouseEvent *e)
566 if (e->button() == Qt::LeftButton) {
571 void VideoWidget::mouseMoveEvent(QMouseEvent *e)
576 float dx = (e->position().x() - last_drag_x) / width();
577 float dy = (e->position().y() - last_drag_y) / height();
579 //zoom_matrix[6] += dx * zoom_matrix[0];
580 //zoom_matrix[7] += dy * zoom_matrix[4];
581 zoom_matrix[6] += dx;
582 zoom_matrix[7] -= dy;
584 video_window->set_zoom_matrix(zoom_matrix);
586 last_drag_x = e->position().x();
587 last_drag_y = e->position().y();
592 // Normalize the matrix so that we never get skew or similar,
593 // and also never can zoom or pan too far out.
594 void VideoWidget::fixup_zoom_matrix()
596 // Correct for any numerical errors (we know the matrix must be orthogonal
597 // and have zero rotation).
598 zoom_matrix[4] = zoom_matrix[0];
599 zoom_matrix[1] = zoom_matrix[2] = zoom_matrix[3] = zoom_matrix[5] = 0.0;
600 zoom_matrix[8] = 1.0;
602 // We can't zoom further out than 1:1. (Perhaps it would be nice to
603 // reuse the last zoom-in point to do this, but the center will have to do
605 if (zoom_matrix[0] < 1.0) {
606 const double zoom = 1.0 / zoom_matrix[0];
607 const double inv_translation_matrix[9] = {
612 const double scale_matrix[9] = {
617 const double translation_matrix[9] = {
622 double tmp1[9], tmp2[9];
623 matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
624 matmul3x3(tmp1, scale_matrix, tmp2);
625 matmul3x3(tmp2, translation_matrix, zoom_matrix);
628 // Looking at the points we'll draw with glVertex2f(), make sure none of them are
629 // inside the square (which would generally mean we've panned ourselves out-of-bounds).
630 // We simply adjust the translation, which is possible because we fixed scaling above.
631 zoom_matrix[6] = min(zoom_matrix[6], 0.0); // Left side (x=0).
632 zoom_matrix[7] = min(zoom_matrix[7], 0.0); // Bottom side (y=0).
633 zoom_matrix[6] = std::max(zoom_matrix[6], 1.0 - zoom_matrix[0]); // Right side (x=1).
634 zoom_matrix[7] = std::max(zoom_matrix[7], 1.0 - zoom_matrix[4]); // Top side (y=1).
637 bool VideoWidget::open(const string &filename)
644 while (running == STARTING) {
645 // Poor man's condition variable...
649 return (running != VIDEO_FILE_ERROR);
652 void VideoWidget::play()
654 if (running != NOT_RUNNING && running != VIDEO_FILE_ERROR) {
655 std::lock_guard<std::mutex> lock(queue_mu);
656 command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
657 producer_thread_should_quit.wakeup();
661 producer_thread_should_quit.unquit();
662 if (producer_thread.joinable()) {
663 producer_thread.join();
665 producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
668 void VideoWidget::pause()
670 if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
673 std::lock_guard<std::mutex> lock(queue_mu);
674 command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
675 producer_thread_should_quit.wakeup();
678 void VideoWidget::seek(int64_t relative_seek_ms)
680 if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
683 std::lock_guard<std::mutex> lock(queue_mu);
684 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
685 producer_thread_should_quit.wakeup();
688 void VideoWidget::seek_frames(int64_t relative_seek_frames)
690 if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
693 std::lock_guard<std::mutex> lock(queue_mu);
694 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
695 producer_thread_should_quit.wakeup();
698 void VideoWidget::seek_absolute(int64_t position_ms)
700 if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
703 std::lock_guard<std::mutex> lock(queue_mu);
704 command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
705 producer_thread_should_quit.wakeup();
708 void VideoWidget::stop()
710 if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
713 producer_thread_should_quit.quit();
714 producer_thread.join();
717 void VideoWidget::producer_thread_func()
719 if (!producer_thread_should_quit.should_quit()) {
720 if (!play_video(pathname)) {
721 running = VIDEO_FILE_ERROR;
723 running = NOT_RUNNING;
728 void VideoWidget::internal_rewind()
730 pts_origin = last_pts = 0;
732 start = next_frame_start = steady_clock::now();
735 template<AVHWDeviceType type>
736 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
738 bool found_config_of_right_type = false;
739 for (int i = 0;; ++i) { // Termination condition inside loop.
740 const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
741 if (config == nullptr) { // End of list.
744 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
745 config->device_type != type) {
746 // Not interesting for us.
750 // We have a config of the right type, but does it actually support
751 // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
752 // is to just replace the pixel format with a software-decoded one,
754 found_config_of_right_type = true;
755 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
756 if (config->pix_fmt == *fmt_ptr) {
757 fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
758 av_hwdevice_get_type_name(type), ctx->codec->name);
759 if (ctx->profile == FF_PROFILE_H264_BASELINE) {
760 fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
761 fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n");
762 fprintf(stderr, " Decoding might fail and fall back to software.\n");
764 return config->pix_fmt;
767 fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
768 unordered_set<AVPixelFormat> seen;
769 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
770 if (!seen.count(*fmt_ptr)) {
771 fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
772 seen.insert(*fmt_ptr);
775 fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
779 if (!found_config_of_right_type) {
780 fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
783 // We found no VA-API formats, so take the first software format.
784 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
785 if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
786 fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
791 // Fallback: Just return anything. (Should never really happen.)
795 AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
796 const std::string &pathname, int video_stream_index,
801 if (!queued_frames.empty()) {
802 AVFrameWithDeleter frame = std::move(queued_frames.front());
803 queued_frames.pop_front();
807 // Read packets until we have a frame or there are none left.
808 bool frame_finished = false;
809 AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
812 AVPacket *pkt = av_packet_alloc();
813 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
814 pkt, av_packet_unref);
817 if (av_read_frame(format_ctx, pkt) == 0) {
818 if (pkt->stream_index == video_stream_index) {
819 if (avcodec_send_packet(video_codec_ctx, pkt) < 0) {
820 fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
822 return AVFrameWithDeleter(nullptr);
826 eof = true; // Or error, but ignore that for the time being.
829 // Decode video, if we have a frame.
830 int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
832 frame_finished = true;
834 } else if (err != AVERROR(EAGAIN)) {
835 fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
837 return AVFrameWithDeleter(nullptr);
842 return video_avframe;
844 return AVFrameWithDeleter(nullptr);
847 int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
849 for (unsigned i = 0; i < ctx->nb_streams; ++i) {
850 if (ctx->streams[i]->codecpar->codec_type == media_type) {
857 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
859 const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
860 return origin + duration_cast<steady_clock::duration>(pts / rate);
863 bool VideoWidget::play_video(const string &pathname)
865 queued_frames.clear();
866 AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
867 /*options=*/nullptr);
868 if (format_ctx == nullptr) {
869 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
873 if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
874 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
878 int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
879 if (video_stream_index == -1) {
880 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
884 // Open video decoder.
885 const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
886 const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
888 video_timebase = format_ctx->streams[video_stream_index]->time_base;
889 AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
890 if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
891 fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
894 if (video_codec == nullptr) {
895 fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
899 // Seemingly, it's not too easy to make something that just initializes
900 // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
901 // without enumerating through several different types.
902 // VA-API and VDPAU will do for now. We prioritize VDPAU for the
903 // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
904 // cards that seems to work, but just hangs when trying to transfer the frame.
906 // Note that we don't actually check codec support beforehand,
907 // so if you have a low-end VDPAU device but a high-end VA-API device,
908 // you lose out on the extra codec support from the latter.
909 AVBufferRef *hw_device_ctx = nullptr;
910 if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
911 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
912 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
913 } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
914 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
915 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
917 fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
920 if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
921 fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
924 unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
925 video_codec_ctx.get(), avcodec_close);
932 int consecutive_errors = 0;
934 while (!producer_thread_should_quit.should_quit()) {
935 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
939 producer_thread_should_quit.sleep_for(hours(1));
944 AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
945 pathname, video_stream_index, &error);
947 if (++consecutive_errors >= 100) {
948 fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
954 consecutive_errors = 0;
956 if (frame == nullptr) {
961 // Sleep until it's time to present this frame.
963 if (last_pts == 0 && pts_origin == 0) {
964 pts_origin = frame->pts;
966 steady_clock::time_point now = steady_clock::now();
967 next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
969 if (duration<double>(now - next_frame_start).count() >= 0.1) {
970 // If we don't have enough CPU to keep up, or if we have a live stream
971 // where the initial origin was somehow wrong, we could be behind indefinitely.
972 fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
974 1e3 * duration<double>(now - next_frame_start).count());
975 pts_origin = frame->pts;
976 start = next_frame_start = now;
978 bool finished_wakeup;
979 finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
980 if (finished_wakeup) {
981 video_window->set_current_frame(make_video_frame(frame.get()));
982 last_frame = steady_clock::now();
986 if (producer_thread_should_quit.should_quit()) break;
989 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
994 // Just paused, so present the frame immediately and then go into deep sleep.
995 video_window->set_current_frame(make_video_frame(frame.get()));
996 last_frame = steady_clock::now();
1001 // If we just seeked, drop this frame on the floor and be done.
1007 store_pts(frame->pts);
1012 void VideoWidget::store_pts(int64_t pts)
1015 last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
1016 post_to_main_thread([this, last_position{last_position.load()}] {
1017 emit position_changed(last_position);
1021 // Taken from Movit (see the comment there for explanation)
1022 float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
1024 float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
1025 if (fabs(local_chroma_pos - 0.5) < 1e-10) {
1026 // x + (-0) can be optimized away freely, as opposed to x + 0.
1029 return (0.5 - local_chroma_pos) / resolution;
1033 shared_ptr<VideoWidget::Frame> VideoWidget::alloc_frame(unsigned width, unsigned height, unsigned chroma_width, unsigned chroma_height)
1035 lock_guard lock(freelist_mu);
1036 for (auto it = frame_freelist.begin(); it != frame_freelist.end(); ++it) {
1037 if ((*it)->width == width &&
1038 (*it)->height == height &&
1039 (*it)->chroma_width == chroma_width &&
1040 (*it)->chroma_height == chroma_height) {
1042 frame_freelist.erase(it);
1043 return shared_ptr<Frame>{frame, free_frame};
1047 Frame *frame = new Frame;
1048 frame->owner = this;
1049 frame->width = width;
1050 frame->height = height;
1051 frame->chroma_width = chroma_width;
1052 frame->chroma_height = chroma_height;
1054 size_t len = frame->width * frame->height + 2 * frame->chroma_width * frame->chroma_height;
1056 while (!video_window->isValid()) {
1062 condition_variable done_cv;
1065 post_to_main_thread([this, &frame, len, &done, &mu, &done_cv]{
1066 video_window->makeCurrent();
1067 auto gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(video_window->context());
1068 gl->glCreateBuffers(1, &frame->pbo);
1069 gl->glNamedBufferStorage(frame->pbo, len, nullptr, GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT);
1070 frame->data = (uint8_t *)gl->glMapNamedBufferRange(frame->pbo, 0, len, GL_MAP_WRITE_BIT | GL_MAP_FLUSH_EXPLICIT_BIT | GL_MAP_PERSISTENT_BIT);
1071 video_window->doneCurrent();
1073 lock_guard lock(mu);
1075 done_cv.notify_all();
1078 unique_lock lock(mu);
1079 done_cv.wait(lock, [&done]{ return done; });
1082 return shared_ptr<Frame>{frame, free_frame};
1085 void VideoWidget::free_frame(VideoWidget::Frame *frame)
1087 VideoWidget *self = frame->owner;
1088 lock_guard lock(self->freelist_mu);
1089 if (self->frame_freelist.size() >= 16) {
1090 GLuint pbo = frame->pbo;
1091 post_to_main_thread([self, pbo]{
1092 self->video_window->makeCurrent();
1093 auto gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(self->video_window->context());
1094 gl->glUnmapNamedBuffer(pbo);
1095 gl->glDeleteBuffers(1, &pbo);
1096 self->video_window->doneCurrent();
1098 delete self->frame_freelist.front();
1099 self->frame_freelist.pop_front();
1101 self->frame_freelist.push_back(frame);
1104 shared_ptr<VideoWidget::Frame> VideoWidget::make_video_frame(const AVFrame *frame)
1106 AVFrameWithDeleter sw_frame;
1108 if (frame->format == AV_PIX_FMT_VAAPI ||
1109 frame->format == AV_PIX_FMT_VDPAU) {
1110 // Get the frame down to the CPU. (TODO: See if we can keep it
1111 // on the GPU all the way, since it will be going up again later.
1112 // However, this only works if the OpenGL GPU is the same one.)
1113 sw_frame = av_frame_alloc_unique();
1114 int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
1116 fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
1118 sw_frame->pts = frame->pts;
1119 sw_frame->pkt_duration = frame->pkt_duration;
1120 frame = sw_frame.get();
1124 if (sws_ctx == nullptr ||
1125 sws_last_width != frame->width ||
1126 sws_last_height != frame->height ||
1127 sws_last_src_format != frame->format) {
1128 sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
1130 sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
1131 frame->width, frame->height, sws_dst_format,
1132 SWS_BICUBIC, nullptr, nullptr, nullptr));
1133 sws_last_width = frame->width;
1134 sws_last_height = frame->height;
1135 sws_last_src_format = frame->format;
1137 if (sws_ctx == nullptr) {
1138 fprintf(stderr, "Could not create scaler context\n");
1142 uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
1143 int linesizes[4] = { 0, 0, 0, 0 };
1144 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
1146 shared_ptr<Frame> video_frame = alloc_frame(
1149 AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w),
1150 AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h));
1152 // We always assume left chroma placement for now.
1153 video_window->set_cbcr_offset(
1154 compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame->chroma_width),
1155 compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame->chroma_height)
1158 pic_data[0] = video_frame->data;
1159 linesizes[0] = frame->width;
1161 pic_data[1] = pic_data[0] + frame->width * frame->height;
1162 linesizes[1] = video_frame->chroma_width;
1164 pic_data[2] = pic_data[1] + video_frame->chroma_width * video_frame->chroma_height;
1165 linesizes[2] = video_frame->chroma_width;
1167 sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
1169 video_frame->need_flush_len = video_frame->width * video_frame->height + 2 * video_frame->chroma_width * video_frame->chroma_height;