git.sesse.net Git - pkanalytics/blob - video_widget.cpp

   1 #include "video_widget.h"
   2
   3 #include <assert.h>
   4 #include <pthread.h>
   5 #include <stdint.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <sys/stat.h>
  10 #include <unistd.h>
  11
  12 extern "C" {
  13 #include <libavcodec/avcodec.h>
  14 #include <libavformat/avformat.h>
  15 #include <libavutil/avutil.h>
  16 #include <libavutil/error.h>
  17 #include <libavutil/frame.h>
  18 #include <libavutil/imgutils.h>
  19 #include <libavutil/mem.h>
  20 #include <libavutil/pixfmt.h>
  21 #include <libavutil/opt.h>
  22 #include <libswscale/swscale.h>
  23 }
  24
  25 #include <chrono>
  26 #include <cstdint>
  27 #include <utility>
  28 #include <vector>
  29 #include <unordered_set>
  30
  31 #include "post_to_main_thread.h"
  32
  33 #include <QOpenGLFunctions>
  34 #include <QWheelEvent>
  35 #include <QMouseEvent>
  36 #include <QMouseEvent>
  37 #include <QHBoxLayout>
  38 #include <QOpenGLFunctions_4_5_Compatibility>
  39 #include <QOpenGLVersionFunctionsFactory>
  40
  41 #define BUFFER_OFFSET(i) ((char *)nullptr + (i))
  42
  43 using namespace std;
  44 using namespace std::chrono;
  45
  46 namespace {
  47
  48 bool is_full_range(const AVPixFmtDescriptor *desc)
  49 {
  50         // This is horrible, but there's no better way that I know of.
  51         return (strchr(desc->name, 'j') != nullptr);
  52 }
  53
  54 AVPixelFormat decide_dst_format(AVPixelFormat src_format)
  55 {
  56         // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
  57         // and be done with it. It's too strange to spend a lot of time on.
  58         // (Let's hope there's no alpha.)
  59         const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
  60         if (src_desc == nullptr ||
  61             src_desc->nb_components != 3 ||
  62             (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
  63                 return AV_PIX_FMT_YUV444P;
  64         }
  65
  66         // The best for us would be Cb and Cr together if possible,
  67         // but FFmpeg doesn't support that except in the special case of
  68         // NV12, so we need to go to planar even for the case of NV12.
  69         // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
  70         // that matches in color range. (This will also include the case of
  71         // the source format already being acceptable.)
  72         bool src_full_range = is_full_range(src_desc);
  73         const char *best_format = "yuv444p";
  74         unsigned best_score = numeric_limits<unsigned>::max();
  75         for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
  76              desc;
  77              desc = av_pix_fmt_desc_next(desc)) {
  78                 // Find planar Y'CbCr formats only.
  79                 if (desc->nb_components != 3) continue;
  80                 if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
  81                 if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
  82                 if (desc->comp[0].plane != 0 ||
  83                     desc->comp[1].plane != 1 ||
  84                     desc->comp[2].plane != 2) continue;
  85
  86                 // 8-bit formats only.
  87                 if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
  88                 if (desc->comp[0].depth != 8) continue;
  89
  90                 // Same or better chroma resolution only.
  91                 int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
  92                 int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
  93                 if (chroma_w_diff < 0 || chroma_h_diff < 0)
  94                         continue;
  95
  96                 // Matching full/limited range only.
  97                 if (is_full_range(desc) != src_full_range)
  98                         continue;
  99
 100                 // Pick something with as little excess chroma resolution as possible.
 101                 unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
 102                 if (score < best_score) {
 103                         best_score = score;
 104                         best_format = desc->name;
 105                 }
 106         }
 107         return av_get_pix_fmt(best_format);
 108 }
 109
 110 }  // namespace
 111
 112 bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
 113 {
 114         // Process any queued commands from other threads.
 115         vector<QueuedCommand> commands;
 116         {
 117                 lock_guard<mutex> lock(queue_mu);
 118                 swap(commands, command_queue);
 119         }
 120
 121         for (const QueuedCommand &cmd : commands) {
 122                 switch (cmd.command) {
 123                 case QueuedCommand::PAUSE:
 124                         paused = true;
 125                         break;
 126                 case QueuedCommand::RESUME:
 127                         paused = false;
 128                         pts_origin = last_pts;
 129                         start = next_frame_start = steady_clock::now();
 130                         break;
 131                 case QueuedCommand::SEEK:
 132                 case QueuedCommand::SEEK_ABSOLUTE:
 133                         // Dealt with below.
 134                         break;
 135                 }
 136         }
 137
 138         // Combine all seeks into one big one. (There are edge cases where this is probably
 139         // subtly wrong, but we'll live with it.)
 140         int64_t base_pts = last_pts;
 141         int64_t relative_seek_ms = 0;
 142         int64_t relative_seek_frames = 0;
 143         for (const QueuedCommand &cmd : commands) {
 144                 if (cmd.command == QueuedCommand::SEEK) {
 145                         relative_seek_ms += cmd.relative_seek_ms;
 146                         relative_seek_frames += cmd.relative_seek_frames;
 147                 } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
 148                         base_pts = av_rescale_q(cmd.seek_ms, AVRational{ 1, 1000 }, video_timebase);
 149                         relative_seek_ms = 0;
 150                         relative_seek_frames = 0;
 151                 }
 152         }
 153         int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
 154         if (relative_seek_ms != 0 && relative_seek_pts == 0) {
 155                 // Just to be sure rounding errors don't move us into nothingness.
 156                 relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
 157         }
 158         int64_t goal_pts = base_pts + relative_seek_pts;
 159         if (goal_pts != last_pts || relative_seek_frames < 0) {
 160                 avcodec_flush_buffers(video_codec_ctx);
 161                 queued_frames.clear();
 162
 163                 // Seek to the last keyframe before this point.
 164                 int64_t seek_pts = goal_pts;
 165                 if (relative_seek_frames < 0) {
 166                         // If we're frame-skipping backwards, add 100 ms of slop for each frame
 167                         // so we're fairly certain we are able to see the ones we want.
 168                         seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
 169                 }
 170                 av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
 171
 172                 // Decode frames until EOF, or until we see something past our seek point.
 173                 std::deque<AVFrameWithDeleter> queue;
 174                 for ( ;; ) {
 175                         bool error = false;
 176                         AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 177                                 pathname, video_stream_index, &error);
 178                         if (frame == nullptr || error) {
 179                                 break;
 180                         }
 181
 182                         int64_t frame_pts = frame->pts;
 183                         if (relative_seek_frames < 0) {
 184                                 // Buffer this frame; don't display it unless we know it's the Nth-latest.
 185                                 queue.push_back(std::move(frame));
 186                                 if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
 187                                         queue.pop_front();
 188                                 }
 189                         }
 190                         if (frame_pts >= goal_pts) {
 191                                 if (relative_seek_frames > 0) {
 192                                         --relative_seek_frames;
 193                                 } else {
 194                                         if (relative_seek_frames < 0) {
 195                                                 // Hope we have the right amount.
 196                                                 // The rest will remain in the queue for when we play forward again.
 197                                                 frame = std::move(queue.front());
 198                                                 queue.pop_front();
 199                                                 queued_frames = std::move(queue);
 200                                         }
 201                                         video_window->set_current_frame(make_video_frame(frame.get()));
 202                                         update();
 203                                         store_pts(frame->pts);
 204                                         break;
 205                                 }
 206                         }
 207                 }
 208
 209                 // NOTE: We keep pause status as-is.
 210
 211                 pts_origin = last_pts;
 212                 start = next_frame_start = last_frame = steady_clock::now();
 213                 if (seeked) {
 214                         *seeked = true;
 215                 }
 216         } else if (relative_seek_frames > 0) {
 217                 // The base PTS is fine, we only need to skip a few frames forwards.
 218                 while (relative_seek_frames > 1) {
 219                         // Eat a frame (ignore errors).
 220                         bool error;
 221                         decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
 222                         --relative_seek_frames;
 223                 }
 224
 225                 // Display the last one.
 226                 bool error;
 227                 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 228                         pathname, video_stream_index, &error);
 229                 if (frame == nullptr || error) {
 230                         return true;
 231                 }
 232                 video_window->set_current_frame(make_video_frame(frame.get()));
 233                 update();
 234                 store_pts(frame->pts);
 235         }
 236         return false;
 237 }
 238
 239 VideoWidget::VideoWidget(QWidget *parent)
 240         : QWidget(parent),
 241           video_window(new VideoWindow(this)) {
 242         setLayout(new QHBoxLayout);
 243         layout()->setContentsMargins(QMargins());
 244         layout()->addWidget(QWidget::createWindowContainer(video_window));
 245
 246         // ...
 247         connect(video_window, &VideoWindow::mouse_wheel, this, &VideoWidget::wheelEvent);
 248         connect(video_window, &VideoWindow::mouse_pressed, this, &VideoWidget::mousePressEvent);
 249         connect(video_window, &VideoWindow::mouse_released, this, &VideoWidget::mouseReleaseEvent);
 250         connect(video_window, &VideoWindow::mouse_moved, this, &VideoWidget::mouseMoveEvent);
 251 }
 252
 253 VideoWidget::~VideoWidget()
 254 {
 255         stop();
 256
 257         // Qt will delete video_window for us after we're gone,
 258         // so make sure its destructor does not try to mess with
 259         // our freelist. The actual freelist frames will leak.
 260         video_window->set_current_frame(nullptr);
 261 }
 262
 263 GLuint compile_shader(QOpenGLFunctions_4_5_Compatibility *gl, const string &shader_src, GLenum type)
 264 {
 265         GLuint obj = gl->glCreateShader(type);
 266         const GLchar* source[] = { shader_src.data() };
 267         const GLint length[] = { (GLint)shader_src.size() };
 268         gl->glShaderSource(obj, 1, source, length);
 269         gl->glCompileShader(obj);
 270
 271         GLchar info_log[4096];
 272         GLsizei log_length = sizeof(info_log) - 1;
 273         gl->glGetShaderInfoLog(obj, log_length, &log_length, info_log);
 274         info_log[log_length] = 0;
 275         if (strlen(info_log) > 0) {
 276                 fprintf(stderr, "Shader compile log: %s\n", info_log);
 277         }
 278
 279         GLint status;
 280         gl->glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
 281         if (status == GL_FALSE) {
 282                 // Add some line numbers to easier identify compile errors.
 283                 string src_with_lines = "/*   1 */ ";
 284                 size_t lineno = 1;
 285                 for (char ch : shader_src) {
 286                         src_with_lines.push_back(ch);
 287                         if (ch == '\n') {
 288                                 char buf[32];
 289                                 snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
 290                                 src_with_lines += buf;
 291                         }
 292                 }
 293
 294                 fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
 295                 exit(1);
 296         }
 297
 298         return obj;
 299 }
 300
 301 void VideoWindow::initializeGL()
 302 {
 303         gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(context());
 304
 305         gl->glDisable(GL_BLEND);
 306         gl->glDisable(GL_DEPTH_TEST);
 307         gl->glDepthMask(GL_FALSE);
 308         gl->glCreateTextures(GL_TEXTURE_2D, 3, tex);
 309
 310         ycbcr_vertex_shader = compile_shader(gl, R"(
 311 #version 450 core
 312
 313 layout(location = 0) in vec2 position;
 314 layout(location = 1) in vec2 texcoord;
 315 out vec2 tc;
 316
 317 void main()
 318 {
 319         // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
 320         //
 321         //   2.000  0.000  0.000 -1.000
 322         //   0.000  2.000  0.000 -1.000
 323         //   0.000  0.000 -2.000 -1.000
 324         //   0.000  0.000  0.000  1.000
 325         gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
 326         tc = texcoord;
 327         tc.y = 1.0f - tc.y;
 328 }
 329 )", GL_VERTEX_SHADER);
 330         ycbcr_fragment_shader = compile_shader(gl, R"(
 331 #version 450 core
 332
 333 layout(location = 0) uniform sampler2D tex_y;
 334 layout(location = 1) uniform sampler2D tex_cb;
 335 layout(location = 2) uniform sampler2D tex_cr;
 336 layout(location = 3) uniform vec2 cbcr_offset;
 337
 338 in vec2 tc;
 339 out vec4 FragColor;
 340
 341 // Computed statically by Movit, for limited-range BT.709.
 342 // (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
 343 const mat3 inv_ycbcr_matrix = mat3(
 344         1.16438f, 1.16438f, 1.16438f,
 345         0.0f, -0.21325f, 2.11240f,
 346         1.79274f, -0.53291f, 0.0f
 347 );
 348
 349 void main()
 350 {
 351         if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
 352                 FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
 353                 return;
 354         }
 355
 356         vec3 ycbcr;
 357         ycbcr.r = texture(tex_y, tc).r;
 358         ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
 359         ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
 360         ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
 361         FragColor.rgb = inv_ycbcr_matrix * ycbcr;
 362         FragColor.a = 1.0f;
 363 }
 364 )", GL_FRAGMENT_SHADER);
 365         ycbcr_program = gl->glCreateProgram();
 366         gl->glAttachShader(ycbcr_program, ycbcr_vertex_shader);
 367         gl->glAttachShader(ycbcr_program, ycbcr_fragment_shader);
 368         gl->glLinkProgram(ycbcr_program);
 369
 370         GLint success;
 371         gl->glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
 372         if (success == GL_FALSE) {
 373                 GLchar error_log[1024] = {0};
 374                 gl->glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
 375                 fprintf(stderr, "Error linking program: %s\n", error_log);
 376                 exit(1);
 377         }
 378
 379         gl->glCreateSamplers(1, &bilinear_sampler);
 380         gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
 381         gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 382         gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 383         gl->glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 384 }
 385
 386 void VideoWindow::resizeGL(int w, int h)
 387 {
 388         gl->glViewport(0, 0, w, h);
 389         display_aspect = double(w) / h;
 390 }
 391
 392 int num_levels(GLuint width, GLuint height)
 393 {
 394         int levels = 1;
 395         while (width > 1 || height > 1) {
 396                 width = max(width / 2, 1u);
 397                 height = max(height / 2, 1u);
 398                 ++levels;
 399         }
 400         return levels;
 401 }
 402
 403 void VideoWindow::paintGL()
 404 {
 405         std::shared_ptr<VideoWidget::Frame> frame;
 406         {
 407                 lock_guard lock(current_frame_mu);
 408                 frame = current_frame;
 409         }
 410         if (frame == nullptr) {
 411                 gl->glClear(GL_COLOR_BUFFER_BIT);
 412                 return;
 413         }
 414
 415         gl->glUseProgram(ycbcr_program);
 416         if (frame->width != last_width || frame->height != last_height) {
 417                 gl->glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
 418         }
 419         if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
 420                 for (GLuint num : { tex[1], tex[2] }) {
 421                         gl->glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
 422                 }
 423         }
 424
 425         gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, frame->pbo);
 426
 427         if (frame->need_flush_len > 0) {
 428                 gl->glFlushMappedNamedBufferRange(frame->pbo, 0, frame->need_flush_len);
 429                 frame->need_flush_len = 0;
 430         }
 431
 432         gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
 433         gl->glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(0));
 434         gl->glGenerateTextureMipmap(tex[0]);
 435
 436         gl->glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(frame->width * frame->height));
 437         gl->glGenerateTextureMipmap(tex[1]);
 438
 439         gl->glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, BUFFER_OFFSET(frame->width * frame->height + frame->chroma_width * frame->chroma_height));
 440         gl->glGenerateTextureMipmap(tex[2]);
 441
 442         gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
 443
 444         gl->glBindTextureUnit(0, tex[0]);
 445         gl->glBindTextureUnit(1, tex[1]);
 446         gl->glBindTextureUnit(2, tex[2]);
 447         gl->glBindSampler(0, bilinear_sampler);
 448         gl->glBindSampler(1, bilinear_sampler);
 449         gl->glBindSampler(2, bilinear_sampler);
 450         gl->glProgramUniform1i(ycbcr_program, 0, 0);
 451         gl->glProgramUniform1i(ycbcr_program, 1, 1);
 452         gl->glProgramUniform1i(ycbcr_program, 2, 2);
 453         gl->glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
 454
 455         float tx1 = 0.0f;
 456         float tx2 = 1.0f;
 457         float ty1 = 0.0f;
 458         float ty2 = 1.0f;
 459
 460         double video_aspect = double(frame->width) / frame->height;
 461         if (display_aspect > video_aspect) {
 462                 double extra_width = frame->height * display_aspect - frame->width;
 463                 tx1 = -0.5 * extra_width / frame->width;
 464                 tx2 = 1.0 + 0.5 * extra_width / frame->width;
 465         } else if (display_aspect < video_aspect) {
 466                 double extra_height = frame->width / display_aspect - frame->height;
 467                 ty1 = -0.5 * extra_height / frame->height;
 468                 ty2 = 1.0 + 0.5 * extra_height / frame->height;
 469         }
 470
 471         gl->glBegin(GL_QUADS);
 472
 473         // (0,0)
 474         gl->glVertexAttrib2f(1, tx1, ty1);
 475         gl->glVertex2f(zoom_matrix[2 * 3 + 0], zoom_matrix[2 * 3 + 1]);
 476
 477         // (0,1)
 478         gl->glVertexAttrib2f(1, tx1, ty2);
 479         gl->glVertex2f(zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0], zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
 480
 481         // (1,1)
 482         gl->glVertexAttrib2f(1, tx2, ty2);
 483         gl->glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0],
 484                    zoom_matrix[1 * 3 + 0] + zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
 485
 486         // (1,0)
 487         gl->glVertexAttrib2f(1, tx2, ty1);
 488         gl->glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[2 * 3 + 0],
 489                    zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 1]);
 490
 491         gl->glEnd();
 492 }
 493
 494 void VideoWindow::set_current_frame(shared_ptr<VideoWidget::Frame> new_frame)
 495 {
 496         {
 497                 lock_guard lock(current_frame_mu);
 498                 current_frame = std::move(new_frame);
 499         }
 500         update();
 501 }
 502
 503 void matmul3x3(const double a[9], const double b[9], double res[9])
 504 {
 505         for (int i = 0; i < 3; ++i) {
 506                 for (int j = 0; j < 3; ++j) {
 507                         double sum = 0.0;
 508                         for (int k = 0; k < 3; ++k) {
 509                                 sum += a[i * 3 + k] * b[k * 3 + j];
 510                         }
 511                         res[i * 3 + j] = sum;
 512                 }
 513         }
 514 }
 515
 516 void VideoWidget::wheelEvent(QWheelEvent *event)
 517 {
 518         int delta = event->angleDelta().y();
 519         if (delta == 0) {
 520                 return;
 521         }
 522         double x = event->position().x() / width();
 523         double y = 1.0 - event->position().y() / height();
 524         double zoom = delta > 0 ? pow(1.005, delta) : pow(1/1.005, -delta);
 525
 526         const double inv_translation_matrix[9] = {
 527                 1.0, 0.0, 0.0,
 528                 0.0, 1.0, 0.0,
 529                 -x, -y, 1.0
 530         };
 531         const double scale_matrix[9] = {
 532                 zoom, 0.0, 0.0,
 533                 0.0, zoom, 0.0,
 534                 0.0, 0.0, 1.0
 535         };
 536         const double translation_matrix[9] = {
 537                 1.0, 0.0, 0.0,
 538                 0.0, 1.0, 0.0,
 539                 x, y, 1.0
 540         };
 541         double tmp1[9], tmp2[9];
 542         matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
 543         matmul3x3(tmp1, scale_matrix, tmp2);
 544         matmul3x3(tmp2, translation_matrix, zoom_matrix);
 545
 546         fixup_zoom_matrix();
 547         video_window->set_zoom_matrix(zoom_matrix);
 548         update();
 549 }
 550
 551 void VideoWidget::mousePressEvent(QMouseEvent *e)
 552 {
 553         if (e->button() == Qt::BackButton) {
 554                 emit mouse_back_clicked();
 555         } else if (e->button() == Qt::ForwardButton) {
 556                 emit mouse_forward_clicked();
 557         } else if (e->button() == Qt::LeftButton) {
 558                 dragging = true;
 559                 last_drag_x = e->position().x();
 560                 last_drag_y = e->position().y();
 561         }
 562 }
 563
 564 void VideoWidget::mouseReleaseEvent(QMouseEvent *e)
 565 {
 566         if (e->button() == Qt::LeftButton) {
 567                 dragging = false;
 568         }
 569 }
 570
 571 void VideoWidget::mouseMoveEvent(QMouseEvent *e)
 572 {
 573         if (!dragging) {
 574                 return;
 575         }
 576         float dx = (e->position().x() - last_drag_x) / width();
 577         float dy = (e->position().y() - last_drag_y) / height();
 578
 579         //zoom_matrix[6] += dx * zoom_matrix[0];
 580         //zoom_matrix[7] += dy * zoom_matrix[4];
 581         zoom_matrix[6] += dx;
 582         zoom_matrix[7] -= dy;
 583         fixup_zoom_matrix();
 584         video_window->set_zoom_matrix(zoom_matrix);
 585
 586         last_drag_x = e->position().x();
 587         last_drag_y = e->position().y();
 588
 589         update();
 590 }
 591
 592 // Normalize the matrix so that we never get skew or similar,
 593 // and also never can zoom or pan too far out.
 594 void VideoWidget::fixup_zoom_matrix()
 595 {
 596         // Correct for any numerical errors (we know the matrix must be orthogonal
 597         // and have zero rotation).
 598         zoom_matrix[4] = zoom_matrix[0];
 599         zoom_matrix[1] = zoom_matrix[2] = zoom_matrix[3] = zoom_matrix[5] = 0.0;
 600         zoom_matrix[8] = 1.0;
 601
 602         // We can't zoom further out than 1:1. (Perhaps it would be nice to
 603         // reuse the last zoom-in point to do this, but the center will have to do
 604         // for now.)
 605         if (zoom_matrix[0] < 1.0) {
 606                 const double zoom = 1.0 / zoom_matrix[0];
 607                 const double inv_translation_matrix[9] = {
 608                         1.0, 0.0, 0.0,
 609                         0.0, 1.0, 0.0,
 610                         -0.5, -0.5, 1.0
 611                 };
 612                 const double scale_matrix[9] = {
 613                         zoom, 0.0, 0.0,
 614                         0.0, zoom, 0.0,
 615                         0.0, 0.0, 1.0
 616                 };
 617                 const double translation_matrix[9] = {
 618                         1.0, 0.0, 0.0,
 619                         0.0, 1.0, 0.0,
 620                         0.5, 0.5, 1.0
 621                 };
 622                 double tmp1[9], tmp2[9];
 623                 matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
 624                 matmul3x3(tmp1, scale_matrix, tmp2);
 625                 matmul3x3(tmp2, translation_matrix, zoom_matrix);
 626         }
 627
 628         // Looking at the points we'll draw with glVertex2f(), make sure none of them are
 629         // inside the square (which would generally mean we've panned ourselves out-of-bounds).
 630         // We simply adjust the translation, which is possible because we fixed scaling above.
 631         zoom_matrix[6] = min(zoom_matrix[6], 0.0);  // Left side (x=0).
 632         zoom_matrix[7] = min(zoom_matrix[7], 0.0);  // Bottom side (y=0).
 633         zoom_matrix[6] = std::max(zoom_matrix[6], 1.0 - zoom_matrix[0]);  // Right side (x=1).
 634         zoom_matrix[7] = std::max(zoom_matrix[7], 1.0 - zoom_matrix[4]);  // Top side (y=1).
 635 }
 636
 637 bool VideoWidget::open(const string &filename)
 638 {
 639         stop();
 640         internal_rewind();
 641         pathname = filename;
 642         play();
 643
 644         while (running == STARTING) {
 645                 // Poor man's condition variable...
 646                 usleep(10000);
 647                 sched_yield();
 648         }
 649         return (running != VIDEO_FILE_ERROR);
 650 }
 651
 652 void VideoWidget::play()
 653 {
 654         if (running != NOT_RUNNING && running != VIDEO_FILE_ERROR) {
 655                 std::lock_guard<std::mutex> lock(queue_mu);
 656                 command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
 657                 producer_thread_should_quit.wakeup();
 658                 return;
 659         }
 660         running = STARTING;
 661         producer_thread_should_quit.unquit();
 662         if (producer_thread.joinable()) {
 663                 producer_thread.join();
 664         }
 665         producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
 666 }
 667
 668 void VideoWidget::pause()
 669 {
 670         if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
 671                 return;
 672         }
 673         std::lock_guard<std::mutex> lock(queue_mu);
 674         command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
 675         producer_thread_should_quit.wakeup();
 676 }
 677
 678 void VideoWidget::seek(int64_t relative_seek_ms)
 679 {
 680         if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
 681                 return;
 682         }
 683         std::lock_guard<std::mutex> lock(queue_mu);
 684         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
 685         producer_thread_should_quit.wakeup();
 686 }
 687
 688 void VideoWidget::seek_frames(int64_t relative_seek_frames)
 689 {
 690         if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
 691                 return;
 692         }
 693         std::lock_guard<std::mutex> lock(queue_mu);
 694         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
 695         producer_thread_should_quit.wakeup();
 696 }
 697
 698 void VideoWidget::seek_absolute(int64_t position_ms)
 699 {
 700         if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
 701                 return;
 702         }
 703         std::lock_guard<std::mutex> lock(queue_mu);
 704         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
 705         producer_thread_should_quit.wakeup();
 706 }
 707
 708 void VideoWidget::stop()
 709 {
 710         if (running == NOT_RUNNING || running == VIDEO_FILE_ERROR) {
 711                 return;
 712         }
 713         producer_thread_should_quit.quit();
 714         producer_thread.join();
 715 }
 716
 717 void VideoWidget::producer_thread_func()
 718 {
 719         if (!producer_thread_should_quit.should_quit()) {
 720                 if (!play_video(pathname)) {
 721                         running = VIDEO_FILE_ERROR;
 722                 } else {
 723                         running = NOT_RUNNING;
 724                 }
 725         }
 726 }
 727
 728 void VideoWidget::internal_rewind()
 729 {
 730         pts_origin = last_pts = 0;
 731         last_position = 0;
 732         start = next_frame_start = steady_clock::now();
 733 }
 734
 735 template<AVHWDeviceType type>
 736 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
 737 {
 738         bool found_config_of_right_type = false;
 739         for (int i = 0;; ++i) {  // Termination condition inside loop.
 740                 const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
 741                 if (config == nullptr) {  // End of list.
 742                         break;
 743                 }
 744                 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
 745                     config->device_type != type) {
 746                         // Not interesting for us.
 747                         continue;
 748                 }
 749
 750                 // We have a config of the right type, but does it actually support
 751                 // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
 752                 // is to just replace the pixel format with a software-decoded one,
 753                 // such as yuv420p.)
 754                 found_config_of_right_type = true;
 755                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 756                         if (config->pix_fmt == *fmt_ptr) {
 757                                 fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
 758                                         av_hwdevice_get_type_name(type), ctx->codec->name);
 759                                 if (ctx->profile == FF_PROFILE_H264_BASELINE) {
 760                                         fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
 761                                         fprintf(stderr, "         Consider encoding it as Constrained Baseline, Main or High instead.\n");
 762                                         fprintf(stderr, "         Decoding might fail and fall back to software.\n");
 763                                 }
 764                                 return config->pix_fmt;
 765                         }
 766                 }
 767                 fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
 768                 unordered_set<AVPixelFormat> seen;
 769                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 770                         if (!seen.count(*fmt_ptr)) {
 771                                 fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
 772                                 seen.insert(*fmt_ptr);
 773                         }
 774                 }
 775                 fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
 776
 777         }
 778
 779         if (!found_config_of_right_type) {
 780                 fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
 781         }
 782
 783         // We found no VA-API formats, so take the first software format.
 784         for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 785                 if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
 786                         fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
 787                         return *fmt_ptr;
 788                 }
 789         }
 790
 791         // Fallback: Just return anything. (Should never really happen.)
 792         return fmt[0];
 793 }
 794
 795 AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
 796         const std::string &pathname, int video_stream_index,
 797         bool *error)
 798 {
 799         *error = false;
 800
 801         if (!queued_frames.empty()) {
 802                 AVFrameWithDeleter frame = std::move(queued_frames.front());
 803                 queued_frames.pop_front();
 804                 return frame;
 805         }
 806
 807         // Read packets until we have a frame or there are none left.
 808         bool frame_finished = false;
 809         AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
 810         bool eof = false;
 811         do {
 812                 AVPacket *pkt = av_packet_alloc();
 813                 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
 814                         pkt, av_packet_unref);
 815                 pkt->data = nullptr;
 816                 pkt->size = 0;
 817                 if (av_read_frame(format_ctx, pkt) == 0) {
 818                         if (pkt->stream_index == video_stream_index) {
 819                                 if (avcodec_send_packet(video_codec_ctx, pkt) < 0) {
 820                                         fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
 821                                         *error = true;
 822                                         return AVFrameWithDeleter(nullptr);
 823                                 }
 824                         }
 825                 } else {
 826                         eof = true;  // Or error, but ignore that for the time being.
 827                 }
 828
 829                 // Decode video, if we have a frame.
 830                 int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
 831                 if (err == 0) {
 832                         frame_finished = true;
 833                         break;
 834                 } else if (err != AVERROR(EAGAIN)) {
 835                         fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
 836                         *error = true;
 837                         return AVFrameWithDeleter(nullptr);
 838                 }
 839         } while (!eof);
 840
 841         if (frame_finished)
 842                 return video_avframe;
 843         else
 844                 return AVFrameWithDeleter(nullptr);
 845 }
 846
 847 int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
 848 {
 849         for (unsigned i = 0; i < ctx->nb_streams; ++i) {
 850                 if (ctx->streams[i]->codecpar->codec_type == media_type) {
 851                         return i;
 852                 }
 853         }
 854         return -1;
 855 }
 856
 857 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
 858 {
 859         const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
 860         return origin + duration_cast<steady_clock::duration>(pts / rate);
 861 }
 862
 863 bool VideoWidget::play_video(const string &pathname)
 864 {
 865         queued_frames.clear();
 866         AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
 867                 /*options=*/nullptr);
 868         if (format_ctx == nullptr) {
 869                 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
 870                 return false;
 871         }
 872
 873         if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
 874                 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
 875                 return false;
 876         }
 877
 878         int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
 879         if (video_stream_index == -1) {
 880                 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
 881                 return false;
 882         }
 883
 884         // Open video decoder.
 885         const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
 886         const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
 887
 888         video_timebase = format_ctx->streams[video_stream_index]->time_base;
 889         AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
 890         if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
 891                 fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
 892                 return false;
 893         }
 894         if (video_codec == nullptr) {
 895                 fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
 896                 return false;
 897         }
 898
 899         // Seemingly, it's not too easy to make something that just initializes
 900         // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
 901         // without enumerating through several different types.
 902         // VA-API and VDPAU will do for now. We prioritize VDPAU for the
 903         // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
 904         // cards that seems to work, but just hangs when trying to transfer the frame.
 905         //
 906         // Note that we don't actually check codec support beforehand,
 907         // so if you have a low-end VDPAU device but a high-end VA-API device,
 908         // you lose out on the extra codec support from the latter.
 909         AVBufferRef *hw_device_ctx = nullptr;
 910         if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
 911                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 912                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
 913         } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
 914                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 915                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
 916         } else {
 917                 fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
 918         }
 919
 920         if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
 921                 fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
 922                 return false;
 923         }
 924         unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
 925                 video_codec_ctx.get(), avcodec_close);
 926
 927         internal_rewind();
 928
 929         running = RUNNING;
 930
 931         // Main loop.
 932         int consecutive_errors = 0;
 933         double rate = 1.0;
 934         while (!producer_thread_should_quit.should_quit()) {
 935                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
 936                         return true;
 937                 }
 938                 if (paused) {
 939                         producer_thread_should_quit.sleep_for(hours(1));
 940                         continue;
 941                 }
 942
 943                 bool error;
 944                 AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
 945                         pathname, video_stream_index, &error);
 946                 if (error) {
 947                         if (++consecutive_errors >= 100) {
 948                                 fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
 949                                 return false;
 950                         } else {
 951                                 continue;
 952                         }
 953                 } else {
 954                         consecutive_errors = 0;
 955                 }
 956                 if (frame == nullptr) {
 957                         // EOF.
 958                         return false;
 959                 }
 960
 961                 // Sleep until it's time to present this frame.
 962                 for ( ;; ) {
 963                         if (last_pts == 0 && pts_origin == 0) {
 964                                 pts_origin = frame->pts;
 965                         }
 966                         steady_clock::time_point now = steady_clock::now();
 967                         next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
 968
 969                         if (duration<double>(now - next_frame_start).count() >= 0.1) {
 970                                 // If we don't have enough CPU to keep up, or if we have a live stream
 971                                 // where the initial origin was somehow wrong, we could be behind indefinitely.
 972                                 fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
 973                                         pathname.c_str(),
 974                                         1e3 * duration<double>(now - next_frame_start).count());
 975                                 pts_origin = frame->pts;
 976                                 start = next_frame_start = now;
 977                         }
 978                         bool finished_wakeup;
 979                         finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
 980                         if (finished_wakeup) {
 981                                 video_window->set_current_frame(make_video_frame(frame.get()));
 982                                 last_frame = steady_clock::now();
 983                                 update();
 984                                 break;
 985                         } else {
 986                                 if (producer_thread_should_quit.should_quit()) break;
 987
 988                                 bool seeked = false;
 989                                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
 990                                         return true;
 991                                 }
 992
 993                                 if (paused) {
 994                                         // Just paused, so present the frame immediately and then go into deep sleep.
 995                                         video_window->set_current_frame(make_video_frame(frame.get()));
 996                                         last_frame = steady_clock::now();
 997                                         update();
 998                                         break;
 999                                 }
1000
1001                                 // If we just seeked, drop this frame on the floor and be done.
1002                                 if (seeked) {
1003                                         break;
1004                                 }
1005                         }
1006                 }
1007                 store_pts(frame->pts);
1008         }
1009         return true;
1010 }
1011
1012 void VideoWidget::store_pts(int64_t pts)
1013 {
1014         last_pts = pts;
1015         last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
1016         post_to_main_thread([this, last_position{last_position.load()}] {
1017                 emit position_changed(last_position);
1018         });
1019 }
1020
1021 // Taken from Movit (see the comment there for explanation)
1022 float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
1023 {
1024         float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
1025         if (fabs(local_chroma_pos - 0.5) < 1e-10) {
1026                 // x + (-0) can be optimized away freely, as opposed to x + 0.
1027                 return -0.0;
1028         } else {
1029                 return (0.5 - local_chroma_pos) / resolution;
1030         }
1031 }
1032
1033 shared_ptr<VideoWidget::Frame> VideoWidget::alloc_frame(unsigned width, unsigned height, unsigned chroma_width, unsigned chroma_height)
1034 {
1035         lock_guard lock(freelist_mu);
1036         for (auto it = frame_freelist.begin(); it != frame_freelist.end(); ++it) {
1037                 if ((*it)->width == width &&
1038                     (*it)->height == height &&
1039                     (*it)->chroma_width == chroma_width &&
1040                     (*it)->chroma_height == chroma_height) {
1041                         Frame *frame = *it;
1042                         frame_freelist.erase(it);
1043                         return shared_ptr<Frame>{frame, free_frame};
1044                 }
1045         }
1046
1047         Frame *frame = new Frame;
1048         frame->owner = this;
1049         frame->width = width;
1050         frame->height = height;
1051         frame->chroma_width = chroma_width;
1052         frame->chroma_height = chroma_height;
1053
1054         size_t len = frame->width * frame->height + 2 * frame->chroma_width * frame->chroma_height;
1055
1056         while (!video_window->isValid()) {
1057                 usleep(100000);
1058         }
1059
1060         // Augh :-)
1061         mutex mu;
1062         condition_variable done_cv;
1063         bool done = false;
1064
1065         post_to_main_thread([this, &frame, len, &done, &mu, &done_cv]{
1066                 video_window->makeCurrent();
1067                 auto gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(video_window->context());
1068                 gl->glCreateBuffers(1, &frame->pbo);
1069                 gl->glNamedBufferStorage(frame->pbo, len, nullptr, GL_MAP_WRITE_BIT | GL_MAP_PERSISTENT_BIT);
1070                 frame->data = (uint8_t *)gl->glMapNamedBufferRange(frame->pbo, 0, len, GL_MAP_WRITE_BIT | GL_MAP_FLUSH_EXPLICIT_BIT | GL_MAP_PERSISTENT_BIT);
1071                 video_window->doneCurrent();
1072
1073                 lock_guard lock(mu);
1074                 done = true;
1075                 done_cv.notify_all();
1076         });
1077         {
1078                 unique_lock lock(mu);
1079                 done_cv.wait(lock, [&done]{ return done; });
1080         }
1081
1082         return shared_ptr<Frame>{frame, free_frame};
1083 }
1084
1085 void VideoWidget::free_frame(VideoWidget::Frame *frame)
1086 {
1087         VideoWidget *self = frame->owner;
1088         lock_guard lock(self->freelist_mu);
1089         if (self->frame_freelist.size() >= 16) {
1090                 GLuint pbo = frame->pbo;
1091                 post_to_main_thread([self, pbo]{
1092                         self->video_window->makeCurrent();
1093                         auto gl = QOpenGLVersionFunctionsFactory::get<QOpenGLFunctions_4_5_Compatibility>(self->video_window->context());
1094                         gl->glUnmapNamedBuffer(pbo);
1095                         gl->glDeleteBuffers(1, &pbo);
1096                         self->video_window->doneCurrent();
1097                 });
1098                 delete self->frame_freelist.front();
1099                 self->frame_freelist.pop_front();
1100         }
1101         self->frame_freelist.push_back(frame);
1102 }
1103
1104 shared_ptr<VideoWidget::Frame> VideoWidget::make_video_frame(const AVFrame *frame)
1105 {
1106         AVFrameWithDeleter sw_frame;
1107
1108         if (frame->format == AV_PIX_FMT_VAAPI ||
1109             frame->format == AV_PIX_FMT_VDPAU) {
1110                 // Get the frame down to the CPU. (TODO: See if we can keep it
1111                 // on the GPU all the way, since it will be going up again later.
1112                 // However, this only works if the OpenGL GPU is the same one.)
1113                 sw_frame = av_frame_alloc_unique();
1114                 int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
1115                 if (err != 0) {
1116                         fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
1117                 } else {
1118                         sw_frame->pts = frame->pts;
1119                         sw_frame->pkt_duration = frame->pkt_duration;
1120                         frame = sw_frame.get();
1121                 }
1122         }
1123
1124         if (sws_ctx == nullptr ||
1125             sws_last_width != frame->width ||
1126             sws_last_height != frame->height ||
1127             sws_last_src_format != frame->format) {
1128                 sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
1129                 sws_ctx.reset(
1130                         sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
1131                                 frame->width, frame->height, sws_dst_format,
1132                                 SWS_BICUBIC, nullptr, nullptr, nullptr));
1133                 sws_last_width = frame->width;
1134                 sws_last_height = frame->height;
1135                 sws_last_src_format = frame->format;
1136         }
1137         if (sws_ctx == nullptr) {
1138                 fprintf(stderr, "Could not create scaler context\n");
1139                 abort();
1140         }
1141
1142         uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
1143         int linesizes[4] = { 0, 0, 0, 0 };
1144         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
1145
1146         shared_ptr<Frame> video_frame = alloc_frame(
1147                 frame->width,
1148                 frame->height,
1149                 AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w),
1150                 AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h));
1151
1152         // We always assume left chroma placement for now.
1153         video_window->set_cbcr_offset(
1154                 compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame->chroma_width),
1155                 compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame->chroma_height)
1156         );
1157
1158         pic_data[0] = video_frame->data;
1159         linesizes[0] = frame->width;
1160
1161         pic_data[1] = pic_data[0] + frame->width * frame->height;
1162         linesizes[1] = video_frame->chroma_width;
1163
1164         pic_data[2] = pic_data[1] + video_frame->chroma_width * video_frame->chroma_height;
1165         linesizes[2] = video_frame->chroma_width;
1166
1167         sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
1168
1169         video_frame->need_flush_len = video_frame->width * video_frame->height + 2 * video_frame->chroma_width * video_frame->chroma_height;
1170
1171         return video_frame;
1172 }
1173