git.sesse.net Git - pkanalytics/blob - video_widget.cpp

   1 #define GL_GLEXT_PROTOTYPES
   2
   3 #include "video_widget.h"
   4
   5 #include <assert.h>
   6 #include <pthread.h>
   7 #include <stdint.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <sys/stat.h>
  12 #include <unistd.h>
  13
  14 extern "C" {
  15 #include <libavcodec/avcodec.h>
  16 #include <libavformat/avformat.h>
  17 #include <libavutil/avutil.h>
  18 #include <libavutil/error.h>
  19 #include <libavutil/frame.h>
  20 #include <libavutil/imgutils.h>
  21 #include <libavutil/mem.h>
  22 #include <libavutil/pixfmt.h>
  23 #include <libavutil/opt.h>
  24 #include <libswscale/swscale.h>
  25 }
  26
  27 #include <chrono>
  28 #include <cstdint>
  29 #include <utility>
  30 #include <vector>
  31 #include <unordered_set>
  32
  33 #include <QOpenGLFunctions>
  34
  35 using namespace std;
  36 using namespace std::chrono;
  37
  38 namespace {
  39
  40 bool is_full_range(const AVPixFmtDescriptor *desc)
  41 {
  42         // This is horrible, but there's no better way that I know of.
  43         return (strchr(desc->name, 'j') != nullptr);
  44 }
  45
  46 AVPixelFormat decide_dst_format(AVPixelFormat src_format)
  47 {
  48         // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
  49         // and be done with it. It's too strange to spend a lot of time on.
  50         // (Let's hope there's no alpha.)
  51         const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
  52         if (src_desc == nullptr ||
  53             src_desc->nb_components != 3 ||
  54             (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
  55                 return AV_PIX_FMT_YUV444P;
  56         }
  57
  58         // The best for us would be Cb and Cr together if possible,
  59         // but FFmpeg doesn't support that except in the special case of
  60         // NV12, so we need to go to planar even for the case of NV12.
  61         // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
  62         // that matches in color range. (This will also include the case of
  63         // the source format already being acceptable.)
  64         bool src_full_range = is_full_range(src_desc);
  65         const char *best_format = "yuv444p";
  66         unsigned best_score = numeric_limits<unsigned>::max();
  67         for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
  68              desc;
  69              desc = av_pix_fmt_desc_next(desc)) {
  70                 // Find planar Y'CbCr formats only.
  71                 if (desc->nb_components != 3) continue;
  72                 if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
  73                 if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
  74                 if (desc->comp[0].plane != 0 ||
  75                     desc->comp[1].plane != 1 ||
  76                     desc->comp[2].plane != 2) continue;
  77
  78                 // 8-bit formats only.
  79                 if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
  80                 if (desc->comp[0].depth != 8) continue;
  81
  82                 // Same or better chroma resolution only.
  83                 int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
  84                 int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
  85                 if (chroma_w_diff < 0 || chroma_h_diff < 0)
  86                         continue;
  87
  88                 // Matching full/limited range only.
  89                 if (is_full_range(desc) != src_full_range)
  90                         continue;
  91
  92                 // Pick something with as little excess chroma resolution as possible.
  93                 unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
  94                 if (score < best_score) {
  95                         best_score = score;
  96                         best_format = desc->name;
  97                 }
  98         }
  99         return av_get_pix_fmt(best_format);
 100 }
 101
 102 }  // namespace
 103
 104 bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
 105 {
 106         // Process any queued commands from other threads.
 107         vector<QueuedCommand> commands;
 108         {
 109                 lock_guard<mutex> lock(queue_mu);
 110                 swap(commands, command_queue);
 111         }
 112
 113         for (const QueuedCommand &cmd : commands) {
 114                 switch (cmd.command) {
 115                 case QueuedCommand::PAUSE:
 116                         paused = true;
 117                         break;
 118                 case QueuedCommand::RESUME:
 119                         paused = false;
 120                         pts_origin = last_pts;
 121                         start = next_frame_start = steady_clock::now();
 122                         break;
 123                 case QueuedCommand::SEEK:
 124                 case QueuedCommand::SEEK_ABSOLUTE:
 125                         // Dealt with below.
 126                         break;
 127                 }
 128         }
 129
 130         // Combine all seeks into one big one. (There are edge cases where this is probably
 131         // subtly wrong, but we'll live with it.)
 132         int64_t base_pts = last_pts;
 133         int64_t relative_seek_ms = 0;
 134         int64_t relative_seek_frames = 0;
 135         for (const QueuedCommand &cmd : commands) {
 136                 if (cmd.command == QueuedCommand::SEEK) {
 137                         relative_seek_ms += cmd.relative_seek_ms;
 138                         relative_seek_frames += cmd.relative_seek_frames;
 139                 } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
 140                         base_pts = cmd.seek_ms;
 141                         relative_seek_ms = 0;
 142                         relative_seek_frames = 0;
 143                 }
 144         }
 145         int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
 146         if (relative_seek_ms != 0 && relative_seek_pts == 0) {
 147                 // Just to be sure rounding errors don't move us into nothingness.
 148                 relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
 149         }
 150         int64_t goal_pts = base_pts + relative_seek_pts;
 151         if (goal_pts != last_pts || relative_seek_frames < 0) {
 152                 avcodec_flush_buffers(video_codec_ctx);
 153                 queued_frames.clear();
 154
 155                 // Seek to the last keyframe before this point.
 156                 int64_t seek_pts = goal_pts;
 157                 if (relative_seek_frames < 0) {
 158                         // If we're frame-skipping backwards, add 100 ms of slop for each frame
 159                         // so we're fairly certain we are able to see the ones we want.
 160                         seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
 161                 }
 162                 av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
 163
 164                 // Decode frames until EOF, or until we see something past our seek point.
 165                 std::deque<AVFrameWithDeleter> queue;
 166                 for ( ;; ) {
 167                         bool error = false;
 168                         AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 169                                 pathname, video_stream_index, &error);
 170                         if (frame == nullptr || error) {
 171                                 break;
 172                         }
 173
 174                         int64_t frame_pts = frame->pts;
 175                         if (relative_seek_frames < 0) {
 176                                 // Buffer this frame; don't display it unless we know it's the Nth-latest.
 177                                 queue.push_back(std::move(frame));
 178                                 if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
 179                                         queue.pop_front();
 180                                 }
 181                         }
 182                         if (frame_pts >= goal_pts) {
 183                                 if (relative_seek_frames > 0) {
 184                                         --relative_seek_frames;
 185                                 } else {
 186                                         if (relative_seek_frames < 0) {
 187                                                 // Hope we have the right amount.
 188                                                 // The rest will remain in the queue for when we play forward again.
 189                                                 frame = std::move(queue.front());
 190                                                 queue.pop_front();
 191                                                 queued_frames = std::move(queue);
 192                                         }
 193                                         current_frame.reset(new Frame(make_video_frame(frame.get())));
 194                                         update();
 195                                         store_pts(frame->pts);
 196                                         break;
 197                                 }
 198                         }
 199                 }
 200
 201                 // NOTE: We keep pause status as-is.
 202
 203                 pts_origin = last_pts;
 204                 start = next_frame_start = last_frame = steady_clock::now();
 205                 if (seeked) {
 206                         *seeked = true;
 207                 }
 208         } else if (relative_seek_frames > 0) {
 209                 // The base PTS is fine, we only need to skip a few frames forwards.
 210                 while (relative_seek_frames > 1) {
 211                         // Eat a frame (ignore errors).
 212                         bool error;
 213                         decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
 214                         --relative_seek_frames;
 215                 }
 216
 217                 // Display the last one.
 218                 bool error;
 219                 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 220                         pathname, video_stream_index, &error);
 221                 if (frame == nullptr || error) {
 222                         return true;
 223                 }
 224                 current_frame.reset(new Frame(make_video_frame(frame.get())));
 225                 update();
 226                 store_pts(frame->pts);
 227         }
 228         return false;
 229 }
 230
 231 VideoWidget::VideoWidget(QWidget *parent)
 232         : QOpenGLWidget(parent) {}
 233
 234 GLuint compile_shader(const string &shader_src, GLenum type)
 235 {
 236         GLuint obj = glCreateShader(type);
 237         const GLchar* source[] = { shader_src.data() };
 238         const GLint length[] = { (GLint)shader_src.size() };
 239         glShaderSource(obj, 1, source, length);
 240         glCompileShader(obj);
 241
 242         GLchar info_log[4096];
 243         GLsizei log_length = sizeof(info_log) - 1;
 244         glGetShaderInfoLog(obj, log_length, &log_length, info_log);
 245         info_log[log_length] = 0;
 246         if (strlen(info_log) > 0) {
 247                 fprintf(stderr, "Shader compile log: %s\n", info_log);
 248         }
 249
 250         GLint status;
 251         glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
 252         if (status == GL_FALSE) {
 253                 // Add some line numbers to easier identify compile errors.
 254                 string src_with_lines = "/*   1 */ ";
 255                 size_t lineno = 1;
 256                 for (char ch : shader_src) {
 257                         src_with_lines.push_back(ch);
 258                         if (ch == '\n') {
 259                                 char buf[32];
 260                                 snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
 261                                 src_with_lines += buf;
 262                         }
 263                 }
 264
 265                 fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
 266                 exit(1);
 267         }
 268
 269         return obj;
 270 }
 271
 272 void VideoWidget::initializeGL()
 273 {
 274         glDisable(GL_BLEND);
 275         glDisable(GL_DEPTH_TEST);
 276         glDepthMask(GL_FALSE);
 277         glCreateTextures(GL_TEXTURE_2D, 3, tex);
 278
 279         ycbcr_vertex_shader = compile_shader(R"(
 280 #version 460 core
 281
 282 layout(location = 0) in vec2 position;
 283 layout(location = 1) in vec2 texcoord;
 284 out vec2 tc;
 285
 286 void main()
 287 {
 288         // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
 289         //
 290         //   2.000  0.000  0.000 -1.000
 291         //   0.000  2.000  0.000 -1.000
 292         //   0.000  0.000 -2.000 -1.000
 293         //   0.000  0.000  0.000  1.000
 294         gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
 295         tc = texcoord;
 296         tc.y = 1.0f - tc.y;
 297 }
 298 )", GL_VERTEX_SHADER);
 299         ycbcr_fragment_shader = compile_shader(R"(
 300 #version 460 core
 301
 302 layout(location = 0) uniform sampler2D tex_y;
 303 layout(location = 1) uniform sampler2D tex_cb;
 304 layout(location = 2) uniform sampler2D tex_cr;
 305 layout(location = 3) uniform vec2 cbcr_offset;
 306
 307 in vec2 tc;
 308 out vec4 FragColor;
 309
 310 // Computed statically by Movit, for limited-range BT.709.
 311 // (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
 312 const mat3 inv_ycbcr_matrix = mat3(
 313         1.16438f, 1.16438f, 1.16438f,
 314         0.0f, -0.21325f, 2.11240f,
 315         1.79274f, -0.53291f, 0.0f
 316 );
 317
 318 void main()
 319 {
 320         if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
 321                 FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
 322                 return;
 323         }
 324
 325         vec3 ycbcr;
 326         ycbcr.r = texture(tex_y, tc).r;
 327         ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
 328         ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
 329         ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
 330         FragColor.rgb = inv_ycbcr_matrix * ycbcr;
 331         FragColor.a = 1.0f;
 332 }
 333 )", GL_FRAGMENT_SHADER);
 334         ycbcr_program = glCreateProgram();
 335         glAttachShader(ycbcr_program, ycbcr_vertex_shader);
 336         glAttachShader(ycbcr_program, ycbcr_fragment_shader);
 337         glLinkProgram(ycbcr_program);
 338
 339         GLint success;
 340         glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
 341         if (success == GL_FALSE) {
 342                 GLchar error_log[1024] = {0};
 343                 glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
 344                 fprintf(stderr, "Error linking program: %s\n", error_log);
 345                 exit(1);
 346         }
 347
 348         glCreateSamplers(1, &bilinear_sampler);
 349         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
 350         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 351         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 352         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 353 }
 354
 355 void VideoWidget::resizeGL(int w, int h)
 356 {
 357         glViewport(0, 0, w, h);
 358         display_aspect = double(w) / h;
 359 }
 360
 361 int num_levels(GLuint width, GLuint height)
 362 {
 363         int levels = 1;
 364         while (width > 1 || height > 1) {
 365                 width = max(width / 2, 1u);
 366                 height = max(height / 2, 1u);
 367                 ++levels;
 368         }
 369         return levels;
 370 }
 371
 372 void VideoWidget::paintGL()
 373 {
 374         std::shared_ptr<Frame> frame = current_frame;
 375         if (frame == nullptr) {
 376                 glClear(GL_COLOR_BUFFER_BIT);
 377                 return;
 378         }
 379
 380         glUseProgram(ycbcr_program);
 381         if (frame->width != last_width || frame->height != last_height) {
 382                 glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
 383         }
 384         if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
 385                 for (GLuint num : { tex[1], tex[2] }) {
 386                         glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
 387                 }
 388         }
 389
 390         glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get());
 391         glGenerateTextureMipmap(tex[0]);
 392
 393         glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height);
 394         glGenerateTextureMipmap(tex[1]);
 395
 396         glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height + frame->chroma_width * frame->chroma_height);
 397         glGenerateTextureMipmap(tex[2]);
 398
 399         glBindTextureUnit(0, tex[0]);
 400         glBindTextureUnit(1, tex[1]);
 401         glBindTextureUnit(2, tex[2]);
 402         glBindSampler(0, bilinear_sampler);
 403         glBindSampler(1, bilinear_sampler);
 404         glBindSampler(2, bilinear_sampler);
 405         glProgramUniform1i(ycbcr_program, 0, 0);
 406         glProgramUniform1i(ycbcr_program, 1, 1);
 407         glProgramUniform1i(ycbcr_program, 2, 2);
 408         glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
 409
 410         float tx1 = 0.0f;
 411         float tx2 = 1.0f;
 412         float ty1 = 0.0f;
 413         float ty2 = 1.0f;
 414
 415         double video_aspect = double(frame->width) / frame->height;
 416         if (display_aspect > video_aspect) {
 417                 double extra_width = frame->height * display_aspect - frame->width;
 418                 tx1 = -0.5 * extra_width / frame->width;
 419                 tx2 = 1.0 + 0.5 * extra_width / frame->width;
 420         } else if (display_aspect < video_aspect) {
 421                 double extra_height = frame->width / display_aspect - frame->height;
 422                 ty1 = -0.5 * extra_height / frame->height;
 423                 ty2 = 1.0 + 0.5 * extra_height / frame->height;
 424         }
 425
 426         glBegin(GL_QUADS);
 427
 428         glVertexAttrib2f(1, tx1, ty1);
 429         glVertex2f(0.0f, 0.0f);
 430
 431         glVertexAttrib2f(1, tx1, ty2);
 432         glVertex2f(0.0f, 1.0f);
 433
 434         glVertexAttrib2f(1, tx2, ty2);
 435         glVertex2f(1.0f, 1.0f);
 436
 437         glVertexAttrib2f(1, tx2, ty1);
 438         glVertex2f(1.0f, 0.0f);
 439
 440         glEnd();
 441 }
 442
 443 void VideoWidget::open(const string &filename)
 444 {
 445         stop();
 446         internal_rewind();
 447         pathname = filename;
 448         play();
 449 }
 450
 451 void VideoWidget::play()
 452 {
 453         if (running) {
 454                 std::lock_guard<std::mutex> lock(queue_mu);
 455                 command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
 456                 producer_thread_should_quit.wakeup();
 457                 return;
 458         }
 459         running = true;
 460         producer_thread_should_quit.unquit();
 461         producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
 462 }
 463
 464 void VideoWidget::pause()
 465 {
 466         if (!running) {
 467                 return;
 468         }
 469         std::lock_guard<std::mutex> lock(queue_mu);
 470         command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
 471         producer_thread_should_quit.wakeup();
 472 }
 473
 474 void VideoWidget::seek(int64_t relative_seek_ms)
 475 {
 476         if (!running) {
 477                 return;
 478         }
 479         std::lock_guard<std::mutex> lock(queue_mu);
 480         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
 481         producer_thread_should_quit.wakeup();
 482 }
 483
 484 void VideoWidget::seek_frames(int64_t relative_seek_frames)
 485 {
 486         if (!running) {
 487                 return;
 488         }
 489         std::lock_guard<std::mutex> lock(queue_mu);
 490         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
 491         producer_thread_should_quit.wakeup();
 492 }
 493
 494 void VideoWidget::seek_absolute(int64_t position_ms)
 495 {
 496         if (!running) {
 497                 return;
 498         }
 499         std::lock_guard<std::mutex> lock(queue_mu);
 500         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
 501         producer_thread_should_quit.wakeup();
 502 }
 503
 504 void VideoWidget::stop()
 505 {
 506         if (!running) {
 507                 return;
 508         }
 509         running = false;
 510         producer_thread_should_quit.quit();
 511         producer_thread.join();
 512 }
 513
 514 void VideoWidget::producer_thread_func()
 515 {
 516         if (!producer_thread_should_quit.should_quit()) {
 517                 if (!play_video(pathname)) {
 518                         // TODO: Send the error back to the UI somehow.
 519                 }
 520         }
 521 }
 522
 523 void VideoWidget::internal_rewind()
 524 {
 525         pts_origin = last_pts = 0;
 526         last_position = 0;
 527         start = next_frame_start = steady_clock::now();
 528 }
 529
 530 template<AVHWDeviceType type>
 531 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
 532 {
 533         bool found_config_of_right_type = false;
 534         for (int i = 0;; ++i) {  // Termination condition inside loop.
 535                 const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
 536                 if (config == nullptr) {  // End of list.
 537                         break;
 538                 }
 539                 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
 540                     config->device_type != type) {
 541                         // Not interesting for us.
 542                         continue;
 543                 }
 544
 545                 // We have a config of the right type, but does it actually support
 546                 // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
 547                 // is to just replace the pixel format with a software-decoded one,
 548                 // such as yuv420p.)
 549                 found_config_of_right_type = true;
 550                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 551                         if (config->pix_fmt == *fmt_ptr) {
 552                                 fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
 553                                         av_hwdevice_get_type_name(type), ctx->codec->name);
 554                                 if (ctx->profile == FF_PROFILE_H264_BASELINE) {
 555                                         fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
 556                                         fprintf(stderr, "         Consider encoding it as Constrained Baseline, Main or High instead.\n");
 557                                         fprintf(stderr, "         Decoding might fail and fall back to software.\n");
 558                                 }
 559                                 return config->pix_fmt;
 560                         }
 561                 }
 562                 fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
 563                 unordered_set<AVPixelFormat> seen;
 564                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 565                         if (!seen.count(*fmt_ptr)) {
 566                                 fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
 567                                 seen.insert(*fmt_ptr);
 568                         }
 569                 }
 570                 fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
 571
 572         }
 573
 574         if (!found_config_of_right_type) {
 575                 fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
 576         }
 577
 578         // We found no VA-API formats, so take the first software format.
 579         for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 580                 if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
 581                         fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
 582                         return *fmt_ptr;
 583                 }
 584         }
 585
 586         // Fallback: Just return anything. (Should never really happen.)
 587         return fmt[0];
 588 }
 589
 590 AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
 591         const std::string &pathname, int video_stream_index,
 592         bool *error)
 593 {
 594         *error = false;
 595
 596         if (!queued_frames.empty()) {
 597                 AVFrameWithDeleter frame = std::move(queued_frames.front());
 598                 queued_frames.pop_front();
 599                 return frame;
 600         }
 601
 602         // Read packets until we have a frame or there are none left.
 603         bool frame_finished = false;
 604         AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
 605         bool eof = false;
 606         do {
 607                 AVPacket pkt;
 608                 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
 609                         &pkt, av_packet_unref);
 610                 av_init_packet(&pkt);
 611                 pkt.data = nullptr;
 612                 pkt.size = 0;
 613                 if (av_read_frame(format_ctx, &pkt) == 0) {
 614                         if (pkt.stream_index == video_stream_index) {
 615                                 if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) {
 616                                         fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
 617                                         *error = true;
 618                                         return AVFrameWithDeleter(nullptr);
 619                                 }
 620                         }
 621                 } else {
 622                         eof = true;  // Or error, but ignore that for the time being.
 623                 }
 624
 625                 // Decode video, if we have a frame.
 626                 int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
 627                 if (err == 0) {
 628                         frame_finished = true;
 629                         break;
 630                 } else if (err != AVERROR(EAGAIN)) {
 631                         fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
 632                         *error = true;
 633                         return AVFrameWithDeleter(nullptr);
 634                 }
 635         } while (!eof);
 636
 637         if (frame_finished)
 638                 return video_avframe;
 639         else
 640                 return AVFrameWithDeleter(nullptr);
 641 }
 642
 643 int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
 644 {
 645         for (unsigned i = 0; i < ctx->nb_streams; ++i) {
 646                 if (ctx->streams[i]->codecpar->codec_type == media_type) {
 647                         return i;
 648                 }
 649         }
 650         return -1;
 651 }
 652
 653 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
 654 {
 655         const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
 656         return origin + duration_cast<steady_clock::duration>(pts / rate);
 657 }
 658
 659 bool VideoWidget::play_video(const string &pathname)
 660 {
 661         queued_frames.clear();
 662         AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
 663                 /*options=*/nullptr);
 664         if (format_ctx == nullptr) {
 665                 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
 666                 return false;
 667         }
 668
 669         if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
 670                 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
 671                 return false;
 672         }
 673
 674         int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
 675         if (video_stream_index == -1) {
 676                 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
 677                 return false;
 678         }
 679
 680         // Open video decoder.
 681         const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
 682         const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
 683
 684         video_timebase = format_ctx->streams[video_stream_index]->time_base;
 685         AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
 686         if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
 687                 fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
 688                 return false;
 689         }
 690         if (video_codec == nullptr) {
 691                 fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
 692                 return false;
 693         }
 694
 695         // Seemingly, it's not too easy to make something that just initializes
 696         // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
 697         // without enumerating through several different types.
 698         // VA-API and VDPAU will do for now. We prioritize VDPAU for the
 699         // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
 700         // cards that seems to work, but just hangs when trying to transfer the frame.
 701         //
 702         // Note that we don't actually check codec support beforehand,
 703         // so if you have a low-end VDPAU device but a high-end VA-API device,
 704         // you lose out on the extra codec support from the latter.
 705         AVBufferRef *hw_device_ctx = nullptr;
 706         if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
 707                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 708                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
 709         } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
 710                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 711                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
 712         } else {
 713                 fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
 714         }
 715
 716         if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
 717                 fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
 718                 return false;
 719         }
 720         unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
 721                 video_codec_ctx.get(), avcodec_close);
 722
 723         internal_rewind();
 724
 725         // Main loop.
 726         int consecutive_errors = 0;
 727         double rate = 1.0;
 728         while (!producer_thread_should_quit.should_quit()) {
 729                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
 730                         return true;
 731                 }
 732                 if (paused) {
 733                         producer_thread_should_quit.sleep_for(hours(1));
 734                         continue;
 735                 }
 736
 737                 bool error;
 738                 AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
 739                         pathname, video_stream_index, &error);
 740                 if (error) {
 741                         if (++consecutive_errors >= 100) {
 742                                 fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
 743                                 return false;
 744                         } else {
 745                                 continue;
 746                         }
 747                 } else {
 748                         consecutive_errors = 0;
 749                 }
 750                 if (frame == nullptr) {
 751                         // EOF.
 752                         return false;
 753                 }
 754
 755                 // Sleep until it's time to present this frame.
 756                 for ( ;; ) {
 757                         if (last_pts == 0 && pts_origin == 0) {
 758                                 pts_origin = frame->pts;
 759                         }
 760                         steady_clock::time_point now = steady_clock::now();
 761                         next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
 762
 763                         if (duration<double>(now - next_frame_start).count() >= 0.1) {
 764                                 // If we don't have enough CPU to keep up, or if we have a live stream
 765                                 // where the initial origin was somehow wrong, we could be behind indefinitely.
 766                                 fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
 767                                         pathname.c_str(),
 768                                         1e3 * duration<double>(now - next_frame_start).count());
 769                                 pts_origin = frame->pts;
 770                                 start = next_frame_start = now;
 771                         }
 772                         bool finished_wakeup;
 773                         finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
 774                         if (finished_wakeup) {
 775                                 current_frame.reset(new Frame(make_video_frame(frame.get())));
 776                                 last_frame = steady_clock::now();
 777                                 update();
 778                                 break;
 779                         } else {
 780                                 if (producer_thread_should_quit.should_quit()) break;
 781
 782                                 bool seeked = false;
 783                                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
 784                                         return true;
 785                                 }
 786
 787                                 if (paused) {
 788                                         // Just paused, so present the frame immediately and then go into deep sleep.
 789                                         current_frame.reset(new Frame(make_video_frame(frame.get())));
 790                                         last_frame = steady_clock::now();
 791                                         update();
 792                                         break;
 793                                 }
 794
 795                                 // If we just seeked, drop this frame on the floor and be done.
 796                                 if (seeked) {
 797                                         break;
 798                                 }
 799                         }
 800                 }
 801                 store_pts(frame->pts);
 802         }
 803         return true;
 804 }
 805
 806 void VideoWidget::store_pts(int64_t pts)
 807 {
 808         last_pts = pts;
 809         last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
 810         emit position_changed(last_position);
 811 }
 812
 813 // Taken from Movit (see the comment there for explanation)
 814 float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
 815 {
 816         float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
 817         if (fabs(local_chroma_pos - 0.5) < 1e-10) {
 818                 // x + (-0) can be optimized away freely, as opposed to x + 0.
 819                 return -0.0;
 820         } else {
 821                 return (0.5 - local_chroma_pos) / resolution;
 822         }
 823 }
 824
 825 VideoWidget::Frame VideoWidget::make_video_frame(const AVFrame *frame)
 826 {
 827         Frame video_frame;
 828         AVFrameWithDeleter sw_frame;
 829
 830         if (frame->format == AV_PIX_FMT_VAAPI ||
 831             frame->format == AV_PIX_FMT_VDPAU) {
 832                 // Get the frame down to the CPU. (TODO: See if we can keep it
 833                 // on the GPU all the way, since it will be going up again later.
 834                 // However, this only works if the OpenGL GPU is the same one.)
 835                 sw_frame = av_frame_alloc_unique();
 836                 int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
 837                 if (err != 0) {
 838                         fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
 839                 } else {
 840                         sw_frame->pts = frame->pts;
 841                         sw_frame->pkt_duration = frame->pkt_duration;
 842                         frame = sw_frame.get();
 843                 }
 844         }
 845
 846         if (sws_ctx == nullptr ||
 847             sws_last_width != frame->width ||
 848             sws_last_height != frame->height ||
 849             sws_last_src_format != frame->format) {
 850                 sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
 851                 sws_ctx.reset(
 852                         sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
 853                                 frame->width, frame->height, sws_dst_format,
 854                                 SWS_BICUBIC, nullptr, nullptr, nullptr));
 855                 sws_last_width = frame->width;
 856                 sws_last_height = frame->height;
 857                 sws_last_src_format = frame->format;
 858         }
 859         if (sws_ctx == nullptr) {
 860                 fprintf(stderr, "Could not create scaler context\n");
 861                 abort();
 862         }
 863
 864         uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
 865         int linesizes[4] = { 0, 0, 0, 0 };
 866         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
 867
 868         video_frame.width = frame->width;
 869         video_frame.height = frame->height;
 870         video_frame.chroma_width = AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w);
 871         video_frame.chroma_height = AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h);
 872
 873         // We always assume left chroma placement for now.
 874         cbcr_offset[0] = compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame.chroma_width);
 875         cbcr_offset[1] = compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame.chroma_height);
 876
 877         size_t len = frame->width * frame->height + 2 * video_frame.chroma_width * video_frame.chroma_height;
 878         video_frame.data.reset(new uint8_t[len]);
 879
 880         pic_data[0] = video_frame.data.get();
 881         linesizes[0] = frame->width;
 882
 883         pic_data[1] = pic_data[0] + frame->width * frame->height;
 884         linesizes[1] = video_frame.chroma_width;
 885
 886         pic_data[2] = pic_data[1] + video_frame.chroma_width * video_frame.chroma_height;
 887         linesizes[2] = video_frame.chroma_width;
 888
 889         sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
 890
 891         return video_frame;
 892 }
 893