git.sesse.net Git - nageru/blob - futatabi/main.cpp

   1 #include <arpa/inet.h>
   2 #include <assert.h>
   3 #include <atomic>
   4 #include <chrono>
   5 #include <condition_variable>
   6 #include <dirent.h>
   7 #include <getopt.h>
   8 #include <memory>
   9 #include <mutex>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string>
  13 #include <sys/stat.h>
  14 #include <sys/types.h>
  15 #include <thread>
  16 #include <unistd.h>
  17 #include <vector>
  18
  19 extern "C" {
  20 #include <libavformat/avformat.h>
  21 }
  22
  23 #include "clip_list.h"
  24 #include "defs.h"
  25 #include "flags.h"
  26 #include "frame.pb.h"
  27 #include "frame_on_disk.h"
  28 #include "mainwindow.h"
  29 #include "player.h"
  30 #include "shared/context.h"
  31 #include "shared/disk_space_estimator.h"
  32 #include "shared/ffmpeg_raii.h"
  33 #include "shared/httpd.h"
  34 #include "shared/metrics.h"
  35 #include "shared/post_to_main_thread.h"
  36 #include "shared/ref_counted_gl_sync.h"
  37 #include "shared/timebase.h"
  38 #include "ui_mainwindow.h"
  39 #include "vaapi_jpeg_decoder.h"
  40
  41 #include <QApplication>
  42 #include <QGLFormat>
  43 #include <QProgressDialog>
  44 #include <QSurfaceFormat>
  45 #include <movit/init.h>
  46 #include <movit/util.h>
  47
  48 using namespace std;
  49 using namespace std::chrono;
  50
  51 constexpr char frame_magic[] = "Ftbifrm0";
  52 constexpr size_t frame_magic_len = 8;
  53
  54 mutex RefCountedGLsync::fence_lock;
  55 atomic<bool> should_quit{ false };
  56
  57 int64_t start_pts = -1;
  58
  59 // TODO: Replace by some sort of GUI control, I guess.
  60 int64_t current_pts = 0;
  61
  62 struct FrameFile {
  63         FILE *fp = nullptr;
  64         unsigned filename_idx;
  65         size_t frames_written_so_far = 0;
  66 };
  67 std::map<int, FrameFile> open_frame_files;
  68
  69 mutex frame_mu;
  70 vector<FrameOnDisk> frames[MAX_STREAMS];  // Under frame_mu.
  71 vector<string> frame_filenames;  // Under frame_mu.
  72
  73 atomic<int64_t> metric_received_frames[MAX_STREAMS]{ { 0 } };
  74 Summary metric_received_frame_size_bytes;
  75
  76 namespace {
  77
  78 FrameOnDisk write_frame(int stream_idx, int64_t pts, const uint8_t *data, size_t size, DB *db)
  79 {
  80         if (open_frame_files.count(stream_idx) == 0) {
  81                 char filename[256];
  82                 snprintf(filename, sizeof(filename), "%s/frames/cam%d-pts%09" PRId64 ".frames",
  83                          global_flags.working_directory.c_str(), stream_idx, pts);
  84                 FILE *fp = fopen(filename, "wb");
  85                 if (fp == nullptr) {
  86                         perror(filename);
  87                         abort();
  88                 }
  89
  90                 lock_guard<mutex> lock(frame_mu);
  91                 unsigned filename_idx = frame_filenames.size();
  92                 frame_filenames.push_back(filename);
  93                 open_frame_files[stream_idx] = FrameFile{ fp, filename_idx, 0 };
  94         }
  95
  96         FrameFile &file = open_frame_files[stream_idx];
  97         unsigned filename_idx = file.filename_idx;
  98         string filename;
  99         {
 100                 lock_guard<mutex> lock(frame_mu);
 101                 filename = frame_filenames[filename_idx];
 102         }
 103
 104         FrameHeaderProto hdr;
 105         hdr.set_stream_idx(stream_idx);
 106         hdr.set_pts(pts);
 107         hdr.set_file_size(size);
 108
 109         string serialized;
 110         if (!hdr.SerializeToString(&serialized)) {
 111                 fprintf(stderr, "Frame header serialization failed.\n");
 112                 abort();
 113         }
 114         uint32_t len = htonl(serialized.size());
 115
 116         if (fwrite(frame_magic, frame_magic_len, 1, file.fp) != 1) {
 117                 perror("fwrite");
 118                 abort();
 119         }
 120         if (fwrite(&len, sizeof(len), 1, file.fp) != 1) {
 121                 perror("fwrite");
 122                 abort();
 123         }
 124         if (fwrite(serialized.data(), serialized.size(), 1, file.fp) != 1) {
 125                 perror("fwrite");
 126                 abort();
 127         }
 128         off_t offset = ftell(file.fp);
 129         if (fwrite(data, size, 1, file.fp) != 1) {
 130                 perror("fwrite");
 131                 abort();
 132         }
 133         fflush(file.fp);  // No fsync(), though. We can accept losing a few frames.
 134         global_disk_space_estimator->report_write(filename, 8 + sizeof(len) + serialized.size() + size, pts);
 135
 136         FrameOnDisk frame;
 137         frame.pts = pts;
 138         frame.filename_idx = filename_idx;
 139         frame.offset = offset;
 140         frame.size = size;
 141
 142         {
 143                 lock_guard<mutex> lock(frame_mu);
 144                 assert(stream_idx < MAX_STREAMS);
 145                 frames[stream_idx].push_back(frame);
 146         }
 147
 148         if (++file.frames_written_so_far >= FRAMES_PER_FILE) {
 149                 size_t size = ftell(file.fp);
 150
 151                 // Start a new file next time.
 152                 if (fclose(file.fp) != 0) {
 153                         perror("fclose");
 154                         abort();
 155                 }
 156                 open_frame_files.erase(stream_idx);
 157
 158                 // Write information about all frames in the finished file to SQLite.
 159                 // (If we crash before getting to do this, we'll be scanning through
 160                 // the file on next startup, and adding it to the database then.)
 161                 // NOTE: Since we don't fsync(), we could in theory get broken data
 162                 // but with the right size, but it would seem unlikely.
 163                 vector<DB::FrameOnDiskAndStreamIdx> frames_this_file;
 164                 {
 165                         lock_guard<mutex> lock(frame_mu);
 166                         for (size_t stream_idx = 0; stream_idx < MAX_STREAMS; ++stream_idx) {
 167                                 for (const FrameOnDisk &frame : frames[stream_idx]) {
 168                                         if (frame.filename_idx == filename_idx) {
 169                                                 frames_this_file.emplace_back(DB::FrameOnDiskAndStreamIdx{ frame, unsigned(stream_idx) });
 170                                         }
 171                                 }
 172                         }
 173                 }
 174
 175                 const char *basename = filename.c_str();
 176                 while (strchr(basename, '/') != nullptr) {
 177                         basename = strchr(basename, '/') + 1;
 178                 }
 179                 db->store_frame_file(basename, size, frames_this_file);
 180         }
 181
 182         return frame;
 183 }
 184
 185 }  // namespace
 186
 187 HTTPD *global_httpd;
 188
 189 void load_existing_frames();
 190 void record_thread_func();
 191
 192 int main(int argc, char **argv)
 193 {
 194         parse_flags(argc, argv);
 195         if (optind == argc) {
 196                 global_flags.stream_source = "multiangle.mp4";
 197                 global_flags.slow_down_input = true;
 198         } else if (optind + 1 == argc) {
 199                 global_flags.stream_source = argv[optind];
 200         } else {
 201                 usage();
 202                 abort();
 203         }
 204
 205         string frame_dir = global_flags.working_directory + "/frames";
 206
 207         if (mkdir(frame_dir.c_str(), 0777) == 0) {
 208                 fprintf(stderr, "%s does not exist, creating it.\n", frame_dir.c_str());
 209         } else if (errno != EEXIST) {
 210                 perror(global_flags.working_directory.c_str());
 211                 abort();
 212         }
 213
 214         avformat_network_init();
 215         global_metrics.set_prefix("futatabi");
 216         global_httpd = new HTTPD;
 217         global_metrics.remove("num_connected_multicam_clients");
 218
 219         QCoreApplication::setAttribute(Qt::AA_ShareOpenGLContexts, true);
 220
 221         QSurfaceFormat fmt;
 222         fmt.setDepthBufferSize(0);
 223         fmt.setStencilBufferSize(0);
 224         fmt.setProfile(QSurfaceFormat::CoreProfile);
 225         fmt.setMajorVersion(4);
 226         fmt.setMinorVersion(5);
 227
 228         // Turn off vsync, since Qt generally gives us at most frame rate
 229         // (display frequency) / (number of QGLWidgets active).
 230         fmt.setSwapInterval(0);
 231
 232         QSurfaceFormat::setDefaultFormat(fmt);
 233
 234         QGLFormat::setDefaultFormat(QGLFormat::fromSurfaceFormat(fmt));
 235
 236         QApplication app(argc, argv);
 237         global_share_widget = new QGLWidget();
 238         if (!global_share_widget->isValid()) {
 239                 fprintf(stderr, "Failed to initialize OpenGL. Futatabi needs at least OpenGL 4.5 to function properly.\n");
 240                 abort();
 241         }
 242
 243         // Initialize Movit.
 244         {
 245                 QSurface *surface = create_surface();
 246                 QOpenGLContext *context = create_context(surface);
 247                 if (!make_current(context, surface)) {
 248                         printf("oops\n");
 249                         abort();
 250                 }
 251                 CHECK(movit::init_movit(MOVIT_SHADER_DIR, movit::MOVIT_DEBUG_OFF));
 252                 delete_context(context);
 253                 // TODO: Delete the surface, too.
 254         }
 255
 256         load_existing_frames();
 257
 258         for (int stream_idx = 0; stream_idx < MAX_STREAMS; ++stream_idx) {
 259                 if (!frames[stream_idx].empty()) {
 260                         assert(start_pts > frames[stream_idx].back().pts);
 261                 }
 262         }
 263
 264         MainWindow main_window;
 265         main_window.show();
 266
 267         global_httpd->add_endpoint("/queue_status", bind(&MainWindow::get_queue_status, &main_window), HTTPD::NO_CORS_POLICY);
 268         global_httpd->start(global_flags.http_port);
 269
 270         init_jpeg_vaapi();
 271
 272         thread record_thread(record_thread_func);
 273
 274         int ret = app.exec();
 275
 276         should_quit = true;
 277         record_thread.join();
 278
 279         return ret;
 280 }
 281
 282 void load_frame_file(const char *filename, const string &basename, unsigned filename_idx, DB *db)
 283 {
 284         struct stat st;
 285         if (stat(filename, &st) == -1) {
 286                 perror(filename);
 287                 abort();
 288         }
 289
 290         vector<DB::FrameOnDiskAndStreamIdx> all_frames = db->load_frame_file(basename, st.st_size, filename_idx);
 291         if (!all_frames.empty()) {
 292                 // We already had this cached in the database, so no need to look in the file.
 293                 for (const DB::FrameOnDiskAndStreamIdx &frame : all_frames) {
 294                         if (frame.stream_idx < MAX_STREAMS) {
 295                                 frames[frame.stream_idx].push_back(frame.frame);
 296                                 start_pts = max(start_pts, frame.frame.pts);
 297                         }
 298                 }
 299                 return;
 300         }
 301
 302         FILE *fp = fopen(filename, "rb");
 303         if (fp == nullptr) {
 304                 perror(filename);
 305                 abort();
 306         }
 307
 308         // Find the actual length of the file, since fseek() past the end of the file
 309         // will succeed without an error.
 310         if (fseek(fp, 0, SEEK_END) == -1) {
 311                 perror("fseek(SEEK_END)");
 312                 abort();
 313         }
 314         off_t file_len = ftell(fp);
 315         if (fseek(fp, 0, SEEK_SET) == -1) {
 316                 perror("fseek(SEEK_SET)");
 317                 abort();
 318         }
 319
 320         size_t magic_offset = 0;
 321         size_t skipped_bytes = 0;
 322         while (!feof(fp) && !ferror(fp)) {
 323                 int ch = getc(fp);
 324                 if (ch == -1) {
 325                         break;
 326                 }
 327                 if (ch != frame_magic[magic_offset++]) {
 328                         skipped_bytes += magic_offset;
 329                         magic_offset = 0;
 330                         continue;
 331                 }
 332                 if (magic_offset < frame_magic_len) {
 333                         // Still reading the magic (hopefully).
 334                         continue;
 335                 }
 336
 337                 // OK, found the magic. Try to parse the frame header.
 338                 magic_offset = 0;
 339
 340                 if (skipped_bytes > 0) {
 341                         fprintf(stderr, "WARNING: %s: Skipped %zu garbage bytes in the middle.\n",
 342                                 filename, skipped_bytes);
 343                         skipped_bytes = 0;
 344                 }
 345
 346                 uint32_t len;
 347                 if (fread(&len, sizeof(len), 1, fp) != 1) {
 348                         fprintf(stderr, "WARNING: %s: Short read when getting length.\n", filename);
 349                         break;
 350                 }
 351
 352                 string serialized;
 353                 serialized.resize(ntohl(len));
 354                 if (fread(&serialized[0], serialized.size(), 1, fp) != 1) {
 355                         fprintf(stderr, "WARNING: %s: Short read when reading frame header (%zu bytes).\n", filename, serialized.size());
 356                         break;
 357                 }
 358
 359                 FrameHeaderProto hdr;
 360                 if (!hdr.ParseFromString(serialized)) {
 361                         fprintf(stderr, "WARNING: %s: Corrupted frame header.\n", filename);
 362                         continue;
 363                 }
 364
 365                 FrameOnDisk frame;
 366                 frame.pts = hdr.pts();
 367                 frame.offset = ftell(fp);
 368                 if (frame.offset == -1) {
 369                         fprintf(stderr, "WARNING: %s: ftell() failed (%s).\n", filename, strerror(errno));
 370                         break;
 371                 }
 372                 frame.filename_idx = filename_idx;
 373                 frame.size = hdr.file_size();
 374
 375                 if (frame.offset + frame.size > file_len ||
 376                     fseek(fp, frame.offset + frame.size, SEEK_SET) == -1) {
 377                         fprintf(stderr, "WARNING: %s: Could not seek past frame (probably truncated).\n", filename);
 378                         break;
 379                 }
 380
 381                 if (hdr.stream_idx() >= 0 && hdr.stream_idx() < MAX_STREAMS) {
 382                         frames[hdr.stream_idx()].push_back(frame);
 383                         start_pts = max(start_pts, hdr.pts());
 384                 }
 385                 all_frames.emplace_back(DB::FrameOnDiskAndStreamIdx{ frame, unsigned(hdr.stream_idx()) });
 386         }
 387
 388         if (skipped_bytes > 0) {
 389                 fprintf(stderr, "WARNING: %s: Skipped %zu garbage bytes at the end.\n",
 390                         filename, skipped_bytes);
 391         }
 392
 393         off_t size = ftell(fp);
 394         fclose(fp);
 395
 396         if (size == -1) {
 397                 fprintf(stderr, "WARNING: %s: ftell() failed (%s).\n", filename, strerror(errno));
 398                 return;
 399         }
 400
 401         db->store_frame_file(basename, size, all_frames);
 402 }
 403
 404 void load_existing_frames()
 405 {
 406         QProgressDialog progress("Scanning frame directory...", "Abort", 0, 1);
 407         progress.setWindowTitle("Futatabi");
 408         progress.setWindowModality(Qt::WindowModal);
 409         progress.setMinimumDuration(1000);
 410         progress.setMaximum(1);
 411         progress.setValue(0);
 412
 413         string frame_dir = global_flags.working_directory + "/frames";
 414         DIR *dir = opendir(frame_dir.c_str());
 415         if (dir == nullptr) {
 416                 perror("frames/");
 417                 start_pts = 0;
 418                 return;
 419         }
 420
 421         vector<string> frame_basenames;
 422         for (;;) {
 423                 errno = 0;
 424                 dirent *de = readdir(dir);
 425                 if (de == nullptr) {
 426                         if (errno != 0) {
 427                                 perror("readdir");
 428                                 abort();
 429                         }
 430                         break;
 431                 }
 432
 433                 if (de->d_type == DT_REG || de->d_type == DT_LNK) {
 434                         string filename = frame_dir + "/" + de->d_name;
 435                         frame_filenames.push_back(filename);
 436                         frame_basenames.push_back(de->d_name);
 437                 }
 438
 439                 if (progress.wasCanceled()) {
 440                         abort();
 441                 }
 442         }
 443         closedir(dir);
 444
 445         progress.setMaximum(frame_filenames.size() + 2);
 446         progress.setValue(1);
 447
 448         progress.setLabelText("Opening database...");
 449         DB db(global_flags.working_directory + "/futatabi.db");
 450
 451         progress.setLabelText("Reading frame files...");
 452         progress.setValue(2);
 453
 454         for (size_t i = 0; i < frame_filenames.size(); ++i) {
 455                 load_frame_file(frame_filenames[i].c_str(), frame_basenames[i], i, &db);
 456                 progress.setValue(i + 3);
 457                 if (progress.wasCanceled()) {
 458                         abort();
 459                 }
 460         }
 461
 462         if (start_pts == -1) {
 463                 start_pts = 0;
 464         } else {
 465                 // Add a gap of one second from the old frames to the new ones.
 466                 start_pts += TIMEBASE;
 467         }
 468         current_pts = start_pts;
 469
 470         for (int stream_idx = 0; stream_idx < MAX_STREAMS; ++stream_idx) {
 471                 sort(frames[stream_idx].begin(), frames[stream_idx].end(),
 472                      [](const auto &a, const auto &b) { return a.pts < b.pts; });
 473         }
 474
 475         db.clean_unused_frame_files(frame_basenames);
 476 }
 477
 478 void record_thread_func()
 479 {
 480         for (unsigned i = 0; i < MAX_STREAMS; ++i) {
 481                 global_metrics.add("received_frames", { { "stream", to_string(i) } }, &metric_received_frames[i]);
 482         }
 483         global_metrics.add("received_frame_size_bytes", &metric_received_frame_size_bytes);
 484
 485         if (global_flags.stream_source.empty() || global_flags.stream_source == "/dev/null") {
 486                 // Save the user from some repetitive messages.
 487                 return;
 488         }
 489
 490         pthread_setname_np(pthread_self(), "ReceiveFrames");
 491
 492         int64_t pts_offset = 0;  // Needs to be initialized due to a spurious GCC warning.
 493         DB db(global_flags.working_directory + "/futatabi.db");
 494
 495         while (!should_quit.load()) {
 496                 auto format_ctx = avformat_open_input_unique(global_flags.stream_source.c_str(), nullptr, nullptr);
 497                 if (format_ctx == nullptr) {
 498                         fprintf(stderr, "%s: Error opening file. Waiting one second and trying again...\n", global_flags.stream_source.c_str());
 499                         sleep(1);
 500                         continue;
 501                 }
 502
 503                 int64_t last_pts = -1;
 504
 505                 while (!should_quit.load()) {
 506                         AVPacket pkt;
 507                         unique_ptr<AVPacket, decltype(av_packet_unref) *> pkt_cleanup(
 508                                 &pkt, av_packet_unref);
 509                         av_init_packet(&pkt);
 510                         pkt.data = nullptr;
 511                         pkt.size = 0;
 512
 513                         // TODO: Make it possible to abort av_read_frame() (use an interrupt callback);
 514                         // right now, should_quit will be ignored if it's hung on I/O.
 515                         if (av_read_frame(format_ctx.get(), &pkt) != 0) {
 516                                 break;
 517                         }
 518                         if (pkt.stream_index >= MAX_STREAMS ||
 519                             format_ctx->streams[pkt.stream_index]->codecpar->codec_type != AVMEDIA_TYPE_VIDEO) {
 520                                 continue;
 521                         }
 522
 523                         ++metric_received_frames[pkt.stream_index];
 524                         metric_received_frame_size_bytes.count_event(pkt.size);
 525
 526                         // Convert pts to our own timebase.
 527                         AVRational stream_timebase = format_ctx->streams[pkt.stream_index]->time_base;
 528                         int64_t pts = av_rescale_q(pkt.pts, stream_timebase, AVRational{ 1, TIMEBASE });
 529
 530                         // Translate offset into our stream.
 531                         if (last_pts == -1) {
 532                                 pts_offset = start_pts - pts;
 533                         }
 534                         pts = std::max(pts + pts_offset, start_pts);
 535
 536                         //fprintf(stderr, "Got a frame from camera %d, pts = %ld, size = %d\n",
 537                         //      pkt.stream_index, pts, pkt.size);
 538                         FrameOnDisk frame = write_frame(pkt.stream_index, pts, pkt.data, pkt.size, &db);
 539
 540                         post_to_main_thread([pkt, frame] {
 541                                 global_mainwindow->display_frame(pkt.stream_index, frame);
 542                         });
 543
 544                         if (last_pts != -1 && global_flags.slow_down_input) {
 545                                 this_thread::sleep_for(microseconds((pts - last_pts) * 1000000 / TIMEBASE));
 546                         }
 547                         last_pts = pts;
 548                         current_pts = pts;
 549                 }
 550
 551                 if (!should_quit.load()) {
 552                         fprintf(stderr, "%s: Hit EOF. Waiting one second and trying again...\n", global_flags.stream_source.c_str());
 553                         sleep(1);
 554                 }
 555
 556                 start_pts = last_pts + TIMEBASE;
 557         }
 558 }