git.sesse.net Git - nageru/blob - futatabi/video_stream.h

   1 #ifndef _VIDEO_STREAM_H
   2 #define _VIDEO_STREAM_H 1
   3
   4 #include <epoxy/gl.h>
   5 #include <stdint.h>
   6
   7 extern "C" {
   8 #include <libavformat/avformat.h>
   9 #include <libavformat/avio.h>
  10 }
  11
  12 #include "frame_on_disk.h"
  13 #include "jpeg_frame_view.h"
  14 #include "queue_spot_holder.h"
  15 #include "shared/ref_counted_gl_sync.h"
  16
  17 #include <atomic>
  18 #include <chrono>
  19 #include <condition_variable>
  20 #include <deque>
  21 #include <functional>
  22 #include <movit/effect_chain.h>
  23 #include <movit/mix_effect.h>
  24 #include <movit/ycbcr_input.h>
  25 #include <mutex>
  26 #include <string>
  27 #include <thread>
  28
  29 class ChromaSubsampler;
  30 class DISComputeFlow;
  31 class Interpolate;
  32 class Mux;
  33 class QSurface;
  34 class QSurfaceFormat;
  35 class YCbCrConverter;
  36
  37 class VideoStream {
  38 public:
  39         VideoStream(AVFormatContext *file_avctx);  // nullptr if output to stream.
  40         ~VideoStream();
  41         void start();
  42         void stop();
  43         void clear_queue();
  44
  45         // “display_func” is called after the frame has been calculated (if needed)
  46         // and has gone out to the stream.
  47         void schedule_original_frame(std::chrono::steady_clock::time_point,
  48                                      int64_t output_pts, std::function<void()> &&display_func,
  49                                      QueueSpotHolder &&queue_spot_holder,
  50                                      FrameOnDisk frame, const std::string &subtitle);
  51         void schedule_faded_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
  52                                   std::function<void()> &&display_func,
  53                                   QueueSpotHolder &&queue_spot_holder,
  54                                   FrameOnDisk frame1, FrameOnDisk frame2,
  55                                   float fade_alpha, const std::string &subtitle);
  56         void schedule_interpolated_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
  57                                          std::function<void(std::shared_ptr<Frame>)> &&display_func,
  58                                          QueueSpotHolder &&queue_spot_holder,
  59                                          FrameOnDisk frame1, FrameOnDisk frame2,
  60                                          float alpha, FrameOnDisk secondary_frame,  // Empty = no secondary (fade) frame.
  61                                          float fade_alpha, const std::string &subtitle);
  62         void schedule_refresh_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
  63                                     std::function<void()> &&display_func,
  64                                     QueueSpotHolder &&queue_spot_holder, const std::string &subtitle);
  65
  66 private:
  67         FrameReader frame_reader;
  68
  69         void encode_thread_func();
  70         std::thread encode_thread;
  71         std::atomic<bool> should_quit{ false };
  72
  73         static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
  74         int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
  75
  76         // Allocated at the very start; if we're empty, we start dropping frames
  77         // (so that we don't build up an infinite interpolation backlog).
  78         struct InterpolatedFrameResources {
  79                 VideoStream *owner;  // Used only for IFRReleaser, below.
  80
  81                 GLuint input_tex;  // Layered (contains both input frames), Y'CbCr.
  82                 GLuint gray_tex;  // Same, but Y only.
  83                 GLuint input_fbos[2];  // For rendering to the two layers of input_tex.
  84
  85                 // Destination textures and FBO if there is a fade.
  86                 GLuint fade_y_output_tex, fade_cbcr_output_tex;
  87                 GLuint fade_fbo;
  88
  89                 GLuint cb_tex, cr_tex;  // Subsampled, final output.
  90
  91                 GLuint pbo;  // For reading the data back.
  92                 void *pbo_contents;  // Persistently mapped.
  93         };
  94         std::mutex queue_lock;
  95         std::deque<std::unique_ptr<InterpolatedFrameResources>> interpolate_resources;  // Under <queue_lock>.
  96         static constexpr size_t num_interpolate_slots = 15;  // Should be larger than Player::max_queued_frames, or we risk mass-dropping frames.
  97
  98         struct IFRReleaser {
  99                 void operator()(InterpolatedFrameResources *ifr) const
 100                 {
 101                         if (ifr != nullptr) {
 102                                 std::lock_guard<std::mutex> lock(ifr->owner->queue_lock);
 103                                 ifr->owner->interpolate_resources.emplace_back(ifr);
 104                         }
 105                 }
 106         };
 107         using BorrowedInterpolatedFrameResources = std::unique_ptr<InterpolatedFrameResources, IFRReleaser>;
 108
 109         struct QueuedFrame {
 110                 std::chrono::steady_clock::time_point local_pts;
 111
 112                 int64_t output_pts;
 113                 enum Type { ORIGINAL, FADED, INTERPOLATED, FADED_INTERPOLATED, REFRESH } type;
 114
 115                 // For original frames only. Made move-only so we know explicitly
 116                 // we don't copy these ~200 kB files around inadvertedly.
 117                 //
 118                 // TODO: Consider using vector<uint8_t> instead, so we save one copy.
 119                 std::unique_ptr<std::string> encoded_jpeg;
 120
 121                 // For everything except original frames.
 122                 FrameOnDisk frame1;
 123
 124                 // For fades only (including fades against interpolated frames).
 125                 FrameOnDisk secondary_frame;
 126
 127                 // For interpolated frames only.
 128                 FrameOnDisk frame2;
 129                 float alpha;
 130                 BorrowedInterpolatedFrameResources resources;
 131                 RefCountedGLsync fence;  // Set when the interpolated image is read back to the CPU.
 132                 GLuint flow_tex, output_tex, cbcr_tex;  // Released in the receiving thread; not really used for anything else. flow_tex will typically even be from a previous frame.
 133                 FrameOnDisk id;
 134
 135                 std::function<void()> display_func;  // Called when the image is done decoding.
 136                 std::function<void(std::shared_ptr<Frame>)> display_decoded_func;  // Same, except for INTERPOLATED and FADED_INTERPOLATED.
 137
 138                 std::string subtitle;  // Blank for none.
 139
 140                 QueueSpotHolder queue_spot_holder;
 141         };
 142         std::deque<QueuedFrame> frame_queue;  // Under <queue_lock>.
 143         std::condition_variable queue_changed;
 144
 145         AVFormatContext *avctx;
 146         std::unique_ptr<Mux> mux;  // To HTTP, or to file.
 147         std::string stream_mux_header;  // Only used in HTTP.
 148         bool seen_sync_markers = false;
 149         bool output_fast_forward;
 150
 151         std::unique_ptr<YCbCrConverter> ycbcr_converter;
 152         std::unique_ptr<YCbCrConverter> ycbcr_semiplanar_converter;
 153
 154         // Frame interpolation.
 155         std::unique_ptr<DISComputeFlow> compute_flow;
 156         std::unique_ptr<Interpolate> interpolate, interpolate_no_split;
 157         std::unique_ptr<ChromaSubsampler> chroma_subsampler;
 158
 159         // Cached flow computation from previous frame, if any.
 160         GLuint last_flow_tex = 0;
 161         FrameOnDisk last_frame1, last_frame2;
 162
 163         std::vector<uint8_t> last_frame;
 164 };
 165
 166 #endif  // !defined(_VIDEO_STREAM_H)