]> git.sesse.net Git - nageru/blob - futatabi/video_stream.h
Reintroduce faster DeckLink shutdown; now with a fix for the UI switcher.
[nageru] / futatabi / video_stream.h
1 #ifndef _VIDEO_STREAM_H
2 #define _VIDEO_STREAM_H 1
3
4 #include <epoxy/gl.h>
5 #include <stdint.h>
6
7 extern "C" {
8 #include <libavformat/avformat.h>
9 #include <libavformat/avio.h>
10 }
11
12 #include "frame_on_disk.h"
13 #include "jpeg_frame_view.h"
14 #include "queue_spot_holder.h"
15 #include "shared/mux.h"
16 #include "shared/ref_counted_gl_sync.h"
17
18 #include <atomic>
19 #include <chrono>
20 #include <condition_variable>
21 #include <deque>
22 #include <functional>
23 #include <movit/effect_chain.h>
24 #include <movit/mix_effect.h>
25 #include <movit/ycbcr_input.h>
26 #include <mutex>
27 #include <string>
28 #include <thread>
29
30 class ChromaSubsampler;
31 class DISComputeFlow;
32 class Interpolate;
33 class QSurface;
34 class QSurfaceFormat;
35 class YCbCrConverter;
36
37 class VideoStream {
38 public:
39         VideoStream(AVFormatContext *file_avctx);  // nullptr if output to stream.
40         ~VideoStream();
41         void start();
42         void stop();
43         void clear_queue();
44
45         // “display_func” is called after the frame has been calculated (if needed)
46         // and has gone out to the stream.
47         void schedule_original_frame(std::chrono::steady_clock::time_point,
48                                      int64_t output_pts, std::function<void()> &&display_func,
49                                      QueueSpotHolder &&queue_spot_holder,
50                                      FrameOnDisk frame, const std::string &subtitle,
51                                      bool include_audio);
52         void schedule_faded_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
53                                   std::function<void()> &&display_func,
54                                   QueueSpotHolder &&queue_spot_holder,
55                                   FrameOnDisk frame1, FrameOnDisk frame2,
56                                   float fade_alpha, const std::string &subtitle);  // Always no audio.
57         void schedule_interpolated_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
58                                          std::function<void(std::shared_ptr<Frame>)> &&display_func,
59                                          QueueSpotHolder &&queue_spot_holder,
60                                          FrameOnDisk frame1, FrameOnDisk frame2,
61                                          float alpha, FrameOnDisk secondary_frame,  // Empty = no secondary (fade) frame.
62                                          float fade_alpha, const std::string &subtitle,
63                                          bool include_audio);
64         void schedule_refresh_frame(std::chrono::steady_clock::time_point, int64_t output_pts,
65                                     std::function<void()> &&display_func,
66                                     QueueSpotHolder &&queue_spot_holder, const std::string &subtitle);  // Always no audio.
67         void schedule_silence(std::chrono::steady_clock::time_point, int64_t output_pts,
68                               int64_t length_pts, QueueSpotHolder &&queue_spot_holder);
69
70 private:
71         struct QueuedFrame;
72
73         FrameReader frame_reader;
74
75         void encode_thread_func();
76         std::thread encode_thread;
77         std::atomic<bool> should_quit{ false };
78
79         static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
80         int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
81         void add_silence(int64_t pts, int64_t length_pts);
82         void add_audio_or_silence(const QueuedFrame &qf);
83
84         // Allocated at the very start; if we're empty, we start dropping frames
85         // (so that we don't build up an infinite interpolation backlog).
86         struct InterpolatedFrameResources {
87                 VideoStream *owner;  // Used only for IFRReleaser, below.
88
89                 GLuint input_tex;  // Layered (contains both input frames), Y'CbCr.
90                 GLuint gray_tex;  // Same, but Y only.
91                 GLuint input_fbos[2];  // For rendering to the two layers of input_tex.
92
93                 // Destination textures and FBO if there is a fade.
94                 GLuint fade_y_output_tex, fade_cbcr_output_tex;
95                 GLuint fade_fbo;
96
97                 GLuint cb_tex, cr_tex;  // Subsampled, final output.
98
99                 GLuint pbo;  // For reading the data back.
100                 void *pbo_contents;  // Persistently mapped.
101         };
102         std::mutex queue_lock;
103         std::deque<std::unique_ptr<InterpolatedFrameResources>> interpolate_resources;  // Under <queue_lock>.
104         static constexpr size_t num_interpolate_slots = 15;  // Should be larger than Player::max_queued_frames, or we risk mass-dropping frames.
105
106         struct IFRReleaser {
107                 void operator()(InterpolatedFrameResources *ifr) const
108                 {
109                         if (ifr != nullptr) {
110                                 std::lock_guard<std::mutex> lock(ifr->owner->queue_lock);
111                                 ifr->owner->interpolate_resources.emplace_back(ifr);
112                         }
113                 }
114         };
115         using BorrowedInterpolatedFrameResources = std::unique_ptr<InterpolatedFrameResources, IFRReleaser>;
116
117         struct QueuedFrame {
118                 std::chrono::steady_clock::time_point local_pts;
119
120                 int64_t output_pts;
121                 enum Type { ORIGINAL, FADED, INTERPOLATED, FADED_INTERPOLATED, REFRESH, SILENCE } type;
122
123                 // For original frames only. Made move-only so we know explicitly
124                 // we don't copy these ~200 kB files around inadvertedly.
125                 std::unique_ptr<std::string> encoded_jpeg;
126
127                 // For everything except original frames and silence.
128                 FrameOnDisk frame1;
129
130                 // For fades only (including fades against interpolated frames).
131                 FrameOnDisk secondary_frame;
132
133                 // For interpolated frames only.
134                 FrameOnDisk frame2;
135                 float alpha;
136                 BorrowedInterpolatedFrameResources resources;
137                 RefCountedGLsync fence;  // Set when the interpolated image is read back to the CPU.
138                 std::chrono::steady_clock::time_point fence_created;
139                 GLuint flow_tex, output_tex, cbcr_tex;  // Released in the receiving thread; not really used for anything else. flow_tex will typically even be from a previous frame.
140                 FrameOnDisk id;
141
142                 std::function<void()> display_func;  // Called when the image is done decoding.
143                 std::function<void(std::shared_ptr<Frame>)> display_decoded_func;  // Same, except for INTERPOLATED and FADED_INTERPOLATED.
144
145                 std::string subtitle;  // Blank for none.
146                 std::string exif_data;  // Blank for none.
147
148                 // Audio, in stereo interleaved 32-bit PCM. If empty and not of type SILENCE, one frame's worth of silence samples
149                 // is synthesized.
150                 std::string audio;
151
152                 // For silence frames only.
153                 int64_t silence_length_pts;
154
155                 QueueSpotHolder queue_spot_holder;
156         };
157         std::deque<QueuedFrame> frame_queue;  // Under <queue_lock>.
158         std::condition_variable queue_changed;
159
160         AVFormatContext *avctx;
161         std::unique_ptr<Mux> mux;  // To HTTP, or to file.
162         std::string stream_mux_header;  // Only used in HTTP.
163         bool seen_sync_markers = false;
164         bool output_fast_forward;
165
166         std::unique_ptr<YCbCrConverter> ycbcr_converter;
167         std::unique_ptr<YCbCrConverter> ycbcr_semiplanar_converter;
168
169         // Frame interpolation.
170         std::unique_ptr<DISComputeFlow> compute_flow;
171         std::unique_ptr<Interpolate> interpolate, interpolate_no_split;
172         std::unique_ptr<ChromaSubsampler> chroma_subsampler;
173
174         // Cached flow computation from previous frame, if any.
175         GLuint last_flow_tex = 0;
176         FrameOnDisk last_frame1, last_frame2;
177
178         std::string last_frame;
179         Mux::WithSubtitles with_subtitles;  // true for streaming, false for export to file.
180 };
181
182 #endif  // !defined(_VIDEO_STREAM_H)