1 #include "video_stream.h"
4 #include <libavformat/avformat.h>
5 #include <libavformat/avio.h>
13 #include "jpeg_frame_view.h"
14 #include "movit/util.h"
19 #include <epoxy/glx.h>
23 extern HTTPD *global_httpd;
27 string read_file(const string &filename)
29 FILE *fp = fopen(filename.c_str(), "rb");
31 perror(filename.c_str());
35 fseek(fp, 0, SEEK_END);
41 fread(&ret[0], len, 1, fp);
48 VideoStream::VideoStream()
50 using namespace movit;
51 // TODO: deduplicate code against JPEGFrameView?
52 ycbcr_convert_chain.reset(new EffectChain(1280, 720));
53 ImageFormat image_format;
54 image_format.color_space = COLORSPACE_sRGB;
55 image_format.gamma_curve = GAMMA_sRGB;
56 ycbcr_format.luma_coefficients = YCBCR_REC_709;
57 ycbcr_format.full_range = false;
58 ycbcr_format.num_levels = 256;
59 ycbcr_format.chroma_subsampling_x = 2;
60 ycbcr_format.chroma_subsampling_y = 1;
61 ycbcr_format.cb_x_position = 0.0f; // H.264 -- _not_ JPEG, even though our input is MJPEG-encoded
62 ycbcr_format.cb_y_position = 0.5f; // Irrelevant.
63 ycbcr_format.cr_x_position = 0.0f;
64 ycbcr_format.cr_y_position = 0.5f;
65 ycbcr_input = (movit::YCbCrInput *)ycbcr_convert_chain->add_input(new YCbCrInput(image_format, ycbcr_format, 1280, 720));
67 ImageFormat inout_format;
68 inout_format.color_space = COLORSPACE_sRGB;
69 inout_format.gamma_curve = GAMMA_sRGB;
72 ycbcr_convert_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
74 ycbcr_convert_chain->set_dither_bits(8);
76 ycbcr_convert_chain->finalize();
79 GLuint input_tex[num_interpolate_slots], gray_tex[num_interpolate_slots];
80 glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, input_tex);
81 glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, gray_tex);
83 constexpr size_t width = 1280, height = 720; // FIXME: adjustable width, height
84 int levels = find_num_levels(width, height);
85 for (size_t i = 0; i < num_interpolate_slots; ++i) {
86 glTextureStorage3D(input_tex[i], levels, GL_RGBA8, width, height, 2);
88 glTextureStorage3D(gray_tex[i], levels, GL_R8, width, height, 2);
91 InterpolatedFrameResources resource;
92 resource.input_tex = input_tex[i];
93 resource.gray_tex = gray_tex[i];
94 glCreateFramebuffers(2, resource.input_fbos);
97 glNamedFramebufferTextureLayer(resource.input_fbos[0], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 0);
99 glNamedFramebufferTextureLayer(resource.input_fbos[1], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 1);
102 GLuint buf = GL_COLOR_ATTACHMENT0;
103 glNamedFramebufferDrawBuffers(resource.input_fbos[0], 1, &buf);
105 glNamedFramebufferDrawBuffers(resource.input_fbos[1], 1, &buf);
108 glCreateBuffers(1, &resource.pbo);
110 glNamedBufferStorage(resource.pbo, width * height * 4, nullptr, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
112 resource.pbo_contents = glMapNamedBufferRange(resource.pbo, 0, width * height * 4, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
113 interpolate_resources.push_back(resource);
118 compute_flow.reset(new DISComputeFlow(width, height, operating_point3));
119 gray.reset(new GrayscaleConversion); // NOTE: Must come after DISComputeFlow, since it sets up the VBO!
120 interpolate.reset(new Interpolate(width, height, operating_point3));
124 VideoStream::~VideoStream() {}
126 void VideoStream::start()
128 AVFormatContext *avctx = avformat_alloc_context();
129 avctx->oformat = av_guess_format("nut", nullptr, nullptr);
131 uint8_t *buf = (uint8_t *)av_malloc(MUX_BUFFER_SIZE);
132 avctx->pb = avio_alloc_context(buf, MUX_BUFFER_SIZE, 1, this, nullptr, nullptr, nullptr);
133 avctx->pb->write_data_type = &VideoStream::write_packet2_thunk;
134 avctx->pb->ignore_boundary_point = 1;
136 Mux::Codec video_codec = Mux::CODEC_MJPEG;
138 avctx->flags = AVFMT_FLAG_CUSTOM_IO;
140 string video_extradata;
142 constexpr int width = 1280, height = 720; // Doesn't matter for MJPEG.
143 stream_mux.reset(new Mux(avctx, width, height, video_codec, video_extradata, /*audio_codec_parameters=*/nullptr, COARSE_TIMEBASE,
144 /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, {}));
147 encode_thread = thread(&VideoStream::encode_thread_func, this);
150 void VideoStream::stop()
152 encode_thread.join();
155 void VideoStream::schedule_original_frame(int64_t output_pts, unsigned stream_idx, int64_t input_pts)
158 qf.type = QueuedFrame::ORIGINAL;
159 qf.output_pts = output_pts;
160 qf.stream_idx = stream_idx;
161 qf.input_first_pts = input_pts;
163 unique_lock<mutex> lock(queue_lock);
164 frame_queue.push_back(qf);
165 queue_nonempty.notify_all();
168 void VideoStream::schedule_interpolated_frame(int64_t output_pts, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha)
170 // Get the temporary OpenGL resources we need for doing the interpolation.
171 InterpolatedFrameResources resources;
173 unique_lock<mutex> lock(queue_lock);
174 if (interpolate_resources.empty()) {
175 fprintf(stderr, "WARNING: Too many interpolated frames already in transit; dropping one.\n");
178 resources = interpolate_resources.front();
179 interpolate_resources.pop_front();
183 qf.type = QueuedFrame::INTERPOLATED;
184 qf.output_pts = output_pts;
185 qf.stream_idx = stream_idx;
186 qf.resources = resources;
190 // Convert frame0 and frame1 to OpenGL textures.
191 // TODO: Deduplicate against JPEGFrameView::setDecodedFrame?
192 for (size_t frame_no = 0; frame_no < 2; ++frame_no) {
193 shared_ptr<Frame> frame = decode_jpeg(filename_for_frame(stream_idx, frame_no == 1 ? input_second_pts : input_first_pts));
194 ycbcr_format.chroma_subsampling_x = frame->chroma_subsampling_x;
195 ycbcr_format.chroma_subsampling_y = frame->chroma_subsampling_y;
196 ycbcr_input->change_ycbcr_format(ycbcr_format);
197 ycbcr_input->set_width(frame->width);
198 ycbcr_input->set_height(frame->height);
199 ycbcr_input->set_pixel_data(0, frame->y.get());
200 ycbcr_input->set_pixel_data(1, frame->cb.get());
201 ycbcr_input->set_pixel_data(2, frame->cr.get());
202 ycbcr_input->set_pitch(0, frame->pitch_y);
203 ycbcr_input->set_pitch(1, frame->pitch_chroma);
204 ycbcr_input->set_pitch(2, frame->pitch_chroma);
205 ycbcr_convert_chain->render_to_fbo(resources.input_fbos[frame_no], 1280, 720);
208 glGenerateTextureMipmap(resources.input_tex);
210 // Compute the interpolated frame.
212 gray->exec(resources.input_tex, resources.gray_tex, 1280, 720, /*num_layers=*/2);
214 glGenerateTextureMipmap(resources.gray_tex);
216 GLuint flow_tex = compute_flow->exec(resources.gray_tex, DISComputeFlow::FORWARD_AND_BACKWARD, DISComputeFlow::DO_NOT_RESIZE_FLOW);
219 qf.output_tex = interpolate->exec(resources.input_tex, flow_tex, 1280, 720, alpha);
222 // Read it down (asynchronously) to the CPU.
223 glPixelStorei(GL_PACK_ROW_LENGTH, 0);
224 glBindBuffer(GL_PIXEL_PACK_BUFFER, resources.pbo);
226 glGetTextureImage(qf.output_tex, 0, GL_RGBA, GL_UNSIGNED_BYTE, 1280 * 720 * 4, nullptr);
228 glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
230 // Set a fence we can wait for to make sure the CPU sees the read.
231 glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
233 qf.fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
236 unique_lock<mutex> lock(queue_lock);
237 frame_queue.push_back(qf);
238 queue_nonempty.notify_all();
241 void VideoStream::encode_thread_func()
243 QSurface *surface = create_surface();
244 QOpenGLContext *context = create_context(surface);
245 bool ok = make_current(context, surface);
247 fprintf(stderr, "Video stream couldn't get an OpenGL context\n");
254 unique_lock<mutex> lock(queue_lock);
255 queue_nonempty.wait(lock, [this]{
256 return !frame_queue.empty();
258 qf = frame_queue.front();
259 frame_queue.pop_front();
262 if (qf.type == QueuedFrame::ORIGINAL) {
263 // Send the JPEG frame on, unchanged.
264 string jpeg = read_file(filename_for_frame(qf.stream_idx, qf.input_first_pts));
266 av_init_packet(&pkt);
267 pkt.stream_index = 0;
268 pkt.data = (uint8_t *)jpeg.data();
269 pkt.size = jpeg.size();
270 stream_mux->add_packet(pkt, qf.output_pts, qf.output_pts);
271 } else if (qf.type == QueuedFrame::INTERPOLATED) {
272 glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED);
274 // DEBUG: Writing the frame to disk.
275 FILE *fp = fopen("inter.ppm", "wb");
276 fprintf(fp, "P6\n%d %d\n255\n", 1280, 720);
277 for (size_t y = 0; y < 720; ++y) {
278 const uint8_t *ptr = (uint8_t *)qf.resources.pbo_contents + (719 - y) * 1280 * 4;
279 for (size_t x = 0; x < 1280; ++x) {
287 // TODO: Release flow and output textures.
289 // Put the frame resources back.
290 unique_lock<mutex> lock(queue_lock);
291 interpolate_resources.push_back(qf.resources);
296 int VideoStream::write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time)
298 VideoStream *video_stream = (VideoStream *)opaque;
299 return video_stream->write_packet2(buf, buf_size, type, time);
302 int VideoStream::write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time)
304 if (type == AVIO_DATA_MARKER_SYNC_POINT || type == AVIO_DATA_MARKER_BOUNDARY_POINT) {
305 seen_sync_markers = true;
306 } else if (type == AVIO_DATA_MARKER_UNKNOWN && !seen_sync_markers) {
307 // We don't know if this is a keyframe or not (the muxer could
308 // avoid marking it), so we just have to make the best of it.
309 type = AVIO_DATA_MARKER_SYNC_POINT;
312 if (type == AVIO_DATA_MARKER_HEADER) {
313 stream_mux_header.append((char *)buf, buf_size);
314 global_httpd->set_header(stream_mux_header);
316 global_httpd->add_data((char *)buf, buf_size, type == AVIO_DATA_MARKER_SYNC_POINT, time, AVRational{ AV_TIME_BASE, 1 });