git.sesse.net Git - nageru/blob - nageru/mjpeg_encoder.h

   1 #ifndef _MJPEG_ENCODER_H
   2 #define _MJPEG_ENCODER_H 1
   3
   4 #include "defs.h"
   5 #include "shared/ffmpeg_raii.h"
   6 #include "shared/httpd.h"
   7 #include "ref_counted_frame.h"
   8
   9 extern "C" {
  10
  11 #include <libavformat/avio.h>
  12
  13 }  // extern "C"
  14
  15 #include <atomic>
  16 #include <bmusb/bmusb.h>
  17 #include <condition_variable>
  18 #include <list>
  19 #include <mutex>
  20 #include <queue>
  21 #include <stdint.h>
  22 #include <string>
  23 #include <thread>
  24
  25 #include <movit/effect.h>
  26 #include <va/va.h>
  27
  28 struct jpeg_compress_struct;
  29 struct VADisplayWithCleanup;
  30 struct VectorDestinationManager;
  31
  32 #define CHECK_VASTATUS(va_status, func)                                 \
  33     if (va_status != VA_STATUS_SUCCESS) {                               \
  34         fprintf(stderr, "%s:%d (%s) failed: %s\n", __func__, __LINE__, func, vaErrorStr(va_status)); \
  35         exit(1);                                                        \
  36     }
  37
  38 class MJPEGEncoder {
  39 public:
  40         MJPEGEncoder(HTTPD *httpd, const std::string &va_display);
  41         ~MJPEGEncoder();
  42         void stop();
  43         void upload_frame(int64_t pts, unsigned card_index, RefCountedFrame frame, const bmusb::VideoFormat &video_format, size_t y_offset, size_t cbcr_offset, std::vector<int32_t> audio, const movit::RGBTriplet &white_balance);
  44         bool using_vaapi() const { return va_dpy != nullptr; }
  45
  46         bool should_encode_mjpeg_for_card(unsigned card_index);
  47
  48 private:
  49         static constexpr int quality = 90;
  50
  51         struct VAResources {
  52                 unsigned width, height;
  53                 VASurfaceID surface;
  54                 VAContextID context;
  55                 VABufferID data_buffer;
  56                 VAImage image;
  57         };
  58
  59         // RAII wrapper to release VAResources on return (even on error).
  60         class ReleaseVAResources {
  61         public:
  62                 ReleaseVAResources() : committed(true) {}
  63
  64                 ReleaseVAResources(MJPEGEncoder *mjpeg, const VAResources &resources)
  65                         : mjpeg(mjpeg), resources(resources) {}
  66
  67                 ReleaseVAResources(ReleaseVAResources &) = delete;
  68
  69                 ReleaseVAResources(ReleaseVAResources &&other)
  70                         : mjpeg(other.mjpeg), resources(other.resources), committed(other.committed) {
  71                         other.commit();
  72                 }
  73
  74                 ReleaseVAResources &operator= (ReleaseVAResources &) = delete;
  75
  76                 ReleaseVAResources &operator= (ReleaseVAResources &&other) {
  77                         if (!committed) {
  78                                 mjpeg->release_va_resources(resources);
  79                         }
  80                         mjpeg = other.mjpeg;
  81                         resources = std::move(other.resources);
  82                         committed = other.committed;
  83                         other.commit();
  84                         return *this;
  85                 }
  86
  87                 ~ReleaseVAResources()
  88                 {
  89                         if (!committed) {
  90                                 mjpeg->release_va_resources(resources);
  91                         }
  92                 }
  93
  94                 void commit() { committed = true; }
  95
  96         private:
  97                 MJPEGEncoder *mjpeg = nullptr;
  98                 VAResources resources;
  99                 bool committed = false;
 100         };
 101
 102         struct QueuedFrame {
 103                 int64_t pts;
 104                 unsigned card_index;
 105                 RefCountedFrame frame;
 106                 bmusb::VideoFormat video_format;
 107                 size_t y_offset, cbcr_offset;
 108                 std::vector<int32_t> audio;
 109                 movit::RGBTriplet white_balance;
 110
 111                 // Only for frames in the process of being encoded by VA-API.
 112                 VAResources resources;
 113                 ReleaseVAResources resource_releaser;
 114         };
 115
 116         void encoder_thread_func();
 117         void va_receiver_thread_func();
 118         void encode_jpeg_va(QueuedFrame &&qf);
 119         std::vector<uint8_t> encode_jpeg_libjpeg(const QueuedFrame &qf);
 120         void write_mjpeg_packet(AVFormatContext *avctx, int64_t pts, unsigned stream_index, const uint8_t *jpeg, size_t jpeg_size);
 121         void write_audio_packet(AVFormatContext *avctx, int64_t pts, unsigned stream_index, const std::vector<int32_t> &audio);
 122         void init_jpeg_422(unsigned width, unsigned height, const movit::RGBTriplet &white_balance, VectorDestinationManager *dest, jpeg_compress_struct *cinfo);
 123         std::vector<uint8_t> get_jpeg_header(unsigned width, unsigned height, const movit::RGBTriplet &white_balance, jpeg_compress_struct *cinfo);
 124         void add_stream(HTTPD::StreamID stream_id);  // Can only be called from the constructor, or the thread owning <streams>.
 125         void update_siphon_streams();  // Same.
 126         void create_ffmpeg_context(HTTPD::StreamID stream_id);
 127
 128         struct WritePacket2Context {
 129                 MJPEGEncoder *mjpeg_encoder;
 130                 HTTPD::StreamID stream_id;
 131         };
 132         std::map<HTTPD::StreamID, WritePacket2Context> ffmpeg_contexts;   // Statically set up, so we never need to worry about dangling pointers.
 133         static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
 134         int write_packet2(HTTPD::StreamID stream_id, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
 135
 136         std::thread encoder_thread, va_receiver_thread;
 137
 138         std::mutex mu;
 139         std::queue<QueuedFrame> frames_to_be_encoded;  // Under mu.
 140         std::condition_variable any_frames_to_be_encoded;  // Governs changes in both frames_to_be_encoded and frames_under_encoding
 141
 142         std::queue<QueuedFrame> frames_encoding;  // Under mu. Used for VA-API only.
 143         std::condition_variable any_frames_encoding;
 144
 145         struct Stream {
 146                 AVFormatContextWithCloser avctx;
 147                 std::string mux_header;
 148         };
 149         std::map<HTTPD::StreamID, Stream> streams;  // Owned by the VA-API receiver thread if VA-API is active, or the encoder thread if not.
 150         HTTPD *httpd;
 151         std::atomic<bool> should_quit{false};
 152         bool running = false;
 153
 154         std::unique_ptr<VADisplayWithCleanup> va_dpy;
 155         VAConfigID config_id;
 156
 157         struct VAKey {
 158                 unsigned width, height;
 159                 movit::RGBTriplet white_balance;
 160
 161                 bool operator< (const VAKey &other) const {
 162                         if (width != other.width)
 163                                 return width < other.width;
 164                         if (height != other.height)
 165                                 return height < other.height;
 166                         if (white_balance.r != other.white_balance.r)
 167                                 return white_balance.r < other.white_balance.r;
 168                         if (white_balance.g != other.white_balance.g)
 169                                 return white_balance.g < other.white_balance.g;
 170                         return white_balance.b < other.white_balance.b;
 171                 }
 172         };
 173         struct VAData {
 174                 std::vector<uint8_t> jpeg_header;
 175                 VAEncPictureParameterBufferJPEG pic_param;
 176                 VAQMatrixBufferJPEG q;
 177                 VAHuffmanTableBufferJPEGBaseline huff;
 178                 VAEncSliceParameterBufferJPEG parms;
 179         };
 180         std::map<VAKey, VAData> va_data_for_parameters;
 181         VAData get_va_data_for_parameters(unsigned width, unsigned height, const movit::RGBTriplet &white_balance);
 182
 183         std::list<VAResources> va_resources_freelist;
 184         std::mutex va_resources_mutex;
 185         VAResources get_va_resources(unsigned width, unsigned height);
 186         void release_va_resources(VAResources resources);
 187
 188         static std::unique_ptr<VADisplayWithCleanup> try_open_va(const std::string &va_display, std::string *error, VAConfigID *config_id);
 189
 190         uint8_t *tmp_y, *tmp_cbcr, *tmp_cb, *tmp_cr;  // Private to the encoder thread. Used by the libjpeg backend only.
 191
 192         std::atomic<int64_t> metric_mjpeg_frames_zero_size_dropped{0};
 193         std::atomic<int64_t> metric_mjpeg_frames_interlaced_dropped{0};
 194         std::atomic<int64_t> metric_mjpeg_frames_unsupported_pixel_format_dropped{0};
 195         std::atomic<int64_t> metric_mjpeg_frames_oversized_dropped{0};
 196         std::atomic<int64_t> metric_mjpeg_overrun_dropped{0};
 197         std::atomic<int64_t> metric_mjpeg_overrun_submitted{0};
 198
 199         friend class PBOFrameAllocator;  // FIXME
 200 };
 201
 202 #endif  // !defined(_MJPEG_ENCODER_H)