git.sesse.net Git - movit/blob - resample_effect.h

   1 #ifndef _MOVIT_RESAMPLE_EFFECT_H
   2 #define _MOVIT_RESAMPLE_EFFECT_H 1
   3
   4 // High-quality image resizing, either up or down.
   5 //
   6 // The default scaling offered by the GPU (and as used in ResizeEffect)
   7 // is bilinear (optionally mipmapped), which is not the highest-quality
   8 // choice, especially for upscaling. ResampleEffect offers the three-lobed
   9 // Lanczos kernel, which is among the most popular choices in image
  10 // processing. While it does have its weaknesses, in particular a certain
  11 // ringing/sharpening effect with artifacts that accumulate over several
  12 // consecutive resizings, it is generally regarded as the best tradeoff.
  13 //
  14 // Works in two passes; first horizontal, then vertical (ResampleEffect,
  15 // which is what the user is intended to use, instantiates two copies of
  16 // SingleResamplePassEffect behind the scenes).
  17
  18 #include <epoxy/gl.h>
  19 #include <assert.h>
  20 #include <stddef.h>
  21 #include <memory>
  22 #include <string>
  23
  24 #include "effect.h"
  25 #include "fp16.h"
  26
  27 namespace movit {
  28
  29 class EffectChain;
  30 class Node;
  31 class SingleResamplePassEffect;
  32 class ResampleComputeEffect;
  33
  34 // Public so that it can be benchmarked externally.
  35 template<class T>
  36 struct Tap {
  37         T weight;
  38         T pos;
  39 };
  40 struct ScalingWeights {
  41         unsigned src_bilinear_samples;
  42         unsigned dst_samples, num_loops;
  43         int int_radius;  // FIXME: really here?
  44         float scaling_factor;  // FIXME: really here?
  45
  46         // Exactly one of these three is set.
  47         std::unique_ptr<Tap<fp16_int_t>[]> bilinear_weights_fp16;
  48         std::unique_ptr<Tap<float>[]> bilinear_weights_fp32;
  49         std::unique_ptr<fp16_int_t[]> raw_weights;
  50 };
  51 enum class BilinearFormatConstraints {
  52         ALLOW_FP16_AND_FP32,
  53         ALLOW_FP32_ONLY
  54 };
  55 ScalingWeights calculate_bilinear_scaling_weights(unsigned src_size, unsigned dst_size, float zoom, float offset, BilinearFormatConstraints constraints);
  56 ScalingWeights calculate_raw_scaling_weights(unsigned src_size, unsigned dst_size, float zoom, float offset);
  57
  58 // A simple manager for support data stored in a 2D texture.
  59 // Consider moving it to a shared location of more classes
  60 // should need similar functionality.
  61 class Support2DTexture {
  62 public:
  63         Support2DTexture();
  64         ~Support2DTexture();
  65
  66         void update(GLint width, GLint height, GLenum internal_format, GLenum format, GLenum type, const GLvoid * data);
  67         GLint get_texnum() const { return texnum; }
  68
  69 private:
  70         GLuint texnum = 0;
  71         GLint last_texture_width = -1, last_texture_height = -1;
  72         GLenum last_texture_internal_format = GL_INVALID_ENUM;
  73 };
  74
  75 class ResampleEffect : public Effect {
  76 public:
  77         ResampleEffect();
  78         ~ResampleEffect();
  79
  80         std::string effect_type_id() const override { return "ResampleEffect"; }
  81
  82         void inform_input_size(unsigned input_num, unsigned width, unsigned height) override;
  83
  84         std::string output_fragment_shader() override {
  85                 assert(false);
  86         }
  87         void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) override {
  88                 assert(false);
  89         }
  90
  91         void rewrite_graph(EffectChain *graph, Node *self) override;
  92         bool set_float(const std::string &key, float value) override;
  93
  94 private:
  95         void update_size();
  96         void update_offset_and_zoom();
  97
  98         // If compute shaders are supported, contains the effect.
  99         // If not, nullptr.
 100         std::unique_ptr<ResampleComputeEffect> compute_effect_owner;
 101         ResampleComputeEffect *compute_effect = nullptr;
 102
 103         // Both of these are owned by us if owns_effects is true (before finalize()),
 104         // and otherwise owned by the EffectChain.
 105         std::unique_ptr<SingleResamplePassEffect> hpass_owner, vpass_owner;
 106         SingleResamplePassEffect *hpass = nullptr, *vpass = nullptr;
 107
 108         int input_width, input_height, output_width, output_height;
 109
 110         float offset_x, offset_y;
 111         float zoom_x, zoom_y;
 112         float zoom_center_x, zoom_center_y;
 113 };
 114
 115 class SingleResamplePassEffect : public Effect {
 116 public:
 117         // If parent is non-nullptr, calls to inform_input_size will be forwarded,
 118         // so that it can inform both passes about the right input and output
 119         // resolutions.
 120         SingleResamplePassEffect(ResampleEffect *parent);
 121         ~SingleResamplePassEffect();
 122         std::string effect_type_id() const override { return "SingleResamplePassEffect"; }
 123
 124         std::string output_fragment_shader() override;
 125
 126         bool needs_texture_bounce() const override { return true; }
 127         bool needs_srgb_primaries() const override { return false; }
 128         AlphaHandling alpha_handling() const override { return INPUT_PREMULTIPLIED_ALPHA_KEEP_BLANK; }
 129
 130         // We specifically do not want mipmaps on the input texture;
 131         // they break minification.
 132         MipmapRequirements needs_mipmaps() const override { return CANNOT_ACCEPT_MIPMAPS; }
 133
 134         void inform_added(EffectChain *chain) override { this->chain = chain; }
 135         void inform_input_size(unsigned input_num, unsigned width, unsigned height) override {
 136                 if (parent != nullptr) {
 137                         parent->inform_input_size(input_num, width, height);
 138                 }
 139         }
 140         bool changes_output_size() const override { return true; }
 141         bool sets_virtual_output_size() const override { return false; }
 142
 143         void get_output_size(unsigned *width, unsigned *height, unsigned *virtual_width, unsigned *virtual_height) const override {
 144                 *virtual_width = *width = this->output_width;
 145                 *virtual_height = *height = this->output_height;
 146         }
 147
 148         void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) override;
 149
 150         enum Direction { HORIZONTAL = 0, VERTICAL = 1 };
 151
 152 private:
 153         void update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num);
 154
 155         ResampleEffect *parent;
 156         EffectChain *chain;
 157         Direction direction;
 158         GLint uniform_sample_tex;
 159         float uniform_num_loops, uniform_slice_height, uniform_sample_x_scale, uniform_sample_x_offset;
 160         float uniform_whole_pixel_offset;
 161         int uniform_num_samples;
 162
 163         int input_width, input_height, output_width, output_height;
 164         float offset, zoom;
 165         int last_input_width, last_input_height, last_output_width, last_output_height;
 166         float last_offset, last_zoom;
 167         int src_bilinear_samples, num_loops;
 168         float slice_height;
 169         Support2DTexture tex;
 170 };
 171
 172 class ResampleComputeEffect : public Effect {
 173 public:
 174         // If parent is non-nullptr, calls to inform_input_size will be forwarded,
 175         // so that it can inform both passes about the right input and output
 176         // resolutions.
 177         ResampleComputeEffect(ResampleEffect *parent);
 178         ~ResampleComputeEffect();
 179         std::string effect_type_id() const override { return "ResampleComputeEffect"; }
 180
 181         std::string output_fragment_shader() override;
 182
 183         // FIXME: This is the primary reason why this doesn't really work;
 184         // there's no good reason why the regular resize should have bounce
 185         // but we shouldn't. (If we did a 2D block instead of 1D columns,
 186         // it would have been different, but we can't, due to the large size
 187         // of the fringe.)
 188         bool needs_texture_bounce() const override { return false; }
 189         bool needs_srgb_primaries() const override { return false; }
 190         AlphaHandling alpha_handling() const override { return INPUT_PREMULTIPLIED_ALPHA_KEEP_BLANK; }
 191
 192         // We specifically do not want mipmaps on the input texture;
 193         // they break minification.
 194         MipmapRequirements needs_mipmaps() const override { return CANNOT_ACCEPT_MIPMAPS; }
 195
 196         void inform_added(EffectChain *chain) override { this->chain = chain; }
 197         void inform_input_size(unsigned input_num, unsigned width, unsigned height) override {
 198                 if (parent != nullptr) {
 199                         parent->inform_input_size(input_num, width, height);
 200                 }
 201         }
 202         bool changes_output_size() const override { return true; }
 203         bool sets_virtual_output_size() const override { return false; }
 204
 205         void get_output_size(unsigned *width, unsigned *height, unsigned *virtual_width, unsigned *virtual_height) const override {
 206                 *virtual_width = *width = this->output_width;
 207                 *virtual_height = *height = this->output_height;
 208         }
 209
 210         bool is_compute_shader() const override { return true; }
 211         void get_compute_dimensions(unsigned output_width, unsigned output_height,
 212                                     unsigned *x, unsigned *y, unsigned *z) const override;
 213
 214         void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) override;
 215
 216 private:
 217         void update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num);
 218
 219         ResampleEffect *parent;
 220         EffectChain *chain;
 221         Support2DTexture tex_horiz, tex_vert;
 222         GLint uniform_sample_tex_horizontal, uniform_sample_tex_vertical;
 223         float uniform_num_x_loops;
 224         int uniform_num_horizontal_filters, uniform_num_vertical_filters;
 225         float uniform_slice_height;
 226         float uniform_horizontal_whole_pixel_offset;
 227         int uniform_vertical_whole_pixel_offset;
 228         int uniform_num_horizontal_samples, uniform_num_vertical_samples;
 229         int uniform_output_samples_per_block;
 230
 231         int input_width, input_height, output_width, output_height;
 232         float offset_x, offset_y, zoom_x, zoom_y;
 233         int last_input_width, last_input_height, last_output_width, last_output_height;
 234         float last_offset_x, last_offset_y, last_zoom_x, last_zoom_y;
 235         int src_horizontal_bilinear_samples;  // Horizontal.
 236         int src_vertical_samples;
 237         float slice_height;
 238         float uniform_inv_input_height, uniform_input_texcoord_y_adjust;
 239         int uniform_vertical_int_radius;
 240         float vertical_scaling_factor;
 241         float uniform_inv_vertical_scaling_factor;
 242 };
 243
 244 }  // namespace movit
 245
 246 #endif // !defined(_MOVIT_RESAMPLE_EFFECT_H)