Fix a bug where combined fp16 weights would be horribly wrong.

[movit] / resample_effect.cpp

diff --git a/resample_effect.cpp b/resample_effect.cpp
index 79d5f21c45543a936f81e33a4ead19ca32d5a74f..244a3e2a6081187f19962b531c7fa86dde46203b 100644 (file)
--- a/resample_effect.cpp
+++ b/resample_effect.cpp
@@ -107,7 +107,7 @@ unsigned combine_samples(const Tap<float> *src, Tap<DestFloat> *dst, float num_s
                float pos2 = src[i + 1].pos;
                assert(pos2 > pos1);
 
-               fp16_int_t pos, total_weight;
+               DestFloat pos, total_weight;
                float sum_sq_error;
                combine_two_samples(w1, w2, pos1, pos2, num_subtexels, inv_num_subtexels, &pos, &total_weight, &sum_sq_error);
 
@@ -142,8 +142,9 @@ void normalize_sum(Tap<T>* vals, unsigned num)
                for (unsigned i = 0; i < num; ++i) {
                        sum += to_fp64(vals[i].weight);
                }
+               double inv_sum = 1.0 / sum;
                for (unsigned i = 0; i < num; ++i) {
-                       vals[i].weight = from_fp64<T>(to_fp64(vals[i].weight) / sum);
+                       vals[i].weight = from_fp64<T>(to_fp64(vals[i].weight) * inv_sum);
                }
        }
 }
@@ -161,14 +162,15 @@ unsigned combine_many_samples(const Tap<float> *weights, unsigned src_size, unsi
 {
        float num_subtexels = src_size / movit_texel_subpixel_precision;
        float inv_num_subtexels = movit_texel_subpixel_precision / src_size;
-       int src_bilinear_samples = 0;
 
-       for (unsigned y = 0; y < dst_samples; ++y) {
-               unsigned num_samples_saved = combine_samples<DestFloat>(weights + y * src_samples, NULL, num_subtexels, inv_num_subtexels, src_samples, UINT_MAX);
-               src_bilinear_samples = max<int>(src_bilinear_samples, src_samples - num_samples_saved);
+       unsigned max_samples_saved = UINT_MAX;
+       for (unsigned y = 0; y < dst_samples && max_samples_saved > 0; ++y) {
+               unsigned num_samples_saved = combine_samples<DestFloat>(weights + y * src_samples, NULL, num_subtexels, inv_num_subtexels, src_samples, max_samples_saved);
+               max_samples_saved = min(max_samples_saved, num_samples_saved);
        }
 
        // Now that we know the right width, actually combine the samples.
+       unsigned src_bilinear_samples = src_samples - max_samples_saved;
        *bilinear_weights = new Tap<DestFloat>[dst_samples * src_bilinear_samples];
        for (unsigned y = 0; y < dst_samples; ++y) {
                Tap<DestFloat> *bilinear_weights_ptr = *bilinear_weights + y * src_bilinear_samples;
@@ -178,8 +180,8 @@ unsigned combine_many_samples(const Tap<float> *weights, unsigned src_size, unsi
                        num_subtexels,
                        inv_num_subtexels,
                        src_samples,
-                       src_samples - src_bilinear_samples);
-               assert(int(src_samples) - int(num_samples_saved) == src_bilinear_samples);
+                       max_samples_saved);
+               assert(num_samples_saved == max_samples_saved);
                normalize_sum(bilinear_weights_ptr, src_bilinear_samples);
        }
        return src_bilinear_samples;