}
// Round to the desired precision. Note that this might take z outside the 0..1 range.
- *offset = from_fp64<DestFloat>(pos1 + z * (pos2 - pos1));
- z = (to_fp64(*offset) - pos1) / (pos2 - pos1);
+ *offset = from_fp32<DestFloat>(pos1 + z * (pos2 - pos1));
+ z = (to_fp32(*offset) - pos1) / (pos2 - pos1);
// Round to the minimum number of bits we have measured earlier.
// The card will do this for us anyway, but if we know what the real z
// w = (a(1-z) + bz) / ((1-z)² + z²)
//
// If z had infinite precision, this would simply reduce to w = w1 + w2.
- *total_weight = from_fp64<DestFloat>((w1 + z * (w2 - w1)) / (z * z + (1 - z) * (1 - z)));
+ *total_weight = from_fp32<DestFloat>((w1 + z * (w2 - w1)) / (z * z + (1 - z) * (1 - z)));
if (sum_sq_error != NULL) {
- float err1 = to_fp64(*total_weight) * (1 - z) - w1;
- float err2 = to_fp64(*total_weight) * z - w2;
+ float err1 = to_fp32(*total_weight) * (1 - z) - w1;
+ float err2 = to_fp32(*total_weight) * z - w2;
*sum_sq_error = err1 * err1 + err2 * err2;
}
}