// Given <ycbcr_format>, compute the values needed to turn Y'CbCr into R'G'B';
// first subtract the returned offset, then left-multiply the returned matrix
// (the scaling is already folded into it).
-void compute_ycbcr_matrix(YCbCrFormat ycbcr_format, float* offset, Matrix3d* ycbcr_to_rgb)
+void compute_ycbcr_matrix(YCbCrFormat ycbcr_format, float* offset, Matrix3d* ycbcr_to_rgb, GLenum type, double *scale_factor)
{
double coeff[3], scale[3];
// Fold in the scaling.
*ycbcr_to_rgb *= Map<const Vector3d>(scale).asDiagonal();
+
+ if (type == GL_UNSIGNED_SHORT) {
+ // For 10-bit or 12-bit packed into 16-bit, we need to scale the values
+ // so that the max value goes from 1023 (or 4095) to 65535. We do this
+ // by folding the scaling into the conversion matrix, so it comes essentially
+ // for free. However, the offset is before the scaling (and thus assumes
+ // correctly scaled values), so we need to adjust that the other way.
+ double scale = 65535.0 / (ycbcr_format.num_levels - 1);
+ offset[0] /= scale;
+ offset[1] /= scale;
+ offset[2] /= scale;
+ *ycbcr_to_rgb *= scale;
+ if (scale_factor != nullptr) {
+ *scale_factor = scale;
+ }
+ } else if (scale_factor != nullptr) {
+ *scale_factor = 1.0;
+ }
}
} // namespace movit