Fixed conversion from RGB to full range BT.709 YCbCr

This was a typo in the formula. The correct formula is as listed on https://en.wikipedia.org/wiki/YCbCr. Without this change, white comes out as 253,255,255.
matte-schwartz · Oct 11, 2024 · 06a6225 · 06a6225
1 parent 3fad657
commit 06a6225
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/rendervulkan.cpp b/src/rendervulkan.cpp
@@ -78,7 +78,7 @@ static constexpr mat3x4 g_rgb2yuv_srgb_to_bt709_limited = {{
 static constexpr mat3x4 g_rgb2yuv_srgb_to_bt709_full = {{
   { 0.2126f, 0.7152f, 0.0722f, 0.0f },
   { -0.1146f, -0.3854f, 0.5000f, 0.5f },
-  { 0.5000f, -0.4542f, -0.0468f, 0.5f },
+  { 0.5000f, -0.4542f, -0.0458f, 0.5f },
 }};
 
 static const mat3x4& colorspace_to_conversion_from_srgb_matrix(EStreamColorspace colorspace) {