From 472687021ea8aa791dfdb806b6f7e6dfa3d414cc Mon Sep 17 00:00:00 2001
From: Tim Moore <tim.moore@cesium.com>
Date: Fri, 13 Dec 2024 15:23:39 +0100
Subject: [PATCH] Calculate I3dm instance rotations by converting basis to a
 matrix
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous approach tried to create rotations directly from the up
and right vectors that encode the instance rotation, but it naïvely
didn't handle rotations of 180 degrees of the up and right vector. It
is reliable to create the rotation matrix, filling in the Z column
by doing a cross-product, and then let the algorithm for converting a
rotation matrix to a quaternion do its robust thing.
---
 .../src/I3dmToGltfConverter.cpp               | 26 +++++--------------
 1 file changed, 7 insertions(+), 19 deletions(-)

diff --git a/Cesium3DTilesContent/src/I3dmToGltfConverter.cpp b/Cesium3DTilesContent/src/I3dmToGltfConverter.cpp
index beb2d3c0f..3e64bd4b8 100644
--- a/Cesium3DTilesContent/src/I3dmToGltfConverter.cpp
+++ b/Cesium3DTilesContent/src/I3dmToGltfConverter.cpp
@@ -143,29 +143,17 @@ glm::vec3 decodeOct32P(const uint16_t rawOct[2]) {
 
 /*
   Calculate the rotation quaternion described by the up, right vectors passed
-  in NORMAL_UP and NORMAL_RIGHT. This is composed of two rotations:
-   + The rotation that takes the up vector to its new position;
-   + The rotation around the new up vector that takes the right vector to its
-  new position.
+  in NORMAL_UP and NORMAL_RIGHT.
 
-  I like to think of each rotation as describing a coordinate frame. The
-  calculation of the second rotation must take place within the first frame.
-
-  The rotations are calculated by finding the rotation that takes one vector to
-  another.
+  There may be a faster method that avoids creating a rotation matrix, but it is
+  hard to get the exceptional cases correct e.g., rotations of 180 degrees about
+  an axis.
  */
 
 glm::quat rotationFromUpRight(const glm::vec3& up, const glm::vec3& right) {
-  // First rotation: up
-  auto upRot = CesiumUtility::Math::rotation(glm::vec3(0.0f, 1.0f, 0.0f), up);
-  // We can rotate a point vector by a quaternion using q * (0, v) *
-  // conj(q). But here we are doing an inverse rotation of the right vector into
-  // the "up frame."
-  glm::quat temp = glm::conjugate(upRot) * glm::quat(0.0f, right) * upRot;
-  glm::vec3 innerRight(temp.x, temp.y, temp.z);
-  glm::quat rightRot =
-      CesiumUtility::Math::rotation(glm::vec3(1.0f, 0.0f, 0.0f), innerRight);
-  return upRot * rightRot;
+  glm::vec3 forward = cross(right, up);
+  glm::mat3x3 rotMat(right, up, forward);
+  return glm::quat(rotMat);
 }
 
 struct ConvertedI3dm {