Use subsuite conclusion for suite failure checks (#73)

projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py

@@ -48,7 +48,7 @@ class SubSuite(OpenTelemetryBase):  # pylint:disable=too-many-instance-attribute
     """Handle test results and tracking of a single sub suite."""
 
     released = False
-    failed = False
+    test_start_exception_caught = False
 
     def __init__(self, etos: ETOS, environment: dict, main_suite_id: str) -> None:
         """Initialize a sub suite."""
@@ -79,6 +79,24 @@ def started(self) -> bool:
                     break
         return bool(self.test_suite_started)
 
+    @property
+    def failed(self) -> bool:
+        """Whether or not this sub suite has failed."""
+        if self.test_start_exception_caught:
+            return True
+        try:
+            conclusion = self.outcome()["conclusion"]
+        except KeyError:
+            msg = (
+                f"Failed to get conclusion for {self.environment.get('name')}. "
+                "Main suite verdict/conclusion may be incorrect!"
+            )
+            exc = RuntimeError(msg)
+            self.logger.error(msg)
+            self._record_exception(exc)
+            conclusion = ""
+        return conclusion != "SUCCESSFUL"
+
     def request_finished_event(self) -> None:
         """Request a test suite finished event for this sub suite."""
         # Prevent ER requests if we know we're not even started.
@@ -116,7 +134,7 @@ def _start(self, identifier: str) -> None:
             try:
                 executor.run_tests(self.environment)
             except TestStartException as exception:
-                self.failed = True
+                self.test_start_exception_caught = True
                 self.logger.error(
                     "Failed to start sub suite: %s", exception.error, extra={"user_log": True}
                 )