Add result to shutdown

projects/etos_suite_runner/src/etos_suite_runner/__main__.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 # -*- coding: utf-8 -*-
 """ETOS suite runner module."""
+
 import os
 import json
 import logging
@@ -32,35 +33,11 @@
 def main():
     """Entry point allowing external calls."""
     etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")
-    etos.config.set("results", [])
     esr = ESR(etos)
     try:
-        esr.run()  # Blocking
-        results = etos.config.get("results") or []
-        result = None
-        for suite_result in results:
-            if suite_result.get("verdict") == "FAILED":
-                result = suite_result
-                # If the verdict on any main suite is FAILED, that is the verdict we set on the
-                # test run, which means that we can break the loop early in that case.
-                break
-            if suite_result.get("verdict") == "INCONCLUSIVE":
-                result = suite_result
-        if len(results) == 0:
-            result = {
-                "conclusion": "Inconclusive",
-                "verdict": "Inconclusive",
-                "description": "Got no results from ESR",
-            }
-        elif result is None:
-            # No suite failed, so lets just pick the first result
-            result = results[0]
-        # Convert, for example, INCONCLUSIVE to Inconclusive to match the controller result struct
-        # TODO Move the result struct to ETOS library and do this conversion on creation
-        result["conclusion"] = result["conclusion"].title().replace("_", "")
-        result["verdict"] = result["verdict"].title()
+        result = esr.run()  # Blocking
         with open("/dev/termination-log", "w", encoding="utf-8") as termination_log:
-            json.dump(result, termination_log)
+            json.dump(result.model_dump(), termination_log)
         LOGGER.info("ESR result: %r", result)
     except:
         result = {

projects/etos_suite_runner/src/etos_suite_runner/esr.py

@@ -15,11 +15,13 @@
 # limitations under the License.
 # -*- coding: utf-8 -*-
 """ETOS suite runner module."""
+
 import logging
 import os
 import signal
 import time
 import threading
+import traceback
 from uuid import uuid4
 
 from eiffellib.events import (
@@ -35,6 +37,8 @@
 import opentelemetry
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
+from etos_suite_runner.lib.result import Result
+
 from .lib.esr_parameters import ESRParameters
 from .lib.exceptions import EnvironmentProviderException
 from .lib.graphql import request_tercc
@@ -206,13 +210,12 @@ def _release_environment(self) -> None:
             if not status:
                 self.logger.error(message)
 
-    def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]:
+    def run_suites(self, triggered: EiffelActivityTriggeredEvent):
         """Start up a suite runner handling multiple suites that execute within test runners.
 
         Will only start the test activity if there's a 'slot' available.
 
         :param triggered: Activity triggered.
-        :return: List of main suite IDs - Used for tests
         """
         context = triggered.meta.event_id
         self.etos.config.set("context", context)
@@ -222,6 +225,7 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]:
         ids = self.params.main_suite_ids()
         for i, suite in enumerate(self.params.test_suite):
             suites.append((ids[i], suite))
+        self.etos.config.set("ids", ids)  # Used for testing
         self.logger.info("Number of test suites to run: %d", len(suites), extra={"user_log": True})
         try:
             self.logger.info("Get test environment.")
@@ -240,7 +244,6 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]:
 
             self.logger.info("Starting ESR.")
             runner.start_suites_and_wait(suites)
-            return [id for id, _ in suites]
         except EnvironmentProviderException as exc:
             # Not running as part of the ETOS Kubernetes controller environment
             if os.getenv("IDENTIFIER") is None:
@@ -267,11 +270,8 @@ def _send_tercc(self, testrun_id: str, iut_id: str) -> None:
         }
         self.etos.events.send(event, links, data)
 
-    def _run(self) -> list[str]:
-        """Run the ESR main loop.
-
-        :return: List of test suites (main suites) that were started.
-        """
+    def _run(self):
+        """Run the ESR main loop."""
         testrun_id = None
         try:
             testrun_id = self.params.testrun_id
@@ -309,11 +309,10 @@ def _run(self) -> list[str]:
             raise
 
         try:
-            ids = self.run_suites(triggered)
+            self.run_suites(triggered)
             self.etos.events.send_activity_finished(
                 triggered, {"conclusion": "SUCCESSFUL"}, {"CONTEXT": context}
             )
-            return ids
         except Exception as exception:  # pylint:disable=broad-except
             reason = str(exception)
             self.logger.exception(
@@ -324,24 +323,56 @@ def _run(self) -> list[str]:
             self._record_exception(exception)
             raise
 
-    def run(self) -> list[str]:
+    def result(self) -> Result:
+        """ESR execution result."""
+        results = self.etos.config.get("results") or []
+        result = {}
+        for suite_result in results:
+            if suite_result.get("verdict") == "FAILED":
+                result = suite_result
+                # If the verdict on any main suite is FAILED, that is the verdict we set on the
+                # test run, which means that we can break the loop early in that case.
+                break
+            if suite_result.get("verdict") == "INCONCLUSIVE":
+                result = suite_result
+        if len(results) == 0:
+            result = {
+                "conclusion": "Inconclusive",
+                "verdict": "Inconclusive",
+                "description": "Got no results from ESR",
+            }
+        elif result == {}:
+            # No suite failed, so lets just pick the first result
+            result = results[0]
+        return Result(**result)
+
+    def run(self) -> Result:
         """Run the ESR main loop.
 
         :return: List of test suites (main suites) that were started.
         """
-        event = {
-            "event": "shutdown",
-            "data": "ESR has finished",
-        }
+        self.etos.config.set("results", [])
+        result = Result(
+            verdict="Inconclusive",
+            conclusion="Failed",
+            description="ESR did not execute",
+        )
         try:
-            return self._run()
-        except Exception as exception:  # pylint:disable=broad-except
+            self._run()
+            result = self.result()
+            return result
+        except Exception:  # pylint:disable=bare-except
+            result = Result(
+                conclusion="Failed",
+                verdict="Inconclusive",
+                description=traceback.format_exc(),
+            )
+            raise
+        finally:
             event = {
                 "event": "shutdown",
-                "data": str(exception),
+                "data": result.model_dump(),
             }
-            raise
-        finally:
             self.event_publisher.publish(event)
 
     def graceful_exit(self, *_) -> None:

projects/etos_suite_runner/src/etos_suite_runner/lib/result.py

@@ -0,0 +1,35 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS result struct."""
+
+from pydantic import BaseModel, field_validator, ValidationInfo
+
+
+# TODO Move the result struct to ETOS library.
+
+
+class Result(BaseModel):
+    """Final result of an ETOS suite run."""
+
+    verdict: str
+    conclusion: str
+    description: str
+
+    @field_validator("verdict", "conclusion")
+    @classmethod
+    def title(cls, v: str, _: ValidationInfo) -> str:
+        """Convert, for example, INCONCLUSIVE to Inconclusive to match the Kubernetes controller."""
+        return v.title()

projects/etos_suite_runner/tests/scenario/test_permutations.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Scenario tests for permutations."""
+
 import json
 import logging
 import os
@@ -33,7 +34,11 @@
 from tests.library.fake_database import FakeDatabase
 from tests.library.fake_server import FakeServer
 from tests.library.handler import Handler
-from tests.scenario.tercc import ARTIFACT_CREATED, PERMUTATION_TERCC, PERMUTATION_TERCC_SUB_SUITES
+from tests.scenario.tercc import (
+    ARTIFACT_CREATED,
+    PERMUTATION_TERCC,
+    PERMUTATION_TERCC_SUB_SUITES,
+)
 
 IUT_PROVIDER = {
     "iut": {
@@ -125,7 +130,8 @@ def setup_providers(self):
         """Setup providers in the fake ETCD database."""
         Config().get("database").put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER))
         Config().get("database").put(
-            "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER)
+            "/environment/provider/execution-space/default",
+            json.dumps(EXECUTION_SPACE_PROVIDER),
         )
         Config().get("database").put(
             "/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER)
@@ -146,7 +152,8 @@ def register_providers(self, testrun_id, host):
             f"/testrun/{testrun_id}/provider/log-area", json.dumps(LOG_AREA_PROVIDER)
         )
         Config().get("database").put(
-            f"/testrun/{testrun_id}/provider/execution-space", json.dumps(EXECUTION_SPACE_PROVIDER)
+            f"/testrun/{testrun_id}/provider/execution-space",
+            json.dumps(EXECUTION_SPACE_PROVIDER),
         )
         Config().get("database").put(
             f"/testrun/{testrun_id}/provider/dataset", json.dumps({"host": host})
@@ -194,19 +201,24 @@ def test_permutation_scenario(self):
             os.environ["ETOS_API"] = server.host
 
             self.logger.info("STEP: Initialize and run ESR.")
-            esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner"))
+            etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")
+            esr = ESR(etos)
 
             try:
                 self.logger.info("STEP: Verify that the ESR executes without errors.")
-                suite_ids = esr.run()
+                result = esr.run()
+                suite_ids = etos.config.get("ids")
 
+                self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config")
                 self.assertEqual(len(suite_ids), 2, "There shall only be two test suite started.")
                 for suite_id in suite_ids:
                     suite_finished = Handler.get_from_db(
                         "EiffelTestSuiteFinishedEvent", {"links.target": suite_id}
                     )
                     self.assertEqual(
-                        len(suite_finished), 1, "There shall only be a single test suite finished."
+                        len(suite_finished),
+                        1,
+                        "There shall only be a single test suite finished.",
                     )
                     outcome = suite_finished[0].get("data", {}).get("outcome", {})
                     self.logger.info(outcome)
@@ -219,6 +231,15 @@ def test_permutation_scenario(self):
                         },
                         f"Wrong outcome {outcome!r}, outcome should be successful.",
                     )
+                self.assertDictEqual(
+                    result.model_dump(),
+                    {
+                        "conclusion": "Successful",
+                        "verdict": "Passed",
+                        "description": "All tests passed.",
+                    },
+                    f"Wrong outcome {result!r}, outcome should be successful.",
+                )
             finally:
                 # If the _get_environment_status method in ESR does not time out before the test
                 # finishes there will be loads of tracebacks in the log. Won't fail the test but
@@ -256,21 +277,28 @@ def test_permutation_scenario_sub_suites(self):
             os.environ["ETOS_API"] = server.host
 
             self.logger.info("STEP: Initialize and run ESR.")
-            esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner"))
+            etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")
+            esr = ESR(etos)
 
             try:
                 self.logger.info("STEP: Verify that the ESR executes without errors.")
-                suite_ids = esr.run()
+                result = esr.run()
+                suite_ids = etos.config.get("ids")
 
+                self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config")
                 self.assertEqual(
-                    len(suite_ids), 2, "There shall only be a single test suite started."
+                    len(suite_ids),
+                    2,
+                    "There shall only be a single test suite started.",
                 )
                 for suite_id in suite_ids:
                     suite_finished = Handler.get_from_db(
                         "EiffelTestSuiteFinishedEvent", {"links.target": suite_id}
                     )
                     self.assertEqual(
-                        len(suite_finished), 1, "There shall only be a single test suite finished."
+                        len(suite_finished),
+                        1,
+                        "There shall only be a single test suite finished.",
                     )
                     outcome = suite_finished[0].get("data", {}).get("outcome", {})
                     self.logger.info(outcome)
@@ -283,6 +311,15 @@ def test_permutation_scenario_sub_suites(self):
                         },
                         f"Wrong outcome {outcome!r}, outcome should be successful.",
                     )
+                self.assertDictEqual(
+                    result.model_dump(),
+                    {
+                        "conclusion": "Successful",
+                        "verdict": "Passed",
+                        "description": "All tests passed.",
+                    },
+                    f"Wrong outcome {result!r}, outcome should be successful.",
+                )
 
             finally:
                 # If the _get_environment_status method in ESR does not time out before the test