From 2c0c79421f2b40fddad3d59c1dd8d6fc2c400aa1 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Mon, 25 Nov 2024 12:37:31 +0100 Subject: [PATCH] Add result to shutdown --- .../src/etos_suite_runner/__main__.py | 29 +------- .../src/etos_suite_runner/esr.py | 71 +++++++++++++------ .../src/etos_suite_runner/lib/result.py | 35 +++++++++ .../tests/scenario/test_permutations.py | 57 ++++++++++++--- .../tests/scenario/test_regular.py | 63 +++++++++++++--- 5 files changed, 188 insertions(+), 67 deletions(-) create mode 100644 projects/etos_suite_runner/src/etos_suite_runner/lib/result.py diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py index 86ab56a..e2ee268 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py @@ -16,6 +16,7 @@ # limitations under the License. # -*- coding: utf-8 -*- """ETOS suite runner module.""" + import os import json import logging @@ -32,35 +33,11 @@ def main(): """Entry point allowing external calls.""" etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") - etos.config.set("results", []) esr = ESR(etos) try: - esr.run() # Blocking - results = etos.config.get("results") or [] - result = None - for suite_result in results: - if suite_result.get("verdict") == "FAILED": - result = suite_result - # If the verdict on any main suite is FAILED, that is the verdict we set on the - # test run, which means that we can break the loop early in that case. - break - if suite_result.get("verdict") == "INCONCLUSIVE": - result = suite_result - if len(results) == 0: - result = { - "conclusion": "Inconclusive", - "verdict": "Inconclusive", - "description": "Got no results from ESR", - } - elif result is None: - # No suite failed, so lets just pick the first result - result = results[0] - # Convert, for example, INCONCLUSIVE to Inconclusive to match the controller result struct - # TODO Move the result struct to ETOS library and do this conversion on creation - result["conclusion"] = result["conclusion"].title().replace("_", "") - result["verdict"] = result["verdict"].title() + result = esr.run() # Blocking with open("/dev/termination-log", "w", encoding="utf-8") as termination_log: - json.dump(result, termination_log) + json.dump(result.model_dump(), termination_log) LOGGER.info("ESR result: %r", result) except: result = { diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 51a9c20..add0bda 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -15,11 +15,13 @@ # limitations under the License. # -*- coding: utf-8 -*- """ETOS suite runner module.""" + import logging import os import signal import time import threading +import traceback from uuid import uuid4 from eiffellib.events import ( @@ -35,6 +37,8 @@ import opentelemetry from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from etos_suite_runner.lib.result import Result + from .lib.esr_parameters import ESRParameters from .lib.exceptions import EnvironmentProviderException from .lib.graphql import request_tercc @@ -206,13 +210,12 @@ def _release_environment(self) -> None: if not status: self.logger.error(message) - def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: + def run_suites(self, triggered: EiffelActivityTriggeredEvent): """Start up a suite runner handling multiple suites that execute within test runners. Will only start the test activity if there's a 'slot' available. :param triggered: Activity triggered. - :return: List of main suite IDs - Used for tests """ context = triggered.meta.event_id self.etos.config.set("context", context) @@ -222,6 +225,7 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: ids = self.params.main_suite_ids() for i, suite in enumerate(self.params.test_suite): suites.append((ids[i], suite)) + self.etos.config.set("ids", ids) # Used for testing self.logger.info("Number of test suites to run: %d", len(suites), extra={"user_log": True}) try: self.logger.info("Get test environment.") @@ -240,7 +244,6 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: self.logger.info("Starting ESR.") runner.start_suites_and_wait(suites) - return [id for id, _ in suites] except EnvironmentProviderException as exc: # Not running as part of the ETOS Kubernetes controller environment if os.getenv("IDENTIFIER") is None: @@ -267,11 +270,8 @@ def _send_tercc(self, testrun_id: str, iut_id: str) -> None: } self.etos.events.send(event, links, data) - def _run(self) -> list[str]: - """Run the ESR main loop. - - :return: List of test suites (main suites) that were started. - """ + def _run(self): + """Run the ESR main loop.""" testrun_id = None try: testrun_id = self.params.testrun_id @@ -309,11 +309,10 @@ def _run(self) -> list[str]: raise try: - ids = self.run_suites(triggered) + self.run_suites(triggered) self.etos.events.send_activity_finished( triggered, {"conclusion": "SUCCESSFUL"}, {"CONTEXT": context} ) - return ids except Exception as exception: # pylint:disable=broad-except reason = str(exception) self.logger.exception( @@ -324,24 +323,56 @@ def _run(self) -> list[str]: self._record_exception(exception) raise - def run(self) -> list[str]: + def result(self) -> Result: + """ESR execution result.""" + results = self.etos.config.get("results") or [] + result = {} + for suite_result in results: + if suite_result.get("verdict") == "FAILED": + result = suite_result + # If the verdict on any main suite is FAILED, that is the verdict we set on the + # test run, which means that we can break the loop early in that case. + break + if suite_result.get("verdict") == "INCONCLUSIVE": + result = suite_result + if len(results) == 0: + result = { + "conclusion": "Inconclusive", + "verdict": "Inconclusive", + "description": "Got no results from ESR", + } + elif result == {}: + # No suite failed, so lets just pick the first result + result = results[0] + return Result(**result) + + def run(self) -> Result: """Run the ESR main loop. :return: List of test suites (main suites) that were started. """ - event = { - "event": "shutdown", - "data": "ESR has finished", - } + self.etos.config.set("results", []) + result = Result( + verdict="Inconclusive", + conclusion="Failed", + description="ESR did not execute", + ) try: - return self._run() - except Exception as exception: # pylint:disable=broad-except + self._run() + result = self.result() + return result + except Exception: # pylint:disable=bare-except + result = Result( + conclusion="Failed", + verdict="Inconclusive", + description=traceback.format_exc(), + ) + raise + finally: event = { "event": "shutdown", - "data": str(exception), + "data": result.model_dump(), } - raise - finally: self.event_publisher.publish(event) def graceful_exit(self, *_) -> None: diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/result.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/result.py new file mode 100644 index 0000000..86d06dc --- /dev/null +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/result.py @@ -0,0 +1,35 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ETOS result struct.""" + +from pydantic import BaseModel, field_validator, ValidationInfo + + +# TODO Move the result struct to ETOS library. + + +class Result(BaseModel): + """Final result of an ETOS suite run.""" + + verdict: str + conclusion: str + description: str + + @field_validator("verdict", "conclusion") + @classmethod + def title(cls, v: str, _: ValidationInfo) -> str: + """Convert, for example, INCONCLUSIVE to Inconclusive to match the Kubernetes controller.""" + return v.title() diff --git a/projects/etos_suite_runner/tests/scenario/test_permutations.py b/projects/etos_suite_runner/tests/scenario/test_permutations.py index ba32351..c77c564 100644 --- a/projects/etos_suite_runner/tests/scenario/test_permutations.py +++ b/projects/etos_suite_runner/tests/scenario/test_permutations.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Scenario tests for permutations.""" + import json import logging import os @@ -33,7 +34,11 @@ from tests.library.fake_database import FakeDatabase from tests.library.fake_server import FakeServer from tests.library.handler import Handler -from tests.scenario.tercc import ARTIFACT_CREATED, PERMUTATION_TERCC, PERMUTATION_TERCC_SUB_SUITES +from tests.scenario.tercc import ( + ARTIFACT_CREATED, + PERMUTATION_TERCC, + PERMUTATION_TERCC_SUB_SUITES, +) IUT_PROVIDER = { "iut": { @@ -125,7 +130,8 @@ def setup_providers(self): """Setup providers in the fake ETCD database.""" Config().get("database").put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) Config().get("database").put( - "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) + "/environment/provider/execution-space/default", + json.dumps(EXECUTION_SPACE_PROVIDER), ) Config().get("database").put( "/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER) @@ -146,7 +152,8 @@ def register_providers(self, testrun_id, host): f"/testrun/{testrun_id}/provider/log-area", json.dumps(LOG_AREA_PROVIDER) ) Config().get("database").put( - f"/testrun/{testrun_id}/provider/execution-space", json.dumps(EXECUTION_SPACE_PROVIDER) + f"/testrun/{testrun_id}/provider/execution-space", + json.dumps(EXECUTION_SPACE_PROVIDER), ) Config().get("database").put( f"/testrun/{testrun_id}/provider/dataset", json.dumps({"host": host}) @@ -194,19 +201,24 @@ def test_permutation_scenario(self): os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") - esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")) + etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") + esr = ESR(etos) try: self.logger.info("STEP: Verify that the ESR executes without errors.") - suite_ids = esr.run() + result = esr.run() + suite_ids = etos.config.get("ids") + self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config") self.assertEqual(len(suite_ids), 2, "There shall only be two test suite started.") for suite_id in suite_ids: suite_finished = Handler.get_from_db( "EiffelTestSuiteFinishedEvent", {"links.target": suite_id} ) self.assertEqual( - len(suite_finished), 1, "There shall only be a single test suite finished." + len(suite_finished), + 1, + "There shall only be a single test suite finished.", ) outcome = suite_finished[0].get("data", {}).get("outcome", {}) self.logger.info(outcome) @@ -219,6 +231,15 @@ def test_permutation_scenario(self): }, f"Wrong outcome {outcome!r}, outcome should be successful.", ) + self.assertDictEqual( + result.model_dump(), + { + "conclusion": "Successful", + "verdict": "Passed", + "description": "All tests passed.", + }, + f"Wrong outcome {result!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but @@ -256,21 +277,28 @@ def test_permutation_scenario_sub_suites(self): os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") - esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")) + etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") + esr = ESR(etos) try: self.logger.info("STEP: Verify that the ESR executes without errors.") - suite_ids = esr.run() + result = esr.run() + suite_ids = etos.config.get("ids") + self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config") self.assertEqual( - len(suite_ids), 2, "There shall only be a single test suite started." + len(suite_ids), + 2, + "There shall only be a single test suite started.", ) for suite_id in suite_ids: suite_finished = Handler.get_from_db( "EiffelTestSuiteFinishedEvent", {"links.target": suite_id} ) self.assertEqual( - len(suite_finished), 1, "There shall only be a single test suite finished." + len(suite_finished), + 1, + "There shall only be a single test suite finished.", ) outcome = suite_finished[0].get("data", {}).get("outcome", {}) self.logger.info(outcome) @@ -283,6 +311,15 @@ def test_permutation_scenario_sub_suites(self): }, f"Wrong outcome {outcome!r}, outcome should be successful.", ) + self.assertDictEqual( + result.model_dump(), + { + "conclusion": "Successful", + "verdict": "Passed", + "description": "All tests passed.", + }, + f"Wrong outcome {result!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test diff --git a/projects/etos_suite_runner/tests/scenario/test_regular.py b/projects/etos_suite_runner/tests/scenario/test_regular.py index faa9fda..a1bf953 100644 --- a/projects/etos_suite_runner/tests/scenario/test_regular.py +++ b/projects/etos_suite_runner/tests/scenario/test_regular.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Scenario tests for the most regular cases.""" + import json import logging import os @@ -126,7 +127,8 @@ def setup_providers(self): """Setup providers in the fake ETCD database.""" Config().get("database").put("/environment/provider/iut/default", json.dumps(IUT_PROVIDER)) Config().get("database").put( - "/environment/provider/execution-space/default", json.dumps(EXECUTION_SPACE_PROVIDER) + "/environment/provider/execution-space/default", + json.dumps(EXECUTION_SPACE_PROVIDER), ) Config().get("database").put( "/environment/provider/log-area/default", json.dumps(LOG_AREA_PROVIDER) @@ -147,7 +149,8 @@ def register_providers(self, testrun_id, host): f"/testrun/{testrun_id}/provider/log-area", json.dumps(LOG_AREA_PROVIDER) ) Config().get("database").put( - f"/testrun/{testrun_id}/provider/execution-space", json.dumps(EXECUTION_SPACE_PROVIDER) + f"/testrun/{testrun_id}/provider/execution-space", + json.dumps(EXECUTION_SPACE_PROVIDER), ) Config().get("database").put( f"/testrun/{testrun_id}/provider/dataset", json.dumps({"host": host}) @@ -195,19 +198,26 @@ def test_full_scenario(self): os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") - esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")) + etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") + esr = ESR(etos) try: self.logger.info("STEP: Verify that the ESR executes without errors.") - suite_ids = esr.run() + result = esr.run() + suite_ids = etos.config.get("ids") + self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config") self.assertEqual( - len(suite_ids), 1, "There shall only be a single test suite started." + len(suite_ids), + 1, + "There shall only be a single test suite started.", ) suite_finished = Handler.get_from_db( "EiffelTestSuiteFinishedEvent", {"links.target": suite_ids[0]} ) self.assertEqual( - len(suite_finished), 1, "There shall only be a single test suite finished." + len(suite_finished), + 1, + "There shall only be a single test suite finished.", ) outcome = suite_finished[0].get("data", {}).get("outcome", {}) self.logger.info(outcome) @@ -220,6 +230,15 @@ def test_full_scenario(self): }, f"Wrong outcome {outcome!r}, outcome should be successful.", ) + self.assertDictEqual( + result.model_dump(), + { + "conclusion": "Successful", + "verdict": "Passed", + "description": "All tests passed.", + }, + f"Wrong outcome {result!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but @@ -258,19 +277,26 @@ def test_full_scenario_sub_suites(self): os.environ["ETOS_API"] = server.host self.logger.info("STEP: Initialize and run ESR.") - esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")) + etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") + esr = ESR(etos) try: self.logger.info("STEP: Verify that the ESR executes without errors.") - suite_ids = esr.run() + result = esr.run() + suite_ids = etos.config.get("ids") + self.assertIsNotNone(suite_ids, "No suite ids added to ETOS config") self.assertEqual( - len(suite_ids), 1, "There shall only be a single test suite started." + len(suite_ids), + 1, + "There shall only be a single test suite started.", ) suite_finished = Handler.get_from_db( "EiffelTestSuiteFinishedEvent", {"links.target": suite_ids[0]} ) self.assertEqual( - len(suite_finished), 1, "There shall only be a single test suite finished." + len(suite_finished), + 1, + "There shall only be a single test suite finished.", ) outcome = suite_finished[0].get("data", {}).get("outcome", {}) self.logger.info(outcome) @@ -283,6 +309,15 @@ def test_full_scenario_sub_suites(self): }, f"Wrong outcome {outcome!r}, outcome should be successful.", ) + self.assertDictEqual( + result.model_dump(), + { + "conclusion": "Successful", + "verdict": "Passed", + "description": "All tests passed.", + }, + f"Wrong outcome {result!r}, outcome should be successful.", + ) finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but @@ -322,7 +357,7 @@ def test_esr_without_recipes(self): self.logger.info("STEP: Initialize and run ESR.") esr = ESR(ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner")) try: - esr.run() + result = esr.run() finished = None for event in Debug().events_published.copy(): if event.meta.type == "EiffelTestSuiteFinishedEvent": @@ -336,9 +371,15 @@ def test_esr_without_recipes(self): assert ( outcome.get("conclusion") == "FAILED" ), "Conclusion was not FAILED when test suite is empty" + assert ( + result.conclusion == "Failed" + ), "Conclusion was not FAILED when test suite is empty" assert ( outcome.get("verdict") == "INCONCLUSIVE" ), "Verdict was not INCONCLUSIVE when test suite is empty" + assert ( + result.verdict == "Inconclusive" + ), "Vertdict was not INCONCLUSIVE when test suite is empty" finally: # If the _get_environment_status method in ESR does not time out before the test # finishes there will be loads of tracebacks in the log. Won't fail the test but