diff --git a/Dockerfile b/Dockerfile index ea9067a..ad30cd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,15 @@ -FROM python:3.9.0-buster AS build +FROM python:3.9-buster AS build COPY . /src WORKDIR /src/projects/etos_suite_runner RUN python3 setup.py bdist_wheel -FROM python:3.9.0-slim-buster +FROM python:3.9-slim-buster COPY --from=build /src/projects/etos_suite_runner/dist/*.whl /tmp # hadolint ignore=DL3013 RUN pip install --no-cache-dir /tmp/*.whl && groupadd -r etos && useradd -r -m -s /bin/false -g etos etos + USER etos LABEL org.opencontainers.image.source=https://github.com/eiffel-community/etos-suite-runner diff --git a/projects/etos_suite_runner/requirements.txt b/projects/etos_suite_runner/requirements.txt index 6ea27e2..1c02a7b 100644 --- a/projects/etos_suite_runner/requirements.txt +++ b/projects/etos_suite_runner/requirements.txt @@ -18,5 +18,8 @@ PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 -etos_lib==4.0.0 -etos_environment_provider~=4.1 +etos_lib==4.2.0 +etos_environment_provider~=4.2 +opentelemetry-api~=1.21 +opentelemetry-exporter-otlp~=1.21 +opentelemetry-sdk~=1.21 diff --git a/projects/etos_suite_runner/setup.cfg b/projects/etos_suite_runner/setup.cfg index c0fb1ba..8ce7775 100644 --- a/projects/etos_suite_runner/setup.cfg +++ b/projects/etos_suite_runner/setup.cfg @@ -28,8 +28,11 @@ install_requires = PyScaffold==3.2.3 packageurl-python~=0.11 cryptography>=42.0.4,<43.0.0 - etos_lib==4.0.0 - etos_environment_provider~=4.1 + etos_lib==4.2.0 + etos_environment_provider~=4.2 + opentelemetry-api~=1.21 + opentelemetry-exporter-otlp~=1.21 + opentelemetry-sdk~=1.21 python_requires = >=3.4 diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py index 3fae8f6..c921c33 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__init__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__init__.py @@ -14,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. """ETOS suite runner module.""" +import logging import os from importlib.metadata import PackageNotFoundError, version +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_NAMESPACE, SERVICE_VERSION, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + from etos_lib.logging.logger import setup_logging try: @@ -30,3 +37,37 @@ ENVIRONMENT = "development" if DEV else "production" os.environ["ENVIRONMENT_PROVIDER_DISABLE_LOGGING"] = "true" setup_logging("ETOS Suite Runner", VERSION, ENVIRONMENT) + + +LOGGER = logging.getLogger(__name__) + +# Setting OTEL_COLLECTOR_HOST will override the default OTEL collector endpoint. +# This is needed because Suite Runner uses the cluster-level OpenTelemetry collector +# instead of a sidecar collector. +if os.getenv("OTEL_COLLECTOR_HOST"): + os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = os.getenv("OTEL_COLLECTOR_HOST") +else: + if "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" in os.environ: + LOGGER.debug("Environment variable OTEL_EXPORTER_OTLP_TRACES_ENDPOINT not used.") + LOGGER.debug("To specify an OpenTelemetry collector host use OTEL_COLLECTOR_HOST.") + del os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] + +if os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"): + LOGGER.info( + "Using OpenTelemetry collector: %s", os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + ) + PROVIDER = TracerProvider( + resource=Resource.create( + { + SERVICE_NAME: "etos-suite-runner", + SERVICE_VERSION: VERSION, + SERVICE_NAMESPACE: ENVIRONMENT, + } + ) + ) + EXPORTER = OTLPSpanExporter() + PROCESSOR = BatchSpanProcessor(EXPORTER) + PROVIDER.add_span_processor(PROCESSOR) + trace.set_tracer_provider(PROVIDER) +else: + LOGGER.info("OpenTelemetry not enabled. OTEL_COLLECTOR_HOST not set.") diff --git a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py index 2891532..5c4e7b7 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/__main__.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/__main__.py @@ -21,6 +21,7 @@ from .esr import ESR + LOGGER = logging.getLogger(__name__) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/esr.py b/projects/etos_suite_runner/src/etos_suite_runner/esr.py index 212e2e9..543dd40 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/esr.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/esr.py @@ -28,16 +28,18 @@ from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas +import opentelemetry from .lib.esr_parameters import ESRParameters from .lib.exceptions import EnvironmentProviderException from .lib.runner import SuiteRunner +from .lib.otel_tracing import get_current_context, OpenTelemetryBase # Remove spam from pika. logging.getLogger("pika").setLevel(logging.WARNING) -class ESR: # pylint:disable=too-many-instance-attributes +class ESR(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Suite runner for ETOS main program. Run this as a daemon on your system in order to trigger test suites within @@ -49,6 +51,7 @@ class ESR: # pylint:disable=too-many-instance-attributes def __init__(self) -> None: """Initialize ESR by creating a rabbitmq publisher.""" self.logger = logging.getLogger("ESR") + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) self.etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"), "ETOS Suite Runner") signal.signal(signal.SIGTERM, self.graceful_exit) self.params = ESRParameters(self.etos) @@ -67,28 +70,38 @@ def _request_environment(self, ids: list[str]) -> None: :param ids: Generated suite runner IDs used to correlate environments and the suite runners. """ - try: - provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) - result = provider.run() - except Exception: - self.params.set_status("FAILURE", "Failed to run environment provider") - self.logger.error( - "Environment provider has failed in creating an environment for test.", - extra={"user_log": True}, - ) - raise - if result.get("error") is not None: - self.params.set_status("FAILURE", result.get("error")) - self.logger.error( - "Environment provider has failed in creating an environment for test.", - extra={"user_log": True}, - ) - else: - self.params.set_status("SUCCESS", result.get("error")) - self.logger.info( - "Environment provider has finished creating an environment for test.", - extra={"user_log": True}, - ) + span_name = "request_environment" + suite_context = get_current_context() + with self.otel_tracer.start_as_current_span( + span_name, + context=suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, + ): + try: + provider = EnvironmentProvider(self.params.tercc.meta.event_id, ids, copy=False) + result = provider.run() + except Exception as exc: + self.params.set_status("FAILURE", "Failed to run environment provider") + self.logger.error( + "Environment provider has failed in creating an environment for test.", + extra={"user_log": True}, + ) + self._record_exception(exc) + raise + if result.get("error") is not None: + self.params.set_status("FAILURE", result.get("error")) + self.logger.error( + "Environment provider has failed in creating an environment for test.", + extra={"user_log": True}, + ) + exc = Exception(str(result.get("error"))) + self._record_exception(exc) + else: + self.params.set_status("SUCCESS", result.get("error")) + self.logger.info( + "Environment provider has finished creating an environment for test.", + extra={"user_log": True}, + ) def _release_environment(self) -> None: """Release an environment from the environment provider.""" @@ -96,11 +109,18 @@ def _release_environment(self) -> None: # Passing variables as keyword argument to make it easier to transition to a function where # jsontas is not required. jsontas = JsonTas() - status, message = release_full_environment( - etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id - ) - if not status: - self.logger.error(message) + span_name = "release_full_environment" + suite_context = get_current_context() + with self.otel_tracer.start_as_current_span( + span_name, + context=suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, + ): + status, message = release_full_environment( + etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id + ) + if not status: + self.logger.error(message) def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: """Start up a suite runner handling multiple suites that execute within test runners. @@ -117,13 +137,11 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: "ESR Docker", {"CONTEXT": context}, image=os.getenv("SUITE_RUNNER") ) runner = SuiteRunner(self.params, self.etos) - ids = [] for suite in self.params.test_suite: suite["test_suite_started_id"] = str(uuid4()) ids.append(suite["test_suite_started_id"]) self.logger.info("Number of test suites to run: %d", len(ids), extra={"user_log": True}) - try: self.logger.info("Get test environment.") threading.Thread( @@ -135,10 +153,11 @@ def run_suites(self, triggered: EiffelActivityTriggeredEvent) -> list[str]: self.logger.info("Starting ESR.") runner.start_suites_and_wait() return ids - except EnvironmentProviderException: + except EnvironmentProviderException as exc: self.logger.info("Release test environment.") self._release_environment() - raise + self._record_exception(exc) + raise exc @staticmethod def verify_input() -> None: @@ -176,7 +195,6 @@ def run(self) -> list[str]: executionType="AUTOMATED", triggers=[{"type": "EIFFEL_EVENT"}], ) - self.verify_input() context = triggered.meta.event_id except: # noqa @@ -211,6 +229,7 @@ def run(self) -> list[str]: "MAJOR", {"CONTEXT": context}, ) + self._record_exception(exception) raise def graceful_exit(self, *_) -> None: diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py index 04bb38e..4210264 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py @@ -16,15 +16,19 @@ """Executor handler module.""" import logging import os -from json import JSONDecodeError +from json import JSONDecodeError, dumps from typing import Union from cryptography.fernet import Fernet from etos_lib import ETOS +from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes +from opentelemetry import trace from requests.auth import HTTPBasicAuth, HTTPDigestAuth from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import HTTPError +from .otel_tracing import OpenTelemetryBase + class TestStartException(Exception): """Exception when starting tests.""" @@ -35,7 +39,7 @@ def __init__(self, message: dict): self.error = message.get("error", "Unknown error when starting tests") -class Executor: # pylint:disable=too-few-public-methods +class Executor(OpenTelemetryBase): # pylint:disable=too-few-public-methods """Executor for launching ETR.""" logger = logging.getLogger("ESR - Executor") @@ -47,6 +51,7 @@ def __init__(self, etos: ETOS) -> None: """ self.etos = etos self.etos.config.set("build_urls", []) + self.tracer = trace.get_tracer(__name__) def __decrypt(self, password: Union[str, dict]) -> str: """Decrypt a password using an encryption key. @@ -89,15 +94,28 @@ def run_tests(self, test_suite: dict) -> None: if request.get("auth"): request["auth"] = self.__auth(**request["auth"]) method = getattr(self.etos.http, request.pop("method").lower()) - try: - response = method(**request) - response.raise_for_status() - except HTTPError as http_error: + span_name = "start_execution_space" + with self.tracer.start_as_current_span(span_name, kind=trace.SpanKind.CLIENT) as span: + span.set_attribute( + SemConvAttributes.EXECUTOR_ID, executor["id"] if "id" in executor else "" + ) + span.set_attribute("http.request.body", dumps(request)) try: - raise TestStartException(http_error.response.json()) from http_error - except JSONDecodeError: - raise TestStartException({"error": http_error.response.text}) from http_error - except RequestsConnectionError as connection_error: - raise TestStartException({"error": str(connection_error)}) from connection_error - self.logger.info("%r", response) - self.logger.debug("%r", response.text) + response = method(**request) + response.raise_for_status() + except HTTPError as http_error: + try: + exc = TestStartException(http_error.response.json()) + self._record_exception(exc) + raise exc from http_error + except JSONDecodeError: + exc = TestStartException({"error": http_error.response.text}) + self._record_exception(exc) + raise exc from http_error + except RequestsConnectionError as connection_error: + exc = TestStartException({"error": str(connection_error)}) + self._record_exception(exc) + raise exc from connection_error + + self.logger.info("%r", response) + self.logger.debug("%r", response.text) diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py new file mode 100644 index 0000000..76e9f63 --- /dev/null +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/otel_tracing.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +"""OpenTelemetry-related code.""" +import logging +import os + +import opentelemetry +from opentelemetry.propagators.textmap import Getter +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + +LOGGER = logging.getLogger(__name__) + + +class OpenTelemetryBase: + """Base functionality for OpenTelemetry data collection.""" + + # pylint: disable=too-few-public-methods + @staticmethod + def _record_exception(exc) -> None: + """Record the given exception to the current OpenTelemetry span.""" + span = opentelemetry.trace.get_current_span() + span.set_attribute("error.type", exc.__class__.__name__) + span.record_exception(exc) + span.set_status(opentelemetry.trace.Status(opentelemetry.trace.StatusCode.ERROR)) + + +class EnvVarContextGetter(Getter): + """OpenTelemetry context getter class for environment variables.""" + + def get(self, carrier, key): + """Get value using the given carrier variable and key.""" + value = os.getenv(carrier) + if value is not None: + pairs = value.split(",") + for pair in pairs: + k, v = pair.split("=", 1) + if k == key: + return [v] + return [] + + def keys(self, carrier): + """Get keys of the given carrier variable.""" + value = os.getenv(carrier) + if value is not None: + return [pair.split("=")[0] for pair in value.split(",") if "=" in pair] + return [] + + +def get_current_context() -> opentelemetry.context.context.Context: + """Get current context propagated via environment variable OTEL_CONTEXT.""" + propagator = TraceContextTextMapPropagator() + ctx = propagator.extract(carrier="OTEL_CONTEXT", getter=EnvVarContextGetter()) + return ctx diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py index fd694c3..7624aaa 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/runner.py @@ -20,12 +20,14 @@ from environment_provider.environment import release_full_environment from etos_lib.logging.logger import FORMAT_CONFIG from jsontas.jsontas import JsonTas +import opentelemetry from .exceptions import EnvironmentProviderException +from .otel_tracing import get_current_context, OpenTelemetryBase from .suite import TestSuite -class SuiteRunner: # pylint:disable=too-few-public-methods +class SuiteRunner(OpenTelemetryBase): # pylint:disable=too-few-public-methods """Test suite runner. Splits test suites into sub suites based on number of products available. @@ -44,6 +46,8 @@ def __init__(self, params, etos): """ self.params = params self.etos = etos + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) + self.otel_suite_context = get_current_context() def _release_environment(self): """Release an environment from the environment provider.""" @@ -51,23 +55,32 @@ def _release_environment(self): # Passing variables as keyword argument to make it easier to transition to a function where # jsontas is not required. jsontas = JsonTas() - status, message = release_full_environment( - etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id - ) - if not status: - self.logger.error(message) + span_name = "release_full_environment" + with self.otel_tracer.start_as_current_span( + span_name, + context=self.otel_suite_context, + kind=opentelemetry.trace.SpanKind.CLIENT, + ): + status, message = release_full_environment( + etos=self.etos, jsontas=jsontas, suite_id=self.params.tercc.meta.event_id + ) + if not status: + self.logger.error(message) def start_suites_and_wait(self): """Get environments and start all test suites.""" try: test_suites = [ - TestSuite(self.etos, self.params, suite) for suite in self.params.test_suite + TestSuite(self.etos, self.params, suite, otel_context=self.otel_suite_context) + for suite in self.params.test_suite ] with ThreadPool() as pool: pool.map(self.run, test_suites) status = self.params.get_status() if status.get("error") is not None: - raise EnvironmentProviderException(status["error"], self.etos.config.get("task_id")) + exc = EnvironmentProviderException(status["error"], self.etos.config.get("task_id")) + self._record_exception(exc) + raise exc finally: self.logger.info("Release the full test environment.") self._release_environment() diff --git a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py index 782066c..9a3284a 100644 --- a/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py +++ b/projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Test suite handler.""" +import json import logging import threading import time @@ -24,7 +25,9 @@ from environment_provider.environment import release_environment from etos_lib import ETOS from etos_lib.logging.logger import FORMAT_CONFIG +from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes from jsontas.jsontas import JsonTas +import opentelemetry from .esr_parameters import ESRParameters from .exceptions import EnvironmentProviderException @@ -37,9 +40,10 @@ request_test_suite_started, ) from .log_filter import DuplicateFilter +from .otel_tracing import OpenTelemetryBase -class SubSuite: # pylint:disable=too-many-instance-attributes +class SubSuite(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Handle test results and tracking of a single sub suite.""" released = False @@ -52,6 +56,7 @@ def __init__(self, etos: ETOS, environment: dict, main_suite_id: str) -> None: self.main_suite_id = main_suite_id self.logger = logging.getLogger(f"SubSuite - {self.environment.get('name')}") self.logger.addFilter(DuplicateFilter(self.logger)) + self.otel_tracer = opentelemetry.trace.get_tracer(__name__) self.test_suite_started = {} self.test_suite_finished = {} @@ -93,36 +98,46 @@ def outcome(self) -> dict: return self.test_suite_finished.get("data", {}).get("testSuiteOutcome", {}) return {} - def start(self, identifier: str) -> None: + def start(self, identifier: str, otel_context: opentelemetry.context.context.Context) -> None: """Start ETR for this sub suite. :param identifier: An identifier for logs in this sub suite. """ - FORMAT_CONFIG.identifier = identifier - self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) - executor = Executor(self.etos) - try: - executor.run_tests(self.environment) - except TestStartException as exception: - self.failed = True - self.logger.error( - "Failed to start sub suite: %s", exception.error, extra={"user_log": True} - ) - raise - self.logger.info("ETR triggered.") - timeout = time.time() + self.etos.debug.default_test_result_timeout - try: - while time.time() < timeout: - time.sleep(10) - if not self.started: - continue - self.logger.info("ETOS test runner has started", extra={"user_log": True}) - self.request_finished_event() - if self.finished: - self.logger.info("ETOS test runner has finished", extra={"user_log": True}) - break - finally: - self.release(identifier) + # OpenTelemetry context needs to be explicitly given here when creating this new span. + # This is because the subsuite is running in a separate thread. + span_name = "execute_testrunner" + with self.otel_tracer.start_as_current_span( + span_name, + context=otel_context, + kind=opentelemetry.trace.SpanKind.CLIENT, + ) as span: + span.set_attribute(SemConvAttributes.SUBSUITE_ID, identifier) + FORMAT_CONFIG.identifier = identifier + self.logger.info("Starting up the ETOS test runner", extra={"user_log": True}) + executor = Executor(self.etos) + try: + executor.run_tests(self.environment) + except TestStartException as exception: + self.failed = True + self.logger.error( + "Failed to start sub suite: %s", exception.error, extra={"user_log": True} + ) + self._record_exception(exception) + raise + self.logger.info("ETR triggered.") + timeout = time.time() + self.etos.debug.default_test_result_timeout + try: + while time.time() < timeout: + time.sleep(10) + if not self.started: + continue + self.logger.info("ETOS test runner has started", extra={"user_log": True}) + self.request_finished_event() + if self.finished: + self.logger.info("ETOS test runner has finished", extra={"user_log": True}) + break + finally: + self.release(identifier) def release(self, testrun_id) -> None: """Release this sub suite.""" @@ -136,19 +151,31 @@ def release(self, testrun_id) -> None: registry = ProviderRegistry(etos=self.etos, jsontas=jsontas, suite_id=testrun_id) self.logger.info(self.environment) - success = release_environment( - etos=self.etos, jsontas=jsontas, provider_registry=registry, sub_suite=self.environment - ) - if not success: - self.logger.exception( - "Failed to check in %r", self.environment["id"], extra={"user_log": True} + + span_name = "release_environment" + with self.otel_tracer.start_as_current_span( + span_name, + kind=opentelemetry.trace.SpanKind.CLIENT, + ) as span: + failure = release_environment( + etos=self.etos, + jsontas=jsontas, + provider_registry=registry, + sub_suite=self.environment, ) - return - self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) - self.released = True + span.set_attribute(SemConvAttributes.TESTRUN_ID, testrun_id) + span.set_attribute(SemConvAttributes.ENVIRONMENT, json.dumps(self.environment)) + if failure is not None: + self.logger.exception( + "Failed to check in %r", self.environment["id"], extra={"user_log": True} + ) + self._record_exception(failure) + return + self.logger.info("Checked in %r", self.environment["id"], extra={"user_log": True}) + self.released = True -class TestSuite: # pylint:disable=too-many-instance-attributes +class TestSuite(OpenTelemetryBase): # pylint:disable=too-many-instance-attributes """Handle the starting and waiting for test suites in ETOS.""" test_suite_started = None @@ -157,7 +184,13 @@ class TestSuite: # pylint:disable=too-many-instance-attributes __activity_triggered = None __activity_finished = None - def __init__(self, etos: ETOS, params: ESRParameters, suite: dict) -> None: + def __init__( + self, + etos: ETOS, + params: ESRParameters, + suite: dict, + otel_context: opentelemetry.context.context.Context = None, + ) -> None: """Initialize a TestSuite instance.""" self.etos = etos self.params = params @@ -165,6 +198,7 @@ def __init__(self, etos: ETOS, params: ESRParameters, suite: dict) -> None: self.logger = logging.getLogger(f"TestSuite - {self.suite.get('name')}") self.logger.addFilter(DuplicateFilter(self.logger)) self.sub_suites = [] + self.otel_context = otel_context @property def sub_suite_environments(self) -> Iterator[dict]: @@ -186,9 +220,11 @@ def sub_suite_environments(self) -> Iterator[dict]: if activity_triggered is None: status = self.params.get_status() if status.get("status") == "FAILURE": - raise EnvironmentProviderException( + exc = EnvironmentProviderException( status.get("error"), self.etos.config.get("task_id") ) + self._record_exception(exc) + raise exc continue activity_finished = self.__environment_activity_finished( activity_triggered["meta"]["id"] @@ -201,16 +237,20 @@ def sub_suite_environments(self) -> Iterator[dict]: yield environment if activity_finished is not None: if activity_finished["data"]["activityOutcome"]["conclusion"] != "SUCCESSFUL": - raise EnvironmentProviderException( + exc = EnvironmentProviderException( activity_finished["data"]["activityOutcome"]["description"], self.etos.config.get("task_id"), ) + self._record_exception(exc) + raise exc if len(environments) > 0: # Must be at least 1 sub suite. return else: # pylint:disable=useless-else-on-loop - raise TimeoutError( + exc = TimeoutError( f"Timed out after {self.etos.config.get('WAIT_FOR_ENVIRONMENT_TIMEOUT')} seconds." ) + self._record_exception(exc) + raise exc @property def all_finished(self) -> bool: @@ -324,7 +364,8 @@ def start(self) -> None: ) self.sub_suites.append(sub_suite) thread = threading.Thread( - target=sub_suite.start, args=(self.params.tercc.meta.event_id,) + target=sub_suite.start, + args=(self.params.tercc.meta.event_id, self.otel_context), ) threads.append(thread) thread.start()