From 60d0a4230af18a374d4302fa44f37daedde7c972 Mon Sep 17 00:00:00 2001 From: Omer Date: Thu, 14 Dec 2023 17:22:44 +0200 Subject: [PATCH] chore(RHTAPWATCH-634): request ODCS compose Implement ODCSRequestor, allowing requesting ODCS for multiple compose files, per number of sources. Signed-off-by: Omer --- generate_compose/compose_generator.py | 2 +- .../odcs_configurations_generator.py | 20 ++- generate_compose/odcs_fetcher.py | 4 +- generate_compose/odcs_requester.py | 23 +++- tests/test_odcs_fetcher.py | 4 +- tests/test_odcs_requester.py | 123 ++++++++++++++++++ 6 files changed, 164 insertions(+), 12 deletions(-) create mode 100644 tests/test_odcs_requester.py diff --git a/generate_compose/compose_generator.py b/generate_compose/compose_generator.py index cdd8c8c..4a69dc0 100644 --- a/generate_compose/compose_generator.py +++ b/generate_compose/compose_generator.py @@ -13,7 +13,7 @@ @dataclass(frozen=True) class ComposeGenerator: """ - Given implementations to all building blocks if a compose generator, generate a + Given implementations to all building blocks of a compose generator, generate a compose and return its reference. :param configurations_generator: an object to generate the configurations used for diff --git a/generate_compose/odcs_configurations_generator.py b/generate_compose/odcs_configurations_generator.py index 192368b..3ef53d8 100644 --- a/generate_compose/odcs_configurations_generator.py +++ b/generate_compose/odcs_configurations_generator.py @@ -1,15 +1,29 @@ """Configurations generator for ODCS compose""" -from dataclasses import dataclass +from dataclasses import dataclass, field + +from odcs.client.odcs import ComposeSourceGeneric # type: ignore from .protocols import ComposeConfigurations, ComposeConfigurationsGenerator @dataclass(frozen=True) -class ODCSConfigurations(ComposeConfigurations): +class ODCSComposeConfig: """ Configurations to be used for requesting an ODCS compose """ + spec: ComposeSourceGeneric + additional_args: dict = field(default_factory=dict) + + +@dataclass +class ODCSComposesConfigs(ComposeConfigurations): + """ + Configurations to be used for requesting multiple ODCS composes + """ + + configs: list[ODCSComposeConfig] + @dataclass(frozen=True) class ODCSConfigurationsGenerator(ComposeConfigurationsGenerator): @@ -21,5 +35,5 @@ class ODCSConfigurationsGenerator(ComposeConfigurationsGenerator): compose_inputs: dict - def __call__(self) -> ODCSConfigurations: + def __call__(self) -> ComposeConfigurations: raise NotImplementedError() diff --git a/generate_compose/odcs_fetcher.py b/generate_compose/odcs_fetcher.py index 6f2bab3..97a541a 100644 --- a/generate_compose/odcs_fetcher.py +++ b/generate_compose/odcs_fetcher.py @@ -5,7 +5,7 @@ import requests -from .odcs_requester import ODCSRequestReference +from .odcs_requester import ODCSRequestReferences from .protocols import ComposeFetcher, ComposeReference @@ -39,7 +39,7 @@ def __call__(self, request_reference: ComposeReference) -> ODCSResultReference: :return: The filesystem path to the downloaded ODCS compose file. """ self.compose_dir_path.mkdir(parents=True, exist_ok=True) - assert isinstance(request_reference, ODCSRequestReference) + assert isinstance(request_reference, ODCSRequestReferences) urls = request_reference.compose_urls for url in urls: with tempfile.NamedTemporaryFile( diff --git a/generate_compose/odcs_requester.py b/generate_compose/odcs_requester.py index d6e6724..e9a6824 100644 --- a/generate_compose/odcs_requester.py +++ b/generate_compose/odcs_requester.py @@ -1,11 +1,14 @@ """Request a new ODCS compose""" from dataclasses import dataclass +from odcs.client.odcs import ODCS # type: ignore + +from .odcs_configurations_generator import ODCSComposesConfigs from .protocols import ComposeConfigurations, ComposeReference, ComposeRequester @dataclass(frozen=True) -class ODCSRequestReference(ComposeReference): +class ODCSRequestReferences(ComposeReference): """ Reference to a remotely-stored compose data """ @@ -16,9 +19,21 @@ class ODCSRequestReference(ComposeReference): @dataclass(frozen=True) class ODCSRequester(ComposeRequester): """ - Request a new ODCS compose based on compose configurations and return a reference + Request new ODCS composes based on compose configurations and return a reference to the remote compose location. """ - def __call__(self, configs: ComposeConfigurations) -> ODCSRequestReference: - raise NotImplementedError() + odcs: ODCS = ODCS("https://odcs.engineering.redhat.com/") + + def __call__(self, compose_configs: ComposeConfigurations) -> ODCSRequestReferences: + assert isinstance(compose_configs, ODCSComposesConfigs) + composes = [ + self.odcs.request_compose(config.spec, **config.additional_args) + for config in compose_configs.configs + ] + for compose in composes: + self.odcs.wait_for_compose(compose["id"]) + + compose_urls = [compose["result_repofile"] for compose in composes] + req_refrences = ODCSRequestReferences(compose_urls=compose_urls) + return req_refrences diff --git a/tests/test_odcs_fetcher.py b/tests/test_odcs_fetcher.py index 43a5670..af3fe33 100644 --- a/tests/test_odcs_fetcher.py +++ b/tests/test_odcs_fetcher.py @@ -6,7 +6,7 @@ import responses from generate_compose.odcs_fetcher import ODCSFetcher, ODCSResultReference -from generate_compose.odcs_requester import ODCSRequestReference +from generate_compose.odcs_requester import ODCSRequestReferences @pytest.mark.parametrize( @@ -46,7 +46,7 @@ ) def test_odcs_fetcher(tmp_path: Path, composes: list[str], urls: list[str]) -> None: """test ODCSFetcher.__call__""" - req_ref = ODCSRequestReference(compose_urls=urls) + req_ref = ODCSRequestReferences(compose_urls=urls) fetcher = ODCSFetcher(compose_dir_path=tmp_path) with responses.RequestsMock() as mock: diff --git a/tests/test_odcs_requester.py b/tests/test_odcs_requester.py new file mode 100644 index 0000000..e5042ea --- /dev/null +++ b/tests/test_odcs_requester.py @@ -0,0 +1,123 @@ +"""test_odcs_requester.py - test odcs_requester""" +from typing import Callable +from unittest.mock import MagicMock, call, create_autospec + +import pytest +from odcs.client.odcs import ODCS, ComposeSourceGeneric # type: ignore +from requests.exceptions import HTTPError + +from generate_compose.odcs_configurations_generator import ( + ODCSComposeConfig, + ODCSComposesConfigs, +) +from generate_compose.odcs_requester import ODCSRequester, ODCSRequestReferences + + +class TestODCSRequester: + """Test odcs_compose_generator.py""" + + @pytest.fixture() + def compose_url(self) -> str: + """Example compose-url, as close to the one expected""" + return "http://download.eng.bos.redhat.com/odcs/prod/odcs-222222" + + @pytest.fixture() + def create_odcs_mock(self, compose_url: str) -> Callable[[int, bool], MagicMock]: + """Create an ODCS mock with specific results for the compose method""" + + def _mock_odcs_compose( + num_of_composes: int = 1, exception: bool = False + ) -> MagicMock: + """Mock for ODCS.compose""" + mock: MagicMock = create_autospec(ODCS) + if exception: + mock.request_compose.side_effect = HTTPError + else: + mock.request_compose.side_effect = [ + {"result_repofile": f"{compose_url}", "id": i} + for i in range(1, num_of_composes + 1) + ] + return mock + + return _mock_odcs_compose + + @pytest.fixture() + def compose_source(self) -> MagicMock: + """Creates a ComposeSource with dynamic typing""" + mock: MagicMock = create_autospec(ComposeSourceGeneric) + return mock + + @pytest.mark.parametrize( + "composes_configs", + [ + pytest.param( + ODCSComposesConfigs([ODCSComposeConfig(spec=compose_source)]), + id="single compose, tag source, no additional arguments", + ), + pytest.param( + ODCSComposesConfigs( + [ + ODCSComposeConfig(spec=compose_source), + ODCSComposeConfig(spec=compose_source), + ] + ), + id="multiple composes, , tag source, no additional arguments", + ), + ], + ) + def test_odcs_requester( + self, + create_odcs_mock: Callable, + compose_url: str, + composes_configs: ODCSComposesConfigs, + ) -> None: + """test ODCSRequester.__call__""" + num_of_composes: int = len(composes_configs.configs) + mock_odcs = create_odcs_mock(num_of_composes) + odcs_requester = ODCSRequester(odcs=mock_odcs) + req_ref = odcs_requester(compose_configs=composes_configs) + + expected_calls = [ + call(composes_configs.configs[0].spec) for _ in range(num_of_composes) + ] + assert mock_odcs.request_compose.call_args_list == expected_calls + + assert mock_odcs.request_compose.call_count == num_of_composes + expected_calls = [ + call(compose_id) for compose_id in range(1, num_of_composes + 1) + ] + mock_odcs.wait_for_compose.assert_has_calls(expected_calls, any_order=False) + assert mock_odcs.wait_for_compose.call_count == num_of_composes + + expected_req_ref = ODCSRequestReferences( + compose_urls=[compose_url] * num_of_composes + ) + assert req_ref == expected_req_ref + + def test_odcs_requester_compose_failure_should_raise( + self, create_odcs_mock: Callable, compose_source: ComposeSourceGeneric + ) -> None: + """test ODCSRequester.__call__ raise an exception when the compose fails""" + mock_odcs = create_odcs_mock(exception=True) + odcs_requester = ODCSRequester(odcs=mock_odcs) + composes_config = ODCSComposesConfigs([ODCSComposeConfig(spec=compose_source)]) + with pytest.raises(HTTPError): + odcs_requester(compose_configs=composes_config) + assert mock_odcs.wait_for_compose.call_count == 0 + + def test_odcs_requester_compose_timeout_failure_should_raise( + self, create_odcs_mock: Callable, compose_source: ComposeSourceGeneric + ) -> None: + """test ODCSRequester.__call__ raise an exception when + waiting for compose throws an exception due to a timeout""" + + mock_odcs = create_odcs_mock(exception=False) + mock_odcs.wait_for_compose.side_effect = RuntimeError + + odcs_requester = ODCSRequester(odcs=mock_odcs) + composes_config = ODCSComposesConfigs([ODCSComposeConfig(spec=compose_source)]) + + with pytest.raises(RuntimeError): + odcs_requester(compose_configs=composes_config) + assert mock_odcs.request_compose.call_count == 1 + assert mock_odcs.wait_for_compose.call_count == 1