diff --git a/generate_compose/odcs_compose_generator.py b/generate_compose/odcs_compose_generator.py index 0ee9540..b81a0c7 100755 --- a/generate_compose/odcs_compose_generator.py +++ b/generate_compose/odcs_compose_generator.py @@ -15,42 +15,33 @@ @click.command() @click.option( - "--compose-file-path", - help="Compose file target path (including filename).", + "--compose-dir-path", + help="Compose files target directory.", type=click.Path(path_type=Path), required=True, ) @click.option( - "--container-yaml-path", - help="Path in which container yaml file is stored", - type=click.Path(path_type=Path), - required=True, -) -@click.option( - "--content-sets-yaml-path", - help="Path in which content_sets yaml file is stored", + "--compose-input-yaml-path", + help="Path in which inputs yaml is stored", type=click.Path(path_type=Path), required=True, ) def main( - compose_file_path: Path, - container_yaml_path: Path, - content_sets_yaml_path: Path, + compose_dir_path: Path, + compose_input_yaml_path: Path, ): """ Get inputs from container and content_sets YAMLs and relay them to an ODCS compose generator that will request a compose and store it in a TBD location. """ - container_data: dict = yaml.safe_load(container_yaml_path.read_text()) - content_sets_data: dict = yaml.safe_load(content_sets_yaml_path.read_text()) + compose_inputs: dict = yaml.safe_load(compose_input_yaml_path.read_text()) compose_generator = ComposeGenerator( configurations_generator=ODCSConfigurationsGenerator( - container_data=container_data, - content_sets_data=content_sets_data, + compose_inputs=compose_inputs, ), requestor=ODCSRequester(), - fetcher=ODCSFetcher(compose_file_path=compose_file_path), + fetcher=ODCSFetcher(compose_dir_path=compose_dir_path), ) compose_generator() diff --git a/generate_compose/odcs_configurations_generator.py b/generate_compose/odcs_configurations_generator.py index 4f950f3..192368b 100644 --- a/generate_compose/odcs_configurations_generator.py +++ b/generate_compose/odcs_configurations_generator.py @@ -16,12 +16,10 @@ class ODCSConfigurationsGenerator(ComposeConfigurationsGenerator): """ Generate odcs configurations based on container and content_sets YAMLs. - :param container_data: data loaded from container.yaml - :param content_sets_data: data loaded from content_sets.yaml + :param compose_inputs: data loaded from compose inputs yaml """ - container_data: dict - content_sets_data: dict + compose_inputs: dict def __call__(self) -> ODCSConfigurations: raise NotImplementedError() diff --git a/generate_compose/odcs_fetcher.py b/generate_compose/odcs_fetcher.py index 29a7497..6f2bab3 100644 --- a/generate_compose/odcs_fetcher.py +++ b/generate_compose/odcs_fetcher.py @@ -1,4 +1,5 @@ """Fetch ready ODCS compose""" +import tempfile from dataclasses import dataclass from pathlib import Path @@ -11,10 +12,10 @@ @dataclass(frozen=True) class ODCSResultReference(ComposeReference): """ - Reference to a locally-stored compose result + Reference to locally-stored compose results """ - compose_file_path: Path + compose_dir_path: Path @dataclass(frozen=True) @@ -22,28 +23,30 @@ class ODCSFetcher(ComposeFetcher): """ Fetch ODCS compose based on a remote compose-reference and store it locally - :param compose_file_path: The Desired path for the compose file. - must include the path to the file as - well as the file's name. + :param compose_dir_path: The Desired path for where compose files should be stored """ - compose_file_path: Path + compose_dir_path: Path def __call__(self, request_reference: ComposeReference) -> ODCSResultReference: """ Fetch the 'ODCS compose' from a remote reference. - :param request_reference: An object containing the url reference for the + :param request_reference: An object containing the url references for the ODCS compose file. :raises HTTPError: If the request for the ODCS compose file failed. :return: The filesystem path to the downloaded ODCS compose file. """ + self.compose_dir_path.mkdir(parents=True, exist_ok=True) assert isinstance(request_reference, ODCSRequestReference) - response = requests.get(request_reference.compose_url, timeout=10) - response.raise_for_status() - - self.compose_file_path.parent.mkdir(parents=True, exist_ok=True) - self.compose_file_path.write_text(response.text, encoding="utf-8") - odcs_result_ref = ODCSResultReference(compose_file_path=self.compose_file_path) - return odcs_result_ref + urls = request_reference.compose_urls + for url in urls: + with tempfile.NamedTemporaryFile( + delete=False, dir=self.compose_dir_path + ) as compose_path: + response = requests.get(url, timeout=10) + response.raise_for_status() + Path(compose_path.name).write_text(response.text, encoding="utf-8") + + return ODCSResultReference(compose_dir_path=self.compose_dir_path) diff --git a/generate_compose/odcs_requester.py b/generate_compose/odcs_requester.py index a5292c8..d6e6724 100644 --- a/generate_compose/odcs_requester.py +++ b/generate_compose/odcs_requester.py @@ -10,7 +10,7 @@ class ODCSRequestReference(ComposeReference): Reference to a remotely-stored compose data """ - compose_url: str + compose_urls: list[str] @dataclass(frozen=True) diff --git a/tests/test_odcs_compose_generator.py b/tests/test_odcs_compose_generator.py index bbc8ba9..24efcc0 100644 --- a/tests/test_odcs_compose_generator.py +++ b/tests/test_odcs_compose_generator.py @@ -17,52 +17,22 @@ class TestODCSComposeGenerator: """Test odcs_compose_generator.py end-to-end""" @pytest.fixture() - def container_data(self) -> dict: - """container.yaml content""" - return { - "compose": {"pulp_repos": True, "signing_intent": "release"}, - "image_build_method": "imagebuilder", - "remote_source": { - "pkg_managers": [], - "ref": "f6ceb2ff45a71938f6949abcd15aa0a1b0f79842", - "repo": "https://github.com/kubevirt/must-gather", - }, - } + def input_data(self) -> list: + """inputs yaml content""" + return [{"spec": {}, "additional_args": {}}] @pytest.fixture() - def container_yaml(self, container_data: dict, tmp_path: Path) -> Path: - """path to container.yaml with content""" - path = tmp_path / "container.yaml" + def input_yaml(self, input_data: list, tmp_path: Path) -> Path: + """path to inputs.yaml with content""" + path = tmp_path / "inputs.yaml" with path.open("w") as file: - yaml.dump(container_data, file) + yaml.dump(input_data, file) return path @pytest.fixture() - def content_sets_data(self) -> dict: - """content_sets.yaml content""" - return { - "x86_64": [ - "rhel-8-for-x86_64-baseos-eus-rpms__8_DOT_6", - "rhel-8-for-x86_64-appstream-eus-rpms__8_DOT_6", - ], - "aarch64": [ - "rhel-8-for-aarch64-baseos-eus-rpms__8_DOT_6", - "rhel-8-for-aarch64-appstream-eus-rpms__8_DOT_6", - ], - } - - @pytest.fixture() - def content_sets_yaml(self, content_sets_data: dict, tmp_path: Path) -> Path: - """path to content_sets.yaml with content""" - path = tmp_path / "content_sets.yml" - with path.open("w") as file: - yaml.dump(content_sets_data, file) - return path - - @pytest.fixture() - def compose_file_path(self, tmp_path: Path) -> Path: + def compose_dir_path(self, tmp_path: Path) -> Path: """Path to where the compose should be stored""" - return tmp_path / "repofile.repo" + return tmp_path @pytest.fixture() def mock_config_generator(self, monkeypatch: MonkeyPatch) -> MagicMock: @@ -99,11 +69,9 @@ def mock_fetcher(self, monkeypatch: MonkeyPatch) -> MagicMock: def test_main( # pylint: disable=too-many-arguments self, - compose_file_path: Path, - container_data: dict, - container_yaml: Path, - content_sets_data: dict, - content_sets_yaml: Path, + compose_dir_path: Path, + input_data: dict, + input_yaml: Path, mock_compose_generator: MagicMock, mock_config_generator: MagicMock, mock_requester: MagicMock, @@ -112,21 +80,17 @@ def test_main( # pylint: disable=too-many-arguments """Test call to odcs_compose_generator.py main function""" odcs_compose_generator.main( # pylint: disable=no-value-for-parameter args=[ - "--compose-file-path", - str(compose_file_path), - "--container-yaml-path", - str(container_yaml), - "--content-sets-yaml-path", - str(content_sets_yaml), + "--compose-dir-path", + str(compose_dir_path), + "--compose-input-yaml-path", + str(input_yaml), ], obj={}, standalone_mode=False, ) - mock_config_generator.assert_called_once_with( - container_data=container_data, content_sets_data=content_sets_data - ) - mock_fetcher.assert_called_once_with(compose_file_path=compose_file_path) + mock_config_generator.assert_called_once_with(compose_inputs=input_data) + mock_fetcher.assert_called_once_with(compose_dir_path=compose_dir_path) mock_compose_generator.assert_called_once_with( configurations_generator=mock_config_generator.return_value, requestor=mock_requester.return_value, diff --git a/tests/test_odcs_fetcher.py b/tests/test_odcs_fetcher.py index 0ca4248..43a5670 100644 --- a/tests/test_odcs_fetcher.py +++ b/tests/test_odcs_fetcher.py @@ -1,40 +1,61 @@ """test_odcs_fetcher.py - test odcs_fetcher""" from pathlib import Path +from textwrap import dedent +import pytest import responses -from generate_compose.odcs_fetcher import ODCSFetcher +from generate_compose.odcs_fetcher import ODCSFetcher, ODCSResultReference from generate_compose.odcs_requester import ODCSRequestReference -def test_odcs_fetcher(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("composes", "urls"), + [ + pytest.param( + [ + dedent( + """ + [odcs-111] + name=compose 1 + """ + ) + ], + ["https://url1"], + id="single compose", + ), + pytest.param( + [ + dedent( + """ + [odcs-111] + name=compose 1 + """ + ), + dedent( + """ + [odcs-222] + name=compose 2 + """ + ), + ], + ["https://url1", "https://url2"], + id="multiple composes", + ), + ], +) +def test_odcs_fetcher(tmp_path: Path, composes: list[str], urls: list[str]) -> None: """test ODCSFetcher.__call__""" - expected_content: str = """ - [odcs-2222222] - name=ODCS repository for compose odcs-2222222 - baseurl=http://download.eng.bos.redhat.com/odcs/prod/odcs-2222222/compose/Temporary/$basearch/os - type=rpm-md - skip_if_unavailable=False - gpgcheck=0 - repo_gpgcheck=0 - enabled=1 - enabled_metadata=1 - """ - mock_compose_url: str = ( - "http://download.eng.bos.redhat.com/odcs/prod/odcs-222222" - "/compose/Temporary/odcs-2222222.repo" - ) - odcs_ref = ODCSRequestReference(compose_url=mock_compose_url) + req_ref = ODCSRequestReference(compose_urls=urls) + fetcher = ODCSFetcher(compose_dir_path=tmp_path) - compose_path = tmp_path / "downloaded_file_dir" / "odcs-2222222.repo" - odcs_fetcher = ODCSFetcher(compose_file_path=compose_path) + with responses.RequestsMock() as mock: + for compose, url in zip(composes, urls): + mock.add(responses.GET, url, body=compose) + out: ODCSResultReference = fetcher(request_reference=req_ref) - with responses.RequestsMock() as req_mock: - req_mock.add(responses.GET, mock_compose_url, body=expected_content) - odcs_result_ref = odcs_fetcher(request_reference=odcs_ref) + files_content = [ + p.read_text(encoding="utf-8") for p in out.compose_dir_path.iterdir() + ] - assert odcs_result_ref.compose_file_path.exists() - odcs_result_ref_content = odcs_result_ref.compose_file_path.read_text( - encoding="utf-8" - ) - assert odcs_result_ref_content == expected_content + assert set(composes) == set(files_content)