From b8aadaa0fc191316a807bc56be1da6e46a8caf65 Mon Sep 17 00:00:00 2001 From: Yftach Herzog Date: Thu, 7 Dec 2023 14:23:45 +0200 Subject: [PATCH] chore(RHTAPWATCH-662): add inputs to odcs compose entrypoint Required inputs when running the odcs entrypoint to specify where the compose should be stored and where the configurations reside. Had to rename the directory name to avoid it having the same name as one of the the modules. Signed-off-by: Yftach Herzog --- .github/workflows/test.yml | 4 +- Pipfile | 3 +- Pipfile.lock | 67 ++++++++- compose_generator/odcs_compose_generator.py | 32 ----- format.sh | 2 +- .../__init__.py | 0 .../compose_generator.py | 0 generate_compose/odcs_compose_generator.py | 59 ++++++++ .../odcs_configurations_generator.py | 0 .../odcs_fetcher.py | 3 + .../odcs_requester.py | 0 .../protocols.py | 0 tests/test_odcs_compose_generator.py | 135 ++++++++++++++++++ tests/test_static_check.py | 2 +- 14 files changed, 268 insertions(+), 39 deletions(-) delete mode 100644 compose_generator/odcs_compose_generator.py rename {compose_generator => generate_compose}/__init__.py (100%) rename {compose_generator => generate_compose}/compose_generator.py (100%) create mode 100755 generate_compose/odcs_compose_generator.py rename {compose_generator => generate_compose}/odcs_configurations_generator.py (100%) rename {compose_generator => generate_compose}/odcs_fetcher.py (90%) rename {compose_generator => generate_compose}/odcs_requester.py (100%) rename {compose_generator => generate_compose}/protocols.py (100%) create mode 100644 tests/test_odcs_compose_generator.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff91fa3..9988c77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,9 @@ jobs: checks: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.11 - name: Install pipenv diff --git a/Pipfile b/Pipfile index 7b9f190..10f4136 100644 --- a/Pipfile +++ b/Pipfile @@ -11,9 +11,10 @@ black = "*" mypy = "*" pylint = "*" isort = "*" +pyyaml = "*" +types-pyyaml = "*" [dev-packages] [requires] python_version = "3.11" -python_full_version = "3.11.6" diff --git a/Pipfile.lock b/Pipfile.lock index 5406ea6..df17fce 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,10 @@ { "_meta": { "hash": { - "sha256": "264c0b231f9d6ee1d77dabeacbb67ff4be3a801496ac82c597640f6aa4779e12" + "sha256": "b28c523e83c5c71a5b4048a70531c50b38fcb2636e8c1f61d62e1eba3d8942d4" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.11.6", "python_version": "3.11" }, "sources": [ @@ -290,6 +289,62 @@ "index": "pypi", "version": "==7.4.3" }, + "pyyaml": { + "hashes": [ + "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290", + "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", + "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" + ], + "index": "pypi", + "version": "==6.0.1" + }, "requests": { "hashes": [ "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", @@ -306,6 +361,14 @@ "markers": "python_version >= '3.7'", "version": "==0.12.3" }, + "types-pyyaml": { + "hashes": [ + "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062", + "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24" + ], + "index": "pypi", + "version": "==6.0.12.12" + }, "typing-extensions": { "hashes": [ "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", diff --git a/compose_generator/odcs_compose_generator.py b/compose_generator/odcs_compose_generator.py deleted file mode 100644 index 4b0179d..0000000 --- a/compose_generator/odcs_compose_generator.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -"""Initialize and use a compose generator using ODCS""" - -from .compose_generator import ComposeGenerator -from .odcs_configurations_generator import ODCSConfigurationsGenerator -from .odcs_fetcher import ODCSFetcher -from .odcs_requester import ODCSRequester - - -def main(): - """ - Get inputs from container and content_sets yamls and relay them to an ODCS - compose generator that will request a compose and store it in a TBD location. - """ - # Get inputs from container and content_sets yamls - container_data = {} - content_sets_data = {} - - compose_generator = ComposeGenerator( - configurations_generator=ODCSConfigurationsGenerator( - container_data=container_data, - content_sets_data=content_sets_data, - ), - requestor=ODCSRequester(), - fetcher=ODCSFetcher(), - ) - compose_generator() - - -if __name__ == "__main__": - main() diff --git a/format.sh b/format.sh index 45c5dfb..632aa4a 100755 --- a/format.sh +++ b/format.sh @@ -1,7 +1,7 @@ #!/bin/bash -ex main() { - local pkgs=("tests" "compose_generator") + local pkgs=("tests" "generate_compose") pipenv run isort --profile black "${pkgs[@]}" pipenv run black "${pkgs[@]}" } diff --git a/compose_generator/__init__.py b/generate_compose/__init__.py similarity index 100% rename from compose_generator/__init__.py rename to generate_compose/__init__.py diff --git a/compose_generator/compose_generator.py b/generate_compose/compose_generator.py similarity index 100% rename from compose_generator/compose_generator.py rename to generate_compose/compose_generator.py diff --git a/generate_compose/odcs_compose_generator.py b/generate_compose/odcs_compose_generator.py new file mode 100755 index 0000000..b5f398c --- /dev/null +++ b/generate_compose/odcs_compose_generator.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +"""Initialize and use a compose generator using ODCS""" + +from pathlib import Path + +import click +import yaml + +from .compose_generator import ComposeGenerator +from .odcs_configurations_generator import ODCSConfigurationsGenerator +from .odcs_fetcher import ODCSFetcher +from .odcs_requester import ODCSRequester + + +@click.command() +@click.option( + "--compose-path", + help="Path to where the compose should be saved.", + type=click.Path(path_type=Path), + required=True, +) +@click.option( + "--container-yaml-path", + help="Path in which container yaml file is stored", + type=click.Path(path_type=Path), + required=True, +) +@click.option( + "--content-sets-yaml-path", + help="Path in which content_sets yaml file is stored", + type=click.Path(path_type=Path), + required=True, +) +def main( + compose_path: Path, + container_yaml_path: Path, + content_sets_yaml_path: Path, +): + """ + Get inputs from container and content_sets yamls and relay them to an ODCS + compose generator that will request a compose and store it in a TBD location. + """ + container_data: dict = yaml.safe_load(container_yaml_path.read_text()) + content_sets_data: dict = yaml.safe_load(content_sets_yaml_path.read_text()) + + compose_generator = ComposeGenerator( + configurations_generator=ODCSConfigurationsGenerator( + container_data=container_data, + content_sets_data=content_sets_data, + ), + requestor=ODCSRequester(), + fetcher=ODCSFetcher(compose_path=compose_path), + ) + compose_generator() + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/compose_generator/odcs_configurations_generator.py b/generate_compose/odcs_configurations_generator.py similarity index 100% rename from compose_generator/odcs_configurations_generator.py rename to generate_compose/odcs_configurations_generator.py diff --git a/compose_generator/odcs_fetcher.py b/generate_compose/odcs_fetcher.py similarity index 90% rename from compose_generator/odcs_fetcher.py rename to generate_compose/odcs_fetcher.py index 51307a3..405c572 100644 --- a/compose_generator/odcs_fetcher.py +++ b/generate_compose/odcs_fetcher.py @@ -1,6 +1,7 @@ """Fetch ready ODCS compose""" from dataclasses import dataclass from pathlib import Path +from typing import Optional from .protocols import ComposeFetcher, ComposeReference @@ -20,5 +21,7 @@ class ODCSFetcher(ComposeFetcher): Fetch ODCS compose based on a remote compose-reference and store it locally """ + compose_path: Optional[Path] + def __call__(self, request_reference: ComposeReference) -> ODCSResultReference: raise NotImplementedError() diff --git a/compose_generator/odcs_requester.py b/generate_compose/odcs_requester.py similarity index 100% rename from compose_generator/odcs_requester.py rename to generate_compose/odcs_requester.py diff --git a/compose_generator/protocols.py b/generate_compose/protocols.py similarity index 100% rename from compose_generator/protocols.py rename to generate_compose/protocols.py diff --git a/tests/test_odcs_compose_generator.py b/tests/test_odcs_compose_generator.py new file mode 100644 index 0000000..ef5d754 --- /dev/null +++ b/tests/test_odcs_compose_generator.py @@ -0,0 +1,135 @@ +"""Test odcs_compose_generator.py end-to-end""" +from pathlib import Path +from unittest.mock import MagicMock, create_autospec + +import pytest +import yaml +from pytest import MonkeyPatch + +from generate_compose import odcs_compose_generator +from generate_compose.compose_generator import ComposeGenerator +from generate_compose.odcs_configurations_generator import ODCSConfigurationsGenerator +from generate_compose.odcs_fetcher import ODCSFetcher +from generate_compose.odcs_requester import ODCSRequester + + +class TestODCSComposeGenerator: + """Test odcs_compose_generator.py end-to-end""" + + @pytest.fixture() + def container_data(self) -> dict: + """container.yaml content""" + return { + "compose": {"pulp_repos": True, "signing_intent": "release"}, + "image_build_method": "imagebuilder", + "remote_source": { + "pkg_managers": [], + "ref": "f6ceb2ff45a71938f6949abcd15aa0a1b0f79842", + "repo": "https://github.com/kubevirt/must-gather", + }, + } + + @pytest.fixture() + def container_yaml(self, container_data: dict, tmp_path: Path) -> Path: + """path to container.yaml with content""" + path = tmp_path / "container.yaml" + with path.open("w") as file: + yaml.dump(container_data, file) + return path + + @pytest.fixture() + def content_sets_data(self) -> dict: + """content_sets.yaml content""" + return { + "x86_64": [ + "rhel-8-for-x86_64-baseos-eus-rpms__8_DOT_6", + "rhel-8-for-x86_64-appstream-eus-rpms__8_DOT_6", + ], + "aarch64": [ + "rhel-8-for-aarch64-baseos-eus-rpms__8_DOT_6", + "rhel-8-for-aarch64-appstream-eus-rpms__8_DOT_6", + ], + } + + @pytest.fixture() + def content_sets_yaml(self, content_sets_data: dict, tmp_path: Path) -> Path: + """path to content_sets.yaml with content""" + path = tmp_path / "content_sets.yml" + with path.open("w") as file: + yaml.dump(content_sets_data, file) + return path + + @pytest.fixture() + def compose_path(self, tmp_path: Path) -> Path: + """Path to where the compose should be stored""" + return tmp_path / "repofile.repo" + + @pytest.fixture() + def mock_config_generator(self, monkeypatch: MonkeyPatch) -> MagicMock: + """Monkey-patched ODCSConfigurationsGenerator""" + mock: MagicMock = create_autospec( + ODCSConfigurationsGenerator, + return_value=create_autospec(ODCSConfigurationsGenerator, instance=True), + ) + monkeypatch.setattr( + odcs_compose_generator, ODCSConfigurationsGenerator.__name__, mock + ) + return mock + + @pytest.fixture() + def mock_compose_generator(self, monkeypatch: MonkeyPatch) -> MagicMock: + """Monkey-patched ComposeGenerator""" + mock: MagicMock = create_autospec(ComposeGenerator) + monkeypatch.setattr(odcs_compose_generator, ComposeGenerator.__name__, mock) + return mock + + @pytest.fixture() + def mock_requester(self, monkeypatch: MonkeyPatch) -> MagicMock: + """Monkey-patched ODCSRequester""" + mock: MagicMock = create_autospec(ODCSRequester) + monkeypatch.setattr(odcs_compose_generator, ODCSRequester.__name__, mock) + return mock + + @pytest.fixture() + def mock_fetcher(self, monkeypatch: MonkeyPatch) -> MagicMock: + """Monkey-patched ODCSFetcher""" + mock: MagicMock = create_autospec(ODCSFetcher) + monkeypatch.setattr(odcs_compose_generator, ODCSFetcher.__name__, mock) + return mock + + def test_main( # pylint: disable=too-many-arguments + self, + compose_path: Path, + container_data: dict, + container_yaml: Path, + content_sets_data: dict, + content_sets_yaml: Path, + mock_compose_generator: MagicMock, + mock_config_generator: MagicMock, + mock_requester: MagicMock, + mock_fetcher: MagicMock, + ) -> None: + """Test call to odcs_compose_generator.py main function""" + odcs_compose_generator.main( # pylint: disable=no-value-for-parameter + args=[ + "--compose-path", + str(compose_path), + "--container-yaml-path", + str(container_yaml), + "--content-sets-yaml-path", + str(content_sets_yaml), + ], + obj={}, + standalone_mode=False, + ) + + mock_config_generator.assert_called_once_with( + container_data=container_data, content_sets_data=content_sets_data + ) + mock_fetcher.assert_called_once_with(compose_path=compose_path) + mock_compose_generator.assert_called_once_with( + configurations_generator=mock_config_generator.return_value, + requestor=mock_requester.return_value, + fetcher=mock_fetcher.return_value, + ) + mock_compose_generator.return_value.assert_called_once_with() diff --git a/tests/test_static_check.py b/tests/test_static_check.py index 6ed98f9..17de8e6 100644 --- a/tests/test_static_check.py +++ b/tests/test_static_check.py @@ -2,7 +2,7 @@ from subprocess import run from typing import Final -PKGS: Final[list[str]] = ["tests", "compose_generator"] +PKGS: Final[list[str]] = ["tests", "generate_compose"] def test_mypy() -> None: