Merge pull request #20 from yftacherzog/RHTAPWATCH-684-multi-compose-…

…support

refactor(RHTAPWATCH-684): support multiple calls to odcs

generate_compose/odcs_compose_generator.py

@@ -15,42 +15,33 @@
 
 @click.command()
 @click.option(
-    "--compose-file-path",
-    help="Compose file target path (including filename).",
+    "--compose-dir-path",
+    help="Compose files target directory.",
     type=click.Path(path_type=Path),
     required=True,
 )
 @click.option(
-    "--container-yaml-path",
-    help="Path in which container yaml file is stored",
-    type=click.Path(path_type=Path),
-    required=True,
-)
-@click.option(
-    "--content-sets-yaml-path",
-    help="Path in which content_sets yaml file is stored",
+    "--compose-input-yaml-path",
+    help="Path in which inputs yaml is stored",
     type=click.Path(path_type=Path),
     required=True,
 )
 def main(
-    compose_file_path: Path,
-    container_yaml_path: Path,
-    content_sets_yaml_path: Path,
+    compose_dir_path: Path,
+    compose_input_yaml_path: Path,
 ):
     """
     Get inputs from container and content_sets YAMLs and relay them to an ODCS
     compose generator that will request a compose and store it in a TBD location.
     """
-    container_data: dict = yaml.safe_load(container_yaml_path.read_text())
-    content_sets_data: dict = yaml.safe_load(content_sets_yaml_path.read_text())
+    compose_inputs: dict = yaml.safe_load(compose_input_yaml_path.read_text())
 
     compose_generator = ComposeGenerator(
         configurations_generator=ODCSConfigurationsGenerator(
-            container_data=container_data,
-            content_sets_data=content_sets_data,
+            compose_inputs=compose_inputs,
         ),
         requestor=ODCSRequester(),
-        fetcher=ODCSFetcher(compose_file_path=compose_file_path),
+        fetcher=ODCSFetcher(compose_dir_path=compose_dir_path),
     )
     compose_generator()
 

generate_compose/odcs_configurations_generator.py

@@ -16,12 +16,10 @@ class ODCSConfigurationsGenerator(ComposeConfigurationsGenerator):
     """
     Generate odcs configurations based on container and content_sets YAMLs.
 
-    :param container_data: data loaded from container.yaml
-    :param content_sets_data: data loaded from content_sets.yaml
+    :param compose_inputs: data loaded from compose inputs yaml
     """
 
-    container_data: dict
-    content_sets_data: dict
+    compose_inputs: dict
 
     def __call__(self) -> ODCSConfigurations:
         raise NotImplementedError()

generate_compose/odcs_fetcher.py

@@ -1,4 +1,5 @@
 """Fetch ready ODCS compose"""
+import tempfile
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -11,39 +12,41 @@
 @dataclass(frozen=True)
 class ODCSResultReference(ComposeReference):
     """
-    Reference to a locally-stored compose result
+    Reference to locally-stored compose results
     """
 
-    compose_file_path: Path
+    compose_dir_path: Path
 
 
 @dataclass(frozen=True)
 class ODCSFetcher(ComposeFetcher):
     """
     Fetch ODCS compose based on a remote compose-reference and store it locally
 
-    :param compose_file_path: The Desired path for the compose file.
-                              must include the path to the file as
-                              well as the file's name.
+    :param compose_dir_path: The Desired path for where compose files should be stored
     """
 
-    compose_file_path: Path
+    compose_dir_path: Path
 
     def __call__(self, request_reference: ComposeReference) -> ODCSResultReference:
         """
         Fetch the 'ODCS compose' from a remote reference.
 
-        :param request_reference: An object containing the url reference for the
+        :param request_reference: An object containing the url references for the
                                   ODCS compose file.
 
         :raises HTTPError: If the request for the ODCS compose file failed.
         :return: The filesystem path to the downloaded ODCS compose file.
         """
+        self.compose_dir_path.mkdir(parents=True, exist_ok=True)
         assert isinstance(request_reference, ODCSRequestReference)
-        response = requests.get(request_reference.compose_url, timeout=10)
-        response.raise_for_status()
-
-        self.compose_file_path.parent.mkdir(parents=True, exist_ok=True)
-        self.compose_file_path.write_text(response.text, encoding="utf-8")
-        odcs_result_ref = ODCSResultReference(compose_file_path=self.compose_file_path)
-        return odcs_result_ref
+        urls = request_reference.compose_urls
+        for url in urls:
+            with tempfile.NamedTemporaryFile(
+                delete=False, dir=self.compose_dir_path
+            ) as compose_path:
+                response = requests.get(url, timeout=10)
+                response.raise_for_status()
+                Path(compose_path.name).write_text(response.text, encoding="utf-8")
+
+        return ODCSResultReference(compose_dir_path=self.compose_dir_path)

generate_compose/odcs_requester.py

@@ -10,7 +10,7 @@ class ODCSRequestReference(ComposeReference):
     Reference to a remotely-stored compose data
     """
 
-    compose_url: str
+    compose_urls: list[str]
 
 
 @dataclass(frozen=True)

tests/test_odcs_compose_generator.py

@@ -17,52 +17,22 @@ class TestODCSComposeGenerator:
     """Test odcs_compose_generator.py end-to-end"""
 
     @pytest.fixture()
-    def container_data(self) -> dict:
-        """container.yaml content"""
-        return {
-            "compose": {"pulp_repos": True, "signing_intent": "release"},
-            "image_build_method": "imagebuilder",
-            "remote_source": {
-                "pkg_managers": [],
-                "ref": "f6ceb2ff45a71938f6949abcd15aa0a1b0f79842",
-                "repo": "https://github.com/kubevirt/must-gather",
-            },
-        }
+    def input_data(self) -> list:
+        """inputs yaml content"""
+        return [{"spec": {}, "additional_args": {}}]
 
     @pytest.fixture()
-    def container_yaml(self, container_data: dict, tmp_path: Path) -> Path:
-        """path to container.yaml with content"""
-        path = tmp_path / "container.yaml"
+    def input_yaml(self, input_data: list, tmp_path: Path) -> Path:
+        """path to inputs.yaml with content"""
+        path = tmp_path / "inputs.yaml"
         with path.open("w") as file:
-            yaml.dump(container_data, file)
+            yaml.dump(input_data, file)
         return path
 
     @pytest.fixture()
-    def content_sets_data(self) -> dict:
-        """content_sets.yaml content"""
-        return {
-            "x86_64": [
-                "rhel-8-for-x86_64-baseos-eus-rpms__8_DOT_6",
-                "rhel-8-for-x86_64-appstream-eus-rpms__8_DOT_6",
-            ],
-            "aarch64": [
-                "rhel-8-for-aarch64-baseos-eus-rpms__8_DOT_6",
-                "rhel-8-for-aarch64-appstream-eus-rpms__8_DOT_6",
-            ],
-        }
-
-    @pytest.fixture()
-    def content_sets_yaml(self, content_sets_data: dict, tmp_path: Path) -> Path:
-        """path to content_sets.yaml with content"""
-        path = tmp_path / "content_sets.yml"
-        with path.open("w") as file:
-            yaml.dump(content_sets_data, file)
-        return path
-
-    @pytest.fixture()
-    def compose_file_path(self, tmp_path: Path) -> Path:
+    def compose_dir_path(self, tmp_path: Path) -> Path:
         """Path to where the compose should be stored"""
-        return tmp_path / "repofile.repo"
+        return tmp_path
 
     @pytest.fixture()
     def mock_config_generator(self, monkeypatch: MonkeyPatch) -> MagicMock:
@@ -99,11 +69,9 @@ def mock_fetcher(self, monkeypatch: MonkeyPatch) -> MagicMock:
 
     def test_main(  # pylint: disable=too-many-arguments
         self,
-        compose_file_path: Path,
-        container_data: dict,
-        container_yaml: Path,
-        content_sets_data: dict,
-        content_sets_yaml: Path,
+        compose_dir_path: Path,
+        input_data: dict,
+        input_yaml: Path,
         mock_compose_generator: MagicMock,
         mock_config_generator: MagicMock,
         mock_requester: MagicMock,
@@ -112,21 +80,17 @@ def test_main(  # pylint: disable=too-many-arguments
         """Test call to odcs_compose_generator.py main function"""
         odcs_compose_generator.main(  # pylint: disable=no-value-for-parameter
             args=[
-                "--compose-file-path",
-                str(compose_file_path),
-                "--container-yaml-path",
-                str(container_yaml),
-                "--content-sets-yaml-path",
-                str(content_sets_yaml),
+                "--compose-dir-path",
+                str(compose_dir_path),
+                "--compose-input-yaml-path",
+                str(input_yaml),
             ],
             obj={},
             standalone_mode=False,
         )
 
-        mock_config_generator.assert_called_once_with(
-            container_data=container_data, content_sets_data=content_sets_data
-        )
-        mock_fetcher.assert_called_once_with(compose_file_path=compose_file_path)
+        mock_config_generator.assert_called_once_with(compose_inputs=input_data)
+        mock_fetcher.assert_called_once_with(compose_dir_path=compose_dir_path)
         mock_compose_generator.assert_called_once_with(
             configurations_generator=mock_config_generator.return_value,
             requestor=mock_requester.return_value,

tests/test_odcs_fetcher.py

@@ -1,40 +1,61 @@
 """test_odcs_fetcher.py - test odcs_fetcher"""
 from pathlib import Path
+from textwrap import dedent
 
+import pytest
 import responses
 
-from generate_compose.odcs_fetcher import ODCSFetcher
+from generate_compose.odcs_fetcher import ODCSFetcher, ODCSResultReference
 from generate_compose.odcs_requester import ODCSRequestReference
 
 
-def test_odcs_fetcher(tmp_path: Path) -> None:
+@pytest.mark.parametrize(
+    ("composes", "urls"),
+    [
+        pytest.param(
+            [
+                dedent(
+                    """
+                    [odcs-111]
+                    name=compose 1
+                    """
+                )
+            ],
+            ["https://url1"],
+            id="single compose",
+        ),
+        pytest.param(
+            [
+                dedent(
+                    """
+                    [odcs-111]
+                    name=compose 1
+                    """
+                ),
+                dedent(
+                    """
+                    [odcs-222]
+                    name=compose 2
+                    """
+                ),
+            ],
+            ["https://url1", "https://url2"],
+            id="multiple composes",
+        ),
+    ],
+)
+def test_odcs_fetcher(tmp_path: Path, composes: list[str], urls: list[str]) -> None:
     """test ODCSFetcher.__call__"""
-    expected_content: str = """
-    [odcs-2222222]
-    name=ODCS repository for compose odcs-2222222
-    baseurl=http://download.eng.bos.redhat.com/odcs/prod/odcs-2222222/compose/Temporary/$basearch/os
-    type=rpm-md
-    skip_if_unavailable=False
-    gpgcheck=0
-    repo_gpgcheck=0
-    enabled=1
-    enabled_metadata=1
-    """
-    mock_compose_url: str = (
-        "http://download.eng.bos.redhat.com/odcs/prod/odcs-222222"
-        "/compose/Temporary/odcs-2222222.repo"
-    )
-    odcs_ref = ODCSRequestReference(compose_url=mock_compose_url)
+    req_ref = ODCSRequestReference(compose_urls=urls)
+    fetcher = ODCSFetcher(compose_dir_path=tmp_path)
 
-    compose_path = tmp_path / "downloaded_file_dir" / "odcs-2222222.repo"
-    odcs_fetcher = ODCSFetcher(compose_file_path=compose_path)
+    with responses.RequestsMock() as mock:
+        for compose, url in zip(composes, urls):
+            mock.add(responses.GET, url, body=compose)
+        out: ODCSResultReference = fetcher(request_reference=req_ref)
 
-    with responses.RequestsMock() as req_mock:
-        req_mock.add(responses.GET, mock_compose_url, body=expected_content)
-        odcs_result_ref = odcs_fetcher(request_reference=odcs_ref)
+    files_content = [
+        p.read_text(encoding="utf-8") for p in out.compose_dir_path.iterdir()
+    ]
 
-    assert odcs_result_ref.compose_file_path.exists()
-    odcs_result_ref_content = odcs_result_ref.compose_file_path.read_text(
-        encoding="utf-8"
-    )
-    assert odcs_result_ref_content == expected_content
+    assert set(composes) == set(files_content)