From 81babf46e9197966ef87d073860ea2d93d5e8b30 Mon Sep 17 00:00:00 2001 From: Jonathan Gangi Date: Fri, 22 Nov 2024 13:38:30 -0300 Subject: [PATCH] feat(SPSTRAT-464): add wrapper for marketplacesvm This commit introduces a wrapper for `pubtools-marketplacesvm` which aims to be eventually used to support VM images pushes on various cloudmarketplaces (such as AWS and Azure). With this wrapper it will be possible to easily integrate the tooling into Tekton CI using the Cloud Staged structure as input. Refers to SPSTRAT-464 Signed-off-by: Jonathan Gangi --- Dockerfile | 5 +- .../marketplacesvm_push_wrapper | 1 + .../marketplacesvm_push_wrapper.py | 149 ++++++++++++++++++ .../test_marketplacesvm_push_wrapper.py | 144 +++++++++++++++++ requirements.txt | 1 + 5 files changed, 299 insertions(+), 1 deletion(-) create mode 120000 pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper create mode 100755 pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py create mode 100644 pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py diff --git a/Dockerfile b/Dockerfile index 9d7c01e..64901f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,8 @@ RUN pip3 install jinja2 \ packageurl-python \ pubtools-content-gateway==${PUBTOOLS_CGW_VERSION} \ pubtools-pulp \ - pubtools-exodus + pubtools-exodus \ + pubtools-marketplacesvm # remove gcc, required only for compiling gssapi indirect dependency of pubtools-pulp via pushsource RUN dnf -y remove gcc @@ -60,6 +61,7 @@ COPY pyxis /home/pyxis COPY utils /home/utils COPY templates /home/templates COPY pubtools-pulp-wrapper /home/pubtools-pulp-wrapper +COPY pubtools-marketplacesvm-wrapper /home/pubtools-marketplacesvm-wrapper COPY developer-portal-wrapper /home/developer-portal-wrapper COPY sbom /home/sbom @@ -77,5 +79,6 @@ ENV HOME=/tekton/home ENV PATH="$PATH:/home/pyxis" ENV PATH="$PATH:/home/utils" ENV PATH="$PATH:/home/pubtools-pulp-wrapper" +ENV PATH="$PATH:/home/pubtools-marketplacesvm-wrapper" ENV PATH="$PATH:/home/developer-portal-wrapper" ENV PATH="$PATH:/home/sbom" diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper new file mode 120000 index 0000000..2903306 --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper @@ -0,0 +1 @@ +marketplacesvm_push_wrapper.py \ No newline at end of file diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py new file mode 100755 index 0000000..72a305d --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Python script to push staged content to various cloud marketplaces. + +This is a simple wrapper pubtools-marketplacesvm-push command that is able to push and publish +content to various cloud marketplaces. This wrapper supports pushing cloud images only from +staged source using the cloud schema. + +For more information please refer to documentation: +* https://github.com/release-engineering/pubtools-marketplacesvm +* https://release-engineering.github.io/pushsource/sources/staged.html#root-destination-cloud-images # noqa:E501 +* https://release-engineering.github.io/pushsource/schema/cloud.html#cloud-schema + +Red Hat Slack channel: +#stratosphere-bau +""" +import argparse +import logging +import os +import re +import subprocess +import sys + +LOG = logging.getLogger("pubtools-marketplacesvm-wrapper") +DEFAULT_LOG_FMT = "%(asctime)s [%(levelname)-8s] %(message)s" +DEFAULT_DATE_FMT = "%Y-%m-%d %H:%M:%S %z" +COMMAND = "pubtools-marketplacesvm-marketplace-push" +CLOUD_MKTS_ENV_VARS_STRICT = ("CLOUD_CREDENTIALS",) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog="marketplacesvm_push_wrapper", + description="Push staged cloud images to various cloud marketplaces.", + ) + + parser.add_argument("--dry-run", action="store_true", help="Log command to be executed") + parser.add_argument( + "--debug", + "-d", + action="count", + default=0, + help=("Show debug logs; can be provided up to three times to enable more logs"), + ) + + parser.add_argument( + "--source", + action="append", + help="Path(s) to staging directory", + required=True, + ) + + parser.add_argument( + "--nochannel", + action="store_true", + help=( + "Do as much as possible without making content available to end-users, then stop." + "May be used to improve the performance of a subsequent full push." + ), + ) + + starmap = parser.add_argument_group("Content mapping settings") + starmap.add_argument( + "--starmap-file", + help="YAML file containing the content mappings on StArMap APIv2 format.", + required=True, + ) + + return parser.parse_args() + + +def get_source_url(stagedirs): + regex = re.compile(r"^/[\w++/*]+$") + for item in stagedirs: + if not regex.match(item): + raise ValueError("Not a valid staging directory: %s" % item) + + return f"staged:{','.join(stagedirs)}" + + +def settings_to_args(parsed): + settings_to_arg_map = { + "starmap_file": "--repo-file", + "source": "", + } + out = ["--offline"] # The "offline" arg is used to prevent invoking a StArMap server + if parsed.nochannel: + out.append("--pre-push") + + for setting, arg in settings_to_arg_map.items(): + if value := getattr(parsed, setting): + if arg: + out.extend([arg]) + out.extend([value]) + + for _ in range(parsed.debug): + out.append("--debug") + + return out + + +def validate_env_vars(args): + assert all([os.getenv(item) for item in CLOUD_MKTS_ENV_VARS_STRICT]), ( + "Provide all required CLOUD_MKTS environment variables: " + f"{', '.join(CLOUD_MKTS_ENV_VARS_STRICT)}" + ) + + args.source = get_source_url(args.source) + return args + + +def main(): + args = validate_env_vars(parse_args()) + + loglevel = logging.DEBUG if args.debug else logging.INFO + + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(loglevel) + + logging.basicConfig( + level=loglevel, + format=DEFAULT_LOG_FMT, + datefmt=DEFAULT_DATE_FMT, + handlers=[stream_handler], + ) + + if args.dry_run: + LOG.info("This is a dry-run!") + + cmd_args = settings_to_args(args) + command = [COMMAND] + cmd_args + cmd_str = " ".join(command) + + if args.dry_run: + LOG.info("Would have run: %s", cmd_str) + else: + try: + LOG.info("Running %s", cmd_str) + subprocess.run(command, check=True) + except subprocess.CalledProcessError: + LOG.exception("Command %s failed, check exception for details", cmd_str) + raise + except Exception as exc: + LOG.exception("Unknown error occurred") + raise RuntimeError from exc + + +if __name__ == "__main__": + main() diff --git a/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py new file mode 100644 index 0000000..2628c9b --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py @@ -0,0 +1,144 @@ +import logging +import os +import sys +from unittest.mock import patch + +import pytest + +import marketplacesvm_push_wrapper + + +@pytest.fixture() +def mock_mkt_env_vars(): + with patch.dict( + os.environ, {k: "test" for k in marketplacesvm_push_wrapper.CLOUD_MKTS_ENV_VARS_STRICT} + ): + yield + + +def test_no_args(capsys): + with pytest.raises(SystemExit): + marketplacesvm_push_wrapper.main() + + _, err = capsys.readouterr() + assert ( + "marketplacesvm_push_wrapper: error: the following arguments are required:" + " --source, --starmap-file" + ) in err + + +def test_dry_run(caplog, mock_mkt_env_vars): + args = [ + "", + "--dry-run", + "--source", + "/test1/starmap", + "--source", + "/test2/starmap", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" in caplog.messages + assert ( + "Would have run: pubtools-marketplacesvm-marketplace-push " + "--offline --repo-file mapping.yaml " + "staged:/test1/starmap,/test2/starmap" + ) in caplog.messages + + +@patch("subprocess.run") +def test_basic_command(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--source", + "/test1/starmap", + "--source", + "/test2/starmap", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--offline --repo-file mapping.yaml " + "staged:/test1/starmap,/test2/starmap" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--offline", + "--repo-file", + "mapping.yaml", + "staged:/test1/starmap,/test2/starmap", + ], + check=True, + ) + + +@patch("subprocess.run") +def test_basic_command_nochannel(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--nochannel", + "--source", + "/test1/starmap", + "--source", + "/test2/starmap", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--offline --pre-push --repo-file mapping.yaml " + "staged:/test1/starmap,/test2/starmap" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--offline", + "--pre-push", + "--repo-file", + "mapping.yaml", + "staged:/test1/starmap,/test2/starmap", + ], + check=True, + ) + + +@pytest.mark.parametrize( + "stageddirs", + [ + ["/foo/bar/"], + ["/a", "/tmp/foo/"], + ["/a/b/c/d/e/f/g/h/i", "/a1/a2/a3/", "/f"], + ], +) +def test_get_source_url(stageddirs): + res = marketplacesvm_push_wrapper.get_source_url(stageddirs) + + assert res == f"staged:{','.join(stageddirs)}" + + +@pytest.mark.parametrize( + "stageddirs", + [["foo"], ["foo/bar"], [r"/\/\/\/\/\/\/"]], +) +def test_get_source_url_invalid(stageddirs): + err = "Not a valid staging directory:" + with pytest.raises(ValueError, match=err): + marketplacesvm_push_wrapper.get_source_url(stageddirs) diff --git a/requirements.txt b/requirements.txt index 4df9895..e6394b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ packageurl-python pubtools-pulp pubtools-exodus pubtools-content-gateway +pubtools-marketplacesvm