diff --git a/Dockerfile b/Dockerfile index 9d7c01e..64901f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,8 @@ RUN pip3 install jinja2 \ packageurl-python \ pubtools-content-gateway==${PUBTOOLS_CGW_VERSION} \ pubtools-pulp \ - pubtools-exodus + pubtools-exodus \ + pubtools-marketplacesvm # remove gcc, required only for compiling gssapi indirect dependency of pubtools-pulp via pushsource RUN dnf -y remove gcc @@ -60,6 +61,7 @@ COPY pyxis /home/pyxis COPY utils /home/utils COPY templates /home/templates COPY pubtools-pulp-wrapper /home/pubtools-pulp-wrapper +COPY pubtools-marketplacesvm-wrapper /home/pubtools-marketplacesvm-wrapper COPY developer-portal-wrapper /home/developer-portal-wrapper COPY sbom /home/sbom @@ -77,5 +79,6 @@ ENV HOME=/tekton/home ENV PATH="$PATH:/home/pyxis" ENV PATH="$PATH:/home/utils" ENV PATH="$PATH:/home/pubtools-pulp-wrapper" +ENV PATH="$PATH:/home/pubtools-marketplacesvm-wrapper" ENV PATH="$PATH:/home/developer-portal-wrapper" ENV PATH="$PATH:/home/sbom" diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper new file mode 120000 index 0000000..2903306 --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper @@ -0,0 +1 @@ +marketplacesvm_push_wrapper.py \ No newline at end of file diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py new file mode 100755 index 0000000..d733d8e --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Python script to push staged content to various cloud marketplaces. + +This is a simple wrapper pubtools-marketplacesvm-push command that is able to push and publish +content to various cloud marketplaces. This wrapper supports pushing cloud images only from +staged source using the cloud schema. + +For more information please refer to documentation: +* https://github.com/release-engineering/pubtools-marketplacesvm +* https://release-engineering.github.io/pushsource/sources/staged.html#root-destination-cloud-images +* https://release-engineering.github.io/pushsource/schema/cloud.html#cloud-schema +""" +import argparse +import logging +import os +import re +import subprocess +import sys + +LOG = logging.getLogger("pubtools-marketplacesvm-wrapper") +DEFAULT_LOG_FMT = "%(asctime)s [%(levelname)-8s] %(message)s" +DEFAULT_DATE_FMT = "%Y-%m-%d %H:%M:%S %z" +COMMAND = "pubtools-marketplacesvm-marketplace-push" +CLOUD_MKTS_ENV_VARS_STRICT = ("CLOUD_CREDENTIALS",) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog="marketplacesvm_push_wrapper", + description="Push staged cloud images to various cloud marketplaces.", + ) + + parser.add_argument("--dry-run", action="store_true", help="Log command to be executed") + parser.add_argument( + "--debug", + "-d", + action="count", + default=0, + help=("Show debug logs; can be provided up to three times to enable more logs"), + ) + + parser.add_argument( + "--source", + action="append", + help="Path(s) to staging directory", + required=True, + ) + + parser.add_argument( + "--nochannel", + action="store_true", + help=( + "Do as much as possible without making content available to end-users, then stop." + "May be used to improve the performance of a subsequent full push." + ), + ) + + starmap = parser.add_argument_group("Content mapping settings") + starmap.add_argument( + "--starmap-file", + help="YAML file containing the content mappings on StArMap APIv2 format.", + required=True, + ) + + return parser.parse_args() + + +def get_source_url(stagedirs): + for item in stagedirs: + if not re.match(r"^/[^,]{1,4000}/starmap/\w+\/*$", item): + raise ValueError("Not a valid staging directory: %s" % item) + + return f"staged:{','.join(stagedirs)}" + + +def settings_to_args(parsed): + settings_to_arg_map = { + 'starmap_file': '--repo-file', + "source": "", + } + out = [] + if parsed.nochannel: + out.append("--pre-push") + + for setting, arg in settings_to_arg_map.items(): + if value := getattr(parsed, setting): + if arg: + out.extend([arg]) + out.extend([value]) + + for _ in range(parsed.debug): + out.append("--debug") + + return out + + +def validate_env_vars(args): + assert all([os.getenv(item) for item in CLOUD_MKTS_ENV_VARS_STRICT]), ( + f"Provide all required CLOUD_MKTS environment variables: " + "{', '.join(CLOUD_MKTS_ENV_VARS_STRICT)}" + ) + + args.source = get_source_url(args.source) + return args + + +def main(): + args = validate_env_vars(parse_args()) + + loglevel = logging.DEBUG if args.debug else logging.INFO + + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(loglevel) + + logging.basicConfig( + level=loglevel, + format=DEFAULT_LOG_FMT, + datefmt=DEFAULT_DATE_FMT, + handlers=[stream_handler], + ) + + if args.dry_run: + LOG.info("This is a dry-run!") + + cmd_args = settings_to_args(args) + command = [COMMAND] + cmd_args + cmd_str = " ".join(command) + + if args.dry_run: + LOG.info("Would have run: %s", cmd_str) + else: + try: + LOG.info("Running %s", cmd_str) + subprocess.run(command, check=True) + except subprocess.CalledProcessError: + LOG.exception("Command %s failed, check exception for details", cmd_str) + raise + except Exception as exc: + LOG.exception("Unknown error occurred") + raise RuntimeError from exc + + +def entrypoint(): + main() + + +if __name__ == "__main__": + entrypoint() diff --git a/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py new file mode 100644 index 0000000..009efce --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py @@ -0,0 +1,118 @@ +import logging +import os +import sys +from unittest.mock import patch + +import pytest + +import marketplacesvm_push_wrapper + + +@pytest.fixture() +def mock_mkt_env_vars(): + with patch.dict( + os.environ, {k: "test" for k in marketplacesvm_push_wrapper.CLOUD_MKTS_ENV_VARS_STRICT} + ): + yield + + +def test_no_args(capsys): + with pytest.raises(SystemExit): + marketplacesvm_push_wrapper.main() + + _, err = capsys.readouterr() + assert ( + "marketplacesvm_push_wrapper: error: the following arguments are required: --source, --starmap-file" + in err + ) + + +def test_dry_run(caplog, mock_mkt_env_vars): + args = [ + "", + "--dry-run", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" in caplog.messages + assert ( + "Would have run: pubtools-marketplacesvm-marketplace-push " + "--repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + +@patch("subprocess.run") +def test_basic_command(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--repo-file", + "mapping.yaml", + "staged:/test/starmap/1,/test/starmap/2", + ], + check=True, + ) + + +@patch("subprocess.run") +def test_basic_command_nochannel(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--nochannel", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--pre-push --repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--pre-push", + "--repo-file", + "mapping.yaml", + "staged:/test/starmap/1,/test/starmap/2", + ], + check=True, + ) diff --git a/requirements.txt b/requirements.txt index 4df9895..e6394b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ packageurl-python pubtools-pulp pubtools-exodus pubtools-content-gateway +pubtools-marketplacesvm