feat(SPSTRAT-464): add wrapper for marketplacesvm

This commit introduces a wrapper for `pubtools-marketplacesvm` which
aims to be eventually used to support VM images pushes on various
cloudmarketplaces (such as AWS and Azure).

With this wrapper it will be possible to easily integrate the tooling
into Tekton CI using the Cloud Staged structure as input.

Refers to SPSTRAT-464

Signed-off-by: Jonathan Gangi <[email protected]>

Dockerfile

@@ -48,7 +48,8 @@ RUN pip3 install jinja2 \
     packageurl-python \
     pubtools-content-gateway==${PUBTOOLS_CGW_VERSION} \
     pubtools-pulp \
-    pubtools-exodus
+    pubtools-exodus \
+    pubtools-marketplacesvm
 
 # remove gcc, required only for compiling gssapi indirect dependency of pubtools-pulp via pushsource
 RUN dnf -y remove gcc
@@ -60,6 +61,7 @@ COPY pyxis /home/pyxis
 COPY utils /home/utils
 COPY templates /home/templates
 COPY pubtools-pulp-wrapper /home/pubtools-pulp-wrapper
+COPY pubtools-marketplacesvm-wrapper /home/pubtools-marketplacesvm-wrapper
 COPY developer-portal-wrapper /home/developer-portal-wrapper
 COPY sbom /home/sbom
 
@@ -77,5 +79,6 @@ ENV HOME=/tekton/home
 ENV PATH="$PATH:/home/pyxis"
 ENV PATH="$PATH:/home/utils"
 ENV PATH="$PATH:/home/pubtools-pulp-wrapper"
+ENV PATH="$PATH:/home/pubtools-marketplacesvm-wrapper"
 ENV PATH="$PATH:/home/developer-portal-wrapper"
 ENV PATH="$PATH:/home/sbom"

pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper

@@ -0,0 +1 @@
+marketplacesvm_push_wrapper.py

pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""
+Python script to push staged content to various cloud marketplaces.
+
+This is a simple wrapper pubtools-marketplacesvm-push command that is able to push and publish
+content to various cloud marketplaces. This wrapper supports pushing cloud images only from
+staged source using the cloud schema.
+
+For more information please refer to documentation:
+* https://github.com/release-engineering/pubtools-marketplacesvm
+* https://release-engineering.github.io/pushsource/sources/staged.html#root-destination-cloud-images  # noqa:E501
+* https://release-engineering.github.io/pushsource/schema/cloud.html#cloud-schema
+
+Red Hat Slack channel:
+#stratosphere-bau
+"""
+import argparse
+import logging
+import os
+import re
+import subprocess
+import sys
+
+LOG = logging.getLogger("pubtools-marketplacesvm-wrapper")
+DEFAULT_LOG_FMT = "%(asctime)s [%(levelname)-8s] %(message)s"
+DEFAULT_DATE_FMT = "%Y-%m-%d %H:%M:%S %z"
+COMMAND = "pubtools-marketplacesvm-marketplace-push"
+CLOUD_MKTS_ENV_VARS_STRICT = ("CLOUD_CREDENTIALS",)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog="marketplacesvm_push_wrapper",
+        description="Push staged cloud images to various cloud marketplaces.",
+    )
+
+    parser.add_argument("--dry-run", action="store_true", help="Log command to be executed")
+    parser.add_argument(
+        "--debug",
+        "-d",
+        action="count",
+        default=0,
+        help=("Show debug logs; can be provided up to three times to enable more logs"),
+    )
+
+    parser.add_argument(
+        "--source",
+        action="append",
+        help="Path(s) to staging directory",
+        required=True,
+    )
+
+    parser.add_argument(
+        "--nochannel",
+        action="store_true",
+        help=(
+            "Do as much as possible without making content available to end-users, then stop."
+            "May be used to improve the performance of a subsequent full push."
+        ),
+    )
+
+    starmap = parser.add_argument_group("Content mapping settings")
+    starmap.add_argument(
+        "--starmap-file",
+        help="YAML file containing the content mappings on StArMap APIv2 format.",
+        required=True,
+    )
+
+    return parser.parse_args()
+
+
+def get_source_url(stagedirs):
+    regex = re.compile(r"^/[\w++/*]+$")
+    for item in stagedirs:
+        if not regex.match(item):
+            raise ValueError("Not a valid staging directory: %s" % item)
+
+    return f"staged:{','.join(stagedirs)}"
+
+
+def settings_to_args(parsed):
+    settings_to_arg_map = {
+        "starmap_file": "--repo-file",
+        "source": "",
+    }
+    out = ["--offline"]  # The "offline" arg is used to prevent invoking a StArMap server
+    if parsed.nochannel:
+        out.append("--pre-push")
+
+    for setting, arg in settings_to_arg_map.items():
+        if value := getattr(parsed, setting):
+            if arg:
+                out.extend([arg])
+            out.extend([value])
+
+    for _ in range(parsed.debug):
+        out.append("--debug")
+
+    return out
+
+
+def validate_env_vars(args):
+    assert all([os.getenv(item) for item in CLOUD_MKTS_ENV_VARS_STRICT]), (
+        "Provide all required CLOUD_MKTS environment variables: "
+        f"{', '.join(CLOUD_MKTS_ENV_VARS_STRICT)}"
+    )
+
+    args.source = get_source_url(args.source)
+    return args
+
+
+def main():
+    args = validate_env_vars(parse_args())
+
+    loglevel = logging.DEBUG if args.debug else logging.INFO
+
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(loglevel)
+
+    logging.basicConfig(
+        level=loglevel,
+        format=DEFAULT_LOG_FMT,
+        datefmt=DEFAULT_DATE_FMT,
+        handlers=[stream_handler],
+    )
+
+    if args.dry_run:
+        LOG.info("This is a dry-run!")
+
+    cmd_args = settings_to_args(args)
+    command = [COMMAND] + cmd_args
+    cmd_str = " ".join(command)
+
+    if args.dry_run:
+        LOG.info("Would have run: %s", cmd_str)
+    else:
+        try:
+            LOG.info("Running %s", cmd_str)
+            subprocess.run(command, check=True)
+        except subprocess.CalledProcessError:
+            LOG.exception("Command %s failed, check exception for details", cmd_str)
+            raise
+        except Exception as exc:
+            LOG.exception("Unknown error occurred")
+            raise RuntimeError from exc
+
+
+if __name__ == "__main__":
+    main()

pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py

@@ -0,0 +1,144 @@
+import logging
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+
+import marketplacesvm_push_wrapper
+
+
+@pytest.fixture()
+def mock_mkt_env_vars():
+    with patch.dict(
+        os.environ, {k: "test" for k in marketplacesvm_push_wrapper.CLOUD_MKTS_ENV_VARS_STRICT}
+    ):
+        yield
+
+
+def test_no_args(capsys):
+    with pytest.raises(SystemExit):
+        marketplacesvm_push_wrapper.main()
+
+    _, err = capsys.readouterr()
+    assert (
+        "marketplacesvm_push_wrapper: error: the following arguments are required:"
+        " --source, --starmap-file"
+    ) in err
+
+
+def test_dry_run(caplog, mock_mkt_env_vars):
+    args = [
+        "",
+        "--dry-run",
+        "--source",
+        "/test1/starmap",
+        "--source",
+        "/test2/starmap",
+        "--starmap-file",
+        "mapping.yaml",
+    ]
+
+    with patch.object(sys, "argv", args):
+        with caplog.at_level(logging.INFO):
+            marketplacesvm_push_wrapper.main()
+            assert "This is a dry-run!" in caplog.messages
+            assert (
+                "Would have run: pubtools-marketplacesvm-marketplace-push "
+                "--offline --repo-file mapping.yaml "
+                "staged:/test1/starmap,/test2/starmap"
+            ) in caplog.messages
+
+
+@patch("subprocess.run")
+def test_basic_command(mock_run, caplog, mock_mkt_env_vars):
+    args = [
+        "",
+        "--source",
+        "/test1/starmap",
+        "--source",
+        "/test2/starmap",
+        "--starmap-file",
+        "mapping.yaml",
+    ]
+
+    with patch.object(sys, "argv", args):
+        with caplog.at_level(logging.INFO):
+            marketplacesvm_push_wrapper.main()
+            assert "This is a dry-run!" not in caplog.messages
+            assert (
+                "Running pubtools-marketplacesvm-marketplace-push "
+                "--offline --repo-file mapping.yaml "
+                "staged:/test1/starmap,/test2/starmap"
+            ) in caplog.messages
+
+    mock_run.assert_called_once_with(
+        [
+            "pubtools-marketplacesvm-marketplace-push",
+            "--offline",
+            "--repo-file",
+            "mapping.yaml",
+            "staged:/test1/starmap,/test2/starmap",
+        ],
+        check=True,
+    )
+
+
+@patch("subprocess.run")
+def test_basic_command_nochannel(mock_run, caplog, mock_mkt_env_vars):
+    args = [
+        "",
+        "--nochannel",
+        "--source",
+        "/test1/starmap",
+        "--source",
+        "/test2/starmap",
+        "--starmap-file",
+        "mapping.yaml",
+    ]
+
+    with patch.object(sys, "argv", args):
+        with caplog.at_level(logging.INFO):
+            marketplacesvm_push_wrapper.main()
+            assert "This is a dry-run!" not in caplog.messages
+            assert (
+                "Running pubtools-marketplacesvm-marketplace-push "
+                "--offline --pre-push --repo-file mapping.yaml "
+                "staged:/test1/starmap,/test2/starmap"
+            ) in caplog.messages
+
+    mock_run.assert_called_once_with(
+        [
+            "pubtools-marketplacesvm-marketplace-push",
+            "--offline",
+            "--pre-push",
+            "--repo-file",
+            "mapping.yaml",
+            "staged:/test1/starmap,/test2/starmap",
+        ],
+        check=True,
+    )
+
+
+@pytest.mark.parametrize(
+    "stageddirs",
+    [
+        ["/foo/bar/"],
+        ["/a", "/tmp/foo/"],
+        ["/a/b/c/d/e/f/g/h/i", "/a1/a2/a3/", "/f"],
+    ],
+)
+def test_get_source_url(stageddirs):
+    res = marketplacesvm_push_wrapper.get_source_url(stageddirs)
+
+    assert res == f"staged:{','.join(stageddirs)}"
+
+
+@pytest.mark.parametrize(
+    "stageddirs",
+    [["foo"], ["foo/bar"], [r"/\/\/\/\/\/\/"]],
+)
+def test_get_source_url_invalid(stageddirs):
+    err = "Not a valid staging directory:"
+    with pytest.raises(ValueError, match=err):
+        marketplacesvm_push_wrapper.get_source_url(stageddirs)

requirements.txt

@@ -6,3 +6,4 @@ packageurl-python
 pubtools-pulp
 pubtools-exodus
 pubtools-content-gateway
+pubtools-marketplacesvm