From b20c9d07d471f4f3586b95723105c1a45df2d670 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 28 Oct 2024 19:17:26 +0100 Subject: [PATCH] podman: always tag/untag the images we work with locally For `podman image inspect` we need to have a tagged image in the local image database. For the buildroot images then (these that will be generated by 'buildah commit' and 'podman save' in the following commits) we would have no tags at all, without a reasonable way to garbage collect them. These buildroot images (and tags too) are rather ephemeral, related to a single Mock command run only. That's why we assign a random tag name to it (UUID string), and why we later "untag" the image to not bloat the local tag/image database. --- mock/docs/site-defaults.cfg | 3 + mock/py/mockbuild/buildroot.py | 7 +- mock/py/mockbuild/config.py | 2 + mock/py/mockbuild/plugins/buildroot_lock.py | 11 ++- mock/py/mockbuild/podman.py | 83 ++++++++++++++++----- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index b4413b7e2..048425cc8 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -401,6 +401,9 @@ # config['rootdir'] = '/var/lib/mock//root/' ## This works in F25+ chroots. This overrides 'use_container_host_hostname' option # config_opts['macros']['%_buildhost'] = 'my.own.hostname' +# +# Each Mock run has a unique UUID +#config_opts["mock_run_uuid"] = str(uuid.uuid4()) ############################################################################# # diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index b495a90e6..3643636c1 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -272,8 +272,10 @@ def _fallback(message): if not self.config["image_skip_pull"]: podman.retry_image_pull(self.config["image_keep_getting"]) else: - getLog().info("Using local image %s (pull skipped)", - self.chroot_image) + podman.read_image_id() + getLog().info("Using local image %s (%s)", + self.chroot_image, podman.image_id) + podman.tag_image() if self.is_bootstrap and self.config["hermetic_build"]: tarball = os.path.join(self.config["offline_local_repository"], @@ -294,6 +296,7 @@ def _fallback(message): raise BootstrapError("Container image architecture check failed") podman.cp(self.make_chroot_path(), self.config["tar_binary"]) + podman.untag() file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), "etc/rpm/macros.image-language-conf")) except _FallbackException as exc: diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index 8aa7cde31..4056edfe7 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -16,6 +16,7 @@ import shlex import socket import sys +import uuid import warnings from templated_dictionary import TemplatedDictionary @@ -411,6 +412,7 @@ def setup_default_config_opts(): config_opts["calculatedeps"] = None config_opts["hermetic_build"] = False + config_opts["mock_run_uuid"] = str(uuid.uuid4()) return config_opts diff --git a/mock/py/mockbuild/plugins/buildroot_lock.py b/mock/py/mockbuild/plugins/buildroot_lock.py index 13d5b74fc..32d267f6f 100644 --- a/mock/py/mockbuild/plugins/buildroot_lock.py +++ b/mock/py/mockbuild/plugins/buildroot_lock.py @@ -7,7 +7,7 @@ import json import os -from mockbuild.podman import Podman +from mockbuild.podman import Podman, PodmanError from mockbuild.installed_packages import query_packages, query_packages_location requires_api_version = "1.1" @@ -101,9 +101,12 @@ def _executor(cmd): # produce lockfiles even if these are useless for hermetic # builds). with self.buildroot.uid_manager.elevated_privileges(): - podman = Podman(self.buildroot, - data["config"]["bootstrap_image"]) - digest = podman.get_image_digest() + try: + podman = Podman(self.buildroot, + data["config"]["bootstrap_image"]) + digest = podman.get_image_digest() + except PodmanError: + digest = "unknown" data["bootstrap"] = { "image_digest": digest, } diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index b13fb87f7..a1d714f7b 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -60,7 +60,7 @@ def __init__(self, buildroot, image): self.buildroot = buildroot self.image = image - self.container_id = None + self.image_id = None getLog().info("Using container image: %s", image) @traceLog() @@ -69,11 +69,33 @@ def pull_image(self): logger = getLog() logger.info("Pulling image: %s", self.image) cmd = [self.podman_binary, "pull", self.image] - out, exit_status = util.do_with_status(cmd, env=self.buildroot.env, - raiseExc=False, returnOutput=1) - if exit_status: - logger.error(out) - return not exit_status + + res = subprocess.run(cmd, env=self.buildroot.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=False) + if res.returncode != 0: + logger.error("%s\n%s", res.stdout, res.stderr) + return False + + # Record the image id for later use. This is needed for the + # oci-image:tarball images that not necessarily have tags/names. + self.image_id = res.stdout.decode("utf-8").strip() + return True + + @property + def _tagged_id(self): + uuid = self.buildroot.config["mock_run_uuid"] + bootstrap = "-bootstrap" if self.buildroot.is_bootstrap else "" + return f"mock{bootstrap}-{uuid}" + + def tag_image(self): + """ + Tag the pulled image as mock-{uuid}, or mock-bootstrap-{uuid}. + """ + cmd = ["podman", "tag", self.image_id, self._tagged_id] + getLog().info("Tagging container image as %s", self._tagged_id) + subprocess.run(cmd, env=self.buildroot.env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) def import_tarball(self, tarball): """ @@ -99,8 +121,8 @@ def mounted_image(self): chroot directory. """ logger = getLog() - cmd_mount = [self.podman_binary, "image", "mount", self.image] - cmd_umount = [self.podman_binary, "image", "umount", self.image] + cmd_mount = [self.podman_binary, "image", "mount", self.image_id] + cmd_umount = [self.podman_binary, "image", "umount", self.image_id] result = subprocess.run(cmd_mount, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, encoding="utf8") @@ -109,13 +131,13 @@ def mounted_image(self): raise PodmanError(message) mountpoint = result.stdout.strip() - logger.info("mounting %s with podman image mount", self.image) + logger.info("mounting %s with podman image mount", self.image_id) try: - logger.info("image %s as %s", self.image, mountpoint) + logger.info("image %s as %s", self.image_id, mountpoint) yield mountpoint finally: logger.info("umounting image %s (%s) with podman image umount", - self.image, mountpoint) + self.image_id, mountpoint) subprocess.run(cmd_umount, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) @@ -123,16 +145,21 @@ def get_image_digest(self): """ Get the "sha256:..." string for the image we work with. """ - check = [self.podman_binary, "image", "inspect", self.image, + the_image = self.image + if the_image.startswith("oci-archive:"): + # We can't query digest from tarball directly, but note + # the image needs to be tagged first! + the_image = self._tagged_id + check = [self.podman_binary, "image", "inspect", the_image, "--format", "{{ .Digest }}"] result = subprocess.run(check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, encoding="utf8") if result.returncode: - raise PodmanError(f"Can't get {self.image} podman image digest: {result.stderr}") + raise PodmanError(f"Can't get {the_image} podman image digest: {result.stderr}") result = result.stdout.strip() if len(result.splitlines()) != 1: - raise PodmanError(f"The digest of {self.image} image is not a single-line string") + raise PodmanError(f"The digest of {the_image} image is not a single-line string") return result def check_native_image_architecture(self): @@ -140,12 +167,12 @@ def check_native_image_architecture(self): Check that self.image has been generated for the current host's architecture. """ - return podman_check_native_image_architecture(self.image, getLog()) + return podman_check_native_image_architecture(self.image_id, getLog()) @traceLog() def cp(self, destination, tar_cmd): """ copy content of container to destination directory """ - getLog().info("Copy content of container %s to %s", self.image, destination) + getLog().info("Copy content of container %s to %s", self.image_id, destination) with self.mounted_image() as mount_path: # pipe-out the temporary mountpoint with the help of tar utility @@ -158,5 +185,27 @@ def cp(self, destination, tar_cmd): tar.communicate() podman.communicate() + def untag(self): + """ + Remove the additional image ID we created - which means the image itself + is garbage-collected if there's no other tag. + """ + cmd = ["podman", "rmi", self._tagged_id] + getLog().info("Removing image %s", self._tagged_id) + subprocess.run(cmd, env=self.buildroot.env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) + + def read_image_id(self): + """ + Given self.image (name), get the image Id. + """ + cmd = ["podman", "image", "inspect", self.image, "--format", + "{{ .Id }}"] + getLog().info("Removing image %s", self.image) + res = subprocess.run(cmd, env=self.buildroot.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=True) + self.image_id = res.stdout.decode("utf-8").strip() + def __repr__(self): - return "Podman({}({}))".format(self.image, self.container_id) + return "Podman({}({}))".format(self.image, self.image_id)