From 2c83b5a41529f560bbba60d513d01448d57e4462 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mon, 2 Dec 2024 11:47:01 +0100 Subject: [PATCH] Use layers digests for comparing podman images Previous method checked default digest provided by podman. This digest is "local" and changed every time image is saved/load or at any other point manifest is modified. This doesn't mean that it is a different image. Viable way for our purposes is to compare that all layers are identical and in the same order. Simple way to distill this into one value is to concatenate individual layers' digests in order of appearance in RootFS. Related to discussion at https://github.com/containers/podman/discussions/24818 --- mock/docs/buildroot-lock-schema-1.0.0.json | 4 +-- mock/py/mockbuild/buildroot.py | 4 +-- mock/py/mockbuild/config.py | 4 +-- mock/py/mockbuild/plugins/buildroot_lock.py | 4 +-- mock/py/mockbuild/podman.py | 33 +++++++++++++++---- mock/tests/test_buildroot_lock.py | 4 +-- .../release-notes-next/podman-digests.feature | 2 ++ 7 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 releng/release-notes-next/podman-digests.feature diff --git a/mock/docs/buildroot-lock-schema-1.0.0.json b/mock/docs/buildroot-lock-schema-1.0.0.json index 9a4cdb25d..b517e9dc8 100644 --- a/mock/docs/buildroot-lock-schema-1.0.0.json +++ b/mock/docs/buildroot-lock-schema-1.0.0.json @@ -82,8 +82,8 @@ "type": "object", "additionalProperties": false, "properties": { - "image_digest": { - "description": "Digest got by the 'podman image inspect --format {{ .Digest }}' command, sha256 string", + "image_layers_digest": { + "description": "SHA256 digest concatenated RootFS layer digests and Config section from 'podman image inspect' command, sha256 string", "type": "string" } } diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index 2cb3eaf5a..5c724e5e0 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -279,11 +279,11 @@ def _fallback(message): "bootstrap.tar") podman.import_tarball(tarball) - digest_expected = self.config.get("image_assert_digest", None) + digest_expected = self.config.get("image_assert_digest", True) if digest_expected: getLog().info("Checking image digest: %s", digest_expected) - digest = podman.get_image_digest() + digest = podman.get_layers_digest() if digest != digest_expected: getLog().warning( f"Expected digest for image {podman.image} is" diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index 22da41426..78662c3dd 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -102,7 +102,7 @@ def setup_default_config_opts(): config_opts['bootstrap_image_ready'] = False config_opts['bootstrap_image_fallback'] = True config_opts['bootstrap_image_keep_getting'] = 120 - config_opts['bootstrap_image_assert_digest'] = None + config_opts['bootstrap_image_assert_digest'] = True config_opts['internal_dev_setup'] = True @@ -786,7 +786,7 @@ def process_hermetic_build_config(cmdline_opts, config_opts): # With hermetic builds, we always assert that we are reproducing the build # with the same image. - config_opts["bootstrap_image_assert_digest"] = data["bootstrap"]["image_digest"] + config_opts["bootstrap_image_assert_digest"] = data["bootstrap"]["image_layers_digest"] @traceLog() diff --git a/mock/py/mockbuild/plugins/buildroot_lock.py b/mock/py/mockbuild/plugins/buildroot_lock.py index 13d5b74fc..f37d5352b 100644 --- a/mock/py/mockbuild/plugins/buildroot_lock.py +++ b/mock/py/mockbuild/plugins/buildroot_lock.py @@ -103,9 +103,9 @@ def _executor(cmd): with self.buildroot.uid_manager.elevated_privileges(): podman = Podman(self.buildroot, data["config"]["bootstrap_image"]) - digest = podman.get_image_digest() + digest = podman.get_layers_digest() data["bootstrap"] = { - "image_digest": digest, + "image_layers_digest": digest, } with open(out_file, "w", encoding="utf-8") as fdlist: diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index 7b96c3c6a..7881ce495 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # vim: noai:ts=4:sw=4:expandtab +import hashlib +import json import os import logging import subprocess @@ -114,21 +116,38 @@ def mounted_image(self): subprocess.run(cmd_umount, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - def get_image_digest(self): + def get_layers_digest(self): """ - Get the "sha256:..." string for the image we work with. + Get sha256 digest of RootFS layers. This must be identical for + all images containing same order of layers, thus it can be used + as the check that we've loaded same image. """ - check = [self.podman_binary, "image", "inspect", self.image, - "--format", "{{ .Digest }}"] + check = [self.podman_binary, "image", "inspect", self.image] result = subprocess.run(check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, encoding="utf8") if result.returncode: raise BootstrapError(f"Can't get {self.image} podman image digest: {result.stderr}") result = result.stdout.strip() - if len(result.splitlines()) != 1: - raise BootstrapError(f"The digest of {self.image} image is not a single-line string") - return result + + try: + data = json.loads(result)[0] + except json.JSONDecodeError as exc: + raise BootstrapError(f"The manifest data of {self.image} " + "are not json-formatted.") from exc + if 'RootFS' not in data: + raise BootstrapError(f"RootFS section of {self.image} is missing.") + if data['RootFS']['Type'] != 'layers': + raise BootstrapError(f"Unexpected format for RootFS in {self.image}.") + + # data which should be sufficient to confirm the image + data = { + 'RootFS': data['RootFS'], + 'Config': data['Config'], + } + # convert to json string with ordered dicts and create hash + data = json.dumps(data, sort_keys=True) + return hashlib.sha256(data.encode()).hexdigest() @traceLog() def cp(self, destination, tar_cmd): diff --git a/mock/tests/test_buildroot_lock.py b/mock/tests/test_buildroot_lock.py index 9a89ee3b2..e0b270369 100644 --- a/mock/tests/test_buildroot_lock.py +++ b/mock/tests/test_buildroot_lock.py @@ -60,7 +60,7 @@ }] }, "bootstrap": { - "image_digest": "sha256:ba1067bef190fbe88f085bd019464a8c0803b7cd1e3f", + "image_layers_digest": "sha256:ba1067bef190fbe88f085bd019464a8c0803b7cd1e3f", }, 'config': { 'bootstrap_image': 'foo', @@ -99,7 +99,7 @@ def _call_method(plugins, buildroot): _, method = plugins.add_hook.call_args[0] podman_obj = MagicMock() - podman_obj.get_image_digest.return_value = EXPECTED_OUTPUT["bootstrap"]["image_digest"] + podman_obj.get_layers_digest.return_value = EXPECTED_OUTPUT["bootstrap"]["image_layers_digest"] podman_cls = MagicMock(return_value=podman_obj) with patch("mockbuild.plugins.buildroot_lock.Podman", side_effect=podman_cls): method() diff --git a/releng/release-notes-next/podman-digests.feature b/releng/release-notes-next/podman-digests.feature new file mode 100644 index 000000000..de84d3814 --- /dev/null +++ b/releng/release-notes-next/podman-digests.feature @@ -0,0 +1,2 @@ +Hermetic build process is enhanced by adding used imaged digests into the +metadata and confirming that exactly same image is used in the next step.