From 538c135deeb679cac8f009af90f88fd86ba6e79c Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 28 Aug 2024 10:38:09 +0200 Subject: [PATCH] Add support for isolated builds There are new options, --calculate-build-dependencies and --isolated-build. New buildroot_lock plugin added. Fixes: #1358 Merges: #1393 Relates: #1380 --- behave/features/environment.py | 1 - behave/features/isolated-build.feature | 22 ++ behave/features/steps/other.py | 46 +++++ behave/testlib.py | 38 +++- docs/Plugin-BuildrootLock.md | 76 +++++++ docs/feature-isolated-builds.md | 194 ++++++++++++++++++ docs/index.md | 2 + mock/docs/buildroot-lock-schema-1.0.0.json | 102 +++++++++ mock/docs/mock.1 | 38 +++- mock/docs/site-defaults.cfg | 11 + mock/etc/mock/isolated-build.cfg | 38 ++++ mock/mock.spec | 14 +- mock/py/mock-isolated-repo.py | 113 ++++++++++ mock/py/mock.py | 28 +++ mock/py/mockbuild/backend.py | 23 ++- mock/py/mockbuild/buildroot.py | 15 ++ mock/py/mockbuild/config.py | 73 ++++++- mock/py/mockbuild/installed_packages.py | 139 +++++++++++++ mock/py/mockbuild/plugins/buildroot_lock.py | 111 ++++++++++ mock/py/mockbuild/podman.py | 24 +++ mock/tests/test_buildroot_lock.py | 151 ++++++++++++++ mock/tests/test_installed_packages.py | 14 ++ .../release-notes-next/isolated-build.feature | 9 + tox.ini | 1 + 24 files changed, 1268 insertions(+), 15 deletions(-) create mode 100644 behave/features/isolated-build.feature create mode 100644 docs/Plugin-BuildrootLock.md create mode 100644 docs/feature-isolated-builds.md create mode 100644 mock/docs/buildroot-lock-schema-1.0.0.json create mode 100644 mock/etc/mock/isolated-build.cfg create mode 100755 mock/py/mock-isolated-repo.py create mode 100644 mock/py/mockbuild/installed_packages.py create mode 100644 mock/py/mockbuild/plugins/buildroot_lock.py create mode 100644 mock/tests/test_buildroot_lock.py create mode 100644 mock/tests/test_installed_packages.py create mode 100644 releng/release-notes-next/isolated-build.feature diff --git a/behave/features/environment.py b/behave/features/environment.py index 68336a383..d30de2804 100644 --- a/behave/features/environment.py +++ b/behave/features/environment.py @@ -44,7 +44,6 @@ def before_all(context): default_config = os.readlink("/etc/mock/default.cfg") context.chroot = default_config[:-4] # drop cfg suffix - context.chroot_used = False context.test_storage = ( "https://github.com/" diff --git a/behave/features/isolated-build.feature b/behave/features/isolated-build.feature new file mode 100644 index 000000000..866c7746f --- /dev/null +++ b/behave/features/isolated-build.feature @@ -0,0 +1,22 @@ +Feature: Mock 5.7+ supports isolated builds + + @isolated_build + Scenario: Isolated build against a DNF5 distribution + Given an unique mock namespace + When deps for python-copr-999-1.src.rpm are calculated against fedora-rawhide-x86_64 + And a local repository is created from lockfile + And an isolated build is retriggered with the lockfile and repository + Then the build succeeds + And the produced lockfile is validated properly + + # We'd like to use centos-stream+epel-9 here right now, but c9s is not + # bootstrap_image_ready=True :-( and rhel-9 packages are CDN (RHSM) + # only. We SHOULD be fine with c10s once container images are available. + # Fedora 40 is DNF4 but not "bootstrap_image_ready" either :-( + #Scenario: Isolated build against a DNF4 distribution + # Given an unique mock namespace + # When deps for mock-test-bump-version-1-0.src.rpm are calculated against centos-stream+epel-10-x86_64 + # And a local repository is created from lockfile + # And an isolated build is retriggered with the lockfile and repository + # Then the build succeeds + # And the produced lockfile is validated properly diff --git a/behave/features/steps/other.py b/behave/features/steps/other.py index 1fa98c8d5..7b340f265 100644 --- a/behave/features/steps/other.py +++ b/behave/features/steps/other.py @@ -2,6 +2,7 @@ import glob import importlib +import json import os import shutil import tempfile @@ -12,14 +13,18 @@ ends_with, equal_to, has_item, + has_entries, has_length, not_, ) from behave import given, when, then # pylint: disable=no-name-in-module +import jsonschema from testlib import no_output, run +# flake8: noqa # pylint: disable=missing-function-docstring,function-redefined +# mypy: disable-error-code="no-redef" def _first_int(string, max_lines=20): @@ -187,3 +192,44 @@ def step_impl(context, call, module, args): def step_impl(context, field, value): assert_that(context.last_method_call_retval[field], equal_to(value)) + + +@when('deps for {srpm} are calculated against {chroot}') +def step_impl(context, srpm, chroot): + url = context.test_storage + srpm + context.mock.calculate_deps(url, chroot) + + +@when('a local repository is created from lockfile') +def step_impl(context): + mock_run = context.mock_runs["calculate-build-deps"][-1] + lockfile = mock_run["lockfile"] + + context.local_repo = tempfile.mkdtemp(prefix="mock-tests-local-repo-") + cmd = ["mock-isolated-repo", "--lockfile", lockfile, "--output-repo", + context.local_repo] + assert_that(run(cmd)[0], equal_to(0)) + + +@when('an isolated build is retriggered with the lockfile and repository') +def step_impl(context): + context.mock.isolated_build() + + +@then('the produced lockfile is validated properly') +def step_impl(context): + mock_run = context.mock_runs["calculate-build-deps"][-1] + lockfile = mock_run["lockfile"] + with open(lockfile, "r", encoding="utf-8") as fd: + lockfile_data = json.load(fd) + + assert_that(lockfile_data["buildroot"]["rpms"], + has_item(has_entries({"name": "filesystem"}))) + + schemafile = os.path.join(os.path.dirname(__file__), '..', '..', '..', + "mock", "docs", + "buildroot-lock-schema-1.0.0.json") + with open(schemafile, "r", encoding="utf-8") as fd: + schema = json.load(fd) + + jsonschema.validate(lockfile_data, schema) diff --git a/behave/testlib.py b/behave/testlib.py index 64c6c81f9..bb8b78cf1 100644 --- a/behave/testlib.py +++ b/behave/testlib.py @@ -3,6 +3,7 @@ from contextlib import contextmanager import io import pipes +import os import subprocess import sys @@ -67,14 +68,13 @@ def __init__(self, context): context.mock_runs = { "init": [], "rebuild": [], + "calculate-build-deps": [], } @property def basecmd(self): """ return the pre-configured mock base command """ cmd = ["mock"] - if self.context.chroot_used: - cmd += ["--chroot", self.context.chroot] if self.context.uniqueext_used: cmd += ["--uniqueext", self.context.uniqueext] for repo in self.context.add_repos: @@ -103,6 +103,40 @@ def rebuild(self, srpms): "srpms": srpms, }] + def calculate_deps(self, srpm, chroot): + """ + Call Mock with --calculate-build-dependencies and produce lockfile + """ + out, err = run_check( + self.basecmd + ["-r", chroot] + ["--calculate-build-dependencies", + srpm]) + self.context.chroot = chroot + self.context.mock_runs["calculate-build-deps"].append({ + "status": 0, + "out": out, + "err": err, + "srpm": srpm, + "chroot": chroot, + "lockfile": os.path.join(self.resultdir, "buildroot_lock.json") + }) + + def isolated_build(self): + """ + From the previous calculate_deps() run, perform isolated build + """ + mock_calc = self.context.mock_runs["calculate-build-deps"][-1] + out, err = run_check(self.basecmd + [ + "--isolated-build", mock_calc["lockfile"], self.context.local_repo, + mock_calc["srpm"] + ]) + self.context.mock_runs["rebuild"].append({ + "status": 0, + "out": out, + "err": err, + }) + # We built into an isolated-build.cfg! + self.context.chroot = "isolated-build" + def clean(self): """ Clean chroot, but keep dnf/yum caches """ run_check(self.basecmd + [ diff --git a/docs/Plugin-BuildrootLock.md b/docs/Plugin-BuildrootLock.md new file mode 100644 index 000000000..540a4221f --- /dev/null +++ b/docs/Plugin-BuildrootLock.md @@ -0,0 +1,76 @@ +--- +layout: default +title: Plugin buildroot_lock +--- + +buildroot_lock Plugin +===================== + +This plugin generates an additional build artifact—the buildroot *lockfile* +(`buildroot_lock.json` file in the result directory). + +The *lockfile* describes both the list of buildroot sources (e.g., a list of +installed RPMs, bootstrap image info, etc.) and a set of Mock configuration +options. Using this information, Mock can later reproduce the buildroot +preparation (see the [Isolated Builds feature page](feature-isolated-builds)). + +This plugin is **disabled** by default but is automatically enabled with the +`--calculate-build-dependencies` option. You can enable it (for all builds) by +this configuration snippet: + +```python +config_opts['plugin_conf']['buildroot_lock_enable'] = True +``` + +**Note:** This plugin does not work with the `--offline` option. + + +Format of the *buildroot_lock.json* file +---------------------------------------- + +The file `buildroot_lock.json` is a JSON file. List of JSON Schema files is +installed together with the Mock RPM package: + + rpm -ql mock | grep schema + /usr/share/doc/mock/buildroot-lock-schema-1.0.0.json + +Currently, we do not provide a compatibility promise. Only the exact same +version of Mock that produced the file is guaranteed to read and process it. +For more information, see [Isolated Builds](feature-isolated-builds). + +Example structure +----------------- + +Contents of such a lockfile may look like: + + { + "version": "1.0.0", + "buildroot": { + "packages": [ + { + "license": "MIT", + "name": "fedora-repos-rawhide", + "version": "42", + "release": "0.1", + "arch": "noarch", + "epoch": null, + "sigmd5": "4378929dac9e51ed8470de5173ae664e", + "signature": null, + "url": "http://ftp.sh.cvut.cz/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/f/fedora-repos-rawhide-42-0.1.noarch.rpm" + }, + ... + ] + }, + "config": { + "target_arch": "x86_64", + "legal_host_arches": [ + "x86_64" + ], + "dist": "rawhide", + "package_manager": "dnf5", + "bootstrap_image": "registry.fedoraproject.org/fedora:rawhide", + "bootstrap_image_ready": true + } + } + +The list of packages is sorted by the `name + arch` pair. diff --git a/docs/feature-isolated-builds.md b/docs/feature-isolated-builds.md new file mode 100644 index 000000000..9fd61aef3 --- /dev/null +++ b/docs/feature-isolated-builds.md @@ -0,0 +1,194 @@ +--- +layout: default +title: Isolated builds with Mock +--- + +Isolated builds with Mock +========================= + +Mock (v5.7+) supports isolated RPM builds, sometimes referred to as "hermetic" +or "offline" builds. For more details, see the +[SLSA "isolated" definition][SLSA]. + +Quick start +----------- + +For the impatient, the TL;DR steps of the HOWTO are as follows: + + # we want to build this package + srpm=your-package.src.rpm + + # we'll create a local repository with pre-fetched RPMs/bootstrap + repo=/tmp/local-repo + + # resolve build deps for the given SRPM, in this case for Fedora Rawhide + mock --calculate-build-dependencies -r fedora-rawhide-x86_64 "$srpm" + + # find the lockfile in Mock's resultdir + lockfile=/var/lib/mock/fedora-rawhide-x86_64/result/buildroot_lock.json + + # create a local RPM repository (+ download bootstrap image) + mock-isolated-repo --lockfile "$lockfile" --output-repo "$repo" + + # perform the isolated build! + mock --isolated-build "$lockfile" "$repo" "$srpm" + +What an "isolated build" is.. +----------------------------- + +The term "isolated build" is often used in different contexts, even within +Mock's terminology. Historically, when we said that "Mock isolates the build," +we typically meant that Mock creates a *buildroot* (also referred to as a *build +directory* or *build chroot*) and runs the (Turing-complete, and thus +potentially insecure) *RPM build* process (i.e., a call to `/usr/bin/rpmbuild`) +inside it. In this sense, Mock "isolates" the RPM build process from the rest +of the system, or protects the system from potential mishaps. However, the +**buildroot preparation** process was never "isolated" in this manner—only the +*RPM build* was. Even the *RPM build* "isolation" was always performed on a +best-effort basis. For more details, see [Mock's Scope](index). + +When we now talk about making builds and the corresponding built artifacts +safer, more predictable, and more reproducible, we refer to the [SLSA +isolation][SLSA] definition. This involves using Mock in an *isolated* +environment, free from unintended external influence. + +Mock itself doesn't aim to provide this level of *isolation*. Mock is still +just a tool that runs in "some" build environment to perform the `SRPM → RPM` +translation. In such an environment, the Mock process can be tampered with by +other processes (potentially even root-owned), and as a result, the artifacts +may be (un)intentionally altered. Therefore, the preparation of the environment +to **run Mock** and the **isolation** itself is the responsibility of a +different tool (for example, `podman run --privileged --network=none`). + +So, what does Mock `--isolated-build` do if it doesn't isolate? +Essentially, it just does less work than it usually does! It optimizes out any +action (primarily during the *buildroot* preparation) that would rely on +"external" factors—specifically, it never expects Internet connectivity. +However, for the eventual build to succeed, **something else** still needs to +perform these omitted actions. Every single component required for *buildroot* +preparation must be prepared in advance for the `mock --isolated-build` +call (within **the** properly *isolated* environment, of course). + + +Challenges +---------- + +You’ve probably noticed that what used to be a simple command—like +`mock -r "$chroot" "$srpm"`—has now become a more complicated set of commands. + +This complexity arises because the *buildroot* in Mock is always prepared by +installing a set of RPMs (Mock calls DNF, DNF calls RPM, ...), which normally +requires a network connection. + +Additionally, it’s not always guaranteed that the DNF/RPM variant on the build +host (e.g., an EPEL 8 host) is sufficient or up-to-date for building the target +distribution (e.g., the newest Fedora Rawhide). Therefore, we need network +access [to obtain the appropriate bootstrap tooling](Feature-bootstrap). + +The [dynamic build dependencies][] further complicate the process. Without +them, we could at least make the `/bin/rpmbuild` fully offline—but with them, +it’s not so simple. Mock needs to interrupt the ongoing *RPM build* process, +resolve additional `%generate_buildrequires` (installing more packages on +demand), restart the *RPM build*, interrupt it again, and so on. This process +also requires a network connection! + +All of this is further complicated by the goal of making the *buildroot* as +*minimal* as possible—the fewer packages installed, the better. We can’t even +afford to install DNF into the buildroot, and as you’ve probably realized, we +definitely don’t want to blindly install all available RPMs. + + +The solution +------------ + +To address the challenges, we needed to separate the online +(`--calculate-build-dependencies`) and offline (`--isolated-build`) tasks +that Mock performs. + +1. **Online Tasks:** These need to be executed first. We let Mock prepare the + *buildroot #1* for the given *SRPM* (using the standard "online" method) and + record its *lockfile*—a list of all the resources obtained from the network + during the process. + + The format of lockfile is defined by provided JSON Schema file(s), see + documentation for the [buildroot_lock plugin](Plugin-BuildrootLock). + + **Note:** The *buildroot* preparation also includes the installation of + dynamic build dependencies! Therefore, we **have to start an RPM build**. + Although we don’t finish the build (we terminate it once the + `%generate_buildrequires` is resolved, before reaching the `%build` phase, + etc.), it must be initiated. + +2. **Offline Repository Creation:** With the *lockfile* from the previous step, + we can easily retrieve the referenced components from the network. The Mock + project provides an example implementation for this step in the + `mock-isolated-repo(1)` utility. This tool downloads all the referenced + components from the internet and places them into a single local + directory—let's call it an *offline repository*. + + **Note:** This step doesn’t necessarily have to be done by the Mock project + itself. The *lockfile* is concise enough for further processing and + validation (e.g., ensuring the set of RPMs and the buildroot image come from + trusted sources) and could be parsed by build-system-specific tools like + [cachi2][] (potentially in the future). + +3. **Offline Build:** With the *srpm* and the *offline repository*, we can + instruct Mock to restart the build using the `--isolated-build + LOCKFILE OFFLINE_REPO SRPM` command. The *lockfile* is still needed at this + stage because it contains some of the configuration options used in step 1 + that must be inherited by the current Mock call. + + This step creates a new *buildroot #2* using the pre-downloaded RPMs in the + *offline repository* (installing them all at once) and then (re)starts the + RPM build process. + +You might notice that some steps are performed twice, specifically downloading +the RPMs (steps 1 and 2) and running the RPM build (steps 1 and 3). This +duplication is a necessary cost (in terms of more resources and time spent on +the build) to ensure that step 3 is _fully offline_. In step 3, the *offline* +RPM build is no longer interrupted by an *online* `%generate_buildrequires` +process—dependencies are already installed! + +Also, while you can calmly experiment with + + + mock --calculate-build-dependencies -r fedora-rawhide-x86_64 "$srpm" + mock --no-clean -r fedora-rawhide-x86_64 "$srpm" + +and it is very close to the TL;DR variant, such an approach is not the same +thing! The *buildroot #1* **was not** prepared by Mock in **isolated** +environment. + +Limitations +----------- + +- We rely heavily on + the [Bootstrap Image feature](Feature-container-for-bootstrap). This allows + us to easily abstract the bootstrap preparation tasks, which would otherwise + depend heavily on the system's RPM/DNF stack, etc. + + For now, we also require the Bootstrap Image to be *ready*. This simplifies + the implementation, as we don't need to recall the set of commands (or list of + packages to install into) needed for bootstrap preparation. + +- It is known fact that *normal builds* and *isolated builds* may result in + slightly different outputs (at least in theory). This issue relates to the + topic of *reproducible builds*. Normally, the *buildroot* is installed using + several DNF commands (RPM transactions), whereas the *isolated* build installs + all dependencies in a single DNF command (RPM transaction). While this + difference might cause the outputs of *normal* and *isolated* builds to vary + (in theory, because the chroot depends on RPM installation order), it OTOH + introduces more determinism! + +- The *lockfile* provides a list of the required RPMs, referenced by URLs. + These URLs point to the corresponding RPM repositories (online) from which + they were installed in step 1. However, in many cases, RPMs are downloaded + from `metalink://` or `mirrorlist://` repositories, meaning the URL might be + selected non-deterministically, and the specific mirrors chosen could be + rather ephemeral. For this reason, users should—for isolated builds—avoid + using mirrored repositories (as in the case of Koji builders) or avoid making + large delays between step 1 and step 2. + +[SLSA]: https://slsa.dev/spec/v1.0/requirements +[dynamic build dependencies]: https://github.com/rpm-software-management/mock/issues/1359 +[cachi2]: https://github.com/containerbuildsystem/cachi2 diff --git a/docs/index.md b/docs/index.md index 40869c762..78964f378 100644 --- a/docs/index.md +++ b/docs/index.md @@ -189,6 +189,7 @@ See a [separate document](Mock-Core-Configs). ## Plugins * [bind_mount](Plugin-BindMount) - bind mountpoints inside the chroot +* [buildroot_lock](Plugin-BuildrootLock) - provide a buildroot lockfile * [ccache](Plugin-CCache) - compiler cache plugin * [chroot_scan](Plugin-ChrootScan) - allows you to retrieve build artifacts from buildroot (e.g. additional logs, coredumps) * [compress_logs](Plugin-CompressLogs) - compress logs @@ -225,6 +226,7 @@ Every plugin has a corresponding wiki page with docs. * [package managers](Feature-package-managers) - supported package managers * [rhel chroots](Feature-rhelchroots) - builds for RHEL * [GPG keys and SSL](feature-gpg-and-ssl) - how to get your GPG keys and SSL certificates to buildroot +* [Isolated (offline) Builds](feature-isolated-builds) - doing offline builds with Mock ## Using Mock outside your git sandbox diff --git a/mock/docs/buildroot-lock-schema-1.0.0.json b/mock/docs/buildroot-lock-schema-1.0.0.json new file mode 100644 index 000000000..9a4cdb25d --- /dev/null +++ b/mock/docs/buildroot-lock-schema-1.0.0.json @@ -0,0 +1,102 @@ +{ + "$id": "https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema-1.0.0.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "object", + "title": "Mock buildroot_lock.json file specification", + "description": "Version 1.0.0; last updated 2024-09-02", + "additionalProperties": false, + "properties": { + "version": { + "description": "Version of the https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema.json schema the document conforms to. Semantic versioned. Mock that implements v2.Y.Z versions no longer reads v1.Y.Z.", + "const": "1.0.0" + }, + "buildroot": { + "description": "The object that describes the Mock buildroot", + "type": "object", + "additionalProperties": false, + "properties": { + "rpms": { + "description": "List of RPM packages installed in the buildroot", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "arch": { + "description": "Architecture for which the package was built, 'noarch' for arch agnostic packages", + "type": "string" + }, + "epoch": { + "description": "Epoch number of the package", + "type": ["string", "null"] + }, + "license": { + "description": "The distribution license(s) of the package", + "type": "string" + }, + "name": { + "description": "Name of the package", + "type": "string" + }, + "release": { + "description": "Release (downstream) number of the package", + "type": "string" + }, + "sigmd5": { + "description": "The SIGMD5 tag from the rpm header.", + "type": "string" + }, + "signature": { + "description": "The signature used to sign the rpm (if any), last 8 characters from the \"rpm -q --qf '%{sigpgp:pgpsig}\n'\" output", + "type": ["string", "null"] + }, + "url": { + "description": "Uniform Resource Locator that points to additional information on the packaged software", + "type": "string" + }, + "version": { + "description": "Version (upstream) of the package", + "type": "string" + } + }, + "required": [ + "arch", + "epoch", + "license", + "name", + "release", + "sigmd5", + "signature", + "url", + "version" + ] + } + } + }, + "required": [ + "rpms" + ] + }, + "bootstrap": { + "description": "The object that describes the Mock bootstrap chroot. Optional, only provided when bootstrap (image) is used.", + "type": "object", + "additionalProperties": false, + "properties": { + "image_digest": { + "description": "Digest got by the 'podman image inspect --format {{ .Digest }}' command, sha256 string", + "type": "string" + } + } + }, + "config": { + "description": "A set of important Mock configuration options used when the buildroot was generated (Mock's internal)", + "type": "object", + "properties": {} + } + }, + "required": [ + "buildroot", + "config", + "version" + ] +} diff --git a/mock/docs/mock.1 b/mock/docs/mock.1 index 18385c33e..aa28b06ee 100644 --- a/mock/docs/mock.1 +++ b/mock/docs/mock.1 @@ -47,6 +47,10 @@ mock [options] \fB\-\-pm\-cmd\fR [\fIarguments ...\fR] mock [options] \fB\-\-yum\-cmd\fR [\fIarguments ...\fR] .LP mock [options] \fB\-\-dnf\-cmd\fR [\fIarguments ...\fR] +.LP +mock [options] \fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +.LP +mock [options] \fB\-\-isolated\-build \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR .SH "DESCRIPTION" @@ -90,6 +94,21 @@ Mock is running some parts of code with root privileges. There are known ways to \fB\-\-buildsrpm\fP Build the specified SRPM either from a spec file and source file/directory or SCM. The chroot (including the results directory) is cleaned first, unless \-\-no\-clean is specified. .TP +\fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +Evaluate and install all the \fISRPM\fR (= file name, path on your system) build +dependencies, including dynamic dependencies in \fI%generate_buildrequires\fR. +This is similar to the \fB\-\-installdeps\fR option which only installs the +static \fIBuildRequires\fR. + +Build chroot-native \fI*.src.rpm\fR and \fI*.nosrc.rpm\fR files (the later only +when \fI%generate_buildrequires\fR is in use!). The \fI*.src.rpm\fR records the +corresponding static list of build dependencies (= \fIBuildRequires\fR). The +\fI*.nosrc.rpm\fR records a full list of build dependencies (static + dynamic). +Test with \fIrpm -qpR [*.src.rpm|*.nosrc.rpm]\fR. + +Additionally, provide a \fIbuildroot_lock.json\fR file; this records the +metadata needed for an isolated build (see also \fB\-\-isolated\-build\fR). +.TP \fB\-\-chain\fR When passing more than one SRPM, it will try to build failed builds if at least one subsequent SRPM succeed. This mimic the behaviour of deprecated mockchain. .TP @@ -135,7 +154,24 @@ Initialize a chroot (clean, install chroot packages, etc.). Do a yum install PACKAGE inside the chroot. No 'clean' is performed. .TP \fB\-\-installdeps\fP -Find out deps for SRPM or RPM, and do a yum install to put them in the chroot. No 'clean' is performed +Find out "static" deps for SRPM or RPM, and do a \fIdnf install\fR to put them +into the buildroot. No 'cleanup' is performed. + +Dynamic build dependencies (\fI%generate_buildrequires\fR specfile section) are +not installed, see \fB\-\-calculate\-build\-dependencies\fR. +.TP +\fB\-\-isolated\-build \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR +Perform an isolated RPM build (i.e., an offline build without the need to access +the Internet at all) from the given \fISRPM\fR (= file name, path on your +system). After running Mock with the \fB\-\-calculate\-build\-dependencies\fR +option to generate the \fILOCKFILE\fR file (typically named +\fIbuildroot_lock.json\fR in the result directory), and then running the +\fImock\-isolated\-repo(1)\fR helper to generate \fIREPO\fR (a directory on the +host that provides RPMs with metadata and a bootstrap image tarball), Mock has +all the necessary information to build RPMs from the given \fISRPM\fR fully +offline. More info in the feature page: + +\fIhttps://rpm-software-management.github.io/mock/feature-isolated-builds\fR .TP \fB\-\-list-chroots\fP List all available chroots names and their description - both system-wide and user ones. diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index b219a5e3b..dcc25512f 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -174,6 +174,11 @@ # Skip the "podman pull" and rely on the image already being in the local cache. #config_opts["bootstrap_image_skip_pull"] = False +# If provided, Mock performs a 'podman image inspect --format {{ .Digest }}' +# call and asserts that the downloaded/imported bootstrap_image has expected +# Digest (SHA256 string). +#config_opts["bootstrap_image_assert_digest"] = None + # anything you specify with 'bootstrap_*' will be copied to bootstrap config # e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become # config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config @@ -653,6 +658,12 @@ # override the default Mock's stack call limit (5000). #config_opts["recursion_limit"] = 5000 +# Mock internals used by the --calculated-build-dependencies and +# --isolated-build options. Please do not set these options in Mock +# configuration files. +# config_opts["calculatedeps"] = None +# config_opts["isolated_build"] = False + # List of usernames (strings) that will be pre-created in buildroot. The UID # and GID in-chroot is going to be the same as on-host. This option is for # example useful for the 'pesign' use-cases that both (a) bind-mount diff --git a/mock/etc/mock/isolated-build.cfg b/mock/etc/mock/isolated-build.cfg new file mode 100644 index 000000000..56bb4b50e --- /dev/null +++ b/mock/etc/mock/isolated-build.cfg @@ -0,0 +1,38 @@ +# used by https://rpm-software-management.github.io/mock/feature-isolated-builds + +config_opts['root'] = 'isolated-build' +config_opts['description'] = 'Isolated Build' + +# Isolated-build configuration file is re-used for multiple chroot +# configurations (particular chroot used depends on the previous +# --calculate-build-dependencies run). That's why Mock automatically runs +# --scrub=all with --isolated-build. It doesn't make sense to waste the time +# with creating caches. +config_opts['plugin_conf']['root_cache_enable'] = False + +config_opts['dnf.conf'] = """ +[main] +keepcache=1 +system_cachedir=/var/cache/dnf +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= +install_weak_deps=0 +metadata_expire=0 +best=1 +protected_packages= + +# repos + +[offline] +name=offline repo +baseurl=file://{{ offline_local_repository }} +enabled=True +skip_if_unavailable=False +""" diff --git a/mock/mock.spec b/mock/mock.spec index 938ec4a41..d27a6af09 100644 --- a/mock/mock.spec +++ b/mock/mock.spec @@ -63,6 +63,8 @@ BuildRequires: python%{python3_pkgversion}-pylint BuildRequires: python%{python3_pkgversion}-rpm BuildRequires: python%{python3_pkgversion}-rpmautospec-core +BuildRequires: argparse-manpage + %if 0%{?fedora} >= 38 # DNF5 stack Recommends: dnf5 @@ -85,6 +87,7 @@ Recommends: podman %if %{with tests} BuildRequires: python%{python3_pkgversion}-distro BuildRequires: python%{python3_pkgversion}-jinja2 +BuildRequires: python%{python3_pkgversion}-jsonschema BuildRequires: python%{python3_pkgversion}-pyroute2 BuildRequires: python%{python3_pkgversion}-pytest BuildRequires: python%{python3_pkgversion}-requests @@ -170,6 +173,9 @@ done # this is what %%sysusers_create_compat will expand to %{_rpmconfigdir}/sysusers.generate-pre.sh mock.conf > sysusers_script +argparse-manpage --pyfile ./py/mock-isolated-repo.py --function _argparser > mock-isolated-repo.1 + + %install #base filesystem mkdir -p %{buildroot}%{_sysconfdir}/mock/eol/templates @@ -178,6 +184,7 @@ mkdir -p %{buildroot}%{_sysconfdir}/mock/templates install -d %{buildroot}%{_bindir} install -d %{buildroot}%{_libexecdir}/mock install mockchain %{buildroot}%{_bindir}/mockchain +install py/mock-isolated-repo.py %{buildroot}%{_bindir}/mock-isolated-repo install py/mock-parse-buildlog.py %{buildroot}%{_bindir}/mock-parse-buildlog install py/mock.py %{buildroot}%{_libexecdir}/mock/mock ln -s consolehelper %{buildroot}%{_bindir}/mock @@ -204,7 +211,7 @@ install -d %{buildroot}%{python_sitelib}/ cp -a py/mockbuild %{buildroot}%{python_sitelib}/ install -d %{buildroot}%{_mandir}/man1 -cp -a docs/mock.1 docs/mock-parse-buildlog.1 %{buildroot}%{_mandir}/man1/ +cp -a docs/mock.1 docs/mock-parse-buildlog.1 mock-isolated-repo.1 %{buildroot}%{_mandir}/man1/ install -d %{buildroot}%{_datadir}/cheat cp -a docs/mock.cheat %{buildroot}%{_datadir}/cheat/mock @@ -212,6 +219,7 @@ install -d %{buildroot}/var/lib/mock install -d %{buildroot}/var/cache/mock mkdir -p %{buildroot}%{_pkgdocdir} +install -p -m 0644 docs/buildroot-lock-schema-*.json %{buildroot}%{_pkgdocdir} install -p -m 0644 docs/site-defaults.cfg %{buildroot}%{_pkgdocdir} mkdir -p %{buildroot}%{_sysusersdir} @@ -239,6 +247,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : %defattr(0644, root, mock) %dir %{_pkgdocdir}/ %doc %{_pkgdocdir}/site-defaults.cfg +%doc %{_pkgdocdir}/buildroot-lock-schema-*.json %{_datadir}/bash-completion/completions/mock %{_datadir}/bash-completion/completions/mock-parse-buildlog @@ -247,6 +256,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # executables %{_bindir}/mock %{_bindir}/mockchain +%{_bindir}/mock-isolated-repo %{_bindir}/mock-parse-buildlog %{_libexecdir}/mock @@ -261,6 +271,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # config files %config(noreplace) %{_sysconfdir}/%{name}/*.ini +%config(noreplace) %{_sysconfdir}/%{name}/isolated-build.cfg %config(noreplace) %{_sysconfdir}/pam.d/%{name} %config(noreplace) %{_sysconfdir}/security/console.apps/%{name} @@ -271,6 +282,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # docs %{_mandir}/man1/mock.1* %{_mandir}/man1/mock-parse-buildlog.1* +%{_mandir}/man1/mock-isolated-repo.1* %{_datadir}/cheat/mock # cache & build dirs diff --git a/mock/py/mock-isolated-repo.py b/mock/py/mock-isolated-repo.py new file mode 100755 index 000000000..049ecf2aa --- /dev/null +++ b/mock/py/mock-isolated-repo.py @@ -0,0 +1,113 @@ +#! /usr/bin/python3 + +""" +Take the JSON provided by Mock, download corresponding RPMs, and put them into +an RPM repository. +""" + +# pylint: disable=invalid-name + +import argparse +import concurrent.futures +import json +import logging +import os +import shutil +import subprocess +import sys + +import requests + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +def download_file(url, outputdir): + """ + Download a single file (pool worker) + """ + file_name = os.path.join(outputdir, os.path.basename(url)) + log.info("Downloading %s", url) + try: + with requests.get(url, stream=True, timeout=30) as response: + if response.status_code != 200: + return False + with open(file_name, "wb") as fd: + shutil.copyfileobj(response.raw, fd) + return True + except: # noqa: E722 + log.exception("Exception raised for %s", url) + raise + + +def _argparser(): + parser = argparse.ArgumentParser( + prog='mock-isolated-repo', + description=( + "Prepare a repository for a `mock --isolated-build` build. " + "Given a Mock buildroot \"lockfile\"\n\n" + " a) create an output repo directory,\n" + " b) download and place all the necessary RPM files there,\n" + " c) create a local RPM repository there (run createrepo), and\n" + " d) dump there also the previously used bootstrap image as " + "a tarball.\n\n" + "Lockfile is a buildroot_lock.json file from Mock's " + "result directory; it is a JSON file generated by the " + "--calculate-build-dependencies option/buildroot_lock " + "plugin."), + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("--lockfile", required=True, + help=( + "Select buildroot_lock.json filename on your system, " + "typically located in the Mock's result directory " + "upon the --calculate-build-dependencies mode " + "execution.")) + parser.add_argument("--output-repo", required=True, + help=( + "Download RPMs into this directory, and then run " + "/bin/createrepo_c utility there to populate the " + "RPM repo metadata.")) + return parser + + +def prepare_image(image_specification, outputdir): + """ + Store the tarball into the same directory where the RPMs are + """ + subprocess.check_output(["podman", "pull", image_specification]) + subprocess.check_output(["podman", "save", "--format=oci-archive", "--quiet", + "-o", os.path.join(outputdir, "bootstrap.tar"), + image_specification]) + + +def _main(): + options = _argparser().parse_args() + + with open(options.lockfile, "r", encoding="utf-8") as fd: + data = json.load(fd) + + try: + os.makedirs(options.output_repo) + except FileExistsError: + pass + + failed = False + urls = [i["url"] for i in data["buildroot"]["rpms"]] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + for i, out in zip(urls, executor.map(download_file, urls, + [options.output_repo for _ in urls])): + if out is False: + log.error("Download failed: %s", i) + failed = True + if failed: + log.error("RPM deps downloading failed") + sys.exit(1) + + subprocess.check_call(["createrepo_c", options.output_repo]) + + prepare_image(data["config"]["bootstrap_image"], options.output_repo) + + +if __name__ == "__main__": + _main() diff --git a/mock/py/mock.py b/mock/py/mock.py index 76e52ff0e..351a82154 100755 --- a/mock/py/mock.py +++ b/mock/py/mock.py @@ -120,6 +120,9 @@ def command_parse(): parser.add_option("--rebuild", action="store_const", const="rebuild", dest="mode", default='__default__', help="rebuild the specified SRPM(s)") + parser.add_option("--calculate-build-dependencies", action="store_const", + const="calculatedeps", dest="mode", + help="Resolve and install static and dynamic build dependencies") parser.add_option("--chain", action="store_const", const="chain", dest="mode", help="build multiple RPMs in chain loop") @@ -390,6 +393,9 @@ def command_parse(): type=str, dest="additional_packages", help=("Additional package to install into the buildroot before " "the build is done. Can be specified multiple times.")) + parser.add_option("--isolated-build", nargs=2, + metavar=("LOCKFILE", "REPO_DIRECTORY"), + help="Perform an isolated (fully offline) SRPM build") (options, args) = parser.parse_known_args() @@ -402,6 +408,20 @@ def command_parse(): else: options.mode = 'rebuild' + if options.isolated_build and options.chroot != 'default': + raise mockbuild.exception.BadCmdline( + "The --isolated-build mode uses a special chroot configuration, " + "you can not select the chroot configuration with the " + "-r/--root option.") + + if options.isolated_build and options.mode != "rebuild": + raise mockbuild.exception.BadCmdline("--rebuild mode needed with --isolated-build") + + options.calculatedeps = None + if options.mode == "calculatedeps": + options.mode = "rebuild" + options.calculatedeps = True + # Optparse.parse_args() eats '--' argument, while argparse doesn't. Do it manually. if args and args[0] == '--': args = args[1:] @@ -673,6 +693,9 @@ def main(): if options.configdir: config_path = options.configdir + if options.isolated_build: + options.chroot = "isolated-build" + config_opts = uidManager.run_in_subprocess_without_privileges( config.load_config, config_path, options.chroot) @@ -946,6 +969,11 @@ def run_command(options, args, config_opts, commands, buildroot): buildroot.remove(*args) elif options.mode == 'rebuild': + if options.isolated_build: + # No caches with isolated builds! Bootstrap is extracted from + # given tarball, buildroot installed from pre-fetched RPMs. + commands.scrub(["all"]) + if config_opts['scm'] or (options.spec and options.sources): srpm = mockbuild.rebuild.do_buildsrpm(config_opts, commands, buildroot, options, args) if srpm: diff --git a/mock/py/mockbuild/backend.py b/mock/py/mockbuild/backend.py index e0cd32413..9649529d3 100644 --- a/mock/py/mockbuild/backend.py +++ b/mock/py/mockbuild/backend.py @@ -721,6 +721,7 @@ def rebuild_package(self, spec_path, timeout, check, dynamic_buildrequires): # --nodeps because rpm in the root may not be able to read rpmdb # created by rpm that created it (outside of chroot) check_opt = [] + calculatedeps = self.config["calculatedeps"] if not check: # this is because EL5/6 does not know --nocheck # when EL5/6 targets are not supported, replace it with --nocheck @@ -782,7 +783,9 @@ def get_command(mode, checkdeps=False): if packages_after == packages_before: success = True for f_buildreqs in buildreqs: - os.remove(f_buildreqs) + if not (success and calculatedeps): + # we want to keep the nosrc.rpm file + os.remove(f_buildreqs) # The first rpmbuild -br already did %prep, so we don't need waste time if '--noprep' not in br_mode: br_mode += ['--noprep'] @@ -806,14 +809,16 @@ def get_command(mode, checkdeps=False): # Unfortunately, we can only do this when using a bootstrap chroot, # because the rpm in the chroot might not understand the rpmdb otherwise. # See https://github.com/rpm-software-management/mock/issues/1246 - checkdeps = dynamic_buildrequires and self.bootstrap_buildroot is not None - self.buildroot.doChroot(get_command(mode, checkdeps=checkdeps), - shell=False, logger=self.buildroot.build_log, timeout=timeout, - uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, - user=self.buildroot.chrootuser, - nspawn_args=self._get_nspawn_args(), - unshare_net=self.private_network, - printOutput=self.config['print_main_output']) + + if not calculatedeps: + checkdeps = dynamic_buildrequires and self.bootstrap_buildroot is not None + self.buildroot.doChroot(get_command(mode, checkdeps=checkdeps), + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, + nspawn_args=self._get_nspawn_args(), + unshare_net=self.private_network, + printOutput=self.config['print_main_output']) results = glob.glob(bd_out + '/RPMS/*.rpm') results += glob.glob(bd_out + '/SRPMS/*.rpm') self.buildroot.final_rpm_list = [os.path.basename(result) for result in results] diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index 801d618a2..7a8d7a79b 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -268,6 +268,21 @@ def _fallback(message): getLog().info("Using local image %s (pull skipped)", self.bootstrap_image) + if self.config["isolated_build"]: + tarball = os.path.join(self.config["offline_local_repository"], + "bootstrap.tar") + podman.import_tarball(tarball) + + digest_expected = self.config.get("image_assert_digest", None) + if digest_expected: + getLog().info("Checking image digest: %s", + digest_expected) + digest = podman.get_image_digest() + if digest != digest_expected: + raise BootstrapError( + f"Expected digest for image {podman.image} is" + f"{digest_expected}, but {digest} found.") + podman.cp(self.make_chroot_path(), self.config["tar_binary"]) file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), "etc/rpm/macros.image-language-conf")) diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index f179bf8f6..0f61f0b63 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -6,6 +6,7 @@ from ast import literal_eval from glob import glob +import json import grp import logging import os @@ -30,7 +31,7 @@ 'ccache', 'selinux', 'package_state', 'chroot_scan', 'lvm_root', 'compress_logs', 'sign', 'pm_request', 'hw_info', 'procenv', 'showrc', 'rpkg_preprocessor', - 'rpmautospec'] + 'rpmautospec', 'buildroot_lock'] def nspawn_supported(): """Detect some situations where the systemd-nspawn chroot code won't work""" @@ -89,6 +90,7 @@ def setup_default_config_opts(): config_opts['bootstrap_image_ready'] = False config_opts['bootstrap_image_fallback'] = True config_opts['bootstrap_image_keep_getting'] = 120 + config_opts['bootstrap_image_assert_digest'] = None config_opts['internal_dev_setup'] = True @@ -170,6 +172,8 @@ def setup_default_config_opts(): 'available_pkgs': False, 'installed_pkgs': True, }, + 'buildroot_lock_enable': False, + 'buildroot_lock_opts': {}, 'pm_request_enable': False, 'pm_request_opts': {}, 'lvm_root_enable': False, @@ -388,6 +392,9 @@ def setup_default_config_opts(): config_opts["recursion_limit"] = 5000 + config_opts["calculatedeps"] = None + config_opts["isolated_build"] = False + return config_opts @@ -398,6 +405,7 @@ def multiply_platform_multiplier(config_opts): if '%_platform_multiplier' not in config_opts["macros"]: config_opts["macros"]["%_platform_multiplier"] = 10 if config_opts["forcearch"] else 1 + @traceLog() def set_config_opts_per_cmdline(config_opts, options, args): "takes processed cmdline args and sets config options." @@ -639,6 +647,11 @@ def set_config_opts_per_cmdline(config_opts, options, args): # which though affects root_cache). config_opts["additional_packages"] = options.additional_packages + config_opts["calculatedeps"] = options.calculatedeps + if config_opts["calculatedeps"]: + config_opts["plugin_conf"]["buildroot_lock_enable"] = True + + process_isolated_build_config(options, config_opts) def check_config(config_opts): if 'root' not in config_opts: @@ -699,6 +712,64 @@ def update_config_from_file(config_opts, config_file): config_opts["config_paths"] = list(new_paths) +def update_config_from_dict(config_opts, updates): + """ + Merge a dictionary into config_opts. No include supported. + """ + for key, value in updates.items(): + config_opts[key] = value + + +def process_isolated_build_config(cmdline_opts, config_opts): + """ + Read the lockfile file generated by the previous + --calculate-build-dependencies run, and adjust the current set of options + in CONFIG_OPTS. + """ + + if not cmdline_opts.isolated_build: + return + + config_opts["isolated_build"] = True + + json_conf, repo_reference = cmdline_opts.isolated_build + with open(json_conf, "r", encoding="utf-8") as fd: + data = json.load(fd) + + if not data["config"].get("bootstrap_image_ready"): + raise exception.BadCmdline( + f"The file {json_conf} did not record the bootstrap_image_ready=True " + "config which means we are not able to prepare the bootstrap chroot " + "in an isolated mode.") + + update_config_from_dict(config_opts, data["config"]) + + final_offline_repo = repo_reference + file_pfx = "file://" + if final_offline_repo.startswith(file_pfx): + final_offline_repo = final_offline_repo[len(file_pfx):] + final_offline_repo = os.path.abspath(final_offline_repo) + if not os.path.exists(os.path.join(final_offline_repo, "repodata")): + raise exception.BadCmdline( + f"The {repo_reference} doesn't seem to be a valid " + "offline RPM repository (RPM metadata not found)") + + config_opts["offline_local_repository"] = final_offline_repo + + # We install all the packages at once (for now?). We could inherit the + # command from the previous "online" run, but it often employs a group + # installation command - and we have no groups in the offline repo. + config_opts["chroot_setup_cmd"] = "install *" + + # The image needs to be prepared on host. Build-systems implementing SLSA 3 + # should make sure the config_opts["bootstrap_image"] is already downloaded. + config_opts["bootstrap_image_skip_pull"] = True + + # With isolated build, we always assert that we are reproducing the build + # with the same image. + config_opts["bootstrap_image_assert_digest"] = data["bootstrap"]["image_digest"] + + @traceLog() def nice_root_alias_error(name, alias_name, arch, no_configs, log): """ diff --git a/mock/py/mockbuild/installed_packages.py b/mock/py/mockbuild/installed_packages.py new file mode 100644 index 000000000..26fce5361 --- /dev/null +++ b/mock/py/mockbuild/installed_packages.py @@ -0,0 +1,139 @@ +""" +Helper methods for getting list of installed packages, and corresponding +packages' metadata +""" + +import os +import subprocess +import mockbuild.exception + + +def _subprocess_executor(command): + """ + We use doOutChroot normally in query_packages* methods, this is a helper + for testing purposes. + """ + return subprocess.check_output(command, env={"LC_ALL": "C"}).decode("utf-8") + + +def query_packages(fields, chrootpath=None, executor=_subprocess_executor): + """ + Query the list of installed packages, including FIELDS metadata, from + CHROOTPATH. + + The FIELDS argument is an array of RPM tags from 'rpm --querytags', without + the '%{}' syntax, for example ['name'] queries for %{name}'. There's an + additional non-standard "signature" field parsed from the standard + "%{sigpgp:pgpsig}" field (the last 8 hex characters). + + CHROOTPATH is the chroot directory with RPM DB. If CHROOTPATH is not + specified, the method uses the rpmdb from host. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + The method returns a list of dictionaries (package metadata info) in a + format documented on + https://docs.pagure.org/koji/content_generator_metadata/#buildroots + For example: + + [{ + "license": "LicenseRef-Fedora-Public-Domain", + "name": "filesystem", + "version": "3.18", + "release": "23.fc41", + "arch": "x86_64", + "epoch": null, + "sigmd5": "dc6edb2b7e390e5f0994267d22b9dc1a", + "signature": null + }] + """ + package_list_cmd = ["rpm", "-qa"] + if chrootpath: + package_list_cmd += ["--root", chrootpath] + package_list_cmd.append("--qf") + + # HACK: Zero-termination is not possible with 'rpm -q --qf QUERYSTRIG', so + # this is a hack. But how likely we can expect the following string in the + # real packages' metadata? + separator = '|/@' + + def _query_key(key): + # The Koji Content Generator's "signature" field can be queried via %{sigpgp} + if key == "signature": + return "sigpgp:pgpsig" + return key + + query_fields = [_query_key(f) for f in fields] + package_list_cmd.append(separator.join(f"%{{{x}}}" for x in query_fields) + "\n") + + def _fixup(package): + """ polish the package's metadata output """ + key = "signature" + if key in package: + if package[key] == "(none)": + package[key] = None + else: + # RSA/SHA256, Mon Jul 29 10:12:32 2024, Key ID 2322d3d94bf0c9db + # Get just last 8 chars ---> ^^^^^^^^ + package[key] = package[key].split()[-1][-8:] + key = "epoch" + if package[key] == "(none)": + package[key] = None + return package + + return [_fixup(p) for p in [dict(zip(fields, line.split(separator))) for + line in + sorted(executor(package_list_cmd).splitlines())] + if p["name"] != "gpg-pubkey"] + + +def query_packages_location(packages, chrootpath=None, executor=_subprocess_executor): + """ + Detect the URLs of the PACKAGES - array of dictionaries (see the output + from query_packages()) in available RPM repositories (/etc/yum.repos.d). + This method modifies PACKAGES in-situ, it adds "url" field to every single + dictionary in the PACKAGES array. + + CHROOTPATH is the chroot directory with RPM DB, if not specified, rpmdb + from host is used. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + Example output: + + [{ + "name": "filesystem", + "version": "3.18", + ... + "url": "https://example.com/fedora-repos-rawhide-42-0.1.noarch.rpm", + ... + }] + """ + + # Note: we do not support YUM in 2024+ + query_locations_cmd = ["/bin/dnf"] + if chrootpath: + query_locations_cmd += [f"--installroot={chrootpath}"] + # The -q is necessary because of and similar: + # https://github.com/rpm-software-management/dnf5/issues/1361 + query_locations_cmd += ["repoquery", "-q", "--location"] + query_locations_cmd += [ + f"{p['name']}-{p['version']}-{p['release']}.{p['arch']}" + for p in packages + ] + location_map = {} + for url in executor(query_locations_cmd).splitlines(): + basename = os.path.basename(url) + # name-arch pair should be unique on the box for every installed package + name, _, _ = basename.rsplit("-", 2) + arch = basename.split(".")[-2] + location_map[f"{name}.{arch}"] = url + + for package in packages: + name_arch = f"{package['name']}.{package['arch']}" + try: + package["url"] = location_map[name_arch] + except KeyError as exc: + raise mockbuild.exception.Error(f"Can't get location for {name_arch}") from exc diff --git a/mock/py/mockbuild/plugins/buildroot_lock.py b/mock/py/mockbuild/plugins/buildroot_lock.py new file mode 100644 index 000000000..6a26638fc --- /dev/null +++ b/mock/py/mockbuild/plugins/buildroot_lock.py @@ -0,0 +1,111 @@ +""" +Produce a lockfile for the prepared buildroot by Mock. Once available, we +should use the DNF built-in command from DNF5: +https://github.com/rpm-software-management/dnf5/issues/833 +""" + +import json +import os + +from mockbuild.podman import Podman +from mockbuild.installed_packages import query_packages, query_packages_location + +requires_api_version = "1.1" + + +def init(plugins, conf, buildroot): + """ The obligatory plugin entry point """ + BuildrootLockfile(plugins, conf, buildroot) + + +class BuildrootLockfile: + """ Produces buildroot_lock.json file in resultdir """ + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.state = buildroot.state + self.conf = conf + self.inst_done = False + plugins.add_hook("postdeps", self.produce_lockfile) + + def produce_lockfile(self): + """ + Upon a request ('produce_lockfile' option set True), generate + the mock-build-environment.json file in resultdir. The file describes + the Mock build environment, and the way to reproduce it. + """ + + filename = "buildroot_lock.json" + statename = "Generating the buildroot lockfile: " + filename + try: + with self.buildroot.uid_manager: + self.state.start(statename) + out_file = os.path.join(self.buildroot.resultdir, filename) + chrootpath = self.buildroot.make_chroot_path() + + # Ḿimic the Koji Content Generator metadata fields: + # https://docs.pagure.org/koji/content_generator_metadata/#buildroots + # + # The query_packages() method below sorts its output according + # to _values_ of the queried RPM headers, so keep name-arch pair + # first to have the output sorted reasonably. + query_fields = ["name", "arch", "license", "version", "release", + "epoch", "sigmd5", "signature"] + + def _executor(cmd): + out, _ = self.buildroot.doOutChroot(cmd, returnOutput=True, + returnStderr=False) + return out + + packages = query_packages(query_fields, chrootpath, _executor) + query_packages_location(packages, chrootpath, _executor) + + data = { + # Try to semver. The future Mock versions (the tool) should + # be able to read older Minor versions of the same Major. + # Anytime we break this assumption, bump the Major. IOW, + # the latest Mock implementing with Major == 1 can read any + # version from the 1.Y.Z range. Mock implementing v2.Y.Z + # no longer reads v1.Y.Z variants. + "version": "1.0.0", + "buildroot": { + "rpms": packages, + }, + # Try to keep this as minimal as possible. If possible, + # implement the config options as DEFAULTS in the + # isolated-build.cfg, or in the + # process_isolated_build_config() method. + "config": {} + } + for cfg_option in [ + # These are hard-coded in the configuration file, but we + # work with a single-config-for-all-arches now. + "target_arch", + "legal_host_arches", + "dist", + "package_manager", + # At this point, we only support isolated builds iff + # bootstrap_image_ready=True, so these two options are + # useful for implementing "assertion" in the + # process_isolated_build_config() method. + "bootstrap_image", + "bootstrap_image_ready", + ]: + if cfg_option in self.buildroot.config: + data["config"][cfg_option] = self.buildroot.config[cfg_option] + + if "bootstrap_image" in data["config"]: + # Optional object, only if bootstrap image used (we still + # produce lockfiles even if these are useless for isolated + # builds). + with self.buildroot.uid_manager.elevated_privileges(): + podman = Podman(self.buildroot, + data["config"]["bootstrap_image"]) + digest = podman.get_image_digest() + data["bootstrap"] = { + "image_digest": digest, + } + + with open(out_file, "w", encoding="utf-8") as fdlist: + fdlist.write(json.dumps(data, indent=4, sort_keys=True) + "\n") + finally: + self.state.finish(statename) diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index 9ece97502..7b96c3c6a 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -70,6 +70,14 @@ def pull_image(self): logger.error(out) return not exit_status + def import_tarball(self, tarball): + """ + Import tarball using podman into the local database. + """ + getLog().info("Loading bootstrap image from %s", tarball) + cmd = [self.podman_binary, "load", "-i", tarball] + util.do_with_status(cmd, env=self.buildroot.env) + def retry_image_pull(self, max_time): """ Try pulling the image multiple times """ @backoff.on_predicate(backoff.expo, lambda x: not x, @@ -106,6 +114,22 @@ def mounted_image(self): subprocess.run(cmd_umount, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + def get_image_digest(self): + """ + Get the "sha256:..." string for the image we work with. + """ + check = [self.podman_binary, "image", "inspect", self.image, + "--format", "{{ .Digest }}"] + result = subprocess.run(check, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False, + encoding="utf8") + if result.returncode: + raise BootstrapError(f"Can't get {self.image} podman image digest: {result.stderr}") + result = result.stdout.strip() + if len(result.splitlines()) != 1: + raise BootstrapError(f"The digest of {self.image} image is not a single-line string") + return result + @traceLog() def cp(self, destination, tar_cmd): """ copy content of container to destination directory """ diff --git a/mock/tests/test_buildroot_lock.py b/mock/tests/test_buildroot_lock.py new file mode 100644 index 000000000..9a89ee3b2 --- /dev/null +++ b/mock/tests/test_buildroot_lock.py @@ -0,0 +1,151 @@ +""" +Test the methods that generate buildroot_lock.json +""" + +import json +import os +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, patch +import jsonschema + +from mockbuild.plugins.buildroot_lock import init +import mockbuild.exception + +# gpg-pubkey packages stay ignored +# fedora-release is normal package with simple license +RPM_OUTPUT = """\ +gpg-pubkey|/@(none)|/@pubkey|/@105ef944|/@65ca83d1|/@(none)|/@(none)|/@(none) +gpg-pubkey|/@(none)|/@pubkey|/@e99d6ad1|/@64d2612c|/@(none)|/@(none)|/@(none) +fedora-release|/@noarch|/@MIT|/@42|/@0.3|/@(none)|/@cf31d87e5e3eac97ff32a98e7e073f37|/@(none) +bash|/@x86_64|/@GPLv3+|/@5.1.8|/@9.el9|/@(none)|/@57e93b1739cc3512f9f29dcaa8a38055|/@RSA/SHA256, Sun Mar 31 12:41:20 2024, Key ID 199e2f91fd431d51 +""" # noqa: E501 + +# cyrus-sasl-lib is not present in the REPOQUERY_OUTPUT +RPM_ONLY_RPM = """\ +cyrus-sasl-lib|/@x86_64|/@BSD with advertising|/@2.1.27|/@21.el9|/@(none)|/@9e1caba09fac94568419b9dfd14fb4c5|/@RSA/SHA256, Mon Sep 12 23:24:22 2022, Key ID 199e2f91fd431d51 +""" # noqa: E501 + +REPOQUERY_OUTPUT = """\ +http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/f/fedora-release-42-0.3.noarch.rpm +https://cdn.redhat.com/content/dist/rhel9/9/x86_64/baseos/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm +""" + +EXPECTED_OUTPUT = { + 'version': '1.0.0', + 'buildroot': { + 'rpms': [{ + 'arch': 'x86_64', + 'epoch': None, + 'license': 'GPLv3+', + 'name': 'bash', + 'release': '9.el9', + 'sigmd5': '57e93b1739cc3512f9f29dcaa8a38055', + 'signature': 'fd431d51', + 'url': 'https://cdn.redhat.com/content/dist/rhel9/9/x86_64' + '/baseos/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm', + 'version': '5.1.8' + }, { + 'arch': 'noarch', + 'epoch': None, + 'license': 'MIT', + 'name': 'fedora-release', + 'release': '0.3', + 'sigmd5': 'cf31d87e5e3eac97ff32a98e7e073f37', + 'signature': None, + 'url': 'http://ftp.fi.muni.cz/pub/linux/fedora/linux' + '/development/rawhide/Everything/x86_64/os' + '/Packages/f/fedora-release-42-0.3.noarch.rpm', + 'version': '42', + }] + }, + "bootstrap": { + "image_digest": "sha256:ba1067bef190fbe88f085bd019464a8c0803b7cd1e3f", + }, + 'config': { + 'bootstrap_image': 'foo', + 'bootstrap_image_ready': True, + "legal_host_arches": ["x86_64"], + "target_arch": "x86_64", + "dist": ".f42", + }, +} + + +def _mock_vars(rpm_out, repoquery_out): + tc = TestCase() + tc.maxDiff = None + buildroot = MagicMock() + buildroot.state = MagicMock() + buildroot.uid_manager = MagicMock() + buildroot.doOutChroot = MagicMock( + side_effect=iter([ + (rpm_out, None), + (repoquery_out, None), + ]) + ) + buildroot.config = EXPECTED_OUTPUT['config'] + buildroot.resultdir = tempfile.mkdtemp(prefix="mock-test-buildroot-lock") + plugins = MagicMock() + plugins.add_hook = MagicMock() + return tc, buildroot, plugins + + +def _call_method(plugins, buildroot): + # initialize the plugin + init(plugins, {}, buildroot) + # obtain the hook method, and call it + plugins.add_hook.assert_called_once() + _, method = plugins.add_hook.call_args[0] + + podman_obj = MagicMock() + podman_obj.get_image_digest.return_value = EXPECTED_OUTPUT["bootstrap"]["image_digest"] + podman_cls = MagicMock(return_value=podman_obj) + with patch("mockbuild.plugins.buildroot_lock.Podman", side_effect=podman_cls): + method() + + +def test_nonexisting_file_in_repo(): + """ + Test the situation when RPM is installed, and no longer available in + repository. + """ + _, buildroot, plugins = _mock_vars( + RPM_OUTPUT + RPM_ONLY_RPM, + REPOQUERY_OUTPUT, + ) + raised = False + try: + _call_method(plugins, buildroot) + except mockbuild.exception.Error as e: + assert e.msg == "Can't get location for cyrus-sasl-lib.x86_64" + raised = True + assert raised + + +def _get_json_schema(): + testdir = os.path.dirname(__file__) + basename = "buildroot-lock-schema-" + EXPECTED_OUTPUT["version"] + ".json" + with open(os.path.join(testdir, "..", "docs", basename), + "r", encoding="utf-8") as fd: + return json.load(fd) + + +def test_buildroot_lock_output(): + """ test the buildroot_lock.json file format """ + tc, buildroot, plugins = _mock_vars(RPM_OUTPUT, REPOQUERY_OUTPUT) + _call_method(plugins, buildroot) + with open(os.path.join(buildroot.resultdir, "buildroot_lock.json"), "r", + encoding="utf-8") as fd: + data = json.load(fd) + tc.assertDictEqual(data, EXPECTED_OUTPUT) + schema = _get_json_schema() + jsonschema.validate(EXPECTED_OUTPUT, schema) + + +def test_json_schema_metadata(): + """ Test basic format of the json schema """ + schema = _get_json_schema() + version = EXPECTED_OUTPUT["version"] + assert "Version " + version + ";" in schema["description"] + assert "schema-" + version + ".json" in schema["$id"] diff --git a/mock/tests/test_installed_packages.py b/mock/tests/test_installed_packages.py new file mode 100644 index 000000000..ba9eb87b4 --- /dev/null +++ b/mock/tests/test_installed_packages.py @@ -0,0 +1,14 @@ +""" +Test the "installed_packages.py" file +""" + +from mockbuild.installed_packages import _subprocess_executor + + +def test_the_default_executor(): + """ + The expected executor output is just stderr + """ + assert "stdout\n" == _subprocess_executor([ + "/bin/sh", "-c", "echo stdout ; echo >&2 stderr" + ]) diff --git a/releng/release-notes-next/isolated-build.feature b/releng/release-notes-next/isolated-build.feature new file mode 100644 index 000000000..e9980912f --- /dev/null +++ b/releng/release-notes-next/isolated-build.feature @@ -0,0 +1,9 @@ +Support for [isolated builds](feature-isolated-builds) has been +[implemented][PR#1393]. This update introduces two new command-line options: +`--calculate-build-deps` and `--isolated-build`, along with the new +`mock-isolated-repo(1)` utility. + +Additionally, this change introduces a new [`buildroot_lock` +plugin](Plugin-BuildrootLock), which generates a new artifact in the buildroot—a +buildroot *lockfile*. Users can enable this plugin explicitly by setting +`config_opts["plugin_conf"]["buildroot_lock_enable"] = True`. diff --git a/tox.ini b/tox.ini index 2775dd093..f737451a6 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ skipsdist = True deps = -rmock/requirements.txt coverage + jsonschema pytest pytest-cov backoff