From 06ce16dddae1e1770af22a34410012d319117a8c Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 28 Aug 2024 10:38:09 +0200 Subject: [PATCH] WIP readability --- docs/Plugin-BuildrootLock.md | 70 +++++++ docs/feature-isolated-builds.md | 182 ++++++++++++++++++ docs/index.md | 2 + mock/docs/mock.1 | 35 +++- mock/docs/site-defaults.cfg | 6 + mock/etc/mock/isolated-build.cfg | 31 +++ mock/mock.spec | 11 +- mock/py/mock-isolated-repo.py | 113 +++++++++++ mock/py/mock.py | 17 ++ mock/py/mockbuild/backend.py | 23 ++- mock/py/mockbuild/buildroot.py | 5 + mock/py/mockbuild/config.py | 68 ++++++- mock/py/mockbuild/installed_packages.py | 137 +++++++++++++ mock/py/mockbuild/plugins/buildroot_lock.py | 93 +++++++++ mock/py/mockbuild/podman.py | 8 + .../release-notes-next/isolated-build.feature | 3 + 16 files changed, 792 insertions(+), 12 deletions(-) create mode 100644 docs/Plugin-BuildrootLock.md create mode 100644 docs/feature-isolated-builds.md create mode 100644 mock/etc/mock/isolated-build.cfg create mode 100755 mock/py/mock-isolated-repo.py create mode 100644 mock/py/mockbuild/installed_packages.py create mode 100644 mock/py/mockbuild/plugins/buildroot_lock.py create mode 100644 releng/release-notes-next/isolated-build.feature diff --git a/docs/Plugin-BuildrootLock.md b/docs/Plugin-BuildrootLock.md new file mode 100644 index 000000000..d8a1e21a4 --- /dev/null +++ b/docs/Plugin-BuildrootLock.md @@ -0,0 +1,70 @@ +--- +layout: default +title: Plugin buildroot_lock +--- + +buildroot_lock Plugin +===================== + +This plugin generates an additional build artifact—the buildroot *lockfile* +(`buildroot.lock` file in the result directory). + +The *lockfile* describes both the list of buildroot sources (e.g., a list of +installed RPMs, bootstrap image info, etc.) and a set of Mock configuration +options. Using this information, Mock can later reproduce the buildroot +preparation (see the [Isolated Builds feature page](feature-isolated-builds)). + +This plugin is **disabled** by default but is automatically enabled with the +`--calculate-build-dependencies` option. You can enable it (for all builds) by +this configuration snippet: + +```python +config_opts['plugin_conf']['buildroot_lock_enable'] = True +``` + +**Note:** This plugin does not work with the `--offline` option. + + +## Format of the *buildroot.lock* file + + +The file `buildroot.lock` is a JSON file. + +Currently, we do not provide a compatibility promise. Only the exact same +version of Mock that produced the file is guaranteed to read and process it. +For more information, see [Isolated Builds](feature-isolated-builds). + +Example contents of such a file: + + { + "version": "0", + "buildroot": { + "packages": [ + { + "license": "MIT", + "name": "fedora-repos-rawhide", + "version": "42", + "release": "0.1", + "arch": "noarch", + "epoch": null, + "nvra": "fedora-repos-rawhide-42-0.1.noarch", + "sigmd5": "4378929dac9e51ed8470de5173ae664e", + "signature": null, + "url": "http://ftp.sh.cvut.cz/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/f/fedora-repos-rawhide-42-0.1.noarch.rpm" + }, + ... + ] + }, + "config": { + "target_arch": "x86_64", + "legal_host_arches": [ + "x86_64" + ], + "dist": "rawhide", + "package_manager": "dnf5", + "bootstrap_image": "registry.fedoraproject.org/fedora:rawhide", + "bootstrap_image_ready": true + } + } + +The list of packages is sorted by the `name + arch` pair. diff --git a/docs/feature-isolated-builds.md b/docs/feature-isolated-builds.md new file mode 100644 index 000000000..faf558c39 --- /dev/null +++ b/docs/feature-isolated-builds.md @@ -0,0 +1,182 @@ +--- +layout: default +title: Isolated builds with Mock +--- + +Isolated builds with Mock +========================= + +Mock (v5.7+) supports isolated RPM builds, sometimes referred to as "hermetic" +or "offline" builds. For more details, see the +[SLSA "isolated" definition][SLSA]. + +Quick start +----------- + +For the impatient, the TL;DR steps of the HOWTO are as follows: + + # we want to build this package + srpm=your-package.src.rpm + + # we'll create a local repository with pre-fetched RPMs/bootstrap + repo=/tmp/local-repo + + # resolve build deps for the given SRPM, in this case for Fedora Rawhide + mock --calculate-build-dependencies -r fedora-rawhide-x86_64 "$srpm" + + # find the lockfile in Mock's resultdir + lockfile=/var/lib/mock/fedora-rawhide-x86_64/result/buildroot.lock + + # create a local RPM repository (+ download bootstrap image) + mock-isolated-repo --lockfile "$lockfile --output-repo "$repo" + + # perform the isolated build! + mock --isolated-build-config "$lockfile" $repo" "$srpm" + +What an "isolated build" is.. +----------------------------- + +The term "isolated build" is often used in different contexts, even within +Mock's terminology. Historically, when we said that "Mock isolates the build," +we typically meant that Mock creates a *buildroot* (also referred to as a *build +directory* or *build chroot*) and runs the (Turing-complete, and thus +potentially insecure) *RPM build* process (i.e., a call to `/usr/bin/rpmbuild`) +inside it. In this sense, Mock "isolates" the RPM build process from the rest +of the system, or protects the system from potential mishaps. However, the +**buildroot preparation** process was never "isolated" in this manner—only the +*RPM build* was. Even the *RPM build* "isolation" was always performed on a +best-effort basis. For more details, see [Mock's Scope](index). + +When we now talk about making builds and the corresponding built artifacts +safer, more predictable, and more reproducible, we refer to the [SLSA +isolation][SLSA] definition. This involves using Mock in an *isolated* +environment, free from unintended external influence. + +Mock itself doesn't aim to provide this level of *isolation*. Mock is still +just a tool that runs in "some" build environment to perform the `SRPM → RPM` +translation. In such an environment, the Mock process can be tampered with by +other processes (potentially even root-owned), and as a result, the artifacts +may be (un)intentionally altered. Therefore, the preparation of the environment +to **run Mock** and the **isolation** itself is the responsibility of a +different tool (for example, `podman run --privileged --network=none`). + +So, what does Mock `--isolated-build-config` do if it doesn't isolate? +Essentially, it just does less work than it usually does! It optimizes out any +action (primarily during the *buildroot* preparation) that would rely on +"external" factors—specifically, it never expects Internet connectivity. +However, for the eventual build to succeed, **something else** still needs to +perform these omitted actions. Every single component required for *buildroot* +preparation must be prepared in advance for the `mock --isolated-build-config` +call (within **the** properly *isolated* environment, of course). + + +Challenges +---------- + +You’ve probably noticed that what used to be a simple command—like +`mock -r "$chroot" "$srpm"`—has now become a more complicated set of commands. + +This complexity arises because the *buildroot* in Mock is always prepared by +installing a set of RPMs (Mock calls DNF, DNF calls RPM, ...), which normally +requires a network connection. + +Additionally, it’s not always guaranteed that the DNF/RPM variant on the build +host (e.g., an EPEL 8 host) is sufficient or up-to-date for building the target +distribution (e.g., the newest Fedora Rawhide). Therefore, we need network +access [to obtain the appropriate bootstrap tooling](Feature-bootstrap). + +The [dynamic build dependencies][] further complicate the process. Without +them, we could at least make the `/bin/rpmbuild` fully offline—but with them, +it’s not so simple. Mock needs to interrupt the ongoing *RPM build* process, +resolve additional `%generate_buildrequires` (installing more packages on +demand), restart the *RPM build*, interrupt it again, and so on. This process +also requires a network connection! + +All of this is further complicated by the goal of making the *buildroot* as +*minimal* as possible—the fewer packages installed, the better. We can’t even +afford to install DNF into the buildroot, and as you’ve probably realized, we +definitely don’t want to blindly install all available RPMs. + + +The solution +------------ + +To address the challenges, we needed to separate the online +(`--calculate-build-dependencies`) and offline (`--isolated-build-config`) tasks +that Mock performs. + +1. **Online Tasks:** These need to be executed first. We let Mock prepare the + *buildroot #1* for the given *SRPM* (using the standard "online" method) and + record its *lockfile*—a list of all the resources obtained from the network + during the process. + + **Note:** The *buildroot* preparation also includes the installation of + dynamic build dependencies! Therefore, we **have to start an RPM build**. + Although we don’t finish the build (we terminate it once the + `%generate_buildrequires` is resolved, before reaching the `%build` phase, + etc.), it must be initiated. + +2. **Offline Repository Creation:** With the *lockfile* from the previous step, + we can easily retrieve the referenced components from the network. The Mock + project provides an example implementation for this step in the + `mock-isolated-repo(1)` utility. This tool downloads all the referenced + components from the internet and places them into a single local + directory—let's call it an *offline repository*. + + **Note:** This step doesn’t necessarily have to be done by the Mock project + itself. The *lockfile* is concise enough for further processing and + validation (e.g., ensuring the set of RPMs and the buildroot image come from + trusted sources) and could be parsed by build-system-specific tools like + [cachi2][] (potentially in the future). + +3. **Offline Build:** With the *srpm* and the *offline repository*, we can + instruct Mock to restart the build using the `--isolated-build-config + LOCKFILE OFFLINE_REPO SRPM` command. The *lockfile* is still needed at this + stage because it contains some of the configuration options used in step 1 + that must be inherited by the current Mock call. + + This step creates a new *buildroot #2* using the pre-downloaded RPMs in the + *offline repository* (installing them all at once) and then (re)starts the + RPM build process. + +You might notice that some steps are performed twice, specifically downloading +the RPMs (steps 1 and 2) and running the RPM build (steps 1 and 3). This +duplication is a necessary cost (in terms of more resources and time spent on +the build) to ensure that step 3 is _fully offline_. In step 3, the *offline* +RPM build is no longer interrupted by an *online* `%generate_buildrequires` +process—dependencies are already installed! + + +Limitations +----------- + +- We rely heavily on + the [Bootstrap Image feature](feature-container-for-bootstrap). This allows + us to easily abstract the bootstrap preparation tasks, which would otherwise + depend heavily on the system's RPM/DNF stack, etc. + + For now, we also require the Bootstrap Image to be *ready*. This simplifies + the implementation, as we don't need to recall the set of commands (or list of + packages to install into) needed for bootstrap preparation. + +- It is known fact that *normal builds* and *isolated builds* may result in + slightly different outputs (at least in theory). This issue relates to the + topic of *reproducible builds*. Normally, the *buildroot* is installed using + several DNF commands (RPM transactions), whereas the *isolated* build installs + all dependencies in a single DNF command (RPM transaction). While this + difference might cause the outputs of *normal* and *isolated* builds to vary + (in theory, because the chroot depends on RPM installation order), it OTOH + introduces more determinism! + +- The *lockfile* provides a list of the required RPMs, referenced by URLs. + These URLs point to the corresponding RPM repositories (online) from which + they were installed in step 1. However, in many cases, RPMs are downloaded + from `metalink://` or `mirrorlist://` repositories, meaning the URL might be + selected non-deterministically, and the specific mirrors chosen could be + rather ephemeral. For this reason, users should—for isolated builds—avoid + using mirrored repositories (as in the case of Koji builders) or avoid making + large delays between step 1 and step 2. + +[SLSA]: https://slsa.dev/spec/v1.0/requirements +[dynamic build dependencies]: https://github.com/rpm-software-management/mock/issues/1359 +[cachi2]: https://github.com/containerbuildsystem/cachi2 diff --git a/docs/index.md b/docs/index.md index 40869c762..78964f378 100644 --- a/docs/index.md +++ b/docs/index.md @@ -189,6 +189,7 @@ See a [separate document](Mock-Core-Configs). ## Plugins * [bind_mount](Plugin-BindMount) - bind mountpoints inside the chroot +* [buildroot_lock](Plugin-BuildrootLock) - provide a buildroot lockfile * [ccache](Plugin-CCache) - compiler cache plugin * [chroot_scan](Plugin-ChrootScan) - allows you to retrieve build artifacts from buildroot (e.g. additional logs, coredumps) * [compress_logs](Plugin-CompressLogs) - compress logs @@ -225,6 +226,7 @@ Every plugin has a corresponding wiki page with docs. * [package managers](Feature-package-managers) - supported package managers * [rhel chroots](Feature-rhelchroots) - builds for RHEL * [GPG keys and SSL](feature-gpg-and-ssl) - how to get your GPG keys and SSL certificates to buildroot +* [Isolated (offline) Builds](feature-isolated-builds) - doing offline builds with Mock ## Using Mock outside your git sandbox diff --git a/mock/docs/mock.1 b/mock/docs/mock.1 index 18385c33e..f49cf8efc 100644 --- a/mock/docs/mock.1 +++ b/mock/docs/mock.1 @@ -47,6 +47,10 @@ mock [options] \fB\-\-pm\-cmd\fR [\fIarguments ...\fR] mock [options] \fB\-\-yum\-cmd\fR [\fIarguments ...\fR] .LP mock [options] \fB\-\-dnf\-cmd\fR [\fIarguments ...\fR] +.LP +mock [options] \fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +.LP +mock [options] \fB\-\-isolated\-build\-config \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR .SH "DESCRIPTION" @@ -90,6 +94,21 @@ Mock is running some parts of code with root privileges. There are known ways to \fB\-\-buildsrpm\fP Build the specified SRPM either from a spec file and source file/directory or SCM. The chroot (including the results directory) is cleaned first, unless \-\-no\-clean is specified. .TP +\fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +Evaluate and install all the \fISRPM\fR (= file name, path on your system) build +dependencies, including dynamic dependencies in \fI%generate_buildrequires\fR. +This is similar to the \fB\-\-installdeps\fR option which only installs the +static \fIBuildRequires\fR. + +Additionally, record the metadata needed for a later isolated Mock build in +an output \fIbuildroot.lock\fR file (see also \fB\-\-isolated\-build\-config\fR). +Examples of such metadata are a list of RPM package URLs to download, the +bootstrap image to fetch, Mock config options to use, etc. + +After \fB\-\-calculate\-build\-dependencies\fR, you may want to use the +\fImock\-isolated\-repo(1)\fR helper to make your box prepared the isolated +build. +.TP \fB\-\-chain\fR When passing more than one SRPM, it will try to build failed builds if at least one subsequent SRPM succeed. This mimic the behaviour of deprecated mockchain. .TP @@ -135,7 +154,21 @@ Initialize a chroot (clean, install chroot packages, etc.). Do a yum install PACKAGE inside the chroot. No 'clean' is performed. .TP \fB\-\-installdeps\fP -Find out deps for SRPM or RPM, and do a yum install to put them in the chroot. No 'clean' is performed +Find out "static" deps for SRPM or RPM, and do a \fIdnf install\fR to put them +into the buildroot. No 'cleanup' is performed. + +Dynamic build dependencies (\fI%generate_buildrequires\fR specfile section) are +not installed, see \fB\-\-calculate\-build\-dependencies\fR. +.TP +\fB\-\-isolated\-build\-config \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR +Perform an isolated RPM build (i.e., an offline build without the need to access +the Internet at all) from the given \fISRPM\fR (= file name, path on your +system). After running Mock with the \fB\-\-calculate\-build\-dependencies\fR +option to generate the \fILOCKFILE\fR file (typically named \fIbuildroot.lock\fR +in the result directory), and then running the \fImock\-isolated\-repo(1)\fR +helper to generate \fIREPO\fR (a directory on the host that provides RPMs with +metadata and a bootstrap image tarball), Mock has all the necessary information +to build RPMs from the given \fISRPM\fR fully offline. .TP \fB\-\-list-chroots\fP List all available chroots names and their description - both system-wide and user ones. diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index b219a5e3b..ed4fc3137 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -653,6 +653,12 @@ # override the default Mock's stack call limit (5000). #config_opts["recursion_limit"] = 5000 +# Mock internals used by the --calculated-build-dependencies and +# --isolated-build-config options. Please do not set these options in Mock +# configuration files +# config_opts["calculatedeps"] = None +# config_opts["isolated_build"] = False + # List of usernames (strings) that will be pre-created in buildroot. The UID # and GID in-chroot is going to be the same as on-host. This option is for # example useful for the 'pesign' use-cases that both (a) bind-mount diff --git a/mock/etc/mock/isolated-build.cfg b/mock/etc/mock/isolated-build.cfg new file mode 100644 index 000000000..074a17263 --- /dev/null +++ b/mock/etc/mock/isolated-build.cfg @@ -0,0 +1,31 @@ +# TODO: link the feature page + +config_opts['root'] = 'isolated-build' +config_opts['description'] = 'Isolated Build' + +config_opts['dnf.conf'] = """ +[main] +keepcache=1 +system_cachedir=/var/cache/dnf +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= +install_weak_deps=0 +metadata_expire=0 +best=1 +protected_packages= + +# repos + +[offline] +name=offline repo +baseurl=file://{{ offline_local_repository }} +enabled=True +skip_if_unavailable=False +""" diff --git a/mock/mock.spec b/mock/mock.spec index 938ec4a41..efb1b6b2b 100644 --- a/mock/mock.spec +++ b/mock/mock.spec @@ -63,6 +63,8 @@ BuildRequires: python%{python3_pkgversion}-pylint BuildRequires: python%{python3_pkgversion}-rpm BuildRequires: python%{python3_pkgversion}-rpmautospec-core +BuildRequires: argparse-manpage + %if 0%{?fedora} >= 38 # DNF5 stack Recommends: dnf5 @@ -170,6 +172,9 @@ done # this is what %%sysusers_create_compat will expand to %{_rpmconfigdir}/sysusers.generate-pre.sh mock.conf > sysusers_script +argparse-manpage --pyfile ./py/mock-isolated-repo.py --function _argparser > mock-isolated-repo.1 + + %install #base filesystem mkdir -p %{buildroot}%{_sysconfdir}/mock/eol/templates @@ -178,6 +183,7 @@ mkdir -p %{buildroot}%{_sysconfdir}/mock/templates install -d %{buildroot}%{_bindir} install -d %{buildroot}%{_libexecdir}/mock install mockchain %{buildroot}%{_bindir}/mockchain +install py/mock-isolated-repo.py %{buildroot}%{_bindir}/mock-isolated-repo install py/mock-parse-buildlog.py %{buildroot}%{_bindir}/mock-parse-buildlog install py/mock.py %{buildroot}%{_libexecdir}/mock/mock ln -s consolehelper %{buildroot}%{_bindir}/mock @@ -204,7 +210,7 @@ install -d %{buildroot}%{python_sitelib}/ cp -a py/mockbuild %{buildroot}%{python_sitelib}/ install -d %{buildroot}%{_mandir}/man1 -cp -a docs/mock.1 docs/mock-parse-buildlog.1 %{buildroot}%{_mandir}/man1/ +cp -a docs/mock.1 docs/mock-parse-buildlog.1 mock-isolated-repo.1 %{buildroot}%{_mandir}/man1/ install -d %{buildroot}%{_datadir}/cheat cp -a docs/mock.cheat %{buildroot}%{_datadir}/cheat/mock @@ -247,6 +253,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # executables %{_bindir}/mock %{_bindir}/mockchain +%{_bindir}/mock-isolated-repo %{_bindir}/mock-parse-buildlog %{_libexecdir}/mock @@ -261,6 +268,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # config files %config(noreplace) %{_sysconfdir}/%{name}/*.ini +%config(noreplace) %{_sysconfdir}/%{name}/isolated-build.cfg %config(noreplace) %{_sysconfdir}/pam.d/%{name} %config(noreplace) %{_sysconfdir}/security/console.apps/%{name} @@ -271,6 +279,7 @@ pylint-3 py/mockbuild/ py/*.py py/mockbuild/plugins/* || : # docs %{_mandir}/man1/mock.1* %{_mandir}/man1/mock-parse-buildlog.1* +%{_mandir}/man1/mock-isolated-repo.1* %{_datadir}/cheat/mock # cache & build dirs diff --git a/mock/py/mock-isolated-repo.py b/mock/py/mock-isolated-repo.py new file mode 100755 index 000000000..df6ad582f --- /dev/null +++ b/mock/py/mock-isolated-repo.py @@ -0,0 +1,113 @@ +#! /usr/bin/python3 + +""" +Take the JSON provided by Mock, download corresponding RPMs, and put them into +an RPM repository. +""" + +# pylint: disable=invalid-name + +import argparse +import concurrent.futures +import json +import logging +import os +import shutil +import subprocess +import sys + +import requests + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +def download_file(url, outputdir): + """ + Download a single file (pool worker) + """ + file_name = os.path.join(outputdir, os.path.basename(url)) + log.info("Downloading %s", url) + try: + with requests.get(url, stream=True, timeout=30) as response: + if response.status_code != 200: + return False + with open(file_name, "wb") as fd: + shutil.copyfileobj(response.raw, fd) + return True + except: # noqa: E722 + log.exception("Exception raised for %s", url) + raise + + +def _argparser(): + parser = argparse.ArgumentParser( + prog='mock-isolated-repo', + description=( + "Prepare a repository for a `mock --isolated-build-config` build. " + "Given a Mock buildroot \"lockfile\"\n\n" + " a) create an output repo directory,\n" + " b) download and place all the necessary RPM files there,\n" + " c) create a local RPM repository there (run createrepo), and\n" + " d) dump there also the previously used bootstrap image as " + "a tarball.\n\n" + "Lockfile is a buildroot.lock file from Mock's " + "result directory; it is a JSON file generated by the " + "--calculate-build-dependencies option/buildroot_lock " + "plugin."), + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument("--lockfile", required=True, + help=( + "Select buildroot.lock filename on your system, " + "typically located in the Mock's result directory " + "upon the --calculate-build-dependencies mode " + "execution.")) + parser.add_argument("--output-repo", required=True, + help=( + "Download RPMs into this directory, and then run " + "/bin/createrepo_c utility there to populate the " + "RPM repo metadata.")) + return parser + + +def prepare_image(image_specification, outputdir): + """ + Store the tarball into the same directory where the RPMs are + """ + subprocess.check_output(["podman", "pull", image_specification]) + subprocess.check_output(["podman", "save", "--format=oci-archive", "--quiet", + "-o", os.path.join(outputdir, "bootstrap.tar"), + image_specification]) + + +def _main(): + options = _argparser().parse_args() + + with open(options.json, "r", encoding="utf-8") as fd: + data = json.load(fd) + + try: + os.makedirs(options.output_repo) + except FileExistsError: + pass + + failed = False + urls = [i["url"] for i in data["buildroot"]["packages"]] + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + for i, out in zip(urls, executor.map(download_file, urls, + [options.output_repo for _ in urls])): + if out is False: + log.error("Download failed: %s", i) + failed = True + if failed: + log.error("RPM deps downloading failed") + sys.exit(1) + + subprocess.check_call(["createrepo_c", options.output_repo]) + + prepare_image(data["config"]["bootstrap_image"], options.output_repo) + + +if __name__ == "__main__": + _main() diff --git a/mock/py/mock.py b/mock/py/mock.py index 76e52ff0e..f211b79d5 100755 --- a/mock/py/mock.py +++ b/mock/py/mock.py @@ -120,6 +120,9 @@ def command_parse(): parser.add_option("--rebuild", action="store_const", const="rebuild", dest="mode", default='__default__', help="rebuild the specified SRPM(s)") + parser.add_option("--calculate-build-dependencies", action="store_const", + const="calculatedeps", dest="mode", + help="Resolve and install static and dynamic build dependencies") parser.add_option("--chain", action="store_const", const="chain", dest="mode", help="build multiple RPMs in chain loop") @@ -390,6 +393,9 @@ def command_parse(): type=str, dest="additional_packages", help=("Additional package to install into the buildroot before " "the build is done. Can be specified multiple times.")) + parser.add_option("--isolated-build-config", nargs=2, + metavar=("LOCKFILE", "REPO_DIRECTORY"), + help="Perform an isolated (fully offline) SRPM build") (options, args) = parser.parse_known_args() @@ -402,6 +408,14 @@ def command_parse(): else: options.mode = 'rebuild' + if options.isolated_build_config and options.mode != "rebuild": + raise mockbuild.exception.BadCmdline("--rebuild mode needed with --isolated-build-config") + + options.calculatedeps = None + if options.mode == "calculatedeps": + options.mode = "rebuild" + options.calculatedeps = True + # Optparse.parse_args() eats '--' argument, while argparse doesn't. Do it manually. if args and args[0] == '--': args = args[1:] @@ -673,6 +687,9 @@ def main(): if options.configdir: config_path = options.configdir + if options.isolated_build_config: + options.chroot = "isolated-build" + config_opts = uidManager.run_in_subprocess_without_privileges( config.load_config, config_path, options.chroot) diff --git a/mock/py/mockbuild/backend.py b/mock/py/mockbuild/backend.py index e0cd32413..9649529d3 100644 --- a/mock/py/mockbuild/backend.py +++ b/mock/py/mockbuild/backend.py @@ -721,6 +721,7 @@ def rebuild_package(self, spec_path, timeout, check, dynamic_buildrequires): # --nodeps because rpm in the root may not be able to read rpmdb # created by rpm that created it (outside of chroot) check_opt = [] + calculatedeps = self.config["calculatedeps"] if not check: # this is because EL5/6 does not know --nocheck # when EL5/6 targets are not supported, replace it with --nocheck @@ -782,7 +783,9 @@ def get_command(mode, checkdeps=False): if packages_after == packages_before: success = True for f_buildreqs in buildreqs: - os.remove(f_buildreqs) + if not (success and calculatedeps): + # we want to keep the nosrc.rpm file + os.remove(f_buildreqs) # The first rpmbuild -br already did %prep, so we don't need waste time if '--noprep' not in br_mode: br_mode += ['--noprep'] @@ -806,14 +809,16 @@ def get_command(mode, checkdeps=False): # Unfortunately, we can only do this when using a bootstrap chroot, # because the rpm in the chroot might not understand the rpmdb otherwise. # See https://github.com/rpm-software-management/mock/issues/1246 - checkdeps = dynamic_buildrequires and self.bootstrap_buildroot is not None - self.buildroot.doChroot(get_command(mode, checkdeps=checkdeps), - shell=False, logger=self.buildroot.build_log, timeout=timeout, - uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, - user=self.buildroot.chrootuser, - nspawn_args=self._get_nspawn_args(), - unshare_net=self.private_network, - printOutput=self.config['print_main_output']) + + if not calculatedeps: + checkdeps = dynamic_buildrequires and self.bootstrap_buildroot is not None + self.buildroot.doChroot(get_command(mode, checkdeps=checkdeps), + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, + nspawn_args=self._get_nspawn_args(), + unshare_net=self.private_network, + printOutput=self.config['print_main_output']) results = glob.glob(bd_out + '/RPMS/*.rpm') results += glob.glob(bd_out + '/SRPMS/*.rpm') self.buildroot.final_rpm_list = [os.path.basename(result) for result in results] diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index 801d618a2..773fa78d3 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -268,6 +268,11 @@ def _fallback(message): getLog().info("Using local image %s (pull skipped)", self.bootstrap_image) + if self.config["isolated_build"]: + tarball = os.path.join(self.config["offline_local_repository"], + "bootstrap.tar") + podman.import_tarball(tarball) + podman.cp(self.make_chroot_path(), self.config["tar_binary"]) file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), "etc/rpm/macros.image-language-conf")) diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index f179bf8f6..d65a58c81 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -6,6 +6,7 @@ from ast import literal_eval from glob import glob +import json import grp import logging import os @@ -30,7 +31,7 @@ 'ccache', 'selinux', 'package_state', 'chroot_scan', 'lvm_root', 'compress_logs', 'sign', 'pm_request', 'hw_info', 'procenv', 'showrc', 'rpkg_preprocessor', - 'rpmautospec'] + 'rpmautospec', 'buildroot_lock'] def nspawn_supported(): """Detect some situations where the systemd-nspawn chroot code won't work""" @@ -170,6 +171,8 @@ def setup_default_config_opts(): 'available_pkgs': False, 'installed_pkgs': True, }, + 'buildroot_lock_enable': False, + 'buildroot_lock_opts': {}, 'pm_request_enable': False, 'pm_request_opts': {}, 'lvm_root_enable': False, @@ -388,6 +391,9 @@ def setup_default_config_opts(): config_opts["recursion_limit"] = 5000 + config_opts["calculatedeps"] = None + config_opts["isolated_build"] = False + return config_opts @@ -398,6 +404,7 @@ def multiply_platform_multiplier(config_opts): if '%_platform_multiplier' not in config_opts["macros"]: config_opts["macros"]["%_platform_multiplier"] = 10 if config_opts["forcearch"] else 1 + @traceLog() def set_config_opts_per_cmdline(config_opts, options, args): "takes processed cmdline args and sets config options." @@ -639,6 +646,11 @@ def set_config_opts_per_cmdline(config_opts, options, args): # which though affects root_cache). config_opts["additional_packages"] = options.additional_packages + config_opts["calculatedeps"] = options.calculatedeps + if config_opts["calculatedeps"]: + config_opts["plugin_conf"]["buildroot_lock_enable"] = True + + process_isolated_build_config(options, config_opts) def check_config(config_opts): if 'root' not in config_opts: @@ -699,6 +711,60 @@ def update_config_from_file(config_opts, config_file): config_opts["config_paths"] = list(new_paths) +def update_config_from_dict(config_opts, updates): + """ + Merge a dictionary into config_opts. No include supported. + """ + for key, value in updates.items(): + config_opts[key] = value + + +def process_isolated_build_config(cmdline_opts, config_opts): + """ + Read the lockfile file generated by the previous + --calculate-build-dependencies run, and adjust the current set of options + in CONFIG_OPTS. + """ + + if not cmdline_opts.isolated_build_config: + return + + config_opts["isolated_build"] = True + + json_conf, repo_reference = cmdline_opts.isolated_build_config + with open(json_conf, "r", encoding="utf-8") as fd: + data = json.load(fd) + + if not data["config"].get("bootstrap_image_ready"): + raise exception.BadCmdline( + f"The file {json_conf} did not record the bootstrap_image_ready=True " + "config which means we are not able to prepare the bootstrap chroot " + "in an isolated mode.") + + update_config_from_dict(config_opts, data["config"]) + + final_offline_repo = repo_reference + file_pfx = "file://" + if final_offline_repo.startswith(file_pfx): + final_offline_repo = final_offline_repo[len(file_pfx):] + final_offline_repo = os.path.abspath(final_offline_repo) + if not os.path.exists(os.path.join(final_offline_repo, "repodata")): + raise exception.BadCmdline( + f"The {repo_reference} doesn't seem to be a valid " + "offline RPM repository (RPM metadata not found)") + + config_opts["offline_local_repository"] = final_offline_repo + + # We install all the packages at once (for now?). We could inherit the + # command from the previous "online" run, but it often employs a group + # installation command - and we have no groups in the offline repo. + config_opts["chroot_setup_cmd"] = "install *" + + # The image needs to be prepared on host. Build-systems implementing SLSA 3 + # should make sure the config_opts["bootstrap_image"] is already downloaded. + config_opts["bootstrap_image_skip_pull"] = True + + @traceLog() def nice_root_alias_error(name, alias_name, arch, no_configs, log): """ diff --git a/mock/py/mockbuild/installed_packages.py b/mock/py/mockbuild/installed_packages.py new file mode 100644 index 000000000..b3ebfb523 --- /dev/null +++ b/mock/py/mockbuild/installed_packages.py @@ -0,0 +1,137 @@ +""" +Helper methods for getting list of installed packages, and corresponding +packages' metadata +""" + +import os +import subprocess +import mockbuild.exception + + +def _subprocess_executor(command): + """ + We use doOutChroot normally in query_packages* methods, this is a helper + for testing purposes. + """ + return subprocess.check_output(command, env={"LC_ALL": "C"}).decode("utf-8") + + +def query_packages(fields, chrootpath=None, executor=_subprocess_executor): + """ + Query the list of installed packages, including FIELDS metadata, from + CHROOTPATH. + + The FIELDS argument is an array of RPM tags from 'rpm --querytags', without + the '%{}' syntax, for example ['name'] queries for %{name}'. There's an + additional non-standard "signature" field parsed from the standard + "%{sigpgp:pgpsig}" field (the last 8 hex characters). + + CHROOTPATH is the chroot directory with RPM DB. If CHROOTPATH is not + specified, the method uses the rpmdb from host. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + The method returns a list of dictionaries (package metadata info) in a + format documented on + https://docs.pagure.org/koji/content_generator_metadata/#buildroots + with some additional fields like nvra. For example: + + [{ + "license": "LicenseRef-Fedora-Public-Domain", + "name": "filesystem", + "version": "3.18", + "release": "23.fc41", + "arch": "x86_64", + "epoch": null, + "nvra": "filesystem-3.18-23.fc41.x86_64", + "sigmd5": "dc6edb2b7e390e5f0994267d22b9dc1a", + "signature": null + }] + """ + package_list_cmd = ["rpm", "-qa"] + if chrootpath: + package_list_cmd += ["--root", chrootpath] + package_list_cmd.append("--qf") + + # HACK: Zero-termination is not possible with 'rpm -q --qf QUERYSTRIG', so + # this is a hack. But how likely we can expect the following string in the + # real packages' metadata? + separator = '|/@' + + def _query_key(key): + # The Koji Content Generator's "signature" field can be queried via %{sigpgp} + if key == "signature": + return "sigpgp:pgpsig" + return key + + query_fields = [_query_key(f) for f in fields] + package_list_cmd.append(separator.join(f"%{{{x}}}" for x in query_fields) + "\n") + + def _fixup(package): + """ polish the package's metadata output """ + key = "signature" + if key in package: + if package[key] == "(none)": + package[key] = None + else: + # RSA/SHA256, Mon Jul 29 10:12:32 2024, Key ID 2322d3d94bf0c9db + # Get just last 8 chars ---> ^^^^^^^^ + package[key] = package[key].split()[-1][-8:] + key = "epoch" + if package[key] == "(none)": + package[key] = None + return package + + return [_fixup(p) for p in [dict(zip(fields, line.split(separator))) for + line in + sorted(executor(package_list_cmd).splitlines())] + if p["name"] != "gpg-pubkey"] + + +def query_packages_location(packages, chrootpath=None, executor=_subprocess_executor): + """ + Detect the URLs of the PACKAGES - array of dictionaries (see the output + from query_packages()) in available RPM repositories (/etc/yum.repos.d). + This method modifies PACKAGES in-situ, it adds "url" field to every single + dictionary in the PACKAGES array. + + CHROOTPATH is the chroot directory with RPM DB, if not specified, rpmdb + from host is used. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + Example output: + + [{ + "name": "filesystem", + "version": "3.18", + ... + "url": "https://example.com/fedora-repos-rawhide-42-0.1.noarch.rpm", + ... + }] + """ + + # Note: we do not support YUM in 2024+ + query_locations_cmd = ["/bin/dnf"] + if chrootpath: + query_locations_cmd += [f"--installroot={chrootpath}"] + query_locations_cmd += ["repoquery", "--location"] + query_locations_cmd += [p["nvra"] for p in packages] + location_map = {} + for url in executor(query_locations_cmd).splitlines(): + if not url.endswith("rpm"): + continue + basename = os.path.basename(url) + # name-arch pair should be unique on the box for every installed package + name, _, _ = basename.rsplit("-", 2) + arch = basename.split(".")[-2] + location_map[f"{name}.{arch}"] = url + + for package in packages: + name_arch = f"{package['name']}.{package['arch']}" + try: + package["url"] = location_map[name_arch] + except KeyError as exc: + raise mockbuild.exception.Error(f"Can't get location for {name_arch}") from exc diff --git a/mock/py/mockbuild/plugins/buildroot_lock.py b/mock/py/mockbuild/plugins/buildroot_lock.py new file mode 100644 index 000000000..e74e92efa --- /dev/null +++ b/mock/py/mockbuild/plugins/buildroot_lock.py @@ -0,0 +1,93 @@ +""" +Produce a lockfile for the prepared buildroot by Mock. Once available, we +should use the DNF built-in command from DNF5: +https://github.com/rpm-software-management/dnf5/issues/833 +""" + +import json +import os + +from mockbuild.installed_packages import query_packages, query_packages_location + +requires_api_version = "1.1" + +def init(plugins, conf, buildroot): + """ The obligatory plugin entry point """ + BuildrootLockfile(plugins, conf, buildroot) + + +class BuildrootLockfile: + """ Produces buildroot.lock (json format) in resultdir """ + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.state = buildroot.state + self.conf = conf + self.inst_done = False + plugins.add_hook("postdeps", self.produce_lockfile) + + def produce_lockfile(self): + """ + Upon a request ('produce_lockfile' option set True), generate + the mock-build-environment.json file in resultdir. The file describes + the Mock build environment, and the way to reproduce it. + """ + + filename = "buildroot.lock" + statename = "Generating the buildroot lockfile: " + filename + try: + with self.buildroot.uid_manager: + self.state.start(statename) + out_file = os.path.join(self.buildroot.resultdir, filename) + chrootpath = self.buildroot.make_chroot_path() + + # Ḿimic the Koji Content Generator metadata fields: + # https://docs.pagure.org/koji/content_generator_metadata/#buildroots + # + # The query_packages() method below sorts its output according + # to _values_ of the queried RPM headers, so keep name-arch pair + # first to have the output sorted reasonably. + query_fields = ["name", "arch", "license", "version", "release", + "epoch", "nvra", "sigmd5", "signature"] + + def _executor(cmd): + out, _ = self.buildroot.doOutChroot(cmd, returnOutput=True) + return out + + packages = query_packages(query_fields, chrootpath, _executor) + query_packages_location(packages, chrootpath, _executor) + + data = { + "version": "0", # too-early stage for compatibility promises + "buildroot": { + "packages": packages, + }, + # Try to keep this as minimal as possible. If possible, + # implement the config options as DEFAULTS in the + # isolated-build.cfg, or in the + # process_isolated_build_config() method. + "config": {} + } + for cfg_option in [ + # These are hard-coded in the configuration file, but we + # work with a single-config-for-all-arches now. + "target_arch", + "legal_host_arches", + "dist", + "package_manager", + # At this point, we only support isolated builds iff + # bootstrap_image_ready=True, so these two options are + # useful for implementing "assertion" in the + # process_isolated_build_config() method. + "bootstrap_image", + "bootstrap_image_ready", + ]: + if cfg_option in self.buildroot.config: + data["config"][cfg_option] = self.buildroot.config[cfg_option] + + if self.buildroot.config['bootstrap_image']: + data["bootstrap_image"] = self.buildroot.config['bootstrap_image'] + + with open(out_file, "w", encoding="utf-8") as fdlist: + fdlist.write(json.dumps(data, indent=4, sort_keys=True) + "\n") + finally: + self.state.finish(statename) diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py index 9ece97502..e46d3d670 100644 --- a/mock/py/mockbuild/podman.py +++ b/mock/py/mockbuild/podman.py @@ -70,6 +70,14 @@ def pull_image(self): logger.error(out) return not exit_status + def import_tarball(self, tarball): + """ + Import tarball using podman into the local database. + """ + getLog().info("Loading bootstrap image from %s", tarball) + cmd = [self.podman_binary, "load", "-i", tarball] + util.do_with_status(cmd, env=self.buildroot.env) + def retry_image_pull(self, max_time): """ Try pulling the image multiple times """ @backoff.on_predicate(backoff.expo, lambda x: not x, diff --git a/releng/release-notes-next/isolated-build.feature b/releng/release-notes-next/isolated-build.feature new file mode 100644 index 000000000..1ca8fd758 --- /dev/null +++ b/releng/release-notes-next/isolated-build.feature @@ -0,0 +1,3 @@ +Isolated build support implemented. There's are two new command-line options +for this, `--calculate-build-deps` and `--isolated-build`. The new feature is +documented on its feature page.