From 1ded6ab047add7b720344332c7e6e0447dc41e23 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 29 Oct 2024 06:38:53 +0100 Subject: [PATCH] buildroot_image: allow using OCI images as the base for buildroot For now we can do this with: --buildroot-image registry.fedoraproject.org/fedora:41 --buildroot-image /path/to/a/tarball.tar This commit generalizes the logic for working with bootstrap image so it can work the same both with bootstrap and buildroot images. Fixes: #1159 Fixes: #1482 --- behave/features/buildroot-image.feature | 19 +++++++++ behave/steps/other.py | 26 +++++++++++- behave/testlib/mock.py | 25 ++++++++++- docs/Feature-buildroot-image.md | 42 +++++++++++++++++++ docs/Plugin-Export-Buildroot-Image.md | 4 +- docs/index.md | 3 +- mock/docs/mock.1 | 5 +++ mock/docs/site-defaults.cfg | 23 ++++++++++ mock/py/mock.py | 7 ++++ mock/py/mockbuild/buildroot.py | 16 ++++--- mock/py/mockbuild/config.py | 18 ++++++++ .../export-import-oci-buildroot-image.feature | 14 +++++++ 12 files changed, 191 insertions(+), 11 deletions(-) create mode 100644 behave/features/buildroot-image.feature create mode 100644 docs/Feature-buildroot-image.md diff --git a/behave/features/buildroot-image.feature b/behave/features/buildroot-image.feature new file mode 100644 index 000000000..8c4edf134 --- /dev/null +++ b/behave/features/buildroot-image.feature @@ -0,0 +1,19 @@ +Feature: Mock 6.0+ supports --bootstrap-image feature and OCI buildroot exports + + @buildroot_image + Scenario: Use image from registry for buildroot preparation + Given an unique mock namespace + Given mock is always executed with "--buildroot-image registry.fedoraproject.org/fedora:rawhide" + When an online source RPM is rebuilt against fedora-rawhide-x86_64 + Then the build succeeds + + @buildroot_image + Scenario: Image from 'export_buildroot_image' works with --buildroot-image + Given an unique mock namespace + Given next mock call uses --enable-plugin=export_buildroot_image option + # No need to do a full build here! + When deps for python-copr-999-1.src.rpm are calculated against fedora-rawhide-x86_64 + And OCI tarball from fedora-rawhide-x86_64 backed up and will be used + And the fedora-rawhide-x86_64 chroot is scrubbed + And an online SRPM python-copr-999-1.src.rpm is rebuilt against fedora-rawhide-x86_64 + Then the build succeeds diff --git a/behave/steps/other.py b/behave/steps/other.py index a353001cf..6a420b720 100644 --- a/behave/steps/other.py +++ b/behave/steps/other.py @@ -166,8 +166,9 @@ def step_impl(context, expected_message): assert_that(err[0], contains_string(expected_message)) -def _rebuild_online(context, chroot=None): - url = context.test_storage + "mock-test-bump-version-1-0.src.rpm" +def _rebuild_online(context, chroot=None, package=None): + package = package or "mock-test-bump-version-1-0.src.rpm" + url = context.test_storage + package if chroot: context.mock.chroot = chroot context.mock.chroot_opt = chroot @@ -183,6 +184,12 @@ def step_impl(context): def step_impl(context, chroot): _rebuild_online(context, chroot) + +@when('an online SRPM {package} is rebuilt against {chroot}') +def step_impl(context, package, chroot): + _rebuild_online(context, chroot, package) + + @then('{output} contains "{text}"') def step_impl(context, output, text): index = 1 if output == "stdout" else 2 @@ -309,3 +316,18 @@ def step_impl(context): break assert_that(tarball.group(), equal_to("mock")) assert_that(tarball.owner(), equal_to(context.current_user)) + + +@when('OCI tarball from {chroot} backed up and will be used') +def step_impl(context, chroot): + resultdir = f"/var/lib/mock/{chroot}-{context.uniqueext}/result" + tarball_base = "buildroot-oci.tar" + tarball = os.path.join(resultdir, tarball_base) + assert os.path.exists(tarball) + shutil.copy(tarball, context.workdir) + context.mock.buildroot_image = os.path.join(context.workdir, tarball_base) + + +@when('the {chroot} chroot is scrubbed') +def step_impl(context, chroot): + context.mock.scrub(chroot) diff --git a/behave/testlib/mock.py b/behave/testlib/mock.py index 0bdb28268..9e44fd6a8 100644 --- a/behave/testlib/mock.py +++ b/behave/testlib/mock.py @@ -30,9 +30,12 @@ def __init__(self, context): context.mock_runs = { "init": [], "rebuild": [], + "scrubs": [], "calculate-build-deps": [], } + self.buildroot_image = None + @property def basecmd(self): """ return the pre-configured mock base command """ @@ -60,6 +63,19 @@ def init(self): }] return out, err + def scrub(self, chroot=None): + """ initialize chroot """ + opts = ["--scrub=all"] + if chroot is not None: + opts += ["-r", chroot] + out, err = run_check(self.basecmd + opts) + self.context.mock_runs['scrubs'] += [{ + "status": 0, + "out": out, + "err": err, + }] + return out, err + def rebuild(self, srpms): """ Rebuild source RPM(s) """ @@ -71,7 +87,14 @@ def rebuild(self, srpms): fd.write(self.context.custom_config) chrootspec = ["-r", str(config_file)] - out, err = run_check(self.basecmd + chrootspec + ["--rebuild"] + srpms) + opts = [] + if self.buildroot_image: + # use and drop + opts += ["--buildroot-image", self.buildroot_image] + self.buildroot_image = None + opts += ["--rebuild"] + srpms + + out, err = run_check(self.basecmd + chrootspec + opts) self.context.mock_runs['rebuild'] += [{ "status": 0, "out": out, diff --git a/docs/Feature-buildroot-image.md b/docs/Feature-buildroot-image.md new file mode 100644 index 000000000..324a2391b --- /dev/null +++ b/docs/Feature-buildroot-image.md @@ -0,0 +1,42 @@ +--- +layout: default +title: Feature buildroot image +--- + +Starting from version v6.0, Mock allows users to use an OCI container image for +pre-creating the buildroot (build chroot). It can be either an online container +image hosted in a registry (or cached locally), or a local image in the form of +a tarball. + +Be cautious when using chroot-compatible images (e.g., it is not advisable to +combine EPEL `ppc64le` images with `fedora-rawhide-x86_64` chroot). + +## Example Use-Case + +1. Mock aggressively caches the build root, so clean up your chroot first: + + ```bash + $ mock -r fedora-rawhide-x86_64 --scrub=all + ``` + +2. Perform any normal Mock operation, but select the OCI image on top of that: + + ```bash + $ mock -r fedora-rawhide-x86_64 \ + --buildroot-image registry.fedoraproject.org/fedora:41 \ + --rebuild /your/src.rpm + ``` + +## Using Exported Buildroot Image + +The [export_buildroot_image](Plugin-Export-Buildroot-Image) plugin allows you to +wrap a prepared buildroot as an OCI archive (tarball). If you have this +tarball, you may select it as well: + +```bash +$ mock -r fedora-rawhide-x86_64 \ + --buildroot-image /tmp/buildroot-oci.tar \ + --rebuild /your/src.rpm +``` + +Again, ensure that you do not combine incompatible chroot and image pairs. diff --git a/docs/Plugin-Export-Buildroot-Image.md b/docs/Plugin-Export-Buildroot-Image.md index 7be7a62d0..50764c765 100644 --- a/docs/Plugin-Export-Buildroot-Image.md +++ b/docs/Plugin-Export-Buildroot-Image.md @@ -29,7 +29,9 @@ The archive has been saved in the result directory: $ ls /var/lib/mock/fedora-rawhide-x86_64/result/*.tar /var/lib/mock/fedora-rawhide-x86_64/result/buildroot-oci.tar -Then, you can try re-running the build without Mock, like this: +You may use this tarball together with [the `--buildroot-image` option +then](Feature-buildroot-image). But also, you can try re-running the +build without Mock, like this: $ chmod a+r /tmp/quick-package/dummy-pkg-20241212_1114-1.src.rpm $ podman run --rm -ti \ diff --git a/docs/index.md b/docs/index.md index 040916c04..354f2e94e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -221,8 +221,9 @@ Every plugin has a corresponding wiki page with docs. ## Features -* [container image for bootstrap](Feature-container-for-bootstrap) - set up bootstrap chroot using Podman. * [bootstrap](Feature-bootstrap) - bootstrapping chroot. I.e., when building F28 on RHEL7, then first install very minimal bootstrap chroot with DNF and rpm from F28 and then use F28's rpm to install final F28 chroot. +* [container image for bootstrap](Feature-container-for-bootstrap) - set up bootstrap chroot using Podman. +* [container image for buildroot](Feature-buildroot-image) - pre-create buildroot from an OCI image * [external dependencies](Feature-external-deps) - use of external dependencies, e.g., `BuildRequires external:pypi:foo`. * [forcearch](Feature-forcearch) - build for foreign architecture using emulated virtualization. * [nosync](Feature-nosync) - speed up build by making `fsync`(2) no-op. diff --git a/mock/docs/mock.1 b/mock/docs/mock.1 index ccae9a18b..129b6a060 100644 --- a/mock/docs/mock.1 +++ b/mock/docs/mock.1 @@ -500,6 +500,11 @@ on non-RPM distributions. This option turns \fB\-\-bootstrap\-chroot\fR on. \fB\-\-no-bootstrap-image\fR don't create bootstrap chroot from container image +.TP +\fB\-\-buildroot\-image\fR \fIBUILDROOT_IMAGE\fR +Use an OCI image (or a local file containing an OCI image as a tarball) as the +base for the buildroot. The image must contain a compatible distribution. + .SH "FILES" .LP \fI/etc/mock/\fP \- default configuration directory diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index 048425cc8..61d890f20 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -395,6 +395,7 @@ ############################################################################# # # Things that you can change, but we dont recommend it: +# # config_opts['chroothome'] = '/builddir' # config_opts['clean'] = True ## you could not really use substitution here so it will not work if overridden: @@ -404,6 +405,28 @@ # # Each Mock run has a unique UUID #config_opts["mock_run_uuid"] = str(uuid.uuid4()) +# +# These OCI buildroot related options are set&used automatically by +# --buildroot-image option logic. The semantics are similar to the *bootstrap* +# counterparts above, e.g., see `config_opts['bootstrap_image']`. +# +# Use OCI image for build chroot initialization. Requires 'buildroot_image' to be set. +#config_opts['use_buildroot_image'] = False +# Initialize buildroot from this OCI image (image reference). +#config_opts['buildroot_image'] = None +# Mock normally tries to pull up2date buildroot image. Set to True if +# you want to use the local image. +#config_opts['buildroot_image_skip_pull'] = False +# No need to intsall any package into the buildroot extracted from an OCI +# image. TODO: not implemented. +#config_opts['buildroot_image_ready'] = False +# If the 'buildroot_image' above can not be used for any reason, fallback to a +# normal DNF installation. If set to False, it leads to hard failure. +#config_opts['buildroot_image_fallback'] = False +# Keep trying 'podman pull' for at most 120s. +#config_opts['buildroot_image_keep_getting'] = 120 +# If set, mock compares the OCI image digest with the one specified here. +#config_opts['buildroot_image_assert_digest'] = None ############################################################################# # diff --git a/mock/py/mock.py b/mock/py/mock.py index 75960d1ce..ec628f1ba 100755 --- a/mock/py/mock.py +++ b/mock/py/mock.py @@ -398,6 +398,13 @@ def command_parse(): parser.add_option('--no-bootstrap-image', dest='usebootstrapimage', action='store_false', help="don't create bootstrap chroot from container image", default=None) + parser.add_option('--buildroot-image', + help=( + "Use an OCI image (or a local file containing an OCI " + "image as a tarball) as the base for the buildroot. " + "The image must contain a compatible distribution " + "(e.g., fedora:41 for fedora-41-x86_64)")) + parser.add_option('--additional-package', action='append', default=[], type=str, dest="additional_packages", help=("Additional package to install into the buildroot before " diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py index 385b48455..6633d0b8c 100644 --- a/mock/py/mockbuild/buildroot.py +++ b/mock/py/mockbuild/buildroot.py @@ -110,8 +110,12 @@ def __init__(self, config, uid_manager, state, plugins, bootstrap_buildroot=None self.env.update(proxy_env) os.environ.update(proxy_env) - self.use_chroot_image = self.config['use_bootstrap_image'] - self.chroot_image = self.config['bootstrap_image'] + image = 'bootstrap' if is_bootstrap else 'buildroot' + self.use_chroot_image = self.config[f"use_{image}_image"] + self.chroot_image = self.config[f"{image}_image"] + self.image_skip_pull = self.config[f"{image}_image_skip_pull"] + self.image_assert_digest = self.config.get(f"{image}_image_assert_digest", None) + self.image_keep_getting = self.config[f"{image}_image_keep_getting"] self.pkg_manager = None self.mounts = mounts.Mounts(self) @@ -239,7 +243,7 @@ def _init_locked(self): @traceLog() def _load_from_container_image(self): - if not self.uses_bootstrap_image or self.chroot_was_initialized: + if not self.use_chroot_image or self.chroot_was_initialized: return if util.mock_host_environment_type() == "docker": @@ -269,15 +273,15 @@ def _fallback(message): podman = Podman(self, self.chroot_image) with _fallback("Can't initialize from container image"): - if not self.config["image_skip_pull"]: - podman.retry_image_pull(self.config["image_keep_getting"]) + if not self.image_skip_pull: + podman.retry_image_pull(self.image_keep_getting) else: podman.read_image_id() getLog().info("Using local image %s (%s)", self.chroot_image, podman.image_id) podman.tag_image() - digest_expected = self.config.get("image_assert_digest", None) + digest_expected = self.image_assert_digest if digest_expected: getLog().info("Checking image digest: %s", digest_expected) diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index 0c540bc67..55c1521fd 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -105,6 +105,14 @@ def setup_default_config_opts(): config_opts['bootstrap_image_keep_getting'] = 120 config_opts['bootstrap_image_assert_digest'] = None + config_opts['use_buildroot_image'] = False + config_opts['buildroot_image'] = None + config_opts['buildroot_image_skip_pull'] = False + config_opts['buildroot_image_ready'] = False + config_opts['buildroot_image_fallback'] = False + config_opts['buildroot_image_keep_getting'] = 120 + config_opts['buildroot_image_assert_digest'] = None + config_opts['internal_dev_setup'] = True # cleanup_on_* only take effect for separate --resultdir @@ -676,6 +684,16 @@ def set_config_opts_per_cmdline(config_opts, options, args): if config_opts["calculatedeps"]: config_opts["plugin_conf"]["buildroot_lock_enable"] = True + if options.buildroot_image: # --buildroot-image option + if os.path.exists(options.buildroot_image): + config_opts["buildroot_image"] = \ + "oci-archive:" + os.path.realpath(options.buildroot_image) + else: + config_opts["buildroot_image"] = options.buildroot_image + + if config_opts["buildroot_image"]: + config_opts["use_buildroot_image"] = True + def check_config(config_opts): if 'root' not in config_opts: raise exception.ConfigError("Error in configuration " diff --git a/releng/release-notes-next/export-import-oci-buildroot-image.feature b/releng/release-notes-next/export-import-oci-buildroot-image.feature index 112ccc930..33f7668ad 100644 --- a/releng/release-notes-next/export-import-oci-buildroot-image.feature +++ b/releng/release-notes-next/export-import-oci-buildroot-image.feature @@ -1,3 +1,17 @@ A new plugin, `export_buildroot_image`, has been added. This plugin can export the Mock chroot as an OCI archive once all the build dependencies have been installed (when the chroot is ready-made for runnign `/bin/rpmbuild -bb`). + +A new complementary feature has been implemented in Mock, and can be enabled +using the following option: + + --buildroot-image /tmp/buildroot-oci.tar + +It allows the use of generated OCI archives as the source for the build chroot, +similar to how `bootstrap_image` is used "as the base" for the bootstrap chroot. + +Additionally, this feature may be used as: + + --buildroot-image registry.access.redhat.com/ubi8/ubi + +Of course, in both cases it is important to use chroot-compatible iamges.