Skip to content

Commit

Permalink
buildroot_image: allow using OCI images as the base for buildroot
Browse files Browse the repository at this point in the history
For now we can do this with:
--buildroot-image registry.fedoraproject.org/fedora:41
--buildroot-image /path/to/a/tarball.tar

This commit generalizes the logic for working with bootstrap image so it
can work the same both with bootstrap and buildroot images.

Fixes: #1159
Fixes: #1482
  • Loading branch information
praiskup committed Dec 14, 2024
1 parent 6e15cb6 commit 1ded6ab
Show file tree
Hide file tree
Showing 12 changed files with 191 additions and 11 deletions.
19 changes: 19 additions & 0 deletions behave/features/buildroot-image.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
Feature: Mock 6.0+ supports --bootstrap-image feature and OCI buildroot exports

@buildroot_image
Scenario: Use image from registry for buildroot preparation
Given an unique mock namespace
Given mock is always executed with "--buildroot-image registry.fedoraproject.org/fedora:rawhide"
When an online source RPM is rebuilt against fedora-rawhide-x86_64
Then the build succeeds

@buildroot_image
Scenario: Image from 'export_buildroot_image' works with --buildroot-image
Given an unique mock namespace
Given next mock call uses --enable-plugin=export_buildroot_image option
# No need to do a full build here!
When deps for python-copr-999-1.src.rpm are calculated against fedora-rawhide-x86_64
And OCI tarball from fedora-rawhide-x86_64 backed up and will be used
And the fedora-rawhide-x86_64 chroot is scrubbed
And an online SRPM python-copr-999-1.src.rpm is rebuilt against fedora-rawhide-x86_64
Then the build succeeds
26 changes: 24 additions & 2 deletions behave/steps/other.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,9 @@ def step_impl(context, expected_message):
assert_that(err[0], contains_string(expected_message))


def _rebuild_online(context, chroot=None):
url = context.test_storage + "mock-test-bump-version-1-0.src.rpm"
def _rebuild_online(context, chroot=None, package=None):
package = package or "mock-test-bump-version-1-0.src.rpm"
url = context.test_storage + package
if chroot:
context.mock.chroot = chroot
context.mock.chroot_opt = chroot
Expand All @@ -183,6 +184,12 @@ def step_impl(context):
def step_impl(context, chroot):
_rebuild_online(context, chroot)


@when('an online SRPM {package} is rebuilt against {chroot}')
def step_impl(context, package, chroot):
_rebuild_online(context, chroot, package)


@then('{output} contains "{text}"')
def step_impl(context, output, text):
index = 1 if output == "stdout" else 2
Expand Down Expand Up @@ -309,3 +316,18 @@ def step_impl(context):
break
assert_that(tarball.group(), equal_to("mock"))
assert_that(tarball.owner(), equal_to(context.current_user))


@when('OCI tarball from {chroot} backed up and will be used')
def step_impl(context, chroot):
resultdir = f"/var/lib/mock/{chroot}-{context.uniqueext}/result"
tarball_base = "buildroot-oci.tar"
tarball = os.path.join(resultdir, tarball_base)
assert os.path.exists(tarball)
shutil.copy(tarball, context.workdir)
context.mock.buildroot_image = os.path.join(context.workdir, tarball_base)


@when('the {chroot} chroot is scrubbed')
def step_impl(context, chroot):
context.mock.scrub(chroot)
25 changes: 24 additions & 1 deletion behave/testlib/mock.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,12 @@ def __init__(self, context):
context.mock_runs = {
"init": [],
"rebuild": [],
"scrubs": [],
"calculate-build-deps": [],
}

self.buildroot_image = None

@property
def basecmd(self):
""" return the pre-configured mock base command """
Expand Down Expand Up @@ -60,6 +63,19 @@ def init(self):
}]
return out, err

def scrub(self, chroot=None):
""" initialize chroot """
opts = ["--scrub=all"]
if chroot is not None:
opts += ["-r", chroot]
out, err = run_check(self.basecmd + opts)
self.context.mock_runs['scrubs'] += [{
"status": 0,
"out": out,
"err": err,
}]
return out, err

def rebuild(self, srpms):
""" Rebuild source RPM(s) """

Expand All @@ -71,7 +87,14 @@ def rebuild(self, srpms):
fd.write(self.context.custom_config)
chrootspec = ["-r", str(config_file)]

out, err = run_check(self.basecmd + chrootspec + ["--rebuild"] + srpms)
opts = []
if self.buildroot_image:
# use and drop
opts += ["--buildroot-image", self.buildroot_image]
self.buildroot_image = None
opts += ["--rebuild"] + srpms

out, err = run_check(self.basecmd + chrootspec + opts)
self.context.mock_runs['rebuild'] += [{
"status": 0,
"out": out,
Expand Down
42 changes: 42 additions & 0 deletions docs/Feature-buildroot-image.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
---
layout: default
title: Feature buildroot image
---

Starting from version v6.0, Mock allows users to use an OCI container image for
pre-creating the buildroot (build chroot). It can be either an online container
image hosted in a registry (or cached locally), or a local image in the form of
a tarball.

Be cautious when using chroot-compatible images (e.g., it is not advisable to
combine EPEL `ppc64le` images with `fedora-rawhide-x86_64` chroot).

## Example Use-Case

1. Mock aggressively caches the build root, so clean up your chroot first:

```bash
$ mock -r fedora-rawhide-x86_64 --scrub=all
```

2. Perform any normal Mock operation, but select the OCI image on top of that:

```bash
$ mock -r fedora-rawhide-x86_64 \
--buildroot-image registry.fedoraproject.org/fedora:41 \
--rebuild /your/src.rpm
```

## Using Exported Buildroot Image

The [export_buildroot_image](Plugin-Export-Buildroot-Image) plugin allows you to
wrap a prepared buildroot as an OCI archive (tarball). If you have this
tarball, you may select it as well:

```bash
$ mock -r fedora-rawhide-x86_64 \
--buildroot-image /tmp/buildroot-oci.tar \
--rebuild /your/src.rpm
```

Again, ensure that you do not combine incompatible chroot and image pairs.
4 changes: 3 additions & 1 deletion docs/Plugin-Export-Buildroot-Image.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ The archive has been saved in the result directory:
$ ls /var/lib/mock/fedora-rawhide-x86_64/result/*.tar
/var/lib/mock/fedora-rawhide-x86_64/result/buildroot-oci.tar

Then, you can try re-running the build without Mock, like this:
You may use this tarball together with [the `--buildroot-image` option
then](Feature-buildroot-image). But also, you can try re-running the
build without Mock, like this:

$ chmod a+r /tmp/quick-package/dummy-pkg-20241212_1114-1.src.rpm
$ podman run --rm -ti \
Expand Down
3 changes: 2 additions & 1 deletion docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,9 @@ Every plugin has a corresponding wiki page with docs.

## Features

* [container image for bootstrap](Feature-container-for-bootstrap) - set up bootstrap chroot using Podman.
* [bootstrap](Feature-bootstrap) - bootstrapping chroot. I.e., when building F28 on RHEL7, then first install very minimal bootstrap chroot with DNF and rpm from F28 and then use F28's rpm to install final F28 chroot.
* [container image for bootstrap](Feature-container-for-bootstrap) - set up bootstrap chroot using Podman.
* [container image for buildroot](Feature-buildroot-image) - pre-create buildroot from an OCI image
* [external dependencies](Feature-external-deps) - use of external dependencies, e.g., `BuildRequires external:pypi:foo`.
* [forcearch](Feature-forcearch) - build for foreign architecture using emulated virtualization.
* [nosync](Feature-nosync) - speed up build by making `fsync`(2) no-op.
Expand Down
5 changes: 5 additions & 0 deletions mock/docs/mock.1
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,11 @@ on non-RPM distributions. This option turns \fB\-\-bootstrap\-chroot\fR on.
\fB\-\-no-bootstrap-image\fR
don't create bootstrap chroot from container image

.TP
\fB\-\-buildroot\-image\fR \fIBUILDROOT_IMAGE\fR
Use an OCI image (or a local file containing an OCI image as a tarball) as the
base for the buildroot. The image must contain a compatible distribution.

.SH "FILES"
.LP
\fI/etc/mock/\fP \- default configuration directory
Expand Down
23 changes: 23 additions & 0 deletions mock/docs/site-defaults.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,7 @@
#############################################################################
#
# Things that you can change, but we dont recommend it:
#
# config_opts['chroothome'] = '/builddir'
# config_opts['clean'] = True
## you could not really use substitution here so it will not work if overridden:
Expand All @@ -404,6 +405,28 @@
#
# Each Mock run has a unique UUID
#config_opts["mock_run_uuid"] = str(uuid.uuid4())
#
# These OCI buildroot related options are set&used automatically by
# --buildroot-image option logic. The semantics are similar to the *bootstrap*
# counterparts above, e.g., see `config_opts['bootstrap_image']`.
#
# Use OCI image for build chroot initialization. Requires 'buildroot_image' to be set.
#config_opts['use_buildroot_image'] = False
# Initialize buildroot from this OCI image (image reference).
#config_opts['buildroot_image'] = None
# Mock normally tries to pull up2date buildroot image. Set to True if
# you want to use the local image.
#config_opts['buildroot_image_skip_pull'] = False
# No need to intsall any package into the buildroot extracted from an OCI
# image. TODO: not implemented.
#config_opts['buildroot_image_ready'] = False
# If the 'buildroot_image' above can not be used for any reason, fallback to a
# normal DNF installation. If set to False, it leads to hard failure.
#config_opts['buildroot_image_fallback'] = False
# Keep trying 'podman pull' for at most 120s.
#config_opts['buildroot_image_keep_getting'] = 120
# If set, mock compares the OCI image digest with the one specified here.
#config_opts['buildroot_image_assert_digest'] = None

#############################################################################
#
Expand Down
7 changes: 7 additions & 0 deletions mock/py/mock.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,13 @@ def command_parse():
parser.add_option('--no-bootstrap-image', dest='usebootstrapimage', action='store_false',
help="don't create bootstrap chroot from container image", default=None)

parser.add_option('--buildroot-image',
help=(
"Use an OCI image (or a local file containing an OCI "
"image as a tarball) as the base for the buildroot. "
"The image must contain a compatible distribution "
"(e.g., fedora:41 for fedora-41-x86_64)"))

parser.add_option('--additional-package', action='append', default=[],
type=str, dest="additional_packages",
help=("Additional package to install into the buildroot before "
Expand Down
16 changes: 10 additions & 6 deletions mock/py/mockbuild/buildroot.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,12 @@ def __init__(self, config, uid_manager, state, plugins, bootstrap_buildroot=None
self.env.update(proxy_env)
os.environ.update(proxy_env)

self.use_chroot_image = self.config['use_bootstrap_image']
self.chroot_image = self.config['bootstrap_image']
image = 'bootstrap' if is_bootstrap else 'buildroot'
self.use_chroot_image = self.config[f"use_{image}_image"]
self.chroot_image = self.config[f"{image}_image"]
self.image_skip_pull = self.config[f"{image}_image_skip_pull"]
self.image_assert_digest = self.config.get(f"{image}_image_assert_digest", None)
self.image_keep_getting = self.config[f"{image}_image_keep_getting"]

self.pkg_manager = None
self.mounts = mounts.Mounts(self)
Expand Down Expand Up @@ -239,7 +243,7 @@ def _init_locked(self):

@traceLog()
def _load_from_container_image(self):
if not self.uses_bootstrap_image or self.chroot_was_initialized:
if not self.use_chroot_image or self.chroot_was_initialized:
return

if util.mock_host_environment_type() == "docker":
Expand Down Expand Up @@ -269,15 +273,15 @@ def _fallback(message):
podman = Podman(self, self.chroot_image)

with _fallback("Can't initialize from container image"):
if not self.config["image_skip_pull"]:
podman.retry_image_pull(self.config["image_keep_getting"])
if not self.image_skip_pull:
podman.retry_image_pull(self.image_keep_getting)
else:
podman.read_image_id()
getLog().info("Using local image %s (%s)",
self.chroot_image, podman.image_id)
podman.tag_image()

digest_expected = self.config.get("image_assert_digest", None)
digest_expected = self.image_assert_digest
if digest_expected:
getLog().info("Checking image digest: %s",
digest_expected)
Expand Down
18 changes: 18 additions & 0 deletions mock/py/mockbuild/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,14 @@ def setup_default_config_opts():
config_opts['bootstrap_image_keep_getting'] = 120
config_opts['bootstrap_image_assert_digest'] = None

config_opts['use_buildroot_image'] = False
config_opts['buildroot_image'] = None
config_opts['buildroot_image_skip_pull'] = False
config_opts['buildroot_image_ready'] = False
config_opts['buildroot_image_fallback'] = False
config_opts['buildroot_image_keep_getting'] = 120
config_opts['buildroot_image_assert_digest'] = None

config_opts['internal_dev_setup'] = True

# cleanup_on_* only take effect for separate --resultdir
Expand Down Expand Up @@ -676,6 +684,16 @@ def set_config_opts_per_cmdline(config_opts, options, args):
if config_opts["calculatedeps"]:
config_opts["plugin_conf"]["buildroot_lock_enable"] = True

if options.buildroot_image: # --buildroot-image option
if os.path.exists(options.buildroot_image):
config_opts["buildroot_image"] = \
"oci-archive:" + os.path.realpath(options.buildroot_image)
else:
config_opts["buildroot_image"] = options.buildroot_image

if config_opts["buildroot_image"]:
config_opts["use_buildroot_image"] = True

def check_config(config_opts):
if 'root' not in config_opts:
raise exception.ConfigError("Error in configuration "
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
A new plugin, `export_buildroot_image`, has been added. This plugin can export
the Mock chroot as an OCI archive once all the build dependencies have been
installed (when the chroot is ready-made for runnign `/bin/rpmbuild -bb`).

A new complementary feature has been implemented in Mock, and can be enabled
using the following option:

--buildroot-image /tmp/buildroot-oci.tar

It allows the use of generated OCI archives as the source for the build chroot,
similar to how `bootstrap_image` is used "as the base" for the bootstrap chroot.

Additionally, this feature may be used as:

--buildroot-image registry.access.redhat.com/ubi8/ubi

Of course, in both cases it is important to use chroot-compatible iamges.

0 comments on commit 1ded6ab

Please sign in to comment.