From d94aa10f0edc75a88e0966c2d65ed3b8427fcaa5 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Thu, 8 Feb 2024 20:32:10 +0100 Subject: [PATCH] A new 'repo_arch' option This option is used for the cases when $basearch DNF-native variable doesn't work as expected in URLs. Mock now also uses the `repo_arch` values for the `man 2 personality` decisions during the bootstrap chroot initialization, as bootstrap is is almost always expected to be "host-native". Closes: #1317 Fixes: #1304 --- .../etc/mock/templates/mageia-branched.tpl | 32 +++++++++---------- .../etc/mock/templates/mageia-cauldron.tpl | 16 +++++----- mock/docs/site-defaults.cfg | 26 +++++++++++++++ mock/py/mock.py | 16 ++++++++++ mock/py/mockbuild/config.py | 6 ++++ mock/py/mockbuild/package_manager.py | 22 +++++-------- mock/tests/test_config_templates.py | 25 +++++++++++++++ releng/release-notes-next/repo-arch.feature | 6 ++++ 8 files changed, 111 insertions(+), 38 deletions(-) create mode 100644 releng/release-notes-next/repo-arch.feature diff --git a/mock-core-configs/etc/mock/templates/mageia-branched.tpl b/mock-core-configs/etc/mock/templates/mageia-branched.tpl index 280fb73b3..9c6e06ec3 100644 --- a/mock-core-configs/etc/mock/templates/mageia-branched.tpl +++ b/mock-core-configs/etc/mock/templates/mageia-branched.tpl @@ -29,10 +29,10 @@ user_agent={{ user_agent }} # repos [mageia] -name=Mageia $releasever - {{ target_arch }} -#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ target_arch }}/media/core/release/ -#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ target_arch }}@§ion=core&repo=release -mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ target_arch }}§ion=core&repo=release +name=Mageia $releasever - {{ repo_arch }} +#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ repo_arch }}/media/core/release/ +#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ repo_arch }}@§ion=core&repo=release +mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ repo_arch }}§ion=core&repo=release fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia @@ -40,10 +40,10 @@ enabled=1 skip_if_unavailable=False [updates] -name=Mageia $releasever - {{ target_arch }} - Updates -#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ target_arch }}/media/core/updates/ -#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ target_arch }}@§ion=core&repo=updates -mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ target_arch }}§ion=core&repo=updates +name=Mageia $releasever - {{ repo_arch }} - Updates +#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ repo_arch }}/media/core/updates/ +#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ repo_arch }}@§ion=core&repo=updates +mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ repo_arch }}§ion=core&repo=updates fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia @@ -51,10 +51,10 @@ enabled=1 skip_if_unavailable=False [mageia-debuginfo] -name=Mageia $releasever - {{ target_arch }} - Debug -#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ target_arch }}/media/debug/core/release/ -#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ target_arch }}@§ion=core&repo=release&debug=true -mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ target_arch }}§ion=core&repo=release&debug=1 +name=Mageia $releasever - {{ repo_arch }} - Debug +#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ repo_arch }}/media/debug/core/release/ +#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ repo_arch }}@§ion=core&repo=release&debug=true +mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ repo_arch }}§ion=core&repo=release&debug=1 fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia @@ -62,10 +62,10 @@ enabled=0 skip_if_unavailable=False [updates-debuginfo] -name=Mageia $releasever - {{ target_arch }} - Updates - Debug -#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ target_arch }}/media/debug/core/updates/ -#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ target_arch }}@§ion=core&repo=updates&debug=true -mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ target_arch }}§ion=core&repo=updates&debug=1 +name=Mageia $releasever - {{ repo_arch }} - Updates - Debug +#baseurl=http://mirrors.kernel.org/mageia/distrib/$releasever/{{ repo_arch }}/media/debug/core/updates/ +#metalink=https://mirrors.mageia.org/metalink?distrib=mageia-$releasever&arch={{ repo_arch }}@§ion=core&repo=updates&debug=true +mirrorlist=https://www.mageia.org/mirrorlist/?release=$releasever&arch={{ repo_arch }}§ion=core&repo=updates&debug=1 fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia diff --git a/mock-core-configs/etc/mock/templates/mageia-cauldron.tpl b/mock-core-configs/etc/mock/templates/mageia-cauldron.tpl index 195830be1..98109a1d2 100644 --- a/mock-core-configs/etc/mock/templates/mageia-cauldron.tpl +++ b/mock-core-configs/etc/mock/templates/mageia-cauldron.tpl @@ -30,10 +30,10 @@ user_agent={{ user_agent }} # repos [mageia-cauldron] -name=Mageia Cauldron - {{ target_arch }} -#baseurl=http://mirrors.kernel.org/mageia/distrib/cauldron/{{ target_arch }}/media/core/release/ -#metalink=https://mirrors.mageia.org/metalink?distrib=cauldron&arch={{ target_arch }}@§ion=core&repo=release -mirrorlist=https://www.mageia.org/mirrorlist/?release=cauldron&arch={{ target_arch }}§ion=core&repo=release +name=Mageia Cauldron - {{ repo_arch }} +#baseurl=http://mirrors.kernel.org/mageia/distrib/cauldron/{{ repo_arch }}/media/core/release/ +#metalink=https://mirrors.mageia.org/metalink?distrib=cauldron&arch={{ repo_arch }}@§ion=core&repo=release +mirrorlist=https://www.mageia.org/mirrorlist/?release=cauldron&arch={{ repo_arch }}§ion=core&repo=release fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia @@ -41,10 +41,10 @@ enabled=1 skip_if_unavailable=False [mageia-cauldron-debuginfo] -name=Mageia Cauldron - {{ target_arch }} - Debug -#baseurl=http://mirrors.kernel.org/mageia/distrib/cauldron/{{ target_arch }}/media/debug/core/release/ -#metalink=https://mirrors.mageia.org/metalink?distrib=cauldron&arch={{ target_arch }}@§ion=core&repo=release&debug=true -mirrorlist=https://www.mageia.org/mirrorlist/?release=cauldron&arch={{ target_arch }}§ion=core&repo=release&debug=1 +name=Mageia Cauldron - {{ repo_arch }} - Debug +#baseurl=http://mirrors.kernel.org/mageia/distrib/cauldron/{{ repo_arch }}/media/debug/core/release/ +#metalink=https://mirrors.mageia.org/metalink?distrib=cauldron&arch={{ repo_arch }}@§ion=core&repo=release&debug=true +mirrorlist=https://www.mageia.org/mirrorlist/?release=cauldron&arch={{ repo_arch }}§ion=core&repo=release&debug=1 fastestmirror=1 gpgcheck=1 gpgkey=file:///usr/share/distribution-gpg-keys/mageia/RPM-GPG-KEY-Mageia diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg index 7bd478c4b..1900047c3 100644 --- a/mock/docs/site-defaults.cfg +++ b/mock/docs/site-defaults.cfg @@ -663,3 +663,29 @@ # example, the host is configured to use FreeIPA-provided subids. # See https://github.com/shadow-maint/shadow/issues/897 # config_opts["use_host_shadow_utils"] = True + +# The `repo_arch` (read-only) option simplifies DNF configuration with Mock for +# non-trivial architecture selection decisions. Typically, we want to use the +# DNF-native `$basearch` variable to instruct DNF to use the appropriate RPM +# architecture for a given Mock config (for cross-arch builds, bootstrap uses a +# different architecture than the target chroot!). However, `$basearch` often +# doesn't work correctly — some distributions do not align the mirror URLs with +# the `$basearch` content (as known by DNF), causing problems with +# cross-distro/cross-architecture builds. The `repo_arch` internal is then +# exported as a `{{ repo_arch }}` Jinja2 placeholder, aiming to help with this +# problem. Simply replace `$basearch` with `{{ repo_arch }}` in your config. +# +# The `repo_arch` thing is not really an "option" but rather a Mock internal +# exported for read-only use-cases. However, the `repo_arch_map` dictionary +# can be used to affect Mock's background decisions. For example, when +# the configuration claims `target_arch=armv7hnl`, but the repo URLs look like +# 'example.com/arm32/', one can use +# `baseurl=example.com/{{ repo_arch }}/` instead of +# `baseurl=example.com/$basearch/`, together with +# `config_opts["repo_arch_map"] = {"armv7hnl": "arm32"}`. +# In such a case, builds on `x86_64` hosts will expand to `example.com/x86_64` +# URL for the bootstrap (native) chroot installation, but also to +# `example.com/arm32` for the target (cross-arch, emulated) chroot +# installation. +#config_opts["repo_arch"] = "Mock internal, e.g. 'x86_64'" +#config_opts["repo_arch_map"] = {} diff --git a/mock/py/mock.py b/mock/py/mock.py index c87326c63..76e52ff0e 100755 --- a/mock/py/mock.py +++ b/mock/py/mock.py @@ -739,6 +739,22 @@ def main(): # disable updating bootstrap chroot bootstrap_buildroot_config['update_before_build'] = False + # Enforce host-native repo architecture for bootstrap chroot (unless + # bootstrap_forcearch=True, which should never be the case). This + # decision affects condPersonality() for DNF calls! + host_arch = config_opts["host_arch"] + if config_opts["use_bootstrap_image"]: + # with bootstrap image, bootstrap is always native + bootstrap_buildroot_config['repo_arch'] = config_opts['repo_arch_map'].get(host_arch, host_arch) + elif host_arch not in config_opts.get("legal_host_arches", []) \ + and not config_opts.get('bootstrap_forcearch'): + # target chroot uses --forcearch, but bootstrap is native + bootstrap_buildroot_config['repo_arch'] = config_opts['repo_arch_map'].get(host_arch, host_arch) + # else: + # We keep the 'repo_arch' copied from target chroot (config_opts['repo_arch']): + # - for the 'multilib' cases (we don't want to use x86_64 repos for i586 chroots) + # - 'bootstrap_forcearch' is set + # disable forcearch in bootstrap, per https://github.com/rpm-software-management/mock/issues/1110 bootstrap_buildroot_config['forcearch'] = None diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py index 2bbd34682..8cf9e129d 100644 --- a/mock/py/mockbuild/config.py +++ b/mock/py/mockbuild/config.py @@ -255,6 +255,8 @@ def setup_default_config_opts(): # dependent on guest OS config_opts['use_host_resolv'] = False config_opts['chroot_setup_cmd'] = ('groupinstall', 'buildsys-build') + config_opts['repo_arch'] = None + config_opts['repo_arch_map'] = {} config_opts['target_arch'] = 'i386' config_opts['releasever'] = None config_opts['rpmbuild_arch'] = None # <-- None means set automatically from target_arch @@ -432,6 +434,10 @@ def set_config_opts_per_cmdline(config_opts, options, args): if options.forcearch: config_opts['forcearch'] = options.forcearch + if not config_opts['repo_arch']: + target = config_opts['target_arch'] + config_opts['repo_arch'] = config_opts['repo_arch_map'].get(target, target) + if not options.clean: config_opts['clean'] = options.clean diff --git a/mock/py/mockbuild/package_manager.py b/mock/py/mockbuild/package_manager.py index 1f939907b..fdaf2d2ed 100644 --- a/mock/py/mockbuild/package_manager.py +++ b/mock/py/mockbuild/package_manager.py @@ -307,30 +307,24 @@ def _execute_mounted(self, *args, **kwargs): try: # either it does not support --installroot (microdnf) or # it is bootstrap image made by container with incomaptible dnf/rpm - if not self.support_installroot or self.is_bootstrap_image: - - personality = kwargs.pop("personality", None) - if self.is_bootstrap_image: - # Multilib fix, see on an example: The host-native - # 64-bit package manager installed in the bootstrap - # chroot (from image) needs to know how to resolve the - # $basearch variable. It would be confused our previous - # 'condPersonality("i386")' call (switched to 32-bit). - # Switch back to 64-bit mode (only the particular DNF - # sub-process). - personality = self.config['host_arch'] + personality = kwargs.pop("personality", None) + if self.buildroot.is_bootstrap and not self.buildroot.config["forcearch"]: + personality = self.buildroot.config["repo_arch"] + if not self.support_installroot or self.is_bootstrap_image: out = util.do(invocation, env=env, chrootPath=self.buildroot.make_chroot_path(), personality=personality, **kwargs) elif self.bootstrap_buildroot is None: + + out = util.do(invocation, env=env, - **kwargs) + personality=personality, **kwargs) else: out = util.do(invocation, env=env, chrootPath=self.bootstrap_buildroot.make_chroot_path(), nspawn_args=self.bootstrap_buildroot.config['nspawn_args'], - **kwargs) + personality=personality, **kwargs) error = None break except Error as e: diff --git a/mock/tests/test_config_templates.py b/mock/tests/test_config_templates.py index 57b284c8d..f9f525b19 100644 --- a/mock/tests/test_config_templates.py +++ b/mock/tests/test_config_templates.py @@ -12,6 +12,31 @@ def test_transitive_expand(): assert config['c'] == 'test test test test' +@pytest.mark.parametrize('setup', [ + # host target bootstrap expected_repo + ('x86_64', 'arm7hl', True, 'x86_64'), + ('x86_64', 'arm7hl', False, 'armhfp'), + ('ppc64le', 'arm7hl', True, 'ppc64le'), + ('ppc64le', 'arm7hl', False, 'armhfp'), +]) +def test_nested_access(setup): + """ + Check that we can access config_opts["foo"]["bar"] items in jinja. + """ + host, target, bootstrap, result = setup + config = TemplatedDictionary() + config["archmap"] = {"i386": "i686", "arm7hl": "armhfp", "x86_64": "x86_64"} + config["host_arch"] = host + config["target_arch"] = target + config["root"] = "foo-bootstrap" if bootstrap else "foo" + config["repo_arch"] = ( + "{% set desired = host_arch if root.endswith('bootstrap') else target_arch %}" + "{{ archmap[desired] if desired in archmap else desired }}" + ) + config['__jinja_expand'] = True + assert config['repo_arch'] == result + + def test_aliases(): config = TemplatedDictionary( alias_spec={ diff --git a/releng/release-notes-next/repo-arch.feature b/releng/release-notes-next/repo-arch.feature new file mode 100644 index 000000000..5c72b2224 --- /dev/null +++ b/releng/release-notes-next/repo-arch.feature @@ -0,0 +1,6 @@ +A new `{{ repo_arch }}` Jinja2 template (templated-dictionary) is provided +by Mock. This variable is usable for DNF config options denoting URLs like +`baseurl=`, `metalink=`, etc. Namely, it can be used instead of the DNF-native +`$basearch` variable which [doesn't work properly for all the +distributions][issue#1304]. The new `config_opts['repo_arch_map']` option has +been added too, if additional tweaks with `repo_arch` template need to be done.