From 77a0e36fbf430706951138fbc4d187417ece20d3 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 21 Sep 2023 18:29:23 +0200 Subject: [PATCH 1/4] Fix data file path detection with new pip Using an editable installation of Kayobe fails on Rocky Linux 9 or Ubuntu with an error such as: ERROR! The requirements file '/home/rocky/kayobe/kayobe/requirements.yml' does not exist. Failed to install Ansible roles from /home/rocky/kayobe/kayobe/utils.py/../requirements.yml via Ansible Galaxy: returncode 1 Control host bootstrap failed - likely Ansible Galaxy flakiness. Sleeping 5 seconds before retrying This is caused by recent changes to how pip manages editable installations. The egg-link file that Kayobe was using to find the source path does not exist anymore. Instead, there is a direct_url.json file under the kayobe dist-info directory that can be parsed. This also includes Ic53efd03cecbd53ad3e3b64b664e084f4e25be0e to work around mocking issues in unit tests. Change-Id: I9dd5b97dec93c0e5393a1e7d9640f85003651b56 Closes-Bug: #2020135 (cherry picked from commit 1847ad3f17a039f29e22f465508991fcb6e1cca7) --- kayobe/utils.py | 22 +++++++++++++++++++ ...stall-data-file-path-743b7a85a5f5db6d.yaml | 5 +++++ 2 files changed, 27 insertions(+) create mode 100644 releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml diff --git a/kayobe/utils.py b/kayobe/utils.py index 727fd783a..ea41750d5 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -14,6 +14,8 @@ import base64 import glob +import importlib_metadata +import json import logging import os import shutil @@ -48,10 +50,30 @@ def _detect_install_prefix(path): return prefix_path +def _get_direct_url(dist): + direct_url = os.path.join(dist._path, 'direct_url.json') + if os.path.isfile(direct_url): + with open(direct_url, 'r') as f: + direct_url_content = json.loads(f.readline().strip()) + url = direct_url_content['url'] + prefix = 'file://' + if url.startswith(prefix): + return url[len(prefix):] + + return None + + def _get_base_path(): override = os.environ.get("KAYOBE_DATA_FILES_PATH") if override: return os.path.join(override) + + kayobe_dist = list(importlib_metadata.Distribution.discover(name="kayobe")) + if kayobe_dist: + direct_url = _get_direct_url(kayobe_dist[0]) + if direct_url: + return direct_url + egg_glob = os.path.join( sys.prefix, 'lib*', 'python*', '*-packages', 'kayobe.egg-link' ) diff --git a/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml b/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml new file mode 100644 index 000000000..dac75efa8 --- /dev/null +++ b/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes detection of data file path when using editable installations with a + recent pip. From d773ed5682bc43531810b59d809231750e9c8205 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 4 Oct 2023 11:13:23 +0200 Subject: [PATCH 2/4] Use importlib.metadata instead of importlib_metadata The importlib.metadata library is part of the Python standard library since Python 3.8. Change-Id: Iecf1d469caaa777c5b253eb0e44dc00692197ef7 (cherry picked from commit 8b465eae49b42e9d8c497ee8afb966cf5b983f7e) --- kayobe/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kayobe/utils.py b/kayobe/utils.py index ea41750d5..b6bccf1e6 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -14,7 +14,7 @@ import base64 import glob -import importlib_metadata +from importlib.metadata import Distribution import json import logging import os @@ -68,7 +68,7 @@ def _get_base_path(): if override: return os.path.join(override) - kayobe_dist = list(importlib_metadata.Distribution.discover(name="kayobe")) + kayobe_dist = list(Distribution.discover(name="kayobe")) if kayobe_dist: direct_url = _get_direct_url(kayobe_dist[0]) if direct_url: From a1fc1b432f7177d2ef17e32cad62bc3c03e488cb Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 5 Oct 2023 09:33:14 +0200 Subject: [PATCH 3/4] CI: Enable Ubuntu jobs again These jobs had been disabled because of RabbitMQ packaging issues [1]. [1] https://review.opendev.org/c/openstack/kolla/+/885857 Change-Id: I6a2a37a4f3c9ddd8d6755ba5c500fad93ab72116 --- zuul.d/project.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index e620cfeb2..f960b39a3 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -13,16 +13,12 @@ - kayobe-tox-molecule - kayobe-overcloud-rocky9 - kayobe-overcloud-centos9s - # NOTE(mattcrees): Job temporarily disabled until fix is merged, see: - # https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-ubuntu-jammy + - kayobe-overcloud-ubuntu-jammy - kayobe-overcloud-tls-rocky9 - kayobe-overcloud-host-configure-rocky9 - kayobe-overcloud-host-configure-centos9s - kayobe-overcloud-host-configure-ubuntu-jammy - # NOTE(mattcrees): Job temporarily disabled until fix is merged, see: - # https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-upgrade-ubuntu-focal + - kayobe-overcloud-upgrade-ubuntu-focal - kayobe-seed-rocky9 - kayobe-seed-ubuntu-jammy - kayobe-seed-images-rocky9 @@ -38,15 +34,11 @@ - kayobe-tox-ansible - kayobe-tox-molecule - kayobe-overcloud-rocky9 - # NOTE(mattcrees): Job temporarily disabled until fix is merged, see: - # https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-ubuntu-jammy + - kayobe-overcloud-ubuntu-jammy - kayobe-overcloud-tls-rocky9 - kayobe-overcloud-host-configure-rocky9 - kayobe-overcloud-host-configure-ubuntu-jammy - # NOTE(mattcrees): Job temporarily disabled until fix is merged, see: - # https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-upgrade-ubuntu-focal + - kayobe-overcloud-upgrade-ubuntu-focal - kayobe-seed-rocky9 - kayobe-seed-ubuntu-jammy - kayobe-seed-upgrade-ubuntu-focal From a1a1f8d3700fa7d274edb47a7e9467c17b923c88 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 10 Oct 2023 08:03:47 +0200 Subject: [PATCH 4/4] bifrost: Populate bifrost host vars on deprovision Closes-Bug: #2038889 Change-Id: I4609494d009c6e1f97d833a9a11529d107b3216f (cherry picked from commit 672d6fa24a978009588e66dd19c623ee244a8f8e) --- kayobe/cli/commands.py | 3 ++- kayobe/tests/unit/cli/test_commands.py | 2 ++ .../bifrost-host-vars-deprovision-525c450cf20f7f71.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index c44b642c0..b7c2a9ab6 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -1106,7 +1106,8 @@ class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command): def take_action(self, parsed_args): self.app.LOG.debug("Deprovisioning overcloud") - playbooks = _build_playbook_list("overcloud-deprovision") + playbooks = _build_playbook_list("kolla-bifrost-hostvars", + "overcloud-deprovision") self.run_kayobe_playbooks(parsed_args, playbooks) diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 9d504854e..418486fab 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1247,6 +1247,8 @@ def test_overcloud_deprovision(self, mock_run): mock.call( mock.ANY, [ + utils.get_data_files_path( + "ansible", "kolla-bifrost-hostvars.yml"), utils.get_data_files_path( "ansible", "overcloud-deprovision.yml"), ], diff --git a/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml new file mode 100644 index 000000000..4b866b30f --- /dev/null +++ b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes failure to run ``kayobe overcloud deprovision`` after Bifrost is + redeployed. + `LP#2038889 `__ +