From b0f8d4fb7753a58be44402fb43866ea7b0505092 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 5 Oct 2023 12:39:42 +0200 Subject: [PATCH 01/47] Remove upgrade jobs following Xena EOL The stable/xena branch was deleted with the end of life of the Xena release [1], which will cause Yoga upgrade jobs to fail. Also re-enable Ubuntu jobs which were disabled due to RabbitMQ packaging issues. [1] https://review.opendev.org/c/openstack/releases/+/894630 Change-Id: I5e662323febe4a1363212d19f57afde4aa35ac6a --- zuul.d/project.yaml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 006772dcd..fdf0035de 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -15,8 +15,8 @@ - kayobe-overcloud-centos9s - kayobe-overcloud-rocky8 - kayobe-overcloud-rocky9 - # - kayobe-overcloud-ubuntu-focal Note(mattcrees): Job temporarily disabled until fix is merged, see: https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-ubuntu-jammy + - kayobe-overcloud-ubuntu-focal + - kayobe-overcloud-ubuntu-jammy - kayobe-overcloud-host-configure-centos8s - kayobe-overcloud-host-configure-centos9s - kayobe-overcloud-host-configure-rocky8 @@ -25,15 +25,11 @@ - kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - - kayobe-overcloud-upgrade-centos8s - # - kayobe-overcloud-upgrade-ubuntu-focal Note(mattcrees): Job temporarily disabled until fix is merged, see: https://review.opendev.org/c/openstack/kolla/+/885857 - kayobe-seed-centos8s - kayobe-seed-rocky8 - kayobe-seed-rocky9 - kayobe-seed-ubuntu-focal - kayobe-seed-ubuntu-jammy - - kayobe-seed-upgrade-centos8s - - kayobe-seed-upgrade-ubuntu-focal - kayobe-seed-vm-centos8s - kayobe-seed-vm-rocky8 - kayobe-seed-vm-rocky9 @@ -54,8 +50,8 @@ - kayobe-overcloud-centos8s - kayobe-overcloud-rocky8 - kayobe-overcloud-rocky9 - # - kayobe-overcloud-ubuntu-focal Note(mattcrees): Job temporarily disabled until fix is merged, see: https://review.opendev.org/c/openstack/kolla/+/885857 - # - kayobe-overcloud-ubuntu-jammy + - kayobe-overcloud-ubuntu-focal + - kayobe-overcloud-ubuntu-jammy - kayobe-overcloud-host-configure-centos8s - kayobe-overcloud-host-configure-rocky8 - kayobe-overcloud-host-configure-rocky9 @@ -63,15 +59,11 @@ - kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - - kayobe-overcloud-upgrade-centos8s - # - kayobe-overcloud-upgrade-ubuntu-focal Note(mattcrees): Job temporarily disabled until fix is merged, see: https://review.opendev.org/c/openstack/kolla/+/885857 - kayobe-seed-centos8s - kayobe-seed-rocky8 - kayobe-seed-rocky9 - kayobe-seed-ubuntu-focal - kayobe-seed-ubuntu-jammy - - kayobe-seed-upgrade-centos8s - - kayobe-seed-upgrade-ubuntu-focal - kayobe-seed-vm-centos8s - kayobe-seed-vm-rocky8 - kayobe-seed-vm-rocky9 From 390fbf1d416f64d9e75856ef17632c9ccd71483d Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 4 Oct 2023 22:23:47 +0200 Subject: [PATCH 02/47] Fix data file path detection with new pip Using an editable installation of Kayobe fails on Rocky Linux 9 or Ubuntu with an error such as: ERROR! The requirements file '/home/rocky/kayobe/kayobe/requirements.yml' does not exist. Failed to install Ansible roles from /home/rocky/kayobe/kayobe/utils.py/../requirements.yml via Ansible Galaxy: returncode 1 Control host bootstrap failed - likely Ansible Galaxy flakiness. Sleeping 5 seconds before retrying This is caused by recent changes to how pip manages editable installations. The egg-link file that Kayobe was using to find the source path does not exist anymore. Instead, there is a direct_url.json file under the kayobe dist-info directory that can be parsed. This also includes Ic53efd03cecbd53ad3e3b64b664e084f4e25be0e to work around mocking issues in unit tests and an amended version of Iecf1d469caaa777c5b253eb0e44dc00692197ef7 to import importlib.metadata or importlib_metadata depending on the Python version. Change-Id: I9dd5b97dec93c0e5393a1e7d9640f85003651b56 Closes-Bug: #2020135 (cherry picked from commit 1847ad3f17a039f29e22f465508991fcb6e1cca7) --- kayobe/utils.py | 25 +++++++++++++++++++ ...stall-data-file-path-743b7a85a5f5db6d.yaml | 5 ++++ requirements.txt | 1 + 3 files changed, 31 insertions(+) create mode 100644 releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml diff --git a/kayobe/utils.py b/kayobe/utils.py index 727fd783a..5864cfff4 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -14,6 +14,11 @@ import base64 import glob +try: + from importlib.metadata import Distribution +except ImportError: # for Python<3.8 + from importlib_metadata import Distribution +import json import logging import os import shutil @@ -48,10 +53,30 @@ def _detect_install_prefix(path): return prefix_path +def _get_direct_url(dist): + direct_url = os.path.join(dist._path, 'direct_url.json') + if os.path.isfile(direct_url): + with open(direct_url, 'r') as f: + direct_url_content = json.loads(f.readline().strip()) + url = direct_url_content['url'] + prefix = 'file://' + if url.startswith(prefix): + return url[len(prefix):] + + return None + + def _get_base_path(): override = os.environ.get("KAYOBE_DATA_FILES_PATH") if override: return os.path.join(override) + + kayobe_dist = list(Distribution.discover(name="kayobe")) + if kayobe_dist: + direct_url = _get_direct_url(kayobe_dist[0]) + if direct_url: + return direct_url + egg_glob = os.path.join( sys.prefix, 'lib*', 'python*', '*-packages', 'kayobe.egg-link' ) diff --git a/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml b/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml new file mode 100644 index 000000000..dac75efa8 --- /dev/null +++ b/releasenotes/notes/editable-install-data-file-path-743b7a85a5f5db6d.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes detection of data file path when using editable installations with a + recent pip. diff --git a/requirements.txt b/requirements.txt index bab4725ed..c5be19e9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pbr>=2.0 # Apache-2.0 Jinja2>3 # BSD ansible>=4,<6.0 # GPLv3 cliff>=3.1.0 # Apache +importlib-metadata;python_version<'3.8' # Apache-2.0 netaddr!=0.7.16,>=0.7.13 # BSD PyYAML>=3.10.0 # MIT selinux # MIT From d2f84443236a2a4745d89a0f4c7956ff8532f193 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 10 Oct 2023 17:25:26 +0200 Subject: [PATCH 03/47] Fix condition for enabling disable-selinux element The overcloud_dib_os_element variable is set to rocky-container when we use rocky as os_distribution, which resulted in skipping the disable-selinux element. Change-Id: Id4486cd5702f95e2b45c291e2cbd930ddb73db62 --- ansible/group_vars/all/overcloud-dib | 2 +- ...disable-selinux-overcloud-dib-rocky-7c381912c3dfbc09.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/disable-selinux-overcloud-dib-rocky-7c381912c3dfbc09.yaml diff --git a/ansible/group_vars/all/overcloud-dib b/ansible/group_vars/all/overcloud-dib index bf5edc44d..9e73ddfe4 100644 --- a/ansible/group_vars/all/overcloud-dib +++ b/ansible/group_vars/all/overcloud-dib @@ -39,7 +39,7 @@ overcloud_dib_os_release: "{{ os_release }}" overcloud_dib_elements_default: - "{{ overcloud_dib_os_element }}" - "cloud-init-datasources" - - "{% if overcloud_dib_os_element in ['centos', 'rocky'] %}disable-selinux{% endif %}" + - "{% if os_distribution in ['centos', 'rocky'] %}disable-selinux{% endif %}" - "enable-serial-console" - "vm" diff --git a/releasenotes/notes/disable-selinux-overcloud-dib-rocky-7c381912c3dfbc09.yaml b/releasenotes/notes/disable-selinux-overcloud-dib-rocky-7c381912c3dfbc09.yaml new file mode 100644 index 000000000..24b3bf78a --- /dev/null +++ b/releasenotes/notes/disable-selinux-overcloud-dib-rocky-7c381912c3dfbc09.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Adds missing ``disable-selinux`` element when building Rocky Linux + overcloud host disk images. From 8e93ecc986b65e73e728f4050f0635023b79c093 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 10 Oct 2023 13:09:12 +0200 Subject: [PATCH 04/47] Add introspection rule to update deploy kernel location Back in the Ussuri release, we changed the Bifrost kernel location to use ipa.kernel instead of ipa.vmlinuz. While this works fine for newly discovered nodes, any node added to Bifrost in Train or earlier will have kept ipa.vmlinuz as its deploy kernel. This can cause issues since upgrading Bifrost or building new deployment images will update ipa.kernel, but not ipa.vmlinuz, resulting in nodes booting with an old kernel and a new ramdisk. Fix by adding a new rule updating the legacy kernel location and documenting how to update node information. Change-Id: I6ae44dcae78424b4638762d015cf24336444707f (cherry picked from commit 44af704836799e9b211e3eccfa5efa72f9713f32) --- ansible/group_vars/all/bifrost | 5 ++++- ansible/group_vars/all/inspector | 12 +++++++++++ ansible/seed-introspection-rules.yml | 1 + etc/kayobe/bifrost.yml | 3 +++ ...update-deploy-kernel-6943f3ad3cb82c51.yaml | 20 +++++++++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/update-deploy-kernel-6943f3ad3cb82c51.yaml diff --git a/ansible/group_vars/all/bifrost b/ansible/group_vars/all/bifrost index d780cfb10..788dddd12 100644 --- a/ansible/group_vars/all/bifrost +++ b/ansible/group_vars/all/bifrost @@ -99,7 +99,7 @@ kolla_bifrost_inspector_port_addition: "{{ inspector_add_ports }}" kolla_bifrost_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" # List of introspection rules for Bifrost's Ironic Inspector service. -kolla_bifrost_inspector_rules: "{{ inspector_rules }}" +kolla_bifrost_inspector_rules: "{{ inspector_rules + [inspector_rule_legacy_deploy_kernel] }}" # Ironic inspector IPMI username to set. kolla_bifrost_inspector_ipmi_username: "{{ ipmi_username }}" @@ -117,6 +117,9 @@ kolla_bifrost_inspector_deploy_kernel: "http://{{ provision_oc_net_name | net_ip # Ironic inspector deployment ramdisk location. kolla_bifrost_inspector_deploy_ramdisk: "http://{{ provision_oc_net_name | net_ip }}:8080/ipa.initramfs" +# Ironic inspector legacy deployment kernel location. +kolla_bifrost_inspector_legacy_deploy_kernel: "http://{{ provision_oc_net_name | net_ip }}:8080/ipa.vmlinuz" + # Timeout of hardware inspection on overcloud nodes, in seconds. Default is # {{ inspector_inspection_timeout }}. kolla_bifrost_inspection_timeout: "{{ inspector_inspection_timeout }}" diff --git a/ansible/group_vars/all/inspector b/ansible/group_vars/all/inspector index a84a98a49..61bc01236 100644 --- a/ansible/group_vars/all/inspector +++ b/ansible/group_vars/all/inspector @@ -119,6 +119,18 @@ inspector_rule_deploy_kernel: path: "driver_info/deploy_kernel" value: "{{ inspector_rule_var_deploy_kernel }}" +# Ironic inspector rule to update deployment kernel from legacy location. +inspector_rule_legacy_deploy_kernel: + description: "Update deploy kernel from legacy" + conditions: + - field: "node://driver_info.deploy_kernel" + op: "eq" + value: "{{ inspector_rule_var_legacy_deploy_kernel }}" + actions: + - action: "set-attribute" + path: "driver_info/deploy_kernel" + value: "{{ inspector_rule_var_deploy_kernel }}" + # Deployment ramdisk referenced by inspector rule. inspector_rule_var_deploy_ramdisk: diff --git a/ansible/seed-introspection-rules.yml b/ansible/seed-introspection-rules.yml index 6c826150b..2b5b0bfab 100644 --- a/ansible/seed-introspection-rules.yml +++ b/ansible/seed-introspection-rules.yml @@ -20,3 +20,4 @@ inspector_rule_var_lldp_switch_port_interface: "{{ kolla_bifrost_inspector_lldp_switch_port_interface }}" inspector_rule_var_deploy_kernel: "{{ kolla_bifrost_inspector_deploy_kernel }}" inspector_rule_var_deploy_ramdisk: "{{ kolla_bifrost_inspector_deploy_ramdisk }}" + inspector_rule_var_legacy_deploy_kernel: "{{ kolla_bifrost_inspector_legacy_deploy_kernel }}" diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index a9eba19dd..d15d18613 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -116,6 +116,9 @@ # Ironic inspector deployment ramdisk location. #kolla_bifrost_inspector_deploy_ramdisk: +# Ironic inspector legacy deployment kernel location. +#kolla_bifrost_inspector_legacy_deploy_kernel: + # Timeout of hardware inspection on overcloud nodes, in seconds. Default is # {{ inspector_inspection_timeout }}. #kolla_bifrost_inspection_timeout: diff --git a/releasenotes/notes/update-deploy-kernel-6943f3ad3cb82c51.yaml b/releasenotes/notes/update-deploy-kernel-6943f3ad3cb82c51.yaml new file mode 100644 index 000000000..c098aec70 --- /dev/null +++ b/releasenotes/notes/update-deploy-kernel-6943f3ad3cb82c51.yaml @@ -0,0 +1,20 @@ +--- +upgrade: + - | + Adds an introspection rule to update the location of the deployment kernel + registered in existing Ironic nodes. Nodes discovered on a deployment + running the Train release or earlier may still be using the ``ipa.vmlinuz`` + kernel, which stays unchanged when deployment images get updated. If only + default introspection rules are in use, existing nodes may be updated from + the Bifrost container with the following command: + + ``OS_CLOUD=bifrost baremetal introspection reprocess $NODE_UUID_OR_NAME`` + + If non-default rules are used, reprocessing may revert any customisation + done by the operator. In this case, a more cautious approach is to update + the deployment kernel location manually: + + ``OS_CLOUD=bifrost baremetal node set --driver-info deploy_kernel= $NODE_UUID_OR_NAME`` + + If the ``kolla_bifrost_inspector_rules`` list is customised, the rule + ``inspector_rule_legacy_deploy_kernel`` should be added to it. From f609ae27a073aac351e703751b7731df9230cc20 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Tue, 10 Oct 2023 08:03:47 +0200 Subject: [PATCH 05/47] bifrost: Populate bifrost host vars on deprovision Closes-Bug: #2038889 Change-Id: I4609494d009c6e1f97d833a9a11529d107b3216f (cherry picked from commit 672d6fa24a978009588e66dd19c623ee244a8f8e) --- kayobe/cli/commands.py | 3 ++- kayobe/tests/unit/cli/test_commands.py | 2 ++ .../bifrost-host-vars-deprovision-525c450cf20f7f71.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 110bc9b3d..5b98adc88 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -1080,7 +1080,8 @@ class OvercloudDeprovision(KayobeAnsibleMixin, VaultMixin, Command): def take_action(self, parsed_args): self.app.LOG.debug("Deprovisioning overcloud") - playbooks = _build_playbook_list("overcloud-deprovision") + playbooks = _build_playbook_list("kolla-bifrost-hostvars", + "overcloud-deprovision") self.run_kayobe_playbooks(parsed_args, playbooks) diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 7e4f440dd..a2442e9f8 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1241,6 +1241,8 @@ def test_overcloud_deprovision(self, mock_run): mock.call( mock.ANY, [ + utils.get_data_files_path( + "ansible", "kolla-bifrost-hostvars.yml"), utils.get_data_files_path( "ansible", "overcloud-deprovision.yml"), ], diff --git a/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml new file mode 100644 index 000000000..4b866b30f --- /dev/null +++ b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes failure to run ``kayobe overcloud deprovision`` after Bifrost is + redeployed. + `LP#2038889 `__ + From cf78d2e7664143e1318dceafaa6a3e80da456fab Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 25 Sep 2023 18:43:07 +0200 Subject: [PATCH 06/47] Fix last return value in _get_base_path The return value was set to the following path which is not valid: kayobe/kayobe/utils.py/.. The intent must have been to use dirname to produce: kayobe/kayobe/.. Change-Id: I9d7d71d42026f3c12da6de9c5ca55dc647554fd0 (cherry picked from commit 810e021d8bdca17b1e217569f9bf24f917f6810d) --- kayobe/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kayobe/utils.py b/kayobe/utils.py index 5864cfff4..6ccc4b6e9 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -91,7 +91,7 @@ def _get_base_path(): return os.path.join(prefix, "share", "kayobe") # Assume uninstalled - return os.path.join(os.path.realpath(__file__), "..") + return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") def galaxy_role_install(role_file, roles_path, force=False): From c6d0829576322b74ec812922bf4e3786c26369e3 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 27 Sep 2023 14:19:21 +0100 Subject: [PATCH 07/47] Add option to skip kolla docker registry login This change adds a variable that can be used to disable kolla docker registry login attempts, even when the registry username and password are set. This is required for deployments using a non-standard containerised registry deployed on the seed during the deploy-container step, since it takes place after the registry login attempt. Change-Id: Ie17ef9ce1147cbaec2e42db932c7d59293b49b1b (cherry picked from commit 57d7764114052b6f69c4eb58a1b6b206067cce74) --- .../roles/deploy-containers/defaults/main.yml | 2 ++ .../roles/deploy-containers/tasks/main.yml | 3 +-- .../reference/seed-custom-containers.rst | 23 +++++++++++++++++++ ...ocker-registry-login-f5b0ba858a35ea39.yaml | 9 ++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/skip-kolla-docker-registry-login-f5b0ba858a35ea39.yaml diff --git a/ansible/roles/deploy-containers/defaults/main.yml b/ansible/roles/deploy-containers/defaults/main.yml index 84c77a7e4..9130164e9 100644 --- a/ansible/roles/deploy-containers/defaults/main.yml +++ b/ansible/roles/deploy-containers/defaults/main.yml @@ -11,3 +11,5 @@ deploy_containers_defaults: restart_policy: "unless-stopped" deploy_containers_docker_api_timeout: 120 + +deploy_containers_registry_attempt_login: "{{ kolla_docker_registry_username is truthy and kolla_docker_registry_password is truthy }}" diff --git a/ansible/roles/deploy-containers/tasks/main.yml b/ansible/roles/deploy-containers/tasks/main.yml index c1ec533a3..a16e6d8d7 100644 --- a/ansible/roles/deploy-containers/tasks/main.yml +++ b/ansible/roles/deploy-containers/tasks/main.yml @@ -6,8 +6,7 @@ password: "{{ kolla_docker_registry_password }}" reauthorize: yes when: - - kolla_docker_registry_username is truthy - - kolla_docker_registry_password is truthy + - deploy_containers_registry_attempt_login | bool - name: Deploy containers (loop) include_tasks: deploy.yml diff --git a/doc/source/configuration/reference/seed-custom-containers.rst b/doc/source/configuration/reference/seed-custom-containers.rst index 4d4ef8e11..8c9738ff0 100644 --- a/doc/source/configuration/reference/seed-custom-containers.rst +++ b/doc/source/configuration/reference/seed-custom-containers.rst @@ -60,3 +60,26 @@ List of Kayobe applied defaults to required docker_container variables: .. literalinclude:: ../../../../ansible/roles/deploy-containers/defaults/main.yml :language: yaml + +Docker registry +=============== + +Seed containers can be pulled from a docker registry deployed on the seed, +since the docker registry deployment step precedes the custom container +deployment step. + +It is also possible to deploy a custom containerised docker registry as a +custom seed container. In this case, basic authentication login attempts can be +disabled by setting + +.. code-block:: yaml + :caption: ``kolla.yml`` + + deploy_containers_registry_attempt_login: false + +Without this setting, the login will fail because the registry has not yet been +deployed. + +More information on deploying a docker registry can be found :ref:`here +`. + diff --git a/releasenotes/notes/skip-kolla-docker-registry-login-f5b0ba858a35ea39.yaml b/releasenotes/notes/skip-kolla-docker-registry-login-f5b0ba858a35ea39.yaml new file mode 100644 index 000000000..7d61e6f9d --- /dev/null +++ b/releasenotes/notes/skip-kolla-docker-registry-login-f5b0ba858a35ea39.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Attempts to log in to the kolla docker registry can be skipped by setting + ``deploy_containers_registry_attempt_login`` to false. + + This is required for deployments using a non-standard registry + deployed on the seed during the deploy-container step, since it takes + place after the registry login attempt. From b0832bf1ffe4a13a90b62a87a82d918c532f336a Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 20 Oct 2023 14:40:59 +0200 Subject: [PATCH 08/47] Bump MichaelRigart.interfaces to v1.14.4 This release includes the following fixes: * Remove nmconnection files generated by cloud-init * Remove ens3 also from NetworkManager config Closes-Bug: #2039975 Change-Id: Id3e7913f4ba9d05cec332dab560fc42d44dd4130 (cherry picked from commit d4502e6ea268bebce926dcf9320b740272e83dfa) --- ...network-manager-cloud-init-config-315c0eb846b72c93.yaml | 7 +++++++ requirements.yml | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/remove-network-manager-cloud-init-config-315c0eb846b72c93.yaml diff --git a/releasenotes/notes/remove-network-manager-cloud-init-config-315c0eb846b72c93.yaml b/releasenotes/notes/remove-network-manager-cloud-init-config-315c0eb846b72c93.yaml new file mode 100644 index 000000000..23667afbf --- /dev/null +++ b/releasenotes/notes/remove-network-manager-cloud-init-config-315c0eb846b72c93.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes conflicts between NetworkManager nmconnection files generated by + ``cloud-init`` and those generated by Kayobe by upgrading the + ``MichaelRigart.interfaces`` role to version 1.14.4. + `LP#2039975 `__ diff --git a/requirements.yml b/requirements.yml index 7271422a6..5ab307e37 100644 --- a/requirements.yml +++ b/requirements.yml @@ -13,8 +13,8 @@ roles: - src: jriguera.configdrive # There are no versioned releases of this role. version: 29871bf3279ef95fc8f7339b9abd13f869980750 - - name: MichaelRigart.interfaces - version: v1.14.1 + - src: MichaelRigart.interfaces + version: v1.14.4 - src: mrlesmithjr.chrony version: v0.1.4 - src: mrlesmithjr.manage_lvm From 3282cda6b152587edb55dd81649ddceb19e0c575 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 20 Oct 2023 14:51:37 +0200 Subject: [PATCH 09/47] Fix MTU of NetworkManager bridge VLAN interfaces We are seeing an issue on Rocky Linux 9 where VLAN interfaces on bridges have their MTU set to MTU-1 after rebooting. This is likely related to how MTU is set by NetworkManager for bridges: it first sets the MTU to MTU-1 before setting it to the correct value [1]. We can work around this issue by not explicitly setting the MTU configuration on the VLAN interface if it is the same as the parent bridge. It will be automatically inherited from the parent. [1] https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/commit/864872e9a8c694ee7472154bf45f869040310b71 Closes-Bug: #2039947 Change-Id: I23366f4de7842e7c2fe40e431fac76f26e9892de (cherry picked from commit ee63b3253dedaf62cc321f5b29080d7e7c459a99) --- ansible/roles/network-redhat/tasks/main.yml | 2 +- kayobe/plugins/filter/networks.py | 23 ++++++++++++++++++- ...-vlan-mtu-workaround-71d48d582b5e23d6.yaml | 6 +++++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/nm-bridge-vlan-mtu-workaround-71d48d582b5e23d6.yaml diff --git a/ansible/roles/network-redhat/tasks/main.yml b/ansible/roles/network-redhat/tasks/main.yml index 24d5b4e92..09f171bf8 100644 --- a/ansible/roles/network-redhat/tasks/main.yml +++ b/ansible/roles/network-redhat/tasks/main.yml @@ -25,7 +25,7 @@ interfaces_ether_interfaces: > {{ network_interfaces | net_select_ethers | - map('net_interface_obj') | + map('net_interface_obj', names=network_interfaces) | list }} interfaces_bridge_interfaces: > {{ network_interfaces | diff --git a/kayobe/plugins/filter/networks.py b/kayobe/plugins/filter/networks.py index 9af4b9dd9..58dbaf9e0 100644 --- a/kayobe/plugins/filter/networks.py +++ b/kayobe/plugins/filter/networks.py @@ -380,7 +380,7 @@ def _validate_rules(rules): @jinja2.pass_context -def net_interface_obj(context, name, inventory_hostname=None): +def net_interface_obj(context, name, inventory_hostname=None, names=None): """Return a dict representation of a network interface. The returned dict is compatible with the interfaces_ether_interfaces @@ -400,6 +400,27 @@ def net_interface_obj(context, name, inventory_hostname=None): netmask = None vlan = net_vlan(context, name, inventory_hostname) mtu = net_mtu(context, name, inventory_hostname) + + # NOTE(priteau): do not pass MTU for VLAN interfaces on bridges when it is + # identical to the parent bridge, to work around a NetworkManager bug. + if names is not None and net_is_vlan_interface(context, name, + inventory_hostname): + # Make a mapping of bridge interfaces and their MTUs + bridge_mtus = {} + for bridge in net_select_bridges(context, names, inventory_hostname): + bridge_interface = net_interface(context, bridge, + inventory_hostname) + bridge_mtus[bridge_interface] = net_mtu(context, bridge, + inventory_hostname) + + # Get parent and check for its MTU if it is a bridge + parent_or_device = get_vlan_parent( + context, name, device, vlan, inventory_hostname) + if parent_or_device in bridge_mtus: + parent_mtu = bridge_mtus[parent_or_device] + if mtu == parent_mtu: + mtu = None + routes = net_routes(context, name, inventory_hostname) if routes: routes = [_route_obj(route) for route in routes] diff --git a/releasenotes/notes/nm-bridge-vlan-mtu-workaround-71d48d582b5e23d6.yaml b/releasenotes/notes/nm-bridge-vlan-mtu-workaround-71d48d582b5e23d6.yaml new file mode 100644 index 000000000..e1568eb7a --- /dev/null +++ b/releasenotes/notes/nm-bridge-vlan-mtu-workaround-71d48d582b5e23d6.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Adds a workaround to avoid NetworkManager setting the MTU of bridge VLAN + interfaces to an incorrect value. + `LP#2039947 `__ From 8a1aa4dc9bcbb22a8189b2bda315c0d55074b76b Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 16 Oct 2023 17:31:05 +0200 Subject: [PATCH 10/47] Resolve slow fact gathering when seed SNAT is off Stop configuring the seed host as gateway when seed SNAT is disabled (which is the default since Xena), otherwise overcloud hosts may take a long time to gather facts until their network configuration is applied (probably because DNS servers are thought to be reachable until the gateway is changed or removed). Closes-Bug: #2039461 Change-Id: Ib847d3420dee374cec067cd8af519b510be04120 (cherry picked from commit 9980a8f891edac8f21d0cde9ee1e756a40c7be3b) --- ansible/kolla-bifrost-hostvars.yml | 8 ++++---- .../configdrive-gateway-62366f613e6eaac9.yaml | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/configdrive-gateway-62366f613e6eaac9.yaml diff --git a/ansible/kolla-bifrost-hostvars.yml b/ansible/kolla-bifrost-hostvars.yml index e43bcd7e6..c74a4f88b 100644 --- a/ansible/kolla-bifrost-hostvars.yml +++ b/ansible/kolla-bifrost-hostvars.yml @@ -19,10 +19,10 @@ ipv4_interface_mac: "{% raw %}{{ extra.pxe_interface_mac | default }}{% endraw %}" ipv4_address: "{{ admin_oc_net_name | net_ip }}" ipv4_subnet_mask: "{{ admin_oc_net_name | net_mask }}" - # If the admin network does not have a gateway defined, use the - # seed as a gateway to allow external access until other networks have - # been configured. - ipv4_gateway: "{{ admin_oc_net_name | net_gateway or admin_oc_net_name | net_ip(seed_host) }}" + # If the admin network does not have a gateway defined and seed SNAT is + # enabled, use the seed as a gateway to allow external access until other + # networks have been configured. Otherwise, do not set any gateway. + ipv4_gateway: "{{ admin_oc_net_name | net_gateway or admin_oc_net_name | net_ip(seed_host) if seed_enable_snat | bool }}" ipv4_nameserver: "{{ resolv_nameservers }}" network_mtu: "{{ admin_oc_net_name | net_mtu or '1500' }}" vlan_id: "{{ '' if admin_oc_net_name == provision_oc_net_name else (admin_oc_net_name | net_vlan) }}" diff --git a/releasenotes/notes/configdrive-gateway-62366f613e6eaac9.yaml b/releasenotes/notes/configdrive-gateway-62366f613e6eaac9.yaml new file mode 100644 index 000000000..f43684f03 --- /dev/null +++ b/releasenotes/notes/configdrive-gateway-62366f613e6eaac9.yaml @@ -0,0 +1,14 @@ +--- +upgrade: + - | + If the admin network does not have a gateway defined and + ``seed_enable_snat`` is ``false``, which is the default, overcloud hosts + will not have a default gateway immediately after provisioning anymore. A + default gateway on another network can still be applied during the host + configuration step. +fixes: + - | + Fixes slow fact gathering in some environments by not configuring the seed + host as the initial default gateway for overcloud hosts when + ``seed_enable_snat`` is ``false``, which is the default. + `LP#2039461 `__ From a25dd2245ca8c04851284350958cebd7fbb89f47 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 20 Oct 2023 14:45:46 +0200 Subject: [PATCH 11/47] Fix Launchpad bug URL Change-Id: I5151e7be611b95f52c05422c03dafe666dba144e (cherry picked from commit 6b1ea167f4175af21860edfca1fdf6b5c5ec60c8) --- .../notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml index 4b866b30f..6d229fb5f 100644 --- a/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml +++ b/releasenotes/notes/bifrost-host-vars-deprovision-525c450cf20f7f71.yaml @@ -3,5 +3,4 @@ fixes: - | Fixes failure to run ``kayobe overcloud deprovision`` after Bifrost is redeployed. - `LP#2038889 `__ - + `LP#2038889 `__ From 76e5438050042a12a0aa8d7bf15f8f128bc0dc3f Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 31 Oct 2023 13:55:11 +0000 Subject: [PATCH 12/47] Adds support for using Dell OS 10 switches with NGS Support for Dell OS 10 was added in Networking Generic Switch in the 2023.1 release[1]. [1] https://review.opendev.org/c/openstack/networking-generic-switch/+/860067 Change-Id: Id3cd5e081dd5c40897ddaada65cfb184d56345b6 (cherry picked from commit 47fb2ae32a35fe3f177fff5d9212a3945dc4334f) --- ansible/kolla-openstack.yml | 1 + releasenotes/notes/dellos10-support-ngs-24ba50803b4bf528.yaml | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 releasenotes/notes/dellos10-support-ngs-24ba50803b4bf528.yaml diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index 6941067ee..e7118145c 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -94,6 +94,7 @@ switch_type_to_device_type: arista: netmiko_arista_eos dellos9: netmiko_dell_force10 + dellos10: netmiko_dell_os10 dell-powerconnect: netmiko_dell_powerconnect junos: netmiko_juniper openvswitch: netmiko_ovs_linux diff --git a/releasenotes/notes/dellos10-support-ngs-24ba50803b4bf528.yaml b/releasenotes/notes/dellos10-support-ngs-24ba50803b4bf528.yaml new file mode 100644 index 000000000..a659fb322 --- /dev/null +++ b/releasenotes/notes/dellos10-support-ngs-24ba50803b4bf528.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds support for using DellOS 10 switches with Networking Generic Switch. From 0f8b8d0804a4de1a0dbff44fe0cfadf607ab4a51 Mon Sep 17 00:00:00 2001 From: Erik Berg Date: Tue, 10 Jan 2023 17:43:43 +0100 Subject: [PATCH 13/47] Allow overriding earlier use of NetworkManager veths have to be in nmconnection format, as nm-settings-ifcfg-rh has no support for reading veth network-scripts, as can be seen by the lack of a table at [1] "Table 37. veth setting". Hence why we're not using NetworkManager with old network-scripts. But if you set interfaces_use_networkmanager to true, you can already start using NM on rhel8 platforms if you so wish. [1] https://networkmanager.dev/docs/api/latest/nm-settings-ifcfg-rh.html Change-Id: Ic4dd537bee826ee89c009bf2c4c95781af19cbad --- ansible/roles/veth/defaults/main.yml | 2 ++ ansible/roles/veth/tasks/RedHat.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible/roles/veth/defaults/main.yml b/ansible/roles/veth/defaults/main.yml index e72b74341..2754b0d0a 100644 --- a/ansible/roles/veth/defaults/main.yml +++ b/ansible/roles/veth/defaults/main.yml @@ -1,3 +1,5 @@ --- +interfaces_use_networkmanager: "{{ ansible_facts.distribution_major_version | int >= 9 }}" + # List of virtual interfaces to configure. veth_interfaces: [] diff --git a/ansible/roles/veth/tasks/RedHat.yml b/ansible/roles/veth/tasks/RedHat.yml index 8315c8269..6b9c48760 100644 --- a/ansible/roles/veth/tasks/RedHat.yml +++ b/ansible/roles/veth/tasks/RedHat.yml @@ -1,6 +1,6 @@ --- - include_tasks: network-scripts.yml - when: ansible_facts.distribution_major_version | int < 9 + when: not interfaces_use_networkmanager - include_tasks: network-manager.yml - when: ansible_facts.distribution_major_version | int >= 9 + when: interfaces_use_networkmanager From 1862781174e041c9af855fd97fdcaac025bda3b1 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 16 Oct 2023 17:15:05 +0100 Subject: [PATCH 14/47] Add seed_deploy_containers_registry_attempt_login Follow on to Ie17ef9ce1147cbaec2e42db932c7d59293b49b1b Adds seed_deploy_containers_registry_attempt_login variable to seed.yml, which acts as a redirection of deploy_containers_registry_attempt_login so that the variable is more descriptive, declared in a better location, and extensible to other groups Change-Id: I86d8f13062ff8e664919cd5d63bc17bdafb32e0c (cherry picked from commit 1476b9a68bc7bda524ea6ec94d701b3aaeebf68f) --- ansible/group_vars/all/seed | 4 ++++ ansible/group_vars/seed/docker-registry | 7 +++++++ .../configuration/reference/seed-custom-containers.rst | 4 ++-- etc/kayobe/seed.yml | 4 ++++ 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 ansible/group_vars/seed/docker-registry diff --git a/ansible/group_vars/all/seed b/ansible/group_vars/all/seed index f1524a2e0..7ec3d4536 100644 --- a/ansible/group_vars/all/seed +++ b/ansible/group_vars/all/seed @@ -123,6 +123,10 @@ seed_users: "{{ users_default }}" # seed_containers: {} +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +seed_deploy_containers_registry_attempt_login: "{{ kolla_docker_registry_username is truthy and kolla_docker_registry_password is truthy }}" + ############################################################################### # Seed node firewalld configuration. diff --git a/ansible/group_vars/seed/docker-registry b/ansible/group_vars/seed/docker-registry new file mode 100644 index 000000000..f439501ec --- /dev/null +++ b/ansible/group_vars/seed/docker-registry @@ -0,0 +1,7 @@ +--- +############################################################################### +# Seed node docker regsitry configuration. + +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +deploy_containers_registry_attempt_login: "{{ seed_deploy_containers_registry_attempt_login }}" diff --git a/doc/source/configuration/reference/seed-custom-containers.rst b/doc/source/configuration/reference/seed-custom-containers.rst index 8c9738ff0..5b3e03cdd 100644 --- a/doc/source/configuration/reference/seed-custom-containers.rst +++ b/doc/source/configuration/reference/seed-custom-containers.rst @@ -73,9 +73,9 @@ custom seed container. In this case, basic authentication login attempts can be disabled by setting .. code-block:: yaml - :caption: ``kolla.yml`` + :caption: ``seed.yml`` - deploy_containers_registry_attempt_login: false + seed_deploy_containers_registry_attempt_login: false Without this setting, the login will fail because the registry has not yet been deployed. diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index f7cacab42..bc86fa627 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,6 +106,10 @@ # #seed_containers: +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +#seed_deploy_containers_registry_attempt_login: + ############################################################################### # Seed node firewalld configuration. From 9b8ed55abe936e99d6dec726b41266284a149943 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 20 Apr 2023 10:25:36 +0100 Subject: [PATCH 15/47] Add retries to get_url and uri tasks These tasks occasionally fail in CI, causing jobs to fail. Change-Id: I89041a641d8cb66c7848fa9ae9264a51aa1e38c8 (cherry picked from commit 2951f26ad1d6d79671f5f94016468af3c43e6143) --- ansible/roles/apt/tasks/keys.yml | 4 ++++ ansible/roles/dell-switch-bmp/tasks/main.yml | 4 ++++ ansible/roles/image-download/tasks/main.yml | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/ansible/roles/apt/tasks/keys.yml b/ansible/roles/apt/tasks/keys.yml index 4c1cda1e0..4b04e824b 100644 --- a/ansible/roles/apt/tasks/keys.yml +++ b/ansible/roles/apt/tasks/keys.yml @@ -17,3 +17,7 @@ mode: 0644 loop: "{{ apt_keys }}" become: true + register: result + until: result is successful + retries: 3 + delay: 5 diff --git a/ansible/roles/dell-switch-bmp/tasks/main.yml b/ansible/roles/dell-switch-bmp/tasks/main.yml index cc7e65894..c02a5a8bc 100644 --- a/ansible/roles/dell-switch-bmp/tasks/main.yml +++ b/ansible/roles/dell-switch-bmp/tasks/main.yml @@ -20,3 +20,7 @@ notify: - Copy Dell switch BMP images become: True + register: result + until: result is successful + retries: 3 + delay: 5 diff --git a/ansible/roles/image-download/tasks/main.yml b/ansible/roles/image-download/tasks/main.yml index 3f905b8bf..85a418ac8 100644 --- a/ansible/roles/image-download/tasks/main.yml +++ b/ansible/roles/image-download/tasks/main.yml @@ -12,6 +12,9 @@ url: "{{ image_download_checksum_url }}" return_content: true register: expected_checksum + until: expected_checksum is successful + retries: 3 + delay: 5 when: - image_download_checksum_url is not none - image_download_checksum_url != "" @@ -29,6 +32,10 @@ # Always download the image if we have no checksum to compare with. force: "{{ expected_checksum is skipped }}" backup: true + register: result + until: result is successful + retries: 3 + delay: 5 when: - image_download_url is not none - image_download_url != "" From f14f10d5c7e49a9a61787ea76c156e72416c985e Mon Sep 17 00:00:00 2001 From: Maksim Malchuk Date: Fri, 14 Oct 2022 17:56:20 +0300 Subject: [PATCH 16/47] Fix an issue when user forgot combine custom passwords Fixes an issue when user forgot to combine 'kolla_ansible_custom_passwords', 'kolla_ansible_default_custom_passwords' and own dictionary with custom passwords in configuration files. Now 'kolla_ansible_extra_custom_passwords' should provide only user custom passwords to add or override in the passwords.yml. Change-Id: I4813a1f6ab9cb566596e806bd0ada6dff342d119 Signed-off-by: Maksim Malchuk (cherry picked from commit 73abf0e57ee3aaeefa53eaf4fb7b1ed5f5c96933) --- ansible/group_vars/all/kolla | 8 ++++- .../configuration/reference/kolla-ansible.rst | 34 ++++++++++++------- etc/kayobe/kolla.yml | 4 +++ ...-passwords-overrides-065fd6bb8eb9689d.yaml | 14 ++++++++ 4 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index 2d2c5a3e5..78590e6c9 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -614,9 +614,15 @@ kolla_ansible_default_custom_passwords: >- if compute_libvirt_enabled | bool and compute_libvirt_enable_sasl | bool else {}) }} +# Dictionary containing extra custom passwords to add or override in the Kolla +# passwords file. +kolla_ansible_extra_custom_passwords: {} + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. -kolla_ansible_custom_passwords: "{{ kolla_ansible_default_custom_passwords }}" +kolla_ansible_custom_passwords: >- + {{ kolla_ansible_default_custom_passwords | + combine(kolla_ansible_extra_custom_passwords) }} ############################################################################### # OpenStack API addresses. diff --git a/doc/source/configuration/reference/kolla-ansible.rst b/doc/source/configuration/reference/kolla-ansible.rst index b6b5ab443..93e17d5fc 100644 --- a/doc/source/configuration/reference/kolla-ansible.rst +++ b/doc/source/configuration/reference/kolla-ansible.rst @@ -578,27 +578,35 @@ variable, if present. The file is generated to ``$KAYOBE_CONFIG_PATH/kolla/passwords.yml``, and should be stored along with other Kayobe configuration files. This file should not be manually modified. -``kolla_ansible_custom_passwords`` - Dictionary containing custom passwords to add or override in the Kolla - passwords file. Default is ``{{ kolla_ansible_default_custom_passwords - }}``, which contains SSH keys for use by Kolla Ansible and Bifrost. - Configuring Custom Passwords ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In order to write additional passwords to ``passwords.yml``, set the kayobe -variable ``kolla_ansible_custom_passwords`` in -``$KAYOBE_CONFIG_PATH/kolla.yml``. +The following variables are used to configure custom passwords: + +* ``kolla_ansible_default_custom_passwords``: Dictionary containing default + custom passwords, required by Kolla Ansible. Contains SSH keys authorized by + kolla user on Kolla hosts, SSH keys authorized in hosts deployed by Bifrost, + Docker Registry password and compute libVirt custom passwords. +* ``kolla_ansible_extra_custom_passwords``: Dictionary containing extra custom + passwords to add or override in the Kolla passwords file. Default is an empty + dictionary. +* ``kolla_ansible_custom_passwords``: Dictionary containing custom passwords to + add or override in the Kolla passwords file. Default is the combination of + the ``kolla_ansible_default_custom_passwords`` and + ``kolla_ansible_extra_custom_passwords``. + +In this example we add our own ``my_custom_password`` and override +``keystone_admin_password``: .. code-block:: yaml :caption: ``$KAYOBE_CONFIG_PATH/kolla.yml`` --- - # Dictionary containing custom passwords to add or override in the Kolla - # passwords file. - kolla_ansible_custom_passwords: > - {{ kolla_ansible_default_custom_passwords | - combine({'my_custom_password': 'correcthorsebatterystaple'}) }} + # Dictionary containing extra custom passwords to add or override in the + # Kolla passwords file. + kolla_ansible_extra_custom_passwords: + my_custom_password: 'correcthorsebatterystaple' + keystone_admin_password: 'superduperstrongpassword' Control Plane Services ====================== diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index a1f0051d0..575f9169a 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -434,6 +434,10 @@ # Kolla passwords file. #kolla_ansible_default_custom_passwords: +# Dictionary containing extra custom passwords to add or override in the Kolla +# passwords file. +#kolla_ansible_extra_custom_passwords: + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. #kolla_ansible_custom_passwords: diff --git a/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml b/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml new file mode 100644 index 000000000..adc5318e7 --- /dev/null +++ b/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml @@ -0,0 +1,14 @@ +--- +fixes: + - | + Fixes an issue when user forgot to combine + ``kolla_ansible_custom_passwords``, + ``kolla_ansible_default_custom_passwords`` and own dictionary with custom + passwords in configuration files. Now + ``kolla_ansible_extra_custom_passwords`` should provide only user custom + passwords to add or override in the passwords.yml. +upgrade: + - | + Now no need to combine ``kolla_ansible_default_custom_passwords`` and + ``kolla_ansible_custom_passwords`` in your custom configuration. Just use + ``kolla_ansible_extra_custom_passwords`` to add or override passwords. From cad48d1d01de7fb08307da52a4f12a037b6f648a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 20 Dec 2022 10:54:49 +0000 Subject: [PATCH 17/47] Fix setting kolla_admin_openrc_cacert Kolla Ansible renamed kolla_internal_fqdn_cacert to kolla_admin_openrc_cacert in Victoria, after which we no longer set the variable correctly in globals.yml. This would lead to a missing OS_CACERT in admin-openrc.sh and public-openrc.sh. This change fixes the issue by renaming the relevant Kayobe variables to match and passing through the correct variable. Backwards compatibility is provided until the end of the deprecation period. kolla_public_openrc_cacert -> kolla_external_fqdn_cacert kolla_admin_openrc_cacert -> kolla_internal_fqdn_cacert Story: 2010486 Task: 47054 Change-Id: I9e1cc20579cf80525d6ef732a1aac99a65bc171b Co-Authored-By: Maksim Malchuk (cherry picked from commit 95729405a38e6292a828c26347406e70132136b2) --- ansible/group_vars/all/kolla | 4 ++-- ansible/roles/kolla-ansible/defaults/main.yml | 4 ++-- .../kolla-ansible/templates/kolla/globals.yml | 3 +-- .../roles/kolla-ansible/tests/test-extras.yml | 2 ++ .../templates/public-openrc.sh.j2 | 4 ++-- .../configuration/reference/kolla-ansible.rst | 22 +++++++++++-------- etc/kayobe/kolla.yml | 4 ++-- ...eprecate-fqdn-cacert-301d5a26ed7107ab.yaml | 13 +++++++++++ 8 files changed, 37 insertions(+), 19 deletions(-) create mode 100644 releasenotes/notes/deprecate-fqdn-cacert-301d5a26ed7107ab.yaml diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index 2d2c5a3e5..32cc2bf43 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -653,7 +653,7 @@ kolla_external_tls_cert: # Path to a CA certificate file to use for the OS_CACERT environment variable # in public-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -kolla_external_fqdn_cacert: +kolla_public_openrc_cacert: "{{ kolla_external_fqdn_cacert | default }}" # Internal API certificate bundle. # @@ -666,7 +666,7 @@ kolla_internal_tls_cert: # Path to a CA certificate file to use for the OS_CACERT environment variable # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -kolla_internal_fqdn_cacert: +kolla_admin_openrc_cacert: "{{ kolla_internal_fqdn_cacert | default }}" ############################################################################### # Proxy configuration diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 450412b8f..fddca5eaf 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -165,8 +165,8 @@ kolla_enable_tls_external: kolla_enable_tls_internal: kolla_external_fqdn_cert: kolla_internal_fqdn_cert: -kolla_external_fqdn_cacert: -kolla_internal_fqdn_cacert: +kolla_public_openrc_cacert: +kolla_admin_openrc_cacert: ############################# # Ironic options diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml index 507854caa..58f8288ef 100644 --- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -207,8 +207,7 @@ kolla_external_fqdn_cert: "{{ kolla_external_fqdn_cert }}" {% if kolla_internal_tls_cert is not none and kolla_internal_tls_cert | length > 0 %} kolla_internal_fqdn_cert: "{{ kolla_internal_fqdn_cert }}" {% endif %} -kolla_external_fqdn_cacert: "{{ kolla_external_fqdn_cacert }}" -kolla_internal_fqdn_cacert: "{{ kolla_internal_fqdn_cacert }}" +kolla_admin_openrc_cacert: "{{ kolla_admin_openrc_cacert }}" ################ # Region options diff --git a/ansible/roles/kolla-ansible/tests/test-extras.yml b/ansible/roles/kolla-ansible/tests/test-extras.yml index 412f3da39..d49d54a3f 100644 --- a/ansible/roles/kolla-ansible/tests/test-extras.yml +++ b/ansible/roles/kolla-ansible/tests/test-extras.yml @@ -121,6 +121,7 @@ kolla_internal_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/internal.pem" kolla_internal_tls_cert: | bogus internal certificate + kolla_admin_openrc_cacert: "{{ temp_path }}/etc/kolla/certificates/ca/foo.crt" kolla_openstack_logging_debug: True grafana_local_admin_user_name: "grafana-admin" kolla_inspector_dhcp_pool_start: "1.2.3.4" @@ -246,6 +247,7 @@ kolla_external_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/external.pem" kolla_enable_tls_internal: True kolla_internal_fqdn_cert: "{{ temp_path }}/etc/kolla/certificates/internal.pem" + kolla_admin_openrc_cacert: "{{ temp_path }}/etc/kolla/certificates/ca/foo.crt" openstack_logging_debug: True grafana_admin_username: "grafana-admin" ironic_dnsmasq_dhcp_range: "1.2.3.4,1.2.3.5,255.255.255.0" diff --git a/ansible/roles/public-openrc/templates/public-openrc.sh.j2 b/ansible/roles/public-openrc/templates/public-openrc.sh.j2 index d0356e800..1c2dd179c 100644 --- a/ansible/roles/public-openrc/templates/public-openrc.sh.j2 +++ b/ansible/roles/public-openrc/templates/public-openrc.sh.j2 @@ -11,8 +11,8 @@ export OS_ENDPOINT_TYPE=publicURL export OS_MANILA_ENDPOINT_TYPE=publicURL {% elif "export OS_MISTRAL_ENDPOINT_TYPE" in line %} export OS_MISTRAL_ENDPOINT_TYPE=publicURL -{% elif "export OS_CACERT" in line and kolla_external_fqdn_cacert is not none %} -export OS_CACERT={{ kolla_external_fqdn_cacert }} +{% elif "export OS_CACERT" in line and kolla_public_openrc_cacert is not none %} +export OS_CACERT={{ kolla_public_openrc_cacert }} {% else %} {{ line }} {% endif %} diff --git a/doc/source/configuration/reference/kolla-ansible.rst b/doc/source/configuration/reference/kolla-ansible.rst index b6b5ab443..912021125 100644 --- a/doc/source/configuration/reference/kolla-ansible.rst +++ b/doc/source/configuration/reference/kolla-ansible.rst @@ -268,10 +268,6 @@ The following variables affect TLS encryption of the public API. A TLS certificate bundle to use for the public API endpoints, if ``kolla_enable_tls_external`` is ``true``. Note that this should be formatted as a literal style block scalar. -``kolla_external_fqdn_cacert`` - Path to a CA certificate file to use for the ``OS_CACERT`` environment - variable in openrc files when TLS is enabled, instead of Kolla Ansible's - default. The following variables affect TLS encryption of the internal API. Currently this requires all Kolla images to be built with the API's root CA trusted. @@ -282,10 +278,18 @@ this requires all Kolla images to be built with the API's root CA trusted. A TLS certificate bundle to use for the internal API endpoints, if ``kolla_enable_tls_internal`` is ``true``. Note that this should be formatted as a literal style block scalar. -``kolla_internal_fqdn_cacert`` + +The following variables affect the generated ``admin-openrc.sh`` and +``public-openrc.sh`` environment files. + +``kolla_public_openrc_cacert`` + Path to a CA certificate file to use for the ``OS_CACERT`` environment + variable in the ``public-openrc.sh`` file when TLS is enabled, instead of + ``kolla_admin_openrc_cacert``. +``kolla_admin_openrc_cacert`` Path to a CA certificate file to use for the ``OS_CACERT`` environment - variable in openrc files when TLS is enabled, instead of Kolla Ansible's - default. + variable in the ``admin-openrc.sh`` and ``public-openrc.sh`` files when TLS + is enabled, instead of Kolla Ansible's default. Example: enabling TLS for the public API ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -302,7 +306,7 @@ Here is an example: -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- - kolla_external_fqdn_cacert: /path/to/ca/certificate/bundle + kolla_admin_openrc_cacert: /path/to/ca/certificate/bundle Example: enabling TLS for the internal API ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -319,7 +323,7 @@ Here is an example: -----BEGIN CERTIFICATE----- ... -----END CERTIFICATE----- - kolla_internal_fqdn_cacert: /path/to/ca/certificate/bundle + kolla_admin_openrc_cacert: /path/to/ca/certificate/bundle Other certificates ------------------ diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index a1f0051d0..164262192 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -473,7 +473,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in public-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_external_fqdn_cacert: +#kolla_public_openrc_cacert: # Internal API certificate bundle. # @@ -486,7 +486,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_internal_fqdn_cacert: +#kolla_admin_openrc_cacert: ############################################################################### # Proxy configuration diff --git a/releasenotes/notes/deprecate-fqdn-cacert-301d5a26ed7107ab.yaml b/releasenotes/notes/deprecate-fqdn-cacert-301d5a26ed7107ab.yaml new file mode 100644 index 000000000..d892cac43 --- /dev/null +++ b/releasenotes/notes/deprecate-fqdn-cacert-301d5a26ed7107ab.yaml @@ -0,0 +1,13 @@ +--- +deprecates: + - | + Renames ``kolla_external_fqdn_cacert`` to ``kolla_public_openrc_cacert`` + and ``kolla_internal_fqdn_cacert`` to ``kolla_admin_openrc_cacert``. This + matches the Kolla Ansible variable name and better reflects their purpose. + The old variable names are still supported until the end of the deprecation + period (2024.2 "D" series release or later). +fixes: + - | + Fixes an issue where the Kolla Ansible variable + ``kolla_admin_openrc_cacert`` was not set to the value of + ``kolla_internal_fqdn_cacert``. From 89994fe7fa1d06401e4aef846f667dbb0000fc5c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 May 2023 14:44:55 +0100 Subject: [PATCH 18/47] dev: Improve error checking for config check functions Various functions in the development/testing scripts rely on 'kayobe configuration dump' to extract the value of flags. If this command fails for any reason, we should exit the script. Currently, some places we continue and return 1, since we check the output against the string 'true'. The to_bool helper function handles failure by checking for a valid boolean output, so let's use that everywhere. Change-Id: I3a5a43fef9c3d68d0db02be12b9f892c437e513d (cherry picked from commit 89fc4fa2792ef256ff7624559a8ea8e2ee2f13fc) --- dev/functions | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/functions b/dev/functions index b790e727f..78379c101 100644 --- a/dev/functions +++ b/dev/functions @@ -233,17 +233,22 @@ function upgrade_kayobe_venv { function is_deploy_image_built_locally { ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images) - [[ $ipa_build_images =~ ^true$ ]] + to_bool "$ipa_build_images" } function is_ironic_enabled { ironic_enabled=$(kayobe configuration dump --host controllers[0] --var-name kolla_enable_ironic) - [[ $ironic_enabled =~ ^true$ ]] + to_bool "$ironic_enabled" } function is_overcloud_host_image_built_by_dib { overcloud_dib_build_host_images=$(kayobe configuration dump --host controllers[0] --var-name overcloud_dib_build_host_images) - [[ $overcloud_dib_build_host_images =~ ^true$ ]] + to_bool "$overcloud_dib_build_host_images" +} + +function is_cinder_enabled { + flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)" + to_bool "$flag" } function environment_setup { @@ -856,11 +861,6 @@ function to_bool { fi } -function is_cinder_enabled { - flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)" - to_bool "$flag" -} - function configure_iptables { # NOTE(wszumski): adapted from the ironic devstack plugin, see: # https://github.com/openstack/ironic/blob/36e87dc5b472d79470b783fbba9ce396e3cbb96e/devstack/lib/ironic#L2132 From f2f85fa7041aa50d5c5444b39c62675e7cf67424 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 12 Jan 2023 14:15:11 +0000 Subject: [PATCH 19/47] Generate local Kolla Ansible config in check mode If running a command in check mode such as kayobe overcloud service deploy --check Kayobe does not generate the local configuration for Kolla Ansible. This can lead to an inaccurate result when comparing with the remote configuration, if there are changes in kayobe-config. For example: * Run kayobe overcloud service deploy * Change a file in etc/kayobe/kolla * Run kayobe overcloud service deploy --check --diff We would expect that the changed file results in a diff against the remote config. However there is no diff displayed. This change fixes the issue by always generating the local Kolla Ansible config, even in check mode. Change-Id: Ic1dd075076ea186b0928bba1a235605c0cd2ec71 Story: 2010526 Task: 47132 (cherry picked from commit 2c0f705e7acab5691591c215e40dea707d12da15) --- kayobe/cli/commands.py | 10 ++--- kayobe/tests/unit/cli/test_commands.py | 37 +++++++++++++++++-- ...ck-mode-local-config-3f8a4ba231a32c1f.yaml | 7 ++++ 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/check-mode-local-config-3f8a4ba231a32c1f.yaml diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 5b98adc88..e1ddd3588 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -104,15 +104,15 @@ def generate_kolla_ansible_config(self, parsed_args, install=False, tags = None if install else "config" playbooks = _build_playbook_list("kolla-ansible") self.run_kayobe_playbooks(parsed_args, playbooks, tags=tags, - ignore_limit=True) + ignore_limit=True, check=False) if service_config: playbooks = _build_playbook_list("kolla-openstack") self.run_kayobe_playbooks(parsed_args, playbooks, - ignore_limit=True) + ignore_limit=True, check=False) if bifrost_config: playbooks = _build_playbook_list("kolla-bifrost") self.run_kayobe_playbooks(parsed_args, playbooks, - ignore_limit=True) + ignore_limit=True, check=False) class KollaAnsibleMixin(object): @@ -244,7 +244,7 @@ def take_action(self, parsed_args): ka_tags = "install" playbooks = _build_playbook_list("kolla-ansible") self.run_kayobe_playbooks(parsed_args, playbooks, tags=ka_tags, - ignore_limit=True) + ignore_limit=True, check=False) if passwords_exist: # If we are bootstrapping a control host for an existing @@ -279,7 +279,7 @@ def take_action(self, parsed_args): self.run_kayobe_playbooks(parsed_args, playbooks, ignore_limit=True) playbooks = _build_playbook_list("kolla-ansible") self.run_kayobe_playbooks(parsed_args, playbooks, tags="install", - ignore_limit=True) + ignore_limit=True, check=False) class ConfigurationDump(KayobeAnsibleMixin, VaultMixin, Command): diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index a2442e9f8..cd52ef4e9 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -60,7 +60,8 @@ def test_control_host_bootstrap(self, mock_run, mock_passwords, mock.ANY, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="install", - ignore_limit=True + ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -93,7 +94,8 @@ def test_control_host_bootstrap_with_passwords( mock.ANY, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags=None, - ignore_limit=True + ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -138,7 +140,8 @@ def test_control_host_upgrade(self, mock_run, mock_prune, mock.ANY, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="install", - ignore_limit=True + ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -509,6 +512,7 @@ def test_seed_host_configure(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -569,6 +573,7 @@ def test_seed_host_configure_wipe_disks(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -872,11 +877,13 @@ def test_seed_service_deploy(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, [utils.get_data_files_path("ansible", "kolla-bifrost.yml")], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -920,11 +927,13 @@ def test_seed_service_upgrade(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, [utils.get_data_files_path("ansible", "kolla-bifrost.yml")], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1177,6 +1186,7 @@ def test_overcloud_inventory_discover(self, mock_run_one, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1275,6 +1285,7 @@ def test_overcloud_facts_gather(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1318,6 +1329,7 @@ def test_overcloud_host_configure(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1374,6 +1386,7 @@ def test_overcloud_host_configure_wipe_disks(self, mock_kolla_run, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1540,6 +1553,7 @@ def test_overcloud_database_backup(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1571,6 +1585,7 @@ def test_overcloud_database_backup_incremental(self, mock_kolla_run, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1601,6 +1616,7 @@ def test_overcloud_database_recover(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1609,6 +1625,7 @@ def test_overcloud_database_recover(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1640,6 +1657,7 @@ def test_overcloud_database_recover_force_host(self, mock_kolla_run, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1648,6 +1666,7 @@ def test_overcloud_database_recover_force_host(self, mock_kolla_run, "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1701,6 +1720,7 @@ def test_overcloud_service_deploy(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags="config", + check=False, ), mock.call( mock.ANY, @@ -1709,6 +1729,7 @@ def test_overcloud_service_deploy(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1766,6 +1787,7 @@ def test_overcloud_service_deploy_containers(self, mock_kolla_run, [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags="config", + check=False, ), mock.call( mock.ANY, @@ -1774,6 +1796,7 @@ def test_overcloud_service_deploy_containers(self, mock_kolla_run, "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1819,6 +1842,7 @@ def test_overcloud_service_prechecks(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags="config", + check=False, ), mock.call( mock.ANY, @@ -1827,6 +1851,7 @@ def test_overcloud_service_prechecks(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -1857,6 +1882,7 @@ def test_overcloud_service_reconfigure(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags="config", + check=False, ), mock.call( mock.ANY, @@ -1865,6 +1891,7 @@ def test_overcloud_service_reconfigure(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1921,6 +1948,7 @@ def test_overcloud_service_stop(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags="config", + check=False, ), mock.call( mock.ANY, @@ -1929,6 +1957,7 @@ def test_overcloud_service_stop(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), mock.call( mock.ANY, @@ -1985,6 +2014,7 @@ def test_overcloud_service_upgrade(self, mock_kolla_run, mock_run): [utils.get_data_files_path("ansible", "kolla-ansible.yml")], ignore_limit=True, tags=None, + check=False, ), mock.call( mock.ANY, @@ -1993,6 +2023,7 @@ def test_overcloud_service_upgrade(self, mock_kolla_run, mock_run): "kolla-openstack.yml"), ], ignore_limit=True, + check=False, ), mock.call( mock.ANY, diff --git a/releasenotes/notes/check-mode-local-config-3f8a4ba231a32c1f.yaml b/releasenotes/notes/check-mode-local-config-3f8a4ba231a32c1f.yaml new file mode 100644 index 000000000..fba1aa222 --- /dev/null +++ b/releasenotes/notes/check-mode-local-config-3f8a4ba231a32c1f.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes an issue where local configuration generation would be skipped when + running in check mode. This would lead to Kolla Ansible checking with stale + configuration. See `story 2010526 + `__ for details. From 52ce15be60678c391b4c9a705ce68cc5d882e623 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 16 Nov 2023 20:47:23 +0100 Subject: [PATCH 20/47] Backport selinux change from zed The disable-selinux role has been renamed to selinux and now supports setting desired state. Previously Kayobe was defaulting to disabling and rebooted the host - to avoid audit logs filling up. This change allows operators to define desired SELinux state and defaults to permissive - to adhere to those site policies that require SELinux to be at least in permissive state. Note that unlike the original patch, this backport keeps the default selinux state as disabled. Co-authored-by: Mark Goddard Change-Id: I42933b0b7d55c69c9f6992e331fafb2e6c42d4d1 (cherry picked from commit caa7cc54ee5919b1b74dcdff21850051641eb9b5) --- ansible/disable-selinux.yml | 9 ---- ansible/infra-vm-host-configure.yml | 2 +- ansible/overcloud-host-configure.yml | 2 +- ansible/roles/disable-selinux/tasks/main.yml | 40 -------------- .../defaults/main.yml | 9 +++- ansible/roles/selinux/tasks/main.yml | 54 +++++++++++++++++++ ansible/seed-host-configure.yml | 2 +- ansible/selinux.yml | 9 ++++ doc/source/configuration/reference/hosts.rst | 11 ++-- .../scenarios/all-in-one/overcloud.rst | 12 ++--- kayobe/cli/commands.py | 6 +-- .../kayobe-infra-vm-base/overrides.yml.j2 | 4 -- .../kayobe-overcloud-base/overrides.yml.j2 | 8 --- .../overrides.yml.j2 | 2 + playbooks/kayobe-seed-base/overrides.yml.j2 | 8 --- .../kayobe-seed-upgrade-base/overrides.yml.j2 | 2 + .../kayobe-seed-vm-base/overrides.yml.j2 | 4 -- ...name-disable-selinux-9053ff36792066bc.yaml | 17 ++++++ 18 files changed, 110 insertions(+), 91 deletions(-) delete mode 100644 ansible/disable-selinux.yml delete mode 100644 ansible/roles/disable-selinux/tasks/main.yml rename ansible/roles/{disable-selinux => selinux}/defaults/main.yml (50%) create mode 100644 ansible/roles/selinux/tasks/main.yml create mode 100644 ansible/selinux.yml create mode 100644 releasenotes/notes/rename-disable-selinux-9053ff36792066bc.yaml diff --git a/ansible/disable-selinux.yml b/ansible/disable-selinux.yml deleted file mode 100644 index 3ce1706c6..000000000 --- a/ansible/disable-selinux.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Disable SELinux and reboot if required - hosts: seed:overcloud:infra-vms - tags: - - disable-selinux - roles: - - role: disable-selinux - disable_selinux_reboot_timeout: "{{ 600 if ansible_facts.virtualization_role == 'host' else 300 }}" - when: ansible_facts.os_family == 'RedHat' diff --git a/ansible/infra-vm-host-configure.yml b/ansible/infra-vm-host-configure.yml index ce7b25c8e..e175757e5 100644 --- a/ansible/infra-vm-host-configure.yml +++ b/ansible/infra-vm-host-configure.yml @@ -9,7 +9,7 @@ - import_playbook: "wipe-disks.yml" - import_playbook: "users.yml" - import_playbook: "dev-tools.yml" -- import_playbook: "disable-selinux.yml" +- import_playbook: "selinux.yml" - import_playbook: "network.yml" - import_playbook: "firewall.yml" - import_playbook: "tuned.yml" diff --git a/ansible/overcloud-host-configure.yml b/ansible/overcloud-host-configure.yml index 31587891b..d43c711e9 100644 --- a/ansible/overcloud-host-configure.yml +++ b/ansible/overcloud-host-configure.yml @@ -9,7 +9,7 @@ - import_playbook: "wipe-disks.yml" - import_playbook: "users.yml" - import_playbook: "dev-tools.yml" -- import_playbook: "disable-selinux.yml" +- import_playbook: "selinux.yml" - import_playbook: "network.yml" - import_playbook: "firewall.yml" - import_playbook: "tuned.yml" diff --git a/ansible/roles/disable-selinux/tasks/main.yml b/ansible/roles/disable-selinux/tasks/main.yml deleted file mode 100644 index 5b777452b..000000000 --- a/ansible/roles/disable-selinux/tasks/main.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: Ensure required packages are installed - package: - name: python3-libselinux - state: present - cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" - update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" - become: True - -- name: Check if SELinux configuration file exists - stat: - path: /etc/selinux/config - register: stat_result - -- name: Ensure SELinux is disabled - selinux: - state: disabled - register: selinux_result - become: True - when: stat_result.stat.exists - -- block: - - name: Set a fact to determine whether we are running locally - set_fact: - is_local: "{{ lookup('pipe', 'hostname') in [ansible_facts.hostname, ansible_facts.nodename] }}" - - - name: Reboot the system to apply SELinux changes (local) - command: shutdown -r now "Applying SELinux changes" - become: True - when: is_local | bool - - - name: Reboot the machine to apply SELinux - reboot: - reboot_timeout: "{{ disable_selinux_reboot_timeout }}" - msg: Applying SELinux changes - become: true - when: not is_local | bool - when: - - disable_selinux_do_reboot | bool - - selinux_result is changed diff --git a/ansible/roles/disable-selinux/defaults/main.yml b/ansible/roles/selinux/defaults/main.yml similarity index 50% rename from ansible/roles/disable-selinux/defaults/main.yml rename to ansible/roles/selinux/defaults/main.yml index 23fd5cd33..b58a0da96 100644 --- a/ansible/roles/disable-selinux/defaults/main.yml +++ b/ansible/roles/selinux/defaults/main.yml @@ -1,7 +1,14 @@ --- +# Target SELinux policy +selinux_policy: targeted + +# Target SELinux state +selinux_state: disabled + # Whether to reboot to apply SELinux config changes. disable_selinux_do_reboot: true +selinux_do_reboot: "{{ disable_selinux_do_reboot }}" # Number of seconds to wait for hosts to become accessible via SSH after being # rebooted. -disable_selinux_reboot_timeout: +selinux_reboot_timeout: diff --git a/ansible/roles/selinux/tasks/main.yml b/ansible/roles/selinux/tasks/main.yml new file mode 100644 index 000000000..54f699303 --- /dev/null +++ b/ansible/roles/selinux/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Ensure required packages are installed + package: + name: python3-libselinux + state: present + cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" + update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + become: True + +- name: Check if SELinux configuration file exists + stat: + path: /etc/selinux/config + register: stat_result + +- name: Ensure desired SELinux state + selinux: + policy: "{{ selinux_policy }}" + state: "{{ selinux_state }}" + register: selinux_result + become: True + when: stat_result.stat.exists + +- block: + - name: Abort SELinux configuration because reboot is disabled + fail: + msg: > + SELinux state change requires a reboot, but selinux_do_reboot is + false. Please run again with selinux_do_reboot set to true to reboot. + when: + - not selinux_do_reboot | bool + + - block: + - name: Set a fact to determine whether we are running locally + set_fact: + is_local: "{{ lookup('pipe', 'hostname') in [ansible_facts.hostname, ansible_facts.nodename] }}" + + - name: Reboot the system to apply SELinux changes (local) + command: shutdown -r now "Applying SELinux changes" + become: True + when: + - is_local | bool + + - name: Reboot the machine to apply SELinux + reboot: + reboot_timeout: "{{ selinux_reboot_timeout }}" + msg: Applying SELinux changes + become: true + when: + - not is_local | bool + when: + - selinux_do_reboot | bool + when: + - stat_result.stat.exists + - selinux_result.reboot_required diff --git a/ansible/seed-host-configure.yml b/ansible/seed-host-configure.yml index 4a89f4f09..b41344eae 100644 --- a/ansible/seed-host-configure.yml +++ b/ansible/seed-host-configure.yml @@ -9,7 +9,7 @@ - import_playbook: "wipe-disks.yml" - import_playbook: "users.yml" - import_playbook: "dev-tools.yml" -- import_playbook: "disable-selinux.yml" +- import_playbook: "selinux.yml" - import_playbook: "network.yml" - import_playbook: "firewall.yml" - import_playbook: "tuned.yml" diff --git a/ansible/selinux.yml b/ansible/selinux.yml new file mode 100644 index 000000000..3ffb2e45e --- /dev/null +++ b/ansible/selinux.yml @@ -0,0 +1,9 @@ +--- +- name: Configure SELinux state and reboot if required + hosts: seed:overcloud:infra-vms + tags: + - selinux + roles: + - role: selinux + selinux_reboot_timeout: "{{ disable_selinux_reboot_timeout | default(600 if ansible_facts.virtualization_role == 'host' else 300) }}" + when: ansible_facts.os_family == 'RedHat' diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 0140979d6..7b3247a73 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -479,15 +479,16 @@ package is added to all ``overcloud`` hosts: SELinux ======= *tags:* - | ``disable-selinux`` + | ``selinux`` .. note:: SELinux applies to CentOS and Rocky systems only. -SELinux is not supported by Kolla Ansible currently, so it is disabled by -Kayobe. If necessary, Kayobe will reboot systems in order to apply a change to +SELinux is not supported by Kolla Ansible currently, so it is set to permissive +by Kayobe. If necessary, it can be configured to disabled by setting +``selinux_state`` to ``disabled``. Kayobe will reboot systems when required for the SELinux configuration. The timeout for waiting for systems to reboot is -``disable_selinux_reboot_timeout``. Alternatively, the reboot may be avoided by -setting ``disable_selinux_do_reboot`` to ``false``. +``selinux_reboot_timeout``. Alternatively, the reboot may be avoided by setting +``selinux_do_reboot`` to ``false``. Network Configuration ===================== diff --git a/doc/source/configuration/scenarios/all-in-one/overcloud.rst b/doc/source/configuration/scenarios/all-in-one/overcloud.rst index 5e3b68eca..2992877ab 100644 --- a/doc/source/configuration/scenarios/all-in-one/overcloud.rst +++ b/doc/source/configuration/scenarios/all-in-one/overcloud.rst @@ -230,16 +230,16 @@ seen in MAAS): controller_bootstrap_user: "cloud-user" -By default, on systems with SELinux enabled, Kayobe will disable SELinux and -reboot the system to apply the change. In a test or development environment -this can be a bit disruptive, particularly when using ephemeral network -configuration. To avoid rebooting the system after disabling SELinux, set -``disable_selinux_do_reboot`` to ``false`` in ``etc/kayobe/globals.yml``. +By default, on systems with SELinux disabled, Kayobe will put SELinux in +permissive mode and reboot the system to apply the change. In a test or +development environment this can be a bit disruptive, particularly when using +ephemeral network configuration. To avoid rebooting the system after enabling +SELinux, set ``selinux_do_reboot`` to ``false`` in ``etc/kayobe/globals.yml``. .. code-block:: yaml :caption: ``etc/kayobe/globals.yml`` - disable_selinux_do_reboot: false + selinux_do_reboot: false In a development environment, we may wish to tune some Kolla Ansible variables. Using QEMU as the virtualisation type will be necessary if KVM is not diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index aa530c04e..325a0dcf0 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -571,7 +571,7 @@ class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. - * Disable SELinux. + * Configure SELinux. * Configure the host's network interfaces. * Configure a firewall. * Configure tuned profile. @@ -878,7 +878,7 @@ class InfraVMHostConfigure(KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. - * Disable SELinux. + * Configure SELinux. * Configure the host's network interfaces. * Configure a firewall. * Configure tuned profile. @@ -1126,7 +1126,7 @@ class OvercloudHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, * Optionally, create a virtualenv for remote target hosts. * Optionally, wipe unmounted disk partitions (--wipe-disks). * Configure user accounts, group associations, and authorised SSH keys. - * Disable SELinux. + * Configure SELinux. * Configure the host's network interfaces. * Configure a firewall. * Configure tuned profile. diff --git a/playbooks/kayobe-infra-vm-base/overrides.yml.j2 b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 index 1a72eb8f6..061d7589c 100644 --- a/playbooks/kayobe-infra-vm-base/overrides.yml.j2 +++ b/playbooks/kayobe-infra-vm-base/overrides.yml.j2 @@ -1,8 +1,4 @@ --- -# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as -# Ansible is run directly on the controller. -disable_selinux_do_reboot: false - # Use the OpenStack infra's Dockerhub mirror. docker_registry_mirrors: - "http://{{ zuul_site_mirror_fqdn }}:8082/" diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 index 67ec1a737..af12ef61e 100644 --- a/playbooks/kayobe-overcloud-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -1,12 +1,4 @@ --- -{% if ansible_facts.distribution_release == "jammy" %} -os_release: "jammy" -{% endif %} - -# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as -# Ansible is run directly on the controller. -disable_selinux_do_reboot: false - # Use the OpenStack infra's Dockerhub mirror. docker_registry_mirrors: - "http://{{ zuul_site_mirror_fqdn }}:8082/" diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 index 5972bdfd6..a4dd3a892 100644 --- a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 @@ -1,6 +1,8 @@ --- # NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as # Ansible is run directly on the controller. +# TODO(priteau): This is needed for the deployment of the previous release. +# Remove when previous_release is zed. disable_selinux_do_reboot: false # Use the OpenStack infra's Dockerhub mirror. diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 index e3be9f7d6..817d20a94 100644 --- a/playbooks/kayobe-seed-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -1,12 +1,4 @@ --- -{% if ansible_facts.distribution_release == "jammy" %} -os_release: "jammy" -{% endif %} - -# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as -# Ansible is run directly on the controller. -disable_selinux_do_reboot: false - # Use the OpenStack infra's Dockerhub mirror. docker_registry_mirrors: - "http://{{ zuul_site_mirror_fqdn }}:8082/" diff --git a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 index 4c9a94716..f378daad7 100644 --- a/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-upgrade-base/overrides.yml.j2 @@ -1,6 +1,8 @@ --- # NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as # Ansible is run directly on the controller. +# TODO(priteau): This is needed for the deployment of the previous release. +# Remove when previous_release is zed. disable_selinux_do_reboot: false # Use the OpenStack infra's Dockerhub mirror. diff --git a/playbooks/kayobe-seed-vm-base/overrides.yml.j2 b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 index 9c5462c73..108efb74f 100644 --- a/playbooks/kayobe-seed-vm-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-vm-base/overrides.yml.j2 @@ -1,8 +1,4 @@ --- -# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as -# Ansible is run directly on the controller. -disable_selinux_do_reboot: false - # Use the OpenStack infra's Dockerhub mirror. docker_registry_mirrors: - "http://{{ zuul_site_mirror_fqdn }}:8082/" diff --git a/releasenotes/notes/rename-disable-selinux-9053ff36792066bc.yaml b/releasenotes/notes/rename-disable-selinux-9053ff36792066bc.yaml new file mode 100644 index 000000000..3bd5c7c92 --- /dev/null +++ b/releasenotes/notes/rename-disable-selinux-9053ff36792066bc.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Adds functionality to configure desired SELinux state (in addition to + disabling SELinux previously). +upgrade: + - | + The ``disable-selinux`` role has been renamed to ``selinux`` and so have + been the related variables. If you set one of them, adapt your + configuration: + + * ``disable_selinux_do_reboot`` becomes ``selinux_do_reboot`` + * ``disable_selinux_reboot_timeout`` becomes ``selinux_reboot_timeout`` + - | + Kayobe still sets SELinux to ``disabled`` by default, unlike in the Zed + 13.0.0 release. Operators may want to set ``selinux_state`` to + ``permissive`` to avoid another reboot in the Zed upgrade. From 4f41f170235da4f83da40fecb18a15a1787362e3 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 20 Nov 2023 17:21:58 +0000 Subject: [PATCH 21/47] Run elasticsearch exporter on opensearch hosts too This synchronises with the kolla-ansible multinode inventory file. Closes-Bug: #2044005 Change-Id: I994fa1b73f2bfa08a386d0b788a842eeb5cc688a --- ansible/roles/kolla-ansible/templates/overcloud-services.j2 | 1 + ...-elasticsearch-exporter-opensearch-cb7791220be58055.yaml | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/notes/prometheus-elasticsearch-exporter-opensearch-cb7791220be58055.yaml diff --git a/ansible/roles/kolla-ansible/templates/overcloud-services.j2 b/ansible/roles/kolla-ansible/templates/overcloud-services.j2 index 4a43b2dd5..da09f19f7 100644 --- a/ansible/roles/kolla-ansible/templates/overcloud-services.j2 +++ b/ansible/roles/kolla-ansible/templates/overcloud-services.j2 @@ -520,6 +520,7 @@ monitoring [prometheus-elasticsearch-exporter:children] elasticsearch +opensearch [prometheus-blackbox-exporter:children] monitoring diff --git a/releasenotes/notes/prometheus-elasticsearch-exporter-opensearch-cb7791220be58055.yaml b/releasenotes/notes/prometheus-elasticsearch-exporter-opensearch-cb7791220be58055.yaml new file mode 100644 index 000000000..f7834d304 --- /dev/null +++ b/releasenotes/notes/prometheus-elasticsearch-exporter-opensearch-cb7791220be58055.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Deploys ``prometheus-elasticsearch-exporter`` on hosts of the + ``opensearch`` group, resolving failures to connect to OpenSearch when the + ``elasticsearch`` and ``opensearch`` groups are different. From 698c281a5914469329ea738e44df306841b6ff64 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Thu, 2 Nov 2023 07:43:22 +0100 Subject: [PATCH 22/47] Improve neutron images regex Match ML2/OVS agents only when OVN is not enabled. Change-Id: Ia2ba7414202de6750b18ada47e7a5bb0b636e7b3 (cherry picked from commit d9ee59ea1bdf21f50f5798278b5f6c75bf638a04) --- ansible/group_vars/all/kolla | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index 8267d4f72..aac866211 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -201,11 +201,11 @@ overcloud_container_image_regex_map: enabled: "{{ kolla_enable_multipathd | bool }}" - regex: ^murano enabled: "{{ kolla_enable_murano | bool }}" - - regex: neutron-server + - regex: "neutron-\\(server\\|metadata-agent\\)" enabled: "{{ kolla_enable_neutron | bool }}" # Neutron SFC agent not currently supported on CentOS binary builds. - - regex: "neutron-\\(dhcp\\|l3\\|metadata\\|linuxbridge\\|openvswitch\\)-agent" - enabled: "{{ kolla_enable_neutron | bool }}" + - regex: "neutron-\\(dhcp\\|l3\\|linuxbridge\\|openvswitch\\)-agent" + enabled: "{{ kolla_enable_neutron | bool and not kolla_enable_ovn | bool}}" - regex: neutron-mlnx-agent enabled: "{{ kolla_enable_neutron_mlnx | bool }}" - regex: neutron-sriov-agent From 0bb3ad8aac24f4716cbdc9840d4d8e7015f075ad Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Thu, 23 Nov 2023 10:54:46 +0000 Subject: [PATCH 23/47] kolla-bifrost: Add precheck for inspection allocation pool Change-Id: I73fad29af6a57b29afea46947d1ae90546253a2a (cherry picked from commit 1e2ac5eb2829dac3f637c7f6c2e00b1c980b22c3) --- ansible/roles/kolla-bifrost/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/roles/kolla-bifrost/tasks/main.yml b/ansible/roles/kolla-bifrost/tasks/main.yml index ab8f1417b..ad5b6b40d 100644 --- a/ansible/roles/kolla-bifrost/tasks/main.yml +++ b/ansible/roles/kolla-bifrost/tasks/main.yml @@ -1,4 +1,12 @@ --- +- name: Check if inspection allocation is defined + assert: + that: + - kolla_bifrost_dhcp_pool_start | length > 0 + - kolla_bifrost_dhcp_pool_end | length > 0 + - kolla_bifrost_dhcp_pool_mask | length > 0 + fail_msg: "Inspection allocation pool for provisioning network is not properly defined" + - name: Ensure the Kolla Bifrost configuration directories exist file: path: "{{ kolla_node_custom_config_path }}/bifrost" From a14d05d537259e8bc9c8933a931a653b64660206 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Tue, 27 Apr 2021 11:32:49 +0100 Subject: [PATCH 24/47] Honor custom ssh args in kayobe-ansible-user.yml Using the raw module will honor ssh args. The slight change behaviour is that it will print an unreachable message and show an unreachable task in the summary. Change-Id: I371e8a583c439264a88dbc4bdae14c472955a063 (cherry picked from commit ffbd1a7833b69b9dfdd56e5e4ff71e9bad925c8b) --- ansible/kayobe-ansible-user.yml | 13 ++++--------- ...nor-ssh-args-for-bootstrap-ba894df14ba58167.yaml | 13 +++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml diff --git a/ansible/kayobe-ansible-user.yml b/ansible/kayobe-ansible-user.yml index 6d081baf6..4568ab01a 100644 --- a/ansible/kayobe-ansible-user.yml +++ b/ansible/kayobe-ansible-user.yml @@ -13,20 +13,15 @@ - kayobe-ansible-user tasks: - name: Check whether the host is accessible via SSH - local_action: - module: command ssh -o BatchMode=yes -p {{ ssh_port }} {{ ssh_user }}@{{ ssh_host }} hostname - failed_when: false + raw: hostname + ignore_unreachable: true changed_when: false check_mode: no register: ssh_result - vars: - ssh_user: "{{ ansible_user }}" - ssh_host: "{{ ansible_host | default(inventory_hostname) }}" - ssh_port: "{{ ansible_ssh_port | default('22') }}" - name: Group hosts requiring kayobe user bootstrapping group_by: - key: kayobe_user_bootstrap_required_{{ ssh_result.rc != 0 }} + key: kayobe_user_bootstrap_required_{{ ssh_result.unreachable | default(false) }} changed_when: false - name: Display a message when bootstrapping is required @@ -34,7 +29,7 @@ msg: > Cannot access host via SSH using Kayobe Ansible user account - attempting bootstrap - when: ssh_result.rc != 0 + when: ssh_result.unreachable | default(false) - name: Ensure python is installed hosts: kayobe_user_bootstrap_required_True diff --git a/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml b/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml new file mode 100644 index 000000000..527d9a178 --- /dev/null +++ b/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml @@ -0,0 +1,13 @@ +--- +fixes: + - | + When determining whether or not a host needs bootstrapping, we attempt to + connect to the host using ansible_user, if the login fails, we then assume + that the host needs bootstrapping. In previous releases we used a manually + crafted ``ssh`` command. This did respect any customisations to the SSH + arguments made through ansible configuration. We now use the raw module so + that these customisations are used when connecting to the host. One + possible use case is to configure a jump host between the control host and + the target hosts. If bootstrapping was needed, hosts will now show as + unreachable in the summary stats at the end of the run. This can safely be + ignored. From 2711492262c4d31dbe6b336a502665346bce63e7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 15 Aug 2023 11:19:41 +0100 Subject: [PATCH 25/47] Fix configuration dump with inline encrypted variables If inline Ansible vault encryption is used to define an encrypted variable in kayobe-config, running 'kayobe configuration dump -l ' fails with the following: Failed to decode config dump YAML file /tmp/tmp_fg1bv_j/localhost.yml: ConstructorError(None, None, "could not determine a constructor for the tag '!vault'", ) This change fixes the error by using the Ansible YAML loader which supports the vault tag. Any vault encrypted variables are sanitised in the dump output. Note that variables in vault encrypted files are not sanitised. Change-Id: I4830500d3c927b0689b6f0bca32c28137916420b Closes-Bug: #2031390 (cherry picked from commit 78702d0e3094e6d6a16a31eaf2517d4e0f25d1c7) --- kayobe/ansible.py | 19 +++++- kayobe/tests/unit/test_ansible.py | 66 ++++++++++++++++++- kayobe/tests/unit/test_utils.py | 54 +++++++++++++++ kayobe/utils.py | 20 +++++- .../config-dump-vault-edc615e475f234ac.yaml | 7 ++ 5 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/config-dump-vault-edc615e475f234ac.yaml diff --git a/kayobe/ansible.py b/kayobe/ansible.py index d8dd80054..312a6cb57 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -21,6 +21,8 @@ import sys import tempfile +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode + from kayobe import exception from kayobe import utils from kayobe import vault @@ -257,6 +259,18 @@ def run_playbook(parsed_args, playbook, *args, **kwargs): return run_playbooks(parsed_args, [playbook], *args, **kwargs) +def _sanitise_hostvar(var): + """Sanitise a host variable.""" + if isinstance(var, AnsibleVaultEncryptedUnicode): + return "******" + # Recursively sanitise dicts and lists. + if isinstance(var, dict): + return {k: _sanitise_hostvar(v) for k, v in var.items()} + if isinstance(var, list): + return [_sanitise_hostvar(v) for v in var] + return var + + def config_dump(parsed_args, host=None, hosts=None, var_name=None, facts=None, extra_vars=None, tags=None, verbose_level=None): dump_dir = tempfile.mkdtemp() @@ -282,7 +296,8 @@ def config_dump(parsed_args, host=None, hosts=None, var_name=None, LOG.debug("Found dump file %s", path) inventory_hostname, ext = os.path.splitext(path) if ext == ".yml": - hvars = utils.read_yaml_file(os.path.join(dump_dir, path)) + dump_file = os.path.join(dump_dir, path) + hvars = utils.read_config_dump_yaml_file(dump_file) if host: return hvars else: @@ -290,7 +305,7 @@ def config_dump(parsed_args, host=None, hosts=None, var_name=None, else: LOG.warning("Unexpected extension on config dump file %s", path) - return hostvars + return {k: _sanitise_hostvar(v) for k, v in hostvars.items()} finally: shutil.rmtree(dump_dir) diff --git a/kayobe/tests/unit/test_ansible.py b/kayobe/tests/unit/test_ansible.py index 1f4a13d52..03f3316c5 100644 --- a/kayobe/tests/unit/test_ansible.py +++ b/kayobe/tests/unit/test_ansible.py @@ -402,7 +402,7 @@ def test_run_playbooks_failure(self, mock_validate, mock_vars, mock_run): ansible.run_playbooks, parsed_args, ["command"]) @mock.patch.object(shutil, 'rmtree') - @mock.patch.object(utils, 'read_yaml_file') + @mock.patch.object(utils, 'read_config_dump_yaml_file') @mock.patch.object(os, 'listdir') @mock.patch.object(ansible, 'run_playbook') @mock.patch.object(tempfile, 'mkdtemp') @@ -440,6 +440,70 @@ def test_config_dump(self, mock_mkdtemp, mock_run, mock_listdir, mock_read, mock.call(os.path.join(dump_dir, "host2.yml")), ]) + @mock.patch.object(shutil, 'rmtree') + @mock.patch.object(utils, 'read_file') + @mock.patch.object(os, 'listdir') + @mock.patch.object(ansible, 'run_playbook') + @mock.patch.object(tempfile, 'mkdtemp') + def test_config_dump_vaulted(self, mock_mkdtemp, mock_run, mock_listdir, + mock_read, mock_rmtree): + parser = argparse.ArgumentParser() + parsed_args = parser.parse_args([]) + dump_dir = "/path/to/dump" + mock_mkdtemp.return_value = dump_dir + mock_listdir.return_value = ["host1.yml", "host2.yml"] + config = """--- +key1: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 633230623736383232323862393364323037343430393530316636363961626361393133646437 + 643438663261356433656365646138666133383032376532310a63323432306431303437623637 + 346236316161343635636230613838316566383933313338636237616338326439616536316639 + 6334343462333062363334300a3930313762313463613537626531313230303731343365643766 + 666436333037 +key2: value2 +key3: + - !vault | + $ANSIBLE_VAULT;1.1;AES256 + 633230623736383232323862393364323037343430393530316636363961626361393133646437 + 643438663261356433656365646138666133383032376532310a63323432306431303437623637 + 346236316161343635636230613838316566383933313338636237616338326439616536316639 + 6334343462333062363334300a3930313762313463613537626531313230303731343365643766 + 666436333037 +""" + config_nested = """--- +key1: + key2: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 633230623736383232323862393364323037343430393530316636363961626361393133646437 + 643438663261356433656365646138666133383032376532310a63323432306431303437623637 + 346236316161343635636230613838316566383933313338636237616338326439616536316639 + 6334343462333062363334300a3930313762313463613537626531313230303731343365643766 + 666436333037 +""" + mock_read.side_effect = [config, config_nested] + result = ansible.config_dump(parsed_args) + expected_result = { + "host1": {"key1": "******", "key2": "value2", "key3": ["******"]}, + "host2": {"key1": {"key2": "******"}}, + } + self.assertEqual(result, expected_result) + dump_config_path = utils.get_data_files_path( + "ansible", "dump-config.yml") + mock_run.assert_called_once_with(parsed_args, + dump_config_path, + extra_vars={ + "dump_path": dump_dir, + }, + check_output=True, tags=None, + verbose_level=None, check=False, + list_tasks=False, diff=False) + mock_rmtree.assert_called_once_with(dump_dir) + mock_listdir.assert_any_call(dump_dir) + mock_read.assert_has_calls([ + mock.call(os.path.join(dump_dir, "host1.yml")), + mock.call(os.path.join(dump_dir, "host2.yml")), + ]) + @mock.patch.object(utils, 'galaxy_role_install', autospec=True) @mock.patch.object(utils, 'is_readable_file', autospec=True) @mock.patch.object(os, 'makedirs', autospec=True) diff --git a/kayobe/tests/unit/test_utils.py b/kayobe/tests/unit/test_utils.py index e1b4e2b02..c784cdad3 100644 --- a/kayobe/tests/unit/test_utils.py +++ b/kayobe/tests/unit/test_utils.py @@ -17,6 +17,7 @@ import unittest from unittest import mock +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode import yaml from kayobe import exception @@ -125,6 +126,59 @@ def test_read_yaml_file_not_yaml(self, mock_read): mock_read.return_value = "[1{!" self.assertRaises(SystemExit, utils.read_yaml_file, "/path/to/file") + @mock.patch.object(utils, "read_file") + def test_read_config_dump_yaml_file(self, mock_read): + config = """--- +key1: value1 +key2: value2 +""" + mock_read.return_value = config + result = utils.read_config_dump_yaml_file("/path/to/file") + self.assertEqual(result, {"key1": "value1", "key2": "value2"}) + mock_read.assert_called_once_with("/path/to/file") + + @mock.patch.object(utils, "read_file") + def test_read_config_dump_yaml_file_vaulted(self, mock_read): + config = """--- +key1: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 633230623736383232323862393364323037343430393530316636363961626361393133646437 + 643438663261356433656365646138666133383032376532310a63323432306431303437623637 + 346236316161343635636230613838316566383933313338636237616338326439616536316639 + 6334343462333062363334300a3930313762313463613537626531313230303731343365643766 + 666436333037 +key2: value2 +key3: + - !vault | + $ANSIBLE_VAULT;1.1;AES256 + 633230623736383232323862393364323037343430393530316636363961626361393133646437 + 643438663261356433656365646138666133383032376532310a63323432306431303437623637 + 346236316161343635636230613838316566383933313338636237616338326439616536316639 + 6334343462333062363334300a3930313762313463613537626531313230303731343365643766 + 666436333037 +""" + mock_read.return_value = config + result = utils.read_config_dump_yaml_file("/path/to/file") + # Can't read the value without an encryption key, so just check type. + self.assertTrue(isinstance(result["key1"], + AnsibleVaultEncryptedUnicode)) + self.assertEqual(result["key2"], "value2") + self.assertTrue(isinstance(result["key3"][0], + AnsibleVaultEncryptedUnicode)) + mock_read.assert_called_once_with("/path/to/file") + + @mock.patch.object(utils, "read_file") + def test_read_config_dump_yaml_file_open_failure(self, mock_read): + mock_read.side_effect = IOError + self.assertRaises(SystemExit, utils.read_config_dump_yaml_file, + "/path/to/file") + + @mock.patch.object(utils, "read_file") + def test_read_config_dump_yaml_file_not_yaml(self, mock_read): + mock_read.return_value = "[1{!" + self.assertRaises(SystemExit, utils.read_config_dump_yaml_file, + "/path/to/file") + @mock.patch.object(subprocess, "check_call") def test_run_command(self, mock_call): output = utils.run_command(["command", "to", "run"]) diff --git a/kayobe/utils.py b/kayobe/utils.py index 6ccc4b6e9..b51e0f98a 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -25,6 +25,7 @@ import subprocess import sys +from ansible.parsing.yaml.loader import AnsibleLoader import yaml from kayobe import exception @@ -154,11 +155,28 @@ def read_yaml_file(path): try: content = read_file(path) except IOError as e: - print("Failed to open config dump file %s: %s" % + print("Failed to open YAML file %s: %s" % (path, repr(e))) sys.exit(1) try: return yaml.safe_load(content) + except yaml.YAMLError as e: + print("Failed to decode YAML file %s: %s" % + (path, repr(e))) + sys.exit(1) + + +def read_config_dump_yaml_file(path): + """Read and decode a configuration dump YAML file.""" + try: + content = read_file(path) + except IOError as e: + print("Failed to open config dump file %s: %s" % + (path, repr(e))) + sys.exit(1) + try: + # AnsibleLoader supports loading vault encrypted variables. + return AnsibleLoader(content).get_single_data() except yaml.YAMLError as e: print("Failed to decode config dump YAML file %s: %s" % (path, repr(e))) diff --git a/releasenotes/notes/config-dump-vault-edc615e475f234ac.yaml b/releasenotes/notes/config-dump-vault-edc615e475f234ac.yaml new file mode 100644 index 000000000..f8e6a2fc7 --- /dev/null +++ b/releasenotes/notes/config-dump-vault-edc615e475f234ac.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes an issue where ``kayobe configuration dump`` would fail when + variables are encrypted using Ansible Vault. Encrypted variables are now + sanitised in the dump output. `LP#2031390 + `__ From e4bcc848bc840f587cf82985ac9e086099ebc56a Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 17 Nov 2023 13:23:36 +0100 Subject: [PATCH 26/47] Disable configuration of SELinux by Kolla Ansible When using the default configuration, bootstrapping servers with Kolla Ansible would revert SELinux from permissive to disabled. Change-Id: I8ad027384d9d062fdd363b10fd7bcebe22d775e0 (cherry picked from commit 22307eb73e97833c25521fda5ff8fc6f173116ad) --- ansible/kolla-ansible.yml | 7 ------- ansible/roles/kolla-ansible/defaults/main.yml | 6 ------ ansible/roles/kolla-ansible/templates/kolla/globals.yml | 6 +++--- .../notes/disable-kolla-selinux-71f76e63776e0aed.yaml | 5 +++++ 4 files changed, 8 insertions(+), 16 deletions(-) create mode 100644 releasenotes/notes/disable-kolla-selinux-71f76e63776e0aed.yaml diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index a469e474d..ccff7c2b0 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -91,13 +91,6 @@ kolla_ansible_passwords_path: "{{ kayobe_env_config_path }}/kolla/passwords.yml" kolla_overcloud_group_vars_path: "{{ kayobe_env_config_path }}/kolla/inventory/group_vars" kolla_ansible_certificates_path: "{{ kayobe_env_config_path }}/kolla/certificates" - # NOTE: This differs from the default SELinux mode in kolla ansible, - # which is permissive. The justification for using this mode is twofold: - # 1. it avoids filling up the audit log - # 2. it avoids an issue seen when using diskimage-builder in the bifrost - # container. - # We could look at making the SELinux mode configurable in future. - kolla_selinux_state: disabled kolla_inspector_dhcp_pool_start: "{{ inspection_net_name | net_inspection_allocation_pool_start }}" kolla_inspector_dhcp_pool_end: "{{ inspection_net_name | net_inspection_allocation_pool_end }}" kolla_inspector_netmask: "{{ inspection_net_name | net_mask }}" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index fddca5eaf..69f79c020 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -278,12 +278,6 @@ kolla_ansible_custom_passwords: {} kolla_external_tls_cert: kolla_internal_tls_cert: -############################################################################### -# SELinux - -# Desired SELinux state. -kolla_selinux_state: - ############################################################################### # NTP diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml index 58f8288ef..c116e8735 100644 --- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -550,9 +550,9 @@ grafana_admin_username: "{{ grafana_local_admin_user_name }}" # Bootstrap-servers - Host Configuration ######################################### -{% if kolla_selinux_state is not none %} -selinux_state: {{ kolla_selinux_state }} -{% endif %} +# Kayobe performs configuration of SELinux, so there is no need for Kolla +# Ansible to repeat this. +change_selinux: false {% if kolla_enable_host_ntp is not none %} enable_host_ntp: {{ kolla_enable_host_ntp | bool }} diff --git a/releasenotes/notes/disable-kolla-selinux-71f76e63776e0aed.yaml b/releasenotes/notes/disable-kolla-selinux-71f76e63776e0aed.yaml new file mode 100644 index 000000000..b6adcd28e --- /dev/null +++ b/releasenotes/notes/disable-kolla-selinux-71f76e63776e0aed.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Disables configuration of SELinux by Kolla Ansible, which could revert + configuration set by Kayobe. From b2dd82da0ec29cb3e1ee6a0277cb0994f149c81c Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 1 Dec 2023 23:29:26 +0100 Subject: [PATCH 27/47] Use latest published CentOS Stream 8 image The image from 20220913 has been removed from cloud.centos.org. Change-Id: I8db8414987b9ae85f794d09f5eadff80b41a9c57 --- ansible/group_vars/all/infra-vms | 4 ++-- ansible/group_vars/all/seed-vm | 4 ++-- etc/kayobe/infra-vms.yml | 2 +- etc/kayobe/seed-vm.yml | 2 +- .../update-centos-cloud-image-latest-b967c585a23d1615.yaml | 5 +++++ 5 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/update-centos-cloud-image-latest-b967c585a23d1615.yaml diff --git a/ansible/group_vars/all/infra-vms b/ansible/group_vars/all/infra-vms index da4d6e537..0cdcbbd1c 100644 --- a/ansible/group_vars/all/infra-vms +++ b/ansible/group_vars/all/infra-vms @@ -48,7 +48,7 @@ infra_vm_root_format: qcow2 # or # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 # when os_distribution is "rocky" and os_release is "9" -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. infra_vm_root_image: >- {%- if os_distribution == 'ubuntu' %} @@ -58,7 +58,7 @@ infra_vm_root_image: >- {%- elif os_distribution == 'rocky' and os_release == '9' %} https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 {%- else -%} - https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2 + https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2 {%- endif %} # Capacity of the infra VM data volume. diff --git a/ansible/group_vars/all/seed-vm b/ansible/group_vars/all/seed-vm index 6e5d3395c..81f5b6500 100644 --- a/ansible/group_vars/all/seed-vm +++ b/ansible/group_vars/all/seed-vm @@ -48,7 +48,7 @@ seed_vm_root_format: qcow2 # or # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 # when os_distribution is "rocky" and os_release is "9" -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. seed_vm_root_image: >- {%- if os_distribution == 'ubuntu' %} @@ -58,7 +58,7 @@ seed_vm_root_image: >- {%- elif os_distribution == 'rocky' and os_release == '9' %} https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 {%- else -%} - https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2 + https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2 {%- endif %} # Capacity of the seed VM data volume. diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index c6efae735..069c0877c 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -37,7 +37,7 @@ # or # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 # when os_distribution is "rocky" and os_release is "9" -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. #infra_vm_root_image: diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index 5209b1b38..24122b033 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -31,7 +31,7 @@ # or # https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 # when os_distribution is "rocky" and os_release is "9" -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. #seed_vm_root_image: diff --git a/releasenotes/notes/update-centos-cloud-image-latest-b967c585a23d1615.yaml b/releasenotes/notes/update-centos-cloud-image-latest-b967c585a23d1615.yaml new file mode 100644 index 000000000..60315e74c --- /dev/null +++ b/releasenotes/notes/update-centos-cloud-image-latest-b967c585a23d1615.yaml @@ -0,0 +1,5 @@ +--- +update: + - | + Updates base CentOS Stream 8 cloud image to + CentOS-Stream-GenericCloud-8-latest.x86_64. From 17f6b8661b9ac011f925a849522f927aad1f9ea8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 28 Nov 2023 13:21:53 +0000 Subject: [PATCH 28/47] CI: Run configuration dump against localhost Since using the to_bool function in more places in I3a5a43fef9c3d68d0db02be12b9f892c437e513d, we are now more strict about the result of the variable dump. If there are no controllers in the inventory, the result will not be a valid boolean and the to_bool function will exit non-zero. This change fixes the issue by running against localhost, which should always be in the inventory. Change-Id: Idcfd9d335f11f6c4d676033128d207f62b363ee9 (cherry picked from commit 0f1f009a853cdb6571059a43f33e81f3134d5f4a) --- dev/functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/functions b/dev/functions index 78379c101..241d07c00 100644 --- a/dev/functions +++ b/dev/functions @@ -232,22 +232,22 @@ function upgrade_kayobe_venv { # Deployment function is_deploy_image_built_locally { - ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images) + ipa_build_images=$(kayobe configuration dump --host localhost --var-name ipa_build_images) to_bool "$ipa_build_images" } function is_ironic_enabled { - ironic_enabled=$(kayobe configuration dump --host controllers[0] --var-name kolla_enable_ironic) + ironic_enabled=$(kayobe configuration dump --host localhost --var-name kolla_enable_ironic) to_bool "$ironic_enabled" } function is_overcloud_host_image_built_by_dib { - overcloud_dib_build_host_images=$(kayobe configuration dump --host controllers[0] --var-name overcloud_dib_build_host_images) + overcloud_dib_build_host_images=$(kayobe configuration dump --host localhost --var-name overcloud_dib_build_host_images) to_bool "$overcloud_dib_build_host_images" } function is_cinder_enabled { - flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)" + flag="$(run_kayobe configuration dump --host localhost --var-name kolla_enable_cinder)" to_bool "$flag" } From 2e309ef0469fd317534ed7ef34a228298a05fe18 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 1 Dec 2023 13:02:19 +0100 Subject: [PATCH 29/47] Remove kolla_enable_host_ntp variable This variable was configuring enable_host_ntp in kolla-ansible, which was removed in the Xena release. Change-Id: I737598d3bbe40b933b4b727eccc3b2a76ed97cdb (cherry picked from commit 8da5472ea84729334b4911080b00cf8929f3111e) --- ansible/kolla-ansible.yml | 1 - ansible/roles/kolla-ansible/defaults/main.yml | 6 ------ ansible/roles/kolla-ansible/templates/kolla/globals.yml | 4 ---- 3 files changed, 11 deletions(-) diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index a469e474d..f3f96320e 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -104,7 +104,6 @@ kolla_inspector_default_gateway: "{{ inspection_net_name | net_inspection_gateway or inspection_net_name | net_gateway }}" kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" - kolla_enable_host_ntp: false docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" kolla_globals_paths_extra: - "{{ kayobe_config_path }}" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index fddca5eaf..ca30bae94 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -284,12 +284,6 @@ kolla_internal_tls_cert: # Desired SELinux state. kolla_selinux_state: -############################################################################### -# NTP - -# Whether to enable the NTP daemon. -kolla_enable_host_ntp: - ############################################################################### # Docker configuration. diff --git a/ansible/roles/kolla-ansible/templates/kolla/globals.yml b/ansible/roles/kolla-ansible/templates/kolla/globals.yml index 58f8288ef..d2d67e879 100644 --- a/ansible/roles/kolla-ansible/templates/kolla/globals.yml +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -554,10 +554,6 @@ grafana_admin_username: "{{ grafana_local_admin_user_name }}" selinux_state: {{ kolla_selinux_state }} {% endif %} -{% if kolla_enable_host_ntp is not none %} -enable_host_ntp: {{ kolla_enable_host_ntp | bool }} -{% endif %} - # Kayobe performs creation of the Kolla Ansible user account, so there is no # need for Kolla Ansible to repeat this. create_kolla_user: false From afd9dec5f7dc28828162cfda84d40c95cf54723d Mon Sep 17 00:00:00 2001 From: Maksim Malchuk Date: Sun, 3 Dec 2023 22:04:27 +0300 Subject: [PATCH 30/47] docs: the page 'Editable installs' has moved Change-Id: I57fe9a6a5037adc341617bd35d3df83c81acc5b0 Signed-off-by: Maksim Malchuk (cherry picked from commit 46bba8eb27d9a601bf74e7367f4f20ef769e1cb7) --- doc/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/installation.rst b/doc/source/installation.rst index fcb260016..a3082e9ae 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -160,7 +160,7 @@ Editable source installation ---------------------------- From Kayobe 5.0.0 onwards it is possible to create an `editable install -`__ +`__ of Kayobe. In an editable install, any changes to the Kayobe source tree will immediately be visible when running any Kayobe commands. To create an editable install, add the ``-e`` flag:: From 192e08da52a2c390806dd7fa67d32a8ca9fb1755 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 9 Jun 2022 15:31:22 +0100 Subject: [PATCH 31/47] docs: add information about enabling TLS to automated guide It only covers the internal API VIP, since we don't have a separate external API in the development environment. Change-Id: I28f2b43ddb8bad6097e690e766f0348b1b8a296b (cherry picked from commit 804c1cb669756abd512477263269def90ba3e077) --- doc/source/contributor/automated.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/doc/source/contributor/automated.rst b/doc/source/contributor/automated.rst index c0e7cc657..4383f0556 100644 --- a/doc/source/contributor/automated.rst +++ b/doc/source/contributor/automated.rst @@ -81,6 +81,30 @@ This can be added using the following commands:: sudo ip l set eth1 up sudo ip l set eth1 master breth1 +Configuration +------------- + +Enable TLS +^^^^^^^^^^ + +Apply the following configuration if you wish to enable TLS for the OpenStack +API: + +Set the following option in ``config/src/kayobe-config/etc/kayobe/kolla.yml``: + +.. code-block:: yaml + + kolla_enable_tls_internal: "yes" + +Set the following options in +``config/src/kayobe-config/etc/kayobe/kolla/globals.yml``: + +.. code-block:: yaml + + kolla_copy_ca_into_containers: "yes" + openstack_cacert: "{% if os_distribution == 'ubuntu' %}/etc/ssl/certs/ca-certificates.crt{% else %}/etc/pki/tls/certs/ca-bundle.crt{% endif %}" + kolla_admin_openrc_cacert: "{% if os_distribution == 'ubuntu' %}/etc/ssl/certs/ca-certificates.crt{% else %}/etc/pki/tls/certs/ca-bundle.crt{% endif %}" + Usage ----- @@ -101,6 +125,10 @@ its dependencies in a Python virtual environment:: changes will not been seen until you reinstall the package. To do this you can run ``./dev/install.sh``. +If you are using TLS and wish to generate self-signed certificates:: + + export KAYOBE_OVERCLOUD_GENERATE_CERTIFICATES=1 + Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control plane:: From 1c45e8b109a00b55275b8cdfd28118df5cdbf807 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 7 Dec 2023 15:01:56 +0100 Subject: [PATCH 32/47] Sync Kolla Ansible feature flags for stable/yoga Change-Id: I42192ff8c2f251855a11bc5306e402707f851325 --- ansible/roles/kolla-ansible/vars/main.yml | 6 ++++++ etc/kayobe/kolla.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/ansible/roles/kolla-ansible/vars/main.yml b/ansible/roles/kolla-ansible/vars/main.yml index 043f32124..73a8e1344 100644 --- a/ansible/roles/kolla-ansible/vars/main.yml +++ b/ansible/roles/kolla-ansible/vars/main.yml @@ -108,6 +108,7 @@ kolla_feature_flags: - gnocchi - gnocchi_statsd - grafana + - grafana_external - hacluster - haproxy - haproxy_memcached @@ -147,6 +148,7 @@ kolla_feature_flags: - keystone_federation - keystone_horizon_policy_file - kibana + - kibana_external - kuryr - loadbalancer - magnum @@ -159,6 +161,8 @@ kolla_feature_flags: - mariabackup - mariadb - masakari + - masakari_hostmonitor + - masakari_instancemonitor - memcached - mistral - monasca @@ -201,6 +205,7 @@ kolla_feature_flags: - placement - prometheus - prometheus_alertmanager + - prometheus_alertmanager_external - prometheus_blackbox_exporter - prometheus_cadvisor - prometheus_ceph_mgr_exporter @@ -210,6 +215,7 @@ kolla_feature_flags: - prometheus_haproxy_exporter - prometheus_libvirt_exporter - prometheus_memcached_exporter + - prometheus_msteams - prometheus_mysqld_exporter - prometheus_node_exporter - prometheus_openstack_exporter diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 94a6e9f29..1734696ea 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -299,6 +299,7 @@ #kolla_enable_gnocchi: #kolla_enable_gnocchi_statsd: #kolla_enable_grafana: +#kolla_enable_grafana_external: #kolla_enable_hacluster: #kolla_enable_haproxy: #kolla_enable_haproxy_memcached: @@ -338,6 +339,7 @@ #kolla_enable_keystone_federation: #kolla_enable_keystone_horizon_policy_file: #kolla_enable_kibana: +#kolla_enable_kibana_external: #kolla_enable_kuryr: #kolla_enable_loadbalancer: #kolla_enable_magnum: @@ -350,6 +352,8 @@ #kolla_enable_mariabackup: #kolla_enable_mariadb: #kolla_enable_masakari: +#kolla_enable_masakari_hostmonitor: +#kolla_enable_masakari_instancemonitor: #kolla_enable_memcached: #kolla_enable_mistral: #kolla_enable_monasca: @@ -392,6 +396,7 @@ #kolla_enable_placement: #kolla_enable_prometheus: #kolla_enable_prometheus_alertmanager: +#kolla_enable_prometheus_alertmanager_external: #kolla_enable_prometheus_blackbox_exporter: #kolla_enable_prometheus_cadvisor: #kolla_enable_prometheus_ceph_mgr_exporter: @@ -401,6 +406,7 @@ #kolla_enable_prometheus_haproxy_exporter: #kolla_enable_prometheus_libvirt_exporter: #kolla_enable_prometheus_memcached_exporter: +#kolla_enable_prometheus_msteams: #kolla_enable_prometheus_mysqld_exporter: #kolla_enable_prometheus_node_exporter: #kolla_enable_prometheus_openstack_exporter: From aa7f7f365f7008085a2b90d6c74e9fcd41e69345 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Thu, 7 Dec 2023 21:06:15 +0100 Subject: [PATCH 33/47] Clean up release notes This is a partial cherry-pick since only some changes were backported. Change-Id: I96f00721c173cf095463424c0ca8779c54943591 (cherry picked from commit 99d03686936dd1378e0d50289dd0ab8f0badcd8d) (cherry picked from commit 5bff84f80fb51c43fe0821e23d49741b849485f8) (cherry picked from commit eb8ff22f6bfffa03e505c6cee57e89a0f741eded) --- ...or-ssh-args-for-bootstrap-ba894df14ba58167.yaml | 14 +++++++------- ...kolla-passwords-overrides-065fd6bb8eb9689d.yaml | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml b/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml index 527d9a178..434408642 100644 --- a/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml +++ b/releasenotes/notes/honor-ssh-args-for-bootstrap-ba894df14ba58167.yaml @@ -2,12 +2,12 @@ fixes: - | When determining whether or not a host needs bootstrapping, we attempt to - connect to the host using ansible_user, if the login fails, we then assume - that the host needs bootstrapping. In previous releases we used a manually - crafted ``ssh`` command. This did respect any customisations to the SSH - arguments made through ansible configuration. We now use the raw module so - that these customisations are used when connecting to the host. One - possible use case is to configure a jump host between the control host and - the target hosts. If bootstrapping was needed, hosts will now show as + connect to the host using ``ansible_user``, if the login fails, we then + assume that the host needs bootstrapping. In previous releases we used a + manually crafted ``ssh`` command. This did not respect any customisations + to the SSH arguments made through Ansible configuration. We now use the raw + module so that these customisations are used when connecting to the host. + One possible use case is to configure a jump host between the control host + and the target hosts. If bootstrapping was needed, hosts will now show as unreachable in the summary stats at the end of the run. This can safely be ignored. diff --git a/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml b/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml index adc5318e7..e66f05799 100644 --- a/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml +++ b/releasenotes/notes/kolla-passwords-overrides-065fd6bb8eb9689d.yaml @@ -6,9 +6,9 @@ fixes: ``kolla_ansible_default_custom_passwords`` and own dictionary with custom passwords in configuration files. Now ``kolla_ansible_extra_custom_passwords`` should provide only user custom - passwords to add or override in the passwords.yml. + passwords to add or override in ``kolla/passwords.yml``. upgrade: - | - Now no need to combine ``kolla_ansible_default_custom_passwords`` and - ``kolla_ansible_custom_passwords`` in your custom configuration. Just use - ``kolla_ansible_extra_custom_passwords`` to add or override passwords. + Introduces a new variable ``kolla_ansible_extra_custom_passwords`` to avoid + the need to combine ``kolla_ansible_default_custom_passwords`` and + ``kolla_ansible_custom_passwords`` when adding or overriding passwords. From 2c625dd590e0b0a1cdcfe0ec6b34dfc147b6b7c6 Mon Sep 17 00:00:00 2001 From: Stig Telfer Date: Tue, 28 Nov 2023 21:55:32 +0000 Subject: [PATCH 34/47] Fix gateway assignment when seed SNAT is disabled Fix a logic issue when seed SNAT is not enabled. In this circumstance Bifrost was generating ConfigDrive data with the default gateway unset even when one is available on the admin network. This regression was introduced in Ib847d3420dee374cec067cd8af519b510be04120 [1]. [1] https://review.opendev.org/c/openstack/kayobe/+/898434 Closes-Bug: #2045926 Change-Id: Ifd9f9cd389ac09448fc91dba74f8f09595698b55 (cherry picked from commit 49f91043c6c3b0549a8625baba1e6e218eb69160) --- ansible/kolla-bifrost-hostvars.yml | 2 +- .../notes/fix-bifrost-gateway-b4bc8a0396dfa935.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fix-bifrost-gateway-b4bc8a0396dfa935.yaml diff --git a/ansible/kolla-bifrost-hostvars.yml b/ansible/kolla-bifrost-hostvars.yml index c74a4f88b..a916c6f88 100644 --- a/ansible/kolla-bifrost-hostvars.yml +++ b/ansible/kolla-bifrost-hostvars.yml @@ -22,7 +22,7 @@ # If the admin network does not have a gateway defined and seed SNAT is # enabled, use the seed as a gateway to allow external access until other # networks have been configured. Otherwise, do not set any gateway. - ipv4_gateway: "{{ admin_oc_net_name | net_gateway or admin_oc_net_name | net_ip(seed_host) if seed_enable_snat | bool }}" + ipv4_gateway: "{{ (admin_oc_net_name | net_gateway) or (admin_oc_net_name | net_ip(seed_host) if seed_enable_snat | bool) }}" ipv4_nameserver: "{{ resolv_nameservers }}" network_mtu: "{{ admin_oc_net_name | net_mtu or '1500' }}" vlan_id: "{{ '' if admin_oc_net_name == provision_oc_net_name else (admin_oc_net_name | net_vlan) }}" diff --git a/releasenotes/notes/fix-bifrost-gateway-b4bc8a0396dfa935.yaml b/releasenotes/notes/fix-bifrost-gateway-b4bc8a0396dfa935.yaml new file mode 100644 index 000000000..b2075e90a --- /dev/null +++ b/releasenotes/notes/fix-bifrost-gateway-b4bc8a0396dfa935.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes gateway assignment when seed SNAT is disabled. In this circumstance + Bifrost was generating ConfigDrive data with the default gateway unset even + when one is available on the admin network. From b9cea681d18bba7cfe4a421468e8ba85fd916a36 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Mon, 4 Dec 2023 09:54:52 +0000 Subject: [PATCH 35/47] Stop NetworkManager from overriding resolv.conf When ``resolv_is_managed`` is set, Kayobe templates resolv.conf. This patch stops NetworkManager from overriding these changes. Closes-Bug: #2044537 Change-Id: I90b61dfe03d53c58327d2b15e70b7b8489bdfb47 (cherry picked from commit 578a257c3858484cd26415f1490f3b4ce1b15ccb) --- ansible/roles/network-redhat/tasks/main.yml | 16 ++++++++++++++++ ...-resolv-conf-management-141788e64d0e82a0.yaml | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 releasenotes/notes/fix-resolv-conf-management-141788e64d0e82a0.yaml diff --git a/ansible/roles/network-redhat/tasks/main.yml b/ansible/roles/network-redhat/tasks/main.yml index 09f171bf8..013019369 100644 --- a/ansible/roles/network-redhat/tasks/main.yml +++ b/ansible/roles/network-redhat/tasks/main.yml @@ -17,6 +17,22 @@ when: resolv_is_managed | bool become: True +- name: Ensure NetworkManager DNS config is present only if required + become: true + community.general.ini_file: + path: /etc/NetworkManager/NetworkManager.conf + section: main + option: "{{ item.option }}" + value: "{{ item.value }}" + state: "{{ 'present' if resolv_is_managed | bool else 'absent'}}" + loop: + - option: dns + value: none + - option: rc-manager + value: unmanaged + when: + - ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version == "9" + - name: Configure network interfaces (RedHat) import_role: name: MichaelRigart.interfaces diff --git a/releasenotes/notes/fix-resolv-conf-management-141788e64d0e82a0.yaml b/releasenotes/notes/fix-resolv-conf-management-141788e64d0e82a0.yaml new file mode 100644 index 000000000..102af06b2 --- /dev/null +++ b/releasenotes/notes/fix-resolv-conf-management-141788e64d0e82a0.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes a bug where NetworkManager would overwrite resolv.conf when + ``resolv_is_managed`` is set to ``True``. + `LP#2044537 `__ From 69ba8785b36e584368180ad4fb461f65aa5e59cc Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 8 Dec 2023 10:35:41 +0100 Subject: [PATCH 36/47] Ensure git is present before checkout tasks The latest Rocky-9-GenericCloud image does not include the git package. Change-Id: I47a54b822d45ab5e1e64fa5e0f8111c0edadcb36 (cherry picked from commit 0df1a461dd8e6439f2e2d4877d94e789785f111a) --- ansible/roles/kolla-ansible/vars/Debian.yml | 1 + ansible/roles/kolla-ansible/vars/RedHat.yml | 1 + ansible/roles/kolla/vars/Debian.yml | 1 + ansible/roles/kolla/vars/RedHat.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/ansible/roles/kolla-ansible/vars/Debian.yml b/ansible/roles/kolla-ansible/vars/Debian.yml index b9b871a4d..bb03e7b9c 100644 --- a/ansible/roles/kolla-ansible/vars/Debian.yml +++ b/ansible/roles/kolla-ansible/vars/Debian.yml @@ -2,6 +2,7 @@ # List of packages to install. kolla_ansible_package_dependencies: - gcc + - git - libffi-dev - libssl-dev - python3-dev diff --git a/ansible/roles/kolla-ansible/vars/RedHat.yml b/ansible/roles/kolla-ansible/vars/RedHat.yml index 6568ed9d1..499084905 100644 --- a/ansible/roles/kolla-ansible/vars/RedHat.yml +++ b/ansible/roles/kolla-ansible/vars/RedHat.yml @@ -2,6 +2,7 @@ # List of packages to install. kolla_ansible_package_dependencies: - gcc + - git - libffi-devel - openssl-devel - python3-devel diff --git a/ansible/roles/kolla/vars/Debian.yml b/ansible/roles/kolla/vars/Debian.yml index 3d7bff90c..7f65900c6 100644 --- a/ansible/roles/kolla/vars/Debian.yml +++ b/ansible/roles/kolla/vars/Debian.yml @@ -2,6 +2,7 @@ # List of packages to install. kolla_package_dependencies: - gcc + - git - libffi-dev - libssl-dev - python3-dev diff --git a/ansible/roles/kolla/vars/RedHat.yml b/ansible/roles/kolla/vars/RedHat.yml index dc2dbb32b..397acf767 100644 --- a/ansible/roles/kolla/vars/RedHat.yml +++ b/ansible/roles/kolla/vars/RedHat.yml @@ -2,6 +2,7 @@ # List of packages to install. kolla_package_dependencies: - gcc + - git - libffi-devel - openssl-devel - python3-devel From 8ecb7db0880d29fdd66f2b660ede5202dbff5289 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 7 Feb 2024 12:00:12 +0000 Subject: [PATCH 37/47] Fix ansible requirements The stable/yoga branch for ansible-collection-kolla no longer exists. This change corrects the reference in requirements.yml to unmaintained/yoga Change-Id: Id3dca1be7f062839e09c585d0162fa908d89c760 --- requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.yml b/requirements.yml index bf147767f..fb80c5ccc 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,7 +2,7 @@ collections: - name: https://opendev.org/openstack/ansible-collection-kolla type: git - version: stable/yoga + version: unmaintained/yoga - name: ansible.netcommon source: https://old-galaxy.ansible.com version: '>=1.0.0,<3.0.0' From c61d47e1e709a302c72faf3df5738b35106db2a0 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 6 Feb 2024 15:06:54 +0000 Subject: [PATCH 38/47] Support credentials for custom DNF repositories This allows use of repositories protected with HTTP basic authentication. Change-Id: I6c9686bdf736d58738fbe88169e62833cc85d2f4 (cherry picked from commit bd6673477f0327a5eb07f0f6e2a647827f742a63) --- ansible/roles/dnf/tasks/custom-repo.yml | 2 ++ releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml diff --git a/ansible/roles/dnf/tasks/custom-repo.yml b/ansible/roles/dnf/tasks/custom-repo.yml index 759594ecf..a20018e92 100644 --- a/ansible/roles/dnf/tasks/custom-repo.yml +++ b/ansible/roles/dnf/tasks/custom-repo.yml @@ -16,12 +16,14 @@ metalink: "{{ item.value.metalink | default(omit)}}" mirrorlist: "{{ item.value.mirrorlist | default(omit)}}" mirrorlist_expire: "{{ item.value.mirrorlist_expire | default(omit)}}" + password: "{{ item.value.password | default(omit) }}" priority: "{{ item.value.priority | default(omit)}}" proxy: "{{ item.value.proxy | default(omit)}}" proxy_password: "{{ item.value.proxy_password | default(omit)}}" proxy_username: "{{ item.value.proxy_username | default(omit)}}" repo_gpgcheck: "{{ item.value.repo_gpgcheck | default(omit)}}" sslverify: "{{ item.value.sslverify | default(omit)}}" + username: "{{ item.value.username | default(omit) }}" state: "{{ item.value.state | default(omit)}}" with_dict: "{{ dnf_custom_repos }}" register: register_dnf_command diff --git a/releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml b/releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml new file mode 100644 index 000000000..50e2f94be --- /dev/null +++ b/releasenotes/notes/dnf-credentials-afc3bf7940cb6e99.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for specifying credentials (username and password) for custom + DNF repositories. From 025a1423547ab84201a2d1ba58e8c856935bf9e9 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 7 Feb 2024 14:29:13 +0000 Subject: [PATCH 39/47] Support auth configuration for Apt repositories This allows use of repositories and proxies protected with HTTP basic authentication. Change-Id: I0ec4ec3e9d60bb1431b44dd6718415214ad80025 (cherry picked from commit f74d95760ed5889260034eda93898eeab6aa52c5) --- ansible/group_vars/all/apt | 9 +++++ ansible/roles/apt/defaults/main.yml | 9 +++++ ansible/roles/apt/files/auth_schema.json | 28 ++++++++++++++++ ansible/roles/apt/tasks/auth.yml | 32 ++++++++++++++++++ ansible/roles/apt/tasks/main.yml | 2 ++ ansible/roles/apt/templates/auth.conf.j2 | 5 +++ doc/source/configuration/reference/hosts.rst | 33 +++++++++++++++++++ etc/kayobe/apt.yml | 11 +++++++ .../overrides.yml.j2 | 5 +++ .../tests/test_overcloud_host_configure.py | 10 ++++++ .../notes/apt-auth-97d0291600836dec.yaml | 5 +++ requirements.txt | 1 + 12 files changed, 150 insertions(+) create mode 100644 ansible/roles/apt/files/auth_schema.json create mode 100644 ansible/roles/apt/tasks/auth.yml create mode 100644 ansible/roles/apt/templates/auth.conf.j2 create mode 100644 releasenotes/notes/apt-auth-97d0291600836dec.yaml diff --git a/ansible/group_vars/all/apt b/ansible/group_vars/all/apt index 46d26de18..6e2867151 100644 --- a/ansible/group_vars/all/apt +++ b/ansible/group_vars/all/apt @@ -45,3 +45,12 @@ apt_repositories: [] # when replacing the distribution repositories via apt_repositories. # Default is false. apt_disable_sources_list: false + +# List of Apt auth configurations. Each item is a dict with the following keys: +# * machine: 'machine' entry in the auth file +# * login: 'login' entry in the auth file +# * password: 'password' entry in the auth file +# * filename: Name of a file in /etc/apt/auth.conf.d in which to store +# the auth configuration. The extension should be ``.conf``. +# Default is an empty list. +apt_auth: [] diff --git a/ansible/roles/apt/defaults/main.yml b/ansible/roles/apt/defaults/main.yml index f818381d7..d604ca86c 100644 --- a/ansible/roles/apt/defaults/main.yml +++ b/ansible/roles/apt/defaults/main.yml @@ -48,3 +48,12 @@ apt_repositories: [] # when replacing the distribution repositories via apt_repositories. # Default is false. apt_disable_sources_list: false + +# List of Apt auth configurations. Each item is a dict with the following keys: +# * machine: 'machine' entry in the auth file +# * login: 'login' entry in the auth file +# * password: 'password' entry in the auth file +# * filename: Name of a file in /etc/apt/auth.conf.d in which to store +# the auth configuration. The extension should be ``.conf``. +# Default is an empty list. +apt_auth: [] diff --git a/ansible/roles/apt/files/auth_schema.json b/ansible/roles/apt/files/auth_schema.json new file mode 100644 index 000000000..977350822 --- /dev/null +++ b/ansible/roles/apt/files/auth_schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "List of Apt auth configurations", + "type": "array", + "items": { + "description": "Apt auth configuration", + "type": "object", + "required": ["machine", "login", "password", "filename"], + "properties": { + "machine": { + "type": "string", + "minLength": 1 + }, + "login": { + "type": "string", + "minLength": 1 + }, + "password": { + "type": "string", + "minLength": 1 + }, + "filename": { + "type": "string", + "minLength": 1 + } + } + } +} diff --git a/ansible/roles/apt/tasks/auth.yml b/ansible/roles/apt/tasks/auth.yml new file mode 100644 index 000000000..9718ebfc1 --- /dev/null +++ b/ansible/roles/apt/tasks/auth.yml @@ -0,0 +1,32 @@ +--- +- name: Validate Apt auth config + ansible.utils.validate: + criteria: "{{ lookup('ansible.builtin.file', 'auth_schema.json') }}" + data: "{{ apt_auth }}" + +- name: Ensure the Apt auth.conf.d directory exists + ansible.builtin.file: + path: "/etc/apt/auth.conf.d" + state: directory + owner: root + group: root + mode: 0755 + become: true + +- name: Configure Apt auth files + ansible.builtin.template: + src: "auth.conf.j2" + dest: "/etc/apt/auth.conf.d/{{ auth.filename }}" + owner: root + group: root + mode: 0600 + become: true + # apt_auth contains sensitive data, so iterate over indices to avoid exposing + # them in Ansible output. + loop: "{{ apt_auth | map(attribute='filename') }}" + loop_control: + index_var: auth_index + vars: + auth: "{{ apt_auth[auth_index] }}" + notify: + - Update apt cache diff --git a/ansible/roles/apt/tasks/main.yml b/ansible/roles/apt/tasks/main.yml index b4cb8f636..7bdf2cf01 100644 --- a/ansible/roles/apt/tasks/main.yml +++ b/ansible/roles/apt/tasks/main.yml @@ -6,3 +6,5 @@ - import_tasks: keys.yml - import_tasks: repos.yml + +- import_tasks: auth.yml diff --git a/ansible/roles/apt/templates/auth.conf.j2 b/ansible/roles/apt/templates/auth.conf.j2 new file mode 100644 index 000000000..a5abd10ae --- /dev/null +++ b/ansible/roles/apt/templates/auth.conf.j2 @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +machine {{ auth.machine }} +login {{ auth.login }} +password {{ auth.password }} diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst index 7b3247a73..9d2e1e030 100644 --- a/doc/source/configuration/reference/hosts.rst +++ b/doc/source/configuration/reference/hosts.rst @@ -442,6 +442,39 @@ that is signed by the key. components: all signed_by: example-key.asc +Apt auth configuration +---------------------- + +Some repositories may require authentication using HTTP basic auth. Apt +supports specifying credentials in URLs in ``sources.list`` files, but these +files must be world-readable. A more secure setup involves writing credentials +to `auth.conf +`__ +files which can have more restrictive permissions. + +Auth configuration is defined by the ``apt_auth`` variable. The format is a +list, with each item mapping to a dict/map with the following items: + +* ``machine``: ``machine`` entry in the auth file +* ``login``: ``machine`` entry in the auth file +* ``password``: ``machine`` entry in the auth file +* ``filename``: Name of a file in ``/etc/apt/auth.conf.d`` in which to store + the auth configuration. The extension should be ``.conf``. + +The default value of ``apt_auth`` is an empty list. + +In the following example, credentials are provided for package repositories at +apt.example.com. + +.. code-block:: yaml + :caption: ``apt.yml`` + + apt_auth: + - machine: apt.example.com + login: my-username + password: my-password + filename: example.conf + Development tools ================= *tags:* diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 34bfdd2ef..b43ca9c08 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -46,6 +46,17 @@ # Default is false. #apt_disable_sources_list: +# List of Apt auth configurations. Each item is a dict with the following keys: +# * machine: 'machine' entry in the auth file +# * login: 'login' entry in the auth file +# * password: 'password' entry in the auth file +# * filename: Name of a file in which to store the auth configuration. The +# extension should be '.conf'. +# * filename: Name of a file in /etc/apt/auth.conf.d in which to store +# the auth configuration. The extension should be ``.conf``. +# Default is an empty list. +#apt_auth: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 index 6c7258195..6269f900f 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-host-configure-base/overrides.yml.j2 @@ -149,6 +149,11 @@ apt_repositories: components: contrib signed_by: td-agent.asc apt_disable_sources_list: true +apt_auth: + - machine: https://apt.example.com + login: foo + password: bar + filename: test.conf {% endif %} {% if ansible_facts.os_family == 'RedHat' %} diff --git a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py index eeb37b9ab..dcc5d6bd6 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py +++ b/playbooks/kayobe-overcloud-host-configure-base/tests/test_overcloud_host_configure.py @@ -219,6 +219,16 @@ def test_apt_custom_package_repository_is_available(host): assert host.package("td-agent").is_installed +@pytest.mark.skipif(not _is_apt(), reason="Apt only supported on Ubuntu") +def test_apt_auth(host): + apt_auth = host.file("/etc/apt/apt.auth.d/test.conf") + assert apt_auth.exists + auth_lines = apt_auth.content_string.split() + assert "machine https://apt.example.com" in auth_lines + assert "login foo" in auth_lines + assert "password bar" in auth_lines + + @pytest.mark.parametrize('repo', ["appstream", "baseos", "extras", "epel"]) @pytest.mark.skipif(not _is_dnf_mirror(), reason="DNF OpenDev mirror only for CentOS 8") def test_dnf_local_package_mirrors(host, repo): diff --git a/releasenotes/notes/apt-auth-97d0291600836dec.yaml b/releasenotes/notes/apt-auth-97d0291600836dec.yaml new file mode 100644 index 000000000..6da3d2e39 --- /dev/null +++ b/releasenotes/notes/apt-auth-97d0291600836dec.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for auth configuration for Apt respositories and proxies using + ``auth.conf`` files. diff --git a/requirements.txt b/requirements.txt index 779c9bd12..80a88283e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ selinux # MIT # INI parsing oslo.config>=5.2.0 # Apache-2.0 paramiko # LGPL +jsonschema<5 # MIT From 3771b2ed1c35774f33e5c1941c8116493f3b032d Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Mon, 12 Feb 2024 15:51:29 +0100 Subject: [PATCH 40/47] Rename occurences of stable/yoga to unmaintained/yoga Pin netaddr to 0.10.1 because 1.0.0 requires Python 3.7, pin pytest to <8 because of dropped path. Disable host-configure jobs for now due to docker devicemapper removal - will be fixed in follow up and re-enabled. Change-Id: I2f6c16642b2d6ed3bb8b8c1f9a2525f9641ac9f3 --- ansible/group_vars/all/openstack | 4 ++-- etc/kayobe/openstack.yml | 2 +- molecule-requirements.txt | 1 + requirements.txt | 2 +- requirements.yml | 2 +- zuul.d/jobs.yaml | 8 -------- zuul.d/project.yaml | 22 +++++++++++----------- 7 files changed, 17 insertions(+), 24 deletions(-) diff --git a/ansible/group_vars/all/openstack b/ansible/group_vars/all/openstack index 0804173e0..5226a4c93 100644 --- a/ansible/group_vars/all/openstack +++ b/ansible/group_vars/all/openstack @@ -5,9 +5,9 @@ # Name of the current OpenStack release. Default is "yoga". openstack_release: "yoga" -# Name of the current OpenStack branch. Default is "stable/yoga". +# Name of the current OpenStack branch. Default is "unmaintained/yoga". openstack_branch: >- - {% if openstack_release != 'master' %}stable/{% endif %}{{ openstack_release | lower }} + {% if openstack_release != 'master' %}unmaintained/{% endif %}{{ openstack_release | lower }} ############################################################################### # OpenStack authentication configuration. diff --git a/etc/kayobe/openstack.yml b/etc/kayobe/openstack.yml index 67ad989b0..f2f4af0be 100644 --- a/etc/kayobe/openstack.yml +++ b/etc/kayobe/openstack.yml @@ -5,7 +5,7 @@ # Name of the current OpenStack release. Default is "yoga". #openstack_release: -# Name of the current OpenStack branch. Default is "stable/yoga". +# Name of the current OpenStack branch. Default is "unmaintained/yoga". #openstack_branch: ############################################################################### diff --git a/molecule-requirements.txt b/molecule-requirements.txt index 839177c3e..ad7150014 100644 --- a/molecule-requirements.txt +++ b/molecule-requirements.txt @@ -6,6 +6,7 @@ ansible-lint>=3.0.0,<6.0.0,!=4.3.0 # MIT docker # Apache-2.0 molecule<3.5.0 # MIT molecule-docker # MIT +pytest<8 # MIT pytest-metadata<2 # MPL pytest-molecule # MIT pytest-testinfra diff --git a/requirements.txt b/requirements.txt index c5be19e9d..4b1c86973 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Jinja2>3 # BSD ansible>=4,<6.0 # GPLv3 cliff>=3.1.0 # Apache importlib-metadata;python_version<'3.8' # Apache-2.0 -netaddr!=0.7.16,>=0.7.13 # BSD +netaddr!=0.7.16,>=0.7.13,<1.0.0 # BSD PyYAML>=3.10.0 # MIT selinux # MIT # INI parsing diff --git a/requirements.yml b/requirements.yml index 5ab307e37..6286caaac 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,7 +2,7 @@ collections: - name: https://opendev.org/openstack/ansible-collection-kolla type: git - version: stable/yoga + version: unmaintained/yoga roles: - src: ahuffman.resolv version: 1.3.1 diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 6c73a51a0..3e96d0ad1 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -59,19 +59,11 @@ required-projects: # Include kayobe to ensure other projects can use this job. - name: openstack/ansible-collection-kolla - # TODO(mgoddard): Remove when kayobe stable/yoga exists. - override-checkout: stable/yoga - name: openstack/kayobe - name: openstack/kayobe-config-dev - name: openstack/kolla - # TODO(mgoddard): Remove when kayobe stable/yoga exists. - override-checkout: stable/yoga - name: openstack/kolla-ansible - # TODO(mgoddard): Remove when kayobe stable/yoga exists. - override-checkout: stable/yoga - name: openstack/requirements - # TODO(mgoddard): Remove when kayobe stable/yoga exists. - override-checkout: stable/yoga - name: openstack/tenks irrelevant-files: - ^.*\.rst$ diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index fdf0035de..033707d34 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -17,12 +17,12 @@ - kayobe-overcloud-rocky9 - kayobe-overcloud-ubuntu-focal - kayobe-overcloud-ubuntu-jammy - - kayobe-overcloud-host-configure-centos8s - - kayobe-overcloud-host-configure-centos9s - - kayobe-overcloud-host-configure-rocky8 - - kayobe-overcloud-host-configure-rocky9 - - kayobe-overcloud-host-configure-ubuntu-focal - - kayobe-overcloud-host-configure-ubuntu-jammy + #- kayobe-overcloud-host-configure-centos8s + #- kayobe-overcloud-host-configure-centos9s + #- kayobe-overcloud-host-configure-rocky8 + #- kayobe-overcloud-host-configure-rocky9 + #- kayobe-overcloud-host-configure-ubuntu-focal + #- kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - kayobe-seed-centos8s @@ -52,11 +52,11 @@ - kayobe-overcloud-rocky9 - kayobe-overcloud-ubuntu-focal - kayobe-overcloud-ubuntu-jammy - - kayobe-overcloud-host-configure-centos8s - - kayobe-overcloud-host-configure-rocky8 - - kayobe-overcloud-host-configure-rocky9 - - kayobe-overcloud-host-configure-ubuntu-focal - - kayobe-overcloud-host-configure-ubuntu-jammy + #- kayobe-overcloud-host-configure-centos8s + #- kayobe-overcloud-host-configure-rocky8 + #- kayobe-overcloud-host-configure-rocky9 + #- kayobe-overcloud-host-configure-ubuntu-focal + #- kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - kayobe-seed-centos8s From bb63164380a6ddbe37251bc9b311dc7b4bdbc52f Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Tue, 13 Feb 2024 22:07:19 +0000 Subject: [PATCH 41/47] CI: support .gitreview of unmaintained branch Change-Id: Ica6c0b204094ed61aef8242c934eb3b7715ce944 (cherry picked from commit bb51db15febc195384a48406cb37571776988ee3) --- dev/functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/functions b/dev/functions index 241d07c00..36bd6190c 100644 --- a/dev/functions +++ b/dev/functions @@ -74,7 +74,7 @@ function config_defaults { if [[ "${BRANCH}" == "" ]]; then SERIES="master" else - SERIES="$(echo ${BRANCH} | sed 's|stable/||')" + SERIES="$(echo ${BRANCH} | sed -E 's,(stable|unmaintained)/,,')" fi # Upper constraints to use when installing Python packages. From 52a8ca7f79997b4301b858aa9fd5a32ebb2982e8 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 5 Feb 2024 16:06:25 +0000 Subject: [PATCH 42/47] Update .gitreview for unmaintained/yoga Change-Id: I0e933fdab088b334344295c546f995c683709913 --- .gitreview | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitreview b/.gitreview index 82071d84f..f9c700c02 100644 --- a/.gitreview +++ b/.gitreview @@ -2,4 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe.git -defaultbranch=stable/yoga +defaultbranch=unmaintained/yoga From 29a4dc397afdc1669b8168af0af0a857413abbd4 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 31 Jan 2024 14:57:02 +0100 Subject: [PATCH 43/47] Deprecate devicemapper due to removal in Docker Docker Engine 25.0 drops support for the devicemapper storage driver after a long deprecation period. Since devicemapper is used in some CI jobs, pin docker-ce package to the latest available 24.x release. The master branch will remove devicemapper support in another patch [1]. Add a deprecation notice indicating to the operator that they must install a compatible version of Docker Engine to keep using devicemapper. Note: this backport is amended to set packages via kolla/globals.yml instead of zzz-overrides.yml. This is required for Zed and earlier releases, which still use kolla bootstrap-servers to install Docker. Note: this backport is amended to support both Focal and Jammy and pins pytest to the latest 7.x release to fix the kayobe-tox-molecule job. [1] https://review.opendev.org/c/openstack/kayobe/+/906386 Change-Id: I6f62081d71232091e26269d4d7877203236c459d (cherry picked from commit bd16d2931d79d6df86f9e400e5387d924d99df30) (cherry picked from commit fbefef43d7d17bc77ba9a0d59dd2f2d04b1c5781) --- ansible/docker-devicemapper.yml | 12 ++++++++++ .../globals.yml.j2 | 8 +++++++ .../pre.yml | 5 +++++ ...-docker-devicemapper-234447328a24094f.yaml | 8 +++++++ zuul.d/project.yaml | 22 +++++++++---------- 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 playbooks/kayobe-overcloud-host-configure-base/globals.yml.j2 create mode 100644 releasenotes/notes/deprecate-docker-devicemapper-234447328a24094f.yaml diff --git a/ansible/docker-devicemapper.yml b/ansible/docker-devicemapper.yml index 70ead14a4..2a7b30275 100644 --- a/ansible/docker-devicemapper.yml +++ b/ansible/docker-devicemapper.yml @@ -5,6 +5,18 @@ - docker - docker-devicemapper tasks: + - name: Warn about deprecation of devicemapper Docker storage driver + fail: + msg: > + Support for using the devicemapper Docker storage driver is + deprecated in Kayobe and will be removed in the Caracal 16.0.0 + release. To keep using devicemapper in the current release, ensure + that the version of Docker Engine installed is below 25.0.0. + # NOTE(priteau): We want this to print a nice big red warning and not to + # fail the run. + ignore_errors: yes + when: docker_storage_driver == 'devicemapper' + - name: Check for existing Docker configuration using devicemapper block: - name: Query Docker daemon for storage driver diff --git a/playbooks/kayobe-overcloud-host-configure-base/globals.yml.j2 b/playbooks/kayobe-overcloud-host-configure-base/globals.yml.j2 new file mode 100644 index 000000000..feff57d91 --- /dev/null +++ b/playbooks/kayobe-overcloud-host-configure-base/globals.yml.j2 @@ -0,0 +1,8 @@ +--- +# Pin Docker package to latest 24.x release before devicemapper was removed +{% if ansible_facts.distribution_release == "jammy" %} +docker_apt_package: "docker-ce=5:24.0.8-1~ubuntu.22.04~jammy" +{% else %} +docker_apt_package: "docker-ce=5:24.0.8-1~ubuntu.20.04~focal" +{% endif %} +docker_yum_package: "docker-ce-24*" diff --git a/playbooks/kayobe-overcloud-host-configure-base/pre.yml b/playbooks/kayobe-overcloud-host-configure-base/pre.yml index 60f81d549..4651d04e7 100644 --- a/playbooks/kayobe-overcloud-host-configure-base/pre.yml +++ b/playbooks/kayobe-overcloud-host-configure-base/pre.yml @@ -25,6 +25,11 @@ src: overrides.yml.j2 dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zzz-overrides.yml" + - name: Ensure kolla-ansible globals.yml override config file exists + template: + src: globals.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml" + # NOTE(mgoddard): Create two loopback devices backed by files. These will # be added to a software RAID volume, then added to an LVM volume group. - name: Ensure a docker storage backing file exists diff --git a/releasenotes/notes/deprecate-docker-devicemapper-234447328a24094f.yaml b/releasenotes/notes/deprecate-docker-devicemapper-234447328a24094f.yaml new file mode 100644 index 000000000..fa8ca3ea4 --- /dev/null +++ b/releasenotes/notes/deprecate-docker-devicemapper-234447328a24094f.yaml @@ -0,0 +1,8 @@ +--- +deprecations: + - | + Support for the ``devicemapper`` Docker storage driver is deprecated + following its removal from Docker Engine 25.0. Support will be fully + removed in the Caracal 16.0.0 release. Operators using ``devicemapper`` + should ensure that a compatible version of Docker Engine is installed (i.e. + release 24.x or below). diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 033707d34..fdf0035de 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -17,12 +17,12 @@ - kayobe-overcloud-rocky9 - kayobe-overcloud-ubuntu-focal - kayobe-overcloud-ubuntu-jammy - #- kayobe-overcloud-host-configure-centos8s - #- kayobe-overcloud-host-configure-centos9s - #- kayobe-overcloud-host-configure-rocky8 - #- kayobe-overcloud-host-configure-rocky9 - #- kayobe-overcloud-host-configure-ubuntu-focal - #- kayobe-overcloud-host-configure-ubuntu-jammy + - kayobe-overcloud-host-configure-centos8s + - kayobe-overcloud-host-configure-centos9s + - kayobe-overcloud-host-configure-rocky8 + - kayobe-overcloud-host-configure-rocky9 + - kayobe-overcloud-host-configure-ubuntu-focal + - kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - kayobe-seed-centos8s @@ -52,11 +52,11 @@ - kayobe-overcloud-rocky9 - kayobe-overcloud-ubuntu-focal - kayobe-overcloud-ubuntu-jammy - #- kayobe-overcloud-host-configure-centos8s - #- kayobe-overcloud-host-configure-rocky8 - #- kayobe-overcloud-host-configure-rocky9 - #- kayobe-overcloud-host-configure-ubuntu-focal - #- kayobe-overcloud-host-configure-ubuntu-jammy + - kayobe-overcloud-host-configure-centos8s + - kayobe-overcloud-host-configure-rocky8 + - kayobe-overcloud-host-configure-rocky9 + - kayobe-overcloud-host-configure-ubuntu-focal + - kayobe-overcloud-host-configure-ubuntu-jammy - kayobe-overcloud-tls-centos8s - kayobe-overcloud-tls-rocky9 - kayobe-seed-centos8s From 9c9d9d6923542eddf832835da9e4e984b701d81a Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Mon, 5 Feb 2024 14:53:17 +0100 Subject: [PATCH 44/47] Reload NetworkManager on DNS config change Initial fix [1] was not reloading NetworkManager config, which caused MichaelRigart.interfaces role to clean up resolv.conf after interface bounce. [1] https://review.opendev.org/c/openstack/kayobe/+/902561 Related-Bug: #2044537 Change-Id: I5644ab2595a30cfff82f03d5ca4aa501a8b36ef8 (cherry picked from commit 2a73c328d39c24772887031c6162b608a980e388) --- ansible/roles/network-redhat/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/roles/network-redhat/tasks/main.yml b/ansible/roles/network-redhat/tasks/main.yml index 013019369..b749ab479 100644 --- a/ansible/roles/network-redhat/tasks/main.yml +++ b/ansible/roles/network-redhat/tasks/main.yml @@ -32,6 +32,15 @@ value: unmanaged when: - ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version == "9" + register: dns_config_task + +- name: Reload NetworkManager with DNS config + become: true + systemd: + name: NetworkManager + state: reloaded + daemon_reload: yes + when: dns_config_task is changed - name: Configure network interfaces (RedHat) import_role: From bcb87d3be78cb02e3d98d491a9bf47e8c044a02a Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Wed, 17 Jan 2024 17:46:06 +0100 Subject: [PATCH 45/47] Switch IPA builds to CentOS Stream 9 for yoga Latest Python 3.6 packages in CentOS Stream 8 break TLS in eventlet, which affects IPA. See [1] for context. [1] https://lists.openstack.org/archives/list/openstack-discuss@lists.openstack.org/thread/NMCPYYHUPG766V5MGUUEKNIDEV6RCELC/ Change-Id: Ib6421e09dd1e8077397433b51a4a707d34b9a2a6 --- ansible/group_vars/all/ipa | 2 +- .../notes/fix-ipa-eventlet-issues-82941ff32f31f2cb.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/fix-ipa-eventlet-issues-82941ff32f31f2cb.yaml diff --git a/ansible/group_vars/all/ipa b/ansible/group_vars/all/ipa index 0d88e4dfe..b7d589208 100644 --- a/ansible/group_vars/all/ipa +++ b/ansible/group_vars/all/ipa @@ -44,7 +44,7 @@ ipa_build_dib_elements: > ipa_build_dib_env_default: # TODO(mgoddard): Use {{ os_release }} here when we use os_distribution # above. - DIB_RELEASE: "8-stream" + DIB_RELEASE: "9-stream" DIB_REPOLOCATION_ironic_python_agent: "{{ ipa_build_source_url }}" DIB_REPOREF_ironic_python_agent: "{{ ipa_build_source_version }}" DIB_REPOREF_requirements: "{{ openstack_branch }}" diff --git a/releasenotes/notes/fix-ipa-eventlet-issues-82941ff32f31f2cb.yaml b/releasenotes/notes/fix-ipa-eventlet-issues-82941ff32f31f2cb.yaml new file mode 100644 index 000000000..1a8c39176 --- /dev/null +++ b/releasenotes/notes/fix-ipa-eventlet-issues-82941ff32f31f2cb.yaml @@ -0,0 +1,8 @@ +--- +upgrade: + - | + Ironic Python Agent images are now built using CentOS Stream 9 by default. +fixes: + - | + Fixes issues with TLS and eventlet affecting Ironic Python Agent. IPA + images should be rebuilt for affected deployments. From 4fce51a502c72b43fd33d03be14060c633cc9a50 Mon Sep 17 00:00:00 2001 From: Zuul Date: Mon, 19 Feb 2024 17:13:44 +0000 Subject: [PATCH 46/47] Merge "Make OVS container build regex independent to OVN" --- ansible/group_vars/all/kolla | 2 +- doc/source/deployment.rst | 6 ++++++ ..._build_OVS_images_while_using_OVN-48471bbaebc953be.yaml | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/add_option_to_build_OVS_images_while_using_OVN-48471bbaebc953be.yaml diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla index d2375b555..a7450a914 100644 --- a/ansible/group_vars/all/kolla +++ b/ansible/group_vars/all/kolla @@ -207,7 +207,7 @@ overcloud_container_image_regex_map: enabled: "{{ kolla_enable_neutron | bool }}" # Neutron SFC agent not currently supported on CentOS binary builds. - regex: "neutron-\\(dhcp\\|l3\\|linuxbridge\\|openvswitch\\)-agent" - enabled: "{{ kolla_enable_neutron | bool and not kolla_enable_ovn | bool}}" + enabled: "{{ kolla_build_neutron_ovs | default(kolla_enable_neutron | bool and not kolla_enable_ovn | bool) }}" - regex: neutron-mlnx-agent enabled: "{{ kolla_enable_neutron_mlnx | bool }}" - regex: neutron-sriov-agent diff --git a/doc/source/deployment.rst b/doc/source/deployment.rst index fc70b587d..91cd967fd 100644 --- a/doc/source/deployment.rst +++ b/doc/source/deployment.rst @@ -534,6 +534,12 @@ image name regular expressions:: (kayobe) $ kayobe overcloud container image build ironic- nova-api +When your environment uses OVN, OVS images will not be built. If you want to +build all Neutron images at the same time, extra variable ``kolla_build_neutron_ovs`` +needs to be set to ``true``:: + + (kayobe) $ kayobe overcloud container image build -e kolla_build_neutron_ovs=true + In order to push images to a registry after they are built, add the ``--push`` argument. diff --git a/releasenotes/notes/add_option_to_build_OVS_images_while_using_OVN-48471bbaebc953be.yaml b/releasenotes/notes/add_option_to_build_OVS_images_while_using_OVN-48471bbaebc953be.yaml new file mode 100644 index 000000000..5f03bfb91 --- /dev/null +++ b/releasenotes/notes/add_option_to_build_OVS_images_while_using_OVN-48471bbaebc953be.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a new variable ``kolla_build_neutron_ovs`` which gives users to have + option to build OVS container images while the system is using OVN. + This is useful when an user wants to build all container images at the + same time. From 360435c57ea565ef65a9a151fcdd6ecba4db2430 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 16 May 2023 14:55:52 +0100 Subject: [PATCH 47/47] Fix firewalld configuration for monitoring hosts There was a stray double quote in the firewalld config for monitoring hosts, resulting in the following error when firewalld is enabled: ValueError: No closing quotation Closes-Bug: 2019867 Change-Id: I201faf14da5d143670250052ab15fc285f24868c (cherry picked from commit b7e95c83d3ea22bdccb5a5440e677c8a36f4cf61) --- ansible/group_vars/monitoring/firewall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/group_vars/monitoring/firewall b/ansible/group_vars/monitoring/firewall index a1b151527..41c3c411b 100644 --- a/ansible/group_vars/monitoring/firewall +++ b/ansible/group_vars/monitoring/firewall @@ -19,7 +19,7 @@ firewalld_zones: > firewalld_default_zone: >- {{ controller_firewalld_default_zone if inventory_hostname in groups['controllers'] else - monitoring_firewalld_default_zone }}" + monitoring_firewalld_default_zone }} # A list of firewall rules to apply. Each item is a dict containing arguments # to pass to the firewalld module. Arguments are omitted if not provided, with @@ -30,4 +30,4 @@ firewalld_default_zone: >- firewalld_rules: > {{ controller_firewalld_rules if inventory_hostname in groups['controllers'] else - monitoring_firewalld_rules }}" + monitoring_firewalld_rules }}