From ded3c040fa97ad77cd74ccb92f9ddcf2b3c692db Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 20 Sep 2023 09:14:17 +0100 Subject: [PATCH 01/61] kayobe-env: Unstick KOLLA_SOURCE_PATH and KOLLA_VENV_PATH The kayobe-env script does not update the KOLLA_SOURCE_PATH and KOLLA_VENV_PATH variables if they are already set. This can lead to dangerous and difficult to diagnose issues where Kayobe uses a different version of Kolla Ansible than expected. This change updates these variables each time the kayobe-env script is sourced. Change-Id: I3b4b0b611750b9c7846ff5f74554aee2f14939e4 Closes-Bug: #2036711 (cherry picked from commit 651b8be1a0a2dd38ab46253a0c4a9c7d617cf7bc) --- kayobe-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kayobe-env b/kayobe-env index 5137927e5..28b1cccdb 100644 --- a/kayobe-env +++ b/kayobe-env @@ -30,8 +30,8 @@ export KOLLA_CONFIG_PATH=$KAYOBE_CONFIG_ROOT/etc/kolla # kayobe/ # kolla-ansible/ base_path=$(realpath $KAYOBE_CONFIG_ROOT/../../) -export KOLLA_SOURCE_PATH=${KOLLA_SOURCE_PATH:-${base_path}/src/kolla-ansible} -export KOLLA_VENV_PATH=${KOLLA_VENV_PATH:-${base_path}/venvs/kolla-ansible} +export KOLLA_SOURCE_PATH=${base_path}/src/kolla-ansible +export KOLLA_VENV_PATH=${base_path}/venvs/kolla-ansible function check_and_export_env { # Look for existing Kayobe environments From d422259ce1ff8fbc800379fe3be5f5d83052a431 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 8 Dec 2023 12:10:48 +0100 Subject: [PATCH 02/61] Synchronise with kayobe stable/yoga Synchronise with kayobe @ 32b12be953d0c2f60970a95b82d0ebf846d0f86f. Change-Id: Ib4b75c084820f16ce245c1e1c50f10ccb2083537 --- etc/kayobe/bifrost.yml | 3 +++ etc/kayobe/compute.yml | 8 ++++---- etc/kayobe/controllers.yml | 8 ++++---- etc/kayobe/globals.yml | 5 ++--- etc/kayobe/infra-vms.yml | 16 +++++++++------- etc/kayobe/kolla.yml | 22 ++++++++++++++++++---- etc/kayobe/monitoring.yml | 6 +++--- etc/kayobe/seed-hypervisor.yml | 4 ++-- etc/kayobe/seed-vm.yml | 10 ++++++---- etc/kayobe/seed.yml | 12 ++++++++---- etc/kayobe/storage.yml | 8 ++++---- 11 files changed, 63 insertions(+), 39 deletions(-) diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index a9eba19dd..d15d18613 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -116,6 +116,9 @@ # Ironic inspector deployment ramdisk location. #kolla_bifrost_inspector_deploy_ramdisk: +# Ironic inspector legacy deployment kernel location. +#kolla_bifrost_inspector_legacy_deploy_kernel: + # Timeout of hardware inspection on overcloud nodes, in seconds. Default is # {{ inspector_inspection_timeout }}. #kolla_bifrost_inspection_timeout: diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index b1d8d6562..57286b40c 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -63,15 +63,15 @@ ############################################################################### # Compute node LVM configuration. -# List of compute volume groups. See mrlesmithjr.manage-lvm role for +# List of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups: -# Default list of compute volume groups. See mrlesmithjr.manage-lvm role for +# Default list of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups_default: -# Additional list of compute volume groups. See mrlesmithjr.manage-lvm role +# Additional list of compute volume groups. See mrlesmithjr.manage_lvm role # for format. #compute_lvm_groups_extra: @@ -82,7 +82,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #compute_lvm_group_data_enabled: -# Compute LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Compute LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_group_data: diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index 983251c6c..4780ec444 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -72,15 +72,15 @@ ############################################################################### # Controller node LVM configuration. -# List of controller volume groups. See mrlesmithjr.manage-lvm role for +# List of controller volume groups. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_groups: -# Default list of controller volume groups. See mrlesmithjr.manage-lvm role for +# Default list of controller volume groups. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_groups_default: -# Additional list of controller volume groups. See mrlesmithjr.manage-lvm role +# Additional list of controller volume groups. See mrlesmithjr.manage_lvm role # for format. #controller_lvm_groups_extra: @@ -91,7 +91,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #controller_lvm_group_data_enabled: -# Controller LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Controller LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #controller_lvm_group_data: diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 16b791548..ecdc5970b 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -4,8 +4,7 @@ ############################################################################### # Local path configuration (Ansible control host). -# Path to Kayobe configuration directory on Ansible control host, with an -# environment path appended if kayobe_environment is set. +# Path to Kayobe configuration directory on Ansible control host. #kayobe_config_path: # Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an @@ -50,7 +49,7 @@ #os_distribution: # OS release. Valid options are "8-stream" when os_distribution is "centos", or -# "8" when os_distribution is "rocky", or "focal" and "jammy" when +# "8" or "9" when os_distribution is "rocky", or "focal" and "jammy" when # os_distribution is "ubuntu". #os_release: diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index a802f5acc..069c0877c 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -32,10 +32,12 @@ # Base image for the infra VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" # when os_distribution is "ubuntu", or -# https://dl.rockylinux.org/pub/rocky/8/images/Rocky-8-GenericCloud.latest.x86_64.qcow2 -# when os_distribution is "rocky", +# https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky" and os_release is "8" # or -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky" and os_release is "9" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. #infra_vm_root_image: @@ -92,15 +94,15 @@ ############################################################################### # Infrastructure VM node LVM configuration. -# List of infrastructure vm volume groups. See mrlesmithjr.manage-lvm role for +# List of infrastructure vm volume groups. See mrlesmithjr.manage_lvm role for # format. #infra_vm_lvm_groups: -# Default list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Default list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_default: -# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_extra: @@ -111,7 +113,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #infra_vm_lvm_group_data_enabled: -# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage-lvm role +# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage_lvm role # for format. #infra_vm_lvm_group_data: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 16714ec39..1734696ea 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -67,8 +67,9 @@ # Kolla configuration. # Kolla base container image distribution. Options are "centos", "debian", -# "ubuntu". Default is -# {{ 'centos' if os_distribution == 'rocky' else os_distribution }}. +# "rocky", "ubuntu". Default is +# {{ 'centos' if (os_distribution == 'rocky' and os_release == '8') else +# os_distribution }}. #kolla_base_distro: # Kolla container image type: binary or source. Default is 'source'. @@ -298,6 +299,7 @@ #kolla_enable_gnocchi: #kolla_enable_gnocchi_statsd: #kolla_enable_grafana: +#kolla_enable_grafana_external: #kolla_enable_hacluster: #kolla_enable_haproxy: #kolla_enable_haproxy_memcached: @@ -337,6 +339,7 @@ #kolla_enable_keystone_federation: #kolla_enable_keystone_horizon_policy_file: #kolla_enable_kibana: +#kolla_enable_kibana_external: #kolla_enable_kuryr: #kolla_enable_loadbalancer: #kolla_enable_magnum: @@ -349,6 +352,8 @@ #kolla_enable_mariabackup: #kolla_enable_mariadb: #kolla_enable_masakari: +#kolla_enable_masakari_hostmonitor: +#kolla_enable_masakari_instancemonitor: #kolla_enable_memcached: #kolla_enable_mistral: #kolla_enable_monasca: @@ -379,6 +384,9 @@ #kolla_enable_nova_ssh: #kolla_enable_octavia: #kolla_enable_octavia_driver_agent: +#kolla_enable_opensearch: +#kolla_enable_opensearch_dashboards: +#kolla_enable_opensearch_dashboards_external: #kolla_enable_openstack_core: #kolla_enable_openvswitch: #kolla_enable_osprofiler: @@ -388,6 +396,7 @@ #kolla_enable_placement: #kolla_enable_prometheus: #kolla_enable_prometheus_alertmanager: +#kolla_enable_prometheus_alertmanager_external: #kolla_enable_prometheus_blackbox_exporter: #kolla_enable_prometheus_cadvisor: #kolla_enable_prometheus_ceph_mgr_exporter: @@ -397,6 +406,7 @@ #kolla_enable_prometheus_haproxy_exporter: #kolla_enable_prometheus_libvirt_exporter: #kolla_enable_prometheus_memcached_exporter: +#kolla_enable_prometheus_msteams: #kolla_enable_prometheus_mysqld_exporter: #kolla_enable_prometheus_node_exporter: #kolla_enable_prometheus_openstack_exporter: @@ -430,6 +440,10 @@ # Kolla passwords file. #kolla_ansible_default_custom_passwords: +# Dictionary containing extra custom passwords to add or override in the Kolla +# passwords file. +#kolla_ansible_extra_custom_passwords: + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. #kolla_ansible_custom_passwords: @@ -469,7 +483,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in public-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_external_fqdn_cacert: +#kolla_public_openrc_cacert: # Internal API certificate bundle. # @@ -482,7 +496,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_internal_fqdn_cacert: +#kolla_admin_openrc_cacert: ############################################################################### # Proxy configuration diff --git a/etc/kayobe/monitoring.yml b/etc/kayobe/monitoring.yml index f332ab938..5468936d3 100644 --- a/etc/kayobe/monitoring.yml +++ b/etc/kayobe/monitoring.yml @@ -63,15 +63,15 @@ ############################################################################### # Monitoring node LVM configuration. -# List of monitoring node volume groups. See mrlesmithjr.manage-lvm role for +# List of monitoring node volume groups. See mrlesmithjr.manage_lvm role for # format. #monitoring_lvm_groups: -# Default list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Default list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_default: -# Additional list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Additional list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_extra: diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index ac72fcd3d..dd8fbca23 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -36,7 +36,7 @@ ############################################################################### # Seed hypervisor node LVM configuration. -# List of seed hypervisor volume groups. See mrlesmithjr.manage-lvm role for +# List of seed hypervisor volume groups. See mrlesmithjr.manage_lvm role for # format. Set to "{{ seed_hypervisor_lvm_groups_with_data }}" to create a # volume group for libvirt storage. #seed_hypervisor_lvm_groups: @@ -45,7 +45,7 @@ # default. #seed_hypervisor_lvm_groups_with_data: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_hypervisor_lvm_group_data: # List of disks for use by seed hypervisor LVM data volume group. Default to an diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index 3856d51f2..24122b033 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -25,11 +25,13 @@ # Base image for the seed VM root volume. Default is # "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" -# when os_distribution is "ubuntu", -# https://dl.rockylinux.org/pub/rocky/8/images/Rocky-8-GenericCloud.latest.x86_64.qcow2 -# when os_distribution is "rocky", +# when os_distribution is "ubuntu", or +# https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky" and os_release is "8" # or -# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220913.0.x86_64.qcow2" +# https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 +# when os_distribution is "rocky" and os_release is "9" +# "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-latest.x86_64.qcow2" # otherwise. #seed_vm_root_image: diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index ade99307d..bc86fa627 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -36,14 +36,14 @@ ############################################################################### # Seed node LVM configuration. -# List of seed volume groups. See mrlesmithjr.manage-lvm role for format. +# List of seed volume groups. See mrlesmithjr.manage_lvm role for format. #seed_lvm_groups: -# Default list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Default list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_default: -# Additional list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Additional list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_extra: @@ -54,7 +54,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #seed_lvm_group_data_enabled: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_lvm_group_data: # List of disks for use by seed LVM data volume group. Default to an invalid @@ -106,6 +106,10 @@ # #seed_containers: +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +#seed_deploy_containers_registry_attempt_login: + ############################################################################### # Seed node firewalld configuration. diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 535666c95..e9e52dfe6 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -68,15 +68,15 @@ ############################################################################### # Storage node LVM configuration. -# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# List of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups: -# Default list of storage volume groups. See mrlesmithjr.manage-lvm role for +# Default list of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups_default: -# Additional list of storage volume groups. See mrlesmithjr.manage-lvm role +# Additional list of storage volume groups. See mrlesmithjr.manage_lvm role # for format. #storage_lvm_groups_extra: @@ -87,7 +87,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #storage_lvm_group_data_enabled: -# Storage LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Storage LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_group_data: From ac809cbf077abaae8b5d544155adb03f1e60f3e0 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 19 Jan 2024 14:34:55 +0000 Subject: [PATCH 03/61] Skip docker registry login by default, login only when pulp is deployed --- etc/kayobe/containers/pulp/post.yml | 7 +++++++ .../environments/aufn-ceph/a-universe-from-nothing.sh | 2 +- etc/kayobe/seed.yml | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml index fec1abb94..967c4e37d 100644 --- a/etc/kayobe/containers/pulp/post.yml +++ b/etc/kayobe/containers/pulp/post.yml @@ -27,3 +27,10 @@ when: - stackhpc_pulp_sync_for_local_container_build | bool - pulp_settings.changed + +- name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh index 1464e6127..03cd23439 100755 --- a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh +++ b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh @@ -87,7 +87,7 @@ kayobe seed vm provision kayobe seed host configure # Deploy local pulp server as a container on the seed VM -kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none -e deploy_containers_registry_attempt_login=False +kayobe seed service deploy --tags seed-deploy-containers --kolla-tags # Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes. $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 96fa86ac0..524f6cff4 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -152,6 +152,10 @@ seed_containers: >- seed_extra_containers: {} +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +seed_deploy_containers_registry_attempt_login: "{{ not seed_pulp_container_enabled | bool }}" + ############################################################################### # Seed node firewalld configuration. From f70698b01429c7f13d1bca0b70f61bebad29f2c0 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 5 Feb 2024 16:06:31 +0000 Subject: [PATCH 04/61] Update .gitreview for unmaintained/yoga Change-Id: Ibd47c684580d35f58b0b715eab2e5110d17bb69a --- .gitreview | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitreview b/.gitreview index 9c0b64bdf..fa721c64d 100644 --- a/.gitreview +++ b/.gitreview @@ -2,4 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe-config.git -defaultbranch=stable/yoga +defaultbranch=unmaintained/yoga From e4f703673b4e5360be7e17dbc880d5d00dc67c60 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 8 Feb 2024 15:48:47 +0000 Subject: [PATCH 05/61] Override kolla_base_distro_version When running the kayobe-automation CI on a Zed/Antelope cloud running Ubuntu host images, the kolla tags to templated incorrectly. For example, a Jammy host gets ``kolla_base_distro_and_version = rocky-jammy``. This is caused by Kayobe's ``globals.yml`` template (``ansible/roles/kolla-ansible/templates/kolla/globals.yml``) setting ``kolla_base_distro_version`` based on Kayobe's variable of the sam name, which references ``os_version``. We can avoid this by explicitly using facts for the version in SKC's ``globals.yml`` template. --- etc/kayobe/kolla/globals.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index b156f050a..7930677f8 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,6 +10,10 @@ enable_docker_repo: "{% raw %}{{ 'overcloud' not in group_names }}{% endraw %}" # This is necessary for os migrations where mixed clouds might be deployed kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}" +# Use facts so this is determined correctly when the control host OS differs +# from os_distribuition. +kolla_base_distro_version: "{% raw %}{{ ansible_facts.distribution_major_version }}{% raw %}" + # Convenience variable for base distro and version string. kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}" From 3217a11baf32ecd6c634613fb336e287d04b3dc5 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Thu, 29 Feb 2024 11:07:42 +0000 Subject: [PATCH 06/61] Update etc/kayobe/kolla/globals.yml Co-authored-by: Mark Goddard --- etc/kayobe/kolla/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 7930677f8..91021deed 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,7 +12,7 @@ kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw % # Use facts so this is determined correctly when the control host OS differs # from os_distribuition. -kolla_base_distro_version: "{% raw %}{{ ansible_facts.distribution_major_version }}{% raw %}" +kolla_base_distro_version: "{% raw %}{{ ansible_facts.distribution_major_version }}{% endraw %}" # Convenience variable for base distro and version string. kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}" From b0743ed2c94d3a54d4071c9cc05033c991cedb03 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Fri, 1 Mar 2024 10:48:33 +0000 Subject: [PATCH 07/61] Correctly map kolla_base_distro_version --- etc/kayobe/kolla/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 91021deed..83bfd9b74 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,7 +12,7 @@ kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw % # Use facts so this is determined correctly when the control host OS differs # from os_distribuition. -kolla_base_distro_version: "{% raw %}{{ ansible_facts.distribution_major_version }}{% endraw %}" +kolla_base_distro_version: "{% raw %}{{ kolla_base_distro_version_default_map[kolla_base_distro] }}{% endraw %}" # Convenience variable for base distro and version string. kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}" From a3a8c101f9ef90bee23a9ea44402d1362b9e5d91 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 1 Mar 2024 11:42:22 +0000 Subject: [PATCH 08/61] Replace references to CentOS with Rocky Linux --- doc/source/contributor/environments/aufn-ceph.rst | 2 +- doc/source/contributor/environments/ci-builder.rst | 2 +- doc/source/operations/tempest.rst | 4 ++-- .../environments/aufn-ceph/configure-local-networking.sh | 2 +- etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/contributor/environments/aufn-ceph.rst b/doc/source/contributor/environments/aufn-ceph.rst index 5fe07b86f..9578efb48 100644 --- a/doc/source/contributor/environments/aufn-ceph.rst +++ b/doc/source/contributor/environments/aufn-ceph.rst @@ -14,7 +14,7 @@ This environment creates a Universe-from-nothing_-style deployment of Kayobe con Prerequisites ============= -* a baremetal node with at least 64GB of RAM running CentOS Stream 8 (or Ubuntu) +* a baremetal node with at least 64GB of RAM running Rocky Linux 9 or Ubuntu Jammy. * access to the test pulp server on SMS lab diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst index f0a6f0ee9..5cbc3371e 100644 --- a/doc/source/contributor/environments/ci-builder.rst +++ b/doc/source/contributor/environments/ci-builder.rst @@ -25,7 +25,7 @@ Access the host via SSH. Install package dependencies. -On CentOS: +On Rocky Linux: .. parsed-literal:: diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst index a3bd4ac1c..ea8503626 100644 --- a/doc/source/operations/tempest.rst +++ b/doc/source/operations/tempest.rst @@ -65,7 +65,7 @@ To install Docker on Ubuntu: sudo apt-get update sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -Installing Docker on CentOS/Rocky: +Installing Docker on Rocky: .. code-block:: bash @@ -99,7 +99,7 @@ Build a Kayobe automation image: git submodule init git submodule update - # If running on Ubuntu, the fact cache can confuse Kayobe in the CentOS-based container + # If running on Ubuntu, the fact cache can confuse Kayobe in the Rocky-based container mv etc/kayobe/facts{,-old} sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . diff --git a/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh b/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh index ab3602d2a..c22bbd518 100755 --- a/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh +++ b/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh @@ -43,7 +43,7 @@ if ! sudo ip l show brcloud >/dev/null 2>&1; then sudo ip l set brcloud up fi -# On CentOS 8, bridges without a port are DOWN, which causes network +# On Rocky Linux, bridges without a port are DOWN, which causes network # configuration to fail. Add a dummy interface and plug it into the bridge. for i in mgmt prov cloud; do if ! sudo ip l show dummy-$i >/dev/null 2>&1; then diff --git a/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml b/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml index 6a1b7ffdf..2f288f030 100644 --- a/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml +++ b/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml @@ -10,5 +10,5 @@ seed_hypervisor_extra_network_interfaces: - "{{ public_net_name }}" - "{{ external_net_names[0] }}" -# Workaround change to cloud-user default login name on CentOS-Stream8 +# Workaround change to cloud-user default login name on Rocky Linux seed_hypervisor_bootstrap_user: "{{ lookup('env', 'USER') }}" From b83cec2ba3cd27ccf6467da1cb34194af4cf9c68 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 1 Mar 2024 11:43:22 +0000 Subject: [PATCH 09/61] docs: Add BASE_IMAGE build-arg for kayobe image build A Rocky Linux 9 base image is required for Zed onwards. --- doc/source/operations/tempest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst index ea8503626..a5991097c 100644 --- a/doc/source/operations/tempest.rst +++ b/doc/source/operations/tempest.rst @@ -101,7 +101,7 @@ Build a Kayobe automation image: git submodule update # If running on Ubuntu, the fact cache can confuse Kayobe in the Rocky-based container mv etc/kayobe/facts{,-old} - sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . + sudo DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=rockylinux:9 --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . Configuration ============= From 100f544225fdbe48802f77de569d7d127cb865f4 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 5 Mar 2024 15:35:03 +0000 Subject: [PATCH 10/61] Fix libvirt error for tenks on Rocky Linux 9 --- etc/kayobe/environments/aufn-ceph/tenks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/environments/aufn-ceph/tenks.yml b/etc/kayobe/environments/aufn-ceph/tenks.yml index 9b0e9e9f4..25eac0374 100644 --- a/etc/kayobe/environments/aufn-ceph/tenks.yml +++ b/etc/kayobe/environments/aufn-ceph/tenks.yml @@ -87,3 +87,9 @@ bridge_type: linuxbridge # No placement service. wait_for_placement: false + +# NOTE(priteau): Disable libvirt_vm_trust_guest_rx_filters, which when enabled +# triggers the following errors when booting baremetal instances with Tenks on +# Libvirt 9: Cannot set interface flags on 'macvtap1': Value too large for +# defined data type +libvirt_vm_trust_guest_rx_filters: false From 4a20e1149ccb0cea2d7a50f73bb5db2eb2afd441 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Wed, 6 Mar 2024 09:26:40 +0000 Subject: [PATCH 11/61] Update etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh --- etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh index 03cd23439..e594ea388 100755 --- a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh +++ b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh @@ -87,7 +87,7 @@ kayobe seed vm provision kayobe seed host configure # Deploy local pulp server as a container on the seed VM -kayobe seed service deploy --tags seed-deploy-containers --kolla-tags +kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none # Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes. $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh From be1b504c68feee30019e73b7f705cb3d1203afe3 Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Wed, 6 Mar 2024 11:35:14 +0100 Subject: [PATCH 12/61] Switch ansible-modules-hashivault back to upstream 5.2.1 version was released with shebang fix [1]. [1]: TerryHowe/ansible-modules-hashivault@f1d30f1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2089c4b3f..ba1a14f12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/yoga ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc;python_version < "3.8" -ansible-modules-hashivault@git+https://github.com/stackhpc/ansible-modules-hashivault@stackhpc-py39;python_version >= "3.8" +ansible-modules-hashivault>=5.2.1;python_version >= "3.8" jmespath From 1455b28f9c9670268ec00c677a9ef239daad809c Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:09:47 +0000 Subject: [PATCH 13/61] Update upgrading docs to include Opensearch issue Adding the known issue with Opensearch to the upgrading docs from the 2023.1 docs. --- doc/source/operations/upgrading.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst index 506e966c5..53df5aef2 100644 --- a/doc/source/operations/upgrading.rst +++ b/doc/source/operations/upgrading.rst @@ -106,6 +106,24 @@ Known issues * The OVN sync repair tool removes metadata ports, breaking OVN load balancers. See `LP#2038091 `__. +* If you run ``kayobe overcloud service upgrade`` twice, it will cause shard + allocation to be disabled in OpenSearch. See `LP#2049512 + `__ for details. + + You can check if this is affecting your system with the following command. If + ``transient.cluster.routing.allocation.enable=none`` is present, shard + allocation is disabled. + + .. code-block:: console + + curl http://:9200/_cluster/settings + + For now, the easiest way to fix this is to turn allocation back on: + + .. code-block:: console + + curl -X PUT http://:9200/_cluster/settings -H 'Content-Type:application/json' -d '{"transient":{"cluster":{"routing":{"allocation":{"enable":"all"}}}}}' + Security baseline ================= From 329197abbbf0a30d52de45a90224b6947a09a528 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 6 Mar 2024 15:32:25 +0000 Subject: [PATCH 14/61] Add Nova Compute Ironic failover procedure Document: - Moving from multiple instances to a single instance - How to re-deploy the service Config changes: - Prompt users to set a static nova-compute-ironic 'host' name. --- doc/source/operations/index.rst | 7 +- doc/source/operations/nova-compute-ironic.rst | 211 ++++++++++++++++++ .../config/nova/nova-compute-ironic.conf | 4 + ...-ironic-failover-doc-a0c4f45b1fb48c4a.yaml | 12 + 4 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 doc/source/operations/nova-compute-ironic.rst create mode 100644 etc/kayobe/kolla/config/nova/nova-compute-ironic.conf create mode 100644 releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst index 38acb60ff..39f1bb847 100644 --- a/doc/source/operations/index.rst +++ b/doc/source/operations/index.rst @@ -7,9 +7,10 @@ This guide is for operators of the StackHPC Kayobe configuration project. .. toctree:: :maxdepth: 1 - upgrading - rabbitmq - octavia hotfix-playbook + nova-compute-ironic + octavia + rabbitmq secret-rotation tempest + upgrading diff --git a/doc/source/operations/nova-compute-ironic.rst b/doc/source/operations/nova-compute-ironic.rst new file mode 100644 index 000000000..e139fa050 --- /dev/null +++ b/doc/source/operations/nova-compute-ironic.rst @@ -0,0 +1,211 @@ +=================== +Nova Compute Ironic +=================== + +This section describes the deployment of the OpenStack Nova Compute +Ironic service. The Nova Compute Ironic service is used to integrate +OpenStack Ironic into Nova as a 'hypervisor' driver. The end users of Nova +can then deploy and manage baremetal hardware, in a similar way to VMs. + +High Availability (HA) +====================== + +The OpenStack Nova Compute service is designed to be installed once on every +hypervisor in an OpenStack deployment. In this configuration, it makes little +sense to run additional service instances. Even if you wanted to, it's not +supported by design. This pattern breaks down with the Ironic baremetal +service, which must run on the OpenStack control plane. It is not feasible +to have a 1:1 mapping of Nova Compute Ironic services to baremetal nodes. + +The obvious HA solution is to run multiple instances of Nova Compute Ironic +on the control plane, so that if one fails, the others can take over. However, +due to assumptions long baked into the Nova source code, this is not trivial. +The HA feature provided by the Nova Compute Ironic service has proven to be +unstable, and the direction upstream is to switch to an active/passive +solution [1]. + +However, challenges still exist with the active/passive solution. Since the +Nova Compute Ironic HA feature is 'always on', one must ensure that only a +single instance (per Ironic conductor group) is ever running. It is not +possible to simply put multiple service instances behind HAProxy and use the +active/passive mode. + +Such problems are commonly solved with a technology such as Pacemaker, or in +the modern world, with a container orchestration engine such as Kubernetes. +Kolla Ansible provides neither, because in general it doesn't need to. Its +goal is simplicity. + +The interim solution is to therefore run a single Nova Compute Ironic +service. If the service goes down, remedial action must be taken before +Ironic nodes can be managed. In many environments the loss of the Ironic +API for short periods is acceptable, providing that it can be easily +resurrected. The purpose of this document is to faciliate that. + +TODO: Add caveats about new sharding mode (not covered here). + +Optimal configuration of Nova Compute Ironic +============================================ + +Determine the current configuration for the site. How many Nova Compute +Ironic instances are running on the control plane? + +.. code-block:: console + + $ openstack compute service list + +Typically you will see either three or one. By default the host will +marked with a postfix, eg. ``controller1-ironic``. If you find more than +one, you will need to remove some instances. You must complete the +following section. + +Moving from multiple Nova Compute Instances to a single instance +---------------------------------------------------------------- + +1. Decide where the single instance should run. Typically, this will be + one of the three control plane hosts. Once you have chosen, set + the following variable in ``etc/kayobe/nova.yml``. Here we have + picked ``controller1``. + + .. code-block:: console + + kolla_nova_compute_ironic_host: controller1 + +2. Ensure that you have organised a maintenance window, during which + there will be no Ironic operations. You will be breaking the Ironic + API. + +3. Perform a database backup. + + .. code-block:: console + + $ kayobe overcloud database backup -vvv + + Check the output of the command, and locate the backup files. + +4. Identify baremetal nodes associated with Nova Compute Ironic instances + that will be removed. You don't need to do anything with these + specifically, it's just for reference later. For example: + + .. code-block:: console + + $ openstack baremetal node list --long -c "Instance Info" | grep controller3-ironic | wc -l + 61 + $ openstack baremetal node list --long -c "Instance Info" | grep controller2-ironic | wc -l + 35 + $ openstack baremetal node list --long -c "Instance Info" | grep controller1-ironic | wc -l + 55 + +5. Disable the redundant Nova Compute Ironic services: + + .. code-block:: console + + $ openstack compute service set controller3-ironic nova-compute --disable + $ openstack compute service set controller2-ironic nova-compute --disable + +6. Delete the redundant Nova Compute Ironic services. You will need the service + ID. For example: + + .. code-block:: console + + $ ID=$(openstack compute service list | grep foo | awk '{print $2}') + $ openstack compute service delete --os-compute-api-version 2.53 $ID + + In older releases, you may hit a bug where the service can't be deleted if it + is not managing any instances. In this case just move on and leave the service + disabled. Eg. + + .. code-block:: console + + $ openstack compute service delete --os-compute-api-version 2.53 c993b57e-f60c-4652-8328-5fb0e17c99c0 + Failed to delete compute service with ID 'c993b57e-f60c-4652-8328-5fb0e17c99c0': HttpException: 500: Server Error for url: + https://acme.pl-2.internal.hpc.is:8774/v2.1/os-services/c993b57e-f60c-4652-8328-5fb0e17c99c0, Unexpected API Error. + Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible. + +7. Remove the Docker containers for the redundant Nova Compute Ironic services: + + .. code-block:: console + + $ ssh controller2 sudo docker rm -f nova_compute_ironic + $ ssh controller3 sudo docker rm -f nova_compute_ironic + +8. Ensure that all Ironic nodes are using the single remaining Nova Compute + Ironic instance. Eg. Baremetal nodes in use by compute instances will not + fail over to the remaining Nova Compute Ironic service. Here, the active + service is running on ``controller1``: + + .. code-block:: console + + $ ssh controller1 + $ sudo docker exec -it mariadb mysql -u nova -p$(sudo grep 'mysql+pymysql://nova:' /etc/kolla/nova-api/nova.conf | awk -F'[:,@]' '{print $3}') + $ MariaDB [(none)]> use nova; + + Proceed with caution. It is good practise to update one record first: + + .. code-block:: console + + $ MariaDB [nova]> update instances set host='controller1-ironic' where uuid=0 and host='controller3-ironic' limit 1; + Query OK, 1 row affected (0.002 sec) + Rows matched: 1 Changed: 1 Warnings: 0 + + At this stage you should go back to step 4 and check that the numbers have + changed as expected. When you are happy, update remaining records for all + services which have been removed: + + .. code-block:: console + + $ MariaDB [nova]> update instances set host='controller1-ironic' where deleted=0 and host='controller3-ironic'; + Query OK, 59 rows affected (0.009 sec) + Rows matched: 59 Changed: 59 Warnings: 0 + $ MariaDB [nova]> update instances set host='controller1-ironic' where deleted=0 and host='controller2-ironic'; + Query OK, 35 rows affected (0.003 sec) + Rows matched: 35 Changed: 35 Warnings: 0 + +9. Repeat step 4. Verify that all Ironic nodes are using the single remaining + Nova Compute Ironic instance. + + +Making it easy to re-deploy Nova Compute Ironic +----------------------------------------------- + +In the previous section we saw that at any given time, a baremetal node is +associated with a single Nova Compute Ironic instance. At this stage, assuming +that you have diligently followed the instructions, you are in the situation +where all Ironic baremetal nodes are managed by a single Nova Compute Ironic +instance. If this service goes down, you will not be able to manage /any/ +baremetal nodes. + +By default, the single remaining Nova Compute Ironic instance will be named +after the host on which it is deployed. The host name is passed to the Nova +Compute Ironic instance via the default section of the ``nova.conf`` file, +using the field: ``host``. + +If you wish to re-deploy this instance, for example because the original host +was permanently mangled in the World Server Throwing Championship [2], you +must ensure that the new instance has the same name as the old one. Simply +setting ``kolla_nova_compute_ironic_host`` to another controller and +re-deploying the service is not enough; the new instance will be named after +the new host. + +To work around this you should set the ``host`` field in ``nova.conf`` to a +constant, such that the new Nova Compute Ironic instance comes up with the +same name as the one it replaces. + +For example, if the original instance resides on ``controller1``, then set the +following in ``etc/kayobe/nova.yml``: + +.. code-block:: console + + kolla_nova_compute_ironic_static_host_name: controller1-ironic + +Note that an ``-ironic`` postfix is added to the hostname. This comes from +a convention in Kolla Ansible. It is worth making this change ahead of time, +even if you don't need to immediately re-deploy the service. + +It is also possible to use an arbitrary ``host`` name, but you will need +to edit the database again. That is an optional exercise left for the reader. +See [1] for further details. + +TODO: Investigate KA bug with assumption about host field. + +[1] https://specs.openstack.org/openstack/nova-specs/specs/2024.1/approved/ironic-shards.html#migrate-from-peer-list-to-shard-key +[2] https://www.cloudfest.com/world-server-throwing-championship diff --git a/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf b/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf new file mode 100644 index 000000000..9f6db7a55 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf @@ -0,0 +1,4 @@ +{% if kolla_enable_ironic|bool and kolla_nova_compute_ironic_host is not none %} +[DEFAULT] +host = {{ kolla_nova_compute_ironic_static_host_name | mandatory('You must set a static host name to help with service failover. See the operations documentation, Ironic section.') }} +{% endif %} diff --git a/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml b/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml new file mode 100644 index 000000000..c5b52984f --- /dev/null +++ b/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Adds basic support and a document explaining how to migrate to a single + nova-compute-ironic instance, and how to re-deploy the instance to another + machine in the event of failure. See the operations / nova-compute-ironic + doc for further details. +upgrade: + - | + Ensure that your deployment has only one nova-compute-ironic service running + per conductor group. See the operations / nova-compute-ironic doc for further + details. From ce924149b618cb05fea0b06c97c23cda9f2a9a2c Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Fri, 8 Dec 2023 16:46:50 +0000 Subject: [PATCH 15/61] Post service deploy hook for OpenStack Capacity --- doc/source/configuration/monitoring.rst | 51 +++++++------------ .../ansible/deploy-os-capacity-exporter.yml | 22 +++++++- .../templates/os_capacity-clouds.yml.j2 | 8 +-- .../post.d/deploy-os-capacity-exporter.yml | 1 + etc/kayobe/stackhpc-monitoring.yml | 8 +-- ...capacity-deploy-hook-b52e87c0819df6fd.yaml | 9 ++++ 6 files changed, 54 insertions(+), 45 deletions(-) create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml create mode 100644 releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 819da9769..f23c7a915 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -141,36 +141,26 @@ OpenStack Capacity ================== OpenStack Capacity allows you to see how much space you have available -in your cloud. StackHPC Kayobe Config includes a playbook for manual -deployment, and it's necessary that some variables are set before -running this playbook. +in your cloud. StackHPC Kayobe Config will deploy OpenStack Capacity +by default on a service deploy, this can be disabled by setting +``stackhpc_enable_os_capacity`` to false. -To successfully deploy OpenStack Capacity, you are required to specify -the OpenStack application credentials in ``kayobe/secrets.yml`` as: +OpenStack Capacity is deployed automatically using a service deploy hook +with the generated kolla-ansible admin credentials, you can override these +by setting the authentication url, username, password, project name and +project domain name in ``stackhpc-monitoring.yml``: .. code-block:: yaml - secrets_os_capacity_credential_id: - secrets_os_capacity_credential_secret: + stackhpc_os_capacity_auth_url: + stackhpc_os_capacity_username: + stackhpc_os_capacity_password: + stackhpc_os_capacity_project_name: + stackhpc_os_capacity_domain_name: + stackhpc_os_capacity_openstack_region_name: -The Keystone authentication URL and OpenStack region can be changed -from their defaults in ``stackhpc-monitoring.yml`` should you need to -set a different OpenStack region for your cloud. The authentication -URL is set to use ``kolla_internal_fqdn`` by default: - -.. code-block:: yaml - - stackhpc_os_capacity_auth_url: - stackhpc_os_capacity_openstack_region_name: - -Additionally, you are required to enable a conditional flag to allow -HAProxy and Prometheus configuration to be templated during deployment. - -.. code-block:: yaml - - stackhpc_enable_os_capacity: true - -If you are deploying in a cloud with internal TLS, you may be required +Additionally, you should ensure these credentials have the correct permissions +for the exporter. If you are deploying in a cloud with internal TLS, you may be required to disable certificate verification for the OpenStack Capacity exporter if your certificate is not signed by a trusted CA. @@ -178,21 +168,14 @@ if your certificate is not signed by a trusted CA. stackhpc_os_capacity_openstack_verify: false -After defining your credentials, you may deploy OpenStack Capacity -using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook +If you've modified your credentials, you will need to re-deploy OpenStack Capacity +using the ``deploy-os-capacity-exporter.yml`` Ansible playbook via Kayobe. .. code-block:: console kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml -It is required that you re-configure the Prometheus, Grafana and HAProxy -services following deployment, to do this run the following Kayobe command. - -.. code-block:: console - - kayobe overcloud service reconfigure -kt grafana,prometheus,loadbalancer - If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus alerts after deployment it's likely the os_exporter secrets have not been set correctly, double check you have entered the correct authentication diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml index 8cff6a89e..978c13e62 100644 --- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -17,14 +17,33 @@ ansible.builtin.file: path: /opt/kayobe/os-capacity/ state: directory + when: stackhpc_enable_os_capacity + + - name: Read admin-openrc credential file + ansible.builtin.command: + cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh" + delegate_to: localhost + register: credential + when: stackhpc_enable_os_capacity + + - name: Set facts for admin credentials + ansible.builtin.set_fact: + stackhpc_os_capacity_auth_url: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_project_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_domain_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_openstack_region_name: "{{ credential.stdout_lines | select('match', '.*OS_REGION_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_username: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_password: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\",'') }}" + when: stackhpc_enable_os_capacity - name: Template clouds.yml ansible.builtin.template: src: templates/os_capacity-clouds.yml.j2 dest: /opt/kayobe/os-capacity/clouds.yaml + when: stackhpc_enable_os_capacity - name: Ensure os_capacity container is running - docker_container: + community.docker.docker_container: name: os_capacity image: ghcr.io/stackhpc/os-capacity:master env: @@ -37,3 +56,4 @@ network_mode: host restart_policy: unless-stopped become: true + when: stackhpc_enable_os_capacity diff --git a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 index a821d6dcb..ef3c8d7a5 100644 --- a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 +++ b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 @@ -2,12 +2,14 @@ clouds: openstack: auth: auth_url: "{{ stackhpc_os_capacity_auth_url }}" - application_credential_id: "{{ secrets_os_capacity_credential_id }}" - application_credential_secret: "{{ secrets_os_capacity_credential_secret }}" + project_name: "{{ stackhpc_os_capacity_project_name }}" + domain_name: "{{ stackhpc_os_capacity_domain_name }}" + username: "{{ stackhpc_os_capacity_username }}" + password: "{{ stackhpc_os_capacity_password }}" region_name: "{{ stackhpc_os_capacity_openstack_region_name }}" interface: "internal" identity_api_version: 3 - auth_type: "v3applicationcredential" + auth_type: "password" {% if not stackhpc_os_capacity_openstack_verify | bool %} verify: False {% endif %} diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml new file mode 120000 index 000000000..0cc70aace --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-os-capacity-exporter.yml \ No newline at end of file diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index 13bf6ba0f..f08e552c3 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -14,13 +14,7 @@ alertmanager_low_memory_threshold_gib: 5 # Whether the OpenStack Capacity exporter is enabled. # Enabling this flag will result in HAProxy configuration and Prometheus scrape # targets being templated during deployment. -stackhpc_enable_os_capacity: false - -# Keystone authentication URL for OpenStack Capacity -stackhpc_os_capacity_auth_url: "http{% if kolla_enable_tls_internal | bool %}s{% endif %}://{{ kolla_internal_fqdn }}:5000" - -# OpenStack region for OpenStack Capacity -stackhpc_os_capacity_openstack_region_name: "RegionOne" +stackhpc_enable_os_capacity: true # Whether TLS certificate verification is enabled for the OpenStack Capacity # exporter during Keystone authentication. diff --git a/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml new file mode 100644 index 000000000..547939199 --- /dev/null +++ b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Automatic deployment for OpenStack Capacity via a Kayobe service + deploy hook using kolla admin credentials. +upgrade: + - | + OpenStack Capacity no longer uses application credentials. Please + delete any previously generated application credentials. \ No newline at end of file From a8cefd05ee2a0e30d752bad5e55391df0a39cdd0 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 8 Mar 2024 11:05:04 +0000 Subject: [PATCH 16/61] squash: Address comments from Alex --- doc/source/operations/nova-compute-ironic.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/source/operations/nova-compute-ironic.rst b/doc/source/operations/nova-compute-ironic.rst index e139fa050..cbed4b753 100644 --- a/doc/source/operations/nova-compute-ironic.rst +++ b/doc/source/operations/nova-compute-ironic.rst @@ -61,12 +61,13 @@ following section. Moving from multiple Nova Compute Instances to a single instance ---------------------------------------------------------------- -1. Decide where the single instance should run. Typically, this will be - one of the three control plane hosts. Once you have chosen, set - the following variable in ``etc/kayobe/nova.yml``. Here we have - picked ``controller1``. +1. Decide where the single instance should run. This should normally be + one of the three OpenStack control plane hosts. For convention, pick + the first one, unless you can think of a good reason not to. Once you + have chosen, set the following variable in ``etc/kayobe/nova.yml``. + Here we have picked ``controller1``. - .. code-block:: console + .. code-block:: yaml kolla_nova_compute_ironic_host: controller1 @@ -193,7 +194,7 @@ same name as the one it replaces. For example, if the original instance resides on ``controller1``, then set the following in ``etc/kayobe/nova.yml``: -.. code-block:: console +.. code-block:: yaml kolla_nova_compute_ironic_static_host_name: controller1-ironic From 417d7acfbe20e2e960d191c978a59870b7d88cc5 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 8 Mar 2024 13:18:13 +0000 Subject: [PATCH 17/61] Expand notes on re-deploying --- doc/source/operations/nova-compute-ironic.rst | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/doc/source/operations/nova-compute-ironic.rst b/doc/source/operations/nova-compute-ironic.rst index cbed4b753..908247678 100644 --- a/doc/source/operations/nova-compute-ironic.rst +++ b/doc/source/operations/nova-compute-ironic.rst @@ -41,7 +41,11 @@ Ironic nodes can be managed. In many environments the loss of the Ironic API for short periods is acceptable, providing that it can be easily resurrected. The purpose of this document is to faciliate that. -TODO: Add caveats about new sharding mode (not covered here). +.. note:: + + The new sharding mode is not covered here and it is assumed that you are + not using it. See [1] for further information. This will be updated in + the future. Optimal configuration of Nova Compute Ironic ============================================ @@ -208,5 +212,90 @@ See [1] for further details. TODO: Investigate KA bug with assumption about host field. +Re-deploying Nova Compute Ironic +-------------------------------- + +The decision to re-deploy Nova Compute Ironic to another host should only be +taken if there is a strong reason to do so. The objective is to minimise +the chance of the old instance starting up alongside the new one. If the +original host has been re-imaged, or physically replaced there is no risk. +However, if the original host has been taken down for non-destructive +maintenance, it is better to avoid re-deploying the service if the end users +can tolerate the wait. If you are forced to re-deploy the service, knowing +that the original instance may start when the host comes back online, you +must plan accordingly. For example, by booting the original host in maintenance +mode and removing the old service before it can start, or by stopping the +new instance before the original one comes back up, and then reverting the +config to move it to the new host. + +There are essentially two scenarios for re-deploying Nova Compute Ironic. +These are described in the following sub-sections: + +Current host is accessible +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Adjust the ``kolla_nova_compute_ironic_host`` variable to point to the +new host, eg. + +.. code-block:: diff + + +kolla_nova_compute_ironic_host: controller2 + -kolla_nova_compute_ironic_host: controller1 + +Remove the old container: + +.. code-block:: console + + $ ssh controller1 sudo docker rm -f nova_compute_ironic + +Deploy the new service: + +.. code-block:: console + + $ kayobe overcloud service deploy -kl controller2 -l controller2 -kt nova + +Verify that the new service appears as 'up' and 'enabled': + +.. code-block:: console + + $ openstack compute service list + +Current host is not accessible +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this case you will need to remove the inaccessible host from the inventory. +For example, in ``etc/kayobe/inventory/hosts``, remove ``controller1`` from +the ``controllers`` group. + +Adjust the ``kolla_nova_compute_ironic_host`` variable to point to the +new host, eg. + +.. code-block:: diff + + +kolla_nova_compute_ironic_host: controller2 + -kolla_nova_compute_ironic_host: controller1 + +Deploy the new service: + +.. code-block:: console + + $ kayobe overcloud service reconfigure -kl controller2 -l controller2 -kt nova + +Verify that the new service appears as 'up' and 'enabled': + +.. code-block:: console + + $ openstack compute service list + +.. note:: + + It is important to stop the original service from starting up again. It is + up to you to prevent this. + +.. note:: + + Once merged, the work on 'Kayobe reliability' may allow this step to run + without modifying the inventory to remove the broken host. + [1] https://specs.openstack.org/openstack/nova-specs/specs/2024.1/approved/ironic-shards.html#migrate-from-peer-list-to-shard-key [2] https://www.cloudfest.com/world-server-throwing-championship From fcfff10a8605bdedd8980a7dc7add7ad0be36c3f Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Fri, 8 Mar 2024 14:16:45 +0000 Subject: [PATCH 18/61] Add Trivy image scanning (#436) Trivy scanning on container image build --------- Co-authored-by: k-s-dean Co-authored-by: Matt Anson Co-authored-by: Alex-Welsh --- .../stackhpc-container-image-build.yml | 131 ++++++++++++++---- etc/kayobe/ansible/docker-registry-login.yml | 11 ++ ...ainer-image-scanning-e5adf2c6b540b502.yaml | 6 + tools/scan-images.sh | 79 +++++++++++ 4 files changed, 201 insertions(+), 26 deletions(-) create mode 100644 etc/kayobe/ansible/docker-registry-login.yml create mode 100644 releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml create mode 100755 tools/scan-images.sh diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index b8afea93e..ad3097d0a 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -38,6 +38,12 @@ on: type: boolean required: false default: true + push-dirty: + description: Push scanned images that have vulnerabilities? + type: boolean + required: false + # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures + default: true env: ANSIBLE_FORCE_COLOR: True @@ -109,7 +115,15 @@ jobs: - name: Install package dependencies run: | sudo apt update - sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget + + - name: Install gh + run: | + sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null + sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh -y - name: Checkout uses: actions/checkout@v4 @@ -127,6 +141,10 @@ jobs: run: | docker ps + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0 + - name: Install Kayobe run: | mkdir -p venvs && @@ -162,65 +180,124 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - - name: Build and push kolla overcloud images + - name: Create build logs output directory + run: mkdir image-build-logs + + - name: Build kolla overcloud images + id: build_overcloud_images + continue-on-error: true run: | - args="${{ github.event.inputs.regexes }}" + args="${{ inputs.regexes }}" args="$args -e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe overcloud container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: github.event.inputs.overcloud == 'true' + if: inputs.overcloud + + - name: Copy overcloud container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log + if: inputs.overcloud - - name: Build and push kolla seed images + - name: Build kolla seed images + id: build_seed_images + continue-on-error: true run: | args="-e kolla_base_distro=${{ matrix.distro }}" args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}" args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" - if ${{ inputs.push }} == 'true'; then - args="$args --push" - fi source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe seed container image build $args env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - if: github.event.inputs.seed == 'true' + if: inputs.seed + + - name: Copy seed container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log + if: inputs.seed - name: Get built container images - run: | - docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images - name: Fail if no images have been built run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi - - name: Upload container images artifact + - name: Scan built container images + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }} + + - name: Move image scan logs to output artifact + run: mv image-scan-output image-build-logs/image-scan-output + + - name: Fail if no images have passed scanning + run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi + if: ${{ !inputs.push-dirty }} + + - name: Copy clean images to push-attempt-images list + run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt + if: inputs.push + + - name: Append dirty images to push list + run: | + cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push && inputs.push-dirty }} + + - name: Push images + run: | + touch image-build-logs/push-failed-images.txt + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml && + + while read -r image; do + # Retries! + for i in {1..5}; do + if docker push $image; then + echo "Pushed $image" + break + elif $i == 5; then + echo "Failed to push $image" + echo $image >> image-build-logs/push-failed-images.txt + else + echo "Failed on retry $i" + sleep 5 + fi; + done + done < image-build-logs/push-attempt-images.txt + shell: bash + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.push + + - name: Upload output artifact uses: actions/upload-artifact@v4 with: - name: ${{ matrix.distro }} container images - path: ${{ matrix.distro }}-container-images + name: ${{ matrix.distro }}-logs + path: image-build-logs retention-days: 7 + if: ${{ !cancelled() }} + + - name: Fail when images failed to build + run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1 + if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} + + - name: Fail when images failed to push + run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi + if: ${{ !cancelled() }} + + - name: Fail when images failed scanning + run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi + if: ${{ !inputs.push-dirty && !cancelled() }} - sync-container-repositories: - name: Trigger container image repository sync - needs: - - container-image-build - if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push - runs-on: ubuntu-latest - permissions: {} - steps: # NOTE(mgoddard): Trigger another CI workflow in the # stackhpc-release-train repository. - name: Trigger container image repository sync run: | filter='${{ inputs.regexes }}' - if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then + if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then filter="$filter bifrost" fi gh workflow run \ @@ -231,7 +308,9 @@ jobs: -f sync-new-images=false env: GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} - name: Display link to container image repository sync workflows run: | echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml" + if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }} diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml new file mode 100644 index 000000000..39ad03600 --- /dev/null +++ b/etc/kayobe/ansible/docker-registry-login.yml @@ -0,0 +1,11 @@ +--- +- name: Login to docker registry + gather_facts: false + hosts: container-image-builders + tasks: + - name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml new file mode 100644 index 000000000..67a99f9c2 --- /dev/null +++ b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Kolla container images created using the + ``stackhpc-container-image-build.yml`` workflow are now automatically + scanned for vulnerablilities. diff --git a/tools/scan-images.sh b/tools/scan-images.sh new file mode 100755 index 000000000..50a04185a --- /dev/null +++ b/tools/scan-images.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Check correct usage +if [[ ! $2 ]]; then + echo "Usage: scan-images.sh " + exit 2 +fi + +set -u + +# Check that trivy is installed +if ! trivy --version; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1' +fi + +# Clear any previous outputs +rm -rf image-scan-output + +# Make a fresh output directory +mkdir -p image-scan-output + +# Get built container images +docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-scanned-container-images.txt + +# Make a file of imagename:tag +images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:) + +# Ensure output files exist +touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt + +# If Trivy detects no vulnerabilities, add the image name to clean-images.txt. +# If there are vulnerabilities detected, add it to dirty-images.txt and +# generate a csv summary +for image in $images; do + filename=$(basename $image | sed 's/:/\./g') + if $(trivy image \ + --quiet \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --output image-scan-output/${filename}.json \ + --ignore-unfixed \ + $image); then + # Clean up the output file for any images with no vulnerabilities + rm -f image-scan-output/${filename}.json + + # Add the image to the clean list + echo "${image}" >> image-scan-output/clean-images.txt + else + # Add the image to the dirty list + echo "${image}" >> image-scan-output/dirty-images.txt + + # Write a header for the summary CSV + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv + + # Write the summary CSV data + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + # Ignore packages with "kernel" in the PkgName + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv + fi +done From 2ec32ceaf155d1876ed15e7849386d44c373683b Mon Sep 17 00:00:00 2001 From: scrungus Date: Fri, 8 Mar 2024 15:39:16 +0000 Subject: [PATCH 19/61] bump magnum-capi-helm version --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 8844a3cbd..ebf8b0929 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -415,7 +415,7 @@ kolla_build_blocks: magnum_base_footer: | RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | head -n -1 | bash {% raw %} - {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.10.0'] %} + {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.11.0'] %} RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }} {% endraw %} # Dict mapping image customization variable names to their values. From ee1aa833a13977a159c5a6a12a062ee38868ad76 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Fri, 8 Mar 2024 15:41:04 +0000 Subject: [PATCH 20/61] Add note about upstream bug --- doc/source/operations/nova-compute-ironic.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/operations/nova-compute-ironic.rst b/doc/source/operations/nova-compute-ironic.rst index 908247678..6cbe00550 100644 --- a/doc/source/operations/nova-compute-ironic.rst +++ b/doc/source/operations/nova-compute-ironic.rst @@ -210,7 +210,13 @@ It is also possible to use an arbitrary ``host`` name, but you will need to edit the database again. That is an optional exercise left for the reader. See [1] for further details. -TODO: Investigate KA bug with assumption about host field. +.. note:: + + There is a bug when overriding the host name in Kolla Ansible, where it + is currently assumed that it will be set to the actual hostname + an + -ironic postfix. The service will come up correctly, but Kolla Ansible + will not detect it. See here: + https://bugs.launchpad.net/kolla-ansible/+bug/2056571 Re-deploying Nova Compute Ironic -------------------------------- From 8d7077f286dfe37b540e1f9cd454677f44b30ac5 Mon Sep 17 00:00:00 2001 From: scrungus Date: Fri, 8 Mar 2024 15:43:41 +0000 Subject: [PATCH 21/61] reno --- .../notes/bump-magnum-capi-helm-6723d89456e6a590.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml new file mode 100644 index 000000000..864edf44b --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Magnum CAPI Helm driver version to v0.11.0 From 2ae28e016a8b2c3614feb6728d3795b95ae1481e Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 8 Mar 2024 21:31:35 +0100 Subject: [PATCH 22/61] Fix Ceph "Objects in the Cluster" dashboard panel The `ceph_cluster_total_objects` metric was removed several releases ago [1]. Use `ceph_pool_objects` which provides per-pool metrics. [1] https://github.com/ceph/ceph-ansible/issues/6032 --- .../grafana/dashboards/ceph/ceph_overview.json | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json index e041d8ff0..e5258168a 100644 --- a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json @@ -1924,23 +1924,25 @@ } ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "ceph_cluster_total_objects", + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pool_objects * on(pool_id) group_left(instance,name) ceph_pool_metadata", "format": "time_series", "interval": "$interval", "intervalFactor": 1, - "legendFormat": "Total", + "legendFormat": "{{name}}", + "range": true, "refId": "A", "step": 300 } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Objects in the Cluster", "tooltip": { "msResolution": false, From 1e33f3da156ce7e1c2e586abccc8e4fe7555f235 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 11 Mar 2024 09:11:06 +0000 Subject: [PATCH 23/61] Fix tempest doc long line --- doc/source/operations/tempest.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst index 82135adf9..b3fa2b5a8 100644 --- a/doc/source/operations/tempest.rst +++ b/doc/source/operations/tempest.rst @@ -277,7 +277,10 @@ command from the base of the ``kayobe-config`` directory: .. code-block:: bash - sudo -E docker run --detach -it --rm --network host -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + sudo -E docker run --name kayobe-automation --detach -it --rm --network host \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack By default, ``no_log`` is set to stop credentials from leaking. This can be disabled by adding ``-e rally_no_sensitive_log=false`` to the end. From e989f4ffb1b38bc491ed605c88f9ae8aa6a6b40f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 11 Mar 2024 09:44:36 +0000 Subject: [PATCH 24/61] CI: Support unmaintained branches in release determination --- .github/workflows/overcloud-host-image-build.yml | 2 +- .github/workflows/overcloud-host-image-promote.yml | 2 +- .github/workflows/overcloud-host-image-upload.yml | 2 +- .github/workflows/stackhpc-ci-cleanup.yml | 2 +- .github/workflows/stackhpc-container-image-build.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index cbccca23e..4c338cda3 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -50,7 +50,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT # Generate a tag to apply to all built overcloud host images. - name: Generate overcloud host image tag diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index 1bd777c8c..d5625f888 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -43,7 +43,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT working-directory: src/kayobe-config - name: Clone StackHPC Kayobe repository diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index 633f423b5..e95531564 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -59,7 +59,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index d0da0c051..a769aa718 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -30,7 +30,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT - name: Install OpenStack client run: | diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index b8afea93e..e41beb3c3 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -59,7 +59,7 @@ jobs: id: openstack_release run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) - echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT # Generate a tag to apply to all built container images. # Without this, each kayobe * container image build command would use a different tag. From 5e0dc70ae9c3b8a57611faec19630131df303232 Mon Sep 17 00:00:00 2001 From: Matt Crees Date: Tue, 12 Mar 2024 14:33:36 +0000 Subject: [PATCH 25/61] Fix Jinja templating in Barbican Vault config The raw tags cause ``ssl_ca_crt_file`` to be templated without a newline on the end. This would give the following misconfiguration: ``` use_ssl = True ssl_ca_crt_file = approle_role_id = approle_secret_id = ``` --- doc/source/configuration/vault.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index 62bdaf24a..4cb39b61b 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -296,7 +296,9 @@ Configure Barbican [vault_plugin] vault_url = https://{{ kolla_internal_vip_address }}:8200 use_ssl = True - ssl_ca_crt_file = {% raw %}{{ openstack_cacert }}{% endraw %} + {% raw %} + ssl_ca_crt_file = {{ openstack_cacert }} + {% endraw %} approle_role_id = {{ secrets_barbican_approle_role_id }} approle_secret_id = {{ secrets_barbican_approle_secret_id }} kv_mountpoint = barbican From 27d147c2ffa8690cbb39895cfe77bb8f0a18a85c Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Fri, 8 Mar 2024 15:38:17 +0000 Subject: [PATCH 26/61] Use StackHPC downstream requirements fork --- etc/kayobe/kolla/kolla-build.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index d88c98ef6..b444eae17 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -9,3 +9,8 @@ base_tag = jammy-20231004 base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} {% endif %} build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }} + +[openstack-base] +type = git +location = https://github.com/stackhpc/requirements +reference = stackhpc/{{ openstack_release }} From 0ada338662318bfeab8209bc8ce398428f974aa3 Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:49:24 +0000 Subject: [PATCH 27/61] Add missing grafana plugins from upstream kolla --- etc/kayobe/kolla.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 969d5fb84..49bc7c8a8 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -320,7 +320,9 @@ kolla_build_blocks: ADD additions-archive / grafana_plugins_install: | RUN grafana-cli plugins install vonage-status-panel \ - && grafana-cli plugins install grafana-piechart-panel + && grafana-cli plugins install grafana-piechart-panel \ + && grafana-cli plugins install grafana-opensearch-datasource \ + && grafana-cli plugins install gnocchixyz-gnocchi-datasource ironic_inspector_header: | ADD additions-archive / magnum_base_footer: | From e054b4de9465beb0817f19febcb0fd325537bc91 Mon Sep 17 00:00:00 2001 From: scrungus Date: Wed, 13 Mar 2024 17:17:49 +0000 Subject: [PATCH 28/61] bump tag --- etc/kayobe/kolla/globals.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 419b1ba72..03e17ae72 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -20,9 +20,9 @@ kayobe_image_tags: rocky: yoga-20231218T141822 ubuntu: yoga-20231107T165648 magnum: - centos: yoga-20240229T120519 - rocky: yoga-20240229T120519 - ubuntu: yoga-20240229T120519 + centos: yoga-20240308T154440 + rocky: yoga-20240308T154440 + ubuntu: yoga-20240308T154440 neutron: centos: yoga-20231114T125927 rocky: yoga-20240105T120257 From 77a950e9eef966def32c6dd6663005b0fbdd20e5 Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:46:49 +0000 Subject: [PATCH 29/61] Bump tags for grafana --- etc/kayobe/kolla-image-tags.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 37a2f0b17..adb956b51 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -18,3 +18,6 @@ kolla_image_tags: neutron: rocky-9: 2023.1-rocky-9-20240202T145927 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T103817 + grafana: + rocky-9: 2023.1-rocky-9-20240313T165255 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240313T165255 From fe51c289de298488b4f29fafd928ef120ce7c89f Mon Sep 17 00:00:00 2001 From: technowhizz <7688823+technowhizz@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:49:33 +0000 Subject: [PATCH 30/61] Add release note --- .../notes/add-grafana-plugins-f4856a30529ac686.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml diff --git a/etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml b/etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml new file mode 100644 index 000000000..b4235388b --- /dev/null +++ b/etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + The grafana image now includes the `gnocchixyz-gnocchi-datasource` and the + `grafana-opensearch-datasource` plugins, which are the default upstream + plugins. From e1f3f8d6f51987f7d4fb874653c698205ef10202 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 15 Mar 2024 10:25:45 +0000 Subject: [PATCH 31/61] Fix releasenote location --- .../notes/add-grafana-plugins-f4856a30529ac686.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {etc/releasenotes => releasenotes}/notes/add-grafana-plugins-f4856a30529ac686.yaml (100%) diff --git a/etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml b/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml similarity index 100% rename from etc/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml rename to releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml From f1564f4a7cf324d370ab6d19a6beb8ec71bf2a5c Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 15 Mar 2024 11:35:08 +0000 Subject: [PATCH 32/61] hotfix: Fix setting containers_list and running without a command --- etc/kayobe/ansible/hotfix-containers.yml | 2 +- etc/kayobe/ansible/run-container-hotfix.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml index b6a811801..23c28a6b9 100644 --- a/etc/kayobe/ansible/hotfix-containers.yml +++ b/etc/kayobe/ansible/hotfix-containers.yml @@ -30,7 +30,7 @@ - name: Set fact for containers list set_fact: - containers_list: host_containers.stdout + containers_list: "{{ host_containers.stdout }}" - name: Fail if no containers match given regex vars: diff --git a/etc/kayobe/ansible/run-container-hotfix.yml b/etc/kayobe/ansible/run-container-hotfix.yml index 582ade5da..de652e451 100644 --- a/etc/kayobe/ansible/run-container-hotfix.yml +++ b/etc/kayobe/ansible/run-container-hotfix.yml @@ -20,3 +20,4 @@ - name: Run container_hotfix_command command: "{{ kolla_container_engine | default('docker')}} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}" + when: container_hotfix_command From 37b387aa3888661b725009e6021676030dbb5bd4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 15 Mar 2024 11:40:24 +0000 Subject: [PATCH 33/61] hotfix: Fix failure message --- etc/kayobe/ansible/hotfix-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml index 23c28a6b9..677105f3e 100644 --- a/etc/kayobe/ansible/hotfix-containers.yml +++ b/etc/kayobe/ansible/hotfix-containers.yml @@ -36,7 +36,7 @@ vars: hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}" fail: - msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers | split('\n') }}" + msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers.stdout_lines }}" when: hotfix_containers == "" - name: Ensure hotfix-files directory exists on the remote host From d2ed09e11fc58ee3658c53cf983075a72d334bb2 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 18 Mar 2024 10:40:11 +0000 Subject: [PATCH 34/61] Run OVN playbook without limit during upgrade --- .../ansible/ovn-fix-chassis-priorities.yml | 31 ++++++++++--------- etc/kayobe/ansible/ubuntu-upgrade.yml | 4 --- tools/ubuntu-upgrade-overcloud.sh | 2 ++ 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml index 36566b6a3..9ba469ce7 100644 --- a/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml +++ b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml @@ -21,22 +21,25 @@ - name: Find OVN DB DB Leader hosts: "{{ ovn_nb_db_group | default('controllers') }}" tasks: - - name: Find the OVN NB DB leader - ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection - changed_when: false - failed_when: false - register: ovn_check_result - check_mode: false + - name: Find OVN DB Leader + when: kolla_enable_ovn | bool + block: + - name: Find the OVN NB DB leader + ansible.builtin.command: docker exec ovn_nb_db ovn-nbctl get-connection + changed_when: false + failed_when: false + register: ovn_check_result + check_mode: false - - name: Group hosts by leader/follower role - ansible.builtin.group_by: - key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}" - changed_when: false + - name: Group hosts by leader/follower role + ansible.builtin.group_by: + key: "ovn_nb_{{ 'leader' if ovn_check_result.rc == 0 else 'follower' }}" + changed_when: false - - name: Assert one leader exists - ansible.builtin.assert: - that: - - groups['ovn_nb_leader'] | default([]) | length == 1 + - name: Assert one leader exists + ansible.builtin.assert: + that: + - groups['ovn_nb_leader'] | default([]) | length == 1 - name: Fix OVN chassis priorities hosts: ovn_nb_leader diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml index 3b477731c..928e1c52d 100644 --- a/etc/kayobe/ansible/ubuntu-upgrade.yml +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -104,7 +104,3 @@ that: - ansible_facts.distribution_major_version == '22' - ansible_facts.distribution_release == 'jammy' - -- name: Run the OVN chassis priority fix playbook - import_playbook: "{{ lookup('ansible.builtin.env', 'KAYOBE_CONFIG_PATH') }}/ansible/ovn-fix-chassis-priorities.yml" - when: kolla_enable_ovn diff --git a/tools/ubuntu-upgrade-overcloud.sh b/tools/ubuntu-upgrade-overcloud.sh index 3e351d6d6..50959c263 100755 --- a/tools/ubuntu-upgrade-overcloud.sh +++ b/tools/ubuntu-upgrade-overcloud.sh @@ -31,4 +31,6 @@ set -x kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml -e os_release=jammy --limit $1 +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ovn-fix-chassis-priorities.yml + kayobe overcloud host configure --limit $1 --kolla-limit $1 -e os_release=jammy From 7067c92e460787a013b1a66ea09a4d73fa4a959a Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 18 Mar 2024 13:42:06 +0000 Subject: [PATCH 35/61] Merge pull request #981 from stackhpc/use-fork-requirements Use StackHPC downstream requirements fork --- etc/kayobe/kolla/kolla-build.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index 2cd9c7a25..d78d0ebe2 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -9,3 +9,8 @@ base_tag = focal-20231003 base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} {% endif %} build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }} + +[openstack-base] +type = git +location = https://github.com/stackhpc/requirements +reference = stackhpc/{{ openstack_release }} From 4ee0f70e8de5d9036b1e0c5db8dfe06bd708ca81 Mon Sep 17 00:00:00 2001 From: scrungus Date: Wed, 20 Mar 2024 12:54:01 +0000 Subject: [PATCH 36/61] feature reno --- releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml index 864edf44b..7fc3cca1a 100644 --- a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml +++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml @@ -1,4 +1,4 @@ --- -upgrade: +features: - | Updates Magnum CAPI Helm driver version to v0.11.0 From d68a23fe0c7a75c7ac927ac8c6ac1b4c89b6a262 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Wed, 20 Mar 2024 11:36:13 +0100 Subject: [PATCH 37/61] Update cephadm collection version --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index aa28f43f6..92c3faecd 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.14.0 + version: 1.15.1 # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the # pulp_glue Python library being installed. - name: pulp.squeezer From aca602e48924e6dc654e6e862af9c44307d862cd Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 20 Mar 2024 13:44:27 +0000 Subject: [PATCH 38/61] Rebuild heat images with yaql 3.0.0 for 2023.1 --- etc/kayobe/kolla-image-tags.yml | 3 +++ .../rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index adb956b51..a457f41ed 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -9,6 +9,9 @@ kolla_image_tags: haproxy_ssh: rocky-9: 2023.1-rocky-9-20240205T162323 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T133905 + heat: + rocky-9: 2023.1-rocky-9-20240319T134201 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240319T134201 letsencrypt: rocky-9: 2023.1-rocky-9-20240205T162323 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T133905 diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml new file mode 100644 index 000000000..da3cb5cbb --- /dev/null +++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The Heat container images are rebuilt with yaql 3.0.0 to include patch for + vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat + services in your system with the current version of Heat images from + StackHPC Release Train. From 9f6a0173b30804b71de4e0fac566cf106b49b89b Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 20 Mar 2024 13:46:29 +0000 Subject: [PATCH 39/61] Rebuild heat images with yaql 3.0.0 for zed --- etc/kayobe/kolla-image-tags.yml | 3 +++ .../rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index a264fdbf1..c26b70d7d 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -6,6 +6,9 @@ kolla_image_tags: openstack: rocky-9: zed-rocky-9-20240202T105829 ubuntu-jammy: zed-ubuntu-jammy-20240129T151534 + heat: + rocky-9: zed-rocky-9-20240320T113114 + ubuntu-jammy: zed-ubuntu-jammy-20240320T113114 magnum: rocky-9: zed-rocky-9-20240301T100039 ubuntu-jammy: zed-ubuntu-jammy-20240301T100039 diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml new file mode 100644 index 000000000..da3cb5cbb --- /dev/null +++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The Heat container images are rebuilt with yaql 3.0.0 to include patch for + vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat + services in your system with the current version of Heat images from + StackHPC Release Train. From 91cc9ec5883f791067e2538e23581bab76eda8ad Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Wed, 20 Mar 2024 13:51:39 +0000 Subject: [PATCH 40/61] Rebuild heat images with yaql 3.0.0 for yoga --- etc/kayobe/kolla/globals.yml | 5 +++++ .../rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml | 7 +++++++ 2 files changed, 12 insertions(+) create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 03e17ae72..e860121d8 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -19,6 +19,10 @@ kayobe_image_tags: centos: yoga-20231107T165648 rocky: yoga-20231218T141822 ubuntu: yoga-20231107T165648 + heat: + centos: yoga-20240320T082414 + rocky: yoga-20240320T082414 + ubuntu: yoga-20240320T082414 magnum: centos: yoga-20240308T154440 rocky: yoga-20240308T154440 @@ -33,6 +37,7 @@ kayobe_image_tags: ubuntu: yoga-20231103T161400 cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}" +heat_tag: "{% raw %}{{ kayobe_image_tags['heat'][kolla_base_distro] }}{% endraw %}" magnum_tag: "{% raw %}{{ kayobe_image_tags['magnum'][kolla_base_distro] }}{% endraw %}" neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}" nova_tag: "{% raw %}{{ kayobe_image_tags['nova'][kolla_base_distro] }}{% endraw %}" diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml new file mode 100644 index 000000000..da3cb5cbb --- /dev/null +++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The Heat container images are rebuilt with yaql 3.0.0 to include patch for + vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat + services in your system with the current version of Heat images from + StackHPC Release Train. From c444a605d2f3f639f64c59188f99f2a8cac16235 Mon Sep 17 00:00:00 2001 From: Scott Davidson <49713135+sd109@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:11:56 +0000 Subject: [PATCH 41/61] Update Magnum CAPI Helm driver version (#1007) * Bump magnum-capi-helm driver to v0.12.0 * Update magnum image tags * Add release note --------- --- etc/kayobe/kolla-image-tags.yml | 4 ++-- etc/kayobe/kolla.yml | 2 +- .../notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index adb956b51..e2692efd3 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -13,8 +13,8 @@ kolla_image_tags: rocky-9: 2023.1-rocky-9-20240205T162323 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T133905 magnum: - rocky-9: 2023.1-rocky-9-20240312T170026 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240312T170026 + rocky-9: 2023.1-rocky-9-20240320T133822 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240320T133822 neutron: rocky-9: 2023.1-rocky-9-20240202T145927 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T103817 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 6bcbdc09a..6db29a0cc 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -328,7 +328,7 @@ kolla_build_blocks: magnum_base_footer: | RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | head -n -1 | bash {% raw %} - {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.11.0'] %} + {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.12.0'] %} RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }} {% endraw %} # Dict mapping image customization variable names to their values. diff --git a/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml b/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml new file mode 100644 index 000000000..6677583fb --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Magnum CAPI Helm driver version to v0.12.0 From 9beb7fb09bb5c51a9261e7bed73564378f5e854e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 21 Mar 2024 13:18:05 +0000 Subject: [PATCH 42/61] Fail on any unparsed Ansible inventory If Ansible is unable to parse an inventory source, by default it will print a warning and continue execution. Typically this highlights an important error that should be addressed. This change modifies the Ansible configuration to error out in this case. --- etc/kayobe/ansible.cfg | 4 ++++ .../notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml | 7 +++++++ 2 files changed, 11 insertions(+) create mode 100644 releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 901001ad8..b38cb8239 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -11,5 +11,9 @@ callbacks_enabled = ansible.posix.profile_tasks # Silence warning about invalid characters found in group names force_valid_group_names = ignore +[inventory] +# Fail when any inventory source cannot be parsed. +any_unparsed_is_failed = True + [ssh_connection] pipelining = True diff --git a/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml new file mode 100644 index 000000000..335691c30 --- /dev/null +++ b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Updates the Ansible configuration to `fail on any unparsed inventory source + `__. + If you are using a separate Ansible configuration for Kolla Ansible, you + may wish to add this setting in ``etc/kayobe/kolla/ansible.cfg``. From 1fd719094c633a48abf73e283deacf55cfd47214 Mon Sep 17 00:00:00 2001 From: Scott Davidson <49713135+sd109@users.noreply.github.com> Date: Fri, 22 Mar 2024 09:28:56 +0000 Subject: [PATCH 43/61] Update docs to reflect upstream Magnum driver changes (#1000) --- doc/source/configuration/magnum-capi.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/source/configuration/magnum-capi.rst b/doc/source/configuration/magnum-capi.rst index 302bf800c..c05a80bcf 100644 --- a/doc/source/configuration/magnum-capi.rst +++ b/doc/source/configuration/magnum-capi.rst @@ -105,16 +105,10 @@ Next, copy the CAPI management cluster's kubeconfig file into your stackhpc-kayo The following config should also be set in your stackhpc-kayobe-config environment: -.. code-block:: ini - :caption: magnum.conf - - [nova_client] - endpoint_type = publicURL - .. code-block:: yaml :caption: kolla/globals.yml - magnum_cluster_api_driver_enabled: true + magnum_capi_helm_driver_enabled: true To apply the configuration, run ``kayobe overcloud service reconfigure -kt magnum``. From 99838a89987a6c4250f4eb0a55ed7d199853b5fe Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 26 Mar 2024 12:42:46 +0000 Subject: [PATCH 44/61] docs: Add an upgrade doc note about Glance show_multiple_locations Operators may decide to replace this option. --- doc/source/operations/upgrading.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst index 53df5aef2..d44f1e917 100644 --- a/doc/source/operations/upgrading.rst +++ b/doc/source/operations/upgrading.rst @@ -94,6 +94,20 @@ custom configuration that uses the Keystone admin port. One such example is the config for Ceph RGW in ``etc/kayobe/cephadm.yml``. Be sure to update any manual references to the old port. +Glance show_multiple_locations disabled +--------------------------------------- + +Kolla Ansible no longer sets ``show_multiple_locations = True`` in Glance by +default when Glance's Ceph RBD backend is enabled. This was applied as a fix +but operators must note that this, in turn, disables Cinder and Nova's +optimisations. In particular, this can increase instance creation times due to +a lack of copy-on-write. + +On the other hand, these optimisations might have been causing other trouble +for operators. Please see `LP#1992153 +`__. Operators relying +on this feature can set the flag themselves using service config overrides. + Known issues ============ From 8f6fcb81af1f53355056ed511d2b18860a2d2125 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 28 Mar 2024 14:15:23 +0000 Subject: [PATCH 45/61] Fix host image builds on Arc runners Arc runners are kubernetes-orchestrated github runners. Host image builds do not work on these runners, so this commit adapts the host image build workflow to spin up a worker VM which executes the build. --- .../workflows/overcloud-host-image-build.yml | 400 +++++++++++------- .github/workflows/stackhpc-ci-cleanup.yml | 20 + .../ansible/openstack-host-image-upload.yml | 54 +++ etc/kayobe/ansible/pulp-host-image-upload.yml | 16 +- .../environments/ci-builder/inventory/hosts | 2 +- etc/kayobe/overcloud-dib.yml | 2 +- 6 files changed, 323 insertions(+), 171 deletions(-) create mode 100644 etc/kayobe/ansible/openstack-host-image-upload.yml diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 4c338cda3..1b6c2e5f9 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -35,21 +35,38 @@ on: env: ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} jobs: overcloud-host-image-build: name: Build overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder] + runs-on: arc-skc-host-image-builder-runner permissions: {} steps: - - uses: actions/checkout@v4 + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@v4 with: path: src/kayobe-config + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + - name: Determine OpenStack release id: openstack_release run: | - BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT # Generate a tag to apply to all built overcloud host images. @@ -62,10 +79,6 @@ jobs: run: | echo "${{ steps.host_image_tag.outputs.host_image_tag }}" - - name: Clean any previous build artifact - run: | - rm -f /tmp/updated_images.txt - - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 with: @@ -73,34 +86,6 @@ jobs: ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} path: src/kayobe - # FIXME: Failed in kolla-ansible : Ensure the latest version of pip is installed - - name: Install dependencies - run: | - sudo dnf -y install python3-virtualenv zstd - - - name: Setup networking - run: | - if ! ip l show breth1 >/dev/null 2>&1; then - sudo ip l add breth1 type bridge - fi - sudo ip l set breth1 up - if ! ip a show breth1 | grep 192.168.33.3/24; then - sudo ip a add 192.168.33.3/24 dev breth1 - fi - if ! ip l show dummy1 >/dev/null 2>&1; then - sudo ip l add dummy1 type dummy - fi - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - - # FIXME: Without this workaround we see the following issue after the runner is power cycled: - # TASK [MichaelRigart.interfaces : RedHat | ensure network service is started and enabled] *** - # Unable to start service network: Job for network.service failed because the control process exited with error code. - # See \"systemctl status network.service\" and \"journalctl -xe\" for details. - - name: Kill dhclient (workaround) - run: | - (sudo killall dhclient || true) && sudo systemctl restart network - - name: Install Kayobe run: | mkdir -p venvs && @@ -110,36 +95,132 @@ jobs: pip install -U pip && pip install ../src/kayobe + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "rocky" + aio_vm_name = "skc-host-image-builder" + # Must be a Rocky Linux 9 host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2" + aio_vm_flavor = "en1.medium" + aio_vm_network = "stackhpc-ci" + aio_vm_subnet = "stackhpc-ci" + aio_vm_interface = "eth0" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + - name: Bootstrap the control host run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe control host bootstrap - - name: Configure the seed host + - name: Configure the seed host (Builder VM) run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure + kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Create bifrost_httpboot Docker volume - run: | - if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then - sudo docker volume create bifrost_httpboot - fi - - - name: Generate clouds.yaml - run: | - cat << EOF > clouds.yaml - ${{ secrets.CLOUDS_YAML }} - EOF - - - name: Install OpenStack client run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Build a CentOS Stream 8 overcloud host image id: build_centos_stream_8 @@ -155,6 +236,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.centos + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_centos_stream_8.outcome == 'failure' + - name: Upload CentOS Stream 8 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -169,18 +260,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' - - name: Upload CentOS Stream 8 overcloud host image to SMS + - name: Upload CentOS Stream 8 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2" \ + -e image_name=overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.centos && steps.build_centos_stream_8.outcome == 'success' @@ -199,6 +288,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_8.outcome == 'failure' + - name: Upload Rocky Linux 8 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -212,19 +311,17 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' - - - name: Upload Rocky Linux 8 overcloud host image to SMS + + - name: Upload Rocky Linux 8 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2" \ + -e image_name=overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success' @@ -243,6 +340,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_9.outcome == 'failure' + - name: Upload Rocky Linux 9 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -256,19 +363,17 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' - - - name: Upload Rocky Linux 9 overcloud host image to SMS + + - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ + -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' @@ -287,6 +392,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_focal.outcome == 'failure' + - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -301,18 +416,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' - - name: Upload Ubuntu Focal 20.04 overcloud host image to SMS + - name: Upload Ubuntu Focal overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2" \ + -e image_name=overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success' @@ -331,6 +444,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -345,83 +468,27 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload Ubuntu Jammy 22.04 overcloud host image to SMS + - name: Upload Ubuntu Jammy overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \ + -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload updated images artifact - uses: actions/upload-artifact@v4 - with: - name: Updated images list - path: /tmp/updated_images.txt - retention-days: 7 - if: steps.build_centos_stream_8.outcome == 'success' || - steps.build_rocky_8.outcome == 'success' || - steps.build_rocky_9.outcome == 'success' || - steps.build_ubuntu_focal.outcome == 'success' || - steps.build_ubuntu_jammy.outcome == 'success' - - - name: Upload CentOS build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: CentOS build logs - path: | - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout - /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stderr - retention-days: 7 - if: steps.build_centos_stream_8.outcome == 'failure' - - - name: Upload Rocky 8 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 8 build logs - path: | - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout - /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stderr - retention-days: 7 - if: steps.build_rocky_8.outcome == 'failure' - - - name: Upload Rocky 9 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 9 build logs - path: | - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr - retention-days: 7 - if: steps.build_rocky_9.outcome == 'failure' - - - name: Upload Ubuntu Focal 20.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Focal 20.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout - /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stderr - retention-days: 7 - if: steps.build_ubuntu_focal.outcome == 'failure' - - - name: Upload Ubuntu Jammy 22.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Jammy 22.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr - retention-days: 7 - if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() - name: Fail if any overcloud host image builds failed run: | @@ -433,7 +500,18 @@ jobs: steps.build_ubuntu_focal.outcome == 'failure' || steps.build_ubuntu_jammy.outcome == 'failure' - - name: Clean up build artifacts - run: | - sudo rm -rf /opt/kayobe/images/ + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs + path: ./logs if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() \ No newline at end of file diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index a769aa718..ed9ec327c 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -55,3 +55,23 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Clean up host image builder instances over 5 hours old + run: | + result=0 + changes_before=$(date -Imin -d -5hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml new file mode 100644 index 000000000..2c92d2446 --- /dev/null +++ b/etc/kayobe/ansible/openstack-host-image-upload.yml @@ -0,0 +1,54 @@ +--- +# This playbook is designed to be used by the overcloud-host-image-build.yml +# GitHub workflow to upload newly-built images to a development cloud for +# testing and use in CI. +- name: Upload an OS image to Glance + hosts: seed + vars: + local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + image_name: "overcloud-{{ os_distribution }}-{{ os_release }}" + tasks: + - block: + - name: Write out clouds.yaml + copy: + content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}" + dest: clouds.yaml + mode: 0600 + + - name: Write out secure.yaml + no_log: true + vars: + os_secrets: + clouds: + openstack: + auth: + application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}" + application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" + copy: + content: "{{ os_secrets | to_nice_yaml }}" + dest: secure.yaml + mode: 0600 + + - name: Ensure dependencies are installed + pip: + name: openstacksdk + + - name: Upload an image to Glance + openstack.cloud.image: + cloud: openstack + name: "{{ image_name }}" + container_format: bare + disk_format: qcow2 + state: present + filename: "{{ local_image_path }}" + + always: + - name: Remove clouds.yaml + file: + path: clouds.yaml + state: absent + + - name: Remove secure.yaml + file: + path: secure.yaml + state: absent diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml index a06897d90..d3a44f133 100644 --- a/etc/kayobe/ansible/pulp-host-image-upload.yml +++ b/etc/kayobe/ansible/pulp-host-image-upload.yml @@ -1,12 +1,12 @@ --- - name: Upload and create a distribution for an image - hosts: localhost + hosts: seed vars: remote_pulp_url: "{{ stackhpc_release_pulp_url }}" remote_pulp_username: "{{ stackhpc_image_repository_username }}" remote_pulp_password: "{{ stackhpc_image_repository_password }}" repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" tasks: - name: Print image tag debug: @@ -74,7 +74,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_latest" - base_path: "{{ base_path }}/latest" + base_path: "{{ pulp_base_path }}/latest" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -86,7 +86,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_{{ host_image_tag }}" - base_path: "{{ base_path }}/{{ host_image_tag }}" + base_path: "{{ pulp_base_path }}/{{ host_image_tag }}" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -95,26 +95,26 @@ - name: Update new images file with versioned path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" create: true - name: Update new images file with latest path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print versioned path debug: - msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print latest path debug: - msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed diff --git a/etc/kayobe/environments/ci-builder/inventory/hosts b/etc/kayobe/environments/ci-builder/inventory/hosts index 49b7be166..33fda8b73 100644 --- a/etc/kayobe/environments/ci-builder/inventory/hosts +++ b/etc/kayobe/environments/ci-builder/inventory/hosts @@ -1,3 +1,3 @@ # A 'seed' host used for building images. [seed] -localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 +builder diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 8f59d58ef..d7f6dbd69 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra: overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.6.0" + version: "v1.6.1" elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See From c1a31acc9d8e85330b664f220051a39e1c2cb9ed Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 2 Apr 2024 14:56:45 +0100 Subject: [PATCH 46/61] Fix AIO connectivity loss in automated script --- etc/kayobe/environments/ci-aio/automated-setup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh index 5129db015..f5468a09a 100644 --- a/etc/kayobe/environments/ci-aio/automated-setup.sh +++ b/etc/kayobe/environments/ci-aio/automated-setup.sh @@ -84,6 +84,10 @@ kayobe overcloud host configure kayobe overcloud service deploy +if type apt; then + sudo cp /run/systemd/network/* /etc/systemd/network +fi + export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe pushd $BASE_PATH/src/kayobe From faaabbb5aa92f547f48f513233fdf24785c9e7b7 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 4 Apr 2024 11:43:40 +0100 Subject: [PATCH 47/61] Fix AIO deploy script --- etc/kayobe/environments/ci-aio/automated-setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh index f5468a09a..84b9b5f09 100644 --- a/etc/kayobe/environments/ci-aio/automated-setup.sh +++ b/etc/kayobe/environments/ci-aio/automated-setup.sh @@ -72,6 +72,10 @@ fi sudo ip l set dummy1 up sudo ip l set dummy1 master breth1 +if type apt; then + sudo cp /run/systemd/network/* /etc/systemd/network +fi + export KAYOBE_VAULT_PASSWORD=$(cat $BASE_PATH/vault-pw) pushd $BASE_PATH/src/kayobe-config source kayobe-env --environment ci-aio @@ -84,10 +88,6 @@ kayobe overcloud host configure kayobe overcloud service deploy -if type apt; then - sudo cp /run/systemd/network/* /etc/systemd/network -fi - export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe pushd $BASE_PATH/src/kayobe From 2d8d500925fd90b7cc83b4fd8af3f53329a2696e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 11:24:55 +0100 Subject: [PATCH 48/61] ci-multinode: Use skc-ci-aio user for ci-multinode env Similar to c338dd9b7cad77c14eb15eb0193d02b0c9ff78b4, but applied to ci-multinode instead of ci-aio. This user only has read-only access to the package and container repositories, so is safer than using the release-train-ci user which has read/write permissions. --- .../environments/ci-multinode/stackhpc-ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index cdb6eb810..ae5768bac 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -11,6 +11,14 @@ kolla_docker_namespace: stackhpc-dev # Host and port of a package repository mirror. # Build and deploy the development Pulp service repositories. stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" +stackhpc_repo_mirror_username: "skc-ci-aio" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 36373536303261313239613761653261663437356566343865383563346334396136653666383765 + 6634396534653865633936653038383132396532386665370a366562383166353966663838316266 + 65333133636330623936623438666632316238376264313234346333346461623765633163353635 + 6565326136313564320a303231383438333062643533333335663034613439393665656162626137 + 65356232656164663831316530333136336362393636656566353635306565626636 # Build and deploy released Pulp repository versions. stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" @@ -66,12 +74,5 @@ stackhpc_include_os_minor_version_in_repo_url: true # Push built images to the development Pulp service registry. stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" -# Username and password of container registry. -stackhpc_docker_registry_username: "stackhpc-kayobe-ci" -stackhpc_docker_registry_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 33356166343730633865363431306535613736663764373034396132356131343066636530393534 - 3262646436663034633131316438633230383330633533350a386365313239303464383636376338 - 61656662333939333063343131633963636431663136643137636664633233633133396339613861 - 3038613063626138610a333566393937643630366564653163613364323965396130613433316537 - 39653335393831633362343934363866346262613166393561666336623062393935 +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" From d77fcb1f64abbe50171d493df05e76e7f03e2a2e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:29:00 +0100 Subject: [PATCH 49/61] ci-multinode: Use Ark package repositories to install packages Similar to e9130b9c51161fdadd676932eae5f2c13f5948a8 but applied to ci-multinode rather than ci-aio. Previously we were using Test Pulp on SMS lab, but this is out of action. Switching to Ark allows CI jobs to run on Leafcloud (or anywhere with Internet access). --- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index ae5768bac..32f8775e1 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -10,7 +10,8 @@ kolla_docker_namespace: stackhpc-dev # Host and port of a package repository mirror. # Build and deploy the development Pulp service repositories. -stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "skc-ci-aio" stackhpc_repo_mirror_password: !vault | $ANSIBLE_VAULT;1.1;AES256 From c57f2c3a7b93d17ed1ccfd31c8c596dd6c2e3064 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:22:07 +0100 Subject: [PATCH 50/61] ci-multinode: Allow rebooting for SELinux state The Yoga overcloud host images currently have SELinux disabled, but the default config enables SELinux in permissive mode on Rocky Linux 9. This change allows the ci-multinode environment to run on these images. --- etc/kayobe/environments/ci-multinode/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml index daecef4f2..fe7285f4c 100644 --- a/etc/kayobe/environments/ci-multinode/globals.yml +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -64,7 +64,7 @@ stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id" ############################################################################### # Avoid a reboot. -disable_selinux_do_reboot: false +disable_selinux_do_reboot: true ############################################################################### # Dummy variable to allow Ansible to accept this file. From e2b2f40cd138a212b2801ed410f791003a0a5fd5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:31:37 +0100 Subject: [PATCH 51/61] ci-multinode: Add API FQDNs to /etc/hosts in fix-networking.yml This avoids using the add-fqdn.yml playbook in terraform-kayobe-multinode, which requires the Terraform/Ansible client to have access to all hosts. --- etc/kayobe/ansible/fix-networking.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 0b14f9ddf..8105db8f4 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -10,11 +10,13 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: - - name: Ensure `hosts` file contains pulp entries + - name: Ensure `hosts` file contains pulp and API entries blockinfile: path: /etc/hosts - marker: "# {mark} Kayobe Pulp entries" + marker: "# {mark} Kayobe entries" block: | 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud 10.205.3.187 pulp-server pulp-server.internal.sms-cloud + 192.168.37.2 internal.infra.mos.{{ root_domain }} + 192.168.39.2 public.infra.mos.{{ root_domain }} become: true From 2ca68f1713b5784fcfc577f37d3b6528f8017060 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:13:17 +0100 Subject: [PATCH 52/61] ci-multinode: Wait for connection in fix-networking.yml This allows us to drop the fix-homedir-ownership.yml playbook in terraform-kayobe-multinode, which also performed the function of waiting for hosts to become reachable. --- etc/kayobe/ansible/fix-networking.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 8105db8f4..01a833264 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -10,6 +10,9 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: + - name: Ensure hosts are reachable + ansible.builtin.wait_for_connection: + - name: Ensure `hosts` file contains pulp and API entries blockinfile: path: /etc/hosts From 33c0d38622136cde0fd46458f837ecd35b18ac40 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:12:05 +0100 Subject: [PATCH 53/61] ci-multinode: Use qemu virtualisation Most multinode environments will use nested virtualisation, and we can't guarantee that nested KVM support is available. Use QEMU as a lowest common denominator. We might consider setting this dynamically based on the hypervisor in future. --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index eab31a1d8..4f9506be0 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -1,4 +1,9 @@ --- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + # Reduce the control plane's memory footprint by limiting the number of worker # processes to two per-service when running in a VM. openstack_service_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 5] | min }}{% endraw %}" From eb1f88ec51ee65a504e064b99e3aadc4ab65cbe0 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 16:03:51 +0100 Subject: [PATCH 54/61] ci-multinode: Set default Ceph release to Quincy on Rocky Linux 9 Pacific is not supported on Rocky Linux 9, so it does not make sense as a default. --- etc/kayobe/environments/ci-multinode/cephadm.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index 7885a5735..4a9d3f448 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -2,6 +2,12 @@ ############################################################################### # Cephadm deployment configuration. +# Ceph release name. +cephadm_ceph_release: "{{ 'quincy' if (ansible_facts['distribution_release'] == 'jammy' or ansible_facts.distribution_major_version == '9') else 'pacific' }}" + +# Ceph container image tag. +cephadm_image_tag: "{{ 'v17.2.7' if cephadm_ceph_release == 'quincy' else 'v16.2.14' }}" + # Ceph OSD specification. cephadm_osd_spec: service_type: osd From 50378160654a5ed5d74cb17180cb3999401d09b5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:09:23 +0100 Subject: [PATCH 55/61] os_capacity: Add tags to playbook, update vault docs Previously the first deployment of a system with a Vault CA for internal TLS and os_capacity enabled would fail when deploying HAProxy. os_capacity deployment requires admin-openrc.sh to exist, but because of the use of -kt haproxy the post-deploy tasks that create it will be skipped. This change fixes the issue by adding an os_capacity tag to the relevant plays, and updating the Vault docs to skip the new tag when deploying HAProxy. --- doc/source/configuration/vault.rst | 2 +- etc/kayobe/ansible/deploy-os-capacity-exporter.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index 4cb39b61b..21268f108 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -111,7 +111,7 @@ Setup HAProxy config for Vault .. code-block:: - kayobe overcloud service deploy -kt haproxy + kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy Setup Vault HA on the overcloud hosts ------------------------------------- diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml index 978c13e62..cc3afa7b0 100644 --- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -1,6 +1,7 @@ --- - name: Remove legacy os_exporter.cfg file hosts: network + tags: os_capacity gather_facts: false tasks: - name: Ensure legacy os_exporter.cfg config file is deleted @@ -11,6 +12,7 @@ - name: Deploy os-capacity exporter hosts: monitoring + tags: os_capacity gather_facts: false tasks: - name: Create os-capacity directory From a6082d0fc3163b4f6ef8a83ba518d8b3722e5ac9 Mon Sep 17 00:00:00 2001 From: Scott Davidson <49713135+sd109@users.noreply.github.com> Date: Thu, 11 Apr 2024 12:18:17 +0100 Subject: [PATCH 56/61] Update Magnum driver from v0.12.0 to v0.13.0 --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 6db29a0cc..adf3081cf 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -328,7 +328,7 @@ kolla_build_blocks: magnum_base_footer: | RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | head -n -1 | bash {% raw %} - {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.12.0'] %} + {% set magnum_capi_packages = ['git+https://github.com/stackhpc/magnum-capi-helm.git@v0.13.0'] %} RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }} {% endraw %} # Dict mapping image customization variable names to their values. From 55b343b63dc8fb07d053ec4d242f468e9cf05142 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 11 Apr 2024 13:54:09 +0100 Subject: [PATCH 57/61] Revert "docs: Add an upgrade doc note about Glance show_multiple_locations" This reverts commit 99838a89987a6c4250f4eb0a55ed7d199853b5fe. It applies to the Zed upgrade, not Antelope. --- doc/source/operations/upgrading.rst | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst index a23495bd6..89f8f6aa8 100644 --- a/doc/source/operations/upgrading.rst +++ b/doc/source/operations/upgrading.rst @@ -162,20 +162,6 @@ environment. This can result in significant changes to the Kolla config. Take extra care when creating the Antelope branch of the kayobe-config and always check the config diff. -Glance show_multiple_locations disabled ---------------------------------------- - -Kolla Ansible no longer sets ``show_multiple_locations = True`` in Glance by -default when Glance's Ceph RBD backend is enabled. This was applied as a fix -but operators must note that this, in turn, disables Cinder and Nova's -optimisations. In particular, this can increase instance creation times due to -a lack of copy-on-write. - -On the other hand, these optimisations might have been causing other trouble -for operators. Please see `LP#1992153 -`__. Operators relying -on this feature can set the flag themselves using service config overrides. - Known issues ============ From 494783864874509a8e3407c72fe528506bbb416c Mon Sep 17 00:00:00 2001 From: Scott Davidson <49713135+sd109@users.noreply.github.com> Date: Thu, 11 Apr 2024 15:43:01 +0100 Subject: [PATCH 58/61] Update Magnum image tags --- etc/kayobe/kolla-image-tags.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 9ff72b2b0..69165cf06 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -16,8 +16,8 @@ kolla_image_tags: rocky-9: 2023.1-rocky-9-20240205T162323 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T133905 magnum: - rocky-9: 2023.1-rocky-9-20240320T133822 - ubuntu-jammy: 2023.1-ubuntu-jammy-20240320T133822 + rocky-9: 2023.1-rocky-9-20240411T125311 + ubuntu-jammy: 2023.1-ubuntu-jammy-20240411T125311 neutron: rocky-9: 2023.1-rocky-9-20240202T145927 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T103817 From f2523ece958ff01dea6f235d57af450008249d0d Mon Sep 17 00:00:00 2001 From: sd109 Date: Mon, 15 Apr 2024 13:29:41 +0100 Subject: [PATCH 59/61] Add release note --- .../notes/bump-magnum-capi-helm-d766b5956de65d31.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml diff --git a/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml b/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml new file mode 100644 index 000000000..eb1e37640 --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml @@ -0,0 +1,4 @@ +--- + features: + - | + Updates Magnum CAPI Helm driver version to v0.13.0 From 680bb722e0c3307de45645e45054393102fb8afb Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Mon, 15 Apr 2024 12:00:18 +0100 Subject: [PATCH 60/61] Update Ubuntu horizon tag to fix CVE-2023-31122 --- etc/kayobe/kolla-image-tags.yml | 2 ++ releasenotes/notes/bump-horizon-694d426decbf7df3.yaml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/notes/bump-horizon-694d426decbf7df3.yaml diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml index 9ff72b2b0..9eeba0c83 100644 --- a/etc/kayobe/kolla-image-tags.yml +++ b/etc/kayobe/kolla-image-tags.yml @@ -12,6 +12,8 @@ kolla_image_tags: heat: rocky-9: 2023.1-rocky-9-20240319T134201 ubuntu-jammy: 2023.1-ubuntu-jammy-20240319T134201 + horizon: + ubuntu-jammy: 2023.1-ubuntu-jammy-20240402T104530 letsencrypt: rocky-9: 2023.1-rocky-9-20240205T162323 ubuntu-jammy: 2023.1-ubuntu-jammy-20240221T133905 diff --git a/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml b/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml new file mode 100644 index 000000000..780797d9e --- /dev/null +++ b/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Update Horizon on Ubuntu to include apache2 package ``2.4.52-1ubuntu4.8`` + which fixes CVE-2023-31122. From b19b42ed1f52bc0d3ea8c30c88972cd8ce23f4ef Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 18 Apr 2024 10:52:39 +0100 Subject: [PATCH 61/61] docs: Remove prometheus and grafana config symlinks These are no longer necessary due to support for kayobe multiple environment merging being backported to Antelope. --- doc/source/configuration/monitoring.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 822a3c02f..069bf4700 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -42,17 +42,6 @@ The configuration options can be found in .. literalinclude:: ../../../etc/kayobe/stackhpc-monitoring.yml :language: yaml -In order to enable stock monitoring configuration within a particular -environment, create the following symbolic links: - -.. code-block:: console - - cd $KAYOBE_CONFIG_PATH - ln -s ../../../../kolla/config/grafana/ environments/$KAYOBE_ENVIRONMENT/kolla/config/ - ln -s ../../../../kolla/config/prometheus/ environments/$KAYOBE_ENVIRONMENT/kolla/config/ - -and commit them to the config repository. - SMART Drive Monitoring ======================