[MCAPI-52] fix: move upgrade cluster to integration

hack/run-integration-tests.sh

@@ -27,7 +27,9 @@ NETWORK_DRIVER=${NETWORK_DRIVER:-calico}
 SONOBUOY_VERSION=${SONOBUOY_VERSION:-0.56.16}
 SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64}
 DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1}
+UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG}
 IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}"
+UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}"
 
 # If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise
 # we download the latest promoted image.
@@ -45,6 +47,21 @@ openstack image create \
   --file=${IMAGE_NAME}.qcow2 \
   ${IMAGE_NAME}
 
+if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
+    if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then
+      curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2
+    else
+      test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1
+    fi
+    # Upload Upgrade image to Glance
+    openstack image create \
+      --disk-format=qcow2 \
+      --container-format=bare \
+      --property os_distro=${OS_DISTRO} \
+      --file=${UPGRADE_IMAGE_NAME}.qcow2 \
+      ${UPGRADE_IMAGE_NAME}
+fi
+
 # Create cluster template
 openstack coe cluster template create \
     --image $(openstack image show ${IMAGE_NAME} -c id -f value) \
@@ -60,6 +77,23 @@ openstack coe cluster template create \
     --label fixed_subnet_cidr=192.168.24.0/24 \
     k8s-${KUBE_TAG};
 
+if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
+    # Create cluster template for upgrade
+    openstack coe cluster template create \
+        --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \
+        --external-network public \
+        --dns-nameserver ${DNS_NAMESERVER} \
+        --master-lb-enabled \
+        --master-flavor m1.large \
+        --flavor m1.large \
+        --network-driver ${NETWORK_DRIVER} \
+        --docker-storage-driver overlay2 \
+        --coe kubernetes \
+        --label kube_tag=${UPGRADE_KUBE_TAG} \
+        --label fixed_subnet_cidr=192.168.24.0/24 \
+        k8s-${UPGRADE_KUBE_TAG};
+fi
+
 # Create cluster
 openstack coe cluster create \
   --cluster-template k8s-${KUBE_TAG} \
@@ -93,6 +127,7 @@ for i in {1..240}; do
     echo "Cluster created"
     break
   else
+    echo "Currtny retry count: $i"
     echo "Cluster status: ${CLUSTER_STATUS}"
     sleep 5
   fi
@@ -120,3 +155,80 @@ if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed";
   echo "Sonobuoy tests failed"
   exit 1
 fi
+
+
+if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
+
+    openstack coe cluster delete k8s-cluster
+    # Wait for cluster to be deleted
+    set +e
+    for i in {1..60}; do
+      openstack coe cluster show k8s-cluster 2>&1
+      exit_status=$?
+      if [ $exit_status -eq 0 ]; then
+          sleep 2
+      else
+          echo "Cluster k8s-cluster deleted."
+          break
+      fi
+    done
+    set -e
+    # Create cluster
+    openstack coe cluster create \
+      --cluster-template k8s-${KUBE_TAG} \
+      --master-count 1 \
+      --node-count 1 \
+      --merge-labels \
+      --label audit_log_enabled=true \
+      k8s-cluster-upgrade
+
+    # Wait for cluster creation to be queued
+    set +e
+    for i in {1..5}; do
+      openstack coe cluster show k8s-cluster-upgrade 2>&1
+      exit_status=$?
+      if [ $exit_status -eq 0 ]; then
+          break
+      else
+          echo "Error: Cluster k8s-cluster-upgrade could not be found."
+          sleep 1
+      fi
+    done
+    set -e
+
+    # Wait for cluster to be "CREATE_COMPLETE".
+    for i in {1..240}; do
+      CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
+      if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
+        echo "Cluster failed to create"
+        exit 1
+      elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then
+        echo "Cluster created"
+        break
+      else
+        echo "Currtny retry count: $i"
+        echo "Cluster status: ${CLUSTER_STATUS}"
+        sleep 5
+      fi
+    done
+
+    # Upgrade cluster
+    openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG}
+    # Wait for cluster to be "UPDATE_COMPLETE".
+    for i in {1..240}; do
+      CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
+      if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
+        echo "Cluster failed to upgrade"
+        exit 1
+      elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then
+        echo "Cluster upgraded"
+        exit 0
+        break
+      else
+        echo "Currtny retry count: $i"
+        echo "Cluster status: ${CLUSTER_STATUS}"
+        sleep 5
+      fi
+    done
+    exit 1
+fi

hack/run-tempest-tests.sh

@@ -0,0 +1,192 @@
+#!/bin/bash -xe
+
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script will run the full functional tests for a given `KUBE_TAG`.  It
+# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE`
+# and then run `sonobuoy` against it.
+
+source /opt/stack/openrc admin admin
+
+OS_DISTRO=${OS_DISTRO:-ubuntu}
+IMAGE_OS=${IMAGE_OS:-ubuntu-2204}
+NETWORK_DRIVER=${NETWORK_DRIVER:-calico}
+DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1}
+UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG}
+IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}"
+UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}"
+
+# If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise
+# we download the latest promoted image.
+if [[ "${BUILD_NEW_IMAGE,,}" != "true" ]]; then
+  curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${IMAGE_NAME}.qcow2
+else
+  test -f ${IMAGE_NAME}.qcow2 || exit 1
+fi
+
+# Upload image to Glance
+openstack image create \
+  --disk-format=qcow2 \
+  --public \
+  --container-format=bare \
+  --property os_distro=${OS_DISTRO} \
+  --file=${IMAGE_NAME}.qcow2 \
+  ${IMAGE_NAME}
+
+if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
+    if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then
+      curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2
+    else
+      test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1
+    fi
+    # Upload Upgrade image to Glance
+    openstack image create \
+      --disk-format=qcow2 \
+      --public \
+      --container-format=bare \
+      --property os_distro=${OS_DISTRO} \
+      --file=${UPGRADE_IMAGE_NAME}.qcow2 \
+      ${UPGRADE_IMAGE_NAME}
+fi
+
+mkdir /tmp/magnum-nodes
+
+pushd /opt/stack/tempest
+echo "Tempest configs:"
+
+cat <<EOF >> /opt/stack/tempest/etc/tempest.conf
+
+[magnum]
+flavor_id = m1.large
+master_flavor_id = m1.large
+copy_logs = true
+network_driver = ${NETWORK_DRIVER}
+image_id = ${IMAGE_OS}-kube-${KUBE_TAG}
+coe = kubernetes
+labels = '{"kube_tag": "${KUBE_TAG}", "fixed_subnet_cidr": "10.0.0.0/26"}'
+docker_storage_driver = overlay2
+
+EOF
+
+if [ ! -d /opt/stack/magnum-tempest-plugin ]; then
+    git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin
+fi
+
+# install magnum-tempest-plugin
+pushd /opt/stack/magnum-tempest-plugin
+$HOME/.local/bin/pip3 install -e .
+popd
+
+echo "Run Tempest against configs:"
+cat /opt/stack/tempest/etc/tempest.conf
+
+echo "Run Tempest tests:"
+/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \
+    --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)'
+    #--exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes'
+popd
+
+
+if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
+
+    # Create cluster template
+    openstack coe cluster template create \
+        --image $(openstack image show ${IMAGE_NAME} -c id -f value) \
+        --external-network public \
+        --dns-nameserver ${DNS_NAMESERVER} \
+        --master-lb-enabled \
+        --master-flavor m1.large \
+        --flavor m1.large \
+        --network-driver ${NETWORK_DRIVER} \
+        --docker-storage-driver overlay2 \
+        --coe kubernetes \
+        --label kube_tag=${KUBE_TAG} \
+        --label fixed_subnet_cidr=192.168.24.0/24 \
+        k8s-${KUBE_TAG};
+
+    # Create cluster template for upgrade
+    openstack coe cluster template create \
+        --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \
+        --external-network public \
+        --dns-nameserver ${DNS_NAMESERVER} \
+        --master-lb-enabled \
+        --master-flavor m1.large \
+        --flavor m1.large \
+        --network-driver ${NETWORK_DRIVER} \
+        --docker-storage-driver overlay2 \
+        --coe kubernetes \
+        --label kube_tag=${UPGRADE_KUBE_TAG} \
+        --label fixed_subnet_cidr=192.168.24.0/24 \
+        k8s-${UPGRADE_KUBE_TAG};
+
+    # Create cluster
+    openstack coe cluster create \
+      --cluster-template k8s-${KUBE_TAG} \
+      --master-count 1 \
+      --node-count 1 \
+      --merge-labels \
+      --label audit_log_enabled=true \
+      k8s-cluster-upgrade
+
+    # Wait for cluster creation to be queued
+    set +e
+    for i in {1..5}; do
+      openstack coe cluster show k8s-cluster-upgrade 2>&1
+      exit_status=$?
+      if [ $exit_status -eq 0 ]; then
+          break
+      else
+          echo "Error: Cluster k8s-cluster-upgrade could not be found."
+          sleep 1
+      fi
+    done
+    set -e
+
+    # Wait for cluster to be "CREATE_COMPLETE".
+    for i in {1..240}; do
+      CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
+      if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
+        echo "Cluster failed to create"
+        exit 1
+      elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then
+        echo "Cluster created"
+        break
+      else
+        echo "Currtny retry count: $i"
+        echo "Cluster status: ${CLUSTER_STATUS}"
+        sleep 5
+      fi
+    done
+
+    # Upgrade cluster
+    openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG}
+    # Wait for cluster to be "UPDATE_COMPLETE".
+    for i in {1..240}; do
+      CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
+      if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
+        echo "Cluster failed to upgrade"
+        exit 1
+      elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then
+        echo "Cluster upgraded"
+        exit 0
+        break
+      else
+        echo "Currtny retry count: $i"
+        echo "Cluster status: ${CLUSTER_STATUS}"
+        sleep 5
+      fi
+    done
+    exit 1
+fi

hack/stack.sh

@@ -18,6 +18,9 @@
 sudo mkdir -p /opt/stack
 sudo chown -R ${USER}. /opt/stack
 
+sudo mkdir -p /etc/tempest/
+sudo chown -R ${USER}. /etc/tempest/
+
 # Clone repository if not present, otherwise update
 if [ ! -f /opt/stack/stack.sh ]; then
     git clone https://git.openstack.org/openstack-dev/devstack /opt/stack
@@ -62,6 +65,7 @@ enable_plugin barbican https://opendev.org/openstack/barbican
 enable_plugin octavia https://opendev.org/openstack/octavia
 enable_plugin ovn-octavia-provider https://opendev.org/openstack/ovn-octavia-provider
 enable_service octavia o-api o-cw o-hm o-hk o-da
+enable_service tempest
 
 # Magnum
 enable_plugin magnum https://opendev.org/openstack/magnum

magnum_cluster_api/tests/functional/conftest.py

@@ -25,6 +25,7 @@ def cluster(
     mock_validate_cluster,
     mock_osc,
     mock_certificates,
+    mock_get_server_group,
 ):
     try:
         ubuntu_driver.create_cluster(context, cluster_obj, 60)