diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh new file mode 100755 index 00000000..46cac1a7 --- /dev/null +++ b/hack/run-tempest-tests.sh @@ -0,0 +1,160 @@ +#!/bin/bash -xe + +# Copyright (c) 2024 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script will run the full functional tests for a given `KUBE_TAG`. It +# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE` +# and then run `sonobuoy` against it. + +source /opt/stack/openrc admin admin + +OS_DISTRO=${OS_DISTRO:-ubuntu} +IMAGE_OS=${IMAGE_OS:-ubuntu-2204} +NETWORK_DRIVER=${NETWORK_DRIVER:-calico} +DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} +UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} +IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" +UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}" + +# If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise +# we download the latest promoted image. +if [[ "${BUILD_NEW_IMAGE,,}" != "true" ]]; then + curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${IMAGE_NAME}.qcow2 +else + test -f ${IMAGE_NAME}.qcow2 || exit 1 +fi + +# Upload image to Glance +openstack image create \ + --disk-format=qcow2 \ + --public \ + --container-format=bare \ + --property os_distro=${OS_DISTRO} \ + --file=${IMAGE_NAME}.qcow2 \ + ${IMAGE_NAME} + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then + curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2 + else + test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1 + fi + # Upload Upgrade image to Glance + openstack image create \ + --disk-format=qcow2 \ + --public \ + --container-format=bare \ + --property os_distro=${OS_DISTRO} \ + --file=${UPGRADE_IMAGE_NAME}.qcow2 \ + ${UPGRADE_IMAGE_NAME} +fi + +pushd /opt/stack/tempest +/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ + --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' +popd + + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + + # Create cluster template + openstack coe cluster template create \ + --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${KUBE_TAG}; + + # Create cluster template for upgrade + openstack coe cluster template create \ + --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${UPGRADE_KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${UPGRADE_KUBE_TAG}; + + # Create cluster + openstack coe cluster create \ + --cluster-template k8s-${KUBE_TAG} \ + --master-count 1 \ + --node-count 1 \ + --merge-labels \ + --label audit_log_enabled=true \ + k8s-cluster-upgrade + + # Wait for cluster creation to be queued + set +e + for i in {1..5}; do + openstack coe cluster show k8s-cluster-upgrade 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + break + else + echo "Error: Cluster k8s-cluster-upgrade could not be found." + sleep 1 + fi + done + set -e + + # Wait for cluster to be "CREATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to create" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then + echo "Cluster created" + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to upgrade" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + exit 0 + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + exit 1 +fi diff --git a/hack/stack.sh b/hack/stack.sh index ebc84c55..39b23d6c 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -18,6 +18,9 @@ sudo mkdir -p /opt/stack sudo chown -R ${USER}. /opt/stack +sudo mkdir -p /etc/tempest/ +sudo chown -R ${USER}. /etc/tempest/ + # Clone repository if not present, otherwise update if [ ! -f /opt/stack/stack.sh ]; then git clone https://git.openstack.org/openstack-dev/devstack /opt/stack @@ -62,6 +65,7 @@ enable_plugin barbican https://opendev.org/openstack/barbican enable_plugin octavia https://opendev.org/openstack/octavia enable_plugin ovn-octavia-provider https://opendev.org/openstack/ovn-octavia-provider enable_service octavia o-api o-cw o-hm o-hk o-da +enable_service tempest # Magnum enable_plugin magnum https://opendev.org/openstack/magnum @@ -87,6 +91,18 @@ MANILA_USE_SERVICE_INSTANCE_PASSWORD=True [cluster_template] kubernetes_allowed_network_drivers = calico,cilium kubernetes_default_network_driver = calico + +[[post-config|$TEMPEST_CONFIG]] +[magnum] +flavor_id = m1.large +master_flavor_id = m1.large +copy_logs = true +network_driver = ${NETWORK_DRIVER:-calico} +image_id = ${IMAGE_OS:-ubuntu-2204}-kube-${KUBE_TAG} +coe = kubernetes +labels = '{\"kube_tag\": \"${KUBE_TAG}\", \"fixed_subnet_cidr\": \"10.0.0.0/26\"}' +docker_storage_driver = overlay2 + EOF # Start DevStack deployment @@ -111,5 +127,14 @@ EOF pip install -U setuptools pip python-magnumclient $HOME/.local/bin/pip3 install -e . +# install magnum-tempest-plugin with fix +git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin +pushd /opt/stack/magnum-tempest-plugin +git fetch https://review.opendev.org/openstack/magnum-tempest-plugin refs/changes/41/935741/1 && git checkout FETCH_HEAD +source /opt/stack/data/venv/bin/activate +#$HOME/.local/bin/pip3 install -e . +pip install . +popd + # Restart Magnum to pick-up new driver sudo systemctl restart devstack@magnum-{api,cond} diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index a45f79f8..746f6647 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -90,7 +90,7 @@ network_driver: cilium - project-template: - name: magnum-cluster-api-ubuntu-2204 + name: magnum-cluster-api-sonobuoy-ubuntu-2204 check: jobs: - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.28.11-calico @@ -101,3 +101,107 @@ - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.30.2-cilium - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-calico - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204 + parent: magnum-cluster-api-tempest + timeout: 10800 + abstract: true + vars: + image_operating_system: ubuntu-2204 + image_os_distro: ubuntu + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.28.11 + upgrade_kube_tag: v1.29.6 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-28-11-1719601167.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.29.6 + upgrade_kube_tag: v1.30.2 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.30.2 + upgrade_kube_tag: v1.31.1 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.31.1 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + vars: + network_driver: cilium + +- project-template: + name: magnum-cluster-api-tempest-ubuntu-2204 + check: + jobs: + - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index edb5b0bb..65c54b8b 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -30,6 +30,17 @@ files: - magnum_cluster_api/cmd/image_builder.py +- job: + name: magnum-cluster-api-tempest + abstract: true + timeout: 7200 + run: zuul.d/playbooks/tempest/run.yml + post-run: zuul.d/playbooks/tempest/post.yml + nodeset: + nodes: + - name: ubuntu-jammy + label: jammy-16c-64g + - job: name: magnum-cluster-api-sonobuoy abstract: true diff --git a/zuul.d/playbooks/tempest/post.yml b/zuul.d/playbooks/tempest/post.yml new file mode 100644 index 00000000..00b5f62b --- /dev/null +++ b/zuul.d/playbooks/tempest/post.yml @@ -0,0 +1,9 @@ +- hosts: all + tasks: + - name: Return built artifacts to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: "tempest Results" + url: "artifacts/tempest.log" diff --git a/zuul.d/playbooks/tempest/run.yml b/zuul.d/playbooks/tempest/run.yml new file mode 100644 index 00000000..d2d6de77 --- /dev/null +++ b/zuul.d/playbooks/tempest/run.yml @@ -0,0 +1,71 @@ +- hosts: all + tasks: + - name: Fetch artifact + get_url: + url: "{{ item.url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + register: fetch_artifact + loop: "{{ zuul.artifacts }}" + when: + - item.name is defined + - item.name == "{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + + - name: Download image + get_url: + url: "{{ image_url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + register: fetch_artifact + retries: 5 + delay: 10 + when: + - image_url is defined + + - name: Fetch artifact for upgrade + get_url: + url: "{{ item.url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact_upgrade + loop: "{{ zuul.artifacts }}" + when: + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + - item.name is defined + - item.name == "{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + + - name: Download image for upgrade + get_url: + url: "{{ upgrade_image_url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact_upgrade + retries: 5 + delay: 10 + when: + - upgrade_image_url is defined + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + + - shell: "./hack/stack.sh" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + IMAGE_OS: "{{ image_operating_system }}" + KUBE_TAG: "{{ kube_tag }}" + NETWORK_DRIVER: "{{ network_driver }}" + + - shell: "./hack/run-tempest-tests.sh" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + IMAGE_OS: "{{ image_operating_system }}" + OS_DISTRO: "{{ image_os_distro }}" + KUBE_TAG: "{{ kube_tag }}" + UPGRADE_KUBE_TAG: "{{ upgrade_kube_tag | default(kube_tag) }}" + BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" + BUILD_NEW_UPGRADE_IMAGE: "{{ fetch_artifact_upgrade.changed }}" + NETWORK_DRIVER: "{{ network_driver }}" + + - name: Copy Tempest results to output folder + copy: + src: "/opt/stack/tempest/tempest.log" + dest: "{{ zuul_output_dir }}/artifacts/tempest.log" + remote_src: true diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index c05ba3ac..04d0bb82 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -9,4 +9,5 @@ - magnum-cluster-api-flatcar - magnum-cluster-api-rockylinux-8 - magnum-cluster-api-rockylinux-9 - - magnum-cluster-api-ubuntu-2204 + - magnum-cluster-api-tempest-ubuntu-2204 + #- magnum-cluster-api-sonobuoy-ubuntu-2204