Skip to content

Commit

Permalink
Add Tempest support
Browse files Browse the repository at this point in the history
  • Loading branch information
ricolin committed Nov 25, 2024
1 parent d4e9ea2 commit a29769f
Show file tree
Hide file tree
Showing 7 changed files with 383 additions and 2 deletions.
160 changes: 160 additions & 0 deletions hack/run-tempest-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
#!/bin/bash -xe

# Copyright (c) 2024 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# This script will run the full functional tests for a given `KUBE_TAG`. It
# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE`
# and then run `sonobuoy` against it.

source /opt/stack/openrc admin admin

OS_DISTRO=${OS_DISTRO:-ubuntu}
IMAGE_OS=${IMAGE_OS:-ubuntu-2204}
NETWORK_DRIVER=${NETWORK_DRIVER:-calico}
DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1}
UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG}
IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}"
UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}"

# If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise
# we download the latest promoted image.
if [[ "${BUILD_NEW_IMAGE,,}" != "true" ]]; then
curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${IMAGE_NAME}.qcow2
else
test -f ${IMAGE_NAME}.qcow2 || exit 1
fi

# Upload image to Glance
openstack image create \
--disk-format=qcow2 \
--public \
--container-format=bare \
--property os_distro=${OS_DISTRO} \
--file=${IMAGE_NAME}.qcow2 \
${IMAGE_NAME}

if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then
if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then
curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2
else
test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1
fi
# Upload Upgrade image to Glance
openstack image create \
--disk-format=qcow2 \
--public \
--container-format=bare \
--property os_distro=${OS_DISTRO} \
--file=${UPGRADE_IMAGE_NAME}.qcow2 \
${UPGRADE_IMAGE_NAME}
fi

pushd /opt/stack/tempest
/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \
--exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)'
popd


if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then

# Create cluster template
openstack coe cluster template create \
--image $(openstack image show ${IMAGE_NAME} -c id -f value) \
--external-network public \
--dns-nameserver ${DNS_NAMESERVER} \
--master-lb-enabled \
--master-flavor m1.large \
--flavor m1.large \
--network-driver ${NETWORK_DRIVER} \
--docker-storage-driver overlay2 \
--coe kubernetes \
--label kube_tag=${KUBE_TAG} \
--label fixed_subnet_cidr=192.168.24.0/24 \
k8s-${KUBE_TAG};

# Create cluster template for upgrade
openstack coe cluster template create \
--image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \
--external-network public \
--dns-nameserver ${DNS_NAMESERVER} \
--master-lb-enabled \
--master-flavor m1.large \
--flavor m1.large \
--network-driver ${NETWORK_DRIVER} \
--docker-storage-driver overlay2 \
--coe kubernetes \
--label kube_tag=${UPGRADE_KUBE_TAG} \
--label fixed_subnet_cidr=192.168.24.0/24 \
k8s-${UPGRADE_KUBE_TAG};

# Create cluster
openstack coe cluster create \
--cluster-template k8s-${KUBE_TAG} \
--master-count 1 \
--node-count 1 \
--merge-labels \
--label audit_log_enabled=true \
k8s-cluster-upgrade

# Wait for cluster creation to be queued
set +e
for i in {1..5}; do
openstack coe cluster show k8s-cluster-upgrade 2>&1
exit_status=$?
if [ $exit_status -eq 0 ]; then
break
else
echo "Error: Cluster k8s-cluster-upgrade could not be found."
sleep 1
fi
done
set -e

# Wait for cluster to be "CREATE_COMPLETE".
for i in {1..240}; do
CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
echo "Cluster failed to create"
exit 1
elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then
echo "Cluster created"
break
else
echo "Currtny retry count: $i"
echo "Cluster status: ${CLUSTER_STATUS}"
sleep 5
fi
done

# Upgrade cluster
openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG}
# Wait for cluster to be "UPDATE_COMPLETE".
for i in {1..240}; do
CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value)
if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then
echo "Cluster failed to upgrade"
exit 1
elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then
echo "Cluster upgraded"
exit 0
break
else
echo "Currtny retry count: $i"
echo "Cluster status: ${CLUSTER_STATUS}"
sleep 5
fi
done
exit 1
fi
25 changes: 25 additions & 0 deletions hack/stack.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
sudo mkdir -p /opt/stack
sudo chown -R ${USER}. /opt/stack

sudo mkdir -p /etc/tempest/
sudo chown -R ${USER}. /etc/tempest/

# Clone repository if not present, otherwise update
if [ ! -f /opt/stack/stack.sh ]; then
git clone https://git.openstack.org/openstack-dev/devstack /opt/stack
Expand Down Expand Up @@ -62,6 +65,7 @@ enable_plugin barbican https://opendev.org/openstack/barbican
enable_plugin octavia https://opendev.org/openstack/octavia
enable_plugin ovn-octavia-provider https://opendev.org/openstack/ovn-octavia-provider
enable_service octavia o-api o-cw o-hm o-hk o-da
enable_service tempest
# Magnum
enable_plugin magnum https://opendev.org/openstack/magnum
Expand All @@ -87,6 +91,18 @@ MANILA_USE_SERVICE_INSTANCE_PASSWORD=True
[cluster_template]
kubernetes_allowed_network_drivers = calico,cilium
kubernetes_default_network_driver = calico
[[post-config|$TEMPEST_CONFIG]]
[magnum]
flavor_id = m1.large
master_flavor_id = m1.large
copy_logs = true
network_driver = ${NETWORK_DRIVER:-calico}
image_id = ${IMAGE_OS:-ubuntu-2204}-kube-${KUBE_TAG}
coe = kubernetes
labels = '{\"kube_tag\": \"${KUBE_TAG}\", \"fixed_subnet_cidr\": \"10.0.0.0/26\"}'
docker_storage_driver = overlay2
EOF

# Start DevStack deployment
Expand All @@ -111,5 +127,14 @@ EOF
pip install -U setuptools pip python-magnumclient
$HOME/.local/bin/pip3 install -e .

# install magnum-tempest-plugin with fix
git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin
pushd /opt/stack/magnum-tempest-plugin
git fetch https://review.opendev.org/openstack/magnum-tempest-plugin refs/changes/41/935741/1 && git checkout FETCH_HEAD
source /opt/stack/data/venv/bin/activate
#$HOME/.local/bin/pip3 install -e .
pip install .
popd

# Restart Magnum to pick-up new driver
sudo systemctl restart devstack@magnum-{api,cond}
106 changes: 105 additions & 1 deletion zuul.d/jobs-ubuntu-2204.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@
network_driver: cilium

- project-template:
name: magnum-cluster-api-ubuntu-2204
name: magnum-cluster-api-sonobuoy-ubuntu-2204
check:
jobs:
- magnum-cluster-api-sonobuoy-ubuntu-2204-v1.28.11-calico
Expand All @@ -101,3 +101,107 @@
- magnum-cluster-api-sonobuoy-ubuntu-2204-v1.30.2-cilium
- magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-calico
- magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-cilium

- job:
name: magnum-cluster-api-tempest-ubuntu-2204
parent: magnum-cluster-api-tempest
timeout: 10800
abstract: true
vars:
image_operating_system: ubuntu-2204
image_os_distro: ubuntu

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11
parent: magnum-cluster-api-tempest-ubuntu-2204
vars:
kube_tag: v1.28.11
upgrade_kube_tag: v1.29.6
image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-28-11-1719601167.qcow2
upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11
vars:
network_driver: calico

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11
vars:
network_driver: cilium

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6
parent: magnum-cluster-api-tempest-ubuntu-2204
vars:
kube_tag: v1.29.6
upgrade_kube_tag: v1.30.2
image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2
upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6
vars:
network_driver: calico

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6
vars:
network_driver: cilium

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2
parent: magnum-cluster-api-tempest-ubuntu-2204
vars:
kube_tag: v1.30.2
upgrade_kube_tag: v1.31.1
image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2
upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2
vars:
network_driver: calico

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2
vars:
network_driver: cilium

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1
parent: magnum-cluster-api-tempest-ubuntu-2204
vars:
kube_tag: v1.31.1
image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1
vars:
network_driver: calico

- job:
name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium
parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1
vars:
network_driver: cilium

- project-template:
name: magnum-cluster-api-tempest-ubuntu-2204
check:
jobs:
- magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico
- magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium
- magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico
- magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium
- magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico
- magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium
- magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico
- magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium
11 changes: 11 additions & 0 deletions zuul.d/jobs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,17 @@
files:
- magnum_cluster_api/cmd/image_builder.py

- job:
name: magnum-cluster-api-tempest
abstract: true
timeout: 7200
run: zuul.d/playbooks/tempest/run.yml
post-run: zuul.d/playbooks/tempest/post.yml
nodeset:
nodes:
- name: ubuntu-jammy
label: jammy-16c-64g

- job:
name: magnum-cluster-api-sonobuoy
abstract: true
Expand Down
9 changes: 9 additions & 0 deletions zuul.d/playbooks/tempest/post.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
- hosts: all
tasks:
- name: Return built artifacts to Zuul
zuul_return:
data:
zuul:
artifacts:
- name: "tempest Results"
url: "artifacts/tempest.log"
Loading

0 comments on commit a29769f

Please sign in to comment.