Merge pull request #13610 from rkachach/fix_issue_pvc_minikube_test #41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Canary integration tests | |
| on: | |
| push: | |
| tags: | |
| - v* | |
| branches: | |
| - master | |
| - release-* | |
| pull_request: | |
| branches: | |
| - master | |
| - release-* | |
| paths-ignore: | |
| - "Documentation/**" | |
| - "design/**" | |
| workflow_dispatch: | |
| inputs: | |
| ceph-image: | |
| description: 'Ceph image for creating Ceph cluster' | |
| default: 'quay.io/ceph/ceph:v18' | |
| defaults: | |
| run: | |
| # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell | |
| shell: bash --noprofile --norc -eo pipefail -x {0} | |
| # cancel the in-progress workflow when PR is refreshed. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| canary: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: deploy cluster | |
| run: tests/scripts/github-action-helper.sh deploy_cluster | |
| - name: setup csi-addons | |
| run: tests/scripts/csiaddons.sh setup_csiaddons | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 2 | |
| - name: wait for ceph mgr to be ready | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr|grep -Eosq \"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\" ; do sleep 1 && echo 'waiting for the manager IP to be available'; done" | |
| mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr) | |
| timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done" | |
| - name: test external script create-external-cluster-resources.py | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data | |
| kubectl -n rook-ceph cp tests/ceph-status-out $toolbox:/etc/ceph/test-data/ | |
| kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources.py $toolbox:/etc/ceph | |
| kubectl -n rook-ceph cp deploy/examples/create-external-cluster-resources-tests.py $toolbox:/etc/ceph | |
| timeout 10 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool; do echo 'waiting for script to succeed' && sleep 1; done" | |
| # print existing client auth | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth ls | |
| - name: test re-running of external script should result in same output | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output1.txt | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool | tee output2.txt | |
| if cmp output1.txt output2.txt; then | |
| echo "files have same output" | |
| rm output1.txt | |
| rm output2.txt | |
| else | |
| echo "re-run with same flags changed the output, result in failure" | |
| rm output1.txt | |
| rm output2.txt | |
| exit 1 | |
| fi | |
| - name: dry run external script create-external-cluster-resources.py | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run | |
| - name: test external script create-external-cluster-resources.py if users already exist with different caps | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # update client.csi-rbd-provisioner csi user caps | |
| # print client.csi-rbd-provisioner user before update | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth caps client.csi-rbd-provisioner mon 'profile rbd, allow command "osd ls"' osd 'profile rbd' mgr 'allow rw' | |
| # print client.csi-rbd-provisioner user after update | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool | |
| # print client.csi-rbd-provisioner user after running script | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-provisioner | |
| - name: run external script create-external-cluster-resources.py unit tests | |
| run: | | |
| kubectl -n rook-ceph exec $(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[0].metadata.name}') -- python3 -m unittest /etc/ceph/create-external-cluster-resources-tests.py | |
| - name: wait for the subvolumegroup to be created | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done" | |
| - name: test subvolumegroup validation | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # pass the correct subvolumegroup and cephfs_filesystem flag name | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs | |
| # pass the subvolumegroup name which doesn't exist | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group | |
| - name: dry run test skip monitoring endpoint | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name=replicapool --dry-run --skip-monitoring-endpoint | |
| - name: test of rados namespace | |
| run: | | |
| kubectl create -f deploy/examples/radosnamespace.yaml | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done" | |
| kubectl delete -f deploy/examples/radosnamespace.yaml | |
| - name: test rados namespace validation | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool | |
| kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosnamespace1 | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosnamespace1 | |
| # test the rados namespace which not exit for replicapool(false testing) | |
| if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then | |
| echo "unexpectedly succeeded after passing the wrong rados namespace: $output" | |
| exit 1 | |
| else | |
| echo "script failed because wrong rados namespace was passed" | |
| fi | |
| - name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true | |
| - name: test external script with restricted_auth_permission flag and with --cluster-name legacy flag | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cluster-name rookstorage --restricted-auth-permission true | |
| - name: test external script with restricted_auth_permission flag | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --cephfs-filesystem-name myfs --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --restricted-auth-permission true | |
| - name: test the upgrade flag | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # print existing client auth | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth ls | |
| # update the existing non-restricted client auth with the new ones | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade | |
| # print upgraded client auth | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth ls | |
| - name: test the upgrade flag for restricted auth user | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # print existing client auth | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool | |
| # restricted auth user need to provide --rbd-data-pool-name, | |
| # --k8s-cluster-name and --run-as-user flag while upgrading | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool | |
| # print upgraded client auth | |
| kubectl -n rook-ceph exec $toolbox -- ceph auth get client.csi-rbd-node-rookstorage-replicapool | |
| - name: validate-rgw-endpoint | |
| run: | | |
| rgw_endpoint=$(kubectl get service -n rook-ceph | awk '/rgw/ {print $3":80"}') | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # pass the valid rgw-endpoint of same ceph cluster | |
| timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" | |
| tests/scripts/github-action-helper.sh check_empty_file output.txt | |
| rm -f output.txt | |
| # pass the invalid rgw-endpoint of different ceph cluster | |
| timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" | |
| if [ -s output.txt ]; then | |
| echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output" | |
| rm -f output.txt | |
| else | |
| echo "no stderr error even wrong endpoint was provided" | |
| rm -f output.txt | |
| exit 1 | |
| fi | |
| # pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path | |
| timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" | |
| tests/scripts/github-action-helper.sh check_empty_file output.txt | |
| rm -f output.txt | |
| # pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls | |
| timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" | |
| tests/scripts/github-action-helper.sh check_empty_file output.txt | |
| rm -f output.txt | |
| - name: validate multisite | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| # create realm | |
| kubectl -n rook-ceph exec $toolbox -- radosgw-admin realm create --rgw-realm=realm1 | |
| # pass correct realm | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm1 | |
| # pass wrong realm | |
| if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm3); then | |
| echo "script run completed with stderr error after passing the wrong realm: $output" | |
| else | |
| echo "script failed because wrong realm was passed" | |
| fi | |
| - name: test enable v2 mon port | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --v2-port-enable | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: validate csi-addons | |
| run: | | |
| tests/scripts/csiaddons.sh verify_crd_created | |
| tests/scripts/csiaddons.sh verify_container_is_running | |
| - name: test osd removal jobs | |
| run: | | |
| kubectl -n rook-ceph delete deploy/rook-ceph-operator | |
| kubectl -n rook-ceph delete deploy/rook-ceph-osd-1 --grace-period=0 --force | |
| sed -i 's/<OSD-IDs>/1/' deploy/examples/osd-purge.yaml | |
| # the CI must force the deletion since we use replica 1 on 2 OSDs | |
| sed -i 's/false/true/' deploy/examples/osd-purge.yaml | |
| sed -i 's|rook/ceph:.*|rook/ceph:local-build|' deploy/examples/osd-purge.yaml | |
| kubectl -n rook-ceph create -f deploy/examples/osd-purge.yaml | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| kubectl -n rook-ceph exec $toolbox -- ceph status | |
| # wait until osd.1 is removed from the osd tree | |
| timeout 120 sh -c "while kubectl -n rook-ceph exec $toolbox -- ceph osd tree|grep -qE 'osd.1'; do echo 'waiting for ceph osd 1 to be purged'; sleep 1; done" | |
| kubectl -n rook-ceph exec $toolbox -- ceph status | |
| kubectl -n rook-ceph exec $toolbox -- ceph osd tree | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| raw-disk: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk as OSD | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only | |
| - name: prepare loop devices for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh prepare_loop_devices 1 | |
| - name: deploy cluster | |
| run: | | |
| export ALLOW_LOOP_DEVICES=true | |
| tests/scripts/github-action-helper.sh deploy_cluster loop | |
| tests/scripts/github-action-helper.sh create_operator_toolbox | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| - name: test toolbox-operator-image pod | |
| run: | | |
| # waiting for toolbox operator image pod to get ready | |
| kubectl -n rook-ceph wait --for=condition=ready pod -l app=rook-ceph-tools-operator-image --timeout=180s | |
| - name: check s5cmd version in the toolbox image | |
| run: | | |
| toolbox=$(kubectl get pod -l app=rook-ceph-tools-operator-image -n rook-ceph -o jsonpath='{.items[*].metadata.name}') | |
| s5cmd_version="$(kubectl -n rook-ceph exec ${toolbox} -- /usr/local/bin/s5cmd version)" | |
| echo ${s5cmd_version} | grep -q "^v2.2.1" || { | |
| echo " Error: the version of s5cmd version in the toolbox is not the expected v2.2.1 but ${s5cmd_version}" | |
| exit 1 | |
| } | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| two-osds-in-device: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk as OSD | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only | |
| - name: deploy cluster | |
| run: tests/scripts/github-action-helper.sh deploy_cluster two_osds_in_device | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| osd-with-metadata-device: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk as OSD | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only | |
| - name: create LV on disk | |
| run: | | |
| dd if=/dev/zero of=test-rook.img bs=1 count=0 seek=10G | |
| # If we use metadata device, both data devices and metadata devices should be logical volumes or raw devices | |
| tests/scripts/github-action-helper.sh create_LV_on_disk $(sudo losetup --find --show test-rook.img) | |
| - name: deploy cluster | |
| run: tests/scripts/github-action-helper.sh deploy_cluster osd_with_metadata_device | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| encryption: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk as OSD | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only | |
| - name: deploy cluster | |
| run: tests/scripts/github-action-helper.sh deploy_cluster encryption | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| lvm: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk as OSD | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only | |
| - name: create LV on disk | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK | |
| - name: deploy cluster | |
| run: tests/scripts/github-action-helper.sh deploy_cluster lvm | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary | |
| pvc: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: prepare loop devices for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh prepare_loop_devices 1 | |
| - name: create cluster prerequisites | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/localPathPV.sh "$BLOCK" | |
| tests/scripts/loopDevicePV.sh 1 | |
| tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy cluster | |
| run: | | |
| yq write -i deploy/examples/operator.yaml "data.ROOK_CEPH_ALLOW_LOOP_DEVICES" --style=double "true" | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 3 | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 3 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 3 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: teardown cluster with cleanup policy | |
| run: | | |
| kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' | |
| kubectl -n rook-ceph delete cephcluster rook-ceph | |
| kubectl -n rook-ceph logs deploy/rook-ceph-operator | |
| tests/scripts/github-action-helper.sh wait_for_cleanup_pod | |
| lsblk | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| sudo head --bytes=60 ${BLOCK}1 | |
| sudo head --bytes=60 ${BLOCK}2 | |
| sudo head --bytes=60 /dev/loop1 | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: pvc | |
| pvc-db: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk | |
| run: tests/scripts/github-action-helper.sh use_local_disk | |
| - name: create bluestore partitions and PVCs | |
| run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs | |
| - name: create cluster prerequisites | |
| run: tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false | |
| cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: pvc-db | |
| pvc-db-wal: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk | |
| run: tests/scripts/github-action-helper.sh use_local_disk | |
| - name: create bluestore partitions and PVCs for wal | |
| run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal | |
| - name: create cluster prerequisites | |
| run: tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy rook | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false | |
| cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| kubectl -n rook-ceph get pods | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: pvc-db-wal | |
| encryption-pvc: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: create cluster prerequisites | |
| run: | | |
| tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| kubectl -n rook-ceph get secrets | |
| sudo lsblk | |
| - name: teardown cluster with cleanup policy | |
| run: | | |
| kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' | |
| kubectl -n rook-ceph delete cephcluster rook-ceph | |
| kubectl -n rook-ceph logs deploy/rook-ceph-operator | |
| tests/scripts/github-action-helper.sh wait_for_cleanup_pod | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| sudo head --bytes=60 ${BLOCK}1 | |
| sudo head --bytes=60 ${BLOCK}2 | |
| sudo lsblk | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: encryption-pvc | |
| encryption-pvc-db: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk | |
| run: tests/scripts/github-action-helper.sh use_local_disk | |
| - name: create bluestore partitions and PVCs | |
| run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs | |
| - name: create cluster prerequisites | |
| run: tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| kubectl -n rook-ceph get pods | |
| kubectl -n rook-ceph get secrets | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: encryption-pvc-db | |
| encryption-pvc-db-wal: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk | |
| run: tests/scripts/github-action-helper.sh use_local_disk | |
| - name: create bluestore partitions and PVCs for wal | |
| run: tests/scripts/github-action-helper.sh create_bluestore_partitions_and_pvcs_for_wal | |
| - name: create cluster prerequisites | |
| run: tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy rook | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| cat tests/manifests/test-on-pvc-db.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| cat tests/manifests/test-on-pvc-wal.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| kubectl patch -n rook-ceph cephcluster rook-ceph --type merge -p '{"spec":{"security":{"keyRotation":{"enabled": true, "schedule":"*/1 * * * *"}}}}' | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| kubectl -n rook-ceph get pods | |
| kubectl -n rook-ceph get secrets | |
| - name: wait and verify key rotation | |
| run: tests/scripts/github-action-helper.sh verify_key_rotation | |
| - name: test osd deployment removal and re-hydration | |
| run: | | |
| kubectl -n rook-ceph delete deploy/rook-ceph-osd-0 | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: encryption-pvc-db-wal | |
| encryption-pvc-kms-vault-token-auth: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: create cluster prerequisites | |
| run: | | |
| tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy vault | |
| run: tests/scripts/deploy-validate-vault.sh deploy | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| cat tests/manifests/test-kms-vault.yaml >> tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| yq merge --inplace --arrays append tests/manifests/test-object.yaml tests/manifests/test-kms-vault-spec-token-auth.yaml | |
| yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_BACKEND_PATH" rook/ver2 | |
| kubectl create -f tests/manifests/test-object.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| tests/scripts/validate_cluster.sh rgw | |
| kubectl -n rook-ceph get pods | |
| kubectl -n rook-ceph get secrets | |
| - name: validate osd vault | |
| run: | | |
| tests/scripts/deploy-validate-vault.sh validate_osd | |
| sudo lsblk | |
| - name: validate rgw vault kv | |
| run: | | |
| tests/scripts/deploy-validate-vault.sh validate_rgw | |
| - name: validate rgw vault transit | |
| run: | | |
| kubectl delete -f tests/manifests/test-object.yaml | |
| yq write -i tests/manifests/test-object.yaml "spec.security.kms.connectionDetails.VAULT_SECRET_ENGINE" transit | |
| timeout 120 bash -c 'while kubectl -n rook-ceph get cephobjectstore my-store; do echo "waiting for objectstore my-store to delete"; sleep 5; done' | |
| echo "wait for rgw pod to be deleted" | |
| kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s | |
| kubectl create -f tests/manifests/test-object.yaml | |
| tests/scripts/validate_cluster.sh rgw | |
| tests/scripts/deploy-validate-vault.sh validate_rgw | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: encryption-pvc-kms-vault-token-auth | |
| encryption-pvc-kms-vault-k8s-auth: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: create cluster prerequisites | |
| run: | | |
| tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: deploy vault | |
| run: KUBERNETES_AUTH=true tests/scripts/deploy-validate-vault.sh deploy | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq merge --inplace --arrays append tests/manifests/test-cluster-on-pvc-encrypted.yaml tests/manifests/test-kms-vault-spec-k8s-auth.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].count" 2 | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].volumeClaimTemplates[0].spec.resources.requests.storage" 6Gi | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: | | |
| tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| kubectl -n rook-ceph get pods | |
| kubectl -n rook-ceph get secrets | |
| - name: validate osd vault | |
| run: | | |
| tests/scripts/deploy-validate-vault.sh validate_osd | |
| sudo lsblk | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: encryption-pvc-kms-vault-k8s-auth | |
| lvm-pvc: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "tests/manifests/test-cluster-on-pvc-encrypted.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: create cluster prerequisites | |
| run: tests/scripts/github-action-helper.sh create_cluster_prerequisites | |
| - name: use local disk | |
| run: tests/scripts/github-action-helper.sh use_local_disk | |
| - name: create LV on disk | |
| run: | | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK | |
| tests/scripts/localPathPV.sh /dev/test-rook-vg/test-rook-lv | |
| - name: deploy cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/operator.yaml | |
| yq write -i tests/manifests/test-cluster-on-pvc-encrypted.yaml "spec.storage.storageClassDeviceSets[0].encrypted" false | |
| kubectl create -f tests/manifests/test-cluster-on-pvc-encrypted.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/toolbox.yaml | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 1 | |
| - name: wait for ceph to be ready | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 1 | |
| - name: check-ownerreferences | |
| run: tests/scripts/github-action-helper.sh check_ownerreferences | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: lvm-pvc | |
| multi-cluster-mirroring: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk into two partitions | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) | |
| tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 | |
| sudo lsblk | |
| - name: deploy first cluster rook | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_first_rook_cluster | |
| cd deploy/examples/ | |
| sed -i "/resources:/,/ # priorityClassName:/d" rbdmirror.yaml | |
| sed -i "/resources:/,/ # priorityClassName:/d" filesystem-mirror.yaml | |
| kubectl create -f rbdmirror.yaml -f filesystem-mirror.yaml | |
| # cephfs-mirroring is a push operation | |
| # running bootstrap create on secondary and bootstrap import on primary. mirror daemons on primary. | |
| - name: deploy second cluster rook | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_second_rook_cluster | |
| cd deploy/examples/ | |
| sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' rbdmirror.yaml | |
| kubectl create -f rbdmirror.yaml | |
| - name: wait for ceph cluster 1 to be ready | |
| run: | | |
| mkdir -p test | |
| tests/scripts/validate_cluster.sh osd 1 | |
| kubectl -n rook-ceph get pods | |
| - name: create replicated mirrored pool on cluster 1 | |
| run: | | |
| cd deploy/examples/ | |
| yq w -i pool-test.yaml spec.mirroring.enabled true | |
| yq w -i pool-test.yaml spec.mirroring.mode image | |
| kubectl create -f pool-test.yaml | |
| timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 1" && sleep 1; done' | |
| - name: create replicated mirrored pool 2 on cluster 1 | |
| run: | | |
| cd deploy/examples/ | |
| yq w -i pool-test.yaml metadata.name replicapool2 | |
| kubectl create -f pool-test.yaml | |
| timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph get cephblockpool replicapool2 -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 1" && sleep 1; done' | |
| yq w -i pool-test.yaml metadata.name replicapool | |
| - name: create replicated mirrored pool on cluster 2 | |
| run: | | |
| cd deploy/examples/ | |
| yq w -i pool-test.yaml metadata.namespace rook-ceph-secondary | |
| kubectl create -f pool-test.yaml | |
| timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool to be created on cluster 2" && sleep 1; done' | |
| - name: create replicated mirrored pool 2 on cluster 2 | |
| run: | | |
| cd deploy/examples/ | |
| yq w -i pool-test.yaml metadata.name replicapool2 | |
| kubectl create -f pool-test.yaml | |
| timeout 180 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for pool replicapool2 to be created on cluster 2" && sleep 1; done' | |
| - name: create images in the pools | |
| run: | | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool create test -s 1G | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool/test snapshot | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -ti -- rbd -p replicapool2 create test -s 1G | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd mirror image enable replicapool2/test snapshot | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test | |
| - name: copy block mirror peer secret into the other cluster for replicapool | |
| run: | | |
| kubectl -n rook-ceph get secret pool-peer-token-replicapool -o yaml > pool-peer-token-replicapool.yaml | |
| yq delete --inplace pool-peer-token-replicapool.yaml metadata.ownerReferences | |
| yq write --inplace pool-peer-token-replicapool.yaml metadata.namespace rook-ceph-secondary | |
| yq write --inplace pool-peer-token-replicapool.yaml metadata.name pool-peer-token-replicapool-config | |
| kubectl create --namespace=rook-ceph-secondary -f pool-peer-token-replicapool.yaml | |
| - name: copy block mirror peer secret into the other cluster for replicapool2 (using cluster global peer) | |
| run: | | |
| kubectl -n rook-ceph get secret cluster-peer-token-my-cluster -o yaml > cluster-peer-token-my-cluster.yaml | |
| yq delete --inplace cluster-peer-token-my-cluster.yaml metadata.ownerReferences | |
| yq write --inplace cluster-peer-token-my-cluster.yaml metadata.namespace rook-ceph-secondary | |
| yq write --inplace cluster-peer-token-my-cluster.yaml metadata.name cluster-peer-token-my-cluster-config | |
| kubectl create --namespace=rook-ceph-secondary -f cluster-peer-token-my-cluster.yaml | |
| - name: add block mirror peer secret to the other cluster for replicapool | |
| run: | | |
| kubectl -n rook-ceph-secondary patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}' | |
| - name: add block mirror peer secret to the other cluster for replicapool2 (using cluster global peer) | |
| run: | | |
| kubectl -n rook-ceph-secondary patch cephblockpool replicapool2 --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["cluster-peer-token-my-cluster-config"]}}}}' | |
| - name: verify image has been mirrored for replicapool | |
| run: | | |
| # let's wait a bit for the image to be present | |
| timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool" && sleep 1; done' | |
| - name: verify image has been mirrored for replicapool2 | |
| run: | | |
| # let's wait a bit for the image to be present | |
| timeout 120 sh -c 'until [ "$(kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- rbd -p replicapool2 ls|grep -c test)" -eq 1 ]; do echo "waiting for image to be mirrored in pool replicapool2" && sleep 1; done' | |
| - name: display cephblockpool and image status | |
| run: | | |
| timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool" && sleep 1; done' | |
| timeout 80 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o jsonpath='{.status.mirroringStatus.summary.daemon_health}'|grep -c OK)" -eq 1 ]; do echo "waiting for mirroring status to be updated in replicapool2" && sleep 1; done' | |
| kubectl -n rook-ceph-secondary get cephblockpool replicapool -o yaml | |
| kubectl -n rook-ceph-secondary get cephblockpool replicapool2 -o yaml | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool info test | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- rbd -p replicapool2 info test | |
| - name: copy block mirror peer secret into the primary cluster for replicapool | |
| run: | | |
| kubectl -n rook-ceph-secondary get secret pool-peer-token-replicapool -o yaml |\ | |
| sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: pool-peer-token-replicapool/name: pool-peer-token-replicapool-config/g' |\ | |
| kubectl create --namespace=rook-ceph -f - | |
| - name: add block mirror peer secret to the primary cluster for replicapool | |
| run: | | |
| kubectl -n rook-ceph patch cephblockpool replicapool --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["pool-peer-token-replicapool-config"]}}}}' | |
| - name: wait for rook-ceph-csi-mapping-config to be updated with cluster ID | |
| run: | | |
| timeout 60 sh -c 'until [ "$(kubectl get cm -n rook-ceph rook-ceph-csi-mapping-config -o jsonpath='{.data.csi-mapping-config-json}' | grep -c "rook-ceph-secondary")" -eq 1 ]; do echo "waiting for rook-ceph-csi-mapping-config to be created with cluster ID mappings" && sleep 1; done' | |
| - name: create replicated mirrored filesystem on cluster 1 | |
| run: | | |
| PRIMARY_YAML=deploy/examples/filesystem-test-primary.yaml | |
| cp deploy/examples/filesystem-test.yaml "$PRIMARY_YAML" | |
| yq merge --inplace --arrays append "$PRIMARY_YAML" tests/manifests/test-fs-mirror-spec.yaml | |
| kubectl create -f "$PRIMARY_YAML" | |
| - name: create replicated mirrored filesystem on cluster 2 | |
| run: | | |
| cd deploy/examples/ | |
| sed -i 's/namespace: rook-ceph/namespace: rook-ceph-secondary/g' filesystem-test.yaml | |
| yq w -i filesystem-test.yaml spec.mirroring.enabled true | |
| kubectl create -f filesystem-test.yaml | |
| - name: wait for filesystem on cluster 1 | |
| run: | | |
| timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done' | |
| - name: wait for filesystem on cluster 2 | |
| run: | | |
| timeout 300 sh -c 'until [ "$(kubectl -n rook-ceph-secondary get cephfilesystem myfs -o jsonpath='{.status.phase}'|grep -c "Ready")" -eq 1 ]; do echo "waiting for filesystem to be created" && sleep 1; done' | |
| - name: copy filesystem mirror peer secret from the secondary cluster to the primary one | |
| run: | | |
| kubectl -n rook-ceph-secondary get secret fs-peer-token-myfs -o yaml |\ | |
| sed '/ownerReferences/,+6d' |\ | |
| sed 's/namespace: rook-ceph-secondary/namespace: rook-ceph/g; s/name: fs-peer-token-myfs/name: fs-peer-token-myfs-config/g' |\ | |
| kubectl create --namespace=rook-ceph -f - | |
| - name: add filesystem mirror peer secret to the primary cluster | |
| run: | | |
| kubectl -n rook-ceph patch cephfilesystem myfs --type merge -p '{"spec":{"mirroring":{"peers": {"secretNames": ["fs-peer-token-myfs-config"]}}}}' | |
| - name: verify fs mirroring is working | |
| run: | | |
| timeout 45 sh -c 'until [ "$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/|grep -c asok)" -gt 3 ]; do echo "waiting for connection to peer" && sleep 1; done' | |
| sockets=$(kubectl -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ls -1 /var/run/ceph/) | |
| status=$(for socket in $sockets; do minikube kubectl -- -n rook-ceph exec -t deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$socket help|awk -F ":" '/get filesystem mirror status/ {print $1}'; done) | |
| if [ "${#status}" -lt 1 ]; then echo "peer addition failed" && exit 1; fi | |
| - name: display cephfilesystem and fs mirror daemon status | |
| run: | | |
| kubectl -n rook-ceph get cephfilesystem myfs -o yaml | |
| # the check is not super ideal since 'mirroring_failed' is only displayed when there is a failure but not when it's working... | |
| timeout 60 sh -c 'while [ "$(kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror daemon status|jq -r '.[0].filesystems[0]'|grep -c "mirroring_failed")" -eq 1 ]; do echo "waiting for filesystem to be mirrored" && sleep 1; done' | |
| - name: Create subvolume on primary cluster | |
| run: | | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume | |
| - name: Create subvolume of same name on secondary cluster | |
| run: | | |
| kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs subvolume create myfs testsubvolume | |
| - name: Deploy Direct Tools pod on primary cluster | |
| run: | | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml | |
| - name: Deploy Direct Tools pod on secondary cluster | |
| run: | | |
| sed -i "s/rook-ceph # namespace/rook-ceph-secondary # namespace/" deploy/examples/direct-mount.yaml | |
| tests/scripts/github-action-helper.sh deploy_manifest_with_local_build deploy/examples/direct-mount.yaml | |
| - name: Configure a directory for snapshot mirroring on primary cluster | |
| run: | | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs snapshot mirror add myfs /volumes/_nogroup/testsubvolume/ | |
| - name: make sure that snapshot mirror is enabled on the secondary cluster | |
| run: | | |
| kubectl exec -n rook-ceph-secondary deploy/rook-ceph-tools -t -- ceph fs snapshot mirror enable myfs | |
| - name: Create 3 snapshots on cluster primary cluster | |
| run: | | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap1 | |
| kubectl exec -n rook-ceph deploy/rook-ceph-tools -t -- ceph fs subvolume snapshot create myfs testsubvolume snap2 | |
| - name: Get the peer and verify the peer synchronization status that snaps have synced on secondary cluster | |
| run: | | |
| exec_fs_mirror='kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror --' | |
| mirror_daemon=$($exec_fs_mirror ls /var/run/ceph/ | grep "fs-mirror" | head -n 1) | |
| # timeout 45 bash -x <<EOF | |
| # while | |
| # clusterfsid=\$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]') | |
| # [ -z "\$clusterfsid" ] | |
| # do echo "Waiting for the clusterfsid to get populated." && sleep 1 | |
| # done | |
| # EOF | |
| # clusterfsid=$($exec_fs_mirror ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror status myfs@1 |jq -r '.peers|keys[]') | |
| # echo $clusterfsid | |
| # kubectl -n rook-ceph-secondary wait pod -l app=rook-direct-mount --for condition=Ready --timeout=400s | |
| # kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mkdir /tmp/registry | |
| # mon_endpoints=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep mon_host /etc/ceph/ceph.conf | awk '{print $3}') | |
| # my_secret=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- grep key /etc/ceph/keyring | awk '{print $3}') | |
| # kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- mount -t ceph -o mds_namespace=myfs,name=admin,secret=$my_secret $mon_endpoints:/ /tmp/registry | |
| # num_snaps_target=$(kubectl -n rook-ceph-secondary exec deploy/rook-direct-mount -- ls -lhsa /tmp/registry/volumes/_nogroup/testsubvolume/.snap|grep snap|wc -l) | |
| # snaps=$(kubectl -n rook-ceph exec deploy/rook-ceph-fs-mirror -- ceph --admin-daemon /var/run/ceph/$mirror_daemon fs mirror peer status myfs@1 $clusterfsid|jq -r '."/volumes/_nogroup/testsubvolume"."snaps_synced"') | |
| # echo "snapshots: $snaps" | |
| # if [ $num_snaps_target = $snaps ] | |
| # then echo "Snapshots have synced." | |
| # else echo "Snapshots have not synced. NEEDS INVESTIGATION" | |
| # fi | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: multi-cluster-mirroring | |
| rgw-multisite-testing: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: run RGW multisite test | |
| uses: ./.github/workflows/rgw-multisite-test | |
| - name: upload test result | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: rgw-multisite-testing | |
| path: test | |
| encryption-pvc-kms-ibm-kp: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: run encryption KMS IBM Key Protect | |
| uses: ./.github/workflows/encryption-pvc-kms-ibm-kp | |
| if: "env.IBM_KP_SERVICE_INSTANCE_ID != '' && env.IBM_KP_SERVICE_API_KEY != ''" | |
| env: | |
| IBM_KP_SERVICE_INSTANCE_ID: ${{ secrets.IBM_INSTANCE_ID }} | |
| IBM_KP_SERVICE_API_KEY: ${{ secrets.IBM_SERVICE_API_KEY }} | |
| with: | |
| ibm-instance-id: ${{ secrets.IBM_INSTANCE_ID }} | |
| ibm-service-api-key: ${{ secrets.IBM_SERVICE_API_KEY }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: upload test result | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: encryption-pvc-kms-ibm-kp | |
| path: test | |
| multus-cluster-network: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.21" | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: install deps | |
| shell: bash --noprofile --norc -eo pipefail -x {0} | |
| run: tests/scripts/github-action-helper.sh install_deps | |
| - name: print k8s cluster status | |
| shell: bash --noprofile --norc -eo pipefail -x {0} | |
| run: tests/scripts/github-action-helper.sh print_k8s_cluster_status | |
| - name: build rook | |
| shell: bash --noprofile --norc -eo pipefail -x {0} | |
| run: tests/scripts/github-action-helper.sh build_rook | |
| - name: validate-yaml | |
| run: tests/scripts/github-action-helper.sh validate_yaml | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: deploy multus | |
| run: tests/scripts/github-action-helper.sh deploy_multus | |
| - name: deploy multus cluster | |
| run: tests/scripts/github-action-helper.sh deploy_multus_cluster | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| - name: wait for ceph-csi configmap to be updated with network namespace | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_csi_configmap_to_be_updated | |
| - name: wait for cephnfs to be ready | |
| run: IS_POD_NETWORK=true IS_MULTUS=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready nfs 1 | |
| - name: check multus connections | |
| run: tests/scripts/github-action-helper.sh test_multus_connections | |
| - name: test ceph-csi-rbd plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_rbd_workload | |
| - name: test ceph-csi-cephfs plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_cephfs_workload | |
| - name: test ceph-csi-nfs plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_nfs_workload | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: canary-multus | |
| csi-hostnetwork-disabled: | |
| runs-on: ubuntu-20.04 | |
| if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')" | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: consider debugging | |
| uses: ./.github/workflows/tmate_debug | |
| with: | |
| use-tmate: ${{ secrets.USE_TMATE }} | |
| - name: setup cluster resources | |
| uses: ./.github/workflows/canary-test-config | |
| - name: set Ceph version in CephCluster manifest | |
| run: tests/scripts/github-action-helper.sh replace_ceph_image "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}" | |
| - name: use local disk and create partitions for osds | |
| run: | | |
| tests/scripts/github-action-helper.sh use_local_disk | |
| tests/scripts/github-action-helper.sh create_partitions_for_osds | |
| - name: deploy CSI hostNetworking disabled cluster | |
| run: tests/scripts/github-action-helper.sh deploy_csi_hostnetwork_disabled_cluster | |
| - name: wait for prepare pod | |
| run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2 | |
| - name: wait for ceph to be ready | |
| run: IS_POD_NETWORK=true tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2 | |
| - name: wait for ceph-csi configmap to be updated with network namespace | |
| run: tests/scripts/github-action-helper.sh wait_for_ceph_csi_configmap_to_be_updated | |
| - name: test ceph-csi-rbd plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_rbd_workload | |
| - name: test ceph-csi-cephfs plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_cephfs_workload | |
| - name: test ceph-csi-nfs plugin restart | |
| run: tests/scripts/github-action-helper.sh test_csi_nfs_workload | |
| - name: collect common logs | |
| if: always() | |
| uses: ./.github/workflows/collect-logs | |
| with: | |
| name: csi-hostnetwork-disabled |