diff --git a/Makefile b/Makefile index 9b4ac60e9..c523c8a79 100644 --- a/Makefile +++ b/Makefile @@ -152,8 +152,8 @@ test: generate fmt vet envtest ## Run tests. rm cover.out.tmp .PHONY: e2etest -e2etest: generate local-release local-deploy chainsaw - GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS) +e2etest: generate local-release local-deploy chainsaw s5cmd + GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) LOCALBIN=$(CACHE_BIN) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS) local-deploy: kind ctlptl tilt kustomize clusterctl $(CTLPTL) apply -f .tilt/ctlptl-config.yaml @@ -316,6 +316,7 @@ NILAWAY ?= $(LOCALBIN)/nilaway GOVULNC ?= $(LOCALBIN)/govulncheck MOCKGEN ?= $(LOCALBIN)/mockgen GOWRAP ?= $(CACHE_BIN)/gowrap +S5CMD ?= $(CACHE_BIN)/s5cmd ## Tool Versions KUSTOMIZE_VERSION ?= v5.4.1 @@ -332,6 +333,7 @@ NILAWAY_VERSION ?= latest GOVULNC_VERSION ?= v1.1.1 MOCKGEN_VERSION ?= v0.4.0 GOWRAP_VERSION ?= v1.3.7 +S5CMD_VERSION ?= v2.2.2 .PHONY: tools tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(KUBECTL) $(CONTROLLER_GEN) $(CONVERSION_GEN) $(TILT) $(KIND) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN) $(GOWRAP) @@ -427,3 +429,7 @@ gowrap: $(GOWRAP) ## Download gowrap locally if necessary. $(GOWRAP): $(CACHE_BIN) GOBIN=$(CACHE_BIN) go install github.com/hexdigest/gowrap/cmd/gowrap@$(GOWRAP_VERSION) +.PHONY: s5cmd +s5cmd: $(S5CMD) +$(S5CMD): $(CACHE_BIN) + GOBIN=$(CACHE_BIN) go install github.com/peak/s5cmd/v2@$(S5CMD_VERSION) diff --git a/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml index e9d4d8e3c..af8749cdf 100755 --- a/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/kubeadm-flatcar-vpcless-capl-cluster/chainsaw-test.yaml @@ -7,9 +7,8 @@ metadata: # Labels to allow the test to be triggered based on selector flag labels: all: - kubeadm: - flavors: flatcar: + flavors: spec: bindings: # A short identifier for the E2E test run diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml index 7648fbccb..8951b4b90 100644 --- a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml @@ -5,3 +5,11 @@ metadata: namespace: kube-system status: replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: etcd-backup + namespace: kube-system +status: + replicas: 1 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml index 4af7d154c..117609842 100644 --- a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml @@ -13,7 +13,7 @@ spec: bindings: # Identifier for the E2E test run - name: run - value: (join('-', ['e2e', 'kdm-ft', env('GIT_REF')])) + value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF')])) - name: cluster # Format the cluster name value: (trim((truncate(($run), `32`)), '-')) @@ -37,15 +37,17 @@ spec: - name: CLUSTERCTL_CONFIG value: (env('CLUSTERCTL_CONFIG')) - name: SSE_KEY - value: (env('SSE_KEY')) + value: "jU$n@AXXf3xseNd&U9ko4J#3B84D6YR6" - name: KONNECTIVITY_AGENT_REPLICAS value: '1' # Here, 1 is set for testing purposes. Default is 3. - name: CLUSTER_AUTOSCALER_VERSION value: 'v1.29.4' + - name: ETCD_BACKUP_SCHEDULE + value: '* * * * *' content: | set -e if [ -z "$SSE_KEY" ]; then - echo "SSE_KEY not set" >&2 + echo "SSE_KEY not set" >&2 exit 1 else clusterctl generate cluster $CLUSTER -n $NAMESPACE \ @@ -203,17 +205,198 @@ spec: namespace: kube-system # Test to check if konnectivity is working - - clusters: - kubeadm-full-cluster: - kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml - name: Testing to check if logs are retrievable + - name: Testing to check if logs are retrievable try: - script: content: | - KUBECONFIG=./kubeadm-full-cluster-kubeconfig.yaml kubectl logs csi-linode-controller-0 -n kube-system + + MAX_RETRIES=5 + RETRY_DELAY=30 # 30 seconds delay before retrying + ATTEMPT=0 + + # retry loop to check if logs are retrievable from a pod + while [ $ATTEMPT -lt $MAX_RETRIES ]; do + KUBECONFIG=./kubeadm-full-cluster-kubeconfig.yaml kubectl logs csi-linode-controller-0 -n kube-system + + if [ -z "$error" ]; then + break + else + ATTEMPT=$((ATTEMPT + 1)) + sleep $RETRY_DELAY + fi + done + check: ($error == null): true + # Test to check if disks are added to the etcd spec + - name: Testing to check if disks are added to the spec + try: + - script: + env: + - name: TARGET_API + value: api.linode.com + - name: TARGET_API_VERSION + value: v4beta + - name: URI + value: linode/instances + - name: FILTER + value: (to_string({"tags":($cluster)})) + content: | + set -e + + MAX_RETRIES=5 + RETRY_DELAY=30 # 30 second delay before retrying + ATTEMPT=0 + ETCD_LABEL="" + ETCD_STATUS="" + + # retry loop to check if etcd disk is attached + while [ $ATTEMPT -lt $MAX_RETRIES ]; do + + # API call to get the linode ID + RESPONSE=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "X-Filter: $FILTER" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI") + + LINODE_ID=$(echo "$RESPONSE" | jq -r '.data[0].id') + + if [ -z "$LINODE_ID" ]; then + echo "Error: LINODE_ID is empty. Response was: $RESPONSE" + exit 1 + fi + + # API call to check the list of disks attached + DISK_RESPONSE=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI/$LINODE_ID/disks?page=1&page_size=100") + + ETCD_LABEL=$(echo "$DISK_RESPONSE" | jq -r '.data[2].label') + ETCD_STATUS=$(echo "$DISK_RESPONSE" | jq -r '.data[2].status') + + if [ "$ETCD_LABEL" == "etcd_disk" ] && [ "$ETCD_STATUS" == "ready" ]; then + break + else + ATTEMPT=$((ATTEMPT + 1)) + sleep $RETRY_DELAY + fi + done + + echo "{ + \"label\": \"$ETCD_LABEL\", + \"status\": \"$ETCD_STATUS\" + }" + + check: + ($error): ~ + (json_parse($stdout)): + label: etcd_disk + status: ready + + # Test to query the object storage bucket and check for data + - name: Testing the object storage buckets + try: + - script: + env: + - name: TARGET_API + value: api.linode.com + - name: TARGET_API_VERSION + value: v4beta + - name: BUCKET_NAME + value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup'])) + - name: URI + value: (join('/', ['object-storage', 'buckets', env('LINODE_REGION')])) + content: | + set -e + + MAX_RETRIES=5 + RETRY_DELAY=40 # 40 second delay before retrying + ATTEMPT=0 + VALID_BUCKET=false + + # retry loop to check if bucket is created and objects are present + while [ $ATTEMPT -lt $MAX_RETRIES ]; do + + # API call to get the bucket details + RESPONSE=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI/$BUCKET_NAME") + + BUCKET_SIZE=$(echo "$RESPONSE" | jq -r '.size') + BUCKET_OBJECTS=$(echo "$RESPONSE" | jq -r '.objects') + + if [ "$BUCKET_SIZE" -gt 0 ] && [ "$BUCKET_OBJECTS" -gt 0 ]; then + VALID_BUCKET=true + break + else + ATTEMPT=$((ATTEMPT + 1)) + sleep $RETRY_DELAY + fi + done + + echo "{ + \"status\": \"$VALID_BUCKET\" + }" + + check: + ($error): ~ + (json_parse($stdout)): + status: "true" + + # Test to check object storage buckets and delete it + - name: Deleting object storage bucket + try: + - script: + env: + - name: CAPL_KUBECONFIG + value: ./kubeadm-full-cluster-kubeconfig.yaml + - name: BUCKET_NAME + value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup'])) + - name: SECRET_NAME + value: (join('-', ['e2e', 'kdmf-tst', env('GIT_REF'), 'etcd-backup-obj-key'])) + - name: BUCKET_ENDPOINT + value: (join('.', [(join('-', [env('LINODE_REGION'), '1'])), 'linodeobjects', 'com' ])) + - name: LOCAL_BIN + value: (env('LOCALBIN')) + - name: BUCKET_REGION + value: env('LINODE_REGION') + - name: TARGET_API + value: api.linode.com + - name: TARGET_API_VERSION + value: v4beta + - name: URI + value: (join('/', ['object-storage', 'buckets', env('LINODE_REGION')])) + content: | + set -e + + # Getting the keys from the CAPL cluster + access_key=$(KUBECONFIG=$CAPL_KUBECONFIG kubectl get secret $SECRET_NAME -n kube-system -o=jsonpath='{.data.access_key}' | base64 -d) + secret_key=$(KUBECONFIG=$CAPL_KUBECONFIG kubectl get secret $SECRET_NAME -n kube-system -o=jsonpath='{.data.secret_key}' | base64 -d) + + #Storing the keys into a config file + cat < .s5cfg + [default] + aws_access_key_id=$access_key + aws_secret_access_key=$secret_key + aws_default_region=$BUCKET_REGION-1 + EOL + + # delete the objects + $LOCAL_BIN/s5cmd --credentials-file .s5cfg --endpoint-url https://$BUCKET_ENDPOINT rm s3://$BUCKET_NAME/etcd-backup/v2/* + + # delete the bucket + curl --request DELETE \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI/$BUCKET_NAME" + + check: + ($error): ~ + # Test to check if child cluster is deleted - name: Testing to see if child cluster is deleted try: @@ -262,5 +445,6 @@ spec: content: | rm -f kubeadm-full-cluster.yaml rm -f kubeadm-full-cluster-kubeconfig.yaml + rm -f .s5cfg check: ($error == null): true