From d82c129f79bd269376383d5206538f83b10f14d9 Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Fri, 27 Dec 2024 23:23:43 +0530 Subject: [PATCH 1/4] Enable Druid to use mock GCS server to run e2e tests * Create gcp secret to use emulator for druid local setup * Add documentation for usage of fake-gcs-server for e2e tests * Add documentation to setup etcd-druid locally using fake-gcs-server * Remove cluster.yaml file & instead embed the port-forward values in kind-up.sh --- Makefile | 12 ++- .../samples/druid_v1alpha1_etcd_fakegcs.yaml | 78 +++++++++++++++++++ config/samples/etcd-secret-fakegcs.yaml | 12 +++ .../getting-started-locally.md | 20 +++-- .../manage-gcs-emulator.md | 47 +++++++++++ docs/development/running-e2e-tests.md | 44 +++++++++-- hack/ci-e2e-kind-gcs.sh | 33 ++++++++ hack/deploy-fakegcs.sh | 12 +++ .../fake-gcs-server/fake-gcs-server.yaml | 45 +++++++++++ .../overlays/gcp/common/files/common.sh | 29 +++++-- .../overlays/gcp/common/gcp.env | 5 +- hack/kind-up.sh | 6 ++ hack/kind/cluster.yaml | 36 --------- test/e2e/utils.go | 9 ++- 14 files changed, 328 insertions(+), 60 deletions(-) create mode 100644 config/samples/druid_v1alpha1_etcd_fakegcs.yaml create mode 100644 config/samples/etcd-secret-fakegcs.yaml create mode 100644 docs/deployment/getting-started-locally/manage-gcs-emulator.md create mode 100755 hack/ci-e2e-kind-gcs.sh create mode 100755 hack/deploy-fakegcs.sh create mode 100644 hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml delete mode 100644 hack/kind/cluster.yaml diff --git a/Makefile b/Makefile index 6c7fa8b43..8a0355900 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,10 @@ ci-e2e-kind: $(GINKGO) $(YQ) $(KIND) ci-e2e-kind-azure: $(GINKGO) @BUCKET_NAME=$(BUCKET_NAME) $(HACK_DIR)/ci-e2e-kind-azure.sh +.PHONY: ci-e2e-kind-gcs +ci-e2e-kind-gcs: $(GINKGO) + @BUCKET_NAME=$(BUCKET_NAME) $(HACK_DIR)/ci-e2e-kind-gcs.sh + # Rules related to binary build, Docker image build and release # ------------------------------------------------------------------------- # Build manager binary @@ -172,7 +176,7 @@ docker-clean: # Rules for locale/remote environment # ------------------------------------------------------------------------- -kind-up kind-down ci-e2e-kind ci-e2e-kind-azure deploy-localstack deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH) +kind-up kind-down ci-e2e-kind ci-e2e-kind-azure deploy-localstack deploy-fakegcs deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH) .PHONY: kind-up kind-up: $(KIND) @@ -218,4 +222,8 @@ deploy-localstack: $(KUBECTL) .PHONY: deploy-azurite deploy-azurite: $(KUBECTL) - ./hack/deploy-azurite.sh \ No newline at end of file + @$(HACK_DIR)/deploy-azurite.sh + +.PHONY: deploy-fakegcs +deploy-fakegcs: $(KUBECTL) + @$(HACK_DIR)/deploy-fakegcs.sh \ No newline at end of file diff --git a/config/samples/druid_v1alpha1_etcd_fakegcs.yaml b/config/samples/druid_v1alpha1_etcd_fakegcs.yaml new file mode 100644 index 000000000..2508f897d --- /dev/null +++ b/config/samples/druid_v1alpha1_etcd_fakegcs.yaml @@ -0,0 +1,78 @@ +apiVersion: druid.gardener.cloud/v1alpha1 +kind: Etcd +metadata: + name: etcd-test + labels: + app: etcd-statefulset + gardener.cloud/role: controlplane + role: test +spec: + selector: + matchLabels: + app: etcd-statefulset + gardener.cloud/role: controlplane + role: test + annotations: + app: etcd-statefulset + gardener.cloud/role: controlplane + # networking.gardener.cloud/to-dns: allowed + # networking.gardener.cloud/to-private-networks: allowed + # networking.gardener.cloud/to-public-networks: allowed + role: test + labels: + app: etcd-statefulset + gardener.cloud/role: controlplane + # networking.gardener.cloud/to-dns: allowed + # networking.gardener.cloud/to-private-networks: allowed + # networking.gardener.cloud/to-public-networks: allowed + role: test + etcd: + metrics: basic + defragmentationSchedule: "0 */24 * * *" + resources: + limits: { cpu: 500m, memory: 1Gi } + requests: { cpu: 100m, memory: 200Mi } + clientPort: 2379 + serverPort: 2380 + quota: 8Gi + # heartbeatDuration: 10s + backup: + port: 8080 + fullSnapshotSchedule: "0 */24 * * *" + resources: + limits: { cpu: 200m, memory: 1Gi } + requests: { cpu: 23m, memory: 128Mi } + garbageCollectionPolicy: Exponential + garbageCollectionPeriod: 43200s + deltaSnapshotPeriod: 300s + deltaSnapshotMemoryLimit: 1Gi + store: + container: etcd-bucket + prefix: etcd-test + provider: gcp + secretRef: + name: etcd-backup-gcp + compression: + enabled: false + policy: "gzip" + leaderElection: + reelectionPeriod: 5s + etcdConnectionTimeout: 5s + + sharedConfig: + autoCompactionMode: periodic + autoCompactionRetention: "30m" + # schedulingConstraints: + # affinity: {} + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: etcd-statefulset + + replicas: 3 + # priorityClassName: priority-class-name + # storageClass: default + # storageCapacity: 10Gi diff --git a/config/samples/etcd-secret-fakegcs.yaml b/config/samples/etcd-secret-fakegcs.yaml new file mode 100644 index 000000000..e2e6135c5 --- /dev/null +++ b/config/samples/etcd-secret-fakegcs.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +data: + bucketName: ZXRjZC1idWNrZXQ= + serviceaccount.json: ZHVtbXk= #dummy + storageAPIEndpoint: aHR0cDovL2Zha2UtZ2NzLmRlZmF1bHQ6ODAwMC9zdG9yYWdlL3YxLw== #http://fake-gcs.default:8000/storage/v1/ + emulatorEnabled: dHJ1ZQ== #true +kind: Secret +metadata: + labels: + app: etcd-statefulset + name: etcd-backup-gcp +type: Opaque \ No newline at end of file diff --git a/docs/deployment/getting-started-locally/getting-started-locally.md b/docs/deployment/getting-started-locally/getting-started-locally.md index 367189261..34959ecc4 100644 --- a/docs/deployment/getting-started-locally/getting-started-locally.md +++ b/docs/deployment/getting-started-locally/getting-started-locally.md @@ -8,13 +8,13 @@ Before we can setup `etcd-druid` and use it to provision `Etcd` clusters, we nee ## 01-Setting up KIND cluster -`etcd-druid` uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to its convenience only. Any other Kubernetes setup would also work. +`etcd-druid` uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to its convenience only. Any other Kubernetes setup would also work. ```bash make kind-up ``` -This command sets up a new Kind cluster and stores the kubeconfig at `./hack/kind/kubeconfig`. Additionally, this command also deploys a local container registry as a docker container. This ensures faster image push/pull times. The local registry can be accessed as `localhost:5001` for pushing and pulling images. +This command sets up a new Kind cluster and stores the kubeconfig at `./hack/kind/kubeconfig`. Additionally, this command also deploys a local container registry as a docker container. This ensures faster image push/pull times. The local registry can be accessed as `localhost:5001` for pushing and pulling images. To target this newly created cluster, set the `KUBECONFIG` environment variable to the kubeconfig file. @@ -70,7 +70,7 @@ make deploy-debug This is similar to `make deploy-dev` but additionally configures containers in pods for debugging as required for each container's runtime technology. The associated debugging ports are exposed and labelled so that they can be port-forwarded to the local machine. Skaffold disables automatic image rebuilding and syncing when using the `debug` mode as compared to `dev` mode. -Go debugging uses [Delve](https://github.com/go-delve/delve). Please see the [skaffold debugging documentation](https://skaffold.dev/docs/workflows/debug/) how to setup your IDE accordingly. +Go debugging uses [Delve](https://github.com/go-delve/delve). Please see the [skaffold debugging documentation](https://skaffold.dev/docs/workflows/debug/) how to setup your IDE accordingly. !!! note Resuming or stopping only a single goroutine (Go Issue [25578](https://github.com/golang/go/issues/25578), [31132](https://github.com/golang/go/issues/31132)) is currently not supported, so the action will cause all the goroutines to get activated or paused. @@ -84,12 +84,13 @@ This means that when a goroutine is paused on a breakpoint, then all the other g !!! info This section is ***Optional*** and is only meant to describe steps to deploy a local object store which can be used for testing and development. If you either do not wish to enable backups or you wish to use remote (infra-provider-specific) object store then this section can be skipped. -An `Etcd` cluster provisioned via etcd-druid provides a capability to take regular delta and full snapshots which are stored in an object store. You can enable this functionality by ensuring that you fill in [spec.backup.store](https://github.com/gardener/etcd-druid/blob/master/config/samples/druid_v1alpha1_etcd.yaml#L49-L54) section of the `Etcd` CR. +An `Etcd` cluster provisioned via etcd-druid provides a capability to take regular delta and full snapshots which are stored in an object store. You can enable this functionality by ensuring that you fill in [spec.backup.store](https://github.com/gardener/etcd-druid/blob/master/config/samples/druid_v1alpha1_etcd.yaml#L49-L54) section of the `Etcd` CR. | Backup Store Variant | Setup Guide | | ----------------------------- | ---------------------------------------------------------- | | Azure Object Storage Emulator | [Manage Azurite](manage-azurite-emulator.md) (Steps 00-03) | | S3 Object Store Emulator | [Manage LocalStack](manage-s3-emulator.md) (Steps 00-03) | +| GCS Object Store Emulator | [Manage GCS Emulator](manage-gcs-emulator.md) (Steps 00-03)| ### Setting up Cloud Provider Object Store Secret @@ -103,6 +104,7 @@ A Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) * All the values in the data field of the secret YAML should in `base64` encoded format. To apply the secret run: + ```bash kubectl apply -f ``` @@ -136,6 +138,7 @@ Brief explanation of the keys: ## 05-Applying Etcd CR Create the Etcd CR (Custom Resource) by applying the Etcd yaml to the cluster + ```bash kubectl apply -f ``` @@ -143,6 +146,7 @@ kubectl apply -f ## 06-Verify the Etcd Cluster To obtain information on the etcd cluster you can invoke the following command: + ```bash kubectl get etcd -o=wide ``` @@ -151,7 +155,7 @@ We adhere to a naming convention for all resources that are provisioned for an ` ### Verify Etcd Pods' Functionality -`etcd-wrapper` uses a [distroless](https://github.com/GoogleContainerTools/distroless) image, which lacks a shell. To interact with etcd, use an [Ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/) as a debug container. Refer to this [documentation](https://github.com/gardener/etcd-wrapper/blob/master/docs/deployment/ops.md#operations--debugging) for building and using an ephemeral container which gets attached to the `etcd-wrapper` pod. +`etcd-wrapper` uses a [distroless](https://github.com/GoogleContainerTools/distroless) image, which lacks a shell. To interact with etcd, use an [Ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/) as a debug container. Refer to this [documentation](https://github.com/gardener/etcd-wrapper/blob/master/docs/deployment/ops.md#operations--debugging) for building and using an ephemeral container which gets attached to the `etcd-wrapper` pod. ```bash # Put a key-value pair into the etcd @@ -169,11 +173,13 @@ For a multi-node etcd cluster, insert the key-value pair using the `etcd` contai ## 08-Cleaning up the setup If you wish to only delete the `Etcd` cluster then you can use the following command: + ```bash kubectl delete etcd ``` -This will add the `deletionTimestamp` to the `Etcd` resource. At the time the creation of the `Etcd` cluster, etcd-druid will add a finalizer to ensure that it cleans up all `Etcd` cluster resources before the CR is removed. +This will add the `deletionTimestamp` to the `Etcd` resource. At the time the creation of the `Etcd` cluster, etcd-druid will add a finalizer to ensure that it cleans up all `Etcd` cluster resources before the CR is removed. + ```yaml finalizers: - druid.gardener.cloud/etcd-druid @@ -182,11 +188,13 @@ This will add the `deletionTimestamp` to the `Etcd` resource. At the time the c etcd-druid will automatically pick up the deletion event and attempt clean up `Etcd` cluster resources. It will only remove the finaliser once all resources have been cleaned up. If you only wish to remove `etcd-druid` but retain the kind cluster then you can use the following make target: + ```bash make undeploy ``` If you wish to delete the kind cluster then you can use the following make target: + ```bash make kind-down ``` diff --git a/docs/deployment/getting-started-locally/manage-gcs-emulator.md b/docs/deployment/getting-started-locally/manage-gcs-emulator.md new file mode 100644 index 000000000..efe23b529 --- /dev/null +++ b/docs/deployment/getting-started-locally/manage-gcs-emulator.md @@ -0,0 +1,47 @@ +# Manage GCS Emulator + +This document is a step-by-step guide on how to configure, deploy and cleanup [GCS Emulator](https://github.com/fsouza/fake-gcs-server), within a [kind](https://kind.sigs.k8s.io/) cluster. GCS Emulator emulates Google Cloud Storage locally, which allows the `Etcd` cluster to interact with GCS. This setup is ideal for local development and testing. + +## 00-Prerequisites + +Ensure that you have setup the development environment as per the [documentation](../../development/prepare-dev-environment.md). + +> **Note:** It is assumed that you have already created kind cluster and the `KUBECONFIG` is pointing to this Kubernetes cluster. + +### Installing gsutil + +To interact with `GCS Emulator` you must also install the [gsutil](https://cloud.google.com/storage/docs/gsutil) utility. Follow the instructions [here](https://cloud.google.com/storage/docs/gsutil_install) to install `gsutil`. + +## 01-Deploy FakeGCS + +Deploy FakeGCS onto the Kubernetes cluster using the command below: + +```bash +make deploy-fakegcs +``` + +## 02-Setup GCS Bucket + +To create a GCS bucket for `Etcd` backup purposes, execute the following command: + +```bash +gsutil -o "Credentials:gs_json_host=127.0.0.1" -o "Credentials:gs_json_port=4443" -o "Boto:https_validate_certificates=False" mb "gs://etcd-bucket" +``` + +## 03-Configure Secret + +Connection details for a GCS Object Store are put into a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/). Apply the Kubernetes Secret manifest through: + +```bash +kubectl apply -f config/samples/etcd-secret-fakegcs.yaml +``` + +> **Note:** The secret created should be referred to in the `Etcd` CR in `spec.backup.store.secretRef`. + +## 04-Cleanup + +To clean the setup, execute the following commands: + +```bash +make kind-down +``` diff --git a/docs/development/running-e2e-tests.md b/docs/development/running-e2e-tests.md index a5e3847e8..82fa672ef 100644 --- a/docs/development/running-e2e-tests.md +++ b/docs/development/running-e2e-tests.md @@ -25,7 +25,7 @@ as described in this document. It's expected that especially the `deploy` step is run against a Kubernetes cluster which doesn't contain an Druid deployment or any left-overs like `druid.gardener.cloud` CRDs. The `deploy` step will likely fail in such scenarios. -> Tip: Create a fresh [KinD](https://kind.sigs.k8s.io/) cluster or a similar one with a small footprint before executing the tests. +> Tip: Create a fresh [KinD](https://kind.sigs.k8s.io/) cluster or a similar one with a small footprint before executing the tests. ## Providers @@ -140,22 +140,56 @@ make \ test-e2e ``` -## e2e test with localstack +## e2e test with local storage emulators [AWS, GCP, AZURE] -The above-mentioned e2e tests need storage from real cloud providers to be setup. But there is a tool named [localstack](https://docs.localstack.cloud/user-guide/aws/s3/) that enables to run e2e test with mock AWS storage. We can also provision KIND cluster for e2e tests. So, together with localstack and KIND cluster, we don't need to depend on any actual cloud provider infrastructure to be setup to run e2e tests. +The above-mentioned e2e tests need storage from real cloud providers to be setup. But there are tools such as [localstack](https://docs.localstack.cloud/user-guide/aws/s3/), [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) and [azurite](https://github.com/Azure/Azurite) that enables to run e2e test with mock AWS storage, mock GCS storage and mock AZURE storage respectively. We can provision a KIND cluster for end-to-end (e2e) tests. By using local emulators alongside the KIND cluster, we eliminate the need for any actual cloud provider infrastructure to be set up for running e2e tests. -### How are the KIND cluster and localstack set up +### How are the KIND cluster and Emulators set up KIND or Kubernetes-In-Docker is a kubernetes cluster that is set up inside a docker container. This cluster is with limited capability as it does not have much compute power. But this cluster can easily be setup inside a container and can be tear down easily just by removing a container. That's why KIND cluster is very easy to use for e2e tests. `Makefile` command helps to spin up a KIND cluster and use the cluster to run e2e tests. +#### Localstack setup + There is a docker image for localstack. The image is deployed as pod inside the KIND cluster through `hack/e2e-test/infrastructure/localstack/localstack.yaml`. `Makefile` takes care of deploying the yaml file in a KIND cluster. The developer needs to run `make ci-e2e-kind` command. This command in turn runs `hack/ci-e2e-kind.sh` which spin up the KIND cluster and deploy localstack in it and then run the e2e tests using localstack as mock AWS storage provider. e2e tests are actually run on host machine but deploy the druid controller inside KIND cluster. Druid controller spawns multinode etcd clusters inside KIND cluster. e2e tests verify whether the druid controller performs its jobs correctly or not. Mock localstack storage is cleaned up after every e2e tests. That's why the e2e tests need to access the localstack pod running inside KIND cluster. The network traffic between host machine and localstack pod is resolved via mapping localstack pod port to host port while setting up the KIND cluster via `hack/e2e-test/infrastructure/kind/cluster.yaml` -### How to execute e2e tests with localstack and KIND cluster +##### How to execute e2e tests with localstack and KIND cluster Run the following `make` command to spin up a KinD cluster, deploy localstack and run the e2e tests with provider `aws`: ```bash make ci-e2e-kind ``` + +#### Fake-GCS-Server setup + +[Fake-gcs-server](https://github.com/fsouza/fake-gcs-server) is run inside a pod using this [docker image](https://hub.docker.com/r/fsouza/fake-gcs-server) in a KIND cluster. + +The user needs to run `make ci-e2e-kind-gcs` to start the e2e tests for druid with GCS emulator as the object storage for etcd backups. The above command internally runs the script `hack/ci-e2e-kind-gcs.sh` which initializes the setup with required steps before going on to create a KIND cluster and deploy fakegcs in it and use that emulator to run e2e tests. + +The `fake-gcs-server` running inside the pod serves HTTP requests at port-8000 and HTTPS requests at port-4443. As the e2e tests runs on the host machine while the emulator runs on KIND, both ports i.e 8000 & 4443 needs to be port-forwarded from the host machine to fake-gcs service running inside the KIND cluster. The port forwardings is defined in the `hack/kind-up.sh` file which is used to setup the KIND cluster. + +##### How to execute e2e tests with fake-gcs-server and KIND cluster + +Run the following `make` command to spin up a KinD cluster, deploy fakegcs and run the e2e tests with provider `gcp`: + +```bash +make ci-e2e-kind-gcs +``` + +#### Azurite setup + +[Azurite](https://github.com/Azure/Azurite) is run inside a pod using this [docker image](mcr.microsoft.com/azure-storage/azurite:latest) in a KIND cluster. + +The user needs to run `make ci-e2e-kind-azure` to start the e2e tests for druid with Azurite as the object storage for etcd backups. The above command internally runs the script `hack/ci-e2e-kind-azure.sh` which initializes the setup with required steps before going on to create a KIND cluster and deploy Azurite in it and use that emulator to run e2e tests. + +The `azurite` running inside the pod serves HTTP requests at port 10000. As the e2e tests runs on the host machine while the emulator runs on KIND cluster, the port 10000 needs to be port-forwarded from the host machine to azurite service running inside the KIND cluster. The port forwardings is defined in the `hack/kind-up.sh` file which is used to setup the KIND cluster. + +##### How to execute e2e tests with Azurite and KIND cluster + +Run the following `make` command to spin up a KinD cluster, deploy Azurite and run the e2e tests with provider `azure`: + +```bash +make ci-e2e-kind-azure +``` diff --git a/hack/ci-e2e-kind-gcs.sh b/hack/ci-e2e-kind-gcs.sh new file mode 100755 index 000000000..c0a3af394 --- /dev/null +++ b/hack/ci-e2e-kind-gcs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +make kind-up + +trap '{ + kind export logs "${ARTIFACTS:-/tmp}/etcd-druid-e2e" --name etcd-druid-e2e || true + make kind-down +}' EXIT + +kubectl wait --for=condition=ready node --all +echo "{ \"serviceaccount.json\": \"\", \"storageAPIEndpoint\": \"http://fake-gcs.default:8000/storage/v1/\", \"emulatorEnabled\": \"true\" }" >/tmp/svc_acc.json +echo -n "http://localhost:8000/storage/v1/" > /tmp/storageAPIEndpoint + +export GOOGLE_STORAGE_API_ENDPOINT="http://localhost:8000/storage/v1/" + +# Deploy fake-gcs and run e2e tests +make deploy-fakegcs +make GCP_SERVICEACCOUNT_JSON_PATH="/tmp/svc_acc.json" \ + GOOGLE_APPLICATION_CREDENTIALS="/tmp/svc_acc.json" \ + GCP_PROJECT_ID="e2e-test" \ + GOOGLE_EMULATOR_ENABLED="true" \ + FAKEGCS_HOST="fake-gcs.default:8000" \ + PROVIDERS="gcp" \ + TEST_ID="$BUCKET_NAME" \ + STEPS="setup,deploy,test" \ + test-e2e \ No newline at end of file diff --git a/hack/deploy-fakegcs.sh b/hack/deploy-fakegcs.sh new file mode 100755 index 000000000..6a80f3d1c --- /dev/null +++ b/hack/deploy-fakegcs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Gardener contributors +# +# SPDX-License-Identifier: Apache-2.0 + +set -o errexit +set -o nounset +set -o pipefail + +kubectl apply -f ./hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml +kubectl rollout status deploy/fake-gcs +kubectl wait --for=condition=ready pod -l app=fake-gcs --timeout=240s \ No newline at end of file diff --git a/hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml b/hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml new file mode 100644 index 000000000..d45933320 --- /dev/null +++ b/hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: fake-gcs + name: fake-gcs +spec: + replicas: 1 + selector: + matchLabels: + app: fake-gcs + strategy: + type: Recreate + template: + metadata: + labels: + app: fake-gcs + spec: + containers: + - name: fake-gcs-server + image: fsouza/fake-gcs-server + args: ["-scheme", "both", "-public-host" , "fake-gcs.default:8000"] + ports: + - containerPort: 4443 + hostPort: 4443 + - containerPort: 8000 + hostPort: 8000 + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: fake-gcs + name: fake-gcs +spec: + ports: + - name: "4443-https" + port: 4443 + targetPort: 4443 + - name: "8000-http" + port: 8000 + targetPort: 8000 + selector: + app: fake-gcs diff --git a/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh b/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh index f733cdc0b..0d488a54d 100644 --- a/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh +++ b/hack/e2e-test/infrastructure/overlays/gcp/common/files/common.sh @@ -3,6 +3,17 @@ # # SPDX-License-Identifier: Apache-2.0 +gsutil_options="" +if [[ -n "${FAKEGCS_HOST}" ]]; then + gs_json_host="${FAKEGCS_HOST%%:*}" + gsutil_options="-o Credentials:gs_json_host=${gs_json_host} -o Credentials:gs_json_port=4443 -o Boto:https_validate_certificates=False" +fi + +project_option="" +if [[ -n "${GCP_PROJECT_ID}" ]]; then + project_option="-p ${GCP_PROJECT_ID}" +fi + # More information at https://cloud.google.com/sdk/docs/install#linux function setup_gcloud() { if $(which gcloud > /dev/null); then @@ -13,7 +24,7 @@ function setup_gcloud() { apt update apt install -y curl apt install -y python3 - curl -Lo "google-cloud-sdk.tar.gz" https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-441.0.0-$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m | sed 's/aarch64/arm/').tar.gz + curl -Lo "google-cloud-sdk.tar.gz" https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-503.0.0-$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m | sed 's/aarch64/arm/').tar.gz tar -xzf google-cloud-sdk.tar.gz ./google-cloud-sdk/install.sh -q export PATH=$PATH:${HOME}/google-cloud-sdk/bin @@ -23,15 +34,19 @@ function setup_gcloud() { function configure_gcloud() { echo "Configuring gcloud..." - gcloud auth activate-service-account --key-file "$GCP_SERVICEACCOUNT_JSON_PATH" --project "$GCP_PROJECT_ID" - echo "Successfully configured gcloud." + if [[ -z "${gsutil_options}" ]]; then + gcloud auth activate-service-account --key-file "$GCP_SERVICEACCOUNT_JSON_PATH" --project "$GCP_PROJECT_ID" + echo "Successfully configured gcloud." + else + echo "Skipping gcloud auth as fake-gcs is used." + fi } function create_gcs_bucket() { - result=$(gsutil list gs://${TEST_ID} 2>&1 || true) + result=$(gsutil ${gsutil_options} list ${project_option} gs://${TEST_ID} 2>&1 || true) if [[ $result == *"404"* ]]; then echo "Creating GCS bucket ${TEST_ID} ..." - gsutil mb -b on gs://${TEST_ID} + gsutil ${gsutil_options} mb ${project_option} -b on gs://${TEST_ID} echo "Successfully created GCS bucket ${TEST_ID} ." else if [[ $result =~ ${TEST_ID} ]] || [[ $result == "" ]]; then @@ -44,12 +59,12 @@ function create_gcs_bucket() { } function delete_gcs_bucket() { - result=$(gsutil list gs://${TEST_ID} 2>&1 || true) + result=$(gsutil ${gsutil_options} list ${project_option} gs://${TEST_ID} 2>&1 || true) if [[ $result == *"404"* ]]; then echo "GCS bucket is already gone." return fi echo "Deleting GCS bucket ${TEST_ID} ..." - gsutil rm -r gs://"${TEST_ID}"/ + gsutil ${gsutil_options} rm ${project_option} -r gs://"${TEST_ID}"/ echo "Successfully deleted GCS bucket ${TEST_ID} ." } diff --git a/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env b/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env index 30a67cf8c..c5b15a84a 100644 --- a/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env +++ b/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env @@ -1,2 +1,5 @@ +GCP_SERVICEACCOUNT_JSON_PATH GCP_PROJECT_ID -TEST_ID \ No newline at end of file +GOOGLE_EMULATOR_ENABLED +FAKEGCS_HOST +TEST_ID diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 98128c965..3851afbb3 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -89,6 +89,12 @@ nodes: - containerPort: 10000 hostPort: 10000 protocol: TCP + - containerPort: 8000 + hostPort: 8000 + protocol: TCP + - containerPort: 4443 + hostPort: 4443 + protocol: TCP EOF if [ "${DEPLOY_REGISTRY}" = true ]; then printf -v reg '[plugins."io.containerd.grpc.v1.cri".registry] diff --git a/hack/kind/cluster.yaml b/hack/kind/cluster.yaml deleted file mode 100644 index ddcc1682e..000000000 --- a/hack/kind/cluster.yaml +++ /dev/null @@ -1,36 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - image: kindest/node:v1.29.2 - # port forward 80 on the host to 80 on this node - extraPortMappings: - # localstack host - - containerPort: 4566 - hostPort: 4566 - # optional: set the bind address on the host - # 0.0.0.0 is the current default - listenAddress: "127.0.0.1" - # optional: set the protocol to one of TCP, UDP, SCTP. - # TCP is the default - protocol: TCP - - containerPort: 10000 - hostPort: 10000 - # container registry - - containerPort: 5001 - hostPort: 5001 - # optional: set the bind address on the host - # 0.0.0.0 is the current default - listenAddress: "127.0.0.1" - # optional: set the protocol to one of TCP, UDP, SCTP. - # TCP is the default - protocol: TCP - # Azurite host - - containerPort: 10000 - hostPort: 10000 - # optional: set the bind address on the host - # 0.0.0.0 is the current default - listenAddress: "127.0.0.1" - # optional: set the protocol to one of TCP, UDP, SCTP. - # TCP is the default - protocol: TCP diff --git a/test/e2e/utils.go b/test/e2e/utils.go index c4316f3c6..ad80741ad 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -385,6 +385,11 @@ func getProviders() ([]TestProvider, error) { }, }, } + fakegcsHost := getEnvOrFallback("FAKEGCS_HOST", "") + if fakegcsHost != "" { + provider.Storage.SecretData["storageAPIEndpoint"] = []byte("http://" + fakegcsHost + "/storage/v1/") + provider.Storage.SecretData["emulatorEnabled"] = []byte("true") + } } case providerLocal: provider = TestProvider{ @@ -406,13 +411,11 @@ func isEmulatorEnabled(provider TestProvider) bool { switch provider.Name { case "aws": return provider.Storage.SecretData["endpoint"] != nil - case "az": + case "az", "gcp": if val, ok := provider.Storage.SecretData["emulatorEnabled"]; ok { return string(val) == "true" } return false - case "gcp": - return false case "local": return false } From 822ba9d76d1ecc4fad0e007b7451dd994ad5c455 Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Tue, 31 Dec 2024 12:54:36 +0530 Subject: [PATCH 2/4] fix tests --- Makefile | 2 +- hack/kind-up.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a0355900..7a037d0de 100644 --- a/Makefile +++ b/Makefile @@ -176,7 +176,7 @@ docker-clean: # Rules for locale/remote environment # ------------------------------------------------------------------------- -kind-up kind-down ci-e2e-kind ci-e2e-kind-azure deploy-localstack deploy-fakegcs deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH) +kind-up kind-down ci-e2e-kind ci-e2e-kind-azure ci-e2e-kind-gcs deploy-localstack deploy-fakegcs deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH) .PHONY: kind-up kind-up: $(KIND) diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 3851afbb3..7b6546fa0 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -76,6 +76,8 @@ function clamp_mss_to_pmtu() { function generate_kind_config() { echo "Generating kind cluster config..." + # Ensure the directory exists + mkdir -p "${KIND_CONFIG_DIR}" cat >"${KIND_CONFIG_DIR}/cluster-config.yaml" < Date: Tue, 31 Dec 2024 14:39:50 +0530 Subject: [PATCH 3/4] temp commit --- hack/kind-up.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 7b6546fa0..3851afbb3 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -76,8 +76,6 @@ function clamp_mss_to_pmtu() { function generate_kind_config() { echo "Generating kind cluster config..." - # Ensure the directory exists - mkdir -p "${KIND_CONFIG_DIR}" cat >"${KIND_CONFIG_DIR}/cluster-config.yaml" < Date: Tue, 31 Dec 2024 17:49:10 +0530 Subject: [PATCH 4/4] Address review comments by @shreyas-s-rao --- .../getting-started-locally/manage-azurite-emulator.md | 5 +++-- .../getting-started-locally/manage-gcs-emulator.md | 4 ++-- .../getting-started-locally/manage-s3-emulator.md | 8 ++++++-- docs/development/running-e2e-tests.md | 2 +- hack/ci-e2e-kind-gcs.sh | 4 +--- hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env | 1 - hack/kind-up.sh | 2 ++ 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/deployment/getting-started-locally/manage-azurite-emulator.md b/docs/deployment/getting-started-locally/manage-azurite-emulator.md index 579022a00..4365ca161 100644 --- a/docs/deployment/getting-started-locally/manage-azurite-emulator.md +++ b/docs/deployment/getting-started-locally/manage-azurite-emulator.md @@ -50,8 +50,9 @@ kubectl apply -f config/samples/etcd-secret-azurite.yaml ## 04-Cleanup -In addition to the kind cluster cleanup you should also unset the environment variable set in step-03 above. +To clean the setup, unset the environment variable set in step-03 above and delete the Azurite deployment: + ```bash unset AZURE_STORAGE_CONNECTION_STRING +kubectl delete -f ./hack/e2e-test/infrastructure/azurite/azurite.yaml ``` - diff --git a/docs/deployment/getting-started-locally/manage-gcs-emulator.md b/docs/deployment/getting-started-locally/manage-gcs-emulator.md index efe23b529..b9b8e0d00 100644 --- a/docs/deployment/getting-started-locally/manage-gcs-emulator.md +++ b/docs/deployment/getting-started-locally/manage-gcs-emulator.md @@ -40,8 +40,8 @@ kubectl apply -f config/samples/etcd-secret-fakegcs.yaml ## 04-Cleanup -To clean the setup, execute the following commands: +To clean the setup, delete the FakeGCS deployment: ```bash -make kind-down +kubectl delete -f ./hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml ``` diff --git a/docs/deployment/getting-started-locally/manage-s3-emulator.md b/docs/deployment/getting-started-locally/manage-s3-emulator.md index 835ee3a39..b69f2ed9e 100644 --- a/docs/deployment/getting-started-locally/manage-s3-emulator.md +++ b/docs/deployment/getting-started-locally/manage-s3-emulator.md @@ -39,11 +39,13 @@ export AWS_DEFAULT_REGION=us-east-2 ``` Create a S3 bucket using the following command: + ```bash aws s3api create-bucket --bucket --region --create-bucket-configuration LocationConstraint= --acl private ``` To verify if the bucket has been created, you can use the following command: + ```bash aws s3api head-bucket --bucket ``` @@ -60,7 +62,9 @@ kubectl apply -f config/samples/etcd-secret-localstack.yaml ## 04-Cleanup -In addition to the kind cluster cleanup you should also unset the environment variable set in step-03 above. +To clean the setup,, unset the environment variable set in step-03 above and delete the LocalStack deployment: + ```bash unset AWS_ENDPOINT_URL_S3 AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION -``` \ No newline at end of file +kubectl delete -f ./hack/e2e-test/infrastructure/localstack/localstack.yaml +``` diff --git a/docs/development/running-e2e-tests.md b/docs/development/running-e2e-tests.md index 82fa672ef..9a926395d 100644 --- a/docs/development/running-e2e-tests.md +++ b/docs/development/running-e2e-tests.md @@ -142,7 +142,7 @@ test-e2e ## e2e test with local storage emulators [AWS, GCP, AZURE] -The above-mentioned e2e tests need storage from real cloud providers to be setup. But there are tools such as [localstack](https://docs.localstack.cloud/user-guide/aws/s3/), [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) and [azurite](https://github.com/Azure/Azurite) that enables to run e2e test with mock AWS storage, mock GCS storage and mock AZURE storage respectively. We can provision a KIND cluster for end-to-end (e2e) tests. By using local emulators alongside the KIND cluster, we eliminate the need for any actual cloud provider infrastructure to be set up for running e2e tests. +The above-mentioned e2e tests need storage from real cloud providers to be setup. But there are tools such as [localstack](https://docs.localstack.cloud/user-guide/aws/s3/), [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) and [azurite](https://github.com/Azure/Azurite) that enable developers to run e2e tests with emulators for AWS S3, Google GCS and Azure ABS respectively. We can provision a KIND cluster for running the e2e tests against. Thus, by using cloud storage emulators alongside the KIND cluster, we eliminate the need for any actual cloud provider infrastructure to be set up for running e2e tests. ### How are the KIND cluster and Emulators set up diff --git a/hack/ci-e2e-kind-gcs.sh b/hack/ci-e2e-kind-gcs.sh index c0a3af394..2c30631e1 100755 --- a/hack/ci-e2e-kind-gcs.sh +++ b/hack/ci-e2e-kind-gcs.sh @@ -15,17 +15,15 @@ trap '{ }' EXIT kubectl wait --for=condition=ready node --all + echo "{ \"serviceaccount.json\": \"\", \"storageAPIEndpoint\": \"http://fake-gcs.default:8000/storage/v1/\", \"emulatorEnabled\": \"true\" }" >/tmp/svc_acc.json echo -n "http://localhost:8000/storage/v1/" > /tmp/storageAPIEndpoint -export GOOGLE_STORAGE_API_ENDPOINT="http://localhost:8000/storage/v1/" - # Deploy fake-gcs and run e2e tests make deploy-fakegcs make GCP_SERVICEACCOUNT_JSON_PATH="/tmp/svc_acc.json" \ GOOGLE_APPLICATION_CREDENTIALS="/tmp/svc_acc.json" \ GCP_PROJECT_ID="e2e-test" \ - GOOGLE_EMULATOR_ENABLED="true" \ FAKEGCS_HOST="fake-gcs.default:8000" \ PROVIDERS="gcp" \ TEST_ID="$BUCKET_NAME" \ diff --git a/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env b/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env index c5b15a84a..3787ea500 100644 --- a/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env +++ b/hack/e2e-test/infrastructure/overlays/gcp/common/gcp.env @@ -1,5 +1,4 @@ GCP_SERVICEACCOUNT_JSON_PATH GCP_PROJECT_ID -GOOGLE_EMULATOR_ENABLED FAKEGCS_HOST TEST_ID diff --git a/hack/kind-up.sh b/hack/kind-up.sh index 3851afbb3..7b6546fa0 100755 --- a/hack/kind-up.sh +++ b/hack/kind-up.sh @@ -76,6 +76,8 @@ function clamp_mss_to_pmtu() { function generate_kind_config() { echo "Generating kind cluster config..." + # Ensure the directory exists + mkdir -p "${KIND_CONFIG_DIR}" cat >"${KIND_CONFIG_DIR}/cluster-config.yaml" <