From ac3e15625c6a906fc5e7f4cc57e007d80ffeb544 Mon Sep 17 00:00:00 2001 From: Seshachalam <104052572+seshachalam-yv@users.noreply.github.com> Date: Thu, 9 Nov 2023 19:18:28 +0530 Subject: [PATCH] Simplify `Localstack` setup by utilizing S3 endpoint support (#713) --- Makefile | 4 +- .../druid_v1alpha1_etcd_localstack.yaml | 78 +++++++++++++++++ config/samples/etcd-secret-localstack.yaml | 15 ++++ .../getting-started-locally-localstack.md | 86 +++++++++++++++++++ hack/ci-e2e-kind.sh | 2 +- hack/deploy-localstack.sh | 37 +------- .../overlays/aws/common/files/common.sh | 15 ++-- 7 files changed, 190 insertions(+), 47 deletions(-) create mode 100644 config/samples/druid_v1alpha1_etcd_localstack.yaml create mode 100644 config/samples/etcd-secret-localstack.yaml create mode 100644 docs/development/getting-started-locally-localstack.md diff --git a/Makefile b/Makefile index 7ffbb9894..d0d65633a 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ deploy-via-kustomize: manifests $(KUSTOMIZE) # Modify the Helm template located at charts/druid/templates if any changes are required .PHONY: deploy deploy: $(SKAFFOLD) - $(SKAFFOLD) run -m etcd-druid + $(SKAFFOLD) run -m etcd-druid --kubeconfig=$(KUBECONFIG_PATH) # Generate manifests e.g. CRD, RBAC etc. .PHONY: manifests @@ -167,7 +167,7 @@ kind-down: $(KIND) .PHONY: deploy-localstack deploy-localstack: $(KUBECTL) - BUCKET_NAME=$(BUCKET_NAME) ./hack/deploy-localstack.sh + ./hack/deploy-localstack.sh .PHONY: ci-e2e-kind ci-e2e-kind: diff --git a/config/samples/druid_v1alpha1_etcd_localstack.yaml b/config/samples/druid_v1alpha1_etcd_localstack.yaml new file mode 100644 index 000000000..4f51553ee --- /dev/null +++ b/config/samples/druid_v1alpha1_etcd_localstack.yaml @@ -0,0 +1,78 @@ +apiVersion: druid.gardener.cloud/v1alpha1 +kind: Etcd +metadata: + name: etcd-test + labels: + app: etcd-statefulset + gardener.cloud/role: controlplane + role: test +spec: + selector: + matchLabels: + app: etcd-statefulset + gardener.cloud/role: controlplane + role: test + annotations: + app: etcd-statefulset + gardener.cloud/role: controlplane + # networking.gardener.cloud/to-dns: allowed + # networking.gardener.cloud/to-private-networks: allowed + # networking.gardener.cloud/to-public-networks: allowed + role: test + labels: + app: etcd-statefulset + gardener.cloud/role: controlplane + # networking.gardener.cloud/to-dns: allowed + # networking.gardener.cloud/to-private-networks: allowed + # networking.gardener.cloud/to-public-networks: allowed + role: test + etcd: + metrics: basic + defragmentationSchedule: "0 */24 * * *" + resources: + limits: { cpu: 500m, memory: 1Gi } + requests: { cpu: 100m, memory: 200Mi } + clientPort: 2379 + serverPort: 2380 + quota: 8Gi + # heartbeatDuration: 10s + backup: + port: 8080 + fullSnapshotSchedule: "0 */24 * * *" + resources: + limits: { cpu: 200m, memory: 1Gi } + requests: { cpu: 23m, memory: 128Mi } + garbageCollectionPolicy: Exponential + garbageCollectionPeriod: 43200s + deltaSnapshotPeriod: 300s + deltaSnapshotMemoryLimit: 1Gi + store: + container: etcd-bucket + prefix: etcd-test + provider: S3 + secretRef: + name: etcd-backup-aws + compression: + enabled: false + policy: "gzip" + leaderElection: + reelectionPeriod: 5s + etcdConnectionTimeout: 5s + + sharedConfig: + autoCompactionMode: periodic + autoCompactionRetention: "30m" + # schedulingConstraints: + # affinity: {} + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: etcd-statefulset + + replicas: 3 + # priorityClassName: priority-class-name + # storageClass: default + # storageCapacity: 10Gi diff --git a/config/samples/etcd-secret-localstack.yaml b/config/samples/etcd-secret-localstack.yaml new file mode 100644 index 000000000..f3c91b753 --- /dev/null +++ b/config/samples/etcd-secret-localstack.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +data: + accessKeyID: QUNDRVNTS0VZQVdTVVNFUg== + bucketName: ZXRjZC1idWNrZXQK + endpoint: aHR0cDovL2xvY2Fsc3RhY2suZGVmYXVsdDo0NTY2 + region: dXMtZWFzdC0y + s3ForcePathStyle: dHJ1ZQ== + secretAccessKey: c0VjcmVUS2V5 +kind: Secret +metadata: + labels: + garden.sapcloud.io/role: controlplane + role: main + name: etcd-backup-aws +type: Opaque \ No newline at end of file diff --git a/docs/development/getting-started-locally-localstack.md b/docs/development/getting-started-locally-localstack.md new file mode 100644 index 000000000..4bcc35120 --- /dev/null +++ b/docs/development/getting-started-locally-localstack.md @@ -0,0 +1,86 @@ +# Getting Started with etcd-druid, LocalStack, and Kind + +This guide provides step-by-step instructions on how to set up etcd-druid with [LocalStack](https://localstack.cloud/) and Kind on your local machine. LocalStack emulates AWS services locally, which allows the etcd cluster to interact with AWS S3 without the need for an actual AWS connection. This setup is ideal for local development and testing. + +## Prerequisites + +- Docker (installed and running) +- AWS CLI (version `>=1.29.0` or `>=2.13.0`) + +## Environment Setup + +### Step 1: Provision the Kind Cluster + +Execute the command below to provision a `kind` cluster. This command also forwards port `4566` from the [kind cluster](hack/e2e-test/infrastructure/kind/cluster.yaml) to your local machine, enabling LocalStack access: + +```bash +make kind-up +``` + +### Step 2: Deploy LocalStack + +Deploy LocalStack onto the Kubernetes cluster using the command below: + +```bash +make deploy-localstack +``` + +### Step 3: Set up an S3 Bucket + +1. Set up the AWS CLI to interact with LocalStack by setting the necessary environment variables. This configuration redirects S3 commands to the LocalStack endpoint and provides the required credentials for authentication: + +```bash +export AWS_ENDPOINT_URL_S3="http://localhost:4566" +export AWS_ACCESS_KEY_ID=ACCESSKEYAWSUSER +export AWS_SECRET_ACCESS_KEY=sEcreTKey +export AWS_DEFAULT_REGION=us-east-2 +``` + +2. Create an S3 bucket for etcd-druid backup purposes: + +```bash +aws s3api create-bucket --bucket etcd-bucket --region us-east-2 --create-bucket-configuration LocationConstraint=us-east-2 --acl private +``` + +### Step 4: Deploy etcd-druid + +Deploy etcd-druid onto the Kind cluster using the command below: + +```bash +make deploy +``` + +### Step 5: Configure etcd with LocalStack Store + +Apply the required Kubernetes manifests to create an etcd custom resource (CR) and a secret for AWS credentials, facilitating LocalStack access: + +```bash +export KUBECONFIG=hack/e2e-test/infrastructure/kind/kubeconfig +kubectl apply -f config/samples/druid_v1alpha1_etcd_localstack.yaml -f config/samples/etcd-secret-localstack.yaml +``` + +### Step 6: Reconcile the etcd + +Initiate etcd reconciliation by annotating the etcd resource with the `gardener.cloud/operation=reconcile` annotation: + +```bash +kubectl annotate etcd etcd-test gardener.cloud/operation=reconcile +``` + +Congratulations! You have successfully configured `etcd-druid`, `LocalStack`, and `kind` on your local machine. Inspect the etcd-druid logs and LocalStack to ensure the setup operates as anticipated. + +To validate the buckets, execute the following command: + +```bash +aws s3 ls etcd-bucket/etcd-test/v2/ +``` + +### Cleanup + +To dismantle the setup, execute the following command: + +```bash +make kind-down +unset AWS_ENDPOINT_URL_S3 AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION KUBECONFIG + +``` \ No newline at end of file diff --git a/hack/ci-e2e-kind.sh b/hack/ci-e2e-kind.sh index e0a515229..d34814059 100755 --- a/hack/ci-e2e-kind.sh +++ b/hack/ci-e2e-kind.sh @@ -27,7 +27,7 @@ kubectl wait --for=condition=ready node --all export AWS_APPLICATION_CREDENTIALS_JSON="/tmp/aws.json" echo "{ \"accessKeyID\": \"ACCESSKEYAWSUSER\", \"secretAccessKey\": \"sEcreTKey\", \"region\": \"us-east-2\", \"endpoint\": \"http://127.0.0.1:4566\", \"s3ForcePathStyle\": true, \"bucketName\": \"${BUCKET_NAME}\" }" >/tmp/aws.json -make deploy-localstack BUCKET_NAME="$BUCKET_NAME" +make deploy-localstack make LOCALSTACK_HOST="localstack.default:4566" \ AWS_ACCESS_KEY_ID="ACCESSKEYAWSUSER" \ AWS_SECRET_ACCESS_KEY="sEcreTKey" \ diff --git a/hack/deploy-localstack.sh b/hack/deploy-localstack.sh index 50ba33221..238081fc9 100755 --- a/hack/deploy-localstack.sh +++ b/hack/deploy-localstack.sh @@ -18,40 +18,5 @@ set -o nounset set -o pipefail kubectl apply -f ./hack/e2e-test/infrastructure/localstack/localstack.yaml - -cat < ${HOME}/.aws/config } function create_s3_bucket() { - result=$(aws ${ENDPOINT_URL} s3api get-bucket-location --bucket ${TEST_ID} 2>&1 || true) + result=$(aws s3api get-bucket-location --bucket ${TEST_ID} 2>&1 || true) if [[ $result == *NoSuchBucket* ]]; then echo "Creating S3 bucket ${TEST_ID} in region ${AWS_REGION}" - aws ${ENDPOINT_URL} s3api create-bucket --bucket ${TEST_ID} --region ${AWS_REGION} --create-bucket-configuration LocationConstraint=${AWS_REGION} --acl private + aws s3api create-bucket --bucket ${TEST_ID} --region ${AWS_REGION} --create-bucket-configuration LocationConstraint=${AWS_REGION} --acl private # Block public access to the S3 bucket - aws ${ENDPOINT_URL} s3api put-public-access-block --bucket ${TEST_ID} --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" + aws s3api put-public-access-block --bucket ${TEST_ID} --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" # Deny non-HTTPS requests to the S3 bucket, except for localstack which is exposed on an HTTP endpoint if [[ -z "${LOCALSTACK_HOST}" ]]; then - aws ${ENDPOINT_URL} s3api put-bucket-policy --bucket ${TEST_ID} --policy "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Principal\":\"*\",\"Action\":\"s3:*\",\"Resource\":[\"arn:aws:s3:::${TEST_ID}\",\"arn:aws:s3:::${TEST_ID}/*\"],\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"false\"},\"NumericLessThan\":{\"s3:TlsVersion\":\"1.2\"}}}]}" + aws s3api put-bucket-policy --bucket ${TEST_ID} --policy "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Deny\",\"Principal\":\"*\",\"Action\":\"s3:*\",\"Resource\":[\"arn:aws:s3:::${TEST_ID}\",\"arn:aws:s3:::${TEST_ID}/*\"],\"Condition\":{\"Bool\":{\"aws:SecureTransport\":\"false\"},\"NumericLessThan\":{\"s3:TlsVersion\":\"1.2\"}}}]}" fi else echo $result @@ -66,10 +65,10 @@ function create_s3_bucket() { function delete_s3_bucket() { echo "About to delete S3 bucket ${TEST_ID}" - result=$(aws ${ENDPOINT_URL} s3api get-bucket-location --bucket ${TEST_ID} 2>&1 || true) + result=$(aws s3api get-bucket-location --bucket ${TEST_ID} 2>&1 || true) if [[ $result == *NoSuchBucket* ]]; then echo "Bucket is already gone." return fi - aws ${ENDPOINT_URL} s3 rb s3://${TEST_ID} --force + aws s3 rb s3://${TEST_ID} --force }