From f4c6ec146d3c8bf67ff9f8e501ab3267c431a278 Mon Sep 17 00:00:00 2001 From: Amol Deodhar Date: Tue, 26 Mar 2024 15:16:13 -0400 Subject: [PATCH] Install etcd-backup-restore for child clusters --- .../linodeobjectstoragebucket_types.go | 5 + cloud/scope/object_storage_bucket.go | 41 +++++- ...r.x-k8s.io_linodeobjectstoragebuckets.yaml | 5 + ...nodeobjectstoragebucket_controller_test.go | 41 +++++- .../etcd-backup-restore.yaml | 135 ++++++++++++++++++ .../etcd-backup-restore/kustomization.yaml | 6 + .../etcd-backup-restore/linode-obj.yaml | 27 ++++ .../etcd-backup-restore/sse-key-secret.yaml | 29 ++++ .../etcd-backup-restore/kustomization.yaml | 17 +++ 9 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 templates/addons/etcd-backup-restore/etcd-backup-restore.yaml create mode 100644 templates/addons/etcd-backup-restore/kustomization.yaml create mode 100644 templates/addons/etcd-backup-restore/linode-obj.yaml create mode 100644 templates/addons/etcd-backup-restore/sse-key-secret.yaml create mode 100644 templates/flavors/etcd-backup-restore/kustomization.yaml diff --git a/api/v1alpha1/linodeobjectstoragebucket_types.go b/api/v1alpha1/linodeobjectstoragebucket_types.go index 995efc277..a806b3fdf 100644 --- a/api/v1alpha1/linodeobjectstoragebucket_types.go +++ b/api/v1alpha1/linodeobjectstoragebucket_types.go @@ -43,6 +43,11 @@ type LinodeObjectStorageBucketSpec struct { // +optional // +kubebuilder:default=0 KeyGeneration *int `json:"keyGeneration,omitempty"` + + // secretType sets the type for the access-keys secret that will be generated by the controller. + // +optional + // +kubebuilder:default=addons.cluster.x-k8s.io/resource-set + SecretType string `json:"secretType,omitempty"` } // LinodeObjectStorageBucketStatus defines the observed state of LinodeObjectStorageBucket diff --git a/cloud/scope/object_storage_bucket.go b/cloud/scope/object_storage_bucket.go index c11c0677a..b8b2867da 100644 --- a/cloud/scope/object_storage_bucket.go +++ b/cloud/scope/object_storage_bucket.go @@ -17,6 +17,18 @@ import ( infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" ) +const bucketDataSecret = `kind: Secret +apiVersion: v1 +metadata: + name: etcd-backup + namespace: kube-system +stringData: + bucket_name: %s + access_key_rw: %s + secret_key_rw: %s + access_key_ro: %s + secret_key_ro: %s` + type ObjectStorageBucketScopeParams struct { Client k8sClient LinodeClientBuilder LinodeObjectStorageClientBuilder @@ -109,18 +121,35 @@ func (s *ObjectStorageBucketScope) GenerateKeySecret(ctx context.Context, keys [ return nil, errors.New("expected two non-nil object storage keys") } } - + var secretStringData map[string]string secretName := fmt.Sprintf(AccessKeyNameTemplate, s.Bucket.Name) - + // If the secret is of ClusterResourceSet type, encapsulate real data in the outer data + if s.Bucket.Spec.SecretType == "addons.cluster.x-k8s.io/resource-set" { + secretStringData = map[string]string{ + "access-keys-secret.yaml": fmt.Sprintf(bucketDataSecret, + s.Bucket.Name, + keys[0].AccessKey, + keys[0].SecretKey, + keys[1].AccessKey, + keys[1].SecretKey, + ), + } + } else { + secretStringData = map[string]string{ + "bucket_name": s.Bucket.Name, + "access_key_rw": keys[0].AccessKey, + "secret_key_rw": keys[0].SecretKey, + "access_key_ro": keys[1].AccessKey, + "secret_key_ro": keys[1].SecretKey, + } + } secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, Namespace: s.Bucket.Namespace, }, - StringData: map[string]string{ - "read_write": keys[0].AccessKey, - "read_only": keys[1].AccessKey, - }, + Type: corev1.SecretType(s.Bucket.Spec.SecretType), + StringData: secretStringData, } scheme := s.Client.Scheme() diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeobjectstoragebuckets.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeobjectstoragebuckets.yaml index b15a44b9b..2332c568f 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeobjectstoragebuckets.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_linodeobjectstoragebuckets.yaml @@ -83,6 +83,11 @@ spec: description: KeyGeneration may be modified to trigger rotations of access keys created for the bucket. type: integer + secretType: + default: addons.cluster.x-k8s.io/resource-set + description: secretType sets the type for the access-keys secret that + will be generated by the controller. + type: string required: - cluster type: object diff --git a/controller/linodeobjectstoragebucket_controller_test.go b/controller/linodeobjectstoragebucket_controller_test.go index 40a4a6552..0f47a6d32 100644 --- a/controller/linodeobjectstoragebucket_controller_test.go +++ b/controller/linodeobjectstoragebucket_controller_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/yaml" infrav1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1" "github.com/linode/cluster-api-provider-linode/cloud/scope" @@ -47,6 +48,22 @@ import ( . "github.com/onsi/gomega" ) +type AccessKeySecret struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + } `yaml:"metadata"` + StringData struct { + Bucket_Name string `yaml:"bucket_name"` + Access_Key_RW string `yaml:"access_key_rw"` + Secret_Key_RW string `yaml:"secret_key_rw"` + Access_Key_RO string `yaml:"access_key_ro"` + Secret_Key_RO string `yaml:"secret_key_ro"` + } `yaml:"stringData"` +} + func mockLinodeClientBuilder(m *mock.MockLinodeObjectStorageClient) scope.LinodeObjectStorageClientBuilder { return func(_ string) (scope.LinodeObjectStorageClient, error) { return m, nil @@ -157,9 +174,15 @@ var _ = Describe("lifecycle", Label("lifecycle"), func() { By("creating a secret with access keys") Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&secret), &secret)).To(Succeed()) - Expect(secret.Data).To(HaveLen(2)) - Expect(string(secret.Data["read_write"])).To(Equal(string("key-0"))) - Expect(string(secret.Data["read_only"])).To(Equal(string("key-1"))) + Expect(secret.Data).To(HaveLen(1)) + var key AccessKeySecret + unMarshallingErr := yaml.Unmarshal(secret.Data["access-keys-secret.yaml"], &key) + Expect(unMarshallingErr).NotTo(HaveOccurred()) + Expect(key.StringData.Bucket_Name).To(Equal("lifecycle")) + Expect(key.StringData.Access_Key_RW).To(Equal("key-0")) + Expect(key.StringData.Secret_Key_RW).To(Equal("")) + Expect(key.StringData.Access_Key_RO).To(Equal("key-1")) + Expect(key.StringData.Secret_Key_RO).To(Equal("")) By("recording the expected events") Expect(<-recorder.Events).To(ContainSubstring("Object storage keys assigned")) @@ -207,9 +230,15 @@ var _ = Describe("lifecycle", Label("lifecycle"), func() { By("re-creating it when it is deleted") Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&secret), &secret)).To(Succeed()) - Expect(secret.Data).To(HaveLen(2)) - Expect(string(secret.Data["read_write"])).To(Equal("key-0")) - Expect(string(secret.Data["read_only"])).To(Equal("key-1")) + Expect(secret.Data).To(HaveLen(1)) + var key AccessKeySecret + unMarshallingErr := yaml.Unmarshal(secret.Data["access-keys-secret.yaml"], &key) + Expect(unMarshallingErr).NotTo(HaveOccurred()) + Expect(key.StringData.Bucket_Name).To(Equal("lifecycle")) + Expect(key.StringData.Access_Key_RW).To(Equal("key-0")) + Expect(key.StringData.Secret_Key_RW).To(Equal("")) + Expect(key.StringData.Access_Key_RO).To(Equal("key-1")) + Expect(key.StringData.Secret_Key_RO).To(Equal("")) By("recording the expected events") Expect(<-recorder.Events).To(ContainSubstring("Object storage keys retrieved")) diff --git a/templates/addons/etcd-backup-restore/etcd-backup-restore.yaml b/templates/addons/etcd-backup-restore/etcd-backup-restore.yaml new file mode 100644 index 000000000..07cbe63ab --- /dev/null +++ b/templates/addons/etcd-backup-restore/etcd-backup-restore.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${CLUSTER_NAME}-etcd-backup-restore-addon +data: + etcd-backup-restore-statefulset.yaml: | + --- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd-backup + namespace: kube-system + spec: + selector: + matchLabels: + app.kubernetes.io/name: etcd + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: etcd + spec: + initContainers: + - name: create-creds-file + image: busybox + command: + - ash + - -c + - | + cat << EOF > /data/credentials-file + { + "accessKeyID": "$${AWS_ACCESS_KEY_ID}", + "secretAccessKey": "$${AWS_SECRET_ACCESS_KEY}", + "region": "$${AWS_REGION}", + "endpoint": "$${AWS_REGION}.linodeobjects.com", + "sseCustomerKey": "$${AWS_SSE_CUSTOMER_KEY}" + } + EOF + volumeMounts: + - name: data-volume + mountPath: /data + env: + - name: "AWS_REGION" + value: ${OBJ_BUCKET_REGION} + - name: "AWS_ACCESS_KEY_ID" + valueFrom: + secretKeyRef: + name: etcd-backup + key: "access_key_rw" + - name: "AWS_SECRET_ACCESS_KEY" + valueFrom: + secretKeyRef: + name: etcd-backup + key: "secret_key_rw" + - name: "AWS_SSE_CUSTOMER_KEY" + valueFrom: + secretKeyRef: + name: sse-key + key: sse-key + containers: + - name: backup-restore + command: + - /etcdbrctl + - snapshot + - --schedule=${ETCD_BACKUP_SCHEDULE:=0 0 * * *} + - --garbage-collection-period=1m + - --storage-provider=S3 + - --store-prefix=etcd-backup + - --insecure-transport=true + - --insecure-skip-tls-verify=true + - --endpoints=https://127.0.0.1:2379 + - --defragmentation-schedule=0 0 */3 * * + - --etcd-connection-timeout=30s + - --delta-snapshot-period=60s + - --delta-snapshot-memory-limit=104857600 + - --compress-snapshots=true + - --compression-policy=gzip + - --cacert=/etc/kubernetes/pki/etcd/ca.crt + - --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt + - --key=/etc/kubernetes/pki/etcd/healthcheck-client.key + image: ${ETCDBR_IMAGE} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + imagePullPolicy: Always + ports: + - containerPort: 8080 + name: server + protocol: TCP + resources: + limits: + cpu: 100m + memory: 1Gi + requests: + cpu: 23m + memory: 128Mi + env: + - name: AWS_APPLICATION_CREDENTIALS_JSON + value: /home/.aws/credentials-file + - name: STORAGE_CONTAINER + value: ${CLUSTER_NAME}-etcd-backup + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: k8s-certs + readOnly: true + - name: data-volume + mountPath: /home/.aws/ + readOnly: true + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/pki + type: DirectoryOrCreate + name: k8s-certs + - name: data-volume + emptyDir: {} + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-etcd-backup-restore +spec: + clusterSelector: + matchLabels: + etcd-backup: "true" + cluster: ${CLUSTER_NAME} + resources: + - kind: ConfigMap + name: ${CLUSTER_NAME}-etcd-backup-restore-addon + strategy: ApplyOnce diff --git a/templates/addons/etcd-backup-restore/kustomization.yaml b/templates/addons/etcd-backup-restore/kustomization.yaml new file mode 100644 index 000000000..8e2c5a957 --- /dev/null +++ b/templates/addons/etcd-backup-restore/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - etcd-backup-restore.yaml + - linode-obj.yaml + - sse-key-secret.yaml diff --git a/templates/addons/etcd-backup-restore/linode-obj.yaml b/templates/addons/etcd-backup-restore/linode-obj.yaml new file mode 100644 index 000000000..2e26a605c --- /dev/null +++ b/templates/addons/etcd-backup-restore/linode-obj.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeObjectStorageBucket +metadata: + labels: + app.kubernetes.io/name: linodeobjectstoragebucket + app.kubernetes.io/instance: ${CLUSTER_NAME}-etcd-backup + app.kubernetes.io/part-of: cluster-api-provider-linode + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-provider-linode + name: ${CLUSTER_NAME}-etcd-backup +spec: + cluster: ${OBJ_BUCKET_REGION} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-etcd-backup-access-keys +spec: + clusterSelector: + matchLabels: + etcd-backup: "true" + cluster: ${CLUSTER_NAME} + resources: + - kind: Secret + name: ${CLUSTER_NAME}-etcd-backup-access-keys + strategy: ApplyOnce diff --git a/templates/addons/etcd-backup-restore/sse-key-secret.yaml b/templates/addons/etcd-backup-restore/sse-key-secret.yaml new file mode 100644 index 000000000..effd0c6ac --- /dev/null +++ b/templates/addons/etcd-backup-restore/sse-key-secret.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Secret +type: addons.cluster.x-k8s.io/resource-set +metadata: + name: ${CLUSTER_NAME}-ssekey +stringData: + sse-key.yaml: |- + kind: Secret + apiVersion: v1 + metadata: + name: sse-key + namespace: kube-system + stringData: + sse-key: ${SSE_KEY} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-ssekey-crs +spec: + clusterSelector: + matchLabels: + etcd-backup: "true" + cluster: ${CLUSTER_NAME} + resources: + - kind: Secret + name: ${CLUSTER_NAME}-ssekey + strategy: ApplyOnce diff --git a/templates/flavors/etcd-backup-restore/kustomization.yaml b/templates/flavors/etcd-backup-restore/kustomization.yaml new file mode 100644 index 000000000..3696694c1 --- /dev/null +++ b/templates/flavors/etcd-backup-restore/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../default + - ../../addons/etcd-backup-restore +patches: + - target: + group: cluster.x-k8s.io + version: v1beta1 + kind: Cluster + patch: |- + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + metadata: + name: ${CLUSTER_NAME} + labels: + etcd-backup: "true"