Install etcd-backup-restore for child clusters

api/v1alpha1/linodeobjectstoragebucket_types.go

@@ -43,6 +43,11 @@ type LinodeObjectStorageBucketSpec struct {
 	// +optional
 	// +kubebuilder:default=0
 	KeyGeneration *int `json:"keyGeneration,omitempty"`
+
+	// secretType sets the type for the access-keys secret that will be generated by the controller.
+	// +optional
+	// +kubebuilder:default=addons.cluster.x-k8s.io/resource-set
+	SecretType string `json:"secretType,omitempty"`
 }
 
 // LinodeObjectStorageBucketStatus defines the observed state of LinodeObjectStorageBucket

cloud/scope/object_storage_bucket.go

@@ -17,6 +17,18 @@ import (
 	infrav1alpha1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1"
 )
 
+const bucketDataSecret = `kind: Secret
+apiVersion: v1
+metadata:
+  name: etcd-backup
+  namespace: kube-system
+stringData:
+  bucket_name: %s
+  access_key_rw: %s
+  secret_key_rw: %s
+  access_key_ro: %s
+  secret_key_ro: %s`
+
 type ObjectStorageBucketScopeParams struct {
 	Client              k8sClient
 	LinodeClientBuilder LinodeObjectStorageClientBuilder
@@ -109,18 +121,35 @@ func (s *ObjectStorageBucketScope) GenerateKeySecret(ctx context.Context, keys [
 			return nil, errors.New("expected two non-nil object storage keys")
 		}
 	}
-
+	var secretStringData map[string]string
 	secretName := fmt.Sprintf(AccessKeyNameTemplate, s.Bucket.Name)
-
+	// If the secret is of ClusterResourceSet type, encapsulate real data in the outer data
+	if s.Bucket.Spec.SecretType == "addons.cluster.x-k8s.io/resource-set" {
+		secretStringData = map[string]string{
+			"access-keys-secret.yaml": fmt.Sprintf(bucketDataSecret,
+				s.Bucket.Name,
+				keys[0].AccessKey,
+				keys[0].SecretKey,
+				keys[1].AccessKey,
+				keys[1].SecretKey,
+			),
+		}
+	} else {
+		secretStringData = map[string]string{
+			"bucket_name":   s.Bucket.Name,
+			"access_key_rw": keys[0].AccessKey,
+			"secret_key_rw": keys[0].SecretKey,
+			"access_key_ro": keys[1].AccessKey,
+			"secret_key_ro": keys[1].SecretKey,
+		}
+	}
 	secret := &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      secretName,
 			Namespace: s.Bucket.Namespace,
 		},
-		StringData: map[string]string{
-			"read_write": keys[0].AccessKey,
-			"read_only":  keys[1].AccessKey,
-		},
+		Type:       corev1.SecretType(s.Bucket.Spec.SecretType),
+		StringData: secretStringData,
 	}
 
 	scheme := s.Client.Scheme()

config/crd/bases/infrastructure.cluster.x-k8s.io_linodeobjectstoragebuckets.yaml

@@ -83,6 +83,11 @@ spec:
                 description: KeyGeneration may be modified to trigger rotations of
                   access keys created for the bucket.
                 type: integer
+              secretType:
+                default: addons.cluster.x-k8s.io/resource-set
+                description: secretType sets the type for the access-keys secret that
+                  will be generated by the controller.
+                type: string
             required:
             - cluster
             type: object

controller/linodeobjectstoragebucket_controller_test.go

@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/yaml"
 
 	infrav1 "github.com/linode/cluster-api-provider-linode/api/v1alpha1"
 	"github.com/linode/cluster-api-provider-linode/cloud/scope"
@@ -47,6 +48,22 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+type AccessKeySecret struct {
+	APIVersion string `yaml:"apiVersion"`
+	Kind       string `yaml:"kind"`
+	Metadata   struct {
+		Name      string `yaml:"name"`
+		Namespace string `yaml:"namespace"`
+	} `yaml:"metadata"`
+	StringData struct {
+		Bucket_Name   string `yaml:"bucket_name"`
+		Access_Key_RW string `yaml:"access_key_rw"`
+		Secret_Key_RW string `yaml:"secret_key_rw"`
+		Access_Key_RO string `yaml:"access_key_ro"`
+		Secret_Key_RO string `yaml:"secret_key_ro"`
+	} `yaml:"stringData"`
+}
+
 func mockLinodeClientBuilder(m *mock.MockLinodeObjectStorageClient) scope.LinodeObjectStorageClientBuilder {
 	return func(_ string) (scope.LinodeObjectStorageClient, error) {
 		return m, nil
@@ -157,9 +174,15 @@ var _ = Describe("lifecycle", Label("lifecycle"), func() {
 
 		By("creating a secret with access keys")
 		Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&secret), &secret)).To(Succeed())
-		Expect(secret.Data).To(HaveLen(2))
-		Expect(string(secret.Data["read_write"])).To(Equal(string("key-0")))
-		Expect(string(secret.Data["read_only"])).To(Equal(string("key-1")))
+		Expect(secret.Data).To(HaveLen(1))
+		var key AccessKeySecret
+		unMarshallingErr := yaml.Unmarshal(secret.Data["access-keys-secret.yaml"], &key)
+		Expect(unMarshallingErr).NotTo(HaveOccurred())
+		Expect(key.StringData.Bucket_Name).To(Equal("lifecycle"))
+		Expect(key.StringData.Access_Key_RW).To(Equal("key-0"))
+		Expect(key.StringData.Secret_Key_RW).To(Equal(""))
+		Expect(key.StringData.Access_Key_RO).To(Equal("key-1"))
+		Expect(key.StringData.Secret_Key_RO).To(Equal(""))
 
 		By("recording the expected events")
 		Expect(<-recorder.Events).To(ContainSubstring("Object storage keys assigned"))
@@ -207,9 +230,15 @@ var _ = Describe("lifecycle", Label("lifecycle"), func() {
 
 		By("re-creating it when it is deleted")
 		Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(&secret), &secret)).To(Succeed())
-		Expect(secret.Data).To(HaveLen(2))
-		Expect(string(secret.Data["read_write"])).To(Equal("key-0"))
-		Expect(string(secret.Data["read_only"])).To(Equal("key-1"))
+		Expect(secret.Data).To(HaveLen(1))
+		var key AccessKeySecret
+		unMarshallingErr := yaml.Unmarshal(secret.Data["access-keys-secret.yaml"], &key)
+		Expect(unMarshallingErr).NotTo(HaveOccurred())
+		Expect(key.StringData.Bucket_Name).To(Equal("lifecycle"))
+		Expect(key.StringData.Access_Key_RW).To(Equal("key-0"))
+		Expect(key.StringData.Secret_Key_RW).To(Equal(""))
+		Expect(key.StringData.Access_Key_RO).To(Equal("key-1"))
+		Expect(key.StringData.Secret_Key_RO).To(Equal(""))
 
 		By("recording the expected events")
 		Expect(<-recorder.Events).To(ContainSubstring("Object storage keys retrieved"))

templates/addons/etcd-backup-restore/etcd-backup-restore.yaml

@@ -0,0 +1,135 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ${CLUSTER_NAME}-etcd-backup-restore-addon
+data:
+  etcd-backup-restore-statefulset.yaml: |
+    ---
+    apiVersion: apps/v1
+    kind: StatefulSet
+    metadata:
+      name: etcd-backup
+      namespace: kube-system
+    spec:
+      selector:
+        matchLabels:
+          app.kubernetes.io/name: etcd
+      updateStrategy:
+        type: RollingUpdate
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: etcd
+        spec:
+          initContainers:
+          - name: create-creds-file
+            image: busybox
+            command:
+            - ash
+            - -c
+            - |
+              cat << EOF > /data/credentials-file
+              {
+                "accessKeyID": "$${AWS_ACCESS_KEY_ID}",
+                "secretAccessKey": "$${AWS_SECRET_ACCESS_KEY}",
+                "region": "$${AWS_REGION}",
+                "endpoint": "$${AWS_REGION}.linodeobjects.com",
+                "sseCustomerKey": "$${AWS_SSE_CUSTOMER_KEY}"
+              }
+              EOF
+            volumeMounts:
+              - name: data-volume
+                mountPath: /data
+            env:
+            - name: "AWS_REGION"
+              value: ${OBJ_BUCKET_REGION}
+            - name: "AWS_ACCESS_KEY_ID"
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup
+                  key: "access_key_rw"
+            - name: "AWS_SECRET_ACCESS_KEY"
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup
+                  key: "secret_key_rw"
+            - name: "AWS_SSE_CUSTOMER_KEY"
+              valueFrom:
+                secretKeyRef:
+                  name: sse-key
+                  key: sse-key
+          containers:
+          - name: backup-restore
+            command:
+            - /etcdbrctl
+            - snapshot
+            - --schedule=${ETCD_BACKUP_SCHEDULE:=0 0 * * *}
+            - --garbage-collection-period=1m
+            - --storage-provider=S3
+            - --store-prefix=etcd-backup
+            - --insecure-transport=true
+            - --insecure-skip-tls-verify=true
+            - --endpoints=https://127.0.0.1:2379
+            - --defragmentation-schedule=0 0 */3 * *
+            - --etcd-connection-timeout=30s
+            - --delta-snapshot-period=60s
+            - --delta-snapshot-memory-limit=104857600
+            - --compress-snapshots=true        
+            - --compression-policy=gzip
+            - --cacert=/etc/kubernetes/pki/etcd/ca.crt
+            - --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt
+            - --key=/etc/kubernetes/pki/etcd/healthcheck-client.key
+            image: ${ETCDBR_IMAGE}
+            securityContext:
+              allowPrivilegeEscalation: false
+              runAsUser: 0
+            imagePullPolicy: Always
+            ports:
+            - containerPort: 8080
+              name: server
+              protocol: TCP
+            resources:
+              limits:
+                cpu: 100m
+                memory: 1Gi
+              requests:
+                cpu: 23m
+                memory: 128Mi
+            env:
+            - name: AWS_APPLICATION_CREDENTIALS_JSON
+              value: /home/.aws/credentials-file
+            - name: STORAGE_CONTAINER
+              value: ${CLUSTER_NAME}-etcd-backup
+            volumeMounts:
+            - mountPath: /etc/kubernetes/pki
+              name: k8s-certs
+              readOnly: true
+            - name: data-volume
+              mountPath: /home/.aws/
+              readOnly: true
+          hostNetwork: true
+          volumes:
+          - hostPath:
+              path: /etc/kubernetes/pki
+              type: DirectoryOrCreate
+            name: k8s-certs
+          - name: data-volume
+            emptyDir: {}
+          tolerations:
+          - key: node-role.kubernetes.io/control-plane
+            effect: NoSchedule
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: ${CLUSTER_NAME}-etcd-backup-restore
+spec:
+  clusterSelector:
+    matchLabels:
+      etcd-backup: "true"
+      cluster: ${CLUSTER_NAME}
+  resources:
+  - kind: ConfigMap
+    name: ${CLUSTER_NAME}-etcd-backup-restore-addon
+  strategy: ApplyOnce

templates/addons/etcd-backup-restore/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+    - etcd-backup-restore.yaml
+    - linode-obj.yaml
+    - sse-key-secret.yaml

templates/addons/etcd-backup-restore/linode-obj.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeObjectStorageBucket
+metadata:
+  labels:
+    app.kubernetes.io/name: linodeobjectstoragebucket
+    app.kubernetes.io/instance: ${CLUSTER_NAME}-etcd-backup
+    app.kubernetes.io/part-of: cluster-api-provider-linode
+    app.kubernetes.io/managed-by: kustomize
+    app.kubernetes.io/created-by: cluster-api-provider-linode
+  name: ${CLUSTER_NAME}-etcd-backup
+spec:
+  cluster: ${OBJ_BUCKET_REGION}
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: ${CLUSTER_NAME}-etcd-backup-access-keys
+spec:
+  clusterSelector:
+    matchLabels:
+      etcd-backup: "true"
+      cluster: ${CLUSTER_NAME}
+  resources:
+  - kind: Secret
+    name: ${CLUSTER_NAME}-etcd-backup-access-keys
+  strategy: ApplyOnce

templates/addons/etcd-backup-restore/sse-key-secret.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: Secret
+type: addons.cluster.x-k8s.io/resource-set
+metadata:
+  name: ${CLUSTER_NAME}-ssekey
+stringData:
+  sse-key.yaml: |-
+    kind: Secret
+    apiVersion: v1
+    metadata:
+      name: sse-key
+      namespace: kube-system
+    stringData:
+      sse-key: ${SSE_KEY}
+---
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: ${CLUSTER_NAME}-ssekey-crs
+spec:
+  clusterSelector:
+    matchLabels:
+      etcd-backup: "true"
+      cluster: ${CLUSTER_NAME}
+  resources:
+  - kind: Secret
+    name: ${CLUSTER_NAME}-ssekey
+  strategy: ApplyOnce

templates/flavors/etcd-backup-restore/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../default
+  - ../../addons/etcd-backup-restore
+patches:
+  - target:
+      group: cluster.x-k8s.io
+      version: v1beta1
+      kind: Cluster
+    patch: |-
+      apiVersion: cluster.x-k8s.io/v1beta1
+      kind: Cluster
+      metadata:
+        name: ${CLUSTER_NAME}
+        labels:
+            etcd-backup: "true"