From 765d55140ea3f781c1b25337ce299ea04857ded3 Mon Sep 17 00:00:00 2001 From: Kirill-Garbar <41306070+Kirill-Garbar@users.noreply.github.com> Date: Thu, 11 Apr 2024 10:03:55 +0200 Subject: [PATCH] Implement mounting user certificates and keys (#125) This PR allows to define user-managed peer, client-server certificates. It implements security specification: ```yaml spec: ... security: tls: peerTrustedCASecret: ca-peer-secret peerSecret: peer-secret serverSecret: server-secret clientTrustedCASecret: ca-client-secret clientSecret: client-secret ... ``` Enabled RBAC and root certificate watching **is not** part of this PR. --------- Co-authored-by: Kirill Garbar --- api/v1alpha1/etcdcluster_types.go | 31 ++ api/v1alpha1/etcdcluster_webhook.go | 47 +++ api/v1alpha1/etcdcluster_webhook_test.go | 86 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 36 +++ charts/etcd-operator/crds/etcd-cluster.yaml | 25 ++ .../crd/bases/etcd.aenix.io_etcdclusters.yaml | 34 +++ config/manager/kustomization.yaml | 5 +- ...tcdcluster-with-external-certificates.yaml | 220 ++++++++++++++ internal/controller/factory/statefulset.go | 271 ++++++++++++++---- .../controller/factory/statefulset_test.go | 109 ++++++- 10 files changed, 793 insertions(+), 71 deletions(-) create mode 100644 examples/manifests/etcdcluster-with-external-certificates.yaml diff --git a/api/v1alpha1/etcdcluster_types.go b/api/v1alpha1/etcdcluster_types.go index adbbb3f4..b9a10c79 100644 --- a/api/v1alpha1/etcdcluster_types.go +++ b/api/v1alpha1/etcdcluster_types.go @@ -41,6 +41,9 @@ type EtcdClusterSpec struct { // +optional PodDisruptionBudgetTemplate *EmbeddedPodDisruptionBudget `json:"podDisruptionBudgetTemplate,omitempty"` Storage StorageSpec `json:"storage"` + // Security describes security settings of etcd (authentication, certificates, rbac) + // +optional + Security *SecuritySpec `json:"security,omitempty"` } const ( @@ -200,6 +203,34 @@ type StorageSpec struct { VolumeClaimTemplate EmbeddedPersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` } +// SecuritySpec defines security settings for etcd. +// +k8s:openapi-gen=true +type SecuritySpec struct { + // Section for user-managed tls certificates + // +optional + TLS TLSSpec `json:"tls,omitempty"` +} + +// TLSSpec defines user-managed certificates names. +type TLSSpec struct { + // Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + // +optional + PeerTrustedCASecret string `json:"peerTrustedCASecret,omitempty"` + // Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + // +optional + PeerSecret string `json:"peerSecret,omitempty"` + // Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + // It is expected to have tls.crt and tls.key fields in the secret. + // +optional + ServerSecret string `json:"serverSecret,omitempty"` + // Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + // +optional + ClientTrustedCASecret string `json:"clientTrustedCASecret,omitempty"` + // Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + // +optional + ClientSecret string `json:"clientSecret,omitempty"` +} + // EmbeddedPersistentVolumeClaim is an embedded version of k8s.io/api/core/v1.PersistentVolumeClaim. // It contains TypeMeta and a reduced ObjectMeta. type EmbeddedPersistentVolumeClaim struct { diff --git a/api/v1alpha1/etcdcluster_webhook.go b/api/v1alpha1/etcdcluster_webhook.go index 44d424ee..69253758 100644 --- a/api/v1alpha1/etcdcluster_webhook.go +++ b/api/v1alpha1/etcdcluster_webhook.go @@ -95,6 +95,11 @@ func (r *EtcdCluster) ValidateCreate() (admission.Warnings, error) { allErrors = append(allErrors, pdbErr...) } + securityErr := r.validateSecurity() + if securityErr != nil { + allErrors = append(allErrors, securityErr...) + } + if errOptions := validateOptions(r); errOptions != nil { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "options"), @@ -139,6 +144,11 @@ func (r *EtcdCluster) ValidateUpdate(old runtime.Object) (admission.Warnings, er warnings = append(warnings, pdbWarnings...) } + securityErr := r.validateSecurity() + if securityErr != nil { + allErrors = append(allErrors, securityErr...) + } + if errOptions := validateOptions(r); errOptions != nil { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "options"), @@ -256,6 +266,43 @@ func (r *EtcdCluster) validatePdb() (admission.Warnings, field.ErrorList) { return warnings, nil } +func (r *EtcdCluster) validateSecurity() field.ErrorList { + + var allErrors field.ErrorList + + if r.Spec.Security == nil { + return nil + } + + security := r.Spec.Security + + if (security.TLS.PeerSecret != "" && security.TLS.PeerTrustedCASecret == "") || + (security.TLS.PeerSecret == "" && security.TLS.PeerTrustedCASecret != "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "tls"), + security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty"), + ) + } + + if (security.TLS.ClientSecret != "" && security.TLS.ClientTrustedCASecret == "") || + (security.TLS.ClientSecret == "" && security.TLS.ClientTrustedCASecret != "") { + + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "security", "tls"), + security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty"), + ) + } + + if len(allErrors) > 0 { + return allErrors + } + + return nil +} + func validateOptions(cluster *EtcdCluster) error { if len(cluster.Spec.Options) == 0 { return nil diff --git a/api/v1alpha1/etcdcluster_webhook_test.go b/api/v1alpha1/etcdcluster_webhook_test.go index ac176551..e9998d1c 100644 --- a/api/v1alpha1/etcdcluster_webhook_test.go +++ b/api/v1alpha1/etcdcluster_webhook_test.go @@ -130,6 +130,92 @@ var _ = Describe("EtcdCluster Webhook", func() { }) }) + Context("Validate Security", func() { + etcdCluster := &EtcdCluster{ + Spec: EtcdClusterSpec{ + Replicas: ptr.To(int32(3)), + Security: &SecuritySpec{}, + }, + } + It("Should admit enabled empty security", func() { + localCluster := etcdCluster.DeepCopy() + err := localCluster.validateSecurity() + Expect(err).To(BeNil()) + }) + + It("Should reject if only one peer secret is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.TLS = TLSSpec{ + PeerTrustedCASecret: "test-peer-ca-cert", + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one peer secret is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.TLS = TLSSpec{ + PeerSecret: "test-peer-cert", + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.peerSecret and spec.security.tls.peerTrustedCASecret must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one client secret is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.TLS = TLSSpec{ + ClientTrustedCASecret: "test-client-ca-cert", + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + + It("Should reject if only one client secret is defined", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security.TLS = TLSSpec{ + ClientTrustedCASecret: "test-client-cert", + } + err := localCluster.validateSecurity() + if Expect(err).NotTo(BeNil()) { + expectedFieldErr := field.Invalid( + field.NewPath("spec", "security", "tls"), + localCluster.Spec.Security.TLS, + "both spec.security.tls.clientSecret and spec.security.tls.clientTrustedCASecret must be filled or empty", + ) + if Expect(err).To(HaveLen(1)) { + Expect(*(err[0])).To(Equal(*expectedFieldErr)) + } + } + }) + }) + Context("Validate PDB", func() { etcdCluster := &EtcdCluster{ Spec: EtcdClusterSpec{ diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index add1e7d7..db42b721 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -173,6 +173,11 @@ func (in *EtcdClusterSpec) DeepCopyInto(out *EtcdClusterSpec) { (*in).DeepCopyInto(*out) } in.Storage.DeepCopyInto(&out.Storage) + if in.Security != nil { + in, out := &in.Security, &out.Security + *out = new(SecuritySpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdClusterSpec. @@ -329,6 +334,22 @@ func (in *PodTemplate) DeepCopy() *PodTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecuritySpec) DeepCopyInto(out *SecuritySpec) { + *out = *in + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecuritySpec. +func (in *SecuritySpec) DeepCopy() *SecuritySpec { + if in == nil { + return nil + } + out := new(SecuritySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { *out = *in @@ -349,3 +370,18 @@ func (in *StorageSpec) DeepCopy() *StorageSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSpec. +func (in *TLSSpec) DeepCopy() *TLSSpec { + if in == nil { + return nil + } + out := new(TLSSpec) + in.DeepCopyInto(out) + return out +} diff --git a/charts/etcd-operator/crds/etcd-cluster.yaml b/charts/etcd-operator/crds/etcd-cluster.yaml index 86e3ab01..4b4ba363 100644 --- a/charts/etcd-operator/crds/etcd-cluster.yaml +++ b/charts/etcd-operator/crds/etcd-cluster.yaml @@ -4318,6 +4318,31 @@ spec: format: int32 minimum: 0 type: integer + security: + description: Security describes security settings of etcd (authentication, certificates, rbac) + properties: + tls: + description: Section for user-managed tls certificates + properties: + clientSecret: + description: Client certificate for etcd-operator to do maintenance. It is expected to have tls.crt and tls.key fields in the secret. + type: string + clientTrustedCASecret: + description: Trusted CA for client certificates that are provided by client to etcd. It is expected to have tls.crt field in the secret. + type: string + peerSecret: + description: Certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt and tls.key fields in the secret. + type: string + peerTrustedCASecret: + description: Trusted CA certificate secret to secure peer-to-peer communication between etcd nodes. It is expected to have tls.crt field in the secret. + type: string + serverSecret: + description: |- + Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + It is expected to have tls.crt and tls.key fields in the secret. + type: string + type: object + type: object storage: description: |- StorageSpec defines the configured storage for a etcd members. diff --git a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml index 1b808fc3..cbfc3188 100644 --- a/config/crd/bases/etcd.aenix.io_etcdclusters.yaml +++ b/config/crd/bases/etcd.aenix.io_etcdclusters.yaml @@ -4571,6 +4571,40 @@ spec: format: int32 minimum: 0 type: integer + security: + description: Security describes security settings of etcd (authentication, + certificates, rbac) + properties: + tls: + description: Section for user-managed tls certificates + properties: + clientSecret: + description: Client certificate for etcd-operator to do maintenance. + It is expected to have tls.crt and tls.key fields in the + secret. + type: string + clientTrustedCASecret: + description: Trusted CA for client certificates that are provided + by client to etcd. It is expected to have tls.crt field + in the secret. + type: string + peerSecret: + description: Certificate secret to secure peer-to-peer communication + between etcd nodes. It is expected to have tls.crt and tls.key + fields in the secret. + type: string + peerTrustedCASecret: + description: Trusted CA certificate secret to secure peer-to-peer + communication between etcd nodes. It is expected to have + tls.crt field in the secret. + type: string + serverSecret: + description: |- + Server certificate secret to secure client-server communication. Is provided to the client who connects to etcd by client port (2379 by default). + It is expected to have tls.crt and tls.key fields in the secret. + type: string + type: object + type: object storage: description: |- StorageSpec defines the configured storage for a etcd members. diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 8335489f..40c41b8b 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -2,4 +2,7 @@ resources: - manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -images: [] +images: +- name: ghcr.io/aenix-io/etcd-operator + newName: ghcr.io/aenix-io/etcd-operator + newTag: latest diff --git a/examples/manifests/etcdcluster-with-external-certificates.yaml b/examples/manifests/etcdcluster-with-external-certificates.yaml new file mode 100644 index 00000000..59c3b43e --- /dev/null +++ b/examples/manifests/etcdcluster-with-external-certificates.yaml @@ -0,0 +1,220 @@ +--- +apiVersion: etcd.aenix.io/v1alpha1 +kind: EtcdCluster +metadata: + name: test + namespace: default +spec: + storage: {} + security: + tls: + peerTrustedCASecret: ca-peer-secret + peerSecret: peer-secret + serverSecret: server-secret + clientTrustedCASecret: ca-client-secret + clientSecret: client-secret +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: default +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-peer + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-peer + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-peer-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-server + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-server + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-server-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ca-certificate-client + namespace: default +spec: + isCA: true + usages: + - "signing" + - "key encipherment" + - "cert sign" + commonName: ca-certificate-client + subject: + organizations: + - ACME Inc. + organizationalUnits: + - Widgets + secretName: ca-client-secret + privateKey: + algorithm: RSA + size: 4096 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-peer + namespace: default +spec: + ca: + secretName: ca-peer-secret +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-server + namespace: default +spec: + ca: + secretName: ca-server-secret +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer-client + namespace: default +spec: + ca: + secretName: ca-client-secret +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: server-certificate + namespace: default +spec: + secretName: server-secret + isCA: false + usages: + - server auth + - signing + - key encipherment + dnsNames: + - test-0 + - test-0.test + - test-0.test.default.svc + - test-0.test.default.svc.cluster.local + - test-1 + - test-1.test + - test-1.test.default.svc + - test-1.test.default.svc.cluster.local + - test-2 + - test-2.test + - test-2.test.default.svc + - test-2.test.default.svc.cluster.local + - test-client + - test-client.default.svc + - test-client.default.svc.cluster.local + - localhost + - "127.0.0.1" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 + issuerRef: + name: ca-issuer-server +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: peer-certificate + namespace: default +spec: + secretName: peer-secret + isCA: false + usages: + - server auth + - client auth + - signing + - key encipherment + dnsNames: + - test-0 + - test-0.test + - test-0.test.default.svc + - test-0.test.default.svc.cluster.local + - test-1 + - test-1.test + - test-1.test.default.svc + - test-1.test.default.svc.cluster.local + - test-2 + - test-2.test + - test-2.test.default.svc + - test-2.test.default.svc.cluster.local + - localhost + - "127.0.0.1" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 + issuerRef: + name: ca-issuer-peer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: client-certificate + namespace: default +spec: + commonName: root + secretName: client-secret + usages: + - "signing" + - "key encipherment" + - "client auth" + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 4096 + issuerRef: + name: ca-issuer-client + kind: Issuer diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index a433643c..f9447901 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -32,6 +32,10 @@ import ( etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" ) +const ( + etcdContainerName = "etcd" +) + func CreateOrUpdateStatefulSet( ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster, @@ -52,6 +56,20 @@ func CreateOrUpdateStatefulSet( podMetadata.Annotations = cluster.Spec.PodTemplate.Annotations + volumeClaimTemplates := []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: GetPVCName(cluster), + Labels: cluster.Spec.Storage.VolumeClaimTemplate.Labels, + Annotations: cluster.Spec.Storage.VolumeClaimTemplate.Annotations, + }, + Spec: cluster.Spec.Storage.VolumeClaimTemplate.Spec, + Status: cluster.Spec.Storage.VolumeClaimTemplate.Status, + }, + } + + volumes := generateVolumes(cluster) + statefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: cluster.Namespace, @@ -81,54 +99,168 @@ func CreateOrUpdateStatefulSet( ServiceAccountName: cluster.Spec.PodTemplate.Spec.ServiceAccountName, ReadinessGates: cluster.Spec.PodTemplate.Spec.ReadinessGates, RuntimeClassName: cluster.Spec.PodTemplate.Spec.RuntimeClassName, + Volumes: volumes, }, }, + VolumeClaimTemplates: volumeClaimTemplates, }, } - statefulSet.Spec.Template.Spec.Volumes = cluster.Spec.PodTemplate.Spec.Volumes - dataVolumeIdx := slices.IndexFunc(statefulSet.Spec.Template.Spec.Volumes, func(volume corev1.Volume) bool { + + if err := ctrl.SetControllerReference(cluster, statefulSet, rscheme); err != nil { + return fmt.Errorf("cannot set controller reference: %w", err) + } + + return reconcileStatefulSet(ctx, rclient, cluster.Name, statefulSet) +} + +func generateVolumes(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Volume { + volumes := []corev1.Volume{} + + var dataVolumeSource corev1.VolumeSource + + if cluster.Spec.Storage.EmptyDir != nil { + dataVolumeSource = corev1.VolumeSource{EmptyDir: cluster.Spec.Storage.EmptyDir} + } else { + dataVolumeSource = corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: GetPVCName(cluster), + }, + } + } + + dataVolumeIdx := slices.IndexFunc(cluster.Spec.PodTemplate.Spec.Volumes, func(volume corev1.Volume) bool { return volume.Name == "data" }) if dataVolumeIdx == -1 { - dataVolumeIdx = len(statefulSet.Spec.Template.Spec.Volumes) - statefulSet.Spec.Template.Spec.Volumes = append( - statefulSet.Spec.Template.Spec.Volumes, - corev1.Volume{Name: "data"}, + dataVolumeIdx = len(cluster.Spec.PodTemplate.Spec.Volumes) + volumes = append( + volumes, + corev1.Volume{}, ) } - if cluster.Spec.Storage.EmptyDir != nil { - statefulSet.Spec.Template.Spec.Volumes[dataVolumeIdx] = corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{EmptyDir: cluster.Spec.Storage.EmptyDir}, - } - } else { - statefulSet.Spec.Template.Spec.Volumes[dataVolumeIdx] = corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: GetPVCName(cluster), + volumes[dataVolumeIdx] = corev1.Volume{ + Name: "data", + VolumeSource: dataVolumeSource, + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { + volumes = append(volumes, + []corev1.Volume{ + { + Name: "peer-trusted-ca-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.TLS.PeerTrustedCASecret, + }, + }, }, - }, + { + Name: "peer-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.TLS.PeerSecret, + }, + }, + }, + }...) + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { + volumes = append(volumes, + []corev1.Volume{ + { + Name: "server-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.TLS.ServerSecret, + }, + }, + }, + }...) + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + volumes = append(volumes, + []corev1.Volume{ + { + Name: "client-trusted-ca-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: cluster.Spec.Security.TLS.ClientTrustedCASecret, + }, + }, + }, + }...) + } + + return volumes + +} + +func generateVolumeMounts(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.VolumeMount { + + volumeMounts := []corev1.VolumeMount{} + + for _, c := range cluster.Spec.PodTemplate.Spec.Containers { + if c.Name == etcdContainerName { + + volumeMounts = c.VolumeMounts + + mountIdx := slices.IndexFunc(volumeMounts, func(mount corev1.VolumeMount) bool { + return mount.Name == "data" + }) + if mountIdx == -1 { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "data", + ReadOnly: false, + MountPath: "/var/run/etcd", + }) + } else { + volumeMounts[mountIdx].ReadOnly = false + volumeMounts[mountIdx].MountPath = "/var/run/etcd" + } } - statefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + + } + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ { - ObjectMeta: metav1.ObjectMeta{ - Name: GetPVCName(cluster), - Labels: cluster.Spec.Storage.VolumeClaimTemplate.Labels, - Annotations: cluster.Spec.Storage.VolumeClaimTemplate.Annotations, - }, - Spec: cluster.Spec.Storage.VolumeClaimTemplate.Spec, - Status: cluster.Spec.Storage.VolumeClaimTemplate.Status, + Name: "peer-trusted-ca-certificate", + ReadOnly: true, + MountPath: "/etc/etcd/pki/peer/ca", }, - } + { + Name: "peer-certificate", + ReadOnly: true, + MountPath: "/etc/etcd/pki/peer/cert", + }, + }...) } - if err := ctrl.SetControllerReference(cluster, statefulSet, rscheme); err != nil { - return fmt.Errorf("cannot set controller reference: %w", err) + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + Name: "server-certificate", + ReadOnly: true, + MountPath: "/etc/etcd/pki/server/cert", + }, + }...) } - return reconcileStatefulSet(ctx, rclient, cluster.Name, statefulSet) + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + + volumeMounts = append(volumeMounts, []corev1.VolumeMount{ + { + Name: "client-trusted-ca-certificate", + ReadOnly: true, + MountPath: "/etc/etcd/pki/client/ca", + }, + }...) + } + + return volumeMounts } func generateEtcdCommand() []string { @@ -138,17 +270,7 @@ func generateEtcdCommand() []string { } func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { - args := []string{ - "--name=$(POD_NAME)", - "--listen-peer-urls=https://0.0.0.0:2380", - // for first version disable TLS for client access - "--listen-client-urls=http://0.0.0.0:2379", - fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", cluster.Name), - "--data-dir=/var/run/etcd/default.etcd", - "--auto-tls", - "--peer-auto-tls", - fmt.Sprintf("--advertise-client-urls=http://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", cluster.Name), - } + args := []string{} for name, value := range cluster.Spec.Options { flag := "--" + name @@ -161,6 +283,51 @@ func generateEtcdArgs(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { args = append(args, fmt.Sprintf("%s=%s", flag, value)) } + peerTlsSettings := []string{"--peer-auto-tls"} + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.PeerSecret != "" { + peerTlsSettings = []string{ + "--peer-trusted-ca-file=/etc/etcd/pki/peer/ca/ca.crt", + "--peer-cert-file=/etc/etcd/pki/peer/cert/tls.crt", + "--peer-key-file=/etc/etcd/pki/peer/cert/tls.key", + "--peer-client-cert-auth", + } + } + + serverTlsSettings := []string{} + serverProtocol := "http" + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ServerSecret != "" { + serverTlsSettings = []string{ + "--cert-file=/etc/etcd/pki/server/cert/tls.crt", + "--key-file=/etc/etcd/pki/server/cert/tls.key", + } + serverProtocol = "https" + } + + clientTlsSettings := []string{} + + if cluster.Spec.Security != nil && cluster.Spec.Security.TLS.ClientSecret != "" { + clientTlsSettings = []string{ + "--trusted-ca-file=/etc/etcd/pki/client/ca/ca.crt", + "--client-cert-auth", + } + } + + args = append(args, []string{ + "--name=$(POD_NAME)", + "--listen-metrics-urls=http://0.0.0.0:2381", + "--listen-peer-urls=https://0.0.0.0:2380", + fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", serverProtocol), + fmt.Sprintf("--initial-advertise-peer-urls=https://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2380", cluster.Name), + "--data-dir=/var/run/etcd/default.etcd", + fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc:2379", serverProtocol, cluster.Name), + }...) + + args = append(args, peerTlsSettings...) + args = append(args, serverTlsSettings...) + args = append(args, clientTlsSettings...) + return args } @@ -186,7 +353,7 @@ func generateContainers(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Conta containers := make([]corev1.Container, 0, len(cluster.Spec.PodTemplate.Spec.Containers)) for _, c := range cluster.Spec.PodTemplate.Spec.Containers { - if c.Name == "etcd" { + if c.Name == etcdContainerName { c.Command = generateEtcdCommand() c.Args = generateEtcdArgs(cluster) c.Ports = mergePorts(c.Ports, []corev1.ContainerPort{ @@ -210,19 +377,7 @@ func generateContainers(cluster *etcdaenixiov1alpha1.EtcdCluster) []corev1.Conta c.LivenessProbe = getLivenessProbe(c.LivenessProbe) c.ReadinessProbe = getReadinessProbe(c.ReadinessProbe) c.Env = mergeEnvs(c.Env, podEnv) - - mountIdx := slices.IndexFunc(c.VolumeMounts, func(mount corev1.VolumeMount) bool { - return mount.Name == "data" - }) - if mountIdx == -1 { - c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ - Name: "data", - MountPath: "/var/run/etcd", - }) - } else { - c.VolumeMounts[mountIdx].ReadOnly = false - c.VolumeMounts[mountIdx].MountPath = "/var/run/etcd" - } + c.VolumeMounts = generateVolumeMounts(cluster) } containers = append(containers, c) @@ -270,7 +425,7 @@ func getStartupProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -283,7 +438,7 @@ func getReadinessProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -296,7 +451,7 @@ func getLivenessProbe(probe *corev1.Probe) *corev1.Probe { ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index 8618f0c2..b57bdc5f 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -117,6 +118,15 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { }, }, } + etcdcluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ + TLS: etcdaenixiov1alpha1.TLSSpec{ + PeerTrustedCASecret: "peer-ca-secret", + PeerSecret: "peer-cert-secret", + ServerSecret: "server-cert-secret", + ClientTrustedCASecret: "client-ca-secret", + ClientSecret: "client-secret", + }, + } sts := &appsv1.StatefulSet{} err := CreateOrUpdateStatefulSet(ctx, etcdcluster, k8sClient, k8sClient.Scheme()) @@ -157,7 +167,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -173,7 +183,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -189,7 +199,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -200,6 +210,45 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { })) }) + By("Checking generated security volumes", func() { + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "peer-trusted-ca-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "peer-ca-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "peer-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "peer-cert-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "server-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "server-cert-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + Expect(sts.Spec.Template.Spec.Volumes).To(ContainElement(v1.Volume{ + Name: "client-trusted-ca-certificate", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "client-ca-secret", + DefaultMode: ptr.To(int32(420)), + }, + }, + })) + }) + By("Deleting the statefulset", func() { Expect(k8sClient.Delete(ctx, sts)).To(Succeed()) }) @@ -259,7 +308,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -275,7 +324,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), Scheme: v1.URISchemeHTTP, }, }, @@ -353,7 +402,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -381,7 +430,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { PeriodSeconds: 3, })) }) - It("should correctly override partial changes ", func() { + It("should correctly override partial changes", func() { probe := getLivenessProbe(&v1.Probe{ InitialDelaySeconds: 7, PeriodSeconds: 3, @@ -390,7 +439,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 7, @@ -406,7 +455,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -443,7 +492,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/readyz?serializable=false", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 7, @@ -459,7 +508,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, PeriodSeconds: 5, @@ -496,7 +545,7 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ Path: "/livez", - Port: intstr.FromInt32(2379), + Port: intstr.FromInt32(2381), }, }, InitialDelaySeconds: 11, @@ -639,5 +688,41 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { } } }) + It("should generate security volumes mounts", func() { + localCluster := etcdCluster.DeepCopy() + localCluster.Spec.Security = &etcdaenixiov1alpha1.SecuritySpec{ + TLS: etcdaenixiov1alpha1.TLSSpec{ + PeerTrustedCASecret: "peer-ca-secret", + PeerSecret: "peer-cert-secret", + ServerSecret: "server-cert-secret", + ClientTrustedCASecret: "client-ca-secret", + ClientSecret: "client-secret", + }, + } + + containers := generateContainers(localCluster) + + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "peer-trusted-ca-certificate", + MountPath: "/etc/etcd/pki/peer/ca", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "peer-certificate", + MountPath: "/etc/etcd/pki/peer/cert", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "server-certificate", + MountPath: "/etc/etcd/pki/server/cert", + ReadOnly: true, + })) + Expect(containers[0].VolumeMounts).To(ContainElement(v1.VolumeMount{ + Name: "client-trusted-ca-certificate", + MountPath: "/etc/etcd/pki/client/ca", + ReadOnly: true, + })) + }) + }) })