From beac1c05fed721d6bce52c465e36cec2971db730 Mon Sep 17 00:00:00 2001 From: prafull01 Date: Thu, 10 Oct 2024 15:24:44 +0530 Subject: [PATCH] Remove the copy-cert init containers --- cmd/self-signer/root.go | 10 +-- .../templates/role-certRotateSelfSigner.yaml | 5 +- cockroachdb/templates/statefulset.yaml | 47 ++++++------ cockroachdb/values.yaml | 4 +- pkg/generator/generate_cert.go | 7 +- pkg/kube/helpers.go | 71 +++++++++++++++++++ 6 files changed, 113 insertions(+), 31 deletions(-) diff --git a/cmd/self-signer/root.go b/cmd/self-signer/root.go index 77f57e66..2be6ae26 100644 --- a/cmd/self-signer/root.go +++ b/cmd/self-signer/root.go @@ -26,6 +26,7 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,8 +34,9 @@ import ( ) var ( - cl client.Client - ctx context.Context + cl client.Client + config *rest.Config + ctx context.Context ) // rootCmd represents the base command when called without any subcommands @@ -71,7 +73,7 @@ func init() { runtimeScheme := runtime.NewScheme() _ = clientgoscheme.AddToScheme(runtimeScheme) - config := controllerruntime.GetConfigOrDie() + config = controllerruntime.GetConfigOrDie() cl, err = client.New(config, client.Options{ Scheme: runtimeScheme, @@ -85,7 +87,7 @@ func init() { func getInitialConfig(caDuration, caExpiry, nodeDuration, nodeExpiry, clientDuration, clientExpiry string) (generator.GenerateCert, error) { - genCert := generator.NewGenerateCert(cl) + genCert := generator.NewGenerateCert(config, cl) if err := genCert.CaCertConfig.SetConfig(caDuration, caExpiry); err != nil { return genCert, err diff --git a/cockroachdb/templates/role-certRotateSelfSigner.yaml b/cockroachdb/templates/role-certRotateSelfSigner.yaml index f0e2b90c..bb3fb497 100644 --- a/cockroachdb/templates/role-certRotateSelfSigner.yaml +++ b/cockroachdb/templates/role-certRotateSelfSigner.yaml @@ -22,6 +22,9 @@ rules: resourceNames: - {{ template "cockroachdb.fullname" . }} - apiGroups: [""] - resources: ["pods"] + resources: ["pods", "pods/logs"] verbs: ["delete", "get"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] {{- end }} diff --git a/cockroachdb/templates/statefulset.yaml b/cockroachdb/templates/statefulset.yaml index 43a1f594..b6eed27d 100644 --- a/cockroachdb/templates/statefulset.yaml +++ b/cockroachdb/templates/statefulset.yaml @@ -51,7 +51,7 @@ spec: {{- end }} {{- end }} serviceAccountName: {{ template "cockroachdb.serviceAccount.name" . }} - {{- if .Values.tls.enabled }} + {{- if and .Values.tls.enabled .Values.statefulset.securityContext.runAsNonRootUser.enabled }} initContainers: - name: copy-certs image: {{ .Values.tls.copyCerts.image | quote }} @@ -266,11 +266,15 @@ spec: - name: datadir mountPath: /cockroach/{{ .Values.conf.path }}/ {{- if .Values.tls.enabled }} + {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }} - name: certs mountPath: /cockroach/cockroach-certs/ - {{- if .Values.tls.certs.provided }} + {{- end }} - name: certs-secret + {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }} mountPath: /cockroach/certs/ + {{- else }} + mountPath: /cockroach/cockroach-certs/ {{- end }} {{- end }} {{- range .Values.statefulset.secretMounts }} @@ -340,34 +344,29 @@ spec: emptyDir: {} {{- end }} {{- if .Values.tls.enabled }} + {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled}} - name: certs emptyDir: {} + {{- end }} {{- if or .Values.tls.certs.provided .Values.tls.certs.certManager .Values.tls.certs.selfSigner.enabled }} - name: certs-secret - {{- if or .Values.tls.certs.tlsSecret .Values.tls.certs.certManager .Values.tls.certs.selfSigner.enabled }} - projected: - sources: - - secret: - {{- if .Values.tls.certs.selfSigner.enabled }} - name: {{ template "cockroachdb.fullname" . }}-node-secret - {{ else }} - name: {{ .Values.tls.certs.nodeSecret }} - {{ end -}} - items: - - key: ca.crt - path: ca.crt - mode: 256 - - key: tls.crt - path: node.crt - mode: 256 - - key: tls.key - path: node.key - mode: 256 - {{- else }} secret: + {{- if .Values.tls.certs.selfSigner.enabled }} + secretName: {{ template "cockroachdb.fullname" . }}-node-secret + {{ else }} secretName: {{ .Values.tls.certs.nodeSecret }} + {{- end }} defaultMode: 256 - {{- end }} + items: + - key: ca.crt + path: ca.crt + mode: 256 + - key: tls.crt + path: node.crt + mode: 256 + - key: tls.key + path: node.key + mode: 256 {{- end }} {{- end }} {{- range .Values.statefulset.secretMounts }} @@ -385,10 +384,12 @@ spec: securityContext: seccompProfile: type: "RuntimeDefault" + {{- if .Values.statefulset.securityContext.runAsNonRootUser.enabled }} fsGroup: 1000 runAsGroup: 1000 runAsUser: 1000 runAsNonRoot: true + {{- end }} {{- end }} {{- end }} {{- if .Values.storage.persistentVolume.enabled }} diff --git a/cockroachdb/values.yaml b/cockroachdb/values.yaml index 78941485..6ad7893e 100644 --- a/cockroachdb/values.yaml +++ b/cockroachdb/values.yaml @@ -283,6 +283,8 @@ statefulset: securityContext: enabled: true + runAsNonRootUser: + enabled: false serviceAccount: # Specifies whether this ServiceAccount should be created. @@ -580,7 +582,7 @@ tls: # Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place. image: repository: cockroachlabs-helm-charts/cockroach-self-signer-cert - tag: "1.5" + tag: "1.6" pullPolicy: IfNotPresent credentials: {} registry: gcr.io diff --git a/pkg/generator/generate_cert.go b/pkg/generator/generate_cert.go index 5aeb6bee..6510b4ff 100644 --- a/pkg/generator/generate_cert.go +++ b/pkg/generator/generate_cert.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cockroachdb/helm-charts/pkg/kube" @@ -52,6 +53,7 @@ func init() { // GenerateCert is the structure containing all the certificate related info type GenerateCert struct { client client.Client + restConfig *rest.Config CertsDir string CaSecret string CAKey string @@ -93,9 +95,10 @@ func (c *certConfig) SetConfig(duration, expiryWindow string) error { return nil } -func NewGenerateCert(cl client.Client) GenerateCert { +func NewGenerateCert(config *rest.Config, cl client.Client) GenerateCert { return GenerateCert{ client: cl, + restConfig: config, CaCertConfig: &certConfig{}, NodeCertConfig: &certConfig{}, ClientCertConfig: &certConfig{}, @@ -370,7 +373,7 @@ func (rc *GenerateCert) generateNodeCert(ctx context.Context, nodeSecretName str return err } - if err = kube.RollingUpdate(ctx, rc.client, rc.DiscoveryServiceName, namespace, rc.ReadinessWait, rc.PodUpdateTimeout); err != nil { + if err = kube.SighupSignalToPods(ctx, rc.restConfig, rc.client, rc.DiscoveryServiceName, namespace); err != nil { return } return nil diff --git a/pkg/kube/helpers.go b/pkg/kube/helpers.go index fbf30e67..c0af58a6 100644 --- a/pkg/kube/helpers.go +++ b/pkg/kube/helpers.go @@ -17,6 +17,7 @@ limitations under the License. package kube import ( + "bytes" "context" "fmt" "strconv" @@ -29,6 +30,10 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -153,6 +158,72 @@ func RollingUpdate(ctx context.Context, cl client.Client, stsName, namespace str return nil } +// SighupSignalToPods sends SIGHUP signal to all the pods in the statefulset. +func SighupSignalToPods(ctx context.Context, config *rest.Config, cl client.Client, stsName, namespace string) error { + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + + var sts v1.StatefulSet + if err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: stsName}, &sts); err != nil { + return err + } + + containerName := sts.Spec.Template.Spec.Containers[0].Name + command := []string{"/bin/bash", "-c", "echo 'Send SIGHUP to cockroach'; kill -s 1 $(ls -l /proc/*/exe | grep cockroach | awk '{print $2}')"} + for i := int32(0); i < sts.Status.Replicas; i++ { + replicaName := stsName + "-" + strconv.Itoa(int(i)) + + stdout, stderr, err := execCommandInPod(clientset, config, namespace, replicaName, containerName, command) + if err != nil { + logrus.Errorf("Failed to send SIGHUP signal to pod [%s], error: %v, stdout: %s, stderr: %s", replicaName, err, stdout, stderr) + } + logrus.Info(stdout) + + // Sleeping for 1 second to allow the pod to receive the signal + time.Sleep(1 * time.Second) + } + + return nil +} + +// execCommandInPod executes the provided command in the given pod and returns the stdout and stderr. +func execCommandInPod(clientset *kubernetes.Clientset, config *rest.Config, namespace, podName, containerName string, command []string) (string, string, error) { + logrus.Infof("Running command %s in pod %s in container %s", command, podName, containerName) + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + Param("container", containerName). + VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return "", "", err + } + + var stdout, stderr bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return stdout.String(), stderr.String(), err + } + + return stdout.String(), stderr.String(), nil +} + func WaitForPodReady(ctx context.Context, cl client.Client, name, namespace string, podUpdateTimeout, podMaxPollingInterval time.Duration) error { f := func() error {