fixed re trigger job issue when public key deleted

Signed-off-by: rchikatw <[email protected]>

controllers/storagecluster/provider_server.go

@@ -454,6 +454,8 @@ func getOnboardingJobObject(instance *ocsv1.StorageCluster) *batchv1.Job {
 			Namespace: instance.Namespace,
 		},
 		Spec: batchv1.JobSpec{
+			// Eligible to delete automatically when job finishes
+			TTLSecondsAfterFinished: util.ToPointer[int32](0),
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
 					RestartPolicy:      corev1.RestartPolicyOnFailure,

controllers/storagecluster/storagecluster_controller.go

@@ -166,13 +166,17 @@ func (r *StorageClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		},
 	}
 
-	onboardingSecretPredicates := builder.WithPredicates(
-		predicate.NewPredicateFuncs(
-			func(client client.Object) bool {
-				return client.GetName() == onboardingTicketPublicKeySecretName
-			},
-		),
-	)
+	secretPredicates := builder.WithPredicates(predicate.Funcs{
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return true
+		},
+		CreateFunc: func(e event.CreateEvent) bool {
+			return e.Object.GetName() != onboardingTicketPublicKeySecretName
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			return e.ObjectNew.GetName() != onboardingTicketPublicKeySecretName
+		},
+	})
 
 	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&ocsv1.StorageCluster{}, builder.WithPredicates(scPredicate)).
@@ -181,6 +185,7 @@ func (r *StorageClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
 		Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
 		Owns(&corev1.ConfigMap{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		Owns(&corev1.Secret{}, secretPredicates).
 		Watches(&ocsv1.StorageProfile{}, enqueueStorageClusterRequest).
 		Watches(
 			&extv1.CustomResourceDefinition{
@@ -190,7 +195,6 @@ func (r *StorageClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			},
 			enqueueStorageClusterRequest,
 		).
-		Watches(&corev1.Secret{}, enqueueStorageClusterRequest, onboardingSecretPredicates).
 		Watches(&ocsv1alpha1.StorageConsumer{}, enqueueStorageClusterRequest, builder.WithPredicates(ocsClientOperatorVersionPredicate))
 	if os.Getenv("SKIP_NOOBAA_CRD_WATCH") != "true" {
 		builder.Owns(&nbv1.NooBaa{})

controllers/util/util.go

@@ -22,3 +22,7 @@ func DetectDuplicateInStringSlice(slice []string) bool {
 	}
 	return false
 }
+
+func ToPointer[T any](value T) *T {
+	return &value
+}

deploy/ocs-operator/manifests/onboarding-secret-generator-role.yaml

@@ -11,3 +11,11 @@ rules:
   - get
   - list
   - create
+  - delete
+- apiGroups:
+  - ocs.openshift.io
+  resources:
+  - storageclusters
+  verbs:
+    - get
+    - list

onboarding/main.go

@@ -4,9 +4,12 @@ import (
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
+	"encoding/json"
 	"encoding/pem"
+	"fmt"
 	"os"
 
+	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
 	"golang.org/x/net/context"
 	corev1 "k8s.io/api/core/v1"
@@ -18,80 +21,131 @@ import (
 )
 
 const (
-	onboardingTicketPublicKeySecretName = "onboarding-ticket-key" //Name of existing public key which is used ocs-operator
-	onboardingPrivateKeySecretName      = "onboarding-private-key"
-	serviceAccountName                  = "onboarding-secret-generator"
+	// Name of existing public key which is used ocs-operator
+	onboardingValidationPublicKeySecretName  = "onboarding-ticket-key"
+	onboardingValidationPrivateKeySecretName = "onboarding-private-key"
 )
 
 func main() {
 	clientset, err := newClient()
 	if err != nil {
-		klog.Error(err, "failed to create controller-runtime client")
-		return
+		klog.Errorf("failed to create clientset: %v", err)
+		os.Exit(1)
 	}
 
 	operatorNamespace, err := util.GetOperatorNamespace()
 	if err != nil {
-		klog.Error(err, "unable to get operator namespace")
+		klog.Errorf("unable to get operator namespace: %v", err)
+		os.Exit(1)
+	}
+
+	// Generate RSA key.
+	privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		klog.Errorf("unable to generate private: %v", err)
+		os.Exit(1)
+	}
+
+	publicKey := &privateKey.PublicKey
+	// Export the keys to pem string
+	privatePem := convertRsaPrivateKeyAsPemStr(privateKey)
+	publicPem, err := convertRsaPublicKeyAsPemStr(publicKey)
+
+	if err != nil {
+		klog.Errorf("failed to convert public key to pem str: %v", err)
+		os.Exit(1)
+	}
+
+	sc := ocsv1.StorageClusterList{}
+	instance, err := clientset.RESTClient().Get().AbsPath(
+		fmt.Sprintf("/apis/%s/%s/namespaces/%s/%s",
+			"ocs.openshift.io",
+			"v1",
+			operatorNamespace,
+			"storageclusters",
+		)).DoRaw(context.TODO())
+
+	if err != nil {
+		klog.Errorf("failed to get storage cluster: %v", err.Error())
+		os.Exit(1)
+	}
+
+	if err := json.Unmarshal(instance, &sc); err != nil {
+		klog.Errorf("failed to parse storage cluster response: %v", err)
+		os.Exit(1)
+	}
+
+	if len(sc.Items) == 0 {
+		klog.Error("no storage cluster found")
+		os.Exit(1)
+	}
+
+	ownerRef := metav1.OwnerReference{
+		UID:        sc.Items[0].UID,
+		APIVersion: sc.Items[0].APIVersion,
+		Kind:       sc.Items[0].Kind,
+		Name:       sc.Items[0].Name,
+	}
+
+	// In situations where there is a risk of one secret being updated and potentially
+	// failing to update another, it is recommended not to rely solely on clientset update mechanisms.
+	// Instead, a safer approach is to delete both secrets and then recreate them simultaneously
+	// to ensure consistency and accuracy of all secrets. By this way it will be easier to diagnose the
+	// issues if one or two secrets do not exist instead of trying to understand if they match
+	err = clientset.CoreV1().
+		Secrets(operatorNamespace).
+		Delete(context.Background(), onboardingValidationPrivateKeySecretName, metav1.DeleteOptions{})
+	if err != nil && !kerrors.IsNotFound(err) {
+		klog.Errorf("failed to delete private secret: %v", err)
+		os.Exit(1)
+	}
+
+	// Delete public key secret
+	err = clientset.CoreV1().
+		Secrets(operatorNamespace).
+		Delete(context.Background(), onboardingValidationPublicKeySecretName, metav1.DeleteOptions{})
+	if err != nil && !kerrors.IsNotFound(err) {
+		klog.Errorf("failed to delete public secret: %v", err)
+		os.Exit(1)
+	}
+
+	privateSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            onboardingValidationPrivateKeySecretName,
+			Namespace:       operatorNamespace,
+			OwnerReferences: []metav1.OwnerReference{ownerRef},
+		},
+		StringData: map[string]string{
+			"key": privatePem,
+		},
+	}
+
+	_, err = clientset.CoreV1().
+		Secrets(operatorNamespace).
+		Create(context.Background(), privateSecret, metav1.CreateOptions{})
+
+	if err != nil {
+		klog.Errorf("failed to create private secret: %v", err)
 		os.Exit(1)
 	}
 
-	// 1. Check public key secret exist or not
-	_, err = clientset.CoreV1().Secrets(operatorNamespace).Get(context.TODO(), onboardingTicketPublicKeySecretName, metav1.GetOptions{})
-
-	if err != nil && kerrors.IsNotFound(err) {
-		// Generate RSA key.
-		var err error
-		privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
-		if err != nil {
-			klog.Error(err, "unable to generate private")
-			os.Exit(1)
-		}
-
-		publicKey := &privateKey.PublicKey
-		// Export the keys to pem string
-		privatePem := convertRsaPrivateKeyAsPemStr(privateKey)
-		publicPem, err := convertRsaPublicKeyAsPemStr(publicKey)
-
-		if err != nil {
-			klog.Error(err, "failed to convert public key to pem str")
-			os.Exit(1)
-		}
-
-		privateSecret := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:        onboardingPrivateKeySecretName,
-				Namespace:   operatorNamespace,
-				Annotations: map[string]string{"kubernetes.io/service-account.name": serviceAccountName},
-			},
-			Type: "kubernetes.io/service-account-token",
-			StringData: map[string]string{
-				"key": privatePem,
-			},
-		}
-
-		_, err = clientset.CoreV1().Secrets(operatorNamespace).Create(context.Background(), privateSecret, metav1.CreateOptions{})
-
-		if err != nil {
-			klog.Error(err, "Failed to create private secret.")
-			os.Exit(1)
-		}
-		publicSecret := &corev1.Secret{
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      onboardingTicketPublicKeySecretName,
-				Namespace: operatorNamespace,
-			},
-			StringData: map[string]string{
-				"key": publicPem,
-			},
-		}
-
-		_, err = clientset.CoreV1().Secrets(operatorNamespace).Create(context.Background(), publicSecret, metav1.CreateOptions{})
-		if err != nil {
-			klog.Error(err, "Failed to create public secret.")
-			os.Exit(1)
-		}
+	publicSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            onboardingValidationPublicKeySecretName,
+			Namespace:       operatorNamespace,
+			OwnerReferences: []metav1.OwnerReference{ownerRef},
+		},
+		StringData: map[string]string{
+			"key": publicPem,
+		},
+	}
 
+	_, err = clientset.CoreV1().
+		Secrets(operatorNamespace).
+		Create(context.Background(), publicSecret, metav1.CreateOptions{})
+	if err != nil {
+		klog.Errorf("failed to create public secret: %v", err)
+		os.Exit(1)
 	}
 
 }
@@ -101,7 +155,8 @@ func newClient() (*kubernetes.Clientset, error) {
 	clientset, err := kubernetes.NewForConfig(config)
 
 	if err != nil {
-		klog.Error(err, "failed to get clientset")
+		klog.Errorf("failed to create clientset: %v", err)
+		os.Exit(1)
 	}
 
 	return clientset, nil

rbac/onboarding-secret-generator-role.yaml

@@ -11,3 +11,11 @@ rules:
   - get
   - list
   - create
+  - delete
+- apiGroups:
+  - ocs.openshift.io
+  resources:
+  - storageclusters
+  verbs:
+    - get
+    - list