Add controller to watch external-kubeconfig-secret

pkg/agent/addon_secret_controller.go

@@ -0,0 +1,93 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-logr/logr"
+	hyperv1beta1 "github.com/openshift/hypershift/api/v1beta1"
+	"github.com/stolostron/hypershift-addon-operator/pkg/util"
+	corev1 "k8s.io/api/core/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+type AddonSecretController struct {
+	spokeClient client.Client
+	log         logr.Logger
+}
+
+var AddonSecretPredicateFunctions = predicate.Funcs{
+	CreateFunc: func(e event.CreateEvent) bool {
+		return false
+	},
+	UpdateFunc: func(e event.UpdateEvent) bool {
+		return false
+	},
+	DeleteFunc: func(e event.DeleteEvent) bool {
+		return e.Object.GetName() == util.ExternalManagedKubeconfigSecretName && strings.HasPrefix(e.Object.GetNamespace(), util.ExternalManagedKubeconfigSecretNsPrefix)
+	},
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (c *AddonSecretController) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}).
+		WithEventFilter(AddonSecretPredicateFunctions).
+		Complete(c)
+}
+
+// Reconcile updates the Hypershift addon status based on the Deployment status.
+func (c *AddonSecretController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	c.log.Info(fmt.Sprintf("reconciling Secret %s", req))
+	defer c.log.Info(fmt.Sprintf("done reconcile Secret %s", req))
+
+	// Get hosted cluster by managed cluster name annotation
+	managedclusterAnnoValue := req.Namespace[11:]
+	c.log.Info("managedclusterAnnoValue=" + managedclusterAnnoValue)
+
+	hc := c.getHostedCluster(ctx, managedclusterAnnoValue)
+	if hc != nil {
+		// Hosted cluster exists but secret is deleted - trigger hostedcluster reconcile to recreate secret
+		if hc.Annotations == nil {
+			annotations := make(map[string]string)
+			hc.Annotations = annotations
+		}
+		hc.Annotations[util.HostedClusterRefreshAnnoKey] = strconv.FormatInt(time.Now().UnixMilli(), 10)
+
+		if err := c.spokeClient.Update(ctx, hc, &client.UpdateOptions{}); err != nil {
+			c.log.Error(err, fmt.Sprintf("failed to update refresh-time annotation in hc %v/%v", req.Namespace, req.Name))
+		}
+	} else {
+		c.log.Info(fmt.Sprintf("No hosted cluster with name or managedcluster-name annotation value = %v", managedclusterAnnoValue))
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (c *AddonSecretController) getHostedCluster(ctx context.Context, managedClusterAnnotationValue string) *hyperv1beta1.HostedCluster {
+	hcs := &hyperv1beta1.HostedClusterList{}
+	if err := c.spokeClient.List(ctx, hcs, &client.ListOptions{}); err != nil {
+		c.log.Error(err, "failed to list the hostedcluster")
+		return nil
+	}
+
+	for _, hc := range hcs.Items {
+		if hc.Annotations[util.ManagedClusterAnnoKey] == managedClusterAnnotationValue {
+			return &hc
+		}
+	}
+
+	for _, hc := range hcs.Items {
+		if hc.Name == managedClusterAnnotationValue {
+			return &hc
+		}
+	}
+
+	return nil
+}

pkg/agent/addon_secret_controller_test.go

@@ -0,0 +1,59 @@
+package agent
+
+import (
+	"context"
+	"testing"
+
+	"github.com/go-logr/zapr"
+	"github.com/stolostron/hypershift-addon-operator/pkg/util"
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestSecretReconcile(t *testing.T) {
+	ctx := context.Background()
+	client := initClient()
+	zapLog, _ := zap.NewDevelopment()
+
+	aCtrl := &AddonSecretController{
+		spokeClient: client,
+		log:         zapr.NewLogger(zapLog),
+	}
+
+	// Create hosted cluster
+	hcNN := types.NamespacedName{Name: "hc1", Namespace: "clusters"}
+	hc := getHostedCluster(hcNN)
+	err := aCtrl.spokeClient.Create(ctx, hc)
+	assert.Nil(t, err, "err nil when hosted cluster is created successfully")
+
+	secretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + "hc1"}
+
+	// Reconcile with annotation
+	_, err = aCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: secretNN})
+	assert.Nil(t, err, "err nil when reconcile was successfully")
+
+	err = aCtrl.spokeClient.Get(ctx, hcNN, hc)
+	assert.Nil(t, err, "is nil when the hosted cluster is found")
+	assert.NotEmpty(t, hc.Annotations[util.HostedClusterRefreshAnnoKey])
+
+	// Create 2nd hosted cluster with managedcluster-name annotation
+	hc2NN := types.NamespacedName{Name: "hc2", Namespace: "clusters"}
+	hc2 := getHostedCluster(hc2NN)
+	annotations := make(map[string]string)
+	annotations[util.ManagedClusterAnnoKey] = "hc1"
+	hc2.Annotations = annotations
+	err = aCtrl.spokeClient.Create(ctx, hc2)
+	assert.Nil(t, err, "err nil when hosted cluster is created successfully")
+
+	// Reconcile with annotation
+	_, err = aCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: secretNN})
+	assert.Nil(t, err, "err nil when reconcile was successfully")
+
+	// managedcluster-name annotation takes precedence, hc2 is updated by the controller this time
+	err = aCtrl.spokeClient.Get(ctx, hc2NN, hc2)
+	assert.Nil(t, err, "is nil when the hosted cluster is found")
+	assert.NotEmpty(t, hc2.Annotations[util.HostedClusterRefreshAnnoKey])
+}

pkg/agent/agent.go

@@ -229,6 +229,16 @@ func (o *AgentOptions) runControllerManager(ctx context.Context) error {
 		return fmt.Errorf("unable to create agent status controller: %s, err: %w", util.AddonStatusControllerName, err)
 	}
 
+	addonSecretController := &AddonSecretController{
+		spokeClient: spokeKubeClient,
+		log:         o.Log.WithName("addon-secret-controller"),
+	}
+
+	if err = addonSecretController.SetupWithManager(mgr); err != nil {
+		metrics.AddonAgentFailedToStartBool.Set(1)
+		return fmt.Errorf("unable to create agent secret controller: %s, err: %w", "addon-secret-controller", err)
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		metrics.AddonAgentFailedToStartBool.Set(1)
 		return fmt.Errorf("unable to set up health check, err: %w", err)
@@ -288,16 +298,16 @@ func (c *agentController) scaffoldHostedclusterSecrets(hcKey types.NamespacedNam
 func (c *agentController) generateExtManagedKubeconfigSecret(ctx context.Context, secretData map[string][]byte, hc hyperv1beta1.HostedCluster) error {
 	// 1. Get hosted cluster's admin kubeconfig secret
 	secret := &corev1.Secret{}
-	secret.SetName("external-managed-kubeconfig")
+	secret.SetName(util.ExternalManagedKubeconfigSecretName)
 	managedClusterAnnoValue, ok := hc.GetAnnotations()[util.ManagedClusterAnnoKey]
 	if !ok || len(managedClusterAnnoValue) == 0 {
 		managedClusterAnnoValue = hc.Name
 	}
-	secret.SetNamespace("klusterlet-" + managedClusterAnnoValue)
+	secret.SetNamespace(util.ExternalManagedKubeconfigSecretNsPrefix + managedClusterAnnoValue)
 	kubeconfigData := secretData["kubeconfig"]
 
 	klusterletNamespace := &corev1.Namespace{}
-	klusterletNamespaceNsn := types.NamespacedName{Name: "klusterlet-" + managedClusterAnnoValue}
+	klusterletNamespaceNsn := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretNsPrefix + managedClusterAnnoValue}
 
 	err := c.spokeClient.Get(ctx, klusterletNamespaceNsn, klusterletNamespace)
 	if err != nil {

pkg/agent/agent_test.go

@@ -106,7 +106,7 @@ kind: Config`)
 	// Create klusterlet namespace
 	klusterletNamespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "klusterlet-" + hc.Name,
+			Name: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name,
 		},
 	}
 	err = aCtrl.hubClient.Create(ctx, klusterletNamespace)
@@ -127,7 +127,7 @@ kind: Config`)
 	err = aCtrl.hubClient.Get(ctx, pwdSecretNN, secret)
 	assert.Nil(t, err, "is nil when the kubeadmin password secret is found")
 
-	kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Name}
+	kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name}
 	err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret)
 	assert.Nil(t, err, "is nil when external-managed-kubeconfig secret is found")
 
@@ -227,7 +227,7 @@ kind: Config`)
 
 	// external-managed-kubeconfig could not be created because there is no klusterlet namespace
 	secret := &corev1.Secret{}
-	kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Name}
+	kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name}
 	err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret)
 	assert.NotNil(t, err, "external-managed-kubeconfig secret not found")
 	assert.Equal(t, true, res.Requeue)
@@ -314,7 +314,7 @@ kind: Config`)
 	// Create klusterlet namespace
 	klusterletNamespace := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: "klusterlet-" + hc.Spec.InfraID,
+			Name: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Spec.InfraID,
 		},
 	}
 	err = aCtrl.hubClient.Create(ctx, klusterletNamespace)
@@ -335,7 +335,7 @@ kind: Config`)
 	err = aCtrl.hubClient.Get(ctx, pwdSecretNN, secret)
 	assert.Nil(t, err, "is nil when the kubeadmin password secret is found")
 
-	kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Spec.InfraID}
+	kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Spec.InfraID}
 	err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret)
 	assert.Nil(t, err, "is nil when external-managed-kubeconfig secret is found")
 

pkg/util/constant.go

@@ -68,6 +68,13 @@ const (
 	HypershiftEnvVarImageClusterApi           = "IMAGE_CLUSTER_API"
 	HypershiftEnvVarImageAgentCapiProvider    = "IMAGE_AGENT_CAPI_PROVIDER"
 
+	// external-managed-kubeconfig secret
+	ExternalManagedKubeconfigSecretName     = "external-managed-kubeconfig"
+	ExternalManagedKubeconfigSecretNsPrefix = "klusterlet-"
+
+	// Hosted cluster refresh-time annotation for triggering reconcile
+	HostedClusterRefreshAnnoKey = "open-cluster-management.io/refresh"
+
 	// AddOnPlacementScore resource name
 	HostedClusterScoresResourceName = "hosted-clusters-score"
 	// AddOnPlacementScore score name