diff --git a/pkg/agent/addon_secret_controller.go b/pkg/agent/addon_secret_controller.go new file mode 100644 index 00000000..b32714a0 --- /dev/null +++ b/pkg/agent/addon_secret_controller.go @@ -0,0 +1,93 @@ +package agent + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/go-logr/logr" + hyperv1beta1 "github.com/openshift/hypershift/api/v1beta1" + "github.com/stolostron/hypershift-addon-operator/pkg/util" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +type AddonSecretController struct { + spokeClient client.Client + log logr.Logger +} + +var AddonSecretPredicateFunctions = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return e.Object.GetName() == util.ExternalManagedKubeconfigSecretName && strings.HasPrefix(e.Object.GetNamespace(), util.ExternalManagedKubeconfigSecretNsPrefix) + }, +} + +// SetupWithManager sets up the controller with the Manager. +func (c *AddonSecretController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Secret{}). + WithEventFilter(AddonSecretPredicateFunctions). + Complete(c) +} + +// Reconcile updates the Hypershift addon status based on the Deployment status. +func (c *AddonSecretController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + c.log.Info(fmt.Sprintf("reconciling Secret %s", req)) + defer c.log.Info(fmt.Sprintf("done reconcile Secret %s", req)) + + // Get hosted cluster by managed cluster name annotation + managedclusterAnnoValue := req.Namespace[11:] + c.log.Info("managedclusterAnnoValue=" + managedclusterAnnoValue) + + hc := c.getHostedCluster(ctx, managedclusterAnnoValue) + if hc != nil { + // Hosted cluster exists but secret is deleted - trigger hostedcluster reconcile to recreate secret + if hc.Annotations == nil { + annotations := make(map[string]string) + hc.Annotations = annotations + } + hc.Annotations[util.HostedClusterRefreshAnnoKey] = strconv.FormatInt(time.Now().UnixMilli(), 10) + + if err := c.spokeClient.Update(ctx, hc, &client.UpdateOptions{}); err != nil { + c.log.Error(err, fmt.Sprintf("failed to update refresh-time annotation in hc %v/%v", req.Namespace, req.Name)) + } + } else { + c.log.Info(fmt.Sprintf("No hosted cluster with name or managedcluster-name annotation value = %v", managedclusterAnnoValue)) + } + + return ctrl.Result{}, nil +} + +func (c *AddonSecretController) getHostedCluster(ctx context.Context, managedClusterAnnotationValue string) *hyperv1beta1.HostedCluster { + hcs := &hyperv1beta1.HostedClusterList{} + if err := c.spokeClient.List(ctx, hcs, &client.ListOptions{}); err != nil { + c.log.Error(err, "failed to list the hostedcluster") + return nil + } + + for _, hc := range hcs.Items { + if hc.Annotations[util.ManagedClusterAnnoKey] == managedClusterAnnotationValue { + return &hc + } + } + + for _, hc := range hcs.Items { + if hc.Name == managedClusterAnnotationValue { + return &hc + } + } + + return nil +} diff --git a/pkg/agent/addon_secret_controller_test.go b/pkg/agent/addon_secret_controller_test.go new file mode 100644 index 00000000..68cb515a --- /dev/null +++ b/pkg/agent/addon_secret_controller_test.go @@ -0,0 +1,59 @@ +package agent + +import ( + "context" + "testing" + + "github.com/go-logr/zapr" + "github.com/stolostron/hypershift-addon-operator/pkg/util" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + ctrl "sigs.k8s.io/controller-runtime" + + "k8s.io/apimachinery/pkg/types" +) + +func TestSecretReconcile(t *testing.T) { + ctx := context.Background() + client := initClient() + zapLog, _ := zap.NewDevelopment() + + aCtrl := &AddonSecretController{ + spokeClient: client, + log: zapr.NewLogger(zapLog), + } + + // Create hosted cluster + hcNN := types.NamespacedName{Name: "hc1", Namespace: "clusters"} + hc := getHostedCluster(hcNN) + err := aCtrl.spokeClient.Create(ctx, hc) + assert.Nil(t, err, "err nil when hosted cluster is created successfully") + + secretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + "hc1"} + + // Reconcile with annotation + _, err = aCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: secretNN}) + assert.Nil(t, err, "err nil when reconcile was successfully") + + err = aCtrl.spokeClient.Get(ctx, hcNN, hc) + assert.Nil(t, err, "is nil when the hosted cluster is found") + assert.NotEmpty(t, hc.Annotations[util.HostedClusterRefreshAnnoKey]) + + // Create 2nd hosted cluster with managedcluster-name annotation + hc2NN := types.NamespacedName{Name: "hc2", Namespace: "clusters"} + hc2 := getHostedCluster(hc2NN) + annotations := make(map[string]string) + annotations[util.ManagedClusterAnnoKey] = "hc1" + hc2.Annotations = annotations + err = aCtrl.spokeClient.Create(ctx, hc2) + assert.Nil(t, err, "err nil when hosted cluster is created successfully") + + // Reconcile with annotation + _, err = aCtrl.Reconcile(ctx, ctrl.Request{NamespacedName: secretNN}) + assert.Nil(t, err, "err nil when reconcile was successfully") + + // managedcluster-name annotation takes precedence, hc2 is updated by the controller this time + err = aCtrl.spokeClient.Get(ctx, hc2NN, hc2) + assert.Nil(t, err, "is nil when the hosted cluster is found") + assert.NotEmpty(t, hc2.Annotations[util.HostedClusterRefreshAnnoKey]) +} diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 1e276d3b..a2f5bcc2 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -229,6 +229,16 @@ func (o *AgentOptions) runControllerManager(ctx context.Context) error { return fmt.Errorf("unable to create agent status controller: %s, err: %w", util.AddonStatusControllerName, err) } + addonSecretController := &AddonSecretController{ + spokeClient: spokeKubeClient, + log: o.Log.WithName("addon-secret-controller"), + } + + if err = addonSecretController.SetupWithManager(mgr); err != nil { + metrics.AddonAgentFailedToStartBool.Set(1) + return fmt.Errorf("unable to create agent secret controller: %s, err: %w", "addon-secret-controller", err) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { metrics.AddonAgentFailedToStartBool.Set(1) return fmt.Errorf("unable to set up health check, err: %w", err) @@ -288,16 +298,16 @@ func (c *agentController) scaffoldHostedclusterSecrets(hcKey types.NamespacedNam func (c *agentController) generateExtManagedKubeconfigSecret(ctx context.Context, secretData map[string][]byte, hc hyperv1beta1.HostedCluster) error { // 1. Get hosted cluster's admin kubeconfig secret secret := &corev1.Secret{} - secret.SetName("external-managed-kubeconfig") + secret.SetName(util.ExternalManagedKubeconfigSecretName) managedClusterAnnoValue, ok := hc.GetAnnotations()[util.ManagedClusterAnnoKey] if !ok || len(managedClusterAnnoValue) == 0 { managedClusterAnnoValue = hc.Name } - secret.SetNamespace("klusterlet-" + managedClusterAnnoValue) + secret.SetNamespace(util.ExternalManagedKubeconfigSecretNsPrefix + managedClusterAnnoValue) kubeconfigData := secretData["kubeconfig"] klusterletNamespace := &corev1.Namespace{} - klusterletNamespaceNsn := types.NamespacedName{Name: "klusterlet-" + managedClusterAnnoValue} + klusterletNamespaceNsn := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretNsPrefix + managedClusterAnnoValue} err := c.spokeClient.Get(ctx, klusterletNamespaceNsn, klusterletNamespace) if err != nil { diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 7bd969c8..6ce9d050 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -106,7 +106,7 @@ kind: Config`) // Create klusterlet namespace klusterletNamespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "klusterlet-" + hc.Name, + Name: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name, }, } err = aCtrl.hubClient.Create(ctx, klusterletNamespace) @@ -127,7 +127,7 @@ kind: Config`) err = aCtrl.hubClient.Get(ctx, pwdSecretNN, secret) assert.Nil(t, err, "is nil when the kubeadmin password secret is found") - kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Name} + kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name} err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret) assert.Nil(t, err, "is nil when external-managed-kubeconfig secret is found") @@ -227,7 +227,7 @@ kind: Config`) // external-managed-kubeconfig could not be created because there is no klusterlet namespace secret := &corev1.Secret{} - kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Name} + kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Name} err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret) assert.NotNil(t, err, "external-managed-kubeconfig secret not found") assert.Equal(t, true, res.Requeue) @@ -314,7 +314,7 @@ kind: Config`) // Create klusterlet namespace klusterletNamespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "klusterlet-" + hc.Spec.InfraID, + Name: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Spec.InfraID, }, } err = aCtrl.hubClient.Create(ctx, klusterletNamespace) @@ -335,7 +335,7 @@ kind: Config`) err = aCtrl.hubClient.Get(ctx, pwdSecretNN, secret) assert.Nil(t, err, "is nil when the kubeadmin password secret is found") - kcExtSecretNN := types.NamespacedName{Name: "external-managed-kubeconfig", Namespace: "klusterlet-" + hc.Spec.InfraID} + kcExtSecretNN := types.NamespacedName{Name: util.ExternalManagedKubeconfigSecretName, Namespace: util.ExternalManagedKubeconfigSecretNsPrefix + hc.Spec.InfraID} err = aCtrl.hubClient.Get(ctx, kcExtSecretNN, secret) assert.Nil(t, err, "is nil when external-managed-kubeconfig secret is found") diff --git a/pkg/util/constant.go b/pkg/util/constant.go index 56624dfe..50751a12 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -68,6 +68,13 @@ const ( HypershiftEnvVarImageClusterApi = "IMAGE_CLUSTER_API" HypershiftEnvVarImageAgentCapiProvider = "IMAGE_AGENT_CAPI_PROVIDER" + // external-managed-kubeconfig secret + ExternalManagedKubeconfigSecretName = "external-managed-kubeconfig" + ExternalManagedKubeconfigSecretNsPrefix = "klusterlet-" + + // Hosted cluster refresh-time annotation for triggering reconcile + HostedClusterRefreshAnnoKey = "open-cluster-management.io/refresh" + // AddOnPlacementScore resource name HostedClusterScoresResourceName = "hosted-clusters-score" // AddOnPlacementScore score name