refactor: kwok installation code

- add functions to create and delete clusterrolebinding to create kwok resources
- refactor kwok install and uninstall fns
- delete manifests in the opposite order of install ]
- add cleaning up left-over kwok installation to future scope
Signed-off-by: vadasambar <[email protected]>

cluster-autoscaler/cloudprovider/kwok/README.md

@@ -39,6 +39,11 @@ As a user, here's what you should do:
 5. Refactor config loading and validation (uses a lot of `if`'s right now; a little difficult to maintain)
 6. Implement `Refresh` (unimplemented right now)
 7. Support customizing annotation used by kwok for managing nodes
+8. Clean-up previous installation of `kwok`
+   * Right now `kwok` is installed when CA starts and uninstalled when CA pod is terminated
+   * If by any chance there is a previous installation of `kwok` at version A and new CA pod starts and installs version B
+   * When the new CA pod terminates it will attempt to delete manifests in version B
+   * If version A had any extra manifests, those manifests would never be deleted
 ### Things about `kwok` provider that you should know
 * If you don't specify a `kwok` release you want to install, `kwok` provider by default creates a `ClusterRoleBinding` called `kwok-provider` which binds the cluster-autoscaler `ServiceAccount` to the built-in `cluster-admin` ClusterRole.  
    * Is there a way to make `kwok` provider use a `ClusterRole` with less permissions than `cluster-admin`?

cluster-autoscaler/cloudprovider/kwok/kwok_config.go

@@ -60,6 +60,16 @@ func getCurrentNamespace() string {
 	return currentNamespace
 }
 
+func getSA() string {
+	sa := os.Getenv("SERVICE_ACCOUNT")
+	if strings.TrimSpace(sa) == "" {
+		klog.Info("env variable 'SERVICE_ACCOUNT' is empty")
+		klog.Fatal("couldn't pod's service account name")
+	}
+
+	return sa
+}
+
 func getConfigMapName() string {
 	configMapName := os.Getenv("KWOK_CONFIG_MAP_NAME")
 	if strings.TrimSpace(configMapName) == "" {

cluster-autoscaler/cloudprovider/kwok/kwok_install.go

@@ -27,7 +27,11 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/google/go-github/v53/github"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/cri-api/pkg/errors"
 	kubectlcmd "k8s.io/kubectl/pkg/cmd"
 )
 
@@ -39,22 +43,77 @@ const (
 	repo            = "kwok"
 	apply    action = "apply"
 	delete   action = "delete"
+	crbName         = "kwok-provider"
 )
 
 type DeployMgr struct {
 	Release string
 }
 
+// InstallClusterRoleBinding creates a ClusterRoleBinding between
+// `cluster-admin` ClusterRole and cluster-autoscaler's ServiceAccount
+func (i *DeployMgr) InstallClusterRoleBinding(kubeClient kubernetes.Interface) error {
+	ns := getCurrentNamespace()
+	sa := getSA()
+	crb := rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{Name: "kwok-provider"},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     "cluster-admin",
+		},
+		Subjects: []rbacv1.Subject{
+			{Kind: rbacv1.ServiceAccountKind, Namespace: ns, Name: sa},
+		},
+	}
+	if _, err := kubeClient.RbacV1().ClusterRoleBindings().
+		Create(context.Background(), &crb, metav1.CreateOptions{}); err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteClusterRoleBinding deletes the ClusterRoleBinding between
+// `cluster-admin` ClusterRole and cluster-autoscaler's ServiceAccount
+func (i *DeployMgr) DeleteClusterRoleBinding(kubeClient kubernetes.Interface) error {
+	if err := kubeClient.RbacV1().ClusterRoleBindings().
+		Delete(context.Background(), crbName, metav1.DeleteOptions{}); !errors.IsNotFound(err) {
+		return err
+	}
+
+	return nil
+}
+
 // UninstallKwok uninstalls kwok >= v0.4.0
 // Based on https://kwok.sigs.k8s.io/docs/user/kwok-in-cluster/#deploy-kwok-in-a-cluster
 func (i *DeployMgr) UninstallKwok() error {
-	return i.kwokKubectl(delete, []string{"--ignore-not-found=true"})
+	stagesCRs := fmt.Sprintf("https://github.com/%s/releases/download/%s/stage-fast.yaml",
+		kwokRepo,
+		i.Release)
+	if err := i.kwokKubectl(stagesCRs, delete, []string{"--ignore-not-found=true"}); err != nil {
+		return err
+	}
+
+	deploymentAndCRDsURL := fmt.Sprintf("https://github.com/%s/releases/download/%s/kwok.yaml",
+		kwokRepo,
+		i.Release)
+	return i.kwokKubectl(deploymentAndCRDsURL, delete, []string{"--ignore-not-found=true"})
 }
 
 // InstallKwok installs kwok >= v0.4.0
 // Based on https://kwok.sigs.k8s.io/docs/user/kwok-in-cluster/#deploy-kwok-in-a-cluster
 func (i *DeployMgr) InstallKwok() error {
-	return i.kwokKubectl(apply, nil)
+	deploymentAndCRDsURL := fmt.Sprintf("https://github.com/%s/releases/download/%s/kwok.yaml",
+		kwokRepo,
+		i.Release)
+	if err := i.kwokKubectl(deploymentAndCRDsURL, apply, nil); err != nil {
+		return err
+	}
+
+	stagesCRs := fmt.Sprintf("https://github.com/%s/releases/download/%s/stage-fast.yaml",
+		kwokRepo,
+		i.Release)
+	return i.kwokKubectl(stagesCRs, apply, nil)
 }
 
 func GetLatestKwokRelease() (string, error) {
@@ -72,11 +131,11 @@ func GetLatestKwokRelease() (string, error) {
 	return rel.GetTagName(), nil
 }
 
-func (i *DeployMgr) kwokKubectl(action action, extraArgs []string) error {
-	deploymentAndCRDs := fmt.Sprintf("https://github.com/%s/releases/download/%s/kwok.yaml", kwokRepo, i.Release)
+func (i *DeployMgr) kwokKubectl(url string, action action, extraArgs []string) error {
+
 	// `kubectl apply` deployment and CRDs
 	cmd := []string{
-		"kubectl", string(action), "-f", deploymentAndCRDs,
+		"kubectl", string(action), "-f", url,
 	}
 	if len(extraArgs) > 0 {
 		cmd = append(cmd, extraArgs...)
@@ -87,20 +146,6 @@ func (i *DeployMgr) kwokKubectl(action action, extraArgs []string) error {
 	}
 	log.Infof("kubectl output: \n%s", string(o))
 
-	stagesCRs := fmt.Sprintf("https://github.com/%s/releases/download/%s/stage-fast.yaml", kwokRepo, i.Release)
-	// `kubectl apply` stages
-	cmd = []string{
-		"kubectl", string(action), "-f", stagesCRs,
-	}
-	if len(extraArgs) > 0 {
-		cmd = append(cmd, extraArgs...)
-	}
-	o, err = runKubectl(cmd...)
-	if err != nil {
-		return err
-	}
-	log.Infof("kubectl output: \n%s", string(o))
-
 	return nil
 }
 

cluster-autoscaler/cloudprovider/kwok/kwok_provider.go

@@ -164,6 +164,11 @@ func (kwok *KwokCloudProvider) Cleanup() error {
 		if err := kwok.deployMgr.UninstallKwok(); err != nil {
 			klog.Fatalf("couldn't uninstall kwok: %v", err)
 		}
+
+		if err := kwok.deployMgr.DeleteClusterRoleBinding(kwok.kubeClient); err != nil {
+			klog.Fatalf("couldn't delete clusterrolebinding '%v': %v", crbName, err)
+		}
+
 	}
 	return nil
 }
@@ -236,9 +241,8 @@ func buildKwokProvider(ko *kwokOptions) (*KwokCloudProvider, error) {
 			Release: kwokConfig.status.kwokRelease,
 		}
 
-		// cleanup older release if any
-		if err := d.UninstallKwok(); err != nil {
-			return nil, fmt.Errorf("couldn't uninstall kwok: %v", err)
+		if err := d.InstallClusterRoleBinding(ko.kubeClient); err != nil {
+			return nil, fmt.Errorf("couldn't create clusterrolebinding to install kwok: %v", err)
 		}
 
 		if err := d.InstallKwok(); err != nil {