feat: add support to install and uninstall kwok

- add option to disable installation
- add option to manually specify kwok release tag
- add future scope in readme
Signed-off-by: vadasambar <[email protected]>

cluster-autoscaler/cloudprovider/kwok/README.md

@@ -31,6 +31,8 @@ As a user, here's what you should do:
 ### What is supported?
 ### What is not supported yet?
 ### Future plans
+1. Support waiting for `kwok` controller's `Deployment` to come up. 
+2. Support merging of static and dynamic node templates
 ### I want a feature
 * Create a new issue and mention `@vadasambar`. SLA: reply within a week until end of 2023 (post which I will think about SLO again and might come up with a new one). 
 

cluster-autoscaler/cloudprovider/kwok/config.go

@@ -158,5 +158,17 @@ func loadConfigFile() *KwokProviderConfig {
 			kwokConfig.ReadNodesFrom)
 	}
 
+	if kwokConfig.install != false {
+		if kwokConfig.Kwok != nil && kwokConfig.Kwok.Release != "" {
+			kwokConfig.status.kwokRelease = kwokConfig.Kwok.Release
+		} else {
+			rel, err := GetLatestKwokRelease()
+			if err != nil {
+				klog.Fatalf("could not get latest kwok release: %v", err)
+			}
+			kwokConfig.status.kwokRelease = rel
+		}
+	}
+
 	return &kwokConfig
 }

cluster-autoscaler/cloudprovider/kwok/install.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kwok
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/google/go-github/v53/github"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	kubectlcmd "k8s.io/kubectl/pkg/cmd"
+)
+
+type action string
+
+const (
+	owner         = "kubernetes-sigs"
+	repo          = "kwok"
+	apply  action = "apply"
+	delete action = "delete"
+)
+
+// UninstallKwok uninstalls kwok from the cluster
+func UninstallKwok(release string) error {
+	return kwokKubectl(release, delete)
+}
+
+// InstallKwok installs kwok in the cluster
+func InstallKwok(release string) error {
+	return kwokKubectl(release, apply)
+}
+
+func GetLatestKwokRelease() (string, error) {
+	// find latest release of `kwok`
+	client := github.NewClient(nil)
+	rel, resp, err := client.Repositories.GetLatestRelease(context.Background(), owner, repo)
+	if err != nil {
+		return "", err
+	}
+
+	if resp.Response.StatusCode != http.StatusOK {
+		log.Fatal("expected 200 response but received", resp.Response.StatusCode)
+	}
+
+	return rel.GetTagName(), nil
+}
+
+// kwokKubectl builds kustomize for kwok and runs `kubectl` on it
+func kwokKubectl(release string, action action) error {
+
+	if release == "" {
+		return fmt.Errorf("release is empty: '%s'", release)
+	}
+
+	// create tmp working directory for kwok
+	tmpDir, err := ioutil.TempDir("", "install-kwok")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// create kustomization file
+	kustomizeTemplate := `
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: registry.k8s.io/kwok/kwok
+  newTag: "{{.latestRelease}}"
+resources:
+- "https://github.com/{{.repo}}/kustomize/kwok?ref={{.latestRelease}}"
+`
+	tmpl, err := template.New("kwok-kustomize").Parse(kustomizeTemplate)
+	if err != nil {
+		return err
+	}
+
+	b := &bytes.Buffer{}
+	err = tmpl.Execute(b, map[string]interface{}{
+		"latestRelease": release,
+		"repo":          owner + "/" + repo,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	log.Infof("latest kwok release is %s", release)
+
+	kustomizationFilePath := filepath.Join(tmpDir, "kustomization.yaml")
+	if err := os.WriteFile(kustomizationFilePath,
+		b.Bytes(),
+		os.FileMode(0600)); err != nil {
+		log.Fatal(err)
+	}
+
+	// `kubectl apply` using kustomize
+	o, err := runKubectl("kubectl", string(action), "-k", tmpDir)
+	if err != nil {
+		return err
+	}
+	log.Infof("kubectl output: \n%s", string(o))
+
+	return nil
+}
+
+// based on argo-workflows executor
+// code ref: https://github.com/argoproj/argo-workflows/blob/545bf3803d6f0c59a4c0a93db23d18001462bf3c/workflow/executor/resource.go#L366
+func runKubectl(args ...string) ([]byte, error) {
+	log.Info(strings.Join(args, " "))
+	os.Args = args
+	var buf bytes.Buffer
+	if err := kubectlcmd.NewKubectlCommand(kubectlcmd.KubectlOptions{
+		Arguments: args,
+		ConfigFlags: genericclioptions.NewConfigFlags(true).
+			WithDeprecatedPasswordFlag().
+			WithDiscoveryBurst(300).
+			WithDiscoveryQPS(50.0),
+		IOStreams: genericclioptions.IOStreams{Out: &buf, ErrOut: os.Stderr},
+	}).Execute(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}

cluster-autoscaler/cloudprovider/kwok/provider.go

@@ -147,6 +147,12 @@ func (kwok *KwokCloudProvider) Cleanup() error {
 			klog.Errorf("error cleaning up node '%v'", node.GetName())
 		}
 	}
+
+	if kwok.config.status.kwokRelease != "" {
+		if err := UninstallKwok(kwok.config.status.kwokRelease); err != nil {
+			klog.Fatalf("couldn't uninstall kwok: %v", err)
+		}
+	}
 	return nil
 }
 
@@ -179,6 +185,12 @@ func BuildKwokCloudProvider(opts config.AutoscalingOptions, do cloudprovider.Nod
 		}
 	}
 
+	if kwok.status.kwokRelease != "" {
+		if err := InstallKwok(kwokConfig.status.kwokRelease); err != nil {
+			klog.Fatalf("couldn't install kwok: %v", err)
+		}
+	}
+
 	return &KwokCloudProvider{
 		nodeGroups:      nodegroups,
 		lister:          lister,

cluster-autoscaler/cloudprovider/kwok/samples/dynamic_nodegroups_config.yaml

@@ -26,4 +26,11 @@ cluster:
       #   availableGPUTypes: 
       #     "nvidia-tesla-k80": {}
 		  #     "nvidia-tesla-p100": {}
-file: {}
+file: {}
+kwok: {} # default: fetch latest release of kwok from github and install it
+# # you can also manually specify which kwok release you want to install
+# kwok:
+#   release: v0.3.0
+# # you can also disable installing kwok in CA code (and install your own kwok release)
+# kwok:
+#   install: false (true if not specified)

cluster-autoscaler/cloudprovider/kwok/samples/static_nodegroups_config.yaml

@@ -26,4 +26,11 @@ file:
       #   gpuLabelKey: "k8s.amazonaws.com/accelerator"
       #   availableGPUTypes: 
       #     "nvidia-tesla-k80": {}
-		  #     "nvidia-tesla-p100": {}
+		  #     "nvidia-tesla-p100": {}
+kwok: {} # default: fetch latest release of kwok from github and install it
+# # you can also manually specify which kwok release you want to install
+# kwok:
+#   release: v0.3.0
+# # you can also disable installing kwok in CA code (and install your own kwok release)
+# kwok:
+#   install: false (true if not specified)

cluster-autoscaler/cloudprovider/kwok/types.go

@@ -77,10 +77,16 @@ type FileConfig struct {
 	Path       string            `json:"path" yaml:"path"`
 }
 
+type KwokConfig struct {
+	Release string `json:"release" yaml:"release"`
+	Install bool   `json:"install" yaml:"install"`
+}
+
 type KwokProviderConfig struct {
 	ReadNodesFrom string         `json:"readNodesFrom" yaml:"readNodesFrom"`
 	Cluster       *ClusterConfig `json:"cluster" yaml:"cluster"`
 	File          *FileConfig    `json:"file" yaml:"file"`
+	Kwok          *KwokConfig    `json:"file" yaml:"file"`
 	status        *GroupingConfig
 }
 
@@ -89,4 +95,5 @@ type GroupingConfig struct {
 	key               string              // annotation or label key
 	gpuLabel          string              // gpu label key
 	availableGPUTypes map[string]struct{} // available gpu types
+	kwokRelease       string              // kwok release to use
 }