feat: wip kwok provider config

- add samples for static and dynamic template nodes
Signed-off-by: vadasambar <[email protected]>

cluster-autoscaler/cloudprovider/kwok/kwok.go

@@ -1,6 +1,3 @@
-//go:build linux
-// +build linux
-
 /*
 Copyright 2023 The Kubernetes Authors.
 
@@ -79,6 +76,7 @@ const (
 // KwokCloudProvider implements CloudProvider interface for kwok
 type KwokCloudProvider struct {
 	nodeGroups      []*NodeGroup
+	config          *KwokProviderConfig
 	resourceLimiter *cloudprovider.ResourceLimiter
 	// TODO(vadasambar): look into abstracting kubeClient
 	// and lister into a single client
@@ -88,6 +86,26 @@ type KwokCloudProvider struct {
 	lister kube_util.NodeLister
 }
 
+type NodegroupsConfig struct {
+	FromNodeLabelKey        string `json:"fromNodeLabelKey" yaml:"fromNodeLabelKey"`
+	FromNodeLabelAnnotation string `json:"fromNodeLabelAnnotation" yaml:"fromNodeLabelAnnotation"`
+}
+
+type ClusterConfig struct {
+	Nodegroups *NodegroupsConfig `json:"nodegroups" yaml:"nodegroups"`
+}
+
+type FileConfig struct {
+	Nodegroups *NodegroupsConfig `json:"nodegroups" yaml:"nodegroups"`
+	Path       string            `json:"path" yaml:"path"`
+}
+
+type KwokProviderConfig struct {
+	ReadNodesFrom string         `json:"readNodesFrom" yaml:"readNodesFrom"`
+	Cluster       *ClusterConfig `json:"cluster" yaml:"cluster"`
+	File          *FileConfig    `json:"file" yaml:"file"`
+}
+
 // Name returns name of the cloud provider.
 func (kwok *KwokCloudProvider) Name() string {
 	return ProviderName
@@ -404,9 +422,52 @@ func BuildKwokCloudProvider(opts config.AutoscalingOptions, do cloudprovider.Nod
 		return n.GetAnnotations()[KwokManagedAnnotation] == "fake"
 	}, stop)
 
-	templatesPath := os.Getenv("KWOK_TEMPLATES_PATH")
+	configPath := os.Getenv("KWOK_CONFIG_PATH")
+	if strings.TrimSpace(configPath) == "" {
+		klog.Fatalf("please specify path to kwok provider config using 'KWOK_CONFIG_PATH' (currently empty)")
+	}
+
+	c, err := os.ReadFile(configPath)
+	if err != nil {
+		klog.Fatalf("failed to load kwok config from path '%s': %v", configPath, err)
+	}
+
+	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(c), 4096)
+	kwokConfig := KwokProviderConfig{}
+	if err := decoder.Decode(&kwokConfig); err != nil {
+		klog.Fatalf("failed to decode kwok config: %v", err)
+	}
+
+	switch kwokConfig.ReadNodesFrom {
+	case "file":
+		if strings.TrimSpace(kwokConfig.File.Path) == "" {
+			klog.Fatalf("please specify 'file.path' in kwok config (currently empty or undefined)")
+		}
+		if strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelKey) == "" &&
+			strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelAnnotation) == "" {
+			klog.Fatalf("please specify either 'file.nodegroups.fromNodeLabelKey' or 'file.nodegroups.fromNodeAnnotation' in kwok config (currently empty or undefined)")
+		}
+		if strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelKey) != "" &&
+			strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelAnnotation) != "" {
+			klog.Fatalf("please specify either 'file.nodegroups.fromNodeLabelKey' or 'file.nodegroups.fromNodeAnnotation' in kwok config (you can't use both)")
+		}
+	case "cluster":
+		if strings.TrimSpace(kwokConfig.Cluster.Nodegroups.FromNodeLabelKey) == "" &&
+			strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelAnnotation) == "" {
+			klog.Fatalf("please specify either 'cluster.nodegroups.fromNodeLabelKey' or 'file.nodegroups.fromNodeAnnotation' in kwok config (currently empty or undefined)")
+		}
+		if strings.TrimSpace(kwokConfig.Cluster.Nodegroups.FromNodeLabelKey) != "" &&
+			strings.TrimSpace(kwokConfig.File.Nodegroups.FromNodeLabelAnnotation) != "" {
+			klog.Fatalf("please specify either 'cluster.nodegroups.fromNodeLabelKey' or 'file.nodegroups.fromNodeAnnotation' in kwok config (you can't use both)")
+		}
+	default:
+		klog.Fatalf("'readNodesFrom' in kwok config is invalid (expected: 'cluster' or 'file'): %s",
+			kwokConfig.ReadNodesFrom)
+	}
+
+	templatesPath := kwokConfig.File.Path
 	if strings.TrimSpace(templatesPath) == "" {
-		klog.Fatal("'KWOK_TEMPLATES_PATH' is not specified or empty")
+		klog.Fatalf("please specify path to kwok provider config using 'KWOK_CONFIG_PATH' (currently empty)")
 	}
 
 	data := readTemplatesFile(templatesPath)

cluster-autoscaler/cloudprovider/kwok/samples/dynamic_nodegroups_config.yaml

@@ -0,0 +1,15 @@
+templateNodesFrom: cluster # possible values: [cluster,file]
+cluster:
+  nodegroups:
+    # if you want to treat nodes with same instance type as a nodegroup
+    # e.g.,
+    # node1: m5.xlarge
+    # node2: c5.xlarge
+    # node3: m5.xlarge
+    # nodegroup1: [node1,node3]
+    # nodegroup2: [node2]
+    fromNodeLabelKey: "node.kubernetes.io/instance-type"
+    # you can either specify fromNodeLabelKey OR fromNodeAnnotation
+    # (both are not allowed)
+    # fromNodeAnnotation: "eks.amazonaws.com/nodegroup"
+file: {}

cluster-autoscaler/cloudprovider/kwok/samples/static_nodegroups_config.yaml

@@ -0,0 +1,17 @@
+
+templateNodesFrom: filepath # possible values: [cluster,file]
+cluster: {}
+file:  
+  path: "/etc/node-templates.yaml"
+  nodegroups:
+    # if you want to treat nodes with same instance type as a nodegroup
+    # e.g.,
+    # node1: m5.xlarge
+    # node2: c5.xlarge
+    # node3: m5.xlarge
+    # nodegroup1: [node1,node3]
+    # nodegroup2: [node2]
+    fromNodeLabelKey: "node.kubernetes.io/instance-type"
+    # you can either specify fromNodeLabelKey OR fromNodeAnnotation
+    # (both are not allowed)
+    # fromNodeAnnotation: "eks.amazonaws.com/nodegroup"