feat: each node pool can now have different init configs

cluster-autoscaler/cloudprovider/hetzner/hetzner_cloud_provider.go

@@ -193,7 +193,6 @@ func BuildHetzner(_ config.AutoscalingOptions, do cloudprovider.NodeGroupDiscove
 
 	validNodePoolName := regexp.MustCompile(`^[a-z0-9A-Z]+[a-z0-9A-Z\-\.\_]*[a-z0-9A-Z]+$|^[a-z0-9A-Z]{1}$`)
 	clusterUpdateLock := sync.Mutex{}
-
 	for _, nodegroupSpec := range do.NodeGroupSpecs {
 		spec, err := createNodePoolSpec(nodegroupSpec)
 		if err != nil {

cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go

@@ -19,6 +19,7 @@ package hetzner
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -45,8 +46,7 @@ type hetznerManager struct {
 	client           *hcloud.Client
 	nodeGroups       map[string]*hetznerNodeGroup
 	apiCallContext   context.Context
-	cloudInit        string
-	image            string
+	clusterConfig    *ClusterConfig
 	sshKey           *hcloud.SSHKey
 	network          *hcloud.Network
 	firewall         *hcloud.Firewall
@@ -57,6 +57,22 @@ type hetznerManager struct {
 	cachedServers    *serversCache
 }
 
+type ClusterConfig struct {
+	ImagesForArch ImageList
+	NodeConfigs   map[string]*NodeConfig
+}
+
+type ImageList struct {
+	Arm64 string
+	Amd64 string
+}
+
+type NodeConfig struct {
+	CloudInit string
+	Taints    []apiv1.Taint
+	Labels    map[string]string
+}
+
 func newManager() (*hetznerManager, error) {
 	token := os.Getenv("HCLOUD_TOKEN")
 	if token == "" {
@@ -72,19 +88,13 @@ func newManager() (*hetznerManager, error) {
 
 	ctx := context.Background()
 
-	cloudInitBase64 := os.Getenv("HCLOUD_CLOUD_INIT")
-	if cloudInitBase64 == "" {
-		return nil, errors.New("`HCLOUD_CLOUD_INIT` is not specified")
-	}
-	cloudInit, err := base64.StdEncoding.DecodeString(cloudInitBase64)
+	clusterConfigBase64 := os.Getenv("HCLOUD_CLUSTER_CONFIG")
+	clusterConfigEnv, err := base64.StdEncoding.DecodeString(clusterConfigBase64)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse cloud init error: %s", err)
 	}
-
-	imageName := os.Getenv("HCLOUD_IMAGE")
-	if imageName == "" {
-		imageName = "ubuntu-20.04"
-	}
+	var clusterConfig *ClusterConfig
+	json.Unmarshal([]byte(string(clusterConfigEnv)), &clusterConfig)
 
 	publicIPv4 := true
 	publicIPv4Str := os.Getenv("HCLOUD_PUBLIC_IPV4")
@@ -141,15 +151,14 @@ func newManager() (*hetznerManager, error) {
 	m := &hetznerManager{
 		client:           client,
 		nodeGroups:       make(map[string]*hetznerNodeGroup),
-		cloudInit:        string(cloudInit),
-		image:            imageName,
 		sshKey:           sshKey,
 		network:          network,
 		firewall:         firewall,
 		createTimeout:    createTimeout,
 		apiCallContext:   ctx,
 		publicIPv4:       publicIPv4,
 		publicIPv6:       publicIPv6,
+		clusterConfig:    clusterConfig,
 		cachedServerType: newServerTypeCache(ctx, client),
 		cachedServers:    newServersCache(ctx, client),
 	}

cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go

@@ -18,7 +18,9 @@ package hetzner
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
+	"maps"
 	"math/rand"
 	"strings"
 	"sync"
@@ -240,6 +242,18 @@ func (n *hetznerNodeGroup) TemplateNodeInfo() (*schedulerframework.NodeInfo, err
 		return nil, err
 	}
 	node.Labels = cloudprovider.JoinStringMaps(node.Labels, nodeGroupLabels)
+	node.Status.Allocatable = node.Status.Capacity
+
+	_, ok := n.manager.clusterConfig.NodeConfigs[n.id]
+	if ok {
+		for _, taint := range n.manager.clusterConfig.NodeConfigs[n.id].Taints {
+			node.Spec.Taints = append(node.Spec.Taints, apiv1.Taint{
+				Key:    taint.Key,
+				Value:  taint.Value,
+				Effect: taint.Effect,
+			})
+		}
+	}
 
 	nodeInfo := schedulerframework.NewNodeInfo(cloudprovider.BuildKubeProxy(n.id))
 	nodeInfo.SetNode(&node)
@@ -325,14 +339,24 @@ func buildNodeGroupLabels(n *hetznerNodeGroup) (map[string]string, error) {
 	if err != nil {
 		return nil, err
 	}
+	klog.V(4).Infof("Build node group label for %s", n.id)
 
-	return map[string]string{
+	labels := map[string]string{
 		apiv1.LabelInstanceType:      n.instanceType,
 		apiv1.LabelTopologyRegion:    n.region,
 		apiv1.LabelArchStable:        archLabel,
 		"csi.hetzner.cloud/location": n.region,
 		nodeGroupLabel:               n.id,
-	}, nil
+	}
+
+	_, ok := n.manager.clusterConfig.NodeConfigs[n.id]
+	if ok {
+		maps.Copy(labels, n.manager.clusterConfig.NodeConfigs[n.id].Labels)
+	}
+
+	klog.V(4).Infof("%s nodegroup labels: %s", n.id, labels)
+
+	return labels, nil
 }
 
 func getMachineTypeResourceList(m *hetznerManager, instanceType string) (apiv1.ResourceList, error) {
@@ -392,10 +416,16 @@ func createServer(n *hetznerNodeGroup) error {
 		return err
 	}
 
+	cloudInit, err := base64.StdEncoding.DecodeString(n.manager.clusterConfig.NodeConfigs[n.id].CloudInit)
+	if err != nil {
+		klog.Errorf("%s - failed to parse cloud init error: %s", n.id, err)
+		return err
+	}
+
 	StartAfterCreate := true
 	opts := hcloud.ServerCreateOpts{
 		Name:             newNodeName(n),
-		UserData:         n.manager.cloudInit,
+		UserData:         string(cloudInit),
 		Location:         &hcloud.Location{Name: n.region},
 		ServerType:       serverType,
 		Image:            image,
@@ -443,7 +473,15 @@ func createServer(n *hetznerNodeGroup) error {
 // server.
 func findImage(n *hetznerNodeGroup, serverType *hcloud.ServerType) (*hcloud.Image, error) {
 	// Select correct image based on server type architecture
-	image, _, err := n.manager.client.Image.GetForArchitecture(context.TODO(), n.manager.image, serverType.Architecture)
+	imageName := "ubuntu-22.04"
+	if serverType.Architecture == hcloud.ArchitectureARM {
+		imageName = n.manager.clusterConfig.ImagesForArch.Arm64
+	}
+
+	if serverType.Architecture == hcloud.ArchitectureX86 {
+		imageName = n.manager.clusterConfig.ImagesForArch.Amd64
+	}
+	image, _, err := n.manager.client.Image.GetForArchitecture(context.TODO(), imageName, serverType.Architecture)
 	if err != nil {
 		// Keep looking for label if image was not found by id or name
 		if !strings.HasPrefix(err.Error(), "image not found") {
@@ -462,12 +500,12 @@ func findImage(n *hetznerNodeGroup, serverType *hcloud.ServerType) (*hcloud.Imag
 		Sort:         []string{"created:desc"},
 		Architecture: []hcloud.Architecture{serverType.Architecture},
 		ListOpts: hcloud.ListOpts{
-			LabelSelector: n.manager.image,
+			LabelSelector: imageName,
 		},
 	})
 
 	if err != nil || len(images) == 0 {
-		return nil, fmt.Errorf("unable to find image %s with architecture %s: %v", n.manager.image, serverType.Architecture, err)
+		return nil, fmt.Errorf("unable to find image %s with architecture %s: %v", imageName, serverType.Architecture, err)
 	}
 
 	return images[0], nil