Reports node taints.

cluster-autoscaler/core/static_autoscaler.go

@@ -955,6 +955,7 @@ func (a *StaticAutoscaler) obtainNodeLists(cp cloudprovider.CloudProvider) ([]*a
 		klog.Errorf("Failed to list ready nodes: %v", err)
 		return nil, nil, caerrors.ToAutoscalerError(caerrors.ApiCallError, err)
 	}
+	a.reportTaintsCount(allNodes)
 
 	// Handle GPU case - allocatable GPU may be equal to 0 up to 15 minutes after
 	// node registers as ready. See https://github.com/kubernetes/kubernetes/issues/54959
@@ -978,6 +979,13 @@ func (a *StaticAutoscaler) updateClusterState(allNodes []*apiv1.Node, nodeInfosF
 	return nil
 }
 
+func (a *StaticAutoscaler) reportTaintsCount(nodes []*apiv1.Node) {
+	foundTaints := taints.CountNodeTaints(nodes, a.taintConfig)
+	for taintType, count := range foundTaints {
+		metrics.ObserveNodeTaintsCount(taintType, float64(count))
+	}
+}
+
 func allPodsAreNew(pods []*apiv1.Pod, currentTime time.Time) bool {
 	if core_utils.GetOldestCreateTime(pods).Add(unschedulablePodTimeBuffer).After(currentTime) {
 		return true

cluster-autoscaler/metrics/metrics.go

@@ -373,6 +373,15 @@ var (
 			Help:      "Number of node groups deleted by Node Autoprovisioning.",
 		},
 	)
+
+	nodeTaintsCount = k8smetrics.NewGaugeVec(
+		&k8smetrics.GaugeOpts{
+			Namespace: caNamespace,
+			Name:      "node_taints_count",
+			Help:      "Number of taints per type used in the cluster.",
+		},
+		[]string{"type"},
+	)
 )
 
 // RegisterAll registers all metrics.
@@ -407,6 +416,7 @@ func RegisterAll(emitPerNodeGroupMetrics bool) {
 	legacyregistry.MustRegister(nodeGroupCreationCount)
 	legacyregistry.MustRegister(nodeGroupDeletionCount)
 	legacyregistry.MustRegister(pendingNodeDeletions)
+	legacyregistry.MustRegister(nodeTaintsCount)
 
 	if emitPerNodeGroupMetrics {
 		legacyregistry.MustRegister(nodesGroupMinNodes)
@@ -615,3 +625,8 @@ func RegisterSkippedScaleUpMemory() {
 func ObservePendingNodeDeletions(value int) {
 	pendingNodeDeletions.Set(float64(value))
 }
+
+// ObserveNodeTaintsCount records the node taints count of given type.
+func ObserveNodeTaintsCount(taintType string, count float64) {
+	nodeTaintsCount.WithLabelValues(taintType).Set(count)
+}

cluster-autoscaler/proposals/metrics.md

@@ -21,17 +21,18 @@ All the metrics are prefixed with `cluster_autoscaler_`.
 
 ### Cluster state
 
-| Metric name | Metric type | Labels | Description |
-| ----------- | ----------- | ------ | ----------- |
-| cluster_safe_to_autoscale | Gauge | | Whether or not cluster is healthy enough for autoscaling. 1 if it is, 0 otherwise. |
-| nodes_count | Gauge | `state`=<node-state> | Number of nodes in cluster. |
-| unschedulable_pods_count | Gauge | | Number of unschedulable ("Pending") pods in the cluster. |
-| node_groups_count | Gauge | `node_group_type`=<node-group-type> | Number of node groups managed by CA. |
-| max_nodes_count | Gauge | | Maximum number of nodes in all node groups. |
-| cluster_cpu_current_cores | Gauge | | | Current number of cores in the cluster, minus deleting nodes. |
-| cpu_limits_cores | Gauge | `direction`=<`minimum` or `maximum`> | Minimum and maximum number of cores in the cluster. |
-| cluster_memory_current_bytes | Gauge | | Current number of bytes of memory in the cluster, minus deleting nodes. |
-| memory_limits_bytes | Gauge | `direction`=<`minimum` or `maximum`> | Minimum and maximum number of bytes of memory in cluster. |
+| Metric name | Metric type | Labels                                     | Description                                                                        |
+| ----------- | ----------- |--------------------------------------------|------------------------------------------------------------------------------------|
+| cluster_safe_to_autoscale | Gauge |                                            | Whether or not cluster is healthy enough for autoscaling. 1 if it is, 0 otherwise. |
+| nodes_count | Gauge | `state`=<node-state>                 | Number of nodes in cluster.                                                        |
+| unschedulable_pods_count | Gauge |                                            | Number of unschedulable ("Pending") pods in the cluster.                           |
+| node_groups_count | Gauge | `node_group_type`=<node-group-type>  | Number of node groups managed by CA.                                               |
+| max_nodes_count | Gauge |                                            | Maximum number of nodes in all node groups.                                        |
+| cluster_cpu_current_cores | Gauge |                                            |                                                                                    | Current number of cores in the cluster, minus deleting nodes. |
+| cpu_limits_cores | Gauge | `direction`=<`minimum` or `maximum`> | Minimum and maximum number of cores in the cluster.                                |
+| cluster_memory_current_bytes | Gauge |                                            | Current number of bytes of memory in the cluster, minus deleting nodes.            |
+| memory_limits_bytes | Gauge | `direction`=<`minimum` or `maximum`> | Minimum and maximum number of bytes of memory in cluster.                          |
+ | node_taints_count | Gauge | `type`=<taint-type>                  | Number of taints per type in cluster.                                              |
 
 * `cluster_safe_to_autoscale` indicates whether cluster is healthy enough for autoscaling. CA stops all operations if significant number of nodes are unready (by default 33% as of CA 0.5.4).
 * `nodes_count` records the total number of nodes, labeled by node state. Possible

cluster-autoscaler/utils/taints/taints.go

@@ -19,6 +19,7 @@ package taints
 import (
 	"context"
 	"fmt"
+	"maps"
 	"strconv"
 	"strings"
 	"time"
@@ -54,17 +55,48 @@ const (
 
 	// AWS: Indicates that a node has volumes stuck in attaching state and hence it is not fit for scheduling more pods
 	awsNodeWithImpairedVolumesTaint = "NodeWithImpairedVolumes"
+
+	// statusNodeTaintReportedType is the value used when reporting node taint count defined as status taint in given taintConfig.
+	statusNodeTaintReportedType = "status-taint"
+
+	// startupNodeTaintReportedType is the value used when reporting node taint count defined as startup taint in given taintConfig.
+	startupNodeTaintReportedType = "startup-taint"
+
+	// unlistedNodeTaintReportedType is the value used when reporting node taint count in case taint key is other than defined in explicitlyReportedNodeTaints and taintConfig.
+	unlistedNodeTaintReportedType = "other"
+)
+
+var (
+	// NodeConditionTaints lists taint keys used as node conditions
+	NodeConditionTaints = TaintKeySet{
+		apiv1.TaintNodeNotReady:                     true,
+		apiv1.TaintNodeUnreachable:                  true,
+		apiv1.TaintNodeUnschedulable:                true,
+		apiv1.TaintNodeMemoryPressure:               true,
+		apiv1.TaintNodeDiskPressure:                 true,
+		apiv1.TaintNodeNetworkUnavailable:           true,
+		apiv1.TaintNodePIDPressure:                  true,
+		cloudproviderapi.TaintExternalCloudProvider: true,
+		cloudproviderapi.TaintNodeShutdown:          true,
+		gkeNodeTerminationHandlerTaint:              true,
+		awsNodeWithImpairedVolumesTaint:             true,
+	}
+
+	// Mutable only in unit tests
+	maxRetryDeadline      time.Duration = 5 * time.Second
+	conflictRetryInterval time.Duration = 750 * time.Millisecond
 )
 
 // TaintKeySet is a set of taint key
 type TaintKeySet map[string]bool
 
 // TaintConfig is a config of taints that require special handling
 type TaintConfig struct {
-	StartupTaints        TaintKeySet
-	StatusTaints         TaintKeySet
-	StartupTaintPrefixes []string
-	StatusTaintPrefixes  []string
+	startupTaints            TaintKeySet
+	statusTaints             TaintKeySet
+	startupTaintPrefixes     []string
+	statusTaintPrefixes      []string
+	explicitlyReportedTaints TaintKeySet
 }
 
 // NewTaintConfig returns the taint config extracted from options
@@ -81,34 +113,41 @@ func NewTaintConfig(opts config.AutoscalingOptions) TaintConfig {
 		statusTaints[taintKey] = true
 	}
 
+	explicitlyReportedTaints := TaintKeySet{
+		ToBeDeletedTaint:       true,
+		DeletionCandidateTaint: true,
+	}
+	maps.Copy(explicitlyReportedTaints, NodeConditionTaints)
+
 	return TaintConfig{
-		StartupTaints:        startupTaints,
-		StatusTaints:         statusTaints,
-		StartupTaintPrefixes: []string{IgnoreTaintPrefix, StartupTaintPrefix},
-		StatusTaintPrefixes:  []string{StatusTaintPrefix},
+		startupTaints:            startupTaints,
+		statusTaints:             statusTaints,
+		startupTaintPrefixes:     []string{IgnoreTaintPrefix, StartupTaintPrefix},
+		statusTaintPrefixes:      []string{StatusTaintPrefix},
+		explicitlyReportedTaints: explicitlyReportedTaints,
 	}
 }
 
-var (
-	// NodeConditionTaints lists taint keys used as node conditions
-	NodeConditionTaints = TaintKeySet{
-		apiv1.TaintNodeNotReady:                     true,
-		apiv1.TaintNodeUnreachable:                  true,
-		apiv1.TaintNodeUnschedulable:                true,
-		apiv1.TaintNodeMemoryPressure:               true,
-		apiv1.TaintNodeDiskPressure:                 true,
-		apiv1.TaintNodeNetworkUnavailable:           true,
-		apiv1.TaintNodePIDPressure:                  true,
-		cloudproviderapi.TaintExternalCloudProvider: true,
-		cloudproviderapi.TaintNodeShutdown:          true,
-		gkeNodeTerminationHandlerTaint:              true,
-		awsNodeWithImpairedVolumesTaint:             true,
+// IsStartupTaint checks whether given taint is a startup taint.
+func (tc TaintConfig) IsStartupTaint(taint string) bool {
+	if _, ok := tc.startupTaints[taint]; ok {
+		return true
 	}
+	return matchesAnyPrefix(tc.startupTaintPrefixes, taint)
+}
 
-	// Mutable only in unit tests
-	maxRetryDeadline      time.Duration = 5 * time.Second
-	conflictRetryInterval time.Duration = 750 * time.Millisecond
-)
+// IsStatusTaint checks whether given taint is a status taint.
+func (tc TaintConfig) IsStatusTaint(taint string) bool {
+	if _, ok := tc.statusTaints[taint]; ok {
+		return true
+	}
+	return matchesAnyPrefix(tc.statusTaintPrefixes, taint)
+}
+
+func (tc TaintConfig) isExplicitlyReportedTaint(taint string) bool {
+	_, ok := tc.explicitlyReportedTaints[taint]
+	return ok
+}
 
 // getKeyShortName converts taint key to short name for logging
 func getKeyShortName(key string) string {
@@ -361,18 +400,8 @@ func SanitizeTaints(taints []apiv1.Taint, taintConfig TaintConfig) []apiv1.Taint
 			continue
 		}
 
-		if _, exists := taintConfig.StartupTaints[taint.Key]; exists {
-			klog.V(4).Infof("Removing startup taint %s, when creating template from node", taint.Key)
-			continue
-		}
-		shouldRemoveBasedOnPrefix := matchesAnyPrefix(taintConfig.StartupTaintPrefixes, taint.Key) || matchesAnyPrefix(taintConfig.StatusTaintPrefixes, taint.Key)
-		if shouldRemoveBasedOnPrefix {
-			klog.V(4).Infof("Removing taint %s based on prefix, when creation template from node", taint.Key)
-			continue
-		}
-
-		if _, exists := taintConfig.StatusTaints[taint.Key]; exists {
-			klog.V(4).Infof("Removing status taint %s, when creating template from node", taint.Key)
+		if taintConfig.IsStartupTaint(taint.Key) || taintConfig.IsStatusTaint(taint.Key) {
+			klog.V(4).Infof("Removing taint %s, when creating template from node", taint.Key)
 			continue
 		}
 
@@ -394,8 +423,7 @@ func FilterOutNodesWithStartupTaints(taintConfig TaintConfig, allNodes, readyNod
 		}
 		ready := true
 		for _, t := range node.Spec.Taints {
-			_, hasStartupTaint := taintConfig.StartupTaints[t.Key]
-			if hasStartupTaint || matchesAnyPrefix(taintConfig.StartupTaintPrefixes, t.Key) {
+			if taintConfig.IsStartupTaint(t.Key) {
 				ready = false
 				nodesWithStartupTaints[node.Name] = kubernetes.GetUnreadyNodeCopy(node, kubernetes.StartupNodes)
 				klog.V(3).Infof("Overriding status of node %v, which seems to have startup taint %q", node.Name, t.Key)
@@ -416,3 +444,33 @@ func FilterOutNodesWithStartupTaints(taintConfig TaintConfig, allNodes, readyNod
 	}
 	return newAllNodes, newReadyNodes
 }
+
+// CountNodeTaints counts used node taints.
+func CountNodeTaints(nodes []*apiv1.Node, taintConfig TaintConfig) map[string]int {
+	foundTaintsCount := make(map[string]int)
+	for _, node := range nodes {
+		for _, taint := range node.Spec.Taints {
+			key := getTaintTypeToReport(taint.Key, taintConfig)
+			foundTaintsCount[key] += 1
+		}
+	}
+	return foundTaintsCount
+}
+
+func getTaintTypeToReport(key string, taintConfig TaintConfig) string {
+	// Track deprecated taints.
+	if strings.HasPrefix(key, IgnoreTaintPrefix) {
+		return IgnoreTaintPrefix
+	}
+
+	if taintConfig.isExplicitlyReportedTaint(key) {
+		return key
+	}
+	if taintConfig.IsStartupTaint(key) {
+		return startupNodeTaintReportedType
+	}
+	if taintConfig.IsStatusTaint(key) {
+		return statusNodeTaintReportedType
+	}
+	return unlistedNodeTaintReportedType
+}