From 98effe0c55feb4faf7d0d8d9a65c15835f9b9f78 Mon Sep 17 00:00:00 2001 From: Artem Minyaylov Date: Wed, 27 Sep 2023 18:22:01 +0000 Subject: [PATCH] Refactor NodeDeleteOptions for use in drainability rules --- cluster-autoscaler/core/autoscaler.go | 10 ++-- .../core/scaledown/actuation/actuator.go | 12 +++-- .../core/scaledown/actuation/drain.go | 10 ++-- .../core/scaledown/legacy/legacy.go | 6 ++- .../core/scaledown/legacy/legacy_test.go | 8 ++-- .../core/scaledown/planner/planner.go | 6 ++- .../core/scaledown/planner/planner_test.go | 13 ++--- cluster-autoscaler/core/static_autoscaler.go | 11 +++-- .../core/static_autoscaler_test.go | 25 +++++----- cluster-autoscaler/main.go | 16 +++---- .../empty_candidates_sorting.go | 15 ++++-- .../empty_candidates_sorting_test.go | 31 ++++++------ cluster-autoscaler/simulator/cluster.go | 12 +++-- cluster-autoscaler/simulator/cluster_test.go | 10 ++-- cluster-autoscaler/simulator/drain.go | 35 ++------------ cluster-autoscaler/simulator/drain_test.go | 7 ++- .../simulator/drainability/context.go | 4 +- .../simulator/options/nodedelete.go | 48 +++++++++++++++++++ 18 files changed, 166 insertions(+), 113 deletions(-) create mode 100644 cluster-autoscaler/simulator/options/nodedelete.go diff --git a/cluster-autoscaler/core/autoscaler.go b/cluster-autoscaler/core/autoscaler.go index 4fd81f3e7436..29d4b841b8ba 100644 --- a/cluster-autoscaler/core/autoscaler.go +++ b/cluster-autoscaler/core/autoscaler.go @@ -31,8 +31,9 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/expander/factory" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" - "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/utils/backoff" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -57,7 +58,8 @@ type AutoscalerOptions struct { DebuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter RemainingPdbTracker pdb.RemainingPdbTracker ScaleUpOrchestrator scaleup.Orchestrator - DeleteOptions simulator.NodeDeleteOptions + DeleteOptions options.NodeDeleteOptions + DrainabilityRules rules.Rules } // Autoscaler is the main component of CA which scales up/down node groups according to its configuration @@ -90,7 +92,9 @@ func NewAutoscaler(opts AutoscalerOptions) (Autoscaler, errors.AutoscalerError) opts.DebuggingSnapshotter, opts.RemainingPdbTracker, opts.ScaleUpOrchestrator, - opts.DeleteOptions), nil + opts.DeleteOptions, + opts.DrainabilityRules, + ), nil } // Initialize default options if not provided. diff --git a/cluster-autoscaler/core/scaledown/actuation/actuator.go b/cluster-autoscaler/core/scaledown/actuation/actuator.go index 247f04767f39..68e07fc5cbe6 100644 --- a/cluster-autoscaler/core/scaledown/actuation/actuator.go +++ b/cluster-autoscaler/core/scaledown/actuation/actuator.go @@ -35,6 +35,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/metrics" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/utilization" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -47,7 +49,8 @@ type Actuator struct { clusterState *clusterstate.ClusterStateRegistry nodeDeletionTracker *deletiontracker.NodeDeletionTracker nodeDeletionScheduler *GroupDeletionScheduler - deleteOptions simulator.NodeDeleteOptions + deleteOptions options.NodeDeleteOptions + drainabilityRules rules.Rules // TODO: Move budget processor to scaledown planner, potentially merge into PostFilteringScaleDownNodeProcessor // This is a larger change to the code structure which impacts some existing actuator unit tests // as well as Cluster Autoscaler implementations that may override ScaleDownSetProcessor @@ -64,15 +67,16 @@ type actuatorNodeGroupConfigGetter interface { } // NewActuator returns a new instance of Actuator. -func NewActuator(ctx *context.AutoscalingContext, csr *clusterstate.ClusterStateRegistry, ndt *deletiontracker.NodeDeletionTracker, deleteOptions simulator.NodeDeleteOptions, configGetter actuatorNodeGroupConfigGetter) *Actuator { +func NewActuator(ctx *context.AutoscalingContext, csr *clusterstate.ClusterStateRegistry, ndt *deletiontracker.NodeDeletionTracker, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules, configGetter actuatorNodeGroupConfigGetter) *Actuator { ndb := NewNodeDeletionBatcher(ctx, csr, ndt, ctx.NodeDeletionBatcherInterval) return &Actuator{ ctx: ctx, clusterState: csr, nodeDeletionTracker: ndt, - nodeDeletionScheduler: NewGroupDeletionScheduler(ctx, ndt, ndb, NewDefaultEvictor(deleteOptions, ndt)), + nodeDeletionScheduler: NewGroupDeletionScheduler(ctx, ndt, ndb, NewDefaultEvictor(deleteOptions, drainabilityRules, ndt)), budgetProcessor: budgets.NewScaleDownBudgetProcessor(ctx), deleteOptions: deleteOptions, + drainabilityRules: drainabilityRules, configGetter: configGetter, nodeDeleteDelayAfterTaint: ctx.NodeDeleteDelayAfterTaint, } @@ -272,7 +276,7 @@ func (a *Actuator) deleteNodesAsync(nodes []*apiv1.Node, nodeGroup cloudprovider continue } - podsToRemove, _, _, err := simulator.GetPodsToMove(nodeInfo, a.deleteOptions, registry, pdbs, time.Now()) + podsToRemove, _, _, err := simulator.GetPodsToMove(nodeInfo, a.deleteOptions, a.drainabilityRules, registry, pdbs, time.Now()) if err != nil { klog.Errorf("Scale-down: couldn't delete node %q, err: %v", node.Name, err) nodeDeleteResult := status.NodeDeleteResult{ResultType: status.NodeDeleteErrorInternal, Err: errors.NewAutoscalerError(errors.InternalError, "GetPodsToMove for %q returned error: %v", node.Name, err)} diff --git a/cluster-autoscaler/core/scaledown/actuation/drain.go b/cluster-autoscaler/core/scaledown/actuation/drain.go index e479abe34a99..6b760603ba17 100644 --- a/cluster-autoscaler/core/scaledown/actuation/drain.go +++ b/cluster-autoscaler/core/scaledown/actuation/drain.go @@ -33,6 +33,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/core/scaledown/status" "k8s.io/autoscaler/cluster-autoscaler/metrics" "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/utils/daemonset" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" pod_util "k8s.io/autoscaler/cluster-autoscaler/utils/pod" @@ -62,11 +64,12 @@ type Evictor struct { DsEvictionEmptyNodeTimeout time.Duration PodEvictionHeadroom time.Duration evictionRegister evictionRegister - deleteOptions simulator.NodeDeleteOptions + deleteOptions options.NodeDeleteOptions + drainabilityRules rules.Rules } // NewDefaultEvictor returns an instance of Evictor using the default parameters. -func NewDefaultEvictor(deleteOptions simulator.NodeDeleteOptions, evictionRegister evictionRegister) Evictor { +func NewDefaultEvictor(deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules, evictionRegister evictionRegister) Evictor { return Evictor{ EvictionRetryTime: DefaultEvictionRetryTime, DsEvictionRetryTime: DefaultDsEvictionRetryTime, @@ -74,6 +77,7 @@ func NewDefaultEvictor(deleteOptions simulator.NodeDeleteOptions, evictionRegist PodEvictionHeadroom: DefaultPodEvictionHeadroom, evictionRegister: evictionRegister, deleteOptions: deleteOptions, + drainabilityRules: drainabilityRules, } } @@ -177,7 +181,7 @@ func (e Evictor) DrainNodeWithPods(ctx *acontext.AutoscalingContext, node *apiv1 // EvictDaemonSetPods creates eviction objects for all DaemonSet pods on the node. func (e Evictor) EvictDaemonSetPods(ctx *acontext.AutoscalingContext, nodeInfo *framework.NodeInfo, timeNow time.Time) error { nodeToDelete := nodeInfo.Node() - _, daemonSetPods, _, err := simulator.GetPodsToMove(nodeInfo, e.deleteOptions, nil, []*policyv1.PodDisruptionBudget{}, timeNow) + _, daemonSetPods, _, err := simulator.GetPodsToMove(nodeInfo, e.deleteOptions, e.drainabilityRules, nil, []*policyv1.PodDisruptionBudget{}, timeNow) if err != nil { return fmt.Errorf("failed to get DaemonSet pods for %s (error: %v)", nodeToDelete.Name, err) } diff --git a/cluster-autoscaler/core/scaledown/legacy/legacy.go b/cluster-autoscaler/core/scaledown/legacy/legacy.go index 65dba06f7f4c..592bf569c0dc 100644 --- a/cluster-autoscaler/core/scaledown/legacy/legacy.go +++ b/cluster-autoscaler/core/scaledown/legacy/legacy.go @@ -32,6 +32,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/metrics" "k8s.io/autoscaler/cluster-autoscaler/processors" "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/utilization" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -55,9 +57,9 @@ type ScaleDown struct { } // NewScaleDown builds new ScaleDown object. -func NewScaleDown(context *context.AutoscalingContext, processors *processors.AutoscalingProcessors, ndt *deletiontracker.NodeDeletionTracker, deleteOptions simulator.NodeDeleteOptions) *ScaleDown { +func NewScaleDown(context *context.AutoscalingContext, processors *processors.AutoscalingProcessors, ndt *deletiontracker.NodeDeletionTracker, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules) *ScaleDown { usageTracker := simulator.NewUsageTracker() - removalSimulator := simulator.NewRemovalSimulator(context.ListerRegistry, context.ClusterSnapshot, context.PredicateChecker, usageTracker, deleteOptions, false) + removalSimulator := simulator.NewRemovalSimulator(context.ListerRegistry, context.ClusterSnapshot, context.PredicateChecker, usageTracker, deleteOptions, drainabilityRules, false) unremovableNodes := unremovable.NewNodes() resourceLimitsFinder := resource.NewLimitsFinder(processors.CustomResourcesProcessor) return &ScaleDown{ diff --git a/cluster-autoscaler/core/scaledown/legacy/legacy_test.go b/cluster-autoscaler/core/scaledown/legacy/legacy_test.go index be6d7565d4b3..1ba61e250dfd 100644 --- a/cluster-autoscaler/core/scaledown/legacy/legacy_test.go +++ b/cluster-autoscaler/core/scaledown/legacy/legacy_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupconfig" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" autoscaler_errors "k8s.io/autoscaler/cluster-autoscaler/utils/errors" appsv1 "k8s.io/api/apps/v1" @@ -1287,14 +1288,13 @@ func newWrapperForTesting(ctx *context.AutoscalingContext, clusterStateRegistry if ndt == nil { ndt = deletiontracker.NewNodeDeletionTracker(0 * time.Second) } - deleteOptions := simulator.NodeDeleteOptions{ + deleteOptions := options.NodeDeleteOptions{ SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, - MinReplicaCount: 0, SkipNodesWithCustomControllerPods: true, } processors := NewTestProcessors(ctx) - sd := NewScaleDown(ctx, processors, ndt, deleteOptions) - actuator := actuation.NewActuator(ctx, clusterStateRegistry, ndt, deleteOptions, processors.NodeGroupConfigProcessor) + sd := NewScaleDown(ctx, processors, ndt, deleteOptions, nil) + actuator := actuation.NewActuator(ctx, clusterStateRegistry, ndt, deleteOptions, nil, processors.NodeGroupConfigProcessor) return NewScaleDownWrapper(sd, actuator) } diff --git a/cluster-autoscaler/core/scaledown/planner/planner.go b/cluster-autoscaler/core/scaledown/planner/planner.go index 0cd4002c4af3..6de613ac8d53 100644 --- a/cluster-autoscaler/core/scaledown/planner/planner.go +++ b/cluster-autoscaler/core/scaledown/planner/planner.go @@ -34,6 +34,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodes" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling" "k8s.io/autoscaler/cluster-autoscaler/simulator/utilization" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -77,7 +79,7 @@ type Planner struct { } // New creates a new Planner object. -func New(context *context.AutoscalingContext, processors *processors.AutoscalingProcessors, deleteOptions simulator.NodeDeleteOptions) *Planner { +func New(context *context.AutoscalingContext, processors *processors.AutoscalingProcessors, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules) *Planner { resourceLimitsFinder := resource.NewLimitsFinder(processors.CustomResourcesProcessor) minUpdateInterval := context.AutoscalingOptions.NodeGroupDefaults.ScaleDownUnneededTime if minUpdateInterval == 0*time.Nanosecond { @@ -87,7 +89,7 @@ func New(context *context.AutoscalingContext, processors *processors.Autoscaling context: context, unremovableNodes: unremovable.NewNodes(), unneededNodes: unneeded.NewNodes(processors.NodeGroupConfigProcessor, resourceLimitsFinder), - rs: simulator.NewRemovalSimulator(context.ListerRegistry, context.ClusterSnapshot, context.PredicateChecker, simulator.NewUsageTracker(), deleteOptions, true), + rs: simulator.NewRemovalSimulator(context.ListerRegistry, context.ClusterSnapshot, context.PredicateChecker, simulator.NewUsageTracker(), deleteOptions, drainabilityRules, true), actuationInjector: scheduling.NewHintingSimulator(context.PredicateChecker), eligibilityChecker: eligibility.NewChecker(processors.NodeGroupConfigProcessor), nodeUtilizationMap: make(map[string]utilization.Info), diff --git a/cluster-autoscaler/core/scaledown/planner/planner_test.go b/cluster-autoscaler/core/scaledown/planner/planner_test.go index d3c14a61b733..4fa54825dc8b 100644 --- a/cluster-autoscaler/core/scaledown/planner/planner_test.go +++ b/cluster-autoscaler/core/scaledown/planner/planner_test.go @@ -37,6 +37,7 @@ import ( . "k8s.io/autoscaler/cluster-autoscaler/core/test" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/utilization" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -492,8 +493,8 @@ func TestUpdateClusterState(t *testing.T) { }, &fake.Clientset{}, registry, provider, nil, nil) assert.NoError(t, err) clustersnapshot.InitializeClusterSnapshotOrDie(t, context.ClusterSnapshot, tc.nodes, tc.pods) - deleteOptions := simulator.NodeDeleteOptions{} - p := New(&context, NewTestProcessors(&context), deleteOptions) + deleteOptions := options.NodeDeleteOptions{} + p := New(&context, NewTestProcessors(&context), deleteOptions, nil) p.eligibilityChecker = &fakeEligibilityChecker{eligible: asMap(tc.eligible)} if tc.isSimulationTimeout { context.AutoscalingOptions.ScaleDownSimulationTimeout = 1 * time.Second @@ -611,8 +612,8 @@ func TestUpdateClusterStatUnneededNodesLimit(t *testing.T) { }, &fake.Clientset{}, nil, provider, nil, nil) assert.NoError(t, err) clustersnapshot.InitializeClusterSnapshotOrDie(t, context.ClusterSnapshot, nodes, nil) - deleteOptions := simulator.NodeDeleteOptions{} - p := New(&context, NewTestProcessors(&context), deleteOptions) + deleteOptions := options.NodeDeleteOptions{} + p := New(&context, NewTestProcessors(&context), deleteOptions, nil) p.eligibilityChecker = &fakeEligibilityChecker{eligible: asMap(nodeNames(nodes))} p.minUpdateInterval = tc.updateInterval p.unneededNodes.Update(previouslyUnneeded, time.Now()) @@ -779,8 +780,8 @@ func TestNodesToDelete(t *testing.T) { }, &fake.Clientset{}, nil, provider, nil, nil) assert.NoError(t, err) clustersnapshot.InitializeClusterSnapshotOrDie(t, context.ClusterSnapshot, allNodes, nil) - deleteOptions := simulator.NodeDeleteOptions{} - p := New(&context, NewTestProcessors(&context), deleteOptions) + deleteOptions := options.NodeDeleteOptions{} + p := New(&context, NewTestProcessors(&context), deleteOptions, nil) p.latestUpdate = time.Now() p.actuationStatus = deletiontracker.NewNodeDeletionTracker(0 * time.Second) p.unneededNodes.Update(allRemovables, time.Now().Add(-1*time.Hour)) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index c4bca6969b9b..b3f7dcb94aeb 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -51,6 +51,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/utils/backoff" caerrors "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -142,7 +144,8 @@ func NewStaticAutoscaler( debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter, remainingPdbTracker pdb.RemainingPdbTracker, scaleUpOrchestrator scaleup.Orchestrator, - deleteOptions simulator.NodeDeleteOptions) *StaticAutoscaler { + deleteOptions options.NodeDeleteOptions, + drainabilityRules rules.Rules) *StaticAutoscaler { clusterStateConfig := clusterstate.ClusterStateRegistryConfig{ MaxTotalUnreadyPercentage: opts.MaxTotalUnreadyPercentage, @@ -169,14 +172,14 @@ func NewStaticAutoscaler( // TODO: Populate the ScaleDownActuator/Planner fields in AutoscalingContext // during the struct creation rather than here. ndt := deletiontracker.NewNodeDeletionTracker(0 * time.Second) - scaleDown := legacy.NewScaleDown(autoscalingContext, processors, ndt, deleteOptions) - actuator := actuation.NewActuator(autoscalingContext, clusterStateRegistry, ndt, deleteOptions, processors.NodeGroupConfigProcessor) + scaleDown := legacy.NewScaleDown(autoscalingContext, processors, ndt, deleteOptions, drainabilityRules) + actuator := actuation.NewActuator(autoscalingContext, clusterStateRegistry, ndt, deleteOptions, drainabilityRules, processors.NodeGroupConfigProcessor) autoscalingContext.ScaleDownActuator = actuator var scaleDownPlanner scaledown.Planner var scaleDownActuator scaledown.Actuator if opts.ParallelDrain { - scaleDownPlanner = planner.New(autoscalingContext, processors, deleteOptions) + scaleDownPlanner = planner.New(autoscalingContext, processors, deleteOptions, drainabilityRules) scaleDownActuator = actuator } else { // TODO: Remove the wrapper once the legacy implementation becomes obsolete. diff --git a/cluster-autoscaler/core/static_autoscaler_test.go b/cluster-autoscaler/core/static_autoscaler_test.go index ace16dde111e..1a6fc141ec46 100644 --- a/cluster-autoscaler/core/static_autoscaler_test.go +++ b/cluster-autoscaler/core/static_autoscaler_test.go @@ -45,6 +45,8 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupconfig" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/utilization" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -149,9 +151,9 @@ func (m *onNodeGroupDeleteMock) Delete(id string) error { return args.Error(0) } -func setUpScaleDownActuator(ctx *context.AutoscalingContext, options config.AutoscalingOptions) { - deleteOptions := simulator.NewNodeDeleteOptions(options) - ctx.ScaleDownActuator = actuation.NewActuator(ctx, nil, deletiontracker.NewNodeDeletionTracker(0*time.Second), deleteOptions, NewTestProcessors(ctx).NodeGroupConfigProcessor) +func setUpScaleDownActuator(ctx *context.AutoscalingContext, autoscalingOptions config.AutoscalingOptions) { + deleteOptions := options.NewNodeDeleteOptions(autoscalingOptions) + ctx.ScaleDownActuator = actuation.NewActuator(ctx, nil, deletiontracker.NewNodeDeletionTracker(0*time.Second), deleteOptions, rules.Default(), NewTestProcessors(ctx).NodeGroupConfigProcessor) } func TestStaticAutoscalerRunOnce(t *testing.T) { @@ -1447,11 +1449,11 @@ func TestStaticAutoscalerUpcomingScaleDownCandidates(t *testing.T) { kubernetes.NewTestPodLister(nil), kubernetes.NewTestPodDisruptionBudgetLister(nil), daemonSetLister, nil, nil, nil, nil) - // Create context with minimal options that guarantee we reach the tested logic. - // We're only testing the input to UpdateClusterState which should be called whenever scale-down is enabled, other options shouldn't matter. - options := config.AutoscalingOptions{ScaleDownEnabled: true} + // Create context with minimal autoscalingOptions that guarantee we reach the tested logic. + // We're only testing the input to UpdateClusterState which should be called whenever scale-down is enabled, other autoscalingOptions shouldn't matter. + autoscalingOptions := config.AutoscalingOptions{ScaleDownEnabled: true} processorCallbacks := newStaticAutoscalerProcessorCallbacks() - ctx, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listerRegistry, provider, processorCallbacks, nil) + ctx, err := NewScaleTestAutoscalingContext(autoscalingOptions, &fake.Clientset{}, listerRegistry, provider, processorCallbacks, nil) assert.NoError(t, err) // Create CSR with unhealthy cluster protection effectively disabled, to guarantee we reach the tested logic. @@ -1459,7 +1461,7 @@ func TestStaticAutoscalerUpcomingScaleDownCandidates(t *testing.T) { csr := clusterstate.NewClusterStateRegistry(provider, csrConfig, ctx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute})) // Setting the Actuator is necessary for testing any scale-down logic, it shouldn't have anything to do in this test. - actuator := actuation.NewActuator(&ctx, csr, deletiontracker.NewNodeDeletionTracker(0*time.Second), simulator.NodeDeleteOptions{}, NewTestProcessors(&ctx).NodeGroupConfigProcessor) + actuator := actuation.NewActuator(&ctx, csr, deletiontracker.NewNodeDeletionTracker(0*time.Second), options.NodeDeleteOptions{}, nil, NewTestProcessors(&ctx).NodeGroupConfigProcessor) ctx.ScaleDownActuator = actuator // Fake planner that keeps track of the scale-down candidates passed to UpdateClusterState. @@ -1847,15 +1849,14 @@ func newScaleDownPlannerAndActuator(t *testing.T, ctx *context.AutoscalingContex ctx.MaxDrainParallelism = 1 ctx.NodeDeletionBatcherInterval = 0 * time.Second ctx.NodeDeleteDelayAfterTaint = 1 * time.Second - deleteOptions := simulator.NodeDeleteOptions{ + deleteOptions := options.NodeDeleteOptions{ SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, - MinReplicaCount: 0, SkipNodesWithCustomControllerPods: true, } ndt := deletiontracker.NewNodeDeletionTracker(0 * time.Second) - sd := legacy.NewScaleDown(ctx, p, ndt, deleteOptions) - actuator := actuation.NewActuator(ctx, cs, ndt, deleteOptions, p.NodeGroupConfigProcessor) + sd := legacy.NewScaleDown(ctx, p, ndt, deleteOptions, nil) + actuator := actuation.NewActuator(ctx, cs, ndt, deleteOptions, nil, p.NodeGroupConfigProcessor) wrapper := legacy.NewScaleDownWrapper(sd, actuator) return wrapper, wrapper } diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index ab21e8f0fae6..ec89b8dece40 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -29,10 +29,6 @@ import ( "syscall" "time" - "k8s.io/autoscaler/cluster-autoscaler/debuggingsnapshot" - "k8s.io/autoscaler/cluster-autoscaler/simulator" - "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" - "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,6 +40,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/config" "k8s.io/autoscaler/cluster-autoscaler/core" "k8s.io/autoscaler/cluster-autoscaler/core/podlistprocessor" + "k8s.io/autoscaler/cluster-autoscaler/debuggingsnapshot" "k8s.io/autoscaler/cluster-autoscaler/estimator" "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/metrics" @@ -54,6 +51,9 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates/emptycandidates" "k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates/previouscandidates" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" + "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" scheduler_util "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" @@ -67,7 +67,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" kube_flag "k8s.io/component-base/cli/flag" componentbaseconfig "k8s.io/component-base/config" - "k8s.io/component-base/config/options" + componentopts "k8s.io/component-base/config/options" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" _ "k8s.io/component-base/logs/json/register" @@ -449,7 +449,7 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter if err != nil { return nil, err } - deleteOptions := simulator.NewNodeDeleteOptions(autoscalingOptions) + deleteOptions := options.NewNodeDeleteOptions(autoscalingOptions) opts := core.AutoscalerOptions{ AutoscalingOptions: autoscalingOptions, @@ -469,7 +469,7 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter if autoscalingOptions.ParallelDrain { sdCandidatesSorting := previouscandidates.NewPreviousCandidates() scaleDownCandidatesComparers = []scaledowncandidates.CandidatesComparer{ - emptycandidates.NewEmptySortingProcessor(emptycandidates.NewNodeInfoGetter(opts.ClusterSnapshot), deleteOptions), + emptycandidates.NewEmptySortingProcessor(emptycandidates.NewNodeInfoGetter(opts.ClusterSnapshot), deleteOptions, rules.Default()), sdCandidatesSorting, } opts.Processors.ScaleDownCandidatesNotifier.Register(sdCandidatesSorting) @@ -563,7 +563,7 @@ func main() { leaderElection := defaultLeaderElectionConfiguration() leaderElection.LeaderElect = true - options.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) + componentopts.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine) featureGate := utilfeature.DefaultMutableFeatureGate loggingConfig := logsapi.NewLoggingConfiguration() diff --git a/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting.go b/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting.go index 38dbeec3071a..8ad745648c2a 100644 --- a/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting.go +++ b/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting.go @@ -22,6 +22,8 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -45,12 +47,17 @@ func NewNodeInfoGetter(c clustersnapshot.ClusterSnapshot) *nodeInfoGetterImpl { // EmptySorting is sorting scale down candidates so that empty nodes appear first. type EmptySorting struct { nodeInfoGetter - deleteOptions simulator.NodeDeleteOptions + deleteOptions options.NodeDeleteOptions + drainabilityRules rules.Rules } // NewEmptySortingProcessor return EmptySorting struct. -func NewEmptySortingProcessor(n nodeInfoGetter, deleteOptions simulator.NodeDeleteOptions) *EmptySorting { - return &EmptySorting{n, deleteOptions} +func NewEmptySortingProcessor(n nodeInfoGetter, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules) *EmptySorting { + return &EmptySorting{ + nodeInfoGetter: n, + deleteOptions: deleteOptions, + drainabilityRules: drainabilityRules, + } } // ScaleDownEarlierThan return true if node1 is empty and node2 isn't. @@ -66,7 +73,7 @@ func (p *EmptySorting) isNodeEmpty(node *apiv1.Node) bool { if err != nil { return false } - podsToRemove, _, _, err := simulator.GetPodsToMove(nodeInfo, p.deleteOptions, nil, nil, time.Now()) + podsToRemove, _, _, err := simulator.GetPodsToMove(nodeInfo, p.deleteOptions, p.drainabilityRules, nil, nil, time.Now()) if err == nil && len(podsToRemove) == 0 { return true } diff --git a/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting_test.go b/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting_test.go index 4aedcd7ea15c..469ddcd81382 100644 --- a/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting_test.go +++ b/cluster-autoscaler/processors/scaledowncandidates/emptycandidates/empty_candidates_sorting_test.go @@ -21,7 +21,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" . "k8s.io/autoscaler/cluster-autoscaler/utils/test" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -62,13 +62,15 @@ func TestScaleDownEarlierThan(t *testing.T) { niGetter := testNodeInfoGetter{map[string]*schedulerframework.NodeInfo{nodeEmptyName: niEmpty, nodeNonEmptyName: niNonEmpty, nodeEmptyName2: niEmpty2}} - deleteOptions := simulator.NodeDeleteOptions{ + deleteOptions := options.NodeDeleteOptions{ SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, - MinReplicaCount: 0, SkipNodesWithCustomControllerPods: true, } - p := EmptySorting{&niGetter, deleteOptions} + p := EmptySorting{ + nodeInfoGetter: &niGetter, + deleteOptions: deleteOptions, + } tests := []struct { name string @@ -95,22 +97,19 @@ func TestScaleDownEarlierThan(t *testing.T) { wantEarlier: true, }, { - name: "Non-empty node is not earlier that node without nodeInfo", - node1: nodeNonEmpty, - node2: noNodeInfoNode, - wantEarlier: false, + name: "Non-empty node is not earlier that node without nodeInfo", + node1: nodeNonEmpty, + node2: noNodeInfoNode, }, { - name: "Node without nodeInfo is not earlier that non-empty node", - node1: noNodeInfoNode, - node2: nodeNonEmpty, - wantEarlier: false, + name: "Node without nodeInfo is not earlier that non-empty node", + node1: noNodeInfoNode, + node2: nodeNonEmpty, }, { - name: "Empty node is not earlier that another empty node", - node1: nodeEmpty, - node2: nodeEmpty2, - wantEarlier: false, + name: "Empty node is not earlier that another empty node", + node1: nodeEmpty, + node2: nodeEmpty2, }, } for _, test := range tests { diff --git a/cluster-autoscaler/simulator/cluster.go b/cluster-autoscaler/simulator/cluster.go index 23ccd037e2fa..ed19fddb5be3 100644 --- a/cluster-autoscaler/simulator/cluster.go +++ b/cluster-autoscaler/simulator/cluster.go @@ -21,6 +21,8 @@ import ( "time" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling" "k8s.io/autoscaler/cluster-autoscaler/utils/drain" @@ -95,19 +97,21 @@ type RemovalSimulator struct { clusterSnapshot clustersnapshot.ClusterSnapshot usageTracker *UsageTracker canPersist bool - deleteOptions NodeDeleteOptions + deleteOptions options.NodeDeleteOptions + drainabilityRules rules.Rules schedulingSimulator *scheduling.HintingSimulator } // NewRemovalSimulator returns a new RemovalSimulator. func NewRemovalSimulator(listers kube_util.ListerRegistry, clusterSnapshot clustersnapshot.ClusterSnapshot, predicateChecker predicatechecker.PredicateChecker, - usageTracker *UsageTracker, deleteOptions NodeDeleteOptions, persistSuccessfulSimulations bool) *RemovalSimulator { + usageTracker *UsageTracker, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules, persistSuccessfulSimulations bool) *RemovalSimulator { return &RemovalSimulator{ listers: listers, clusterSnapshot: clusterSnapshot, usageTracker: usageTracker, canPersist: persistSuccessfulSimulations, deleteOptions: deleteOptions, + drainabilityRules: drainabilityRules, schedulingSimulator: scheduling.NewHintingSimulator(predicateChecker), } } @@ -159,7 +163,7 @@ func (r *RemovalSimulator) SimulateNodeRemoval( return nil, &UnremovableNode{Node: nodeInfo.Node(), Reason: UnexpectedError} } - podsToRemove, daemonSetPods, blockingPod, err := GetPodsToMove(nodeInfo, r.deleteOptions, r.listers, pdbs, timestamp) + podsToRemove, daemonSetPods, blockingPod, err := GetPodsToMove(nodeInfo, r.deleteOptions, r.drainabilityRules, r.listers, pdbs, timestamp) if err != nil { klog.V(2).Infof("node %s cannot be removed: %v", nodeName, err) if blockingPod != nil { @@ -193,7 +197,7 @@ func (r *RemovalSimulator) FindEmptyNodesToRemove(candidates []string, timestamp continue } // Should block on all pods - podsToRemove, _, _, err := GetPodsToMove(nodeInfo, r.deleteOptions, nil, nil, timestamp) + podsToRemove, _, _, err := GetPodsToMove(nodeInfo, r.deleteOptions, r.drainabilityRules, nil, nil, timestamp) if err == nil && len(podsToRemove) == 0 { result = append(result, node) } diff --git a/cluster-autoscaler/simulator/cluster_test.go b/cluster-autoscaler/simulator/cluster_test.go index 9a8b745466be..8f035a9fa454 100644 --- a/cluster-autoscaler/simulator/cluster_test.go +++ b/cluster-autoscaler/simulator/cluster_test.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" "k8s.io/autoscaler/cluster-autoscaler/utils/drain" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -59,7 +60,7 @@ func TestFindEmptyNodes(t *testing.T) { clusterSnapshot := clustersnapshot.NewBasicClusterSnapshot() clustersnapshot.InitializeClusterSnapshotOrDie(t, clusterSnapshot, []*apiv1.Node{nodes[0], nodes[1], nodes[2], nodes[3]}, []*apiv1.Pod{pod1, pod2}) testTime := time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) - r := NewRemovalSimulator(nil, clusterSnapshot, nil, nil, testDeleteOptions(), false) + r := NewRemovalSimulator(nil, clusterSnapshot, nil, nil, testDeleteOptions(), nil, false) emptyNodes := r.FindEmptyNodesToRemove(nodeNames, testTime) assert.Equal(t, []string{nodeNames[0], nodeNames[2], nodeNames[3]}, emptyNodes) } @@ -206,7 +207,7 @@ func TestFindNodesToRemove(t *testing.T) { destinations = append(destinations, node.Name) } clustersnapshot.InitializeClusterSnapshotOrDie(t, clusterSnapshot, test.allNodes, test.pods) - r := NewRemovalSimulator(registry, clusterSnapshot, predicateChecker, tracker, testDeleteOptions(), false) + r := NewRemovalSimulator(registry, clusterSnapshot, predicateChecker, tracker, testDeleteOptions(), nil, false) toRemove, unremovable := r.FindNodesToRemove(test.candidates, destinations, time.Now(), []*policyv1.PodDisruptionBudget{}) fmt.Printf("Test scenario: %s, found len(toRemove)=%v, expected len(test.toRemove)=%v\n", test.name, len(toRemove), len(test.toRemove)) assert.Equal(t, toRemove, test.toRemove) @@ -215,11 +216,10 @@ func TestFindNodesToRemove(t *testing.T) { } } -func testDeleteOptions() NodeDeleteOptions { - return NodeDeleteOptions{ +func testDeleteOptions() options.NodeDeleteOptions { + return options.NodeDeleteOptions{ SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, - MinReplicaCount: 0, SkipNodesWithCustomControllerPods: true, } } diff --git a/cluster-autoscaler/simulator/drain.go b/cluster-autoscaler/simulator/drain.go index 1e9b62f2653e..b25323da98eb 100644 --- a/cluster-autoscaler/simulator/drain.go +++ b/cluster-autoscaler/simulator/drain.go @@ -24,41 +24,15 @@ import ( policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/autoscaler/cluster-autoscaler/config" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/utils/drain" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" pod_util "k8s.io/autoscaler/cluster-autoscaler/utils/pod" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) -// NodeDeleteOptions contains various options to customize how draining will behave -type NodeDeleteOptions struct { - // SkipNodesWithSystemPods tells if nodes with pods from kube-system should be deleted (except for DaemonSet or mirror pods) - SkipNodesWithSystemPods bool - // SkipNodesWithLocalStorage tells if nodes with pods with local storage, e.g. EmptyDir or HostPath, should be deleted - SkipNodesWithLocalStorage bool - // SkipNodesWithCustomControllerPods tells if nodes with custom-controller owned pods should be skipped from deletion (skip if 'true') - SkipNodesWithCustomControllerPods bool - // MinReplicaCount controls the minimum number of replicas that a replica set or replication controller should have - // to allow their pods deletion in scale down - MinReplicaCount int - // DrainabilityRules contain a list of checks that are used to verify whether a pod can be drained from node. - DrainabilityRules rules.Rules -} - -// NewNodeDeleteOptions returns new node delete options extracted from autoscaling options -func NewNodeDeleteOptions(opts config.AutoscalingOptions) NodeDeleteOptions { - return NodeDeleteOptions{ - SkipNodesWithSystemPods: opts.SkipNodesWithSystemPods, - SkipNodesWithLocalStorage: opts.SkipNodesWithLocalStorage, - MinReplicaCount: opts.MinReplicaCount, - SkipNodesWithCustomControllerPods: opts.SkipNodesWithCustomControllerPods, - DrainabilityRules: rules.Default(), - } -} - // GetPodsToMove returns a list of pods that should be moved elsewhere // and a list of DaemonSet pods that should be evicted if the node // is drained. Raises error if there is an unreplicated pod. @@ -67,15 +41,14 @@ func NewNodeDeleteOptions(opts config.AutoscalingOptions) NodeDeleteOptions { // If listers is not nil it checks whether RC, DS, Jobs and RS that created these pods // still exist. // TODO(x13n): Rewrite GetPodsForDeletionOnNodeDrain into a set of DrainabilityRules. -func GetPodsToMove(nodeInfo *schedulerframework.NodeInfo, deleteOptions NodeDeleteOptions, listers kube_util.ListerRegistry, - pdbs []*policyv1.PodDisruptionBudget, timestamp time.Time) (pods []*apiv1.Pod, daemonSetPods []*apiv1.Pod, blockingPod *drain.BlockingPod, err error) { +func GetPodsToMove(nodeInfo *schedulerframework.NodeInfo, deleteOptions options.NodeDeleteOptions, drainabilityRules rules.Rules, listers kube_util.ListerRegistry, pdbs []*policyv1.PodDisruptionBudget, timestamp time.Time) (pods []*apiv1.Pod, daemonSetPods []*apiv1.Pod, blockingPod *drain.BlockingPod, err error) { var drainPods, drainDs []*apiv1.Pod - drainabilityRules := deleteOptions.DrainabilityRules if drainabilityRules == nil { drainabilityRules = rules.Default() } drainCtx := &drainability.DrainContext{ - Pdbs: pdbs, + Pdbs: pdbs, + DeleteOptions: deleteOptions, } for _, podInfo := range nodeInfo.Pods { pod := podInfo.Pod diff --git a/cluster-autoscaler/simulator/drain_test.go b/cluster-autoscaler/simulator/drain_test.go index f20cfe492536..3000a912dd74 100644 --- a/cluster-autoscaler/simulator/drain_test.go +++ b/cluster-autoscaler/simulator/drain_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability/rules" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" "k8s.io/autoscaler/cluster-autoscaler/utils/drain" . "k8s.io/autoscaler/cluster-autoscaler/utils/test" "k8s.io/kubernetes/pkg/kubelet/types" @@ -305,14 +306,12 @@ func TestGetPodsToMove(t *testing.T) { } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - deleteOptions := NodeDeleteOptions{ + deleteOptions := options.NodeDeleteOptions{ SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, - MinReplicaCount: 0, SkipNodesWithCustomControllerPods: true, - DrainabilityRules: tc.rules, } - p, d, b, err := GetPodsToMove(schedulerframework.NewNodeInfo(tc.pods...), deleteOptions, nil, tc.pdbs, testTime) + p, d, b, err := GetPodsToMove(schedulerframework.NewNodeInfo(tc.pods...), deleteOptions, tc.rules, nil, tc.pdbs, testTime) if tc.wantErr { assert.Error(t, err) } else { diff --git a/cluster-autoscaler/simulator/drainability/context.go b/cluster-autoscaler/simulator/drainability/context.go index 80bfa8cf9782..a36d02578d52 100644 --- a/cluster-autoscaler/simulator/drainability/context.go +++ b/cluster-autoscaler/simulator/drainability/context.go @@ -18,9 +18,11 @@ package drainability import ( policyv1 "k8s.io/api/policy/v1" + "k8s.io/autoscaler/cluster-autoscaler/simulator/options" ) // DrainContext contains parameters for drainability rules. type DrainContext struct { - Pdbs []*policyv1.PodDisruptionBudget + Pdbs []*policyv1.PodDisruptionBudget + DeleteOptions options.NodeDeleteOptions } diff --git a/cluster-autoscaler/simulator/options/nodedelete.go b/cluster-autoscaler/simulator/options/nodedelete.go new file mode 100644 index 000000000000..947095d6eb78 --- /dev/null +++ b/cluster-autoscaler/simulator/options/nodedelete.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "k8s.io/autoscaler/cluster-autoscaler/config" +) + +// NodeDeleteOptions contains various options to customize how draining will behave +type NodeDeleteOptions struct { + // SkipNodesWithSystemPods is true if nodes with kube-system pods should be + // deleted (except for DaemonSet or mirror pods). + SkipNodesWithSystemPods bool + // SkipNodesWithLocalStorage is true if nodes with pods using local storage + // (e.g. EmptyDir or HostPath) should be deleted. + SkipNodesWithLocalStorage bool + // SkipNodesWithCustomControllerPods is true if nodes with + // custom-controller-owned pods should be skipped. + SkipNodesWithCustomControllerPods bool + // MinReplicaCount determines the minimum number of replicas that a replica + // set or replication controller should have to allow pod deletion during + // scale down. + MinReplicaCount int +} + +// NewNodeDeleteOptions returns new node delete options extracted from autoscaling options. +func NewNodeDeleteOptions(opts config.AutoscalingOptions) NodeDeleteOptions { + return NodeDeleteOptions{ + SkipNodesWithSystemPods: opts.SkipNodesWithSystemPods, + SkipNodesWithLocalStorage: opts.SkipNodesWithLocalStorage, + MinReplicaCount: opts.MinReplicaCount, + SkipNodesWithCustomControllerPods: opts.SkipNodesWithCustomControllerPods, + } +}