From e60f525ec619147dc6f6b97f8e8d4ab027810188 Mon Sep 17 00:00:00 2001 From: zhifei92 Date: Fri, 28 Jun 2024 11:23:02 +0800 Subject: [PATCH] feat: support MaxNoOfPodsToEvictTotal --- pkg/api/types.go | 3 ++ pkg/api/v1alpha1/types.go | 3 ++ pkg/api/v1alpha1/zz_generated.deepcopy.go | 5 +++ pkg/api/v1alpha2/types.go | 3 ++ pkg/api/v1alpha2/zz_generated.conversion.go | 2 ++ pkg/api/v1alpha2/zz_generated.deepcopy.go | 5 +++ pkg/api/zz_generated.deepcopy.go | 5 +++ pkg/descheduler/descheduler.go | 1 + pkg/descheduler/evictions/errors.go | 12 +++++++ pkg/descheduler/evictions/evictions.go | 33 ++++++++++++------- pkg/descheduler/evictions/options.go | 6 ++++ .../lownodeutilization_test.go | 31 ++++++++++++++--- .../nodeutilization/nodeutilization.go | 17 +++++++--- .../plugins/podlifetime/pod_lifetime.go | 2 ++ .../plugins/podlifetime/pod_lifetime_test.go | 15 ++++++++- .../removeduplicates/removeduplicates.go | 2 ++ .../plugins/removefailedpods/failedpods.go | 2 ++ .../toomanyrestarts.go | 2 ++ .../pod_antiaffinity.go | 2 ++ .../pod_antiaffinity_test.go | 12 ++++++- .../node_affinity.go | 2 ++ .../node_affinity_test.go | 15 ++++++++- .../node_taint.go | 2 ++ .../node_taint_test.go | 25 ++++++++++++-- .../topologyspreadconstraint.go | 2 ++ 25 files changed, 184 insertions(+), 25 deletions(-) diff --git a/pkg/api/types.go b/pkg/api/types.go index f874c3dd8..c729b5e2a 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -38,6 +38,9 @@ type DeschedulerPolicy struct { // MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace. MaxNoOfPodsToEvictPerNamespace *uint + + // MaxNoOfPodsToTotal restricts maximum of pods to be evicted total. + MaxNoOfPodsToEvictTotal *uint } // Namespaces carries a list of included/excluded namespaces diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index fb0ccc28a..11011e968 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -52,6 +52,9 @@ type DeschedulerPolicy struct { // MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace. MaxNoOfPodsToEvictPerNamespace *uint `json:"maxNoOfPodsToEvictPerNamespace,omitempty"` + + // MaxNoOfPodsToTotal restricts maximum of pods to be evicted total. + MaxNoOfPodsToEvictTotal *uint `json:"maxNoOfPodsToEvictTotal,omitempty"` } type ( diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index fb57c89f7..33d1e39f6 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -77,6 +77,11 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) { *out = new(uint) **out = **in } + if in.MaxNoOfPodsToEvictTotal != nil { + in, out := &in.MaxNoOfPodsToEvictTotal, &out.MaxNoOfPodsToEvictTotal + *out = new(uint) + **out = **in + } return } diff --git a/pkg/api/v1alpha2/types.go b/pkg/api/v1alpha2/types.go index 72c7949d2..d663efb86 100644 --- a/pkg/api/v1alpha2/types.go +++ b/pkg/api/v1alpha2/types.go @@ -37,6 +37,9 @@ type DeschedulerPolicy struct { // MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace. MaxNoOfPodsToEvictPerNamespace *uint `json:"maxNoOfPodsToEvictPerNamespace,omitempty"` + + // MaxNoOfPodsToTotal restricts maximum of pods to be evicted total. + MaxNoOfPodsToEvictTotal *uint `json:"maxNoOfPodsToEvictTotal,omitempty"` } type DeschedulerProfile struct { diff --git a/pkg/api/v1alpha2/zz_generated.conversion.go b/pkg/api/v1alpha2/zz_generated.conversion.go index c40500043..a13151ed2 100644 --- a/pkg/api/v1alpha2/zz_generated.conversion.go +++ b/pkg/api/v1alpha2/zz_generated.conversion.go @@ -104,6 +104,7 @@ func autoConvert_v1alpha2_DeschedulerPolicy_To_api_DeschedulerPolicy(in *Desched out.NodeSelector = (*string)(unsafe.Pointer(in.NodeSelector)) out.MaxNoOfPodsToEvictPerNode = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictPerNode)) out.MaxNoOfPodsToEvictPerNamespace = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictPerNamespace)) + out.MaxNoOfPodsToEvictTotal = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictTotal)) return nil } @@ -122,6 +123,7 @@ func autoConvert_api_DeschedulerPolicy_To_v1alpha2_DeschedulerPolicy(in *api.Des out.NodeSelector = (*string)(unsafe.Pointer(in.NodeSelector)) out.MaxNoOfPodsToEvictPerNode = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictPerNode)) out.MaxNoOfPodsToEvictPerNamespace = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictPerNamespace)) + out.MaxNoOfPodsToEvictTotal = (*uint)(unsafe.Pointer(in.MaxNoOfPodsToEvictTotal)) return nil } diff --git a/pkg/api/v1alpha2/zz_generated.deepcopy.go b/pkg/api/v1alpha2/zz_generated.deepcopy.go index af73d5023..a0129fd8e 100644 --- a/pkg/api/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha2/zz_generated.deepcopy.go @@ -51,6 +51,11 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) { *out = new(uint) **out = **in } + if in.MaxNoOfPodsToEvictTotal != nil { + in, out := &in.MaxNoOfPodsToEvictTotal, &out.MaxNoOfPodsToEvictTotal + *out = new(uint) + **out = **in + } return } diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index c4abcbf94..2f84cf1ed 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -51,6 +51,11 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) { *out = new(uint) **out = **in } + if in.MaxNoOfPodsToEvictTotal != nil { + in, out := &in.MaxNoOfPodsToEvictTotal, &out.MaxNoOfPodsToEvictTotal + *out = new(uint) + **out = **in + } return } diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index a60667e66..9b685d140 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -102,6 +102,7 @@ func newDescheduler(rs *options.DeschedulerServer, deschedulerPolicy *api.Desche WithPolicyGroupVersion(evictionPolicyGroupVersion). WithMaxPodsToEvictPerNode(deschedulerPolicy.MaxNoOfPodsToEvictPerNode). WithMaxPodsToEvictPerNamespace(deschedulerPolicy.MaxNoOfPodsToEvictPerNamespace). + WithMaxPodsToEvictTotal(deschedulerPolicy.MaxNoOfPodsToEvictTotal). WithDryRun(rs.DryRun). WithMetricsEnabled(!rs.DisableMetrics), ) diff --git a/pkg/descheduler/evictions/errors.go b/pkg/descheduler/evictions/errors.go index e8323ce78..1b5c40f69 100644 --- a/pkg/descheduler/evictions/errors.go +++ b/pkg/descheduler/evictions/errors.go @@ -31,3 +31,15 @@ func NewEvictionNamespaceLimitError(namespace string) *EvictionNamespaceLimitErr } var _ error = &EvictionNamespaceLimitError{} + +type EvictionTotalLimitError struct{} + +func (e EvictionTotalLimitError) Error() string { + return "maximum number of evicted pods total reached" +} + +func NewEvictionTotalLimitError() *EvictionTotalLimitError { + return &EvictionTotalLimitError{} +} + +var _ error = &EvictionTotalLimitError{} diff --git a/pkg/descheduler/evictions/evictions.go b/pkg/descheduler/evictions/evictions.go index be4b5ba6e..0149bb5ae 100644 --- a/pkg/descheduler/evictions/evictions.go +++ b/pkg/descheduler/evictions/evictions.go @@ -47,8 +47,10 @@ type PodEvictor struct { dryRun bool maxPodsToEvictPerNode *uint maxPodsToEvictPerNamespace *uint - nodepodCount nodePodEvictedCount + maxPodsToEvictTotal *uint + nodePodCount nodePodEvictedCount namespacePodCount namespacePodEvictCount + totalPodCount uint metricsEnabled bool eventRecorder events.EventRecorder } @@ -69,29 +71,27 @@ func NewPodEvictor( dryRun: options.dryRun, maxPodsToEvictPerNode: options.maxPodsToEvictPerNode, maxPodsToEvictPerNamespace: options.maxPodsToEvictPerNamespace, + maxPodsToEvictTotal: options.maxPodsToEvictTotal, metricsEnabled: options.metricsEnabled, - nodepodCount: make(nodePodEvictedCount), + nodePodCount: make(nodePodEvictedCount), namespacePodCount: make(namespacePodEvictCount), } } // NodeEvicted gives a number of pods evicted for node func (pe *PodEvictor) NodeEvicted(node *v1.Node) uint { - return pe.nodepodCount[node.Name] + return pe.nodePodCount[node.Name] } // TotalEvicted gives a number of pods evicted through all nodes func (pe *PodEvictor) TotalEvicted() uint { - var total uint - for _, count := range pe.nodepodCount { - total += count - } - return total + return pe.totalPodCount } func (pe *PodEvictor) ResetCounters() { - pe.nodepodCount = make(nodePodEvictedCount) + pe.nodePodCount = make(nodePodEvictedCount) pe.namespacePodCount = make(namespacePodEvictCount) + pe.totalPodCount = 0 } func (pe *PodEvictor) SetClient(client clientset.Interface) { @@ -115,8 +115,18 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, opts EvictOptio ctx, span = tracing.Tracer().Start(ctx, "EvictPod", trace.WithAttributes(attribute.String("podName", pod.Name), attribute.String("podNamespace", pod.Namespace), attribute.String("reason", opts.Reason), attribute.String("operation", tracing.EvictOperation))) defer span.End() + if pe.maxPodsToEvictTotal != nil && pe.totalPodCount+1 > *pe.maxPodsToEvictTotal { + err := NewEvictionTotalLimitError() + if pe.metricsEnabled { + metrics.PodsEvicted.With(map[string]string{"result": err.Error(), "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc() + } + span.AddEvent("Eviction Failed", trace.WithAttributes(attribute.String("node", pod.Spec.NodeName), attribute.String("err", err.Error()))) + klog.ErrorS(err, "Error evicting pod", "limit", *pe.maxPodsToEvictTotal) + return err + } + if pod.Spec.NodeName != "" { - if pe.maxPodsToEvictPerNode != nil && pe.nodepodCount[pod.Spec.NodeName]+1 > *pe.maxPodsToEvictPerNode { + if pe.maxPodsToEvictPerNode != nil && pe.nodePodCount[pod.Spec.NodeName]+1 > *pe.maxPodsToEvictPerNode { err := NewEvictionNodeLimitError(pod.Spec.NodeName) if pe.metricsEnabled { metrics.PodsEvicted.With(map[string]string{"result": err.Error(), "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc() @@ -149,9 +159,10 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, opts EvictOptio } if pod.Spec.NodeName != "" { - pe.nodepodCount[pod.Spec.NodeName]++ + pe.nodePodCount[pod.Spec.NodeName]++ } pe.namespacePodCount[pod.Namespace]++ + pe.totalPodCount++ if pe.metricsEnabled { metrics.PodsEvicted.With(map[string]string{"result": "success", "strategy": opts.StrategyName, "namespace": pod.Namespace, "node": pod.Spec.NodeName, "profile": opts.ProfileName}).Inc() diff --git a/pkg/descheduler/evictions/options.go b/pkg/descheduler/evictions/options.go index 6bcac58c2..267854ee4 100644 --- a/pkg/descheduler/evictions/options.go +++ b/pkg/descheduler/evictions/options.go @@ -9,6 +9,7 @@ type Options struct { dryRun bool maxPodsToEvictPerNode *uint maxPodsToEvictPerNamespace *uint + maxPodsToEvictTotal *uint metricsEnabled bool } @@ -39,6 +40,11 @@ func (o *Options) WithMaxPodsToEvictPerNamespace(maxPodsToEvictPerNamespace *uin return o } +func (o *Options) WithMaxPodsToEvictTotal(maxPodsToEvictTotal *uint) *Options { + o.maxPodsToEvictTotal = maxPodsToEvictTotal + return o +} + func (o *Options) WithMetricsEnabled(metricsEnabled bool) *Options { o.metricsEnabled = metricsEnabled return o diff --git a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go index 6c8f656e9..ab167b667 100644 --- a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go +++ b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go @@ -979,11 +979,14 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) { }, } + var uint0, uint1 uint = 0, 1 tests := []struct { - name string - nodes []*v1.Node - pods []*v1.Pod - evictionsExpected uint + name string + nodes []*v1.Node + pods []*v1.Pod + maxPodsToEvictPerNode *uint + maxPodsToEvictTotal *uint + evictionsExpected uint }{ { name: "No taints", @@ -1039,6 +1042,26 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) { }, evictionsExpected: 1, }, + { + name: "Pod which tolerates node taint, set maxPodsToEvictTotal(0), should not be expelled", + nodes: []*v1.Node{n1, n3withTaints}, + pods: []*v1.Pod{ + // Node 1 pods + test.BuildTestPod(fmt.Sprintf("pod_1_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_2_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_3_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_4_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_5_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_6_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + test.BuildTestPod(fmt.Sprintf("pod_7_%s", n1.Name), 200, 0, n1.Name, test.SetRSOwnerRef), + podThatToleratesTaint, + // Node 3 pods + test.BuildTestPod(fmt.Sprintf("pod_9_%s", n3withTaints.Name), 200, 0, n3withTaints.Name, test.SetRSOwnerRef), + }, + maxPodsToEvictPerNode: &uint1, + maxPodsToEvictTotal: &uint0, + evictionsExpected: 0, + }, } for _, item := range tests { diff --git a/pkg/framework/plugins/nodeutilization/nodeutilization.go b/pkg/framework/plugins/nodeutilization/nodeutilization.go index ada8c125c..b3a352aad 100644 --- a/pkg/framework/plugins/nodeutilization/nodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/nodeutilization.go @@ -274,8 +274,14 @@ func evictPodsFromSourceNodes( klog.V(1).InfoS("Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers") // sort the evictable Pods based on priority. This also sorts them based on QoS. If there are multiple pods with same priority, they are sorted based on QoS tiers. podutil.SortPodsBasedOnPriorityLowToHigh(removablePods) - evictPods(ctx, evictableNamespaces, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, evictOptions, continueEviction) - + err := evictPods(ctx, evictableNamespaces, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, evictOptions, continueEviction) + if err != nil { + switch err.(type) { + case *evictions.EvictionTotalLimitError: + return + default: + } + } } } @@ -289,7 +295,7 @@ func evictPods( podEvictor frameworktypes.Evictor, evictOptions evictions.EvictOptions, continueEviction continueEvictionCond, -) { +) error { var excludedNamespaces sets.Set[string] if evictableNamespaces != nil { excludedNamespaces = sets.New(evictableNamespaces.Exclude...) @@ -349,13 +355,14 @@ func evictPods( continue } switch err.(type) { - case *evictions.EvictionNodeLimitError: - return + case *evictions.EvictionNodeLimitError, *evictions.EvictionTotalLimitError: + return err default: klog.Errorf("eviction failed: %v", err) } } } + return nil } // sortNodesByUsage sorts nodes based on usage according to the given plugin. diff --git a/pkg/framework/plugins/podlifetime/pod_lifetime.go b/pkg/framework/plugins/podlifetime/pod_lifetime.go index f70c8886b..79f711f31 100644 --- a/pkg/framework/plugins/podlifetime/pod_lifetime.go +++ b/pkg/framework/plugins/podlifetime/pod_lifetime.go @@ -140,6 +140,8 @@ loop: switch err.(type) { case *evictions.EvictionNodeLimitError: continue loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go index 9b613c539..e0ca64843 100644 --- a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go +++ b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go @@ -156,6 +156,7 @@ func TestPodLifeTime(t *testing.T) { ignorePvcPods bool maxPodsToEvictPerNode *uint maxPodsToEvictPerNamespace *uint + maxPodsToEvictTotal *uint applyPodsFunc func(pods []*v1.Pod) }{ { @@ -314,6 +315,17 @@ func TestPodLifeTime(t *testing.T) { maxPodsToEvictPerNamespace: utilptr.To[uint](1), expectedEvictedPodCount: 1, }, + { + description: "1 Oldest pod should be evicted when maxPodsToEvictTotal is set to 1", + args: &PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + }, + pods: []*v1.Pod{p1, p2, p9}, + nodes: []*v1.Node{node1}, + maxPodsToEvictPerNamespace: utilptr.To[uint](2), + maxPodsToEvictTotal: utilptr.To[uint](1), + expectedEvictedPodCount: 1, + }, { description: "1 Oldest pod should be evicted when maxPodsToEvictPerNode is set to 1", args: &PodLifeTimeArgs{ @@ -560,7 +572,8 @@ func TestPodLifeTime(t *testing.T) { eventRecorder, evictions.NewOptions(). WithMaxPodsToEvictPerNode(tc.maxPodsToEvictPerNode). - WithMaxPodsToEvictPerNamespace(tc.maxPodsToEvictPerNamespace), + WithMaxPodsToEvictPerNamespace(tc.maxPodsToEvictPerNamespace). + WithMaxPodsToEvictTotal(tc.maxPodsToEvictTotal), ) defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ diff --git a/pkg/framework/plugins/removeduplicates/removeduplicates.go b/pkg/framework/plugins/removeduplicates/removeduplicates.go index c419c8da9..3d92ba8b0 100644 --- a/pkg/framework/plugins/removeduplicates/removeduplicates.go +++ b/pkg/framework/plugins/removeduplicates/removeduplicates.go @@ -217,6 +217,8 @@ func (r *RemoveDuplicates) Balance(ctx context.Context, nodes []*v1.Node) *frame switch err.(type) { case *evictions.EvictionNodeLimitError: continue loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removefailedpods/failedpods.go b/pkg/framework/plugins/removefailedpods/failedpods.go index 1376bce19..8b1badc26 100644 --- a/pkg/framework/plugins/removefailedpods/failedpods.go +++ b/pkg/framework/plugins/removefailedpods/failedpods.go @@ -111,6 +111,8 @@ func (d *RemoveFailedPods) Deschedule(ctx context.Context, nodes []*v1.Node) *fr switch err.(type) { case *evictions.EvictionNodeLimitError: break loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts.go b/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts.go index 9e3e2ea9a..fb9656971 100644 --- a/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts.go +++ b/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts.go @@ -131,6 +131,8 @@ func (d *RemovePodsHavingTooManyRestarts) Deschedule(ctx context.Context, nodes switch err.(type) { case *evictions.EvictionNodeLimitError: break loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity.go b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity.go index 3e9983564..476566a18 100644 --- a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity.go +++ b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity.go @@ -112,6 +112,8 @@ loop: switch err.(type) { case *evictions.EvictionNodeLimitError: continue loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go index ab81a8b3b..c6d434d04 100644 --- a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go +++ b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go @@ -121,6 +121,7 @@ func TestPodAntiAffinity(t *testing.T) { description string maxPodsToEvictPerNode *uint maxNoOfPodsToEvictPerNamespace *uint + maxNoOfPodsToEvictTotal *uint pods []*v1.Pod expectedEvictedPodCount uint nodeFit bool @@ -146,6 +147,14 @@ func TestPodAntiAffinity(t *testing.T) { nodes: []*v1.Node{node1}, expectedEvictedPodCount: 3, }, + { + description: "Maximum pods to evict (maxNoOfPodsToEvictTotal)", + maxNoOfPodsToEvictPerNamespace: &uint3, + maxNoOfPodsToEvictTotal: &uint1, + pods: []*v1.Pod{p1, p2, p3, p4}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 1, + }, { description: "Evict only 1 pod after sorting", pods: []*v1.Pod{p5, p6, p7}, @@ -237,7 +246,8 @@ func TestPodAntiAffinity(t *testing.T) { eventRecorder, evictions.NewOptions(). WithMaxPodsToEvictPerNode(test.maxPodsToEvictPerNode). - WithMaxPodsToEvictPerNamespace(test.maxNoOfPodsToEvictPerNamespace), + WithMaxPodsToEvictPerNamespace(test.maxNoOfPodsToEvictPerNamespace). + WithMaxPodsToEvictTotal(test.maxNoOfPodsToEvictTotal), ) defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ diff --git a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity.go b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity.go index 19c676e0e..586ce2218 100644 --- a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity.go +++ b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity.go @@ -144,6 +144,8 @@ func (d *RemovePodsViolatingNodeAffinity) processNodes(ctx context.Context, node switch err.(type) { case *evictions.EvictionNodeLimitError: break loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go index 0c660491e..200193a10 100644 --- a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go +++ b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go @@ -119,6 +119,7 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { expectedEvictedPodCount uint maxPodsToEvictPerNode *uint maxNoOfPodsToEvictPerNamespace *uint + maxNoOfPodsToEvictTotal *uint args RemovePodsViolatingNodeAffinityArgs nodefit bool }{ @@ -237,6 +238,17 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { nodes: []*v1.Node{nodeWithoutLabels, nodeWithLabels}, maxNoOfPodsToEvictPerNamespace: &uint1, }, + { + description: "Pod is scheduled on node without matching labels, another schedulable node available, maxNoOfPodsToEvictTotal set to 0, should not be evicted [required affinity]", + expectedEvictedPodCount: 0, + args: RemovePodsViolatingNodeAffinityArgs{ + NodeAffinityType: []string{"requiredDuringSchedulingIgnoredDuringExecution"}, + }, + pods: addPodsToNode(nodeWithoutLabels, nil, "requiredDuringSchedulingIgnoredDuringExecution"), + nodes: []*v1.Node{nodeWithoutLabels, nodeWithLabels}, + maxNoOfPodsToEvictPerNamespace: &uint1, + maxNoOfPodsToEvictTotal: &uint0, + }, { description: "Pod is scheduled on node without matching labels, another schedulable node available, maxNoOfPodsToEvictPerNamespace set to 1, should be evicted [preferred affinity]", expectedEvictedPodCount: 1, @@ -363,7 +375,8 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { eventRecorder, evictions.NewOptions(). WithMaxPodsToEvictPerNode(tc.maxPodsToEvictPerNode). - WithMaxPodsToEvictPerNamespace(tc.maxNoOfPodsToEvictPerNamespace), + WithMaxPodsToEvictPerNamespace(tc.maxNoOfPodsToEvictPerNamespace). + WithMaxPodsToEvictTotal(tc.maxNoOfPodsToEvictTotal), ) defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ diff --git a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go index 9736fe2f5..e447a4662 100644 --- a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go +++ b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go @@ -129,6 +129,8 @@ func (d *RemovePodsViolatingNodeTaints) Deschedule(ctx context.Context, nodes [] switch err.(type) { case *evictions.EvictionNodeLimitError: break loop + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) } diff --git a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go index 6cafcf5c5..b09177403 100644 --- a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go +++ b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go @@ -177,7 +177,7 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { p15 = addTolerationToPod(p15, "testTaint", "test", 1, v1.TaintEffectNoSchedule) p15 = addTolerationToPod(p15, "testingTaint", "testing", 1, v1.TaintEffectNoSchedule) - var uint1 uint = 1 + var uint1, uint2 uint = 1, 2 tests := []struct { description string @@ -187,6 +187,7 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { evictSystemCriticalPods bool maxPodsToEvictPerNode *uint maxNoOfPodsToEvictPerNamespace *uint + maxNoOfPodsToEvictTotal *uint expectedEvictedPodCount uint nodeFit bool includePreferNoSchedule bool @@ -209,6 +210,16 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { evictSystemCriticalPods: false, expectedEvictedPodCount: 1, // p4 gets evicted }, + { + description: "Only number of Pods not tolerating node taint should be evicted", + pods: []*v1.Pod{p1, p5, p6}, + nodes: []*v1.Node{node1}, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + maxPodsToEvictPerNode: &uint2, + maxNoOfPodsToEvictTotal: &uint1, + expectedEvictedPodCount: 1, // p5 or p6 gets evicted + }, { description: "Only number of Pods not tolerating node taint should be evicted", pods: []*v1.Pod{p1, p5, p6}, @@ -227,6 +238,15 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { maxNoOfPodsToEvictPerNamespace: &uint1, expectedEvictedPodCount: 1, // p5 or p6 gets evicted }, + { + description: "Only number of Pods not tolerating node taint should be evicted", + pods: []*v1.Pod{p1, p5, p6}, + nodes: []*v1.Node{node1}, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + maxNoOfPodsToEvictPerNamespace: &uint1, + expectedEvictedPodCount: 1, // p5 or p6 gets evicted + }, { description: "Critical pods not tolerating node taint should not be evicted", pods: []*v1.Pod{p7, p8, p9, p10}, @@ -408,7 +428,8 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { eventRecorder, evictions.NewOptions(). WithMaxPodsToEvictPerNode(tc.maxPodsToEvictPerNode). - WithMaxPodsToEvictPerNamespace(tc.maxNoOfPodsToEvictPerNamespace), + WithMaxPodsToEvictPerNamespace(tc.maxNoOfPodsToEvictPerNamespace). + WithMaxPodsToEvictTotal(tc.maxNoOfPodsToEvictTotal), ) defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ diff --git a/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint.go b/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint.go index 171a9b7ef..304c691b5 100644 --- a/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint.go +++ b/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint.go @@ -242,6 +242,8 @@ func (d *RemovePodsViolatingTopologySpreadConstraint) Balance(ctx context.Contex switch err.(type) { case *evictions.EvictionNodeLimitError: nodeLimitExceeded[pod.Spec.NodeName] = true + case *evictions.EvictionTotalLimitError: + return nil default: klog.Errorf("eviction failed: %v", err) }