diff --git a/pkg/api/types.go b/pkg/api/types.go index a586d0f93..d02aced2a 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -43,6 +43,9 @@ type DeschedulerPolicy struct { // MaxNoOfPodsToEvictPerNode restricts maximum of pods to be evicted per node. MaxNoOfPodsToEvictPerNode *uint + + // MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace. + MaxNoOfPodsToEvictPerNamespace *uint } type StrategyName string diff --git a/pkg/api/v1alpha1/types.go b/pkg/api/v1alpha1/types.go index 700afdaa6..db45f2389 100644 --- a/pkg/api/v1alpha1/types.go +++ b/pkg/api/v1alpha1/types.go @@ -43,6 +43,9 @@ type DeschedulerPolicy struct { // MaxNoOfPodsToEvictPerNode restricts maximum of pods to be evicted per node. MaxNoOfPodsToEvictPerNode *int `json:"maxNoOfPodsToEvictPerNode,omitempty"` + + // MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace. + MaxNoOfPodsToEvictPerNamespace *int `json:"maxNoOfPodsToEvictPerNamespace,omitempty"` } type StrategyName string diff --git a/pkg/api/v1alpha1/zz_generated.conversion.go b/pkg/api/v1alpha1/zz_generated.conversion.go index 05cc815f6..8e4aa990c 100644 --- a/pkg/api/v1alpha1/zz_generated.conversion.go +++ b/pkg/api/v1alpha1/zz_generated.conversion.go @@ -143,6 +143,13 @@ func autoConvert_v1alpha1_DeschedulerPolicy_To_api_DeschedulerPolicy(in *Desched } else { out.MaxNoOfPodsToEvictPerNode = nil } + if in.MaxNoOfPodsToEvictPerNamespace != nil { + in, out := &in.MaxNoOfPodsToEvictPerNamespace, &out.MaxNoOfPodsToEvictPerNamespace + *out = new(uint) + **out = uint(**in) + } else { + out.MaxNoOfPodsToEvictPerNamespace = nil + } return nil } @@ -164,6 +171,13 @@ func autoConvert_api_DeschedulerPolicy_To_v1alpha1_DeschedulerPolicy(in *api.Des } else { out.MaxNoOfPodsToEvictPerNode = nil } + if in.MaxNoOfPodsToEvictPerNamespace != nil { + in, out := &in.MaxNoOfPodsToEvictPerNamespace, &out.MaxNoOfPodsToEvictPerNamespace + *out = new(int) + **out = int(**in) + } else { + out.MaxNoOfPodsToEvictPerNamespace = nil + } return nil } diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index d78798d6f..839a6ef7a 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -62,6 +62,11 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) { *out = new(int) **out = **in } + if in.MaxNoOfPodsToEvictPerNamespace != nil { + in, out := &in.MaxNoOfPodsToEvictPerNamespace, &out.MaxNoOfPodsToEvictPerNamespace + *out = new(int) + **out = **in + } return } diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 8e850d8d0..2335974b1 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -62,6 +62,11 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) { *out = new(uint) **out = **in } + if in.MaxNoOfPodsToEvictPerNamespace != nil { + in, out := &in.MaxNoOfPodsToEvictPerNamespace, &out.MaxNoOfPodsToEvictPerNamespace + *out = new(uint) + **out = **in + } return } diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index f9b5cd4e0..a773e26f2 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -128,6 +128,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer evictionPolicyGroupVersion, rs.DryRun, deschedulerPolicy.MaxNoOfPodsToEvictPerNode, + deschedulerPolicy.MaxNoOfPodsToEvictPerNamespace, nodes, evictLocalStoragePods, evictSystemCriticalPods, diff --git a/pkg/descheduler/evictions/evictions.go b/pkg/descheduler/evictions/evictions.go index 02ab18317..c7449708b 100644 --- a/pkg/descheduler/evictions/evictions.go +++ b/pkg/descheduler/evictions/evictions.go @@ -46,17 +46,20 @@ const ( // nodePodEvictedCount keeps count of pods evicted on node type nodePodEvictedCount map[*v1.Node]uint +type namespacePodEvictCount map[string]uint type PodEvictor struct { - client clientset.Interface - nodes []*v1.Node - policyGroupVersion string - dryRun bool - maxPodsToEvictPerNode *uint - nodepodCount nodePodEvictedCount - evictLocalStoragePods bool - evictSystemCriticalPods bool - ignorePvcPods bool + client clientset.Interface + nodes []*v1.Node + policyGroupVersion string + dryRun bool + maxPodsToEvictPerNode *uint + maxPodsToEvictPerNamespace *uint + nodepodCount nodePodEvictedCount + namespacePodCount namespacePodEvictCount + evictLocalStoragePods bool + evictSystemCriticalPods bool + ignorePvcPods bool } func NewPodEvictor( @@ -64,27 +67,31 @@ func NewPodEvictor( policyGroupVersion string, dryRun bool, maxPodsToEvictPerNode *uint, + maxPodsToEvictPerNamespace *uint, nodes []*v1.Node, evictLocalStoragePods bool, evictSystemCriticalPods bool, ignorePvcPods bool, ) *PodEvictor { var nodePodCount = make(nodePodEvictedCount) + var namespacePodCount = make(namespacePodEvictCount) for _, node := range nodes { // Initialize podsEvicted till now with 0. nodePodCount[node] = 0 } return &PodEvictor{ - client: client, - nodes: nodes, - policyGroupVersion: policyGroupVersion, - dryRun: dryRun, - maxPodsToEvictPerNode: maxPodsToEvictPerNode, - nodepodCount: nodePodCount, - evictLocalStoragePods: evictLocalStoragePods, - evictSystemCriticalPods: evictSystemCriticalPods, - ignorePvcPods: ignorePvcPods, + client: client, + nodes: nodes, + policyGroupVersion: policyGroupVersion, + dryRun: dryRun, + maxPodsToEvictPerNode: maxPodsToEvictPerNode, + maxPodsToEvictPerNamespace: maxPodsToEvictPerNamespace, + nodepodCount: nodePodCount, + namespacePodCount: namespacePodCount, + evictLocalStoragePods: evictLocalStoragePods, + evictSystemCriticalPods: evictSystemCriticalPods, + ignorePvcPods: ignorePvcPods, } } @@ -111,10 +118,15 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, node *v1.Node, reason += " (" + strings.Join(reasons, ", ") + ")" } if pe.maxPodsToEvictPerNode != nil && pe.nodepodCount[node]+1 > *pe.maxPodsToEvictPerNode { - metrics.PodsEvicted.With(map[string]string{"result": "maximum number reached", "strategy": strategy, "namespace": pod.Namespace}).Inc() + metrics.PodsEvicted.With(map[string]string{"result": "maximum number of pods per node reached", "strategy": strategy, "namespace": pod.Namespace}).Inc() return false, fmt.Errorf("Maximum number %v of evicted pods per %q node reached", *pe.maxPodsToEvictPerNode, node.Name) } + if pe.maxPodsToEvictPerNamespace != nil && pe.namespacePodCount[pod.Namespace]+1 > *pe.maxPodsToEvictPerNamespace { + metrics.PodsEvicted.With(map[string]string{"result": "maximum number of pods per namespace reached", "strategy": strategy, "namespace": pod.Namespace}).Inc() + return false, fmt.Errorf("Maximum number %v of evicted pods per %q namespace reached", *pe.maxPodsToEvictPerNamespace, pod.Namespace) + } + err := evictPod(ctx, pe.client, pod, pe.policyGroupVersion, pe.dryRun) if err != nil { // err is used only for logging purposes @@ -124,6 +136,7 @@ func (pe *PodEvictor) EvictPod(ctx context.Context, pod *v1.Pod, node *v1.Node, } pe.nodepodCount[node]++ + pe.namespacePodCount[pod.Namespace]++ if pe.dryRun { klog.V(1).InfoS("Evicted pod in dry run mode", "pod", klog.KObj(pod), "reason", reason) } else { diff --git a/pkg/descheduler/strategies/duplicates_test.go b/pkg/descheduler/strategies/duplicates_test.go index df9e77e18..e2b9520f2 100644 --- a/pkg/descheduler/strategies/duplicates_test.go +++ b/pkg/descheduler/strategies/duplicates_test.go @@ -275,6 +275,7 @@ func TestFindDuplicatePods(t *testing.T) { "v1", false, nil, + nil, testCase.nodes, false, false, @@ -682,6 +683,7 @@ func TestRemoveDuplicatesUniformly(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, nil, + nil, testCase.nodes, false, false, diff --git a/pkg/descheduler/strategies/failedpods_test.go b/pkg/descheduler/strategies/failedpods_test.go index 32c64f7b4..366f3db13 100644 --- a/pkg/descheduler/strategies/failedpods_test.go +++ b/pkg/descheduler/strategies/failedpods_test.go @@ -222,6 +222,7 @@ func TestRemoveFailedPods(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, nil, + nil, tc.nodes, false, false, diff --git a/pkg/descheduler/strategies/node_affinity_test.go b/pkg/descheduler/strategies/node_affinity_test.go index dd450bfaf..b0c12668c 100644 --- a/pkg/descheduler/strategies/node_affinity_test.go +++ b/pkg/descheduler/strategies/node_affinity_test.go @@ -104,12 +104,13 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { var uint1 uint = 1 tests := []struct { - description string - nodes []*v1.Node - pods []v1.Pod - strategy api.DeschedulerStrategy - expectedEvictedPodCount uint - maxPodsToEvictPerNode *uint + description string + nodes []*v1.Node + pods []v1.Pod + strategy api.DeschedulerStrategy + expectedEvictedPodCount uint + maxPodsToEvictPerNode *uint + maxNoOfPodsToEvictPerNamespace *uint }{ { description: "Invalid strategy type, should not evict any pods", @@ -155,6 +156,22 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { nodes: []*v1.Node{nodeWithoutLabels, nodeWithLabels}, maxPodsToEvictPerNode: &uint1, }, + { + description: "Pod is scheduled on node without matching labels, another schedulable node available, maxNoOfPodsToEvictPerNamespace set to 1, should not be evicted", + expectedEvictedPodCount: 1, + strategy: requiredDuringSchedulingIgnoredDuringExecutionStrategy, + pods: addPodsToNode(nodeWithoutLabels, nil), + nodes: []*v1.Node{nodeWithoutLabels, nodeWithLabels}, + maxNoOfPodsToEvictPerNamespace: &uint1, + }, + { + description: "Pod is scheduled on node without matching labels, another schedulable node available, maxNoOfPodsToEvictPerNamespace set to 1, no pod evicted since pod terminting", + expectedEvictedPodCount: 1, + strategy: requiredDuringSchedulingIgnoredDuringExecutionStrategy, + pods: addPodsToNode(nodeWithoutLabels, &metav1.Time{}), + nodes: []*v1.Node{nodeWithoutLabels, nodeWithLabels}, + maxNoOfPodsToEvictPerNamespace: &uint1, + }, { description: "Pod is scheduled on node without matching labels, but no node where pod fits is available, should not evict", expectedEvictedPodCount: 0, @@ -184,6 +201,7 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, tc.maxPodsToEvictPerNode, + tc.maxNoOfPodsToEvictPerNamespace, tc.nodes, false, false, diff --git a/pkg/descheduler/strategies/node_taint_test.go b/pkg/descheduler/strategies/node_taint_test.go index 868cfbaf4..1c54f21f5 100644 --- a/pkg/descheduler/strategies/node_taint_test.go +++ b/pkg/descheduler/strategies/node_taint_test.go @@ -119,14 +119,15 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { var uint1 uint = 1 tests := []struct { - description string - nodes []*v1.Node - pods []v1.Pod - evictLocalStoragePods bool - evictSystemCriticalPods bool - maxPodsToEvictPerNode *uint - expectedEvictedPodCount uint - nodeFit bool + description string + nodes []*v1.Node + pods []v1.Pod + evictLocalStoragePods bool + evictSystemCriticalPods bool + maxPodsToEvictPerNode *uint + maxNoOfPodsToEvictPerNamespace *uint + expectedEvictedPodCount uint + nodeFit bool }{ { @@ -154,6 +155,15 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { maxPodsToEvictPerNode: &uint1, expectedEvictedPodCount: 1, //p5 or p6 gets evicted }, + { + description: "Only number of Pods not tolerating node taint should be evicted", + pods: []v1.Pod{*p1, *p5, *p6}, + nodes: []*v1.Node{node1}, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + maxNoOfPodsToEvictPerNamespace: &uint1, + expectedEvictedPodCount: 1, //p5 or p6 gets evicted + }, { description: "Critical pods not tolerating node taint should not be evicted", pods: []v1.Pod{*p7, *p8, *p9, *p10}, @@ -228,6 +238,7 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, tc.maxPodsToEvictPerNode, + tc.maxNoOfPodsToEvictPerNamespace, tc.nodes, tc.evictLocalStoragePods, tc.evictSystemCriticalPods, diff --git a/pkg/descheduler/strategies/nodeutilization/highnodeutilization_test.go b/pkg/descheduler/strategies/nodeutilization/highnodeutilization_test.go index 547debc24..ff6dc06eb 100644 --- a/pkg/descheduler/strategies/nodeutilization/highnodeutilization_test.go +++ b/pkg/descheduler/strategies/nodeutilization/highnodeutilization_test.go @@ -566,6 +566,7 @@ func TestHighNodeUtilization(t *testing.T) { "v1", false, nil, + nil, nodes, false, false, @@ -756,6 +757,7 @@ func TestHighNodeUtilizationWithTaints(t *testing.T) { "policy/v1", false, &item.evictionsExpected, + nil, item.nodes, false, false, diff --git a/pkg/descheduler/strategies/nodeutilization/lownodeutilization_test.go b/pkg/descheduler/strategies/nodeutilization/lownodeutilization_test.go index b12c47daf..d26c0840c 100644 --- a/pkg/descheduler/strategies/nodeutilization/lownodeutilization_test.go +++ b/pkg/descheduler/strategies/nodeutilization/lownodeutilization_test.go @@ -778,6 +778,7 @@ func TestLowNodeUtilization(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, nil, + nil, nodes, false, false, @@ -1078,6 +1079,7 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, &item.evictionsExpected, + nil, item.nodes, false, false, diff --git a/pkg/descheduler/strategies/pod_antiaffinity_test.go b/pkg/descheduler/strategies/pod_antiaffinity_test.go index b6767d6b4..0a71262ee 100644 --- a/pkg/descheduler/strategies/pod_antiaffinity_test.go +++ b/pkg/descheduler/strategies/pod_antiaffinity_test.go @@ -103,12 +103,13 @@ func TestPodAntiAffinity(t *testing.T) { var uint3 uint = 3 tests := []struct { - description string - maxPodsToEvictPerNode *uint - pods []v1.Pod - expectedEvictedPodCount uint - nodeFit bool - nodes []*v1.Node + description string + maxPodsToEvictPerNode *uint + maxNoOfPodsToEvictPerNamespace *uint + pods []v1.Pod + expectedEvictedPodCount uint + nodeFit bool + nodes []*v1.Node }{ { description: "Maximum pods to evict - 0", @@ -123,6 +124,13 @@ func TestPodAntiAffinity(t *testing.T) { nodes: []*v1.Node{node1}, expectedEvictedPodCount: 3, }, + { + description: "Maximum pods to evict (maxPodsToEvictPerNamespace=3) - 3", + maxNoOfPodsToEvictPerNamespace: &uint3, + pods: []v1.Pod{*p1, *p2, *p3, *p4}, + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 3, + }, { description: "Evict only 1 pod after sorting", pods: []v1.Pod{*p5, *p6, *p7}, @@ -182,6 +190,7 @@ func TestPodAntiAffinity(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, test.maxPodsToEvictPerNode, + test.maxNoOfPodsToEvictPerNamespace, test.nodes, false, false, diff --git a/pkg/descheduler/strategies/pod_lifetime_test.go b/pkg/descheduler/strategies/pod_lifetime_test.go index d66e19fa3..35399b7c8 100644 --- a/pkg/descheduler/strategies/pod_lifetime_test.go +++ b/pkg/descheduler/strategies/pod_lifetime_test.go @@ -276,6 +276,7 @@ func TestPodLifeTime(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, nil, + nil, tc.nodes, false, false, diff --git a/pkg/descheduler/strategies/toomanyrestarts_test.go b/pkg/descheduler/strategies/toomanyrestarts_test.go index 66d604b6a..29cf38dd6 100644 --- a/pkg/descheduler/strategies/toomanyrestarts_test.go +++ b/pkg/descheduler/strategies/toomanyrestarts_test.go @@ -117,11 +117,12 @@ func TestRemovePodsHavingTooManyRestarts(t *testing.T) { var uint3 uint = 3 tests := []struct { - description string - nodes []*v1.Node - strategy api.DeschedulerStrategy - expectedEvictedPodCount uint - maxPodsToEvictPerNode *uint + description string + nodes []*v1.Node + strategy api.DeschedulerStrategy + expectedEvictedPodCount uint + maxPodsToEvictPerNode *uint + maxNoOfPodsToEvictPerNamespace *uint }{ { description: "All pods have total restarts under threshold, no pod evictions", @@ -178,6 +179,13 @@ func TestRemovePodsHavingTooManyRestarts(t *testing.T) { expectedEvictedPodCount: 3, maxPodsToEvictPerNode: &uint3, }, + { + description: "All pods have total restarts equals threshold(maxNoOfPodsToEvictPerNamespace=3), 3 pod evictions", + strategy: createStrategy(true, true, 1, false), + nodes: []*v1.Node{node1}, + expectedEvictedPodCount: 3, + maxNoOfPodsToEvictPerNamespace: &uint3, + }, { description: "All pods have total restarts equals threshold(maxPodsToEvictPerNode=3) but the only other node is tained, 0 pod evictions", strategy: createStrategy(true, true, 1, true), @@ -206,6 +214,7 @@ func TestRemovePodsHavingTooManyRestarts(t *testing.T) { policyv1.SchemeGroupVersion.String(), false, tc.maxPodsToEvictPerNode, + tc.maxNoOfPodsToEvictPerNamespace, tc.nodes, false, false, diff --git a/pkg/descheduler/strategies/topologyspreadconstraint_test.go b/pkg/descheduler/strategies/topologyspreadconstraint_test.go index e586a8204..de4ae74ca 100644 --- a/pkg/descheduler/strategies/topologyspreadconstraint_test.go +++ b/pkg/descheduler/strategies/topologyspreadconstraint_test.go @@ -887,6 +887,7 @@ func TestTopologySpreadConstraint(t *testing.T) { "v1", false, nil, + nil, tc.nodes, false, false, diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 26f3dfce4..56bff2b53 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -145,6 +145,7 @@ func TestRemoveDuplicates(t *testing.T) { evictionPolicyGroupVersion, false, nil, + nil, nodes, true, false, diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 60c740782..d05bd4035 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -167,6 +167,7 @@ func runPodLifetimeStrategy( evictionPolicyGroupVersion, false, nil, + nil, nodes, false, evictCritical, @@ -1291,6 +1292,7 @@ func initPodEvictorOrFail(t *testing.T, clientSet clientset.Interface, nodes []* evictionPolicyGroupVersion, false, nil, + nil, nodes, true, false, diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index 3580f9511..c782fb426 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -135,6 +135,7 @@ func TestTooManyRestarts(t *testing.T) { evictionPolicyGroupVersion, false, nil, + nil, nodes, true, false,