From f47c2c44071a5c73759fcaf5ba413dfdfe278307 Mon Sep 17 00:00:00 2001 From: Lucas Severo Alves Date: Mon, 29 Aug 2022 17:11:30 +0200 Subject: [PATCH] add new preevectionfilter plugin with args --- hack/update-generated-deep-copies.sh | 2 +- hack/verify-deep-copies.sh | 2 +- pkg/api/types.go | 5 + pkg/api/zz_generated.deepcopy.go | 21 + pkg/descheduler/descheduler.go | 37 +- pkg/descheduler/evictions/evictions.go | 182 ----- pkg/descheduler/evictions/evictions_test.go | 594 ---------------- pkg/framework/fake/fake.go | 2 +- .../plugins/defaultevictor/defaultevictor.go | 193 ++++++ .../defaultevictor/defaultevictor_test.go | 636 ++++++++++++++++++ pkg/framework/plugins/defaultevictor/types.go | 37 + .../defaultevictor/zz_generated.deepcopy.go | 60 ++ .../highnodeutilization_test.go | 65 +- .../lownodeutilization_test.go | 67 +- .../plugins/podlifetime/pod_lifetime_test.go | 28 +- .../removeduplicates/removeduplicates_test.go | 64 +- .../removefailedpods/failedpods_test.go | 30 +- .../toomanyrestarts_test.go | 30 +- .../pod_antiaffinity_test.go | 33 +- .../node_affinity_test.go | 32 +- .../node_taint_test.go | 34 +- .../topologyspreadconstraint_test.go | 34 +- pkg/framework/types.go | 9 + pkg/utils/priority.go | 20 + test/e2e/e2e_duplicatepods_test.go | 40 +- test/e2e/e2e_failedpods_test.go | 29 +- test/e2e/e2e_test.go | 57 +- test/e2e/e2e_toomanyrestarts_test.go | 28 +- test/e2e/e2e_topologyspreadconstraint_test.go | 27 +- 29 files changed, 1426 insertions(+), 972 deletions(-) create mode 100644 pkg/framework/plugins/defaultevictor/defaultevictor.go create mode 100644 pkg/framework/plugins/defaultevictor/defaultevictor_test.go create mode 100644 pkg/framework/plugins/defaultevictor/types.go create mode 100644 pkg/framework/plugins/defaultevictor/zz_generated.deepcopy.go diff --git a/hack/update-generated-deep-copies.sh b/hack/update-generated-deep-copies.sh index 5a71d1554..880961f70 100755 --- a/hack/update-generated-deep-copies.sh +++ b/hack/update-generated-deep-copies.sh @@ -5,6 +5,6 @@ go build -o "${OS_OUTPUT_BINPATH}/deepcopy-gen" "k8s.io/code-generator/cmd/deepc ${OS_OUTPUT_BINPATH}/deepcopy-gen \ --go-header-file "hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "${PRJ_PREFIX}/pkg/apis/componentconfig,${PRJ_PREFIX}/pkg/apis/componentconfig/v1alpha1,${PRJ_PREFIX}/pkg/api,${PRJ_PREFIX}/pkg/api/v1alpha1" \ + --input-dirs "${PRJ_PREFIX}/pkg/apis/componentconfig,${PRJ_PREFIX}/pkg/apis/componentconfig/v1alpha1,${PRJ_PREFIX}/pkg/api,${PRJ_PREFIX}/pkg/api/v1alpha1,${PRJ_PREFIX}/pkg/framework/plugins/defaultevictor/" \ --output-file-base zz_generated.deepcopy diff --git a/hack/verify-deep-copies.sh b/hack/verify-deep-copies.sh index 254a8e465..817b1046e 100755 --- a/hack/verify-deep-copies.sh +++ b/hack/verify-deep-copies.sh @@ -20,7 +20,7 @@ go build -o "${OS_OUTPUT_BINPATH}/deepcopy-gen" "k8s.io/code-generator/cmd/deepc ${OS_OUTPUT_BINPATH}/deepcopy-gen \ --go-header-file "hack/boilerplate/boilerplate.go.txt" \ - --input-dirs "./pkg/apis/componentconfig,./pkg/apis/componentconfig/v1alpha1,./pkg/api,./pkg/api/v1alpha1" \ + --input-dirs "./pkg/apis/componentconfig,./pkg/apis/componentconfig/v1alpha1,./pkg/api,./pkg/api/v1alpha1,./pkg/framework/plugins/defaultevictor/" \ --output-file-base zz_generated.deepcopy popd > /dev/null 2>&1 diff --git a/pkg/api/types.go b/pkg/api/types.go index 4880eaa75..43aefeffd 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -126,3 +126,8 @@ type FailedPods struct { Reasons []string IncludingInitContainers bool } + +type PriorityThreshold struct { + Value *int32 + Name string +} diff --git a/pkg/api/zz_generated.deepcopy.go b/pkg/api/zz_generated.deepcopy.go index 5d1407380..2c360367f 100644 --- a/pkg/api/zz_generated.deepcopy.go +++ b/pkg/api/zz_generated.deepcopy.go @@ -248,6 +248,27 @@ func (in *PodsHavingTooManyRestarts) DeepCopy() *PodsHavingTooManyRestarts { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PriorityThreshold) DeepCopyInto(out *PriorityThreshold) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PriorityThreshold. +func (in *PriorityThreshold) DeepCopy() *PriorityThreshold { + if in == nil { + return nil + } + out := new(PriorityThreshold) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RemoveDuplicates) DeepCopyInto(out *RemoveDuplicates) { *out = *in diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index 1558d321c..35df742e5 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -44,6 +44,7 @@ import ( nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/utils" ) @@ -88,7 +89,7 @@ func Run(ctx context.Context, rs *options.DeschedulerServer) error { return runFn() } -type strategyFunction func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor, evictorFilter *evictions.EvictorFilter, getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc) +type strategyFunction func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor, evictorFilter framework.EvictorPlugin, getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc) func cachedClient( realClient clientset.Interface, @@ -170,7 +171,7 @@ func cachedClient( // can evict a pod without importing a specific pod evictor type evictorImpl struct { podEvictor *evictions.PodEvictor - evictorFilter *evictions.EvictorFilter + evictorFilter framework.EvictorPlugin } var _ framework.Evictor = &evictorImpl{} @@ -376,23 +377,33 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer continue } - evictorFilter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - evictLocalStoragePods, - evictSystemCriticalPods, - ignorePvcPods, - evictBarePods, - evictions.WithNodeFit(nodeFit), - evictions.WithPriorityThreshold(thresholdPriority), + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: evictLocalStoragePods, + EvictSystemCriticalPods: evictSystemCriticalPods, + IgnorePvcPods: ignorePvcPods, + EvictFailedBarePods: evictBarePods, + NodeFit: nodeFit, + PriorityThreshold: &api.PriorityThreshold{ + Value: &thresholdPriority, + }, + } + + evictorFilter, _ := defaultevictor.New( + defaultevictorArgs, + &handleImpl{ + clientSet: rs.Client, + getPodsAssignedToNodeFunc: getPodsAssignedToNode, + sharedInformerFactory: sharedInformerFactory, + }, ) + handle := &handleImpl{ clientSet: rs.Client, getPodsAssignedToNodeFunc: getPodsAssignedToNode, sharedInformerFactory: sharedInformerFactory, evictor: &evictorImpl{ podEvictor: podEvictor, - evictorFilter: evictorFilter, + evictorFilter: evictorFilter.(framework.EvictorPlugin), }, } @@ -404,7 +415,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer if pgFnc, exists := pluginsMap[string(name)]; exists { pgFnc(childCtx, nodes, params, handle) } else { - f(childCtx, rs.Client, strategy, nodes, podEvictor, evictorFilter, getPodsAssignedToNode) + f(childCtx, rs.Client, strategy, nodes, podEvictor, evictorFilter.(framework.EvictorPlugin), getPodsAssignedToNode) } } } else { diff --git a/pkg/descheduler/evictions/evictions.go b/pkg/descheduler/evictions/evictions.go index a2936cdf1..0f9c1e497 100644 --- a/pkg/descheduler/evictions/evictions.go +++ b/pkg/descheduler/evictions/evictions.go @@ -24,23 +24,14 @@ import ( policy "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/events" "k8s.io/klog/v2" "sigs.k8s.io/descheduler/metrics" - nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" - podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" - "sigs.k8s.io/descheduler/pkg/utils" eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" ) -const ( - evictPodAnnotationKey = "descheduler.alpha.kubernetes.io/evict" -) - // nodePodEvictedCount keeps count of pods evicted on node type nodePodEvictedCount map[string]uint type namespacePodEvictCount map[string]uint @@ -203,176 +194,3 @@ func evictPod(ctx context.Context, client clientset.Interface, pod *v1.Pod, poli } return err } - -type Options struct { - priority *int32 - nodeFit bool - labelSelector labels.Selector -} - -// WithPriorityThreshold sets a threshold for pod's priority class. -// Any pod whose priority class is lower is evictable. -func WithPriorityThreshold(priority int32) func(opts *Options) { - return func(opts *Options) { - var p int32 = priority - opts.priority = &p - } -} - -// WithNodeFit sets whether or not to consider taints, node selectors, -// and pod affinity when evicting. A pod whose tolerations, node selectors, -// and affinity match a node other than the one it is currently running on -// is evictable. -func WithNodeFit(nodeFit bool) func(opts *Options) { - return func(opts *Options) { - opts.nodeFit = nodeFit - } -} - -// WithLabelSelector sets whether or not to apply label filtering when evicting. -// Any pod matching the label selector is considered evictable. -func WithLabelSelector(labelSelector labels.Selector) func(opts *Options) { - return func(opts *Options) { - opts.labelSelector = labelSelector - } -} - -type constraint func(pod *v1.Pod) error - -type EvictorFilter struct { - constraints []constraint -} - -func NewEvictorFilter( - nodes []*v1.Node, - nodeIndexer podutil.GetPodsAssignedToNodeFunc, - evictLocalStoragePods bool, - evictSystemCriticalPods bool, - ignorePvcPods bool, - evictFailedBarePods bool, - opts ...func(opts *Options), -) *EvictorFilter { - options := &Options{} - for _, opt := range opts { - opt(options) - } - - ev := &EvictorFilter{} - if evictFailedBarePods { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - ownerRefList := podutil.OwnerRef(pod) - // Enable evictFailedBarePods to evict bare pods in failed phase - if len(ownerRefList) == 0 && pod.Status.Phase != v1.PodFailed { - return fmt.Errorf("pod does not have any ownerRefs and is not in failed phase") - } - return nil - }) - } else { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - ownerRefList := podutil.OwnerRef(pod) - // Moved from IsEvictable function for backward compatibility - if len(ownerRefList) == 0 { - return fmt.Errorf("pod does not have any ownerRefs") - } - return nil - }) - } - if !evictSystemCriticalPods { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - // Moved from IsEvictable function to allow for disabling - if utils.IsCriticalPriorityPod(pod) { - return fmt.Errorf("pod has system critical priority") - } - return nil - }) - - if options.priority != nil { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - if IsPodEvictableBasedOnPriority(pod, *options.priority) { - return nil - } - return fmt.Errorf("pod has higher priority than specified priority class threshold") - }) - } - } - if !evictLocalStoragePods { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - if utils.IsPodWithLocalStorage(pod) { - return fmt.Errorf("pod has local storage and descheduler is not configured with evictLocalStoragePods") - } - return nil - }) - } - if ignorePvcPods { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - if utils.IsPodWithPVC(pod) { - return fmt.Errorf("pod has a PVC and descheduler is configured to ignore PVC pods") - } - return nil - }) - } - if options.nodeFit { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - if !nodeutil.PodFitsAnyOtherNode(nodeIndexer, pod, nodes) { - return fmt.Errorf("pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable") - } - return nil - }) - } - if options.labelSelector != nil && !options.labelSelector.Empty() { - ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { - if !options.labelSelector.Matches(labels.Set(pod.Labels)) { - return fmt.Errorf("pod labels do not match the labelSelector filter in the policy parameter") - } - return nil - }) - } - - return ev -} - -// IsEvictable decides when a pod is evictable -func (ef *EvictorFilter) Filter(pod *v1.Pod) bool { - checkErrs := []error{} - - ownerRefList := podutil.OwnerRef(pod) - if utils.IsDaemonsetPod(ownerRefList) { - checkErrs = append(checkErrs, fmt.Errorf("pod is a DaemonSet pod")) - } - - if utils.IsMirrorPod(pod) { - checkErrs = append(checkErrs, fmt.Errorf("pod is a mirror pod")) - } - - if utils.IsStaticPod(pod) { - checkErrs = append(checkErrs, fmt.Errorf("pod is a static pod")) - } - - if utils.IsPodTerminating(pod) { - checkErrs = append(checkErrs, fmt.Errorf("pod is terminating")) - } - - for _, c := range ef.constraints { - if err := c(pod); err != nil { - checkErrs = append(checkErrs, err) - } - } - - if len(checkErrs) > 0 && !HaveEvictAnnotation(pod) { - klog.V(4).InfoS("Pod lacks an eviction annotation and fails the following checks", "pod", klog.KObj(pod), "checks", errors.NewAggregate(checkErrs).Error()) - return false - } - - return true -} - -// HaveEvictAnnotation checks if the pod have evict annotation -func HaveEvictAnnotation(pod *v1.Pod) bool { - _, found := pod.ObjectMeta.Annotations[evictPodAnnotationKey] - return found -} - -// IsPodEvictableBasedOnPriority checks if the given pod is evictable based on priority resolved from pod Spec. -func IsPodEvictableBasedOnPriority(pod *v1.Pod, priority int32) bool { - return pod.Spec.Priority == nil || *pod.Spec.Priority < priority -} diff --git a/pkg/descheduler/evictions/evictions_test.go b/pkg/descheduler/evictions/evictions_test.go index ffae40a38..2892a16a7 100644 --- a/pkg/descheduler/evictions/evictions_test.go +++ b/pkg/descheduler/evictions/evictions_test.go @@ -23,7 +23,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" @@ -70,599 +69,6 @@ func TestEvictPod(t *testing.T) { } } -func TestIsEvictable(t *testing.T) { - n1 := test.BuildTestNode("node1", 1000, 2000, 13, nil) - lowPriority := int32(800) - highPriority := int32(900) - - nodeTaintKey := "hardware" - nodeTaintValue := "gpu" - - nodeLabelKey := "datacenter" - nodeLabelValue := "east" - type testCase struct { - description string - pods []*v1.Pod - nodes []*v1.Node - evictFailedBarePods bool - evictLocalStoragePods bool - evictSystemCriticalPods bool - priorityThreshold *int32 - nodeFit bool - result bool - } - - testCases := []testCase{ - { - description: "Failed pod eviction with no ownerRefs", - pods: []*v1.Pod{ - test.BuildTestPod("bare_pod_failed", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Status.Phase = v1.PodFailed - }), - }, - evictFailedBarePods: false, - result: false, - }, { - description: "Normal pod eviction with no ownerRefs and evictFailedBarePods enabled", - pods: []*v1.Pod{test.BuildTestPod("bare_pod", 400, 0, n1.Name, nil)}, - evictFailedBarePods: true, - result: false, - }, { - description: "Failed pod eviction with no ownerRefs", - pods: []*v1.Pod{ - test.BuildTestPod("bare_pod_failed_but_can_be_evicted", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Status.Phase = v1.PodFailed - }), - }, - evictFailedBarePods: true, - result: true, - }, { - description: "Normal pod eviction with normal ownerRefs", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Normal pod eviction with normal ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p2", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Normal pod eviction with replicaSet ownerRefs", - pods: []*v1.Pod{ - test.BuildTestPod("p3", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Normal pod eviction with replicaSet ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p4", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.ObjectMeta.OwnerReferences = test.GetReplicaSetOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Normal pod eviction with statefulSet ownerRefs", - pods: []*v1.Pod{ - test.BuildTestPod("p18", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Normal pod eviction with statefulSet ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p19", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.ObjectMeta.OwnerReferences = test.GetStatefulSetOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod not evicted because it is bound to a PV and evictLocalStoragePods = false", - pods: []*v1.Pod{ - test.BuildTestPod("p5", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Volumes = []v1.Volume{ - { - Name: "sample", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, - EmptyDir: &v1.EmptyDirVolumeSource{ - SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, - }, - }, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: false, - }, { - description: "Pod is evicted because it is bound to a PV and evictLocalStoragePods = true", - pods: []*v1.Pod{ - test.BuildTestPod("p6", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Volumes = []v1.Volume{ - { - Name: "sample", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, - EmptyDir: &v1.EmptyDirVolumeSource{ - SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, - }, - }, - } - }), - }, - evictLocalStoragePods: true, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod is evicted because it is bound to a PV and evictLocalStoragePods = false, but it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p7", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Volumes = []v1.Volume{ - { - Name: "sample", - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, - EmptyDir: &v1.EmptyDirVolumeSource{ - SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, - }, - }, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod not evicted becasuse it is part of a daemonSet", - pods: []*v1.Pod{ - test.BuildTestPod("p8", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: false, - }, { - description: "Pod is evicted becasuse it is part of a daemonSet, but it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p9", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod not evicted becasuse it is a mirror poddsa", - pods: []*v1.Pod{ - test.BuildTestPod("p10", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Annotations = test.GetMirrorPodAnnotation() - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: false, - }, { - description: "Pod is evicted becasuse it is a mirror pod, but it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p11", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Annotations = test.GetMirrorPodAnnotation() - pod.Annotations["descheduler.alpha.kubernetes.io/evict"] = "true" - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod not evicted becasuse it has system critical priority", - pods: []*v1.Pod{ - test.BuildTestPod("p12", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - priority := utils.SystemCriticalPriority - pod.Spec.Priority = &priority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: false, - }, { - description: "Pod is evicted becasuse it has system critical priority, but it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p13", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - priority := utils.SystemCriticalPriority - pod.Spec.Priority = &priority - pod.Annotations = map[string]string{ - "descheduler.alpha.kubernetes.io/evict": "true", - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - result: true, - }, { - description: "Pod not evicted becasuse it has a priority higher than the configured priority threshold", - pods: []*v1.Pod{ - test.BuildTestPod("p14", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Priority = &highPriority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - priorityThreshold: &lowPriority, - result: false, - }, { - description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p15", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.Spec.Priority = &highPriority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - priorityThreshold: &lowPriority, - result: true, - }, { - description: "Pod is evicted becasuse it has system critical priority, but evictSystemCriticalPods = true", - pods: []*v1.Pod{ - test.BuildTestPod("p16", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - priority := utils.SystemCriticalPriority - pod.Spec.Priority = &priority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: true, - result: true, - }, { - description: "Pod is evicted becasuse it has system critical priority, but evictSystemCriticalPods = true and it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p16", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - priority := utils.SystemCriticalPriority - pod.Spec.Priority = &priority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: true, - result: true, - }, { - description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but evictSystemCriticalPods = true", - pods: []*v1.Pod{ - test.BuildTestPod("p17", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Priority = &highPriority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: true, - priorityThreshold: &lowPriority, - result: true, - }, { - description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but evictSystemCriticalPods = true and it has scheduler.alpha.kubernetes.io/evict annotation", - pods: []*v1.Pod{ - test.BuildTestPod("p17", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} - pod.Spec.Priority = &highPriority - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: true, - priorityThreshold: &lowPriority, - result: true, - }, { - description: "Pod with no tolerations running on normal node, all other nodes tainted", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { - node.Spec.Taints = []v1.Taint{ - { - Key: nodeTaintKey, - Value: nodeTaintValue, - Effect: v1.TaintEffectNoSchedule, - }, - } - }), - test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { - node.Spec.Taints = []v1.Taint{ - { - Key: nodeTaintKey, - Value: nodeTaintValue, - Effect: v1.TaintEffectNoSchedule, - }, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: false, - }, { - description: "Pod with correct tolerations running on normal node, all other nodes tainted", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.Tolerations = []v1.Toleration{ - { - Key: nodeTaintKey, - Value: nodeTaintValue, - Effect: v1.TaintEffectNoSchedule, - }, - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { - node.Spec.Taints = []v1.Taint{ - { - Key: nodeTaintKey, - Value: nodeTaintValue, - Effect: v1.TaintEffectNoSchedule, - }, - } - }), - test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { - node.Spec.Taints = []v1.Taint{ - { - Key: nodeTaintKey, - Value: nodeTaintValue, - Effect: v1.TaintEffectNoSchedule, - }, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: true, - }, { - description: "Pod with incorrect node selector", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.NodeSelector = map[string]string{ - nodeLabelKey: "fail", - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: false, - }, { - description: "Pod with correct node selector", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.NodeSelector = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: true, - }, { - description: "Pod with correct node selector, but only available node doesn't have enough CPU", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.NodeSelector = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2-TEST", 10, 16, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestNode("node3-TEST", 10, 16, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: false, - }, { - description: "Pod with correct node selector, and one node has enough memory", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.NodeSelector = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestPod("node2-pod-10GB-mem", 20, 10, "node2", func(pod *v1.Pod) { - pod.ObjectMeta.Labels = map[string]string{ - "test": "true", - } - }), - test.BuildTestPod("node3-pod-10GB-mem", 20, 10, "node3", func(pod *v1.Pod) { - pod.ObjectMeta.Labels = map[string]string{ - "test": "true", - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 100, 16, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestNode("node3", 100, 20, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: true, - }, { - description: "Pod with correct node selector, but both nodes don't have enough memory", - pods: []*v1.Pod{ - test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { - pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() - pod.Spec.NodeSelector = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestPod("node2-pod-10GB-mem", 10, 10, "node2", func(pod *v1.Pod) { - pod.ObjectMeta.Labels = map[string]string{ - "test": "true", - } - }), - test.BuildTestPod("node3-pod-10GB-mem", 10, 10, "node3", func(pod *v1.Pod) { - pod.ObjectMeta.Labels = map[string]string{ - "test": "true", - } - }), - }, - nodes: []*v1.Node{ - test.BuildTestNode("node2", 100, 16, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - test.BuildTestNode("node3", 100, 16, 10, func(node *v1.Node) { - node.ObjectMeta.Labels = map[string]string{ - nodeLabelKey: nodeLabelValue, - } - }), - }, - evictLocalStoragePods: false, - evictSystemCriticalPods: false, - nodeFit: true, - result: false, - }, - } - - for _, test := range testCases { - - t.Run(test.description, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - nodes := append(test.nodes, n1) - - var objs []runtime.Object - for _, node := range test.nodes { - objs = append(objs, node) - } - for _, pod := range test.pods { - objs = append(objs, pod) - } - - fakeClient := fake.NewSimpleClientset(objs...) - - sharedInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0) - podInformer := sharedInformerFactory.Core().V1().Pods() - - getPodsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer) - if err != nil { - t.Errorf("Build get pods assigned to node function error: %v", err) - } - - sharedInformerFactory.Start(ctx.Done()) - sharedInformerFactory.WaitForCacheSync(ctx.Done()) - - var opts []func(opts *Options) - if test.priorityThreshold != nil { - opts = append(opts, WithPriorityThreshold(*test.priorityThreshold)) - } - if test.nodeFit { - opts = append(opts, WithNodeFit(true)) - } - - evictorFilter := NewEvictorFilter( - nodes, - getPodsAssignedToNode, - test.evictLocalStoragePods, - test.evictSystemCriticalPods, - false, - test.evictFailedBarePods, - opts..., - ) - - result := evictorFilter.Filter(test.pods[0]) - if result != test.result { - t.Errorf("IsEvictable should return for pod %s %t, but it returns %t", test.pods[0].Name, test.result, result) - } - }) - } -} func TestPodTypes(t *testing.T) { n1 := test.BuildTestNode("node1", 1000, 2000, 9, nil) p1 := test.BuildTestPod("p1", 400, 0, n1.Name, nil) diff --git a/pkg/framework/fake/fake.go b/pkg/framework/fake/fake.go index dba0d1b5e..98e71e4d5 100644 --- a/pkg/framework/fake/fake.go +++ b/pkg/framework/fake/fake.go @@ -16,7 +16,7 @@ type HandleImpl struct { ClientsetImpl clientset.Interface GetPodsAssignedToNodeFuncImpl podutil.GetPodsAssignedToNodeFunc SharedInformerFactoryImpl informers.SharedInformerFactory - EvictorFilterImpl *evictions.EvictorFilter + EvictorFilterImpl framework.EvictorPlugin PodEvictorImpl *evictions.PodEvictor } diff --git a/pkg/framework/plugins/defaultevictor/defaultevictor.go b/pkg/framework/plugins/defaultevictor/defaultevictor.go new file mode 100644 index 000000000..04d51ba77 --- /dev/null +++ b/pkg/framework/plugins/defaultevictor/defaultevictor.go @@ -0,0 +1,193 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultevictor + +import ( + // "context" + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework" + "sigs.k8s.io/descheduler/pkg/utils" +) + +const ( + PluginName = "DefaultEvictor" + evictPodAnnotationKey = "descheduler.alpha.kubernetes.io/evict" +) + +var _ framework.EvictorPlugin = &DefaultEvictor{} + +type constraint func(pod *v1.Pod) error + +// DefaultEvictor is the first EvictorPlugin, which defines the default extension points of the +// pre-baked evictor that is shipped. +// Even though we name this plugin DefaultEvictor, it does not actually evict anything, +// This plugin is only meant to customize other actions (extension points) of the evictor, +// like filtering, sorting, and other ones that might be relevant in the future +type DefaultEvictor struct { + constraints []constraint +} + +// IsPodEvictableBasedOnPriority checks if the given pod is evictable based on priority resolved from pod Spec. +func IsPodEvictableBasedOnPriority(pod *v1.Pod, priority int32) bool { + return pod.Spec.Priority == nil || *pod.Spec.Priority < priority +} + +// HaveEvictAnnotation checks if the pod have evict annotation +func HaveEvictAnnotation(pod *v1.Pod) bool { + _, found := pod.ObjectMeta.Annotations[evictPodAnnotationKey] + return found +} + +// New builds plugin from its arguments while passing a handle +func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error) { + defaultEvictorArgs, ok := args.(*DefaultEvictorArgs) + if !ok { + return nil, fmt.Errorf("want args to be of type defaultEvictorFilterArgs, got %T", args) + } + + ev := &DefaultEvictor{} + + if defaultEvictorArgs.EvictFailedBarePods { + klog.V(1).InfoS("Warning: EvictFailedBarePods is set to True. This could cause eviction of pods without ownerReferences.") + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + ownerRefList := podutil.OwnerRef(pod) + // Enable evictFailedBarePods to evict bare pods in failed phase + if len(ownerRefList) == 0 && pod.Status.Phase != v1.PodFailed { + return fmt.Errorf("pod does not have any ownerRefs and is not in failed phase") + } + return nil + }) + } else { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + ownerRefList := podutil.OwnerRef(pod) + if len(ownerRefList) == 0 { + return fmt.Errorf("pod does not have any ownerRefs") + } + return nil + }) + } + if !defaultEvictorArgs.EvictSystemCriticalPods { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + if utils.IsCriticalPriorityPod(pod) { + return fmt.Errorf("pod has system critical priority") + } + return nil + }) + + if defaultEvictorArgs.PriorityThreshold != nil && (defaultEvictorArgs.PriorityThreshold.Value != nil || len(defaultEvictorArgs.PriorityThreshold.Name) > 0) { + thresholdPriority, err := utils.GetPriorityValueFromPriorityThreshold(context.TODO(), handle.ClientSet(), defaultEvictorArgs.PriorityThreshold) + if err != nil { + return nil, fmt.Errorf("failed to get priority threshold: %v", err) + } + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + if IsPodEvictableBasedOnPriority(pod, thresholdPriority) { + return nil + } + return fmt.Errorf("pod has higher priority than specified priority class threshold") + }) + } + } else { + klog.V(1).InfoS("Warning: EvictSystemCriticalPods is set to True. This could cause eviction of Kubernetes system pods.") + } + if !defaultEvictorArgs.EvictLocalStoragePods { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + if utils.IsPodWithLocalStorage(pod) { + return fmt.Errorf("pod has local storage and descheduler is not configured with evictLocalStoragePods") + } + return nil + }) + } + if defaultEvictorArgs.IgnorePvcPods { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + if utils.IsPodWithPVC(pod) { + return fmt.Errorf("pod has a PVC and descheduler is configured to ignore PVC pods") + } + return nil + }) + } + if defaultEvictorArgs.NodeFit { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + nodes, err := nodeutil.ReadyNodes(context.TODO(), handle.ClientSet(), handle.SharedInformerFactory().Core().V1().Nodes(), defaultEvictorArgs.NodeSelector) + if err != nil { + return fmt.Errorf("could not list nodes when processing NodeFit") + } + if !nodeutil.PodFitsAnyOtherNode(handle.GetPodsAssignedToNodeFunc(), pod, nodes) { + return fmt.Errorf("pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable") + } + return nil + }) + } + if defaultEvictorArgs.LabelSelector != nil && !defaultEvictorArgs.LabelSelector.Empty() { + ev.constraints = append(ev.constraints, func(pod *v1.Pod) error { + if !defaultEvictorArgs.LabelSelector.Matches(labels.Set(pod.Labels)) { + return fmt.Errorf("pod labels do not match the labelSelector filter in the policy parameter") + } + return nil + }) + } + + return ev, nil +} + +// Name retrieves the plugin name +func (d *DefaultEvictor) Name() string { + return PluginName +} + +func (d *DefaultEvictor) Filter(pod *v1.Pod) bool { + checkErrs := []error{} + + if HaveEvictAnnotation(pod) { + return true + } + + ownerRefList := podutil.OwnerRef(pod) + if utils.IsDaemonsetPod(ownerRefList) { + checkErrs = append(checkErrs, fmt.Errorf("pod is a DaemonSet pod")) + } + + if utils.IsMirrorPod(pod) { + checkErrs = append(checkErrs, fmt.Errorf("pod is a mirror pod")) + } + + if utils.IsStaticPod(pod) { + checkErrs = append(checkErrs, fmt.Errorf("pod is a static pod")) + } + + if utils.IsPodTerminating(pod) { + checkErrs = append(checkErrs, fmt.Errorf("pod is terminating")) + } + + for _, c := range d.constraints { + if err := c(pod); err != nil { + checkErrs = append(checkErrs, err) + } + } + + if len(checkErrs) > 0 { + klog.V(4).InfoS("Pod fails the following checks", "pod", klog.KObj(pod), "checks", errors.NewAggregate(checkErrs).Error()) + return false + } + + return true +} diff --git a/pkg/framework/plugins/defaultevictor/defaultevictor_test.go b/pkg/framework/plugins/defaultevictor/defaultevictor_test.go new file mode 100644 index 000000000..00c0a550e --- /dev/null +++ b/pkg/framework/plugins/defaultevictor/defaultevictor_test.go @@ -0,0 +1,636 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultevictor + +import ( + "context" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/descheduler/pkg/api" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework" + frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/utils" + "sigs.k8s.io/descheduler/test" +) + +func TestDefaultEvictorFilter(t *testing.T) { + n1 := test.BuildTestNode("node1", 1000, 2000, 13, nil) + lowPriority := int32(800) + highPriority := int32(900) + + nodeTaintKey := "hardware" + nodeTaintValue := "gpu" + + nodeLabelKey := "datacenter" + nodeLabelValue := "east" + type testCase struct { + description string + pods []*v1.Pod + nodes []*v1.Node + evictFailedBarePods bool + evictLocalStoragePods bool + evictSystemCriticalPods bool + priorityThreshold *int32 + nodeFit bool + result bool + } + + testCases := []testCase{ + { + description: "Failed pod eviction with no ownerRefs", + pods: []*v1.Pod{ + test.BuildTestPod("bare_pod_failed", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Status.Phase = v1.PodFailed + }), + }, + evictFailedBarePods: false, + result: false, + }, { + description: "Normal pod eviction with no ownerRefs and evictFailedBarePods enabled", + pods: []*v1.Pod{test.BuildTestPod("bare_pod", 400, 0, n1.Name, nil)}, + evictFailedBarePods: true, + result: false, + }, { + description: "Failed pod eviction with no ownerRefs", + pods: []*v1.Pod{ + test.BuildTestPod("bare_pod_failed_but_can_be_evicted", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Status.Phase = v1.PodFailed + }), + }, + evictFailedBarePods: true, + result: true, + }, { + description: "Normal pod eviction with normal ownerRefs", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Normal pod eviction with normal ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p2", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Normal pod eviction with replicaSet ownerRefs", + pods: []*v1.Pod{ + test.BuildTestPod("p3", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Normal pod eviction with replicaSet ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p4", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.ObjectMeta.OwnerReferences = test.GetReplicaSetOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Normal pod eviction with statefulSet ownerRefs", + pods: []*v1.Pod{ + test.BuildTestPod("p18", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Normal pod eviction with statefulSet ownerRefs and descheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p19", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.ObjectMeta.OwnerReferences = test.GetStatefulSetOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod not evicted because it is bound to a PV and evictLocalStoragePods = false", + pods: []*v1.Pod{ + test.BuildTestPod("p5", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, + }, + }, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: false, + }, { + description: "Pod is evicted because it is bound to a PV and evictLocalStoragePods = true", + pods: []*v1.Pod{ + test.BuildTestPod("p6", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, + }, + }, + } + }), + }, + evictLocalStoragePods: true, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod is evicted because it is bound to a PV and evictLocalStoragePods = false, but it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p7", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)}, + }, + }, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod not evicted becasuse it is part of a daemonSet", + pods: []*v1.Pod{ + test.BuildTestPod("p8", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: false, + }, { + description: "Pod is evicted becasuse it is part of a daemonSet, but it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p9", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.ObjectMeta.OwnerReferences = test.GetDaemonSetOwnerRefList() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod not evicted becasuse it is a mirror poddsa", + pods: []*v1.Pod{ + test.BuildTestPod("p10", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Annotations = test.GetMirrorPodAnnotation() + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: false, + }, { + description: "Pod is evicted becasuse it is a mirror pod, but it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p11", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Annotations = test.GetMirrorPodAnnotation() + pod.Annotations["descheduler.alpha.kubernetes.io/evict"] = "true" + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod not evicted becasuse it has system critical priority", + pods: []*v1.Pod{ + test.BuildTestPod("p12", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + priority := utils.SystemCriticalPriority + pod.Spec.Priority = &priority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: false, + }, { + description: "Pod is evicted becasuse it has system critical priority, but it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p13", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + priority := utils.SystemCriticalPriority + pod.Spec.Priority = &priority + pod.Annotations = map[string]string{ + "descheduler.alpha.kubernetes.io/evict": "true", + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + result: true, + }, { + description: "Pod not evicted becasuse it has a priority higher than the configured priority threshold", + pods: []*v1.Pod{ + test.BuildTestPod("p14", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Priority = &highPriority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + priorityThreshold: &lowPriority, + result: false, + }, { + description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p15", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.Spec.Priority = &highPriority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + priorityThreshold: &lowPriority, + result: true, + }, { + description: "Pod is evicted becasuse it has system critical priority, but evictSystemCriticalPods = true", + pods: []*v1.Pod{ + test.BuildTestPod("p16", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + priority := utils.SystemCriticalPriority + pod.Spec.Priority = &priority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: true, + result: true, + }, { + description: "Pod is evicted becasuse it has system critical priority, but evictSystemCriticalPods = true and it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p16", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + priority := utils.SystemCriticalPriority + pod.Spec.Priority = &priority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: true, + result: true, + }, { + description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but evictSystemCriticalPods = true", + pods: []*v1.Pod{ + test.BuildTestPod("p17", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Priority = &highPriority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: true, + priorityThreshold: &lowPriority, + result: true, + }, { + description: "Pod is evicted becasuse it has a priority higher than the configured priority threshold, but evictSystemCriticalPods = true and it has scheduler.alpha.kubernetes.io/evict annotation", + pods: []*v1.Pod{ + test.BuildTestPod("p17", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"} + pod.Spec.Priority = &highPriority + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: true, + priorityThreshold: &lowPriority, + result: true, + }, { + description: "Pod with no tolerations running on normal node, all other nodes tainted", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { + node.Spec.Taints = []v1.Taint{ + { + Key: nodeTaintKey, + Value: nodeTaintValue, + Effect: v1.TaintEffectNoSchedule, + }, + } + }), + test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { + node.Spec.Taints = []v1.Taint{ + { + Key: nodeTaintKey, + Value: nodeTaintValue, + Effect: v1.TaintEffectNoSchedule, + }, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: false, + }, { + description: "Pod with correct tolerations running on normal node, all other nodes tainted", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.Tolerations = []v1.Toleration{ + { + Key: nodeTaintKey, + Value: nodeTaintValue, + Effect: v1.TaintEffectNoSchedule, + }, + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { + node.Spec.Taints = []v1.Taint{ + { + Key: nodeTaintKey, + Value: nodeTaintValue, + Effect: v1.TaintEffectNoSchedule, + }, + } + }), + test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { + node.Spec.Taints = []v1.Taint{ + { + Key: nodeTaintKey, + Value: nodeTaintValue, + Effect: v1.TaintEffectNoSchedule, + }, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: true, + }, { + description: "Pod with incorrect node selector", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.NodeSelector = map[string]string{ + nodeLabelKey: "fail", + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: false, + }, { + description: "Pod with correct node selector", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.NodeSelector = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: true, + }, { + description: "Pod with correct node selector, but only available node doesn't have enough CPU", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.NodeSelector = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2-TEST", 10, 16, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestNode("node3-TEST", 10, 16, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: false, + }, { + description: "Pod with correct node selector, and one node has enough memory", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.NodeSelector = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestPod("node2-pod-10GB-mem", 20, 10, "node2", func(pod *v1.Pod) { + pod.ObjectMeta.Labels = map[string]string{ + "test": "true", + } + }), + test.BuildTestPod("node3-pod-10GB-mem", 20, 10, "node3", func(pod *v1.Pod) { + pod.ObjectMeta.Labels = map[string]string{ + "test": "true", + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 100, 16, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestNode("node3", 100, 20, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: true, + }, { + description: "Pod with correct node selector, but both nodes don't have enough memory", + pods: []*v1.Pod{ + test.BuildTestPod("p1", 12, 8, n1.Name, func(pod *v1.Pod) { + pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList() + pod.Spec.NodeSelector = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestPod("node2-pod-10GB-mem", 10, 10, "node2", func(pod *v1.Pod) { + pod.ObjectMeta.Labels = map[string]string{ + "test": "true", + } + }), + test.BuildTestPod("node3-pod-10GB-mem", 10, 10, "node3", func(pod *v1.Pod) { + pod.ObjectMeta.Labels = map[string]string{ + "test": "true", + } + }), + }, + nodes: []*v1.Node{ + test.BuildTestNode("node2", 100, 16, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + test.BuildTestNode("node3", 100, 16, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeLabelKey: nodeLabelValue, + } + }), + }, + evictLocalStoragePods: false, + evictSystemCriticalPods: false, + nodeFit: true, + result: false, + }, + } + + for _, test := range testCases { + + t.Run(test.description, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var objs []runtime.Object + for _, node := range test.nodes { + objs = append(objs, node) + } + for _, pod := range test.pods { + objs = append(objs, pod) + } + + fakeClient := fake.NewSimpleClientset(objs...) + + sharedInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0) + podInformer := sharedInformerFactory.Core().V1().Pods() + + getPodsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer) + if err != nil { + t.Errorf("Build get pods assigned to node function error: %v", err) + } + + sharedInformerFactory.Start(ctx.Done()) + sharedInformerFactory.WaitForCacheSync(ctx.Done()) + + // var opts []func(opts *FilterOptions) + // if test.priorityThreshold != nil { + // opts = append(opts, WithPriorityThreshold(*test.priorityThreshold)) + // } + // if test.nodeFit { + // opts = append(opts, WithNodeFit(true)) + // } + + defaultEvictorArgs := &DefaultEvictorArgs{ + EvictLocalStoragePods: test.evictLocalStoragePods, + EvictSystemCriticalPods: test.evictSystemCriticalPods, + IgnorePvcPods: false, + EvictFailedBarePods: test.evictFailedBarePods, + PriorityThreshold: &api.PriorityThreshold{ + Value: test.priorityThreshold, + }, + NodeFit: test.nodeFit, + } + + evictorPlugin, err := New( + defaultEvictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + + result := evictorPlugin.(framework.EvictorPlugin).Filter(test.pods[0]) + if result != test.result { + t.Errorf("Filter should return for pod %s %t, but it returns %t", test.pods[0].Name, test.result, result) + } + + }) + } +} diff --git a/pkg/framework/plugins/defaultevictor/types.go b/pkg/framework/plugins/defaultevictor/types.go new file mode 100644 index 000000000..24bf0b251 --- /dev/null +++ b/pkg/framework/plugins/defaultevictor/types.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultevictor + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/descheduler/pkg/api" +) + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// DefaultEvictorArgs holds arguments used to configure DefaultEvictor plugin. +type DefaultEvictorArgs struct { + metav1.TypeMeta + + NodeSelector string + EvictLocalStoragePods bool + EvictSystemCriticalPods bool + IgnorePvcPods bool + EvictFailedBarePods bool + LabelSelector labels.Selector + PriorityThreshold *api.PriorityThreshold + NodeFit bool +} diff --git a/pkg/framework/plugins/defaultevictor/zz_generated.deepcopy.go b/pkg/framework/plugins/defaultevictor/zz_generated.deepcopy.go new file mode 100644 index 000000000..a9ff801aa --- /dev/null +++ b/pkg/framework/plugins/defaultevictor/zz_generated.deepcopy.go @@ -0,0 +1,60 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package defaultevictor + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" + api "sigs.k8s.io/descheduler/pkg/api" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DefaultEvictorArgs) DeepCopyInto(out *DefaultEvictorArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.LabelSelector != nil { + out.LabelSelector = in.LabelSelector.DeepCopySelector() + } + if in.PriorityThreshold != nil { + in, out := &in.PriorityThreshold, &out.PriorityThreshold + *out = new(api.PriorityThreshold) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefaultEvictorArgs. +func (in *DefaultEvictorArgs) DeepCopy() *DefaultEvictorArgs { + if in == nil { + return nil + } + out := new(DefaultEvictorArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DefaultEvictorArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/framework/plugins/nodeutilization/highnodeutilization_test.go b/pkg/framework/plugins/nodeutilization/highnodeutilization_test.go index 6de7650c9..1ba9b9e22 100644 --- a/pkg/framework/plugins/nodeutilization/highnodeutilization_test.go +++ b/pkg/framework/plugins/nodeutilization/highnodeutilization_test.go @@ -36,6 +36,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/utils" "sigs.k8s.io/descheduler/test" ) @@ -494,20 +495,33 @@ func TestHighNodeUtilization(t *testing.T) { eventRecorder, ) + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: true, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - testCase.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(true), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := NewHighNodeUtilization(&componentconfig.HighNodeUtilizationArgs{ @@ -634,19 +648,32 @@ func TestHighNodeUtilizationWithTaints(t *testing.T) { eventRecorder, ) + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - item.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := NewHighNodeUtilization(&componentconfig.HighNodeUtilizationArgs{ diff --git a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go index 407eb7a74..eccffefb6 100644 --- a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go +++ b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/descheduler/pkg/apis/componentconfig" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" v1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1" @@ -760,20 +761,33 @@ func TestLowNodeUtilization(t *testing.T) { eventRecorder, ) + defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: true, + } + + evictorFilter, err := defaultevictor.New( + defaultEvictorFilterArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - test.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(true), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := NewLowNodeUtilization(&componentconfig.LowNodeUtilizationArgs{ @@ -920,20 +934,33 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) { eventRecorder, ) + defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: true, + } + + evictorFilter, err := defaultevictor.New( + defaultEvictorFilterArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - item.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(true), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := NewLowNodeUtilization(&componentconfig.LowNodeUtilizationArgs{ diff --git a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go index 357783b20..63e05549f 100644 --- a/pkg/framework/plugins/podlifetime/pod_lifetime_test.go +++ b/pkg/framework/plugins/podlifetime/pod_lifetime_test.go @@ -33,6 +33,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/test" ) @@ -356,19 +357,30 @@ func TestPodLifeTime(t *testing.T) { eventRecorder, ) - evictorFilter := evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - false, - false, - tc.ignorePvcPods, - false, + defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: tc.ignorePvcPods, + EvictFailedBarePods: false, + } + + evictorFilter, err := defaultevictor.New( + defaultEvictorFilterArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + plugin, err := New(tc.args, &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }) if err != nil { diff --git a/pkg/framework/plugins/removeduplicates/removeduplicates_test.go b/pkg/framework/plugins/removeduplicates/removeduplicates_test.go index 931ef3a0b..1c25b82d0 100644 --- a/pkg/framework/plugins/removeduplicates/removeduplicates_test.go +++ b/pkg/framework/plugins/removeduplicates/removeduplicates_test.go @@ -18,11 +18,13 @@ package removeduplicates import ( "context" + "testing" + "k8s.io/client-go/tools/events" "sigs.k8s.io/descheduler/pkg/apis/componentconfig" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" - "testing" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" v1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -324,20 +326,29 @@ func TestFindDuplicatePods(t *testing.T) { nodeFit := testCase.nodefit + defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: nodeFit, + } + + evictorFilter, _ := defaultevictor.New( + defaultEvictorFilterArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - testCase.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(nodeFit), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := New(&componentconfig.RemoveDuplicatesArgs{ @@ -763,19 +774,32 @@ func TestRemoveDuplicatesUniformly(t *testing.T) { eventRecorder, ) + defaultEvictorFilterArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + evictorFilter, err := defaultevictor.New( + defaultEvictorFilterArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - testCase.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := New(&componentconfig.RemoveDuplicatesArgs{}, diff --git a/pkg/framework/plugins/removefailedpods/failedpods_test.go b/pkg/framework/plugins/removefailedpods/failedpods_test.go index 6e0b2b160..624e0c483 100644 --- a/pkg/framework/plugins/removefailedpods/failedpods_test.go +++ b/pkg/framework/plugins/removefailedpods/failedpods_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/test" ) @@ -301,16 +302,27 @@ func TestRemoveFailedPods(t *testing.T) { eventRecorder, ) - evictorFilter := evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(tc.nodeFit), + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: tc.nodeFit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + plugin, err := New(&componentconfig.RemoveFailedPodsArgs{ Reasons: tc.args.Reasons, MinPodLifetimeSeconds: tc.args.MinPodLifetimeSeconds, @@ -322,7 +334,7 @@ func TestRemoveFailedPods(t *testing.T) { &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), SharedInformerFactoryImpl: sharedInformerFactory, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }, diff --git a/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts_test.go b/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts_test.go index ad552fc29..a975f343d 100644 --- a/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts_test.go +++ b/pkg/framework/plugins/removepodshavingtoomanyrestarts/toomanyrestarts_test.go @@ -34,6 +34,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/test" ) @@ -262,22 +263,33 @@ func TestRemovePodsHavingTooManyRestarts(t *testing.T) { eventRecorder, ) - evictorFilter := evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(tc.nodeFit), + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: tc.nodeFit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + plugin, err := New( &tc.args, &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), SharedInformerFactoryImpl: sharedInformerFactory, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }) diff --git a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go index f0787f306..094338c63 100644 --- a/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go +++ b/pkg/framework/plugins/removepodsviolatinginterpodantiaffinity/pod_antiaffinity_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/utils" "sigs.k8s.io/descheduler/test" ) @@ -226,20 +227,34 @@ func TestPodAntiAffinity(t *testing.T) { false, eventRecorder, ) + + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: test.nodeFit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, SharedInformerFactoryImpl: sharedInformerFactory, - EvictorFilterImpl: evictions.NewEvictorFilter( - test.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(test.nodeFit), - ), + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), } plugin, err := New( &componentconfig.RemovePodsViolatingInterPodAntiAffinityArgs{}, diff --git a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go index 87261755f..2adbd7033 100644 --- a/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go +++ b/pkg/framework/plugins/removepodsviolatingnodeaffinity/node_affinity_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/test" ) @@ -226,20 +227,33 @@ func TestRemovePodsViolatingNodeAffinity(t *testing.T) { eventRecorder, ) + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: tc.nodefit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, SharedInformerFactoryImpl: sharedInformerFactory, - EvictorFilterImpl: evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(tc.nodefit), - ), + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), } plugin, err := New( diff --git a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go index 122f7051a..bddf0bfeb 100644 --- a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go +++ b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go @@ -34,6 +34,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/utils" "sigs.k8s.io/descheduler/test" ) @@ -363,20 +364,33 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { eventRecorder, ) + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: tc.evictLocalStoragePods, + EvictSystemCriticalPods: tc.evictSystemCriticalPods, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: tc.nodeFit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - tc.evictLocalStoragePods, - tc.evictSystemCriticalPods, - false, - false, - evictions.WithNodeFit(tc.nodeFit), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := New(&componentconfig.RemovePodsViolatingNodeTaintsArgs{ diff --git a/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint_test.go b/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint_test.go index 806073e50..86dcf37bd 100644 --- a/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint_test.go +++ b/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint/topologyspreadconstraint_test.go @@ -21,6 +21,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/test" ) @@ -1172,20 +1173,33 @@ func TestTopologySpreadConstraint(t *testing.T) { eventRecorder, ) + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: tc.nodeFit, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: fakeClient, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - false, - false, - false, - false, - evictions.WithNodeFit(tc.nodeFit), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := New( diff --git a/pkg/framework/types.go b/pkg/framework/types.go index 26998b7f0..b2ffdfb36 100644 --- a/pkg/framework/types.go +++ b/pkg/framework/types.go @@ -70,3 +70,12 @@ type BalancePlugin interface { Plugin Balance(ctx context.Context, nodes []*v1.Node) *Status } + +// EvictorPlugin defines extension points for a general evictor behavior +// Even though we name this plugin interface EvictorPlugin, it does not actually evict anything, +// This plugin is only meant to customize other actions (extension points) of the evictor, +// like filtering, sorting, and other ones that might be relevant in the future +type EvictorPlugin interface { + Plugin + Filter(pod *v1.Pod) bool +} diff --git a/pkg/utils/priority.go b/pkg/utils/priority.go index 709273dfc..058de5ea8 100644 --- a/pkg/utils/priority.go +++ b/pkg/utils/priority.go @@ -72,3 +72,23 @@ func GetPriorityFromStrategyParams(ctx context.Context, client clientset.Interfa } return } + +// GetPriorityValueFromPriorityThreshold gets priority from the given PriorityThreshold. +// It will return SystemCriticalPriority by default. +func GetPriorityValueFromPriorityThreshold(ctx context.Context, client clientset.Interface, priorityThreshold *api.PriorityThreshold) (priority int32, err error) { + if priorityThreshold == nil { + return SystemCriticalPriority, nil + } + if priorityThreshold.Value != nil { + priority = *priorityThreshold.Value + } else { + priority, err = GetPriorityFromPriorityClass(ctx, client, priorityThreshold.Name) + if err != nil { + return 0, fmt.Errorf("unable to get priority value from the priority class: %v", err) + } + } + if priority > SystemCriticalPriority { + return 0, fmt.Errorf("priority threshold can't be greater than %d", SystemCriticalPriority) + } + return +} diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index fbfb1baec..f7554f7a1 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -18,12 +18,14 @@ package e2e import ( "context" + "strings" + "testing" + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates" - "strings" - "testing" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -154,20 +156,34 @@ func TestRemoveDuplicates(t *testing.T) { false, eventRecorder, ) + + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + NodeFit: false, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientSet, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, + ) + + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + handle := &frameworkfake.HandleImpl{ ClientsetImpl: clientSet, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictions.NewEvictorFilter( - workerNodes, - getPodsAssignedToNode, - true, - false, - false, - false, - evictions.WithNodeFit(false), - ), - SharedInformerFactoryImpl: sharedInformerFactory, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), + SharedInformerFactoryImpl: sharedInformerFactory, } plugin, err := removeduplicates.New(&componentconfig.RemoveDuplicatesArgs{}, diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index 6a1b86115..e052601d3 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -14,9 +14,9 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" "sigs.k8s.io/descheduler/pkg/apis/componentconfig" - "sigs.k8s.io/descheduler/pkg/descheduler/evictions" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removefailedpods" ) @@ -78,15 +78,26 @@ func TestFailedPods(t *testing.T) { podEvictor := initPodEvictorOrFail(t, clientSet, getPodsAssignedToNode, nodes) - filter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - true, - false, - false, - false, + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientSet, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + t.Logf("Running RemoveFailedPods strategy for %s", name) plugin, err := removefailedpods.New(&componentconfig.RemoveFailedPodsArgs{ @@ -100,7 +111,7 @@ func TestFailedPods(t *testing.T) { &frameworkfake.HandleImpl{ ClientsetImpl: clientSet, PodEvictorImpl: podEvictor, - EvictorFilterImpl: filter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), SharedInformerFactoryImpl: sharedInformerFactory, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }, diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index e2319f239..cccb2463a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -51,6 +51,7 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization" "sigs.k8s.io/descheduler/pkg/framework/plugins/podlifetime" "sigs.k8s.io/descheduler/pkg/utils" @@ -200,16 +201,28 @@ func runPodLifetimePlugin( } } - evictorFilter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - false, - evictCritical, - false, - false, - evictions.WithPriorityThreshold(thresholdPriority), + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: false, + EvictSystemCriticalPods: evictCritical, + IgnorePvcPods: false, + EvictFailedBarePods: false, + PriorityThreshold: &api.PriorityThreshold{ + Value: &thresholdPriority, + }, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientset, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + maxPodLifeTimeSeconds := uint(1) plugin, err := podlifetime.New(&componentconfig.PodLifeTimeArgs{ @@ -219,7 +232,7 @@ func runPodLifetimePlugin( }, &frameworkfake.HandleImpl{ ClientsetImpl: clientset, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }) if err != nil { @@ -346,16 +359,22 @@ func TestLowNodeUtilization(t *testing.T) { // Run LowNodeUtilization plugin podEvictor := initPodEvictorOrFail(t, clientSet, getPodsAssignedToNode, nodes) - evictorFilter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - true, - false, - false, - false, + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + evictorFilter, _ := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientSet, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + }, ) - podFilter, err := podutil.NewOptions().WithFilter(evictorFilter.Filter).BuildFilterFunc() + podFilter, err := podutil.NewOptions().WithFilter(evictorFilter.(framework.EvictorPlugin).Filter).BuildFilterFunc() if err != nil { t.Errorf("Error initializing pod filter function, %v", err) } @@ -371,7 +390,7 @@ func TestLowNodeUtilization(t *testing.T) { ClientsetImpl: clientSet, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), SharedInformerFactoryImpl: sharedInformerFactory, } @@ -390,7 +409,7 @@ func TestLowNodeUtilization(t *testing.T) { waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace) - podFilter, err = podutil.NewOptions().WithFilter(evictorFilter.Filter).BuildFilterFunc() + podFilter, err = podutil.NewOptions().WithFilter(evictorFilter.(framework.EvictorPlugin).Filter).BuildFilterFunc() if err != nil { t.Errorf("Error initializing pod filter function, %v", err) } diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index 42499f14a..2e436ba03 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -36,6 +36,7 @@ import ( eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodshavingtoomanyrestarts" ) @@ -156,21 +157,32 @@ func TestTooManyRestarts(t *testing.T) { eventRecorder, ) - evictorFilter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - true, - false, - false, - false, + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + evictorFilter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientSet, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + SharedInformerFactoryImpl: sharedInformerFactory, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + plugin, err := removepodshavingtoomanyrestarts.New( &tc.args, &frameworkfake.HandleImpl{ ClientsetImpl: clientSet, PodEvictorImpl: podEvictor, - EvictorFilterImpl: evictorFilter, + EvictorFilterImpl: evictorFilter.(framework.EvictorPlugin), SharedInformerFactoryImpl: sharedInformerFactory, GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index c3b3c63b9..e436006a2 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -11,9 +11,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/descheduler/pkg/apis/componentconfig" - "sigs.k8s.io/descheduler/pkg/descheduler/evictions" "sigs.k8s.io/descheduler/pkg/framework" frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint" ) @@ -85,14 +85,23 @@ func TestTopologySpreadConstraint(t *testing.T) { // Run TopologySpreadConstraint strategy t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) - filter := evictions.NewEvictorFilter( - nodes, - getPodsAssignedToNode, - true, - false, - false, - false, + defaultevictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + + filter, err := defaultevictor.New( + defaultevictorArgs, + &frameworkfake.HandleImpl{ + ClientsetImpl: clientSet, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + }, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } plugin, err := removepodsviolatingtopologyspreadconstraint.New(&componentconfig.RemovePodsViolatingTopologySpreadConstraintArgs{ IncludeSoftConstraints: tc.constraint != v1.DoNotSchedule, @@ -100,7 +109,7 @@ func TestTopologySpreadConstraint(t *testing.T) { &frameworkfake.HandleImpl{ ClientsetImpl: clientSet, PodEvictorImpl: podEvictor, - EvictorFilterImpl: filter, + EvictorFilterImpl: filter.(framework.EvictorPlugin), GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, }, )