descheduler: wire the metrics collector with the framework handle

Node utilization refactoring

cmd/descheduler/app/options/options.go

@@ -26,6 +26,8 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	componentbaseconfig "k8s.io/component-base/config"
 	componentbaseoptions "k8s.io/component-base/config/options"
+	metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
+
 	"sigs.k8s.io/descheduler/pkg/apis/componentconfig"
 	"sigs.k8s.io/descheduler/pkg/apis/componentconfig/v1alpha1"
 	deschedulerscheme "sigs.k8s.io/descheduler/pkg/descheduler/scheme"
@@ -42,6 +44,7 @@ type DeschedulerServer struct {
 
 	Client         clientset.Interface
 	EventClient    clientset.Interface
+	MetricsClient  metricsclient.Interface
 	SecureServing  *apiserveroptions.SecureServingOptionsWithLoopback
 	DisableMetrics bool
 	EnableHTTP2    bool

kubernetes/base/rbac.yaml

@@ -24,7 +24,7 @@ rules:
   verbs: ["get", "watch", "list"]
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
-  verbs: ["create"]
+  verbs: ["create", "update"]
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]
   resourceNames: ["descheduler"]

pkg/api/types.go

@@ -41,6 +41,9 @@ type DeschedulerPolicy struct {
 
 	// MaxNoOfPodsToTotal restricts maximum of pods to be evicted total.
 	MaxNoOfPodsToEvictTotal *uint
+
+	// MetricsCollector configures collection of metrics about actual resource utilization
+	MetricsCollector MetricsCollector
 }
 
 // Namespaces carries a list of included/excluded namespaces
@@ -84,3 +87,10 @@ type PluginSet struct {
 	Enabled  []string
 	Disabled []string
 }
+
+// MetricsCollector configures collection of metrics about actual resource utilization
+type MetricsCollector struct {
+	// Enabled metrics collection from kubernetes metrics.
+	// Later, the collection can be extended to other providers.
+	Enabled bool
+}

pkg/api/v1alpha2/types.go

@@ -40,6 +40,9 @@ type DeschedulerPolicy struct {
 
 	// MaxNoOfPodsToTotal restricts maximum of pods to be evicted total.
 	MaxNoOfPodsToEvictTotal *uint `json:"maxNoOfPodsToEvictTotal,omitempty"`
+
+	// MetricsCollector configures collection of metrics about actual resource utilization
+	MetricsCollector MetricsCollector `json:"metricsCollector,omitempty"`
 }
 
 type DeschedulerProfile struct {
@@ -66,3 +69,10 @@ type PluginSet struct {
 	Enabled  []string `json:"enabled"`
 	Disabled []string `json:"disabled"`
 }
+
+// MetricsCollector configures collection of metrics about actual resource utilization
+type MetricsCollector struct {
+	// Enabled metrics collection from kubernetes metrics.
+	// Later, the collection can be extended to other providers.
+	Enabled bool
+}

pkg/descheduler/client/client.go

@@ -21,6 +21,7 @@ import (
 
 	clientset "k8s.io/client-go/kubernetes"
 	componentbaseconfig "k8s.io/component-base/config"
+	metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
 
 	// Ensure to load all auth plugins.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -28,7 +29,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-func CreateClient(clientConnection componentbaseconfig.ClientConnectionConfiguration, userAgt string) (clientset.Interface, error) {
+func createConfig(clientConnection componentbaseconfig.ClientConnectionConfiguration, userAgt string) (*rest.Config, error) {
 	var cfg *rest.Config
 	if len(clientConnection.Kubeconfig) != 0 {
 		master, err := GetMasterFromKubeconfig(clientConnection.Kubeconfig)
@@ -56,9 +57,28 @@ func CreateClient(clientConnection componentbaseconfig.ClientConnectionConfigura
 		cfg = rest.AddUserAgent(cfg, userAgt)
 	}
 
+	return cfg, nil
+}
+
+func CreateClient(clientConnection componentbaseconfig.ClientConnectionConfiguration, userAgt string) (clientset.Interface, error) {
+	cfg, err := createConfig(clientConnection, userAgt)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create config: %v", err)
+	}
+
 	return clientset.NewForConfig(cfg)
 }
 
+func CreateMetricsClient(clientConnection componentbaseconfig.ClientConnectionConfiguration, userAgt string) (metricsclient.Interface, error) {
+	cfg, err := createConfig(clientConnection, userAgt)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create config: %v", err)
+	}
+
+	// Create the metrics clientset to access the metrics.k8s.io API
+	return metricsclient.NewForConfig(cfg)
+}
+
 func GetMasterFromKubeconfig(filename string) (string, error) {
 	config, err := clientcmd.LoadFromFile(filename)
 	if err != nil {

pkg/descheduler/descheduler.go

@@ -25,41 +25,39 @@ import (
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
+	v1 "k8s.io/api/core/v1"
+	policy "k8s.io/api/policy/v1"
+	schedulingv1 "k8s.io/api/scheduling/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/informers"
+	clientset "k8s.io/client-go/kubernetes"
+	fakeclientset "k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/events"
 	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/klog/v2"
 
-	v1 "k8s.io/api/core/v1"
-	policy "k8s.io/api/policy/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
-	utilversion "k8s.io/apimachinery/pkg/util/version"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
-	fakeclientset "k8s.io/client-go/kubernetes/fake"
-	listersv1 "k8s.io/client-go/listers/core/v1"
-	schedulingv1 "k8s.io/client-go/listers/scheduling/v1"
-	core "k8s.io/client-go/testing"
-
-	"sigs.k8s.io/descheduler/pkg/descheduler/client"
-	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
-	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
-	"sigs.k8s.io/descheduler/pkg/tracing"
-	"sigs.k8s.io/descheduler/pkg/utils"
-	"sigs.k8s.io/descheduler/pkg/version"
-
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
 	"sigs.k8s.io/descheduler/metrics"
 	"sigs.k8s.io/descheduler/pkg/api"
+	"sigs.k8s.io/descheduler/pkg/descheduler/client"
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
+	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
+	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
 	"sigs.k8s.io/descheduler/pkg/framework/pluginregistry"
 	frameworkprofile "sigs.k8s.io/descheduler/pkg/framework/profile"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
+	"sigs.k8s.io/descheduler/pkg/tracing"
+	"sigs.k8s.io/descheduler/pkg/utils"
+	"sigs.k8s.io/descheduler/pkg/version"
 )
 
 type eprunner func(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status
@@ -71,24 +69,70 @@ type profileRunner struct {
 
 type descheduler struct {
 	rs                     *options.DeschedulerServer
-	podLister              listersv1.PodLister
-	nodeLister             listersv1.NodeLister
-	namespaceLister        listersv1.NamespaceLister
-	priorityClassLister    schedulingv1.PriorityClassLister
+	ir                     *informerResources
 	getPodsAssignedToNode  podutil.GetPodsAssignedToNodeFunc
 	sharedInformerFactory  informers.SharedInformerFactory
 	deschedulerPolicy      *api.DeschedulerPolicy
 	eventRecorder          events.EventRecorder
 	podEvictor             *evictions.PodEvictor
 	podEvictionReactionFnc func(*fakeclientset.Clientset) func(action core.Action) (bool, runtime.Object, error)
+	metricsCollector       *metricscollector.MetricsCollector
+}
+
+type informerResources struct {
+	sharedInformerFactory informers.SharedInformerFactory
+	resourceToInformer    map[schema.GroupVersionResource]informers.GenericInformer
+}
+
+func newInformerResources(sharedInformerFactory informers.SharedInformerFactory) *informerResources {
+	return &informerResources{
+		sharedInformerFactory: sharedInformerFactory,
+		resourceToInformer:    make(map[schema.GroupVersionResource]informers.GenericInformer),
+	}
+}
+
+func (ir *informerResources) Uses(resources ...schema.GroupVersionResource) error {
+	for _, resource := range resources {
+		informer, err := ir.sharedInformerFactory.ForResource(resource)
+		if err != nil {
+			return err
+		}
+
+		ir.resourceToInformer[resource] = informer
+	}
+	return nil
+}
+
+// CopyTo Copy informer subscriptions to the new factory and objects to the fake client so that the backing caches are populated for when listers are used.
+func (ir *informerResources) CopyTo(fakeClient *fakeclientset.Clientset, newFactory informers.SharedInformerFactory) error {
+	for resource, informer := range ir.resourceToInformer {
+		_, err := newFactory.ForResource(resource)
+		if err != nil {
+			return fmt.Errorf("error getting resource %s: %w", resource, err)
+		}
+
+		objects, err := informer.Lister().List(labels.Everything())
+		if err != nil {
+			return fmt.Errorf("error listing %s: %w", informer, err)
+		}
+
+		for _, object := range objects {
+			fakeClient.Tracker().Add(object)
+		}
+	}
+	return nil
 }
 
 func newDescheduler(rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, eventRecorder events.EventRecorder, sharedInformerFactory informers.SharedInformerFactory) (*descheduler, error) {
 	podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
-	podLister := sharedInformerFactory.Core().V1().Pods().Lister()
-	nodeLister := sharedInformerFactory.Core().V1().Nodes().Lister()
-	namespaceLister := sharedInformerFactory.Core().V1().Namespaces().Lister()
-	priorityClassLister := sharedInformerFactory.Scheduling().V1().PriorityClasses().Lister()
+
+	ir := newInformerResources(sharedInformerFactory)
+	ir.Uses(v1.SchemeGroupVersion.WithResource("pods"),
+		v1.SchemeGroupVersion.WithResource("nodes"),
+		// Future work could be to let each plugin declare what type of resources it needs; that way dry runs would stay
+		// consistent with the real runs without having to keep the list here in sync.
+		v1.SchemeGroupVersion.WithResource("namespaces"),                // Used by the defaultevictor plugin
+		schedulingv1.SchemeGroupVersion.WithResource("priorityclasses")) // Used by the defaultevictor plugin
 
 	getPodsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer)
 	if err != nil {
@@ -107,18 +151,21 @@ func newDescheduler(rs *options.DeschedulerServer, deschedulerPolicy *api.Desche
 			WithMetricsEnabled(!rs.DisableMetrics),
 	)
 
+	var metricsCollector *metricscollector.MetricsCollector
+	if deschedulerPolicy.MetricsCollector.Enabled {
+		metricsCollector = metricscollector.NewMetricsCollector(rs.Client, rs.MetricsClient)
+	}
+
 	return &descheduler{
 		rs:                     rs,
-		podLister:              podLister,
-		nodeLister:             nodeLister,
-		namespaceLister:        namespaceLister,
-		priorityClassLister:    priorityClassLister,
+		ir:                     ir,
 		getPodsAssignedToNode:  getPodsAssignedToNode,
 		sharedInformerFactory:  sharedInformerFactory,
 		deschedulerPolicy:      deschedulerPolicy,
 		eventRecorder:          eventRecorder,
 		podEvictor:             podEvictor,
 		podEvictionReactionFnc: podEvictionReactionFnc,
+		metricsCollector:       metricsCollector,
 	}, nil
 }
 
@@ -146,13 +193,14 @@ func (d *descheduler) runDeschedulerLoop(ctx context.Context, nodes []*v1.Node)
 		fakeClient := fakeclientset.NewSimpleClientset()
 		// simulate a pod eviction by deleting a pod
 		fakeClient.PrependReactor("create", "pods", d.podEvictionReactionFnc(fakeClient))
-		err := cachedClient(d.rs.Client, fakeClient, d.podLister, d.nodeLister, d.namespaceLister, d.priorityClassLister)
+		fakeSharedInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0)
+
+		err := d.ir.CopyTo(fakeClient, fakeSharedInformerFactory)
 		if err != nil {
 			return err
 		}
 
 		// create a new instance of the shared informer factor from the cached client
-		fakeSharedInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0)
 		// register the pod informer, otherwise it will not get running
 		d.getPodsAssignedToNode, err = podutil.BuildGetPodsAssignedToNodeFunc(fakeSharedInformerFactory.Core().V1().Pods().Informer())
 		if err != nil {
@@ -197,6 +245,7 @@ func (d *descheduler) runProfiles(ctx context.Context, client clientset.Interfac
 			frameworkprofile.WithSharedInformerFactory(d.sharedInformerFactory),
 			frameworkprofile.WithPodEvictor(d.podEvictor),
 			frameworkprofile.WithGetPodsAssignedToNodeFnc(d.getPodsAssignedToNode),
+			frameworkprofile.WithMetricsCollector(d.metricsCollector),
 		)
 		if err != nil {
 			klog.ErrorS(err, "unable to create a profile", "profile", profile.Name)
@@ -261,6 +310,14 @@ func Run(ctx context.Context, rs *options.DeschedulerServer) error {
 		return err
 	}
 
+	if deschedulerPolicy.MetricsCollector.Enabled {
+		metricsClient, err := client.CreateMetricsClient(clientConnection, "descheduler")
+		if err != nil {
+			return err
+		}
+		rs.MetricsClient = metricsClient
+	}
+
 	runFn := func() error {
 		return RunDeschedulerStrategies(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion)
 	}
@@ -336,62 +393,6 @@ func podEvictionReactionFnc(fakeClient *fakeclientset.Clientset) func(action cor
 	}
 }
 
-func cachedClient(
-	realClient clientset.Interface,
-	fakeClient *fakeclientset.Clientset,
-	podLister listersv1.PodLister,
-	nodeLister listersv1.NodeLister,
-	namespaceLister listersv1.NamespaceLister,
-	priorityClassLister schedulingv1.PriorityClassLister,
-) error {
-	klog.V(3).Infof("Pulling resources for the cached client from the cluster")
-	pods, err := podLister.List(labels.Everything())
-	if err != nil {
-		return fmt.Errorf("unable to list pods: %v", err)
-	}
-
-	for _, item := range pods {
-		if _, err := fakeClient.CoreV1().Pods(item.Namespace).Create(context.TODO(), item, metav1.CreateOptions{}); err != nil {
-			return fmt.Errorf("unable to copy pod: %v", err)
-		}
-	}
-
-	nodes, err := nodeLister.List(labels.Everything())
-	if err != nil {
-		return fmt.Errorf("unable to list nodes: %v", err)
-	}
-
-	for _, item := range nodes {
-		if _, err := fakeClient.CoreV1().Nodes().Create(context.TODO(), item, metav1.CreateOptions{}); err != nil {
-			return fmt.Errorf("unable to copy node: %v", err)
-		}
-	}
-
-	namespaces, err := namespaceLister.List(labels.Everything())
-	if err != nil {
-		return fmt.Errorf("unable to list namespaces: %v", err)
-	}
-
-	for _, item := range namespaces {
-		if _, err := fakeClient.CoreV1().Namespaces().Create(context.TODO(), item, metav1.CreateOptions{}); err != nil {
-			return fmt.Errorf("unable to copy namespace: %v", err)
-		}
-	}
-
-	priorityClasses, err := priorityClassLister.List(labels.Everything())
-	if err != nil {
-		return fmt.Errorf("unable to list priorityclasses: %v", err)
-	}
-
-	for _, item := range priorityClasses {
-		if _, err := fakeClient.SchedulingV1().PriorityClasses().Create(context.TODO(), item, metav1.CreateOptions{}); err != nil {
-			return fmt.Errorf("unable to copy priorityclass: %v", err)
-		}
-	}
-
-	return nil
-}
-
 func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string) error {
 	var span trace.Span
 	ctx, span = tracing.Tracer().Start(ctx, "RunDeschedulerStrategies")
@@ -424,11 +425,18 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 	sharedInformerFactory.Start(ctx.Done())
 	sharedInformerFactory.WaitForCacheSync(ctx.Done())
 
+	go func() {
+		klog.V(2).Infof("Starting metrics collector")
+		descheduler.metricsCollector.Run(ctx)
+		klog.V(2).Infof("Stopped metrics collector")
+	}()
+
 	wait.NonSlidingUntil(func() {
 		// A next context is created here intentionally to avoid nesting the spans via context.
 		sCtx, sSpan := tracing.Tracer().Start(ctx, "NonSlidingUntil")
 		defer sSpan.End()
-		nodes, err := nodeutil.ReadyNodes(sCtx, rs.Client, descheduler.nodeLister, nodeSelector)
+
+		nodes, err := nodeutil.ReadyNodes(sCtx, rs.Client, descheduler.sharedInformerFactory.Core().V1().Nodes().Lister(), nodeSelector)
 		if err != nil {
 			sSpan.AddEvent("Failed to detect ready nodes", trace.WithAttributes(attribute.String("err", err.Error())))
 			klog.Error(err)

pkg/descheduler/descheduler_test.go

@@ -16,11 +16,16 @@ import (
 	fakeclientset "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
 	"k8s.io/klog/v2"
+	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
+	fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake"
 	utilptr "k8s.io/utils/ptr"
+
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
 	"sigs.k8s.io/descheduler/pkg/api"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 	"sigs.k8s.io/descheduler/pkg/framework/pluginregistry"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
+	"sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingnodetaints"
 	"sigs.k8s.io/descheduler/pkg/utils"
@@ -33,6 +38,7 @@ func initPluginRegistry() {
 	pluginregistry.Register(removeduplicates.PluginName, removeduplicates.New, &removeduplicates.RemoveDuplicates{}, &removeduplicates.RemoveDuplicatesArgs{}, removeduplicates.ValidateRemoveDuplicatesArgs, removeduplicates.SetDefaults_RemoveDuplicatesArgs, pluginregistry.PluginRegistry)
 	pluginregistry.Register(defaultevictor.PluginName, defaultevictor.New, &defaultevictor.DefaultEvictor{}, &defaultevictor.DefaultEvictorArgs{}, defaultevictor.ValidateDefaultEvictorArgs, defaultevictor.SetDefaults_DefaultEvictorArgs, pluginregistry.PluginRegistry)
 	pluginregistry.Register(removepodsviolatingnodetaints.PluginName, removepodsviolatingnodetaints.New, &removepodsviolatingnodetaints.RemovePodsViolatingNodeTaints{}, &removepodsviolatingnodetaints.RemovePodsViolatingNodeTaintsArgs{}, removepodsviolatingnodetaints.ValidateRemovePodsViolatingNodeTaintsArgs, removepodsviolatingnodetaints.SetDefaults_RemovePodsViolatingNodeTaintsArgs, pluginregistry.PluginRegistry)
+	pluginregistry.Register(nodeutilization.LowNodeUtilizationPluginName, nodeutilization.NewLowNodeUtilization, &nodeutilization.LowNodeUtilization{}, &nodeutilization.LowNodeUtilizationArgs{}, nodeutilization.ValidateLowNodeUtilizationArgs, nodeutilization.SetDefaults_LowNodeUtilizationArgs, pluginregistry.PluginRegistry)
 }
 
 func removePodsViolatingNodeTaintsPolicy() *api.DeschedulerPolicy {
@@ -99,6 +105,44 @@ func removeDuplicatesPolicy() *api.DeschedulerPolicy {
 	}
 }
 
+func lowNodeUtilizationPolicy(thresholds, targetThresholds api.ResourceThresholds, metricsEnabled bool) *api.DeschedulerPolicy {
+	return &api.DeschedulerPolicy{
+		Profiles: []api.DeschedulerProfile{
+			{
+				Name: "Profile",
+				PluginConfigs: []api.PluginConfig{
+					{
+						Name: nodeutilization.LowNodeUtilizationPluginName,
+						Args: &nodeutilization.LowNodeUtilizationArgs{
+							Thresholds:       thresholds,
+							TargetThresholds: targetThresholds,
+							MetricsUtilization: nodeutilization.MetricsUtilization{
+								MetricsServer: metricsEnabled,
+							},
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: &defaultevictor.DefaultEvictorArgs{},
+					},
+				},
+				Plugins: api.Plugins{
+					Filter: api.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Balance: api.PluginSet{
+						Enabled: []string{
+							nodeutilization.LowNodeUtilizationPluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func initDescheduler(t *testing.T, ctx context.Context, internalDeschedulerPolicy *api.DeschedulerPolicy, objects ...runtime.Object) (*options.DeschedulerServer, *descheduler, *fakeclientset.Clientset) {
 	client := fakeclientset.NewSimpleClientset(objects...)
 	eventClient := fakeclientset.NewSimpleClientset(objects...)
@@ -539,3 +583,76 @@ func TestDeschedulingLimits(t *testing.T) {
 		})
 	}
 }
+
+func TestLoadAwareDescheduling(t *testing.T) {
+	initPluginRegistry()
+
+	ownerRef1 := test.GetReplicaSetOwnerRefList()
+	updatePod := func(pod *v1.Pod) {
+		pod.ObjectMeta.OwnerReferences = ownerRef1
+	}
+
+	ctx := context.Background()
+	node1 := test.BuildTestNode("n1", 2000, 3000, 10, taintNodeNoSchedule)
+	node2 := test.BuildTestNode("n2", 2000, 3000, 10, nil)
+	nodes := []*v1.Node{node1, node2}
+
+	p1 := test.BuildTestPod("p1", 300, 0, node1.Name, updatePod)
+	p2 := test.BuildTestPod("p2", 300, 0, node1.Name, updatePod)
+	p3 := test.BuildTestPod("p3", 300, 0, node1.Name, updatePod)
+	p4 := test.BuildTestPod("p4", 300, 0, node1.Name, updatePod)
+	p5 := test.BuildTestPod("p5", 300, 0, node1.Name, updatePod)
+
+	ctxCancel, cancel := context.WithCancel(ctx)
+	_, descheduler, client := initDescheduler(
+		t,
+		ctxCancel,
+		lowNodeUtilizationPolicy(
+			api.ResourceThresholds{
+				v1.ResourceCPU:  30,
+				v1.ResourcePods: 30,
+			},
+			api.ResourceThresholds{
+				v1.ResourceCPU:  50,
+				v1.ResourcePods: 50,
+			},
+			true, // enabled metrics utilization
+		),
+		node1, node2, p1, p2, p3, p4, p5)
+	defer cancel()
+
+	nodemetricses := []*v1beta1.NodeMetrics{
+		test.BuildNodeMetrics("n1", 2400, 3000),
+		test.BuildNodeMetrics("n2", 400, 0),
+	}
+
+	podmetricses := []*v1beta1.PodMetrics{
+		test.BuildPodMetrics("p1", 400, 0),
+		test.BuildPodMetrics("p2", 400, 0),
+		test.BuildPodMetrics("p3", 400, 0),
+		test.BuildPodMetrics("p4", 400, 0),
+		test.BuildPodMetrics("p5", 400, 0),
+	}
+
+	var metricsObjs []runtime.Object
+	for _, nodemetrics := range nodemetricses {
+		metricsObjs = append(metricsObjs, nodemetrics)
+	}
+	for _, podmetrics := range podmetricses {
+		metricsObjs = append(metricsObjs, podmetrics)
+	}
+
+	metricsClientset := fakemetricsclient.NewSimpleClientset(metricsObjs...)
+	descheduler.metricsCollector = metricscollector.NewMetricsCollector(client, metricsClientset)
+	descheduler.metricsCollector.Collect(ctx)
+
+	err := descheduler.runDeschedulerLoop(ctx, nodes)
+	if err != nil {
+		t.Fatalf("Unable to run a descheduling loop: %v", err)
+	}
+	totalEs := descheduler.podEvictor.TotalEvicted()
+	if totalEs != 2  {
+		t.Fatalf("Expected %v evictions in total, got %v instead", 2, totalEs)
+	}
+	t.Logf("Total evictions: %v", totalEs)
+}

pkg/descheduler/metricscollector/metricscollector.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
+	metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	utilptr "k8s.io/utils/ptr"
+)
+
+const (
+	beta float64 = 0.9
+)
+
+type MetricsCollector struct {
+	clientset        kubernetes.Interface
+	metricsClientset metricsclient.Interface
+
+	nodes map[string]map[v1.ResourceName]*resource.Quantity
+
+	mu sync.Mutex
+}
+
+func NewMetricsCollector(clientset kubernetes.Interface, metricsClientset metricsclient.Interface) *MetricsCollector {
+	return &MetricsCollector{
+		clientset:        clientset,
+		metricsClientset: metricsClientset,
+		nodes:            make(map[string]map[v1.ResourceName]*resource.Quantity),
+	}
+}
+
+func (mc *MetricsCollector) Run(ctx context.Context) {
+	wait.NonSlidingUntil(func() {
+		mc.Collect(ctx)
+	}, 5*time.Second, ctx.Done())
+}
+
+func weightedAverage(prevValue, value int64) int64 {
+	return int64(math.Floor(beta*float64(prevValue) + (1-beta)*float64(value)))
+}
+
+func (mc *MetricsCollector) NodeUsage(node *v1.Node) (map[v1.ResourceName]*resource.Quantity, error) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	if _, exists := mc.nodes[node.Name]; !exists {
+		klog.V(4).Infof("unable to find node %q in the collected metrics", node.Name)
+		return nil, fmt.Errorf("unable to find node %q in the collected metrics", node.Name)
+	}
+	return map[v1.ResourceName]*resource.Quantity{
+		v1.ResourceCPU:    utilptr.To[resource.Quantity](mc.nodes[node.Name][v1.ResourceCPU].DeepCopy()),
+		v1.ResourceMemory: utilptr.To[resource.Quantity](mc.nodes[node.Name][v1.ResourceMemory].DeepCopy()),
+	}, nil
+}
+
+func (mc *MetricsCollector) MetricsClient() metricsclient.Interface {
+	return mc.metricsClientset
+}
+
+func (mc *MetricsCollector) Collect(ctx context.Context) error {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	nodes, err := mc.clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to list nodes: %v", err)
+	}
+
+	for _, node := range nodes.Items {
+		metrics, err := mc.metricsClientset.MetricsV1beta1().NodeMetricses().Get(context.TODO(), node.Name, metav1.GetOptions{})
+		if err != nil {
+			fmt.Printf("Error fetching metrics for node %s: %v\n", node.Name, err)
+			// No entry -> duplicate the previous value -> do nothing as beta*PV + (1-beta)*PV = PV
+			continue
+		}
+
+		if _, exists := mc.nodes[node.Name]; !exists {
+			mc.nodes[node.Name] = map[v1.ResourceName]*resource.Quantity{
+				v1.ResourceCPU:    utilptr.To[resource.Quantity](metrics.Usage.Cpu().DeepCopy()),
+				v1.ResourceMemory: utilptr.To[resource.Quantity](metrics.Usage.Memory().DeepCopy()),
+			}
+		} else {
+			// get MilliValue to reduce loss of precision
+			mc.nodes[node.Name][v1.ResourceCPU].SetMilli(
+				weightedAverage(mc.nodes[node.Name][v1.ResourceCPU].MilliValue(), metrics.Usage.Cpu().MilliValue()),
+			)
+			mc.nodes[node.Name][v1.ResourceMemory].SetMilli(
+				weightedAverage(mc.nodes[node.Name][v1.ResourceMemory].MilliValue(), metrics.Usage.Memory().MilliValue()),
+			)
+		}
+
+		// Display CPU and memory usage
+		// fmt.Printf("%s: %vm, %vMi\n", node.Name, metrics.Usage.Cpu().MilliValue(), metrics.Usage.Memory().Value()/(1024*1024))
+		// fmt.Printf("%s: %vm, %vMi\n", node.Name, mc.nodes[node.Name][v1.ResourceCPU].MilliValue(), mc.nodes[node.Name][v1.ResourceMemory].Value()/(1024*1024))
+	}
+
+	fmt.Printf("--\n")
+
+	return nil
+}

pkg/descheduler/metricscollector/metricscollector_test.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metricscollector
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	fakeclientset "k8s.io/client-go/kubernetes/fake"
+	fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake"
+	"sigs.k8s.io/descheduler/test"
+)
+
+func TestMetricsCollector1(t *testing.T) {
+	kubeconfig := os.Getenv("KUBECONFIG")
+
+	// Use the kubeconfig to build the Kubernetes client
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Create the standard Kubernetes clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	// Create the metrics clientset to access the metrics.k8s.io API
+	metricsClientset, err := metricsclient.NewForConfig(config)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	collector := NewMetricsCollector(clientset, metricsClientset)
+	collector.Run(context.TODO())
+	// collector.Collect(context.TODO())
+}
+
+func checkCpuNodeUsage(t *testing.T, usage map[v1.ResourceName]*resource.Quantity, millicpu int64) {
+	t.Logf("current node cpu usage: %v\n", usage[v1.ResourceCPU].MilliValue())
+	if usage[v1.ResourceCPU].MilliValue() != millicpu {
+		t.Fatalf("cpu node usage expected to be %v, got %v instead", millicpu, usage[v1.ResourceCPU].MilliValue())
+	}
+}
+
+func TestMetricsCollector2(t *testing.T) {
+	gvr := schema.GroupVersionResource{Group: "metrics.k8s.io", Version: "v1beta1", Resource: "nodemetricses"}
+
+	n1 := test.BuildTestNode("n1", 2000, 3000, 10, nil)
+	n2 := test.BuildTestNode("n2", 2000, 3000, 10, nil)
+	n3 := test.BuildTestNode("n3", 2000, 3000, 10, nil)
+
+	n1metrics := test.BuildNodeMetrics("n1", 400, 1714978816)
+	n2metrics := test.BuildNodeMetrics("n2", 1400, 1714978816)
+	n3metrics := test.BuildNodeMetrics("n3", 300, 1714978816)
+
+	clientset := fakeclientset.NewSimpleClientset(n1, n2, n3)
+	metricsClientset := fakemetricsclient.NewSimpleClientset(n1metrics, n2metrics, n3metrics)
+
+	t.Logf("Set initial node cpu usage to 1400")
+	collector := NewMetricsCollector(clientset, metricsClientset)
+	collector.Collect(context.TODO())
+	nodesUsage, _ := collector.NodeUsage(n2)
+	checkCpuNodeUsage(t, nodesUsage, 1400)
+
+	t.Logf("Set current node cpu usage to 500")
+	n2metrics.Usage[v1.ResourceCPU] = *resource.NewMilliQuantity(500, resource.DecimalSI)
+	metricsClientset.Tracker().Update(gvr, n2metrics, "")
+	collector.Collect(context.TODO())
+	nodesUsage, _ = collector.NodeUsage(n2)
+	checkCpuNodeUsage(t, nodesUsage, 1310)
+
+	t.Logf("Set current node cpu usage to 500")
+	n2metrics.Usage[v1.ResourceCPU] = *resource.NewMilliQuantity(900, resource.DecimalSI)
+	metricsClientset.Tracker().Update(gvr, n2metrics, "")
+	collector.Collect(context.TODO())
+	nodesUsage, _ = collector.NodeUsage(n2)
+	checkCpuNodeUsage(t, nodesUsage, 1268)
+}

pkg/descheduler/node/node.go

@@ -218,7 +218,12 @@ func fitsRequest(nodeIndexer podutil.GetPodsAssignedToNodeFunc, pod *v1.Pod, nod
 		resourceNames = append(resourceNames, name)
 	}
 
-	availableResources, err := nodeAvailableResources(nodeIndexer, node, resourceNames)
+	availableResources, err := nodeAvailableResources(nodeIndexer, node, resourceNames,
+		func(pod *v1.Pod) (v1.ResourceList, error) {
+			req, _ := utils.PodRequestsAndLimits(pod)
+			return req, nil
+		},
+	)
 	if err != nil {
 		return false, err
 	}
@@ -239,12 +244,15 @@ func fitsRequest(nodeIndexer podutil.GetPodsAssignedToNodeFunc, pod *v1.Pod, nod
 }
 
 // nodeAvailableResources returns resources mapped to the quanitity available on the node.
-func nodeAvailableResources(nodeIndexer podutil.GetPodsAssignedToNodeFunc, node *v1.Node, resourceNames []v1.ResourceName) (map[v1.ResourceName]*resource.Quantity, error) {
+func nodeAvailableResources(nodeIndexer podutil.GetPodsAssignedToNodeFunc, node *v1.Node, resourceNames []v1.ResourceName, podUtilization podutil.PodUtilizationFnc) (map[v1.ResourceName]*resource.Quantity, error) {
 	podsOnNode, err := podutil.ListPodsOnANode(node.Name, nodeIndexer, nil)
 	if err != nil {
 		return nil, err
 	}
-	nodeUtilization := NodeUtilization(podsOnNode, resourceNames)
+	nodeUtilization, err := NodeUtilization(podsOnNode, resourceNames, podUtilization)
+	if err != nil {
+		return nil, err
+	}
 	remainingResources := map[v1.ResourceName]*resource.Quantity{
 		v1.ResourceCPU:    resource.NewMilliQuantity(node.Status.Allocatable.Cpu().MilliValue()-nodeUtilization[v1.ResourceCPU].MilliValue(), resource.DecimalSI),
 		v1.ResourceMemory: resource.NewQuantity(node.Status.Allocatable.Memory().Value()-nodeUtilization[v1.ResourceMemory].Value(), resource.BinarySI),
@@ -265,31 +273,34 @@ func nodeAvailableResources(nodeIndexer podutil.GetPodsAssignedToNodeFunc, node
 }
 
 // NodeUtilization returns the resources requested by the given pods. Only resources supplied in the resourceNames parameter are calculated.
-func NodeUtilization(pods []*v1.Pod, resourceNames []v1.ResourceName) map[v1.ResourceName]*resource.Quantity {
-	totalReqs := map[v1.ResourceName]*resource.Quantity{
+func NodeUtilization(pods []*v1.Pod, resourceNames []v1.ResourceName, podUtilization podutil.PodUtilizationFnc) (map[v1.ResourceName]*resource.Quantity, error) {
+	totalUtilization := map[v1.ResourceName]*resource.Quantity{
 		v1.ResourceCPU:    resource.NewMilliQuantity(0, resource.DecimalSI),
 		v1.ResourceMemory: resource.NewQuantity(0, resource.BinarySI),
 		v1.ResourcePods:   resource.NewQuantity(int64(len(pods)), resource.DecimalSI),
 	}
 	for _, name := range resourceNames {
 		if !IsBasicResource(name) {
-			totalReqs[name] = resource.NewQuantity(0, resource.DecimalSI)
+			totalUtilization[name] = resource.NewQuantity(0, resource.DecimalSI)
 		}
 	}
 
 	for _, pod := range pods {
-		req, _ := utils.PodRequestsAndLimits(pod)
+		podUtil, err := podUtilization(pod)
+		if err != nil {
+			return nil, err
+		}
 		for _, name := range resourceNames {
-			quantity, ok := req[name]
+			quantity, ok := podUtil[name]
 			if ok && name != v1.ResourcePods {
 				// As Quantity.Add says: Add adds the provided y quantity to the current value. If the current value is zero,
 				// the format of the quantity will be updated to the format of y.
-				totalReqs[name].Add(quantity)
+				totalUtilization[name].Add(quantity)
 			}
 		}
 	}
 
-	return totalReqs
+	return totalUtilization, nil
 }
 
 // IsBasicResource checks if resource is basic native.

pkg/descheduler/pod/pods.go

@@ -39,6 +39,9 @@ type FilterFunc func(*v1.Pod) bool
 // as input and returns the pods that assigned to the node.
 type GetPodsAssignedToNodeFunc func(string, FilterFunc) ([]*v1.Pod, error)
 
+// PodUtilizationFnc is a function for getting pod's utilization. E.g. requested resources of utilization from metrics.
+type PodUtilizationFnc func(pod *v1.Pod) (v1.ResourceList, error)
+
 // WrapFilterFuncs wraps a set of FilterFunc in one.
 func WrapFilterFuncs(filters ...FilterFunc) FilterFunc {
 	return func(pod *v1.Pod) bool {

pkg/framework/fake/fake.go

@@ -8,6 +8,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
 )
@@ -18,6 +19,7 @@ type HandleImpl struct {
 	SharedInformerFactoryImpl     informers.SharedInformerFactory
 	EvictorFilterImpl             frameworktypes.EvictorPlugin
 	PodEvictorImpl                *evictions.PodEvictor
+	MetricsCollectorImpl              *metricscollector.MetricsCollector
 }
 
 var _ frameworktypes.Handle = &HandleImpl{}
@@ -26,6 +28,10 @@ func (hi *HandleImpl) ClientSet() clientset.Interface {
 	return hi.ClientsetImpl
 }
 
+func (hi *HandleImpl) MetricsCollector() *metricscollector.MetricsCollector {
+	return hi.MetricsCollectorImpl
+}
+
 func (hi *HandleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc {
 	return hi.GetPodsAssignedToNodeFuncImpl
 }

pkg/framework/plugins/nodeutilization/highnodeutilization.go

@@ -38,9 +38,13 @@ const HighNodeUtilizationPluginName = "HighNodeUtilization"
 // Note that CPU/Memory requests are used to calculate nodes' utilization and not the actual resource usage.
 
 type HighNodeUtilization struct {
-	handle    frameworktypes.Handle
-	args      *HighNodeUtilizationArgs
-	podFilter func(pod *v1.Pod) bool
+	handle                   frameworktypes.Handle
+	args                     *HighNodeUtilizationArgs
+	podFilter                func(pod *v1.Pod) bool
+	underutilizationCriteria []interface{}
+	resourceNames            []v1.ResourceName
+	targetThresholds         api.ResourceThresholds
+	usageSnapshot            usageClient
 }
 
 var _ frameworktypes.BalancePlugin = &HighNodeUtilization{}
@@ -52,6 +56,21 @@ func NewHighNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (
 		return nil, fmt.Errorf("want args to be of type HighNodeUtilizationArgs, got %T", args)
 	}
 
+	targetThresholds := make(api.ResourceThresholds)
+	setDefaultForThresholds(highNodeUtilizatioArgs.Thresholds, targetThresholds)
+	resourceNames := getResourceNames(targetThresholds)
+
+	underutilizationCriteria := []interface{}{
+		"CPU", highNodeUtilizatioArgs.Thresholds[v1.ResourceCPU],
+		"Mem", highNodeUtilizatioArgs.Thresholds[v1.ResourceMemory],
+		"Pods", highNodeUtilizatioArgs.Thresholds[v1.ResourcePods],
+	}
+	for name := range highNodeUtilizatioArgs.Thresholds {
+		if !nodeutil.IsBasicResource(name) {
+			underutilizationCriteria = append(underutilizationCriteria, string(name), int64(highNodeUtilizatioArgs.Thresholds[name]))
+		}
+	}
+
 	podFilter, err := podutil.NewOptions().
 		WithFilter(handle.Evictor().Filter).
 		BuildFilterFunc()
@@ -60,9 +79,13 @@ func NewHighNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (
 	}
 
 	return &HighNodeUtilization{
-		handle:    handle,
-		args:      highNodeUtilizatioArgs,
-		podFilter: podFilter,
+		handle:                   handle,
+		args:                     highNodeUtilizatioArgs,
+		resourceNames:            resourceNames,
+		targetThresholds:         targetThresholds,
+		underutilizationCriteria: underutilizationCriteria,
+		podFilter:                podFilter,
+		usageSnapshot:            newRequestedUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc()),
 	}, nil
 }
 
@@ -73,15 +96,15 @@ func (h *HighNodeUtilization) Name() string {
 
 // Balance extension point implementation for the plugin
 func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status {
-	thresholds := h.args.Thresholds
-	targetThresholds := make(api.ResourceThresholds)
-
-	setDefaultForThresholds(thresholds, targetThresholds)
-	resourceNames := getResourceNames(targetThresholds)
+	if err := h.usageSnapshot.capture(nodes); err != nil {
+		return &frameworktypes.Status{
+			Err: fmt.Errorf("error getting node usage: %v", err),
+		}
+	}
 
 	sourceNodes, highNodes := classifyNodes(
-		getNodeUsage(nodes, resourceNames, h.handle.GetPodsAssignedToNodeFunc()),
-		getNodeThresholds(nodes, thresholds, targetThresholds, resourceNames, h.handle.GetPodsAssignedToNodeFunc(), false),
+		getNodeUsage(nodes, h.usageSnapshot),
+		getNodeThresholds(nodes, h.args.Thresholds, h.targetThresholds, h.resourceNames, false, h.usageSnapshot),
 		func(node *v1.Node, usage NodeUsage, threshold NodeThresholds) bool {
 			return isNodeWithLowUtilization(usage, threshold.lowResourceThreshold)
 		},
@@ -94,18 +117,7 @@ func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fr
 		})
 
 	// log message in one line
-	keysAndValues := []interface{}{
-		"CPU", thresholds[v1.ResourceCPU],
-		"Mem", thresholds[v1.ResourceMemory],
-		"Pods", thresholds[v1.ResourcePods],
-	}
-	for name := range thresholds {
-		if !nodeutil.IsBasicResource(name) {
-			keysAndValues = append(keysAndValues, string(name), int64(thresholds[name]))
-		}
-	}
-
-	klog.V(1).InfoS("Criteria for a node below target utilization", keysAndValues...)
+	klog.V(1).InfoS("Criteria for a node below target utilization", h.underutilizationCriteria...)
 	klog.V(1).InfoS("Number of underutilized nodes", "totalNumber", len(sourceNodes))
 
 	if len(sourceNodes) == 0 {
@@ -147,8 +159,10 @@ func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fr
 		h.handle.Evictor(),
 		evictions.EvictOptions{StrategyName: HighNodeUtilizationPluginName},
 		h.podFilter,
-		resourceNames,
-		continueEvictionCond)
+		h.resourceNames,
+		continueEvictionCond,
+		h.usageSnapshot,
+	)
 
 	return nil
 }

pkg/framework/plugins/nodeutilization/lownodeutilization.go

@@ -24,6 +24,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/descheduler/pkg/api"
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
 	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
@@ -36,9 +38,13 @@ const LowNodeUtilizationPluginName = "LowNodeUtilization"
 // to calculate nodes' utilization and not the actual resource usage.
 
 type LowNodeUtilization struct {
-	handle    frameworktypes.Handle
-	args      *LowNodeUtilizationArgs
-	podFilter func(pod *v1.Pod) bool
+	handle                   frameworktypes.Handle
+	args                     *LowNodeUtilizationArgs
+	podFilter                func(pod *v1.Pod) bool
+	underutilizationCriteria []interface{}
+	overutilizationCriteria  []interface{}
+	resourceNames            []v1.ResourceName
+	usageSnapshot            usageClient
 }
 
 var _ frameworktypes.BalancePlugin = &LowNodeUtilization{}
@@ -50,6 +56,30 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f
 		return nil, fmt.Errorf("want args to be of type LowNodeUtilizationArgs, got %T", args)
 	}
 
+	setDefaultForLNUThresholds(lowNodeUtilizationArgsArgs.Thresholds, lowNodeUtilizationArgsArgs.TargetThresholds, lowNodeUtilizationArgsArgs.UseDeviationThresholds)
+
+	underutilizationCriteria := []interface{}{
+		"CPU", lowNodeUtilizationArgsArgs.Thresholds[v1.ResourceCPU],
+		"Mem", lowNodeUtilizationArgsArgs.Thresholds[v1.ResourceMemory],
+		"Pods", lowNodeUtilizationArgsArgs.Thresholds[v1.ResourcePods],
+	}
+	for name := range lowNodeUtilizationArgsArgs.Thresholds {
+		if !nodeutil.IsBasicResource(name) {
+			underutilizationCriteria = append(underutilizationCriteria, string(name), int64(lowNodeUtilizationArgsArgs.Thresholds[name]))
+		}
+	}
+
+	overutilizationCriteria := []interface{}{
+		"CPU", lowNodeUtilizationArgsArgs.TargetThresholds[v1.ResourceCPU],
+		"Mem", lowNodeUtilizationArgsArgs.TargetThresholds[v1.ResourceMemory],
+		"Pods", lowNodeUtilizationArgsArgs.TargetThresholds[v1.ResourcePods],
+	}
+	for name := range lowNodeUtilizationArgsArgs.TargetThresholds {
+		if !nodeutil.IsBasicResource(name) {
+			overutilizationCriteria = append(overutilizationCriteria, string(name), int64(lowNodeUtilizationArgsArgs.TargetThresholds[name]))
+		}
+	}
+
 	podFilter, err := podutil.NewOptions().
 		WithFilter(handle.Evictor().Filter).
 		BuildFilterFunc()
@@ -57,10 +87,23 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f
 		return nil, fmt.Errorf("error initializing pod filter function: %v", err)
 	}
 
+	resourceNames := getResourceNames(lowNodeUtilizationArgsArgs.Thresholds)
+
+	var usageSnapshot usageClient
+	if lowNodeUtilizationArgsArgs.MetricsUtilization.MetricsServer {
+		usageSnapshot = newActualUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc(), handle.MetricsCollector())
+	} else {
+		usageSnapshot = newRequestedUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc())
+	}
+
 	return &LowNodeUtilization{
-		handle:    handle,
-		args:      lowNodeUtilizationArgsArgs,
-		podFilter: podFilter,
+		handle:                   handle,
+		args:                     lowNodeUtilizationArgsArgs,
+		underutilizationCriteria: underutilizationCriteria,
+		overutilizationCriteria:  overutilizationCriteria,
+		resourceNames:            resourceNames,
+		podFilter:                podFilter,
+		usageSnapshot:            usageSnapshot,
 	}, nil
 }
 
@@ -71,43 +114,15 @@ func (l *LowNodeUtilization) Name() string {
 
 // Balance extension point implementation for the plugin
 func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status {
-	useDeviationThresholds := l.args.UseDeviationThresholds
-	thresholds := l.args.Thresholds
-	targetThresholds := l.args.TargetThresholds
-
-	// check if Pods/CPU/Mem are set, if not, set them to 100
-	if _, ok := thresholds[v1.ResourcePods]; !ok {
-		if useDeviationThresholds {
-			thresholds[v1.ResourcePods] = MinResourcePercentage
-			targetThresholds[v1.ResourcePods] = MinResourcePercentage
-		} else {
-			thresholds[v1.ResourcePods] = MaxResourcePercentage
-			targetThresholds[v1.ResourcePods] = MaxResourcePercentage
+	if err := l.usageSnapshot.capture(nodes); err != nil {
+		return &frameworktypes.Status{
+			Err: fmt.Errorf("error getting node usage: %v", err),
 		}
 	}
-	if _, ok := thresholds[v1.ResourceCPU]; !ok {
-		if useDeviationThresholds {
-			thresholds[v1.ResourceCPU] = MinResourcePercentage
-			targetThresholds[v1.ResourceCPU] = MinResourcePercentage
-		} else {
-			thresholds[v1.ResourceCPU] = MaxResourcePercentage
-			targetThresholds[v1.ResourceCPU] = MaxResourcePercentage
-		}
-	}
-	if _, ok := thresholds[v1.ResourceMemory]; !ok {
-		if useDeviationThresholds {
-			thresholds[v1.ResourceMemory] = MinResourcePercentage
-			targetThresholds[v1.ResourceMemory] = MinResourcePercentage
-		} else {
-			thresholds[v1.ResourceMemory] = MaxResourcePercentage
-			targetThresholds[v1.ResourceMemory] = MaxResourcePercentage
-		}
-	}
-	resourceNames := getResourceNames(thresholds)
 
 	lowNodes, sourceNodes := classifyNodes(
-		getNodeUsage(nodes, resourceNames, l.handle.GetPodsAssignedToNodeFunc()),
-		getNodeThresholds(nodes, thresholds, targetThresholds, resourceNames, l.handle.GetPodsAssignedToNodeFunc(), useDeviationThresholds),
+		getNodeUsage(nodes, l.usageSnapshot),
+		getNodeThresholds(nodes, l.args.Thresholds, l.args.TargetThresholds, l.resourceNames, l.args.UseDeviationThresholds, l.usageSnapshot),
 		// The node has to be schedulable (to be able to move workload there)
 		func(node *v1.Node, usage NodeUsage, threshold NodeThresholds) bool {
 			if nodeutil.IsNodeUnschedulable(node) {
@@ -122,31 +137,11 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra
 	)
 
 	// log message for nodes with low utilization
-	underutilizationCriteria := []interface{}{
-		"CPU", thresholds[v1.ResourceCPU],
-		"Mem", thresholds[v1.ResourceMemory],
-		"Pods", thresholds[v1.ResourcePods],
-	}
-	for name := range thresholds {
-		if !nodeutil.IsBasicResource(name) {
-			underutilizationCriteria = append(underutilizationCriteria, string(name), int64(thresholds[name]))
-		}
-	}
-	klog.V(1).InfoS("Criteria for a node under utilization", underutilizationCriteria...)
+	klog.V(1).InfoS("Criteria for a node under utilization", l.underutilizationCriteria...)
 	klog.V(1).InfoS("Number of underutilized nodes", "totalNumber", len(lowNodes))
 
 	// log message for over utilized nodes
-	overutilizationCriteria := []interface{}{
-		"CPU", targetThresholds[v1.ResourceCPU],
-		"Mem", targetThresholds[v1.ResourceMemory],
-		"Pods", targetThresholds[v1.ResourcePods],
-	}
-	for name := range targetThresholds {
-		if !nodeutil.IsBasicResource(name) {
-			overutilizationCriteria = append(overutilizationCriteria, string(name), int64(targetThresholds[name]))
-		}
-	}
-	klog.V(1).InfoS("Criteria for a node above target utilization", overutilizationCriteria...)
+	klog.V(1).InfoS("Criteria for a node above target utilization", l.overutilizationCriteria...)
 	klog.V(1).InfoS("Number of overutilized nodes", "totalNumber", len(sourceNodes))
 
 	if len(lowNodes) == 0 {
@@ -194,8 +189,41 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra
 		l.handle.Evictor(),
 		evictions.EvictOptions{StrategyName: LowNodeUtilizationPluginName},
 		l.podFilter,
-		resourceNames,
-		continueEvictionCond)
+		l.resourceNames,
+		continueEvictionCond,
+		l.usageSnapshot,
+	)
 
 	return nil
 }
+
+func setDefaultForLNUThresholds(thresholds, targetThresholds api.ResourceThresholds, useDeviationThresholds bool) {
+	// check if Pods/CPU/Mem are set, if not, set them to 100
+	if _, ok := thresholds[v1.ResourcePods]; !ok {
+		if useDeviationThresholds {
+			thresholds[v1.ResourcePods] = MinResourcePercentage
+			targetThresholds[v1.ResourcePods] = MinResourcePercentage
+		} else {
+			thresholds[v1.ResourcePods] = MaxResourcePercentage
+			targetThresholds[v1.ResourcePods] = MaxResourcePercentage
+		}
+	}
+	if _, ok := thresholds[v1.ResourceCPU]; !ok {
+		if useDeviationThresholds {
+			thresholds[v1.ResourceCPU] = MinResourcePercentage
+			targetThresholds[v1.ResourceCPU] = MinResourcePercentage
+		} else {
+			thresholds[v1.ResourceCPU] = MaxResourcePercentage
+			targetThresholds[v1.ResourceCPU] = MaxResourcePercentage
+		}
+	}
+	if _, ok := thresholds[v1.ResourceMemory]; !ok {
+		if useDeviationThresholds {
+			thresholds[v1.ResourceMemory] = MinResourcePercentage
+			targetThresholds[v1.ResourceMemory] = MinResourcePercentage
+		} else {
+			thresholds[v1.ResourceMemory] = MaxResourcePercentage
+			targetThresholds[v1.ResourceMemory] = MaxResourcePercentage
+		}
+	}
+}

pkg/framework/plugins/nodeutilization/lownodeutilization_test.go

@@ -18,8 +18,12 @@ package nodeutilization
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"net"
+	"net/http"
 	"testing"
+	"time"
 
 	"sigs.k8s.io/descheduler/pkg/api"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
@@ -32,10 +36,18 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
+	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
+	fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake"
 
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 	"sigs.k8s.io/descheduler/pkg/utils"
 	"sigs.k8s.io/descheduler/test"
+
+	promapi "github.com/prometheus/client_golang/api"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
 )
 
 func TestLowNodeUtilization(t *testing.T) {
@@ -48,14 +60,17 @@ func TestLowNodeUtilization(t *testing.T) {
 	notMatchingNodeSelectorValue := "east"
 
 	testCases := []struct {
-		name                         string
-		useDeviationThresholds       bool
-		thresholds, targetThresholds api.ResourceThresholds
-		nodes                        []*v1.Node
-		pods                         []*v1.Pod
-		expectedPodsEvicted          uint
-		evictedPods                  []string
-		evictableNamespaces          *api.Namespaces
+		name                           string
+		useDeviationThresholds         bool
+		thresholds, targetThresholds   api.ResourceThresholds
+		nodes                          []*v1.Node
+		pods                           []*v1.Pod
+		nodemetricses                  []*v1beta1.NodeMetrics
+		podmetricses                   []*v1beta1.PodMetrics
+		expectedPodsEvicted            uint
+		expectedPodsWithMetricsEvicted uint
+		evictedPods                    []string
+		evictableNamespaces            *api.Namespaces
 	}{
 		{
 			name: "no evictable pods",
@@ -103,7 +118,20 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 0,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 2401, 1714978816),
+				test.BuildNodeMetrics(n2NodeName, 401, 1714978816),
+				test.BuildNodeMetrics(n3NodeName, 10, 1714978816),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            0,
+			expectedPodsWithMetricsEvicted: 0,
 		},
 		{
 			name: "without priorities",
@@ -153,7 +181,20 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 4,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            4,
+			expectedPodsWithMetricsEvicted: 4,
 		},
 		{
 			name: "without priorities, but excluding namespaces",
@@ -218,12 +259,25 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
 			evictableNamespaces: &api.Namespaces{
 				Exclude: []string{
 					"namespace1",
 				},
 			},
-			expectedPodsEvicted: 0,
+			expectedPodsEvicted:            0,
+			expectedPodsWithMetricsEvicted: 0,
 		},
 		{
 			name: "without priorities, but include only default namespace",
@@ -283,12 +337,25 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
 			evictableNamespaces: &api.Namespaces{
 				Include: []string{
 					"default",
 				},
 			},
-			expectedPodsEvicted: 2,
+			expectedPodsEvicted:            2,
+			expectedPodsWithMetricsEvicted: 2,
 		},
 		{
 			name: "without priorities stop when cpu capacity is depleted",
@@ -306,14 +373,14 @@ func TestLowNodeUtilization(t *testing.T) {
 				test.BuildTestNode(n3NodeName, 4000, 3000, 10, test.SetNodeUnschedulable),
 			},
 			pods: []*v1.Pod{
-				test.BuildTestPod("p1", 400, 300, n1NodeName, test.SetRSOwnerRef),
-				test.BuildTestPod("p2", 400, 300, n1NodeName, test.SetRSOwnerRef),
-				test.BuildTestPod("p3", 400, 300, n1NodeName, test.SetRSOwnerRef),
-				test.BuildTestPod("p4", 400, 300, n1NodeName, test.SetRSOwnerRef),
-				test.BuildTestPod("p5", 400, 300, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef),
 				// These won't be evicted.
-				test.BuildTestPod("p6", 400, 300, n1NodeName, test.SetDSOwnerRef),
-				test.BuildTestPod("p7", 400, 300, n1NodeName, func(pod *v1.Pod) {
+				test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
+				test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) {
 					// A pod with local storage.
 					test.SetNormalOwnerRef(pod)
 					pod.Spec.Volumes = []v1.Volume{
@@ -330,17 +397,29 @@ func TestLowNodeUtilization(t *testing.T) {
 					// A Mirror Pod.
 					pod.Annotations = test.GetMirrorPodAnnotation()
 				}),
-				test.BuildTestPod("p8", 400, 300, n1NodeName, func(pod *v1.Pod) {
+				test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) {
 					// A Critical Pod.
 					test.SetNormalOwnerRef(pod)
 					pod.Namespace = "kube-system"
 					priority := utils.SystemCriticalPriority
 					pod.Spec.Priority = &priority
 				}),
-				test.BuildTestPod("p9", 400, 2100, n2NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			// 4 pods available for eviction based on v1.ResourcePods, only 3 pods can be evicted before cpu is depleted
-			expectedPodsEvicted: 3,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 0, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            4,
+			expectedPodsWithMetricsEvicted: 4,
 		},
 		{
 			name: "with priorities",
@@ -410,7 +489,20 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 4,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            4,
+			expectedPodsWithMetricsEvicted: 4,
 		},
 		{
 			name: "without priorities evicting best-effort pods only",
@@ -478,8 +570,21 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 4,
-			evictedPods:         []string{"p1", "p2", "p4", "p5"},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            4,
+			expectedPodsWithMetricsEvicted: 4,
+			evictedPods:                    []string{"p1", "p2", "p4", "p5"},
 		},
 		{
 			name: "with extended resource",
@@ -558,8 +663,21 @@ func TestLowNodeUtilization(t *testing.T) {
 					test.SetPodExtendedResourceRequest(pod, extendedResource, 1)
 				}),
 			},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
 			// 4 pods available for eviction based on v1.ResourcePods, only 3 pods can be evicted before extended resource is depleted
-			expectedPodsEvicted: 3,
+			expectedPodsEvicted:            3,
+			expectedPodsWithMetricsEvicted: 0,
 		},
 		{
 			name: "with extended resource in some of nodes",
@@ -586,8 +704,21 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef),
 			},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
 			// 0 pods available for eviction because there's no enough extended resource in node2
-			expectedPodsEvicted: 0,
+			expectedPodsEvicted:            0,
+			expectedPodsWithMetricsEvicted: 0,
 		},
 		{
 			name: "without priorities, but only other node is unschedulable",
@@ -636,7 +767,19 @@ func TestLowNodeUtilization(t *testing.T) {
 					pod.Spec.Priority = &priority
 				}),
 			},
-			expectedPodsEvicted: 0,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            0,
+			expectedPodsWithMetricsEvicted: 0,
 		},
 		{
 			name: "without priorities, but only other node doesn't match pod node selector for p4 and p5",
@@ -701,7 +844,17 @@ func TestLowNodeUtilization(t *testing.T) {
 					pod.Spec.Priority = &priority
 				}),
 			},
-			expectedPodsEvicted: 3,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+			},
+			expectedPodsEvicted:            3,
+			expectedPodsWithMetricsEvicted: 3,
 		},
 		{
 			name: "without priorities, but only other node doesn't match pod node affinity for p4 and p5",
@@ -795,7 +948,17 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 3,
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+			},
+			expectedPodsEvicted:            3,
+			expectedPodsWithMetricsEvicted: 3,
 		},
 		{
 			name: "deviation thresholds",
@@ -847,71 +1010,210 @@ func TestLowNodeUtilization(t *testing.T) {
 				}),
 				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
 			},
-			expectedPodsEvicted: 2,
-			evictedPods:         []string{},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+				test.BuildNodeMetrics(n3NodeName, 11, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 401, 0),
+				test.BuildPodMetrics("p2", 401, 0),
+				test.BuildPodMetrics("p3", 401, 0),
+				test.BuildPodMetrics("p4", 401, 0),
+				test.BuildPodMetrics("p5", 401, 0),
+			},
+			expectedPodsEvicted:            2,
+			expectedPodsWithMetricsEvicted: 2,
+			evictedPods:                    []string{},
+		},
+		{
+			name: "without priorities different evictions for requested and actual resources",
+			thresholds: api.ResourceThresholds{
+				v1.ResourceCPU:  30,
+				v1.ResourcePods: 30,
+			},
+			targetThresholds: api.ResourceThresholds{
+				v1.ResourceCPU:  50,
+				v1.ResourcePods: 50,
+			},
+			nodes: []*v1.Node{
+				test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
+				test.BuildTestNode(n2NodeName, 4000, 3000, 10, func(node *v1.Node) {
+					node.ObjectMeta.Labels = map[string]string{
+						nodeSelectorKey: notMatchingNodeSelectorValue,
+					}
+				}),
+			},
+			pods: []*v1.Pod{
+				test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				// These won't be evicted.
+				test.BuildTestPod("p4", 400, 0, n1NodeName, func(pod *v1.Pod) {
+					// A pod with affinity to run in the "west" datacenter upon scheduling
+					test.SetNormalOwnerRef(pod)
+					pod.Spec.Affinity = &v1.Affinity{
+						NodeAffinity: &v1.NodeAffinity{
+							RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+								NodeSelectorTerms: []v1.NodeSelectorTerm{
+									{
+										MatchExpressions: []v1.NodeSelectorRequirement{
+											{
+												Key:      nodeSelectorKey,
+												Operator: "In",
+												Values:   []string{nodeSelectorValue},
+											},
+										},
+									},
+								},
+							},
+						},
+					}
+				}),
+				test.BuildTestPod("p5", 400, 0, n1NodeName, func(pod *v1.Pod) {
+					// A pod with affinity to run in the "west" datacenter upon scheduling
+					test.SetNormalOwnerRef(pod)
+					pod.Spec.Affinity = &v1.Affinity{
+						NodeAffinity: &v1.NodeAffinity{
+							RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
+								NodeSelectorTerms: []v1.NodeSelectorTerm{
+									{
+										MatchExpressions: []v1.NodeSelectorRequirement{
+											{
+												Key:      nodeSelectorKey,
+												Operator: "In",
+												Values:   []string{nodeSelectorValue},
+											},
+										},
+									},
+								},
+							},
+						},
+					}
+				}),
+				test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
+				test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) {
+					// A pod with local storage.
+					test.SetNormalOwnerRef(pod)
+					pod.Spec.Volumes = []v1.Volume{
+						{
+							Name: "sample",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{Path: "somePath"},
+								EmptyDir: &v1.EmptyDirVolumeSource{
+									SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI),
+								},
+							},
+						},
+					}
+					// A Mirror Pod.
+					pod.Annotations = test.GetMirrorPodAnnotation()
+				}),
+				test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) {
+					// A Critical Pod.
+					test.SetNormalOwnerRef(pod)
+					pod.Namespace = "kube-system"
+					priority := utils.SystemCriticalPriority
+					pod.Spec.Priority = &priority
+				}),
+				test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef),
+			},
+			nodemetricses: []*v1beta1.NodeMetrics{
+				test.BuildNodeMetrics(n1NodeName, 3201, 0),
+				test.BuildNodeMetrics(n2NodeName, 401, 0),
+			},
+			podmetricses: []*v1beta1.PodMetrics{
+				test.BuildPodMetrics("p1", 801, 0),
+				test.BuildPodMetrics("p2", 801, 0),
+				test.BuildPodMetrics("p3", 801, 0),
+			},
+			expectedPodsEvicted:            3,
+			expectedPodsWithMetricsEvicted: 2,
 		},
 	}
 
 	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
+		testFnc := func(metricsEnabled bool, expectedPodsEvicted uint) func(t *testing.T) {
+			return func(t *testing.T) {
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
 
-			var objs []runtime.Object
-			for _, node := range tc.nodes {
-				objs = append(objs, node)
-			}
-			for _, pod := range tc.pods {
-				objs = append(objs, pod)
-			}
-			fakeClient := fake.NewSimpleClientset(objs...)
+				var objs []runtime.Object
+				for _, node := range tc.nodes {
+					objs = append(objs, node)
+				}
+				for _, pod := range tc.pods {
+					objs = append(objs, pod)
+				}
+				var metricsObjs []runtime.Object
+				for _, nodemetrics := range tc.nodemetricses {
+					metricsObjs = append(metricsObjs, nodemetrics)
+				}
+				for _, podmetrics := range tc.podmetricses {
+					metricsObjs = append(metricsObjs, podmetrics)
+				}
 
-			podsForEviction := make(map[string]struct{})
-			for _, pod := range tc.evictedPods {
-				podsForEviction[pod] = struct{}{}
-			}
+				fakeClient := fake.NewSimpleClientset(objs...)
+				metricsClientset := fakemetricsclient.NewSimpleClientset(metricsObjs...)
+				collector := metricscollector.NewMetricsCollector(fakeClient, metricsClientset)
+				err := collector.Collect(ctx)
+				if err != nil {
+					t.Fatalf("unable to collect metrics: %v", err)
+				}
 
-			evictionFailed := false
-			if len(tc.evictedPods) > 0 {
-				fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
-					getAction := action.(core.CreateAction)
-					obj := getAction.GetObject()
-					if eviction, ok := obj.(*policy.Eviction); ok {
-						if _, exists := podsForEviction[eviction.Name]; exists {
-							return true, obj, nil
+				podsForEviction := make(map[string]struct{})
+				for _, pod := range tc.evictedPods {
+					podsForEviction[pod] = struct{}{}
+				}
+
+				evictionFailed := false
+				if len(tc.evictedPods) > 0 {
+					fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
+						getAction := action.(core.CreateAction)
+						obj := getAction.GetObject()
+						if eviction, ok := obj.(*policy.Eviction); ok {
+							if _, exists := podsForEviction[eviction.Name]; exists {
+								return true, obj, nil
+							}
+							evictionFailed = true
+							return true, nil, fmt.Errorf("pod %q was unexpectedly evicted", eviction.Name)
 						}
-						evictionFailed = true
-						return true, nil, fmt.Errorf("pod %q was unexpectedly evicted", eviction.Name)
-					}
-					return true, obj, nil
-				})
-			}
+						return true, obj, nil
+					})
+				}
 
-			handle, podEvictor, err := frameworktesting.InitFrameworkHandle(ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil)
-			if err != nil {
-				t.Fatalf("Unable to initialize a framework handle: %v", err)
-			}
+				handle, podEvictor, err := frameworktesting.InitFrameworkHandle(ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil)
+				if err != nil {
+					t.Fatalf("Unable to initialize a framework handle: %v", err)
+				}
+				handle.MetricsCollectorImpl = collector
 
-			plugin, err := NewLowNodeUtilization(&LowNodeUtilizationArgs{
-				Thresholds:             tc.thresholds,
-				TargetThresholds:       tc.targetThresholds,
-				UseDeviationThresholds: tc.useDeviationThresholds,
-				EvictableNamespaces:    tc.evictableNamespaces,
-			},
-				handle)
-			if err != nil {
-				t.Fatalf("Unable to initialize the plugin: %v", err)
-			}
-			plugin.(frameworktypes.BalancePlugin).Balance(ctx, tc.nodes)
+				plugin, err := NewLowNodeUtilization(&LowNodeUtilizationArgs{
+					Thresholds:             tc.thresholds,
+					TargetThresholds:       tc.targetThresholds,
+					UseDeviationThresholds: tc.useDeviationThresholds,
+					EvictableNamespaces:    tc.evictableNamespaces,
+					MetricsUtilization: MetricsUtilization{
+						MetricsServer: metricsEnabled,
+					},
+				},
+					handle)
+				if err != nil {
+					t.Fatalf("Unable to initialize the plugin: %v", err)
+				}
+				plugin.(frameworktypes.BalancePlugin).Balance(ctx, tc.nodes)
 
-			podsEvicted := podEvictor.TotalEvicted()
-			if tc.expectedPodsEvicted != podsEvicted {
-				t.Errorf("Expected %v pods to be evicted but %v got evicted", tc.expectedPodsEvicted, podsEvicted)
+				podsEvicted := podEvictor.TotalEvicted()
+				if expectedPodsEvicted != podsEvicted {
+					t.Errorf("Expected %v pods to be evicted but %v got evicted", expectedPodsEvicted, podsEvicted)
+				}
+				if evictionFailed {
+					t.Errorf("Pod evictions failed unexpectedly")
+				}
 			}
-			if evictionFailed {
-				t.Errorf("Pod evictions failed unexpectedly")
-			}
-		})
+		}
+		t.Run(tc.name, testFnc(false, tc.expectedPodsEvicted))
+		t.Run(tc.name+" with metrics enabled", testFnc(true, tc.expectedPodsWithMetricsEvicted))
 	}
 }
 
@@ -1067,3 +1369,62 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) {
 		})
 	}
 }
+
+func TestLowNodeUtilizationWithMetrics(t *testing.T) {
+	return
+	roundTripper := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
+	}
+
+	AuthToken := "eyJhbGciOiJSUzI1NiIsImtpZCI6IkNoTW9tT2w2cWtzR2V0dURZdjBqdnBSdmdWM29lWmc3dWpfNW0yaDc2NHMifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTcyODk5MjY3NywiaWF0IjoxNzI4OTg5MDc3LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMiLCJqdGkiOiJkNDY3ZjVmMy0xNGVmLTRkMjItOWJkNC1jMGM1Mzk3NzYyZDgiLCJrdWJlcm5ldGVzLmlvIjp7Im5hbWVzcGFjZSI6Im9wZW5zaGlmdC1tb25pdG9yaW5nIiwic2VydmljZWFjY291bnQiOnsibmFtZSI6InByb21ldGhldXMtazhzIiwidWlkIjoiNjY4NDllMGItYTAwZC00NjUzLWE5NTItNThiYTE1MTk4NTlkIn19LCJuYmYiOjE3Mjg5ODkwNzcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.J1i6-oRAC9J8mqrlZPKGA-CU5PbUzhm2QxAWFnu65-NXR3e252mesybwtjkwxUtTLKrsYHQXwEsG5rGcQsvMcGK9RC9y5z33DFj8tPPwOGLJYJ-s5cTImTqKtWRXzTlcrsrUYTYApfrOsEyXwyfDow4PCslZjR3cd5FMRbvXNqHLg26nG_smApR4wc6kXy7xxlRuGhxu-dUiscQP56njboOK61JdTG8F3FgOayZnKk1jGeVdIhXClqGWJyokk-ZM3mMK1MxzGXY0tLbe37V4B7g3NDiH651BUcicfDSky46yfcAYxMDbZgpK2TByWApAllN0wixz2WsFfyBVu_Q5xtZ9Gi9BUHSa5ioRiBK346W4Bdmr9ala5ldIXDa59YE7UB34DsCHyqvzRx_Sj76hLzy2jSOk7RsL0fM8sDoJL4ROdi-3Jtr5uPY593I8H8qeQvFS6PQfm0bUZqVKrrLoCK_uk9guH4a6K27SlD-Utk3dpsjbmrwcjBxm-zd_LE9YyQ734My00Pcy9D5eNio3gESjGsHqGFc_haq4ZCiVOCkbdmABjpPEL6K7bs1GMZbHt1CONL0-LzymM8vgGNj0grjpG8-5AF8ZuSqR7pbZSV_NO2nKkmrwpILCw0Joqp6V3C9pP9nXWHIDyVMxMK870zxzt_qCoPRLCAujQQn6e0U"
+	client, err := promapi.NewClient(promapi.Config{
+		Address:      "https://prometheus-k8s-openshift-monitoring.apps.jchaloup-20241015-3.group-b.devcluster.openshift.com",
+		RoundTripper: config.NewAuthorizationCredentialsRoundTripper("Bearer", config.NewInlineSecret(AuthToken), roundTripper),
+	})
+	if err != nil {
+		t.Fatalf("prom client error: %v", err)
+	}
+
+	// pod:container_cpu_usage:sum
+	// container_memory_usage_bytes
+
+	v1api := promv1.NewAPI(client)
+	ctx := context.TODO()
+	// promQuery := "avg_over_time(kube_pod_container_resource_requests[1m])"
+	promQuery := "kube_pod_container_resource_requests"
+	results, warnings, err := v1api.Query(ctx, promQuery, time.Now())
+	fmt.Printf("results: %#v\n", results)
+	for _, sample := range results.(model.Vector) {
+		fmt.Printf("sample: %#v\n", sample)
+	}
+	fmt.Printf("warnings: %v\n", warnings)
+	fmt.Printf("err: %v\n", err)
+
+	result := model.Value(
+		&model.Vector{
+			&model.Sample{
+				Metric: model.Metric{
+					"container": "kube-controller-manager",
+					"endpoint":  "https-main",
+					"job":       "kube-state-metrics",
+					"namespace": "openshift-kube-controller-manager",
+					"node":      "ip-10-0-72-168.us-east-2.compute.internal",
+					"pod":       "kube-controller-manager-ip-10-0-72-168.us-east-2.compute.internal",
+					"resource":  "cpu",
+					"service":   "kube-state-metrics",
+					"uid":       "ae46c09f-ade7-4133-9ee8-cf45ac78ca6d",
+					"unit":      "core",
+				},
+				Value:     0.06,
+				Timestamp: 1728991761711,
+			},
+		},
+	)
+	fmt.Printf("result: %#v\n", result)
+}

pkg/framework/plugins/nodeutilization/nodeutilization.go

@@ -78,14 +78,14 @@ func getNodeThresholds(
 	nodes []*v1.Node,
 	lowThreshold, highThreshold api.ResourceThresholds,
 	resourceNames []v1.ResourceName,
-	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
 	useDeviationThresholds bool,
+	usageClient usageClient,
 ) map[string]NodeThresholds {
 	nodeThresholdsMap := map[string]NodeThresholds{}
 
 	averageResourceUsagePercent := api.ResourceThresholds{}
 	if useDeviationThresholds {
-		averageResourceUsagePercent = averageNodeBasicresources(nodes, getPodsAssignedToNode, resourceNames)
+		averageResourceUsagePercent = averageNodeBasicresources(nodes, usageClient)
 	}
 
 	for _, node := range nodes {
@@ -121,22 +121,15 @@ func getNodeThresholds(
 
 func getNodeUsage(
 	nodes []*v1.Node,
-	resourceNames []v1.ResourceName,
-	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
+	usageClient usageClient,
 ) []NodeUsage {
 	var nodeUsageList []NodeUsage
 
 	for _, node := range nodes {
-		pods, err := podutil.ListPodsOnANode(node.Name, getPodsAssignedToNode, nil)
-		if err != nil {
-			klog.V(2).InfoS("Node will not be processed, error accessing its pods", "node", klog.KObj(node), "err", err)
-			continue
-		}
-
 		nodeUsageList = append(nodeUsageList, NodeUsage{
 			node:    node,
-			usage:   nodeutil.NodeUtilization(pods, resourceNames),
-			allPods: pods,
+			usage:   usageClient.nodeUtilization(node.Name),
+			allPods: usageClient.pods(node.Name),
 		})
 	}
 
@@ -226,6 +219,7 @@ func evictPodsFromSourceNodes(
 	podFilter func(pod *v1.Pod) bool,
 	resourceNames []v1.ResourceName,
 	continueEviction continueEvictionCond,
+	usageSnapshot usageClient,
 ) {
 	// upper bound on total number of pods/cpu/memory and optional extended resources to be moved
 	totalAvailableUsage := map[v1.ResourceName]*resource.Quantity{
@@ -243,6 +237,10 @@ func evictPodsFromSourceNodes(
 				totalAvailableUsage[name] = resource.NewQuantity(0, resource.DecimalSI)
 			}
 			totalAvailableUsage[name].Add(*node.thresholds.highResourceThreshold[name])
+			if _, exists := node.usage[name]; !exists {
+				klog.Errorf("unable to find %q resource in node's %q usage, terminating eviction", name, node.node.Name)
+				return
+			}
 			totalAvailableUsage[name].Sub(*node.usage[name])
 		}
 	}
@@ -274,7 +272,7 @@ func evictPodsFromSourceNodes(
 		klog.V(1).InfoS("Evicting pods based on priority, if they have same priority, they'll be evicted based on QoS tiers")
 		// sort the evictable Pods based on priority. This also sorts them based on QoS. If there are multiple pods with same priority, they are sorted based on QoS tiers.
 		podutil.SortPodsBasedOnPriorityLowToHigh(removablePods)
-		err := evictPods(ctx, evictableNamespaces, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, evictOptions, continueEviction)
+		err := evictPods(ctx, evictableNamespaces, removablePods, node, totalAvailableUsage, taintsOfDestinationNodes, podEvictor, evictOptions, continueEviction, usageSnapshot)
 		if err != nil {
 			switch err.(type) {
 			case *evictions.EvictionTotalLimitError:
@@ -295,6 +293,7 @@ func evictPods(
 	podEvictor frameworktypes.Evictor,
 	evictOptions evictions.EvictOptions,
 	continueEviction continueEvictionCond,
+	usageSnapshot usageClient,
 ) error {
 	var excludedNamespaces sets.Set[string]
 	if evictableNamespaces != nil {
@@ -320,18 +319,21 @@ func evictPods(
 			if !preEvictionFilterWithOptions(pod) {
 				continue
 			}
+			podUsage, err := usageSnapshot.podUsage(pod)
+			if err != nil {
+				klog.Errorf("unable to get pod usage for %v/%v: %v", pod.Namespace, pod.Name, err)
+				continue
+			}
 			err = podEvictor.Evict(ctx, pod, evictOptions)
 			if err == nil {
 				klog.V(3).InfoS("Evicted pods", "pod", klog.KObj(pod))
-
 				for name := range totalAvailableUsage {
 					if name == v1.ResourcePods {
 						nodeInfo.usage[name].Sub(*resource.NewQuantity(1, resource.DecimalSI))
 						totalAvailableUsage[name].Sub(*resource.NewQuantity(1, resource.DecimalSI))
 					} else {
-						quantity := utils.GetResourceRequestQuantity(pod, name)
-						nodeInfo.usage[name].Sub(quantity)
-						totalAvailableUsage[name].Sub(quantity)
+						nodeInfo.usage[name].Sub(*podUsage[name])
+						totalAvailableUsage[name].Sub(*podUsage[name])
 					}
 				}
 
@@ -437,17 +439,12 @@ func classifyPods(pods []*v1.Pod, filter func(pod *v1.Pod) bool) ([]*v1.Pod, []*
 	return nonRemovablePods, removablePods
 }
 
-func averageNodeBasicresources(nodes []*v1.Node, getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc, resourceNames []v1.ResourceName) api.ResourceThresholds {
+func averageNodeBasicresources(nodes []*v1.Node, usageClient usageClient) api.ResourceThresholds {
 	total := api.ResourceThresholds{}
 	average := api.ResourceThresholds{}
 	numberOfNodes := len(nodes)
 	for _, node := range nodes {
-		pods, err := podutil.ListPodsOnANode(node.Name, getPodsAssignedToNode, nil)
-		if err != nil {
-			numberOfNodes--
-			continue
-		}
-		usage := nodeutil.NodeUtilization(pods, resourceNames)
+		usage := usageClient.nodeUtilization(node.Name)
 		nodeCapacity := node.Status.Capacity
 		if len(node.Status.Allocatable) > 0 {
 			nodeCapacity = node.Status.Allocatable

pkg/framework/plugins/nodeutilization/types.go

@@ -28,6 +28,7 @@ type LowNodeUtilizationArgs struct {
 	Thresholds             api.ResourceThresholds `json:"thresholds"`
 	TargetThresholds       api.ResourceThresholds `json:"targetThresholds"`
 	NumberOfNodes          int                    `json:"numberOfNodes,omitempty"`
+	MetricsUtilization     MetricsUtilization     `json:metricsUtilization,omitempty`
 
 	// Naming this one differently since namespaces are still
 	// considered while considering resources used by pods
@@ -41,10 +42,19 @@ type LowNodeUtilizationArgs struct {
 type HighNodeUtilizationArgs struct {
 	metav1.TypeMeta `json:",inline"`
 
-	Thresholds    api.ResourceThresholds `json:"thresholds"`
-	NumberOfNodes int                    `json:"numberOfNodes,omitempty"`
+	Thresholds         api.ResourceThresholds `json:"thresholds"`
+	NumberOfNodes      int                    `json:"numberOfNodes,omitempty"`
+	MetricsUtilization MetricsUtilization     `json:metricsUtilization,omitempty`
+
 	// Naming this one differently since namespaces are still
 	// considered while considering resources used by pods
 	// but then filtered out before eviction
 	EvictableNamespaces *api.Namespaces `json:"evictableNamespaces,omitempty"`
 }
+
+// MetricsUtilization allow to consume actual resource utilization from metrics
+type MetricsUtilization struct {
+	// metricsServer enables metrics from a kubernetes metrics server.
+	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
+	MetricsServer bool `json:"metricsServer,omitempty"`
+}

pkg/framework/plugins/nodeutilization/usageclients.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeutilization
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	utilptr "k8s.io/utils/ptr"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
+	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
+	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
+	"sigs.k8s.io/descheduler/pkg/utils"
+)
+
+type usageClient interface {
+	nodeUtilization(node string) map[v1.ResourceName]*resource.Quantity
+	nodes() []*v1.Node
+	pods(node string) []*v1.Pod
+	capture(nodes []*v1.Node) error
+	podUsage(pod *v1.Pod) (map[v1.ResourceName]*resource.Quantity, error)
+}
+
+type requestedUsageClient struct {
+	resourceNames         []v1.ResourceName
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc
+
+	_nodes           []*v1.Node
+	_pods            map[string][]*v1.Pod
+	_nodeUtilization map[string]map[v1.ResourceName]*resource.Quantity
+}
+
+var _ usageClient = &requestedUsageClient{}
+
+func newRequestedUsageSnapshot(
+	resourceNames []v1.ResourceName,
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
+) *requestedUsageClient {
+	return &requestedUsageClient{
+		resourceNames:         resourceNames,
+		getPodsAssignedToNode: getPodsAssignedToNode,
+	}
+}
+
+func (s *requestedUsageClient) nodeUtilization(node string) map[v1.ResourceName]*resource.Quantity {
+	return s._nodeUtilization[node]
+}
+
+func (s *requestedUsageClient) nodes() []*v1.Node {
+	return s._nodes
+}
+
+func (s *requestedUsageClient) pods(node string) []*v1.Pod {
+	return s._pods[node]
+}
+
+func (s *requestedUsageClient) podUsage(pod *v1.Pod) (map[v1.ResourceName]*resource.Quantity, error) {
+	usage := make(map[v1.ResourceName]*resource.Quantity)
+	for _, resourceName := range s.resourceNames {
+		usage[resourceName] = utilptr.To[resource.Quantity](utils.GetResourceRequestQuantity(pod, resourceName).DeepCopy())
+	}
+	return usage, nil
+}
+
+func (s *requestedUsageClient) capture(nodes []*v1.Node) error {
+	s._nodeUtilization = make(map[string]map[v1.ResourceName]*resource.Quantity)
+	s._pods = make(map[string][]*v1.Pod)
+	capturedNodes := []*v1.Node{}
+
+	for _, node := range nodes {
+		pods, err := podutil.ListPodsOnANode(node.Name, s.getPodsAssignedToNode, nil)
+		if err != nil {
+			klog.V(2).InfoS("Node will not be processed, error accessing its pods", "node", klog.KObj(node), "err", err)
+			continue
+		}
+
+		nodeUsage, err := nodeutil.NodeUtilization(pods, s.resourceNames, func(pod *v1.Pod) (v1.ResourceList, error) {
+			req, _ := utils.PodRequestsAndLimits(pod)
+			return req, nil
+		})
+		if err != nil {
+			return err
+		}
+
+		// store the snapshot of pods from the same (or the closest) node utilization computation
+		s._pods[node.Name] = pods
+		s._nodeUtilization[node.Name] = nodeUsage
+		capturedNodes = append(capturedNodes, node)
+	}
+
+	s._nodes = capturedNodes
+
+	return nil
+}
+
+type actualUsageClient struct {
+	resourceNames         []v1.ResourceName
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc
+	metricsCollector      *metricscollector.MetricsCollector
+
+	_nodes           []*v1.Node
+	_pods            map[string][]*v1.Pod
+	_nodeUtilization map[string]map[v1.ResourceName]*resource.Quantity
+}
+
+var _ usageClient = &actualUsageClient{}
+
+func newActualUsageSnapshot(
+	resourceNames []v1.ResourceName,
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
+	metricsCollector *metricscollector.MetricsCollector,
+) *actualUsageClient {
+	return &actualUsageClient{
+		resourceNames:         resourceNames,
+		getPodsAssignedToNode: getPodsAssignedToNode,
+		metricsCollector:      metricsCollector,
+	}
+}
+
+func (client *actualUsageClient) nodeUtilization(node string) map[v1.ResourceName]*resource.Quantity {
+	return client._nodeUtilization[node]
+}
+
+func (client *actualUsageClient) nodes() []*v1.Node {
+	return client._nodes
+}
+
+func (client *actualUsageClient) pods(node string) []*v1.Pod {
+	return client._pods[node]
+}
+
+func (client *actualUsageClient) podUsage(pod *v1.Pod) (map[v1.ResourceName]*resource.Quantity, error) {
+	// It's not efficient to keep track of all pods in a cluster when only their fractions is evicted.
+	// Thus, take the current pod metrics without computing any softening (like e.g. EWMA).
+	podMetrics, err := client.metricsCollector.MetricsClient().MetricsV1beta1().PodMetricses(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("unable to get podmetrics for %q/%q: %v", pod.Namespace, pod.Name, err)
+	}
+
+	totalUsage := make(map[v1.ResourceName]*resource.Quantity)
+	for _, container := range podMetrics.Containers {
+		for _, resourceName := range client.resourceNames {
+			if _, exists := container.Usage[resourceName]; !exists {
+				continue
+			}
+			if totalUsage[resourceName] == nil {
+				totalUsage[resourceName] = utilptr.To[resource.Quantity](container.Usage[resourceName].DeepCopy())
+			} else {
+				totalUsage[resourceName].Add(container.Usage[resourceName])
+			}
+		}
+	}
+
+	return totalUsage, nil
+}
+
+func (client *actualUsageClient) capture(nodes []*v1.Node) error {
+	client._nodeUtilization = make(map[string]map[v1.ResourceName]*resource.Quantity)
+	client._pods = make(map[string][]*v1.Pod)
+	capturedNodes := []*v1.Node{}
+
+	for _, node := range nodes {
+		pods, err := podutil.ListPodsOnANode(node.Name, client.getPodsAssignedToNode, nil)
+		if err != nil {
+			klog.V(2).InfoS("Node will not be processed, error accessing its pods", "node", klog.KObj(node), "err", err)
+			continue
+		}
+
+		nodeUsage, err := client.metricsCollector.NodeUsage(node)
+		if err != nil {
+			return err
+		}
+		nodeUsage[v1.ResourcePods] = resource.NewQuantity(int64(len(pods)), resource.DecimalSI)
+
+		// store the snapshot of pods from the same (or the closest) node utilization computation
+		client._pods[node.Name] = pods
+		client._nodeUtilization[node.Name] = nodeUsage
+		capturedNodes = append(capturedNodes, node)
+	}
+
+	client._nodes = capturedNodes
+
+	return nil
+}

pkg/framework/plugins/nodeutilization/usageclients_test.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeutilization
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"k8s.io/client-go/informers"
+	fakeclientset "k8s.io/client-go/kubernetes/fake"
+	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
+	fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
+	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
+	"sigs.k8s.io/descheduler/test"
+)
+
+var gvr = schema.GroupVersionResource{Group: "metrics.k8s.io", Version: "v1beta1", Resource: "nodemetricses"}
+
+func updateMetricsAndCheckNodeUtilization(
+	t *testing.T,
+	ctx context.Context,
+	newValue, expectedValue int64,
+	metricsClientset *fakemetricsclient.Clientset,
+	collector *metricscollector.MetricsCollector,
+	usageSnapshot usageClient,
+	nodes []*v1.Node,
+	nodeName string,
+	nodemetrics *v1beta1.NodeMetrics,
+) {
+	t.Logf("Set current node cpu usage to %v", newValue)
+	nodemetrics.Usage[v1.ResourceCPU] = *resource.NewMilliQuantity(newValue, resource.DecimalSI)
+	metricsClientset.Tracker().Update(gvr, nodemetrics, "")
+	err := collector.Collect(ctx)
+	if err != nil {
+		t.Fatalf("failed to capture metrics: %v", err)
+	}
+	err = usageSnapshot.capture(nodes)
+	if err != nil {
+		t.Fatalf("failed to capture a snapshot: %v", err)
+	}
+	nodeUtilization := usageSnapshot.nodeUtilization(nodeName)
+	t.Logf("current node cpu usage: %v\n", nodeUtilization[v1.ResourceCPU].MilliValue())
+	if nodeUtilization[v1.ResourceCPU].MilliValue() != expectedValue {
+		t.Fatalf("cpu node usage expected to be %v, got %v instead", expectedValue, nodeUtilization[v1.ResourceCPU].MilliValue())
+	}
+	pods := usageSnapshot.pods(nodeName)
+	fmt.Printf("pods: %#v\n", pods)
+	if len(pods) != 2 {
+		t.Fatalf("expected 2 pods for node %v, got %v instead", nodeName, len(pods))
+	}
+	capturedNodes := usageSnapshot.nodes()
+	if len(capturedNodes) != 3 {
+		t.Fatalf("expected 3 captured node, got %v instead", len(capturedNodes))
+	}
+}
+
+func TestActualUsageClient(t *testing.T) {
+	n1 := test.BuildTestNode("n1", 2000, 3000, 10, nil)
+	n2 := test.BuildTestNode("n2", 2000, 3000, 10, nil)
+	n3 := test.BuildTestNode("n3", 2000, 3000, 10, nil)
+
+	p1 := test.BuildTestPod("p1", 400, 0, n1.Name, nil)
+	p21 := test.BuildTestPod("p21", 400, 0, n2.Name, nil)
+	p22 := test.BuildTestPod("p22", 400, 0, n2.Name, nil)
+	p3 := test.BuildTestPod("p3", 400, 0, n3.Name, nil)
+
+	nodes := []*v1.Node{n1, n2, n3}
+
+	n1metrics := test.BuildNodeMetrics("n1", 400, 1714978816)
+	n2metrics := test.BuildNodeMetrics("n2", 1400, 1714978816)
+	n3metrics := test.BuildNodeMetrics("n3", 300, 1714978816)
+
+	clientset := fakeclientset.NewSimpleClientset(n1, n2, n3, p1, p21, p22, p3)
+	metricsClientset := fakemetricsclient.NewSimpleClientset(n1metrics, n2metrics, n3metrics)
+
+	ctx := context.TODO()
+
+	resourceNames := []v1.ResourceName{
+		v1.ResourceCPU,
+		v1.ResourceMemory,
+	}
+
+	sharedInformerFactory := informers.NewSharedInformerFactory(clientset, 0)
+	podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
+	podsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer)
+	if err != nil {
+		t.Fatalf("Build get pods assigned to node function error: %v", err)
+	}
+
+	sharedInformerFactory.Start(ctx.Done())
+	sharedInformerFactory.WaitForCacheSync(ctx.Done())
+
+	collector := metricscollector.NewMetricsCollector(clientset, metricsClientset)
+
+	usageSnapshot := newActualUsageSnapshot(
+		resourceNames,
+		podsAssignedToNode,
+		collector,
+	)
+
+	updateMetricsAndCheckNodeUtilization(t, ctx,
+		1400, 1400,
+		metricsClientset, collector, usageSnapshot, nodes, n2.Name, n2metrics,
+	)
+
+	updateMetricsAndCheckNodeUtilization(t, ctx,
+		500, 1310,
+		metricsClientset, collector, usageSnapshot, nodes, n2.Name, n2metrics,
+	)
+
+	updateMetricsAndCheckNodeUtilization(t, ctx,
+		900, 1269,
+		metricsClientset, collector, usageSnapshot, nodes, n2.Name, n2metrics,
+	)
+}

pkg/framework/profile/profile.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 
 	"k8s.io/klog/v2"
 )
@@ -67,6 +68,7 @@ func (ei *evictorImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.Ev
 // handleImpl implements the framework handle which gets passed to plugins
 type handleImpl struct {
 	clientSet                 clientset.Interface
+	metricsCollector          *metricscollector.MetricsCollector
 	getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
 	sharedInformerFactory     informers.SharedInformerFactory
 	evictor                   *evictorImpl
@@ -79,6 +81,10 @@ func (hi *handleImpl) ClientSet() clientset.Interface {
 	return hi.clientSet
 }
 
+func (hi *handleImpl) MetricsCollector() *metricscollector.MetricsCollector {
+	return hi.metricsCollector
+}
+
 // GetPodsAssignedToNodeFunc retrieves GetPodsAssignedToNodeFunc implementation
 func (hi *handleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc {
 	return hi.getPodsAssignedToNodeFunc
@@ -128,6 +134,7 @@ type handleImplOpts struct {
 	sharedInformerFactory     informers.SharedInformerFactory
 	getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
 	podEvictor                *evictions.PodEvictor
+	metricsCollector          *metricscollector.MetricsCollector
 }
 
 // WithClientSet sets clientSet for the scheduling frameworkImpl.
@@ -155,6 +162,12 @@ func WithGetPodsAssignedToNodeFnc(getPodsAssignedToNodeFunc podutil.GetPodsAssig
 	}
 }
 
+func WithMetricsCollector(metricsCollector *metricscollector.MetricsCollector) Option {
+	return func(o *handleImplOpts) {
+		o.metricsCollector = metricsCollector
+	}
+}
+
 func getPluginConfig(pluginName string, pluginConfigs []api.PluginConfig) (*api.PluginConfig, int) {
 	for idx, pluginConfig := range pluginConfigs {
 		if pluginConfig.Name == pluginName {
@@ -253,6 +266,7 @@ func NewProfile(config api.DeschedulerProfile, reg pluginregistry.Registry, opts
 			profileName: config.Name,
 			podEvictor:  hOpts.podEvictor,
 		},
+		metricsCollector: hOpts.metricsCollector,
 	}
 
 	pluginNames := append(config.Plugins.Deschedule.Enabled, config.Plugins.Balance.Enabled...)

pkg/framework/types/types.go

@@ -22,6 +22,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
@@ -36,6 +37,7 @@ type Handle interface {
 	Evictor() Evictor
 	GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc
 	SharedInformerFactory() informers.SharedInformerFactory
+	MetricsCollector() *metricscollector.MetricsCollector
 }
 
 // Evictor defines an interface for filtering and evicting pods

test/e2e/e2e_duplicatepods_test.go

@@ -21,30 +21,68 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	componentbaseconfig "k8s.io/component-base/config"
 	utilptr "k8s.io/utils/ptr"
 
 	"sigs.k8s.io/descheduler/pkg/api"
+	apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2"
 	"sigs.k8s.io/descheduler/pkg/descheduler/client"
-	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates"
-	frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing"
-	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
 )
 
+func removeDuplicatesPolicy(removeDuplicatesArgs *removeduplicates.RemoveDuplicatesArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy {
+	return &apiv1alpha2.DeschedulerPolicy{
+		Profiles: []apiv1alpha2.DeschedulerProfile{
+			{
+				Name: removeduplicates.PluginName + "Profile",
+				PluginConfigs: []apiv1alpha2.PluginConfig{
+					{
+						Name: removeduplicates.PluginName,
+						Args: runtime.RawExtension{
+							Object: removeDuplicatesArgs,
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: runtime.RawExtension{
+							Object: evictorArgs,
+						},
+					},
+				},
+				Plugins: apiv1alpha2.Plugins{
+					Filter: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Balance: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							removeduplicates.PluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestRemoveDuplicates(t *testing.T) {
 	ctx := context.Background()
 
 	clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
 	if err != nil {
-		t.Errorf("Error during client creation with %v", err)
+		t.Errorf("Error during kubernetes client creation with %v", err)
 	}
 
 	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
@@ -62,67 +100,33 @@ func TestRemoveDuplicates(t *testing.T) {
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
 
 	t.Log("Creating duplicates pods")
-
-	deploymentObj := &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "duplicate-pod",
-			Namespace: testNamespace.Name,
-			Labels:    map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"},
-		},
-		Spec: appsv1.DeploymentSpec{
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"},
-				},
-				Spec: v1.PodSpec{
-					SecurityContext: &v1.PodSecurityContext{
-						RunAsNonRoot: utilptr.To(true),
-						RunAsUser:    utilptr.To[int64](1000),
-						RunAsGroup:   utilptr.To[int64](1000),
-						SeccompProfile: &v1.SeccompProfile{
-							Type: v1.SeccompProfileTypeRuntimeDefault,
-						},
-					},
-					Containers: []v1.Container{{
-						Name:            "pause",
-						ImagePullPolicy: "Always",
-						Image:           "registry.k8s.io/pause",
-						Ports:           []v1.ContainerPort{{ContainerPort: 80}},
-						SecurityContext: &v1.SecurityContext{
-							AllowPrivilegeEscalation: utilptr.To(false),
-							Capabilities: &v1.Capabilities{
-								Drop: []v1.Capability{
-									"ALL",
-								},
-							},
-						},
-					}},
-				},
-			},
-		},
-	}
+	testLabel := map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}
+	deploymentObj := buildTestDeployment("duplicate-pod", testNamespace.Name, 0, testLabel, nil)
 
 	tests := []struct {
-		description             string
+		name                    string
 		replicasNum             int
 		beforeFunc              func(deployment *appsv1.Deployment)
-		expectedEvictedPodCount uint
-		minReplicas             uint
+		expectedEvictedPodCount int
+		removeDuplicatesArgs    *removeduplicates.RemoveDuplicatesArgs
+		evictorArgs             *defaultevictor.DefaultEvictorArgs
 	}{
 		{
-			description: "Evict Pod even Pods schedule to specific node",
+			name:        "Evict Pod even Pods schedule to specific node",
 			replicasNum: 4,
 			beforeFunc: func(deployment *appsv1.Deployment) {
 				deployment.Spec.Replicas = utilptr.To[int32](4)
 				deployment.Spec.Template.Spec.NodeName = workerNodes[0].Name
 			},
 			expectedEvictedPodCount: 2,
+			removeDuplicatesArgs:    &removeduplicates.RemoveDuplicatesArgs{},
+			evictorArgs: &defaultevictor.DefaultEvictorArgs{
+				EvictLocalStoragePods: true,
+				MinReplicas:           3,
+			},
 		},
 		{
-			description: "Evict Pod even Pods with local storage",
+			name:        "Evict Pod even Pods with local storage",
 			replicasNum: 5,
 			beforeFunc: func(deployment *appsv1.Deployment) {
 				deployment.Spec.Replicas = utilptr.To[int32](5)
@@ -138,19 +142,28 @@ func TestRemoveDuplicates(t *testing.T) {
 				}
 			},
 			expectedEvictedPodCount: 2,
+			removeDuplicatesArgs:    &removeduplicates.RemoveDuplicatesArgs{},
+			evictorArgs: &defaultevictor.DefaultEvictorArgs{
+				EvictLocalStoragePods: true,
+				MinReplicas:           3,
+			},
 		},
 		{
-			description: "Ignores eviction with minReplicas of 4",
+			name:        "Ignores eviction with minReplicas of 4",
 			replicasNum: 3,
 			beforeFunc: func(deployment *appsv1.Deployment) {
 				deployment.Spec.Replicas = utilptr.To[int32](3)
 			},
 			expectedEvictedPodCount: 0,
-			minReplicas:             4,
+			removeDuplicatesArgs:    &removeduplicates.RemoveDuplicatesArgs{},
+			evictorArgs: &defaultevictor.DefaultEvictorArgs{
+				EvictLocalStoragePods: true,
+				MinReplicas:           4,
+			},
 		},
 	}
 	for _, tc := range tests {
-		t.Run(tc.description, func(t *testing.T) {
+		t.Run(tc.name, func(t *testing.T) {
 			t.Logf("Creating deployment %v in %v namespace", deploymentObj.Name, deploymentObj.Namespace)
 			tc.beforeFunc(deploymentObj)
 
@@ -158,52 +171,93 @@ func TestRemoveDuplicates(t *testing.T) {
 			if err != nil {
 				t.Logf("Error creating deployment: %v", err)
 				if err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
-					LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"})).String(),
+					LabelSelector: labels.SelectorFromSet(deploymentObj.Labels).String(),
 				}); err != nil {
 					t.Fatalf("Unable to delete deployment: %v", err)
 				}
 				return
 			}
-			defer clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{})
-			waitForPodsRunning(ctx, t, clientSet, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, tc.replicasNum, testNamespace.Name)
+			defer func() {
+				clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{})
+				waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace)
+			}()
+			waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace)
 
-			// Run removeduplicates plugin
-			evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet)
-			if err != nil || len(evictionPolicyGroupVersion) == 0 {
-				t.Fatalf("Error creating eviction policy group %v", err)
+			preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+
+			// Deploy the descheduler with the configured policy
+			tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{
+				Include: []string{testNamespace.Name},
 			}
-
-			handle, podEvictor, err := frameworktesting.InitFrameworkHandle(
-				ctx,
-				clientSet,
-				nil,
-				defaultevictor.DefaultEvictorArgs{
-					EvictLocalStoragePods: true,
-					MinReplicas:           tc.minReplicas,
-				},
-				nil,
-			)
+			deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs))
 			if err != nil {
-				t.Fatalf("Unable to initialize a framework handle: %v", err)
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 			}
 
-			plugin, err := removeduplicates.New(&removeduplicates.RemoveDuplicatesArgs{
-				Namespaces: &api.Namespaces{
-					Include: []string{testNamespace.Name},
-				},
-			},
-				handle,
-			)
+			t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
+			_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
 			if err != nil {
-				t.Fatalf("Unable to initialize the plugin: %v", err)
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 			}
-			t.Log("Running removeduplicates plugin")
-			plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes)
 
-			waitForTerminatingPodsToDisappear(ctx, t, clientSet, testNamespace.Name)
-			actualEvictedPodCount := podEvictor.TotalEvicted()
-			if actualEvictedPodCount != tc.expectedEvictedPodCount {
-				t.Errorf("Test error for description: %s. Unexpected number of pods have been evicted, got %v, expected %v", tc.description, actualEvictedPodCount, tc.expectedEvictedPodCount)
+			defer func() {
+				t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
+				err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+				}
+			}()
+
+			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
+			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+			}
+
+			deschedulerPodName := ""
+			defer func() {
+				if deschedulerPodName != "" {
+					printPodLogs(ctx, t, clientSet, deschedulerPodName)
+				}
+
+				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
+				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
+				}
+
+				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
+			}()
+
+			t.Logf("Waiting for the descheduler pod running")
+			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+			if len(deschedulerPods) != 0 {
+				deschedulerPodName = deschedulerPods[0].Name
+			}
+
+			// Run RemoveDuplicates strategy
+			var meetsExpectations bool
+			var actualEvictedPodCount int
+			if err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+				currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+				actualEvictedPod := preRunNames.Difference(currentRunNames)
+				actualEvictedPodCount = actualEvictedPod.Len()
+				t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount)
+				if actualEvictedPodCount != tc.expectedEvictedPodCount {
+					t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount)
+					return false, nil
+				}
+				meetsExpectations = true
+				return true, nil
+			}); err != nil {
+				t.Errorf("Error waiting for descheduler running: %v", err)
+			}
+
+			if !meetsExpectations {
+				t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount)
+			} else {
+				t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name)
 			}
 		})
 	}

test/e2e/e2e_failedpods_test.go

@@ -11,130 +11,216 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	componentbaseconfig "k8s.io/component-base/config"
 	utilptr "k8s.io/utils/ptr"
 
+	"sigs.k8s.io/descheduler/pkg/api"
+	apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2"
 	"sigs.k8s.io/descheduler/pkg/descheduler/client"
-	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
-	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/removefailedpods"
-	frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing"
-	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
-	"sigs.k8s.io/descheduler/test"
 )
 
-var oneHourPodLifetimeSeconds uint = 3600
+var (
+	oneHourPodLifetimeSeconds   uint = 3600
+	oneSecondPodLifetimeSeconds uint = 1
+)
+
+func removeFailedPodsPolicy(removeFailedPodsArgs *removefailedpods.RemoveFailedPodsArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy {
+	return &apiv1alpha2.DeschedulerPolicy{
+		Profiles: []apiv1alpha2.DeschedulerProfile{
+			{
+				Name: removefailedpods.PluginName + "Profile",
+				PluginConfigs: []apiv1alpha2.PluginConfig{
+					{
+						Name: removefailedpods.PluginName,
+						Args: runtime.RawExtension{
+							Object: removeFailedPodsArgs,
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: runtime.RawExtension{
+							Object: evictorArgs,
+						},
+					},
+				},
+				Plugins: apiv1alpha2.Plugins{
+					Filter: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Deschedule: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							removefailedpods.PluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
 
 func TestFailedPods(t *testing.T) {
 	ctx := context.Background()
 
 	clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
 	if err != nil {
-		t.Errorf("Error during client creation with %v", err)
+		t.Errorf("Error during kubernetes client creation with %v", err)
 	}
 
-	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
-	if err != nil {
-		t.Errorf("Error listing node with %v", err)
-	}
-	nodes, _ := splitNodesAndWorkerNodes(nodeList.Items)
 	t.Log("Creating testing namespace")
 	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
 	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
 		t.Fatalf("Unable to create ns %v", testNamespace.Name)
 	}
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
-	testCases := map[string]struct {
-		expectedEvictedCount uint
-		args                 *removefailedpods.RemoveFailedPodsArgs
+
+	tests := []struct {
+		name                    string
+		expectedEvictedPodCount int
+		removeFailedPodsArgs    *removefailedpods.RemoveFailedPodsArgs
 	}{
-		"test-failed-pods-default-args": {
-			expectedEvictedCount: 1,
-			args:                 &removefailedpods.RemoveFailedPodsArgs{},
-		},
-		"test-failed-pods-reason-unmatched": {
-			expectedEvictedCount: 0,
-			args: &removefailedpods.RemoveFailedPodsArgs{
-				Reasons: []string{"ReasonDoesNotMatch"},
+		{
+			name:                    "test-failed-pods-default-args",
+			expectedEvictedPodCount: 1,
+			removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{
+				MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds,
 			},
 		},
-		"test-failed-pods-min-age-unmet": {
-			expectedEvictedCount: 0,
-			args: &removefailedpods.RemoveFailedPodsArgs{
+		{
+			name:                    "test-failed-pods-reason-unmatched",
+			expectedEvictedPodCount: 0,
+			removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{
+				Reasons:               []string{"ReasonDoesNotMatch"},
+				MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds,
+			},
+		},
+		{
+			name:                    "test-failed-pods-min-age-unmet",
+			expectedEvictedPodCount: 0,
+			removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{
 				MinPodLifetimeSeconds: &oneHourPodLifetimeSeconds,
 			},
 		},
-		"test-failed-pods-exclude-job-kind": {
-			expectedEvictedCount: 0,
-			args: &removefailedpods.RemoveFailedPodsArgs{
-				ExcludeOwnerKinds: []string{"Job"},
+		{
+			name:                    "test-failed-pods-exclude-job-kind",
+			expectedEvictedPodCount: 0,
+			removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{
+				ExcludeOwnerKinds:     []string{"Job"},
+				MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds,
 			},
 		},
 	}
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			job := initFailedJob(name, testNamespace.Namespace)
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			job := initFailedJob(tc.name, testNamespace.Namespace)
 			t.Logf("Creating job %s in %s namespace", job.Name, job.Namespace)
 			jobClient := clientSet.BatchV1().Jobs(testNamespace.Name)
 			if _, err := jobClient.Create(ctx, job, metav1.CreateOptions{}); err != nil {
-				t.Fatalf("Error creating Job %s: %v", name, err)
+				t.Fatalf("Error creating Job %s: %v", tc.name, err)
 			}
 			deletePropagationPolicy := metav1.DeletePropagationForeground
-			defer jobClient.Delete(ctx, job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy})
+			defer func() {
+				jobClient.Delete(ctx, job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy})
+				waitForPodsToDisappear(ctx, t, clientSet, job.Labels, job.Namespace)
+			}()
 			waitForJobPodPhase(ctx, t, clientSet, job, v1.PodFailed)
 
-			evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet)
-			if err != nil || len(evictionPolicyGroupVersion) == 0 {
-				t.Fatalf("Error detecting eviction policy group: %v", err)
+			preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+
+			// Deploy the descheduler with the configured policy
+			evictorArgs := &defaultevictor.DefaultEvictorArgs{
+				EvictLocalStoragePods:   true,
+				EvictSystemCriticalPods: false,
+				IgnorePvcPods:           false,
+				EvictFailedBarePods:     false,
+			}
+			tc.removeFailedPodsArgs.Namespaces = &api.Namespaces{
+				Include: []string{testNamespace.Name},
 			}
 
-			handle, podEvictor, err := frameworktesting.InitFrameworkHandle(
-				ctx,
-				clientSet,
-				evictions.NewOptions().
-					WithPolicyGroupVersion(evictionPolicyGroupVersion),
-				defaultevictor.DefaultEvictorArgs{
-					EvictLocalStoragePods: true,
-				},
-				nil,
-			)
+			deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs))
 			if err != nil {
-				t.Fatalf("Unable to initialize a framework handle: %v", err)
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 			}
 
-			t.Logf("Running RemoveFailedPods strategy for %s", name)
-
-			plugin, err := removefailedpods.New(&removefailedpods.RemoveFailedPodsArgs{
-				Reasons:                 tc.args.Reasons,
-				MinPodLifetimeSeconds:   tc.args.MinPodLifetimeSeconds,
-				IncludingInitContainers: tc.args.IncludingInitContainers,
-				ExcludeOwnerKinds:       tc.args.ExcludeOwnerKinds,
-				LabelSelector:           tc.args.LabelSelector,
-				Namespaces:              tc.args.Namespaces,
-			},
-				handle,
-			)
+			t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
+			_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
 			if err != nil {
-				t.Fatalf("Unable to initialize the plugin: %v", err)
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 			}
 
-			plugin.(frameworktypes.DeschedulePlugin).Deschedule(ctx, nodes)
-			t.Logf("Finished RemoveFailedPods strategy for %s", name)
+			defer func() {
+				t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
+				err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+				}
+			}()
 
-			if actualEvictedCount := podEvictor.TotalEvicted(); actualEvictedCount == tc.expectedEvictedCount {
-				t.Logf("Total of %d Pods were evicted for %s", actualEvictedCount, name)
+			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
+			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+			}
+
+			deschedulerPodName := ""
+			defer func() {
+				if deschedulerPodName != "" {
+					printPodLogs(ctx, t, clientSet, deschedulerPodName)
+				}
+
+				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
+				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
+				}
+
+				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
+			}()
+
+			t.Logf("Waiting for the descheduler pod running")
+			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+			if len(deschedulerPods) != 0 {
+				deschedulerPodName = deschedulerPods[0].Name
+			}
+
+			// Run RemoveDuplicates strategy
+			var meetsExpectations bool
+			var actualEvictedPodCount int
+			if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+				currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+				actualEvictedPod := preRunNames.Difference(currentRunNames)
+				actualEvictedPodCount = actualEvictedPod.Len()
+				t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount)
+				if actualEvictedPodCount != tc.expectedEvictedPodCount {
+					t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount)
+					return false, nil
+				}
+				meetsExpectations = true
+				return true, nil
+			}); err != nil {
+				t.Errorf("Error waiting for descheduler running: %v", err)
+			}
+
+			if !meetsExpectations {
+				t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount)
 			} else {
-				t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedCount, tc.expectedEvictedCount)
+				t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name)
 			}
 		})
 	}
 }
 
 func initFailedJob(name, namespace string) *batchv1.Job {
-	podSpec := test.MakePodSpec("", nil)
+	podSpec := makePodSpec("", nil)
 	podSpec.Containers[0].Command = []string{"/bin/false"}
 	podSpec.RestartPolicy = v1.RestartPolicyNever
 	labelsSet := labels.Set{"test": name, "name": name}

test/e2e/e2e_leaderelection_test.go

@@ -30,17 +30,60 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
-	utilptr "k8s.io/utils/ptr"
-	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
-	"sigs.k8s.io/descheduler/pkg/descheduler"
+	componentbaseconfig "k8s.io/component-base/config"
+
+	"sigs.k8s.io/descheduler/pkg/api"
+	apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2"
+	"sigs.k8s.io/descheduler/pkg/descheduler/client"
+	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
+	"sigs.k8s.io/descheduler/pkg/framework/plugins/podlifetime"
 )
 
+func podlifetimePolicy(podLifeTimeArgs *podlifetime.PodLifeTimeArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy {
+	return &apiv1alpha2.DeschedulerPolicy{
+		Profiles: []apiv1alpha2.DeschedulerProfile{
+			{
+				Name: podlifetime.PluginName + "Profile",
+				PluginConfigs: []apiv1alpha2.PluginConfig{
+					{
+						Name: podlifetime.PluginName,
+						Args: runtime.RawExtension{
+							Object: podLifeTimeArgs,
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: runtime.RawExtension{
+							Object: evictorArgs,
+						},
+					},
+				},
+				Plugins: apiv1alpha2.Plugins{
+					Filter: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Deschedule: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							podlifetime.PluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestLeaderElection(t *testing.T) {
-	descheduler.SetupPlugins()
 	ctx := context.Background()
 
-	clientSet, _, _, _ := initializeClient(ctx, t)
+	clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
+	if err != nil {
+		t.Errorf("Error during kubernetes client creation with %v", err)
+	}
 
 	ns1 := "e2e-" + strings.ToLower(t.Name()+"-a")
 	ns2 := "e2e-" + strings.ToLower(t.Name()+"-b")
@@ -59,51 +102,28 @@ func TestLeaderElection(t *testing.T) {
 	}
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace2.Name, metav1.DeleteOptions{})
 
-	deployment1, err := createDeployment(ctx, clientSet, ns1, 5, t)
+	testLabel := map[string]string{"test": "leaderelection", "name": "test-leaderelection"}
+	deployment1 := buildTestDeployment("leaderelection", ns1, 5, testLabel, nil)
+	err = createDeployment(t, ctx, clientSet, deployment1)
 	if err != nil {
 		t.Fatalf("create deployment 1: %v", err)
 	}
-	defer clientSet.AppsV1().Deployments(deployment1.Namespace).Delete(ctx, deployment1.Name, metav1.DeleteOptions{})
 
-	deployment2, err := createDeployment(ctx, clientSet, ns2, 5, t)
+	deployment2 := buildTestDeployment("leaderelection", ns2, 5, testLabel, nil)
+	err = createDeployment(t, ctx, clientSet, deployment2)
 	if err != nil {
 		t.Fatalf("create deployment 2: %v", err)
 	}
-	defer clientSet.AppsV1().Deployments(deployment2.Namespace).Delete(ctx, deployment2.Name, metav1.DeleteOptions{})
+	defer func() {
+		clientSet.AppsV1().Deployments(deployment1.Namespace).Delete(ctx, deployment1.Name, metav1.DeleteOptions{})
+		clientSet.AppsV1().Deployments(deployment2.Namespace).Delete(ctx, deployment2.Name, metav1.DeleteOptions{})
+	}()
 
-	waitForPodsRunning(ctx, t, clientSet, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, 5, ns1)
+	waitForPodsRunning(ctx, t, clientSet, deployment1.Labels, 5, deployment1.Namespace)
+	podListAOrg := getCurrentPodNames(ctx, clientSet, ns1, t)
 
-	podListAOrg := getPodNameList(ctx, clientSet, ns1, t)
-
-	waitForPodsRunning(ctx, t, clientSet, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, 5, ns2)
-
-	podListBOrg := getPodNameList(ctx, clientSet, ns2, t)
-
-	s1, err := options.NewDeschedulerServer()
-	if err != nil {
-		t.Fatalf("unable to initialize server: %v", err)
-	}
-	s1.Client = clientSet
-	s1.DeschedulingInterval = 5 * time.Second
-	s1.LeaderElection.LeaderElect = true
-	s1.LeaderElection.RetryPeriod = metav1.Duration{
-		Duration: time.Second,
-	}
-	s1.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
-	s1.PolicyConfigFile = "./policy_leaderelection_a.yaml"
-
-	s2, err := options.NewDeschedulerServer()
-	if err != nil {
-		t.Fatalf("unable to initialize server: %v", err)
-	}
-	s2.Client = clientSet
-	s2.DeschedulingInterval = 5 * time.Second
-	s2.LeaderElection.LeaderElect = true
-	s2.LeaderElection.RetryPeriod = metav1.Duration{
-		Duration: time.Second,
-	}
-	s2.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG")
-	s2.PolicyConfigFile = "./policy_leaderelection_b.yaml"
+	waitForPodsRunning(ctx, t, clientSet, deployment2.Labels, 5, deployment2.Namespace)
+	podListBOrg := getCurrentPodNames(ctx, clientSet, ns2, t)
 
 	// Delete the descheduler lease
 	err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
@@ -114,36 +134,42 @@ func TestLeaderElection(t *testing.T) {
 	}
 	t.Logf("Removed kube-system/descheduler lease")
 
-	t.Log("starting deschedulers")
-
-	go func() {
-		err := descheduler.Run(ctx, s1)
-		if err != nil {
-			t.Errorf("unable to start descheduler: %v", err)
-			return
-		}
-	}()
-
+	t.Log("Starting deschedulers")
+	pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1)
 	time.Sleep(1 * time.Second)
-
-	go func() {
-		err := descheduler.Run(ctx, s2)
-		if err != nil {
-			t.Errorf("unable to start descheduler: %v", err)
-			return
+	pod2Name, deploy2, cm2 := startDeschedulerServer(t, ctx, clientSet, ns2)
+	defer func() {
+		for _, podName := range []string{pod1Name, pod2Name} {
+			printPodLogs(ctx, t, clientSet, podName)
 		}
-	}()
 
-	defer clientSet.CoordinationV1().Leases(s1.LeaderElection.ResourceNamespace).Delete(ctx, s1.LeaderElection.ResourceName, metav1.DeleteOptions{})
-	defer clientSet.CoordinationV1().Leases(s2.LeaderElection.ResourceNamespace).Delete(ctx, s2.LeaderElection.ResourceName, metav1.DeleteOptions{})
+		for _, deploy := range []*appsv1.Deployment{deploy1, deploy2} {
+			t.Logf("Deleting %q deployment...", deploy.Name)
+			err = clientSet.AppsV1().Deployments(deploy.Namespace).Delete(ctx, deploy.Name, metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("Unable to delete %q deployment: %v", deploy.Name, err)
+			}
+
+			waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace)
+		}
+
+		for _, cm := range []*v1.ConfigMap{cm1, cm2} {
+			t.Logf("Deleting %q CM...", cm.Name)
+			err = clientSet.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{})
+			if err != nil {
+				t.Fatalf("Unable to delete %q CM: %v", cm.Name, err)
+			}
+		}
+
+		clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{})
+	}()
 
 	// wait for a while so all the pods are 5 seconds older
 	time.Sleep(7 * time.Second)
 
 	// validate only pods from e2e-testleaderelection-a namespace are evicted.
-	podListA := getPodNameList(ctx, clientSet, ns1, t)
-
-	podListB := getPodNameList(ctx, clientSet, ns2, t)
+	podListA := getCurrentPodNames(ctx, clientSet, ns1, t)
+	podListB := getCurrentPodNames(ctx, clientSet, ns2, t)
 
 	left := reflect.DeepEqual(podListAOrg, podListA)
 	right := reflect.DeepEqual(podListBOrg, podListB)
@@ -165,73 +191,78 @@ func TestLeaderElection(t *testing.T) {
 	}
 }
 
-func createDeployment(ctx context.Context, clientSet clientset.Interface, namespace string, replicas int32, t *testing.T) (*appsv1.Deployment, error) {
-	deployment := &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "leaderelection",
-			Namespace: namespace,
-			Labels:    map[string]string{"test": "leaderelection", "name": "test-leaderelection"},
-		},
-		Spec: appsv1.DeploymentSpec{
-			Replicas: utilptr.To[int32](replicas),
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"test": "leaderelection", "name": "test-leaderelection"},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{"test": "leaderelection", "name": "test-leaderelection"},
-				},
-				Spec: v1.PodSpec{
-					SecurityContext: &v1.PodSecurityContext{
-						RunAsNonRoot: utilptr.To(true),
-						RunAsUser:    utilptr.To[int64](1000),
-						RunAsGroup:   utilptr.To[int64](1000),
-						SeccompProfile: &v1.SeccompProfile{
-							Type: v1.SeccompProfileTypeRuntimeDefault,
-						},
-					},
-					Containers: []v1.Container{{
-						Name:            "pause",
-						ImagePullPolicy: "Always",
-						Image:           "registry.k8s.io/pause",
-						Ports:           []v1.ContainerPort{{ContainerPort: 80}},
-						SecurityContext: &v1.SecurityContext{
-							AllowPrivilegeEscalation: utilptr.To(false),
-							Capabilities: &v1.Capabilities{
-								Drop: []v1.Capability{
-									"ALL",
-								},
-							},
-						},
-					}},
-				},
-			},
-		},
-	}
-
+func createDeployment(t *testing.T, ctx context.Context, clientSet clientset.Interface, deployment *appsv1.Deployment) error {
 	t.Logf("Creating deployment %v for namespace %s", deployment.Name, deployment.Namespace)
-	deployment, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
+	_, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{})
 	if err != nil {
 		t.Logf("Error creating deployment: %v", err)
 		if err = clientSet.AppsV1().Deployments(deployment.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
-			LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "leaderelection", "name": "test-leaderelection"})).String(),
+			LabelSelector: labels.SelectorFromSet(deployment.Labels).String(),
 		}); err != nil {
 			t.Fatalf("Unable to delete deployment: %v", err)
 		}
-		return nil, fmt.Errorf("create deployment %v", err)
+		return fmt.Errorf("create deployment %v", err)
 	}
-	return deployment, nil
+	return nil
 }
 
-func getPodNameList(ctx context.Context, clientSet clientset.Interface, namespace string, t *testing.T) []string {
-	podList, err := clientSet.CoreV1().Pods(namespace).List(
-		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "leaderelection", "name": "test-leaderelection"})).String()})
+func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clientset.Interface, testName string) (string, *appsv1.Deployment, *v1.ConfigMap) {
+	var maxLifeTime uint = 5
+	podLifeTimeArgs := &podlifetime.PodLifeTimeArgs{
+		MaxPodLifeTimeSeconds: &maxLifeTime,
+		Namespaces: &api.Namespaces{
+			Include: []string{testName},
+		},
+	}
+
+	// Deploy the descheduler with the configured policy
+	evictorArgs := &defaultevictor.DefaultEvictorArgs{
+		EvictLocalStoragePods:   true,
+		EvictSystemCriticalPods: false,
+		IgnorePvcPods:           false,
+		EvictFailedBarePods:     false,
+	}
+	deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs))
+	deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName)
 	if err != nil {
-		t.Fatalf("Unable to list pods from ns: %s: %v", namespace, err)
+		t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 	}
-	podNames := make([]string, len(podList.Items))
-	for i, pod := range podList.Items {
-		podNames[i] = pod.Name
+
+	t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name)
+	_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 	}
-	return podNames
+
+	deschedulerDeploymentObj := deschedulerDeployment(testName)
+	deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName)
+	args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args
+	deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect", "--leader-elect-retry-period", "1s")
+	deschedulerDeploymentObj.Spec.Template.Spec.Volumes = []v1.Volume{
+		{
+			Name: "policy-volume",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: deschedulerPolicyConfigMapObj.Name,
+					},
+				},
+			},
+		},
+	}
+
+	t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+	_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+	if err != nil {
+		t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+	}
+
+	t.Logf("Waiting for the descheduler pod running")
+	var podName string
+	pods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+	if len(pods) != 0 {
+		podName = pods[0].Name
+	}
+
+	return podName, deschedulerDeploymentObj, deschedulerPolicyConfigMapObj
 }

test/e2e/e2e_test.go

@@ -27,8 +27,6 @@ import (
 	"testing"
 	"time"
 
-	"sigs.k8s.io/yaml"
-
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	schedulingv1 "k8s.io/api/scheduling/v1"
@@ -44,6 +42,7 @@ import (
 	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/klog/v2"
 	utilptr "k8s.io/utils/ptr"
+	"sigs.k8s.io/yaml"
 
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
 	"sigs.k8s.io/descheduler/pkg/api"
@@ -63,7 +62,6 @@ import (
 	frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
 	"sigs.k8s.io/descheduler/pkg/utils"
-	"sigs.k8s.io/descheduler/test"
 )
 
 func isClientRateLimiterError(err error) bool {
@@ -195,67 +193,6 @@ func printPodLogs(ctx context.Context, t *testing.T, kubeClient clientset.Interf
 	}
 }
 
-func waitForDeschedulerPodRunning(t *testing.T, ctx context.Context, kubeClient clientset.Interface, testName string) string {
-	deschedulerPodName := ""
-	if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
-		podList, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
-			LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "descheduler", "test": testName})).String(),
-		})
-		if err != nil {
-			t.Logf("Unable to list pods: %v", err)
-			if isClientRateLimiterError(err) {
-				return false, nil
-			}
-			return false, err
-		}
-
-		runningPods := []*v1.Pod{}
-		for _, item := range podList.Items {
-			if item.Status.Phase != v1.PodRunning {
-				continue
-			}
-			pod := item
-			runningPods = append(runningPods, &pod)
-		}
-
-		if len(runningPods) != 1 {
-			t.Logf("Expected a single running pod, got %v instead", len(runningPods))
-			return false, nil
-		}
-
-		deschedulerPodName = runningPods[0].Name
-		t.Logf("Found a descheduler pod running: %v", deschedulerPodName)
-		return true, nil
-	}); err != nil {
-		t.Fatalf("Error waiting for a running descheduler: %v", err)
-	}
-	return deschedulerPodName
-}
-
-func waitForDeschedulerPodAbsent(t *testing.T, ctx context.Context, kubeClient clientset.Interface, testName string) {
-	if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
-		podList, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
-			LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "descheduler", "test": testName})).String(),
-		})
-		if err != nil {
-			t.Logf("Unable to list pods: %v", err)
-			if isClientRateLimiterError(err) {
-				return false, nil
-			}
-			return false, err
-		}
-
-		if len(podList.Items) > 0 {
-			t.Logf("Found a descheduler pod. Waiting until it gets deleted")
-			return false, nil
-		}
-
-		return true, nil
-	}); err != nil {
-		t.Fatalf("Error waiting for a descheduler to disapear: %v", err)
-	}
-}
-
 func TestMain(m *testing.M) {
 	if os.Getenv("DESCHEDULER_IMAGE") == "" {
 		klog.Errorf("DESCHEDULER_IMAGE env is not set")
@@ -297,7 +234,7 @@ func RcByNameContainer(name, namespace string, replicas int32, labels map[string
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: labels,
 				},
-				Spec: test.MakePodSpec(priorityClassName, gracePeriod),
+				Spec: makePodSpec(priorityClassName, gracePeriod),
 			},
 		},
 	}
@@ -329,12 +266,83 @@ func DsByNameContainer(name, namespace string, labels map[string]string, gracePe
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: labels,
 				},
-				Spec: test.MakePodSpec("", gracePeriod),
+				Spec: makePodSpec("", gracePeriod),
 			},
 		},
 	}
 }
 
+func buildTestDeployment(name, namespace string, replicas int32, testLabel map[string]string, apply func(deployment *appsv1.Deployment)) *appsv1.Deployment {
+	deployment := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    testLabel,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: utilptr.To[int32](replicas),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: testLabel,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: testLabel,
+				},
+				Spec: makePodSpec("", utilptr.To[int64](0)),
+			},
+		},
+	}
+
+	if apply != nil {
+		apply(deployment)
+	}
+
+	return deployment
+}
+
+func makePodSpec(priorityClassName string, gracePeriod *int64) v1.PodSpec {
+	return v1.PodSpec{
+		SecurityContext: &v1.PodSecurityContext{
+			RunAsNonRoot: utilptr.To(true),
+			RunAsUser:    utilptr.To[int64](1000),
+			RunAsGroup:   utilptr.To[int64](1000),
+			SeccompProfile: &v1.SeccompProfile{
+				Type: v1.SeccompProfileTypeRuntimeDefault,
+			},
+		},
+		Containers: []v1.Container{{
+			Name:            "pause",
+			ImagePullPolicy: "IfNotPresent",
+			Image:           "registry.k8s.io/pause",
+			Ports:           []v1.ContainerPort{{ContainerPort: 80}},
+			Resources: v1.ResourceRequirements{
+				Limits: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("100m"),
+					v1.ResourceMemory: resource.MustParse("200Mi"),
+				},
+				Requests: v1.ResourceList{
+					v1.ResourceCPU:    resource.MustParse("100m"),
+					v1.ResourceMemory: resource.MustParse("100Mi"),
+				},
+			},
+			SecurityContext: &v1.SecurityContext{
+				AllowPrivilegeEscalation: utilptr.To(false),
+				Capabilities: &v1.Capabilities{
+					Drop: []v1.Capability{
+						"ALL",
+					},
+				},
+			},
+		}},
+		PriorityClassName:             priorityClassName,
+		TerminationGracePeriodSeconds: gracePeriod,
+	}
+}
+
 func initializeClient(ctx context.Context, t *testing.T) (clientset.Interface, informers.SharedInformerFactory, listersv1.NodeLister, podutil.GetPodsAssignedToNodeFunc) {
 	clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
 	if err != nil {
@@ -1705,6 +1713,10 @@ func waitForPodRunning(ctx context.Context, t *testing.T, clientSet clientset.In
 	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
 		podItem, err := clientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
 		if err != nil {
+			t.Logf("Unable to list pods: %v", err)
+			if isClientRateLimiterError(err) {
+				return false, nil
+			}
 			return false, err
 		}
 
@@ -1719,28 +1731,62 @@ func waitForPodRunning(ctx context.Context, t *testing.T, clientSet clientset.In
 	}
 }
 
-func waitForPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, desireRunningPodNum int, namespace string) {
-	if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+func waitForPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, desiredRunningPodNum int, namespace string) []*v1.Pod {
+	desiredRunningPods := make([]*v1.Pod, desiredRunningPodNum)
+	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
 		podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
 			LabelSelector: labels.SelectorFromSet(labelMap).String(),
 		})
 		if err != nil {
-			return false, err
-		}
-		if len(podList.Items) != desireRunningPodNum {
-			t.Logf("Waiting for %v pods to be running, got %v instead", desireRunningPodNum, len(podList.Items))
-			return false, nil
-		}
-		for _, pod := range podList.Items {
-			if pod.Status.Phase != v1.PodRunning {
-				t.Logf("Pod %v not running yet, is %v instead", pod.Name, pod.Status.Phase)
+			t.Logf("Unable to list pods: %v", err)
+			if isClientRateLimiterError(err) {
 				return false, nil
 			}
+			return false, err
 		}
+		runningPods := []*v1.Pod{}
+		for _, item := range podList.Items {
+			if item.Status.Phase != v1.PodRunning {
+				continue
+			}
+			pod := item
+			runningPods = append(runningPods, &pod)
+		}
+
+		if len(runningPods) != desiredRunningPodNum {
+			t.Logf("Waiting for %v pods to be running, got %v instead", desiredRunningPodNum, len(runningPods))
+			return false, nil
+		}
+		desiredRunningPods = runningPods
+
 		return true, nil
 	}); err != nil {
 		t.Fatalf("Error waiting for pods running: %v", err)
 	}
+	return desiredRunningPods
+}
+
+func waitForPodsToDisappear(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, namespace string) {
+	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+		podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+			LabelSelector: labels.SelectorFromSet(labelMap).String(),
+		})
+		if err != nil {
+			t.Logf("Unable to list pods: %v", err)
+			if isClientRateLimiterError(err) {
+				return false, nil
+			}
+			return false, err
+		}
+
+		if len(podList.Items) > 0 {
+			t.Logf("Found a existing pod. Waiting until it gets deleted")
+			return false, nil
+		}
+		return true, nil
+	}); err != nil {
+		t.Fatalf("Error waiting for pods to disappear: %v", err)
+	}
 }
 
 func splitNodesAndWorkerNodes(nodes []v1.Node) ([]*v1.Node, []*v1.Node) {
@@ -1756,8 +1802,8 @@ func splitNodesAndWorkerNodes(nodes []v1.Node) ([]*v1.Node, []*v1.Node) {
 	return allNodes, workerNodes
 }
 
-func getCurrentPodNames(t *testing.T, ctx context.Context, kubeClient clientset.Interface, namespace string) []string {
-	podList, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+func getCurrentPodNames(ctx context.Context, clientSet clientset.Interface, namespace string, t *testing.T) []string {
+	podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
 	if err != nil {
 		t.Logf("Unable to list pods: %v", err)
 		return nil

test/e2e/e2e_toomanyrestarts_test.go

@@ -32,7 +32,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	componentbaseconfig "k8s.io/component-base/config"
-	utilptr "k8s.io/utils/ptr"
 
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
 	"sigs.k8s.io/descheduler/pkg/api"
@@ -104,50 +103,10 @@ func TestTooManyRestarts(t *testing.T) {
 	}
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
 
-	deploymentObj := &appsv1.Deployment{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "restart-pod",
-			Namespace: testNamespace.Name,
-			Labels:    map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"},
-		},
-		Spec: appsv1.DeploymentSpec{
-			Replicas: utilptr.To[int32](deploymentReplicas),
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"},
-				},
-				Spec: v1.PodSpec{
-					SecurityContext: &v1.PodSecurityContext{
-						RunAsNonRoot: utilptr.To(true),
-						RunAsUser:    utilptr.To[int64](1000),
-						RunAsGroup:   utilptr.To[int64](1000),
-						SeccompProfile: &v1.SeccompProfile{
-							Type: v1.SeccompProfileTypeRuntimeDefault,
-						},
-					},
-					Containers: []v1.Container{{
-						Name:            "pause",
-						ImagePullPolicy: "Always",
-						Image:           "registry.k8s.io/pause",
-						Command:         []string{"/bin/sh"},
-						Args:            []string{"-c", "sleep 1s && exit 1"},
-						Ports:           []v1.ContainerPort{{ContainerPort: 80}},
-						SecurityContext: &v1.SecurityContext{
-							AllowPrivilegeEscalation: utilptr.To(false),
-							Capabilities: &v1.Capabilities{
-								Drop: []v1.Capability{
-									"ALL",
-								},
-							},
-						},
-					}},
-				},
-			},
-		},
-	}
+	deploymentObj := buildTestDeployment("restart-pod", testNamespace.Name, deploymentReplicas, map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, func(deployment *appsv1.Deployment) {
+		deployment.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh"}
+		deployment.Spec.Template.Spec.Containers[0].Args = []string{"-c", "sleep 1s && exit 1"}
+	})
 
 	t.Logf("Creating deployment %v", deploymentObj.Name)
 	_, err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).Create(ctx, deploymentObj, metav1.CreateOptions{})
@@ -190,7 +149,7 @@ func TestTooManyRestarts(t *testing.T) {
 			rs.Client = clientSet
 			rs.EventClient = clientSet
 
-			preRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...)
+			preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
 			// Deploy the descheduler with the configured policy
 			deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy)
 			if err != nil {
@@ -228,15 +187,18 @@ func TestTooManyRestarts(t *testing.T) {
 				if err != nil {
 					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
 				}
-				waitForDeschedulerPodAbsent(t, ctx, clientSet, testNamespace.Name)
+				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
 			}()
 
 			t.Logf("Waiting for the descheduler pod running")
-			deschedulerPodName = waitForDeschedulerPodRunning(t, ctx, clientSet, testNamespace.Name)
+			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+			if len(deschedulerPods) != 0 {
+				deschedulerPodName = deschedulerPods[0].Name
+			}
 
 			// Run RemovePodsHavingTooManyRestarts strategy
 			if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) {
-				currentRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...)
+				currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
 				actualEvictedPod := preRunNames.Difference(currentRunNames)
 				actualEvictedPodCount := uint(actualEvictedPod.Len())
 				t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount)

test/e2e/e2e_topologyspreadconstraint_test.go

@@ -6,30 +6,70 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	componentbaseconfig "k8s.io/component-base/config"
 
+	"sigs.k8s.io/descheduler/pkg/api"
+	apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2"
 	"sigs.k8s.io/descheduler/pkg/descheduler/client"
-	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
-	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
 	"sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint"
-	frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing"
-	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
-	"sigs.k8s.io/descheduler/test"
 )
 
 const zoneTopologyKey string = "topology.kubernetes.io/zone"
 
+func topologySpreadConstraintPolicy(constraintArgs *removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs,
+	evictorArgs *defaultevictor.DefaultEvictorArgs,
+) *apiv1alpha2.DeschedulerPolicy {
+	return &apiv1alpha2.DeschedulerPolicy{
+		Profiles: []apiv1alpha2.DeschedulerProfile{
+			{
+				Name: removepodsviolatingtopologyspreadconstraint.PluginName + "Profile",
+				PluginConfigs: []apiv1alpha2.PluginConfig{
+					{
+						Name: removepodsviolatingtopologyspreadconstraint.PluginName,
+						Args: runtime.RawExtension{
+							Object: constraintArgs,
+						},
+					},
+					{
+						Name: defaultevictor.PluginName,
+						Args: runtime.RawExtension{
+							Object: evictorArgs,
+						},
+					},
+				},
+				Plugins: apiv1alpha2.Plugins{
+					Filter: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							defaultevictor.PluginName,
+						},
+					},
+					Balance: apiv1alpha2.PluginSet{
+						Enabled: []string{
+							removepodsviolatingtopologyspreadconstraint.PluginName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func TestTopologySpreadConstraint(t *testing.T) {
 	ctx := context.Background()
+
 	clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "")
 	if err != nil {
-		t.Errorf("Error during client creation with %v", err)
+		t.Errorf("Error during kubernetes client creation with %v", err)
 	}
 
 	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
@@ -44,14 +84,16 @@ func TestTopologySpreadConstraint(t *testing.T) {
 	}
 	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})
 
-	testCases := map[string]struct {
-		expectedEvictedCount     uint
+	testCases := []struct {
+		name                     string
+		expectedEvictedPodCount  int
 		replicaCount             int
 		topologySpreadConstraint v1.TopologySpreadConstraint
 	}{
-		"test-topology-spread-hard-constraint": {
-			expectedEvictedCount: 1,
-			replicaCount:         4,
+		{
+			name:                    "test-topology-spread-hard-constraint",
+			expectedEvictedPodCount: 1,
+			replicaCount:            4,
 			topologySpreadConstraint: v1.TopologySpreadConstraint{
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
@@ -63,9 +105,10 @@ func TestTopologySpreadConstraint(t *testing.T) {
 				WhenUnsatisfiable: v1.DoNotSchedule,
 			},
 		},
-		"test-topology-spread-soft-constraint": {
-			expectedEvictedCount: 1,
-			replicaCount:         4,
+		{
+			name:                    "test-topology-spread-soft-constraint",
+			expectedEvictedPodCount: 1,
+			replicaCount:            4,
 			topologySpreadConstraint: v1.TopologySpreadConstraint{
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
@@ -77,9 +120,10 @@ func TestTopologySpreadConstraint(t *testing.T) {
 				WhenUnsatisfiable: v1.ScheduleAnyway,
 			},
 		},
-		"test-node-taints-policy-honor": {
-			expectedEvictedCount: 1,
-			replicaCount:         4,
+		{
+			name:                    "test-node-taints-policy-honor",
+			expectedEvictedPodCount: 1,
+			replicaCount:            4,
 			topologySpreadConstraint: v1.TopologySpreadConstraint{
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
@@ -92,9 +136,10 @@ func TestTopologySpreadConstraint(t *testing.T) {
 				WhenUnsatisfiable: v1.DoNotSchedule,
 			},
 		},
-		"test-node-affinity-policy-ignore": {
-			expectedEvictedCount: 1,
-			replicaCount:         4,
+		{
+			name:                    "test-node-affinity-policy-ignore",
+			expectedEvictedPodCount: 1,
+			replicaCount:            4,
 			topologySpreadConstraint: v1.TopologySpreadConstraint{
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
@@ -107,9 +152,10 @@ func TestTopologySpreadConstraint(t *testing.T) {
 				WhenUnsatisfiable:  v1.DoNotSchedule,
 			},
 		},
-		"test-match-label-keys": {
-			expectedEvictedCount: 0,
-			replicaCount:         4,
+		{
+			name:                    "test-match-label-keys",
+			expectedEvictedPodCount: 0,
+			replicaCount:            4,
 			topologySpreadConstraint: v1.TopologySpreadConstraint{
 				LabelSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
@@ -123,106 +169,172 @@ func TestTopologySpreadConstraint(t *testing.T) {
 			},
 		},
 	}
-	for name, tc := range testCases {
-		t.Run(name, func(t *testing.T) {
-			t.Logf("Creating Deployment %s with %d replicas", name, tc.replicaCount)
-			deployment := test.BuildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) {
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Logf("Creating Deployment %s with %d replicas", tc.name, tc.replicaCount)
+			deployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels
+			deployLabels["name"] = tc.name
+			deployment := buildTestDeployment(tc.name, testNamespace.Name, int32(tc.replicaCount), deployLabels, func(d *appsv1.Deployment) {
 				d.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{tc.topologySpreadConstraint}
 			})
 			if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil {
-				t.Fatalf("Error creating Deployment %s %v", name, err)
+				t.Fatalf("Error creating Deployment %s %v", tc.name, err)
 			}
-			defer test.DeleteDeployment(ctx, t, clientSet, deployment)
-			test.WaitForDeploymentPodsRunning(ctx, t, clientSet, deployment)
+			defer func() {
+				clientSet.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{})
+				waitForPodsToDisappear(ctx, t, clientSet, deployment.Labels, deployment.Namespace)
+			}()
+			waitForPodsRunning(ctx, t, clientSet, deployment.Labels, tc.replicaCount, deployment.Namespace)
 
 			// Create a "Violator" Deployment that has the same label and is forced to be on the same node using a nodeSelector
-			violatorDeploymentName := name + "-violator"
-			violatorCount := tc.topologySpreadConstraint.MaxSkew + 1
-			violatorDeployment := test.BuildTestDeployment(violatorDeploymentName, testNamespace.Name, violatorCount, tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) {
+			violatorDeploymentName := tc.name + "-violator"
+			violatorDeployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels
+			violatorDeployLabels["name"] = violatorDeploymentName
+			violatorDeployment := buildTestDeployment(violatorDeploymentName, testNamespace.Name, tc.topologySpreadConstraint.MaxSkew+1, violatorDeployLabels, func(d *appsv1.Deployment) {
 				d.Spec.Template.Spec.NodeSelector = map[string]string{zoneTopologyKey: workerNodes[0].Labels[zoneTopologyKey]}
 			})
-			if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, violatorDeployment, metav1.CreateOptions{}); err != nil {
-				t.Fatalf("Error creating Deployment %s: %v", violatorDeploymentName, err)
-			}
-			defer test.DeleteDeployment(ctx, t, clientSet, violatorDeployment)
-			test.WaitForDeploymentPodsRunning(ctx, t, clientSet, violatorDeployment)
-
-			evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet)
-			if err != nil || len(evictionPolicyGroupVersion) == 0 {
-				t.Fatalf("Error detecting eviction policy group: %v", err)
-			}
-
-			handle, podEvictor, err := frameworktesting.InitFrameworkHandle(
-				ctx,
-				clientSet,
-				evictions.NewOptions().
-					WithPolicyGroupVersion(evictionPolicyGroupVersion),
-				defaultevictor.DefaultEvictorArgs{
-					EvictLocalStoragePods: true,
-				},
-				nil,
-			)
-			if err != nil {
-				t.Fatalf("Unable to initialize a framework handle: %v", err)
+			if _, err := clientSet.AppsV1().Deployments(violatorDeployment.Namespace).Create(ctx, violatorDeployment, metav1.CreateOptions{}); err != nil {
+				t.Fatalf("Error creating Deployment %s: %v", violatorDeployment.Name, err)
 			}
+			defer func() {
+				clientSet.AppsV1().Deployments(violatorDeployment.Namespace).Delete(ctx, violatorDeployment.Name, metav1.DeleteOptions{})
+				waitForPodsToDisappear(ctx, t, clientSet, violatorDeployment.Labels, violatorDeployment.Namespace)
+			}()
+			waitForPodsRunning(ctx, t, clientSet, violatorDeployment.Labels, int(*violatorDeployment.Spec.Replicas), violatorDeployment.Namespace)
 
 			// Run TopologySpreadConstraint strategy
-			t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", name)
+			t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", tc.name)
 
-			plugin, err := removepodsviolatingtopologyspreadconstraint.New(&removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs{
+			preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+
+			evictorArgs := &defaultevictor.DefaultEvictorArgs{
+				EvictLocalStoragePods:   true,
+				EvictSystemCriticalPods: false,
+				IgnorePvcPods:           false,
+				EvictFailedBarePods:     false,
+			}
+			constraintArgs := &removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs{
 				Constraints: []v1.UnsatisfiableConstraintAction{tc.topologySpreadConstraint.WhenUnsatisfiable},
-			},
-				handle,
-			)
+				Namespaces: &api.Namespaces{
+					Include: []string{testNamespace.Name},
+				},
+			}
+			deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs))
 			if err != nil {
-				t.Fatalf("Unable to initialize the plugin: %v", err)
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
 			}
 
-			plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes)
-			t.Logf("Finished RemovePodsViolatingTopologySpreadConstraint strategy for %s", name)
+			t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name)
+			_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+			}
 
-			t.Logf("Wait for terminating pods of %s to disappear", name)
-			waitForTerminatingPodsToDisappear(ctx, t, clientSet, deployment.Namespace)
+			defer func() {
+				t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
+				err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
+				}
+			}()
+			deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
+			t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
+			_, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{})
+			if err != nil {
+				t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err)
+			}
 
-			if totalEvicted := podEvictor.TotalEvicted(); totalEvicted == tc.expectedEvictedCount {
-				t.Logf("Total of %d Pods were evicted for %s", totalEvicted, name)
+			deschedulerPodName := ""
+			defer func() {
+				if deschedulerPodName != "" {
+					printPodLogs(ctx, t, clientSet, deschedulerPodName)
+				}
+
+				t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
+				err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
+				if err != nil {
+					t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
+				}
+				waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
+			}()
+
+			t.Logf("Waiting for the descheduler pod running")
+			deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
+			if len(deschedulerPods) != 0 {
+				deschedulerPodName = deschedulerPods[0].Name
+			}
+
+			// Run RemovePodsHavingTooManyRestarts strategy
+			var meetsEvictedExpectations bool
+			var actualEvictedPodCount int
+			t.Logf("Check whether the number of evicted pods meets the expectation")
+			if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+				currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
+				actualEvictedPod := preRunNames.Difference(currentRunNames)
+				actualEvictedPodCount = actualEvictedPod.Len()
+				t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount)
+				if actualEvictedPodCount != tc.expectedEvictedPodCount {
+					t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount)
+					return false, nil
+				}
+				meetsEvictedExpectations = true
+				return true, nil
+			}); err != nil {
+				t.Errorf("Error waiting for descheduler running: %v", err)
+			}
+
+			if !meetsEvictedExpectations {
+				t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount)
 			} else {
-				t.Fatalf("Expected %d evictions but got %d for %s TopologySpreadConstraint", tc.expectedEvictedCount, totalEvicted, name)
+				t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name)
 			}
 
-			if tc.expectedEvictedCount == 0 {
+			if tc.expectedEvictedPodCount == 0 {
 				return
 			}
 
-			// Ensure recently evicted Pod are rescheduled and running before asserting for a balanced topology spread
-			test.WaitForDeploymentPodsRunning(ctx, t, clientSet, deployment)
+			var meetsSkewExpectations bool
+			var skewVal int
+			t.Logf("Check whether the skew meets the expectation")
+			if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) {
+				listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()}
+				pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions)
+				if err != nil {
+					t.Errorf("Error listing pods for %s: %v", tc.name, err)
+				}
 
-			listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()}
-			pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions)
-			if err != nil {
-				t.Errorf("Error listing pods for %s: %v", name, err)
+				nodePodCountMap := make(map[string]int)
+				for _, pod := range pods.Items {
+					nodePodCountMap[pod.Spec.NodeName]++
+				}
+
+				if len(nodePodCountMap) != len(workerNodes) {
+					t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", tc.name, len(nodePodCountMap))
+					return false, nil
+				}
+
+				skewVal = getSkewValPodDistribution(nodePodCountMap)
+				if skewVal > int(tc.topologySpreadConstraint.MaxSkew) {
+					t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", tc.name, tc.topologySpreadConstraint.MaxSkew, skewVal)
+					return false, nil
+				}
+
+				meetsSkewExpectations = true
+				return true, nil
+			}); err != nil {
+				t.Errorf("Error waiting for descheduler running: %v", err)
 			}
 
-			nodePodCountMap := make(map[string]int)
-			for _, pod := range pods.Items {
-				nodePodCountMap[pod.Spec.NodeName]++
+			if !meetsSkewExpectations {
+				t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", tc.name, tc.topologySpreadConstraint.MaxSkew, skewVal)
+			} else {
+				t.Logf("Pods for %s were distributed in line with max skew of %d", tc.name, tc.topologySpreadConstraint.MaxSkew)
 			}
-
-			if len(nodePodCountMap) != len(workerNodes) {
-				t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", name, len(nodePodCountMap))
-			}
-
-			min, max := getMinAndMaxPodDistribution(nodePodCountMap)
-			if max-min > int(tc.topologySpreadConstraint.MaxSkew) {
-				t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, max-min)
-			}
-
-			t.Logf("Pods for %s were distributed in line with max skew of %d", name, tc.topologySpreadConstraint.MaxSkew)
 		})
 	}
 }
 
-func getMinAndMaxPodDistribution(nodePodCountMap map[string]int) (int, int) {
+func getSkewValPodDistribution(nodePodCountMap map[string]int) int {
 	min := math.MaxInt32
 	max := math.MinInt32
 	for _, podCount := range nodePodCountMap {
@@ -234,7 +346,7 @@ func getMinAndMaxPodDistribution(nodePodCountMap map[string]int) (int, int) {
 		}
 	}
 
-	return min, max
+	return max - min
 }
 
 func nodeInclusionPolicyRef(policy v1.NodeInclusionPolicy) *v1.NodeInclusionPolicy {

test/e2e/policy_leaderelection_a.yaml

@@ -1,15 +0,0 @@
-apiVersion: "descheduler/v1alpha2"
-kind: "DeschedulerPolicy"
-profiles:
-  - name: ProfileName
-    pluginConfig:
-    - name: "PodLifeTime"
-      args:
-        maxPodLifeTimeSeconds: 5
-        namespaces:
-          include:
-          - "e2e-testleaderelection-a"
-    plugins:
-      deschedule:
-        enabled:
-          - "PodLifeTime"

test/e2e/policy_leaderelection_b.yaml

@@ -1,15 +0,0 @@
-apiVersion: "descheduler/v1alpha2"
-kind: "DeschedulerPolicy"
-profiles:
-  - name: ProfileName
-    pluginConfig:
-    - name: "PodLifeTime"
-      args:
-        maxPodLifeTimeSeconds: 5
-        namespaces:
-          include:
-          - "e2e-testleaderelection-b"
-    plugins:
-      deschedule:
-        enabled:
-          - "PodLifeTime"

test/e2e/test_clientconnection_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	componentbaseconfig "k8s.io/component-base/config"
+
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
 	deschedulerapi "sigs.k8s.io/descheduler/pkg/api"
 	"sigs.k8s.io/descheduler/pkg/descheduler"

test/test_utils.go

@@ -31,45 +31,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/metrics/pkg/apis/metrics/v1beta1"
 	utilptr "k8s.io/utils/ptr"
 )
 
-func BuildTestDeployment(name, namespace string, replicas int32, labels map[string]string, apply func(deployment *appsv1.Deployment)) *appsv1.Deployment {
-	// Add "name": name to the labels, overwriting if it exists.
-	labels["name"] = name
-
-	deployment := &appsv1.Deployment{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Deployment",
-			APIVersion: "apps/v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
-		},
-		Spec: appsv1.DeploymentSpec{
-			Replicas: utilptr.To[int32](replicas),
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"name": name,
-				},
-			},
-			Template: v1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: labels,
-				},
-				Spec: MakePodSpec("", utilptr.To[int64](0)),
-			},
-		},
-	}
-
-	if apply != nil {
-		apply(deployment)
-	}
-
-	return deployment
-}
-
 // BuildTestPod creates a test pod with given parameters.
 func BuildTestPod(name string, cpu, memory int64, nodeName string, apply func(*v1.Pod)) *v1.Pod {
 	pod := &v1.Pod{
@@ -103,6 +68,26 @@ func BuildTestPod(name string, cpu, memory int64, nodeName string, apply func(*v
 	return pod
 }
 
+// BuildPodMetrics creates a test podmetrics with given parameters.
+func BuildPodMetrics(name string, millicpu, mem int64) *v1beta1.PodMetrics {
+	return &v1beta1.PodMetrics{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: "default",
+		},
+		Window: metav1.Duration{Duration: 20010000000},
+		Containers: []v1beta1.ContainerMetrics{
+			{
+				Name: "container-1",
+				Usage: v1.ResourceList{
+					v1.ResourceCPU:    *resource.NewMilliQuantity(millicpu, resource.DecimalSI),
+					v1.ResourceMemory: *resource.NewQuantity(mem, resource.BinarySI),
+				},
+			},
+		},
+	}
+}
+
 // GetMirrorPodAnnotation returns the annotation needed for mirror pod.
 func GetMirrorPodAnnotation() map[string]string {
 	return map[string]string{
@@ -171,42 +156,16 @@ func BuildTestNode(name string, millicpu, mem, pods int64, apply func(*v1.Node))
 	return node
 }
 
-func MakePodSpec(priorityClassName string, gracePeriod *int64) v1.PodSpec {
-	return v1.PodSpec{
-		SecurityContext: &v1.PodSecurityContext{
-			RunAsNonRoot: utilptr.To(true),
-			RunAsUser:    utilptr.To[int64](1000),
-			RunAsGroup:   utilptr.To[int64](1000),
-			SeccompProfile: &v1.SeccompProfile{
-				Type: v1.SeccompProfileTypeRuntimeDefault,
-			},
+func BuildNodeMetrics(name string, millicpu, mem int64) *v1beta1.NodeMetrics {
+	return &v1beta1.NodeMetrics{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Window: metav1.Duration{Duration: 20010000000},
+		Usage: v1.ResourceList{
+			v1.ResourceCPU:    *resource.NewMilliQuantity(millicpu, resource.DecimalSI),
+			v1.ResourceMemory: *resource.NewQuantity(mem, resource.BinarySI),
 		},
-		Containers: []v1.Container{{
-			Name:            "pause",
-			ImagePullPolicy: "Never",
-			Image:           "registry.k8s.io/pause",
-			Ports:           []v1.ContainerPort{{ContainerPort: 80}},
-			Resources: v1.ResourceRequirements{
-				Limits: v1.ResourceList{
-					v1.ResourceCPU:    resource.MustParse("100m"),
-					v1.ResourceMemory: resource.MustParse("200Mi"),
-				},
-				Requests: v1.ResourceList{
-					v1.ResourceCPU:    resource.MustParse("100m"),
-					v1.ResourceMemory: resource.MustParse("100Mi"),
-				},
-			},
-			SecurityContext: &v1.SecurityContext{
-				AllowPrivilegeEscalation: utilptr.To(false),
-				Capabilities: &v1.Capabilities{
-					Drop: []v1.Capability{
-						"ALL",
-					},
-				},
-			},
-		}},
-		PriorityClassName:             priorityClassName,
-		TerminationGracePeriodSeconds: gracePeriod,
 	}
 }
 
@@ -316,30 +275,6 @@ func DeleteDeployment(ctx context.Context, t *testing.T, clientSet clientset.Int
 	}
 }
 
-func WaitForDeploymentPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, deployment *appsv1.Deployment) {
-	if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(c context.Context) (bool, error) {
-		podList, err := clientSet.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{
-			LabelSelector: labels.SelectorFromSet(deployment.Spec.Template.ObjectMeta.Labels).String(),
-		})
-		if err != nil {
-			return false, err
-		}
-		if len(podList.Items) != int(*deployment.Spec.Replicas) {
-			t.Logf("Waiting for %v pods to be created, got %v instead", *deployment.Spec.Replicas, len(podList.Items))
-			return false, nil
-		}
-		for _, pod := range podList.Items {
-			if pod.Status.Phase != v1.PodRunning {
-				t.Logf("Pod %v not running yet, is %v instead", pod.Name, pod.Status.Phase)
-				return false, nil
-			}
-		}
-		return true, nil
-	}); err != nil {
-		t.Fatalf("Error waiting for pods running: %v", err)
-	}
-}
-
 func SetPodAntiAffinity(inputPod *v1.Pod, labelKey, labelValue string) {
 	inputPod.Spec.Affinity = &v1.Affinity{
 		PodAntiAffinity: &v1.PodAntiAffinity{

vendor/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake/fake_nodemetrics.go

@@ -0,0 +1,77 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+	v1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"
+)
+
+// FakeNodeMetricses implements NodeMetricsInterface
+type FakeNodeMetricses struct {
+	Fake *FakeMetricsV1beta1
+}
+
+var nodemetricsesResource = v1beta1.SchemeGroupVersion.WithResource("nodemetricses")
+
+var nodemetricsesKind = v1beta1.SchemeGroupVersion.WithKind("NodeMetrics")
+
+// Get takes name of the nodeMetrics, and returns the corresponding nodeMetrics object, and an error if there is any.
+func (c *FakeNodeMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.NodeMetrics, err error) {
+	emptyResult := &v1beta1.NodeMetrics{}
+	obj, err := c.Fake.
+		Invokes(testing.NewRootGetActionWithOptions(nodemetricsesResource, name, options), emptyResult)
+	if obj == nil {
+		return emptyResult, err
+	}
+	return obj.(*v1beta1.NodeMetrics), err
+}
+
+// List takes label and field selectors, and returns the list of NodeMetricses that match those selectors.
+func (c *FakeNodeMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.NodeMetricsList, err error) {
+	emptyResult := &v1beta1.NodeMetricsList{}
+	obj, err := c.Fake.
+		Invokes(testing.NewRootListActionWithOptions(nodemetricsesResource, nodemetricsesKind, opts), emptyResult)
+	if obj == nil {
+		return emptyResult, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1beta1.NodeMetricsList{ListMeta: obj.(*v1beta1.NodeMetricsList).ListMeta}
+	for _, item := range obj.(*v1beta1.NodeMetricsList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested nodeMetricses.
+func (c *FakeNodeMetricses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewRootWatchActionWithOptions(nodemetricsesResource, opts))
+}

vendor/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1/fake/fake_podmetrics.go

@@ -0,0 +1,81 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+	v1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"
+)
+
+// FakePodMetricses implements PodMetricsInterface
+type FakePodMetricses struct {
+	Fake *FakeMetricsV1beta1
+	ns   string
+}
+
+var podmetricsesResource = v1beta1.SchemeGroupVersion.WithResource("podmetricses")
+
+var podmetricsesKind = v1beta1.SchemeGroupVersion.WithKind("PodMetrics")
+
+// Get takes name of the podMetrics, and returns the corresponding podMetrics object, and an error if there is any.
+func (c *FakePodMetricses) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.PodMetrics, err error) {
+	emptyResult := &v1beta1.PodMetrics{}
+	obj, err := c.Fake.
+		Invokes(testing.NewGetActionWithOptions(podmetricsesResource, c.ns, name, options), emptyResult)
+
+	if obj == nil {
+		return emptyResult, err
+	}
+	return obj.(*v1beta1.PodMetrics), err
+}
+
+// List takes label and field selectors, and returns the list of PodMetricses that match those selectors.
+func (c *FakePodMetricses) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.PodMetricsList, err error) {
+	emptyResult := &v1beta1.PodMetricsList{}
+	obj, err := c.Fake.
+		Invokes(testing.NewListActionWithOptions(podmetricsesResource, podmetricsesKind, c.ns, opts), emptyResult)
+
+	if obj == nil {
+		return emptyResult, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1beta1.PodMetricsList{ListMeta: obj.(*v1beta1.PodMetricsList).ListMeta}
+	for _, item := range obj.(*v1beta1.PodMetricsList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested podMetricses.
+func (c *FakePodMetricses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewWatchActionWithOptions(podmetricsesResource, c.ns, opts))
+
+}