descheduler/test/e2e/e2e_test.go

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"fmt"
	"math"
	"os"
	"sort"
	"strings"
	"testing"
	"time"

	v1 "k8s.io/api/core/v1"
	schedulingv1 "k8s.io/api/scheduling/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/informers"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	v1qos "k8s.io/kubectl/pkg/util/qos"

	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
	"sigs.k8s.io/descheduler/pkg/api"
	deschedulerapi "sigs.k8s.io/descheduler/pkg/api"
	"sigs.k8s.io/descheduler/pkg/descheduler"
	"sigs.k8s.io/descheduler/pkg/descheduler/client"
	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
	eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
	"sigs.k8s.io/descheduler/pkg/descheduler/strategies"
	"sigs.k8s.io/descheduler/pkg/descheduler/strategies/nodeutilization"
	"sigs.k8s.io/descheduler/pkg/utils"
)

func MakePodSpec(priorityClassName string, gracePeriod *int64) v1.PodSpec {
	return v1.PodSpec{
		Containers: []v1.Container{{
			Name:            "pause",
			ImagePullPolicy: "Never",
			Image:           "kubernetes/pause",
			Ports:           []v1.ContainerPort{{ContainerPort: 80}},
			Resources: v1.ResourceRequirements{
				Limits: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("100m"),
					v1.ResourceMemory: resource.MustParse("200Mi"),
				},
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("100m"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
		}},
		PriorityClassName:             priorityClassName,
		TerminationGracePeriodSeconds: gracePeriod,
	}
}

// RcByNameContainer returns a ReplicationController with specified name and container
func RcByNameContainer(name, namespace string, replicas int32, labels map[string]string, gracePeriod *int64, priorityClassName string) *v1.ReplicationController {
	zeroGracePeriod := int64(0)

	// Add "name": name to the labels, overwriting if it exists.
	labels["name"] = name
	if gracePeriod == nil {
		gracePeriod = &zeroGracePeriod
	}
	return &v1.ReplicationController{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ReplicationController",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
		},
		Spec: v1.ReplicationControllerSpec{
			Replicas: func(i int32) *int32 { return &i }(replicas),
			Selector: map[string]string{
				"name": name,
			},
			Template: &v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: labels,
				},
				Spec: MakePodSpec(priorityClassName, gracePeriod),
			},
		},
	}
}

func initializeClient(t *testing.T) (clientset.Interface, coreinformers.NodeInformer, podutil.GetPodsAssignedToNodeFunc, chan struct{}) {
	clientSet, err := client.CreateClient(os.Getenv("KUBECONFIG"))
	if err != nil {
		t.Errorf("Error during client creation with %v", err)
	}

	stopChannel := make(chan struct{})

	sharedInformerFactory := informers.NewSharedInformerFactory(clientSet, 0)
	nodeInformer := sharedInformerFactory.Core().V1().Nodes()
	podInformer := sharedInformerFactory.Core().V1().Pods()

	getPodsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer)
	if err != nil {
		t.Errorf("build get pods assigned to node function error: %v", err)
	}

	sharedInformerFactory.Start(stopChannel)
	sharedInformerFactory.WaitForCacheSync(stopChannel)

	waitForNodesReady(context.Background(), t, clientSet, nodeInformer)
	return clientSet, nodeInformer, getPodsAssignedToNode, stopChannel
}

func waitForNodesReady(ctx context.Context, t *testing.T, clientSet clientset.Interface, nodeInformer coreinformers.NodeInformer) {
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
		if err != nil {
			return false, err
		}

		readyNodes, err := nodeutil.ReadyNodes(ctx, clientSet, nodeInformer, "")
		if err != nil {
			return false, err
		}
		if len(nodeList.Items) != len(readyNodes) {
			t.Logf("%v/%v nodes are ready. Waiting for all nodes to be ready...", len(readyNodes), len(nodeList.Items))
			return false, nil
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for nodes to be ready: %v", err)
	}
}

func runPodLifetimeStrategy(
	ctx context.Context,
	t *testing.T,
	clientset clientset.Interface,
	nodeInformer coreinformers.NodeInformer,
	namespaces *deschedulerapi.Namespaces,
	priorityClass string,
	priority *int32,
	evictCritical bool,
	labelSelector *metav1.LabelSelector,
	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
) {
	// Run descheduler.
	evictionPolicyGroupVersion, err := eutils.SupportEviction(clientset)
	if err != nil || len(evictionPolicyGroupVersion) == 0 {
		t.Fatalf("%v", err)
	}

	nodes, err := nodeutil.ReadyNodes(ctx, clientset, nodeInformer, "")
	if err != nil {
		t.Fatalf("%v", err)
	}

	maxPodLifeTimeSeconds := uint(1)
	strategies.PodLifeTime(
		ctx,
		clientset,
		deschedulerapi.DeschedulerStrategy{
			Enabled: true,
			Params: &deschedulerapi.StrategyParameters{
				PodLifeTime:                &deschedulerapi.PodLifeTime{MaxPodLifeTimeSeconds: &maxPodLifeTimeSeconds},
				Namespaces:                 namespaces,
				ThresholdPriority:          priority,
				ThresholdPriorityClassName: priorityClass,
				LabelSelector:              labelSelector,
			},
		},
		nodes,
		evictions.NewPodEvictor(
			clientset,
			evictionPolicyGroupVersion,
			false,
			nil,
			nil,
			nodes,
			false,
			evictCritical,
			false,
			false,
			false,
		),
		getPodsAssignedToNode,
	)
}

func getPodNames(pods []v1.Pod) []string {
	names := []string{}
	for _, pod := range pods {
		names = append(names, pod.Name)
	}
	return names
}

func intersectStrings(lista, listb []string) []string {
	commonNames := []string{}

	for _, stra := range lista {
		for _, strb := range listb {
			if stra == strb {
				commonNames = append(commonNames, stra)
				break
			}
		}
	}

	return commonNames
}

func TestLowNodeUtilization(t *testing.T) {
	ctx := context.Background()

	clientSet, _, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		t.Errorf("Error listing node with %v", err)
	}

	nodes, workerNodes := splitNodesAndWorkerNodes(nodeList.Items)

	t.Log("Creating testing namespace")
	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	// Make all worker nodes resource balanced
	cleanUp, err := createBalancedPodForNodes(t, ctx, clientSet, testNamespace.Name, workerNodes, 0.5)
	if err != nil {
		t.Fatalf("Unable to create load balancing pods: %v", err)
	}
	defer cleanUp()

	t.Log("Creating pods each consuming 10% of node's allocatable")
	nodeCpu := workerNodes[0].Status.Allocatable[v1.ResourceCPU]
	tenthOfCpu := int64(float64((&nodeCpu).MilliValue()) * 0.1)

	t.Log("Creating pods all bound to a single node")
	for i := 0; i < 4; i++ {
		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      fmt.Sprintf("lnu-pod-%v", i),
				Namespace: testNamespace.Name,
				Labels:    map[string]string{"test": "node-utilization", "name": "test-rc-node-utilization"},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{{
					Name:            "pause",
					ImagePullPolicy: "Never",
					Image:           "kubernetes/pause",
					Ports:           []v1.ContainerPort{{ContainerPort: 80}},
					Resources: v1.ResourceRequirements{
						Limits: v1.ResourceList{
							v1.ResourceCPU: *resource.NewMilliQuantity(tenthOfCpu, resource.DecimalSI),
						},
						Requests: v1.ResourceList{
							v1.ResourceCPU: *resource.NewMilliQuantity(tenthOfCpu, resource.DecimalSI),
						},
					},
				}},
				Affinity: &v1.Affinity{
					NodeAffinity: &v1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
							NodeSelectorTerms: []v1.NodeSelectorTerm{
								{
									MatchFields: []v1.NodeSelectorRequirement{
										{Key: "metadata.name", Operator: v1.NodeSelectorOpIn, Values: []string{workerNodes[0].Name}},
									},
								},
							},
						},
					},
				},
			},
		}

		t.Logf("Creating pod %v in %v namespace for node %v", pod.Name, pod.Namespace, workerNodes[0].Name)
		_, err := clientSet.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
		if err != nil {
			t.Logf("Error creating LNU pods: %v", err)
			if err = clientSet.CoreV1().Pods(pod.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
				LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "node-utilization", "name": "test-rc-node-utilization"})).String(),
			}); err != nil {
				t.Fatalf("Unable to delete LNU pods: %v", err)
			}
			return
		}
	}

	t.Log("Creating RC with 4 replicas owning the created pods")
	rc := RcByNameContainer("test-rc-node-utilization", testNamespace.Name, int32(4), map[string]string{"test": "node-utilization"}, nil, "")
	if _, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Create(ctx, rc, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Error creating RC %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rc)
	waitForRCPodsRunning(ctx, t, clientSet, rc)

	// Run LowNodeUtilization strategy
	podEvictor := initPodEvictorOrFail(t, clientSet, nodes)

	podFilter, err := podutil.NewOptions().WithFilter(podEvictor.Evictable().IsEvictable).BuildFilterFunc()
	if err != nil {
		t.Errorf("Error initializing pod filter function, %v", err)
	}

	podsOnMosttUtilizedNode, err := podutil.ListPodsOnANode(workerNodes[0].Name, getPodsAssignedToNode, podFilter)
	if err != nil {
		t.Errorf("Error listing pods on a node %v", err)
	}
	podsBefore := len(podsOnMosttUtilizedNode)

	t.Log("Running LowNodeUtilization strategy")
	nodeutilization.LowNodeUtilization(
		ctx,
		clientSet,
		deschedulerapi.DeschedulerStrategy{
			Enabled: true,
			Params: &deschedulerapi.StrategyParameters{
				NodeResourceUtilizationThresholds: &deschedulerapi.NodeResourceUtilizationThresholds{
					Thresholds: deschedulerapi.ResourceThresholds{
						v1.ResourceCPU: 70,
					},
					TargetThresholds: deschedulerapi.ResourceThresholds{
						v1.ResourceCPU: 80,
					},
				},
			},
		},
		workerNodes,
		podEvictor,
		getPodsAssignedToNode,
	)

	waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace)

	podFilter, err = podutil.NewOptions().WithFilter(podEvictor.Evictable().IsEvictable).BuildFilterFunc()
	if err != nil {
		t.Errorf("Error initializing pod filter function, %v", err)
	}

	podsOnMosttUtilizedNode, err = podutil.ListPodsOnANode(workerNodes[0].Name, getPodsAssignedToNode, podFilter)
	if err != nil {
		t.Errorf("Error listing pods on a node %v", err)
	}
	podsAfter := len(podsOnMosttUtilizedNode)

	if podsAfter >= podsBefore {
		t.Fatalf("No pod has been evicted from %v node", workerNodes[0].Name)
	}
	t.Logf("Number of pods on node %v changed from %v to %v", workerNodes[0].Name, podsBefore, podsAfter)
}

// TODO(jchaloup): add testcases for two included/excluded namespaces

func TestNamespaceConstraintsInclude(t *testing.T) {
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	rc := RcByNameContainer("test-rc-podlifetime", testNamespace.Name, 5, map[string]string{"test": "podlifetime-include"}, nil, "")
	if _, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Create(ctx, rc, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating deployment %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rc)

	// wait for a while so all the pods are at least few seconds older
	time.Sleep(5 * time.Second)

	// it's assumed all new pods are named differently from currently running -> no name collision
	podList, err := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if len(podList.Items) != 5 {
		t.Fatalf("Expected 5 replicas, got %v instead", len(podList.Items))
	}

	initialPodNames := getPodNames(podList.Items)
	sort.Strings(initialPodNames)
	t.Logf("Existing pods: %v", initialPodNames)

	t.Logf("set the strategy to delete pods from %v namespace", rc.Namespace)
	runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, &deschedulerapi.Namespaces{
		Include: []string{rc.Namespace},
	}, "", nil, false, nil, getPodsAssignedToNode)

	// All pods are supposed to be deleted, wait until all the old pods are deleted
	if err := wait.PollImmediate(time.Second, 20*time.Second, func() (bool, error) {
		podList, err := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
		if err != nil {
			return false, nil
		}

		includePodNames := getPodNames(podList.Items)
		// validate all pod were deleted
		if len(intersectStrings(initialPodNames, includePodNames)) > 0 {
			t.Logf("Waiting until %v pods get deleted", intersectStrings(initialPodNames, includePodNames))
			// check if there's at least one pod not in Terminating state
			for _, pod := range podList.Items {
				// In case podList contains newly created pods
				if len(intersectStrings(initialPodNames, []string{pod.Name})) == 0 {
					continue
				}
				if pod.DeletionTimestamp == nil {
					t.Logf("Pod %v not in terminating state", pod.Name)
					return false, nil
				}
			}
			t.Logf("All %v pods are terminating", intersectStrings(initialPodNames, includePodNames))
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods to be deleted: %v", err)
	}
}

func TestNamespaceConstraintsExclude(t *testing.T) {
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	rc := RcByNameContainer("test-rc-podlifetime", testNamespace.Name, 5, map[string]string{"test": "podlifetime-exclude"}, nil, "")
	if _, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Create(ctx, rc, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating deployment %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rc)

	// wait for a while so all the pods are at least few seconds older
	time.Sleep(5 * time.Second)

	// it's assumed all new pods are named differently from currently running -> no name collision
	podList, err := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if len(podList.Items) != 5 {
		t.Fatalf("Expected 5 replicas, got %v instead", len(podList.Items))
	}

	initialPodNames := getPodNames(podList.Items)
	sort.Strings(initialPodNames)
	t.Logf("Existing pods: %v", initialPodNames)

	t.Logf("set the strategy to delete pods from namespaces except the %v namespace", rc.Namespace)
	runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, &deschedulerapi.Namespaces{
		Exclude: []string{rc.Namespace},
	}, "", nil, false, nil, getPodsAssignedToNode)

	t.Logf("Waiting 10s")
	time.Sleep(10 * time.Second)
	podList, err = clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods after running strategy: %v", err)
	}

	excludePodNames := getPodNames(podList.Items)
	sort.Strings(excludePodNames)
	t.Logf("Existing pods: %v", excludePodNames)

	// validate no pods were deleted
	if len(intersectStrings(initialPodNames, excludePodNames)) != 5 {
		t.Fatalf("None of %v pods are expected to be deleted", initialPodNames)
	}
}

func TestEvictSystemCriticalPriority(t *testing.T) {
	testEvictSystemCritical(t, false)
}

func TestEvictSystemCriticalPriorityClass(t *testing.T) {
	testEvictSystemCritical(t, true)
}

func testEvictSystemCritical(t *testing.T, isPriorityClass bool) {
	var highPriority = int32(1000)
	var lowPriority = int32(500)
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	// create two priority classes
	highPriorityClass := &schedulingv1.PriorityClass{
		ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name()) + "-highpriority"},
		Value:      highPriority,
	}
	if _, err := clientSet.SchedulingV1().PriorityClasses().Create(ctx, highPriorityClass, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Error creating priorityclass %s: %v", highPriorityClass.Name, err)
	}
	defer clientSet.SchedulingV1().PriorityClasses().Delete(ctx, highPriorityClass.Name, metav1.DeleteOptions{})

	lowPriorityClass := &schedulingv1.PriorityClass{
		ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name()) + "-lowpriority"},
		Value:      lowPriority,
	}
	if _, err := clientSet.SchedulingV1().PriorityClasses().Create(ctx, lowPriorityClass, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Error creating priorityclass %s: %v", lowPriorityClass.Name, err)
	}
	defer clientSet.SchedulingV1().PriorityClasses().Delete(ctx, lowPriorityClass.Name, metav1.DeleteOptions{})

	// Create a replication controller with the "system-node-critical" priority class (this gives the pods a priority of 2000001000)
	rcCriticalPriority := RcByNameContainer("test-rc-podlifetime-criticalpriority", testNamespace.Name, 3,
		map[string]string{"test": "podlifetime-criticalpriority"}, nil, "system-node-critical")
	if _, err := clientSet.CoreV1().ReplicationControllers(rcCriticalPriority.Namespace).Create(ctx, rcCriticalPriority, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %s: %v", rcCriticalPriority.Name, err)
	}
	defer deleteRC(ctx, t, clientSet, rcCriticalPriority)

	// create two RCs with different priority classes in the same namespace
	rcHighPriority := RcByNameContainer("test-rc-podlifetime-highpriority", testNamespace.Name, 3,
		map[string]string{"test": "podlifetime-highpriority"}, nil, highPriorityClass.Name)
	if _, err := clientSet.CoreV1().ReplicationControllers(rcHighPriority.Namespace).Create(ctx, rcHighPriority, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %s: %v", rcHighPriority.Name, err)
	}
	defer deleteRC(ctx, t, clientSet, rcHighPriority)

	rcLowPriority := RcByNameContainer("test-rc-podlifetime-lowpriority", testNamespace.Name, 3,
		map[string]string{"test": "podlifetime-lowpriority"}, nil, lowPriorityClass.Name)
	if _, err := clientSet.CoreV1().ReplicationControllers(rcLowPriority.Namespace).Create(ctx, rcLowPriority, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %s: %v", rcLowPriority.Name, err)
	}
	defer deleteRC(ctx, t, clientSet, rcLowPriority)

	// wait for a while so all the pods are at least few seconds older
	time.Sleep(5 * time.Second)

	// it's assumed all new pods are named differently from currently running -> no name collision
	podListCriticalPriority, err := clientSet.CoreV1().Pods(rcCriticalPriority.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcCriticalPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	podListHighPriority, err := clientSet.CoreV1().Pods(rcHighPriority.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcHighPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}
	podListLowPriority, err := clientSet.CoreV1().Pods(rcLowPriority.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcLowPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if (len(podListHighPriority.Items) + len(podListLowPriority.Items) + len(podListCriticalPriority.Items)) != 9 {
		t.Fatalf("Expected 9 replicas, got %v instead", len(podListHighPriority.Items)+len(podListLowPriority.Items)+len(podListCriticalPriority.Items))
	}

	initialPodNames := append(getPodNames(podListHighPriority.Items), getPodNames(podListLowPriority.Items)...)
	initialPodNames = append(initialPodNames, getPodNames(podListCriticalPriority.Items)...)
	sort.Strings(initialPodNames)
	t.Logf("Existing pods: %v", initialPodNames)

	if isPriorityClass {
		runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, highPriorityClass.Name, nil, true, nil, getPodsAssignedToNode)
	} else {
		runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, "", &highPriority, true, nil, getPodsAssignedToNode)
	}

	// All pods are supposed to be deleted, wait until all pods in the test namespace are terminating
	t.Logf("All pods in the test namespace, no matter their priority (including system-node-critical and system-cluster-critical), will be deleted")
	if err := wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) {
		podList, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, metav1.ListOptions{})
		if err != nil {
			return false, nil
		}
		currentPodNames := getPodNames(podList.Items)
		// validate all pod were deleted
		if len(intersectStrings(initialPodNames, currentPodNames)) > 0 {
			t.Logf("Waiting until %v pods get deleted", intersectStrings(initialPodNames, currentPodNames))
			// check if there's at least one pod not in Terminating state
			for _, pod := range podList.Items {
				// In case podList contains newly created pods
				if len(intersectStrings(initialPodNames, []string{pod.Name})) == 0 {
					continue
				}
				if pod.DeletionTimestamp == nil {
					t.Logf("Pod %v not in terminating state", pod.Name)
					return false, nil
				}
			}
			t.Logf("All %v pods are terminating", intersectStrings(initialPodNames, currentPodNames))
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods to be deleted: %v", err)
	}
}

func TestThresholdPriority(t *testing.T) {
	testPriority(t, false)
}

func TestThresholdPriorityClass(t *testing.T) {
	testPriority(t, true)
}

func testPriority(t *testing.T, isPriorityClass bool) {
	var highPriority = int32(1000)
	var lowPriority = int32(500)
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	// create two priority classes
	highPriorityClass := &schedulingv1.PriorityClass{
		ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name()) + "-highpriority"},
		Value:      highPriority,
	}
	if _, err := clientSet.SchedulingV1().PriorityClasses().Create(ctx, highPriorityClass, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Error creating priorityclass %s: %v", highPriorityClass.Name, err)
	}
	defer clientSet.SchedulingV1().PriorityClasses().Delete(ctx, highPriorityClass.Name, metav1.DeleteOptions{})

	lowPriorityClass := &schedulingv1.PriorityClass{
		ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name()) + "-lowpriority"},
		Value:      lowPriority,
	}
	if _, err := clientSet.SchedulingV1().PriorityClasses().Create(ctx, lowPriorityClass, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Error creating priorityclass %s: %v", lowPriorityClass.Name, err)
	}
	defer clientSet.SchedulingV1().PriorityClasses().Delete(ctx, lowPriorityClass.Name, metav1.DeleteOptions{})

	// create two RCs with different priority classes in the same namespace
	rcHighPriority := RcByNameContainer("test-rc-podlifetime-highpriority", testNamespace.Name, 5,
		map[string]string{"test": "podlifetime-highpriority"}, nil, highPriorityClass.Name)
	if _, err := clientSet.CoreV1().ReplicationControllers(rcHighPriority.Namespace).Create(ctx, rcHighPriority, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %s: %v", rcHighPriority.Name, err)
	}
	defer deleteRC(ctx, t, clientSet, rcHighPriority)

	rcLowPriority := RcByNameContainer("test-rc-podlifetime-lowpriority", testNamespace.Name, 5,
		map[string]string{"test": "podlifetime-lowpriority"}, nil, lowPriorityClass.Name)
	if _, err := clientSet.CoreV1().ReplicationControllers(rcLowPriority.Namespace).Create(ctx, rcLowPriority, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %s: %v", rcLowPriority.Name, err)
	}
	defer deleteRC(ctx, t, clientSet, rcLowPriority)

	// wait for a while so all the pods are at least few seconds older
	time.Sleep(5 * time.Second)

	// it's assumed all new pods are named differently from currently running -> no name collision
	podListHighPriority, err := clientSet.CoreV1().Pods(rcHighPriority.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcHighPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}
	podListLowPriority, err := clientSet.CoreV1().Pods(rcLowPriority.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcLowPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if len(podListHighPriority.Items)+len(podListLowPriority.Items) != 10 {
		t.Fatalf("Expected 10 replicas, got %v instead", len(podListHighPriority.Items)+len(podListLowPriority.Items))
	}

	expectReservePodNames := getPodNames(podListHighPriority.Items)
	expectEvictPodNames := getPodNames(podListLowPriority.Items)
	sort.Strings(expectReservePodNames)
	sort.Strings(expectEvictPodNames)
	t.Logf("Pods not expected to be evicted: %v, pods expected to be evicted: %v", expectReservePodNames, expectEvictPodNames)

	if isPriorityClass {
		t.Logf("set the strategy to delete pods with priority lower than priority class %s", highPriorityClass.Name)
		runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, highPriorityClass.Name, nil, false, nil, getPodsAssignedToNode)
	} else {
		t.Logf("set the strategy to delete pods with priority lower than %d", highPriority)
		runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, "", &highPriority, false, nil, getPodsAssignedToNode)
	}

	t.Logf("Waiting 10s")
	time.Sleep(10 * time.Second)
	// check if all pods with high priority class are not evicted
	podListHighPriority, err = clientSet.CoreV1().Pods(rcHighPriority.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcHighPriority.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods after running strategy: %v", err)
	}

	excludePodNames := getPodNames(podListHighPriority.Items)
	sort.Strings(excludePodNames)
	t.Logf("Existing high priority pods: %v", excludePodNames)

	// validate no pods were deleted
	if len(intersectStrings(expectReservePodNames, excludePodNames)) != 5 {
		t.Fatalf("None of %v high priority pods are expected to be deleted", expectReservePodNames)
	}

	//check if all pods with low priority class are evicted
	if err := wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) {
		podListLowPriority, err := clientSet.CoreV1().Pods(rcLowPriority.Namespace).List(
			ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcLowPriority.Spec.Template.Labels).String()})
		if err != nil {
			return false, nil
		}

		includePodNames := getPodNames(podListLowPriority.Items)
		// validate all pod were deleted
		if len(intersectStrings(expectEvictPodNames, includePodNames)) > 0 {
			t.Logf("Waiting until %v low priority pods get deleted", intersectStrings(expectEvictPodNames, includePodNames))
			// check if there's at least one pod not in Terminating state
			for _, pod := range podListLowPriority.Items {
				// In case podList contains newly created pods
				if len(intersectStrings(expectEvictPodNames, []string{pod.Name})) == 0 {
					continue
				}
				if pod.DeletionTimestamp == nil {
					t.Logf("Pod %v not in terminating state", pod.Name)
					return false, nil
				}
			}
			t.Logf("All %v pods are terminating", intersectStrings(expectEvictPodNames, includePodNames))
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods to be deleted: %v", err)
	}
}

func TestPodLabelSelector(t *testing.T) {
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	// create two replicationControllers with different labels
	rcEvict := RcByNameContainer("test-rc-podlifetime-evict", testNamespace.Name, 5, map[string]string{"test": "podlifetime-evict"}, nil, "")
	if _, err := clientSet.CoreV1().ReplicationControllers(rcEvict.Namespace).Create(ctx, rcEvict, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rcEvict)

	rcReserve := RcByNameContainer("test-rc-podlifetime-reserve", testNamespace.Name, 5, map[string]string{"test": "podlifetime-reserve"}, nil, "")
	if _, err := clientSet.CoreV1().ReplicationControllers(rcReserve.Namespace).Create(ctx, rcReserve, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating rc %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rcReserve)

	// wait for a while so all the pods are at least few seconds older
	time.Sleep(5 * time.Second)

	// it's assumed all new pods are named differently from currently running -> no name collision
	podListEvict, err := clientSet.CoreV1().Pods(rcEvict.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcEvict.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}
	podListReserve, err := clientSet.CoreV1().Pods(rcReserve.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcReserve.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if len(podListEvict.Items)+len(podListReserve.Items) != 10 {
		t.Fatalf("Expected 10 replicas, got %v instead", len(podListEvict.Items)+len(podListReserve.Items))
	}

	expectReservePodNames := getPodNames(podListReserve.Items)
	expectEvictPodNames := getPodNames(podListEvict.Items)
	sort.Strings(expectReservePodNames)
	sort.Strings(expectEvictPodNames)
	t.Logf("Pods not expected to be evicted: %v, pods expected to be evicted: %v", expectReservePodNames, expectEvictPodNames)

	t.Logf("set the strategy to delete pods with label test:podlifetime-evict")
	runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, "", nil, false, &metav1.LabelSelector{MatchLabels: map[string]string{"test": "podlifetime-evict"}}, getPodsAssignedToNode)

	t.Logf("Waiting 10s")
	time.Sleep(10 * time.Second)
	// check if all pods without target label are not evicted
	podListReserve, err = clientSet.CoreV1().Pods(rcReserve.Namespace).List(
		ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcReserve.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods after running strategy: %v", err)
	}

	reservedPodNames := getPodNames(podListReserve.Items)
	sort.Strings(reservedPodNames)
	t.Logf("Existing reserved pods: %v", reservedPodNames)

	// validate no pods were deleted
	if len(intersectStrings(expectReservePodNames, reservedPodNames)) != 5 {
		t.Fatalf("None of %v unevictable pods are expected to be deleted", expectReservePodNames)
	}

	//check if all selected pods are evicted
	if err := wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) {
		podListEvict, err := clientSet.CoreV1().Pods(rcEvict.Namespace).List(
			ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rcEvict.Spec.Template.Labels).String()})
		if err != nil {
			return false, nil
		}

		newPodNames := getPodNames(podListEvict.Items)
		// validate all pod were deleted
		if len(intersectStrings(expectEvictPodNames, newPodNames)) > 0 {
			t.Logf("Waiting until %v selected pods get deleted", intersectStrings(expectEvictPodNames, newPodNames))
			// check if there's at least one pod not in Terminating state
			for _, pod := range podListEvict.Items {
				// In case podList contains newly created pods
				if len(intersectStrings(expectEvictPodNames, []string{pod.Name})) == 0 {
					continue
				}
				if pod.DeletionTimestamp == nil {
					t.Logf("Pod %v not in terminating state", pod.Name)
					return false, nil
				}
			}
			t.Logf("All %v pods are terminating", intersectStrings(expectEvictPodNames, newPodNames))
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods to be deleted: %v", err)
	}
}

func TestEvictAnnotation(t *testing.T) {
	ctx := context.Background()

	clientSet, nodeInformer, getPodsAssignedToNode, stopCh := initializeClient(t)
	defer close(stopCh)

	nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		t.Errorf("Error listing node with %v", err)
	}

	var nodes []*v1.Node
	for i := range nodeList.Items {
		node := nodeList.Items[i]
		nodes = append(nodes, &node)
	}

	testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}}
	if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil {
		t.Fatalf("Unable to create ns %v", testNamespace.Name)
	}
	defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})

	t.Log("Create RC with pods with local storage which require descheduler.alpha.kubernetes.io/evict annotation to be set for eviction")
	rc := RcByNameContainer("test-rc-evict-annotation", testNamespace.Name, int32(5), map[string]string{"test": "annotation"}, nil, "")
	rc.Spec.Template.Annotations = map[string]string{"descheduler.alpha.kubernetes.io/evict": "true"}
	rc.Spec.Template.Spec.Volumes = []v1.Volume{
		{
			Name: "sample",
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{
					SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI)},
			},
		},
	}

	if _, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Create(ctx, rc, metav1.CreateOptions{}); err != nil {
		t.Errorf("Error creating deployment %v", err)
	}
	defer deleteRC(ctx, t, clientSet, rc)

	t.Logf("Waiting 10s to make pods 10s old")
	time.Sleep(10 * time.Second)

	podList, err := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
	if err != nil {
		t.Fatalf("Unable to list pods: %v", err)
	}

	if len(podList.Items) != 5 {
		t.Fatalf("Expected 5 replicas, got %v instead", len(podList.Items))
	}

	initialPodNames := getPodNames(podList.Items)
	sort.Strings(initialPodNames)
	t.Logf("Existing pods: %v", initialPodNames)

	t.Log("Running PodLifetime strategy")
	runPodLifetimeStrategy(ctx, t, clientSet, nodeInformer, nil, "", nil, false, nil, getPodsAssignedToNode)

	if err := wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
		podList, err = clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
		if err != nil {
			return false, fmt.Errorf("Unable to list pods after running strategy: %v", err)
		}

		excludePodNames := getPodNames(podList.Items)
		sort.Strings(excludePodNames)
		t.Logf("Existing pods: %v", excludePodNames)

		// validate no pods were deleted
		if len(intersectStrings(initialPodNames, excludePodNames)) > 0 {
			t.Logf("Not every pods was evicted")
			return false, nil
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods to be deleted: %v", err)
	}
}

func TestDeschedulingInterval(t *testing.T) {
	ctx := context.Background()
	clientSet, err := client.CreateClient(os.Getenv("KUBECONFIG"))
	if err != nil {
		t.Errorf("Error during client creation with %v", err)
	}

	// By default, the DeschedulingInterval param should be set to 0, meaning Descheduler only runs once then exits
	s, err := options.NewDeschedulerServer()
	if err != nil {
		t.Fatalf("Unable to initialize server: %v", err)
	}
	s.Client = clientSet

	deschedulerPolicy := &api.DeschedulerPolicy{}

	c := make(chan bool, 1)
	go func() {
		evictionPolicyGroupVersion, err := eutils.SupportEviction(s.Client)
		if err != nil || len(evictionPolicyGroupVersion) == 0 {
			t.Errorf("Error when checking support for eviction: %v", err)
		}
		if err := descheduler.RunDeschedulerStrategies(ctx, s, deschedulerPolicy, evictionPolicyGroupVersion); err != nil {
			t.Errorf("Error running descheduler strategies: %+v", err)
		}
		c <- true
	}()

	select {
	case <-c:
		// successfully returned
	case <-time.After(3 * time.Minute):
		t.Errorf("descheduler.Run timed out even without descheduling-interval set")
	}
}

func waitForRCPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, rc *v1.ReplicationController) {
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		podList, err := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{
			LabelSelector: labels.SelectorFromSet(labels.Set(rc.Spec.Template.ObjectMeta.Labels)).String(),
		})
		if err != nil {
			return false, err
		}
		if len(podList.Items) != int(*rc.Spec.Replicas) {
			t.Logf("Waiting for %v pods to be created, got %v instead", *rc.Spec.Replicas, len(podList.Items))
			return false, nil
		}
		for _, pod := range podList.Items {
			if pod.Status.Phase != v1.PodRunning {
				t.Logf("Pod %v not running yet, is %v instead", pod.Name, pod.Status.Phase)
				return false, nil
			}
		}
		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pods running: %v", err)
	}
}

func waitForTerminatingPodsToDisappear(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) {
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
		if err != nil {
			return false, err
		}
		for _, pod := range podList.Items {
			if pod.DeletionTimestamp != nil {
				t.Logf("Pod %v still terminating", pod.Name)
				return false, nil
			}
		}
		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for terminating pods to disappear: %v", err)
	}
}

func deleteRC(ctx context.Context, t *testing.T, clientSet clientset.Interface, rc *v1.ReplicationController) {
	// set number of replicas to 0
	rcdeepcopy := rc.DeepCopy()
	rcdeepcopy.Spec.Replicas = func(i int32) *int32 { return &i }(0)
	if _, err := clientSet.CoreV1().ReplicationControllers(rcdeepcopy.Namespace).Update(ctx, rcdeepcopy, metav1.UpdateOptions{}); err != nil {
		t.Fatalf("Error updating replica controller %v", err)
	}

	// wait 30 seconds until all pods are deleted
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		scale, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).GetScale(ctx, rc.Name, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		return scale.Spec.Replicas == 0, nil
	}); err != nil {
		t.Fatalf("Error deleting rc pods %v", err)
	}

	if err := wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
		podList, _ := clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()})
		t.Logf("Waiting for %v RC pods to disappear, still %v remaining", rc.Name, len(podList.Items))
		if len(podList.Items) > 0 {
			return false, nil
		}
		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for rc pods to disappear: %v", err)
	}

	if err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Delete(ctx, rc.Name, metav1.DeleteOptions{}); err != nil {
		t.Fatalf("Error deleting rc %v", err)
	}

	if err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
		_, err := clientSet.CoreV1().ReplicationControllers(rc.Namespace).Get(ctx, rc.Name, metav1.GetOptions{})
		if err != nil && strings.Contains(err.Error(), "not found") {
			return true, nil
		}
		return false, nil
	}); err != nil {
		t.Fatalf("Error deleting rc %v", err)
	}
}

var balancePodLabel = map[string]string{"podname": "priority-balanced-memory"}

// track min memory limit based on crio minimum. pods cannot set a limit lower than this
// see: https://github.com/cri-o/cri-o/blob/29805b13e9a43d9d22628553db337ce1c1bec0a8/internal/config/cgmgr/cgmgr.go#L23
// see: https://bugzilla.redhat.com/show_bug.cgi?id=1595256
var crioMinMemLimit = 12 * 1024 * 1024

var podRequestedResource = &v1.ResourceRequirements{
	Limits: v1.ResourceList{
		v1.ResourceMemory: resource.MustParse("100Mi"),
		v1.ResourceCPU:    resource.MustParse("100m"),
	},
	Requests: v1.ResourceList{
		v1.ResourceMemory: resource.MustParse("100Mi"),
		v1.ResourceCPU:    resource.MustParse("100m"),
	},
}

// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio.
// TODO(jchaloup): The function is updated version of what under https://github.com/kubernetes/kubernetes/blob/84483a5/test/e2e/scheduling/priorities.go#L478.
//                 Import it once the function is moved under k8s.io/components-helper repo and modified to work for both priority and predicates cases.
func createBalancedPodForNodes(
	t *testing.T,
	ctx context.Context,
	cs clientset.Interface,
	ns string,
	nodes []*v1.Node,
	ratio float64,
) (func(), error) {
	cleanUp := func() {
		// Delete all remaining pods
		err := cs.CoreV1().Pods(ns).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
		})
		if err != nil {
			t.Logf("Failed to delete memory balanced pods: %v.", err)
		} else {
			err := wait.PollImmediate(2*time.Second, time.Minute, func() (bool, error) {
				podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
					LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
				})
				if err != nil {
					t.Logf("Failed to list memory balanced pods: %v.", err)
					return false, nil
				}
				if len(podList.Items) > 0 {
					return false, nil
				}
				return true, nil
			})
			if err != nil {
				t.Logf("Failed to wait until all memory balanced pods are deleted: %v.", err)
			}
		}
	}

	// find the max, if the node has the max,use the one, if not,use the ratio parameter
	var maxCPUFraction, maxMemFraction float64 = ratio, ratio
	var cpuFractionMap = make(map[string]float64)
	var memFractionMap = make(map[string]float64)

	for _, node := range nodes {
		cpuFraction, memFraction, _, _ := computeCPUMemFraction(t, cs, node, podRequestedResource)
		cpuFractionMap[node.Name] = cpuFraction
		memFractionMap[node.Name] = memFraction
		if cpuFraction > maxCPUFraction {
			maxCPUFraction = cpuFraction
		}
		if memFraction > maxMemFraction {
			maxMemFraction = memFraction
		}
	}

	// we need the max one to keep the same cpu/mem use rate
	ratio = math.Max(maxCPUFraction, maxMemFraction)
	for _, node := range nodes {
		memAllocatable, found := node.Status.Allocatable[v1.ResourceMemory]
		if !found {
			t.Fatalf("Failed to get node's Allocatable[v1.ResourceMemory]")
		}
		memAllocatableVal := memAllocatable.Value()

		cpuAllocatable, found := node.Status.Allocatable[v1.ResourceCPU]
		if !found {
			t.Fatalf("Failed to get node's Allocatable[v1.ResourceCPU]")
		}
		cpuAllocatableMil := cpuAllocatable.MilliValue()

		needCreateResource := v1.ResourceList{}
		cpuFraction := cpuFractionMap[node.Name]
		memFraction := memFractionMap[node.Name]
		needCreateResource[v1.ResourceCPU] = *resource.NewMilliQuantity(int64((ratio-cpuFraction)*float64(cpuAllocatableMil)), resource.DecimalSI)

		// add crioMinMemLimit to ensure that all pods are setting at least that much for a limit, while keeping the same ratios
		needCreateResource[v1.ResourceMemory] = *resource.NewQuantity(int64((ratio-memFraction)*float64(memAllocatableVal)+float64(crioMinMemLimit)), resource.BinarySI)

		var gracePeriod = int64(1)
		// Don't set OwnerReferences to avoid pod eviction
		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "filler-pod-" + string(uuid.NewUUID()),
				Namespace: ns,
				Labels:    balancePodLabel,
			},
			Spec: v1.PodSpec{
				Affinity: &v1.Affinity{
					NodeAffinity: &v1.NodeAffinity{
						RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
							NodeSelectorTerms: []v1.NodeSelectorTerm{
								{
									MatchFields: []v1.NodeSelectorRequirement{
										{Key: "metadata.name", Operator: v1.NodeSelectorOpIn, Values: []string{node.Name}},
									},
								},
							},
						},
					},
				},
				Containers: []v1.Container{
					{
						Name:  "pause",
						Image: "kubernetes/pause",
						Resources: v1.ResourceRequirements{
							Limits:   needCreateResource,
							Requests: needCreateResource,
						},
					},
				},
				// PriorityClassName:             conf.PriorityClassName,
				TerminationGracePeriodSeconds: &gracePeriod,
			},
		}

		t.Logf("Creating pod %v in %v namespace for node %v", pod.Name, pod.Namespace, node.Name)
		_, err := cs.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
		if err != nil {
			t.Logf("Error creating filler pod: %v", err)
			return cleanUp, err
		}

		waitForPodRunning(ctx, t, cs, pod)
	}

	for _, node := range nodes {
		t.Log("Compute Cpu, Mem Fraction after create balanced pods.")
		computeCPUMemFraction(t, cs, node, podRequestedResource)
	}

	return cleanUp, nil
}

func computeCPUMemFraction(t *testing.T, cs clientset.Interface, node *v1.Node, resourceReq *v1.ResourceRequirements) (float64, float64, int64, int64) {
	t.Logf("ComputeCPUMemFraction for node: %v", node.Name)
	totalRequestedCPUResource := resourceReq.Requests.Cpu().MilliValue()
	totalRequestedMemResource := resourceReq.Requests.Memory().Value()
	allpods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		t.Fatalf("Expect error of invalid, got : %v", err)
	}
	for _, pod := range allpods.Items {
		if pod.Spec.NodeName == node.Name {
			req, _ := utils.PodRequestsAndLimits(&pod)
			if _, ok := req[v1.ResourceCPU]; !ok {
				req[v1.ResourceCPU] = *resource.NewMilliQuantity(0, resource.DecimalSI)
			}
			if _, ok := req[v1.ResourceMemory]; !ok {
				req[v1.ResourceMemory] = *resource.NewQuantity(0, resource.BinarySI)
			}

			cpuRequest := req[v1.ResourceCPU]
			memoryRequest := req[v1.ResourceMemory]

			t.Logf("Pod for on the node: %v, Cpu: %v, Mem: %v", pod.Name, (&cpuRequest).MilliValue(), (&memoryRequest).Value())
			// Ignore best effort pods while computing fractions as they won't be taken in account by scheduler.
			if v1qos.GetPodQOS(&pod) == v1.PodQOSBestEffort {
				continue
			}
			totalRequestedCPUResource += (&cpuRequest).MilliValue()
			totalRequestedMemResource += (&memoryRequest).Value()
		}
	}
	cpuAllocatable, found := node.Status.Allocatable[v1.ResourceCPU]
	if !found {
		t.Fatalf("Failed to get node's Allocatable[v1.ResourceCPU]")
	}
	cpuAllocatableMil := cpuAllocatable.MilliValue()

	floatOne := float64(1)
	cpuFraction := float64(totalRequestedCPUResource) / float64(cpuAllocatableMil)
	if cpuFraction > floatOne {
		cpuFraction = floatOne
	}
	memAllocatable, found := node.Status.Allocatable[v1.ResourceMemory]
	if !found {
		t.Fatalf("Failed to get node's Allocatable[v1.ResourceMemory]")
	}
	memAllocatableVal := memAllocatable.Value()
	memFraction := float64(totalRequestedMemResource) / float64(memAllocatableVal)
	if memFraction > floatOne {
		memFraction = floatOne
	}

	t.Logf("Node: %v, totalRequestedCPUResource: %v, cpuAllocatableMil: %v, cpuFraction: %v", node.Name, totalRequestedCPUResource, cpuAllocatableMil, cpuFraction)
	t.Logf("Node: %v, totalRequestedMemResource: %v, memAllocatableVal: %v, memFraction: %v", node.Name, totalRequestedMemResource, memAllocatableVal, memFraction)

	return cpuFraction, memFraction, cpuAllocatableMil, memAllocatableVal
}

func waitForPodRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, pod *v1.Pod) {
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		podItem, err := clientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
		if err != nil {
			return false, err
		}

		if podItem.Status.Phase != v1.PodRunning {
			t.Logf("Pod %v not running yet, is %v instead", podItem.Name, podItem.Status.Phase)
			return false, nil
		}

		return true, nil
	}); err != nil {
		t.Fatalf("Error waiting for pod running: %v", err)
	}
}

func splitNodesAndWorkerNodes(nodes []v1.Node) ([]*v1.Node, []*v1.Node) {
	var allNodes []*v1.Node
	var workerNodes []*v1.Node
	for i := range nodes {
		node := nodes[i]
		allNodes = append(allNodes, &node)
		if _, exists := node.Labels["node-role.kubernetes.io/master"]; !exists {
			workerNodes = append(workerNodes, &node)
		}
	}
	return allNodes, workerNodes
}

func initPodEvictorOrFail(t *testing.T, clientSet clientset.Interface, nodes []*v1.Node) *evictions.PodEvictor {
	evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet)
	if err != nil || len(evictionPolicyGroupVersion) == 0 {
		t.Fatalf("Error creating eviction policy group: %v", err)
	}
	return evictions.NewPodEvictor(
		clientSet,
		evictionPolicyGroupVersion,
		false,
		nil,
		nil,
		nodes,
		true,
		false,
		false,
		false,
		false,
	)
}