From b7b352780e259118bbb5094f4b253fb14e084457 Mon Sep 17 00:00:00 2001 From: Jan Chaloupka Date: Tue, 5 Nov 2024 17:03:12 +0100 Subject: [PATCH] LowNodeUtilization: test metrics based utilization --- .../metricscollector/metricscollector.go | 4 +- pkg/framework/fake/fake.go | 6 + .../nodeutilization/highnodeutilization.go | 4 +- .../nodeutilization/lownodeutilization.go | 14 +- .../lownodeutilization_test.go | 529 +++++++++++++++--- .../nodeutilization/nodeutilization.go | 8 +- .../plugins/nodeutilization/types.go | 14 +- pkg/framework/profile/profile.go | 6 + pkg/framework/types/types.go | 2 + test/test_utils.go | 20 + 10 files changed, 513 insertions(+), 94 deletions(-) diff --git a/pkg/descheduler/metricscollector/metricscollector.go b/pkg/descheduler/metricscollector/metricscollector.go index 07e3f5dea..9b06e7e25 100644 --- a/pkg/descheduler/metricscollector/metricscollector.go +++ b/pkg/descheduler/metricscollector/metricscollector.go @@ -115,8 +115,8 @@ func (mc *MetricsCollector) Collect(ctx context.Context) error { } // Display CPU and memory usage - fmt.Printf("%s: %vm, %vMi\n", node.Name, metrics.Usage.Cpu().MilliValue(), metrics.Usage.Memory().Value()/(1024*1024)) - fmt.Printf("%s: %vm, %vMi\n", node.Name, mc.nodes[node.Name][v1.ResourceCPU].MilliValue(), mc.nodes[node.Name][v1.ResourceMemory].Value()/(1024*1024)) + // fmt.Printf("%s: %vm, %vMi\n", node.Name, metrics.Usage.Cpu().MilliValue(), metrics.Usage.Memory().Value()/(1024*1024)) + // fmt.Printf("%s: %vm, %vMi\n", node.Name, mc.nodes[node.Name][v1.ResourceCPU].MilliValue(), mc.nodes[node.Name][v1.ResourceMemory].Value()/(1024*1024)) } fmt.Printf("--\n") diff --git a/pkg/framework/fake/fake.go b/pkg/framework/fake/fake.go index a16132d40..1a89f4d4a 100644 --- a/pkg/framework/fake/fake.go +++ b/pkg/framework/fake/fake.go @@ -8,6 +8,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + "sigs.k8s.io/descheduler/pkg/descheduler/metricscollector" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) @@ -18,6 +19,7 @@ type HandleImpl struct { SharedInformerFactoryImpl informers.SharedInformerFactory EvictorFilterImpl frameworktypes.EvictorPlugin PodEvictorImpl *evictions.PodEvictor + MetricsCollectorImpl *metricscollector.MetricsCollector } var _ frameworktypes.Handle = &HandleImpl{} @@ -26,6 +28,10 @@ func (hi *HandleImpl) ClientSet() clientset.Interface { return hi.ClientsetImpl } +func (hi *HandleImpl) MetricsCollector() *metricscollector.MetricsCollector { + return hi.MetricsCollectorImpl +} + func (hi *HandleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc { return hi.GetPodsAssignedToNodeFuncImpl } diff --git a/pkg/framework/plugins/nodeutilization/highnodeutilization.go b/pkg/framework/plugins/nodeutilization/highnodeutilization.go index 2a15d9e14..127fe59b1 100644 --- a/pkg/framework/plugins/nodeutilization/highnodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/highnodeutilization.go @@ -160,7 +160,9 @@ func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fr evictions.EvictOptions{StrategyName: HighNodeUtilizationPluginName}, h.podFilter, h.resourceNames, - continueEvictionCond) + continueEvictionCond, + h.usageSnapshot, + ) return nil } diff --git a/pkg/framework/plugins/nodeutilization/lownodeutilization.go b/pkg/framework/plugins/nodeutilization/lownodeutilization.go index 1977fbb34..7640c05f2 100644 --- a/pkg/framework/plugins/nodeutilization/lownodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/lownodeutilization.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" + "sigs.k8s.io/descheduler/pkg/api" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" @@ -88,6 +89,13 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f resourceNames := getResourceNames(lowNodeUtilizationArgsArgs.Thresholds) + var usageSnapshot usageClient + if lowNodeUtilizationArgsArgs.MetricsUtilization.MetricsServer { + usageSnapshot = newActualUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc(), handle.MetricsCollector()) + } else { + usageSnapshot = newRequestedUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc()) + } + return &LowNodeUtilization{ handle: handle, args: lowNodeUtilizationArgsArgs, @@ -95,7 +103,7 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f overutilizationCriteria: overutilizationCriteria, resourceNames: resourceNames, podFilter: podFilter, - usageSnapshot: newRequestedUsageSnapshot(resourceNames, handle.GetPodsAssignedToNodeFunc()), + usageSnapshot: usageSnapshot, }, nil } @@ -182,7 +190,9 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra evictions.EvictOptions{StrategyName: LowNodeUtilizationPluginName}, l.podFilter, l.resourceNames, - continueEvictionCond) + continueEvictionCond, + l.usageSnapshot, + ) return nil } diff --git a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go index a99acc7b6..d2e6c2f60 100644 --- a/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go +++ b/pkg/framework/plugins/nodeutilization/lownodeutilization_test.go @@ -18,8 +18,12 @@ package nodeutilization import ( "context" + "crypto/tls" "fmt" + "net" + "net/http" "testing" + "time" "sigs.k8s.io/descheduler/pkg/api" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" @@ -32,10 +36,18 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" + "k8s.io/metrics/pkg/apis/metrics/v1beta1" + fakemetricsclient "k8s.io/metrics/pkg/client/clientset/versioned/fake" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + "sigs.k8s.io/descheduler/pkg/descheduler/metricscollector" "sigs.k8s.io/descheduler/pkg/utils" "sigs.k8s.io/descheduler/test" + + promapi "github.com/prometheus/client_golang/api" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" ) func TestLowNodeUtilization(t *testing.T) { @@ -48,14 +60,17 @@ func TestLowNodeUtilization(t *testing.T) { notMatchingNodeSelectorValue := "east" testCases := []struct { - name string - useDeviationThresholds bool - thresholds, targetThresholds api.ResourceThresholds - nodes []*v1.Node - pods []*v1.Pod - expectedPodsEvicted uint - evictedPods []string - evictableNamespaces *api.Namespaces + name string + useDeviationThresholds bool + thresholds, targetThresholds api.ResourceThresholds + nodes []*v1.Node + pods []*v1.Pod + nodemetricses []*v1beta1.NodeMetrics + podmetricses []*v1beta1.PodMetrics + expectedPodsEvicted uint + expectedPodsWithMetricsEvicted uint + evictedPods []string + evictableNamespaces *api.Namespaces }{ { name: "no evictable pods", @@ -103,7 +118,20 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 0, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 2401, 1714978816), + test.BuildNodeMetrics(n2NodeName, 401, 1714978816), + test.BuildNodeMetrics(n3NodeName, 10, 1714978816), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 0, + expectedPodsWithMetricsEvicted: 0, }, { name: "without priorities", @@ -153,7 +181,20 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 4, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 4, + expectedPodsWithMetricsEvicted: 4, }, { name: "without priorities, but excluding namespaces", @@ -218,12 +259,25 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, evictableNamespaces: &api.Namespaces{ Exclude: []string{ "namespace1", }, }, - expectedPodsEvicted: 0, + expectedPodsEvicted: 0, + expectedPodsWithMetricsEvicted: 0, }, { name: "without priorities, but include only default namespace", @@ -283,12 +337,25 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, evictableNamespaces: &api.Namespaces{ Include: []string{ "default", }, }, - expectedPodsEvicted: 2, + expectedPodsEvicted: 2, + expectedPodsWithMetricsEvicted: 2, }, { name: "without priorities stop when cpu capacity is depleted", @@ -306,14 +373,14 @@ func TestLowNodeUtilization(t *testing.T) { test.BuildTestNode(n3NodeName, 4000, 3000, 10, test.SetNodeUnschedulable), }, pods: []*v1.Pod{ - test.BuildTestPod("p1", 400, 300, n1NodeName, test.SetRSOwnerRef), - test.BuildTestPod("p2", 400, 300, n1NodeName, test.SetRSOwnerRef), - test.BuildTestPod("p3", 400, 300, n1NodeName, test.SetRSOwnerRef), - test.BuildTestPod("p4", 400, 300, n1NodeName, test.SetRSOwnerRef), - test.BuildTestPod("p5", 400, 300, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef), // These won't be evicted. - test.BuildTestPod("p6", 400, 300, n1NodeName, test.SetDSOwnerRef), - test.BuildTestPod("p7", 400, 300, n1NodeName, func(pod *v1.Pod) { + test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef), + test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) { // A pod with local storage. test.SetNormalOwnerRef(pod) pod.Spec.Volumes = []v1.Volume{ @@ -330,17 +397,29 @@ func TestLowNodeUtilization(t *testing.T) { // A Mirror Pod. pod.Annotations = test.GetMirrorPodAnnotation() }), - test.BuildTestPod("p8", 400, 300, n1NodeName, func(pod *v1.Pod) { + test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) { // A Critical Pod. test.SetNormalOwnerRef(pod) pod.Namespace = "kube-system" priority := utils.SystemCriticalPriority pod.Spec.Priority = &priority }), - test.BuildTestPod("p9", 400, 2100, n2NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - // 4 pods available for eviction based on v1.ResourcePods, only 3 pods can be evicted before cpu is depleted - expectedPodsEvicted: 3, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 0, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 4, + expectedPodsWithMetricsEvicted: 4, }, { name: "with priorities", @@ -410,7 +489,20 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 4, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 4, + expectedPodsWithMetricsEvicted: 4, }, { name: "without priorities evicting best-effort pods only", @@ -478,8 +570,21 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 4, - evictedPods: []string{"p1", "p2", "p4", "p5"}, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 4, + expectedPodsWithMetricsEvicted: 4, + evictedPods: []string{"p1", "p2", "p4", "p5"}, }, { name: "with extended resource", @@ -558,8 +663,21 @@ func TestLowNodeUtilization(t *testing.T) { test.SetPodExtendedResourceRequest(pod, extendedResource, 1) }), }, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, // 4 pods available for eviction based on v1.ResourcePods, only 3 pods can be evicted before extended resource is depleted - expectedPodsEvicted: 3, + expectedPodsEvicted: 3, + expectedPodsWithMetricsEvicted: 0, }, { name: "with extended resource in some of nodes", @@ -586,8 +704,21 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef), }, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, // 0 pods available for eviction because there's no enough extended resource in node2 - expectedPodsEvicted: 0, + expectedPodsEvicted: 0, + expectedPodsWithMetricsEvicted: 0, }, { name: "without priorities, but only other node is unschedulable", @@ -636,7 +767,19 @@ func TestLowNodeUtilization(t *testing.T) { pod.Spec.Priority = &priority }), }, - expectedPodsEvicted: 0, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 0, + expectedPodsWithMetricsEvicted: 0, }, { name: "without priorities, but only other node doesn't match pod node selector for p4 and p5", @@ -701,7 +844,17 @@ func TestLowNodeUtilization(t *testing.T) { pod.Spec.Priority = &priority }), }, - expectedPodsEvicted: 3, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + }, + expectedPodsEvicted: 3, + expectedPodsWithMetricsEvicted: 3, }, { name: "without priorities, but only other node doesn't match pod node affinity for p4 and p5", @@ -795,7 +948,17 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 3, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + }, + expectedPodsEvicted: 3, + expectedPodsWithMetricsEvicted: 3, }, { name: "deviation thresholds", @@ -847,71 +1010,210 @@ func TestLowNodeUtilization(t *testing.T) { }), test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef), }, - expectedPodsEvicted: 2, - evictedPods: []string{}, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + test.BuildNodeMetrics(n3NodeName, 11, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 401, 0), + test.BuildPodMetrics("p2", 401, 0), + test.BuildPodMetrics("p3", 401, 0), + test.BuildPodMetrics("p4", 401, 0), + test.BuildPodMetrics("p5", 401, 0), + }, + expectedPodsEvicted: 2, + expectedPodsWithMetricsEvicted: 2, + evictedPods: []string{}, + }, + { + name: "without priorities different evictions for requested and actual resources", + thresholds: api.ResourceThresholds{ + v1.ResourceCPU: 30, + v1.ResourcePods: 30, + }, + targetThresholds: api.ResourceThresholds{ + v1.ResourceCPU: 50, + v1.ResourcePods: 50, + }, + nodes: []*v1.Node{ + test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil), + test.BuildTestNode(n2NodeName, 4000, 3000, 10, func(node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + nodeSelectorKey: notMatchingNodeSelectorValue, + } + }), + }, + pods: []*v1.Pod{ + test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef), + test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef), + // These won't be evicted. + test.BuildTestPod("p4", 400, 0, n1NodeName, func(pod *v1.Pod) { + // A pod with affinity to run in the "west" datacenter upon scheduling + test.SetNormalOwnerRef(pod) + pod.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: nodeSelectorKey, + Operator: "In", + Values: []string{nodeSelectorValue}, + }, + }, + }, + }, + }, + }, + } + }), + test.BuildTestPod("p5", 400, 0, n1NodeName, func(pod *v1.Pod) { + // A pod with affinity to run in the "west" datacenter upon scheduling + test.SetNormalOwnerRef(pod) + pod.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: nodeSelectorKey, + Operator: "In", + Values: []string{nodeSelectorValue}, + }, + }, + }, + }, + }, + }, + } + }), + test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef), + test.BuildTestPod("p7", 400, 0, n1NodeName, func(pod *v1.Pod) { + // A pod with local storage. + test.SetNormalOwnerRef(pod) + pod.Spec.Volumes = []v1.Volume{ + { + Name: "sample", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{Path: "somePath"}, + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI), + }, + }, + }, + } + // A Mirror Pod. + pod.Annotations = test.GetMirrorPodAnnotation() + }), + test.BuildTestPod("p8", 400, 0, n1NodeName, func(pod *v1.Pod) { + // A Critical Pod. + test.SetNormalOwnerRef(pod) + pod.Namespace = "kube-system" + priority := utils.SystemCriticalPriority + pod.Spec.Priority = &priority + }), + test.BuildTestPod("p9", 0, 0, n2NodeName, test.SetRSOwnerRef), + }, + nodemetricses: []*v1beta1.NodeMetrics{ + test.BuildNodeMetrics(n1NodeName, 3201, 0), + test.BuildNodeMetrics(n2NodeName, 401, 0), + }, + podmetricses: []*v1beta1.PodMetrics{ + test.BuildPodMetrics("p1", 801, 0), + test.BuildPodMetrics("p2", 801, 0), + test.BuildPodMetrics("p3", 801, 0), + }, + expectedPodsEvicted: 3, + expectedPodsWithMetricsEvicted: 2, }, } for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + testFnc := func(metricsEnabled bool, expectedPodsEvicted uint) func(t *testing.T) { + return func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - var objs []runtime.Object - for _, node := range tc.nodes { - objs = append(objs, node) - } - for _, pod := range tc.pods { - objs = append(objs, pod) - } - fakeClient := fake.NewSimpleClientset(objs...) + var objs []runtime.Object + for _, node := range tc.nodes { + objs = append(objs, node) + } + for _, pod := range tc.pods { + objs = append(objs, pod) + } + var metricsObjs []runtime.Object + for _, nodemetrics := range tc.nodemetricses { + metricsObjs = append(metricsObjs, nodemetrics) + } + for _, podmetrics := range tc.podmetricses { + metricsObjs = append(metricsObjs, podmetrics) + } - podsForEviction := make(map[string]struct{}) - for _, pod := range tc.evictedPods { - podsForEviction[pod] = struct{}{} - } + fakeClient := fake.NewSimpleClientset(objs...) + metricsClientset := fakemetricsclient.NewSimpleClientset(metricsObjs...) + collector := metricscollector.NewMetricsCollector(fakeClient, metricsClientset) + err := collector.Collect(ctx) + if err != nil { + t.Fatalf("unable to collect metrics: %v", err) + } - evictionFailed := false - if len(tc.evictedPods) > 0 { - fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { - getAction := action.(core.CreateAction) - obj := getAction.GetObject() - if eviction, ok := obj.(*policy.Eviction); ok { - if _, exists := podsForEviction[eviction.Name]; exists { - return true, obj, nil + podsForEviction := make(map[string]struct{}) + for _, pod := range tc.evictedPods { + podsForEviction[pod] = struct{}{} + } + + evictionFailed := false + if len(tc.evictedPods) > 0 { + fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) { + getAction := action.(core.CreateAction) + obj := getAction.GetObject() + if eviction, ok := obj.(*policy.Eviction); ok { + if _, exists := podsForEviction[eviction.Name]; exists { + return true, obj, nil + } + evictionFailed = true + return true, nil, fmt.Errorf("pod %q was unexpectedly evicted", eviction.Name) } - evictionFailed = true - return true, nil, fmt.Errorf("pod %q was unexpectedly evicted", eviction.Name) - } - return true, obj, nil - }) - } + return true, obj, nil + }) + } - handle, podEvictor, err := frameworktesting.InitFrameworkHandle(ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil) - if err != nil { - t.Fatalf("Unable to initialize a framework handle: %v", err) - } + handle, podEvictor, err := frameworktesting.InitFrameworkHandle(ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil) + if err != nil { + t.Fatalf("Unable to initialize a framework handle: %v", err) + } + handle.MetricsCollectorImpl = collector - plugin, err := NewLowNodeUtilization(&LowNodeUtilizationArgs{ - Thresholds: tc.thresholds, - TargetThresholds: tc.targetThresholds, - UseDeviationThresholds: tc.useDeviationThresholds, - EvictableNamespaces: tc.evictableNamespaces, - }, - handle) - if err != nil { - t.Fatalf("Unable to initialize the plugin: %v", err) - } - plugin.(frameworktypes.BalancePlugin).Balance(ctx, tc.nodes) + plugin, err := NewLowNodeUtilization(&LowNodeUtilizationArgs{ + Thresholds: tc.thresholds, + TargetThresholds: tc.targetThresholds, + UseDeviationThresholds: tc.useDeviationThresholds, + EvictableNamespaces: tc.evictableNamespaces, + MetricsUtilization: MetricsUtilization{ + MetricsServer: metricsEnabled, + }, + }, + handle) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } + plugin.(frameworktypes.BalancePlugin).Balance(ctx, tc.nodes) - podsEvicted := podEvictor.TotalEvicted() - if tc.expectedPodsEvicted != podsEvicted { - t.Errorf("Expected %v pods to be evicted but %v got evicted", tc.expectedPodsEvicted, podsEvicted) + podsEvicted := podEvictor.TotalEvicted() + if expectedPodsEvicted != podsEvicted { + t.Errorf("Expected %v pods to be evicted but %v got evicted", expectedPodsEvicted, podsEvicted) + } + if evictionFailed { + t.Errorf("Pod evictions failed unexpectedly") + } } - if evictionFailed { - t.Errorf("Pod evictions failed unexpectedly") - } - }) + } + t.Run(tc.name, testFnc(false, tc.expectedPodsEvicted)) + t.Run(tc.name+" with metrics enabled", testFnc(true, tc.expectedPodsWithMetricsEvicted)) } } @@ -1067,3 +1369,62 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) { }) } } + +func TestLowNodeUtilizationWithMetrics(t *testing.T) { + return + roundTripper := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + AuthToken := "eyJhbGciOiJSUzI1NiIsImtpZCI6IkNoTW9tT2w2cWtzR2V0dURZdjBqdnBSdmdWM29lWmc3dWpfNW0yaDc2NHMifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTcyODk5MjY3NywiaWF0IjoxNzI4OTg5MDc3LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMiLCJqdGkiOiJkNDY3ZjVmMy0xNGVmLTRkMjItOWJkNC1jMGM1Mzk3NzYyZDgiLCJrdWJlcm5ldGVzLmlvIjp7Im5hbWVzcGFjZSI6Im9wZW5zaGlmdC1tb25pdG9yaW5nIiwic2VydmljZWFjY291bnQiOnsibmFtZSI6InByb21ldGhldXMtazhzIiwidWlkIjoiNjY4NDllMGItYTAwZC00NjUzLWE5NTItNThiYTE1MTk4NTlkIn19LCJuYmYiOjE3Mjg5ODkwNzcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.J1i6-oRAC9J8mqrlZPKGA-CU5PbUzhm2QxAWFnu65-NXR3e252mesybwtjkwxUtTLKrsYHQXwEsG5rGcQsvMcGK9RC9y5z33DFj8tPPwOGLJYJ-s5cTImTqKtWRXzTlcrsrUYTYApfrOsEyXwyfDow4PCslZjR3cd5FMRbvXNqHLg26nG_smApR4wc6kXy7xxlRuGhxu-dUiscQP56njboOK61JdTG8F3FgOayZnKk1jGeVdIhXClqGWJyokk-ZM3mMK1MxzGXY0tLbe37V4B7g3NDiH651BUcicfDSky46yfcAYxMDbZgpK2TByWApAllN0wixz2WsFfyBVu_Q5xtZ9Gi9BUHSa5ioRiBK346W4Bdmr9ala5ldIXDa59YE7UB34DsCHyqvzRx_Sj76hLzy2jSOk7RsL0fM8sDoJL4ROdi-3Jtr5uPY593I8H8qeQvFS6PQfm0bUZqVKrrLoCK_uk9guH4a6K27SlD-Utk3dpsjbmrwcjBxm-zd_LE9YyQ734My00Pcy9D5eNio3gESjGsHqGFc_haq4ZCiVOCkbdmABjpPEL6K7bs1GMZbHt1CONL0-LzymM8vgGNj0grjpG8-5AF8ZuSqR7pbZSV_NO2nKkmrwpILCw0Joqp6V3C9pP9nXWHIDyVMxMK870zxzt_qCoPRLCAujQQn6e0U" + client, err := promapi.NewClient(promapi.Config{ + Address: "https://prometheus-k8s-openshift-monitoring.apps.jchaloup-20241015-3.group-b.devcluster.openshift.com", + RoundTripper: config.NewAuthorizationCredentialsRoundTripper("Bearer", config.NewInlineSecret(AuthToken), roundTripper), + }) + if err != nil { + t.Fatalf("prom client error: %v", err) + } + + // pod:container_cpu_usage:sum + // container_memory_usage_bytes + + v1api := promv1.NewAPI(client) + ctx := context.TODO() + // promQuery := "avg_over_time(kube_pod_container_resource_requests[1m])" + promQuery := "kube_pod_container_resource_requests" + results, warnings, err := v1api.Query(ctx, promQuery, time.Now()) + fmt.Printf("results: %#v\n", results) + for _, sample := range results.(model.Vector) { + fmt.Printf("sample: %#v\n", sample) + } + fmt.Printf("warnings: %v\n", warnings) + fmt.Printf("err: %v\n", err) + + result := model.Value( + &model.Vector{ + &model.Sample{ + Metric: model.Metric{ + "container": "kube-controller-manager", + "endpoint": "https-main", + "job": "kube-state-metrics", + "namespace": "openshift-kube-controller-manager", + "node": "ip-10-0-72-168.us-east-2.compute.internal", + "pod": "kube-controller-manager-ip-10-0-72-168.us-east-2.compute.internal", + "resource": "cpu", + "service": "kube-state-metrics", + "uid": "ae46c09f-ade7-4133-9ee8-cf45ac78ca6d", + "unit": "core", + }, + Value: 0.06, + Timestamp: 1728991761711, + }, + }, + ) + fmt.Printf("result: %#v\n", result) +} diff --git a/pkg/framework/plugins/nodeutilization/nodeutilization.go b/pkg/framework/plugins/nodeutilization/nodeutilization.go index b3a3afcde..d650dd12a 100644 --- a/pkg/framework/plugins/nodeutilization/nodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/nodeutilization.go @@ -18,7 +18,6 @@ package nodeutilization import ( "context" - "fmt" "math" "sort" @@ -238,6 +237,10 @@ func evictPodsFromSourceNodes( totalAvailableUsage[name] = resource.NewQuantity(0, resource.DecimalSI) } totalAvailableUsage[name].Add(*node.thresholds.highResourceThreshold[name]) + if _, exists := node.usage[name]; !exists { + klog.Errorf("unable to find %q resource in node's %q usage, terminating eviction", name, node.node.Name) + return + } totalAvailableUsage[name].Sub(*node.usage[name]) } } @@ -299,7 +302,6 @@ func evictPods( if continueEviction(nodeInfo, totalAvailableUsage) { for _, pod := range inputPods { - fmt.Printf("pods: %v\n", pod.Name) if !utils.PodToleratesTaints(pod, taintsOfLowNodes) { klog.V(3).InfoS("Skipping eviction for pod, doesn't tolerate node taint", "pod", klog.KObj(pod)) continue @@ -319,7 +321,7 @@ func evictPods( } podUsage, err := usageSnapshot.podUsage(pod) if err != nil { - klog.ErrorS(err, "unable to get pod usage for %v/%v: %v", pod.Namespace, pod.Name, err) + klog.Errorf("unable to get pod usage for %v/%v: %v", pod.Namespace, pod.Name, err) continue } err = podEvictor.Evict(ctx, pod, evictOptions) diff --git a/pkg/framework/plugins/nodeutilization/types.go b/pkg/framework/plugins/nodeutilization/types.go index 4823b0695..f4a5924b5 100644 --- a/pkg/framework/plugins/nodeutilization/types.go +++ b/pkg/framework/plugins/nodeutilization/types.go @@ -28,6 +28,7 @@ type LowNodeUtilizationArgs struct { Thresholds api.ResourceThresholds `json:"thresholds"` TargetThresholds api.ResourceThresholds `json:"targetThresholds"` NumberOfNodes int `json:"numberOfNodes,omitempty"` + MetricsUtilization MetricsUtilization `json:metricsUtilization,omitempty` // Naming this one differently since namespaces are still // considered while considering resources used by pods @@ -41,10 +42,19 @@ type LowNodeUtilizationArgs struct { type HighNodeUtilizationArgs struct { metav1.TypeMeta `json:",inline"` - Thresholds api.ResourceThresholds `json:"thresholds"` - NumberOfNodes int `json:"numberOfNodes,omitempty"` + Thresholds api.ResourceThresholds `json:"thresholds"` + NumberOfNodes int `json:"numberOfNodes,omitempty"` + MetricsUtilization MetricsUtilization `json:metricsUtilization,omitempty` + // Naming this one differently since namespaces are still // considered while considering resources used by pods // but then filtered out before eviction EvictableNamespaces *api.Namespaces `json:"evictableNamespaces,omitempty"` } + +// MetricsUtilization allow to consume actual resource utilization from metrics +type MetricsUtilization struct { + // metricsServer enables metrics from a kubernetes metrics server. + // Please see https://kubernetes-sigs.github.io/metrics-server/ for more. + MetricsServer bool `json:"metricsServer,omitempty"` +} diff --git a/pkg/framework/profile/profile.go b/pkg/framework/profile/profile.go index 5cac3ea27..044de1198 100644 --- a/pkg/framework/profile/profile.go +++ b/pkg/framework/profile/profile.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + "sigs.k8s.io/descheduler/pkg/descheduler/metricscollector" "k8s.io/klog/v2" ) @@ -67,6 +68,7 @@ func (ei *evictorImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.Ev // handleImpl implements the framework handle which gets passed to plugins type handleImpl struct { clientSet clientset.Interface + metricsCollector *metricscollector.MetricsCollector getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc sharedInformerFactory informers.SharedInformerFactory evictor *evictorImpl @@ -79,6 +81,10 @@ func (hi *handleImpl) ClientSet() clientset.Interface { return hi.clientSet } +func (hi *handleImpl) MetricsCollector() *metricscollector.MetricsCollector { + return hi.metricsCollector +} + // GetPodsAssignedToNodeFunc retrieves GetPodsAssignedToNodeFunc implementation func (hi *handleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc { return hi.getPodsAssignedToNodeFunc diff --git a/pkg/framework/types/types.go b/pkg/framework/types/types.go index 1d5e1d95c..98950a99d 100644 --- a/pkg/framework/types/types.go +++ b/pkg/framework/types/types.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" + "sigs.k8s.io/descheduler/pkg/descheduler/metricscollector" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" @@ -36,6 +37,7 @@ type Handle interface { Evictor() Evictor GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc SharedInformerFactory() informers.SharedInformerFactory + MetricsCollector() *metricscollector.MetricsCollector } // Evictor defines an interface for filtering and evicting pods diff --git a/test/test_utils.go b/test/test_utils.go index 04b0ab694..cf0e7f6ab 100644 --- a/test/test_utils.go +++ b/test/test_utils.go @@ -68,6 +68,26 @@ func BuildTestPod(name string, cpu, memory int64, nodeName string, apply func(*v return pod } +// BuildPodMetrics creates a test podmetrics with given parameters. +func BuildPodMetrics(name string, millicpu, mem int64) *v1beta1.PodMetrics { + return &v1beta1.PodMetrics{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Window: metav1.Duration{Duration: 20010000000}, + Containers: []v1beta1.ContainerMetrics{ + { + Name: "container-1", + Usage: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(millicpu, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(mem, resource.BinarySI), + }, + }, + }, + } +} + // GetMirrorPodAnnotation returns the annotation needed for mirror pod. func GetMirrorPodAnnotation() map[string]string { return map[string]string{