From 95a631f6a50f06a3486b20332368db6720167dd4 Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Thu, 20 Mar 2025 11:02:47 +0100 Subject: [PATCH] feat: move classifier to its own package move the classifier to its own package. introduces a generic way of classifying usages against thresholds. --- .../nodeutilization/classifier/classifier.go | 84 ++ .../classifier/classifier_test.go | 739 ++++++++++++++++++ .../nodeutilization/highnodeutilization.go | 33 +- .../nodeutilization/lownodeutilization.go | 35 +- .../nodeutilization/nodeutilization.go | 61 +- .../nodeutilization/nodeutilization_test.go | 363 +++++++++ 6 files changed, 1241 insertions(+), 74 deletions(-) create mode 100644 pkg/framework/plugins/nodeutilization/classifier/classifier.go create mode 100644 pkg/framework/plugins/nodeutilization/classifier/classifier_test.go diff --git a/pkg/framework/plugins/nodeutilization/classifier/classifier.go b/pkg/framework/plugins/nodeutilization/classifier/classifier.go new file mode 100644 index 000000000..65c69420f --- /dev/null +++ b/pkg/framework/plugins/nodeutilization/classifier/classifier.go @@ -0,0 +1,84 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package classifier + +// Classifier is a function that classifies a resource usage based on a limit. +// The function should return true if the resource usage matches the classifier +// intent. +type Classifier[K comparable, V any] func(K, V, V) bool + +// Comparer is a function that compares two objects. This function should return +// -1 if the first object is less than the second, 0 if they are equal, and 1 if +// the first object is greater than the second. Of course this is a simplification +// and any value between -1 and 1 can be returned. +type Comparer[V any] func(V, V) int + +// Values is a map of values indexed by a comparable key. An example of this +// can be a list of resources indexed by a node name. +type Values[K comparable, V any] map[K]V + +// Limits is a map of list of limits indexed by a comparable key. Each limit +// inside the list requires a classifier to evaluate. +type Limits[K comparable, V any] map[K][]V + +// Classify is a function that classifies based on classifier functions. This +// function receives Values, a list of n Limits (indexed by name), and a list +// of n Classifiers. The classifier at n position is called to evaluate the +// limit at n position. The first classifier to return true will receive the +// value, at this point the loop will break and the next value will be +// evaluated. This function returns a slice of maps, each position in the +// returned slice correspond to one of the classifiers (e.g. if n limits +// and classifiers are provided, the returned slice will have n maps). +func Classify[K comparable, V any]( + values Values[K, V], limits Limits[K, V], classifiers ...Classifier[K, V], +) []map[K]V { + result := make([]map[K]V, len(classifiers)) + for i := range classifiers { + result[i] = make(map[K]V) + } + + for index, usage := range values { + for i, limit := range limits[index] { + if len(classifiers) <= i { + continue + } + if !classifiers[i](index, usage, limit) { + continue + } + result[i][index] = usage + break + } + } + + return result +} + +// ForMap is a function that returns a classifier that compares all values in a +// map. The function receives a Comparer function that is used to compare all +// the map values. The returned Classifier will return true only if the +// provided Comparer function returns a value less than 0 for all the values. +func ForMap[K, I comparable, V any, M ~map[I]V](cmp Comparer[V]) Classifier[K, M] { + return func(_ K, usages, limits M) bool { + for idx, usage := range usages { + if limit, ok := limits[idx]; ok { + if cmp(usage, limit) >= 0 { + return false + } + } + } + return true + } +} diff --git a/pkg/framework/plugins/nodeutilization/classifier/classifier_test.go b/pkg/framework/plugins/nodeutilization/classifier/classifier_test.go new file mode 100644 index 000000000..030bfd260 --- /dev/null +++ b/pkg/framework/plugins/nodeutilization/classifier/classifier_test.go @@ -0,0 +1,739 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package classifier + +import ( + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestClassifySimple(t *testing.T) { + for _, tt := range []struct { + name string + usage map[string]int + limits map[string][]int + classifiers []Classifier[string, int] + expected []map[string]int + }{ + { + name: "empty", + usage: map[string]int{}, + limits: map[string][]int{}, + expected: []map[string]int{}, + }, + { + name: "one under one over", + usage: map[string]int{ + "node1": 2, + "node2": 8, + }, + limits: map[string][]int{ + "node1": {4, 6}, + "node2": {4, 6}, + }, + expected: []map[string]int{ + {"node1": 2}, + {"node2": 8}, + }, + classifiers: []Classifier[string, int]{ + func(_ string, usage, limit int) bool { + return usage < limit + }, + func(_ string, usage, limit int) bool { + return usage > limit + }, + }, + }, + { + name: "randomly positioned over utilized", + usage: map[string]int{ + "node1": 2, + "node2": 8, + "node3": 2, + "node4": 8, + "node5": 8, + "node6": 2, + "node7": 2, + "node8": 8, + "node9": 8, + }, + limits: map[string][]int{ + "node1": {4, 6}, + "node2": {4, 6}, + "node3": {4, 6}, + "node4": {4, 6}, + "node5": {4, 6}, + "node6": {4, 6}, + "node7": {4, 6}, + "node8": {4, 6}, + "node9": {4, 6}, + }, + expected: []map[string]int{ + { + "node1": 2, + "node3": 2, + "node6": 2, + "node7": 2, + }, + { + "node2": 8, + "node4": 8, + "node5": 8, + "node8": 8, + "node9": 8, + }, + }, + classifiers: []Classifier[string, int]{ + func(_ string, usage, limit int) bool { + return usage < limit + }, + func(_ string, usage, limit int) bool { + return usage > limit + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + result := Classify(tt.usage, tt.limits, tt.classifiers...) + if !reflect.DeepEqual(result, tt.expected) { + t.Fatalf("unexpected result: %v", result) + } + }) + } +} + +func TestClassify_pointers(t *testing.T) { + for _, tt := range []struct { + name string + usage map[string]map[v1.ResourceName]*resource.Quantity + limits map[string][]map[v1.ResourceName]*resource.Quantity + classifiers []Classifier[string, map[v1.ResourceName]*resource.Quantity] + expected []map[string]map[v1.ResourceName]*resource.Quantity + }{ + { + name: "empty", + usage: map[string]map[v1.ResourceName]*resource.Quantity{}, + limits: map[string][]map[v1.ResourceName]*resource.Quantity{}, + expected: []map[string]map[v1.ResourceName]*resource.Quantity{}, + }, + { + name: "single underutilized", + usage: map[string]map[v1.ResourceName]*resource.Quantity{ + "node1": { + v1.ResourceCPU: ptr.To(resource.MustParse("2")), + v1.ResourceMemory: ptr.To(resource.MustParse("2Gi")), + }, + }, + limits: map[string][]map[v1.ResourceName]*resource.Quantity{ + "node1": { + { + v1.ResourceCPU: ptr.To(resource.MustParse("4")), + v1.ResourceMemory: ptr.To(resource.MustParse("4Gi")), + }, + }, + }, + expected: []map[string]map[v1.ResourceName]*resource.Quantity{ + { + "node1": { + v1.ResourceCPU: ptr.To(resource.MustParse("2")), + v1.ResourceMemory: ptr.To(resource.MustParse("2Gi")), + }, + }, + }, + classifiers: []Classifier[string, map[v1.ResourceName]*resource.Quantity]{ + ForMap[string, v1.ResourceName, *resource.Quantity, map[v1.ResourceName]*resource.Quantity]( + func(usage, limit *resource.Quantity) int { + return usage.Cmp(*limit) + }, + ), + }, + }, + { + name: "single underutilized and properly utilized", + usage: map[string]map[v1.ResourceName]*resource.Quantity{ + "node1": { + v1.ResourceCPU: ptr.To(resource.MustParse("2")), + v1.ResourceMemory: ptr.To(resource.MustParse("2Gi")), + }, + "node2": { + v1.ResourceCPU: ptr.To(resource.MustParse("5")), + v1.ResourceMemory: ptr.To(resource.MustParse("5Gi")), + }, + "node3": { + v1.ResourceCPU: ptr.To(resource.MustParse("8")), + v1.ResourceMemory: ptr.To(resource.MustParse("8Gi")), + }, + }, + limits: map[string][]map[v1.ResourceName]*resource.Quantity{ + "node1": { + { + v1.ResourceCPU: ptr.To(resource.MustParse("4")), + v1.ResourceMemory: ptr.To(resource.MustParse("4Gi")), + }, + { + v1.ResourceCPU: ptr.To(resource.MustParse("16")), + v1.ResourceMemory: ptr.To(resource.MustParse("16Gi")), + }, + }, + "node2": { + { + v1.ResourceCPU: ptr.To(resource.MustParse("4")), + v1.ResourceMemory: ptr.To(resource.MustParse("4Gi")), + }, + { + v1.ResourceCPU: ptr.To(resource.MustParse("16")), + v1.ResourceMemory: ptr.To(resource.MustParse("16Gi")), + }, + }, + "node3": { + { + v1.ResourceCPU: ptr.To(resource.MustParse("4")), + v1.ResourceMemory: ptr.To(resource.MustParse("4Gi")), + }, + { + v1.ResourceCPU: ptr.To(resource.MustParse("16")), + v1.ResourceMemory: ptr.To(resource.MustParse("16Gi")), + }, + }, + }, + expected: []map[string]map[v1.ResourceName]*resource.Quantity{ + { + "node1": { + v1.ResourceCPU: ptr.To(resource.MustParse("2")), + v1.ResourceMemory: ptr.To(resource.MustParse("2Gi")), + }, + }, + {}, + }, + classifiers: []Classifier[string, map[v1.ResourceName]*resource.Quantity]{ + ForMap[string, v1.ResourceName, *resource.Quantity, map[v1.ResourceName]*resource.Quantity]( + func(usage, limit *resource.Quantity) int { + return usage.Cmp(*limit) + }, + ), + ForMap[string, v1.ResourceName, *resource.Quantity, map[v1.ResourceName]*resource.Quantity]( + func(usage, limit *resource.Quantity) int { + return limit.Cmp(*usage) + }, + ), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + result := Classify(tt.usage, tt.limits, tt.classifiers...) + if !reflect.DeepEqual(result, tt.expected) { + t.Fatalf("unexpected result: %v", result) + } + }) + } +} + +func TestClassify(t *testing.T) { + for _, tt := range []struct { + name string + usage map[string]v1.ResourceList + limits map[string][]v1.ResourceList + classifiers []Classifier[string, v1.ResourceList] + expected []map[string]v1.ResourceList + }{ + { + name: "empty", + usage: map[string]v1.ResourceList{}, + limits: map[string][]v1.ResourceList{}, + expected: []map[string]v1.ResourceList{}, + }, + { + name: "single underutilized", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + }, + }, + { + name: "less classifiers than limits", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + "node2": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + "node3": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + }, + }, + { + name: "more classifiers than limits", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("20"), + v1.ResourceMemory: resource.MustParse("20"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("50"), + v1.ResourceMemory: resource.MustParse("50"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("80"), + v1.ResourceMemory: resource.MustParse("80"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("30"), + v1.ResourceMemory: resource.MustParse("30"), + }, + }, + "node2": { + { + v1.ResourceCPU: resource.MustParse("30"), + v1.ResourceMemory: resource.MustParse("30"), + }, + }, + "node3": { + { + v1.ResourceCPU: resource.MustParse("30"), + v1.ResourceMemory: resource.MustParse("30"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node1": { + v1.ResourceCPU: resource.MustParse("20"), + v1.ResourceMemory: resource.MustParse("20"), + }, + }, + {}, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + { + name: "single underutilized and properly utilized", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("5Gi"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + "node2": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + "node3": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("16"), + v1.ResourceMemory: resource.MustParse("16Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + {}, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + { + name: "single underutilized and multiple over utilized nodes", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + "node2": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + "node3": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node1": { + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + { + "node2": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("8"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + { + name: "over and under at the same time", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + "node2": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + {}, + {}, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + { + name: "only memory over utilized", + usage: map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("8Gi"), + }, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + { + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + { + v1.ResourceCPU: resource.MustParse("6"), + v1.ResourceMemory: resource.MustParse("6Gi"), + }, + }, + }, + expected: []map[string]v1.ResourceList{ + {}, + {}, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + { + name: "randomly positioned over utilized", + usage: map[string]v1.ResourceList{ + "node1": {v1.ResourceCPU: resource.MustParse("8")}, + "node2": {v1.ResourceCPU: resource.MustParse("2")}, + "node3": {v1.ResourceCPU: resource.MustParse("8")}, + "node4": {v1.ResourceCPU: resource.MustParse("2")}, + "node5": {v1.ResourceCPU: resource.MustParse("8")}, + "node6": {v1.ResourceCPU: resource.MustParse("8")}, + "node7": {v1.ResourceCPU: resource.MustParse("8")}, + "node8": {v1.ResourceCPU: resource.MustParse("2")}, + "node9": {v1.ResourceCPU: resource.MustParse("5")}, + }, + limits: map[string][]v1.ResourceList{ + "node1": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node2": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node3": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node4": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node5": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node6": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node7": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node8": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + "node9": { + {v1.ResourceCPU: resource.MustParse("4")}, + {v1.ResourceCPU: resource.MustParse("6")}, + }, + }, + expected: []map[string]v1.ResourceList{ + { + "node2": {v1.ResourceCPU: resource.MustParse("2")}, + "node4": {v1.ResourceCPU: resource.MustParse("2")}, + "node8": {v1.ResourceCPU: resource.MustParse("2")}, + }, + { + "node1": {v1.ResourceCPU: resource.MustParse("8")}, + "node3": {v1.ResourceCPU: resource.MustParse("8")}, + "node5": {v1.ResourceCPU: resource.MustParse("8")}, + "node6": {v1.ResourceCPU: resource.MustParse("8")}, + "node7": {v1.ResourceCPU: resource.MustParse("8")}, + }, + }, + classifiers: []Classifier[string, v1.ResourceList]{ + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return usage.Cmp(limit) + }, + ), + ForMap[string, v1.ResourceName, resource.Quantity, v1.ResourceList]( + func(usage, limit resource.Quantity) int { + return limit.Cmp(usage) + }, + ), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + result := Classify(tt.usage, tt.limits, tt.classifiers...) + if !reflect.DeepEqual(result, tt.expected) { + t.Fatalf("unexpected result: %v", result) + } + }) + } +} diff --git a/pkg/framework/plugins/nodeutilization/highnodeutilization.go b/pkg/framework/plugins/nodeutilization/highnodeutilization.go index f030a15ed..992f8516a 100644 --- a/pkg/framework/plugins/nodeutilization/highnodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/highnodeutilization.go @@ -28,6 +28,7 @@ import ( nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/classifier" "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/normalizer" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) @@ -144,24 +145,22 @@ func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fr // classify nodes in two groups: underutilized and schedulable. we will // later try to move pods from the first group to the second. - nodeGroups := classifyNodeUsage( + nodeGroups := classifier.Classify( usage, thresholds, - []classifierFnc{ - // underutilized nodes. - func(nodeName string, usage, threshold api.ResourceThresholds) bool { - return isNodeBelowThreshold(usage, threshold) - }, - // schedulable nodes. - func(nodeName string, usage, threshold api.ResourceThresholds) bool { - if nodeutil.IsNodeUnschedulable(nodesMap[nodeName]) { - klog.V(2).InfoS( - "Node is unschedulable", - "node", klog.KObj(nodesMap[nodeName]), - ) - return false - } - return true - }, + // underutilized nodes. + func(nodeName string, usage, threshold api.ResourceThresholds) bool { + return isNodeBelowThreshold(usage, threshold) + }, + // schedulable nodes. + func(nodeName string, usage, threshold api.ResourceThresholds) bool { + if nodeutil.IsNodeUnschedulable(nodesMap[nodeName]) { + klog.V(2).InfoS( + "Node is unschedulable", + "node", klog.KObj(nodesMap[nodeName]), + ) + return false + } + return true }, ) diff --git a/pkg/framework/plugins/nodeutilization/lownodeutilization.go b/pkg/framework/plugins/nodeutilization/lownodeutilization.go index fcbbdf56c..edb6b3d9a 100644 --- a/pkg/framework/plugins/nodeutilization/lownodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/lownodeutilization.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/descheduler/pkg/descheduler/evictions" nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/classifier" "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/normalizer" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) @@ -185,25 +186,23 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra // classify nodes in under and over utilized. we will later try to move // pods from the overutilized nodes to the underutilized ones. - nodeGroups := classifyNodeUsage( + nodeGroups := classifier.Classify( usage, thresholds, - []classifierFnc{ - // underutilization criteria processing. nodes that are - // underutilized but aren't schedulable are ignored. - func(nodeName string, usage, threshold api.ResourceThresholds) bool { - if nodeutil.IsNodeUnschedulable(nodesMap[nodeName]) { - klog.V(2).InfoS( - "Node is unschedulable, thus not considered as underutilized", - "node", klog.KObj(nodesMap[nodeName]), - ) - return false - } - return isNodeBelowThreshold(usage, threshold) - }, - // overutilization criteria evaluation. - func(nodeName string, usage, threshold api.ResourceThresholds) bool { - return isNodeAboveThreshold(usage, threshold) - }, + // underutilization criteria processing. nodes that are + // underutilized but aren't schedulable are ignored. + func(nodeName string, usage, threshold api.ResourceThresholds) bool { + if nodeutil.IsNodeUnschedulable(nodesMap[nodeName]) { + klog.V(2).InfoS( + "Node is unschedulable, thus not considered as underutilized", + "node", klog.KObj(nodesMap[nodeName]), + ) + return false + } + return isNodeBelowThreshold(usage, threshold) + }, + // overutilization criteria evaluation. + func(nodeName string, usage, threshold api.ResourceThresholds) bool { + return isNodeAboveThreshold(usage, threshold) }, ) diff --git a/pkg/framework/plugins/nodeutilization/nodeutilization.go b/pkg/framework/plugins/nodeutilization/nodeutilization.go index 9432cc500..62995b8c0 100644 --- a/pkg/framework/plugins/nodeutilization/nodeutilization.go +++ b/pkg/framework/plugins/nodeutilization/nodeutilization.go @@ -38,28 +38,40 @@ import ( "sigs.k8s.io/descheduler/pkg/utils" ) -// []NodeUsage is a snapshot, so allPods can not be read any time to avoid breaking consistency between the node's actual usage and available pods +// []NodeUsage is a snapshot, so allPods can not be read any time to avoid +// breaking consistency between the node's actual usage and available pods. // // New data model: // - node usage: map[string]api.ReferencedResourceList // - thresholds: map[string]api.ReferencedResourceList // - all pods: map[string][]*v1.Pod -// After classification: -// - each group will have its own (smaller) node usage and thresholds and allPods -// Both node usage and thresholds are needed to compute the remaining resources that can be evicted/can accepted evicted pods // -// 1. translate node usages into percentages as float or int64 (how much precision is lost?, maybe use BigInt?) -// 2. produce thresholds (if they need to be computed, otherwise use user provided, they are already in percentages) -// 3. classify nodes into groups -// 4. produces a list of nodes (sorted as before) that have the node usage, the threshold (only one this time) and the snapshottted pod list present - +// After classification: +// - each group will have its own (smaller) node usage and thresholds and +// allPods. +// +// Both node usage and thresholds are needed to compute the remaining resources +// that can be evicted/can accepted evicted pods. +// +// 1. translate node usages into percentages as float or int64 (how much +// precision is lost?, maybe use BigInt?). +// 2. produce thresholds (if they need to be computed, otherwise use user +// provided, they are already in percentages). +// 3. classify nodes into groups. +// 4. produces a list of nodes (sorted as before) that have the node usage, +// the threshold (only one this time) and the snapshottted pod list +// present. +// // Data wise // Produce separated maps for: // - nodes: map[string]*v1.Node // - node usage: map[string]api.ReferencedResourceList // - thresholds: map[string][]api.ReferencedResourceList // - pod list: map[string][]*v1.Pod -// Once the nodes are classified produce the original []NodeInfo so the code is not that much changed (postponing further refactoring once it is needed) +// +// Once the nodes are classified produce the original []NodeInfo so the code is +// not that much changed (postponing further refactoring once it is needed). + const ( // MetricResource is a special resource name we use to keep track of a // metric obtained from a third party entity. @@ -118,35 +130,6 @@ func getNodeUsageSnapshot( return nodesMap, nodesUsageMap, podListMap } -// classifierFnc is a function that classifies a node based on its usage and -// thresholds. returns true if it belongs to the group the classifier -// represents. -type classifierFnc func(string, api.ResourceThresholds, api.ResourceThresholds) bool - -// classifyNodeUsage classify nodes into different groups based on classifiers. -// returns one group for each classifier. -func classifyNodeUsage( - nodeUsageAsNodeThresholds map[string]api.ResourceThresholds, - nodeThresholdsMap map[string][]api.ResourceThresholds, - classifiers []classifierFnc, -) []map[string]api.ResourceThresholds { - nodeGroups := make([]map[string]api.ResourceThresholds, len(classifiers)) - for i := range len(classifiers) { - nodeGroups[i] = make(map[string]api.ResourceThresholds) - } - - for nodeName, nodeUsage := range nodeUsageAsNodeThresholds { - for idx, classFnc := range classifiers { - if classFnc(nodeName, nodeUsage, nodeThresholdsMap[nodeName][idx]) { - nodeGroups[idx][nodeName] = nodeUsage - break - } - } - } - - return nodeGroups -} - // usageToKeysAndValues converts a ReferencedResourceList into a list of // keys and values. this is useful for logging. func usageToKeysAndValues(usage api.ReferencedResourceList) []any { diff --git a/pkg/framework/plugins/nodeutilization/nodeutilization_test.go b/pkg/framework/plugins/nodeutilization/nodeutilization_test.go index 3404ada0e..af9d49de5 100644 --- a/pkg/framework/plugins/nodeutilization/nodeutilization_test.go +++ b/pkg/framework/plugins/nodeutilization/nodeutilization_test.go @@ -17,13 +17,17 @@ limitations under the License. package nodeutilization import ( + "math" "reflect" "testing" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/descheduler/pkg/api" + "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/classifier" + "sigs.k8s.io/descheduler/pkg/framework/plugins/nodeutilization/normalizer" ) func BuildTestNodeInfo(name string, apply func(*NodeInfo)) *NodeInfo { @@ -214,3 +218,362 @@ func TestResourceUsageToResourceThreshold(t *testing.T) { }) } } + +func ResourceListUsageNormalizer(usages, totals v1.ResourceList) api.ResourceThresholds { + result := api.ResourceThresholds{} + for rname, value := range usages { + total, ok := totals[rname] + if !ok { + continue + } + + used, avail := value.Value(), total.Value() + if rname == v1.ResourceCPU { + used, avail = value.MilliValue(), total.MilliValue() + } + + pct := math.Max(math.Min(float64(used)/float64(avail)*100, 100), 0) + result[rname] = api.Percentage(pct) + } + return result +} + +// This is a test for thresholds being defined as deviations from the average +// usage. This is expected to be a little longer test case. We are going to +// comment the steps to make it easier to follow. +func TestClassificationUsingDeviationThresholds(t *testing.T) { + // These are the two thresholds defined by the user. These thresholds + // mean that our low limit will be 5 pct points below the average and + // the high limit will be 5 pct points above the average. + userDefinedThresholds := map[string]api.ResourceThresholds{ + "low": {v1.ResourceCPU: 5, v1.ResourceMemory: 5}, + "high": {v1.ResourceCPU: 5, v1.ResourceMemory: 5}, + } + + // Create a fake total amount of resources for all nodes. We define + // the total amount to 1000 for both memory and cpu. This is so we + // can easily calculate (manually) the percentage of usages here. + nodesTotal := normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1000"), + v1.ResourceMemory: resource.MustParse("1000"), + }, + ) + + // Create a fake usage per server per resource. We are aiming to + // have the average of these resources in 50%. When applying the + // thresholds we should obtain the low threhold at 45% and the high + // threshold at 55%. + nodesUsage := map[string]v1.ResourceList{ + "node1": { + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("100"), + }, + "node2": { + v1.ResourceCPU: resource.MustParse("480"), + v1.ResourceMemory: resource.MustParse("480"), + }, + "node3": { + v1.ResourceCPU: resource.MustParse("520"), + v1.ResourceMemory: resource.MustParse("520"), + }, + "node4": { + v1.ResourceCPU: resource.MustParse("500"), + v1.ResourceMemory: resource.MustParse("500"), + }, + "node5": { + v1.ResourceCPU: resource.MustParse("900"), + v1.ResourceMemory: resource.MustParse("900"), + }, + } + + // Normalize the usage to percentages and then calculate the average + // among all nodes. + usage := normalizer.Normalize(nodesUsage, nodesTotal, ResourceListUsageNormalizer) + average := normalizer.Average(usage) + + // Create the thresholds by first applying the deviations and then + // replicating once for each node. Thresholds are supposed to be per + // node even though the user provides them only once. This is by + // design as it opens the possibility for further implementations of + // thresholds per node. + thresholds := normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + []api.ResourceThresholds{ + normalizer.Sum(average, normalizer.Negate(userDefinedThresholds["low"])), + normalizer.Sum(average, userDefinedThresholds["high"]), + }, + ) + + // Classify the nodes according to the thresholds. Nodes below the low + // threshold (45%) are underutilized, nodes above the high threshold + // (55%) are overutilized and nodes in between are properly utilized. + result := classifier.Classify( + usage, thresholds, + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(usage - limit) + }, + ), + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(limit - usage) + }, + ), + ) + + // we expect the node1 to be undertilized (10%), node2, node3 and node4 + // to be properly utilized (48%, 52% and 50% respectively) and node5 to + // be overutilized (90%). + expected := []map[string]api.ResourceThresholds{ + {"node1": {v1.ResourceCPU: 10, v1.ResourceMemory: 10}}, + {"node5": {v1.ResourceCPU: 90, v1.ResourceMemory: 90}}, + } + + if !reflect.DeepEqual(result, expected) { + t.Fatalf("unexpected result: %v, expecting: %v", result, expected) + } +} + +// This is almost a copy of TestUsingDeviationThresholds but we are using +// pointers here. This is for making sure our generic types are in check. To +// understand this code better read comments on TestUsingDeviationThresholds. +func TestUsingDeviationThresholdsWithPointers(t *testing.T) { + userDefinedThresholds := map[string]api.ResourceThresholds{ + "low": {v1.ResourceCPU: 5, v1.ResourceMemory: 5}, + "high": {v1.ResourceCPU: 5, v1.ResourceMemory: 5}, + } + + nodesTotal := normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + map[v1.ResourceName]*resource.Quantity{ + v1.ResourceCPU: ptr.To(resource.MustParse("1000")), + v1.ResourceMemory: ptr.To(resource.MustParse("1000")), + }, + ) + + nodesUsage := map[string]map[v1.ResourceName]*resource.Quantity{ + "node1": { + v1.ResourceCPU: ptr.To(resource.MustParse("100")), + v1.ResourceMemory: ptr.To(resource.MustParse("100")), + }, + "node2": { + v1.ResourceCPU: ptr.To(resource.MustParse("480")), + v1.ResourceMemory: ptr.To(resource.MustParse("480")), + }, + "node3": { + v1.ResourceCPU: ptr.To(resource.MustParse("520")), + v1.ResourceMemory: ptr.To(resource.MustParse("520")), + }, + "node4": { + v1.ResourceCPU: ptr.To(resource.MustParse("500")), + v1.ResourceMemory: ptr.To(resource.MustParse("500")), + }, + "node5": { + v1.ResourceCPU: ptr.To(resource.MustParse("900")), + v1.ResourceMemory: ptr.To(resource.MustParse("900")), + }, + } + + ptrNormalizer := func( + usages, totals map[v1.ResourceName]*resource.Quantity, + ) api.ResourceThresholds { + newUsages := v1.ResourceList{} + for name, usage := range usages { + newUsages[name] = *usage + } + newTotals := v1.ResourceList{} + for name, total := range totals { + newTotals[name] = *total + } + return ResourceListUsageNormalizer(newUsages, newTotals) + } + + usage := normalizer.Normalize(nodesUsage, nodesTotal, ptrNormalizer) + average := normalizer.Average(usage) + + thresholds := normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + []api.ResourceThresholds{ + normalizer.Sum(average, normalizer.Negate(userDefinedThresholds["low"])), + normalizer.Sum(average, userDefinedThresholds["high"]), + }, + ) + + result := classifier.Classify( + usage, thresholds, + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(usage - limit) + }, + ), + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(limit - usage) + }, + ), + ) + + expected := []map[string]api.ResourceThresholds{ + {"node1": {v1.ResourceCPU: 10, v1.ResourceMemory: 10}}, + {"node5": {v1.ResourceCPU: 90, v1.ResourceMemory: 90}}, + } + + if !reflect.DeepEqual(result, expected) { + t.Fatalf("unexpected result: %v, expecting: %v", result, expected) + } +} + +func TestNormalizeAndClassify(t *testing.T) { + for _, tt := range []struct { + name string + usage map[string]v1.ResourceList + totals map[string]v1.ResourceList + thresholds map[string][]api.ResourceThresholds + expected []map[string]api.ResourceThresholds + classifiers []classifier.Classifier[string, api.ResourceThresholds] + }{ + { + name: "happy path test", + usage: map[string]v1.ResourceList{ + "node1": { + // underutilized on cpu and memory. + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10"), + }, + "node2": { + // overutilized on cpu and memory. + v1.ResourceCPU: resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), + }, + "node3": { + // properly utilized on cpu and memory. + v1.ResourceCPU: resource.MustParse("50"), + v1.ResourceMemory: resource.MustParse("50"), + }, + "node4": { + // underutilized on cpu and overutilized on memory. + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("90"), + }, + }, + totals: normalizer.Replicate( + []string{"node1", "node2", "node3", "node4"}, + v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("100"), + }, + ), + thresholds: normalizer.Replicate( + []string{"node1", "node2", "node3", "node4"}, + []api.ResourceThresholds{ + {v1.ResourceCPU: 20, v1.ResourceMemory: 20}, + {v1.ResourceCPU: 80, v1.ResourceMemory: 80}, + }, + ), + expected: []map[string]api.ResourceThresholds{ + { + "node1": {v1.ResourceCPU: 10, v1.ResourceMemory: 10}, + }, + { + "node2": {v1.ResourceCPU: 90, v1.ResourceMemory: 90}, + }, + }, + classifiers: []classifier.Classifier[string, api.ResourceThresholds]{ + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(usage - limit) + }, + ), + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(limit - usage) + }, + ), + }, + }, + { + name: "three thresholds", + usage: map[string]v1.ResourceList{ + "node1": { + // match for the first classifier. + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("10"), + }, + "node2": { + // match for the third classifier. + v1.ResourceCPU: resource.MustParse("90"), + v1.ResourceMemory: resource.MustParse("90"), + }, + "node3": { + // match fo the second classifier. + v1.ResourceCPU: resource.MustParse("40"), + v1.ResourceMemory: resource.MustParse("40"), + }, + "node4": { + // matches no classifier. + v1.ResourceCPU: resource.MustParse("10"), + v1.ResourceMemory: resource.MustParse("90"), + }, + "node5": { + // match for the first classifier. + v1.ResourceCPU: resource.MustParse("11"), + v1.ResourceMemory: resource.MustParse("18"), + }, + }, + totals: normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("100"), + }, + ), + thresholds: normalizer.Replicate( + []string{"node1", "node2", "node3", "node4", "node5"}, + []api.ResourceThresholds{ + {v1.ResourceCPU: 20, v1.ResourceMemory: 20}, + {v1.ResourceCPU: 50, v1.ResourceMemory: 50}, + {v1.ResourceCPU: 80, v1.ResourceMemory: 80}, + }, + ), + expected: []map[string]api.ResourceThresholds{ + { + "node1": {v1.ResourceCPU: 10, v1.ResourceMemory: 10}, + "node5": {v1.ResourceCPU: 11, v1.ResourceMemory: 18}, + }, + { + "node3": {v1.ResourceCPU: 40, v1.ResourceMemory: 40}, + }, + { + "node2": {v1.ResourceCPU: 90, v1.ResourceMemory: 90}, + }, + }, + classifiers: []classifier.Classifier[string, api.ResourceThresholds]{ + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(usage - limit) + }, + ), + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(usage - limit) + }, + ), + classifier.ForMap[string, v1.ResourceName, api.Percentage, api.ResourceThresholds]( + func(usage, limit api.Percentage) int { + return int(limit - usage) + }, + ), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + pct := normalizer.Normalize(tt.usage, tt.totals, ResourceListUsageNormalizer) + res := classifier.Classify(pct, tt.thresholds, tt.classifiers...) + if !reflect.DeepEqual(res, tt.expected) { + t.Fatalf("unexpected result: %v, expecting: %v", res, tt.expected) + } + }) + } +}