From 704f6d449612c3030b6ebb7d420a1d1e21a086e6 Mon Sep 17 00:00:00 2001 From: Jan Chaloupka Date: Wed, 15 Jun 2022 11:26:17 +0200 Subject: [PATCH] Migrate RemovePodsViolatingNodeTaints into a plugin --- pkg/apis/componentconfig/types_pluginargs.go | 34 +++++ .../validation/validation_pluginargs.go | 33 +++++ .../componentconfig/zz_generated.deepcopy.go | 42 ++++++ pkg/descheduler/descheduler.go | 91 +++++++++++- pkg/descheduler/strategy_migration.go | 57 ++++++++ pkg/framework/fake/fake.go | 45 ++++++ .../node_taint.go | 129 ++++++++++++++++++ .../node_taint_test.go | 60 +++++--- pkg/framework/types.go | 72 ++++++++++ 9 files changed, 541 insertions(+), 22 deletions(-) create mode 100644 pkg/apis/componentconfig/types_pluginargs.go create mode 100644 pkg/apis/componentconfig/validation/validation_pluginargs.go create mode 100644 pkg/descheduler/strategy_migration.go create mode 100644 pkg/framework/fake/fake.go create mode 100644 pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go rename pkg/{descheduler/strategies => framework/plugins/removepodsviolatingnodetaints}/node_taint_test.go (91%) create mode 100644 pkg/framework/types.go diff --git a/pkg/apis/componentconfig/types_pluginargs.go b/pkg/apis/componentconfig/types_pluginargs.go new file mode 100644 index 000000000..7eca584c6 --- /dev/null +++ b/pkg/apis/componentconfig/types_pluginargs.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package componentconfig + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/descheduler/pkg/api" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// RemovePodsViolatingNodeTaintsArgs holds arguments used to configure the RemovePodsViolatingNodeTaints plugin. +type RemovePodsViolatingNodeTaintsArgs struct { + metav1.TypeMeta + + Namespaces *api.Namespaces + LabelSelector *metav1.LabelSelector + IncludePreferNoSchedule bool + ExcludedTaints []string +} diff --git a/pkg/apis/componentconfig/validation/validation_pluginargs.go b/pkg/apis/componentconfig/validation/validation_pluginargs.go new file mode 100644 index 000000000..ebd67f073 --- /dev/null +++ b/pkg/apis/componentconfig/validation/validation_pluginargs.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "fmt" + + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" +) + +// ValidateRemovePodsViolatingNodeTaintsArgs validates RemovePodsViolatingNodeTaints arguments +func ValidateRemovePodsViolatingNodeTaintsArgs(args *componentconfig.RemovePodsViolatingNodeTaintsArgs) error { + // At most one of include/exclude can be set + if args.Namespaces != nil && len(args.Namespaces.Include) > 0 && len(args.Namespaces.Exclude) > 0 { + return fmt.Errorf("only one of Include/Exclude namespaces can be set") + } + + return nil +} diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index f450e704c..9d7267fc9 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -22,7 +22,9 @@ limitations under the License. package componentconfig import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + api "sigs.k8s.io/descheduler/pkg/api" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -51,3 +53,43 @@ func (in *DeschedulerConfiguration) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemovePodsViolatingNodeTaintsArgs) DeepCopyInto(out *RemovePodsViolatingNodeTaintsArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = new(api.Namespaces) + (*in).DeepCopyInto(*out) + } + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.ExcludedTaints != nil { + in, out := &in.ExcludedTaints, &out.ExcludedTaints + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemovePodsViolatingNodeTaintsArgs. +func (in *RemovePodsViolatingNodeTaintsArgs) DeepCopy() *RemovePodsViolatingNodeTaintsArgs { + if in == nil { + return nil + } + out := new(RemovePodsViolatingNodeTaintsArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RemovePodsViolatingNodeTaintsArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index 15d138ff0..f5fd5c997 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -45,6 +45,8 @@ import ( podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" "sigs.k8s.io/descheduler/pkg/descheduler/strategies" "sigs.k8s.io/descheduler/pkg/descheduler/strategies/nodeutilization" + "sigs.k8s.io/descheduler/pkg/framework" + "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingnodetaints" "sigs.k8s.io/descheduler/pkg/utils" ) @@ -166,6 +168,59 @@ func cachedClient( return fakeClient, nil } +// evictorImpl implements the Evictor interface so plugins +// can evict a pod without importing a specific pod evictor +type evictorImpl struct { + podEvictor *evictions.PodEvictor + evictorFilter *evictions.EvictorFilter +} + +var _ framework.Evictor = &evictorImpl{} + +// Filter checks if a pod can be evicted +func (ei *evictorImpl) Filter(pod *v1.Pod) bool { + return ei.evictorFilter.Filter(pod) +} + +// Evict evicts a pod (no pre-check performed) +func (ei *evictorImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.EvictOptions) bool { + return ei.podEvictor.EvictPod(ctx, pod, opts) +} + +func (ei *evictorImpl) NodeLimitExceeded(node *v1.Node) bool { + return ei.podEvictor.NodeLimitExceeded(node) +} + +// handleImpl implements the framework handle which gets passed to plugins +type handleImpl struct { + clientSet clientset.Interface + getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc + sharedInformerFactory informers.SharedInformerFactory + evictor *evictorImpl +} + +var _ framework.Handle = &handleImpl{} + +// ClientSet retrieves kube client set +func (hi *handleImpl) ClientSet() clientset.Interface { + return hi.clientSet +} + +// GetPodsAssignedToNodeFunc retrieves GetPodsAssignedToNodeFunc implementation +func (hi *handleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc { + return hi.getPodsAssignedToNodeFunc +} + +// SharedInformerFactory retrieves shared informer factory +func (hi *handleImpl) SharedInformerFactory() informers.SharedInformerFactory { + return hi.sharedInformerFactory +} + +// Evictor retrieves evictor so plugins can filter and evict pods +func (hi *handleImpl) Evictor() framework.Evictor { + return hi.evictor +} + func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string) error { sharedInformerFactory := informers.NewSharedInformerFactory(rs.Client, 0) nodeInformer := sharedInformerFactory.Core().V1().Nodes() @@ -194,7 +249,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer "HighNodeUtilization": nodeutilization.HighNodeUtilization, "RemovePodsViolatingInterPodAntiAffinity": strategies.RemovePodsViolatingInterPodAntiAffinity, "RemovePodsViolatingNodeAffinity": strategies.RemovePodsViolatingNodeAffinity, - "RemovePodsViolatingNodeTaints": strategies.RemovePodsViolatingNodeTaints, + "RemovePodsViolatingNodeTaints": nil, "RemovePodsHavingTooManyRestarts": strategies.RemovePodsHavingTooManyRestarts, "PodLifeTime": strategies.PodLifeTime, "RemovePodsViolatingTopologySpreadConstraint": strategies.RemovePodsViolatingTopologySpreadConstraint, @@ -290,17 +345,28 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer for name, strategy := range deschedulerPolicy.Strategies { if f, ok := strategyFuncs[name]; ok { if strategy.Enabled { + params := strategy.Params + if params == nil { + params = &api.StrategyParameters{} + } + nodeFit := false if name != "PodLifeTime" { - if strategy.Params != nil { - nodeFit = strategy.Params.NodeFit - } + nodeFit = params.NodeFit + } + + // TODO(jchaloup): once all strategies are migrated move this check under + // the default evictor args validation + if params.ThresholdPriority != nil && params.ThresholdPriorityClassName != "" { + klog.V(1).ErrorS(fmt.Errorf("priority threshold misconfigured"), "only one of priorityThreshold fields can be set", "pluginName", removepodsviolatingnodetaints.PluginName) + continue } thresholdPriority, err := utils.GetPriorityFromStrategyParams(ctx, rs.Client, strategy.Params) if err != nil { klog.ErrorS(err, "Failed to get threshold priority from strategy's params") continue } + evictorFilter := evictions.NewEvictorFilter( nodes, getPodsAssignedToNode, @@ -311,11 +377,26 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer evictions.WithNodeFit(nodeFit), evictions.WithPriorityThreshold(thresholdPriority), ) + handle := &handleImpl{ + clientSet: rs.Client, + getPodsAssignedToNodeFunc: getPodsAssignedToNode, + sharedInformerFactory: sharedInformerFactory, + evictor: &evictorImpl{ + podEvictor: podEvictor, + evictorFilter: evictorFilter, + }, + } + // TODO: strategyName should be accessible from within the strategy using a framework // handle or function which the Evictor has access to. For migration/in-progress framework // work, we are currently passing this via context. To be removed // (See discussion thread https://github.com/kubernetes-sigs/descheduler/pull/885#discussion_r919962292) - f(context.WithValue(ctx, "strategyName", string(name)), rs.Client, strategy, nodes, podEvictor, evictorFilter, getPodsAssignedToNode) + childCtx := context.WithValue(ctx, "strategyName", string(name)) + if pgFnc, exists := pluginsMap[string(name)]; exists { + pgFnc(childCtx, nodes, params, handle) + } else { + f(childCtx, rs.Client, strategy, nodes, podEvictor, evictorFilter, getPodsAssignedToNode) + } } } else { klog.ErrorS(fmt.Errorf("unknown strategy name"), "skipping strategy", "strategy", name) diff --git a/pkg/descheduler/strategy_migration.go b/pkg/descheduler/strategy_migration.go new file mode 100644 index 000000000..0bbcee012 --- /dev/null +++ b/pkg/descheduler/strategy_migration.go @@ -0,0 +1,57 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package descheduler + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/descheduler/pkg/api" + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" + "sigs.k8s.io/descheduler/pkg/apis/componentconfig/validation" + "sigs.k8s.io/descheduler/pkg/framework" + "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingnodetaints" +) + +// Once all strategies are migrated the arguments get read from the configuration file +// without any wiring. Keeping the wiring here so the descheduler can still use +// the v1alpha1 configuration during the strategy migration to plugins. + +var pluginsMap = map[string]func(ctx context.Context, nodes []*v1.Node, params *api.StrategyParameters, handle *handleImpl){ + "RemovePodsViolatingNodeTaints": func(ctx context.Context, nodes []*v1.Node, params *api.StrategyParameters, handle *handleImpl) { + args := &componentconfig.RemovePodsViolatingNodeTaintsArgs{ + Namespaces: params.Namespaces, + LabelSelector: params.LabelSelector, + IncludePreferNoSchedule: params.IncludePreferNoSchedule, + ExcludedTaints: params.ExcludedTaints, + } + if err := validation.ValidateRemovePodsViolatingNodeTaintsArgs(args); err != nil { + klog.V(1).ErrorS(err, "unable to validate plugin arguments", "pluginName", removepodsviolatingnodetaints.PluginName) + return + } + pg, err := removepodsviolatingnodetaints.New(args, handle) + if err != nil { + klog.V(1).ErrorS(err, "unable to initialize a plugin", "pluginName", removepodsviolatingnodetaints.PluginName) + return + } + status := pg.(framework.DeschedulePlugin).Deschedule(ctx, nodes) + if status != nil && status.Err != nil { + klog.V(1).ErrorS(err, "plugin finished with error", "pluginName", removepodsviolatingnodetaints.PluginName) + } + }, +} diff --git a/pkg/framework/fake/fake.go b/pkg/framework/fake/fake.go new file mode 100644 index 000000000..dba0d1b5e --- /dev/null +++ b/pkg/framework/fake/fake.go @@ -0,0 +1,45 @@ +package fake + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework" +) + +type HandleImpl struct { + ClientsetImpl clientset.Interface + GetPodsAssignedToNodeFuncImpl podutil.GetPodsAssignedToNodeFunc + SharedInformerFactoryImpl informers.SharedInformerFactory + EvictorFilterImpl *evictions.EvictorFilter + PodEvictorImpl *evictions.PodEvictor +} + +var _ framework.Handle = &HandleImpl{} + +func (hi *HandleImpl) ClientSet() clientset.Interface { + return hi.ClientsetImpl +} +func (hi *HandleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc { + return hi.GetPodsAssignedToNodeFuncImpl +} +func (hi *HandleImpl) SharedInformerFactory() informers.SharedInformerFactory { + return hi.SharedInformerFactoryImpl +} +func (hi *HandleImpl) Evictor() framework.Evictor { + return hi +} +func (hi *HandleImpl) Filter(pod *v1.Pod) bool { + return hi.EvictorFilterImpl.Filter(pod) +} +func (hi *HandleImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.EvictOptions) bool { + return hi.PodEvictorImpl.EvictPod(ctx, pod, opts) +} +func (hi *HandleImpl) NodeLimitExceeded(node *v1.Node) bool { + return hi.PodEvictorImpl.NodeLimitExceeded(node) +} diff --git a/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go new file mode 100644 index 000000000..3c3ec4182 --- /dev/null +++ b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint.go @@ -0,0 +1,129 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package removepodsviolatingnodetaints + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework" + "sigs.k8s.io/descheduler/pkg/utils" +) + +const PluginName = "RemovePodsViolatingNodeTaints" + +// RemovePodsViolatingNodeTaints evicts pods on the node which violate NoSchedule Taints on nodes +type RemovePodsViolatingNodeTaints struct { + handle framework.Handle + args *componentconfig.RemovePodsViolatingNodeTaintsArgs + taintFilterFnc func(taint *v1.Taint) bool + podFilter podutil.FilterFunc +} + +var _ framework.Plugin = &RemovePodsViolatingNodeTaints{} +var _ framework.DeschedulePlugin = &RemovePodsViolatingNodeTaints{} + +// New builds plugin from its arguments while passing a handle +func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error) { + nodeTaintsArgs, ok := args.(*componentconfig.RemovePodsViolatingNodeTaintsArgs) + if !ok { + return nil, fmt.Errorf("want args to be of type RemovePodsViolatingNodeTaintsArgs, got %T", args) + } + + var includedNamespaces, excludedNamespaces sets.String + if nodeTaintsArgs.Namespaces != nil { + includedNamespaces = sets.NewString(nodeTaintsArgs.Namespaces.Include...) + excludedNamespaces = sets.NewString(nodeTaintsArgs.Namespaces.Exclude...) + } + + podFilter, err := podutil.NewOptions(). + WithFilter(handle.Evictor().Filter). + WithNamespaces(includedNamespaces). + WithoutNamespaces(excludedNamespaces). + WithLabelSelector(nodeTaintsArgs.LabelSelector). + BuildFilterFunc() + if err != nil { + return nil, fmt.Errorf("error initializing pod filter function: %v", err) + } + + excludedTaints := sets.NewString(nodeTaintsArgs.ExcludedTaints...) + excludeTaint := func(taint *v1.Taint) bool { + // Exclude taints by key *or* key=value + return excludedTaints.Has(taint.Key) || (taint.Value != "" && excludedTaints.Has(fmt.Sprintf("%s=%s", taint.Key, taint.Value))) + } + + taintFilterFnc := func(taint *v1.Taint) bool { return (taint.Effect == v1.TaintEffectNoSchedule) && !excludeTaint(taint) } + if nodeTaintsArgs.IncludePreferNoSchedule { + taintFilterFnc = func(taint *v1.Taint) bool { + return (taint.Effect == v1.TaintEffectNoSchedule || taint.Effect == v1.TaintEffectPreferNoSchedule) && !excludeTaint(taint) + } + } + + return &RemovePodsViolatingNodeTaints{ + handle: handle, + podFilter: podFilter, + args: nodeTaintsArgs, + taintFilterFnc: taintFilterFnc, + }, nil +} + +// Name retrieves the plugin name +func (d *RemovePodsViolatingNodeTaints) Name() string { + return PluginName +} + +// Deschedule extension point implementation for the plugin +func (d *RemovePodsViolatingNodeTaints) Deschedule(ctx context.Context, nodes []*v1.Node) *framework.Status { + for _, node := range nodes { + klog.V(1).InfoS("Processing node", "node", klog.KObj(node)) + pods, err := podutil.ListAllPodsOnANode(node.Name, d.handle.GetPodsAssignedToNodeFunc(), d.podFilter) + if err != nil { + // no pods evicted as error encountered retrieving evictable Pods + return &framework.Status{ + Err: fmt.Errorf("error listing pods on a node: %v", err), + } + } + totalPods := len(pods) + skipNode := false + for i := 0; i < totalPods; i++ { + if !utils.TolerationsTolerateTaintsWithFilter( + pods[i].Spec.Tolerations, + node.Spec.Taints, + d.taintFilterFnc, + ) { + klog.V(2).InfoS("Not all taints with NoSchedule effect are tolerated after update for pod on node", "pod", klog.KObj(pods[i]), "node", klog.KObj(node)) + d.handle.Evictor().Evict(ctx, pods[i], evictions.EvictOptions{}) + if d.handle.Evictor().NodeLimitExceeded(node) { + skipNode = true + break + } + } + } + if skipNode { + continue + } + } + return nil +} diff --git a/pkg/descheduler/strategies/node_taint_test.go b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go similarity index 91% rename from pkg/descheduler/strategies/node_taint_test.go rename to pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go index 7263dd8e2..4326da789 100644 --- a/pkg/descheduler/strategies/node_taint_test.go +++ b/pkg/framework/plugins/removepodsviolatingnodetaints/node_taint_test.go @@ -1,4 +1,20 @@ -package strategies +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package removepodsviolatingnodetaints import ( "context" @@ -12,9 +28,11 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" - "sigs.k8s.io/descheduler/pkg/api" + "sigs.k8s.io/descheduler/pkg/apis/componentconfig" "sigs.k8s.io/descheduler/pkg/descheduler/evictions" podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" + "sigs.k8s.io/descheduler/pkg/framework" + frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake" "sigs.k8s.io/descheduler/pkg/utils" "sigs.k8s.io/descheduler/test" ) @@ -341,25 +359,33 @@ func TestDeletePodsViolatingNodeTaints(t *testing.T) { false, ) - strategy := api.DeschedulerStrategy{ - Params: &api.StrategyParameters{ - NodeFit: tc.nodeFit, - IncludePreferNoSchedule: tc.includePreferNoSchedule, - ExcludedTaints: tc.excludedTaints, - }, + handle := &frameworkfake.HandleImpl{ + ClientsetImpl: fakeClient, + GetPodsAssignedToNodeFuncImpl: getPodsAssignedToNode, + PodEvictorImpl: podEvictor, + EvictorFilterImpl: evictions.NewEvictorFilter( + tc.nodes, + getPodsAssignedToNode, + tc.evictLocalStoragePods, + tc.evictSystemCriticalPods, + false, + false, + evictions.WithNodeFit(tc.nodeFit), + ), + SharedInformerFactoryImpl: sharedInformerFactory, } - evictorFilter := evictions.NewEvictorFilter( - tc.nodes, - getPodsAssignedToNode, - tc.evictLocalStoragePods, - tc.evictSystemCriticalPods, - false, - false, - evictions.WithNodeFit(tc.nodeFit), + plugin, err := New(&componentconfig.RemovePodsViolatingNodeTaintsArgs{ + IncludePreferNoSchedule: tc.includePreferNoSchedule, + ExcludedTaints: tc.excludedTaints, + }, + handle, ) + if err != nil { + t.Fatalf("Unable to initialize the plugin: %v", err) + } - RemovePodsViolatingNodeTaints(ctx, fakeClient, strategy, tc.nodes, podEvictor, evictorFilter, getPodsAssignedToNode) + plugin.(framework.DeschedulePlugin).Deschedule(ctx, tc.nodes) actualEvictedPodCount := podEvictor.TotalEvicted() if actualEvictedPodCount != tc.expectedEvictedPodCount { t.Errorf("Test %#v failed, Unexpected no of pods evicted: pods evicted: %d, expected: %d", tc.description, actualEvictedPodCount, tc.expectedEvictedPodCount) diff --git a/pkg/framework/types.go b/pkg/framework/types.go new file mode 100644 index 000000000..26998b7f0 --- /dev/null +++ b/pkg/framework/types.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" +) + +// Handle provides handles used by plugins to retrieve a kubernetes client set, +// evictor interface, shared informer factory and other instruments shared +// across plugins. +type Handle interface { + // ClientSet returns a kubernetes clientSet. + ClientSet() clientset.Interface + Evictor() Evictor + GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc + SharedInformerFactory() informers.SharedInformerFactory +} + +// Evictor defines an interface for filtering and evicting pods +// while abstracting away the specific pod evictor/evictor filter. +type Evictor interface { + // Filter checks if a pod can be evicted + Filter(*v1.Pod) bool + // Evict evicts a pod (no pre-check performed) + Evict(context.Context, *v1.Pod, evictions.EvictOptions) bool + // NodeLimitExceeded checks if the number of evictions for a node was exceeded + NodeLimitExceeded(node *v1.Node) bool +} + +// Status describes result of an extension point invocation +type Status struct { + Err error +} + +// Plugin is the parent type for all the descheduling framework plugins. +type Plugin interface { + Name() string +} + +// DeschedulePlugin defines an extension point for a general descheduling operation +type DeschedulePlugin interface { + Plugin + Deschedule(ctx context.Context, nodes []*v1.Node) *Status +} + +// BalancePlugin defines an extension point for balancing pods across a cluster +type BalancePlugin interface { + Plugin + Balance(ctx context.Context, nodes []*v1.Node) *Status +}