mirror of
https://github.com/kubernetes-sigs/descheduler.git
synced 2026-01-26 05:14:13 +01:00
bcc6c8eb2a
* Descheduling profile * Fake plugin + profile unit testing * Rename Profile config type into DeschedulerProfile To avoid resamblance with profileImpl * First run deschedule, then balance extension points
311 lines
10 KiB
Go
311 lines
10 KiB
Go
/*
|
|
Copyright 2023 The Kubernetes Authors.
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package profile
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"sigs.k8s.io/descheduler/metrics"
|
|
"sigs.k8s.io/descheduler/pkg/api"
|
|
"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
|
|
podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
|
|
"sigs.k8s.io/descheduler/pkg/framework"
|
|
"sigs.k8s.io/descheduler/pkg/framework/pluginregistry"
|
|
"sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor"
|
|
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/util/errors"
|
|
"k8s.io/client-go/informers"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
|
|
"k8s.io/klog/v2"
|
|
)
|
|
|
|
// evictorImpl implements the Evictor interface so plugins
|
|
// can evict a pod without importing a specific pod evictor
|
|
type evictorImpl struct {
|
|
podEvictor *evictions.PodEvictor
|
|
evictorFilter framework.EvictorPlugin
|
|
}
|
|
|
|
var _ framework.Evictor = &evictorImpl{}
|
|
|
|
// Filter checks if a pod can be evicted
|
|
func (ei *evictorImpl) Filter(pod *v1.Pod) bool {
|
|
return ei.evictorFilter.Filter(pod)
|
|
}
|
|
|
|
// PreEvictionFilter checks if pod can be evicted right before eviction
|
|
func (ei *evictorImpl) PreEvictionFilter(pod *v1.Pod) bool {
|
|
return ei.evictorFilter.PreEvictionFilter(pod)
|
|
}
|
|
|
|
// Evict evicts a pod (no pre-check performed)
|
|
func (ei *evictorImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.EvictOptions) bool {
|
|
return ei.podEvictor.EvictPod(ctx, pod, opts)
|
|
}
|
|
|
|
func (ei *evictorImpl) NodeLimitExceeded(node *v1.Node) bool {
|
|
return ei.podEvictor.NodeLimitExceeded(node)
|
|
}
|
|
|
|
// handleImpl implements the framework handle which gets passed to plugins
|
|
type handleImpl struct {
|
|
clientSet clientset.Interface
|
|
getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
|
|
sharedInformerFactory informers.SharedInformerFactory
|
|
evictor *evictorImpl
|
|
}
|
|
|
|
var _ framework.Handle = &handleImpl{}
|
|
|
|
// ClientSet retrieves kube client set
|
|
func (hi *handleImpl) ClientSet() clientset.Interface {
|
|
return hi.clientSet
|
|
}
|
|
|
|
// GetPodsAssignedToNodeFunc retrieves GetPodsAssignedToNodeFunc implementation
|
|
func (hi *handleImpl) GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc {
|
|
return hi.getPodsAssignedToNodeFunc
|
|
}
|
|
|
|
// SharedInformerFactory retrieves shared informer factory
|
|
func (hi *handleImpl) SharedInformerFactory() informers.SharedInformerFactory {
|
|
return hi.sharedInformerFactory
|
|
}
|
|
|
|
// Evictor retrieves evictor so plugins can filter and evict pods
|
|
func (hi *handleImpl) Evictor() framework.Evictor {
|
|
return hi.evictor
|
|
}
|
|
|
|
type profileImpl struct {
|
|
profileName string
|
|
podEvictor *evictions.PodEvictor
|
|
|
|
deschedulePlugins []framework.DeschedulePlugin
|
|
balancePlugins []framework.BalancePlugin
|
|
}
|
|
|
|
// Option for the handleImpl.
|
|
type Option func(*handleImplOpts)
|
|
|
|
type handleImplOpts struct {
|
|
clientSet clientset.Interface
|
|
sharedInformerFactory informers.SharedInformerFactory
|
|
getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
|
|
podEvictor *evictions.PodEvictor
|
|
}
|
|
|
|
// WithClientSet sets clientSet for the scheduling frameworkImpl.
|
|
func WithClientSet(clientSet clientset.Interface) Option {
|
|
return func(o *handleImplOpts) {
|
|
o.clientSet = clientSet
|
|
}
|
|
}
|
|
|
|
func WithSharedInformerFactory(sharedInformerFactory informers.SharedInformerFactory) Option {
|
|
return func(o *handleImplOpts) {
|
|
o.sharedInformerFactory = sharedInformerFactory
|
|
}
|
|
}
|
|
|
|
func WithPodEvictor(podEvictor *evictions.PodEvictor) Option {
|
|
return func(o *handleImplOpts) {
|
|
o.podEvictor = podEvictor
|
|
}
|
|
}
|
|
|
|
func WithGetPodsAssignedToNodeFnc(getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc) Option {
|
|
return func(o *handleImplOpts) {
|
|
o.getPodsAssignedToNodeFunc = getPodsAssignedToNodeFunc
|
|
}
|
|
}
|
|
|
|
func getPluginConfig(pluginName string, pluginConfigs []api.PluginConfig) (*api.PluginConfig, int) {
|
|
for idx, pluginConfig := range pluginConfigs {
|
|
if pluginConfig.Name == pluginName {
|
|
return &pluginConfig, idx
|
|
}
|
|
}
|
|
return nil, 0
|
|
}
|
|
|
|
func buildPlugin(config api.DeschedulerProfile, pluginName string, handle *handleImpl, reg pluginregistry.Registry) (framework.Plugin, error) {
|
|
pc, _ := getPluginConfig(pluginName, config.PluginConfigs)
|
|
if pc == nil {
|
|
klog.ErrorS(fmt.Errorf("unable to get plugin config"), "skipping plugin", "plugin", pluginName, "profile", config.Name)
|
|
return nil, fmt.Errorf("unable to find %q plugin config", pluginName)
|
|
}
|
|
|
|
registryPlugin, ok := reg[pluginName]
|
|
if !ok {
|
|
klog.ErrorS(fmt.Errorf("unable to find plugin in the pluginsMap"), "skipping plugin", "plugin", pluginName)
|
|
return nil, fmt.Errorf("unable to find %q plugin in the pluginsMap", pluginName)
|
|
}
|
|
pg, err := registryPlugin.PluginBuilder(pc.Args, handle)
|
|
if err != nil {
|
|
klog.ErrorS(err, "unable to initialize a plugin", "pluginName", pluginName)
|
|
return nil, fmt.Errorf("unable to initialize %q plugin: %v", pluginName, err)
|
|
}
|
|
return pg, nil
|
|
}
|
|
|
|
func NewProfile(config api.DeschedulerProfile, reg pluginregistry.Registry, opts ...Option) (*profileImpl, error) {
|
|
hOpts := &handleImplOpts{}
|
|
for _, optFnc := range opts {
|
|
optFnc(hOpts)
|
|
}
|
|
|
|
if hOpts.clientSet == nil {
|
|
return nil, fmt.Errorf("clientSet missing")
|
|
}
|
|
|
|
if hOpts.sharedInformerFactory == nil {
|
|
return nil, fmt.Errorf("sharedInformerFactory missing")
|
|
}
|
|
|
|
if hOpts.podEvictor == nil {
|
|
return nil, fmt.Errorf("podEvictor missing")
|
|
}
|
|
|
|
evictorPlugin, err := buildPlugin(config, defaultevictor.PluginName, &handleImpl{
|
|
clientSet: hOpts.clientSet,
|
|
getPodsAssignedToNodeFunc: hOpts.getPodsAssignedToNodeFunc,
|
|
sharedInformerFactory: hOpts.sharedInformerFactory,
|
|
}, reg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to build %v plugin: %v", defaultevictor.PluginName, err)
|
|
}
|
|
if evictorPlugin == nil {
|
|
return nil, fmt.Errorf("empty plugin build for %v plugin: %v", defaultevictor.PluginName, err)
|
|
}
|
|
|
|
handle := &handleImpl{
|
|
clientSet: hOpts.clientSet,
|
|
getPodsAssignedToNodeFunc: hOpts.getPodsAssignedToNodeFunc,
|
|
sharedInformerFactory: hOpts.sharedInformerFactory,
|
|
evictor: &evictorImpl{
|
|
podEvictor: hOpts.podEvictor,
|
|
evictorFilter: evictorPlugin.(framework.EvictorPlugin),
|
|
},
|
|
}
|
|
|
|
deschedulePlugins := []framework.DeschedulePlugin{}
|
|
balancePlugins := []framework.BalancePlugin{}
|
|
|
|
descheduleEnabled := make(map[string]struct{})
|
|
balanceEnabled := make(map[string]struct{})
|
|
for _, name := range config.Plugins.Deschedule.Enabled {
|
|
descheduleEnabled[name] = struct{}{}
|
|
}
|
|
for _, name := range config.Plugins.Balance.Enabled {
|
|
balanceEnabled[name] = struct{}{}
|
|
}
|
|
|
|
// Assuming only a list of enabled extension points.
|
|
// Later, when a default list of plugins and their extension points is established,
|
|
// compute the list of enabled extension points as (DefaultEnabled + Enabled - Disabled)
|
|
for _, plugin := range append(config.Plugins.Deschedule.Enabled, config.Plugins.Balance.Enabled...) {
|
|
pg, err := buildPlugin(config, plugin, handle, reg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to build %v plugin: %v", plugin, err)
|
|
}
|
|
if pg != nil {
|
|
// pg can be of any of each type, or both
|
|
|
|
if _, exists := descheduleEnabled[plugin]; exists {
|
|
_, ok := pg.(framework.DeschedulePlugin)
|
|
if ok {
|
|
deschedulePlugins = append(deschedulePlugins, pg.(framework.DeschedulePlugin))
|
|
}
|
|
}
|
|
|
|
if _, exists := balanceEnabled[plugin]; exists {
|
|
_, ok := pg.(framework.BalancePlugin)
|
|
if ok {
|
|
balancePlugins = append(balancePlugins, pg.(framework.BalancePlugin))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return &profileImpl{
|
|
profileName: config.Name,
|
|
podEvictor: hOpts.podEvictor,
|
|
deschedulePlugins: deschedulePlugins,
|
|
balancePlugins: balancePlugins,
|
|
}, nil
|
|
}
|
|
|
|
func (d profileImpl) RunDeschedulePlugins(ctx context.Context, nodes []*v1.Node) *framework.Status {
|
|
errs := []error{}
|
|
for _, pl := range d.deschedulePlugins {
|
|
evicted := d.podEvictor.TotalEvicted()
|
|
// TODO: strategyName should be accessible from within the strategy using a framework
|
|
// handle or function which the Evictor has access to. For migration/in-progress framework
|
|
// work, we are currently passing this via context. To be removed
|
|
// (See discussion thread https://github.com/kubernetes-sigs/descheduler/pull/885#discussion_r919962292)
|
|
strategyStart := time.Now()
|
|
childCtx := context.WithValue(ctx, "strategyName", pl.Name())
|
|
status := pl.Deschedule(childCtx, nodes)
|
|
metrics.DeschedulerStrategyDuration.With(map[string]string{"strategy": pl.Name(), "profile": d.profileName}).Observe(time.Since(strategyStart).Seconds())
|
|
|
|
if status != nil && status.Err != nil {
|
|
errs = append(errs, fmt.Errorf("plugin %q finished with error: %v", pl.Name(), status.Err))
|
|
}
|
|
klog.V(1).InfoS("Total number of pods evicted", "extension point", "Deschedule", "evictedPods", d.podEvictor.TotalEvicted()-evicted)
|
|
}
|
|
|
|
aggrErr := errors.NewAggregate(errs)
|
|
if aggrErr == nil {
|
|
return &framework.Status{}
|
|
}
|
|
|
|
return &framework.Status{
|
|
Err: fmt.Errorf("%v", aggrErr.Error()),
|
|
}
|
|
}
|
|
|
|
func (d profileImpl) RunBalancePlugins(ctx context.Context, nodes []*v1.Node) *framework.Status {
|
|
errs := []error{}
|
|
for _, pl := range d.balancePlugins {
|
|
evicted := d.podEvictor.TotalEvicted()
|
|
// TODO: strategyName should be accessible from within the strategy using a framework
|
|
// handle or function which the Evictor has access to. For migration/in-progress framework
|
|
// work, we are currently passing this via context. To be removed
|
|
// (See discussion thread https://github.com/kubernetes-sigs/descheduler/pull/885#discussion_r919962292)
|
|
strategyStart := time.Now()
|
|
childCtx := context.WithValue(ctx, "strategyName", pl.Name())
|
|
status := pl.Balance(childCtx, nodes)
|
|
metrics.DeschedulerStrategyDuration.With(map[string]string{"strategy": pl.Name(), "profile": d.profileName}).Observe(time.Since(strategyStart).Seconds())
|
|
|
|
if status != nil && status.Err != nil {
|
|
errs = append(errs, fmt.Errorf("plugin %q finished with error: %v", pl.Name(), status.Err))
|
|
}
|
|
klog.V(1).InfoS("Total number of pods evicted", "extension point", "Balance", "evictedPods", d.podEvictor.TotalEvicted()-evicted)
|
|
}
|
|
|
|
aggrErr := errors.NewAggregate(errs)
|
|
if aggrErr == nil {
|
|
return &framework.Status{}
|
|
}
|
|
|
|
return &framework.Status{
|
|
Err: fmt.Errorf("%v", aggrErr.Error()),
|
|
}
|
|
}
|