Sok3t/descheduler
1
0
Fork 0
mirror of https://github.com/kubernetes-sigs/descheduler.git synced 2026-01-26 05:14:13 +01:00
Code Packages Releases Activity

metrics: enable loop duration and strategy duration metrics (#1041)

Browse Source 
* metrics: enable loop duration and strategy duration metrics

* metrics: enable loop duration and strategy duration metrics
This commit is contained in:
Harsha Narayana
2023-02-04 09:38:28 +05:30
committed by GitHub
parent 904b8835e1
commit 93a014ef5e
2 changed files with 53 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
metrics/metrics.go
View File

@@ -48,9 +48,29 @@ var (
},
)
DeschedulerLoopDuration = metrics.NewHistogramVec(
&metrics.HistogramOpts{
Subsystem: DeschedulerSubsystem,
Name: "descheduler_loop_duration_seconds",
Help: "Time taken to complete a full descheduling cycle",
StabilityLevel: metrics.ALPHA,
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500},
}, []string{})
DeschedulerStrategyDuration = metrics.NewHistogramVec(
&metrics.HistogramOpts{
Subsystem: DeschedulerSubsystem,
Name: "descheduler_strategy_duration_seconds",
Help: "Time taken to complete Each strategy of the descheduling operation",
StabilityLevel: metrics.ALPHA,
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25, 50, 100},
}, []string{"strategy", "profile"})
metricsList = []metrics.Registerable{
PodsEvicted,
buildInfo,
DeschedulerLoopDuration,
DeschedulerStrategyDuration,
}
)

49
pkg/descheduler/descheduler.go
View File

@@ -19,6 +19,7 @@ package descheduler
import (
"context"
"fmt"
"time"
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/klog/v2"
@@ -50,6 +51,16 @@ import (
"sigs.k8s.io/descheduler/pkg/utils"
)
type enabledDeschedulePluginEntry struct {
Plugin framework.DeschedulePlugin
Profile string
}
type enabledBalancePluginEntry struct {
Plugin framework.BalancePlugin
Profile string
}
func Run(ctx context.Context, rs *options.DeschedulerServer) error {
metrics.Register()
@@ -268,6 +279,8 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
defer eventBroadcaster.Shutdown()
wait.NonSlidingUntil(func() {
loopStartDuration := time.Now()
defer metrics.DeschedulerLoopDuration.With(map[string]string{}).Observe(time.Since(loopStartDuration).Seconds())
nodes, err := nodeutil.ReadyNodes(ctx, rs.Client, nodeLister, nodeSelector)
if err != nil {
klog.V(1).InfoS("Unable to get ready nodes", "err", err)
@@ -323,8 +336,8 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
eventRecorder,
)
var enabledDeschedulePlugins []framework.DeschedulePlugin
var enabledBalancePlugins []framework.BalancePlugin
var enabledDeschedulePlugins []enabledDeschedulePluginEntry
var enabledBalancePlugins []enabledBalancePluginEntry
// Build plugins
for _, profile := range deschedulerPolicy.Profiles {
@@ -374,7 +387,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
}
if pg != nil {
// pg can be of any of each type, or both
enabledDeschedulePlugins, enabledBalancePlugins = includeProfilePluginsByType(enabledDeschedulePlugins, enabledBalancePlugins, pg)
enabledDeschedulePlugins, enabledBalancePlugins = includeProfilePluginsByType(enabledDeschedulePlugins, enabledBalancePlugins, pg, profile.Name)
}
}
}
@@ -385,10 +398,12 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
// handle or function which the Evictor has access to. For migration/in-progress framework
// work, we are currently passing this via context. To be removed
// (See discussion thread https://github.com/kubernetes-sigs/descheduler/pull/885#discussion_r919962292)
childCtx := context.WithValue(ctx, "strategyName", pg.Name())
status := pg.Deschedule(childCtx, nodes)
strategyStart := time.Now()
childCtx := context.WithValue(ctx, "strategyName", pg.Plugin.Name())
status := pg.Plugin.Deschedule(childCtx, nodes)
metrics.DeschedulerStrategyDuration.With(map[string]string{"strategy": pg.Plugin.Name(), "profile": pg.Profile}).Observe(time.Since(strategyStart).Seconds())
if status != nil && status.Err != nil {
klog.ErrorS(status.Err, "plugin finished with error", "pluginName", pg.Name())
klog.ErrorS(status.Err, "plugin finished with error", "pluginName", pg.Plugin.Name())
}
}
@@ -397,10 +412,12 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
// handle or function which the Evictor has access to. For migration/in-progress framework
// work, we are currently passing this via context. To be removed
// (See discussion thread https://github.com/kubernetes-sigs/descheduler/pull/885#discussion_r919962292)
childCtx := context.WithValue(ctx, "strategyName", pg.Name())
status := pg.Balance(childCtx, nodes)
strategyStart := time.Now()
childCtx := context.WithValue(ctx, "strategyName", pg.Plugin.Name())
status := pg.Plugin.Balance(childCtx, nodes)
metrics.DeschedulerStrategyDuration.With(map[string]string{"strategy": pg.Plugin.Name(), "profile": pg.Profile}).Observe(time.Since(strategyStart).Seconds())
if status != nil && status.Err != nil {
klog.ErrorS(status.Err, "plugin finished with error", "pluginName", pg.Name())
klog.ErrorS(status.Err, "plugin finished with error", "pluginName", pg.Plugin.Name())
}
}
@@ -415,24 +432,24 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
return nil
}
func includeProfilePluginsByType(enabledDeschedulePlugins []framework.DeschedulePlugin, enabledBalancePlugins []framework.BalancePlugin, pg framework.Plugin) ([]framework.DeschedulePlugin, []framework.BalancePlugin) {
enabledDeschedulePlugins = includeDeschedule(enabledDeschedulePlugins, pg)
enabledBalancePlugins = includeBalance(enabledBalancePlugins, pg)
func includeProfilePluginsByType(enabledDeschedulePlugins []enabledDeschedulePluginEntry, enabledBalancePlugins []enabledBalancePluginEntry, pg framework.Plugin, profile string) ([]enabledDeschedulePluginEntry, []enabledBalancePluginEntry) {
enabledDeschedulePlugins = includeDeschedule(enabledDeschedulePlugins, pg, profile)
enabledBalancePlugins = includeBalance(enabledBalancePlugins, pg, profile)
return enabledDeschedulePlugins, enabledBalancePlugins
}
func includeDeschedule(enabledDeschedulePlugins []framework.DeschedulePlugin, pg framework.Plugin) []framework.DeschedulePlugin {
func includeDeschedule(enabledDeschedulePlugins []enabledDeschedulePluginEntry, pg framework.Plugin, profile string) []enabledDeschedulePluginEntry {
_, ok := pg.(framework.DeschedulePlugin)
if ok {
enabledDeschedulePlugins = append(enabledDeschedulePlugins, pg.(framework.DeschedulePlugin))
enabledDeschedulePlugins = append(enabledDeschedulePlugins, enabledDeschedulePluginEntry{Plugin: pg.(framework.DeschedulePlugin), Profile: profile})
}
return enabledDeschedulePlugins
}
func includeBalance(enabledBalancePlugins []framework.BalancePlugin, pg framework.Plugin) []framework.BalancePlugin {
func includeBalance(enabledBalancePlugins []enabledBalancePluginEntry, pg framework.Plugin, profile string) []enabledBalancePluginEntry {
_, ok := pg.(framework.BalancePlugin)
if ok {
enabledBalancePlugins = append(enabledBalancePlugins, pg.(framework.BalancePlugin))
enabledBalancePlugins = append(enabledBalancePlugins, enabledBalancePluginEntry{Plugin: pg.(framework.BalancePlugin), Profile: profile})
}
return enabledBalancePlugins
}