From 57a04aae9f9040c062e3c1996031376569d74536 Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Mon, 24 Feb 2025 15:41:18 +0100 Subject: [PATCH] chore: add descheduler plugin example This commit adds a sample plugin implementation as follow: This directory provides an example plugin for the Kubernetes Descheduler, demonstrating how to evict pods based on custom criteria. The plugin targets pods based on: * **Name Regex:** Pods matching a specified regular expression. * **Age:** Pods older than a defined duration. * **Namespace:** Pods within or outside a given list of namespaces (inclusion or exclusion). To incorporate this plugin into your Descheduler build, you must register it within the Descheduler's plugin registry. Follow these steps: 1. **Register the Plugin:** * Modify the `pkg/descheduler/setupplugins.go` file. * Add the following registration line to the end of the `RegisterDefaultPlugins()` function: ```go pluginregistry.Register( example.PluginName, example.New, &example.Example{}, &example.ExampleArgs{}, example.ValidateExampleArgs, example.SetDefaults_Example, registry, ) ``` 2. **Generate Code:** * If you modify the plugin's code, execute `make gen` before rebuilding the Descheduler. This ensures generated code is up-to-date. 3. **Rebuild the Descheduler:** * Build the descheduler with your changes. Configure the plugin's behavior using the Descheduler's policy configuration. Here's an example: ```yaml apiVersion: descheduler/v1alpha2 kind: DeschedulerPolicy profiles: - name: LifecycleAndUtilization plugins: deschedule: enabled: - Example pluginConfig: - name: Example args: regex: ^descheduler-test.*$ maxAge: 3m namespaces: include: - default ``` - `regex: ^descheduler-test.*$`: Evicts pods whose names match the regular expression `^descheduler-test.*$`. - `maxAge: 3m`: Evicts pods older than 3 minutes. - `namespaces.include: - default`: Evicts pods within the default namespace. This configuration will cause the plugin to evict pods that meet all three criteria: matching the `regex`, exceeding the `maxAge`, and residing in the specified namespace. --- pkg/framework/plugins/example/README.md | 90 ++++++++++ pkg/framework/plugins/example/defaults.go | 36 ++++ pkg/framework/plugins/example/docs.go | 16 ++ pkg/framework/plugins/example/example.go | 170 ++++++++++++++++++ pkg/framework/plugins/example/register.go | 31 ++++ pkg/framework/plugins/example/types.go | 45 +++++ pkg/framework/plugins/example/validation.go | 45 +++++ .../plugins/example/zz_generated.deepcopy.go | 57 ++++++ .../plugins/example/zz_generated.defaults.go | 33 ++++ 9 files changed, 523 insertions(+) create mode 100644 pkg/framework/plugins/example/README.md create mode 100644 pkg/framework/plugins/example/defaults.go create mode 100644 pkg/framework/plugins/example/docs.go create mode 100644 pkg/framework/plugins/example/example.go create mode 100644 pkg/framework/plugins/example/register.go create mode 100644 pkg/framework/plugins/example/types.go create mode 100644 pkg/framework/plugins/example/validation.go create mode 100644 pkg/framework/plugins/example/zz_generated.deepcopy.go create mode 100644 pkg/framework/plugins/example/zz_generated.defaults.go diff --git a/pkg/framework/plugins/example/README.md b/pkg/framework/plugins/example/README.md new file mode 100644 index 000000000..6313928ae --- /dev/null +++ b/pkg/framework/plugins/example/README.md @@ -0,0 +1,90 @@ +# Descheduler Plugin: Example Implementation + +This directory provides an example plugin for the Kubernetes Descheduler, +demonstrating how to evict pods based on custom criteria. The plugin targets +pods based on: + +* **Name Regex:** Pods matching a specified regular expression. +* **Age:** Pods older than a defined duration. +* **Namespace:** Pods within or outside a given list of namespaces (inclusion + or exclusion). + +## Building and Integrating the Plugin + +To incorporate this plugin into your Descheduler build, you must register it +within the Descheduler's plugin registry. Follow these steps: + +1. **Register the Plugin:** + * Modify the `pkg/descheduler/setupplugins.go` file. + * Add the following registration line to the end of the + `RegisterDefaultPlugins()` function: + + ```go + pluginregistry.Register( + example.PluginName, + example.New, + &example.Example{}, + &example.ExampleArgs{}, + example.ValidateExampleArgs, + example.SetDefaults_Example, + registry, + ) + ``` + +2. **Generate Code:** + * If you modify the plugin's code, execute `make gen` before rebuilding the + Descheduler. This ensures generated code is up-to-date. + +3. **Rebuild the Descheduler:** + * Build the descheduler with your changes. + +## Plugin Configuration + +Configure the plugin's behavior using the Descheduler's policy configuration. +Here's an example: + +```yaml +apiVersion: descheduler/v1alpha2 +kind: DeschedulerPolicy +profiles: +- name: LifecycleAndUtilization + plugins: + deschedule: + enabled: + - Example + pluginConfig: + - name: Example + args: + regex: ^descheduler-test.*$ + maxAge: 3m + namespaces: + include: + - default +``` + +## Explanation + +- `regex: ^descheduler-test.*$`: Evicts pods whose names match the regular + expression `^descheduler-test.*$`. +- `maxAge: 3m`: Evicts pods older than 3 minutes. +- `namespaces.include: - default`: Evicts pods within the default namespace. + +This configuration will cause the plugin to evict pods that meet all three +criteria: matching the `regex`, exceeding the `maxAge`, and residing in the +specified namespace. + +## Notes + +- This plugin is configured through the `ExampleArgs` struct, which defines the + plugin's parameters. +- Plugins must implement a function to validate and another to set the default + values for their `Args` struct. +- The fields in the `ExampleArgs` struct reflect directly into the + `DeschedulerPolicy` configuration. +- Plugins must comply with the `DeschedulePlugin` interface to be registered + with the Descheduler. +- The main functionality of the plugin is implemented in the `Deschedule()` + method, which is called by the Descheduler when the plugin is executed. +- A good amount of descheduling logic can be achieved by means of filters. +- Whenever a change in the Plugin's configuration is made the developer should + regenerate the code by running `make gen`. diff --git a/pkg/framework/plugins/example/defaults.go b/pkg/framework/plugins/example/defaults.go new file mode 100644 index 000000000..13df58ae9 --- /dev/null +++ b/pkg/framework/plugins/example/defaults.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +func addDefaultingFuncs(scheme *runtime.Scheme) error { + return RegisterDefaults(scheme) +} + +// SetDefaults_Example sets the default arguments for the Example plugin. On +// this case we set the default regex to match only empty strings (this should +// not ever match anything). The default maximum age for pods is set to 5 +// minutes. +func SetDefaults_Example(obj runtime.Object) { + args := obj.(*ExampleArgs) + if args.Regex == "" { + args.Regex = "^$" + } + if args.MaxAge == "" { + args.MaxAge = "5m" + } +} diff --git a/pkg/framework/plugins/example/docs.go b/pkg/framework/plugins/example/docs.go new file mode 100644 index 000000000..dbb28ff36 --- /dev/null +++ b/pkg/framework/plugins/example/docs.go @@ -0,0 +1,16 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:defaulter-gen=TypeMeta + +package example diff --git a/pkg/framework/plugins/example/example.go b/pkg/framework/plugins/example/example.go new file mode 100644 index 000000000..1ea5cd178 --- /dev/null +++ b/pkg/framework/plugins/example/example.go @@ -0,0 +1,170 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + "context" + "fmt" + "regexp" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + fwtypes "sigs.k8s.io/descheduler/pkg/framework/types" + + "sigs.k8s.io/descheduler/pkg/descheduler/evictions" + podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod" +) + +// PluginName is used when registering the plugin. You need to choose a unique +// name across all plugins. This name is used to identify the plugin config in +// the descheduler policy. +const PluginName = "Example" + +// We need to ensure that the plugin struct complies with the DeschedulePlugin +// interface. This prevent unexpected changes that may render this type +// incompatible. +var _ fwtypes.DeschedulePlugin = &Example{} + +// Example is our plugin (implementing the DeschedulePlugin interface). This +// plugin will evict pods that match a regex and are older than a certain age. +type Example struct { + handle fwtypes.Handle + args *ExampleArgs + podFilter podutil.FilterFunc +} + +// New builds a plugin instance from its arguments. Arguments are passed in as +// a runtime.Object. Handle is used by plugins to retrieve a kubernetes client +// set, evictor interface, shared informer factory and other instruments shared +// across different plugins. +func New(args runtime.Object, handle fwtypes.Handle) (fwtypes.Plugin, error) { + // make sure we are receiving the right argument type. + exampleArgs, ok := args.(*ExampleArgs) + if !ok { + return nil, fmt.Errorf("args must be of type ExampleArgs, got %T", args) + } + + // we can use the included and excluded namespaces to filter the pods we want + // to evict. + var includedNamespaces, excludedNamespaces sets.Set[string] + if exampleArgs.Namespaces != nil { + includedNamespaces = sets.New(exampleArgs.Namespaces.Include...) + excludedNamespaces = sets.New(exampleArgs.Namespaces.Exclude...) + } + + // here we create a pod filter that will return only pods that can be + // evicted (according to the evictor and inside the namespaces we want). + // NOTE: here we could also add a function to filter out by the regex and + // age but for sake of the example we are keeping it simple and filtering + // those out in the Deschedule() function. + podFilter, err := podutil.NewOptions(). + WithNamespaces(includedNamespaces). + WithoutNamespaces(excludedNamespaces). + WithFilter( + podutil.WrapFilterFuncs( + handle.Evictor().Filter, + handle.Evictor().PreEvictionFilter, + ), + ). + BuildFilterFunc() + if err != nil { + return nil, fmt.Errorf("error initializing pod filter function: %v", err) + } + + return &Example{ + handle: handle, + podFilter: podFilter, + args: exampleArgs, + }, nil +} + +// Name returns the plugin name. +func (d *Example) Name() string { + return PluginName +} + +// Deschedule is the function where most of the logic around eviction is laid +// down. Here we go through all pods in all nodes and evict the ones that match +// the regex and are older than the maximum age. This function receives a list +// of nodes we need to process. +func (d *Example) Deschedule(ctx context.Context, nodes []*v1.Node) *fwtypes.Status { + var podsToEvict []*v1.Pod + logger := klog.FromContext(ctx) + logger.Info("Example plugin starting descheduling") + + re, err := regexp.Compile(d.args.Regex) + if err != nil { + err = fmt.Errorf("fail to compile regex: %w", err) + return &fwtypes.Status{Err: err} + } + + duration, err := time.ParseDuration(d.args.MaxAge) + if err != nil { + err = fmt.Errorf("fail to parse max age: %w", err) + return &fwtypes.Status{Err: err} + } + + // here we create an auxiliar filter to remove all pods that don't + // match the provided regex or are still too young to be evicted. + // This filter will be used when we list all pods on a node. This + // filter here could have been part of the podFilter but we are + // keeping it separate for the sake of the example. + filter := func(pod *v1.Pod) bool { + if !re.MatchString(pod.Name) { + return false + } + deadline := pod.CreationTimestamp.Add(duration) + return time.Now().After(deadline) + } + + // go node by node getting all pods that we can evict. + for _, node := range nodes { + // ListAllPodsOnANode is a helper function that retrieves all + // pods filtering out the ones we can't evict. We merge the + // default filters with the one we created above. + pods, err := podutil.ListAllPodsOnANode( + node.Name, + d.handle.GetPodsAssignedToNodeFunc(), + podutil.WrapFilterFuncs(d.podFilter, filter), + ) + if err != nil { + err = fmt.Errorf("fail to list pods: %w", err) + return &fwtypes.Status{Err: err} + } + + // as we have already filtered out pods that don't match the + // regex or are too young we can simply add them all to the + // eviction list. + podsToEvict = append(podsToEvict, pods...) + } + + // evict all the pods. + for _, pod := range podsToEvict { + logger.Info("Example plugin evicting pod", "pod", klog.KObj(pod)) + opts := evictions.EvictOptions{StrategyName: PluginName} + if err := d.handle.Evictor().Evict(ctx, pod, opts); err != nil { + logger.Error(err, "unable to evict pod", "pod", klog.KObj(pod)) + } + } + + logger.Info("Example plugin finished descheduling") + return nil +} diff --git a/pkg/framework/plugins/example/register.go b/pkg/framework/plugins/example/register.go new file mode 100644 index 000000000..54fa207c7 --- /dev/null +++ b/pkg/framework/plugins/example/register.go @@ -0,0 +1,31 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder() + localSchemeBuilder = &SchemeBuilder + AddToScheme = localSchemeBuilder.AddToScheme +) + +func init() { + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(addDefaultingFuncs) +} diff --git a/pkg/framework/plugins/example/types.go b/pkg/framework/plugins/example/types.go new file mode 100644 index 000000000..fb4095f4d --- /dev/null +++ b/pkg/framework/plugins/example/types.go @@ -0,0 +1,45 @@ +/* +Copyright 2025 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/descheduler/pkg/api" +) + +// +k8s:deepcopy-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ExampleArgs holds a list of arguments used to configure the plugin. For this +// simple example we only care about a regex, a maximum age and possibly a list +// of namespaces to which we want to apply the descheduler. This plugin evicts +// pods that match a given regular expression and are older than the maximum +// allowed age. Most of the fields here were defined as strings so we can +// validate them somewhere else (show you a better implementation example). +type ExampleArgs struct { + metav1.TypeMeta `json:",inline"` + + // Regex is a regular expression we use to match against pod names. If + // the pod name matches the regex it will be evicted. This is expected + // to be a valid regular expression (according to go's regexp package). + Regex string `json:"regex"` + + // MaxAge is the maximum age a pod can have before it is considered for + // eviction. This is expected to be a valid time.Duration. + MaxAge string `json:"maxAge"` + + // Namespaces allows us to filter on which namespaces we want to apply + // the descheduler. + Namespaces *api.Namespaces `json:"namespaces,omitempty"` +} diff --git a/pkg/framework/plugins/example/validation.go b/pkg/framework/plugins/example/validation.go new file mode 100644 index 000000000..ebb58f232 --- /dev/null +++ b/pkg/framework/plugins/example/validation.go @@ -0,0 +1,45 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + "fmt" + "regexp" + "time" + + "k8s.io/apimachinery/pkg/runtime" +) + +// ValidateExampleArgs validates if the plugin arguments are correct (we have +// everything we need). On this case we only validate if we have a valid +// regular expression and maximum age. +func ValidateExampleArgs(obj runtime.Object) error { + args := obj.(*ExampleArgs) + if args.Regex == "" { + return fmt.Errorf("regex argument must be set") + } + + if _, err := regexp.Compile(args.Regex); err != nil { + return fmt.Errorf("invalid regex: %v", err) + } + + if _, err := time.ParseDuration(args.MaxAge); err != nil { + return fmt.Errorf("invalid max age: %v", err) + } + + return nil +} diff --git a/pkg/framework/plugins/example/zz_generated.deepcopy.go b/pkg/framework/plugins/example/zz_generated.deepcopy.go new file mode 100644 index 000000000..92eaf82dc --- /dev/null +++ b/pkg/framework/plugins/example/zz_generated.deepcopy.go @@ -0,0 +1,57 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package example + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" + api "sigs.k8s.io/descheduler/pkg/api" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExampleArgs) DeepCopyInto(out *ExampleArgs) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = new(api.Namespaces) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExampleArgs. +func (in *ExampleArgs) DeepCopy() *ExampleArgs { + if in == nil { + return nil + } + out := new(ExampleArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExampleArgs) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/framework/plugins/example/zz_generated.defaults.go b/pkg/framework/plugins/example/zz_generated.defaults.go new file mode 100644 index 000000000..16f62cd5b --- /dev/null +++ b/pkg/framework/plugins/example/zz_generated.defaults.go @@ -0,0 +1,33 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by defaulter-gen. DO NOT EDIT. + +package example + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + return nil +}