DefaultEvictor: introduce no-eviction policy

NoEvictionPolicy dictates whether a no-eviction policy is prefered or mandatory.
Needs to be used with caution as this will give users ability to protect their pods
from eviction. Which might work against enfored policies. E.g. plugins evicting pods
violating security policies.

pkg/framework/plugins/defaultevictor/defaultevictor.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
 
+	evictionutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
 	nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
@@ -140,6 +141,10 @@ func (d *DefaultEvictor) Filter(pod *v1.Pod) bool {
 		return true
 	}
 
+	if d.args.NoEvictionPolicy == MandatoryNoEvictionPolicy && evictionutils.HaveNoEvictionAnnotation(pod) {
+		return false
+	}
+
 	if utils.IsMirrorPod(pod) {
 		checkErrs = append(checkErrs, fmt.Errorf("pod is a mirror pod"))
 	}

pkg/framework/plugins/defaultevictor/defaultevictor_test.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
 	"sigs.k8s.io/descheduler/pkg/api"
+	evictionutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
 	frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
@@ -51,6 +52,7 @@ type testCase struct {
 	minPodAge               *metav1.Duration
 	result                  bool
 	ignorePodsWithoutPDB    bool
+	noEvictionPolicy        NoEvictionPolicy
 }
 
 func TestDefaultEvictorPreEvictionFilter(t *testing.T) {
@@ -358,6 +360,29 @@ func TestDefaultEvictorFilter(t *testing.T) {
 				}),
 			},
 			result: true,
+		}, {
+			description: "Normal pod eviction with normal ownerRefs and " + evictionutils.SoftNoEvictionAnnotationKey + " annotation (preference)",
+			pods: []*v1.Pod{
+				test.BuildTestPod("p2", 400, 0, n1.Name, func(pod *v1.Pod) {
+					pod.Annotations = map[string]string{evictionutils.SoftNoEvictionAnnotationKey: ""}
+					pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList()
+				}),
+			},
+			evictLocalStoragePods:   false,
+			evictSystemCriticalPods: false,
+			result:                  true,
+		}, {
+			description: "Normal pod eviction with normal ownerRefs and " + evictionutils.SoftNoEvictionAnnotationKey + " annotation (mandatory)",
+			pods: []*v1.Pod{
+				test.BuildTestPod("p2", 400, 0, n1.Name, func(pod *v1.Pod) {
+					pod.Annotations = map[string]string{evictionutils.SoftNoEvictionAnnotationKey: ""}
+					pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList()
+				}),
+			},
+			evictLocalStoragePods:   false,
+			evictSystemCriticalPods: false,
+			noEvictionPolicy:        MandatoryNoEvictionPolicy,
+			result:                  false,
 		}, {
 			description: "Normal pod eviction with replicaSet ownerRefs",
 			pods: []*v1.Pod{
@@ -831,6 +856,7 @@ func initializePlugin(ctx context.Context, test testCase) (frameworktypes.Plugin
 		MinReplicas:          test.minReplicas,
 		MinPodAge:            test.minPodAge,
 		IgnorePodsWithoutPDB: test.ignorePodsWithoutPDB,
+		NoEvictionPolicy:     test.noEvictionPolicy,
 	}
 
 	evictorPlugin, err := New(

pkg/framework/plugins/defaultevictor/types.go

@@ -37,4 +37,23 @@ type DefaultEvictorArgs struct {
 	MinReplicas             uint                   `json:"minReplicas,omitempty"`
 	MinPodAge               *metav1.Duration       `json:"minPodAge,omitempty"`
 	IgnorePodsWithoutPDB    bool                   `json:"ignorePodsWithoutPDB,omitempty"`
+	NoEvictionPolicy        NoEvictionPolicy       `json:"noEvictionPolicy,omitempty"`
 }
+
+// NoEvictionPolicy dictates whether a no-eviction policy is preferred or mandatory.
+// Needs to be used with caution as this will give users ability to protect their pods
+// from eviction. Which might work against enfored policies. E.g. plugins evicting pods
+// violating security policies.
+type NoEvictionPolicy string
+
+const (
+	// PreferredNoEvictionPolicy interprets the no-eviction policy as a preference.
+	// Meaning the annotation will get ignored by the DefaultEvictor plugin.
+	// Yet, plugins may optionally sort their pods based on the annotation
+	// and focus on evicting pods that do not set the annotation.
+	PreferredNoEvictionPolicy NoEvictionPolicy = "Preferred"
+
+	// MandatoryNoEvictionPolicy interprets the no-eviction policy as mandatory.
+	// Every pod carying the annotation will get excluded from eviction.
+	MandatoryNoEvictionPolicy NoEvictionPolicy = "Mandatory"
+)

pkg/framework/plugins/defaultevictor/validation.go

@@ -25,12 +25,18 @@ func ValidateDefaultEvictorArgs(obj runtime.Object) error {
 	args := obj.(*DefaultEvictorArgs)
 
 	if args.PriorityThreshold != nil && args.PriorityThreshold.Value != nil && len(args.PriorityThreshold.Name) > 0 {
-		return fmt.Errorf("priority threshold misconfigured, only one of priorityThreshold fields can be set, got %v", args)
+		return fmt.Errorf("priority threshold misconfigured, only one of priorityThreshold fields can be set")
 	}
 
 	if args.MinReplicas == 1 {
 		klog.V(4).Info("DefaultEvictor minReplicas must be greater than 1 to check for min pods during eviction. This check will be ignored during eviction.")
 	}
 
+	if args.NoEvictionPolicy != "" {
+		if args.NoEvictionPolicy != PreferredNoEvictionPolicy && args.NoEvictionPolicy != MandatoryNoEvictionPolicy {
+			return fmt.Errorf("noEvictionPolicy accepts only %q values", []NoEvictionPolicy{PreferredNoEvictionPolicy, MandatoryNoEvictionPolicy})
+		}
+	}
+
 	return nil
 }

pkg/framework/plugins/defaultevictor/validation_test.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaultevictor
+
+import (
+	"fmt"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilptr "k8s.io/utils/ptr"
+	"sigs.k8s.io/descheduler/pkg/api"
+)
+
+func TestValidateDefaultEvictorArgs(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    *DefaultEvictorArgs
+		errInfo error
+	}{
+		{
+			name: "passing invalid priority",
+			args: &DefaultEvictorArgs{
+				PriorityThreshold: &api.PriorityThreshold{
+					Value: utilptr.To[int32](1),
+					Name:  "priority-name",
+				},
+			},
+			errInfo: fmt.Errorf("priority threshold misconfigured, only one of priorityThreshold fields can be set"),
+		}, {
+			name: "passing invalid no eviction policy",
+			args: &DefaultEvictorArgs{
+				NoEvictionPolicy: "invalid-no-eviction-policy",
+			},
+			errInfo: fmt.Errorf("noEvictionPolicy accepts only %q values", []NoEvictionPolicy{PreferredNoEvictionPolicy, MandatoryNoEvictionPolicy}),
+		},
+	}
+
+	for _, testCase := range tests {
+		t.Run(testCase.name, func(t *testing.T) {
+			validateErr := ValidateDefaultEvictorArgs(runtime.Object(testCase.args))
+			if validateErr == nil || testCase.errInfo == nil {
+				if validateErr != testCase.errInfo {
+					t.Errorf("expected validity of plugin config: %q but got %q instead", testCase.errInfo, validateErr)
+				}
+			} else if validateErr.Error() != testCase.errInfo.Error() {
+				t.Errorf("expected validity of plugin config: %q but got %q instead", testCase.errInfo, validateErr)
+			}
+		})
+	}
+}