Merge pull request #722 from Dentrax/feature/leaderelection

feat(leaderelection): impl leader election for HA Deployment

pkg/apis/componentconfig/types.go

@@ -53,6 +53,9 @@ type DeschedulerConfiguration struct {
 	// IgnorePVCPods sets whether PVC pods should be allowed to be evicted
 	IgnorePVCPods bool
 
+	// LeaderElection starts Deployment using leader election loop
+	LeaderElection componentbaseconfig.LeaderElectionConfiguration
+
 	// Logging specifies the options of logging.
 	// Refer [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) for more information.
 	Logging componentbaseconfig.LoggingConfiguration

pkg/apis/componentconfig/v1alpha1/types.go

@@ -54,6 +54,9 @@ type DeschedulerConfiguration struct {
 	// IgnorePVCPods sets whether PVC pods should be allowed to be evicted
 	IgnorePVCPods bool `json:"ignorePvcPods,omitempty"`
 
+	// LeaderElection starts Deployment using leader election loop
+	LeaderElection componentbaseconfig.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+
 	// Logging specifies the options of logging.
 	// Refer [Logs Options](https://github.com/kubernetes/component-base/blob/master/logs/options.go) for more information.
 	Logging componentbaseconfig.LoggingConfiguration `json:"logging,omitempty"`

pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go

@@ -58,6 +58,7 @@ func autoConvert_v1alpha1_DeschedulerConfiguration_To_componentconfig_Deschedule
 	out.MaxNoOfPodsToEvictPerNode = in.MaxNoOfPodsToEvictPerNode
 	out.EvictLocalStoragePods = in.EvictLocalStoragePods
 	out.IgnorePVCPods = in.IgnorePVCPods
+	out.LeaderElection = in.LeaderElection
 	out.Logging = in.Logging
 	return nil
 }
@@ -76,6 +77,7 @@ func autoConvert_componentconfig_DeschedulerConfiguration_To_v1alpha1_Deschedule
 	out.MaxNoOfPodsToEvictPerNode = in.MaxNoOfPodsToEvictPerNode
 	out.EvictLocalStoragePods = in.EvictLocalStoragePods
 	out.IgnorePVCPods = in.IgnorePVCPods
+	out.LeaderElection = in.LeaderElection
 	out.Logging = in.Logging
 	return nil
 }

pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go

@@ -29,6 +29,7 @@ import (
 func (in *DeschedulerConfiguration) DeepCopyInto(out *DeschedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
+	out.LeaderElection = in.LeaderElection
 	in.Logging.DeepCopyInto(&out.Logging)
 	return
 }

pkg/apis/componentconfig/zz_generated.deepcopy.go

@@ -29,6 +29,7 @@ import (
 func (in *DeschedulerConfiguration) DeepCopyInto(out *DeschedulerConfiguration) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
+	out.LeaderElection = in.LeaderElection
 	in.Logging.DeepCopyInto(&out.Logging)
 	return
 }

pkg/descheduler/descheduler.go

@@ -69,7 +69,22 @@ func Run(ctx context.Context, rs *options.DeschedulerServer) error {
 		return err
 	}
 
-	return RunDeschedulerStrategies(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion)
+	runFn := func() error {
+		return RunDeschedulerStrategies(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion)
+	}
+
+	if rs.LeaderElection.LeaderElect && rs.DeschedulingInterval.Seconds() == 0 {
+		return fmt.Errorf("leaderElection must be used with deschedulingInterval")
+	}
+
+	if rs.LeaderElection.LeaderElect && !rs.DryRun {
+		if err := NewLeaderElection(runFn, rsclient, &rs.LeaderElection, ctx); err != nil {
+			return fmt.Errorf("leaderElection: %w", err)
+		}
+		return nil
+	}
+
+	return runFn()
 }
 
 type strategyFunction func(ctx context.Context, client clientset.Interface, strategy api.DeschedulerStrategy, nodes []*v1.Node, podEvictor *evictions.PodEvictor, getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc)

pkg/descheduler/leaderelection.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package descheduler
+
+import (
+	"context"
+	"fmt"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	componentbaseconfig "k8s.io/component-base/config"
+	"k8s.io/klog/v2"
+	"os"
+)
+
+// NewLeaderElection starts the leader election code loop
+func NewLeaderElection(
+	run func() error,
+	client clientset.Interface,
+	LeaderElectionConfig *componentbaseconfig.LeaderElectionConfiguration,
+	ctx context.Context,
+) error {
+	var id string
+
+	if hostname, err := os.Hostname(); err != nil {
+		// on errors, make sure we're unique
+		id = string(uuid.NewUUID())
+	} else {
+		// add a uniquifier so that two processes on the same host don't accidentally both become active
+		id = hostname + "_" + string(uuid.NewUUID())
+	}
+
+	klog.V(3).Infof("Assigned unique lease holder id: %s", id)
+
+	if len(LeaderElectionConfig.ResourceNamespace) == 0 {
+		return fmt.Errorf("namespace may not be empty")
+	}
+
+	if len(LeaderElectionConfig.ResourceName) == 0 {
+		return fmt.Errorf("name may not be empty")
+	}
+
+	lock, err := resourcelock.New(
+		LeaderElectionConfig.ResourceLock,
+		LeaderElectionConfig.ResourceNamespace,
+		LeaderElectionConfig.ResourceName,
+		client.CoreV1(),
+		client.CoordinationV1(),
+		resourcelock.ResourceLockConfig{
+			Identity: id,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("unable to create leader election lock: %v", err)
+	}
+
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock:            lock,
+		ReleaseOnCancel: true,
+		LeaseDuration:   LeaderElectionConfig.LeaseDuration.Duration,
+		RenewDeadline:   LeaderElectionConfig.RenewDeadline.Duration,
+		RetryPeriod:     LeaderElectionConfig.RetryPeriod.Duration,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				klog.V(1).InfoS("Started leading")
+				err := run()
+				if err != nil {
+					klog.Error(err)
+				}
+			},
+			OnStoppedLeading: func() {
+				klog.V(1).InfoS("Leader lost")
+			},
+			OnNewLeader: func(identity string) {
+				// Just got the lock
+				if identity == id {
+					return
+				}
+				klog.V(1).Infof("New leader elected: %v", identity)
+			},
+		},
+	})
+	return nil
+}