Merge pull request #1533 from ingvagabund/node-utilization-util-snapshot

[lownodeutilization]: Actual utilization: integration with Prometheus

README.md

@@ -129,14 +129,18 @@ These are top level keys in the Descheduler Policy that you can use to configure
 | `metricsProviders`                 | `[]object` | `nil`       | Enables various metrics providers like Kubernetes [Metrics Server](https://kubernetes-sigs.github.io/metrics-server/)      |
 | `evictionFailureEventNotification` | `bool`   | `false`       | Enables eviction failure event notification.                                                                               |
 | `gracePeriodSeconds`               | `int`    | `0`           | The duration in seconds before the object should be deleted. The value zero indicates delete immediately.                  |
+| `prometheus` |`object`| `nil` | Configures collection of Prometheus metrics for actual resource utilization |
+| `prometheus.url` |`string`| `nil` | Points to a Prometheus server url |
+| `prometheus.authToken` |`object`| `nil` | Sets Prometheus server authentication token. If not specified in cluster authentication token from the container's file system is read. |
+| `prometheus.authToken.secretReference` |`object`| `nil` | Read the authentication token from a kubernetes secret (the secret is expected to contain the token under `prometheusAuthToken` data key) |
+| `prometheus.authToken.secretReference.namespace` |`string`| `nil` | Authentication token kubernetes secret namespace (currently, the RBAC configuration permits retrieving secrets from the `kube-system` namespace. If the secret needs to be accessed from a different namespace, the existing RBAC rules must be explicitly extended. |
+| `prometheus.authToken.secretReference.name` |`string`| `nil` | Authentication token kubernetes secret name |
 
 The descheduler currently allows to configure a metric collection of Kubernetes Metrics through `metricsProviders` field.
-The previous way of setting `metricsCollector` field is deprecated. There is currently one source to configure:
-```
-metricsProviders:
-- source: KubernetesMetrics
-```
-The list can be extended with other metrics providers in the future.
+The previous way of setting `metricsCollector` field is deprecated. There are currently two sources to configure:
+- `KubernetesMetrics`: enables metrics collection from Kubernetes Metrics server
+- `Prometheus`: enables metrics collection from Prometheus server
+
 In general, each plugin can consume metrics from a different provider so multiple distinct providers can be configured in parallel.
 
 
@@ -174,9 +178,15 @@ maxNoOfPodsToEvictPerNode: 5000 # you don't need to set this, unlimited if not s
 maxNoOfPodsToEvictPerNamespace: 5000 # you don't need to set this, unlimited if not set
 maxNoOfPodsToEvictTotal: 5000 # you don't need to set this, unlimited if not set
 gracePeriodSeconds: 60 # you don't need to set this, 0 if not set
-# you don't need to set this, Kubernetes metrics are not collected if not set
+# you don't need to set this, metrics are not collected if not set
 metricsProviders:
-- source: KubernetesMetrics
+- source: Prometheus
+  prometheus:
+    url: http://prometheus-kube-prometheus-prometheus.prom.svc.cluster.local
+    authToken:
+      secretReference:
+        namespace: "kube-system"
+        name: "authtoken"
 profiles:
   - name: ProfileName
     pluginConfig:
@@ -303,6 +313,10 @@ like `kubectl top`) may differ from the calculated consumption, due to these com
 actual usage metrics. Metrics-based descheduling can be enabled by setting `metricsUtilization.metricsServer` field (deprecated)
 or `metricsUtilization.source` field to `KubernetesMetrics`.
 In order to have the plugin consume the metrics the metric provider needs to be configured as well.
+Alternatively, it is possible to create a prometheus client and configure a prometheus query to consume
+metrics outside of the kubernetes metrics server. The query is expected to return a vector of values for
+each node. The values are expected to be any real number within <0; 1> interval. During eviction only
+a single pod is evicted at most from each overutilized node. There's currently no support for evicting more.
 See `metricsProviders` field at [Top Level configuration](#top-level-configuration) for available options.
 
 **Parameters:**
@@ -318,6 +332,7 @@ See `metricsProviders` field at [Top Level configuration](#top-level-configurati
 |`metricsUtilization`|object|
 |`metricsUtilization.metricsServer` (deprecated)|bool|
 |`metricsUtilization.source`|string|
+|`metricsUtilization.prometheus.query`|string|
 
 
 **Example:**
@@ -338,8 +353,10 @@ profiles:
           "cpu" : 50
           "memory": 50
           "pods": 50
-        metricsUtilization:
-          source: KubernetesMetrics
+        # metricsUtilization:
+        #   source: Prometheus
+        #   prometheus:
+        #     query: instance:node_cpu:rate:sum
         evictionLimits:
           node: 5
     plugins:

cmd/descheduler/app/options/options.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
 	"github.com/spf13/pflag"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,6 +55,7 @@ type DeschedulerServer struct {
 	Client            clientset.Interface
 	EventClient       clientset.Interface
 	MetricsClient     metricsclient.Interface
+	PrometheusClient  promapi.Client
 	SecureServing     *apiserveroptions.SecureServingOptionsWithLoopback
 	SecureServingInfo *apiserver.SecureServingInfo
 	DisableMetrics    bool

go.mod

@@ -5,6 +5,8 @@ go 1.23.3
 require (
 	github.com/client9/misspell v0.3.4
 	github.com/google/go-cmp v0.6.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/common v0.55.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	go.opentelemetry.io/otel v1.28.0
@@ -71,17 +73,17 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mmarkdown/mmark v2.0.40+incompatible // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/openshift/custom-resource-status v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect

go.sum

@@ -177,6 +177,8 @@ github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
@@ -209,6 +211,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=

kubernetes/base/rbac.yaml

@@ -36,6 +36,15 @@ rules:
   resources: ["nodes", "pods"]
   verbs: ["get", "list"]
 ---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: descheduler-role
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "watch"]
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -54,3 +63,16 @@ subjects:
   - name: descheduler-sa
     kind: ServiceAccount
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: descheduler-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: descheduler-role
+subjects:
+  - name: descheduler-sa
+    kind: ServiceAccount
+    namespace: kube-system

pkg/api/types.go

@@ -115,6 +115,9 @@ const (
 	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
 	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
 	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+
+	// KubernetesMetrics enables metrics from a Prometheus metrics server.
+	PrometheusMetrics MetricsSource = "Prometheus"
 )
 
 // MetricsCollector configures collection of metrics about actual resource utilization
@@ -128,7 +131,32 @@ type MetricsCollector struct {
 type MetricsProvider struct {
 	// Source enables metrics from Kubernetes metrics server.
 	Source MetricsSource
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus *Prometheus
 }
 
 // ReferencedResourceList is an adaption of v1.ResourceList with resources as references
 type ReferencedResourceList = map[v1.ResourceName]*resource.Quantity
+
+type Prometheus struct {
+	URL string
+	// authToken used for authentication with the prometheus server.
+	// If not set the in cluster authentication token for the descheduler service
+	// account is read from the container's file system.
+	AuthToken *AuthToken
+}
+
+type AuthToken struct {
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference *SecretReference
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string
+	// name is the name of the secret.
+	Name string
+}

pkg/api/v1alpha2/types.go

@@ -90,6 +90,9 @@ const (
 	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
 	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
 	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+
+	// KubernetesMetrics enables metrics from a Prometheus metrics server.
+	PrometheusMetrics MetricsSource = "Prometheus"
 )
 
 // MetricsCollector configures collection of metrics about actual resource utilization
@@ -103,4 +106,29 @@ type MetricsCollector struct {
 type MetricsProvider struct {
 	// Source enables metrics from Kubernetes metrics server.
 	Source MetricsSource `json:"source,omitempty"`
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus *Prometheus `json:"prometheus,omitempty"`
+}
+
+type Prometheus struct {
+	URL string `json:"url,omitempty"`
+	// authToken used for authentication with the prometheus server.
+	// If not set the in cluster authentication token for the descheduler service
+	// account is read from the container's file system.
+	AuthToken *AuthToken `json:"authToken,omitempty"`
+}
+
+type AuthToken struct {
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference *SecretReference `json:"secretReference,omitempty"`
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string `json:"namespace,omitempty"`
+	// name is the name of the secret.
+	Name string `json:"name,omitempty"`
 }

pkg/api/v1alpha2/zz_generated.conversion.go

@@ -36,6 +36,16 @@ func init() {
 // RegisterConversions adds conversion functions to the given scheme.
 // Public to allow building arbitrary schemes.
 func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddGeneratedConversionFunc((*AuthToken)(nil), (*api.AuthToken)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_AuthToken_To_api_AuthToken(a.(*AuthToken), b.(*api.AuthToken), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*api.AuthToken)(nil), (*AuthToken)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_api_AuthToken_To_v1alpha2_AuthToken(a.(*api.AuthToken), b.(*AuthToken), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*DeschedulerProfile)(nil), (*api.DeschedulerProfile)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_DeschedulerProfile_To_api_DeschedulerProfile(a.(*DeschedulerProfile), b.(*api.DeschedulerProfile), scope)
 	}); err != nil {
@@ -91,6 +101,26 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*Prometheus)(nil), (*api.Prometheus)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_Prometheus_To_api_Prometheus(a.(*Prometheus), b.(*api.Prometheus), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*api.Prometheus)(nil), (*Prometheus)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_api_Prometheus_To_v1alpha2_Prometheus(a.(*api.Prometheus), b.(*Prometheus), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*SecretReference)(nil), (*api.SecretReference)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_SecretReference_To_api_SecretReference(a.(*SecretReference), b.(*api.SecretReference), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*api.SecretReference)(nil), (*SecretReference)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_api_SecretReference_To_v1alpha2_SecretReference(a.(*api.SecretReference), b.(*SecretReference), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddConversionFunc((*api.DeschedulerPolicy)(nil), (*DeschedulerPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_api_DeschedulerPolicy_To_v1alpha2_DeschedulerPolicy(a.(*api.DeschedulerPolicy), b.(*DeschedulerPolicy), scope)
 	}); err != nil {
@@ -109,6 +139,26 @@ func RegisterConversions(s *runtime.Scheme) error {
 	return nil
 }
 
+func autoConvert_v1alpha2_AuthToken_To_api_AuthToken(in *AuthToken, out *api.AuthToken, s conversion.Scope) error {
+	out.SecretReference = (*api.SecretReference)(unsafe.Pointer(in.SecretReference))
+	return nil
+}
+
+// Convert_v1alpha2_AuthToken_To_api_AuthToken is an autogenerated conversion function.
+func Convert_v1alpha2_AuthToken_To_api_AuthToken(in *AuthToken, out *api.AuthToken, s conversion.Scope) error {
+	return autoConvert_v1alpha2_AuthToken_To_api_AuthToken(in, out, s)
+}
+
+func autoConvert_api_AuthToken_To_v1alpha2_AuthToken(in *api.AuthToken, out *AuthToken, s conversion.Scope) error {
+	out.SecretReference = (*SecretReference)(unsafe.Pointer(in.SecretReference))
+	return nil
+}
+
+// Convert_api_AuthToken_To_v1alpha2_AuthToken is an autogenerated conversion function.
+func Convert_api_AuthToken_To_v1alpha2_AuthToken(in *api.AuthToken, out *AuthToken, s conversion.Scope) error {
+	return autoConvert_api_AuthToken_To_v1alpha2_AuthToken(in, out, s)
+}
+
 func autoConvert_v1alpha2_DeschedulerPolicy_To_api_DeschedulerPolicy(in *DeschedulerPolicy, out *api.DeschedulerPolicy, s conversion.Scope) error {
 	if in.Profiles != nil {
 		in, out := &in.Profiles, &out.Profiles
@@ -225,6 +275,7 @@ func Convert_api_MetricsCollector_To_v1alpha2_MetricsCollector(in *api.MetricsCo
 
 func autoConvert_v1alpha2_MetricsProvider_To_api_MetricsProvider(in *MetricsProvider, out *api.MetricsProvider, s conversion.Scope) error {
 	out.Source = api.MetricsSource(in.Source)
+	out.Prometheus = (*api.Prometheus)(unsafe.Pointer(in.Prometheus))
 	return nil
 }
 
@@ -235,6 +286,7 @@ func Convert_v1alpha2_MetricsProvider_To_api_MetricsProvider(in *MetricsProvider
 
 func autoConvert_api_MetricsProvider_To_v1alpha2_MetricsProvider(in *api.MetricsProvider, out *MetricsProvider, s conversion.Scope) error {
 	out.Source = MetricsSource(in.Source)
+	out.Prometheus = (*Prometheus)(unsafe.Pointer(in.Prometheus))
 	return nil
 }
 
@@ -339,3 +391,47 @@ func autoConvert_api_Plugins_To_v1alpha2_Plugins(in *api.Plugins, out *Plugins,
 func Convert_api_Plugins_To_v1alpha2_Plugins(in *api.Plugins, out *Plugins, s conversion.Scope) error {
 	return autoConvert_api_Plugins_To_v1alpha2_Plugins(in, out, s)
 }
+
+func autoConvert_v1alpha2_Prometheus_To_api_Prometheus(in *Prometheus, out *api.Prometheus, s conversion.Scope) error {
+	out.URL = in.URL
+	out.AuthToken = (*api.AuthToken)(unsafe.Pointer(in.AuthToken))
+	return nil
+}
+
+// Convert_v1alpha2_Prometheus_To_api_Prometheus is an autogenerated conversion function.
+func Convert_v1alpha2_Prometheus_To_api_Prometheus(in *Prometheus, out *api.Prometheus, s conversion.Scope) error {
+	return autoConvert_v1alpha2_Prometheus_To_api_Prometheus(in, out, s)
+}
+
+func autoConvert_api_Prometheus_To_v1alpha2_Prometheus(in *api.Prometheus, out *Prometheus, s conversion.Scope) error {
+	out.URL = in.URL
+	out.AuthToken = (*AuthToken)(unsafe.Pointer(in.AuthToken))
+	return nil
+}
+
+// Convert_api_Prometheus_To_v1alpha2_Prometheus is an autogenerated conversion function.
+func Convert_api_Prometheus_To_v1alpha2_Prometheus(in *api.Prometheus, out *Prometheus, s conversion.Scope) error {
+	return autoConvert_api_Prometheus_To_v1alpha2_Prometheus(in, out, s)
+}
+
+func autoConvert_v1alpha2_SecretReference_To_api_SecretReference(in *SecretReference, out *api.SecretReference, s conversion.Scope) error {
+	out.Namespace = in.Namespace
+	out.Name = in.Name
+	return nil
+}
+
+// Convert_v1alpha2_SecretReference_To_api_SecretReference is an autogenerated conversion function.
+func Convert_v1alpha2_SecretReference_To_api_SecretReference(in *SecretReference, out *api.SecretReference, s conversion.Scope) error {
+	return autoConvert_v1alpha2_SecretReference_To_api_SecretReference(in, out, s)
+}
+
+func autoConvert_api_SecretReference_To_v1alpha2_SecretReference(in *api.SecretReference, out *SecretReference, s conversion.Scope) error {
+	out.Namespace = in.Namespace
+	out.Name = in.Name
+	return nil
+}
+
+// Convert_api_SecretReference_To_v1alpha2_SecretReference is an autogenerated conversion function.
+func Convert_api_SecretReference_To_v1alpha2_SecretReference(in *api.SecretReference, out *SecretReference, s conversion.Scope) error {
+	return autoConvert_api_SecretReference_To_v1alpha2_SecretReference(in, out, s)
+}

pkg/api/v1alpha2/zz_generated.deepcopy.go

@@ -25,6 +25,27 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthToken) DeepCopyInto(out *AuthToken) {
+	*out = *in
+	if in.SecretReference != nil {
+		in, out := &in.SecretReference, &out.SecretReference
+		*out = new(SecretReference)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthToken.
+func (in *AuthToken) DeepCopy() *AuthToken {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthToken)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) {
 	*out = *in
@@ -69,7 +90,9 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) {
 	if in.MetricsProviders != nil {
 		in, out := &in.MetricsProviders, &out.MetricsProviders
 		*out = make([]MetricsProvider, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.GracePeriodSeconds != nil {
 		in, out := &in.GracePeriodSeconds, &out.GracePeriodSeconds
@@ -140,6 +163,11 @@ func (in *MetricsCollector) DeepCopy() *MetricsCollector {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MetricsProvider) DeepCopyInto(out *MetricsProvider) {
 	*out = *in
+	if in.Prometheus != nil {
+		in, out := &in.Prometheus, &out.Prometheus
+		*out = new(Prometheus)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -217,3 +245,40 @@ func (in *Plugins) DeepCopy() *Plugins {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Prometheus) DeepCopyInto(out *Prometheus) {
+	*out = *in
+	if in.AuthToken != nil {
+		in, out := &in.AuthToken, &out.AuthToken
+		*out = new(AuthToken)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Prometheus.
+func (in *Prometheus) DeepCopy() *Prometheus {
+	if in == nil {
+		return nil
+	}
+	out := new(Prometheus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/api/zz_generated.deepcopy.go

@@ -25,6 +25,27 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthToken) DeepCopyInto(out *AuthToken) {
+	*out = *in
+	if in.SecretReference != nil {
+		in, out := &in.SecretReference, &out.SecretReference
+		*out = new(SecretReference)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthToken.
+func (in *AuthToken) DeepCopy() *AuthToken {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthToken)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) {
 	*out = *in
@@ -69,7 +90,9 @@ func (in *DeschedulerPolicy) DeepCopyInto(out *DeschedulerPolicy) {
 	if in.MetricsProviders != nil {
 		in, out := &in.MetricsProviders, &out.MetricsProviders
 		*out = make([]MetricsProvider, len(*in))
-		copy(*out, *in)
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
 	}
 	if in.GracePeriodSeconds != nil {
 		in, out := &in.GracePeriodSeconds, &out.GracePeriodSeconds
@@ -161,6 +184,11 @@ func (in *MetricsCollector) DeepCopy() *MetricsCollector {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MetricsProvider) DeepCopyInto(out *MetricsProvider) {
 	*out = *in
+	if in.Prometheus != nil {
+		in, out := &in.Prometheus, &out.Prometheus
+		*out = new(Prometheus)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -288,6 +316,27 @@ func (in *PriorityThreshold) DeepCopy() *PriorityThreshold {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Prometheus) DeepCopyInto(out *Prometheus) {
+	*out = *in
+	if in.AuthToken != nil {
+		in, out := &in.AuthToken, &out.AuthToken
+		*out = new(AuthToken)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Prometheus.
+func (in *Prometheus) DeepCopy() *Prometheus {
+	if in == nil {
+		return nil
+	}
+	out := new(Prometheus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in ResourceThresholds) DeepCopyInto(out *ResourceThresholds) {
 	{
@@ -309,3 +358,19 @@ func (in ResourceThresholds) DeepCopy() ResourceThresholds {
 	in.DeepCopyInto(out)
 	return *out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecretReference) DeepCopyInto(out *SecretReference) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference.
+func (in *SecretReference) DeepCopy() *SecretReference {
+	if in == nil {
+		return nil
+	}
+	out := new(SecretReference)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/descheduler/client/client.go

@@ -17,17 +17,30 @@ limitations under the License.
 package client
 
 import (
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	promapi "github.com/prometheus/client_golang/api"
+	"github.com/prometheus/common/config"
 
 	// Ensure to load all auth plugins.
 	clientset "k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
 	componentbaseconfig "k8s.io/component-base/config"
 	metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
 )
 
+var K8sPodCAFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
 func createConfig(clientConnection componentbaseconfig.ClientConnectionConfiguration, userAgt string) (*rest.Config, error) {
 	var cfg *rest.Config
 	if len(clientConnection.Kubeconfig) != 0 {
@@ -94,3 +107,61 @@ func GetMasterFromKubeconfig(filename string) (string, error) {
 	}
 	return "", fmt.Errorf("failed to get master address from kubeconfig: cluster information not found")
 }
+
+func loadCAFile(filepath string) (*x509.CertPool, error) {
+	caCert, err := ioutil.ReadFile(filepath)
+	if err != nil {
+		return nil, err
+	}
+
+	caCertPool := x509.NewCertPool()
+	if ok := caCertPool.AppendCertsFromPEM(caCert); !ok {
+		return nil, fmt.Errorf("failed to append CA certificate to the pool")
+	}
+
+	return caCertPool, nil
+}
+
+func CreatePrometheusClient(prometheusURL, authToken string) (promapi.Client, *http.Transport, error) {
+	// Retrieve Pod CA cert
+	caCertPool, err := loadCAFile(K8sPodCAFilePath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error loading CA file: %v", err)
+	}
+
+	// Get Prometheus Host
+	u, err := url.Parse(prometheusURL)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Error parsing prometheus URL: %v", err)
+	}
+	t := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConns:        100,
+		IdleConnTimeout:     90 * time.Second,
+		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig: &tls.Config{
+			RootCAs:    caCertPool,
+			ServerName: u.Host,
+		},
+	}
+	roundTripper := transport.NewBearerAuthRoundTripper(
+		authToken,
+		t,
+	)
+
+	if authToken != "" {
+		client, err := promapi.NewClient(promapi.Config{
+			Address:      prometheusURL,
+			RoundTripper: config.NewAuthorizationCredentialsRoundTripper("Bearer", config.NewInlineSecret(authToken), roundTripper),
+		})
+		return client, t, err
+	}
+	client, err := promapi.NewClient(promapi.Config{
+		Address: prometheusURL,
+	})
+	return client, t, err
+}

pkg/descheduler/descheduler.go

@@ -20,9 +20,12 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"net/http"
 	"strconv"
 	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
+
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 
@@ -30,18 +33,24 @@ import (
 	policy "k8s.io/api/policy/v1"
 	policyv1 "k8s.io/api/policy/v1"
 	schedulingv1 "k8s.io/api/scheduling/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	fakeclientset "k8s.io/client-go/kubernetes/fake"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/rest"
 	core "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/workqueue"
 	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/klog/v2"
 
@@ -62,6 +71,11 @@ import (
 	"sigs.k8s.io/descheduler/pkg/version"
 )
 
+const (
+	prometheusAuthTokenSecretKey = "prometheusAuthToken"
+	workQueueKey                 = "key"
+)
+
 type eprunner func(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status
 
 type profileRunner struct {
@@ -70,15 +84,21 @@ type profileRunner struct {
 }
 
 type descheduler struct {
-	rs                     *options.DeschedulerServer
-	ir                     *informerResources
-	getPodsAssignedToNode  podutil.GetPodsAssignedToNodeFunc
-	sharedInformerFactory  informers.SharedInformerFactory
-	deschedulerPolicy      *api.DeschedulerPolicy
-	eventRecorder          events.EventRecorder
-	podEvictor             *evictions.PodEvictor
-	podEvictionReactionFnc func(*fakeclientset.Clientset) func(action core.Action) (bool, runtime.Object, error)
-	metricsCollector       *metricscollector.MetricsCollector
+	rs                                *options.DeschedulerServer
+	ir                                *informerResources
+	getPodsAssignedToNode             podutil.GetPodsAssignedToNodeFunc
+	sharedInformerFactory             informers.SharedInformerFactory
+	namespacedSecretsLister           corev1listers.SecretNamespaceLister
+	deschedulerPolicy                 *api.DeschedulerPolicy
+	eventRecorder                     events.EventRecorder
+	podEvictor                        *evictions.PodEvictor
+	podEvictionReactionFnc            func(*fakeclientset.Clientset) func(action core.Action) (bool, runtime.Object, error)
+	metricsCollector                  *metricscollector.MetricsCollector
+	prometheusClient                  promapi.Client
+	previousPrometheusClientTransport *http.Transport
+	queue                             workqueue.RateLimitingInterface
+	currentPrometheusAuthToken        string
+	metricsProviders                  map[api.MetricsSource]*api.MetricsProvider
 }
 
 type informerResources struct {
@@ -125,8 +145,15 @@ func (ir *informerResources) CopyTo(fakeClient *fakeclientset.Clientset, newFact
 	return nil
 }
 
-func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, eventRecorder events.EventRecorder, sharedInformerFactory informers.SharedInformerFactory,
-) (*descheduler, error) {
+func metricsProviderListToMap(providersList []api.MetricsProvider) map[api.MetricsSource]*api.MetricsProvider {
+	providersMap := make(map[api.MetricsSource]*api.MetricsProvider)
+	for _, provider := range providersList {
+		providersMap[provider.Source] = &provider
+	}
+	return providersMap
+}
+
+func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, eventRecorder events.EventRecorder, sharedInformerFactory, namespacedSharedInformerFactory informers.SharedInformerFactory) (*descheduler, error) {
 	podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
 
 	ir := newInformerResources(sharedInformerFactory)
@@ -165,20 +192,7 @@ func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedu
 		return nil, err
 	}
 
-	var metricsCollector *metricscollector.MetricsCollector
-	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
-		nodeSelector := labels.Everything()
-		if deschedulerPolicy.NodeSelector != nil {
-			sel, err := labels.Parse(*deschedulerPolicy.NodeSelector)
-			if err != nil {
-				return nil, err
-			}
-			nodeSelector = sel
-		}
-		metricsCollector = metricscollector.NewMetricsCollector(sharedInformerFactory.Core().V1().Nodes().Lister(), rs.MetricsClient, nodeSelector)
-	}
-
-	return &descheduler{
+	desch := &descheduler{
 		rs:                     rs,
 		ir:                     ir,
 		getPodsAssignedToNode:  getPodsAssignedToNode,
@@ -187,8 +201,148 @@ func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedu
 		eventRecorder:          eventRecorder,
 		podEvictor:             podEvictor,
 		podEvictionReactionFnc: podEvictionReactionFnc,
-		metricsCollector:       metricsCollector,
-	}, nil
+		prometheusClient:       rs.PrometheusClient,
+		queue:                  workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "descheduler"}),
+		metricsProviders:       metricsProviderListToMap(deschedulerPolicy.MetricsProviders),
+	}
+
+	if rs.MetricsClient != nil {
+		nodeSelector := labels.Everything()
+		if deschedulerPolicy.NodeSelector != nil {
+			sel, err := labels.Parse(*deschedulerPolicy.NodeSelector)
+			if err != nil {
+				return nil, err
+			}
+			nodeSelector = sel
+		}
+		desch.metricsCollector = metricscollector.NewMetricsCollector(sharedInformerFactory.Core().V1().Nodes().Lister(), rs.MetricsClient, nodeSelector)
+	}
+
+	prometheusProvider := desch.metricsProviders[api.PrometheusMetrics]
+	if prometheusProvider != nil && prometheusProvider.Prometheus != nil && prometheusProvider.Prometheus.AuthToken != nil {
+		authTokenSecret := prometheusProvider.Prometheus.AuthToken.SecretReference
+		if authTokenSecret == nil || authTokenSecret.Namespace == "" {
+			return nil, fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
+		}
+		if namespacedSharedInformerFactory == nil {
+			return nil, fmt.Errorf("namespacedSharedInformerFactory not configured")
+		}
+		namespacedSharedInformerFactory.Core().V1().Secrets().Informer().AddEventHandler(desch.eventHandler())
+		desch.namespacedSecretsLister = namespacedSharedInformerFactory.Core().V1().Secrets().Lister().Secrets(authTokenSecret.Namespace)
+	}
+
+	return desch, nil
+}
+
+func (d *descheduler) reconcileInClusterSAToken() error {
+	// Read the sa token and assume it has the sufficient permissions to authenticate
+	cfg, err := rest.InClusterConfig()
+	if err == nil {
+		if d.currentPrometheusAuthToken != cfg.BearerToken {
+			klog.V(2).Infof("Creating Prometheus client (with SA token)")
+			prometheusClient, transport, err := client.CreatePrometheusClient(d.metricsProviders[api.PrometheusMetrics].Prometheus.URL, cfg.BearerToken)
+			if err != nil {
+				return fmt.Errorf("unable to create a prometheus client: %v", err)
+			}
+			d.prometheusClient = prometheusClient
+			if d.previousPrometheusClientTransport != nil {
+				d.previousPrometheusClientTransport.CloseIdleConnections()
+			}
+			d.previousPrometheusClientTransport = transport
+			d.currentPrometheusAuthToken = cfg.BearerToken
+		}
+		return nil
+	}
+	if err == rest.ErrNotInCluster {
+		return nil
+	}
+	return fmt.Errorf("unexpected error when reading in cluster config: %v", err)
+}
+
+func (d *descheduler) runAuthenticationSecretReconciler(ctx context.Context) {
+	defer utilruntime.HandleCrash()
+	defer d.queue.ShutDown()
+
+	klog.Infof("Starting authentication secret reconciler")
+	defer klog.Infof("Shutting down authentication secret reconciler")
+
+	go wait.UntilWithContext(ctx, d.runAuthenticationSecretReconcilerWorker, time.Second)
+
+	<-ctx.Done()
+}
+
+func (d *descheduler) runAuthenticationSecretReconcilerWorker(ctx context.Context) {
+	for d.processNextWorkItem(ctx) {
+	}
+}
+
+func (d *descheduler) processNextWorkItem(ctx context.Context) bool {
+	dsKey, quit := d.queue.Get()
+	if quit {
+		return false
+	}
+	defer d.queue.Done(dsKey)
+
+	err := d.sync()
+	if err == nil {
+		d.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	d.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+func (d *descheduler) sync() error {
+	prometheusConfig := d.metricsProviders[api.PrometheusMetrics].Prometheus
+	if prometheusConfig == nil || prometheusConfig.AuthToken == nil || prometheusConfig.AuthToken.SecretReference == nil {
+		return fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
+	}
+	ns := prometheusConfig.AuthToken.SecretReference.Namespace
+	name := prometheusConfig.AuthToken.SecretReference.Name
+	secretObj, err := d.namespacedSecretsLister.Get(name)
+	if err != nil {
+		// clear the token if the secret is not found
+		if apierrors.IsNotFound(err) {
+			d.currentPrometheusAuthToken = ""
+			if d.previousPrometheusClientTransport != nil {
+				d.previousPrometheusClientTransport.CloseIdleConnections()
+			}
+			d.previousPrometheusClientTransport = nil
+			d.prometheusClient = nil
+		}
+		return fmt.Errorf("unable to get %v/%v secret", ns, name)
+	}
+	authToken := string(secretObj.Data[prometheusAuthTokenSecretKey])
+	if authToken == "" {
+		return fmt.Errorf("prometheus authentication token secret missing %q data or empty", prometheusAuthTokenSecretKey)
+	}
+	if d.currentPrometheusAuthToken == authToken {
+		return nil
+	}
+
+	klog.V(2).Infof("authentication secret token updated, recreating prometheus client")
+	prometheusClient, transport, err := client.CreatePrometheusClient(prometheusConfig.URL, authToken)
+	if err != nil {
+		return fmt.Errorf("unable to create a prometheus client: %v", err)
+	}
+	d.prometheusClient = prometheusClient
+	if d.previousPrometheusClientTransport != nil {
+		d.previousPrometheusClientTransport.CloseIdleConnections()
+	}
+	d.previousPrometheusClientTransport = transport
+	d.currentPrometheusAuthToken = authToken
+	return nil
+}
+
+func (d *descheduler) eventHandler() cache.ResourceEventHandler {
+	return cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { d.queue.Add(workQueueKey) },
+		UpdateFunc: func(old, new interface{}) { d.queue.Add(workQueueKey) },
+		DeleteFunc: func(obj interface{}) { d.queue.Add(workQueueKey) },
+	}
 }
 
 func (d *descheduler) runDeschedulerLoop(ctx context.Context, nodes []*v1.Node) error {
@@ -268,6 +422,7 @@ func (d *descheduler) runProfiles(ctx context.Context, client clientset.Interfac
 			frameworkprofile.WithPodEvictor(d.podEvictor),
 			frameworkprofile.WithGetPodsAssignedToNodeFnc(d.getPodsAssignedToNode),
 			frameworkprofile.WithMetricsCollector(d.metricsCollector),
+			frameworkprofile.WithPrometheusClient(d.prometheusClient),
 		)
 		if err != nil {
 			klog.ErrorS(err, "unable to create a profile", "profile", profile.Name)
@@ -332,7 +487,7 @@ func Run(ctx context.Context, rs *options.DeschedulerServer) error {
 		return err
 	}
 
-	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
+	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || metricsProviderListToMap(deschedulerPolicy.MetricsProviders)[api.KubernetesMetrics] != nil {
 		metricsClient, err := client.CreateMetricsClient(clientConnection, "descheduler")
 		if err != nil {
 			return err
@@ -415,6 +570,14 @@ func podEvictionReactionFnc(fakeClient *fakeclientset.Clientset) func(action cor
 	}
 }
 
+type tokenReconciliation int
+
+const (
+	noReconciliation tokenReconciliation = iota
+	inClusterReconciliation
+	secretReconciliation
+)
+
 func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string) error {
 	var span trace.Span
 	ctx, span = tracing.Tracer().Start(ctx, "RunDeschedulerStrategies")
@@ -436,7 +599,22 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 	eventBroadcaster, eventRecorder := utils.GetRecorderAndBroadcaster(ctx, eventClient)
 	defer eventBroadcaster.Shutdown()
 
-	descheduler, err := newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, sharedInformerFactory)
+	var namespacedSharedInformerFactory informers.SharedInformerFactory
+	metricProviderTokenReconciliation := noReconciliation
+
+	prometheusProvider := metricsProviderListToMap(deschedulerPolicy.MetricsProviders)[api.PrometheusMetrics]
+	if prometheusProvider != nil && prometheusProvider.Prometheus != nil && prometheusProvider.Prometheus.URL != "" {
+		if prometheusProvider.Prometheus.AuthToken != nil {
+			// Will get reconciled
+			namespacedSharedInformerFactory = informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields), informers.WithNamespace(prometheusProvider.Prometheus.AuthToken.SecretReference.Namespace))
+			metricProviderTokenReconciliation = secretReconciliation
+		} else {
+			// Use the sa token and assume it has the sufficient permissions to authenticate
+			metricProviderTokenReconciliation = inClusterReconciliation
+		}
+	}
+
+	descheduler, err := newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, sharedInformerFactory, namespacedSharedInformerFactory)
 	if err != nil {
 		span.AddEvent("Failed to create new descheduler", trace.WithAttributes(attribute.String("err", err.Error())))
 		return err
@@ -445,10 +623,17 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 	defer cancel()
 
 	sharedInformerFactory.Start(ctx.Done())
+	if metricProviderTokenReconciliation == secretReconciliation {
+		namespacedSharedInformerFactory.Start(ctx.Done())
+	}
+
 	sharedInformerFactory.WaitForCacheSync(ctx.Done())
 	descheduler.podEvictor.WaitForEventHandlersSync(ctx)
+	if metricProviderTokenReconciliation == secretReconciliation {
+		namespacedSharedInformerFactory.WaitForCacheSync(ctx.Done())
+	}
 
-	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
+	if descheduler.metricsCollector != nil {
 		go func() {
 			klog.V(2).Infof("Starting metrics collector")
 			descheduler.metricsCollector.Run(ctx)
@@ -462,7 +647,19 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 		}
 	}
 
+	if metricProviderTokenReconciliation == secretReconciliation {
+		go descheduler.runAuthenticationSecretReconciler(ctx)
+	}
+
 	wait.NonSlidingUntil(func() {
+		if metricProviderTokenReconciliation == inClusterReconciliation {
+			// Read the sa token and assume it has the sufficient permissions to authenticate
+			if err := descheduler.reconcileInClusterSAToken(); err != nil {
+				klog.ErrorS(err, "unable to reconcile an in cluster SA token")
+				return
+			}
+		}
+
 		// A next context is created here intentionally to avoid nesting the spans via context.
 		sCtx, sSpan := tracing.Tracer().Start(ctx, "NonSlidingUntil")
 		defer sSpan.End()

pkg/descheduler/descheduler_test.go

@@ -193,7 +193,7 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
 	sharedInformerFactory := informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields))
 	eventBroadcaster, eventRecorder := utils.GetRecorderAndBroadcaster(ctx, client)
 
-	descheduler, err := newDescheduler(ctx, rs, internalDeschedulerPolicy, "v1", eventRecorder, sharedInformerFactory)
+	descheduler, err := newDescheduler(ctx, rs, internalDeschedulerPolicy, "v1", eventRecorder, sharedInformerFactory, nil)
 	if err != nil {
 		eventBroadcaster.Shutdown()
 		t.Fatalf("Unable to create a descheduler instance: %v", err)

pkg/descheduler/policyconfig.go

@@ -19,6 +19,7 @@ package descheduler
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -154,11 +155,42 @@ func validateDeschedulerConfiguration(in api.DeschedulerPolicy, registry pluginr
 			}
 		}
 	}
-	if len(in.MetricsProviders) > 1 {
-		errorsInPolicy = append(errorsInPolicy, fmt.Errorf("only a single metrics provider can be set, got %v instead", len(in.MetricsProviders)))
+	providers := map[api.MetricsSource]api.MetricsProvider{}
+	for _, provider := range in.MetricsProviders {
+		if _, ok := providers[provider.Source]; ok {
+			errorsInPolicy = append(errorsInPolicy, fmt.Errorf("metric provider %q is already configured, each source can be configured only once", provider.Source))
+		} else {
+			providers[provider.Source] = provider
+		}
 	}
-	if len(in.MetricsProviders) > 0 && in.MetricsCollector != nil && in.MetricsCollector.Enabled {
+	if _, exists := providers[api.KubernetesMetrics]; exists && in.MetricsCollector != nil && in.MetricsCollector.Enabled {
 		errorsInPolicy = append(errorsInPolicy, fmt.Errorf("it is not allowed to combine metrics provider when metrics collector is enabled"))
 	}
+	if prometheusConfig, exists := providers[api.PrometheusMetrics]; exists {
+		if prometheusConfig.Prometheus == nil {
+			errorsInPolicy = append(errorsInPolicy, fmt.Errorf("prometheus configuration is required when prometheus source is enabled"))
+		} else {
+			if prometheusConfig.Prometheus.URL == "" {
+				errorsInPolicy = append(errorsInPolicy, fmt.Errorf("prometheus URL is required when prometheus is enabled"))
+			} else {
+				u, err := url.Parse(prometheusConfig.Prometheus.URL)
+				if err != nil {
+					errorsInPolicy = append(errorsInPolicy, fmt.Errorf("error parsing prometheus URL: %v", err))
+				} else if u.Scheme != "https" {
+					errorsInPolicy = append(errorsInPolicy, fmt.Errorf("prometheus URL's scheme is not https, got %q instead", u.Scheme))
+				}
+			}
+
+			if prometheusConfig.Prometheus.AuthToken != nil {
+				secretRef := prometheusConfig.Prometheus.AuthToken.SecretReference
+				if secretRef == nil {
+					errorsInPolicy = append(errorsInPolicy, fmt.Errorf("prometheus authToken secret is expected to be set when authToken field is"))
+				} else if secretRef.Name == "" || secretRef.Namespace == "" {
+					errorsInPolicy = append(errorsInPolicy, fmt.Errorf("prometheus authToken secret reference does not set both namespace and name"))
+				}
+			}
+		}
+	}
+
 	return utilerrors.NewAggregate(errorsInPolicy)
 }

pkg/descheduler/policyconfig_test.go

@@ -212,14 +212,14 @@ func TestValidateDeschedulerConfiguration(t *testing.T) {
 			result: fmt.Errorf("[in profile RemoveFailedPods: only one of Include/Exclude namespaces can be set, in profile RemovePodsViolatingTopologySpreadConstraint: only one of Include/Exclude namespaces can be set]"),
 		},
 		{
-			description: "Too many metrics providers error",
+			description: "Duplicit metrics providers error",
 			deschedulerPolicy: api.DeschedulerPolicy{
 				MetricsProviders: []api.MetricsProvider{
 					{Source: api.KubernetesMetrics},
 					{Source: api.KubernetesMetrics},
 				},
 			},
-			result: fmt.Errorf("only a single metrics provider can be set, got 2 instead"),
+			result: fmt.Errorf("metric provider \"KubernetesMetrics\" is already configured, each source can be configured only once"),
 		},
 		{
 			description: "Too many metrics providers error",
@@ -233,12 +233,145 @@ func TestValidateDeschedulerConfiguration(t *testing.T) {
 			},
 			result: fmt.Errorf("it is not allowed to combine metrics provider when metrics collector is enabled"),
 		},
+		{
+			description: "missing prometheus url error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source:     api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus URL is required when prometheus is enabled"),
+		},
+		{
+			description: "prometheus url is not valid error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "http://example.com:-80",
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("error parsing prometheus URL: parse \"http://example.com:-80\": invalid port \":-80\" after host"),
+		},
+		{
+			description: "prometheus url does not have https error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "http://example.com:80",
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus URL's scheme is not https, got \"http\" instead"),
+		},
+		{
+			description: "prometheus authtoken with no secret reference error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL:       "https://example.com:80",
+							AuthToken: &api.AuthToken{},
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus authToken secret is expected to be set when authToken field is"),
+		},
+		{
+			description: "prometheus authtoken with empty secret reference error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "https://example.com:80",
+							AuthToken: &api.AuthToken{
+								SecretReference: &api.SecretReference{},
+							},
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus authToken secret reference does not set both namespace and name"),
+		},
+		{
+			description: "prometheus authtoken missing secret reference namespace error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "https://example.com:80",
+							AuthToken: &api.AuthToken{
+								SecretReference: &api.SecretReference{
+									Name: "secretname",
+								},
+							},
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus authToken secret reference does not set both namespace and name"),
+		},
+		{
+			description: "prometheus authtoken missing secret reference name error",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "https://example.com:80",
+							AuthToken: &api.AuthToken{
+								SecretReference: &api.SecretReference{
+									Namespace: "secretnamespace",
+								},
+							},
+						},
+					},
+				},
+			},
+			result: fmt.Errorf("prometheus authToken secret reference does not set both namespace and name"),
+		},
+		{
+			description: "valid prometheus authtoken secret reference",
+			deschedulerPolicy: api.DeschedulerPolicy{
+				MetricsProviders: []api.MetricsProvider{
+					{
+						Source: api.PrometheusMetrics,
+						Prometheus: &api.Prometheus{
+							URL: "https://example.com:80",
+							AuthToken: &api.AuthToken{
+								SecretReference: &api.SecretReference{
+									Name:      "secretname",
+									Namespace: "secretnamespace",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			result := validateDeschedulerConfiguration(tc.deschedulerPolicy, pluginregistry.PluginRegistry)
-			if result.Error() != tc.result.Error() {
+			if result == nil && tc.result != nil || result != nil && tc.result == nil {
+				t.Errorf("test '%s' failed. expected \n'%s', got \n'%s'", tc.description, tc.result, result)
+			} else if result == nil && tc.result == nil {
+				return
+			} else if result.Error() != tc.result.Error() {
 				t.Errorf("test '%s' failed. expected \n'%s', got \n'%s'", tc.description, tc.result, result)
 			}
 		})

pkg/framework/fake/fake.go

@@ -11,6 +11,8 @@ import (
 	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
+
+	promapi "github.com/prometheus/client_golang/api"
 )
 
 type HandleImpl struct {
@@ -20,6 +22,7 @@ type HandleImpl struct {
 	EvictorFilterImpl             frameworktypes.EvictorPlugin
 	PodEvictorImpl                *evictions.PodEvictor
 	MetricsCollectorImpl          *metricscollector.MetricsCollector
+	PrometheusClientImpl          promapi.Client
 }
 
 var _ frameworktypes.Handle = &HandleImpl{}
@@ -28,6 +31,10 @@ func (hi *HandleImpl) ClientSet() clientset.Interface {
 	return hi.ClientsetImpl
 }
 
+func (hi *HandleImpl) PrometheusClient() promapi.Client {
+	return hi.PrometheusClientImpl
+}
+
 func (hi *HandleImpl) MetricsCollector() *metricscollector.MetricsCollector {
 	return hi.MetricsCollectorImpl
 }

pkg/framework/plugins/nodeutilization/highnodeutilization.go

@@ -95,7 +95,7 @@ func (h *HighNodeUtilization) Name() string {
 
 // Balance extension point implementation for the plugin
 func (h *HighNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status {
-	if err := h.usageClient.sync(nodes); err != nil {
+	if err := h.usageClient.sync(ctx, nodes); err != nil {
 		return &frameworktypes.Status{
 			Err: fmt.Errorf("error getting node usage: %v", err),
 		}

pkg/framework/plugins/nodeutilization/lownodeutilization.go

@@ -55,7 +55,23 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f
 		return nil, fmt.Errorf("want args to be of type LowNodeUtilizationArgs, got %T", args)
 	}
 
-	setDefaultForLNUThresholds(lowNodeUtilizationArgsArgs.Thresholds, lowNodeUtilizationArgsArgs.TargetThresholds, lowNodeUtilizationArgsArgs.UseDeviationThresholds)
+	metricsUtilization := lowNodeUtilizationArgsArgs.MetricsUtilization
+	if metricsUtilization != nil && metricsUtilization.Source == api.PrometheusMetrics {
+		if metricsUtilization.Prometheus != nil && metricsUtilization.Prometheus.Query != "" {
+			uResourceNames := getResourceNames(lowNodeUtilizationArgsArgs.Thresholds)
+			oResourceNames := getResourceNames(lowNodeUtilizationArgsArgs.TargetThresholds)
+			if len(uResourceNames) != 1 || uResourceNames[0] != MetricResource {
+				return nil, fmt.Errorf("thresholds are expected to specify a single instance of %q resource, got %v instead", MetricResource, uResourceNames)
+			}
+			if len(oResourceNames) != 1 || oResourceNames[0] != MetricResource {
+				return nil, fmt.Errorf("targetThresholds are expected to specify a single instance of %q resource, got %v instead", MetricResource, oResourceNames)
+			}
+		} else {
+			return nil, fmt.Errorf("prometheus query is missing")
+		}
+	} else {
+		setDefaultForLNUThresholds(lowNodeUtilizationArgsArgs.Thresholds, lowNodeUtilizationArgsArgs.TargetThresholds, lowNodeUtilizationArgsArgs.UseDeviationThresholds)
+	}
 
 	underutilizationCriteria := []interface{}{
 		"CPU", lowNodeUtilizationArgsArgs.Thresholds[v1.ResourceCPU],
@@ -90,11 +106,23 @@ func NewLowNodeUtilization(args runtime.Object, handle frameworktypes.Handle) (f
 
 	var usageClient usageClient
 	// MetricsServer is deprecated, removed once dropped
-	if lowNodeUtilizationArgsArgs.MetricsUtilization != nil && (lowNodeUtilizationArgsArgs.MetricsUtilization.MetricsServer || lowNodeUtilizationArgsArgs.MetricsUtilization.Source == api.KubernetesMetrics) {
-		if handle.MetricsCollector() == nil {
-			return nil, fmt.Errorf("metrics client not initialized")
+	if metricsUtilization != nil {
+		switch {
+		case metricsUtilization.MetricsServer, metricsUtilization.Source == api.KubernetesMetrics:
+			if handle.MetricsCollector() == nil {
+				return nil, fmt.Errorf("metrics client not initialized")
+			}
+			usageClient = newActualUsageClient(resourceNames, handle.GetPodsAssignedToNodeFunc(), handle.MetricsCollector())
+		case metricsUtilization.Source == api.PrometheusMetrics:
+			if handle.PrometheusClient() == nil {
+				return nil, fmt.Errorf("prometheus client not initialized")
+			}
+			usageClient = newPrometheusUsageClient(handle.GetPodsAssignedToNodeFunc(), handle.PrometheusClient(), metricsUtilization.Prometheus.Query)
+		case metricsUtilization.Source != "":
+			return nil, fmt.Errorf("unrecognized metrics source")
+		default:
+			return nil, fmt.Errorf("metrics source is empty")
 		}
-		usageClient = newActualUsageClient(resourceNames, handle.GetPodsAssignedToNodeFunc(), handle.MetricsCollector())
 	} else {
 		usageClient = newRequestedUsageClient(resourceNames, handle.GetPodsAssignedToNodeFunc())
 	}
@@ -117,7 +145,7 @@ func (l *LowNodeUtilization) Name() string {
 
 // Balance extension point implementation for the plugin
 func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status {
-	if err := l.usageClient.sync(nodes); err != nil {
+	if err := l.usageClient.sync(ctx, nodes); err != nil {
 		return &frameworktypes.Status{
 			Err: fmt.Errorf("error getting node usage: %v", err),
 		}
@@ -126,12 +154,20 @@ func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *fra
 	nodesMap, nodesUsageMap, podListMap := getNodeUsageSnapshot(nodes, l.usageClient)
 	var nodeThresholdsMap map[string][]api.ResourceThresholds
 	if l.args.UseDeviationThresholds {
-		nodeThresholdsMap = getNodeThresholdsFromAverageNodeUsage(nodes, l.usageClient, l.args.Thresholds, l.args.TargetThresholds)
+		thresholds, average := getNodeThresholdsFromAverageNodeUsage(nodes, l.usageClient, l.args.Thresholds, l.args.TargetThresholds)
+		klog.InfoS("Average utilization through all nodes", "utilization", average)
+		// All nodes are expected to have the same thresholds
+		for nodeName := range thresholds {
+			klog.InfoS("Underutilization threshold based on average utilization", "threshold", thresholds[nodeName][0])
+			klog.InfoS("Overutilization threshold based on average utilization", "threshold", thresholds[nodeName][1])
+			break
+		}
+		nodeThresholdsMap = thresholds
 	} else {
 		nodeThresholdsMap = getStaticNodeThresholds(nodes, l.args.Thresholds, l.args.TargetThresholds)
 	}
-	nodesUsageAsNodeThresholdsMap := nodeUsageToResourceThresholds(nodesUsageMap, nodesMap)
 
+	nodesUsageAsNodeThresholdsMap := nodeUsageToResourceThresholds(nodesUsageMap, nodesMap)
 	nodeGroups := classifyNodeUsage(
 		nodesUsageAsNodeThresholdsMap,
 		nodeThresholdsMap,

pkg/framework/plugins/nodeutilization/lownodeutilization_test.go

@@ -41,6 +41,8 @@ import (
 	frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
 	"sigs.k8s.io/descheduler/pkg/utils"
 	"sigs.k8s.io/descheduler/test"
+
+	"github.com/prometheus/common/model"
 )
 
 func TestLowNodeUtilization(t *testing.T) {
@@ -1257,9 +1259,9 @@ func TestLowNodeUtilization(t *testing.T) {
 				}
 				handle.MetricsCollectorImpl = collector
 
-				var metricsSource api.MetricsSource = ""
+				var metricsUtilization *MetricsUtilization
 				if metricsEnabled {
-					metricsSource = api.KubernetesMetrics
+					metricsUtilization = &MetricsUtilization{Source: api.KubernetesMetrics}
 				}
 
 				plugin, err := NewLowNodeUtilization(&LowNodeUtilizationArgs{
@@ -1268,9 +1270,7 @@ func TestLowNodeUtilization(t *testing.T) {
 					UseDeviationThresholds: tc.useDeviationThresholds,
 					EvictionLimits:         tc.evictionLimits,
 					EvictableNamespaces:    tc.evictableNamespaces,
-					MetricsUtilization: &MetricsUtilization{
-						Source: metricsSource,
-					},
+					MetricsUtilization:     metricsUtilization,
 				},
 					handle)
 				if err != nil {
@@ -1444,3 +1444,241 @@ func TestLowNodeUtilizationWithTaints(t *testing.T) {
 		})
 	}
 }
+
+func withLocalStorage(pod *v1.Pod) {
+	// A pod with local storage.
+	test.SetNormalOwnerRef(pod)
+	pod.Spec.Volumes = []v1.Volume{
+		{
+			Name: "sample",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{Path: "somePath"},
+				EmptyDir: &v1.EmptyDirVolumeSource{
+					SizeLimit: resource.NewQuantity(int64(10), resource.BinarySI),
+				},
+			},
+		},
+	}
+	// A Mirror Pod.
+	pod.Annotations = test.GetMirrorPodAnnotation()
+}
+
+func withCriticalPod(pod *v1.Pod) {
+	// A Critical Pod.
+	test.SetNormalOwnerRef(pod)
+	pod.Namespace = "kube-system"
+	priority := utils.SystemCriticalPriority
+	pod.Spec.Priority = &priority
+}
+
+func TestLowNodeUtilizationWithPrometheusMetrics(t *testing.T) {
+	n1NodeName := "n1"
+	n2NodeName := "n2"
+	n3NodeName := "n3"
+
+	testCases := []struct {
+		name                string
+		samples             model.Vector
+		nodes               []*v1.Node
+		pods                []*v1.Pod
+		expectedPodsEvicted uint
+		evictedPods         []string
+		args                *LowNodeUtilizationArgs
+	}{
+		{
+			name: "with instance:node_cpu:rate:sum query",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					MetricResource: 30,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					MetricResource: 50,
+				},
+				MetricsUtilization: &MetricsUtilization{
+					Source: api.PrometheusMetrics,
+					Prometheus: &Prometheus{
+						Query: "instance:node_cpu:rate:sum",
+					},
+				},
+			},
+			samples: model.Vector{
+				sample("instance:node_cpu:rate:sum", n1NodeName, 0.5695757575757561),
+				sample("instance:node_cpu:rate:sum", n2NodeName, 0.4245454545454522),
+				sample("instance:node_cpu:rate:sum", n3NodeName, 0.20381818181818104),
+			},
+			nodes: []*v1.Node{
+				test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
+				test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
+				test.BuildTestNode(n3NodeName, 4000, 3000, 10, nil),
+			},
+			pods: []*v1.Pod{
+				test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				// These won't be evicted.
+				test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
+				test.BuildTestPod("p7", 400, 0, n1NodeName, withLocalStorage),
+				test.BuildTestPod("p8", 400, 0, n1NodeName, withCriticalPod),
+				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
+			},
+			expectedPodsEvicted: 1,
+		},
+		{
+			name: "with instance:node_cpu:rate:sum query with more evictions",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					MetricResource: 30,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					MetricResource: 50,
+				},
+				EvictionLimits: &api.EvictionLimits{
+					Node: ptr.To[uint](3),
+				},
+				MetricsUtilization: &MetricsUtilization{
+					Source: api.PrometheusMetrics,
+					Prometheus: &Prometheus{
+						Query: "instance:node_cpu:rate:sum",
+					},
+				},
+			},
+			samples: model.Vector{
+				sample("instance:node_cpu:rate:sum", n1NodeName, 0.5695757575757561),
+				sample("instance:node_cpu:rate:sum", n2NodeName, 0.4245454545454522),
+				sample("instance:node_cpu:rate:sum", n3NodeName, 0.20381818181818104),
+			},
+			nodes: []*v1.Node{
+				test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
+				test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
+				test.BuildTestNode(n3NodeName, 4000, 3000, 10, nil),
+			},
+			pods: []*v1.Pod{
+				test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				// These won't be evicted.
+				test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
+				test.BuildTestPod("p7", 400, 0, n1NodeName, withLocalStorage),
+				test.BuildTestPod("p8", 400, 0, n1NodeName, withCriticalPod),
+				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
+			},
+			expectedPodsEvicted: 3,
+		},
+		{
+			name: "with instance:node_cpu:rate:sum query with deviation",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					MetricResource: 5,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					MetricResource: 5,
+				},
+				EvictionLimits: &api.EvictionLimits{
+					Node: ptr.To[uint](2),
+				},
+				UseDeviationThresholds: true,
+				MetricsUtilization: &MetricsUtilization{
+					Source: api.PrometheusMetrics,
+					Prometheus: &Prometheus{
+						Query: "instance:node_cpu:rate:sum",
+					},
+				},
+			},
+			samples: model.Vector{
+				sample("instance:node_cpu:rate:sum", n1NodeName, 0.5695757575757561),
+				sample("instance:node_cpu:rate:sum", n2NodeName, 0.4245454545454522),
+				sample("instance:node_cpu:rate:sum", n3NodeName, 0.20381818181818104),
+			},
+			nodes: []*v1.Node{
+				test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
+				test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
+				test.BuildTestNode(n3NodeName, 4000, 3000, 10, nil),
+			},
+			pods: []*v1.Pod{
+				test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef),
+				// These won't be evicted.
+				test.BuildTestPod("p6", 400, 0, n1NodeName, test.SetDSOwnerRef),
+				test.BuildTestPod("p7", 400, 0, n1NodeName, withLocalStorage),
+				test.BuildTestPod("p8", 400, 0, n1NodeName, withCriticalPod),
+				test.BuildTestPod("p9", 400, 0, n2NodeName, test.SetRSOwnerRef),
+			},
+			expectedPodsEvicted: 2,
+		},
+	}
+
+	for _, tc := range testCases {
+		testFnc := func(metricsEnabled bool, expectedPodsEvicted uint) func(t *testing.T) {
+			return func(t *testing.T) {
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+
+				var objs []runtime.Object
+				for _, node := range tc.nodes {
+					objs = append(objs, node)
+				}
+				for _, pod := range tc.pods {
+					objs = append(objs, pod)
+				}
+
+				fakeClient := fake.NewSimpleClientset(objs...)
+
+				podsForEviction := make(map[string]struct{})
+				for _, pod := range tc.evictedPods {
+					podsForEviction[pod] = struct{}{}
+				}
+
+				evictionFailed := false
+				if len(tc.evictedPods) > 0 {
+					fakeClient.Fake.AddReactor("create", "pods", func(action core.Action) (bool, runtime.Object, error) {
+						getAction := action.(core.CreateAction)
+						obj := getAction.GetObject()
+						if eviction, ok := obj.(*policy.Eviction); ok {
+							if _, exists := podsForEviction[eviction.Name]; exists {
+								return true, obj, nil
+							}
+							evictionFailed = true
+							return true, nil, fmt.Errorf("pod %q was unexpectedly evicted", eviction.Name)
+						}
+						return true, obj, nil
+					})
+				}
+
+				handle, podEvictor, err := frameworktesting.InitFrameworkHandle(ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil)
+				if err != nil {
+					t.Fatalf("Unable to initialize a framework handle: %v", err)
+				}
+
+				handle.PrometheusClientImpl = &fakePromClient{
+					result:   tc.samples,
+					dataType: model.ValVector,
+				}
+				plugin, err := NewLowNodeUtilization(tc.args, handle)
+				if err != nil {
+					t.Fatalf("Unable to initialize the plugin: %v", err)
+				}
+
+				status := plugin.(frameworktypes.BalancePlugin).Balance(ctx, tc.nodes)
+				if status != nil {
+					t.Fatalf("Balance.err: %v", status.Err)
+				}
+
+				podsEvicted := podEvictor.TotalEvicted()
+				if expectedPodsEvicted != podsEvicted {
+					t.Errorf("Expected %v pods to be evicted but %v got evicted", expectedPodsEvicted, podsEvicted)
+				}
+				if evictionFailed {
+					t.Errorf("Pod evictions failed unexpectedly")
+				}
+			}
+		}
+		t.Run(tc.name, testFnc(false, tc.expectedPodsEvicted))
+	}
+}

pkg/framework/plugins/nodeutilization/nodeutilization.go

@@ -57,6 +57,7 @@ import (
 // - thresholds: map[string][]api.ReferencedResourceList
 // - pod list: map[string][]*v1.Pod
 // Once the nodes are classified produce the original []NodeInfo so the code is not that much changed (postponing further refactoring once it is needed)
+const MetricResource = v1.ResourceName("MetricResource")
 
 // NodeUsage stores a node's info, pods on it, thresholds and its resource usage
 type NodeUsage struct {
@@ -94,20 +95,30 @@ func normalizePercentage(percent api.Percentage) api.Percentage {
 	return percent
 }
 
+func nodeCapacity(node *v1.Node, nodeUsage api.ReferencedResourceList) v1.ResourceList {
+	capacity := node.Status.Capacity
+	if len(node.Status.Allocatable) > 0 {
+		capacity = node.Status.Allocatable
+	}
+	// the usage captures the metrics resource
+	if _, ok := nodeUsage[MetricResource]; ok {
+		// Make ResourceMetrics 100% => 100 points
+		capacity[MetricResource] = *resource.NewQuantity(int64(100), resource.DecimalSI)
+	}
+	return capacity
+}
+
 func getNodeThresholdsFromAverageNodeUsage(
 	nodes []*v1.Node,
 	usageClient usageClient,
 	lowSpan, highSpan api.ResourceThresholds,
-) map[string][]api.ResourceThresholds {
+) (map[string][]api.ResourceThresholds, api.ResourceThresholds) {
 	total := api.ResourceThresholds{}
 	average := api.ResourceThresholds{}
 	numberOfNodes := len(nodes)
 	for _, node := range nodes {
 		usage := usageClient.nodeUtilization(node.Name)
-		nodeCapacity := node.Status.Capacity
-		if len(node.Status.Allocatable) > 0 {
-			nodeCapacity = node.Status.Allocatable
-		}
+		nodeCapacity := nodeCapacity(node, usage)
 		for resource, value := range usage {
 			nodeCapacityValue := nodeCapacity[resource]
 			if resource == v1.ResourceCPU {
@@ -138,7 +149,7 @@ func getNodeThresholdsFromAverageNodeUsage(
 			highThreshold,
 		}
 	}
-	return nodeThresholds
+	return nodeThresholds, average
 }
 
 func getStaticNodeThresholds(
@@ -216,10 +227,7 @@ func roundTo2Decimals(percentage float64) float64 {
 }
 
 func resourceUsagePercentages(nodeUsage api.ReferencedResourceList, node *v1.Node, round bool) api.ResourceThresholds {
-	nodeCapacity := node.Status.Capacity
-	if len(node.Status.Allocatable) > 0 {
-		nodeCapacity = node.Status.Allocatable
-	}
+	nodeCapacity := nodeCapacity(node, nodeUsage)
 
 	resourceUsagePercentage := api.ResourceThresholds{}
 	for resourceName, resourceUsage := range nodeUsage {
@@ -395,16 +403,29 @@ func evictPods(
 			if !preEvictionFilterWithOptions(pod) {
 				continue
 			}
+
+			// In case podUsage does not support resource counting (e.g. provided metric
+			// does not quantify pod resource utilization).
+			unconstrainedResourceEviction := false
 			podUsage, err := usageClient.podUsage(pod)
 			if err != nil {
-				klog.Errorf("unable to get pod usage for %v/%v: %v", pod.Namespace, pod.Name, err)
-				continue
+				if _, ok := err.(*notSupportedError); !ok {
+					klog.Errorf("unable to get pod usage for %v/%v: %v", pod.Namespace, pod.Name, err)
+					continue
+				}
+				unconstrainedResourceEviction = true
 			}
 			err = podEvictor.Evict(ctx, pod, evictOptions)
 			if err == nil {
+				if maxNoOfPodsToEvictPerNode == nil && unconstrainedResourceEviction {
+					klog.V(3).InfoS("Currently, only a single pod eviction is allowed")
+					break
+				}
 				evictionCounter++
 				klog.V(3).InfoS("Evicted pods", "pod", klog.KObj(pod))
-
+				if unconstrainedResourceEviction {
+					continue
+				}
 				for name := range totalAvailableUsage {
 					if name == v1.ResourcePods {
 						nodeInfo.usage[name].Sub(*resource.NewQuantity(1, resource.DecimalSI))

pkg/framework/plugins/nodeutilization/types.go

@@ -55,13 +55,24 @@ type HighNodeUtilizationArgs struct {
 }
 
 // MetricsUtilization allow to consume actual resource utilization from metrics
+// +k8s:deepcopy-gen=true
 type MetricsUtilization struct {
 	// metricsServer enables metrics from a kubernetes metrics server.
 	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
-	// Deprecated. Use MetricsSource instead.
+	// Deprecated. Use Source instead.
 	MetricsServer bool `json:"metricsServer,omitempty"`
 
 	// source enables the plugin to consume metrics from a metrics source.
 	// Currently only KubernetesMetrics available.
 	Source api.MetricsSource `json:"source,omitempty"`
+
+	// prometheus enables metrics collection through a prometheus query.
+	Prometheus *Prometheus `json:"prometheus,omitempty"`
+}
+
+type Prometheus struct {
+	// query returning a vector of samples, each sample labeled with `instance`
+	// corresponding to a node name with each sample value as a real number
+	// in <0; 1> interval.
+	Query string `json:"query,omitempty"`
 }

pkg/framework/plugins/nodeutilization/usageclients.go

@@ -19,7 +19,11 @@ package nodeutilization
 import (
 	"context"
 	"fmt"
+	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,11 +37,33 @@ import (
 	"sigs.k8s.io/descheduler/pkg/utils"
 )
 
+type UsageClientType int
+
+const (
+	requestedUsageClientType UsageClientType = iota
+	actualUsageClientType
+	prometheusUsageClientType
+)
+
+type notSupportedError struct {
+	usageClientType UsageClientType
+}
+
+func (e notSupportedError) Error() string {
+	return "maximum number of evicted pods per node reached"
+}
+
+func newNotSupportedError(usageClientType UsageClientType) *notSupportedError {
+	return &notSupportedError{
+		usageClientType: usageClientType,
+	}
+}
+
 type usageClient interface {
 	// Both low/high node utilization plugins are expected to invoke sync right
 	// after Balance method is invoked. There's no cache invalidation so each
 	// Balance is expected to get the latest data by invoking sync.
-	sync(nodes []*v1.Node) error
+	sync(ctx context.Context, nodes []*v1.Node) error
 	nodeUtilization(node string) api.ReferencedResourceList
 	pods(node string) []*v1.Pod
 	podUsage(pod *v1.Pod) (api.ReferencedResourceList, error)
@@ -79,7 +105,7 @@ func (s *requestedUsageClient) podUsage(pod *v1.Pod) (api.ReferencedResourceList
 	return usage, nil
 }
 
-func (s *requestedUsageClient) sync(nodes []*v1.Node) error {
+func (s *requestedUsageClient) sync(ctx context.Context, nodes []*v1.Node) error {
 	s._nodeUtilization = make(map[string]api.ReferencedResourceList)
 	s._pods = make(map[string][]*v1.Pod)
 
@@ -165,7 +191,7 @@ func (client *actualUsageClient) podUsage(pod *v1.Pod) (api.ReferencedResourceLi
 	return totalUsage, nil
 }
 
-func (client *actualUsageClient) sync(nodes []*v1.Node) error {
+func (client *actualUsageClient) sync(ctx context.Context, nodes []*v1.Node) error {
 	client._nodeUtilization = make(map[string]api.ReferencedResourceList)
 	client._pods = make(map[string][]*v1.Pod)
 
@@ -200,3 +226,95 @@ func (client *actualUsageClient) sync(nodes []*v1.Node) error {
 
 	return nil
 }
+
+type prometheusUsageClient struct {
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc
+	promClient            promapi.Client
+	promQuery             string
+
+	_pods            map[string][]*v1.Pod
+	_nodeUtilization map[string]map[v1.ResourceName]*resource.Quantity
+}
+
+var _ usageClient = &actualUsageClient{}
+
+func newPrometheusUsageClient(
+	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc,
+	promClient promapi.Client,
+	promQuery string,
+) *prometheusUsageClient {
+	return &prometheusUsageClient{
+		getPodsAssignedToNode: getPodsAssignedToNode,
+		promClient:            promClient,
+		promQuery:             promQuery,
+	}
+}
+
+func (client *prometheusUsageClient) nodeUtilization(node string) map[v1.ResourceName]*resource.Quantity {
+	return client._nodeUtilization[node]
+}
+
+func (client *prometheusUsageClient) pods(node string) []*v1.Pod {
+	return client._pods[node]
+}
+
+func (client *prometheusUsageClient) podUsage(pod *v1.Pod) (map[v1.ResourceName]*resource.Quantity, error) {
+	return nil, newNotSupportedError(prometheusUsageClientType)
+}
+
+func NodeUsageFromPrometheusMetrics(ctx context.Context, promClient promapi.Client, promQuery string) (map[string]map[v1.ResourceName]*resource.Quantity, error) {
+	results, warnings, err := promv1.NewAPI(promClient).Query(ctx, promQuery, time.Now())
+	if err != nil {
+		return nil, fmt.Errorf("unable to capture prometheus metrics: %v", err)
+	}
+	if len(warnings) > 0 {
+		klog.Infof("prometheus metrics warnings: %v", warnings)
+	}
+
+	if results.Type() != model.ValVector {
+		return nil, fmt.Errorf("expected query results to be of type %q, got %q instead", model.ValVector, results.Type())
+	}
+
+	nodeUsages := make(map[string]map[v1.ResourceName]*resource.Quantity)
+	for _, sample := range results.(model.Vector) {
+		nodeName, exists := sample.Metric["instance"]
+		if !exists {
+			return nil, fmt.Errorf("The collected metrics sample is missing 'instance' key")
+		}
+		if sample.Value < 0 || sample.Value > 1 {
+			return nil, fmt.Errorf("The collected metrics sample for %q has value %v outside of <0; 1> interval", string(nodeName), sample.Value)
+		}
+		nodeUsages[string(nodeName)] = map[v1.ResourceName]*resource.Quantity{
+			MetricResource: resource.NewQuantity(int64(sample.Value*100), resource.DecimalSI),
+		}
+	}
+
+	return nodeUsages, nil
+}
+
+func (client *prometheusUsageClient) sync(ctx context.Context, nodes []*v1.Node) error {
+	client._nodeUtilization = make(map[string]map[v1.ResourceName]*resource.Quantity)
+	client._pods = make(map[string][]*v1.Pod)
+
+	nodeUsages, err := NodeUsageFromPrometheusMetrics(ctx, client.promClient, client.promQuery)
+	if err != nil {
+		return err
+	}
+
+	for _, node := range nodes {
+		if _, exists := nodeUsages[node.Name]; !exists {
+			return fmt.Errorf("unable to find metric entry for %v", node.Name)
+		}
+		pods, err := podutil.ListPodsOnANode(node.Name, client.getPodsAssignedToNode, nil)
+		if err != nil {
+			klog.V(2).InfoS("Node will not be processed, error accessing its pods", "node", klog.KObj(node), "err", err)
+			return fmt.Errorf("error accessing %q node's pods: %v", node.Name, err)
+		}
+
+		// store the snapshot of pods from the same (or the closest) node utilization computation
+		client._pods[node.Name] = pods
+		client._nodeUtilization[node.Name] = nodeUsages[node.Name]
+	}
+
+	return nil
+}

pkg/framework/plugins/nodeutilization/usageclients_test.go

@@ -18,9 +18,14 @@ package nodeutilization
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/url"
 	"testing"
 
+	"github.com/prometheus/common/model"
+
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/labels"
@@ -58,9 +63,9 @@ func updateMetricsAndCheckNodeUtilization(
 	if err != nil {
 		t.Fatalf("failed to capture metrics: %v", err)
 	}
-	err = usageClient.sync(nodes)
+	err = usageClient.sync(ctx, nodes)
 	if err != nil {
-		t.Fatalf("failed to capture a snapshot: %v", err)
+		t.Fatalf("failed to sync a snapshot: %v", err)
 	}
 	nodeUtilization := usageClient.nodeUtilization(nodeName)
 	t.Logf("current node cpu usage: %v\n", nodeUtilization[v1.ResourceCPU].MilliValue())
@@ -137,3 +142,158 @@ func TestActualUsageClient(t *testing.T) {
 		metricsClientset, collector, usageClient, nodes, n2.Name, n2metrics,
 	)
 }
+
+type fakePromClient struct {
+	result   interface{}
+	dataType model.ValueType
+}
+
+type fakePayload struct {
+	Status string      `json:"status"`
+	Data   queryResult `json:"data"`
+}
+
+type queryResult struct {
+	Type   model.ValueType `json:"resultType"`
+	Result interface{}     `json:"result"`
+}
+
+func (client *fakePromClient) URL(ep string, args map[string]string) *url.URL {
+	return &url.URL{}
+}
+
+func (client *fakePromClient) Do(ctx context.Context, request *http.Request) (*http.Response, []byte, error) {
+	jsonData, err := json.Marshal(fakePayload{
+		Status: "success",
+		Data: queryResult{
+			Type:   client.dataType,
+			Result: client.result,
+		},
+	})
+
+	return &http.Response{StatusCode: 200}, jsonData, err
+}
+
+func sample(metricName, nodeName string, value float64) *model.Sample {
+	return &model.Sample{
+		Metric: model.Metric{
+			"__name__": model.LabelValue(metricName),
+			"instance": model.LabelValue(nodeName),
+		},
+		Value:     model.SampleValue(value),
+		Timestamp: 1728991761711,
+	}
+}
+
+func TestPrometheusUsageClient(t *testing.T) {
+	n1 := test.BuildTestNode("ip-10-0-17-165.ec2.internal", 2000, 3000, 10, nil)
+	n2 := test.BuildTestNode("ip-10-0-51-101.ec2.internal", 2000, 3000, 10, nil)
+	n3 := test.BuildTestNode("ip-10-0-94-25.ec2.internal", 2000, 3000, 10, nil)
+
+	nodes := []*v1.Node{n1, n2, n3}
+
+	p1 := test.BuildTestPod("p1", 400, 0, n1.Name, nil)
+	p21 := test.BuildTestPod("p21", 400, 0, n2.Name, nil)
+	p22 := test.BuildTestPod("p22", 400, 0, n2.Name, nil)
+	p3 := test.BuildTestPod("p3", 400, 0, n3.Name, nil)
+
+	tests := []struct {
+		name      string
+		result    interface{}
+		dataType  model.ValueType
+		nodeUsage map[string]int64
+		err       error
+	}{
+		{
+			name:     "valid data",
+			dataType: model.ValVector,
+			result: model.Vector{
+				sample("instance:node_cpu:rate:sum", "ip-10-0-51-101.ec2.internal", 0.20381818181818104),
+				sample("instance:node_cpu:rate:sum", "ip-10-0-17-165.ec2.internal", 0.4245454545454522),
+				sample("instance:node_cpu:rate:sum", "ip-10-0-94-25.ec2.internal", 0.5695757575757561),
+			},
+			nodeUsage: map[string]int64{
+				"ip-10-0-51-101.ec2.internal": 20,
+				"ip-10-0-17-165.ec2.internal": 42,
+				"ip-10-0-94-25.ec2.internal":  56,
+			},
+		},
+		{
+			name:     "invalid data missing instance label",
+			dataType: model.ValVector,
+			result: model.Vector{
+				&model.Sample{
+					Metric: model.Metric{
+						"__name__": model.LabelValue("instance:node_cpu:rate:sum"),
+					},
+					Value:     model.SampleValue(0.20381818181818104),
+					Timestamp: 1728991761711,
+				},
+			},
+			err: fmt.Errorf("The collected metrics sample is missing 'instance' key"),
+		},
+		{
+			name:     "invalid data value out of range",
+			dataType: model.ValVector,
+			result: model.Vector{
+				sample("instance:node_cpu:rate:sum", "ip-10-0-51-101.ec2.internal", 1.20381818181818104),
+			},
+			err: fmt.Errorf("The collected metrics sample for \"ip-10-0-51-101.ec2.internal\" has value 1.203818181818181 outside of <0; 1> interval"),
+		},
+		{
+			name:     "invalid data not a vector",
+			dataType: model.ValScalar,
+			result: model.Scalar{
+				Value:     model.SampleValue(0.20381818181818104),
+				Timestamp: 1728991761711,
+			},
+			err: fmt.Errorf("expected query results to be of type \"vector\", got \"scalar\" instead"),
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			pClient := &fakePromClient{
+				result:   tc.result,
+				dataType: tc.dataType,
+			}
+
+			clientset := fakeclientset.NewSimpleClientset(n1, n2, n3, p1, p21, p22, p3)
+
+			ctx := context.TODO()
+			sharedInformerFactory := informers.NewSharedInformerFactory(clientset, 0)
+			podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
+			podsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer)
+			if err != nil {
+				t.Fatalf("Build get pods assigned to node function error: %v", err)
+			}
+
+			sharedInformerFactory.Start(ctx.Done())
+			sharedInformerFactory.WaitForCacheSync(ctx.Done())
+
+			prometheusUsageClient := newPrometheusUsageClient(podsAssignedToNode, pClient, "instance:node_cpu:rate:sum")
+			err = prometheusUsageClient.sync(ctx, nodes)
+			if tc.err == nil {
+				if err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+			} else {
+				if err == nil {
+					t.Fatalf("unexpected %q error, got nil instead", tc.err)
+				} else if err.Error() != tc.err.Error() {
+					t.Fatalf("expected %q error, got %q instead", tc.err, err)
+				}
+				return
+			}
+
+			for _, node := range nodes {
+				nodeUtil := prometheusUsageClient.nodeUtilization(node.Name)
+				if nodeUtil[MetricResource].Value() != tc.nodeUsage[node.Name] {
+					t.Fatalf("expected %q node utilization to be %v, got %v instead", node.Name, tc.nodeUsage[node.Name], nodeUtil[MetricResource])
+				} else {
+					t.Logf("%v node utilization: %v", node.Name, nodeUtil[MetricResource])
+				}
+			}
+		})
+	}
+}

pkg/framework/plugins/nodeutilization/validation.go

@@ -44,6 +44,17 @@ func ValidateLowNodeUtilizationArgs(obj runtime.Object) error {
 	if err != nil {
 		return err
 	}
+	if args.MetricsUtilization != nil {
+		if args.MetricsUtilization.Source == api.KubernetesMetrics && args.MetricsUtilization.MetricsServer {
+			return fmt.Errorf("it is not allowed to set both %q source and metricsServer", api.KubernetesMetrics)
+		}
+		if args.MetricsUtilization.Source == api.KubernetesMetrics && args.MetricsUtilization.Prometheus != nil {
+			return fmt.Errorf("prometheus configuration is not allowed to set when source is set to %q", api.KubernetesMetrics)
+		}
+		if args.MetricsUtilization.Source == api.PrometheusMetrics && (args.MetricsUtilization.Prometheus == nil || args.MetricsUtilization.Prometheus.Query == "") {
+			return fmt.Errorf("prometheus query is required when metrics source is set to %q", api.PrometheusMetrics)
+		}
+	}
 	return nil
 }
 

pkg/framework/plugins/nodeutilization/validation_test.go

@@ -183,6 +183,65 @@ func TestValidateLowNodeUtilizationPluginConfig(t *testing.T) {
 			},
 			errInfo: nil,
 		},
+		{
+			name: "setting both kubernetes metrics source and metricsserver",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    20,
+					v1.ResourceMemory: 20,
+					extendedResource:  20,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    80,
+					v1.ResourceMemory: 80,
+					extendedResource:  80,
+				},
+				MetricsUtilization: &MetricsUtilization{
+					MetricsServer: true,
+					Source:        api.KubernetesMetrics,
+				},
+			},
+			errInfo: fmt.Errorf("it is not allowed to set both \"KubernetesMetrics\" source and metricsServer"),
+		},
+		{
+			name: "missing prometheus query",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    20,
+					v1.ResourceMemory: 20,
+					extendedResource:  20,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    80,
+					v1.ResourceMemory: 80,
+					extendedResource:  80,
+				},
+				MetricsUtilization: &MetricsUtilization{
+					Source: api.PrometheusMetrics,
+				},
+			},
+			errInfo: fmt.Errorf("prometheus query is required when metrics source is set to \"Prometheus\""),
+		},
+		{
+			name: "prometheus set when source set to kubernetes metrics",
+			args: &LowNodeUtilizationArgs{
+				Thresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    20,
+					v1.ResourceMemory: 20,
+					extendedResource:  20,
+				},
+				TargetThresholds: api.ResourceThresholds{
+					v1.ResourceCPU:    80,
+					v1.ResourceMemory: 80,
+					extendedResource:  80,
+				},
+				MetricsUtilization: &MetricsUtilization{
+					Source:     api.KubernetesMetrics,
+					Prometheus: &Prometheus{},
+				},
+			},
+			errInfo: fmt.Errorf("prometheus configuration is not allowed to set when source is set to \"KubernetesMetrics\""),
+		},
 	}
 
 	for _, testCase := range tests {
@@ -190,10 +249,10 @@ func TestValidateLowNodeUtilizationPluginConfig(t *testing.T) {
 			validateErr := ValidateLowNodeUtilizationArgs(runtime.Object(testCase.args))
 			if validateErr == nil || testCase.errInfo == nil {
 				if validateErr != testCase.errInfo {
-					t.Errorf("expected validity of plugin config: %v but got %v instead", testCase.errInfo, validateErr)
+					t.Errorf("expected validity of plugin config: %q but got %q instead", testCase.errInfo, validateErr)
 				}
 			} else if validateErr.Error() != testCase.errInfo.Error() {
-				t.Errorf("expected validity of plugin config: %v but got %v instead", testCase.errInfo, validateErr)
+				t.Errorf("expected validity of plugin config: %q but got %q instead", testCase.errInfo, validateErr)
 			}
 		})
 	}

pkg/framework/plugins/nodeutilization/zz_generated.deepcopy.go

@@ -84,7 +84,7 @@ func (in *LowNodeUtilizationArgs) DeepCopyInto(out *LowNodeUtilizationArgs) {
 	if in.MetricsUtilization != nil {
 		in, out := &in.MetricsUtilization, &out.MetricsUtilization
 		*out = new(MetricsUtilization)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.EvictableNamespaces != nil {
 		in, out := &in.EvictableNamespaces, &out.EvictableNamespaces
@@ -116,3 +116,24 @@ func (in *LowNodeUtilizationArgs) DeepCopyObject() runtime.Object {
 	}
 	return nil
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricsUtilization) DeepCopyInto(out *MetricsUtilization) {
+	*out = *in
+	if in.Prometheus != nil {
+		in, out := &in.Prometheus, &out.Prometheus
+		*out = new(Prometheus)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsUtilization.
+func (in *MetricsUtilization) DeepCopy() *MetricsUtilization {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricsUtilization)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/framework/profile/profile.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 
@@ -68,6 +69,7 @@ func (ei *evictorImpl) Evict(ctx context.Context, pod *v1.Pod, opts evictions.Ev
 // handleImpl implements the framework handle which gets passed to plugins
 type handleImpl struct {
 	clientSet                 clientset.Interface
+	prometheusClient          promapi.Client
 	metricsCollector          *metricscollector.MetricsCollector
 	getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
 	sharedInformerFactory     informers.SharedInformerFactory
@@ -81,6 +83,10 @@ func (hi *handleImpl) ClientSet() clientset.Interface {
 	return hi.clientSet
 }
 
+func (hi *handleImpl) PrometheusClient() promapi.Client {
+	return hi.prometheusClient
+}
+
 func (hi *handleImpl) MetricsCollector() *metricscollector.MetricsCollector {
 	return hi.metricsCollector
 }
@@ -131,6 +137,7 @@ type Option func(*handleImplOpts)
 
 type handleImplOpts struct {
 	clientSet                 clientset.Interface
+	prometheusClient          promapi.Client
 	sharedInformerFactory     informers.SharedInformerFactory
 	getPodsAssignedToNodeFunc podutil.GetPodsAssignedToNodeFunc
 	podEvictor                *evictions.PodEvictor
@@ -144,6 +151,13 @@ func WithClientSet(clientSet clientset.Interface) Option {
 	}
 }
 
+// WithPrometheusClient sets Prometheus client for the scheduling frameworkImpl.
+func WithPrometheusClient(prometheusClient promapi.Client) Option {
+	return func(o *handleImplOpts) {
+		o.prometheusClient = prometheusClient
+	}
+}
+
 func WithSharedInformerFactory(sharedInformerFactory informers.SharedInformerFactory) Option {
 	return func(o *handleImplOpts) {
 		o.sharedInformerFactory = sharedInformerFactory
@@ -267,6 +281,7 @@ func NewProfile(config api.DeschedulerProfile, reg pluginregistry.Registry, opts
 			podEvictor:  hOpts.podEvictor,
 		},
 		metricsCollector: hOpts.metricsCollector,
+		prometheusClient: hOpts.prometheusClient,
 	}
 
 	pluginNames := append(config.Plugins.Deschedule.Enabled, config.Plugins.Balance.Enabled...)

pkg/framework/types/types.go

@@ -26,6 +26,8 @@ import (
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
 	"sigs.k8s.io/descheduler/pkg/descheduler/metricscollector"
 	podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
+
+	promapi "github.com/prometheus/client_golang/api"
 )
 
 // Handle provides handles used by plugins to retrieve a kubernetes client set,
@@ -34,6 +36,7 @@ import (
 type Handle interface {
 	// ClientSet returns a kubernetes clientSet.
 	ClientSet() clientset.Interface
+	PrometheusClient() promapi.Client
 	Evictor() Evictor
 	GetPodsAssignedToNodeFunc() podutil.GetPodsAssignedToNodeFunc
 	SharedInformerFactory() informers.SharedInformerFactory

vendor/github.com/jpillora/backoff/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Jaime Pillora
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/jpillora/backoff/README.md

@@ -0,0 +1,119 @@
+# Backoff
+
+A simple exponential backoff counter in Go (Golang)
+
+[![GoDoc](https://godoc.org/github.com/jpillora/backoff?status.svg)](https://godoc.org/github.com/jpillora/backoff) [![Circle CI](https://circleci.com/gh/jpillora/backoff.svg?style=shield)](https://circleci.com/gh/jpillora/backoff)
+
+### Install
+
+```
+$ go get -v github.com/jpillora/backoff
+```
+
+### Usage
+
+Backoff is a `time.Duration` counter. It starts at `Min`. After every call to `Duration()` it is  multiplied by `Factor`. It is capped at `Max`. It returns to `Min` on every call to `Reset()`. `Jitter` adds randomness ([see below](#example-using-jitter)). Used in conjunction with the `time` package.
+
+---
+
+#### Simple example
+
+``` go
+
+b := &backoff.Backoff{
+	//These are the defaults
+	Min:    100 * time.Millisecond,
+	Max:    10 * time.Second,
+	Factor: 2,
+	Jitter: false,
+}
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+
+fmt.Printf("Reset!\n")
+b.Reset()
+
+fmt.Printf("%s\n", b.Duration())
+```
+
+```
+100ms
+200ms
+400ms
+Reset!
+100ms
+```
+
+---
+
+#### Example using `net` package
+
+``` go
+b := &backoff.Backoff{
+    Max:    5 * time.Minute,
+}
+
+for {
+	conn, err := net.Dial("tcp", "example.com:5309")
+	if err != nil {
+		d := b.Duration()
+		fmt.Printf("%s, reconnecting in %s", err, d)
+		time.Sleep(d)
+		continue
+	}
+	//connected
+	b.Reset()
+	conn.Write([]byte("hello world!"))
+	// ... Read ... Write ... etc
+	conn.Close()
+	//disconnected
+}
+
+```
+
+---
+
+#### Example using `Jitter`
+
+Enabling `Jitter` adds some randomization to the backoff durations. [See Amazon's writeup of performance gains using jitter](http://www.awsarchitectureblog.com/2015/03/backoff.html). Seeding is not necessary but doing so gives repeatable results.
+
+```go
+import "math/rand"
+
+b := &backoff.Backoff{
+	Jitter: true,
+}
+
+rand.Seed(42)
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+
+fmt.Printf("Reset!\n")
+b.Reset()
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+```
+
+```
+100ms
+106.600049ms
+281.228155ms
+Reset!
+100ms
+104.381845ms
+214.957989ms
+```
+
+#### Documentation
+
+https://godoc.org/github.com/jpillora/backoff
+
+#### Credits
+
+Forked from [some JavaScript](https://github.com/segmentio/backo) written by [@tj](https://github.com/tj)

vendor/github.com/jpillora/backoff/backoff.go

@@ -0,0 +1,100 @@
+// Package backoff provides an exponential-backoff implementation.
+package backoff
+
+import (
+	"math"
+	"math/rand"
+	"sync/atomic"
+	"time"
+)
+
+// Backoff is a time.Duration counter, starting at Min. After every call to
+// the Duration method the current timing is multiplied by Factor, but it
+// never exceeds Max.
+//
+// Backoff is not generally concurrent-safe, but the ForAttempt method can
+// be used concurrently.
+type Backoff struct {
+	attempt uint64
+	// Factor is the multiplying factor for each increment step
+	Factor float64
+	// Jitter eases contention by randomizing backoff steps
+	Jitter bool
+	// Min and Max are the minimum and maximum values of the counter
+	Min, Max time.Duration
+}
+
+// Duration returns the duration for the current attempt before incrementing
+// the attempt counter. See ForAttempt.
+func (b *Backoff) Duration() time.Duration {
+	d := b.ForAttempt(float64(atomic.AddUint64(&b.attempt, 1) - 1))
+	return d
+}
+
+const maxInt64 = float64(math.MaxInt64 - 512)
+
+// ForAttempt returns the duration for a specific attempt. This is useful if
+// you have a large number of independent Backoffs, but don't want use
+// unnecessary memory storing the Backoff parameters per Backoff. The first
+// attempt should be 0.
+//
+// ForAttempt is concurrent-safe.
+func (b *Backoff) ForAttempt(attempt float64) time.Duration {
+	// Zero-values are nonsensical, so we use
+	// them to apply defaults
+	min := b.Min
+	if min <= 0 {
+		min = 100 * time.Millisecond
+	}
+	max := b.Max
+	if max <= 0 {
+		max = 10 * time.Second
+	}
+	if min >= max {
+		// short-circuit
+		return max
+	}
+	factor := b.Factor
+	if factor <= 0 {
+		factor = 2
+	}
+	//calculate this duration
+	minf := float64(min)
+	durf := minf * math.Pow(factor, attempt)
+	if b.Jitter {
+		durf = rand.Float64()*(durf-minf) + minf
+	}
+	//ensure float64 wont overflow int64
+	if durf > maxInt64 {
+		return max
+	}
+	dur := time.Duration(durf)
+	//keep within bounds
+	if dur < min {
+		return min
+	}
+	if dur > max {
+		return max
+	}
+	return dur
+}
+
+// Reset restarts the current attempt counter at zero.
+func (b *Backoff) Reset() {
+	atomic.StoreUint64(&b.attempt, 0)
+}
+
+// Attempt returns the current attempt counter value.
+func (b *Backoff) Attempt() float64 {
+	return float64(atomic.LoadUint64(&b.attempt))
+}
+
+// Copy returns a backoff with equals constraints as the original
+func (b *Backoff) Copy() *Backoff {
+	return &Backoff{
+		Factor: b.Factor,
+		Jitter: b.Jitter,
+		Min:    b.Min,
+		Max:    b.Max,
+	}
+}

vendor/github.com/mwitkow/go-conntrack/.gitignore

@@ -0,0 +1,163 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+### Go template
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+

vendor/github.com/mwitkow/go-conntrack/.travis.yml

@@ -0,0 +1,17 @@
+sudo: false
+language: go
+go:
+- "1.8"
+- "1.9"
+- "1.10"
+- "1.11"
+- "1.12"
+
+install:
+- go get github.com/stretchr/testify
+- go get github.com/prometheus/client_golang/prometheus
+- go get golang.org/x/net/context
+- go get golang.org/x/net/trace
+
+script:
+- go test -v ./...

vendor/github.com/mwitkow/go-conntrack/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/mwitkow/go-conntrack/README.md

@@ -0,0 +1,88 @@
+# Go tracing and monitoring (Prometheus) for `net.Conn`
+
+[![Travis Build](https://travis-ci.org/mwitkow/go-conntrack.svg)](https://travis-ci.org/mwitkow/go-conntrack)
+[![Go Report Card](https://goreportcard.com/badge/github.com/mwitkow/go-conntrack)](http://goreportcard.com/report/mwitkow/go-conntrack)
+[![GoDoc](http://img.shields.io/badge/GoDoc-Reference-blue.svg)](https://godoc.org/github.com/mwitkow/go-conntrack)
+[![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE)
+
+[Prometheus](https://prometheus.io/) monitoring and [`x/net/trace`](https://godoc.org/golang.org/x/net/trace#EventLog) tracing wrappers `net.Conn`, both inbound (`net.Listener`) and outbound (`net.Dialer`).
+
+## Why?
+
+Go standard library does a great job of doing "the right" things with your connections: `http.Transport` pools outbound ones, and `http.Server` sets good *Keep Alive* defaults.
+However, it is still easy to get it wrong, see the excellent [*The complete guide to Go net/http timeouts*](https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/). 
+
+That's why you should be able to monitor (using Prometheus) how many connections your Go frontend servers have inbound, and how big are the connection pools to your backends. You should also be able to inspect your connection without `ssh` and `netstat`.
+
+![Events page with connections](https://raw.githubusercontent.com/mwitkow/go-conntrack/images/events.png)
+
+## How to use?
+
+All of these examples can be found in [`example/server.go`](example/server.go):
+
+### Conntrack Dialer for HTTP DefaultClient
+ 
+Most often people use the default `http.DefaultClient` that uses `http.DefaultTransport`. The easiest way to make sure all your outbound connections monitored and trace is:
+
+```go
+http.DefaultTransport.(*http.Transport).DialContext = conntrack.NewDialContextFunc(
+    conntrack.DialWithTracing(),
+    conntrack.DialWithDialer(&net.Dialer{
+        Timeout:   30 * time.Second,
+        KeepAlive: 30 * time.Second,
+    }),
+)
+```
+
+#### Dialer Name
+
+Tracked outbound connections are organised by *dialer name* (with `default` being default). The *dialer name* is used for monitoring (`dialer_name` label) and tracing (`net.ClientConn.<dialer_name>` family). 
+
+You can pass `conntrack.WithDialerName()` to `NewDialContextFunc` to set the name for the dialer. Moreover, you can set the *dialer name* per invocation of the dialer, by passing it in the `Context`. For example using the [`ctxhttp`](https://godoc.org/golang.org/x/net/context/ctxhttp) lib:
+
+```go
+callCtx := conntrack.DialNameToContext(parentCtx, "google")
+ctxhttp.Get(callCtx, http.DefaultClient, "https://www.google.com")
+```
+
+### Conntrack Listener for HTTP Server
+
+Tracked inbound connections are organised by *listener name* (with `default` being default). The *listener name* is used for monitoring (`listener_name` label) and tracing (`net.ServerConn.<listener_name>` family). For example, a simple `http.Server` can be instrumented like this:
+
+```go
+listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
+listener = conntrack.NewListener(listener, 
+    conntrack.TrackWithName("http"), 
+    conntrack.TrackWithTracing(),
+    conntrack.TrackWithTcpKeepAlive(5 * time.Minutes))
+httpServer.Serve(listener) 
+```
+
+Note, the `TrackWithTcpKeepAlive`. The default `http.ListenAndServe` adds a tcp keep alive wrapper to inbound TCP connections. `conntrack.NewListener` allows you to do that without another layer of wrapping.
+
+#### TLS server example
+
+The standard lobrary `http.ListenAndServerTLS` does a lot to bootstrap TLS connections, including supporting HTTP2 negotiation. Unfortunately, that is hard to do if you want to provide your own `net.Listener`. That's why this repo comes with `connhelpers` package, which takes care of configuring `tls.Config` for that use case. Here's an example of use:
+
+```go
+listener, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
+listener = conntrack.NewListener(listener, 
+    conntrack.TrackWithName("https"), 
+    conntrack.TrackWithTracing(),
+    conntrack.TrackWithTcpKeepAlive(5 * time.Minutes))
+tlsConfig, err := connhelpers.TlsConfigForServerCerts(*tlsCertFilePath, *tlsKeyFilePath)
+tlsConfig, err = connhelpers.TlsConfigWithHttp2Enabled(tlsConfig)
+tlsListener := tls.NewListener(listener, tlsConfig)
+httpServer.Serve(listener) 
+```
+
+# Status
+
+This code is used by Improbable's HTTP frontending and proxying stack for debuging and monitoring of established user connections.
+
+Additional tooling will be added if needed, and contributions are welcome.
+
+#License
+
+`go-conntrack` is released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
+

vendor/github.com/mwitkow/go-conntrack/dialer_reporter.go

@@ -0,0 +1,108 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package conntrack
+
+import (
+	"context"
+	"net"
+	"os"
+	"syscall"
+
+	prom "github.com/prometheus/client_golang/prometheus"
+)
+
+type failureReason string
+
+const (
+	failedResolution  = "resolution"
+	failedConnRefused = "refused"
+	failedTimeout     = "timeout"
+	failedUnknown     = "unknown"
+)
+
+var (
+	dialerAttemptedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "dialer_conn_attempted_total",
+			Help:      "Total number of connections attempted by the given dialer a given name.",
+		}, []string{"dialer_name"})
+
+	dialerConnEstablishedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "dialer_conn_established_total",
+			Help:      "Total number of connections successfully established by the given dialer a given name.",
+		}, []string{"dialer_name"})
+
+	dialerConnFailedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "dialer_conn_failed_total",
+			Help:      "Total number of connections failed to dial by the dialer a given name.",
+		}, []string{"dialer_name", "reason"})
+
+	dialerConnClosedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "dialer_conn_closed_total",
+			Help:      "Total number of connections closed which originated from the dialer of a given name.",
+		}, []string{"dialer_name"})
+)
+
+func init() {
+	prom.MustRegister(dialerAttemptedTotal)
+	prom.MustRegister(dialerConnEstablishedTotal)
+	prom.MustRegister(dialerConnFailedTotal)
+	prom.MustRegister(dialerConnClosedTotal)
+}
+
+// preRegisterDialerMetrics pre-populates Prometheus labels for the given dialer name, to avoid Prometheus missing labels issue.
+func PreRegisterDialerMetrics(dialerName string) {
+	dialerAttemptedTotal.WithLabelValues(dialerName)
+	dialerConnEstablishedTotal.WithLabelValues(dialerName)
+	for _, reason := range []failureReason{failedTimeout, failedResolution, failedConnRefused, failedUnknown} {
+		dialerConnFailedTotal.WithLabelValues(dialerName, string(reason))
+	}
+	dialerConnClosedTotal.WithLabelValues(dialerName)
+}
+
+func reportDialerConnAttempt(dialerName string) {
+	dialerAttemptedTotal.WithLabelValues(dialerName).Inc()
+}
+
+func reportDialerConnEstablished(dialerName string) {
+	dialerConnEstablishedTotal.WithLabelValues(dialerName).Inc()
+}
+
+func reportDialerConnClosed(dialerName string) {
+	dialerConnClosedTotal.WithLabelValues(dialerName).Inc()
+}
+
+func reportDialerConnFailed(dialerName string, err error) {
+	if netErr, ok := err.(*net.OpError); ok {
+		switch nestErr := netErr.Err.(type) {
+		case *net.DNSError:
+			dialerConnFailedTotal.WithLabelValues(dialerName, string(failedResolution)).Inc()
+			return
+		case *os.SyscallError:
+			if nestErr.Err == syscall.ECONNREFUSED {
+				dialerConnFailedTotal.WithLabelValues(dialerName, string(failedConnRefused)).Inc()
+			}
+			dialerConnFailedTotal.WithLabelValues(dialerName, string(failedUnknown)).Inc()
+			return
+		}
+		if netErr.Timeout() {
+			dialerConnFailedTotal.WithLabelValues(dialerName, string(failedTimeout)).Inc()
+		}
+	} else if err == context.Canceled || err == context.DeadlineExceeded {
+		dialerConnFailedTotal.WithLabelValues(dialerName, string(failedTimeout)).Inc()
+		return
+	}
+	dialerConnFailedTotal.WithLabelValues(dialerName, string(failedUnknown)).Inc()
+}

vendor/github.com/mwitkow/go-conntrack/dialer_wrapper.go

@@ -0,0 +1,166 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package conntrack
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	"golang.org/x/net/trace"
+)
+
+var (
+	dialerNameKey = "conntrackDialerKey"
+)
+
+type dialerOpts struct {
+	name                  string
+	monitoring            bool
+	tracing               bool
+	parentDialContextFunc dialerContextFunc
+}
+
+type dialerOpt func(*dialerOpts)
+
+type dialerContextFunc func(context.Context, string, string) (net.Conn, error)
+
+// DialWithName sets the name of the dialer for tracking and monitoring.
+// This is the name for the dialer (default is `default`), but for `NewDialContextFunc` can be overwritten from the
+// Context using `DialNameToContext`.
+func DialWithName(name string) dialerOpt {
+	return func(opts *dialerOpts) {
+		opts.name = name
+	}
+}
+
+// DialWithoutMonitoring turns *off* Prometheus monitoring for this dialer.
+func DialWithoutMonitoring() dialerOpt {
+	return func(opts *dialerOpts) {
+		opts.monitoring = false
+	}
+}
+
+// DialWithTracing turns *on* the /debug/events tracing of the dial calls.
+func DialWithTracing() dialerOpt {
+	return func(opts *dialerOpts) {
+		opts.tracing = true
+	}
+}
+
+// DialWithDialer allows you to override the `net.Dialer` instance used to actually conduct the dials.
+func DialWithDialer(parentDialer *net.Dialer) dialerOpt {
+	return DialWithDialContextFunc(parentDialer.DialContext)
+}
+
+// DialWithDialContextFunc allows you to override func gets used for the actual dialing. The default is `net.Dialer.DialContext`.
+func DialWithDialContextFunc(parentDialerFunc dialerContextFunc) dialerOpt {
+	return func(opts *dialerOpts) {
+		opts.parentDialContextFunc = parentDialerFunc
+	}
+}
+
+// DialNameFromContext returns the name of the dialer from the context of the DialContext func, if any.
+func DialNameFromContext(ctx context.Context) string {
+	val, ok := ctx.Value(dialerNameKey).(string)
+	if !ok {
+		return ""
+	}
+	return val
+}
+
+// DialNameToContext returns a context that will contain a dialer name override.
+func DialNameToContext(ctx context.Context, dialerName string) context.Context {
+	return context.WithValue(ctx, dialerNameKey, dialerName)
+}
+
+// NewDialContextFunc returns a `DialContext` function that tracks outbound connections.
+// The signature is compatible with `http.Tranport.DialContext` and is meant to be used there.
+func NewDialContextFunc(optFuncs ...dialerOpt) func(context.Context, string, string) (net.Conn, error) {
+	opts := &dialerOpts{name: defaultName, monitoring: true, parentDialContextFunc: (&net.Dialer{}).DialContext}
+	for _, f := range optFuncs {
+		f(opts)
+	}
+	if opts.monitoring {
+		PreRegisterDialerMetrics(opts.name)
+	}
+	return func(ctx context.Context, network string, addr string) (net.Conn, error) {
+		name := opts.name
+		if ctxName := DialNameFromContext(ctx); ctxName != "" {
+			name = ctxName
+		}
+		return dialClientConnTracker(ctx, network, addr, name, opts)
+	}
+}
+
+// NewDialFunc returns a `Dial` function that tracks outbound connections.
+// The signature is compatible with `http.Tranport.Dial` and is meant to be used there for Go < 1.7.
+func NewDialFunc(optFuncs ...dialerOpt) func(string, string) (net.Conn, error) {
+	dialContextFunc := NewDialContextFunc(optFuncs...)
+	return func(network string, addr string) (net.Conn, error) {
+		return dialContextFunc(context.TODO(), network, addr)
+	}
+}
+
+type clientConnTracker struct {
+	net.Conn
+	opts       *dialerOpts
+	dialerName string
+	event      trace.EventLog
+	mu         sync.Mutex
+}
+
+func dialClientConnTracker(ctx context.Context, network string, addr string, dialerName string, opts *dialerOpts) (net.Conn, error) {
+	var event trace.EventLog
+	if opts.tracing {
+		event = trace.NewEventLog(fmt.Sprintf("net.ClientConn.%s", dialerName), fmt.Sprintf("%v", addr))
+	}
+	if opts.monitoring {
+		reportDialerConnAttempt(dialerName)
+	}
+	conn, err := opts.parentDialContextFunc(ctx, network, addr)
+	if err != nil {
+		if event != nil {
+			event.Errorf("failed dialing: %v", err)
+			event.Finish()
+		}
+		if opts.monitoring {
+			reportDialerConnFailed(dialerName, err)
+		}
+		return nil, err
+	}
+	if event != nil {
+		event.Printf("established: %s -> %s", conn.LocalAddr(), conn.RemoteAddr())
+	}
+	if opts.monitoring {
+		reportDialerConnEstablished(dialerName)
+	}
+	tracker := &clientConnTracker{
+		Conn:       conn,
+		opts:       opts,
+		dialerName: dialerName,
+		event:      event,
+	}
+	return tracker, nil
+}
+
+func (ct *clientConnTracker) Close() error {
+	err := ct.Conn.Close()
+	ct.mu.Lock()
+	if ct.event != nil {
+		if err != nil {
+			ct.event.Errorf("failed closing: %v", err)
+		} else {
+			ct.event.Printf("closing")
+		}
+		ct.event.Finish()
+		ct.event = nil
+	}
+	ct.mu.Unlock()
+	if ct.opts.monitoring {
+		reportDialerConnClosed(ct.dialerName)
+	}
+	return err
+}

vendor/github.com/mwitkow/go-conntrack/listener_reporter.go

@@ -0,0 +1,43 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package conntrack
+
+import prom "github.com/prometheus/client_golang/prometheus"
+
+var (
+	listenerAcceptedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "listener_conn_accepted_total",
+			Help:      "Total number of connections opened to the listener of a given name.",
+		}, []string{"listener_name"})
+
+	listenerClosedTotal = prom.NewCounterVec(
+		prom.CounterOpts{
+			Namespace: "net",
+			Subsystem: "conntrack",
+			Name:      "listener_conn_closed_total",
+			Help:      "Total number of connections closed that were made to the listener of a given name.",
+		}, []string{"listener_name"})
+)
+
+func init() {
+	prom.MustRegister(listenerAcceptedTotal)
+	prom.MustRegister(listenerClosedTotal)
+}
+
+// preRegisterListener pre-populates Prometheus labels for the given listener name, to avoid Prometheus missing labels issue.
+func preRegisterListenerMetrics(listenerName string) {
+	listenerAcceptedTotal.WithLabelValues(listenerName)
+	listenerClosedTotal.WithLabelValues(listenerName)
+}
+
+func reportListenerConnAccepted(listenerName string) {
+	listenerAcceptedTotal.WithLabelValues(listenerName).Inc()
+}
+
+func reportListenerConnClosed(listenerName string) {
+	listenerClosedTotal.WithLabelValues(listenerName).Inc()
+}

vendor/github.com/mwitkow/go-conntrack/listener_wrapper.go

@@ -0,0 +1,158 @@
+// Copyright 2016 Michal Witkowski. All Rights Reserved.
+// See LICENSE for licensing terms.
+
+package conntrack
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/jpillora/backoff"
+	"golang.org/x/net/trace"
+)
+
+const (
+	defaultName = "default"
+)
+
+type listenerOpts struct {
+	name         string
+	monitoring   bool
+	tracing      bool
+	tcpKeepAlive time.Duration
+	retryBackoff *backoff.Backoff
+}
+
+type listenerOpt func(*listenerOpts)
+
+// TrackWithName sets the name of the Listener for use in tracking and monitoring.
+func TrackWithName(name string) listenerOpt {
+	return func(opts *listenerOpts) {
+		opts.name = name
+	}
+}
+
+// TrackWithoutMonitoring turns *off* Prometheus monitoring for this listener.
+func TrackWithoutMonitoring() listenerOpt {
+	return func(opts *listenerOpts) {
+		opts.monitoring = false
+	}
+}
+
+// TrackWithTracing turns *on* the /debug/events tracing of the live listener connections.
+func TrackWithTracing() listenerOpt {
+	return func(opts *listenerOpts) {
+		opts.tracing = true
+	}
+}
+
+// TrackWithRetries enables retrying of temporary Accept() errors, with the given backoff between attempts.
+// Concurrent accept calls that receive temporary errors have independent backoff scaling.
+func TrackWithRetries(b backoff.Backoff) listenerOpt {
+	return func(opts *listenerOpts) {
+		opts.retryBackoff = &b
+	}
+}
+
+// TrackWithTcpKeepAlive makes sure that any `net.TCPConn` that get accepted have a keep-alive.
+// This is useful for HTTP servers in order for, for example laptops, to not use up resources on the
+// server while they don't utilise their connection.
+// A value of 0 disables it.
+func TrackWithTcpKeepAlive(keepalive time.Duration) listenerOpt {
+	return func(opts *listenerOpts) {
+		opts.tcpKeepAlive = keepalive
+	}
+}
+
+type connTrackListener struct {
+	net.Listener
+	opts *listenerOpts
+}
+
+// NewListener returns the given listener wrapped in connection tracking listener.
+func NewListener(inner net.Listener, optFuncs ...listenerOpt) net.Listener {
+	opts := &listenerOpts{
+		name:       defaultName,
+		monitoring: true,
+		tracing:    false,
+	}
+	for _, f := range optFuncs {
+		f(opts)
+	}
+	if opts.monitoring {
+		preRegisterListenerMetrics(opts.name)
+	}
+	return &connTrackListener{
+		Listener: inner,
+		opts:     opts,
+	}
+}
+
+func (ct *connTrackListener) Accept() (net.Conn, error) {
+	// TODO(mwitkow): Add monitoring of failed accept.
+	var (
+		conn net.Conn
+		err  error
+	)
+	for attempt := 0; ; attempt++ {
+		conn, err = ct.Listener.Accept()
+		if err == nil || ct.opts.retryBackoff == nil {
+			break
+		}
+		if t, ok := err.(interface{ Temporary() bool }); !ok || !t.Temporary() {
+			break
+		}
+		time.Sleep(ct.opts.retryBackoff.ForAttempt(float64(attempt)))
+	}
+	if err != nil {
+		return nil, err
+	}
+	if tcpConn, ok := conn.(*net.TCPConn); ok && ct.opts.tcpKeepAlive > 0 {
+		tcpConn.SetKeepAlive(true)
+		tcpConn.SetKeepAlivePeriod(ct.opts.tcpKeepAlive)
+	}
+	return newServerConnTracker(conn, ct.opts), nil
+}
+
+type serverConnTracker struct {
+	net.Conn
+	opts  *listenerOpts
+	event trace.EventLog
+	mu    sync.Mutex
+}
+
+func newServerConnTracker(inner net.Conn, opts *listenerOpts) net.Conn {
+	tracker := &serverConnTracker{
+		Conn: inner,
+		opts: opts,
+	}
+	if opts.tracing {
+		tracker.event = trace.NewEventLog(fmt.Sprintf("net.ServerConn.%s", opts.name), fmt.Sprintf("%v", inner.RemoteAddr()))
+		tracker.event.Printf("accepted: %v -> %v", inner.RemoteAddr(), inner.LocalAddr())
+	}
+	if opts.monitoring {
+		reportListenerConnAccepted(opts.name)
+	}
+	return tracker
+}
+
+func (ct *serverConnTracker) Close() error {
+	err := ct.Conn.Close()
+	ct.mu.Lock()
+	if ct.event != nil {
+		if err != nil {
+			ct.event.Errorf("failed closing: %v", err)
+		} else {
+			ct.event.Printf("closing")
+		}
+		ct.event.Finish()
+		ct.event = nil
+	}
+	ct.mu.Unlock()
+	if ct.opts.monitoring {
+		reportListenerConnClosed(ct.opts.name)
+	}
+	return err
+}

vendor/github.com/prometheus/client_golang/api/client.go

@@ -0,0 +1,156 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package api provides clients for the HTTP APIs.
+package api
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+// DefaultRoundTripper is used if no RoundTripper is set in Config.
+var DefaultRoundTripper http.RoundTripper = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	DialContext: (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}).DialContext,
+	TLSHandshakeTimeout: 10 * time.Second,
+}
+
+// Config defines configuration parameters for a new client.
+type Config struct {
+	// The address of the Prometheus to connect to.
+	Address string
+
+	// Client is used by the Client to drive HTTP requests. If not provided,
+	// a new one based on the provided RoundTripper (or DefaultRoundTripper) will be used.
+	Client *http.Client
+
+	// RoundTripper is used by the Client to drive HTTP requests. If not
+	// provided, DefaultRoundTripper will be used.
+	RoundTripper http.RoundTripper
+}
+
+func (cfg *Config) roundTripper() http.RoundTripper {
+	if cfg.RoundTripper == nil {
+		return DefaultRoundTripper
+	}
+	return cfg.RoundTripper
+}
+
+func (cfg *Config) client() http.Client {
+	if cfg.Client == nil {
+		return http.Client{
+			Transport: cfg.roundTripper(),
+		}
+	}
+	return *cfg.Client
+}
+
+func (cfg *Config) validate() error {
+	if cfg.Client != nil && cfg.RoundTripper != nil {
+		return errors.New("api.Config.RoundTripper and api.Config.Client are mutually exclusive")
+	}
+	return nil
+}
+
+// Client is the interface for an API client.
+type Client interface {
+	URL(ep string, args map[string]string) *url.URL
+	Do(context.Context, *http.Request) (*http.Response, []byte, error)
+}
+
+// NewClient returns a new Client.
+//
+// It is safe to use the returned Client from multiple goroutines.
+func NewClient(cfg Config) (Client, error) {
+	u, err := url.Parse(cfg.Address)
+	if err != nil {
+		return nil, err
+	}
+	u.Path = strings.TrimRight(u.Path, "/")
+
+	if err := cfg.validate(); err != nil {
+		return nil, err
+	}
+
+	return &httpClient{
+		endpoint: u,
+		client:   cfg.client(),
+	}, nil
+}
+
+type httpClient struct {
+	endpoint *url.URL
+	client   http.Client
+}
+
+func (c *httpClient) URL(ep string, args map[string]string) *url.URL {
+	p := path.Join(c.endpoint.Path, ep)
+
+	for arg, val := range args {
+		arg = ":" + arg
+		p = strings.ReplaceAll(p, arg, val)
+	}
+
+	u := *c.endpoint
+	u.Path = p
+
+	return &u
+}
+
+func (c *httpClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, error) {
+	if ctx != nil {
+		req = req.WithContext(ctx)
+	}
+	resp, err := c.client.Do(req)
+	defer func() {
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var body []byte
+	done := make(chan struct{})
+	go func() {
+		var buf bytes.Buffer
+		_, err = buf.ReadFrom(resp.Body)
+		body = buf.Bytes()
+		close(done)
+	}()
+
+	select {
+	case <-ctx.Done():
+		<-done
+		err = resp.Body.Close()
+		if err == nil {
+			err = ctx.Err()
+		}
+	case <-done:
+	}
+
+	return resp, body, err
+}

vendor/github.com/prometheus/common/config/config.go

@@ -0,0 +1,101 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package no longer handles safe yaml parsing. In order to
+// ensure correct yaml unmarshalling, use "yaml.UnmarshalStrict()".
+
+package config
+
+import (
+	"encoding/json"
+	"net/http"
+	"path/filepath"
+)
+
+const secretToken = "<secret>"
+
+// Secret special type for storing secrets.
+type Secret string
+
+// MarshalSecretValue if set to true will expose Secret type
+// through the marshal interfaces. Useful for outside projects
+// that load and marshal the Prometheus config.
+var MarshalSecretValue bool = false
+
+// MarshalYAML implements the yaml.Marshaler interface for Secrets.
+func (s Secret) MarshalYAML() (interface{}, error) {
+	if MarshalSecretValue {
+		return string(s), nil
+	}
+	if s != "" {
+		return secretToken, nil
+	}
+	return nil, nil
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface for Secrets.
+func (s *Secret) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain Secret
+	return unmarshal((*plain)(s))
+}
+
+// MarshalJSON implements the json.Marshaler interface for Secret.
+func (s Secret) MarshalJSON() ([]byte, error) {
+	if MarshalSecretValue {
+		return json.Marshal(string(s))
+	}
+	if len(s) == 0 {
+		return json.Marshal("")
+	}
+	return json.Marshal(secretToken)
+}
+
+type ProxyHeader map[string][]Secret
+
+func (h *ProxyHeader) HTTPHeader() http.Header {
+	if h == nil || *h == nil {
+		return nil
+	}
+
+	header := make(http.Header)
+
+	for name, values := range *h {
+		var s []string
+		if values != nil {
+			s = make([]string, 0, len(values))
+			for _, value := range values {
+				s = append(s, string(value))
+			}
+		}
+		header[name] = s
+	}
+
+	return header
+}
+
+// DirectorySetter is a config type that contains file paths that may
+// be relative to the file containing the config.
+type DirectorySetter interface {
+	// SetDirectory joins any relative file paths with dir.
+	// Any paths that are empty or absolute remain unchanged.
+	SetDirectory(dir string)
+}
+
+// JoinDir joins dir and path if path is relative.
+// If path is empty or absolute, it is returned unchanged.
+func JoinDir(dir, path string) string {
+	if path == "" || filepath.IsAbs(path) {
+		return path
+	}
+	return filepath.Join(dir, path)
+}

vendor/github.com/prometheus/common/config/headers.go

@@ -0,0 +1,140 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package no longer handles safe yaml parsing. In order to
+// ensure correct yaml unmarshalling, use "yaml.UnmarshalStrict()".
+
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+)
+
+// reservedHeaders that change the connection, are set by Prometheus, or can
+// be changed otherwise.
+var reservedHeaders = map[string]struct{}{
+	"Authorization":                       {},
+	"Host":                                {},
+	"Content-Encoding":                    {},
+	"Content-Length":                      {},
+	"Content-Type":                        {},
+	"User-Agent":                          {},
+	"Connection":                          {},
+	"Keep-Alive":                          {},
+	"Proxy-Authenticate":                  {},
+	"Proxy-Authorization":                 {},
+	"Www-Authenticate":                    {},
+	"Accept-Encoding":                     {},
+	"X-Prometheus-Remote-Write-Version":   {},
+	"X-Prometheus-Remote-Read-Version":    {},
+	"X-Prometheus-Scrape-Timeout-Seconds": {},
+
+	// Added by SigV4.
+	"X-Amz-Date":           {},
+	"X-Amz-Security-Token": {},
+	"X-Amz-Content-Sha256": {},
+}
+
+// Headers represents the configuration for HTTP headers.
+type Headers struct {
+	Headers map[string]Header `yaml:",inline"`
+	dir     string
+}
+
+// Header represents the configuration for a single HTTP header.
+type Header struct {
+	Values  []string `yaml:"values,omitempty" json:"values,omitempty"`
+	Secrets []Secret `yaml:"secrets,omitempty" json:"secrets,omitempty"`
+	Files   []string `yaml:"files,omitempty" json:"files,omitempty"`
+}
+
+func (h Headers) MarshalJSON() ([]byte, error) {
+	// Inline the Headers map when serializing JSON because json encoder doesn't support "inline" directive.
+	return json.Marshal(h.Headers)
+}
+
+// SetDirectory records the directory to make headers file relative to the
+// configuration file.
+func (h *Headers) SetDirectory(dir string) {
+	if h == nil {
+		return
+	}
+	h.dir = dir
+}
+
+// Validate validates the Headers config.
+func (h *Headers) Validate() error {
+	for n, header := range h.Headers {
+		if _, ok := reservedHeaders[http.CanonicalHeaderKey(n)]; ok {
+			return fmt.Errorf("setting header %q is not allowed", http.CanonicalHeaderKey(n))
+		}
+		for _, v := range header.Files {
+			f := JoinDir(h.dir, v)
+			_, err := os.ReadFile(f)
+			if err != nil {
+				return fmt.Errorf("unable to read header %q from file %s: %w", http.CanonicalHeaderKey(n), f, err)
+			}
+		}
+	}
+	return nil
+}
+
+// NewHeadersRoundTripper returns a RoundTripper that sets HTTP headers on
+// requests as configured.
+func NewHeadersRoundTripper(config *Headers, next http.RoundTripper) http.RoundTripper {
+	if len(config.Headers) == 0 {
+		return next
+	}
+	return &headersRoundTripper{
+		config: config,
+		next:   next,
+	}
+}
+
+type headersRoundTripper struct {
+	next   http.RoundTripper
+	config *Headers
+}
+
+// RoundTrip implements http.RoundTripper.
+func (rt *headersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = cloneRequest(req)
+	for n, h := range rt.config.Headers {
+		for _, v := range h.Values {
+			req.Header.Add(n, v)
+		}
+		for _, v := range h.Secrets {
+			req.Header.Add(n, string(v))
+		}
+		for _, v := range h.Files {
+			f := JoinDir(rt.config.dir, v)
+			b, err := os.ReadFile(f)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read headers file %s: %w", f, err)
+			}
+			req.Header.Add(n, strings.TrimSpace(string(b)))
+		}
+	}
+	return rt.next.RoundTrip(req)
+}
+
+// CloseIdleConnections implements closeIdler.
+func (rt *headersRoundTripper) CloseIdleConnections() {
+	if ci, ok := rt.next.(closeIdler); ok {
+		ci.CloseIdleConnections()
+	}
+}

vendor/golang.org/x/net/http/httpproxy/proxy.go

@@ -0,0 +1,367 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package httpproxy provides support for HTTP proxy determination
+// based on environment variables, as provided by net/http's
+// ProxyFromEnvironment function.
+//
+// The API is not subject to the Go 1 compatibility promise and may change at
+// any time.
+package httpproxy
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/idna"
+)
+
+// Config holds configuration for HTTP proxy settings. See
+// FromEnvironment for details.
+type Config struct {
+	// HTTPProxy represents the value of the HTTP_PROXY or
+	// http_proxy environment variable. It will be used as the proxy
+	// URL for HTTP requests unless overridden by NoProxy.
+	HTTPProxy string
+
+	// HTTPSProxy represents the HTTPS_PROXY or https_proxy
+	// environment variable. It will be used as the proxy URL for
+	// HTTPS requests unless overridden by NoProxy.
+	HTTPSProxy string
+
+	// NoProxy represents the NO_PROXY or no_proxy environment
+	// variable. It specifies a string that contains comma-separated values
+	// specifying hosts that should be excluded from proxying. Each value is
+	// represented by an IP address prefix (1.2.3.4), an IP address prefix in
+	// CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*).
+	// An IP address prefix and domain name can also include a literal port
+	// number (1.2.3.4:80).
+	// A domain name matches that name and all subdomains. A domain name with
+	// a leading "." matches subdomains only. For example "foo.com" matches
+	// "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com".
+	// A single asterisk (*) indicates that no proxying should be done.
+	// A best effort is made to parse the string and errors are
+	// ignored.
+	NoProxy string
+
+	// CGI holds whether the current process is running
+	// as a CGI handler (FromEnvironment infers this from the
+	// presence of a REQUEST_METHOD environment variable).
+	// When this is set, ProxyForURL will return an error
+	// when HTTPProxy applies, because a client could be
+	// setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy.
+	CGI bool
+}
+
+// config holds the parsed configuration for HTTP proxy settings.
+type config struct {
+	// Config represents the original configuration as defined above.
+	Config
+
+	// httpsProxy is the parsed URL of the HTTPSProxy if defined.
+	httpsProxy *url.URL
+
+	// httpProxy is the parsed URL of the HTTPProxy if defined.
+	httpProxy *url.URL
+
+	// ipMatchers represent all values in the NoProxy that are IP address
+	// prefixes or an IP address in CIDR notation.
+	ipMatchers []matcher
+
+	// domainMatchers represent all values in the NoProxy that are a domain
+	// name or hostname & domain name
+	domainMatchers []matcher
+}
+
+// FromEnvironment returns a Config instance populated from the
+// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the
+// lowercase versions thereof).
+//
+// The environment values may be either a complete URL or a
+// "host[:port]", in which case the "http" scheme is assumed. An error
+// is returned if the value is a different form.
+func FromEnvironment() *Config {
+	return &Config{
+		HTTPProxy:  getEnvAny("HTTP_PROXY", "http_proxy"),
+		HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"),
+		NoProxy:    getEnvAny("NO_PROXY", "no_proxy"),
+		CGI:        os.Getenv("REQUEST_METHOD") != "",
+	}
+}
+
+func getEnvAny(names ...string) string {
+	for _, n := range names {
+		if val := os.Getenv(n); val != "" {
+			return val
+		}
+	}
+	return ""
+}
+
+// ProxyFunc returns a function that determines the proxy URL to use for
+// a given request URL. Changing the contents of cfg will not affect
+// proxy functions created earlier.
+//
+// A nil URL and nil error are returned if no proxy is defined in the
+// environment, or a proxy should not be used for the given request, as
+// defined by NO_PROXY.
+//
+// As a special case, if req.URL.Host is "localhost" or a loopback address
+// (with or without a port number), then a nil URL and nil error will be returned.
+func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) {
+	// Preprocess the Config settings for more efficient evaluation.
+	cfg1 := &config{
+		Config: *cfg,
+	}
+	cfg1.init()
+	return cfg1.proxyForURL
+}
+
+func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) {
+	var proxy *url.URL
+	if reqURL.Scheme == "https" {
+		proxy = cfg.httpsProxy
+	} else if reqURL.Scheme == "http" {
+		proxy = cfg.httpProxy
+		if proxy != nil && cfg.CGI {
+			return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
+		}
+	}
+	if proxy == nil {
+		return nil, nil
+	}
+	if !cfg.useProxy(canonicalAddr(reqURL)) {
+		return nil, nil
+	}
+
+	return proxy, nil
+}
+
+func parseProxy(proxy string) (*url.URL, error) {
+	if proxy == "" {
+		return nil, nil
+	}
+
+	proxyURL, err := url.Parse(proxy)
+	if err != nil || proxyURL.Scheme == "" || proxyURL.Host == "" {
+		// proxy was bogus. Try prepending "http://" to it and
+		// see if that parses correctly. If not, we fall
+		// through and complain about the original one.
+		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
+			return proxyURL, nil
+		}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
+	}
+	return proxyURL, nil
+}
+
+// useProxy reports whether requests to addr should use a proxy,
+// according to the NO_PROXY or no_proxy environment variable.
+// addr is always a canonicalAddr with a host and port.
+func (cfg *config) useProxy(addr string) bool {
+	if len(addr) == 0 {
+		return true
+	}
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+	if host == "localhost" {
+		return false
+	}
+	ip := net.ParseIP(host)
+	if ip != nil {
+		if ip.IsLoopback() {
+			return false
+		}
+	}
+
+	addr = strings.ToLower(strings.TrimSpace(host))
+
+	if ip != nil {
+		for _, m := range cfg.ipMatchers {
+			if m.match(addr, port, ip) {
+				return false
+			}
+		}
+	}
+	for _, m := range cfg.domainMatchers {
+		if m.match(addr, port, ip) {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *config) init() {
+	if parsed, err := parseProxy(c.HTTPProxy); err == nil {
+		c.httpProxy = parsed
+	}
+	if parsed, err := parseProxy(c.HTTPSProxy); err == nil {
+		c.httpsProxy = parsed
+	}
+
+	for _, p := range strings.Split(c.NoProxy, ",") {
+		p = strings.ToLower(strings.TrimSpace(p))
+		if len(p) == 0 {
+			continue
+		}
+
+		if p == "*" {
+			c.ipMatchers = []matcher{allMatch{}}
+			c.domainMatchers = []matcher{allMatch{}}
+			return
+		}
+
+		// IPv4/CIDR, IPv6/CIDR
+		if _, pnet, err := net.ParseCIDR(p); err == nil {
+			c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet})
+			continue
+		}
+
+		// IPv4:port, [IPv6]:port
+		phost, pport, err := net.SplitHostPort(p)
+		if err == nil {
+			if len(phost) == 0 {
+				// There is no host part, likely the entry is malformed; ignore.
+				continue
+			}
+			if phost[0] == '[' && phost[len(phost)-1] == ']' {
+				phost = phost[1 : len(phost)-1]
+			}
+		} else {
+			phost = p
+		}
+		// IPv4, IPv6
+		if pip := net.ParseIP(phost); pip != nil {
+			c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport})
+			continue
+		}
+
+		if len(phost) == 0 {
+			// There is no host part, likely the entry is malformed; ignore.
+			continue
+		}
+
+		// domain.com or domain.com:80
+		// foo.com matches bar.foo.com
+		// .domain.com or .domain.com:port
+		// *.domain.com or *.domain.com:port
+		if strings.HasPrefix(phost, "*.") {
+			phost = phost[1:]
+		}
+		matchHost := false
+		if phost[0] != '.' {
+			matchHost = true
+			phost = "." + phost
+		}
+		if v, err := idnaASCII(phost); err == nil {
+			phost = v
+		}
+		c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost})
+	}
+}
+
+var portMap = map[string]string{
+	"http":   "80",
+	"https":  "443",
+	"socks5": "1080",
+}
+
+// canonicalAddr returns url.Host but always with a ":port" suffix
+func canonicalAddr(url *url.URL) string {
+	addr := url.Hostname()
+	if v, err := idnaASCII(addr); err == nil {
+		addr = v
+	}
+	port := url.Port()
+	if port == "" {
+		port = portMap[url.Scheme]
+	}
+	return net.JoinHostPort(addr, port)
+}
+
+// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
+// return true if the string includes a port.
+func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
+
+func idnaASCII(v string) (string, error) {
+	// TODO: Consider removing this check after verifying performance is okay.
+	// Right now punycode verification, length checks, context checks, and the
+	// permissible character tests are all omitted. It also prevents the ToASCII
+	// call from salvaging an invalid IDN, when possible. As a result it may be
+	// possible to have two IDNs that appear identical to the user where the
+	// ASCII-only version causes an error downstream whereas the non-ASCII
+	// version does not.
+	// Note that for correct ASCII IDNs ToASCII will only do considerably more
+	// work, but it will not cause an allocation.
+	if isASCII(v) {
+		return v, nil
+	}
+	return idna.Lookup.ToASCII(v)
+}
+
+func isASCII(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= utf8.RuneSelf {
+			return false
+		}
+	}
+	return true
+}
+
+// matcher represents the matching rule for a given value in the NO_PROXY list
+type matcher interface {
+	// match returns true if the host and optional port or ip and optional port
+	// are allowed
+	match(host, port string, ip net.IP) bool
+}
+
+// allMatch matches on all possible inputs
+type allMatch struct{}
+
+func (a allMatch) match(host, port string, ip net.IP) bool {
+	return true
+}
+
+type cidrMatch struct {
+	cidr *net.IPNet
+}
+
+func (m cidrMatch) match(host, port string, ip net.IP) bool {
+	return m.cidr.Contains(ip)
+}
+
+type ipMatch struct {
+	ip   net.IP
+	port string
+}
+
+func (m ipMatch) match(host, port string, ip net.IP) bool {
+	if m.ip.Equal(ip) {
+		return m.port == "" || m.port == port
+	}
+	return false
+}
+
+type domainMatch struct {
+	host string
+	port string
+
+	matchHost bool
+}
+
+func (m domainMatch) match(host, port string, ip net.IP) bool {
+	if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) {
+		return m.port == "" || m.port == port
+	}
+	return false
+}

vendor/golang.org/x/oauth2/clientcredentials/clientcredentials.go

@@ -0,0 +1,124 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package clientcredentials implements the OAuth2.0 "client credentials" token flow,
+// also known as the "two-legged OAuth 2.0".
+//
+// This should be used when the client is acting on its own behalf or when the client
+// is the resource owner. It may also be used when requesting access to protected
+// resources based on an authorization previously arranged with the authorization
+// server.
+//
+// See https://tools.ietf.org/html/rfc6749#section-4.4
+package clientcredentials // import "golang.org/x/oauth2/clientcredentials"
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/internal"
+)
+
+// Config describes a 2-legged OAuth2 flow, with both the
+// client application information and the server's endpoint URLs.
+type Config struct {
+	// ClientID is the application's ID.
+	ClientID string
+
+	// ClientSecret is the application's secret.
+	ClientSecret string
+
+	// TokenURL is the resource server's token endpoint
+	// URL. This is a constant specific to each server.
+	TokenURL string
+
+	// Scope specifies optional requested permissions.
+	Scopes []string
+
+	// EndpointParams specifies additional parameters for requests to the token endpoint.
+	EndpointParams url.Values
+
+	// AuthStyle optionally specifies how the endpoint wants the
+	// client ID & client secret sent. The zero value means to
+	// auto-detect.
+	AuthStyle oauth2.AuthStyle
+
+	// authStyleCache caches which auth style to use when Endpoint.AuthStyle is
+	// the zero value (AuthStyleAutoDetect).
+	authStyleCache internal.LazyAuthStyleCache
+}
+
+// Token uses client credentials to retrieve a token.
+//
+// The provided context optionally controls which HTTP client is used. See the oauth2.HTTPClient variable.
+func (c *Config) Token(ctx context.Context) (*oauth2.Token, error) {
+	return c.TokenSource(ctx).Token()
+}
+
+// Client returns an HTTP client using the provided token.
+// The token will auto-refresh as necessary.
+//
+// The provided context optionally controls which HTTP client
+// is returned. See the oauth2.HTTPClient variable.
+//
+// The returned Client and its Transport should not be modified.
+func (c *Config) Client(ctx context.Context) *http.Client {
+	return oauth2.NewClient(ctx, c.TokenSource(ctx))
+}
+
+// TokenSource returns a TokenSource that returns t until t expires,
+// automatically refreshing it as necessary using the provided context and the
+// client ID and client secret.
+//
+// Most users will use Config.Client instead.
+func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource {
+	source := &tokenSource{
+		ctx:  ctx,
+		conf: c,
+	}
+	return oauth2.ReuseTokenSource(nil, source)
+}
+
+type tokenSource struct {
+	ctx  context.Context
+	conf *Config
+}
+
+// Token refreshes the token by using a new client credentials request.
+// tokens received this way do not include a refresh token
+func (c *tokenSource) Token() (*oauth2.Token, error) {
+	v := url.Values{
+		"grant_type": {"client_credentials"},
+	}
+	if len(c.conf.Scopes) > 0 {
+		v.Set("scope", strings.Join(c.conf.Scopes, " "))
+	}
+	for k, p := range c.conf.EndpointParams {
+		// Allow grant_type to be overridden to allow interoperability with
+		// non-compliant implementations.
+		if _, ok := v[k]; ok && k != "grant_type" {
+			return nil, fmt.Errorf("oauth2: cannot overwrite parameter %q", k)
+		}
+		v[k] = p
+	}
+
+	tk, err := internal.RetrieveToken(c.ctx, c.conf.ClientID, c.conf.ClientSecret, c.conf.TokenURL, v, internal.AuthStyle(c.conf.AuthStyle), c.conf.authStyleCache.Get())
+	if err != nil {
+		if rErr, ok := err.(*internal.RetrieveError); ok {
+			return nil, (*oauth2.RetrieveError)(rErr)
+		}
+		return nil, err
+	}
+	t := &oauth2.Token{
+		AccessToken:  tk.AccessToken,
+		TokenType:    tk.TokenType,
+		RefreshToken: tk.RefreshToken,
+		Expiry:       tk.Expiry,
+	}
+	return t.WithExtra(tk.Raw), nil
+}

vendor/modules.txt

@@ -174,6 +174,9 @@ github.com/inconshreveable/mousetrap
 # github.com/josharian/intern v1.0.0
 ## explicit; go 1.5
 github.com/josharian/intern
+# github.com/jpillora/backoff v1.0.0
+## explicit; go 1.13
+github.com/jpillora/backoff
 # github.com/json-iterator/go v1.1.12
 ## explicit; go 1.12
 github.com/json-iterator/go
@@ -196,6 +199,9 @@ github.com/modern-go/reflect2
 # github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 ## explicit
 github.com/munnerz/goautoneg
+# github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
+## explicit
+github.com/mwitkow/go-conntrack
 # github.com/openshift/custom-resource-status v1.1.2
 ## explicit; go 1.12
 github.com/openshift/custom-resource-status/conditions/v1
@@ -204,6 +210,8 @@ github.com/openshift/custom-resource-status/conditions/v1
 github.com/pkg/errors
 # github.com/prometheus/client_golang v1.19.1
 ## explicit; go 1.20
+github.com/prometheus/client_golang/api
+github.com/prometheus/client_golang/api/prometheus/v1
 github.com/prometheus/client_golang/prometheus
 github.com/prometheus/client_golang/prometheus/collectors
 github.com/prometheus/client_golang/prometheus/internal
@@ -216,6 +224,7 @@ github.com/prometheus/client_golang/prometheus/testutil/promlint/validations
 github.com/prometheus/client_model/go
 # github.com/prometheus/common v0.55.0
 ## explicit; go 1.20
+github.com/prometheus/common/config
 github.com/prometheus/common/expfmt
 github.com/prometheus/common/model
 # github.com/prometheus/procfs v0.15.1
@@ -360,6 +369,7 @@ golang.org/x/mod/semver
 ## explicit; go 1.18
 golang.org/x/net/context
 golang.org/x/net/http/httpguts
+golang.org/x/net/http/httpproxy
 golang.org/x/net/http2
 golang.org/x/net/http2/hpack
 golang.org/x/net/idna
@@ -369,6 +379,7 @@ golang.org/x/net/websocket
 # golang.org/x/oauth2 v0.23.0
 ## explicit; go 1.18
 golang.org/x/oauth2
+golang.org/x/oauth2/clientcredentials
 golang.org/x/oauth2/internal
 # golang.org/x/sync v0.10.0
 ## explicit; go 1.18