From 0777837a135ffe120ad1db6e3fc67db2d2b0c179 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Wed, 27 May 2020 13:08:17 +0100 Subject: [PATCH 1/3] Add helm chart --- .github/workflows/release.yaml | 30 ++++++++++ Makefile | 10 +++- charts/descheduler/.helmignore | 22 +++++++ charts/descheduler/Chart.yaml | 16 +++++ charts/descheduler/README.md | 59 +++++++++++++++++++ charts/descheduler/templates/NOTES.txt | 1 + charts/descheduler/templates/_helpers.tpl | 56 ++++++++++++++++++ charts/descheduler/templates/clusterrole.yaml | 21 +++++++ .../templates/clusterrolebinding.yaml | 16 +++++ charts/descheduler/templates/configmap.yaml | 11 ++++ charts/descheduler/templates/cronjob.yaml | 53 +++++++++++++++++ .../descheduler/templates/serviceaccount.yaml | 8 +++ charts/descheduler/values.yaml | 59 +++++++++++++++++++ 13 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yaml create mode 100644 charts/descheduler/.helmignore create mode 100644 charts/descheduler/Chart.yaml create mode 100644 charts/descheduler/README.md create mode 100644 charts/descheduler/templates/NOTES.txt create mode 100644 charts/descheduler/templates/_helpers.tpl create mode 100644 charts/descheduler/templates/clusterrole.yaml create mode 100644 charts/descheduler/templates/clusterrolebinding.yaml create mode 100644 charts/descheduler/templates/configmap.yaml create mode 100644 charts/descheduler/templates/cronjob.yaml create mode 100644 charts/descheduler/templates/serviceaccount.yaml create mode 100644 charts/descheduler/values.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..54af98699 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,30 @@ +name: Release Charts + +on: + push: + tags: + - chart-* + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Fetch history + run: git fetch --prune --unshallow + + - name: Add dependency chart repos + run: | + helm repo add stable https://kubernetes-charts.storage.googleapis.com/ + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.0.0-rc.2 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/Makefile b/Makefile index e30478ef8..6eab6b029 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,8 @@ IMAGE_GCLOUD:=$(REGISTRY)/descheduler:$(VERSION) # In the future binaries can be uploaded to # GCS bucket gs://k8s-staging-descheduler. +HAS_HELM := $(shell which helm) + all: build build: @@ -62,7 +64,7 @@ push: push-container-to-gcloud clean: rm -rf _output -verify: verify-gofmt verify-vendor lint +verify: verify-gofmt verify-vendor lint lint-chart verify-gofmt: ./hack/verify-gofmt.sh @@ -87,3 +89,9 @@ ifndef HAS_GOLANGCI curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b ./_output/bin ${GOLANGCI_VERSION} endif ./_output/bin/golangci-lint run + +lint-chart: +ifndef HAS_HELM + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && chmod 700 ./get_helm.sh && ./get_helm.sh +endif + helm lint ./charts/descheduler diff --git a/charts/descheduler/.helmignore b/charts/descheduler/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/charts/descheduler/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/descheduler/Chart.yaml b/charts/descheduler/Chart.yaml new file mode 100644 index 000000000..7629fb9e5 --- /dev/null +++ b/charts/descheduler/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +name: descheduler +version: 1.0.0 +appVersion: 0.18.0 +description: Descheduler for Kubernetes is used to rebalance clusters by evicting pods that can potentially be scheduled on better nodes. In the current implementation, descheduler does not schedule replacement of evicted pods but relies on the default scheduler for that. +keywords: +- kubernetes +- descheduler +- kube-scheduler +home: https://github.com/kubernetes-sigs/descheduler +icon: https://kubernetes.io/images/favicon.png +sources: +- https://github.com/kubernetes-sigs/descheduler +maintainers: +- name: stevehipwell + email: steve.hipwell@github.com diff --git a/charts/descheduler/README.md b/charts/descheduler/README.md new file mode 100644 index 000000000..741d24407 --- /dev/null +++ b/charts/descheduler/README.md @@ -0,0 +1,59 @@ +# Descheduler for Kubernetes + +[Descheduler](https://github.com/kubernetes-sigs/descheduler/) for Kubernetes is used to rebalance clusters by evicting pods that can potentially be scheduled on better nodes. In the current implementation, descheduler does not schedule replacement of evicted pods but relies on the default scheduler for that. + +## TL;DR: + +```shell +helm repo add descheduler https://kubernetes-sigs.github.io/descheduler/ +$ helm install descheduler/descheduler --name my-release +``` + +## Introduction + +This chart bootstraps a [desheduler](https://github.com/kubernetes-sigs/descheduler/) cron job on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.14+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```shell +helm install --name my-release descheduler/descheduler +``` + +The command deploys _descheduler_ on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```shell +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the _descheduler_ chart and their default values. + +| Parameter | Description | Default | +| ------------------------------ | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | +| `image.repository` | Docker repository to use | `us.gcr.io/k8s-artifacts-prod/descheduler/descheduler` | +| `image.tag` | Docker tag to use | `v[chart appVersion]` | +| `image.pullPolicy` | Docker image pull policy | `IfNotPresent` | +| `nameOverride` | String to partially override `descheduler.fullname` template (will prepend the release name) | `""` | +| `fullnameOverride` | String to fully override `descheduler.fullname` template | `""` | +| `schedule` | The cron schedule to run the _descheduler_ job on | `"*/2 * * * *"` | +| `cmdOptions` | The options to pass to the _descheduler_ command | _see values.yaml_ | +| `deschedulerPolicy.strategies` | The _descheduler_ strategies to apply | _see values.yaml_ | +| `priorityClassName` | The name of the priority class to add to pods | `system-cluster-critical` | +| `rbac.create` | If `true`, create & use RBAC resources | `true` | +| `serviceAccount.create` | If `true`, create a service account for the cron job | `true` | +| `serviceAccount.name` | The name of the service account to use, if not set and create is true a name is generated using the fullname template | `nil` | diff --git a/charts/descheduler/templates/NOTES.txt b/charts/descheduler/templates/NOTES.txt new file mode 100644 index 000000000..f43d29ed6 --- /dev/null +++ b/charts/descheduler/templates/NOTES.txt @@ -0,0 +1 @@ +Descheduler installed as a cron job. diff --git a/charts/descheduler/templates/_helpers.tpl b/charts/descheduler/templates/_helpers.tpl new file mode 100644 index 000000000..68df7d3f9 --- /dev/null +++ b/charts/descheduler/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "descheduler.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "descheduler.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "descheduler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "descheduler.labels" -}} +app.kubernetes.io/name: {{ include "descheduler.name" . }} +helm.sh/chart: {{ include "descheduler.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "descheduler.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "descheduler.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/descheduler/templates/clusterrole.yaml b/charts/descheduler/templates/clusterrole.yaml new file mode 100644 index 000000000..b63ef308f --- /dev/null +++ b/charts/descheduler/templates/clusterrole.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "descheduler.fullname" . }} + labels: + {{- include "descheduler.labels" . | nindent 4 }} +rules: +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list", "delete"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +{{- end -}} diff --git a/charts/descheduler/templates/clusterrolebinding.yaml b/charts/descheduler/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..6cbc62004 --- /dev/null +++ b/charts/descheduler/templates/clusterrolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "descheduler.fullname" . }} + labels: + {{- include "descheduler.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "descheduler.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "descheduler.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/descheduler/templates/configmap.yaml b/charts/descheduler/templates/configmap.yaml new file mode 100644 index 000000000..a8b620a85 --- /dev/null +++ b/charts/descheduler/templates/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "descheduler.fullname" . }} + labels: + {{- include "descheduler.labels" . | nindent 4 }} +data: + policy.yaml: | + apiVersion: "descheduler/v1alpha1" + kind: "DeschedulerPolicy" +{{ toYaml .Values.deschedulerPolicy | trim | indent 4 }} diff --git a/charts/descheduler/templates/cronjob.yaml b/charts/descheduler/templates/cronjob.yaml new file mode 100644 index 000000000..3bdb9be5d --- /dev/null +++ b/charts/descheduler/templates/cronjob.yaml @@ -0,0 +1,53 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ template "descheduler.fullname" . }} + labels: + {{- include "descheduler.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.schedule | quote }} + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + metadata: + name: {{ template "descheduler.fullname" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} + {{- .Values.podAnnotations | toYaml | nindent 12 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ include "descheduler.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- .Values.podLabels | toYaml | nindent 12 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + serviceAccountName: {{ template "descheduler.serviceAccountName" . }} + restartPolicy: "Never" + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "v%s" .Chart.AppVersion) }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "/bin/descheduler" + args: + - "--policy-config-file" + - "/policy-dir/policy.yaml" + {{- range $key, $value := .Values.cmdOptions }} + - {{ printf "--%s" $key | quote }} + {{- if $value }} + - {{ $value | quote }} + {{- end }} + {{- end }} + volumeMounts: + - mountPath: /policy-dir + name: policy-volume + volumes: + - name: policy-volume + configMap: + name: {{ template "descheduler.fullname" . }} diff --git a/charts/descheduler/templates/serviceaccount.yaml b/charts/descheduler/templates/serviceaccount.yaml new file mode 100644 index 000000000..8149ce072 --- /dev/null +++ b/charts/descheduler/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "descheduler.serviceAccountName" . }} + labels: + {{- include "descheduler.labels" . | nindent 4 }} +{{- end -}} diff --git a/charts/descheduler/values.yaml b/charts/descheduler/values.yaml new file mode 100644 index 000000000..9ecb9e47d --- /dev/null +++ b/charts/descheduler/values.yaml @@ -0,0 +1,59 @@ +# Default values for descheduler. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: us.gcr.io/k8s-artifacts-prod/descheduler/descheduler + # Overrides the image tag whose default is the chart version + tag: "" + pullPolicy: IfNotPresent + +nameOverride: "" +fullnameOverride: "" + +schedule: "*/2 * * * *" + +cmdOptions: + v: 3 + # evict-local-storage-pods: + # max-pods-to-evict-per-node: 10 + # node-selector: "key1=value1,key2=value2" + +deschedulerPolicy: + strategies: + RemoveDuplicates: + enabled: true + RemovePodsViolatingNodeTaints: + enabled: true + RemovePodsViolatingNodeAffinity: + enabled: true + params: + nodeAffinityType: + - requiredDuringSchedulingIgnoredDuringExecution + RemovePodsViolatingInterPodAntiAffinity: + enabled: true + LowNodeUtilization: + enabled: true + params: + nodeResourceUtilizationThresholds: + thresholds: + cpu: 20 + memory: 20 + pods: 20 + targetThresholds: + cpu: 50 + memory: 50 + pods: 50 + +priorityClassName: system-cluster-critical + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: From 60e41670af038007263a0712ebc141f0abc2fb48 Mon Sep 17 00:00:00 2001 From: Sean Malloy Date: Fri, 17 Jul 2020 00:49:31 -0500 Subject: [PATCH 2/3] Update Release Documentation For Helm Charts The descheduler now has an official helm chart. This change documents the process to release a new helm chart artifact. --- docs/release-guide.md | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/release-guide.md b/docs/release-guide.md index f50ddf611..99f34d316 100644 --- a/docs/release-guide.md +++ b/docs/release-guide.md @@ -1,6 +1,8 @@ # Release Guide -## Semi-automatic +## Container Image + +### Semi-automatic 1. Make sure your repo is clean by git's standards 2. Create a release branch `git checkout -b release-1.18` (not required for patch releases) @@ -11,7 +13,7 @@ 7. Publish release 8. Email `kubernetes-sig-scheduling@googlegroups.com` to announce the release -## Manual +### Manual 1. Make sure your repo is clean by git's standards 2. Create a release branch `git checkout -b release-1.18` (not required for patch releases) @@ -24,7 +26,7 @@ 9. Publish release 10. Email `kubernetes-sig-scheduling@googlegroups.com` to announce the release -## Notes +### Notes See [post-descheduler-push-images dashboard](https://testgrid.k8s.io/sig-scheduling#post-descheduler-push-images) for staging registry image build job status. List images in staging registry. @@ -46,3 +48,19 @@ Pull image from the staging registry. ``` docker pull gcr.io/k8s-staging-descheduler/descheduler:v20200206-0.9.0-94-ge2a23f284 ``` + +## Helm Chart +Helm chart releases are managed by a separate set of git tags that are prefixed with `chart-*`. Example git tag name is `chart-0.18.0`. Released versions of the +helm charts are stored in the `gh-pages` branch of this repo. The [chart-releaser-action GitHub Action](https://github.com/helm/chart-releaser-action) is setup to +build and push the helm charts to the `gh-pages` branch when a `chart-*` git tag is created. + +The major and minor version of the chart matches the descheduler major and minor versions. For example descheduler helm chart version chart-0.18.0 corresponds +to descheduler version v0.18.0. The patch version of the descheduler helm chart and the patcher version of the descheduler will not necessarily match. The patch +version of the descheduler helm chart is used to version changes specific to the helm chart. + +1. Merge all helm chart changes into the appropriate release branch(i.e. release-1.18) + 1. Ensure that `appVersion` in file `charts/descheduler/Chart.yaml` matches the descheduler version(no `v` prefix) + 2. Ensure that `version` in file `charts/descheduler/Chart.yaml` has been incremented. This is the chart version. +2. Make sure your repo is clean by git's standards +3. Create the tag and push it `git checkout release-1.18; CHART_VERSION=chart-0.18.0; git tag $CHART_VERSION; git push origin $CHART_VERSION` +4. Verify the new helm artifact has been successfully pushed to the `gh-pages` branch From 870a518db449ea41a5b2cf5cc5a6e3b4729c9a5c Mon Sep 17 00:00:00 2001 From: Sean Malloy Date: Sat, 18 Jul 2020 23:07:17 -0500 Subject: [PATCH 3/3] Update Helm Chart Version To 0.18.0 --- charts/descheduler/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/descheduler/Chart.yaml b/charts/descheduler/Chart.yaml index 7629fb9e5..6a789fe67 100644 --- a/charts/descheduler/Chart.yaml +++ b/charts/descheduler/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: descheduler -version: 1.0.0 +version: 0.18.0 appVersion: 0.18.0 description: Descheduler for Kubernetes is used to rebalance clusters by evicting pods that can potentially be scheduled on better nodes. In the current implementation, descheduler does not schedule replacement of evicted pods but relies on the default scheduler for that. keywords: