From 55cf45a6ba1f6e68912b457399c100d70d2c68c3 Mon Sep 17 00:00:00 2001 From: Sean Malloy Date: Fri, 13 Dec 2019 00:35:05 -0600 Subject: [PATCH 1/2] Streamline Deployment Docs For End Users The k8s YAML manifests for deploying the descheduler as a k8s job were duplicated across the "examples" and "kubernetes" directories and also in README.md. This change consolidates the YAML manifests into the "kubernetes" directory and simplifies the installation instructions for end users in README.md. Additionally a k8s CronJob has been added. --- README.md | 112 +++++----------------------------- examples/descheduler-job.yaml | 27 -------- examples/descheduler.yaml | 61 ------------------ kubernetes/configmap.yaml | 1 + kubernetes/cronjob.yaml | 35 +++++++++++ kubernetes/job.yaml | 6 +- 6 files changed, 55 insertions(+), 187 deletions(-) delete mode 100644 examples/descheduler-job.yaml delete mode 100644 examples/descheduler.yaml create mode 100644 kubernetes/cronjob.yaml diff --git a/README.md b/README.md index 8809e7d84..8ca64ed13 100644 --- a/README.md +++ b/README.md @@ -47,120 +47,40 @@ For more information about available options run: $ ./_output/bin/descheduler --help ``` -## Running Descheduler as a Job Inside of a Pod +## Running Descheduler as a Job or CronJob -Descheduler can be run as a job inside of a pod. It has the advantage of +The descheduler can be run as a job or cronjob inside of a pod. It has the advantage of being able to be run multiple times without needing user intervention. -Descheduler pod is run as a critical pod to avoid being evicted by itself, -or by kubelet due to an eviction event. Since critical pods are created in -`kube-system` namespace, descheduler job and its pod will also be created +The descheduler pod is run as a critical pod to avoid being evicted by itself, +or by the kubelet due to an eviction event. Since critical pods are created in the +`kube-system` namespace, the descheduler job and its pod will also be created in `kube-system` namespace. -### Create a container image +### Setup RBAC -First we create a simple Docker image utilizing the Dockerfile found in the root directory: +To give necessary permissions for the descheduler to work in a pod. ``` -$ make dev-image +$ kubectl create -f kubernetes/rbac.yaml ``` -This creates an image based off the binary we've built before. To build both the -binary and image in one step you can run the following command: - -``` -$ make image -``` - -This eliminates the need to have Go installed locally and builds the binary -within it's own container. - -### Create a cluster role - -To give necessary permissions for the descheduler to work in a pod, create a cluster role: - -``` -$ cat << EOF| kubectl create -f - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: descheduler-cluster-role -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "watch", "list"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list", "delete"] -- apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] -EOF -``` - -### Create the service account which will be used to run the job: - -``` -$ kubectl create sa descheduler-sa -n kube-system -``` - -### Bind the cluster role to the service account: - -``` -$ kubectl create clusterrolebinding descheduler-cluster-role-binding \ - --clusterrole=descheduler-cluster-role \ - --serviceaccount=kube-system:descheduler-sa -``` ### Create a configmap to store descheduler policy -Descheduler policy is created as a ConfigMap in `kube-system` namespace -so that it can be mounted as a volume inside pod. - ``` -$ kubectl create configmap descheduler-policy-configmap \ - -n kube-system --from-file= -``` -### Create the job specification (descheduler-job.yaml) - -``` -apiVersion: batch/v1 -kind: Job -metadata: - name: descheduler-job - namespace: kube-system -spec: - parallelism: 1 - completions: 1 - template: - metadata: - name: descheduler-pod - spec: - priorityClassName: system-cluster-critical - containers: - - name: descheduler - image: descheduler - volumeMounts: - - mountPath: /policy-dir - name: policy-volume - command: ["/bin/descheduler", "--policy-config-file", "/policy-dir/policy.yaml", "-v", "1"] - restartPolicy: "Never" - serviceAccountName: descheduler-sa - volumes: - - name: policy-volume - configMap: - name: descheduler-policy-configmap +$ kubectl create -f kubernetes/configmap.yaml ``` -Please note that the pod template is configured with critical pod annotation, and -the policy `policy-file` is mounted as a volume from the config map. +### Create a Job or CronJob -### Run the descheduler as a job in a pod: +As a Job. ``` -$ kubectl create -f descheduler-job.yaml +$ kubectl create -f kubernetes/job.yaml ``` -### Examples - -See [descheduler.yaml](examples/descheduler.yaml) and [descheduler-job.yaml](examples/descheduler-job.yaml) for a combined yaml file of the above steps. +Or as a CronJob. +``` +$ kubectl create -f kubernetes/cronjob.yaml +``` ## Policy and Strategies diff --git a/examples/descheduler-job.yaml b/examples/descheduler-job.yaml deleted file mode 100644 index 83ad7c649..000000000 --- a/examples/descheduler-job.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: descheduler-job - namespace: kube-system -spec: - parallelism: 1 - completions: 1 - template: - metadata: - name: descheduler-pod - spec: - priorityClassName: system-cluster-critical - containers: - - name: descheduler - image: docker.io/aveshagarwal/descheduler:0.9.0 - volumeMounts: - - mountPath: /policy-dir - name: policy-volume - command: ["/bin/descheduler", "--policy-config-file", "/policy-dir/policy.yaml", "-v", "1"] - restartPolicy: "Never" - serviceAccountName: descheduler-sa - volumes: - - name: policy-volume - configMap: - name: descheduler-policy-configmap \ No newline at end of file diff --git a/examples/descheduler.yaml b/examples/descheduler.yaml deleted file mode 100644 index 3a27cdb46..000000000 --- a/examples/descheduler.yaml +++ /dev/null @@ -1,61 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: descheduler-cluster-role -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list", "delete"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: descheduler-sa - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: descheduler-user -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: descheduler-cluster-role -subjects: - - kind: ServiceAccount - name: descheduler-sa - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: descheduler-policy-configmap - namespace: kube-system -data: - policy.yaml: | - apiVersion: "descheduler/v1alpha1" - kind: "DeschedulerPolicy" - strategies: - "RemoveDuplicates": - enabled: true - "RemovePodsViolatingInterPodAntiAffinity": - enabled: true - "LowNodeUtilization": - enabled: true - params: - nodeResourceUtilizationThresholds: - thresholds: - "cpu" : 20 - "memory": 20 - "pods": 20 - targetThresholds: - "cpu" : 50 - "memory": 50 - "pods": 50 \ No newline at end of file diff --git a/kubernetes/configmap.yaml b/kubernetes/configmap.yaml index b91cfa152..8e346ddf5 100644 --- a/kubernetes/configmap.yaml +++ b/kubernetes/configmap.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: ConfigMap metadata: diff --git a/kubernetes/cronjob.yaml b/kubernetes/cronjob.yaml new file mode 100644 index 000000000..dcb05715c --- /dev/null +++ b/kubernetes/cronjob.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: descheduler-cronjob + namespace: kube-system +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + metadata: + name: descheduler-pod + spec: + priorityClassName: system-cluster-critical + containers: + - name: descheduler + image: docker.io/aveshagarwal/descheduler:0.9.0 # TODO: change to official gcr.io registry when ready + volumeMounts: + - mountPath: /policy-dir + name: policy-volume + command: + - "/bin/descheduler" + args: + - "--policy-config-file" + - "/policy-dir/policy.yaml" + - "--v" + - "3" + restartPolicy: "Never" + serviceAccountName: descheduler-sa + volumes: + - name: policy-volume + configMap: + name: descheduler-policy-configmap diff --git a/kubernetes/job.yaml b/kubernetes/job.yaml index 126a4ea54..6b66cff17 100644 --- a/kubernetes/job.yaml +++ b/kubernetes/job.yaml @@ -1,3 +1,4 @@ +--- apiVersion: batch/v1 kind: Job metadata: @@ -9,12 +10,11 @@ spec: template: metadata: name: descheduler-pod - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" spec: + priorityClassName: system-cluster-critical containers: - name: descheduler - image: descheduler:latest + image: docker.io/aveshagarwal/descheduler:0.9.0 # TODO: change to official gcr.io registry when ready volumeMounts: - mountPath: /policy-dir name: policy-volume From fb8cdc10c7d32ac183d8f44a7d83bcbb809bd755 Mon Sep 17 00:00:00 2001 From: Sean Malloy Date: Sat, 25 Jan 2020 22:39:22 -0600 Subject: [PATCH 2/2] Update Container Image Registry Change the Job and CronJob YAML manifests to use container image registry us.gcr.io/k8s-artifacts-prod/descheduler. This is the new official location for the descheduler container image that is very close to being all setup. --- kubernetes/cronjob.yaml | 2 +- kubernetes/job.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/cronjob.yaml b/kubernetes/cronjob.yaml index dcb05715c..b7b95d189 100644 --- a/kubernetes/cronjob.yaml +++ b/kubernetes/cronjob.yaml @@ -16,7 +16,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: descheduler - image: docker.io/aveshagarwal/descheduler:0.9.0 # TODO: change to official gcr.io registry when ready + image: us.gcr.io/k8s-artifacts-prod/descheduler:v0.10.0 volumeMounts: - mountPath: /policy-dir name: policy-volume diff --git a/kubernetes/job.yaml b/kubernetes/job.yaml index 6b66cff17..bc3ed27ab 100644 --- a/kubernetes/job.yaml +++ b/kubernetes/job.yaml @@ -14,7 +14,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: descheduler - image: docker.io/aveshagarwal/descheduler:0.9.0 # TODO: change to official gcr.io registry when ready + image: us.gcr.io/k8s-artifacts-prod/descheduler:v0.10.0 volumeMounts: - mountPath: /policy-dir name: policy-volume