diff --git a/README.md b/README.md index 10ebae94f..ff9a1af7c 100644 --- a/README.md +++ b/README.md @@ -47,120 +47,40 @@ For more information about available options run: $ ./_output/bin/descheduler --help ``` -## Running Descheduler as a Job Inside of a Pod +## Running Descheduler as a Job or CronJob -Descheduler can be run as a job inside of a pod. It has the advantage of +The descheduler can be run as a job or cronjob inside of a pod. It has the advantage of being able to be run multiple times without needing user intervention. -Descheduler pod is run as a critical pod to avoid being evicted by itself, -or by kubelet due to an eviction event. Since critical pods are created in -`kube-system` namespace, descheduler job and its pod will also be created +The descheduler pod is run as a critical pod to avoid being evicted by itself, +or by the kubelet due to an eviction event. Since critical pods are created in the +`kube-system` namespace, the descheduler job and its pod will also be created in `kube-system` namespace. -### Create a container image +### Setup RBAC -First we create a simple Docker image utilizing the Dockerfile found in the root directory: +To give necessary permissions for the descheduler to work in a pod. ``` -$ make dev-image +$ kubectl create -f kubernetes/rbac.yaml ``` -This creates an image based off the binary we've built before. To build both the -binary and image in one step you can run the following command: - -``` -$ make image -``` - -This eliminates the need to have Go installed locally and builds the binary -within it's own container. - -### Create a cluster role - -To give necessary permissions for the descheduler to work in a pod, create a cluster role: - -``` -$ cat << EOF| kubectl create -f - -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: descheduler-cluster-role -rules: -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "watch", "list"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list", "delete"] -- apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] -EOF -``` - -### Create the service account which will be used to run the job: - -``` -$ kubectl create sa descheduler-sa -n kube-system -``` - -### Bind the cluster role to the service account: - -``` -$ kubectl create clusterrolebinding descheduler-cluster-role-binding \ - --clusterrole=descheduler-cluster-role \ - --serviceaccount=kube-system:descheduler-sa -``` ### Create a configmap to store descheduler policy -Descheduler policy is created as a ConfigMap in `kube-system` namespace -so that it can be mounted as a volume inside pod. - ``` -$ kubectl create configmap descheduler-policy-configmap \ - -n kube-system --from-file= -``` -### Create the job specification (descheduler-job.yaml) - -``` -apiVersion: batch/v1 -kind: Job -metadata: - name: descheduler-job - namespace: kube-system -spec: - parallelism: 1 - completions: 1 - template: - metadata: - name: descheduler-pod - spec: - priorityClassName: system-cluster-critical - containers: - - name: descheduler - image: descheduler - volumeMounts: - - mountPath: /policy-dir - name: policy-volume - command: ["/bin/descheduler", "--policy-config-file", "/policy-dir/policy.yaml", "-v", "1"] - restartPolicy: "Never" - serviceAccountName: descheduler-sa - volumes: - - name: policy-volume - configMap: - name: descheduler-policy-configmap +$ kubectl create -f kubernetes/configmap.yaml ``` -Please note that the pod template is configured with critical pod annotation, and -the policy `policy-file` is mounted as a volume from the config map. +### Create a Job or CronJob -### Run the descheduler as a job in a pod: +As a Job. ``` -$ kubectl create -f descheduler-job.yaml +$ kubectl create -f kubernetes/job.yaml ``` -### Examples - -See [descheduler.yaml](examples/descheduler.yaml) and [descheduler-job.yaml](examples/descheduler-job.yaml) for a combined yaml file of the above steps. +Or as a CronJob. +``` +$ kubectl create -f kubernetes/cronjob.yaml +``` ## Policy and Strategies diff --git a/examples/descheduler-job.yaml b/examples/descheduler-job.yaml deleted file mode 100644 index 83ad7c649..000000000 --- a/examples/descheduler-job.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: descheduler-job - namespace: kube-system -spec: - parallelism: 1 - completions: 1 - template: - metadata: - name: descheduler-pod - spec: - priorityClassName: system-cluster-critical - containers: - - name: descheduler - image: docker.io/aveshagarwal/descheduler:0.9.0 - volumeMounts: - - mountPath: /policy-dir - name: policy-volume - command: ["/bin/descheduler", "--policy-config-file", "/policy-dir/policy.yaml", "-v", "1"] - restartPolicy: "Never" - serviceAccountName: descheduler-sa - volumes: - - name: policy-volume - configMap: - name: descheduler-policy-configmap \ No newline at end of file diff --git a/examples/descheduler.yaml b/examples/descheduler.yaml deleted file mode 100644 index 3a27cdb46..000000000 --- a/examples/descheduler.yaml +++ /dev/null @@ -1,61 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: descheduler-cluster-role -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list", "delete"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: descheduler-sa - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: descheduler-user -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: descheduler-cluster-role -subjects: - - kind: ServiceAccount - name: descheduler-sa - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: descheduler-policy-configmap - namespace: kube-system -data: - policy.yaml: | - apiVersion: "descheduler/v1alpha1" - kind: "DeschedulerPolicy" - strategies: - "RemoveDuplicates": - enabled: true - "RemovePodsViolatingInterPodAntiAffinity": - enabled: true - "LowNodeUtilization": - enabled: true - params: - nodeResourceUtilizationThresholds: - thresholds: - "cpu" : 20 - "memory": 20 - "pods": 20 - targetThresholds: - "cpu" : 50 - "memory": 50 - "pods": 50 \ No newline at end of file diff --git a/kubernetes/configmap.yaml b/kubernetes/configmap.yaml index b91cfa152..8e346ddf5 100644 --- a/kubernetes/configmap.yaml +++ b/kubernetes/configmap.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: ConfigMap metadata: diff --git a/kubernetes/cronjob.yaml b/kubernetes/cronjob.yaml new file mode 100644 index 000000000..b7b95d189 --- /dev/null +++ b/kubernetes/cronjob.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: descheduler-cronjob + namespace: kube-system +spec: + schedule: "*/2 * * * *" + concurrencyPolicy: "Forbid" + jobTemplate: + spec: + template: + metadata: + name: descheduler-pod + spec: + priorityClassName: system-cluster-critical + containers: + - name: descheduler + image: us.gcr.io/k8s-artifacts-prod/descheduler:v0.10.0 + volumeMounts: + - mountPath: /policy-dir + name: policy-volume + command: + - "/bin/descheduler" + args: + - "--policy-config-file" + - "/policy-dir/policy.yaml" + - "--v" + - "3" + restartPolicy: "Never" + serviceAccountName: descheduler-sa + volumes: + - name: policy-volume + configMap: + name: descheduler-policy-configmap diff --git a/kubernetes/job.yaml b/kubernetes/job.yaml index 126a4ea54..bc3ed27ab 100644 --- a/kubernetes/job.yaml +++ b/kubernetes/job.yaml @@ -1,3 +1,4 @@ +--- apiVersion: batch/v1 kind: Job metadata: @@ -9,12 +10,11 @@ spec: template: metadata: name: descheduler-pod - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" spec: + priorityClassName: system-cluster-critical containers: - name: descheduler - image: descheduler:latest + image: us.gcr.io/k8s-artifacts-prod/descheduler:v0.10.0 volumeMounts: - mountPath: /policy-dir name: policy-volume