diff --git a/.travis.yml b/.travis.yml index 24a53b87c..5b4edade1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,4 +4,4 @@ go: script: - hack/verify-gofmt.sh - make build -- make test +- make test-unit diff --git a/Makefile b/Makefile index 147ed8f82..8f37c0e03 100644 --- a/Makefile +++ b/Makefile @@ -41,5 +41,8 @@ image: clean: rm -rf _output -test: +test-unit: ./test/run-unit-tests.sh + +test-e2e: + ./test/run-e2e-tests.sh diff --git a/hack/e2e-gce/gcloud_create_cluster.sh b/hack/e2e-gce/gcloud_create_cluster.sh new file mode 100755 index 000000000..8e4be025f --- /dev/null +++ b/hack/e2e-gce/gcloud_create_cluster.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +echo "Make sure that uuid package is installed" + +master_uuid=$(uuid) +node1_uuid=$(uuid) +node2_uuid=$(uuid) +kube_apiserver_port=6443 +kube_version=1.9.4 + +DESCHEDULER_ROOT=$(dirname "${BASH_SOURCE}")/../../ +E2E_GCE_HOME=$DESCHEDULER_ROOT/hack/e2e-gce + + +create_cluster() { + echo "#################### Creating instances ##########################" + gcloud compute instances create descheduler-$master_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b + # Keeping the --zone here so as to make sure that e2e's can run locally. + echo "gcloud compute instances delete descheduler-$master_uuid --zone=us-east1-b --quiet" > $E2E_GCE_HOME/delete_cluster.sh + + gcloud compute instances create descheduler-$node1_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b + echo "gcloud compute instances delete descheduler-$node1_uuid --zone=us-east1-b --quiet" >> $E2E_GCE_HOME/delete_cluster.sh + + gcloud compute instances create descheduler-$node2_uuid --image="ubuntu-1604-xenial-v20180306" --image-project="ubuntu-os-cloud" --zone=us-east1-b + echo "gcloud compute instances delete descheduler-$node2_uuid --zone=us-east1-b --quiet" >> $E2E_GCE_HOME/delete_cluster.sh + + # Delete the firewall port created for master. + echo "gcloud compute firewall-rules delete kubeapiserver-$master_uuid --quiet" >> $E2E_GCE_HOME/delete_cluster.sh + chmod 755 $E2E_GCE_HOME/delete_cluster.sh +} + + +generate_kubeadm_instance_files() { + # TODO: Check if they have come up. awk $6 contains the state(RUNNING or not). + master_public_ip=$(gcloud compute instances list | grep $master_uuid|awk '{print $5}') + node1_public_ip=$(gcloud compute instances list | grep $node1_uuid|awk '{print $5}') + node2_public_ip=$(gcloud compute instances list | grep $node2_uuid|awk '{print $5}') + echo "kubeadm init --kubernetes-version=${kube_version} --apiserver-advertise-address=${master_public_ip}" --skip-preflight-checks --pod-network-cidr=10.96.0.0/12 > $E2E_GCE_HOME/kubeadm_install.sh +} + + +transfer_install_files() { + gcloud compute scp $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$master_uuid:/tmp --zone=us-east1-b + gcloud compute scp $E2E_GCE_HOME/kubeadm_install.sh descheduler-$master_uuid:/tmp --zone=us-east1-b + gcloud compute scp $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$node1_uuid:/tmp --zone=us-east1-b + gcloud compute scp $E2E_GCE_HOME/kubeadm_preinstall.sh descheduler-$node2_uuid:/tmp --zone=us-east1-b +} + + +install_kube() { + # Docker installation. + gcloud compute ssh descheduler-$master_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b + gcloud compute ssh descheduler-$node1_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b + gcloud compute ssh descheduler-$node2_uuid --command "sudo apt-get update; sudo apt-get install -y docker.io" --zone=us-east1-b + # kubeadm installation. + # 1. Transfer files to master, nodes. + transfer_install_files + # 2. Install kubeadm. + #TODO: Add rm /tmp/kubeadm_install.sh + # Open port for kube API server + gcloud compute firewall-rules create kubeapiserver-$master_uuid --allow tcp:6443 --source-tags=descheduler-$master_uuid --source-ranges=0.0.0.0/0 --description="Opening api server port" + + gcloud compute ssh descheduler-$master_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b + kubeadm_join_command=$(gcloud compute ssh descheduler-$master_uuid --command "sudo chmod 755 /tmp/kubeadm_install.sh; sudo /tmp/kubeadm_install.sh" --zone=us-east1-b|grep 'kubeadm join') + + # Copy the kubeconfig file onto /tmp for e2e tests. + gcloud compute ssh descheduler-$master_uuid --command "sudo cp /etc/kubernetes/admin.conf /tmp; sudo chmod 777 /tmp/admin.conf" --zone=us-east1-b + gcloud compute scp descheduler-$master_uuid:/tmp/admin.conf /tmp/admin.conf --zone=us-east1-b + + # Postinstall on master, need to add a network plugin for kube-dns to come to running state. + gcloud compute ssh descheduler-$master_uuid --command "sudo kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml --kubeconfig /etc/kubernetes/admin.conf" --zone=us-east1-b + echo $kubeadm_join_command > $E2E_GCE_HOME/kubeadm_join.sh + + # Copy kubeadm_join to every node. + #TODO: Put these in a loop, so that extension becomes possible. + gcloud compute ssh descheduler-$node1_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b + gcloud compute scp $E2E_GCE_HOME/kubeadm_join.sh descheduler-$node1_uuid:/tmp --zone=us-east1-b + gcloud compute ssh descheduler-$node1_uuid --command "sudo chmod 755 /tmp/kubeadm_join.sh; sudo /tmp/kubeadm_join.sh" --zone=us-east1-b + + gcloud compute ssh descheduler-$node2_uuid --command "sudo chmod 755 /tmp/kubeadm_preinstall.sh; sudo /tmp/kubeadm_preinstall.sh" --zone=us-east1-b + gcloud compute scp $E2E_GCE_HOME/kubeadm_join.sh descheduler-$node2_uuid:/tmp --zone=us-east1-b + gcloud compute ssh descheduler-$node2_uuid --command "sudo chmod 755 /tmp/kubeadm_join.sh; sudo /tmp/kubeadm_join.sh" --zone=us-east1-b + +} + + +create_cluster + +generate_kubeadm_instance_files + +install_kube diff --git a/hack/e2e-gce/gcloud_sdk_configure.sh b/hack/e2e-gce/gcloud_sdk_configure.sh new file mode 100755 index 000000000..8dd8ed1ca --- /dev/null +++ b/hack/e2e-gce/gcloud_sdk_configure.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + + +gcloud auth activate-service-account --key-file "${GCE_SA_CREDS}" +gcloud config set project $GCE_PROJECT_ID +gcloud config set compute/zone $GCE_ZONE diff --git a/hack/e2e-gce/install_gcloud.sh b/hack/e2e-gce/install_gcloud.sh new file mode 100755 index 000000000..688ff1eb6 --- /dev/null +++ b/hack/e2e-gce/install_gcloud.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-176.0.0-linux-x86_64.tar.gz + +tar -xvzf google-cloud-sdk-176.0.0-linux-x86_64.tar.gz + +./google-cloud-sdk/install.sh -q diff --git a/hack/e2e-gce/kubeadm_preinstall.sh b/hack/e2e-gce/kubeadm_preinstall.sh new file mode 100644 index 000000000..157834615 --- /dev/null +++ b/hack/e2e-gce/kubeadm_preinstall.sh @@ -0,0 +1,11 @@ +apt-get update +apt-get install -y docker.io + +apt-get update && apt-get install -y apt-transport-https +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +cat </etc/apt/sources.list.d/kubernetes.list +deb http://apt.kubernetes.io/ kubernetes-xenial main +EOF +apt-get update +apt-get install -y kubelet kubeadm kubectl +exit 0 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 000000000..b83f5e270 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/golang/glog" + "testing" + "time" + + "github.com/kubernetes-incubator/descheduler/cmd/descheduler/app/options" + deschedulerapi "github.com/kubernetes-incubator/descheduler/pkg/api" + "github.com/kubernetes-incubator/descheduler/pkg/descheduler/client" + eutils "github.com/kubernetes-incubator/descheduler/pkg/descheduler/evictions/utils" + nodeutil "github.com/kubernetes-incubator/descheduler/pkg/descheduler/node" + podutil "github.com/kubernetes-incubator/descheduler/pkg/descheduler/pod" + "github.com/kubernetes-incubator/descheduler/pkg/descheduler/strategies" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/api/testapi" +) + +func MakePodSpec() v1.PodSpec { + return v1.PodSpec{ + Containers: []v1.Container{{ + Name: "pause", + Image: "kubernetes/pause", + Ports: []v1.ContainerPort{{ContainerPort: 80}}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("500Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("500Mi"), + }, + }, + }}, + } +} + +// RcByNameContainer returns a ReplicationControoler with specified name and container +func RcByNameContainer(name string, replicas int32, labels map[string]string, gracePeriod *int64) *v1.ReplicationController { + + zeroGracePeriod := int64(0) + + // Add "name": name to the labels, overwriting if it exists. + labels["name"] = name + if gracePeriod == nil { + gracePeriod = &zeroGracePeriod + } + return &v1.ReplicationController{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + APIVersion: testapi.Groups[v1.GroupName].GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.ReplicationControllerSpec{ + Replicas: func(i int32) *int32 { return &i }(replicas), + Selector: map[string]string{ + "name": name, + }, + Template: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: MakePodSpec(), + }, + }, + } +} + +// startEndToEndForLowNodeUtilization tests the lownode utilization strategy. +func startEndToEndForLowNodeUtilization(clientset clientset.Interface) { + var thresholds = make(deschedulerapi.ResourceThresholds) + var targetThresholds = make(deschedulerapi.ResourceThresholds) + thresholds[v1.ResourceMemory] = 20 + thresholds[v1.ResourcePods] = 20 + thresholds[v1.ResourceCPU] = 85 + targetThresholds[v1.ResourceMemory] = 20 + targetThresholds[v1.ResourcePods] = 20 + targetThresholds[v1.ResourceCPU] = 90 + // Run descheduler. + evictionPolicyGroupVersion, err := eutils.SupportEviction(clientset) + if err != nil || len(evictionPolicyGroupVersion) == 0 { + glog.Fatalf("%v", err) + } + stopChannel := make(chan struct{}) + nodes, err := nodeutil.ReadyNodes(clientset, "", stopChannel) + if err != nil { + glog.Fatalf("%v", err) + } + nodeUtilizationThresholds := deschedulerapi.NodeResourceUtilizationThresholds{Thresholds: thresholds, TargetThresholds: targetThresholds} + nodeUtilizationStrategyParams := deschedulerapi.StrategyParameters{NodeResourceUtilizationThresholds: nodeUtilizationThresholds} + lowNodeUtilizationStrategy := deschedulerapi.DeschedulerStrategy{Enabled: true, Params: nodeUtilizationStrategyParams} + ds := &options.DeschedulerServer{Client: clientset} + strategies.LowNodeUtilization(ds, lowNodeUtilizationStrategy, evictionPolicyGroupVersion, nodes) + time.Sleep(10 * time.Second) + + return +} + +func TestE2E(t *testing.T) { + // If we have reached here, it means cluster would have been already setup and the kubeconfig file should + // be in /tmp directory. + clientSet, err := client.CreateClient("/tmp/admin.conf") + if err != nil { + t.Errorf("Error during client creation with %v", err) + } + nodeList, err := clientSet.Core().Nodes().List(metav1.ListOptions{}) + if err != nil { + t.Errorf("Error listing node with %v", err) + } + // Assumption: We would have 3 node cluster by now. Kubeadm brings all the master components onto master node. + // So, the last node would have least utilization. + leastLoadedNode := nodeList.Items[2] + rc := RcByNameContainer("test-rc", int32(15), map[string]string{"test": "app"}, nil) + _, err = clientSet.CoreV1().ReplicationControllers("default").Create(rc) + if err != nil { + t.Errorf("Error creating deployment %v", err) + } + podsOnleastUtilizedNode, err := podutil.ListPodsOnANode(clientSet, &leastLoadedNode) + if err != nil { + t.Errorf("Error listing pods on a node %v", err) + } + podsBefore := len(podsOnleastUtilizedNode) + t.Log("Eviction of pods starting") + startEndToEndForLowNodeUtilization(clientSet) + podsOnleastUtilizedNode, err = podutil.ListPodsOnANode(clientSet, &leastLoadedNode) + if err != nil { + t.Errorf("Error listing pods on a node %v", err) + } + podsAfter := len(podsOnleastUtilizedNode) + if podsBefore > podsAfter { + t.Fatalf("We should have see more pods on this node as per kubeadm's way of installing %v, %v", podsBefore, podsAfter) + } +} diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh new file mode 100755 index 000000000..2e365078e --- /dev/null +++ b/test/run-e2e-tests.sh @@ -0,0 +1,19 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# This just run e2e tests. +go test github.com/kubernetes-incubator/descheduler/test/e2e/ -v + diff --git a/test/run-unit-tests.sh b/test/run-unit-tests.sh index a9f5ce422..ae85f23a0 100755 --- a/test/run-unit-tests.sh +++ b/test/run-unit-tests.sh @@ -14,6 +14,6 @@ #!/bin/bash -# run unit tests -go test $(go list github.com/kubernetes-incubator/descheduler/... | grep -v github.com/kubernetes-incubator/descheduler/vendor/) +# This just run unit-tests. Ignoring the current directory so as to avoid running e2e tests. +go test $(go list github.com/kubernetes-incubator/descheduler/... | grep -v github.com/kubernetes-incubator/descheduler/vendor/| grep -v github.com/kubernetes-incubator/descheduler/test/)