e2e: add delete apiserver test

e2e/deleteapi_test.go

@@ -0,0 +1,62 @@
+package e2e
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestDeleteAPI(t *testing.T) {
+	apiPods, err := client.CoreV1().Pods("kube-system").List(metav1.ListOptions{LabelSelector: "k8s-app=kube-apiserver"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// delete any api-server pods
+	for i, pod := range apiPods.Items {
+		err := client.CoreV1().Pods("kube-system").Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{})
+		if err != nil {
+			// TODO: if HA we should be able to successfully
+			// delete all. Until then just log if we can't delete
+			// something that isn't the first pod.
+			if i == 0 {
+				t.Fatalf("error deleting api-server pod: %v", err)
+			} else {
+				t.Logf("error deleting api-server pod: %v", err)
+			}
+		}
+	}
+
+	// wait for api-server to go down by waiting until listing pods returns
+	// errors. This is potentially error prone, but without waiting for the
+	// apiserver to go down the next step will return sucess before the
+	// apiserver is ever destroyed.
+	waitDestroy := func() error {
+		// only checking api being down , specific function not important
+		_, err := client.Discovery().ServerVersion()
+
+		if err == nil {
+			return fmt.Errorf("waiting for apiserver to go down: %v", err)
+		}
+		return nil
+	}
+
+	if err := retry(100, 500*time.Millisecond, waitDestroy); err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until api server is back up
+	waitAPI := func() error {
+		// only checking for presence of api returning, specific function not important
+		_, err := client.Discovery().ServerVersion()
+		if err != nil {
+			return fmt.Errorf("waiting for apiserver to return: %v", err)
+		}
+		return nil
+	}
+	if err := retry(30, 10*time.Second, waitAPI); err != nil {
+		t.Fatal(err)
+	}
+}

e2e/etcdscale_test.go

@@ -132,8 +132,8 @@ func resizeSelfHostedEtcd(c kubernetes.Interface, size int) error {
 		return nil
 	}
 
-	if err := retry(15, 10*time.Second, podsReady); err != nil {
-		return fmt.Errorf("Waited 150 seconds for etcd to scale: %v", err)
+	if err := retry(31, 10*time.Second, podsReady); err != nil {
+		return fmt.Errorf("Waited 300 seconds for etcd to scale: %v", err)
 	}
 
 	return nil

hack/terraform-quickstart/iam.tf

@@ -1,6 +1,10 @@
 resource "aws_iam_instance_profile" "bk_profile" {
   name_prefix = "bootkube_e2e_profile"
-  role        = "${aws_iam_role.bk_role.name}"
+  role        = "${aws_iam_role.bk_role.id}"
+
+  provisioner "local-exec" {
+    command = "sleep 90"
+  }
 }
 
 resource "aws_iam_role" "bk_role" {

hack/terraform-quickstart/main.tf

@@ -28,10 +28,10 @@ resource "aws_instance" "worker_node" {
 }
 
 resource "aws_instance" "master_node" {
-  ami           = "${data.aws_ami.coreos_ami.image_id}"
-  instance_type = "m3.medium"
-  key_name      = "${var.ssh_key}"
-  count         = "${var.additional_masters}"
+  ami                  = "${data.aws_ami.coreos_ami.image_id}"
+  instance_type        = "m3.medium"
+  key_name             = "${var.ssh_key}"
+  count                = "${var.additional_masters}"
   iam_instance_profile = "${aws_iam_instance_profile.bk_profile.id}"
 
   tags {