Improves(e2e): improves integration tests and to better run locally

test/e2e/basic_workflow_test.go

@@ -25,99 +25,80 @@ import (
 	"gotest.tools/assert"
 )
 
-var (
-	e env
-	k kn
-)
-
-const (
-	KnDefaultTestImage string = "gcr.io/knative-samples/helloworld-go"
-)
-
-func Setup(t *testing.T) func(t *testing.T) {
-	e = buildEnv(t)
-	k = kn{t, e.Namespace, Logger{}}
-	CreateTestNamespace(t, e.Namespace)
-	return Teardown
-}
-
-func Teardown(t *testing.T) {
-	DeleteTestNamespace(t, e.Namespace)
-}
-
 func TestBasicWorkflow(t *testing.T) {
-	teardown := Setup(t)
-	defer teardown(t)
+	test := NewE2eTest(t)
+	test.Setup(t)
+	defer test.Teardown(t)
 
 	t.Run("returns no service before running tests", func(t *testing.T) {
-		testServiceListEmpty(t, k)
+		test.testServiceListEmpty(t)
 	})
 
 	t.Run("create hello service and returns no error", func(t *testing.T) {
-		testServiceCreate(t, k, "hello")
+		test.testServiceCreate(t, "hello")
 	})
 
 	t.Run("returns valid info about hello service", func(t *testing.T) {
-		testServiceList(t, k, "hello")
-		testServiceDescribe(t, k, "hello")
+		test.testServiceList(t, "hello")
+		test.testServiceDescribe(t, "hello")
 	})
 
 	t.Run("update hello service's configuration and returns no error", func(t *testing.T) {
-		testServiceUpdate(t, k, "hello", []string{"--env", "TARGET=kn", "--port", "8888"})
+		test.testServiceUpdate(t, "hello", []string{"--env", "TARGET=kn", "--port", "8888"})
 	})
 
 	t.Run("create another service and returns no error", func(t *testing.T) {
-		testServiceCreate(t, k, "svc2")
+		test.testServiceCreate(t, "svc2")
 	})
 
 	t.Run("returns a list of revisions associated with hello and svc2 services", func(t *testing.T) {
-		testRevisionListForService(t, k, "hello")
-		testRevisionListForService(t, k, "svc2")
+		test.testRevisionListForService(t, "hello")
+		test.testRevisionListForService(t, "svc2")
 	})
 
 	t.Run("returns a list of routes associated with hello and svc2 services", func(t *testing.T) {
-		testRouteList(t, k)
-		testRouteListWithArgument(t, k, "hello")
+		test.testRouteList(t)
+		test.testRouteListWithArgument(t, "hello")
 	})
 
 	t.Run("delete hello and svc2 services and returns no error", func(t *testing.T) {
-		testServiceDelete(t, k, "hello")
-		testServiceDelete(t, k, "svc2")
+		test.testServiceDelete(t, "hello")
+		test.testServiceDelete(t, "svc2")
 	})
 
 	t.Run("returns no service after completing tests", func(t *testing.T) {
-		testServiceListEmpty(t, k)
+		test.testServiceListEmpty(t)
 	})
 }
 
 // Private test functions
 
-func testServiceListEmpty(t *testing.T, k kn) {
-	out, err := k.RunWithOpts([]string{"service", "list"}, runOpts{NoNamespace: false})
+func (test *e2eTest) testServiceListEmpty(t *testing.T) {
+	out, err := test.kn.RunWithOpts([]string{"service", "list"}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
 	assert.Check(t, util.ContainsAll(out, "No resources found."))
 }
 
-func testServiceCreate(t *testing.T, k kn, serviceName string) {
-	out, err := k.RunWithOpts([]string{"service", "create",
+func (test *e2eTest) testServiceCreate(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "create",
 		fmt.Sprintf("%s", serviceName),
 		"--image", KnDefaultTestImage}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
-	assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully created in namespace", k.namespace, "OK"))
+	assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully created in namespace", test.kn.namespace, "OK"))
 }
 
-func testServiceList(t *testing.T, k kn, serviceName string) {
-	out, err := k.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
+func (test *e2eTest) testServiceList(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "list", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
 	expectedOutput := fmt.Sprintf("%s", serviceName)
 	assert.Check(t, util.ContainsAll(out, expectedOutput))
 }
 
-func testRevisionListForService(t *testing.T, k kn, serviceName string) {
-	out, err := k.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false})
+func (test *e2eTest) testRevisionListForService(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"revision", "list", "-s", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
 	outputLines := strings.Split(out, "\n")
@@ -129,8 +110,8 @@ func testRevisionListForService(t *testing.T, k kn, serviceName string) {
 	}
 }
 
-func testServiceDescribe(t *testing.T, k kn, serviceName string) {
-	out, err := k.RunWithOpts([]string{"service", "describe", serviceName}, runOpts{NoNamespace: false})
+func (test *e2eTest) testServiceDescribe(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "describe", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
 	expectedOutputHeader := `apiVersion: serving.knative.dev/v1alpha1
@@ -139,37 +120,37 @@ metadata:`
 	expectedOutput := `generation: 1
   name: %s
   namespace: %s`
-	expectedOutput = fmt.Sprintf(expectedOutput, serviceName, k.namespace)
+	expectedOutput = fmt.Sprintf(expectedOutput, serviceName, test.kn.namespace)
 	assert.Check(t, util.ContainsAll(out, expectedOutputHeader, expectedOutput))
 }
 
-func testServiceUpdate(t *testing.T, k kn, serviceName string, args []string) {
-	out, err := k.RunWithOpts(append([]string{"service", "update", serviceName}, args...), runOpts{NoNamespace: false})
+func (test *e2eTest) testServiceUpdate(t *testing.T, serviceName string, args []string) {
+	out, err := test.kn.RunWithOpts(append([]string{"service", "update", serviceName}, args...), runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
 	expectedOutput := fmt.Sprintf("Service '%s' updated", serviceName)
 	assert.Check(t, util.ContainsAll(out, expectedOutput))
 }
 
-func testRouteList(t *testing.T, k kn) {
-	out, err := k.RunWithOpts([]string{"route", "list"}, runOpts{})
+func (test *e2eTest) testRouteList(t *testing.T) {
+	out, err := test.kn.RunWithOpts([]string{"route", "list"}, runOpts{})
 	assert.NilError(t, err)
 
 	expectedHeaders := []string{"NAME", "URL", "AGE", "CONDITIONS", "TRAFFIC"}
 	assert.Check(t, util.ContainsAll(out, expectedHeaders...))
 }
 
-func testRouteListWithArgument(t *testing.T, k kn, routeName string) {
-	out, err := k.RunWithOpts([]string{"route", "list", routeName}, runOpts{})
+func (test *e2eTest) testRouteListWithArgument(t *testing.T, routeName string) {
+	out, err := test.kn.RunWithOpts([]string{"route", "list", routeName}, runOpts{})
 	assert.NilError(t, err)
 
 	expectedOutput := fmt.Sprintf("100%% -> %s", routeName)
 	assert.Check(t, util.ContainsAll(out, routeName, expectedOutput))
 }
 
-func testServiceDelete(t *testing.T, k kn, serviceName string) {
-	out, err := k.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false})
+func (test *e2eTest) testServiceDelete(t *testing.T, serviceName string) {
+	out, err := test.kn.RunWithOpts([]string{"service", "delete", serviceName}, runOpts{NoNamespace: false})
 	assert.NilError(t, err)
 
-	assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully deleted in namespace", k.namespace))
+	assert.Check(t, util.ContainsAll(out, "Service", serviceName, "successfully deleted in namespace", test.kn.namespace))
 }

test/e2e/common.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 )
 
 type runOpts struct {
@@ -35,22 +36,53 @@ type runOpts struct {
 	Redact       bool
 }
 
+const (
+	KnDefaultTestImage string        = "gcr.io/knative-samples/helloworld-go"
+	MaxRetries         int           = 10
+	RetrySleepDuration time.Duration = 30 * time.Second
+)
+
+type e2eTest struct {
+	env env
+	kn  kn
+}
+
+func NewE2eTest(t *testing.T) *e2eTest {
+	return &e2eTest{
+		env: buildEnv(t),
+	}
+}
+
+// Setup set up an enviroment for kn integration test returns the Teardown cleanup function
+func (test *e2eTest) Setup(t *testing.T) {
+	test.env.Namespace = fmt.Sprintf("%s%d", test.env.Namespace, namespaceCount)
+	namespaceCount++
+	test.kn = kn{t, test.env.Namespace, Logger{}}
+	test.CreateTestNamespace(t, test.env.Namespace)
+}
+
+// Teardown clean up
+func (test *e2eTest) Teardown(t *testing.T) {
+	test.DeleteTestNamespace(t, test.env.Namespace)
+}
+
 // CreateTestNamespace creates and tests a namesspace creation invoking kubectl
-func CreateTestNamespace(t *testing.T, namespace string) {
-	kubectl := kubectl{t, Logger{}}
-	out, err := kubectl.RunWithOpts([]string{"create", "namespace", namespace}, runOpts{})
+func (test *e2eTest) CreateTestNamespace(t *testing.T, namespace string) {
+	logger := Logger{}
+	expectedOutputRegexp := fmt.Sprintf("namespace?.+%s.+created", namespace)
+	out, err := createNamespace(t, namespace, MaxRetries, logger)
 	if err != nil {
-		t.Fatalf(fmt.Sprintf("Error executing 'kubectl create namespace' command. Error: %s", err.Error()))
+		logger.Fatalf("Could not create namespace, giving up")
 	}
 
-	expectedOutputRegexp := fmt.Sprintf("namespace?.+%s.+created", namespace)
+	// check that last output indeed show created namespace
 	if !matchRegexp(t, expectedOutputRegexp, out) {
 		t.Fatalf("Expected output incorrect, expecting to include:\n%s\n Instead found:\n%s\n", expectedOutputRegexp, out)
 	}
 }
 
 // CreateTestNamespace deletes and tests a namesspace deletion invoking kubectl
-func DeleteTestNamespace(t *testing.T, namespace string) {
+func (test *e2eTest) DeleteTestNamespace(t *testing.T, namespace string) {
 	kubectl := kubectl{t, Logger{}}
 	out, err := kubectl.RunWithOpts([]string{"delete", "namespace", namespace}, runOpts{})
 	if err != nil {
@@ -63,7 +95,61 @@ func DeleteTestNamespace(t *testing.T, namespace string) {
 	}
 }
 
+// WaitForNamespaceDeleted wait until namespace is deleted
+func (test *e2eTest) WaitForNamespaceDeleted(t *testing.T, namespace string) {
+	logger := Logger{}
+	deleted := checkNamespaceDeleted(t, namespace, MaxRetries, logger)
+	if !deleted {
+		t.Fatalf(fmt.Sprintf("Error deleting namespace %s, timed out", namespace))
+	}
+}
+
 // Private functions
+func checkNamespaceDeleted(t *testing.T, namespace string, maxRetries int, logger Logger) bool {
+	kubectlGetNamespace := func() (string, error) {
+		kubectl := kubectl{t, logger}
+		return kubectl.RunWithOpts([]string{"get", "namespace"}, runOpts{})
+	}
+
+	retries := 0
+	for retries < MaxRetries {
+		output, _ := kubectlGetNamespace()
+		if !strings.Contains(output, namespace) {
+			return true
+		}
+
+		retries++
+		logger.Debugf("Namespace is terminating, waiting %ds, and trying again: %d of %d\n", int(RetrySleepDuration.Seconds()), retries, maxRetries)
+		time.Sleep(RetrySleepDuration)
+	}
+
+	return true
+}
+
+func createNamespace(t *testing.T, namespace string, maxRetries int, logger Logger) (string, error) {
+	kubectlCreateNamespace := func() (string, error) {
+		kubectl := kubectl{t, logger}
+		return kubectl.RunWithOpts([]string{"create", "namespace", namespace}, runOpts{AllowError: true})
+	}
+
+	var (
+		retries int
+		err     error
+		out     string
+	)
+
+	for retries < maxRetries {
+		out, err := kubectlCreateNamespace()
+		if err == nil {
+			return out, nil
+		}
+		retries++
+		logger.Debugf("Could not create namespace, waiting %ds, and trying again: %d of %d\n", int(RetrySleepDuration.Seconds()), retries, maxRetries)
+		time.Sleep(RetrySleepDuration)
+	}
+
+	return out, err
+}
 
 func runCLIWithOpts(cli string, args []string, opts runOpts, logger Logger) (string, error) {
 	logger.Debugf("Running '%s'...\n", cmdCLIDesc(cli, args))

test/e2e/env.go

@@ -26,6 +26,8 @@ type env struct {
 
 const defaultKnE2ENamespace = "kne2etests"
 
+var namespaceCount = 0
+
 func buildEnv(t *testing.T) env {
 	env := env{
 		Namespace: os.Getenv("KN_E2E_NAMESPACE"),

test/e2e/revision_workflow_test.go

@@ -25,31 +25,34 @@ import (
 )
 
 func TestRevisionWorkflow(t *testing.T) {
-	teardown := Setup(t)
-	defer teardown(t)
+	test := NewE2eTest(t)
+	test.Setup(t)
+	defer test.Teardown(t)
 
 	t.Run("create hello service and returns no error", func(t *testing.T) {
-		testServiceCreate(t, k, "hello")
+		test.testServiceCreate(t, "hello")
 	})
 
 	t.Run("delete latest revision from hello service and returns no error", func(t *testing.T) {
-		testDeleteRevision(t, k, "hello")
+		test.testDeleteRevision(t, "hello")
 	})
 
 	t.Run("delete hello service and returns no error", func(t *testing.T) {
-		testServiceDelete(t, k, "hello")
+		test.testServiceDelete(t, "hello")
 	})
 }
 
-func testDeleteRevision(t *testing.T, k kn, serviceName string) {
-	revName, err := k.RunWithOpts([]string{"revision", "list", "-o=jsonpath={.items[0].metadata.name}"}, runOpts{})
+// Private
+
+func (test *e2eTest) testDeleteRevision(t *testing.T, serviceName string) {
+	revName, err := test.kn.RunWithOpts([]string{"revision", "list", "-o=jsonpath={.items[0].metadata.name}"}, runOpts{})
 	assert.NilError(t, err)
 	if strings.Contains(revName, "No resources found.") {
 		t.Errorf("Could not find revision name.")
 	}
 
-	out, err := k.RunWithOpts([]string{"revision", "delete", revName}, runOpts{})
+	out, err := test.kn.RunWithOpts([]string{"revision", "delete", revName}, runOpts{})
 	assert.NilError(t, err)
 
-	assert.Check(t, util.ContainsAll(out, "Revision", revName, "deleted", "namespace", k.namespace))
+	assert.Check(t, util.ContainsAll(out, "Revision", revName, "deleted", "namespace", test.kn.namespace))
 }