Added envvar (CLEANUPDEPLOYMENTS) to cleanup failed tests.

Jenkinsfile.groovy

@@ -78,6 +78,7 @@ def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) {
         timestamps {
             withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { 
                 withEnv([
+                "CLEANDEPLOYMENTS=1",
                 "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}",
                 "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}",
                 "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}",

Makefile

@@ -80,7 +80,7 @@ TESTLENGTHOPTIONS := -test.short
 TESTTIMEOUT := 20m
 ifeq ($(LONG), 1)
 	TESTLENGTHOPTIONS :=
-	TESTTIMEOUT := 60m
+	TESTTIMEOUT := 120m
 endif
 ifdef VERBOSE
 	TESTVERBOSEOPTIONS := -v 

scripts/kube_run_tests.sh

@@ -15,5 +15,6 @@ kubectl --namespace ${DEPLOYMENTNAMESPACE} \
     --image=${IMAGEID} \
     --env="ENTERPRISEIMAGE=${ENTERPRISEIMAGE}" \
     --env="TEST_NAMESPACE=${DEPLOYMENTNAMESPACE}" \
+    --env="CLEANDEPLOYMENTS=${CLEANDEPLOYMENTS}" \
     -- \
     -test.v -test.timeout $TESTTIMEOUT $TESTLENGTHOPTIONS $TESTOPTIONS

tests/auth_test.go

@@ -33,6 +33,7 @@ func TestAuthenticationSingleDefaultSecret(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -86,6 +87,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -106,7 +108,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) {
 
 	// Secret must still exist
 	if _, err := waitUntilSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns, nil, time.Second); err != nil {
-		t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.JWTSecretName, err)
+		t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.GetJWTSecretName(), err)
 	}
 
 	// Cleanup secret
@@ -132,6 +134,7 @@ func TestAuthenticationNoneSingle(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -169,6 +172,7 @@ func TestAuthenticationClusterDefaultSecret(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -222,6 +226,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -242,7 +247,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) {
 
 	// Secret must still exist
 	if _, err := waitUntilSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns, nil, time.Second); err != nil {
-		t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.JWTSecretName, err)
+		t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.GetJWTSecretName(), err)
 	}
 
 	// Cleanup secret
@@ -268,6 +273,7 @@ func TestAuthenticationNoneCluster(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {

tests/deployments_test.go

@@ -84,6 +84,7 @@ func deploymentSubTest(t *testing.T, mode api.DeploymentMode, engine api.Storage
 	// Create deployment
 	_, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate)
 	require.NoError(t, err, fmt.Sprintf("Create deployment failed: %v", err))
+	defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace)
 
 	// Wait for deployment to be ready
 	deployment, err := waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())
@@ -124,9 +125,11 @@ func TestMultiDeployment(t *testing.T) {
 	// Create deployments
 	_, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate1)
 	require.NoError(t, err, fmt.Sprintf("Deployment creation failed: %v", err))
+	defer deferedCleanupDeployment(deploymentClient, deploymentTemplate1.GetName(), k8sNameSpace)
 
 	_, err = deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate2)
 	require.NoError(t, err, fmt.Sprintf("Deployment creation failed: %v", err))
+	defer deferedCleanupDeployment(deploymentClient, deploymentTemplate2.GetName(), k8sNameSpace)
 
 	// Wait for deployments to be ready
 	deployment1, err := waitUntilDeployment(deploymentClient, deploymentTemplate1.GetName(), k8sNameSpace, deploymentIsReady())

tests/environments_test.go

@@ -75,6 +75,7 @@ func TestProduction(t *testing.T) {
 		// REVIEW - should the test already fail here
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace)
 
 	_, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())
 	assert.Error(t, err, fmt.Sprintf("Deployment is up and running when it should not! There are not enough nodes(%d) for all DBServers(%d) in production modes.", numNodes, dbserverCount))

tests/immutable_test.go

@@ -54,6 +54,7 @@ func TestImmutableFields(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {

tests/member_resilience_test.go

@@ -34,6 +34,7 @@ func TestMemberResilienceAgents(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -134,6 +135,7 @@ func TestMemberResilienceCoordinators(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -230,6 +232,7 @@ func TestMemberResilienceDBServers(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {

tests/resilience_test.go

@@ -37,6 +37,7 @@ func TestResiliencePod(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -117,6 +118,7 @@ func TestResiliencePVC(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -201,6 +203,7 @@ func TestResiliencePVDBServer(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -304,6 +307,7 @@ func TestResilienceService(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {

tests/rocksdb_encryption_test.go

@@ -47,6 +47,7 @@ func TestRocksDBEncryptionSingle(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady())

tests/scale_test.go

@@ -31,6 +31,7 @@ func TestScaleClusterNonTLS(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {
@@ -101,6 +102,7 @@ func TestScaleCluster(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
 
 	// Wait for deployment to be ready
 	if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil {

tests/test_util.go

@@ -71,6 +71,12 @@ func getEnterpriseImageOrSkip(t *testing.T) string {
 	return image
 }
 
+// shouldCleanDeployments returns true when deployments created
+// by tests should be removed, even when the test fails.
+func shouldCleanDeployments() bool {
+	return os.Getenv("CLEANDEPLOYMENTS") != ""
+}
+
 // mustNewKubeClient creates a kubernetes client
 // failing the test on errors.
 func mustNewKubeClient(t *testing.T) kubernetes.Interface {
@@ -307,6 +313,18 @@ func removeDeployment(cli versioned.Interface, deploymentName, ns string) error
 	return nil
 }
 
+// deferedCleanupDeployment removes a deployment when shouldCleanDeployments return true.
+// This function is intended to be used in a defer statement.
+func deferedCleanupDeployment(cli versioned.Interface, deploymentName, ns string) error {
+	if !shouldCleanDeployments() {
+		return nil
+	}
+	if err := removeDeployment(cli, deploymentName, ns); err != nil {
+		return maskAny(err)
+	}
+	return nil
+}
+
 // removeSecret removes a secret
 func removeSecret(cli kubernetes.Interface, secretName, ns string) error {
 	if err := cli.CoreV1().Secrets(ns).Delete(secretName, nil); err != nil && k8sutil.IsNotFound(err) {

tests/upgrade_test.go

@@ -99,6 +99,7 @@ func upgradeSubTest(t *testing.T, mode api.DeploymentMode, engine api.StorageEng
 	if err != nil {
 		t.Fatalf("Create deployment failed: %v", err)
 	}
+	defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace)
 
 	// Wait for deployment to be ready
 	deployment, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())