Update the e2e tests to run in the WCP cluster (vmware-tanzu#868)

* Update the e2e test cases
* Add a script to run e2e test in the WCP cluster

---------

Signed-off-by: Wenqi Qiu <[email protected]>

hack/test-e2e.sh

@@ -0,0 +1,63 @@
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+NSX_OPERATOR_DIR='/root/nsx-operator'
+NSX_OPERATOR_CI_DIR='/root/nsx-operator-ci'
+rm -fr $NSX_OPERATOR_DIR $NSX_OPERATOR_CI_DIR
+git clone https://github.com/vmware-tanzu/nsx-operator/ $NSX_OPERATOR_DIR
+cd $NSX_OPERATOR_DIR
+export GO111MODULE=on
+export GOPATH=/root/go
+export GOROOT=/usr/local/go
+export PATH=$GOROOT/bin:$PATH
+go version
+go env
+
+# if manual trigger, use the default email and author
+if [ -z ${ghprbActualCommitAuthorEmail+x} ]; then
+    ghprbActualCommitAuthorEmail="[email protected]"
+fi
+if [ -z ${ghprbActualCommitAuthor+x} ]; then
+    ghprbActualCommitAuthor="manually_trigger"
+fi
+git config user.email ${ghprbActualCommitAuthorEmail}
+git config user.name ${ghprbActualCommitAuthor}
+git remote add pr $ghprbAuthorRepoGitUrl
+git fetch pr $ghprbSourceBranch:$ghprbSourceBranch
+git checkout $ghprbSourceBranch
+
+
+# go build -o $NSX_OPERATOR_DIR/bin/manager cmd/main.go
+make build
+md5sum $NSX_OPERATOR_DIR/bin/manager /etc/vmware/wcp/tls/manager
+
+kubectl scale deployment nsx-ncp -n vmware-system-nsx --replicas=0
+sleep 15
+hostname=$(hostname)
+kubectl patch deployment nsx-ncp -n vmware-system-nsx -p "{\"spec\": {\"template\": {\"spec\": {\"nodeSelector\": {\"kubernetes.io/hostname\": \"$hostname\"}}}}}"
+cp $NSX_OPERATOR_DIR/bin/manager /etc/vmware/wcp/tls/
+md5sum $NSX_OPERATOR_DIR/bin/manager /etc/vmware/wcp/tls/manager
+chmod 777 /etc/vmware/wcp/tls/manager
+
+kubectl apply -f $NSX_OPERATOR_DIR/build/yaml/crd/vpc/
+kubectl scale deployment nsx-ncp -n vmware-system-nsx --replicas=1
+sleep 35
+
+pod_name=$(kubectl get pods -n  vmware-system-nsx -o jsonpath="{.items[0].metadata.name}")
+mkdir -p /etc/nsx-ujo/vc
+mkdir -p /etc/ncp/
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /etc/nsx-ujo/ncp.ini > /etc/ncp.ini
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /etc/ncp/lb-default.cert > /etc/ncp/lb-default.cert
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /etc/ncp/lb-default.key > /etc/ncp/lb-default.key
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat  /etc/nsx-ujo/nsx_manager_certificate_0 >  /etc/nsx-ujo/nsx_manager_certificate_0
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat  /var/run/secrets/kubernetes.io/serviceaccount/token > /var/run/secrets/kubernetes.io/serviceaccount/token
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /var/run/secrets/kubernetes.io/serviceaccount/ca.crt  > /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /etc/nsx-ujo/vc/username > /etc/nsx-ujo/vc/username
+kubectl exec $pod_name -c nsx-ncp -n vmware-system-nsx -- cat /etc/nsx-ujo/vc/password > /etc/nsx-ujo/vc/password
+
+cp -r /root/test $NSX_OPERATOR_DIR/
+# Use the -run parameter to run only specific test cases within certain modules.
+# -run '.*Subnet.*|.*SubnetSet.*|.*SecurityPolicy.*|.*NetworkInfo.*'
+e2e=true go test -v ./test/e2e -coverpkg=./pkg/... -remote.kubeconfig /root/.kube/config -operator-cfg-path /etc/ncp.ini -test.timeout 15m -coverprofile cover-e2e.out

test/e2e/framework.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/client/clientset/versioned"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 	"github.com/vmware-tanzu/nsx-operator/test/e2e/providers"
 )
 
@@ -79,15 +80,6 @@ type TestData struct {
 
 var testData *TestData
 
-// Temporarily disable traffic check
-/*
-type PodIPs struct {
-	ipv4      *net.IP
-	ipv6      *net.IP
-	ipStrings []string
-}
-*/
-
 func initProvider() error {
 	providerFactory := map[string]func(string) (providers.ProviderInterface, error){
 		"remote": providers.NewRemoteProvider,
@@ -439,7 +431,11 @@ func (data *TestData) getCRResource(timeout time.Duration, crtype, crname, names
 		if err != nil || rc != 0 {
 			return false, fmt.Errorf("error when running the following command `%s` on master Node: %v, %s", uid_cmd, err, stdout)
 		} else {
-			uid := strings.Split(stdout, ":")[1]
+			parts := strings.Split(stdout, ":")
+			if len(parts) < 2 {
+				return false, nil
+			}
+			uid := parts[1]
 			ret = uid
 		}
 		return true, nil
@@ -675,16 +671,20 @@ func applyYAML(filename string, ns string) error {
 	}
 	var stdout, stderr bytes.Buffer
 	command := exec.Command("bash", "-c", cmd)
-	log.Printf("Applying YAML file %s", filename)
 	command.Stdout = &stdout
 	command.Stderr = &stderr
+
+	log.Printf("Applying YAML file: %s, Namespace: %s", filename, ns)
+
 	err := command.Run()
+	outStr, errStr := stdout.String(), stderr.String()
+
+	log.Printf("YAML file %s applied. Output: '%s', Error: '%s'", filename, outStr, errStr)
+
 	if err != nil {
-		log.Printf("Error when applying YAML file %s: %v", filename, err)
-		return err
+		log.Printf("Failed to apply YAML file %s: %v", filename, err)
+		return fmt.Errorf("failed to apply YAML: %w", err)
 	}
-	outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes())
-	log.Printf("YAML file %s applied with output: '%s' and error: '%s'", cmd, outStr, errStr)
 	return nil
 }
 
@@ -811,6 +811,9 @@ func (data *TestData) waitForResourceExistByPath(pathPolicy string, shouldExist
 			if !shouldExist {
 				return true, nil
 			}
+			if err == util.HttpNotFoundError && shouldExist {
+				return false, nil
+			}
 			return false, err
 		}
 		id, ok := resp["id"].(string)

test/e2e/manifest/testVPC/customize_networkconfig.yaml

@@ -6,8 +6,8 @@ metadata:
   name: selfdefinedconfig
 spec:
   defaultSubnetSize: 32
-  nsxProject: /orgs/default/projects/nsx_operator_e2e_test
+  nsxProject: /orgs/default/projects/project-quality
   privateIPs:
     - 172.29.0.0/16
     - 172.39.0.0/16
-  vpcConnectivityProfile: /orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default
+  vpcConnectivityProfile: /orgs/default/projects/project-quality/vpc-connectivity-profiles/default

test/e2e/manifest/testVPC/customize_networkconfig_updated.yaml

@@ -6,9 +6,9 @@ metadata:
   name: selfdefinedconfig
 spec:
   defaultSubnetSize: 32
-  nsxProject: /orgs/default/projects/nsx_operator_e2e_test
+  nsxProject: /orgs/default/projects/project-quality
   privateIPs:
     - 172.29.0.0/16
     - 172.39.0.0/16
     - 172.49.0.0/16
-  vpcConnectivityProfile: /orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default
+  vpcConnectivityProfile: /orgs/default/projects/project-quality/vpc-connectivity-profiles/default

test/e2e/manifest/testVPC/default_networkconfig.yaml

@@ -10,8 +10,8 @@ metadata:
     nsx.vmware.com/default: "true"
 spec:
   defaultSubnetSize: 32
-  nsxProject: /orgs/default/projects/nsx_operator_e2e_test
+  nsxProject: /orgs/default/projects/project-quality
   privateIPs:
     - 172.28.0.0/16
     - 172.38.0.0/16
-  vpcConnectivityProfile: /orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default
+  vpcConnectivityProfile: /orgs/default/projects/project-quality/vpc-connectivity-profiles/default

test/e2e/manifest/testVPC/system_networkconfig.yaml

@@ -7,8 +7,8 @@ metadata:
   name: system
 spec:
   defaultSubnetSize: 32
-  nsxProject: /orgs/default/projects/nsx_operator_e2e_test
+  nsxProject: /orgs/default/projects/project-quality
   privateIPs:
     - 172.27.0.0/16
     - 172.37.0.0/16
-  vpcConnectivityProfile: /orgs/default/projects/nsx_operator_e2e_test/vpc-connectivity-profiles/default
+  vpcConnectivityProfile: /orgs/default/projects/project-quality/vpc-connectivity-profiles/default