Add option to subctl verify to run basic connectivity tests

Select the tests to run based on Ginkgo labels instead of test description patterns. The basic tests are specified via a new label defined in shipyard. Fixes submariner-io#697 Signed-off-by: Tom Pantelis <[email protected]>
tpantelis · Oct 18, 2023 · 53cd5e5 · 53cd5e5
1 parent 84ae622
commit 53cd5e5
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 47 deletions.
diff --git a/cmd/subctl/verify.go b/cmd/subctl/verify.go
@@ -32,7 +32,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/spf13/cobra"
 	"github.com/submariner-io/admiral/pkg/reporter"
-	_ "github.com/submariner-io/lighthouse/test/e2e/discovery"
+	"github.com/submariner-io/lighthouse/test/e2e/discovery"
 	_ "github.com/submariner-io/lighthouse/test/e2e/framework"
 	"github.com/submariner-io/shipyard/test/e2e"
 	"github.com/submariner-io/shipyard/test/e2e/framework"
@@ -42,9 +42,9 @@ import (
 	"github.com/submariner-io/subctl/internal/exit"
 	"github.com/submariner-io/subctl/internal/restconfig"
 	"github.com/submariner-io/subctl/pkg/cluster"
-	_ "github.com/submariner-io/submariner/test/e2e/compliance"
-	_ "github.com/submariner-io/submariner/test/e2e/dataplane"
-	_ "github.com/submariner-io/submariner/test/e2e/redundancy"
+	"github.com/submariner-io/submariner/test/e2e/compliance"
+	"github.com/submariner-io/submariner/test/e2e/dataplane"
+	"github.com/submariner-io/submariner/test/e2e/redundancy"
 	"k8s.io/client-go/rest"
 )
 
@@ -86,7 +86,7 @@ The following verifications are deemed disruptive:
 				toContextPresent, err := verifyRestConfigProducer.RunOnSelectedPrefixedContext(
 					"to",
 					func(toClusterInfo *cluster.Info, _ string, status reporter.Interface) error {
-						return runVerify(fromClusterInfo, toClusterInfo, namespace, determinePatternsToVerify())
+						return runVerify(fromClusterInfo, toClusterInfo, namespace, determineSpecLabelsToVerify())
 					}, status)
 
 				if toContextPresent {
@@ -149,21 +149,23 @@ func checkVerifyArguments(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("--connection-timeout must be >=20")
 	}
 
-	if _, _, err := getVerifyPatterns(verifyOnly, true); err != nil {
+	if _, _, err := getVerifySpecLabels(verifyOnly, true); err != nil {
 		return err
 	}
 
 	return checkNoArguments(cmd, args)
 }
 
-var verifyE2EPatterns = map[string]string{
-	component.Connectivity:     "\\[dataplane",
-	component.ServiceDiscovery: "\\[discovery",
-	"compliance":               "\\[compliance]",
+var verifyE2ESpecLabels = map[string]string{
+	component.Connectivity: dataplane.TestLabel,
+	fmt.Sprintf("%s-%s", framework.BasicTestLabel, component.Connectivity): fmt.Sprintf("%s&&%s",
+		dataplane.TestLabel, framework.BasicTestLabel),
+	component.ServiceDiscovery: discovery.TestLabel,
+	"compliance":               compliance.TestLabel,
 }
 
-var verifyE2EDisruptivePatterns = map[string]string{
-	"gateway-failover": "\\[redundancy",
+var verifyE2EDisruptiveSpecLabels = map[string]string{
+	"gateway-failover": redundancy.TestLabel,
 }
 
 type verificationType int
@@ -175,8 +177,8 @@ const (
 )
 
 func disruptiveVerificationNames() []string {
-	names := make([]string, 0, len(verifyE2EDisruptivePatterns))
-	for n := range verifyE2EDisruptivePatterns {
+	names := make([]string, 0, len(verifyE2EDisruptiveSpecLabels))
+	for n := range verifyE2EDisruptiveSpecLabels {
 		names = append(names, n)
 	}
 
@@ -189,7 +191,7 @@ func extractDisruptiveVerifications(csv string) []string {
 	verifications := strings.Split(csv, ",")
 	for _, verification := range verifications {
 		verification = strings.Trim(strings.ToLower(verification), " ")
-		if _, ok := verifyE2EDisruptivePatterns[verification]; ok {
+		if _, ok := verifyE2EDisruptiveSpecLabels[verification]; ok {
 			disruptive = append(disruptive, verification)
 		}
 	}
@@ -200,60 +202,60 @@ func extractDisruptiveVerifications(csv string) []string {
 func getAllVerifyKeys() []string {
 	keys := []string{}
 
-	for k := range verifyE2EPatterns {
+	for k := range verifyE2ESpecLabels {
 		keys = append(keys, k)
 	}
 
-	for k := range verifyE2EDisruptivePatterns {
+	for k := range verifyE2EDisruptiveSpecLabels {
 		keys = append(keys, k)
 	}
 
 	return keys
 }
 
-func getVerifyPattern(key string) (verificationType, string) {
-	if pattern, ok := verifyE2EPatterns[key]; ok {
+func getVerifySpecLabel(key string) (verificationType, string) {
+	if pattern, ok := verifyE2ESpecLabels[key]; ok {
 		return normalVerification, pattern
 	}
 
-	if pattern, ok := verifyE2EDisruptivePatterns[key]; ok {
+	if pattern, ok := verifyE2EDisruptiveSpecLabels[key]; ok {
 		return disruptiveVerification, pattern
 	}
 
 	return unknownVerification, ""
 }
 
-func getVerifyPatterns(csv string, includeDisruptive bool) ([]string, []string, error) {
-	outputPatterns := []string{}
+func getVerifySpecLabels(csv string, includeDisruptive bool) ([]string, []string, error) {
+	outputLabels := []string{}
 	outputVerifications := []string{}
 
 	verifications := strings.Split(csv, ",")
 	for _, verification := range verifications {
 		verification = strings.Trim(strings.ToLower(verification), " ")
 
-		vtype, pattern := getVerifyPattern(verification)
+		vtype, label := getVerifySpecLabel(verification)
 		switch vtype {
 		case unknownVerification:
 			return nil, nil, fmt.Errorf("unknown verification %q", verification)
 		case normalVerification:
-			outputPatterns = append(outputPatterns, pattern)
+			outputLabels = append(outputLabels, label)
 			outputVerifications = append(outputVerifications, verification)
 		case disruptiveVerification:
 			if includeDisruptive {
-				outputPatterns = append(outputPatterns, pattern)
+				outputLabels = append(outputLabels, label)
 				outputVerifications = append(outputVerifications, verification)
 			}
 		}
 	}
 
-	if len(outputPatterns) == 0 {
+	if len(outputLabels) == 0 {
 		return nil, nil, fmt.Errorf("please specify at least one verification to be performed")
 	}
 
-	return outputPatterns, outputVerifications, nil
+	return outputLabels, outputVerifications, nil
 }
 
-func determinePatternsToVerify() []string {
+func determineSpecLabelsToVerify() []string {
 	disruptive := extractDisruptiveVerifications(verifyOnly)
 	if !disruptiveTests && len(disruptive) > 0 {
 		err := survey.AskOne(&survey.Confirm{
@@ -271,17 +273,17 @@ prompt for confirmation therefore you must specify --enable-disruptive to run th
 		}
 	}
 
-	patterns, verifications, err := getVerifyPatterns(verifyOnly, disruptiveTests)
+	labels, verifications, err := getVerifySpecLabels(verifyOnly, disruptiveTests)
 	if err != nil {
 		exit.WithMessage(err.Error())
 	}
 
 	fmt.Printf("Performing the following verifications: %s\n", strings.Join(verifications, ", "))
 
-	return patterns
+	return labels
 }
 
-func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, patterns []string) error {
+func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, specLabels []string) error {
 	framework.RestConfigs = []*rest.Config{fromClusterInfo.RestConfig, toClusterInfo.RestConfig}
 	framework.TestContext.ClusterIDs = []string{fromClusterInfo.Name, toClusterInfo.Name}
 	framework.TestContext.KubeContexts = []string{fromClusterInfo.Name, toClusterInfo.Name}
@@ -296,7 +298,7 @@ func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, p
 	framework.TestContext.KubeConfig = "not-used"
 
 	suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
-	suiteConfig.FocusStrings = patterns
+	suiteConfig.LabelFilter = fmt.Sprintf("%s", strings.Join(specLabels, ","))
 	suiteConfig.RandomSeed = 1
 	reporterConfig.Verbose = verboseConnectivityVerification
 	reporterConfig.JUnitReport = junitReport

diff --git a/go.mod b/go.mod
@@ -10,17 +10,17 @@ require (
 	github.com/google/go-github/v54 v54.0.0
 	github.com/gophercloud/utils v0.0.0-20210909165623-d7085207ff6d
 	github.com/mattn/go-isatty v0.0.19
-	github.com/onsi/ginkgo/v2 v2.12.0
-	github.com/onsi/gomega v1.27.10
+	github.com/onsi/ginkgo/v2 v2.13.0
+	github.com/onsi/gomega v1.28.0
 	github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352
 	github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7
-	github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f
-	github.com/submariner-io/shipyard v0.16.0-m4
-	github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc
+	github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f
+	github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4
+	github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6
 	github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c
 	github.com/uw-labs/lichen v0.1.7
 	golang.org/x/net v0.17.0

diff --git a/go.sum b/go.sum
@@ -424,15 +424,15 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
-github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
-github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
+github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c=
+github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8=
 github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00 h1:sYXq/GVWN0Un+6eEGd3vft4dY+M3i0RSL3GJhvQBOGY=
 github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -518,12 +518,12 @@ github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352 h1:3oQ
 github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352/go.mod h1:Zb/vxLUvvPivyyL3wSYadlyWRGNc5hRuk5NRCGHlt2g=
 github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7 h1:8e+FUBmmHCftibX+x2m3SBJopf5XySNbYJF8rVrSN9c=
 github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7/go.mod h1:VZA3+Z0RtUeJy34aSLDXrKAYbgRa0WVqazTGN2OnOs4=
-github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f h1:i8j+QjZp46BdiLGS85L+Z5bjUcBS4VRmmqa9sBfuMYM=
-github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f/go.mod h1:pJOVA8F5WUcRd/YfwtCT0OLV2G9bjw9qp9x31IvPnNI=
-github.com/submariner-io/shipyard v0.16.0-m4 h1:UhxS3w3C+c2kVUrJVH4VMjbhkrgTjzo8oPlo/ANbjvI=
-github.com/submariner-io/shipyard v0.16.0-m4/go.mod h1:4brXpjvD+OL3/hd8+laET47FeoOsQzkQ74aprhEyfhE=
-github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc h1:AQ1NSA/jjmv7h8w1chWrusr/1GO0GCut5H6NTiaET00=
-github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc/go.mod h1:m1BOR2L7TrR5m25DPVGF6sURBOE2KAnu1RTwM/7j9g8=
+github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f h1:MV4PPJ/rVPeiInoMPR5c7aUz0iCR+R+hL8xtMSLa2Yo=
+github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f/go.mod h1:GLccS5EPVAW6HIFF6reO+qn2vJvM5RFtRx1IJ3zV/2o=
+github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4 h1:toajDp31eWHV2cL+oFdKVdZrrcojX7EC5HcpG5/Qjj8=
+github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4/go.mod h1:1zPFbxQbgZZXvV2rukb1EliGog4+OlAEhbU5aLHwpXA=
+github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6 h1:h39KRqoLCfBQmkSjXcomgDNaD+jXvfPIpqWT+cY/rxI=
+github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6/go.mod h1:4GjiSYK+kvM5QwJZV7YKMq1ufNxyfUto4Yy9uriOEBw=
 github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c h1:93yW5AvbCQ6JXrmGbaZbqqjOnk59rlp7lR2Ytk70wjk=
 github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c/go.mod h1:fwyc7Bu+jlxUReOobXGh6BUJ2vwM+EN+5e2ftLv59zc=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=