Skip to content

Commit

Permalink
Add option to subctl verify to run basic connectivity tests
Browse files Browse the repository at this point in the history 
Select the tests to run based on Ginkgo labels instead of test
description patterns. The basic tests are specified via a new label
defined in shipyard. From the CLI, the basic tests are invoked via
'--only basic-connectivity'.

Fixes submariner-io#697

Signed-off-by: Tom Pantelis <[email protected]>
  • Loading branch information
@tpantelis
tpantelis committed Nov 2, 2023
1 parent 08b9835 commit cef0d25
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 47 deletions.
66 changes: 34 additions & 32 deletions cmd/subctl/verify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
"github.com/onsi/ginkgo/v2"
"github.com/spf13/cobra"
"github.com/submariner-io/admiral/pkg/reporter"
_ "github.com/submariner-io/lighthouse/test/e2e/discovery"
"github.com/submariner-io/lighthouse/test/e2e/discovery"
_ "github.com/submariner-io/lighthouse/test/e2e/framework"
"github.com/submariner-io/shipyard/test/e2e"
"github.com/submariner-io/shipyard/test/e2e/framework"
Expand All @@ -42,9 +42,9 @@ import (
"github.com/submariner-io/subctl/internal/exit"
"github.com/submariner-io/subctl/internal/restconfig"
"github.com/submariner-io/subctl/pkg/cluster"
_ "github.com/submariner-io/submariner/test/e2e/compliance"
_ "github.com/submariner-io/submariner/test/e2e/dataplane"
_ "github.com/submariner-io/submariner/test/e2e/redundancy"
"github.com/submariner-io/submariner/test/e2e/compliance"
"github.com/submariner-io/submariner/test/e2e/dataplane"
"github.com/submariner-io/submariner/test/e2e/redundancy"
"k8s.io/client-go/rest"
)

Expand Down Expand Up @@ -86,7 +86,7 @@ The following verifications are deemed disruptive:
toContextPresent, err := verifyRestConfigProducer.RunOnSelectedPrefixedContext(
"to",
func(toClusterInfo *cluster.Info, _ string, status reporter.Interface) error {
return runVerify(fromClusterInfo, toClusterInfo, namespace, determinePatternsToVerify())
return runVerify(fromClusterInfo, toClusterInfo, namespace, determineSpecLabelsToVerify())
}, status)

if toContextPresent {
Expand Down Expand Up @@ -149,21 +149,23 @@ func checkVerifyArguments(cmd *cobra.Command, args []string) error {
return fmt.Errorf("--connection-timeout must be >=20")
}

if _, _, err := getVerifyPatterns(verifyOnly, true); err != nil {
if _, _, err := getVerifySpecLabels(verifyOnly, true); err != nil {
return err
}

return checkNoArguments(cmd, args)
}

var verifyE2EPatterns = map[string]string{
component.Connectivity: "\\[dataplane",
component.ServiceDiscovery: "\\[discovery",
"compliance": "\\[compliance]",
var verifyE2ESpecLabels = map[string]string{
component.Connectivity: dataplane.TestLabel,
fmt.Sprintf("%s-%s", framework.BasicTestLabel, component.Connectivity): fmt.Sprintf("%s&&%s",
dataplane.TestLabel, framework.BasicTestLabel),
component.ServiceDiscovery: discovery.TestLabel,
"compliance": compliance.TestLabel,
}

var verifyE2EDisruptivePatterns = map[string]string{
"gateway-failover": "\\[redundancy",
var verifyE2EDisruptiveSpecLabels = map[string]string{
"gateway-failover": redundancy.TestLabel,
}

type verificationType int
Expand All @@ -175,8 +177,8 @@ const (
)

func disruptiveVerificationNames() []string {
names := make([]string, 0, len(verifyE2EDisruptivePatterns))
for n := range verifyE2EDisruptivePatterns {
names := make([]string, 0, len(verifyE2EDisruptiveSpecLabels))
for n := range verifyE2EDisruptiveSpecLabels {
names = append(names, n)
}

Expand All @@ -189,7 +191,7 @@ func extractDisruptiveVerifications(csv string) []string {
verifications := strings.Split(csv, ",")
for _, verification := range verifications {
verification = strings.Trim(strings.ToLower(verification), " ")
if _, ok := verifyE2EDisruptivePatterns[verification]; ok {
if _, ok := verifyE2EDisruptiveSpecLabels[verification]; ok {
disruptive = append(disruptive, verification)
}
}
Expand All @@ -200,60 +202,60 @@ func extractDisruptiveVerifications(csv string) []string {
func getAllVerifyKeys() []string {
keys := []string{}

for k := range verifyE2EPatterns {
for k := range verifyE2ESpecLabels {
keys = append(keys, k)
}

for k := range verifyE2EDisruptivePatterns {
for k := range verifyE2EDisruptiveSpecLabels {
keys = append(keys, k)
}

return keys
}

func getVerifyPattern(key string) (verificationType, string) {
if pattern, ok := verifyE2EPatterns[key]; ok {
func getVerifySpecLabel(key string) (verificationType, string) {
if pattern, ok := verifyE2ESpecLabels[key]; ok {
return normalVerification, pattern
}

if pattern, ok := verifyE2EDisruptivePatterns[key]; ok {
if pattern, ok := verifyE2EDisruptiveSpecLabels[key]; ok {
return disruptiveVerification, pattern
}

return unknownVerification, ""
}

func getVerifyPatterns(csv string, includeDisruptive bool) ([]string, []string, error) {
outputPatterns := []string{}
func getVerifySpecLabels(csv string, includeDisruptive bool) ([]string, []string, error) {
outputLabels := []string{}
outputVerifications := []string{}

verifications := strings.Split(csv, ",")
for _, verification := range verifications {
verification = strings.Trim(strings.ToLower(verification), " ")

vtype, pattern := getVerifyPattern(verification)
vtype, label := getVerifySpecLabel(verification)
switch vtype {
case unknownVerification:
return nil, nil, fmt.Errorf("unknown verification %q", verification)
case normalVerification:
outputPatterns = append(outputPatterns, pattern)
outputLabels = append(outputLabels, label)
outputVerifications = append(outputVerifications, verification)
case disruptiveVerification:
if includeDisruptive {
outputPatterns = append(outputPatterns, pattern)
outputLabels = append(outputLabels, label)
outputVerifications = append(outputVerifications, verification)
}
}
}

if len(outputPatterns) == 0 {
if len(outputLabels) == 0 {
return nil, nil, fmt.Errorf("please specify at least one verification to be performed")
}

return outputPatterns, outputVerifications, nil
return outputLabels, outputVerifications, nil
}

func determinePatternsToVerify() []string {
func determineSpecLabelsToVerify() []string {
disruptive := extractDisruptiveVerifications(verifyOnly)
if !disruptiveTests && len(disruptive) > 0 {
err := survey.AskOne(&survey.Confirm{
Expand All @@ -271,17 +273,17 @@ prompt for confirmation therefore you must specify --enable-disruptive to run th
}
}

patterns, verifications, err := getVerifyPatterns(verifyOnly, disruptiveTests)
labels, verifications, err := getVerifySpecLabels(verifyOnly, disruptiveTests)
if err != nil {
exit.WithMessage(err.Error())
}

fmt.Printf("Performing the following verifications: %s\n", strings.Join(verifications, ", "))

return patterns
return labels
}

func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, patterns []string) error {
func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, specLabels []string) error {
framework.RestConfigs = []*rest.Config{fromClusterInfo.RestConfig, toClusterInfo.RestConfig}
framework.TestContext.ClusterIDs = []string{fromClusterInfo.Name, toClusterInfo.Name}
framework.TestContext.KubeContexts = []string{fromClusterInfo.Name, toClusterInfo.Name}
Expand All @@ -296,7 +298,7 @@ func runVerify(fromClusterInfo, toClusterInfo *cluster.Info, namespace string, p
framework.TestContext.KubeConfig = "not-used"

suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
suiteConfig.FocusStrings = patterns
suiteConfig.LabelFilter = strings.Join(specLabels, ",")
suiteConfig.RandomSeed = 1
reporterConfig.Verbose = verboseConnectivityVerification
reporterConfig.JUnitReport = junitReport
Expand Down
10 changes: 5 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,17 @@ require (
github.com/google/go-github/v54 v54.0.0
github.com/gophercloud/utils v0.0.0-20210909165623-d7085207ff6d
github.com/mattn/go-isatty v0.0.19
github.com/onsi/ginkgo/v2 v2.12.0
github.com/onsi/gomega v1.27.10
github.com/onsi/ginkgo/v2 v2.13.0
github.com/onsi/gomega v1.28.0
github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00
github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352
github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f
github.com/submariner-io/shipyard v0.16.0-m4
github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f
github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4
github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6
github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c
github.com/uw-labs/lichen v0.1.7
golang.org/x/net v0.17.0
Expand Down
20 changes: 10 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,15 +424,15 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c=
github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8=
github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00 h1:sYXq/GVWN0Un+6eEGd3vft4dY+M3i0RSL3GJhvQBOGY=
github.com/openshift/api v0.0.0-20230714214528-de6ad7979b00/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
Expand Down Expand Up @@ -518,12 +518,12 @@ github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352 h1:3oQ
github.com/submariner-io/admiral v0.16.0-m4.0.20231011054946-6287acf98352/go.mod h1:Zb/vxLUvvPivyyL3wSYadlyWRGNc5hRuk5NRCGHlt2g=
github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7 h1:8e+FUBmmHCftibX+x2m3SBJopf5XySNbYJF8rVrSN9c=
github.com/submariner-io/cloud-prepare v0.16.0-m4.0.20231010134947-f76b728243d7/go.mod h1:VZA3+Z0RtUeJy34aSLDXrKAYbgRa0WVqazTGN2OnOs4=
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f h1:i8j+QjZp46BdiLGS85L+Z5bjUcBS4VRmmqa9sBfuMYM=
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231010200214-a72dce76d30f/go.mod h1:pJOVA8F5WUcRd/YfwtCT0OLV2G9bjw9qp9x31IvPnNI=
github.com/submariner-io/shipyard v0.16.0-m4 h1:UhxS3w3C+c2kVUrJVH4VMjbhkrgTjzo8oPlo/ANbjvI=
github.com/submariner-io/shipyard v0.16.0-m4/go.mod h1:4brXpjvD+OL3/hd8+laET47FeoOsQzkQ74aprhEyfhE=
github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc h1:AQ1NSA/jjmv7h8w1chWrusr/1GO0GCut5H6NTiaET00=
github.com/submariner-io/submariner v0.16.0-m4.0.20231010141121-3d3cc9ce23cc/go.mod h1:m1BOR2L7TrR5m25DPVGF6sURBOE2KAnu1RTwM/7j9g8=
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f h1:MV4PPJ/rVPeiInoMPR5c7aUz0iCR+R+hL8xtMSLa2Yo=
github.com/submariner-io/lighthouse v0.16.0-m4.0.20231017060950-d4193c3e8a5f/go.mod h1:GLccS5EPVAW6HIFF6reO+qn2vJvM5RFtRx1IJ3zV/2o=
github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4 h1:toajDp31eWHV2cL+oFdKVdZrrcojX7EC5HcpG5/Qjj8=
github.com/submariner-io/shipyard v0.16.0-m4.0.20231017114407-11b7ad52c6a4/go.mod h1:1zPFbxQbgZZXvV2rukb1EliGog4+OlAEhbU5aLHwpXA=
github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6 h1:h39KRqoLCfBQmkSjXcomgDNaD+jXvfPIpqWT+cY/rxI=
github.com/submariner-io/submariner v0.16.0-m4.0.20231017181939-daf50e5352a6/go.mod h1:4GjiSYK+kvM5QwJZV7YKMq1ufNxyfUto4Yy9uriOEBw=
github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c h1:93yW5AvbCQ6JXrmGbaZbqqjOnk59rlp7lR2Ytk70wjk=
github.com/submariner-io/submariner-operator v0.16.0-m4.0.20231010180358-7d33f167894c/go.mod h1:fwyc7Bu+jlxUReOobXGh6BUJ2vwM+EN+5e2ftLv59zc=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down
4 changes: 4 additions & 0 deletions scripts/test/system.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,10 @@ _subctl benchmark latency "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFI

_subctl benchmark throughput "${KUBECONFIGS_DIR}"/kind-config-cluster1 "${KUBECONFIGS_DIR}"/kind-config-cluster2 && exit 1

# Test subctl verify basic

_subctl verify --context cluster1 --tocontext cluster2 --only basic-connectivity --verbose

# Test subctl cloud prepare invocations

# test non deploy subctl for deploy commands
Expand Down

0 comments on commit cef0d25

Please sign in to comment.