roachtest: expand tag functionality to allow filtering by conjunctive…

… tags.

oday, tests are matched if they have any of the specified tags. This patch introduces the ability
to match tests which have all specified tags, whilst attempting to maintain backward compatibility.

Epic: none
Fixes: cockroachdb#96655

Release note: None

pkg/BUILD.bazel

@@ -3383,8 +3383,8 @@ test_suite(
     name = "ccl_tests",
     tags = [
         "-broken_in_bazel",
-        "ccl_test",
         "-integration",
+        "ccl_test",
     ],
     tests = ALL_TESTS,
 )

pkg/cmd/roachtest/README.md

@@ -27,6 +27,15 @@ acceptance/cli/node-status [server]
 [...]
 ```
 
+The list can be filtered by passing a regular expression to the `list` command which will match against the test name.
+Multiple `tag:` prefixed args can be specified to further narrow by which tags are present for a test. The following 
+will list all tests with name containing `admission` where test tags match `(weekly && aws) || my-tag`
+
+```
+roachtest list admission tag:weekly,aws tag:my-tag
+```
+
+
 ## Getting the binaries
 
 To run a test, the `roachtest run` command is used. Since a test typically

pkg/cmd/roachtest/registry/filter.go

@@ -19,9 +19,15 @@ import (
 // See NewTestFilter.
 type TestFilter struct {
 	Name *regexp.Regexp
-	Tag  *regexp.Regexp
-	// RawTag is the string representation of the regexps in tag.
-	RawTag     []string
+	// Multiple `tag:` parameters can be passed for which only one needs to match, but the
+	// value of a single `tag:` parameter can be a comma-separated list of tags which all need
+	// to match.
+	// e.g. `tag:foo,bar` matches tests with tags `foo` and `bar`, and `tag:foo tag:bar` matches
+	// tests with either tag `foo` or tag `bar`.
+	//
+	// This set contains each tag, so the above examples would be represented as `["foo,bar"]` and
+	// `["foo", "bar"]` respectively..
+	Tags       map[string]struct{}
 	RunSkipped bool
 }
 
@@ -31,22 +37,15 @@ type TestFilter struct {
 // name.
 func NewTestFilter(filter []string, runSkipped bool) *TestFilter {
 	var name []string
-	var tag []string
-	var rawTag []string
+	tags := make(map[string]struct{})
 	for _, v := range filter {
 		if strings.HasPrefix(v, "tag:") {
-			tag = append(tag, strings.TrimPrefix(v, "tag:"))
-			rawTag = append(rawTag, v)
+			tags[strings.TrimPrefix(v, "tag:")] = struct{}{}
 		} else {
 			name = append(name, v)
 		}
 	}
 
-	if len(tag) == 0 {
-		tag = []string{DefaultTag}
-		rawTag = []string{"tag:" + DefaultTag}
-	}
-
 	makeRE := func(strs []string) *regexp.Regexp {
 		switch len(strs) {
 		case 0:
@@ -63,8 +62,7 @@ func NewTestFilter(filter []string, runSkipped bool) *TestFilter {
 
 	return &TestFilter{
 		Name:       makeRE(name),
-		Tag:        makeRE(tag),
-		RawTag:     rawTag,
+		Tags:       tags,
 		RunSkipped: runSkipped,
 	}
 }

pkg/cmd/roachtest/registry/test_spec.go

@@ -13,6 +13,7 @@ package registry
 import (
 	"context"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
@@ -127,17 +128,19 @@ func (t *TestSpec) Match(filter *TestFilter) MatchType {
 	if !filter.Name.MatchString(t.Name) {
 		return FailedFilter
 	}
-	if len(t.Tags) == 0 {
-		if !filter.Tag.MatchString("default") {
-			return FailedTags
-		}
+
+	if len(filter.Tags) == 0 {
 		return Matched
 	}
-	for _, t := range t.Tags {
-		if filter.Tag.MatchString(t) {
+
+	testTags := stringSliceToSet(t.Tags)
+	for tag := range filter.Tags {
+		// If the tag is a single CSV e.g. "foo,bar,baz", we match all the tags
+		if matchesAll(testTags, strings.Split(tag, ",")) {
 			return Matched
 		}
 	}
+
 	return FailedTags
 }
 
@@ -148,3 +151,20 @@ func PromSub(raw string) string {
 	invalidPromRE := regexp.MustCompile("[^a-zA-Z0-9_]")
 	return invalidPromRE.ReplaceAllLiteralString(raw, "_")
 }
+
+func matchesAll(testTags map[string]struct{}, filterTags []string) bool {
+	for _, s := range filterTags {
+		if _, ok := testTags[s]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func stringSliceToSet(slice []string) map[string]struct{} {
+	set := make(map[string]struct{})
+	for _, s := range slice {
+		set[s] = struct{}{}
+	}
+	return set
+}

pkg/cmd/roachtest/test_test.go

@@ -55,14 +55,17 @@ func TestMatchOrSkip(t *testing.T) {
 		expected registry.MatchType
 	}{
 		{nil, "foo", nil, registry.Matched},
-		{nil, "foo", []string{"bar"}, registry.FailedTags},
-		{[]string{"tag:b"}, "foo", []string{"bar"}, registry.Matched},
+		{nil, "foo", []string{"bar"}, registry.Matched},
+		{[]string{"tag:bar"}, "foo", []string{"bar"}, registry.Matched},
+		// Partial tag match is not supported
+		{[]string{"tag:b"}, "foo", []string{"bar"}, registry.FailedTags},
 		{[]string{"tag:b"}, "foo", nil, registry.FailedTags},
-		{[]string{"tag:default"}, "foo", nil, registry.Matched},
 		{[]string{"tag:f"}, "foo", []string{"bar"}, registry.FailedTags},
-		{[]string{"f"}, "foo", []string{"bar"}, registry.FailedTags},
+		// Specifying no tag filters matches all tags.
+		{[]string{"f"}, "foo", []string{"bar"}, registry.Matched},
 		{[]string{"f"}, "bar", []string{"bar"}, registry.FailedFilter},
-		{[]string{"f", "tag:b"}, "foo", []string{"bar"}, registry.Matched},
+		{[]string{"f", "tag:bar"}, "foo", []string{"bar"}, registry.Matched},
+		{[]string{"f", "tag:b"}, "foo", []string{"bar"}, registry.FailedTags},
 		{[]string{"f", "tag:f"}, "foo", []string{"bar"}, registry.FailedTags},
 	}
 	for _, c := range testCases {

pkg/cmd/roachtest/tests/awsdms.go

@@ -191,7 +191,7 @@ func registerAWSDMS(r registry.Registry) {
 		Name:    "awsdms",
 		Owner:   registry.OwnerSQLSessions, // TODO(otan): add a migrations OWNERS team
 		Cluster: r.MakeClusterSpec(1),
-		Tags:    []string{`default`, `awsdms`},
+		Tags:    []string{`default`, `awsdms`, `aws`},
 		Run:     runAWSDMS,
 	})
 }

pkg/cmd/roachtest/tests/backup.go

@@ -509,16 +509,18 @@ func registerBackup(r registry.Registry) {
 	for _, item := range []struct {
 		kmsProvider string
 		machine     string
+		tags        []string
 	}{
 		{kmsProvider: "GCS", machine: spec.GCE},
-		{kmsProvider: "AWS", machine: spec.AWS},
+		{kmsProvider: "AWS", machine: spec.AWS, tags: []string{"aws"}},
 	} {
 		item := item
 		r.Add(registry.TestSpec{
 			Name:              fmt.Sprintf("backup/KMS/%s/%s", item.kmsProvider, KMSSpec.String()),
 			Owner:             registry.OwnerDisasterRecovery,
 			Cluster:           KMSSpec,
 			EncryptionSupport: registry.EncryptionMetamorphic,
+			Tags:              item.tags,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				if c.Spec().Cloud != item.machine {
 					t.Skip("backupKMS roachtest is only configured to run on "+item.machine, "")

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -299,6 +299,9 @@ type replicationTestSpec struct {
 	// If non-empty, the test will be skipped with the supplied reason.
 	skip string
 
+	// tags are used to categorize the test.
+	tags []string
+
 	// fields below are instantiated at runtime
 	setup   *c2cSetup
 	t       test.Test
@@ -570,6 +573,7 @@ func registerClusterToCluster(r registry.Registry) {
 			timeout:            1 * time.Hour,
 			additionalDuration: 10 * time.Minute,
 			cutover:            5 * time.Minute,
+			tags:               []string{"aws"},
 		},
 		{
 			name:               "c2c/UnitTest",
@@ -608,6 +612,7 @@ func registerClusterToCluster(r registry.Registry) {
 			Cluster:         r.MakeClusterSpec(sp.dstNodes+sp.srcNodes+1, clusterOps...),
 			Timeout:         sp.timeout,
 			Skip:            sp.skip,
+			Tags:            sp.tags,
 			RequiresLicense: true,
 			Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				sp.setupC2C(ctx, t, c)

pkg/cmd/roachtest/tests/kv.go

@@ -297,6 +297,12 @@ func registerKV(r registry.Registry) {
 			encryption = registry.EncryptionAlwaysEnabled
 		}
 		cSpec := r.MakeClusterSpec(opts.nodes+1, spec.CPU(opts.cpus), spec.SSD(opts.ssds), spec.RAID0(opts.raid0))
+
+		// All the kv0|95 tests should run on AWS by default
+		if opts.tags == nil && (opts.readPercent == 95 || opts.readPercent == 0) {
+			opts.tags = []string{"aws"}
+		}
+
 		var skip string
 		if opts.ssds != 0 && cSpec.Cloud != spec.GCE {
 			skip = fmt.Sprintf("multi-store tests are not supported on cloud %s", cSpec.Cloud)

pkg/cmd/roachtest/tests/restore.go

@@ -124,6 +124,7 @@ func registerRestore(r registry.Registry) {
 		Owner:   registry.OwnerDisasterRecovery,
 		Cluster: withPauseSpecs.hardware.makeClusterSpecs(r),
 		Timeout: withPauseSpecs.timeout,
+		Tags:    []string{`aws`},
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 
 			if c.Spec().Cloud != withPauseSpecs.backup.cloud {
@@ -274,6 +275,7 @@ func registerRestore(r registry.Registry) {
 			hardware: makeHardwareSpecs(hardwareSpecs{}),
 			backup:   makeBackupSpecs(backupSpecs{}),
 			timeout:  1 * time.Hour,
+			tags:     []string{"aws"},
 		},
 		{
 			// Note that the default specs in makeHardwareSpecs() spin up restore tests in aws,
@@ -295,20 +297,23 @@ func registerRestore(r registry.Registry) {
 			hardware: makeHardwareSpecs(hardwareSpecs{nodes: 8}),
 			backup:   makeBackupSpecs(backupSpecs{}),
 			timeout:  1 * time.Hour,
+			tags:     []string{"aws"},
 		},
 		{
 			// Benchmarks if per node throughput doubles if the vcpu count doubles
 			// relative to default.
 			hardware: makeHardwareSpecs(hardwareSpecs{cpus: 16}),
 			backup:   makeBackupSpecs(backupSpecs{}),
 			timeout:  1 * time.Hour,
+			tags:     []string{"aws"},
 		},
 		{
 			// Ensures we can restore a 48 length incremental chain.
 			// Also benchmarks per node throughput for a long chain.
 			hardware: makeHardwareSpecs(hardwareSpecs{}),
 			backup:   makeBackupSpecs(backupSpecs{backupsIncluded: 48}),
 			timeout:  1 * time.Hour,
+			tags:     []string{"aws"},
 		},
 		{
 			// The nightly 8TB Restore test.
@@ -317,6 +322,7 @@ func registerRestore(r registry.Registry) {
 				version:  "v22.2.1",
 				workload: tpceRestore{customers: 500000}}),
 			timeout: 5 * time.Hour,
+			tags:    []string{"aws"},
 		},
 		{
 			// The weekly 32TB Restore test.

pkg/cmd/roachtest/tests/schemachange_random_load.go

@@ -33,6 +33,7 @@ type randomLoadBenchSpec struct {
 	Nodes       int
 	Ops         int
 	Concurrency int
+	Tags        []string
 }
 
 func registerSchemaChangeRandomLoad(r registry.Registry) {
@@ -66,6 +67,7 @@ func registerSchemaChangeRandomLoad(r registry.Registry) {
 		Nodes:       3,
 		Ops:         2000,
 		Concurrency: 1,
+		Tags:        []string{"aws"},
 	})
 
 	registerRandomLoadBenchSpec(r, randomLoadBenchSpec{
@@ -97,6 +99,7 @@ func registerRandomLoadBenchSpec(r registry.Registry, b randomLoadBenchSpec) {
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			runSchemaChangeRandomLoad(ctx, t, c, b.Ops, b.Concurrency)
 		},
+		Tags: b.Tags,
 	})
 }
 

pkg/cmd/roachtest/tests/tpcc.go

@@ -500,7 +500,7 @@ func registerTPCC(r registry.Registry) {
 		// running with the max supported warehouses.
 		Name:              "tpcc/headroom/" + headroomSpec.String(),
 		Owner:             registry.OwnerTestEng,
-		Tags:              []string{`default`, `release_qualification`},
+		Tags:              []string{`default`, `release_qualification`, `aws`},
 		Cluster:           headroomSpec,
 		EncryptionSupport: registry.EncryptionMetamorphic,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
@@ -834,6 +834,7 @@ func registerTPCC(r registry.Registry) {
 
 		LoadWarehouses: gceOrAws(cloud, 3500, 3900),
 		EstimatedMax:   gceOrAws(cloud, 2900, 3500),
+		Tags:           []string{`aws`},
 	})
 	registerTPCCBenchSpec(r, tpccBenchSpec{
 		Nodes: 12,
@@ -889,6 +890,7 @@ func registerTPCC(r registry.Registry) {
 		LoadWarehouses:    gceOrAws(cloud, 3500, 3900),
 		EstimatedMax:      gceOrAws(cloud, 2900, 3500),
 		EncryptionEnabled: true,
+		Tags:              []string{`aws`},
 	})
 	registerTPCCBenchSpec(r, tpccBenchSpec{
 		Nodes: 12,

pkg/cmd/roachtest/tests/ycsb.go

@@ -108,6 +108,7 @@ func registerYCSB(r registry.Registry) {
 				Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 					runYCSB(ctx, t, c, wl, cpus, false /* rangeTombstone */)
 				},
+				Tags: []string{`aws`},
 			})
 
 			if wl == "A" {
@@ -129,6 +130,7 @@ func registerYCSB(r registry.Registry) {
 					Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 						runYCSB(ctx, t, c, wl, cpus, true /* rangeTombstone */)
 					},
+					Tags: []string{`aws`},
 				})
 			}
 		}