Add ability to split integration tests into different groups (#3544)

* Add ability to shard integration tests across muliple hosts. * Change to Group. * Add to dev guide. * Fix magefile. * Apply suggestions from code review Co-authored-by: Craig MacKenzie <[email protected]> * Add define.Default constant. * Remove isolate. * Fix tests. * Update groups. * Adjust groups. * Adjust some groups, add debugging SSH key. * Only windows default group with all logs. * Remove windows specific. * Fix add_cloud_metadata allowed errors. * Another allowed error. * Yet another error. --------- Co-authored-by: Craig MacKenzie <[email protected]> (cherry picked from commit 8a8abd0) # Conflicts: # magefile.go # pkg/testing/runner/config.go # pkg/testing/runner/runner.go # testing/integration/beats_serverless_test.go # testing/integration/install_unprivileged_test.go # testing/integration/logs_ingestion_test.go # testing/integration/upgrade_fleet_test.go
elastic · Nov 29, 2023 · 86ed266 · 86ed266
1 parent c28a222
commit 86ed266
Show file tree

Hide file tree

Showing 30 changed files with 1,651 additions and 214 deletions.
diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md
@@ -67,6 +67,16 @@ between, and it can be very specific or not very specific.
 > **_NOTE:_**  This only filters down the tests based on the platform. It will not execute a tests on a platform unless
 > the test defines as supporting it.
 
+#### Selecting specific group
+
+By default, the runner will run all test groups. Each group runs on a dedicated machine instance. When working on groups of tests it's better to limit to a specific
+group of tests instead of running all tests. This can be done by using the `TEST_GROUPS="default upgrade-standalone"`
+environment variable. This variable can take multiple groups with a space between.
+
+- `TEST_GROUPS="default" mage integration:test` to execute only tests in the "default" group.
+- `TEST_GROUPS="default upgrade-standalone" mage integration:test` to execute only tests in the "default" or
+"upgrade-standalone" group.
+
 #### Passing additional go test flags
 
 When running the tests we can pass additional go test flag using the env variable `GOTEST_FLAGS`.
@@ -168,6 +178,17 @@ the `github.com/elastic/elastic-agent/pkg/testing/define` package for the test
 framework's API and the `github.com/elastic/elastic-agent/pkg/testing/tools`
 package for helper utilities.
 
+### Test group
+
+Every `define.Require` must define a `Group` that it belongs too. Each group is executed on a separate instance with all tests with in the same group executed
+on the same instance. Placing similar tests in the same group allows those tests to run on its own instance
+as well as provides a way for a developer to select a specific group of tests with `TEST_GROUP="{group-name}"`.
+
+Grouping tests is another way of spreading out the testing load across multiple instances. The more groups that
+are defined the more instances will be provisioned to complete all tests. A balance between a small good set of
+groups is better than a ton of groups each executing a small set of tests, as the time to set up an instance can
+out weight the benefits of creating another group.
+
 ### Test namespaces
 
 Every test has access to its own unique namespace (a string value). This namespace can

diff --git a/magefile.go b/magefile.go
@@ -1773,6 +1773,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 	_ = os.MkdirAll(diagDir, 0755)
 
 	cfg := runner.Config{
+<<<<<<< HEAD
 		AgentVersion:      agentVersion,
 		AgentStackVersion: agentStackVersion,
 		BuildDir:          agentBuildDir,
@@ -1787,6 +1788,24 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
 		Timestamp:         timestamp,
 		TestFlags:         goTestFlags,
 		ExtraEnv:          extraEnv,
+=======
+		AgentVersion:   agentVersion,
+		StackVersion:   agentStackVersion,
+		BuildDir:       agentBuildDir,
+		GOVersion:      goVersion,
+		RepoDir:        repoDir,
+		DiagnosticsDir: diagDir,
+		StateDir:       ".integration-cache",
+		Platforms:      testPlatforms(),
+		Groups:         testGroups(),
+		Matrix:         matrix,
+		SingleTest:     singleTest,
+		VerboseMode:    mg.Verbose(),
+		Timestamp:      timestamp,
+		TestFlags:      goTestFlags,
+		ExtraEnv:       extraEnv,
+		BinaryName:     binaryName,
+>>>>>>> 8a8abd046a (Add ability to split integration tests into different groups (#3544))
 	}
 	ogcCfg := ogc.Config{
 		ServiceTokenPath: serviceTokenPath,
@@ -1871,6 +1890,20 @@ func testPlatforms() []string {
 	return platforms
 }
 
+func testGroups() []string {
+	groupsStr := os.Getenv("TEST_GROUPS")
+	if groupsStr == "" {
+		return nil
+	}
+	var groups []string
+	for _, g := range strings.Split(groupsStr, " ") {
+		if g != "" {
+			groups = append(groups, g)
+		}
+	}
+	return groups
+}
+
 // Pre-requisite: user must have the gcloud CLI installed
 func authGCP(ctx context.Context) error {
 	// We only need the service account token to exist.

diff --git a/pkg/testing/define/batch.go b/pkg/testing/define/batch.go
@@ -41,15 +41,16 @@ var defaultOS = []OS{
 
 // Batch is a grouping of tests that all have the same requirements.
 type Batch struct {
+	// Group must be set on each test to define which group the tests belongs.
+	// Tests that are in the same group are executed on the same runner.
+	Group string `json:"group"`
+
 	// OS defines the operating systems this test batch needs.
 	OS OS `json:"os"`
 
 	// Stack defines the stack required for this batch.
 	Stack *Stack `json:"stack,omitempty"`
 
-	// Isolate defines that this batch is isolated to a single test.
-	Isolate bool `json:"isolate"`
-
 	// Tests define the set of packages and tests that do not require sudo
 	// privileges to be performed.
 	Tests []BatchPackageTests `json:"tests"`
@@ -177,15 +178,12 @@ func appendTest(batches []Batch, tar testActionResult, req Requirements) []Batch
 	}
 	for _, o := range set {
 		var batch Batch
-		batchIdx := -1
-		if !req.Isolate {
-			batchIdx = findBatchIdx(batches, o, req.Stack)
-		}
+		batchIdx := findBatchIdx(batches, req.Group, o, req.Stack)
 		if batchIdx == -1 {
 			// new batch required
 			batch = Batch{
+				Group:     req.Group,
 				OS:        o,
-				Isolate:   req.Isolate,
 				Tests:     nil,
 				SudoTests: nil,
 			}
@@ -241,10 +239,10 @@ func appendPackageTest(tests []BatchPackageTests, pkg string, name string, stack
 	return tests
 }
 
-func findBatchIdx(batches []Batch, os OS, stack *Stack) int {
+func findBatchIdx(batches []Batch, group string, os OS, stack *Stack) int {
 	for i, b := range batches {
-		if b.Isolate {
-			// never add to an isolate batch
+		if b.Group != group {
+			// must be in the same group
 			continue
 		}
 		if b.OS.Type != os.Type || b.OS.Arch != os.Arch {

diff --git a/pkg/testing/define/batch_test.go b/pkg/testing/define/batch_test.go
@@ -93,6 +93,7 @@ func TestBatch(t *testing.T) {
 	}
 	expected := []Batch{
 		{
+			Group: Default,
 			OS: OS{
 				Type: Darwin,
 				Arch: AMD64,
@@ -101,6 +102,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Darwin,
 				Arch: ARM64,
@@ -109,6 +111,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: darwinSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Linux,
 				Arch: AMD64,
@@ -117,6 +120,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type:    Linux,
 				Arch:    ARM64,
@@ -152,6 +156,7 @@ func TestBatch(t *testing.T) {
 			SudoTests: linuxSudoTests,
 		},
 		{
+			Group: Default,
 			OS: OS{
 				Type: Windows,
 				Arch: AMD64,
@@ -160,170 +165,47 @@ func TestBatch(t *testing.T) {
 			SudoTests: windowsSudoTests,
 		},
 		{
+			Group: "one",
 			OS: OS{
-				Type: Darwin,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: ARM64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: ARM64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Windows,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestAnyIsolate",
-						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
-						{
-							Name: "TestDarwinIsolate",
-						},
-					},
-				},
+				Type:    Linux,
+				Arch:    ARM64,
+				Version: "20.04",
+				Distro:  "ubuntu",
 			},
-		},
-		{
-			OS: OS{
-				Type: Darwin,
-				Arch: ARM64,
+			Stack: &Stack{
+				Version: "8.8.0",
 			},
-			Isolate: true,
 			Tests: []BatchPackageTests{
 				{
 					Name: pkgName,
 					Tests: []BatchPackageTest{
 						{
-							Name: "TestDarwinIsolate",
+							Name:  "TestGroup_One_One",
+							Stack: true,
 						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Linux,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
 						{
-							Name: "TestLinuxIsolate",
+							Name:  "TestGroup_One_Two",
+							Stack: true,
 						},
 					},
 				},
 			},
 		},
 		{
+			Group: "two",
 			OS: OS{
 				Type: Linux,
 				Arch: ARM64,
 			},
-			Isolate: true,
 			Tests: []BatchPackageTests{
 				{
 					Name: pkgName,
 					Tests: []BatchPackageTest{
 						{
-							Name: "TestLinuxIsolate",
+							Name: "TestGroup_Two_One",
 						},
-					},
-				},
-			},
-		},
-		{
-			OS: OS{
-				Type: Windows,
-				Arch: AMD64,
-			},
-			Isolate: true,
-			Tests: []BatchPackageTests{
-				{
-					Name: pkgName,
-					Tests: []BatchPackageTest{
 						{
-							Name: "TestWindowsIsolate",
+							Name: "TestGroup_Two_Two",
 						},
 					},
 				},
@@ -344,6 +226,7 @@ var testLinuxLocalTests = []BatchPackageTest{
 
 var testLinuxLocalBatch = []Batch{
 	{
+		Group: Default,
 		OS: OS{
 			Type: "linux",
 			Arch: "amd64",
@@ -356,6 +239,7 @@ var testLinuxLocalBatch = []Batch{
 		},
 	},
 	{
+		Group: Default,
 		OS: OS{
 			Type: "linux",
 			Arch: "arm64",