Cherry-pick elastic#17773 to 7.x: [ci] Add 'mage test' to each project (

elastic#17956)

* Remove legacy Windows CI mage targets and files

GoTestUnit and Fields are not longer required for Windows CI tests.

* Ensure all projects have a test target

Ensure that each project has a test target. Additionally, if the project has unit tests then
make sure it has an unitTest target. And likewise, an integTest target if the project has
integration tests.

This also ensures that for python notetests that the NOSE_TESTMATCH environment
variable can be set to select a subset of tests to execute. For example
`NOSE_TESTMATCH=test_kafka mage pythonIntegTest`.

* Fix metricbeat dashboard test

Mage writes dashboards to build/kibana. The old Makefile convention was to use _meta/kibana.generated.

* Add kafka-python to python requirements

Each project is sharing the same python virtual environment as an optimization when using
Mage. Metricbeat had one additional dependency than what was listed in libbeat's requirements.txt.

* Update libbeat integ tests to work under mage

The mage integTest environment does not create a test.env file to pass to docker-compose. So
move the variables into the docker-compose.yml file. If variables from a file are wanted at a
later time then the built-in support for a ".env" file in docker-compose should be used instead.

The python integration test was written to require code coverage instrumentation in the Go
binaries. I updated the test to work when code coverage is not enabled.

* Decrease verbosity of the chown function when using mage -v
* Test Metricbeat using mage directly
* Ignore generators/_templates for goimports

This contains templates rather than .go files despite their file extensions.

* Add integTest stub to Packetbeat
* Add integTest to Heartbeat magefile

(cherry picked from commit 62b1e90)

Jenkinsfile

@@ -65,7 +65,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Elastic Agent x-pack Linux", "-C x-pack/elastic-agent testsuite")
+            mageTarget("Elastic Agent x-pack Linux", "x-pack/elastic-agent", "build test")
           }
         }
 
@@ -79,7 +79,7 @@ pipeline {
             }
           }
           steps {
-            mageTargetWin("Elastic Agent x-pack Windows Unit test", "x-pack/elastic-agent", "unitTest")
+            mageTargetWin("Elastic Agent x-pack Windows Unit test", "x-pack/elastic-agent", "build unitTest")
           }
         }
 
@@ -93,7 +93,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Elastic Agent x-pack Mac OS X", "TEST_ENVIRONMENT=0 -C x-pack/elastic-agent testsuite")
+            mageTarget("Elastic Agent x-pack Mac OS X", "x-pack/elastic-agent", "build unitTest")
           }
         }
 
@@ -133,7 +133,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Filebeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C filebeat testsuite")
+            mageTarget("Filebeat oss Mac OS X", "filebeat", "build unitTest")
           }
         }
         stage('Filebeat Windows'){
@@ -146,7 +146,7 @@ pipeline {
             }
           }
           steps {
-            mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "unitTest")
+            mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "build unitTest")
           }
         }
         stage('Heartbeat'){
@@ -174,7 +174,7 @@ pipeline {
                 }
               }
               steps {
-                makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite")
+                mageTarget("Heartbeat oss Mac OS X", "heartbeat", "build unitTest")
               }
             }
             stage('Heartbeat Windows'){
@@ -187,7 +187,7 @@ pipeline {
                 }
               }
               steps {
-                mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "unitTest")
+                mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "build unitTest")
               }
             }
           }
@@ -222,7 +222,7 @@ pipeline {
                 }
               }
               steps {
-                makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite")
+                mageTarget("Auditbeat oss Mac OS X", "auditbeat", "build unitTest")
               }
             }
             stage('Auditbeat Windows'){
@@ -235,7 +235,7 @@ pipeline {
                 }
               }
               steps {
-                mageTargetWin("Auditbeat Windows Unit test", "auditbeat", "unitTest")
+                mageTargetWin("Auditbeat Windows Unit test", "auditbeat", "build unitTest")
               }
             }
           }
@@ -293,7 +293,7 @@ pipeline {
             makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite")
           }
         }
-        stage('Metricbeat Unit tests'){
+        stage('Metricbeat OSS Unit tests'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
           when {
@@ -303,10 +303,10 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Metricbeat Unit tests", "-C metricbeat unit-tests coverage-report")
+            mageTarget("Metricbeat OSS linux/amd64 (unitTest)", "metricbeat", "build unitTest")
           }
         }
-        stage('Metricbeat Integration tests'){
+        stage('Metricbeat OSS Integration tests'){
           agent { label 'ubuntu && immutable' }
           options { skipDefaultCheckout() }
           when {
@@ -316,20 +316,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Metricbeat Integration tests", "-C metricbeat integration-tests-environment coverage-report")
-          }
-        }
-        stage('Metricbeat System tests'){
-          agent { label 'ubuntu && immutable' }
-          options { skipDefaultCheckout() }
-          when {
-            beforeAgent true
-            expression {
-              return env.BUILD_METRICBEAT != "false"
-            }
-          }
-          steps {
-            makeTarget("Metricbeat System tests", "-C metricbeat update system-tests-environment coverage-report")
+            mageTarget("Metricbeat OSS linux/amd64 (integTest)", "metricbeat", "integTest")
           }
         }
         stage('Metricbeat x-pack'){
@@ -342,7 +329,7 @@ pipeline {
             }
           }
           steps {
-            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "update build test")
+            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
           }
         }
         stage('Metricbeat crosscompile'){
@@ -355,7 +342,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Metricbeat oss crosscompile", "-C metricbeat crosscompile")
+            makeTarget("Metricbeat OSS crosscompile", "-C metricbeat crosscompile")
           }
         }
         stage('Metricbeat Mac OS X'){
@@ -368,7 +355,7 @@ pipeline {
             }
           }
           steps {
-            makeTarget("Metricbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C metricbeat testsuite")
+            mageTarget("Metricbeat OSS Mac OS X", "metricbeat", "build unitTest")
           }
         }
         stage('Metricbeat Windows'){
@@ -381,7 +368,7 @@ pipeline {
             }
           }
           steps {
-            mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "unitTest")
+            mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "build unitTest")
           }
         }
         stage('Packetbeat'){
@@ -443,7 +430,7 @@ pipeline {
                 }
               }
               steps {
-                mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "unitTest")
+                mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "build unitTest")
               }
             }
           }
@@ -458,7 +445,7 @@ pipeline {
             }
           }
           steps {
-            mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "unitTest")
+            mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "build unitTest")
           }
         }
         stage('Functionbeat'){
@@ -489,7 +476,7 @@ pipeline {
                 }
               }
               steps {
-                mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "update build test")
+                mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "build unitTest")
               }
             }
             stage('Functionbeat Windows'){
@@ -502,7 +489,7 @@ pipeline {
                 }
               }
               steps {
-                mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "unitTest")
+                mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "build unitTest")
               }
             }
           }

auditbeat/magefile.go

@@ -45,12 +45,6 @@ func init() {
 	devtools.BeatDescription = "Audit the activities of users and processes on your system."
 }
 
-// Aliases provides compatibility with CI while we transition all Beats
-// to having common testing targets.
-var Aliases = map[string]interface{}{
-	"goTestUnit": unittest.GoUnitTest, // dev-tools/jenkins_ci.ps1 uses this.
-}
-
 // Build builds the Beat binary.
 func Build() error {
 	return devtools.Build(devtools.DefaultBuildArgs())

dev-tools/mage/common.go

@@ -803,3 +803,21 @@ func ParseVersion(version string) (major, minor, patch int, err error) {
 	patch, _ = strconv.Atoi(data["patch"])
 	return
 }
+
+// ListMatchingEnvVars returns all of the environment variables names that begin
+// with prefix.
+func ListMatchingEnvVars(prefixes ...string) []string {
+	var vars []string
+	for _, v := range os.Environ() {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(v, prefix) {
+				eqIdx := strings.Index(v, "=")
+				if eqIdx != -1 {
+					vars = append(vars, v[:eqIdx])
+				}
+				break
+			}
+		}
+	}
+	return vars
+}

dev-tools/mage/crossbuild.go

@@ -296,7 +296,8 @@ func DockerChown(path string) {
 // chownPaths will chown the file and all of the dirs specified in the path.
 func chownPaths(uid, gid int, path string) error {
 	start := time.Now()
-	defer log.Printf("chown took: %v", time.Now().Sub(start))
+	numFixed := 0
+	defer log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
 
 	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -315,10 +316,10 @@ func chownPaths(uid, gid int, path string) error {
 			return nil
 		}
 
-		log.Printf("chown file: %v", name)
 		if err := os.Chown(name, uid, gid); err != nil {
 			return errors.Wrapf(err, "failed to chown path=%v", name)
 		}
+		numFixed++
 		return nil
 	})
 }

dev-tools/mage/fmt.go

@@ -51,10 +51,12 @@ func Format() {
 }
 
 // GoImports executes goimports against all .go files in and below the CWD. It
-// ignores vendor/ directories.
+// ignores vendor/ and generator/_templates/ directories.
 func GoImports() error {
 	goFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
-		return filepath.Ext(path) == ".go" && !strings.Contains(path, "vendor/")
+		return filepath.Ext(path) == ".go" &&
+			!strings.Contains(path, "vendor/") &&
+			!strings.Contains(path, "generator/_templates/")
 	})
 	if err != nil {
 		return err

dev-tools/mage/pkg_test.go

@@ -103,7 +103,7 @@ func TestDumpVariables(t *testing.T) {
 }
 
 func TestLoadSpecs(t *testing.T) {
-	pkgs, err := LoadSpecs("files/packages.yml")
+	pkgs, err := LoadSpecs("../packaging/packages.yml")
 	if err != nil {
 		t.Fatal(err)
 	}

dev-tools/mage/target/integtest/integtest.go

@@ -70,13 +70,17 @@ func GoIntegTest(ctx context.Context) error {
 	}, whitelistedEnvVars...)
 }
 
-// PythonIntegTest executes the python system tests in the integration environment (Docker).
+// PythonIntegTest executes the python system tests in the integration
+// environment (Docker).
+// Use NOSE_TESTMATCH=pattern to only run tests matching the specified pattern.
+// Use any other NOSE_* environment variable to influence the behavior of
+// nosetests.
 func PythonIntegTest(ctx context.Context) error {
 	if !devtools.IsInIntegTestEnv() {
 		mg.SerialDeps(pythonTestDeps...)
 	}
 	return devtools.RunIntegTest("pythonIntegTest", func() error {
 		mg.Deps(devtools.BuildSystemTestBinary)
 		return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs())
-	}, whitelistedEnvVars...)
+	}, append(whitelistedEnvVars, devtools.ListMatchingEnvVars("NOSE_")...)...)
 }

dev-tools/mage/target/integtest/notests/integtest.go

@@ -19,7 +19,7 @@ package notests
 
 import "fmt"
 
-// IntegTest method fallbacks to GoIntegTest()
+// IntegTest executes integration tests (it uses Docker to run the tests).
 func IntegTest() {
 	GoIntegTest()
 }