Merge branch 'master' into agent-empty-config

.ci/scripts/install-terraform.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"

.ci/scripts/terraform-cleanup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+DIRECTORY=${1:-.}
+
+FAILED=0
+for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
+  cd $(dirname $tfstate)
+  if ! terraform destroy -auto-approve; then
+    FAILED=1
+  fi
+  cd -
+done
+
+exit $FAILED

.gitignore

@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
 *.test
 *.prof
 *.pyc
+
+# Terraform
+*.terraform
+*.tfstate*

CHANGELOG.next.asciidoc

@@ -210,6 +210,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add keystore support for autodiscover static configurations. {pull]16306[16306]
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191]
+- Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
 
 *Auditbeat*
 

Jenkinsfile

@@ -15,9 +15,11 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
     DOCKER_COMPOSE_VERSION = "1.21.0"
+    TERRAFORM_VERSION = "0.12.24"
     PIPELINE_LOG_LEVEL = "INFO"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
   }
   options {
@@ -36,6 +38,11 @@ pipeline {
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
     booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.')
+
+    booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
+    string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
+
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
@@ -352,8 +359,30 @@ pipeline {
               return env.BUILD_METRICBEAT_XPACK != "false"
             }
           }
-          steps {
-            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+          stages {
+            stage('Prepare cloud integration tests environments'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                startCloudTestEnv('x-pack-metricbeat', [
+                   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+                ])
+              }
+            }
+            stage('Metricbeat x-pack'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                withCloudTestEnv() {
+                  mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+                }
+              }
+            }
+          }
+          post {
+            cleanup {
+              terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+            }
           }
         }
         stage('Metricbeat crosscompile'){
@@ -683,7 +712,7 @@ def withBeatsEnv(boolean archive, Closure body) {
     "TEST_COVERAGE=true",
     "RACE_DETECTOR=true",
     "PYTHON_ENV=${WORKSPACE}/python-env",
-    "TEST_TAGS=oracle",
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
     "DOCKER_PULL=0",
   ]) {
     deleteDir()
@@ -750,6 +779,7 @@ def installTools() {
   if(isUnix()) {
     retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
     retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
+    retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
     retry(i) { sh(label: "Install Mage", script: "make mage") }
   } else {
     retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
@@ -821,6 +851,7 @@ def dumpFilteredEnvironment(){
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"
   echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}"
+  echo "TEST_TAGS: ${env.TEST_TAGS}"
   echo "GOX_OS: ${env.GOX_OS}"
   echo "GOX_OSARCH: ${env.GOX_OSARCH}"
   echo "GOX_FLAGS: ${env.GOX_FLAGS}"
@@ -907,6 +938,98 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
+// withCloudTestEnv executes a closure with credentials for cloud test
+// environments.
+def withCloudTestEnv(Closure body) {
+  def maskedVars = []
+  def testTags = "${env.TEST_TAGS}"
+
+  // AWS
+  if (params.allCloudTests || params.awsCloudTests) {
+    testTags = "${testTags},aws"
+    def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
+    if (!aws.containsKey('access_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'")
+    }
+    if (!aws.containsKey('secret_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
+    }
+    maskedVars.addAll([
+      [var: "AWS_REGION", password: params.awsRegion],
+      [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
+      [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
+    ])
+  }
+
+  withEnv([
+    "TEST_TAGS=${testTags}",
+  ]) {
+    withEnvMask(vars: maskedVars) {
+      body()
+    }
+  }
+}
+
+def terraformInit(String directory) {
+  dir(directory) {
+    sh(label: "Terraform Init on ${directory}", script: "terraform init")
+  }
+}
+
+def terraformApply(String directory) {
+  terraformInit(directory)
+  dir(directory) {
+    sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+  }
+}
+
+// Start testing environment on cloud using terraform. Terraform files are
+// stashed so they can be used by other stages. They are also archived in
+// case manual cleanup is needed.
+//
+// Example:
+//   startCloudTestEnv('x-pack-metricbeat', [
+//     [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+//   ])
+//   ...
+//   terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+def startCloudTestEnv(String name, environments = []) {
+  withCloudTestEnv() {
+    withBeatsEnv(false) {
+      def runAll = params.runAllCloudTests
+      try {
+        for (environment in environments) {
+          if (environment.cond || runAll) {
+            retry(2) {
+              terraformApply(environment.dir)
+            }
+          }
+        }
+      } finally {
+        // Archive terraform states in case manual cleanup is needed.
+        archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
+      }
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+    }
+  }
+}
+
+
+// Looks for all terraform states in directory and runs terraform destroy for them,
+// it uses terraform states previously stashed by startCloudTestEnv.
+def terraformCleanup(String stashName, String directory) {
+  stage("Remove cloud scenarios in ${directory}"){
+    withCloudTestEnv() {
+      withBeatsEnv(false) {
+        unstash "terraform-${stashName}"
+        retry(2) {
+          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
+        }
+      }
+    }
+  }
+}
+
 def loadConfigEnvVars(){
   def empty = []
   env.GO_VERSION = readFile(".go-version").trim()

auditbeat/docs/fields.asciidoc

@@ -2458,7 +2458,8 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Hostname of the agent.
+Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
+
 
 type: keyword
 

dev-tools/mage/common.go

@@ -833,3 +833,28 @@ func ListMatchingEnvVars(prefixes ...string) []string {
 	}
 	return vars
 }
+
+// IntegrationTestEnvVars returns the names of environment variables needed to configure
+// connections to integration test environments.
+func IntegrationTestEnvVars() []string {
+	// Environment variables that can be configured with paths to files
+	// with authentication information.
+	vars := []string{
+		"AWS_SHARED_CREDENTIAL_FILE",
+		"AZURE_AUTH_LOCATION",
+		"GOOGLE_APPLICATION_CREDENTIALS",
+	}
+	// Environment variables with authentication information.
+	prefixes := []string{
+		"AWS_",
+		"AZURE_",
+
+		// Accepted by terraform, but not by many clients, including Beats
+		"GOOGLE_",
+		"GCLOUD_",
+	}
+	for _, prefix := range prefixes {
+		vars = append(vars, ListMatchingEnvVars(prefix)...)
+	}
+	return vars
+}

dev-tools/mage/crossbuild.go

@@ -297,7 +297,9 @@ func DockerChown(path string) {
 func chownPaths(uid, gid int, path string) error {
 	start := time.Now()
 	numFixed := 0
-	defer log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
+	defer func() {
+		log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
+	}()
 
 	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
 		if err != nil {

dev-tools/mage/gotest.go

@@ -156,7 +156,9 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 		foundModule = true
 
 		// Set MODULE because only want that modules tests to run inside the testing environment.
-		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()})
+		env := map[string]string{"MODULE": fi.Name()}
+		passThroughEnvs(env, IntegrationTestEnvVars()...)
+		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env)
 		if err != nil {
 			return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
 		}
@@ -454,5 +456,10 @@ func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error {
 	if len(binArgs.InputFiles) > 0 {
 		args = append(args, binArgs.InputFiles...)
 	}
+
+	start := time.Now()
+	defer func() {
+		log.Printf("BuildSystemTestGoBinary (go %v) took %v.", strings.Join(args, " "), time.Since(start))
+	}()
 	return sh.RunV("go", args...)
 }

dev-tools/mage/integtest.go

@@ -185,37 +185,30 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 		if err != nil {
 			return nil, errors.Wrapf(err, "%s tester failed on Use", t.Name())
 		}
-		if use {
-			// Create the steps for the specific runner.
-			var runnerSteps IntegrationTestSteps
-			requirements := t.StepRequirements()
-			if requirements != nil {
-				runnerSteps = append(runnerSteps, requirements...)
-			}
-			runnerSteps = append(runnerSteps, steps...)
-
-			// Create the custom env for the runner.
-			env := map[string]string{}
-			for k, v := range passInEnv {
-				env[k] = v
-			}
-			env[insideIntegrationTestEnvVar] = "true"
-			passThroughEnvs(env, defaultPassthroughEnvVars...)
-			if mg.Verbose() {
-				env["MAGEFILE_VERBOSE"] = "1"
-			}
-			if UseVendor {
-				env["GOFLAGS"] = "-mod=vendor"
-			}
-
-			runner := &IntegrationRunner{
-				steps:  runnerSteps,
-				tester: t,
-				dir:    dir,
-				env:    env,
-			}
-			runners = append(runners, runner)
+		if !use {
+			continue
+		}
+		runner, err := initRunner(t, dir, passInEnv)
+		if err != nil {
+			return nil, errors.Wrapf(err, "initializing %s runner", t.Name())
+		}
+		runners = append(runners, runner)
+	}
+	// Keep support for modules that don't have a local environment defined at the module
+	// level (system, stack and cloud modules by now)
+	if len(runners) == 0 {
+		if mg.Verbose() {
+			fmt.Printf(">> No runner found in %s, using docker\n", path)
+		}
+		tester, ok := globalIntegrationTesters["docker"]
+		if !ok {
+			return nil, fmt.Errorf("docker integration test runner not registered")
+		}
+		runner, err := initRunner(tester, dir, passInEnv)
+		if err != nil {
+			return nil, errors.Wrapf(err, "initializing docker runner")
 		}
+		runners = append(runners, runner)
 	}
 	return runners, nil
 }
@@ -230,6 +223,10 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	if !ok {
 		return nil, fmt.Errorf("docker integration test runner not registered")
 	}
+	return initRunner(tester, cwd, nil, passThroughEnvVars...)
+}
+
+func initRunner(tester IntegrationTester, dir string, passInEnv map[string]string, passThroughEnvVars ...string) (*IntegrationRunner, error) {
 	var runnerSteps IntegrationTestSteps
 	requirements := tester.StepRequirements()
 	if requirements != nil {
@@ -240,8 +237,11 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	env := map[string]string{
 		insideIntegrationTestEnvVar: "true",
 	}
-	passThroughEnvs(env, defaultPassthroughEnvVars...)
+	for name, value := range passInEnv {
+		env[name] = value
+	}
 	passThroughEnvs(env, passThroughEnvVars...)
+	passThroughEnvs(env, defaultPassthroughEnvVars...)
 	if mg.Verbose() {
 		env["MAGEFILE_VERBOSE"] = "1"
 	}
@@ -252,7 +252,7 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	runner := &IntegrationRunner{
 		steps:  runnerSteps,
 		tester: tester,
-		dir:    cwd,
+		dir:    dir,
 		env:    env,
 	}
 	return runner, nil