Merge remote-tracking branch 'upstream/master' into feature/fix-warni…

…ngs-archive

* upstream/master: (274 commits)
  Test export commands in all Beats (elastic#20016)
  [Ingest Manager] Allow using drop path for agent build (elastic#20019)
  [DOCS] Remove include for deleted file in monitoring docs (elastic#20038)
  Removing xpack.monitoring.* settings (elastic#18608)
  [Heartbeat] Add service_name option for APM integration (elastic#19932)
  [Elastic Agent] Fix merging of fleet.yml. Add --staging to enroll cmd. (elastic#20026)
  Stricter stalebot configuration for PRs (elastic#20004)
  [ci] Favor direct mage invocation on CI (elastic#19960)
  Add cloudwatch input into Filebeat configure inputs documentation (elastic#19973)
  [Filebeat] remove delimiter \n from log line in s3 input (elastic#19972)
  [Metricbeat] Update MySQL dashboard (elastic#19913)
  Packetbeat process monitor: Ignore missing /proc/net/tcp6 (elastic#19945)
  [CI] fix MODULE variable cornercases (elastic#19985)
  Ignore timestamp in fortinet/clientendpoint and netscout/sightline (elastic#19998)
  add 7.9 to .backportrc.json (elastic#19952)
  Update internal links to external (elastic#19947)
  Remove Dynamic Script Compilations warning in Cisco module (elastic#19840)
  [Elastic Agent] Fix RPM and DEB packaging for Elastic Agent (elastic#19959)
  [Ingest Manager] Do not compare err with custom type (elastic#19980)
  Fix nanocore sum for non default intervals on Kubernetes Overview Dashboard (elastic#19675)
  ...

.backportrc.json

@@ -0,0 +1,7 @@
+{
+  "upstream": "elastic/beats",
+  "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+  "labels": ["backport"],
+  "autoAssign": true,
+  "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"
+}

.ci/apm-beats-update.groovy

@@ -2,7 +2,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'linux && immutable' }
+  agent none
   environment {
     REPO = 'apm-server'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
@@ -27,63 +27,77 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
+    upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
   }
   stages {
-    /**
-     Checkout the code and stash it, to use it on other stages.
-    */
-    stage('Checkout') {
-      options { skipDefaultCheckout() }
-      steps {
-        deleteDir()
-        gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
-        script {
-          dir("${BEATS_DIR}"){
-            env.GO_VERSION = readFile(".go-version").trim()
-            def regexps =[
-              "^devtools/mage.*",
-              "^libbeat/scripts/Makefile",
-            ]
-            env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
-            // Skip all the stages except docs for PR's with asciidoc changes only
-            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
-          }
-        }
-      }
-    }
-    /**
-    updates beats updates the framework part and go parts of beats.
-    Then build and test.
-    Finally archive the results.
-    */
-    stage('Update Beats') {
-      options { skipDefaultCheckout() }
+    stage('Filter build') {
+      agent { label 'ubuntu && immutable' }
       when {
         beforeAgent true
         anyOf {
-          branch 'master'
-          branch "\\d+\\.\\d+"
-          branch "v\\d?"
-          tag "v\\d+\\.\\d+\\.\\d+*"
-          allOf {
-            expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
-            changeRequest()
+          triggeredBy cause: "IssueCommentCause"
+          expression {
+            def ret = isUserTrigger() || isUpstreamTrigger()
+            if(!ret){
+              currentBuild.result = 'NOT_BUILT'
+              currentBuild.description = "The build has been skipped"
+              currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
+              echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
+            }
+            return ret
+          }
+        }
+      }
+      /**
+       Checkout the code and stash it, to use it on other stages.
+      */
+      stage('Checkout') {
+        steps {
+          deleteDir()
+          gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
+          script {
+            dir("${BEATS_DIR}"){
+              env.GO_VERSION = readFile(".go-version").trim()
+              def regexps =[
+                "^devtools/mage.*",
+                "^libbeat/scripts/Makefile",
+              ]
+              env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
+              // Skip all the stages except docs for PR's with asciidoc changes only
+              env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+            }
           }
-
         }
       }
-      steps {
-        withGithubNotify(context: 'Check Apm Server Beats Update') {
-          beatsUpdate()
+      /**
+      updates beats updates the framework part and go parts of beats.
+      Then build and test.
+      Finally archive the results.
+      */
+      stage('Update Beats') {
+        options { skipDefaultCheckout() }
+        when {
+          beforeAgent true
+          anyOf {
+            branch 'master'
+            branch "\\d+\\.\\d+"
+            branch "v\\d?"
+            tag "v\\d+\\.\\d+\\.\\d+*"
+            allOf {
+              expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+              changeRequest()
+            }
+
+          }
+        }
+        steps {
+          withGithubNotify(context: 'Check Apm Server Beats Update') {
+            beatsUpdate()
+          }
         }
       }
     }
   }
-  // post {
-  //   cleanup {
-  //     notifyBuildResult()
-  //   }
-  // }
 }
 
 def beatsUpdate() {

.ci/build-docker-images.groovy

@@ -3,18 +3,16 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'linux && immutable' }
+  agent { label 'ubuntu-16 && immutable' }
   environment {
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
     DOCKER_REGISTRY = 'docker.elastic.co'
     DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
-    GOPATH = "${env.WORKSPACE}"
     HOME = "${env.WORKSPACE}"
-    JOB_GCS_BUCKET = credentials('gcs-bucket')
     NOTIFY_TO = credentials('notify-to')
-    PATH = "${env.GOPATH}/bin:${env.PATH}"
-    PIPELINE_LOG_LEVEL='INFO'
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -31,26 +29,20 @@ pipeline {
   }
   parameters {
     booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images")
+    string(name: 'BRANCH_REFERENCE', defaultValue: "master", description: "Git branch/tag to use")
   }
   stages {
     stage('Checkout') {
       steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "${params.BRANCH_REFERENCE}",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
         dir("${BASE_DIR}"){
-          git("https://github.com/elastic/${REPO}.git")
+          setEnvVar("GO_VERSION", readFile(file: ".go-version")?.trim())
         }
-        script {
-          dir("${BASE_DIR}"){
-            env.GO_VERSION = readFile(".go-version").trim()
-          }
-        }
-      }
-    }
-    stage('Install dependencies') {
-      when {
-        expression { return params.RELEASE_TEST_IMAGES }
-      }
-      steps {
-        sh(label: 'Install virtualenv', script: 'pip install --user virtualenv')
       }
     }
     stage('Metricbeat Test Docker images'){
@@ -62,9 +54,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/metricbeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }
@@ -78,9 +73,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/metricbeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }
@@ -94,9 +92,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/filebeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/filebeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }

.ci/jobs/apm-beats-update.yml

@@ -0,0 +1,60 @@
+- job:
+    name: Beats/apm-beats-update
+    display-name: 'APM Server Beats update'
+    description: 'Check if the Beats lib update works on the APM server master branch'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    prune-dead-branches: true
+    number-to-keep: 10
+    days-to-keep: 30
+    script-path: '.ci/apm-beats-update.groovy'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        discover-tags: true
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        disable-pr-notifications: true
+        notification-context: 'apm-beats-update'
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: github-app-beats-ci
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - skip-initial-build: true
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: 30
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        - change-request:
+            ignore-target-only-changes: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true

.ci/jobs/beats-test-infra.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: Beats/beats-test-infra
+    display-name: 'Beats test infra'
+    description: 'Pipeline to run the test infra'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: .ci/validateWorkersBeatsCi.groovy
+        scm:
+        - git:
+            url: [email protected]:elastic/apm-pipeline-library.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/apm-test-pipeline.git
+            branches:
+                - master
+    triggers:
+        - timed: 'H H(3-4) * * 1-5'

.ci/jobs/beats-windows-mbp.yml

@@ -1,7 +1,7 @@
 ---
 - job:
     name: Beats/beats-windows-mbp
-    display-name: Beats Pipeline for Windows 
+    display-name: Beats Pipeline for Windows
     description: Jenkins pipeline for the Beats project running on windows agents.
     view: Beats
     project-type: multibranch
@@ -11,13 +11,14 @@
           branch-discovery: no-pr
           discover-pr-forks-strategy: merge-current
           discover-pr-forks-trust: permission
-          discover-pr-origin: merge-current          
+          discover-pr-origin: merge-current
           discover-tags: false
-          head-filter-regex: 'master'
-          notification-context: 'beats-ci'
+          # Run MBP for the master branch and PRs
+          head-filter-regex: '(master|PR-.*)'
+          notification-context: 'beats-ci/windows'
           repo: beats
           repo-owner: elastic
-          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          credentials-id: github-app-beats-ci
           ssh-checkout:
             credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
           build-strategies:
@@ -26,7 +27,18 @@
                 ignore-tags-newer-than: -1
             - regular-branches: true
             - change-request:
-                ignore-target-only-changes: false
+                ignore-target-only-changes: true
+          property-strategies:
+            # Builds for PRs won't be triggered automatically.
+            # Only the master branch will be triggered automatically.
+            named-branches:
+              defaults:
+                - suppress-scm-triggering: true
+              exceptions:
+                - exception:
+                  branch-name: master
+                  properties:
+                    - suppress-scm-triggering: false
           clean:
             after: true
             before: true