Merge remote-tracking branch 'upstream/master' into feature/add-githu…

…b-for-macos

* upstream/master: (172 commits)
  [Elastic Agent] Fix issue with ensureServiceToken. (elastic#29800)
  [Winlogbeat] Add provider name to Security routing pipeline check (elastic#29781)
  Add summary to journeys which don't emit journey:end (early node subprocess exits) (elastic#29606)
  Prepare 8.0.0-rc1 changelog (elastic#29795) (elastic#29806)
  Change docker image from CentOS 7 to Ubuntu 20.04 (elastic#29681)
  libbeat/processors/add_process_metadata: implement a process cache eviction policy (elastic#29717)
  [Automation] Update elastic stack version to 8.1.0-7004acda for testing (elastic#29783)
  Missing changelog entry for elastic#29773 (elastic#29791)
  Add a readme for k8s autodiscover provider (elastic#28213)
  Remove overriding of index pattern on the Kubernetes overview dashboard (elastic#29676)
  jjbb: remove obsoleted branches (<7.16) (elastic#29707)
  Add k8s metadata in state_cronjob metricset (elastic#29572)
  ibmmq: Fix timestamp parsing (elastic#29773)
  Do not add date to index if `@meta.index` is set (elastic#29775)
  ci: uses aliases for the branches (elastic#29706)
  Filebeat tests: Restore `@timestamp` field validation (elastic#29772)
  Forward port 7.16.3 changelog to master (elastic#29777)
  auditd: Store program arguments in process.args array (elastic#29601)
  System/socket: Support kernel_clone() replacement for _do_fork() (elastic#29744)
  Do not mention removal if version is not specified in `cfgwarn` messages (elastic#29727)
  ...

.ci/beats-tester-bc.groovy

@@ -7,7 +7,7 @@ pipeline {
   environment {
     BASE_DIR = 'src/github.com/elastic/beats'
     PIPELINE_LOG_LEVEL = "INFO"
-    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main'
     BASE_URL = "https://staging.elastic.co/${params.version}/downloads"
     APM_BASE_URL = "${env.BASE_URL}/apm-server"
     BEATS_BASE_URL = "${env.BASE_URL}/beats"

.ci/beats-tester.groovy

@@ -7,7 +7,7 @@ pipeline {
   environment {
     BASE_DIR = 'src/github.com/elastic/beats'
     PIPELINE_LOG_LEVEL = "INFO"
-    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
+    BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/main'
   }
   options {
     timeout(time: 1, unit: 'HOURS')

.ci/jobs/apm-beats-update.yml

@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'
@@ -36,10 +36,7 @@
                 name: 'master'
                 case-sensitive: true
             - regex-name:
-                regex: '7\.[x789]'
-                case-sensitive: true
-            - regex-name:
-                regex: '7\.1\d'
+                regex: '7\.1[6789]'
                 case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'

.ci/jobs/beats-release-changelog.yml

@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main

.ci/jobs/beats-release-minor-major.yml

@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main

.ci/jobs/beats-release-patch.yml

@@ -17,4 +17,4 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
             branches:
-                - master
+                - main

.ci/jobs/beats-tester.yml

@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-tester'
           repo: 'beats'
@@ -32,7 +32,7 @@
                   name: 'master'
                   case-sensitive: true
               - regex-name:
-                  regex: '7\.([x9]|1\d+)'
+                  regex: '7\.1[6789]'
                   case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|6\.[89]|7\.16|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|6\.[89]|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'
@@ -39,10 +39,7 @@
                 regex: '6\.[89]'
                 case-sensitive: true
             - regex-name:
-                regex: '7\.[x789]'
-                case-sensitive: true
-            - regex-name:
-                regex: '7\.1\d'
+                regex: '7\.1[6789]'
                 case-sensitive: true
             - regex-name:
                 regex: '8\.\d+'

.ci/jobs/packaging.yml

@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+          head-filter-regex: '(master|7\.1[6789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'
@@ -32,10 +32,7 @@
                   name: 'master'
                   case-sensitive: true
               - regex-name:
-                  regex: '7\.[x789]'
-                  case-sensitive: true
-              - regex-name:
-                  regex: '7\.1\d'
+                  regex: '7\.1[6789]'
                   case-sensitive: true
               - regex-name:
                   regex: '8\.\d+'

.ci/packaging.groovy

@@ -101,7 +101,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'winlogbeat',
@@ -111,7 +110,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/functionbeat',
                    'x-pack/heartbeat',
-                  // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
                   'x-pack/osquerybeat',
                   'x-pack/packetbeat',
@@ -199,7 +197,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'x-pack/auditbeat',
@@ -277,8 +274,6 @@ def pushCIDockerImages(Map args = [:]) {
       tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
       tagAndPush(beatName: 'heartbeat', arch: arch)
-    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
-      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
       tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {

.ci/packer_cache.sh

@@ -21,7 +21,7 @@ function dockerPullCommonImages() {
   docker.elastic.co/observability-ci/database-enterprise:12.2.0.1
   docker.elastic.co/beats-dev/fpm:1.11.0
   golang:1.14.12-stretch
-  centos:7
+  ubuntu:20.04
   "
   for image in ${DOCKER_IMAGES} ; do
     (retry 2 docker pull ${image}) || echo "Error pulling ${image} Docker image. Continuing."

.ci/schedule-daily.groovy

@@ -20,9 +20,7 @@ pipeline {
   stages {
     stage('Nighly beats builds') {
       steps {
-        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
-        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.15')
+        runBuilds(quietPeriodFactor: 2000, branches: ['master', '8.<minor>', '7.<minor>', '7.<next-minor>'])
       }
     }
   }
@@ -33,7 +31,37 @@ pipeline {
   }
 }
 
-def runBuild(Map args = [:]) {
-  def jobName = args.job
-  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+def runBuilds(Map args = [:]) {
+  def branches = []
+  // Expand macros and filter duplicated matches.
+  args.branches.each { branch ->
+    def branchName = getBranchName(branch)
+    if (!branches.contains(branchName)) {
+      branches << branchName
+    }
+  }
+
+  def quietPeriod = 0
+  branches.each { branch ->
+    build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+    // Increate the quiet period for the next iteration
+    quietPeriod += args.quietPeriodFactor
+  }
+}
+
+def getBranchName(branch) {
+  // special macro to look for the latest minor version
+  if (branch.contains('8.<minor>')) {
+   return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor8())
+  }
+  if (branch.contains('8.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor8())
+  }
+  if (branch.contains('7.<minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor7())
+  }
+  if (branch.contains('7.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor7())
+  }
+  return branch
 }

.ci/schedule-weekly.groovy

@@ -20,9 +20,7 @@ pipeline {
   stages {
     stage('Weekly beats builds') {
       steps {
-        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.16')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.15')
+        runBuilds(quietPeriodFactor: 1000, branches: ['master', '8.<minor>', '7.<minor>', '7.<next-minor>'])
       }
     }
   }
@@ -33,7 +31,37 @@ pipeline {
   }
 }
 
-def runBuild(Map args = [:]) {
-  def jobName = args.job
-  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+def runBuilds(Map args = [:]) {
+  def branches = []
+  // Expand macros and filter duplicated matches.
+  args.branches.each { branch ->
+    def branchName = getBranchName(branch)
+    if (!branches.contains(branchName)) {
+      branches << branchName
+    }
+  }
+
+  def quietPeriod = 0
+  branches.each { branch ->
+    build(quietPeriod: quietPeriod, job: "Beats/beats/${branch}", parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+    // Increate the quiet period for the next iteration
+    quietPeriod += args.quietPeriodFactor
+  }
+}
+
+def getBranchName(branch) {
+  // special macro to look for the latest minor version
+  if (branch.contains('8.<minor>')) {
+   return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor8())
+  }
+  if (branch.contains('8.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor8())
+  }
+  if (branch.contains('7.<minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getCurrentMinorReleaseFor7())
+  }
+  if (branch.contains('7.<next-minor>')) {
+    return bumpUtils.getMajorMinor(bumpUtils.getNextMinorReleaseFor7())
+  }
+  return branch
 }

.ci/scripts/kind-setup.sh

@@ -1,23 +1,5 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-kind create cluster --image kindest/node:${K8S_VERSION} --config - <<EOF
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: ClusterConfiguration
-    scheduler:
-      extraArgs:
-        bind-address: "0.0.0.0"
-        port: "10251"
-        secure-port: "10259"
-    controllerManager:
-      extraArgs:
-        bind-address: "0.0.0.0"
-        port: "10252"
-        secure-port: "10257"
-EOF
+kind create cluster --image kindest/node:${K8S_VERSION}
 kubectl cluster-info

.github/workflows/elastic-agent-project-board.yml

@@ -0,0 +1,46 @@
+name: Add to Elastic Agent Data Plane or Control Plane Board
+on:
+  issues:
+    types:
+      - labeled
+jobs:
+  add_to_data_plane-project:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.label.name == 'Team:Elastic-Agent-Data-Plane'
+    steps:
+      - uses: octokit/[email protected]
+        id: add_to_project
+        with:
+          headers: '{"GraphQL-Features": "projects_next_graphql"}'
+          query: |
+            mutation add_to_project($projectid:String!,$contentid:String!) {
+               updateIssue(input: {id:$contentid, projectIds:$projectid}) {
+                clientMutationId
+              }
+             }
+          projectid: ${{ env.PROJECT_ID }}
+          contentid: ${{ github.event.issue.node_id }}
+        env:
+          PROJECT_ID: "PRO_kwDOAGc3Zs4AzG8z"
+          GITHUB_TOKEN: ${{ secrets.ELASTIC_AGENT_PROJECT_BOARD_TOKEN }}
+  add_to_control_plane-project:
+    runs-on: ubuntu-latest
+    if: |
+      github.event.label.name == 'Team:Elastic-Agent-Control-Plane'
+    steps:
+      - uses: octokit/[email protected]
+        id: add_to_project
+        with:
+          headers: '{"GraphQL-Features": "projects_next_graphql"}'
+          query: |
+             mutation add_to_project($projectid:String!,$contentid:String!) {
+               updateIssue(input: {id:$contentid, projectIds:$projectid}) {
+                clientMutationId
+              }
+             }
+          projectid: ${{ env.PROJECT_ID }}
+          contentid: ${{ github.event.issue.node_id }}
+        env:
+          PROJECT_ID: "PRO_kwDOAGc3Zs4AzG9E"
+          GITHUB_TOKEN: ${{ secrets.ELASTIC_AGENT_PROJECT_BOARD_TOKEN }}