Merge remote-tracking branch 'upstream/master' into feature/fix-warni…

…ngs-archive

* upstream/master: (119 commits)
  Update filebeat input docs (elastic#19110)
  Add ECS fields from log pipeline of PostgreSQL (elastic#19127)
  Init package libbeat/statestore (elastic#19117)
  [Ingest Manager] Retryable downloads of beats (elastic#19102)
  [DOCS] Add output.console to Functionbeat doc and Functionbeat reference file (elastic#18965)
  Add compatibility info (elastic#18929)
  Set ecszap version to v0.2.0 (elastic#19106)
  [filebeat][httpjson] Fix unit test function call (elastic#19124)
  [Filebeat][httpjson] Adds oauth2 support for httpjson input (elastic#18892)
  Allow host.* fields to be disabled in Suricata module (elastic#19107)
  Make selector string casing configurable (elastic#18854)
  Switch the GRPC communication where Agent is running the server and the beats are connecting back to Agent (elastic#18973)
  Disable host.* fields by default for netflow module (elastic#19087)
  Automatically fill zube teams on backports if available (elastic#18924)
  Fix crash on vsphere module (elastic#19078)
  [Ingest Manager] Download snapshot artifacts from snapshots repo (elastic#18685)
  [Ingest Manager] Basic Elastic Agent documentation (elastic#19030)
  Make user.id a string in system/users, in line with ECS (elastic#19019)
  [docs] Add 7.8 release highlights placeholder file (elastic#18493)
  Fix translate_sid's empty target field handling (elastic#18991)
  ...

.ci/build-docker-images.groovy

@@ -0,0 +1,110 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    GOPATH = "${env.WORKSPACE}"
+    HOME = "${env.WORKSPACE}"
+    JOB_GCS_BUCKET = credentials('gcs-bucket')
+    NOTIFY_TO = credentials('notify-to')
+    PATH = "${env.GOPATH}/bin:${env.PATH}"
+    PIPELINE_LOG_LEVEL='INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    cron 'H H(0-5) * * 1-5'
+  }
+  parameters {
+    booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images")
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        dir("${BASE_DIR}"){
+          git("https://github.com/elastic/${REPO}.git")
+        }
+        script {
+          dir("${BASE_DIR}"){
+            env.GO_VERSION = readFile(".go-version").trim()
+          }
+        }
+      }
+    }
+    stage('Install dependencies') {
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        sh(label: 'Install virtualenv', script: 'pip install --user virtualenv')
+      }
+    }
+    stage('Metricbeat Test Docker images'){
+      options {
+        warnError('Metricbeat Test Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        dir("${HOME}/${BASE_DIR}"){
+          retry(3){
+            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/metricbeat'")
+          }
+        }
+      }
+    }
+    stage('Metricbeat x-pack Test Docker images'){
+      options {
+        warnError('Metricbeat x-pack Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        dir("${HOME}/${BASE_DIR}"){
+          retry(3){
+            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/metricbeat'")
+          }
+        }
+      }
+    }
+    stage('Filebeat x-pack Test Docker images'){
+      options {
+        warnError('Filebeat x-pack Test Docker images failed')
+      }
+      when {
+        expression { return params.RELEASE_TEST_IMAGES }
+      }
+      steps {
+        dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
+        dir("${HOME}/${BASE_DIR}"){
+          retry(3){
+            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/filebeat'")
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

.ci/jobs/beats-windows-mbp.yml

@@ -0,0 +1,44 @@
+---
+- job:
+    name: Beats/beats-windows-mbp
+    display-name: Beats Pipeline for Windows 
+    description: Jenkins pipeline for the Beats project running on windows agents.
+    view: Beats
+    project-type: multibranch
+    script-path: .ci/windows.groovy
+    scm:
+      - github:
+          branch-discovery: no-pr
+          discover-pr-forks-strategy: merge-current
+          discover-pr-forks-trust: permission
+          discover-pr-origin: merge-current          
+          discover-tags: false
+          head-filter-regex: 'master'
+          notification-context: 'beats-ci'
+          repo: beats
+          repo-owner: elastic
+          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          ssh-checkout:
+            credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          build-strategies:
+            - tags:
+                ignore-tags-older-than: -1
+                ignore-tags-newer-than: -1
+            - regular-branches: true
+            - change-request:
+                ignore-target-only-changes: false
+          clean:
+            after: true
+            before: true
+          prune: true
+          shallow-clone: true
+          depth: 4
+          do-not-fetch-tags: true
+          submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+          timeout: '15'
+          use-author: true
+          wipe-workspace: 'True'

.ci/jobs/build-it-docker-images.yml

@@ -0,0 +1,22 @@
+---
+- job:
+    name: Beats/build-it-docker-images
+    display-name: ITs Docker images
+    description: Job to pre-build docker images used in integration tests.
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+      script-path: .ci/build-docker-images.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - 'master'
+    triggers:
+      - timed: 'H H(0-5) * * 1-5'

.ci/jobs/defaults.yml

@@ -0,0 +1,20 @@
+
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: beats-ci
+
+##### JOB DEFAULTS
+
+- job:
+    logrotate:
+      numToKeep: 20
+    node: linux
+    concurrent: true
+    publishers:
+      - email:
+          recipients: [email protected]
+    periodic-folder-trigger: 1w
+    prune-dead-branches: true

.ci/scripts/build-beats-integrations-test-images.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -exo pipefail
+
+#
+# Install the given go version using the gimme script.
+#
+# Parameters:
+#   - GO_VERSION - that's the version which will be installed and enabled.
+#   - BEAT_BASE_DIR - that's the base directory of the Beat.
+#
+
+readonly GO_VERSION="${1?Please define the Go version to be used}"
+readonly BEAT_BASE_DIR="${2?Please define the location of the Beat directory}"
+
+function build_test_images() {
+    local baseDir="${1}"
+
+    cd "${baseDir}"
+    mage compose:buildSupportedVersions
+}
+
+function install_go() {
+    local goVersion="${1}"
+
+    # Install Go using the same travis approach
+    echo "Installing ${goVersion} with gimme."
+    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=${goVersion} bash)"
+}
+
+function install_mage() {
+    local baseDir="${1}"
+
+    cd "${baseDir}"
+    make mage
+}
+
+function push_test_images() {
+    local baseDir="${1}"
+
+    cd "${baseDir}"
+    mage compose:pushSupportedVersions
+}
+
+function main() {
+    install_go "${GO_VERSION}"
+    install_mage "${BEAT_BASE_DIR}"
+
+    build_test_images "${BEAT_BASE_DIR}"
+    push_test_images "${BEAT_BASE_DIR}"
+}
+
+main "$@"