Merge remote-tracking branch 'upstream/master' into feature/windows-o…

…ne-beats * upstream/master: (105 commits) ci: enable packaging job (elastic#19536) ci: disable upstream trigger on PRs for the packaging job (elastic#19490) Implement memlog on-disk handling (elastic#19408) fix go.mod for PR elastic#19423 (elastic#19521) [MetricBeat] add param `aws_partition` to support aws-cn, aws-us-gov regions (elastic#19423) Input v2 stateless manager (elastic#19406) Input v2 compatibility layer (elastic#19401) [Elastic Agent] Fix artifact downloading to allow endpoint-security to be downloaded (elastic#19503) fix: ignore target changes on scans (elastic#19510) Add more helpers to pipeline/testing package (elastic#19405) Report dependencies in CSV format (elastic#19506) [Filebeat] Fix reference leak in TCP and Unix socket inputs (elastic#19459) Cursor input skeleton (elastic#19378) Add changelog. (elastic#19495) [DOC] Typo in Kerberos (elastic#19265) Remove accidentally commited unused NOTICE template (elastic#19485) [Elastic Agent] Support the install, control, and uninstall of Endpoint (elastic#19248) [Filebeat][httpjson] Add split_events_by config setting (elastic#19246) ci: disabling packaging job until we fix it (elastic#19481) Fix golang.org/x/tools to release1.13 (elastic#19478) ...
v1v · Jul 2, 2020 · c57b3ae · c57b3ae
2 parents d74b1b7 + d6f23ff
commit c57b3ae
Show file tree

Hide file tree

Showing 11,341 changed files with 178,085 additions and 2,999,373 deletions.
diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy
@@ -9,12 +9,10 @@ pipeline {
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
     DOCKER_REGISTRY = 'docker.elastic.co'
     DOCKER_REGISTRY_SECRET = 'secret/observability-team/ci/docker-registry/prod'
-    GOPATH = "${env.WORKSPACE}"
     HOME = "${env.WORKSPACE}"
-    JOB_GCS_BUCKET = credentials('gcs-bucket')
     NOTIFY_TO = credentials('notify-to')
-    PATH = "${env.GOPATH}/bin:${env.PATH}"
-    PIPELINE_LOG_LEVEL='INFO'
+    PIPELINE_LOG_LEVEL = 'INFO'
+    JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -31,26 +29,20 @@ pipeline {
   }
   parameters {
     booleanParam(name: "RELEASE_TEST_IMAGES", defaultValue: "true", description: "If it's needed to build & push Beats' test images")
+    string(name: 'BRANCH_REFERENCE', defaultValue: "master", description: "Git branch/tag to use")
   }
   stages {
     stage('Checkout') {
       steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}",
+          branch: "${params.BRANCH_REFERENCE}",
+          repo: "https://github.com/elastic/${REPO}.git",
+          credentialsId: "${JOB_GIT_CREDENTIALS}"
+        )
         dir("${BASE_DIR}"){
-          git("https://github.com/elastic/${REPO}.git")
+          setEnvVar("GO_VERSION", readFile(file: ".go-version")?.trim())
         }
-        script {
-          dir("${BASE_DIR}"){
-            env.GO_VERSION = readFile(".go-version").trim()
-          }
-        }
-      }
-    }
-    stage('Install dependencies') {
-      when {
-        expression { return params.RELEASE_TEST_IMAGES }
-      }
-      steps {
-        sh(label: 'Install virtualenv', script: 'pip install --user virtualenv')
       }
     }
     stage('Metricbeat Test Docker images'){
@@ -62,9 +54,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/metricbeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }
@@ -78,9 +73,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/metricbeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/metricbeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }
@@ -94,9 +92,12 @@ pipeline {
       }
       steps {
         dockerLogin(secret: "${env.DOCKER_REGISTRY_SECRET}", registry: "${env.DOCKER_REGISTRY}")
-        dir("${HOME}/${BASE_DIR}"){
-          retry(3){
-            sh(label: 'Build ', script: ".ci/scripts/build-beats-integrations-test-images.sh '${GO_VERSION}' '${HOME}/${BASE_DIR}/x-pack/filebeat'")
+        withMageEnv(){
+          dir("${BASE_DIR}/x-pack/filebeat"){
+            retryWithSleep(retries: 3, seconds: 5, backoff: true){
+              sh(label: 'Build', script: "mage compose:buildSupportedVersions");
+              sh(label: 'Push', script: "mage compose:pushSupportedVersions");
+            }
           }
         }
       }

diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml
@@ -0,0 +1,62 @@
+- job:
+    name: Beats/apm-beats-update
+    display-name: 'APM Server Beats update'
+    description: 'Check if the Beats lib update works on the APM server master branch'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    periodic-folder-trigger: 1w
+    prune-dead-branches: true
+    number-to-keep: 10
+    days-to-keep: 30
+    script-path: '.ci/apm-beats-update.groovy'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        discover-tags: true
+        disable-pr-notifications: true
+        notification-context: 'apm-beats-update'
+        property-strategies:
+          all-branches:
+          - suppress-scm-triggering: true
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        - change-request:
+            ignore-target-only-changes: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true
diff --git a/.ci/jobs/beats-test-infra.yml b/.ci/jobs/beats-test-infra.yml
@@ -0,0 +1,22 @@
+---
+- job:
+    name: Beats/beats-test-infra
+    display-name: 'Beats test infra'
+    description: 'Pipeline to run the test infra'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: .ci/validateWorkersBeatsCi.groovy
+        scm:
+        - git:
+            url: [email protected]:elastic/apm-pipeline-library.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/apm-test-pipeline.git
+            branches:
+                - master
+    triggers:
+        - timed: 'H H(3-4) * * 1-5'
diff --git a/.ci/jobs/beats-windows-mbp.yml b/.ci/jobs/beats-windows-mbp.yml
@@ -1,7 +1,7 @@
 ---
 - job:
     name: Beats/beats-windows-mbp
-    display-name: Beats Pipeline for Windows 
+    display-name: Beats Pipeline for Windows
     description: Jenkins pipeline for the Beats project running on windows agents.
     view: Beats
     project-type: multibranch
@@ -11,7 +11,7 @@
           branch-discovery: no-pr
           discover-pr-forks-strategy: merge-current
           discover-pr-forks-trust: permission
-          discover-pr-origin: merge-current          
+          discover-pr-origin: merge-current
           discover-tags: false
           # Run MBP for the master branch and PRs
           head-filter-regex: '(master|PR-.*)'
@@ -27,10 +27,10 @@
                 ignore-tags-newer-than: -1
             - regular-branches: true
             - change-request:
-                ignore-target-only-changes: false
+                ignore-target-only-changes: true
           property-strategies:
             # Builds for PRs won't be triggered automatically.
-            # Only the master branch will be triggered automatically. 
+            # Only the master branch will be triggered automatically.
             named-branches:
               defaults:
                 - suppress-scm-triggering: true

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -0,0 +1,62 @@
+---
+- job:
+    name: Beats/beats
+    display-name: 'Beats (replacement)'
+    description: 'Beats Main Pipeline'
+    view: Beats
+    concurrent: true
+    project-type: multibranch
+    periodic-folder-trigger: 1w
+    prune-dead-branches: true
+    number-to-keep: 10
+    days-to-keep: 30
+    script-path: 'Jenkinsfile'
+    triggers: []
+    wrappers: []
+    scm:
+    - github:
+        branch-discovery: 'no-pr'
+        discover-pr-forks-strategy: 'merge-current'
+        discover-pr-forks-trust: 'permission'
+        discover-pr-origin: 'merge-current'
+        discover-tags: true
+        notification-context: "beats-ci"
+        property-strategies:
+          all-branches:
+          - suppress-scm-triggering: true
+        repo: 'beats'
+        repo-owner: 'elastic'
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - change-request:
+            ignore-target-only-changes: true
+        - named-branches:
+            - exact-name:
+                name: 'master'
+                case-sensitive: true
+            - regex-name:
+                regex: '7\.[x789]'
+                case-sensitive: true
+            - regex-name:
+                regex: '8\.\d+'
+                case-sensitive: true
+        clean:
+            after: true
+            before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+            disable: false
+            recursive: true
+            parent-credentials: true
+            timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: true
diff --git a/.ci/jobs/build-it-docker-images.yml b/.ci/jobs/build-it-docker-images.yml
@@ -5,6 +5,11 @@
     description: Job to pre-build docker images used in integration tests.
     view: Beats
     project-type: pipeline
+    parameters:
+      - string:
+          name: BRANCH_REFERENCE
+          default: master
+          description: the Git branch specifier
     pipeline-scm:
       script-path: .ci/build-docker-images.groovy
       scm:
@@ -17,6 +22,6 @@
             credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
             reference-repo: /var/lib/jenkins/.git-references/beats.git
             branches:
-              - 'master'
+              - $BRANCH_REFERENCE
     triggers:
       - timed: 'H H(0-5) * * 1-5'
diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml
@@ -0,0 +1,6 @@
+---
+#https://docs.openstack.org/infra/jenkins-job-builder/project_folder.html
+- job:
+    name: Beats
+    description: Beats
+    project-type: folder
diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml
@@ -11,7 +11,7 @@
           branch-discovery: no-pr
           discover-pr-forks-strategy: merge-current
           discover-pr-forks-trust: permission
-          discover-pr-origin: merge-current          
+          discover-pr-origin: merge-current
           discover-tags: true
           notification-context: 'beats-ci'
           repo: golang-crossbuild
@@ -25,7 +25,7 @@
                 ignore-tags-newer-than: -1
             - regular-branches: true
             - change-request:
-                ignore-target-only-changes: false
+                ignore-target-only-changes: true
           clean:
             after: true
             before: true

diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -0,0 +1,54 @@
+---
+- job:
+    name: Beats/packaging
+    display-name: Beats Packaging
+    description: Make the packages for beats and publish them on a GCS bucket.
+    view: Beats
+    disabled: false
+    project-type: multibranch
+    script-path: .ci/packaging.groovy
+    scm:
+      - github:
+          branch-discovery: 'no-pr'
+          discover-pr-forks-strategy: 'merge-current'
+          discover-pr-forks-trust: 'permission'
+          discover-pr-origin: 'merge-current'
+          discover-tags: true
+          disable-pr-notifications: false
+          notification-context: 'beats-packaging'
+          repo: 'beats'
+          repo-owner: 'elastic'
+          credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+          ssh-checkout:
+            credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          build-strategies:
+          - tags:
+              ignore-tags-older-than: 30
+              ignore-tags-newer-than: -1
+          - named-branches:
+              - exact-name:
+                  name: 'master'
+                  case-sensitive: true
+              - regex-name:
+                  regex: '7\.[x789]'
+                  case-sensitive: true
+              - regex-name:
+                  regex: '8\.\d+'
+                  case-sensitive: true
+          - change-request:
+              ignore-target-only-changes: true
+          clean:
+              after: true
+              before: true
+          prune: true
+          shallow-clone: true
+          depth: 3
+          do-not-fetch-tags: true
+          submodule:
+              disable: false
+              recursive: true
+              parent-credentials: true
+              timeout: 100
+          timeout: '15'
+          use-author: true
+          wipe-workspace: true