CI: split stages in categories (#25112) (#25258)

(cherry picked from commit f4c570b) Co-authored-by: Victor Martinez <[email protected]>
elastic · Apr 30, 2021 · 1114678 · 1114678
1 parent 839aa77
commit 1114678
Show file tree

Hide file tree

Showing 23 changed files with 341 additions and 78 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -65,6 +65,11 @@ pipeline {
             retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') }
           }
         }
+        withMageEnv(version: "${env.GO_VERSION}"){
+          dir("${BASE_DIR}"){
+            setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
+          }
+        }
       }
     }
     stage('Lint'){
@@ -77,15 +82,11 @@ pipeline {
           withGithubNotify(context: "Lint") {
             withBeatsEnv(archive: false, id: "lint") {
               dumpVariables()
-              setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
               whenTrue(env.ONLY_DOCS == 'true') {
                 cmd(label: "make check", script: "make check")
               }
               whenTrue(env.ONLY_DOCS == 'false') {
-                cmd(label: "make check-python", script: "make check-python")
-                cmd(label: "make check-go", script: "make check-go")
-                cmd(label: "make notice", script: "make notice")
-                cmd(label: "Check for changes", script: "make check-no-changes")
+                runLinting()
               }
             }
           }
@@ -108,28 +109,48 @@ pipeline {
         }
       }
       steps {
-        deleteDir()
-        unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-        dir("${BASE_DIR}"){
-          script {
-            def mapParallelTasks = [:]
-            def content = readYaml(file: 'Jenkinsfile.yml')
-            if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
-              error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
-            } else {
-              content['projects'].each { projectName ->
-                generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
-                  mapParallelTasks["${k}"] = v
-                }
-              }
-              notifyBuildReason()
-              parallel(mapParallelTasks)
-            }
+        runBuildAndTest(filterStage: 'mandatory')
+      }
+    }
+    stage('Extended') {
+      options { skipDefaultCheckout() }
+      when {
+        // Always when running builds on branches/tags
+        // On a PR basis, skip if changes are only related to docs.
+        // Always when forcing the input parameter
+        anyOf {
+          not { changeRequest() }                           // If no PR
+          allOf {                                           // If PR and no docs changes
+            expression { return env.ONLY_DOCS == "false" }
+            changeRequest()
           }
+          expression { return params.runAllStages }         // If UI forced
         }
       }
+      steps {
+        runBuildAndTest(filterStage: 'extended')
+      }
     }
     stage('Packaging') {
+      options { skipDefaultCheckout() }
+      when {
+        // Always when running builds on branches/tags
+        // On a PR basis, skip if changes are only related to docs.
+        // Always when forcing the input parameter
+        anyOf {
+          not { changeRequest() }                           // If no PR
+          allOf {                                           // If PR and no docs changes
+            expression { return env.ONLY_DOCS == "false" }
+            changeRequest()
+          }
+          expression { return params.runAllStages }         // If UI forced
+        }
+      }
+      steps {
+        runBuildAndTest(filterStage: 'packaging')
+      }
+    }
+    stage('Packaging-Pipeline') {
       agent none
       options { skipDefaultCheckout() }
       when {
@@ -166,6 +187,46 @@ VERSION=${env.VERSION}-SNAPSHOT""")
   }
 }
 
+def runLinting() {
+  def mapParallelTasks = [:]
+  def content = readYaml(file: 'Jenkinsfile.yml')
+  content['projects'].each { projectName ->
+    generateStages(project: projectName, changeset: content['changeset'], filterStage: 'lint').each { k,v ->
+      mapParallelTasks["${k}"] = v
+    }
+  }
+  mapParallelTasks['default'] = {
+                                cmd(label: "make check-python", script: "make check-python")
+                                cmd(label: "make check-go", script: "make check-go")
+                                cmd(label: "make notice", script: "make notice")
+                                cmd(label: "Check for changes", script: "make check-no-changes")
+                              }
+
+  parallel(mapParallelTasks)
+}
+
+def runBuildAndTest(Map args = [:]) {
+  def filterStage = args.get('filterStage', 'mandatory')
+  deleteDir()
+  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+  dir("${BASE_DIR}"){
+    def mapParallelTasks = [:]
+    def content = readYaml(file: 'Jenkinsfile.yml')
+    if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
+      error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
+    } else {
+      content['projects'].each { projectName ->
+        generateStages(project: projectName, changeset: content['changeset'], filterStage: filterStage).each { k,v ->
+          mapParallelTasks["${k}"] = v
+        }
+      }
+      notifyBuildReason()
+      parallel(mapParallelTasks)
+    }
+  }
+}
+
+
 /**
 * There are only two supported branches, master and 7.x
 */
@@ -191,21 +252,21 @@ def getBranchIndice(String compare) {
   return 'master'
 }
 
-
 /**
 * This method is the one used for running the parallel stages, therefore
 * its arguments are passed by the beatsStages step.
 */
 def generateStages(Map args = [:]) {
   def projectName = args.project
+  def filterStage = args.get('filterStage', 'all')
   def changeset = args.changeset
   def mapParallelStages = [:]
   def fileName = "${projectName}/Jenkinsfile.yml"
   if (fileExists(fileName)) {
     def content = readYaml(file: fileName)
     // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation.
     if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) {
-      mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this))
+      mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this), filterStage: filterStage)
     }
   } else {
     log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml")

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -13,13 +13,14 @@ when:
     tags: true                 ## for all the tags
 platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
-    Lint:
+    lint:
         make: |
           make -C auditbeat check;
           make -C auditbeat update;
           make -C x-pack/auditbeat check;
           make -C x-pack/auditbeat update;
           make check-no-changes;
+        stage: lint
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
@@ -33,10 +34,13 @@ stages:
                 - "armTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended
     build:
         mage: "mage build test"
+        stage: mandatory
     crosscompile:
         make: "make -C auditbeat crosscompile"
+        stage: mandatory
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
@@ -50,46 +54,56 @@ stages:
                 - "macosTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended
     windows:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-2019"
-            #- "windows-7-32-bit" https://github.com/elastic/beats/issues/19831
-            #- "windows-2008-r2" https://github.com/elastic/beats/issues/19799
+        stage: mandatory
     windows-2016:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-2016"
+        stage: extended
     windows-2012:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-2012-r2"
+        stage: extended
     windows-10:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-10"
+        stage: extended
     windows-8:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-8"
+        stage: extended
+    #windows-2008:  See https://github.com/elastic/beats/issues/19799
+    #    mage: "mage build unitTest"
+    #    platforms:             ## override default labels in this specific stage.
+    #        - "windows-2008-r2"
+    #    stage: extended
     #windows-7:  See https://github.com/elastic/beats/issues/19831
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-7"
-    #    when:                  ## Override the top-level when.
-    #        comments:
-    #            - "/test filebeat for windows-7"
-    #        labels:
-    #            - "windows-7"
-    #        branches: true     ## for all the branches
-    #        tags: true         ## for all the tags
+    #    stage: extended
+    #windows-7-32:  See https://github.com/elastic/beats/issues/19831
+    #    mage: "mage build unitTest"
+    #    platforms:             ## override default labels in this specific stage.
+    #        - "windows-7-32-bit"
+    #    stage: extended
     packaging-linux:
         packaging-linux: "mage package"
         e2e:
             enabled: false
+        stage: packaging
     packaging-arm:
         packaging-arm: "mage package"
         e2e:
             enabled: false
         platforms:             ## override default label in this specific stage.
           - "arm"
+        stage: packaging
diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml
@@ -13,8 +13,10 @@ when:
 platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
     lint:
-       make: |
-        make -C deploy/kubernetes all;
-        make check-no-changes;
+        make: |
+            make -C deploy/kubernetes all;
+            make check-no-changes;
+        stage: lint
     k8sTest:
         k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"
+        stage: mandatory
diff --git a/dev-tools/Jenkinsfile.yml b/dev-tools/Jenkinsfile.yml
@@ -13,4 +13,5 @@ when:
 platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
     lint:
-       make: "make -C dev-tools check"
+        make: "make -C dev-tools check"
+        stage: lint
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -13,13 +13,14 @@ when:
     tags: true                 ## for all the tags
 platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
-    Lint:
+    lint:
         make: |
           make -C filebeat check;
           make -C filebeat update;
           make -C x-pack/filebeat check;
           make -C x-pack/filebeat update;
           make check-no-changes;
+        stage: lint
     arm:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
@@ -33,9 +34,11 @@ stages:
                 - "armTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
+        stage: mandatory
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.
@@ -49,45 +52,48 @@ stages:
                 - "macosTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended
     windows:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-2019"
             #- "windows-2008-r2"  https://github.com/elastic/beats/issues/19795
+        stage: mandatory
     windows-2016:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-2016"
+        stage: extended
     windows-10:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-10"
+        stage: extended
     windows-8:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-8"
+        stage: extended
     #windows-7:  See https://github.com/elastic/beats/issues/22317
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.
     #        - "windows-7"
     #    when:                  ## Override the top-level when.
-    #        comments:
-    #            - "/test filebeat for windows-7"
-    #        labels:
-    #            - "windows-7"
-    #        branches: true     ## for all the branches
-    #        tags: true         ## for all the tags
+    #    stage: packaging
     windows-7-32:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+        stage: extended
     packaging-linux:
         packaging-linux: "mage package"
         e2e:
             enabled: false
+        stage: packaging
     packaging-arm:
         packaging-arm: "mage package"
         e2e:
             enabled: false
         platforms:             ## override default label in this specific stage.
           - "arm"
+        stage: packaging
diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml
@@ -17,8 +17,10 @@ platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
     metricbeat-test:
         make: "make -C generator/_templates/metricbeat test test-package"
+        stage: mandatory
     beat-test:
         make: "make -C generator/_templates/beat test test-package"
+        stage: mandatory
     macos-metricbeat:
         make: "make -C generator/_templates/metricbeat test"
         platforms:             ## override default label in this specific stage.
@@ -32,6 +34,7 @@ stages:
                 - "macosTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended
     macos-beat:
         make: "make -C generator/_templates/beat test"
         platforms:             ## override default label in this specific stage.
@@ -45,3 +48,4 @@ stages:
                 - "macosTest"
             branches: true     ## for all the branches
             tags: true         ## for all the tags
+        stage: extended