feat: add a new step to run the e2e tests for certain parts of Beats (e…

…lastic#21100)

* feat: add a new step to run the e2e tests for certain parts of Beats

We are going to trigger the tests for those parts affected by the
elastic-agent, filebeat, or metricbeat, because those are the ones we
verify  in the e2e-testing suite

* chore: do not include heartbeat

* feat: trigger the e2e tests

* fix: use relative path

* chore: use proper target branch name for PRs

* chore: use different tag

* fix: use proper env variable

* chore: pass github checks context to downstream job

* chore: revert shared lib version

Co-authored-by: Victor Martinez <[email protected]>

* chore: add BASE_DIR env variable

Co-authored-by: Victor Martinez <[email protected]>

* chore: remove duplicated env

* ffix: add param comma separator

* fix: wrong copy&paste

* chore: move e2e GH check out of the release context

* chore: simplify conditional logic

* chore: refine execution of test suites

* fix: use proper parameter name

* chore: set metricbeat version

* chore: remove slack notifications on PRs

* chore: update parameter

* chore: run multiple test suites per beat type

Co-authored-by: Victor Martinez <[email protected]>

.ci/packaging.groovy

@@ -5,12 +5,14 @@
 pipeline {
   agent none
   environment {
-    BASE_DIR = 'src/github.com/elastic/beats'
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
     SNAPSHOT = "true"
     PIPELINE_LOG_LEVEL = "INFO"
   }
@@ -119,6 +121,7 @@ pipeline {
                     release()
                     pushCIDockerImages()
                   }
+                  runE2ETestForPackages()
                 }
               }
               stage('Package Mac OS'){
@@ -208,6 +211,25 @@ def tagAndPush(name){
   }
 }
 
+def runE2ETestForPackages(){
+  def suite = ''
+
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
+    if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
+      suite = 'helm,ingest-manager'
+    } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
+      suite = ''
+    } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+      suite = 'ingest-manager'
+    } else {
+      echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
+      return
+    }
+
+    triggerE2ETests(suite)
+  }
+}
+
 def release(){
   withBeatsEnv(){
     dir("${env.BEATS_FOLDER}") {
@@ -217,6 +239,32 @@ def release(){
   }
 }
 
+def triggerE2ETests(String suite) {
+  echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")
+
+  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
+  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  build(job: "${e2eTestsPipeline}",
+    parameters: [
+      booleanParam(name: 'forceSkipGitChecks', value: true),
+      booleanParam(name: 'forceSkipPresubmit', value: true),
+      booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+      booleanParam(name: 'USE_CI_SNAPSHOTS', value: true),
+      string(name: 'ELASTIC_AGENT_VERSION', value: "pr-${env.CHANGE_ID}"),
+      string(name: 'METRICBEAT_VERSION', value: "pr-${env.CHANGE_ID}"),
+      string(name: 'runTestsSuites', value: suite),
+      string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+      string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+      string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+    ],
+    propagate: false,
+    wait: false
+  )
+
+  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
+  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],