[CI] Pipeline 2.0 for monorepos (#20104)

.ci/scripts/generate_build_table.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+if __name__ == "__main__":
+
+    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|---------|------------|------|")
+    for root, dirs, files in os.walk("."):
+        dirs.sort()
+        for file in files:
+            if file.endswith("Jenkinsfile.yml") and root != ".":
+                with open(os.path.join(root, file), 'r') as f:
+                    doc = yaml.load(f, Loader=yaml.FullLoader)
+                module = root.replace(".{}".format(os.sep), '')
+                platforms = [doc["platform"]]
+                when = ""
+                if "branches" in doc["when"]:
+                    when = f"{when}/:palm_tree:"
+                if "changeset" in doc["when"]:
+                    when = f"{when}/:file_folder:"
+                if "comments" in doc["when"]:
+                    when = f"{when}/:speech_balloon:"
+                if "labels" in doc["when"]:
+                    when = f"{when}/:label:"
+                if "parameters" in doc["when"]:
+                    when = f"{when}/:smiley:"
+                if "tags" in doc["when"]:
+                    when = f"{when}/:taco:"
+                for stage in doc["stages"]:
+                    withModule = False
+                    if "make" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["make"]
+                    if "mage" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["mage"]
+                    if "platforms" in doc["stages"][stage]:
+                        platforms = doc["stages"][stage]["platforms"]
+                    if "withModule" in doc["stages"][stage]:
+                        withModule = doc["stages"][stage]["withModule"]
+                    if "when" in doc["stages"][stage]:
+                        when = f"{when}/:star:"
+                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
+                        module, stage, command, withModule, platforms, when))
+
+print("> :palm_tree: -> Git Branch based")
+print("> :label: -> GitHub Pull Request Label based")
+print("> :file_folder: -> Changeset based")
+print("> :speech_balloon: -> GitHub Pull Request comment based")
+print("> :taco: -> Git tag based")
+print("> :smiley: -> Manual UI interaction based")
+print("> :star: -> More specific cases based")

.ci/scripts/get-vendor-dependencies.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Given the go module it will list all the dependencies that will be later on
+# used by the CI to enable/disable specific stages as long as the changeset
+# matches any of those patterns.
+#
+
+GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"}
+BEATS=${1:?"parameter missing."}
+eval "$(gvm "${GO_VERSION}")"
+
+go list -deps ./"${BEATS}" \
+| grep 'elastic/beats' \
+| sort \
+| sed -e "s#github.com/elastic/beats/v7/##g" \
+| awk '{print "^" $1 "/.*"}'

.ci/scripts/install-tools.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+.ci/scripts/install-go.sh
+.ci/scripts/install-docker-compose.sh
+.ci/scripts/install-terraform.sh
+make mage