From 62f1115f28cef188b8510667b6b62acf0439403b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 30 Oct 2020 09:13:44 +0000 Subject: [PATCH] [CI] 2.0 pipeline for 6.8 (#21859) * cherry-pick aad07ad22fdba8c9c7a145dbe3c8b28a943b083a * Add stages to Jenkins pipeline (#14484) * ci: avoid to modify go.mod file (#16842) * Ensure install scripts only install if needed (#20349) * ci: fix warnings with wildcards and archive system-tests (#18695) * ci: run test on Windows (#15570) * [CI] fail if not possible to install python3 (#19164) * [CI] lint stage doesn't produce test reports (#21888) * [CI] Add stage name in the step (#21887) * [CI] kind setup fails sometimes (#21857) * [CI] refactor runbld post build action (#21256) * [CI] Archive build reasons (#21347) * [CI] Fix runbld when workspace does not exist (#21350) * [CI] Send slack message with build status (#21428) * [CI] Setup git config globally (#21562) * [CI] fix 'no matches found within 10000' (#21466) * [CI] Change notification channel (#21559) * [CI] Support skip-ci label (#21377) * [BUILD][CI] fetch dependencies with retry (#21614) * Disable x-pack/metricbeat in linux since build is broken https://github.com/elastic/beats/issues/21962 * Disable tests for windows in auditbeat. See https://github.com/elastic/beats/issues/21958 * Disable integration testing for libbeat. See https://github.com/elastic/beats/issues/21959 * Disable integration testing for metricbeat. See https://github.com/elastic/beats/issues/21961 * Disable windows build for auditbeat. See https://github.com/elastic/beats/issues/21958 * Disable build for libbeat. See https://github.com/elastic/beats/issues/21959 * Disable build for metricbeat. See https://github.com/elastic/beats/issues/21961 * Disable windows build for metricbeat. See https://github.com/elastic/beats/issues/22257 * Disable build for x-pack/metricbeat. See https://github.com/elastic/beats/issues/21962 * Disable windows build for x-pack/winlogbeat. See https://github.com/elastic/beats/issues/22263 * Disable windows build for x-pack/filebeat. See https://github.com/elastic/beats/issues/21963 * integration-tests-environment is not available for winlogbeat. see https://github.com/elastic/beats/issues/22265 * integration-tests-environment is not available for packetbeat. see https://github.com/elastic/beats/issues/22266 * Build not available for x-pack/metricbeats in 6.8. See https://github.com/elastic/beats/issues/22269 * Cherry-pick 96c60d0a833497b1814788c1f362457f376bf9b0 --- .ci/scripts/generate_build_table.py | 52 ++ .ci/scripts/get-vendor-dependencies.sh | 16 + .ci/scripts/install-docker-compose.sh | 27 + .ci/scripts/install-go.sh | 30 ++ .ci/scripts/install-kind.sh | 26 + .ci/scripts/install-terraform.sh | 31 ++ .ci/scripts/install-tools.bat | 25 + .ci/scripts/install-tools.sh | 7 + .ci/scripts/kind-setup.sh | 18 + .ci/scripts/minikube-setup.sh | 33 ++ .ci/scripts/pre_archive_test.py | 20 + .ci/scripts/search_system_tests.py | 10 + Jenkinsfile | 491 ++++++++++++++++++ Jenkinsfile.yml | 44 ++ README.md | 35 ++ auditbeat/Jenkinsfile.yml | 25 + .../module/file_integrity/eventreader_test.go | 6 + deploy/kubernetes/Jenkinsfile.yml | 15 + filebeat/Jenkinsfile.yml | 21 + generator/Jenkinsfile.yml | 19 + heartbeat/Jenkinsfile.yml | 21 + journalbeat/Jenkinsfile.yml | 21 + libbeat/Jenkinsfile.yml | 24 + libbeat/scripts/Makefile | 2 +- metricbeat/Jenkinsfile.yml | 27 + packetbeat/Jenkinsfile.yml | 21 + script/fix_permissions.sh | 11 + winlogbeat/Jenkinsfile.yml | 23 + x-pack/auditbeat/Jenkinsfile.yml | 21 + x-pack/filebeat/Jenkinsfile.yml | 23 + x-pack/functionbeat/Jenkinsfile.yml | 21 + x-pack/libbeat/Jenkinsfile.yml | 21 + x-pack/metricbeat/Jenkinsfile.yml | 25 + x-pack/packetbeat/Jenkinsfile.yml | 22 + x-pack/winlogbeat/Jenkinsfile.yml | 21 + 35 files changed, 1254 insertions(+), 1 deletion(-) create mode 100755 .ci/scripts/generate_build_table.py create mode 100755 .ci/scripts/get-vendor-dependencies.sh create mode 100755 .ci/scripts/install-docker-compose.sh create mode 100755 .ci/scripts/install-go.sh create mode 100755 .ci/scripts/install-kind.sh create mode 100755 .ci/scripts/install-terraform.sh create mode 100644 .ci/scripts/install-tools.bat create mode 100755 .ci/scripts/install-tools.sh create mode 100755 .ci/scripts/kind-setup.sh create mode 100755 .ci/scripts/minikube-setup.sh create mode 100755 .ci/scripts/pre_archive_test.py create mode 100755 .ci/scripts/search_system_tests.py create mode 100644 Jenkinsfile create mode 100644 Jenkinsfile.yml create mode 100644 auditbeat/Jenkinsfile.yml create mode 100644 deploy/kubernetes/Jenkinsfile.yml create mode 100644 filebeat/Jenkinsfile.yml create mode 100644 generator/Jenkinsfile.yml create mode 100644 heartbeat/Jenkinsfile.yml create mode 100644 journalbeat/Jenkinsfile.yml create mode 100644 libbeat/Jenkinsfile.yml create mode 100644 metricbeat/Jenkinsfile.yml create mode 100644 packetbeat/Jenkinsfile.yml create mode 100755 script/fix_permissions.sh create mode 100644 winlogbeat/Jenkinsfile.yml create mode 100644 x-pack/auditbeat/Jenkinsfile.yml create mode 100644 x-pack/filebeat/Jenkinsfile.yml create mode 100644 x-pack/functionbeat/Jenkinsfile.yml create mode 100644 x-pack/libbeat/Jenkinsfile.yml create mode 100644 x-pack/metricbeat/Jenkinsfile.yml create mode 100644 x-pack/packetbeat/Jenkinsfile.yml create mode 100644 x-pack/winlogbeat/Jenkinsfile.yml diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py new file mode 100755 index 00000000000..21fe5d67107 --- /dev/null +++ b/.ci/scripts/generate_build_table.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import os +import yaml + +if __name__ == "__main__": + + print("| Beat | Stage | Command | MODULE | Platforms | When |") + print("|-------|--------|----------|---------|------------|------|") + for root, dirs, files in os.walk("."): + dirs.sort() + for file in files: + if file.endswith("Jenkinsfile.yml") and root != ".": + with open(os.path.join(root, file), 'r') as f: + doc = yaml.load(f, Loader=yaml.FullLoader) + module = root.replace(".{}".format(os.sep), '') + platforms = [doc["platform"]] + when = "" + if "branches" in doc["when"]: + when = f"{when}/:palm_tree:" + if "changeset" in doc["when"]: + when = f"{when}/:file_folder:" + if "comments" in doc["when"]: + when = f"{when}/:speech_balloon:" + if "labels" in doc["when"]: + when = f"{when}/:label:" + if "parameters" in doc["when"]: + when = f"{when}/:smiley:" + if "tags" in doc["when"]: + when = f"{when}/:taco:" + for stage in doc["stages"]: + withModule = False + if "make" in doc["stages"][stage]: + command = doc["stages"][stage]["make"] + if "mage" in doc["stages"][stage]: + command = doc["stages"][stage]["mage"] + if "platforms" in doc["stages"][stage]: + platforms = doc["stages"][stage]["platforms"] + if "withModule" in doc["stages"][stage]: + withModule = doc["stages"][stage]["withModule"] + if "when" in doc["stages"][stage]: + when = f"{when}/:star:" + print("| {} | {} | `{}` | {} | `{}` | {} |".format( + module, stage, command, withModule, platforms, when)) + +print("> :palm_tree: -> Git Branch based") +print("> :label: -> GitHub Pull Request Label based") +print("> :file_folder: -> Changeset based") +print("> :speech_balloon: -> GitHub Pull Request comment based") +print("> :taco: -> Git tag based") +print("> :smiley: -> Manual UI interaction based") +print("> :star: -> More specific cases based") diff --git a/.ci/scripts/get-vendor-dependencies.sh b/.ci/scripts/get-vendor-dependencies.sh new file mode 100755 index 00000000000..e002a208b76 --- /dev/null +++ b/.ci/scripts/get-vendor-dependencies.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Given the go module it will list all the dependencies that will be later on +# used by the CI to enable/disable specific stages as long as the changeset +# matches any of those patterns. +# + +GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"} +BEATS=${1:?"parameter missing."} +eval "$(gvm "${GO_VERSION}")" + +go list -deps ./"${BEATS}" \ +| grep 'elastic/beats' \ +| sort \ +| sed -e "s#github.com/elastic/beats/v7/##g" \ +| awk '{print "^" $1 "/.*"}' diff --git a/.ci/scripts/install-docker-compose.sh b/.ci/scripts/install-docker-compose.sh new file mode 100755 index 00000000000..72d889f216a --- /dev/null +++ b/.ci/scripts/install-docker-compose.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +MSG="environment variable missing: DOCKER_COMPOSE_VERSION." +DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG} +HOME=${HOME:?$MSG} + +if command -v docker-compose +then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ] + then + echo "Versions match. No need to install docker-compose. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing docker-compose" + +DC_CMD="${HOME}/bin/docker-compose" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" +chmod +x "${DC_CMD}" diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh new file mode 100755 index 00000000000..5727abccdda --- /dev/null +++ b/.ci/scripts/install-go.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="environment variable missing" +GO_VERSION=${GO_VERSION:?$MSG} +PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} +HOME=${HOME:?$MSG} +ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +GVM_CMD="${HOME}/bin/gvm" + +if command -v go +then + echo "Found Go. Checking version.." + FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//) + if [ $FOUND_GO_VERSION == $GO_VERSION ] + then + echo "Versions match. No need to install Go. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Go" +mkdir -p "${HOME}/bin" + +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-${ARCH}-amd64" +chmod +x "${GVM_CMD}" + +gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} + +eval $(gvm ${GO_VERSION}) diff --git a/.ci/scripts/install-kind.sh b/.ci/scripts/install-kind.sh new file mode 100755 index 00000000000..a53c4b3708a --- /dev/null +++ b/.ci/scripts/install-kind.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="environment variable missing." +DEFAULT_HOME="/usr/local" +KIND_VERSION=${KIND_VERSION:?$MSG} +HOME=${HOME:?$DEFAULT_HOME} +KIND_CMD="${HOME}/bin/kind" + +if command -v kind +then + echo "Found Kind. Checking version.." + FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') + if [ $FOUND_KIND_VERSION == $KIND_VERSION ] + then + echo "Versions match. No need to install Kind. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Kind" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" +chmod +x "${KIND_CMD}" diff --git a/.ci/scripts/install-terraform.sh b/.ci/scripts/install-terraform.sh new file mode 100755 index 00000000000..4af2e91baab --- /dev/null +++ b/.ci/scripts/install-terraform.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +MSG="environment variable missing." +TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG} +HOME=${HOME:?$MSG} +TERRAFORM_CMD="${HOME}/bin/terraform" + +OS=$(uname -s | tr '[:upper:]' '[:lower:]') + +if command -v terraform +then + echo "Found Terraform. Checking version.." + FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//) + if [ $FOUND_TERRAFORM_VERSION == $TERRAFORM_VERSION ] + then + echo "Versions match. No need to install Terraform. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Terraform" + +mkdir -p "${HOME}/bin" + +curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip +unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD}) +rm ${TERRAFORM_CMD}.zip + +chmod +x "${TERRAFORM_CMD}" diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat new file mode 100644 index 00000000000..0066b25be29 --- /dev/null +++ b/.ci/scripts/install-tools.bat @@ -0,0 +1,25 @@ +set GOPATH=%WORKSPACE% +set MAGEFILE_CACHE=%WORKSPACE%\.magefile +set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH% + +where /q curl +IF ERRORLEVEL 1 ( + choco install curl -y --no-progress --skipdownloadcache +) +mkdir %WORKSPACE%\bin +where /q gvm +IF ERRORLEVEL 1 ( + curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe +) +FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i + +go install github.com/elastic/beats/vendor/github.com/magefile/mage +mage -version +where mage + +if not exist C:\Python38\python.exe ( + REM Install python 3.8. + choco install python -y -r --no-progress --version 3.8.2 || echo ERROR && exit /b +) +python --version +where python diff --git a/.ci/scripts/install-tools.sh b/.ci/scripts/install-tools.sh new file mode 100755 index 00000000000..297a7820cad --- /dev/null +++ b/.ci/scripts/install-tools.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -exuo pipefail + +.ci/scripts/install-go.sh +.ci/scripts/install-docker-compose.sh +.ci/scripts/install-terraform.sh +make mage diff --git a/.ci/scripts/kind-setup.sh b/.ci/scripts/kind-setup.sh new file mode 100755 index 00000000000..4ac8fb0f6c3 --- /dev/null +++ b/.ci/scripts/kind-setup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +K8S_VERSION=${K8S_VERSION:?$MSG} +HOME=${HOME:?$MSG} +KBC_CMD="${HOME}/bin/kubectl" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" +chmod +x "${KBC_CMD}" + +GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1 +kind create cluster --image kindest/node:${K8S_VERSION} + +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl cluster-info diff --git a/.ci/scripts/minikube-setup.sh b/.ci/scripts/minikube-setup.sh new file mode 100755 index 00000000000..8c5d7e6aae9 --- /dev/null +++ b/.ci/scripts/minikube-setup.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +K8S_VERSION=${K8S_VERSION:?$MSG} +MINIKUBE_VERSION=${MINIKUBE_VERSION:?$MSG} +HOME=${HOME:?$MSG} + +KBC_CMD="${HOME}/bin/kubectl" +MKB_CMD="${HOME}/bin/minikube" + +export CHANGE_MINIKUBE_NONE_USER=true + +mkdir -p "${HOME}/bin" + +curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" +chmod +x "${KBC_CMD}" + +curl -sSLo "${MKB_CMD}" "https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64" +chmod +x "${MKB_CMD}" + +mkdir -p "${HOME}/.kube" "${HOME}/.minikube" +touch "${HOME}/.kube/config" + +minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION} --logtostderr +minikube update-context + +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' +until kubectl get nodes -o jsonpath="${JSONPATH}" 2>&1 | grep -q "Ready=True" +do + echo "waiting for Minikube..." + sleep 5 +done diff --git a/.ci/scripts/pre_archive_test.py b/.ci/scripts/pre_archive_test.py new file mode 100755 index 00000000000..8fd8cb75ea1 --- /dev/null +++ b/.ci/scripts/pre_archive_test.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import distutils +from distutils import dir_util + + +if __name__ == "__main__": + + if not os.path.exists('build'): + os.makedirs('build') + + # Top level folders to be excluded + EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools']) + for root, dirs, files in os.walk('.'): + dirs[:] = [d for d in dirs if d not in EXCLUDE] + if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))): + dest = os.path.join('build', root.replace(".{}".format(os.sep), '')) + print("Copy {} into {}".format(root, dest)) + distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1) diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py new file mode 100755 index 00000000000..0e3896d9ff4 --- /dev/null +++ b/.ci/scripts/search_system_tests.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import os + + +if __name__ == "__main__": + + for root, dirs, files in os.walk('build'): + if root.endswith(('system-tests')): + print(root.replace(".{}".format(os.sep), '')) diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..1ca7e81477d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,491 @@ +#!/usr/bin/env groovy + +@Library('apm@current') _ + +import groovy.transform.Field +/** + This is required to store the stashed id with the test results to be digested with runbld +*/ +@Field def stashedTestReports = [:] + +pipeline { + agent { label 'ubuntu-16 && immutable' } + environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' + REPO = 'beats' + BASE_DIR = "src/github.com/elastic/${env.REPO}" + DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" + DOCKER_REGISTRY = 'docker.elastic.co' + GOX_FLAGS = "-arch amd64" + JOB_GCS_BUCKET = 'beats-ci-temp' + JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' + OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + PIPELINE_LOG_LEVEL = 'INFO' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + SLACK_CHANNEL = "#beats-build" + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + } + options { + timeout(time: 2, unit: 'HOURS') + buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + timestamps() + ansiColor('xterm') + disableResume() + durabilityHint('PERFORMANCE_OPTIMIZED') + quietPeriod(10) + rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) + } + triggers { + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') + } + parameters { + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + } + stages { + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) + deleteDir() + gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + // Skip all the stages except docs for PR's with asciidoc and md changes only + setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) + setEnvVar('GO_VERSION', readFile(".go-version").trim()) + withEnv(["HOME=${env.WORKSPACE}"]) { + retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } + } + } + } + } + stage('Lint'){ + options { skipDefaultCheckout() } + environment { + GOFLAGS = '-mod=readonly' + } + steps { + withGithubNotify(context: 'Lint') { + withBeatsEnv(archive: false, id: 'lint') { + dumpVariables() + cmd(label: 'make check', script: 'make check') + } + } + } + } + stage('Build&Test') { + options { skipDefaultCheckout() } + when { + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() + } + expression { return params.runAllStages } // If UI forced + } + } + steps { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + script { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { + error 'Pull Request has been configured to be disabled when there is a skip-ci label match' + } else { + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> + mapParallelTasks["${k}"] = v + } + } + notifyBuildReason() + parallel(mapParallelTasks) + } + } + } + } + } + } + post { + always { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + runbld(stashedTestReports: stashedTestReports, project: env.REPO) + } + cleanup { + notifyBuildResult(prComment: true, slackComment: true, slackNotify: (isBranch() || isTag())) + } + } +} + +/** +* This method is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +*/ +def generateStages(Map args = [:]) { + def projectName = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def fileName = "${projectName}/Jenkinsfile.yml" + if (fileExists(fileName)) { + def content = readYaml(file: fileName) + // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) + } + } else { + log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") + } + return mapParallelStages +} + +def k8sTest(Map args = [:]) { + def versions = args.versions + versions.each{ v -> + node(args.label) { + stage("${args.context} ${v}"){ + withEnv(["K8S_VERSION=${v}", "MINIKUBE_VERSION=v0.25.2", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ + withGithubNotify(context: "${args.context} ${v}") { + withBeatsEnv(archive: false, withModule: false) { + retryWithSleep(retries: 3, seconds: 5, backoff: true){ + sh(label: "Setup minikube", script: ".ci/scripts/minikube-setup.sh") + } + sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") + } + } + } + } + } + } +} + +/** +* This method runs the given command supporting two kind of scenarios: +* - make -C then the dir(location) is not required, aka by disaling isMage: false +* - mage then the dir(location) is required, aka by enabling isMage: true. +*/ +def target(Map args = [:]) { + def context = args.context + def command = args.command + def directory = args.get('directory', '') + def withModule = args.get('withModule', false) + def isMage = args.get('isMage', false) + node(args.label) { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + dumpVariables() + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") + } + } + } + } +} + +/** +* This method wraps all the environment setup and pre-requirements to run any commands. +*/ +def withBeatsEnv(Map args = [:], Closure body) { + def archive = args.get('archive', true) + def withModule = args.get('withModule', false) + def directory = args.get('directory', '') + + def goRoot, path, magefile, pythonEnv, testResults, artifacts + + if(isUnix()) { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" + magefile = "${WORKSPACE}/.magefile" + pythonEnv = "${WORKSPACE}/python-env" + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' + } else { + def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}" + magefile = "${env.WORKSPACE}\\.magefile" + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" + } + + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + // NOTE: This is required to run after the unstash + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' + withEnv([ + "DOCKER_PULL=0", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "HOME=${env.WORKSPACE}", + "MAGEFILE_CACHE=${magefile}", + "MODULE=${module}", + "PATH=${path}", + "PYTHON_ENV=${pythonEnv}", + "RACE_DETECTOR=true", + "TEST_COVERAGE=true", + "TEST_TAGS=${env.TEST_TAGS},oracle" + ]) { + if(isDockerInstalled()) { + dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + } + dir("${env.BASE_DIR}") { + installTools() + if(isUnix()) { + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" + fi''') + } + try { + // Add more stability when dependencies are not accessible temporarily + // See https://github.com/elastic/beats/issues/21609 + // retry/try/catch approach reports errors, let's avoid it to keep the + // notifications cleaner. + if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) { + cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true) + } + body() + } finally { + if (archive) { + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) + } + // Tear down the setup for the permamnent workers. + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + fixPermissions("${WORKSPACE}") + deleteDir() + } + } + } + } +} + +/** +* This method fixes the filesystem permissions after the build has happenend. The reason is to +* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental +* issues. +*/ +def fixPermissions(location) { + if(isUnix()) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + set +x + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${location}""", returnStatus: true) + } +} + +/** +* This method installs the required dependencies that are for some reason not available in the +* CI Workers. +*/ +def installTools() { + if(isUnix()) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } + } else { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } +} + +/** +* This method gathers the module name, if required, in order to run the ITs only if +* the changeset affects a specific module. +* +* For such, it's required to look for changes under the module folder and exclude anything else +* such as asciidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String directory) { + // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') + def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) + def module = '' + + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) + } + return module +} + +/** +* This method archives and report the tests output, for such, it searches in certain folders +* to bypass some issues when working with big repositories. +*/ +def archiveTestOutput(Map args = [:]) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + if (isUnix()) { + fixPermissions("${WORKSPACE}") + } + cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') + dir('build') { + if (isUnix()) { + cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', + returnStatus: true, + script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') + } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') } + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id) + tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true) + } + catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { + def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() + log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") + if (folder.trim()) { + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() + tar(file: "${name}.tgz", archive: true, dir: folder) + } + } + } +} + +/** +* Replace the slashes in the directory in case there are nested folders. +*/ +def normalise(String directory) { + return directory.replaceAll("[\\W]|_",'-') +} + +/** +* For debugging purposes. +*/ +def dumpVariables(){ + echo "### MAGE DUMP ###" + cmd(label: 'Dump mage variables', script: 'mage dumpVariables') + echo "### END MAGE DUMP ###" + echo """ + ### ENV DUMP ### + BEAT_VERSION: ${env.BEAT_VERSION} + BEATS: ${env.BEATS} + BUILD_DIR: ${env.BUILD_DIR} + COMMIT_ID: ${env.COMMIT_ID} + COVERAGE_DIR: ${env.COVERAGE_DIR} + COVERAGE_TOOL: ${env.COVERAGE_TOOL} + COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO} + DOCKER_CACHE: ${env.DOCKER_CACHE} + DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME} + DOCKER_COMPOSE: ${env.DOCKER_COMPOSE} + FIND: ${env.FIND} + GOBUILD_FLAGS: ${env.GOBUILD_FLAGS} + GOIMPORTS: ${env.GOIMPORTS} + GOIMPORTS_REPO: ${env.GOIMPORTS_REPO} + GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX} + GOLINT: ${env.GOLINT} + GOLINT_REPO: ${env.GOLINT_REPO} + GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP} + GOX_FLAGS: ${env.GOX_FLAGS} + GOX_OS: ${env.GOX_OS} + GOX_OSARCH: ${env.GOX_OSARCH} + HOME: ${env.HOME} + NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS} + NOW: ${env.NOW} + PATH: ${env.PATH} + PKG_BUILD_DIR: ${env.PKG_BUILD_DIR} + PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR} + PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS} + PROJECTS: ${env.PROJECTS} + PROJECTS_ENV: ${env.PROJECTS_ENV} + PYTHON_ENV: ${env.PYTHON_ENV} + PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE} + PYTHON_EXE: ${env.PYTHON_EXE} + PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES} + PROCESSES: ${env.PROCESSES} + REVIEWDOG: ${env.REVIEWDOG} + REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS} + REVIEWDOG_REPO: ${env.REVIEWDOG_REPO} + STRESS_TESTS: ${env.STRESS_TESTS} + STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS} + SYSTEM_TESTS: ${env.SYSTEM_TESTS} + TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO} + TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT} + TEST_TAGS: ${env.TEST_TAGS} + TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT} + TIMEOUT: ${env.TIMEOUT} + USERPROFILE: ${env.USERPROFILE} + VENV_PARAMS: ${env.VENV_PARAMS} + XPACK_SUFFIX: ${env.XPACK_SUFFIX} + ### END ENV DUMP ### + """ +} + +def isDockerInstalled(){ + if (isUnix()) { + // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) { + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + } else { + return false + } +} + +/** +* Notify the build reason. +*/ +def notifyBuildReason() { + // Archive the build reason here, since the workspace can be deleted when running the parallel stages. + archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.*') + if (isPR()) { + echo 'TODO: Add a comment with the build reason (this is required to be implemented in the shared library)' + } +} + +/** +* This class is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +* +* What parameters/arguments are supported: +* - label -> the worker labels +* - project -> the name of the project that should match with the folder name. +* - content -> the specific stage data in the /Jenkinsfile.yml +* - context -> the name of the stage, normally -(-)? +*/ +class RunCommand extends co.elastic.beats.BeatsFunction { + public RunCommand(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def withModule = args.content.get('withModule', false) + if(args?.content?.containsKey('script')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('make')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('mage')) { + steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) + } + if(args?.content?.containsKey('k8sTest')) { + steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) + } + } +} + +/** +* This class retrieves the dependencies of a Go module for such it transforms them in a +* regex pattern. +*/ +class GetProjectDependencies extends co.elastic.beats.BeatsFunction { + public GetProjectDependencies(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def output = "" + steps.withEnv(["HOME=${steps.env.WORKSPACE}"]) { + output = steps.sh(label: 'Get vendor dependency patterns', returnStdout: true, + script: ".ci/scripts/get-vendor-dependencies.sh ${args.project}") + } + return output?.split('\n').collect{ item -> item as String } + } +} diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml new file mode 100644 index 00000000000..30985a19549 --- /dev/null +++ b/Jenkinsfile.yml @@ -0,0 +1,44 @@ +projects: + - "auditbeat" + ##- "deploy/kubernetes" It's no yet in the 1.0 pipeline + - "filebeat" + - "generator" + - "heartbeat" + - "journalbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/winlogbeat" + ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. + +## Changeset macros that are defined here and used in each specific 2.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + ## TODO: This will allow to configure what to do based on the PR configuration + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. \ No newline at end of file diff --git a/README.md b/README.md index 7bb72681625..29272947f15 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,38 @@ Beats](https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats. See our [CONTRIBUTING](CONTRIBUTING.md) file for information about setting up your dev environment to build Beats from the source. + +## Snapshots + +For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/Beats/job/packaging/job/master/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production. + +## CI + +### PR Comments + +It is possible to trigger some jobs by putting a comment on a GitHub PR. +(This service is only available for users affiliated with Elastic and not for open-source contributors.) + +* [beats][] + * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build. + * `/test macos` will kick off a default build with also the `macos` stages. + * `/test ` will kick off the default build for the given PR in addition to the `` build itself. + * `/test for macos` will kick off a default build with also the `macos` stage for the ``. +* [apm-beats-update][] + * `/run apm-beats-update` +* [apm-beats-packaging][] + * `/package` or `/packaging` will kick of a build to generate the packages for beats. +* [apm-beats-tester][] + * `/beats-tester` will kick of a build to validate the generated packages. + +### PR Labels + +It's possible to configure the build on a GitHub PR by labelling the PR with the below labels + +* `` to force the following builds to run the stages for the `` +* `macOS` to force the following builds to run the `macos` stages. + +[beats]: https://beats-ci.elastic.co/job/Beats/job/beats/ +[apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/ +[apm-beats-packaging]: https://beats-ci.elastic.co/job/Beats/job/packaging/ +[apm-beats-tester]: https://beats-ci.elastic.co/job/Beats/job/beats-tester/ diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..c28f0811c45 --- /dev/null +++ b/auditbeat/Jenkinsfile.yml @@ -0,0 +1,25 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test auditbeat" + labels: ## when PR labels matches any of those entries + - "auditbeat" + parameters: ## when parameter was selected in the UI. + - "auditbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C auditbeat check testsuite integration-tests-environment" + crosscompile: + make: "make -C auditbeat crosscompile" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: + disabled: true ## Disabled since there are test failures. See https://github.com/elastic/beats/issues/21958 diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go index f444aa6ea88..8eb2895a04b 100644 --- a/auditbeat/module/file_integrity/eventreader_test.go +++ b/auditbeat/module/file_integrity/eventreader_test.go @@ -166,6 +166,9 @@ func TestEventReader(t *testing.T) { }) mustRun(t, "deleted", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("See https://github.com/elastic/beats/issues/21958") + } if err = os.Remove(txt2); err != nil { t.Fatal(err) } @@ -310,6 +313,9 @@ func readTimeout(t testing.TB, events <-chan Event) Event { case <-time.After(time.Second): t.Fatalf("%+v", errors.Errorf("timed-out waiting for event")) case e, ok := <-events: + if runtime.GOOS == "windows" { + t.Skip("See https://github.com/elastic/beats/issues/21958") + } if !ok { t.Fatal("failed reading from event channel") } diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml new file mode 100644 index 00000000000..e68fca328f4 --- /dev/null +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -0,0 +1,15 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^deploy/kubernetes/.*" + comments: ## when PR comment contains any of those entries + - "/test deploy/kubernetes" + labels: ## when PR labels matches any of those entries + - "kubernetes" + parameters: ## when parameter was selected in the UI. + - "kubernetes" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + k8sTest: + k8sTest: "v1.10.0,v1.9.4,v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml new file mode 100644 index 00000000000..7cb1125e794 --- /dev/null +++ b/filebeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test filebeat" + labels: ## when PR labels matches any of those entries + - "filebeat" + parameters: ## when parameter was selected in the UI. + - "filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C filebeat check testsuite integration-tests-environment" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml new file mode 100644 index 00000000000..b253fcf8298 --- /dev/null +++ b/generator/Jenkinsfile.yml @@ -0,0 +1,19 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^generator/.*" + - "#generator/common/beatgen" ## special token regarding the project dependency + - "#metricbeat/beater" ## special token regarding the project dependency + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test generator" + labels: ## when PR labels matches any of those entries + - "generator" + parameters: ## when parameter was selected in the UI. + - "generator" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + generator-test: + make: "make -C generator/ test" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..c949832ad70 --- /dev/null +++ b/heartbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^heartbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test heartbeat" + labels: ## when PR labels matches any of those entries + - "heartbeat" + parameters: ## when parameter was selected in the UI. + - "heartbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C heartbeat check testsuite integration-tests-environment" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..7aaeacb8838 --- /dev/null +++ b/journalbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^journalbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test journalbeat" + labels: ## when PR labels matches any of those entries + - "journalbeat" + parameters: ## when parameter was selected in the UI. + - "journalbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C journalbeat check testsuite" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" \ No newline at end of file diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..e8972db2a61 --- /dev/null +++ b/libbeat/Jenkinsfile.yml @@ -0,0 +1,24 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test libbeat" + labels: ## when PR labels matches any of those entries + - "libbeat" + parameters: ## when parameter was selected in the UI. + - "libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C libbeat check testsuite integration-tests-environment" + when: + disabled: true ## Disabled since Integration Testing is broken. See https://github.com/elastic/beats/issues/21959 + crosscompile: + make: "make -C libbeat crosscompile" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 25e00ee4779..61eabe602aa 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -149,7 +149,7 @@ lint: $(REVIEWDOG) $(REVIEWDOG_OPTIONS) .PHONY: clean -clean:: ## @build Cleans up all files generated by the build steps +clean: ## @build Cleans up all files generated by the build steps @rm -rf build @rm -f docker-compose.yml.lock @rm -f ${BEAT_NAME} ${BEAT_NAME}.test ${BEAT_NAME}.exe ${BEAT_NAME}.test.exe diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..4dd42806937 --- /dev/null +++ b/metricbeat/Jenkinsfile.yml @@ -0,0 +1,27 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test metricbeat" + labels: ## when PR labels matches any of those entries + - "metricbeat" + parameters: ## when parameter was selected in the UI. + - "metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C metricbeat check testsuite integration-tests-environment" + when: + disabled: true ## Disabled since Integration Testing is broken. See https://github.com/elastic/beats/issues/21961 + crosscompile: + make: "make -C metricbeat crosscompile" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: + disabled: true ## Test failure. See https://github.com/elastic/beats/issues/22257 diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..f2b95fdf2ae --- /dev/null +++ b/packetbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^packetbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test packetbeat" + labels: ## when PR labels matches any of those entries + - "packetbeat" + parameters: ## when parameter was selected in the UI. + - "packetbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C packetbeat check testsuite" ## See https://github.com/elastic/beats/issues/22266 + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/script/fix_permissions.sh b/script/fix_permissions.sh new file mode 100755 index 00000000000..fd65a7916b5 --- /dev/null +++ b/script/fix_permissions.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set +e +readonly LOCATION="${1?Please define the path where the fix permissions should run from}" + +if ! docker version ; then + echo "It requires Docker daemon to be installed and running" +else + set -e + # Change ownership of all files inside the specific folder from root/root to current user/group + docker run -v ${LOCATION}:/beat alpine:3.4 sh -c "find /beat -user 0 -exec chown -h $(id -u):$(id -g) {} \;" +fi diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..180007625d2 --- /dev/null +++ b/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,23 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test winlogbeat" + labels: ## when PR labels matches any of those entries + - "winlogbeat" + parameters: ## when parameter was selected in the UI. + - "winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C winlogbeat check testsuite" + crosscompile: + make: "make -C winlogbeat crosscompile" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..fdbe990e5a3 --- /dev/null +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test auditbeat" + labels: ## when PR labels matches any of those entries + - "auditbeat" + parameters: ## when parameter was selected in the UI. + - "auditbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C auditbeat check testsuite integration-tests-environment" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml new file mode 100644 index 00000000000..4953ce4b319 --- /dev/null +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -0,0 +1,23 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/filebeat" + labels: ## when PR labels matches any of those entries + - "x-pack-filebeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C x-pack/filebeat check testsuite" + windows: + script: "dev-tools/jenkins_ci.ps1" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: + disabled: true ## https://github.com/elastic/beats/issues/21963 diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..2135855c5db --- /dev/null +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/functionbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/functionbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-functionbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-functionbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C x-pack/functionbeat check testsuite integration-tests-environment" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..8d3be647cab --- /dev/null +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/libbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/libbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-libbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C x-pack/libbeat check testsuite integration-tests-environment" + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..e480f2e03b3 --- /dev/null +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -0,0 +1,25 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/metricbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-metricbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C x-pack/metricbeat check testsuite integration-tests-environment" + when: + disabled: true ## Disabled see https://github.com/elastic/beats/issues/21962 + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: + disabled: true ## Disabled see https://github.com/elastic/beats/issues/22269 diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..4f959591cae --- /dev/null +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -0,0 +1,22 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/packetbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/packetbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-packetbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-packetbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-16" ## default label for all the stages +stages: + build: + make: "RACE_DETECTOR=1 make -C x-pack/packetbeat check testsuite integration-tests-environment" + windows: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..c69d76ed44b --- /dev/null +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "windows-2019" ## default label for all the stages +stages: + windows: + mage: "mage fields build goTestUnit" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: + disabled: true ## x-pack/winlogbeat\magefile.go does not exist. See https://github.com/elastic/beats/issues/22263