Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/windows-7-64
Browse files Browse the repository at this point in the history
* upstream/master: (364 commits)
  Add vendoring to Google Cloud Functions again (elastic#21070)
  [Elastic Agent] Add fleet.host.id for sending to endpoint. (elastic#21042)
  Do not need Google credentials before using it (elastic#21072)
  [Filebeat][New Module] Zoom webhook module (elastic#20414)
  Add support for GMT timezone offset in decode_cef (elastic#20993)
  Filebeat: Fix random error on harvester close (elastic#21048)
  Add ingress controller dashboards (elastic#21052)
  Fix loggers in composable module. (elastic#21047)
  [Ingest Manager] Increase kibana client timeout to 5 minutes (elastic#21037)
  Add changelog. (elastic#21041)
  [Elastic Agent] Add support for EQL based conditions (elastic#20994)
  Disable Kafka metricsets based on Jolokia (elastic#20989)
  Update apm agent (elastic#21031)
  Add container ECS fields in kubernetes metadata (elastic#20984)
  Sanitize event.host in Metricbeat (elastic#21022)
  Update api-keys.asciidoc - API key prerequisites (elastic#21026)
  [Filebeat][suricata] Map x509 for suricata/eve fileset (elastic#20973)
  [Filebeat][santa] Map x509 fields in santa module (elastic#20976)
  [Filebeat][fortinet] Map x509 ecs fields for fortinet fw fileset (elastic#20983)
  Bump zeek kerberos/ssl/x509 ecs version (elastic#21003)
  ...
  • Loading branch information
v1v committed Sep 14, 2020
2 parents 98f985b + 50ea11a commit fcc41b4
Show file tree
Hide file tree
Showing 1,892 changed files with 397,209 additions and 38,869 deletions.
2 changes: 1 addition & 1 deletion .backportrc.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"upstream": "elastic/beats",
"branches": [{ "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
"branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
"labels": ["backport"],
"autoAssign": true,
"prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"
Expand Down
108 changes: 57 additions & 51 deletions .ci/apm-beats-update.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
@Library('apm@current') _

pipeline {
agent none
agent { label 'master' }
environment {
REPO = 'apm-server'
BASE_DIR = "src/github.com/elastic/${env.REPO}"
Expand All @@ -27,69 +27,75 @@ pipeline {
}
triggers {
issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-18 && immutable' }
when {
beforeAgent true
expression {
def ret = isCommentTrigger() || isUserTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment")
anyOf {
triggeredBy cause: "IssueCommentCause"
expression {
def ret = isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
}
return ret
}
return ret
}
}
/**
Checkout the code and stash it, to use it on other stages.
*/
stage('Checkout') {
steps {
deleteDir()
gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
script {
dir("${BEATS_DIR}"){
env.GO_VERSION = readFile(".go-version").trim()
def regexps =[
"^devtools/mage.*",
"^libbeat/scripts/Makefile",
]
env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
// Skip all the stages except docs for PR's with asciidoc changes only
env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
stages {
/**
Checkout the code and stash it, to use it on other stages.
*/
stage('Checkout') {
options { skipDefaultCheckout() }
steps {
deleteDir()
gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
script {
dir("${BEATS_DIR}"){
env.GO_VERSION = readFile(".go-version").trim()
def regexps =[
"^devtools/mage.*",
"^libbeat/scripts/Makefile",
]
env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
// Skip all the stages except docs for PR's with asciidoc changes only
env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
}
}
}
}
}
/**
updates beats updates the framework part and go parts of beats.
Then build and test.
Finally archive the results.
*/
stage('Update Beats') {
options { skipDefaultCheckout() }
when {
beforeAgent true
anyOf {
branch 'master'
branch "\\d+\\.\\d+"
branch "v\\d?"
tag "v\\d+\\.\\d+\\.\\d+*"
allOf {
expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
changeRequest()
}
/**
updates beats updates the framework part and go parts of beats.
Then build and test.
Finally archive the results.
*/
stage('Update Beats') {
options { skipDefaultCheckout() }
when {
beforeAgent true
anyOf {
branch 'master'
branch "\\d+\\.\\d+"
branch "v\\d?"
tag "v\\d+\\.\\d+\\.\\d+*"
allOf {
expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
changeRequest()
}

}
}
}
steps {
withGithubNotify(context: 'Check Apm Server Beats Update') {
beatsUpdate()
steps {
withGithubNotify(context: 'Check Apm Server Beats Update') {
beatsUpdate()
}
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions .ci/jobs/apm-beats-update.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
discover-tags: true
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
disable-pr-notifications: true
notification-context: 'apm-beats-update'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
59 changes: 59 additions & 0 deletions .ci/jobs/beats-mbp-2.0.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
---
- job:
name: Beats/beats-mbp-2.0
display-name: 'Beats (2.0)'
description: 'Beats Main Pipeline 2.0'
view: Beats
concurrent: true
project-type: multibranch
prune-dead-branches: true
days-to-keep: 30
script-path: '.ci/Jenkinsfile'
triggers: []
wrappers: []
scm:
- github:
branch-discovery: 'no-pr'
discover-pr-forks-strategy: 'merge-current'
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
discover-tags: true
disable-pr-notifications: true
notification-context: "beats-ci-2.0"
repo: 'beats'
repo-owner: 'elastic'
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
- tags:
ignore-tags-older-than: -1
ignore-tags-newer-than: 365
- change-request:
ignore-target-only-changes: true
- named-branches:
- exact-name:
name: 'master'
case-sensitive: true
- regex-name:
regex: '7\.[x789]'
case-sensitive: true
- regex-name:
regex: '8\.\d+'
case-sensitive: true
clean:
after: true
before: true
prune: true
shallow-clone: true
depth: 3
do-not-fetch-tags: true
submodule:
disable: false
recursive: true
parent-credentials: true
timeout: 100
timeout: '15'
use-author: true
wipe-workspace: true
4 changes: 2 additions & 2 deletions .ci/jobs/beats-windows-mbp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
discover-tags: false
# Run MBP for the master branch and PRs
head-filter-regex: '(master|PR-.*)'
notification-context: 'beats-ci'
notification-context: 'beats-ci/windows'
repo: beats
repo-owner: elastic
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
12 changes: 4 additions & 8 deletions .ci/jobs/beats.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
---
- job:
name: Beats/beats
display-name: 'Beats (replacement)'
display-name: 'Beats'
description: 'Beats Main Pipeline'
view: Beats
concurrent: true
project-type: multibranch
prune-dead-branches: true
number-to-keep: 10
days-to-keep: 30
script-path: 'Jenkinsfile'
triggers: []
Expand All @@ -18,21 +17,18 @@
discover-pr-forks-strategy: 'merge-current'
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
discover-tags: true
notification-context: "beats-ci"
property-strategies:
all-branches:
- suppress-scm-triggering: true
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
- tags:
ignore-tags-older-than: -1
ignore-tags-newer-than: -1
ignore-tags-newer-than: 365
- change-request:
ignore-target-only-changes: true
- named-branches:
Expand Down
4 changes: 2 additions & 2 deletions .ci/jobs/packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
discover-tags: true
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
disable-pr-notifications: true
notification-context: 'beats-packaging'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
53 changes: 40 additions & 13 deletions .ci/packaging.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pipeline {
triggers {
issueCommentTrigger('(?i)^\\/packag[ing|e]$')
// disable upstream trigger on a PR basis
upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
}
parameters {
booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
Expand All @@ -37,15 +37,18 @@ pipeline {
agent { label 'ubuntu && immutable' }
when {
beforeAgent true
expression {
def ret = isCommentTrigger() || isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
anyOf {
triggeredBy cause: "IssueCommentCause"
expression {
def ret = isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
}
return ret
}
return ret
}
}
stages {
Expand Down Expand Up @@ -76,7 +79,7 @@ pipeline {
'x-pack/dockerlogbeat',
'x-pack/filebeat',
'x-pack/functionbeat',
// 'x-pack/heartbeat',
'x-pack/heartbeat',
// 'x-pack/journalbeat',
'x-pack/metricbeat',
// 'x-pack/packetbeat',
Expand Down Expand Up @@ -158,7 +161,6 @@ def pushCIDockerImages(){
} else if ("${env.BEATS_FOLDER}" == "filebeat") {
tagAndPush('filebeat-oss')
} else if ("${env.BEATS_FOLDER}" == "heartbeat"){
tagAndPush('heartbeat')
tagAndPush('heartbeat-oss')
} else if ("${env.BEATS_FOLDER}" == "journalbeat"){
tagAndPush('journalbeat')
Expand All @@ -174,6 +176,8 @@ def pushCIDockerImages(){
tagAndPush('elastic-agent')
} else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
tagAndPush('filebeat')
} else if ("${env.BEATS_FOLDER}" == "x-pack/heartbeat"){
tagAndPush('heartbeat')
} else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
tagAndPush('metricbeat')
}
Expand All @@ -185,8 +189,14 @@ def tagAndPush(name){
if("${env.SNAPSHOT}" == "true"){
libbetaVer += "-SNAPSHOT"
}

def tagName = "${libbetaVer}"
if (isPR()) {
tagName = "pr-${env.CHANGE_ID}"
}

def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}"
def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
retry(3){
Expand Down Expand Up @@ -219,7 +229,12 @@ def withMacOSEnv(Closure body){
}

def publishPackages(baseDir){
googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
def bucketUri = "gs://${JOB_GCS_BUCKET}/snapshots"
if (isPR()) {
bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
}
def beatsFolderName = getBeatsName(baseDir)
googleStorageUpload(bucket: "${bucketUri}/${beatsFolderName}",
credentialsId: "${JOB_GCS_CREDENTIALS}",
pathPrefix: "${baseDir}/build/distributions/",
pattern: "${baseDir}/build/distributions/**/*",
Expand All @@ -228,6 +243,18 @@ def publishPackages(baseDir){
)
}

/**
* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
* therefore the storage bucket in GCP should follow the same folder structure.
* This is required by https://github.com/elastic/beats-tester
* e.g.
* baseDir=name -> return name
* baseDir=name1/name2/name3-> return name2
*/
def getBeatsName(baseDir) {
return baseDir.replace('x-pack/', '')
}

def withBeatsEnv(Closure body) {
withMageEnv(){
withEnv([
Expand Down
Loading

0 comments on commit fcc41b4

Please sign in to comment.