Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…
Browse files Browse the repository at this point in the history
…ne-2.0

* upstream/master: (33 commits)
  Stop running agent container as root by default (elastic#21213)
  Stop running auditbeat container as root by default (elastic#21202)
  Fix autodiscover flaky tests (elastic#21242)
  [Ingest Manager] Enabled dev builds (elastic#21241)
  Fix librpm installation in auditbeat build (elastic#21239)
  Fix prometheus default config (elastic#21253)
  Fix dev guide test command (elastic#21254)
  Move aws lambda metricset to GA (elastic#21255)
  [Docs] Typo in table syntax (elastic#20227)
  [ECS] Adds related.hosts to capture all hostnames and host identifiers on an event. (elastic#21160)
  Add recursive split to httpjson (elastic#21214)
  [DOCS] Add beat specific start widgets (elastic#21217)
  Fix timestamp handling in remote_write (elastic#21166)
  Fix aws, azure and googlecloud compute dashboards (elastic#21098)
  Add acceptable event log keys to winlog (elastic#21205)
  Add elastic-agent to gitignore (elastic#21219)
  Add cloudfoundry tags to events (elastic#21177)
  [Ingest Manager] Agent includes pgp file (elastic#19480)
  Add compatibility note about ingress-controller-v0.34.1 (elastic#21209)
  [Ingest Manager] Support for UPGRADE_ACTION (elastic#21002)
  ...
  • Loading branch information
v1v committed Sep 24, 2020
2 parents d636f2c + a9db1b1 commit 778c037
Show file tree
Hide file tree
Showing 280 changed files with 13,118 additions and 3,013 deletions.
106 changes: 106 additions & 0 deletions .ci/beats-tester.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#!/usr/bin/env groovy

@Library('apm@current') _

pipeline {
agent none
environment {
BASE_DIR = 'src/github.com/elastic/beats'
PIPELINE_LOG_LEVEL = "INFO"
BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
}
options {
timeout(time: 1, unit: 'HOURS')
buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
timestamps()
ansiColor('xterm')
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
disableConcurrentBuilds()
}
triggers {
issueCommentTrigger('(?i)^\\/beats-tester$')
upstream("Beats/packaging/${env.JOB_BASE_NAME}")
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
when {
beforeAgent true
anyOf {
triggeredBy cause: "IssueCommentCause"
expression {
def ret = isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
}
return ret
}
}
}
stages {
stage('Checkout') {
options { skipDefaultCheckout() }
steps {
deleteDir()
gitCheckout(basedir: "${BASE_DIR}")
setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim())
}
}
stage('Build master') {
options { skipDefaultCheckout() }
when { branch 'master' }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
stage('Build *.x branch') {
options { skipDefaultCheckout() }
when { branch '*.x' }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
stage('Build PullRequest') {
options { skipDefaultCheckout() }
when { changeRequest() }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT",
apm: "https://storage.googleapis.com/apm-ci-artifacts/jobs/pull-requests/pr-${env.CHANGE_ID}",
beats: "https://storage.googleapis.com/beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}")
}
}
stage('Build release branch') {
options { skipDefaultCheckout() }
when {
not {
allOf {
branch comparator: 'REGEXP', pattern: '(master|.*x)'
changeRequest()
}
}
}
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
}
}
}
}

def runBeatsTesterJob(Map args = [:]) {
if (args.apm && args.beats) {
build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
parameters: [
string(name: 'APM_URL_BASE', value: args.apm),
string(name: 'BEATS_URL_BASE', value: args.beats),
string(name: 'VERSION', value: args.version)
])
} else {
build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ])
}
}
56 changes: 56 additions & 0 deletions .ci/jobs/beats-tester.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
---
- job:
name: Beats/beats-tester
display-name: Beats Tester
description: Run the beats-tester
view: Beats
disabled: false
project-type: multibranch
script-path: .ci/beats-tester.groovy
scm:
- github:
branch-discovery: 'no-pr'
discover-pr-forks-strategy: 'merge-current'
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
discover-tags: true
head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
disable-pr-notifications: true
notification-context: 'beats-tester'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
- skip-initial-build: true
- tags:
ignore-tags-older-than: -1
ignore-tags-newer-than: 30
- named-branches:
- exact-name:
name: 'master'
case-sensitive: true
- regex-name:
regex: '7\.([x9]|1\d+)'
case-sensitive: true
- regex-name:
regex: '8\.\d+'
case-sensitive: true
- change-request:
ignore-target-only-changes: true
clean:
after: true
before: true
prune: true
shallow-clone: true
depth: 3
do-not-fetch-tags: true
submodule:
disable: false
recursive: true
parent-credentials: true
timeout: 100
timeout: '15'
use-author: true
wipe-workspace: true
14 changes: 14 additions & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Remove the deprecated `xpack.monitoring.*` settings. Going forward only `monitoring.*` settings may be used. {issue}9424[9424] {pull}18608[18608]
- Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
- Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]

*Auditbeat*

- File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
- Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685]
- Change network.direction values to ECS recommended values (inbound, outbound). {issue}12445[12445] {pull}20695[20695]
- Docker container needs to be explicitly run as user root for auditing. {pull}21202[21202]

*Filebeat*

Expand Down Expand Up @@ -171,6 +173,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811]
- [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]

*Auditbeat*

Expand Down Expand Up @@ -264,6 +267,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
- Provide backwards compatibility for the `append` processor when Elasticsearch is less than 7.10.0. {pull}21159[21159]

*Heartbeat*

Expand Down Expand Up @@ -345,6 +349,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Disable Kafka metricsets based on Jolokia by default. They require a different configuration. {pull}20989[20989]
- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095]
- Handle missing counters in the application_pool metricset. {pull}21071[21071]
- Fix timestamp handling in remote_write. {pull}21166[21166]
- Fix remote_write flaky test. {pull}21173[21173]
- Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]

*Packetbeat*

Expand All @@ -362,6 +369,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
- Fix catchall bucket config errors by adding more validation. {issue}17572[16282] {pull}20887[16287]

==== Added

Expand Down Expand Up @@ -428,6 +436,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
- Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
- Added experimental dataset `sophos/utm`. {pull}20820[20820]
- Add Cloud Foundry tags in related events. {pull}21177[21177]

*Auditbeat*

Expand Down Expand Up @@ -573,12 +582,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Convert httpjson to v2 input {pull}20226[20226]
- Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
- Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
- Added new properties field support for event.outcome in azure module {pull}20998[20998]
- Improve Zeek Kerberos module with `x509` ECS mappings {pull}20958[20958]
- Improve Fortinet firewall module with `x509` ECS mappings {pull}20983[20983]
- Improve Santa module with `x509` ECS mappings {pull}20976[20976]
- Improve Suricata Eve module with `x509` ECS mappings {pull}20973[20973]
- Added new module for Zoom webhooks {pull}20414[20414]
- Add type and sub_type to panw panos fileset {pull}20912[20912]
- Always attempt community_id processor on zeek module {pull}21155[21155]
- Add related.hosts ecs field to all modules {pull}21160[21160]

*Heartbeat*

Expand Down Expand Up @@ -694,6 +706,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649]
- Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
- Add host inventory metrics to system module. {pull}20415[20415]
- Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
- Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
- Request prometheus endpoints to be gzipped by default {pull}20766[20766]
Expand All @@ -702,6 +715,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
- Sanitize `event.host`. {pull}21022[21022]
- Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]

*Packetbeat*

Expand Down
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ It is possible to trigger some jobs by putting a comment on a GitHub PR.
* `/run apm-beats-update`
* [apm-beats-packaging][]
* `/package` or `/packaging` will kick of a build to generate the packages for beats.
* [apm-beats-tester][]
* `/beats-tester` will kick of a build to validate the generated packages.

### PR Labels

Expand All @@ -118,3 +120,4 @@ It's possible to configure the build on a GitHub PR by labelling the PR with the
[beats]: https://beats-ci.elastic.co/job/Beats/job/beats/
[apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/
[apm-beats-packaging]: https://beats-ci.elastic.co/job/Beats/job/packaging/
[apm-beats-tester]: https://beats-ci.elastic.co/job/Beats/job/beats-tester/
2 changes: 1 addition & 1 deletion auditbeat/docs/running-on-docker.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ It is also essential to run {beatname_uc} in the host PID namespace.

["source","sh",subs="attributes"]
----
docker run --cap-add=AUDIT_CONTROL,AUDIT_READ --pid=host {dockerimage}
docker run --cap-add=AUDIT_CONTROL --cap-add=AUDIT_READ --user=root --pid=host {dockerimage}
----
2 changes: 1 addition & 1 deletion auditbeat/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func Package() {

// TestPackages tests the generated packages (i.e. file modes, owners, groups).
func TestPackages() error {
return devtools.TestPackages(devtools.WithRootUserContainer())
return devtools.TestPackages()
}

// Update is an alias for running fields, dashboards, config, includes.
Expand Down
1 change: 0 additions & 1 deletion auditbeat/scripts/mage/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ func CustomizePackaging(pkgFlavor PackagingFlavor) {
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfig)
sampleRulesTarget = "/etc/{{.BeatName}}/" + defaultSampleRulesTarget
case devtools.Docker:
args.Spec.ExtraVar("user", "root")
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}
Expand Down
1 change: 1 addition & 0 deletions dev-tools/mage/crossbuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ func (b GolangCrossBuilder) Build() error {
"--env", "MAGEFILE_VERBOSE="+verbose,
"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
"--env", fmt.Sprintf("DEV=%v", DevBuild),
"-v", repoInfo.RootDir+":"+mountPoint,
"-w", workDir,
image,
Expand Down
7 changes: 7 additions & 0 deletions dev-tools/mage/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ var (
BeatProjectType ProjectType

Snapshot bool
DevBuild bool

versionQualified bool
versionQualifier string
Expand Down Expand Up @@ -117,6 +118,11 @@ func init() {
panic(errors.Wrap(err, "failed to parse SNAPSHOT env value"))
}

DevBuild, err = strconv.ParseBool(EnvOr("DEV", "false"))
if err != nil {
panic(errors.Wrap(err, "failed to parse DEV env value"))
}

versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
}

Expand Down Expand Up @@ -165,6 +171,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
"BeatURL": BeatURL,
"BeatUser": BeatUser,
"Snapshot": Snapshot,
"DEV": DevBuild,
"Qualifier": versionQualifier,
}

Expand Down
4 changes: 2 additions & 2 deletions dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ shared:
buildFrom: 'centos:7'
dockerfile: 'Dockerfile.elastic-agent.tmpl'
docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
user: 'root'
user: '{{ .BeatName }}'
linux_capabilities: ''
files:
'elastic-agent.yml':
Expand Down Expand Up @@ -509,7 +509,7 @@ shared:
- &docker_ubi_spec
extra_vars:
image_name: '{{.BeatName}}-ubi8'
from: 'registry.access.redhat.com/ubi8/ubi-minimal'
from: 'docker.elastic.co/ubi8/ubi-minimal'

- &elastic_docker_spec
extra_vars:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
chown -R root:root {{ $beatHome }} && \
find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
find {{ $beatHome }}/data -type d -exec chmod 0770 {} \; && \
find {{ $beatHome }}/data -type f -exec chmod 0660 {} \; && \
rm {{ $beatBinary }} && \
ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
chmod 0750 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
Expand All @@ -21,7 +23,7 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
{{- range $i, $modulesd := .ModulesDirs }}
chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
{{- end }}
chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs
true

FROM {{ .from }}

Expand Down Expand Up @@ -69,6 +71,10 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint

COPY --from=home {{ $beatHome }} {{ $beatHome }}

# Elastic Agent needs group permissions in the home itself to be able to
# create fleet.yml when running as non-root.
RUN chmod 0770 {{ $beatHome }}

RUN mkdir /licenses
COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
Expand Down
2 changes: 1 addition & 1 deletion docs/devguide/modules-dev-guide.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,6 @@ locally for a specific module, using the following procedure under Filebeat dire

. Run an Elasticsearch instance locally using docker: `docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT`
. Create python env: `make python-env`
. Source python env: `./build/python-env/bin/activate`
. Source python env: `source ./build/python-env/bin/activate`
. Create the testing binary: `make filebeat.test`
. Run the test, ie: `GENERATE=1 INTEGRATION_TESTS=1 BEAT_STRICT_PERMS=false TESTING_FILEBEAT_MODULES=nginx pytest tests/system/test_modules.py`
6 changes: 3 additions & 3 deletions filebeat/docs/filebeat-general-options.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ filebeat.registry.path: registry
-------------------------------------------------------------------------------------

NOTE: The registry is only updated when new events are flushed and not on a predefined period.
That means in case there are some states where the TTL expired, these are only removed when new event are processed.
That means in case there are some states where the TTL expired, these are only removed when new events are processed.

NOTE: The registry stores it's data in the subdirectory filebeat/data.json. It
NOTE: The registry stores its data in the subdirectory filebeat/data.json. It
also contains a meta data file named filebeat/meta.json. The meta file contains
the file format version number.

NOTE: The content stored in filebeat/data.json is compatible to the old registry file data format.
NOTE: The content stored in filebeat/data.json is compatible with the old registry file data format.

[float]
==== `registry.file_permissions`
Expand Down
Loading

0 comments on commit 778c037

Please sign in to comment.