diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml new file mode 100644 index 00000000000..e66b6be82e1 --- /dev/null +++ b/.ci/jobs/golang-crossbuild-mbp.yml @@ -0,0 +1,43 @@ +--- +- job: + name: Beats/golang-crossbuild-mbp + display-name: Pipeline for golang-crossbuild + description: Jenkins pipeline for the golang-crossbuild project. + view: Beats + project-type: multibranch + script-path: Jenkinsfile + scm: + - github: + branch-discovery: no-pr + discover-pr-forks-strategy: merge-current + discover-pr-forks-trust: permission + discover-pr-origin: merge-current + discover-tags: true + notification-context: 'beats-ci' + repo: golang-crossbuild + repo-owner: elastic + credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken + ssh-checkout: + credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba + build-strategies: + - tags: + ignore-tags-older-than: -1 + ignore-tags-newer-than: -1 + - regular-branches: true + - change-request: + ignore-target-only-changes: false + clean: + after: true + before: true + prune: true + shallow-clone: true + depth: 4 + do-not-fetch-tags: true + submodule: + disable: false + recursive: true + parent-credentials: true + timeout: 100 + timeout: '15' + use-author: true + wipe-workspace: 'True' diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat index 54bcaf5e8a4..d5d48b1dcd0 100644 --- a/.ci/scripts/install-tools.bat +++ b/.ci/scripts/install-tools.bat @@ -20,7 +20,7 @@ where mage if not exist C:\Python38\python.exe ( REM Install python 3.8. - choco install python -y -r --no-progress --version 3.8.2 + choco install python -y -r --no-progress --version 3.8.2 || echo ERROR && exit /b ) python --version where python diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index c4bcc67e780..cf93fe5a690 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -46,6 +46,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only. Your magefile.go will require a change to adapt the devtool API. See the pull request for more details. {pull}18148[18148] - The Elasticsearch client settings expect the API key to be raw (not base64-encoded). {issue}18939[18939] {pull}18945[18945] +- `management.ConfigManager` has been renamed to `management.Manager`. {pull}19114[19114] +- `UpdateStatus` has been added to the `management.Manager` interface. {pull}19114[19114] ==== Bugfixes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7f1df7c0d85..93da0220287 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -39,10 +39,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d * CEF {pull}18223[18223] * PANW {pull}18223[18223] * Cisco {pull}18753[18753] +* CrowdStrike {pull}19132[19132] * iptables {pull}18756[18756] * Checkpoint {pull}18754[18754] * Netflow {pull}19087[19087] +* Zeek {pull}19113[19113] (`forwarded` tag is not included by default) * Suricata {pull}19107[19107] (`forwarded` tag is not included by default) +* CoreDNS {pull}19134[19134] (`forwarded` tag is not included by default) +* Envoy Proxy {pull}19134[19134] (`forwarded` tag is not included by default) - Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359] - Adds check on `` config option value for the azure input `resource_manager_endpoint`. {pull}18890[18890] - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] @@ -241,6 +245,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add missing network.sent_packets_count metric into compute metricset in googlecloud module. {pull}18802[18802] - Fix compute and pubsub dashboard for googlecloud module. {issue}18962[18962] {pull}18980[18980] - Fix crash on vsphere module when Host information is not available. {issue}18996[18996] {pull}19078[19078] +- Fix incorrect usage of hints builder when exposed port is a substring of the hint {pull}19052[19052] *Packetbeat* diff --git a/Jenkinsfile b/Jenkinsfile index aafa025b9f6..69ceb5cb387 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -704,7 +704,7 @@ pipeline { stage('Generators Metricbeat Linux'){ steps { // FIXME see https://github.com/elastic/beats/issues/18132 - catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { + catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') { makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test") makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test-package") } @@ -713,7 +713,7 @@ pipeline { stage('Generators Beat Linux'){ steps { // FIXME see https://github.com/elastic/beats/issues/18132 - catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { + catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') { makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test") makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test-package") } @@ -730,7 +730,7 @@ pipeline { } steps { // FIXME see https://github.com/elastic/beats/issues/18132 - catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { + catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') { makeTarget(context: "Generators Metricbeat Mac OS X", target: "-C generator/_templates/metricbeat test") } } @@ -751,7 +751,7 @@ pipeline { } steps { // FIXME see https://github.com/elastic/beats/issues/18132 - catchError(buildResult: 'SUCCESS', message: 'Ignore error temporally', stageResult: 'UNSTABLE') { + catchError(buildResult: 'SUCCESS', message: 'Ignore error temporarily', stageResult: 'UNSTABLE') { makeTarget(context: "Generators Beat Mac OS X", target: "-C generator/_templates/beat test") } } diff --git a/NOTICE.txt b/NOTICE.txt index 811df0e8f3c..baeaa60356f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -7843,8 +7843,7 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: go.elastic.co/ecszap -Version: v0.1.1 -Revision: cdd95a104193 +Version: v0.2.0 License type (autodetected): Apache-2.0 ./vendor/go.elastic.co/ecszap/LICENSE: -------------------------------------------------------------------- diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl index 52b0870d719..a38ea8701a3 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl @@ -5,10 +5,8 @@ FROM {{ .from }} # Installing jq needs to be installed after epel-release and cannot be in the same yum install command. -RUN yum -y --setopt=tsflags=nodocs update && \ - yum install epel-release -y && \ - yum install jq -y && \ - yum clean all +RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && yum install --setopt=tsflags=nodocs -y epel-release && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) +RUN for iter in {1..10}; do yum update -y && yum install -y jq && yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; (exit $exit_code) LABEL \ org.label-schema.build-date="{{ date }}" \ diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 91bfc27b665..10448b5a93f 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -3,7 +3,6 @@ set -eo pipefail # Environment variables used -# FLEET_CONFIG_ID - config related to new token [defaul] # FLEET_ENROLLMENT_TOKEN - existing enrollment token to be used for enroll # FLEET_ENROLL - if set to 1 enroll will be performed # FLEET_SETUP - if set to 1 fleet setup will be performed @@ -49,8 +48,9 @@ function enroll(){ if [ $exitCode -ne 0 ]; then exit $exitCode fi + + apikey=$(echo $enrollResp | jq -r '.item.api_key') fi - apikey=$(echo $enrollResp | jq -r '.item.api_key') echo $apikey ./{{ .BeatName }} enroll ${KIBANA_HOST:-http://localhost:5601} $apikey -f diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index fb94a26762a..7de7b5d541f 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -114,7 +114,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) { haveEnabledInputs = true } - if !config.ConfigInput.Enabled() && !config.ConfigModules.Enabled() && !haveEnabledInputs && config.Autodiscover == nil && !b.ConfigManager.Enabled() { + if !config.ConfigInput.Enabled() && !config.ConfigModules.Enabled() && !haveEnabledInputs && config.Autodiscover == nil && !b.Manager.Enabled() { if !b.InSetupCmd { return nil, errors.New("no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?") } diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index 1ad682fc496..817fea40b6d 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -86,7 +86,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { return err } - if b.ConfigManager.Enabled() { + if b.Manager.Enabled() { bt.RunCentralMgmtMonitors(b) } diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 75585ba8992..f5da2db9d0e 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -66,7 +66,7 @@ type Beat struct { Fields []byte // Data from fields.yml - ConfigManager management.ConfigManager // config manager + Manager management.Manager // manager Keystore keystore.Keystore } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index d962207463a..f320b81c42c 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -330,12 +330,12 @@ func (b *Beat) createBeater(bt beat.Creator) (beat.Beater, error) { // Report central management state mgmt := monitoring.GetNamespace("state").GetRegistry().NewRegistry("management") - monitoring.NewBool(mgmt, "enabled").Set(b.ConfigManager.Enabled()) + monitoring.NewBool(mgmt, "enabled").Set(b.Manager.Enabled()) debugf("Initializing output plugins") outputEnabled := b.Config.Output.IsSet() && b.Config.Output.Config().Enabled() if !outputEnabled { - if b.ConfigManager.Enabled() { + if b.Manager.Enabled() { logp.Info("Output is configured through Central Management") } else { msg := "No outputs are defined. Please define one under the output section." @@ -462,8 +462,8 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { logp.Info("%s start running.", b.Info.Beat) // Launch config manager - b.ConfigManager.Start(beater.Stop) - defer b.ConfigManager.Stop() + b.Manager.Start(beater.Stop) + defer b.Manager.Stop() return beater.Run(&b.Beat) } @@ -643,12 +643,12 @@ func (b *Beat) configure(settings Settings) error { logp.Info("Beat ID: %v", b.Info.ID) // initialize config manager - b.ConfigManager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID) + b.Manager, err = management.Factory(b.Config.Management)(b.Config.Management, reload.Register, b.Beat.Info.ID) if err != nil { return err } - if err := b.ConfigManager.CheckRawConfig(b.RawConfig); err != nil { + if err := b.Manager.CheckRawConfig(b.RawConfig); err != nil { return err } diff --git a/libbeat/management/management.go b/libbeat/management/management.go index 690c3dba7f7..52e03fb6943 100644 --- a/libbeat/management/management.go +++ b/libbeat/management/management.go @@ -18,11 +18,36 @@ package management import ( + "sync" + "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Status describes the current status of the beat. +type Status int + +const ( + // Unknown is initial status when none has been reported. + Unknown Status = iota + // Starting is status describing application is starting. + Starting + // Configuring is status describing application is configuring. + Configuring + // Running is status describing application is running. + Running + // Degraded is status describing application is degraded. + Degraded + // Failed is status describing application is failed. This status should + // only be used in the case the beat should stop running as the failure + // cannot be recovered. + Failed + // Stopping is status describing application is stopping. + Stopping ) // Namespace is the feature namespace for queue definition. @@ -33,19 +58,28 @@ var DebugK = "centralmgmt" var centralMgmtKey = "x-pack-cm" -// ConfigManager interacts with the beat to update configurations -// from an external source -type ConfigManager interface { - // Enabled returns true if config manager is enabled +// StatusReporter provides a method to update current status of the beat. +type StatusReporter interface { + // UpdateStatus called when the status of the beat has changed. + UpdateStatus(status Status, msg string) +} + +// Manager interacts with the beat to provide status updates and to receive +// configurations. +type Manager interface { + StatusReporter + + // Enabled returns true if manager is enabled. Enabled() bool - // Start the config manager - Start(func()) + // Start the config manager giving it a stopFunc callback + // so the beat can be told when to stop. + Start(stopFunc func()) - // Stop the config manager + // Stop the config manager. Stop() - // CheckRawConfig check settings are correct before launching the beat + // CheckRawConfig check settings are correct before launching the beat. CheckRawConfig(cfg *common.Config) error } @@ -53,7 +87,7 @@ type ConfigManager interface { type PluginFunc func(*common.Config) FactoryFunc // FactoryFunc for creating a config manager -type FactoryFunc func(*common.Config, *reload.Registry, uuid.UUID) (ConfigManager, error) +type FactoryFunc func(*common.Config, *reload.Registry, uuid.UUID) (Manager, error) // Register a config manager func Register(name string, fn PluginFunc, stability feature.Stability) { @@ -91,13 +125,32 @@ func defaultModeConfig() *modeConfig { } // nilManager, fallback when no manager is present -type nilManager struct{} +type nilManager struct { + logger *logp.Logger + lock sync.Mutex + status Status + msg string +} -func nilFactory(*common.Config, *reload.Registry, uuid.UUID) (ConfigManager, error) { - return nilManager{}, nil +func nilFactory(*common.Config, *reload.Registry, uuid.UUID) (Manager, error) { + log := logp.NewLogger("mgmt") + return &nilManager{ + logger: log, + status: Unknown, + msg: "", + }, nil } -func (nilManager) Enabled() bool { return false } -func (nilManager) Start(_ func()) {} -func (nilManager) Stop() {} -func (nilManager) CheckRawConfig(cfg *common.Config) error { return nil } +func (*nilManager) Enabled() bool { return false } +func (*nilManager) Start(_ func()) {} +func (*nilManager) Stop() {} +func (*nilManager) CheckRawConfig(cfg *common.Config) error { return nil } +func (n *nilManager) UpdateStatus(status Status, msg string) { + n.lock.Lock() + defer n.lock.Unlock() + if n.status != status || n.msg != msg { + n.status = status + n.msg = msg + n.logger.Infof("Status change to %s: %s", status, msg) + } +} diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index b366789ba27..c90b4e55419 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -19,6 +19,7 @@ package hints import ( "fmt" + "strconv" "strings" "github.com/elastic/go-ucfg" @@ -187,7 +188,7 @@ func (m *metricHints) getHostsWithPort(hints common.MapStr, port int) ([]string, // Only pick hosts that have ${data.port} or the port on current event. This will make // sure that incorrect meta mapping doesn't happen for _, h := range thosts { - if strings.Contains(h, "data.port") || strings.Contains(h, fmt.Sprintf(":%d", port)) || + if strings.Contains(h, "data.port") || m.checkHostPort(h, port) || // Use the event that has no port config if there is a ${data.host}:9090 like input (port == 0 && strings.Contains(h, "data.host")) { result = append(result, h) @@ -202,6 +203,27 @@ func (m *metricHints) getHostsWithPort(hints common.MapStr, port int) ([]string, return result, true } +func (m *metricHints) checkHostPort(h string, p int) bool { + port := strconv.Itoa(p) + + index := strings.LastIndex(h, ":"+port) + // Check if host contains :port. If not then return false + if index == -1 { + return false + } + + // Check if the host ends with :port. Return true if yes + end := index + len(port) + 1 + if end == len(h) { + return true + } + + // Check if the character immediately after :port. If its not a number then return true. + // This is to avoid adding :80 as a valid host for an event that has port=8080 + // Also ensure that port=3306 and hint="tcp(${data.host}:3306)/" is valid + return h[end] < '0' || h[end] > '9' +} + func (m *metricHints) getNamespace(hints common.MapStr) string { return builder.GetHintString(hints, m.Key, namespace) } diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index a6dddb6b7a1..f7159323640 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -290,6 +290,45 @@ func TestGenerateHints(t *testing.T) { "enabled": true, }, }, + { + message: "Module, namespace, host hint shouldn't return when port isn't the same has hint", + event: bus.Event{ + "host": "1.2.3.4", + "port": 80, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "${data.host}:8080", + }, + }, + }, + len: 0, + }, + { + message: "Non http URLs with valid host port combination should return a valid config", + event: bus.Event{ + "host": "1.2.3.4", + "port": 3306, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "tcp(${data.host}:3306)/", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "hosts": []interface{}{"tcp(1.2.3.4:3306)/"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + }, + }, } for _, test := range tests { mockRegister := mb.NewRegister() diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index fbf9d23110f..bdd45ac3693 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -134,7 +134,7 @@ func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbe return nil, errors.Wrap(err, "error reading configuration file") } - dynamicCfgEnabled := config.ConfigModules.Enabled() || config.Autodiscover != nil || b.ConfigManager.Enabled() + dynamicCfgEnabled := config.ConfigModules.Enabled() || config.Autodiscover != nil || b.Manager.Enabled() if !dynamicCfgEnabled && len(config.Modules) == 0 { return nil, mb.ErrEmptyConfig } diff --git a/vendor/modules.txt b/vendor/modules.txt index f0695bcffcf..86eee7cb174 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -861,7 +861,7 @@ go.elastic.co/apm/transport/transporttest go.elastic.co/apm/module/apmelasticsearch # go.elastic.co/apm/module/apmhttp v1.7.2 go.elastic.co/apm/module/apmhttp -# go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 +# go.elastic.co/ecszap v0.2.0 go.elastic.co/ecszap go.elastic.co/ecszap/internal # go.elastic.co/fastjson v1.0.0 diff --git a/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc new file mode 100644 index 00000000000..2068e94f342 --- /dev/null +++ b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc @@ -0,0 +1,21 @@ +[[elastic-agent-cmd-options]] += Command line options + +experimental[] + +The `elastic-agent run` command provides flags that alter the behavior of an +agent: + +`-path.home`:: +The home directory of the {agent}. `path.home` determines the location of the +configuration files and data directory. + +`-c`:: +The configuration file to load. If not specified, {agent} uses +`{path.home}/elastic-agent.yml`. + +`-path.data`:: +The data directory used by {agent} to store downloaded artifacts. Also stores +logs for any {beats} started and managed by {agent}. ++ +If not specified, {agent} uses `{path.home}/data`. diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc new file mode 100644 index 00000000000..2300f7f9621 --- /dev/null +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc @@ -0,0 +1,11 @@ +[[elastic-agent-configuration-example]] += Configuration example + +experimental[] + +The following example shows a full list of configuration options: + +[source,yaml] +---- +include::elastic-agent_configuration_example.yml[] +---- diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc new file mode 100644 index 00000000000..fc34d15503c --- /dev/null +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc @@ -0,0 +1,111 @@ +[[elastic-agent-configuration]] += Configuration settings + +experimental[] + +By default {agent} runs in standalone mode to ingest system data and send it to +a local {es} instance running on port 9200. It uses the demo credentials of the +`elastic` user. It's also configured to monitor all {beats} managed by the agent +and send the {beats} logs and metrics to the same {es) instance. + +To alter this behavior, configure the output and other configuration settings: + +* <> +* <> +* <> + +[float] +[[elastic-agent-output-configuration]] +== Output settings + +Specify one or more outputs. Specifying multiple outputs allows you to pair +each data source with a different output. + +IMPORTANT: {agent} currently works with the {es} output only. + +Example output configuration: + +[source,yaml] +------------------------------------------------------------------------------------- +outputs: + default: + type: elasticsearch + hosts: [127.0.0.1:9200] + username: elastic + password: changeme + + monitoring: + type: elasticsearch + api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + hosts: ["localhost:9200"] + ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" +------------------------------------------------------------------------------------- + +This example configures two outputs: `default` and `monitoring`. +Notice that they use different authentication methods. The first one uses a +username and password pair, and the second one contains an API key. + +[NOTE] +============== +A default output configuration is required. +============== + +[float] +[[elastic-agent-monitoring-configuration]] +== {beats} monitoring settings + +{agent} monitors {beats} by default. To disable or change monitoring +settings, set options under `settings.monitoring`: + +[source,yaml] +------------------------------------------------------------------------------------- +settings.monitoring: + # enabled turns on monitoring of running processes + enabled: true + # enables log monitoring + logs: true + # enables metrics monitoring + metrics: true + # specifies output to be used + use_output: monitoring +------------------------------------------------------------------------------------- + + +To disable monitoring, set `settings.monitoring.enabled` to `false`. When set to +`false`, {beats} monitoring is turned off, and all other options in this section +are ignored. + +To enable monitoring, set `settings.monitoring.enabled` to `true`. Also set the +`logs` and `metrics` settings to control whether logs, metrics, or both are +collected. If neither setting is specified, monitoring is disabled. Set +`use_output` to specify the output to which monitoring events are sent. + +[[elastic-agent-datasource-configuration]] +== Datasource settings + +By default {agent} collects system metrics, such as cpu, memory, network, and +filesystem metrics, and sends them to the default output. For example: + + +[source,yaml] +------------------------------------------------------------------------------------- +datasources: + - namespace: default + use_output: default + inputs: + - type: system/metrics + streams: + - metricset: cpu + dataset: system.cpu + - metricset: memory + dataset: system.memory + - metricset: network + dataset: system.network + - metricset: filesystem + dataset: system.filesystem +------------------------------------------------------------------------------------- + +If `use_output` is not specified, the `default` output is used. + +//For more examples, see +//<>. diff --git a/x-pack/elastic-agent/docs/elastic-agent.asciidoc b/x-pack/elastic-agent/docs/elastic-agent.asciidoc index f2a092c7d47..92cbfdd7e42 100644 --- a/x-pack/elastic-agent/docs/elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent.asciidoc @@ -1,242 +1,25 @@ [[elastic-agent-installation-configuration]] -== Get started with {beatname_uc} += Manage your {agent}s -++++ -Get started -++++ +experimental[] -Elastic Agent is a single, unified agent that you can deploy to hosts or containers to collect data and send it to the {stack}. Behind the scenes, Elastic Agent runs the {beats} shippers or Endpoint required for your configuration. +{agent} is a single, unified agent that you can deploy to hosts or containers to +collect data and send it to the {stack}. Behind the scenes, {agent} runs the +{beats} shippers or Endpoint required for your configuration. + +To learn how to install, configure, and run your {agent}s, see: * <> -* <> +* <> * <> * <> -[[elastic-agent-installation]] -== Install Elastic Agent - -=== Step 1: Unpack archive - - -[[mac]] -*mac:* - -ifeval::["{release-state}"=="unreleased"] - -Version {version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -["source","sh",subs="attributes,callouts"] ------------------------------------------------- -curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{version}-darwin-x86_64.tar.gz -tar xzvf elastic-agent-{version}-darwin-x86_64.tar.gz ------------------------------------------------- - -endif::[] - -[[linux]] -*linux:* - -ifeval::["{release-state}"=="unreleased"] - -Version {version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -["source","sh",subs="attributes,callouts"] ------------------------------------------------- -curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{version}-linux-x86_64.tar.gz -tar xzvf elastic-agent-{version}-linux-x86_64.tar.gz ------------------------------------------------- - -endif::[] - -[[win]] -*win:* - -ifeval::["{release-state}"=="unreleased"] - -Version {version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -. Download the Elastic Agent Windows zip file from the -https://www.elastic.co/downloads/beats/elastic-agent[downloads page]. - -. Extract the contents of the zip file into `C:\Program Files`. - -. Rename the `elastic-agent--windows` directory to `Elastic-Agent`. - -. Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). - -. From the PowerShell prompt, run the following commands to install Filebeat as a -Windows service: -+ -[source,shell] ----------------------------------------------------------------------- -PS > cd 'C:\Program Files\Elastic-Agent' -PS C:\Program Files\Elastic-Agent> .\install-service-elastic-agent.ps1 ----------------------------------------------------------------------- - -NOTE: If script execution is disabled on your system, you need to set the execution policy for the current session to allow the script to run. For example: `PowerShell.exe -ExecutionPolicy UnRestricted -File .\install-service-elastic-agent.ps1`. - -endif::[] - -=== Step 2: Run Elastic Agent - -If Elastic Agent is not installed as an auto-starting service, start it manually: - - -[source,shell] ----------------------------------------------------------------------- -$ ./elastic-agent run ----------------------------------------------------------------------- - -[[elastic-agent-execution-modes]] -== Execution modes - -Elastic Agent runs in two modes: standalone or fleet. The two modes differ in how you configure and manage the agent. -[float] -=== Standalone mode - -With _standalone mode_, you manually configure and manage the agent locally. Each agent is configured to be in standalone mode by default after installation. -At startup, Elastic Agent reads the configuration file specified by the `-c` argument or uses the default configuration, `elastic-agent.yml`, which is located in the same directory as the agent. - -For configuration options see `elastic-agent_configuration_example.yml` - -=== Fleet mode - -With _fleet mode_, you manage Elastic Agent remotely. The agent uses a trusted {kib} instance to retrieve configurations and report agent events. This trusted {kib} instance must have Ingest Manager and Fleet enabled. - -To create a trusted communication channel between Elastic Agent and {kib}, you enroll the agent to Fleet. - -To enroll an Elastic Agent to Fleet: - - -. Stop the agent. - -. Enroll the agent: -+ -[source,shell] ----------------------------------------------------------------------- -$ ./elastic-agent http://localhost:5601 $token ----------------------------------------------------------------------- -+ -`$token` is an enrollment token acquired from Fleet. - -[[elastic-agent-cmd-options]] -== Command line options - -The `elastic-agent run` command provides flags that alter the behavior of an agent. - -==== `-path.home` - -The home directory of the Elastic Agent. `path.home` determines the location of the configuration files and data directory. - -==== `-c` - -The configuration file to load. -If not specified, Elastic Agent uses `{path.home}/elastic-agent.yml`. - - -==== `-path.data` - -The data directory used by Elastic Agent to store downloaded artifacts. Also stores logs for any Beats started and managed by Elastic Agent. - -If not specified, Elastic Agent uses `{path.home}/data`. - -[[elastic-agent-configuration]] -== Configure Elastic Agent - -By default Elastic Agent runs in standalone mode to ingest system data and send it to a local {es} instance running on port 9200. It uses the demo credentials of the `elastic` user. It's also configured to monitor all Beats managed by the agent and send the Beats logs and metrics to the same {es) instance. - -To alter this behavior, configure the output. - -=== Configure the output - -Elastic Agent enables definition of multiple outputs where each data source can be paired with different output. - -At the moment Elastic Agent works only with Elasticsearch output. -Sample configuration can look like the example below: - -[source,yaml] -------------------------------------------------------------------------------------- -outputs: - default: - type: elasticsearch - hosts: [127.0.0.1:9200] - username: elastic - password: changeme - - monitoring: - type: elasticsearch - api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw - hosts: ["localhost:9200"] - ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" -------------------------------------------------------------------------------------- - -This example configures two outputs: `default` and `monitoring`. -Notice that they use different authentication methods. The first one uses a username and password pair, and the second one contains an api key. - -[NOTE] -============== -A default output configuration is required. -============== - -=== Configure Beats monitoring - -Elastic Agent is monitoring _Beats_ by default. To disable or change monitoring settings, set options under `settings.monitoring`: - -[source,yaml] -------------------------------------------------------------------------------------- -settings.monitoring: - # enabled turns on monitoring of running processes - enabled: true - # enables log monitoring - logs: true - # enables metrics monitoring - metrics: true - # specifies output to be used - use_output: monitoring -------------------------------------------------------------------------------------- - - -To disable monitoring, set `settings.monitoring.enabled` to `false`. When set to `false`, Beats monitoring is turned off, and all other options in this section are ignored. -To enable monitoring, set `settings.monitoring.enabled` to `true`. Also set the `logs` and `metrics` settings to control whether logs, metrics, or both are collected. If neither setting is specified, monitoring is disabled. - - -Set `use_output` to specify the output to which monitoring events are sent. - -=== Specify data sources - -By default Elastic Agent collects system metrics, such as cpu, memory, network, and filesystem metrics, and sends them to the default output. For example: +include::install-elastic-agent.asciidoc[leveloffset=+1] +include::run-elastic-agent.asciidoc[leveloffset=+1] -[source,yaml] -------------------------------------------------------------------------------------- -datasources: - - namespace: default - use_output: default - inputs: - - type: system/metrics - streams: - - metricset: cpu - dataset: system.cpu - - metricset: memory - dataset: system.memory - - metricset: network - dataset: system.network - - metricset: filesystem - dataset: system.filesystem -------------------------------------------------------------------------------------- +include::elastic-agent-command-line.asciidoc[leveloffset=+1] -If `use_output` is not specified, the `default` output is used. +include::elastic-agent-configuration.asciidoc[leveloffset=+1] -For more examples, see `elastic-agent_configuration_example.yml` +//include::elastic-agent-configuration-example.asciidoc[leveloffset=+1] diff --git a/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc new file mode 100644 index 00000000000..06cca040135 --- /dev/null +++ b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc @@ -0,0 +1,81 @@ +:release-state: released + +[[elastic-agent-installation]] += Install {agent} + +experimental[] + +Download and install the agent on each system you want to monitor. + +To download and install {elastic-agent}, use the commands that work with your +system: + +//TODO: Replace with tabbed panels when the code is stable (might be after 7.8). + +*mac:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {agent} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +---- +curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{version}-darwin-x86_64.tar.gz +tar xzvf elastic-agent-{version}-darwin-x86_64.tar.gz +---- + +endif::[] + +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {agent} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +---- +curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{version}-linux-x86_64.tar.gz +tar xzvf elastic-agent-{version}-linux-x86_64.tar.gz +---- + +endif::[] + +*win:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {agent} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +. Download the {agent} Windows zip file from the +https://www.elastic.co/downloads/beats/elastic-agent[downloads page]. + +. Extract the contents of the zip file into `C:\Program Files`. + +. Rename the `elastic-agent--windows` directory to `Elastic-Agent`. + +. Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). + +. From the PowerShell prompt, run the following commands to install Filebeat as a +Windows service: ++ +[source,shell] +---- +PS > cd 'C:\Program Files\Elastic-Agent' +PS C:\Program Files\Elastic-Agent> .\install-service-elastic-agent.ps1 +---- + +NOTE: If script execution is disabled on your system, you need to set the execution policy for the current session to allow the script to run. For example: `PowerShell.exe -ExecutionPolicy UnRestricted -File .\install-service-elastic-agent.ps1`. + +endif::[] diff --git a/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc new file mode 100644 index 00000000000..560b3eb19d8 --- /dev/null +++ b/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc @@ -0,0 +1,65 @@ +[[run-elastic-agent]] += Run {agent} + +experimental[] + +{agent} runs in two modes: standalone or fleet. The two modes differ in how you +configure and manage the agent. + +[float] +[[standalone-mode]] +== Run in standalone mode (default) + +With _standalone mode_, you manually configure and manage the agent locally. +Each agent is configured to be in standalone mode by default after installation. + +If {agent} is installed as an auto-starting service, it will run automatically +when you restart your system. + +To start {agent} manually, run: + +[source,shell] +---- +./elastic-agent run +---- + +If no configuration file is specified, {agent} uses the default configuration, +`elastic-agent.yml`, which is located in the same directory as {agent}. Specify +the `-c` flag to use a different configuration file. + +For configuration options, see <>. + +//<> + +[float] +[[fleet-mode]] +== Run in {fleet} mode + +With _fleet mode_, you manage {agent} remotely. The agent uses a trusted {kib} +instance to retrieve configurations and report agent events. This trusted {kib} +instance must have {ingest-manager} and {fleet} enabled. + +To create a trusted communication channel between {agent} and {kib}, enroll the +agent to {fleet}. + +To enroll an {agent} to {fleet}: + +. Stop the agent, if it's already running. + +. Go the **{fleet}** tab in {ingest-manager}, and click **Enroll new agent** to +generate a token. See <> for detailed steps. + +. Enroll the agent: ++ +[source,shell] +---- +./elastic-agent enroll http://localhost:5601 $token +---- ++ +Where `$token` is an enrollment token acquired from {fleet}. + +To start {agent}, run: +[source,shell] +---- +./elastic-agent run +---- diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 719902e74a7..e7397d03c6a 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -423,7 +423,14 @@ func (Demo) NoEnroll() error { } func runAgent(env map[string]string) error { - supportedEnvs := map[string]int{"FLEET_CONFIG_ID": 0, "FLEET_ENROLLMENT_TOKEN": 0, "FLEET_ENROLL": 0, "FLEET_SETUP": 0, "FLEET_TOKEN_NAME": 0, "KIBANA_HOST": 0, "KIBANA_PASSWORD": 0, "KIBANA_USERNAME": 0} + prevPlatforms := os.Getenv("PLATFORMS") + defer os.Setenv("PLATFORMS", prevPlatforms) + + // setting this improves build time + os.Setenv("PLATFORMS", "+all linux/amd64") + devtools.Platforms = devtools.NewPlatformList("+all linux/amd64") + + supportedEnvs := map[string]int{"FLEET_ENROLLMENT_TOKEN": 0, "FLEET_ENROLL": 0, "FLEET_SETUP": 0, "FLEET_TOKEN_NAME": 0, "KIBANA_HOST": 0, "KIBANA_PASSWORD": 0, "KIBANA_USERNAME": 0} tag := dockerTag() dockerImageOut, err := sh.Output("docker", "image", "ls") @@ -435,7 +442,6 @@ func runAgent(env map[string]string) error { if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ - "linux-x86.tar.gz", "linux-x86_64.tar.gz", }, devtools.UseElasticAgentDemoPackaging) @@ -451,7 +457,11 @@ func runAgent(env map[string]string) error { } // prepare env variables - var envs []string + envs := []string{ + // providing default kibana to be fixed for os-es if not provided + "KIBANA_HOST=http://localhost:5601", + } + envs = append(envs, os.Environ()...) for k, v := range env { envs = append(envs, fmt.Sprintf("%s=%s", k, v)) @@ -465,6 +475,9 @@ func runAgent(env map[string]string) error { continue } + // fix value + e = fmt.Sprintf("%s=%s", parts[0], fixOsEnv(parts[0], parts[1])) + dockerCmdArgs = append(dockerCmdArgs, "-e", e) } @@ -521,6 +534,18 @@ func dockerTag() string { return tagBase } +func fixOsEnv(k, v string) string { + switch k { + case "KIBANA_HOST": + // network host works in a weird way here + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return strings.Replace(strings.ToLower(v), "localhost", "host.docker.internal", 1) + } + } + + return v +} + func buildVars() map[string]string { vars := make(map[string]string) diff --git a/x-pack/filebeat/input/cloudfoundry/input_integration_test.go b/x-pack/filebeat/input/cloudfoundry/input_integration_test.go new file mode 100644 index 00000000000..ac31a07132b --- /dev/null +++ b/x-pack/filebeat/input/cloudfoundry/input_integration_test.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package cloudfoundry + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" +) + +func TestInput(t *testing.T) { + config := common.MustNewConfigFrom(cftest.GetConfigFromEnv(t)) + + events := make(chan beat.Event) + connector := channel.ConnectorFunc(func(*common.Config, beat.ClientConfig) (channel.Outleter, error) { + return newOutleter(events), nil + }) + + inputCtx := input.Context{Done: make(chan struct{})} + + input, err := NewInput(config, connector, inputCtx) + require.NoError(t, err) + + go input.Run() + defer input.Stop() + + select { + case e := <-events: + t.Logf("Event received: %+v", e) + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for events") + } +} + +type outleter struct { + events chan<- beat.Event + done chan struct{} +} + +func newOutleter(events chan<- beat.Event) *outleter { + return &outleter{ + events: events, + done: make(chan struct{}), + } +} + +func (o *outleter) Close() error { + close(o.done) + return nil +} + +func (o *outleter) Done() <-chan struct{} { + return o.done +} + +func (o *outleter) OnEvent(e beat.Event) bool { + select { + case o.events <- e: + return true + default: + return false + } +} diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml index b2f0ebe4519..be7f27f551f 100644 --- a/x-pack/filebeat/module/coredns/log/config/coredns.yml +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -3,7 +3,8 @@ paths: {{ range $i, $path := .paths }} - {{$path}} {{ end }} -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_fields: target: '' diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 9382953fd37..9f5de43be3b 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -12,6 +12,9 @@ multiline.match: after multiline.max_lines: 5000 multiline.timeout: 10 +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + processors: - script: lang: javascript diff --git a/x-pack/filebeat/module/crowdstrike/falcon/manifest.yml b/x-pack/filebeat/module/crowdstrike/falcon/manifest.yml index b3d3edbb641..ab5f880e3a3 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/manifest.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/manifest.yml @@ -4,5 +4,7 @@ var: - name: paths default: - /var/log/crowdstrike/falconhoseclient/output + - name: tags + default: [forwarded] input: config/falcon.yml diff --git a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-audit-events.log-expected.json b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-audit-events.log-expected.json index e5466024247..3aae7f3f80e 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-audit-events.log-expected.json +++ b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-audit-events.log-expected.json @@ -1,97 +1,72 @@ [ { "@timestamp": "2020-02-27T19:12:14.000Z", - "service.type": "crowdstrike", - "input.type": "log", + "crowdstrike.event.HostnameField": "hostnameofmachine", + "crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de", + "crowdstrike.event.StartTimestamp": 1582830734, + "crowdstrike.event.UserName": "first.last@company.com", "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 1045, - "crowdstrike.metadata.eventType": "RemoteResponseSessionStartEvent", "crowdstrike.metadata.eventCreationTime": 1582830734000, + "crowdstrike.metadata.eventType": "RemoteResponseSessionStartEvent", + "crowdstrike.metadata.offset": 1045, "crowdstrike.metadata.version": "1.0", - "crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de", - "crowdstrike.event.HostnameField": "hostnameofmachine", - "crowdstrike.event.UserName": "first.last@company.com", - "crowdstrike.event.StartTimestamp": 1582830734, - "event.module": "crowdstrike", + "event.action": "remote_response_session_start_event", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "remote_response_session_start_event", - "event.type": [ "start" ], + "event.module": "crowdstrike", "event.outcome": "unknown", - "message": "Remote response session started", - "host.name": "hostnameofmachine", - "user.name": "first.last@company.com", - "user.email": "first.last@company.com", - "agent.type": "falcon", + "event.type": [ + "start" + ], "fileset.name": "falcon", - "log.file.path": "falcon-events.log", + "input.type": "log", "log.flags": [ "multiline" ], - "log.offset": 0 + "log.offset": 0, + "message": "Remote response session started", + "service.type": "crowdstrike", + "tags": [ + "forwarded" + ], + "user.email": "first.last@company.com", + "user.name": "first.last@company.com" }, { "@timestamp": "2020-02-27T19:12:52.000Z", - "crowdstrike.metadata.offset": 1046, - "crowdstrike.metadata.eventType": "RemoteResponseSessionEndEvent", - "crowdstrike.metadata.eventCreationTime": 1582830772000, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de", + "crowdstrike.event.EndTimestamp": 1582830772, "crowdstrike.event.HostnameField": "hostnameofmachine", + "crowdstrike.event.SessionId": "6020260b-0398-4d41-999d-5531b55522de", "crowdstrike.event.UserName": "first.last@company.com", - "crowdstrike.event.EndTimestamp": 1582830772, - "user.name": "first.last@company.com", - "user.email": "first.last@company.com", - "fileset.name": "falcon", - "service.type": "crowdstrike", - "input.type": "log", - "event.module": "crowdstrike", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1582830772000, + "crowdstrike.metadata.eventType": "RemoteResponseSessionEndEvent", + "crowdstrike.metadata.offset": 1046, + "crowdstrike.metadata.version": "1.0", + "event.action": "remote_response_session_end_event", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "remote_response_session_end_event", - "event.type": ["end"], + "event.module": "crowdstrike", "event.outcome": "unknown", - "message": "Remote response session ended", - "host.name": "hostnameofmachine", - "log.file.path": "falcon-events.log", + "event.type": [ + "end" + ], + "fileset.name": "falcon", + "input.type": "log", "log.flags": [ "multiline" ], "log.offset": 457, - "agent.type": "falcon" + "message": "Remote response session ended", + "service.type": "crowdstrike", + "tags": [ + "forwarded" + ], + "user.email": "first.last@company.com", + "user.name": "first.last@company.com" }, { "@timestamp": "2020-02-12T21:29:10.710Z", - "message": "Crowdstrike Streaming API", - "source.ip": "10.10.0.8", - "input.type": "log", - "event.module": "crowdstrike", - "event.dataset": "crowdstrike.falcon_audit", - "event.kind": "event", - "event.action": "stream_started", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", - "agent.type": "falcon", - "user.name": "api-client-id:1234567890abcdefghijklmnopqrstuvwxyz", - "log.offset": 910, - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" - ], - "service.type": "crowdstrike", - "fileset.name": "falcon", - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 0, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581542950710, - "crowdstrike.event.UserIp": "10.10.0.8", - "crowdstrike.event.OperationName": "streamStarted", - "crowdstrike.event.ServiceName": "Crowdstrike Streaming API", - "crowdstrike.event.Success": true, - "crowdstrike.event.UTCTimestamp": 1581542950, "crowdstrike.event.AuditKeyValues": [ { "Key": "APIClientID", @@ -114,175 +89,215 @@ "ValueString": "[UserActivityAuditEvent HashSpreadingEvent RemoteResponseSessionStartEvent RemoteResponseSessionEndEvent DetectionSummaryEvent AuthActivityAuditEvent]" } ], - "crowdstrike.event.UserId": "api-client-id:1234567890abcdefghijklmnopqrstuvwxyz" - }, - { - "@timestamp": "2020-02-12T21:39:37.147Z", - "log.offset": 2152, - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" + "crowdstrike.event.OperationName": "streamStarted", + "crowdstrike.event.ServiceName": "Crowdstrike Streaming API", + "crowdstrike.event.Success": true, + "crowdstrike.event.UTCTimestamp": 1581542950, + "crowdstrike.event.UserId": "api-client-id:1234567890abcdefghijklmnopqrstuvwxyz", + "crowdstrike.event.UserIp": "10.10.0.8", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581542950710, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 0, + "crowdstrike.metadata.version": "1.0", + "event.action": "stream_started", + "event.category": [ + "authentication" ], - "source.ip": "192.168.6.8", - "fileset.name": "falcon", - "service.type": "crowdstrike", - "input.type": "log", - "event.module": "crowdstrike", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "two_factor_authenticate", - "event.type": ["change"], - "event.category": ["authentication"], + "event.module": "crowdstrike", "event.outcome": "success", - "crowdstrike.metadata.eventCreationTime": 1581543577147, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 1, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 910, + "message": "Crowdstrike Streaming API", + "service.type": "crowdstrike", + "source.ip": "10.10.0.8", + "tags": [ + "forwarded" + ], + "user.name": "api-client-id:1234567890abcdefghijklmnopqrstuvwxyz" + }, + { + "@timestamp": "2020-02-12T21:39:37.147Z", + "crowdstrike.event.OperationName": "twoFactorAuthenticate", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581543577147, "crowdstrike.event.UserId": "alice@company.com", "crowdstrike.event.UserIp": "192.168.6.8", - "crowdstrike.event.OperationName": "twoFactorAuthenticate", - "agent.type": "falcon", - "user.name": "alice@company.com", - "user.email": "alice@company.com", - "message": "CrowdStrike Authentication" - }, - { - "@timestamp": "2020-02-12T22:14:37.554Z", - "log.flags": [ - "multiline" + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581543577147, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 1, + "crowdstrike.metadata.version": "1.0", + "event.action": "two_factor_authenticate", + "event.category": [ + "authentication" ], - "log.offset": 2645, - "log.file.path": "falcon-events.log", - "fileset.name": "falcon", - "service.type": "crowdstrike", - "event.module": "crowdstrike", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "two_factor_authenticate", - "event.type": ["change"], - "event.category": ["authentication"], + "event.module": "crowdstrike", "event.outcome": "success", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 2, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581545677554, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.event.UserId": "bob@company.com", - "crowdstrike.event.UserIp": "192.168.6.3", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 2152, + "message": "CrowdStrike Authentication", + "service.type": "crowdstrike", + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" + }, + { + "@timestamp": "2020-02-12T22:14:37.554Z", "crowdstrike.event.OperationName": "twoFactorAuthenticate", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581545677554, - "user.name": "bob@company.com", - "user.email": "bob@company.com", + "crowdstrike.event.UserId": "bob@company.com", + "crowdstrike.event.UserIp": "192.168.6.3", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581545677554, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 2, + "crowdstrike.metadata.version": "1.0", + "event.action": "two_factor_authenticate", + "event.category": [ + "authentication" + ], + "event.dataset": "crowdstrike.falcon_audit", + "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 2645, "message": "CrowdStrike Authentication", + "service.type": "crowdstrike", "source.ip": "192.168.6.3", - "input.type": "log", - "agent.type": "falcon" + "tags": [ + "forwarded" + ], + "user.email": "bob@company.com", + "user.name": "bob@company.com" }, { "@timestamp": "2020-02-12T22:24:08.000Z", - "input.type": "log", - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 3, - "crowdstrike.metadata.eventType": "UserActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581546248000, - "crowdstrike.event.ServiceName": "groups", "crowdstrike.event.AuditKeyValues": [ { - "ValueString": "3c80ce30b9654cb4bd15beec6a517e65", - "Key": "group_id" + "Key": "group_id", + "ValueString": "3c80ce30b9654cb4bd15beec6a517e65" }, { "Key": "action_name", "ValueString": "add_group_member" } ], + "crowdstrike.event.OperationName": "update_group", + "crowdstrike.event.ServiceName": "groups", "crowdstrike.event.UTCTimestamp": 1581546248, "crowdstrike.event.UserId": "chris@company.com", "crowdstrike.event.UserIp": "192.168.6.13", - "crowdstrike.event.OperationName": "update_group", - "log.offset": 3136, - "log.file.path": "falcon-events.log", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581546248000, + "crowdstrike.metadata.eventType": "UserActivityAuditEvent", + "crowdstrike.metadata.offset": 3, + "crowdstrike.metadata.version": "1.0", + "event.action": "user_activity_audit_event", + "event.category": [ + "iam" + ], + "event.dataset": "crowdstrike.falcon_audit", + "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "unknown", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", "log.flags": [ "multiline" ], - "service.type": "crowdstrike", - "fileset.name": "falcon", - "agent.type": "falcon", - "user.name": "chris@company.com", - "user.email": "chris@company.com", + "log.offset": 3136, "message": "update_group", + "service.type": "crowdstrike", "source.ip": "192.168.6.13", - "event.kind": "event", - "event.action": "user_activity_audit_event", - "event.type": ["change"], - "event.outcome": "unknown", - "event.category": ["iam"], - "event.module": "crowdstrike", - "event.dataset": "crowdstrike.falcon_audit" + "tags": [ + "forwarded" + ], + "user.email": "chris@company.com", + "user.name": "chris@company.com" }, { "@timestamp": "2020-02-13T13:41:52.140Z", - "source.ip": "192.168.6.8", - "user.name": "alice@company.com", - "user.email": "alice@company.com", - "service.type": "crowdstrike", - "input.type": "log", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 4, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581601312140, - "crowdstrike.metadata.version": "1.0", "crowdstrike.event.AuditKeyValues": [ { "Key": "target_name", "ValueString": "alice@company.com" } ], - "crowdstrike.event.UserId": "alice@company.com", - "crowdstrike.event.UserIp": "192.168.6.8", "crowdstrike.event.OperationName": "requestResetPassword", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581601312140, - "agent.type": "falcon", - "message": "CrowdStrike Authentication", - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" + "crowdstrike.event.UserId": "alice@company.com", + "crowdstrike.event.UserIp": "192.168.6.8", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581601312140, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 4, + "crowdstrike.metadata.version": "1.0", + "event.action": "request_reset_password", + "event.category": [ + "authentication" ], - "log.offset": 3858, - "event.module": "crowdstrike", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "request_reset_password", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", - "fileset.name": "falcon" - }, - { - "@timestamp": "2020-02-13T13:42:21.730Z", - "event.kind": "event", - "event.action": "two_factor_authenticate", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", - "event.dataset": "crowdstrike.falcon_audit", "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], "fileset.name": "falcon", - "agent.type": "falcon", - "user.name": "alice@company.com", - "user.email": "alice@company.com", "input.type": "log", - "source.ip": "192.168.6.8", + "log.flags": [ + "multiline" + ], + "log.offset": 3858, + "message": "CrowdStrike Authentication", "service.type": "crowdstrike", + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" + }, + { + "@timestamp": "2020-02-13T13:42:21.730Z", "crowdstrike.event.OperationName": "twoFactorAuthenticate", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, @@ -290,183 +305,200 @@ "crowdstrike.event.UserId": "alice@company.com", "crowdstrike.event.UserIp": "192.168.6.8", "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 5, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", "crowdstrike.metadata.eventCreationTime": 1581601341730, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 5, "crowdstrike.metadata.version": "1.0", - "message": "CrowdStrike Authentication", - "log.offset": 4506, - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" - ] - }, - { - "@timestamp": "2020-02-13T13:45:20.236Z", - "user.name": "alice@company.com", - "user.email": "alice@company.com", - "log.offset": 5003, - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" + "event.action": "two_factor_authenticate", + "event.category": [ + "authentication" ], - "event.action": "change_password", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", - "event.module": "crowdstrike", "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], "fileset.name": "falcon", - "agent.type": "falcon", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 4506, "message": "CrowdStrike Authentication", - "source.ip": "192.168.6.8", "service.type": "crowdstrike", - "input.type": "log", - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 6, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581601520236, - "crowdstrike.event.Success": true, - "crowdstrike.event.UTCTimestamp": 1581601520236, + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" + }, + { + "@timestamp": "2020-02-13T13:45:20.236Z", "crowdstrike.event.AuditKeyValues": [ { "Key": "target_name", "ValueString": "first.last@company.com" } ], + "crowdstrike.event.OperationName": "changePassword", + "crowdstrike.event.ServiceName": "CrowdStrike Authentication", + "crowdstrike.event.Success": true, + "crowdstrike.event.UTCTimestamp": 1581601520236, "crowdstrike.event.UserId": "alice@company.com", "crowdstrike.event.UserIp": "192.168.6.8", - "crowdstrike.event.OperationName": "changePassword", - "crowdstrike.event.ServiceName": "CrowdStrike Authentication" - }, - { - "@timestamp": "2020-02-13T13:46:12.362Z", - "log.offset": 5657, - "log.file.path": "falcon-events.log", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581601520236, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 6, + "crowdstrike.metadata.version": "1.0", + "event.action": "change_password", + "event.category": [ + "authentication" + ], + "event.dataset": "crowdstrike.falcon_audit", + "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", "log.flags": [ "multiline" ], + "log.offset": 5003, + "message": "CrowdStrike Authentication", "service.type": "crowdstrike", - "input.type": "log", - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581601572362, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 7, + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" + }, + { + "@timestamp": "2020-02-13T13:46:12.362Z", + "crowdstrike.event.OperationName": "userAuthenticate", + "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581601572362, "crowdstrike.event.UserId": "alice@company.com", "crowdstrike.event.UserIp": "192.168.6.8", - "crowdstrike.event.OperationName": "userAuthenticate", - "crowdstrike.event.ServiceName": "CrowdStrike Authentication", - "message": "CrowdStrike Authentication", - "source.ip": "192.168.6.8", - "event.module": "crowdstrike", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581601572362, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 7, + "crowdstrike.metadata.version": "1.0", + "event.action": "user_authenticate", + "event.category": [ + "authentication" + ], "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "user_authenticate", - "event.type": ["change"], - "event.category": ["authentication"], + "event.module": "crowdstrike", "event.outcome": "success", + "event.type": [ + "change" + ], "fileset.name": "falcon", - "agent.type": "falcon", - "user.name": "alice@company.com", - "user.email": "alice@company.com" + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 5657, + "message": "CrowdStrike Authentication", + "service.type": "crowdstrike", + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" }, { "@timestamp": "2020-02-13T13:50:14.754Z", - "input.type": "log", - "crowdstrike.metadata.eventCreationTime": 1581601814754, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 8, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.event.OperationName": "twoFactorAuthenticate", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581601814754, "crowdstrike.event.UserId": "alice@company.com", "crowdstrike.event.UserIp": "192.168.6.8", - "crowdstrike.event.OperationName": "twoFactorAuthenticate", - "agent.type": "falcon", - "source.ip": "192.168.6.8", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581601814754, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 8, + "crowdstrike.metadata.version": "1.0", + "event.action": "two_factor_authenticate", + "event.category": [ + "authentication" + ], "event.dataset": "crowdstrike.falcon_audit", "event.kind": "event", - "event.action": "two_factor_authenticate", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], "fileset.name": "falcon", - "service.type": "crowdstrike", - "user.name": "alice@company.com", - "user.email": "alice@company.com", - "message": "CrowdStrike Authentication", - "log.offset": 6149, - "log.file.path": "falcon-events.log", + "input.type": "log", "log.flags": [ "multiline" - ] + ], + "log.offset": 6149, + "message": "CrowdStrike Authentication", + "service.type": "crowdstrike", + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" }, { "@timestamp": "2020-02-13T13:50:20.289Z", - "agent.type": "falcon", - "event.action": "self_accept_eula", - "event.type": ["change"], - "event.category": ["authentication"], - "event.outcome": "success", - "event.module": "crowdstrike", - "event.dataset": "crowdstrike.falcon_audit", - "event.kind": "event", - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 9, - "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", - "crowdstrike.metadata.eventCreationTime": 1581601820289, - "crowdstrike.event.UserId": "alice@company.com", - "crowdstrike.event.UserIp": "192.168.6.8", "crowdstrike.event.OperationName": "selfAcceptEula", "crowdstrike.event.ServiceName": "CrowdStrike Authentication", "crowdstrike.event.Success": true, "crowdstrike.event.UTCTimestamp": 1581601820289, + "crowdstrike.event.UserId": "alice@company.com", + "crowdstrike.event.UserIp": "192.168.6.8", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581601820289, + "crowdstrike.metadata.eventType": "AuthActivityAuditEvent", + "crowdstrike.metadata.offset": 9, + "crowdstrike.metadata.version": "1.0", + "event.action": "self_accept_eula", + "event.category": [ + "authentication" + ], + "event.dataset": "crowdstrike.falcon_audit", + "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "success", + "event.type": [ + "change" + ], "fileset.name": "falcon", - "service.type": "crowdstrike", "input.type": "log", - "user.name": "alice@company.com", - "user.email": "alice@company.com", - "message": "CrowdStrike Authentication", - "log.file.path": "falcon-events.log", "log.flags": [ "multiline" ], "log.offset": 6642, - "source.ip": "192.168.6.8" + "message": "CrowdStrike Authentication", + "service.type": "crowdstrike", + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" }, { "@timestamp": "2020-02-13T14:14:22.000Z", - "agent.type": "falcon", - "message": "detection_update", - "source.ip": "192.168.6.8", - "input.type": "log", - "event.dataset": "crowdstrike.falcon_audit", - "event.kind": "event", - "event.action": "user_activity_audit_event", - "event.type": ["change"], - "event.outcome": "unknown", - "event.category": ["iam"], - "event.module": "crowdstrike", - "fileset.name": "falcon", - "crowdstrike.metadata.eventCreationTime": 1581603262000, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 10, - "crowdstrike.metadata.eventType": "UserActivityAuditEvent", - "crowdstrike.event.UTCTimestamp": 1581603262, - "crowdstrike.event.UserId": "alice@company.com", - "crowdstrike.event.UserIp": "192.168.6.8", - "crowdstrike.event.OperationName": "detection_update", - "crowdstrike.event.ServiceName": "detections", "crowdstrike.event.AuditKeyValues": [ { "Key": "detection_id", @@ -485,13 +517,40 @@ "ValueString": "first.last@company.com" } ], - "log.offset": 7128, - "log.file.path": "falcon-events.log", + "crowdstrike.event.OperationName": "detection_update", + "crowdstrike.event.ServiceName": "detections", + "crowdstrike.event.UTCTimestamp": 1581603262, + "crowdstrike.event.UserId": "alice@company.com", + "crowdstrike.event.UserIp": "192.168.6.8", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1581603262000, + "crowdstrike.metadata.eventType": "UserActivityAuditEvent", + "crowdstrike.metadata.offset": 10, + "crowdstrike.metadata.version": "1.0", + "event.action": "user_activity_audit_event", + "event.category": [ + "iam" + ], + "event.dataset": "crowdstrike.falcon_audit", + "event.kind": "event", + "event.module": "crowdstrike", + "event.outcome": "unknown", + "event.type": [ + "change" + ], + "fileset.name": "falcon", + "input.type": "log", "log.flags": [ "multiline" ], + "log.offset": 7128, + "message": "detection_update", "service.type": "crowdstrike", - "user.name": "alice@company.com", - "user.email": "alice@company.com" + "source.ip": "192.168.6.8", + "tags": [ + "forwarded" + ], + "user.email": "alice@company.com", + "user.name": "alice@company.com" } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json index fddd89e4fea..e4f8a56d58c 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json +++ b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json @@ -1,113 +1,125 @@ [ { "@timestamp": "2020-02-19T08:30:00.000Z", - "process.pid": 38684386611, - "process.name": "explorer.exe", - "process.command_line": "C:\\Windows\\Explorer.EXE", - "process.executable": "C:\\Windows\\Explorer.EXE", - "process.args": ["C:\\Windows\\Explorer.EXE"], - "event.dataset": "crowdstrike.falcon_endpoint", - "event.kind": "alert", - "event.action": "Prevention, process killed.", - "event.type": ["info"], - "event.category": ["malware"], - "event.severity": 4, - "event.module": "crowdstrike", - "event.url": "https://falcon.crowdstrike.com/ec86abd353824e96765ecbe18eb4f0b4", - "event.outcome": "unknown", - "service.type": "crowdstrike", - "user.name": "alice", - "user.domain": "CORP-DOMAIN", - "rule.description": "Terminated a process related to the deletion of backups, which is often indicative of ransomware activity.", - "rule.name": "Process Terminated", - "log.flags": [ - "multiline" - ], - "log.offset": 0, - "log.file.path": "falcon-events.log", - "source.ip": "192.168.12.51", - "agent.type": "falcon", - "host.name": "alice-laptop", - "message": "Terminated a process related to the deletion of backups, which is often indicative of ransomware activity.", - "fileset.name": "falcon", - "input.type": "log", - "file.hash.md5": "ac4c51eb24aa95b77f705ab159189e24", - "file.hash.sha256": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a", - "threat.tactic.name": "malware", - "threat.technique.name": "ransomware", - "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 294564, - "crowdstrike.metadata.eventType": "DetectionSummaryEvent", - "crowdstrike.metadata.eventCreationTime": 1582101000000, - "crowdstrike.metadata.version": "1.0", - "crowdstrike.event.ParentProcessId": 38682494050, - "crowdstrike.event.SHA256String": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a", - "crowdstrike.event.SensorId": "7c808b4c8878433287eea53d4a8c3268", - "crowdstrike.event.LocalIP": "192.168.12.51", - "crowdstrike.event.FalconHostLink": "https://falcon.crowdstrike.com/ec86abd353824e96765ecbe18eb4f0b4", - "crowdstrike.event.Tactic": "Malware", - "crowdstrike.event.ProcessEndTime": 0, - "crowdstrike.event.Severity": 4, "crowdstrike.event.CommandLine": "C:\\Windows\\Explorer.EXE", - "crowdstrike.event.Technique": "Ransomware", - "crowdstrike.event.Objective": "Falcon Detection Method", - "crowdstrike.event.ProcessId": 38684386611, + "crowdstrike.event.ComputerName": "alice-laptop", "crowdstrike.event.DetectDescription": "Terminated a process related to the deletion of backups, which is often indicative of ransomware activity.", + "crowdstrike.event.DetectId": "ldt:ec86abd353824e96765ecbe18eb4f0b4:38655257584", + "crowdstrike.event.DetectName": "Process Terminated", + "crowdstrike.event.FalconHostLink": "https://falcon.crowdstrike.com/ec86abd353824e96765ecbe18eb4f0b4", + "crowdstrike.event.FileName": "explorer.exe", + "crowdstrike.event.FilePath": "\\Device\\HarddiskVolume1\\Windows", + "crowdstrike.event.LocalIP": "192.168.12.51", + "crowdstrike.event.MACAddress": "00-00-00-11-22-33", + "crowdstrike.event.MD5String": "ac4c51eb24aa95b77f705ab159189e24", + "crowdstrike.event.MachineDomain": "CORP-DOMAIN", + "crowdstrike.event.Objective": "Falcon Detection Method", + "crowdstrike.event.ParentProcessId": 38682494050, "crowdstrike.event.PatternDispositionDescription": "Prevention, process killed.", - "crowdstrike.event.PatternDispositionFlags.Indicator": false, "crowdstrike.event.PatternDispositionFlags.Detect": false, + "crowdstrike.event.PatternDispositionFlags.InddetMask": false, + "crowdstrike.event.PatternDispositionFlags.Indicator": false, + "crowdstrike.event.PatternDispositionFlags.KillParent": false, "crowdstrike.event.PatternDispositionFlags.KillProcess": true, "crowdstrike.event.PatternDispositionFlags.KillSubProcess": false, - "crowdstrike.event.PatternDispositionFlags.KillParent": false, "crowdstrike.event.PatternDispositionFlags.OperationBlocked": false, + "crowdstrike.event.PatternDispositionFlags.PolicyDisabled": false, "crowdstrike.event.PatternDispositionFlags.ProcessBlocked": false, - "crowdstrike.event.PatternDispositionFlags.InddetMask": false, - "crowdstrike.event.PatternDispositionFlags.SensorOnly": false, - "crowdstrike.event.PatternDispositionFlags.Rooting": false, - "crowdstrike.event.PatternDispositionFlags.QuarantineMachine": false, "crowdstrike.event.PatternDispositionFlags.QuarantineFile": false, - "crowdstrike.event.PatternDispositionFlags.PolicyDisabled": false, - "crowdstrike.event.FileName": "explorer.exe", - "crowdstrike.event.MachineDomain": "CORP-DOMAIN", + "crowdstrike.event.PatternDispositionFlags.QuarantineMachine": false, + "crowdstrike.event.PatternDispositionFlags.Rooting": false, + "crowdstrike.event.PatternDispositionFlags.SensorOnly": false, "crowdstrike.event.PatternDispositionValue": 16, - "crowdstrike.event.ComputerName": "alice-laptop", - "crowdstrike.event.UserName": "alice", - "crowdstrike.event.MD5String": "ac4c51eb24aa95b77f705ab159189e24", - "crowdstrike.event.DetectId": "ldt:ec86abd353824e96765ecbe18eb4f0b4:38655257584", - "crowdstrike.event.MACAddress": "00-00-00-11-22-33", + "crowdstrike.event.ProcessEndTime": 0, + "crowdstrike.event.ProcessId": 38684386611, "crowdstrike.event.ProcessStartTime": 1536846339, - "crowdstrike.event.DetectName": "Process Terminated", + "crowdstrike.event.SHA256String": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a", + "crowdstrike.event.SensorId": "7c808b4c8878433287eea53d4a8c3268", + "crowdstrike.event.Severity": 4, "crowdstrike.event.SeverityName": "High", - "crowdstrike.event.FilePath": "\\Device\\HarddiskVolume1\\Windows" - }, - { - "@timestamp": "2020-03-04T04:17:56.766Z", - "log.offset": 2063, - "log.file.path": "falcon-events.log", - "log.flags": [ - "multiline" + "crowdstrike.event.Tactic": "Malware", + "crowdstrike.event.Technique": "Ransomware", + "crowdstrike.event.UserName": "alice", + "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.metadata.eventCreationTime": 1582101000000, + "crowdstrike.metadata.eventType": "DetectionSummaryEvent", + "crowdstrike.metadata.offset": 294564, + "crowdstrike.metadata.version": "1.0", + "event.action": "Prevention, process killed.", + "event.category": [ + "malware" ], - "event.module": "crowdstrike", "event.dataset": "crowdstrike.falcon_endpoint", "event.kind": "alert", - "event.type": ["info"], - "event.category": ["malware"], - "event.action": "incident", - "event.url": "https://falcon.crowdstrike.com/crowdscore/incidents/details/inc:8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "event.module": "crowdstrike", "event.outcome": "unknown", + "event.severity": 4, + "event.type": [ + "info" + ], + "event.url": "https://falcon.crowdstrike.com/ec86abd353824e96765ecbe18eb4f0b4", + "file.hash.md5": "ac4c51eb24aa95b77f705ab159189e24", + "file.hash.sha256": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a", + "fileset.name": "falcon", "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 0, + "message": "Terminated a process related to the deletion of backups, which is often indicative of ransomware activity.", + "process.args": [ + "C:\\Windows\\Explorer.EXE" + ], + "process.command_line": "C:\\Windows\\Explorer.EXE", + "process.executable": "C:\\Windows\\Explorer.EXE", + "process.name": "explorer.exe", + "process.pid": 38684386611, + "rule.description": "Terminated a process related to the deletion of backups, which is often indicative of ransomware activity.", + "rule.name": "Process Terminated", + "service.type": "crowdstrike", + "source.ip": "192.168.12.51", + "tags": [ + "forwarded" + ], + "threat.tactic.name": "malware", + "threat.technique.name": "ransomware", + "user.domain": "CORP-DOMAIN", + "user.name": "alice" + }, + { + "@timestamp": "2020-03-04T04:17:56.766Z", + "crowdstrike.event.FalconHostLink": "https://falcon.crowdstrike.com/crowdscore/incidents/details/inc:8f69fe9e-b995-4204-95ad-44f9bcf75b6b", + "crowdstrike.event.FineScore": 1.2, + "crowdstrike.event.IncidentEndTime": 1583295470, + "crowdstrike.event.IncidentStartTime": 1583295228, + "crowdstrike.event.State": "open", "crowdstrike.metadata.customerIDString": "8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.metadata.offset": 1824, - "crowdstrike.metadata.eventType": "IncidentSummaryEvent", "crowdstrike.metadata.eventCreationTime": 1583295476766, + "crowdstrike.metadata.eventType": "IncidentSummaryEvent", + "crowdstrike.metadata.offset": 1824, "crowdstrike.metadata.version": "1.0", - "crowdstrike.event.IncidentStartTime": 1583295228, - "crowdstrike.event.IncidentEndTime": 1583295470, - "crowdstrike.event.FalconHostLink": "https://falcon.crowdstrike.com/crowdscore/incidents/details/inc:8f69fe9e-b995-4204-95ad-44f9bcf75b6b", - "crowdstrike.event.State": "open", - "crowdstrike.event.FineScore": 1.2, - "message": "Incident score 1.2", + "event.action": "incident", + "event.category": [ + "malware" + ], + "event.dataset": "crowdstrike.falcon_endpoint", + "event.kind": "alert", + "event.module": "crowdstrike", + "event.outcome": "unknown", + "event.type": [ + "info" + ], + "event.url": "https://falcon.crowdstrike.com/crowdscore/incidents/details/inc:8f69fe9e-b995-4204-95ad-44f9bcf75b6b", "fileset.name": "falcon", - "service.type": "crowdstrike" + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 2063, + "message": "Incident score 1.2", + "service.type": "crowdstrike", + "tags": [ + "forwarded" + ] } -] +] \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml index b2f0ebe4519..be7f27f551f 100644 --- a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml +++ b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml @@ -3,7 +3,8 @@ paths: {{ range $i, $path := .paths }} - {{$path}} {{ end }} -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - add_fields: target: '' diff --git a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml index 474e4d94ffa..6b6fcf216f2 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index 2fd07156cd8..b13737a882d 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml index 4ee5d46f72a..e99deb6c68f 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml index cba0c8ccf17..e0eb1831b77 100644 --- a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml +++ b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml index 6bed5fbef2d..33890a55071 100644 --- a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml +++ b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index 9a137ca2d9f..a6457935937 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - rename: diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml index d4b8abc101a..49e69972ef6 100644 --- a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml +++ b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml index 413ccb7959b..9f2046e7822 100644 --- a/x-pack/filebeat/module/zeek/files/config/files.yml +++ b/x-pack/filebeat/module/zeek/files/config/files.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml index cd380c29057..9be20cbd85b 100644 --- a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml +++ b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml index 38f7241b314..6f517c69320 100644 --- a/x-pack/filebeat/module/zeek/http/config/http.yml +++ b/x-pack/filebeat/module/zeek/http/config/http.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false @@ -62,7 +63,7 @@ processors: - from: "zeek.http.password" to: "url.password" - + - from: "zeek.http.user_agent" to: "user_agent.original" diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml index cb52e756362..9293393192f 100644 --- a/x-pack/filebeat/module/zeek/intel/config/intel.yml +++ b/x-pack/filebeat/module/zeek/intel/config/intel.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - rename: diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml index 0b6991603a9..5125e82fac2 100644 --- a/x-pack/filebeat/module/zeek/irc/config/irc.yml +++ b/x-pack/filebeat/module/zeek/irc/config/irc.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml index 5a03d269d8e..79b6b6c94c0 100644 --- a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml +++ b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml index 923177268c6..c6a6c3bf9e1 100644 --- a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml +++ b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml index da2b4b02d92..b3fd2c5484f 100644 --- a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml +++ b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml index 3bf628c0029..71eef51d512 100644 --- a/x-pack/filebeat/module/zeek/notice/config/notice.yml +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml index 07b4960364a..dcf1cb46f6f 100644 --- a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml +++ b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml index f2355d34ad8..874a0fde6d9 100644 --- a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml +++ b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml index e6464ba8601..3df430d7dc9 100644 --- a/x-pack/filebeat/module/zeek/pe/config/pe.yml +++ b/x-pack/filebeat/module/zeek/pe/config/pe.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml index b80a2177a04..d133dbc014c 100644 --- a/x-pack/filebeat/module/zeek/radius/config/radius.yml +++ b/x-pack/filebeat/module/zeek/radius/config/radius.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml index fad5d1f4234..1ea1c28520c 100644 --- a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml +++ b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml index aa0f303e28e..d0eade39d0e 100644 --- a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml +++ b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml index c289ff7103b..7355ba353c7 100644 --- a/x-pack/filebeat/module/zeek/sip/config/sip.yml +++ b/x-pack/filebeat/module/zeek/sip/config/sip.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml index b613880302f..693c439454a 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml index 7c188a39f4d..16fb3d36964 100644 --- a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml +++ b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml index ca6ffa00ada..168458ba1d0 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml index 6ec75151295..acc02bf91df 100644 --- a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml +++ b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml index f3a670a9d2d..684ecd01d06 100644 --- a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml +++ b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml index 4d3060ae657..9ce64d99076 100644 --- a/x-pack/filebeat/module/zeek/socks/config/socks.yml +++ b/x-pack/filebeat/module/zeek/socks/config/socks.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml index 5657816c73a..453c6b0fae5 100644 --- a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml +++ b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml index 74336b8428c..e140ba94e2c 100644 --- a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml +++ b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/stats/config/stats.yml b/x-pack/filebeat/module/zeek/stats/config/stats.yml index ff8ca3cf95e..cdf243f7a45 100644 --- a/x-pack/filebeat/module/zeek/stats/config/stats.yml +++ b/x-pack/filebeat/module/zeek/stats/config/stats.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml index 662968b502e..16c6ebb6187 100644 --- a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml +++ b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml index fc1c4b7230f..13a2a37cc69 100644 --- a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml +++ b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml index 8781fa6dcd6..ac636e9e7c0 100644 --- a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml +++ b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml index 9631d283de4..5807f95927b 100644 --- a/x-pack/filebeat/module/zeek/weird/config/weird.yml +++ b/x-pack/filebeat/module/zeek/weird/config/weird.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/filebeat/module/zeek/x509/config/x509.yml b/x-pack/filebeat/module/zeek/x509/config/x509.yml index cb70bd594e7..34ab8fc651f 100644 --- a/x-pack/filebeat/module/zeek/x509/config/x509.yml +++ b/x-pack/filebeat/module/zeek/x509/config/x509.yml @@ -4,7 +4,8 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] -tags: {{.tags}} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} json.keys_under_root: false diff --git a/x-pack/libbeat/common/cloudfoundry/test/config.go b/x-pack/libbeat/common/cloudfoundry/test/config.go new file mode 100644 index 00000000000..c4024caa5ae --- /dev/null +++ b/x-pack/libbeat/common/cloudfoundry/test/config.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package test + +import ( + "os" + "testing" +) + +func GetConfigFromEnv(t *testing.T) map[string]interface{} { + t.Helper() + + config := map[string]interface{}{ + "api_address": lookupEnv(t, "CLOUDFOUNDRY_API_ADDRESS"), + "client_id": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_ID"), + "client_secret": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_SECRET"), + + "ssl.verification_mode": "none", + } + + optionalConfig(config, "uaa_address", "CLOUDFOUNDRY_UAA_ADDRESS") + optionalConfig(config, "rlp_address", "CLOUDFOUNDRY_RLP_ADDRESS") + optionalConfig(config, "doppler_address", "CLOUDFOUNDRY_DOPPLER_ADDRESS") + + if t.Failed() { + t.FailNow() + } + + return config +} + +func lookupEnv(t *testing.T, name string) string { + value, ok := os.LookupEnv(name) + if !ok { + t.Errorf("Environment variable %s is not set", name) + } + return value +} + +func optionalConfig(config map[string]interface{}, key string, envVar string) { + if value, ok := os.LookupEnv(envVar); ok { + config[key] = value + } +} diff --git a/x-pack/libbeat/management/fleet/manager.go b/x-pack/libbeat/management/fleet/manager.go index 55903480f06..9b20b17bc48 100644 --- a/x-pack/libbeat/management/fleet/manager.go +++ b/x-pack/libbeat/management/fleet/manager.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "sort" + "sync" "github.com/gofrs/uuid" "github.com/pkg/errors" @@ -35,12 +36,15 @@ type Manager struct { registry *reload.Registry blacklist *xmanagement.ConfigBlacklist client *client.Client + lock sync.Mutex + status management.Status + msg string stopFunc func() } // NewFleetManager returns a X-Pack Beats Fleet Management manager. -func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.ConfigManager, error) { +func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { @@ -51,7 +55,7 @@ func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID } // NewFleetManagerWithConfig returns a X-Pack Beats Fleet Management manager. -func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.ConfigManager, error) { +func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { log := logp.NewLogger(management.DebugK) m := &Manager{ @@ -122,15 +126,28 @@ func (cm *Manager) CheckRawConfig(cfg *common.Config) error { return nil } +// UpdateStatus updates the manager with the current status for the beat. +func (cm *Manager) UpdateStatus(status management.Status, msg string) { + cm.lock.Lock() + defer cm.lock.Unlock() + + if cm.status != status || cm.msg != msg { + cm.status = status + cm.msg = msg + cm.client.Status(statusToProtoStatus(status), msg) + cm.logger.Infof("Status change to %s: %s", status, msg) + } +} + func (cm *Manager) OnConfig(s string) { - cm.client.Status(proto.StateObserved_CONFIGURING, "Updating configuration") + cm.UpdateStatus(management.Configuring, "Updating configuration") var configMap common.MapStr uconfig, err := common.NewConfigFrom(s) if err != nil { err = errors.Wrap(err, "config blocks unsuccessfully generated") cm.logger.Error(err) - cm.client.Status(proto.StateObserved_FAILED, err.Error()) + cm.UpdateStatus(management.Failed, err.Error()) return } @@ -138,7 +155,7 @@ func (cm *Manager) OnConfig(s string) { if err != nil { err = errors.Wrap(err, "config blocks unsuccessfully generated") cm.logger.Error(err) - cm.client.Status(proto.StateObserved_FAILED, err.Error()) + cm.UpdateStatus(management.Failed, err.Error()) return } @@ -146,14 +163,14 @@ func (cm *Manager) OnConfig(s string) { if err != nil { err = errors.Wrap(err, "could not apply the configuration") cm.logger.Error(err) - cm.client.Status(proto.StateObserved_FAILED, err.Error()) + cm.UpdateStatus(management.Failed, err.Error()) return } if errs := cm.apply(blocks); !errs.IsEmpty() { err = errors.Wrap(err, "could not apply the configuration") cm.logger.Error(err) - cm.client.Status(proto.StateObserved_FAILED, err.Error()) + cm.UpdateStatus(management.Failed, err.Error()) return } @@ -285,3 +302,25 @@ func (cm *Manager) toConfigBlocks(cfg common.MapStr) (api.ConfigBlocks, error) { return res, nil } + +func statusToProtoStatus(status management.Status) proto.StateObserved_Status { + switch status { + case management.Unknown: + // unknown is reported as healthy, as the status is unknown + return proto.StateObserved_HEALTHY + case management.Starting: + return proto.StateObserved_STARTING + case management.Configuring: + return proto.StateObserved_CONFIGURING + case management.Running: + return proto.StateObserved_HEALTHY + case management.Degraded: + return proto.StateObserved_DEGRADED + case management.Failed: + return proto.StateObserved_FAILED + case management.Stopping: + return proto.StateObserved_STOPPING + } + // unknown status, still reported as healthy + return proto.StateObserved_HEALTHY +} diff --git a/x-pack/libbeat/management/fleet/manager_test.go b/x-pack/libbeat/management/fleet/manager_test.go index 7af72a04291..7810886018a 100644 --- a/x-pack/libbeat/management/fleet/manager_test.go +++ b/x-pack/libbeat/management/fleet/manager_test.go @@ -7,9 +7,13 @@ package fleet import ( "testing" - "github.com/elastic/beats/v7/libbeat/common" + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/management" ) func TestConfigBlocks(t *testing.T) { @@ -53,6 +57,16 @@ output: } } +func TestStatusToProtoStatus(t *testing.T) { + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Unknown)) + assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(management.Starting)) + assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(management.Configuring)) + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Running)) + assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(management.Degraded)) + assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(management.Failed)) + assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(management.Stopping)) +} + type dummyReloadable struct{} func (dummyReloadable) Reload(config *reload.ConfigWithMeta) error { diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index b6b75b373ac..e9dbf7511a4 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -43,7 +43,7 @@ type ConfigManager struct { } // NewConfigManager returns a X-Pack Beats Central Management manager -func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.ConfigManager, error) { +func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { @@ -54,7 +54,7 @@ func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID } // NewConfigManagerWithConfig returns a X-Pack Beats Central Management manager -func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.ConfigManager, error) { +func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { var client *api.Client var cache *Cache var blacklist *ConfigBlacklist @@ -152,6 +152,11 @@ func (cm *ConfigManager) CheckRawConfig(cfg *common.Config) error { return nil } +// UpdateStatus updates the manager with the current status for the beat. +func (cm *ConfigManager) UpdateStatus(_ management.Status, _ string) { + // do nothing; no longer under development and has been deprecated +} + func (cm *ConfigManager) worker() { defer cm.wg.Done() diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go new file mode 100644 index 00000000000..b05683b487e --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package container + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func TestFetch(t *testing.T) { + config := mtest.GetConfig(t, "container") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(60*time.Second, 1, ms) + + require.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config := mtest.GetConfig(t, "container") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(60*time.Second, 1, ms) + + require.NotEmpty(t, events) + + beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mbtest.WriteEventToDataJSON(t, beatEvent, "") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go new file mode 100644 index 00000000000..6a87ce6f951 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package counter + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func TestFetch(t *testing.T) { + config := mtest.GetConfig(t, "counter") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + + require.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config := mtest.GetConfig(t, "counter") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + + require.NotEmpty(t, events) + + beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mbtest.WriteEventToDataJSON(t, beatEvent, "") +} diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go new file mode 100644 index 00000000000..c0756c57cca --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mtest + +import ( + "testing" + + cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" +) + +func GetConfig(t *testing.T, metricset string) map[string]interface{} { + t.Helper() + + config := cftest.GetConfigFromEnv(t) + config["module"] = "cloudfoundry" + config["metricsets"] = []string{metricset} + + return config +} diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go new file mode 100644 index 00000000000..03d11bb6b7e --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package value + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func TestFetch(t *testing.T) { + config := mtest.GetConfig(t, "value") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + + require.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config := mtest.GetConfig(t, "value") + + ms := mbtest.NewPushMetricSetV2(t, config) + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + + require.NotEmpty(t, events) + + beatEvent := mbtest.StandardizeEvent(ms, events[0]) + mbtest.WriteEventToDataJSON(t, beatEvent, "") +}