diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 02adc1a06b8..2eec1486ee3 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -135,9 +135,11 @@ pipeline { 'linux/386', 'linux/arm64', 'linux/armv7', - 'linux/ppc64le', - 'linux/mips64', - 'linux/s390x', + // The platforms above are disabled temporarly as crossbuild images are + // not available. See: https://github.com/elastic/golang-crossbuild/issues/71 + //'linux/ppc64le', + //'linux/mips64', + //'linux/s390x', 'windows/amd64', 'windows/386', (params.macos ? '' : 'darwin/amd64'), @@ -344,7 +346,7 @@ def triggerE2ETests(String suite) { ] if (isPR()) { def version = "pr-${env.CHANGE_ID}" - parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true)) + parameters.push(booleanParam(name: 'ELASTIC_AGENT_USE_CI_SNAPSHOTS', value: true)) parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}")) parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}")) } diff --git a/.go-version b/.go-version index 52e779f28fa..4ed70fac17d 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.14.7 +1.14.12 diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 0756e36fdf5..edec465f854 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -57,6 +57,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162] - Metricbeat module builders call host parser only once when instantiating light modules. {pull}20149[20149] +- Fix export dashboard command when running against Elastic Cloud hosted Kibana. {pull}22746[22746] ==== Added @@ -102,4 +103,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Go version to 1.14.7. {pull}20508[20508] - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576] - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827] -- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267] \ No newline at end of file +- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 46d6a9f5b30..000eaf8415a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -215,6 +215,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed documentation for commands in beats dev guide {pull}22194[22194] - Fix parsing of expired licences. {issue}21112[21112] {pull}22180[22180] - Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438] +- Fix FileVersion contained in Windows exe files. {pull}22581[22581] +- Fix index template loading when the new index format is selected. {issue}22482[22482] {pull}22682[22682] + *Auditbeat* @@ -232,6 +235,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325] - system/socket: Fixed a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690] - file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282] +- auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673] +- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693] *Filebeat* @@ -332,6 +337,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377] +- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] *Heartbeat* @@ -341,6 +347,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Heartbeat* - The `service_name` monitor option is being replaced with `service.name` which is more correct. We will support the old option till 8.0. {pull}20330[20330] +- Fix exit on monitors with `enabled: false` {pull}22829[22829] *Journalbeat* @@ -435,6 +442,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change Session ID type from int to string {pull}22359[22359] - Fix filesystem types on Windows in filesystem metricset. {pull}22531[22531] - Fix failiures caused by custom beat names with more than 15 characters {pull}22550[22550] +- Stop generating NaN values from Cloud Foundry module to avoid errors in outputs. {pull}22634[22634] +- Update NATS dashboards to leverage connection and route metricsets {pull}22646[22646] +- Fix `logstash` module when `xpack.enabled: true` is set from emitting redundant events. {pull}22808[22808] *Packetbeat* @@ -531,6 +541,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328] - Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439] - Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521] +- Fix polling node when it is not ready and monitor by hostname {pull}22666[22666] *Auditbeat* @@ -716,12 +727,17 @@ from being added to events by default. {pull}18159[18159] - Add SSL option to checkpoint module {pull}19560[19560] - Add max_number_of_messages config into s3 input. {pull}21993[21993] - Update Okta documentation for new stateful restarts. {pull}22091[22091] +- Rename googlecloud module to gcp module. {pull}22214[22214] - Rename awscloudwatch input to aws-cloudwatch. {pull}22228[22228] - Rename google-pubsub input to gcp-pubsub. {pull}22213[22213] - Copy tag names from MISP data into events. {pull}21664[21664] - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291] - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] +- Add platform logs in the azure filebeat module. {pull}22371[22371] - Added `event.ingested` field to data from the Netflow module. {pull}22412[22412] +- Improve panw ECS url fields mapping. {pull}22481[22481] +- Improve Nats filebeat dashboard. {pull}22726[22726] +- Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699] *Heartbeat* @@ -841,8 +857,11 @@ same journal. {pull}18467[18467] - Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] - Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703] - Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325] +- Add awsfargate module task_stats metricset to monitor AWS ECS Fargate. {pull}22034[22034] - Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445] - Add unit file states to system/service {pull}22557[22557] +- Add io.ops in fields exported by system.diskio. {pull}22066[22066] +- `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732] *Packetbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 2fb10bbfa5b..e0fb587f184 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,7 +35,7 @@ pipeline { } options { timeout(time: 3, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) + buildDiscarder(logRotator(numToKeepStr: '60', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') disableResume() @@ -216,7 +216,7 @@ def generateStages(Map args = [:]) { } def cloud(Map args = [:]) { - node(args.label) { + withNode(args.label) { startCloudTestEnv(name: args.directory, dirs: args.dirs) } withCloudTestEnv() { @@ -231,7 +231,7 @@ def cloud(Map args = [:]) { def k8sTest(Map args = [:]) { def versions = args.versions versions.each{ v -> - node(args.label) { + withNode(args.label) { stage("${args.context} ${v}"){ withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ withGithubNotify(context: "${args.context} ${v}") { @@ -292,7 +292,7 @@ def runCommand(Map args = [:]) { def directory = args.get('directory', '') def withModule = args.get('withModule', false) def isMage = args.get('isMage', false) - node(args.label) { + withNode(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { dumpVariables() @@ -306,6 +306,16 @@ def runCommand(Map args = [:]) { } } +/** +* This method wraps the node call with some latency to avoid the known issue with the scalabitity in gobld. +*/ +def withNode(String label, Closure body) { + sleep randomNumber(min: 10, max: 200) + node(label) { + body() + } +} + /** * This method wraps all the environment setup and pre-requirements to run any commands. */ diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 0db572a2d1f..c6d4c0c0735 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/auditbeat/module/auditd/audit_linux.go b/auditbeat/module/auditd/audit_linux.go index 1586eaeaffa..a2c9e004877 100644 --- a/auditbeat/module/auditd/audit_linux.go +++ b/auditbeat/module/auditd/audit_linux.go @@ -163,7 +163,11 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) { ms.log.Errorw("Failure creating audit monitoring client", "error", err) } go func() { - defer client.Close() + defer func() { // Close the most recently allocated "client" instance. + if client != nil { + client.Close() + } + }() timer := time.NewTicker(lostEventsUpdateInterval) defer timer.Stop() for { @@ -175,6 +179,15 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) { ms.updateKernelLostMetric(status.Lost) } else { ms.log.Error("get status request failed:", err) + if err = client.Close(); err != nil { + ms.log.Errorw("Error closing audit monitoring client", "error", err) + } + client, err = libaudit.NewAuditClient(nil) + if err != nil { + ms.log.Errorw("Failure creating audit monitoring client", "error", err) + reporter.Error(err) + return + } } } } diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index e4b93625b75..dd0a2fd56c2 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -782,7 +782,7 @@ func binaryExtension(goos string) string { return "" } -var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P\d)+\.(?P\d)+(?:\.(?P\d)+.*)?$`) +var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P\d+)\.(?P\d+)(?:\.(?P\d+).*)?$`) // ParseVersion extracts the major, minor, and optional patch number from a // version string. diff --git a/dev-tools/mage/common_test.go b/dev-tools/mage/common_test.go index e4ce81505ad..8b6ac8bd732 100644 --- a/dev-tools/mage/common_test.go +++ b/dev-tools/mage/common_test.go @@ -33,6 +33,8 @@ func TestParseVersion(t *testing.T) { {"1.2.3-SNAPSHOT", 1, 2, 3}, {"1.2.3rc1", 1, 2, 3}, {"1.2", 1, 2, 0}, + {"7.10.0", 7, 10, 0}, + {"10.01.22", 10, 1, 22}, } for _, tc := range tests { diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index 7a25b9906af..d9707991a5f 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 18968522656..5c6244588fd 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -37,7 +37,7 @@ grouped in the following categories: * <> * <> * <> -* <> +* <> * <> * <> * <> @@ -3055,6 +3055,153 @@ type: keyword -- +[float] +=== platformlogs + +Fields for Azure platform logs. + + + +*`azure.platformlogs.operation_name`*:: ++ +-- +Operation name + + +type: keyword + +-- + +*`azure.platformlogs.result_type`*:: ++ +-- +Result type + + +type: keyword + +-- + +*`azure.platformlogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.platformlogs.category`*:: ++ +-- +Category + + +type: keyword + +-- + +*`azure.platformlogs.event_category`*:: ++ +-- +Event Category + + +type: keyword + +-- + +*`azure.platformlogs.status`*:: ++ +-- +Status + + +type: keyword + +-- + +*`azure.platformlogs.ccpNamespace`*:: ++ +-- +ccpNamespace + + +type: keyword + +-- + +*`azure.platformlogs.Cloud`*:: ++ +-- +Cloud + + +type: keyword + +-- + +*`azure.platformlogs.Environment`*:: ++ +-- +Environment + + +type: keyword + +-- + +*`azure.platformlogs.EventTimeString`*:: ++ +-- +EventTimeString + + +type: keyword + +-- + +*`azure.platformlogs.Caller`*:: ++ +-- +Caller + + +type: keyword + +-- + +*`azure.platformlogs.ScaleUnit`*:: ++ +-- +ScaleUnit + + +type: keyword + +-- + +*`azure.platformlogs.ActivityId`*:: ++ +-- +ActivityId + + +type: keyword + +-- + +*`azure.platformlogs.properties.*`*:: ++ +-- +Properties + + +type: object + +-- + [float] === signinlogs @@ -68452,8 +68599,8 @@ type: integer -- -[[exported-fields-googlecloud]] -== Google Cloud fields +[[exported-fields-gcp]] +== Google Cloud Platform (GCP) fields Module for handling logs from Google Cloud. diff --git a/filebeat/docs/howto/howto.asciidoc b/filebeat/docs/howto/howto.asciidoc index 14675aae3ca..bd4774940b2 100644 --- a/filebeat/docs/howto/howto.asciidoc +++ b/filebeat/docs/howto/howto.asciidoc @@ -5,7 +5,7 @@ -- Learn how to perform common {beatname_uc} configuration tasks. -* < +* <> * <<{beatname_lc}-template>> * <> * <> diff --git a/filebeat/docs/images/filebeat-googlecloud-audit.png b/filebeat/docs/images/filebeat-gcp-audit.png similarity index 100% rename from filebeat/docs/images/filebeat-googlecloud-audit.png rename to filebeat/docs/images/filebeat-gcp-audit.png diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index 30e0ec38f46..69633f6836d 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -64,4 +64,6 @@ include::./faq.asciidoc[] include::{libbeat-dir}/contributing-to-beats.asciidoc[] +include::redirects.asciidoc[] + diff --git a/filebeat/docs/inputs/input-common-unix-options.asciidoc b/filebeat/docs/inputs/input-common-unix-options.asciidoc index f73278944a6..9f97d84017e 100644 --- a/filebeat/docs/inputs/input-common-unix-options.asciidoc +++ b/filebeat/docs/inputs/input-common-unix-options.asciidoc @@ -14,7 +14,14 @@ The maximum size of the message received over the socket. The default is `20MiB` [id="{beatname_lc}-input-{type}-unix-path"] ==== `path` -The path to the Unix socket that will receive event streams. +The path to the Unix socket that will receive events. + +[float] +[id="{beatname_lc}-input-{type}-unix-socket-type"] +==== `socket_type` + +The type to of the Unix socket that will receive events. Valid values +are `stream` and `datagram`. The default is `stream`. [float] [id="{beatname_lc}-input-{type}-unix-group"] diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 45010618214..af8c2413185 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -24,6 +24,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -46,6 +49,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/filebeat/docs/modules/googlecloud.asciidoc b/filebeat/docs/modules/gcp.asciidoc similarity index 85% rename from filebeat/docs/modules/googlecloud.asciidoc rename to filebeat/docs/modules/gcp.asciidoc index bc0e62e93b8..ee700d81281 100644 --- a/filebeat/docs/modules/googlecloud.asciidoc +++ b/filebeat/docs/modules/gcp.asciidoc @@ -2,10 +2,10 @@ This file is generated! See scripts/docs_collector.py //// -[[filebeat-module-googlecloud]] +[[filebeat-module-gcp]] [role="xpack"] -:modulename: googlecloud +:modulename: gcp :has-dashboards: false == Google Cloud module @@ -29,18 +29,18 @@ include::../include/config-option-intro.asciidoc[] ==== `audit` fileset settings [role="screenshot"] -image::./images/filebeat-googlecloud-audit.png[] +image::./images/filebeat-gcp-audit.png[] Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp audit: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-audit - var.subscription_name: filebeat-googlecloud-audit-sub + var.topic: gcp-vpc-audit + var.subscription_name: filebeat-gcp-audit-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- @@ -80,12 +80,12 @@ Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp vpcflow: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-flowlogs - var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + var.topic: gcp-vpc-flowlogs + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- @@ -125,12 +125,12 @@ Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp firewall: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-firewall - var.subscription_name: filebeat-googlecloud-vpc-firewall-sub + var.topic: gcp-vpc-firewall + var.subscription_name: filebeat-gcp-vpc-firewall-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- @@ -170,5 +170,5 @@ field. Defaults to `false`, meaning the original message is not saved. === Fields For a description of each field in the module, see the -<> section. +<> section. diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 6c862dc2c77..d3a02fee862 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -22,7 +22,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> - * <> + * <> * <> * <> * <> @@ -91,7 +91,7 @@ include::modules/elasticsearch.asciidoc[] include::modules/envoyproxy.asciidoc[] include::modules/f5.asciidoc[] include::modules/fortinet.asciidoc[] -include::modules/googlecloud.asciidoc[] +include::modules/gcp.asciidoc[] include::modules/gsuite.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/ibmmq.asciidoc[] diff --git a/filebeat/docs/redirects.asciidoc b/filebeat/docs/redirects.asciidoc new file mode 100644 index 00000000000..7a41406099b --- /dev/null +++ b/filebeat/docs/redirects.asciidoc @@ -0,0 +1,10 @@ +["appendix",role="exclude",id="redirects"] += Deleted pages + +The following pages have moved or been deleted. + +[role="exclude",id="filebeat-module-googlecloud"] +== Google Cloud module + +See <>. + diff --git a/filebeat/input/syslog/config.go b/filebeat/input/syslog/config.go index ff009bfb1dd..ff97abd8e14 100644 --- a/filebeat/input/syslog/config.go +++ b/filebeat/input/syslog/config.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/filebeat/harvester" "github.com/elastic/beats/v7/filebeat/inputsource" - netcommon "github.com/elastic/beats/v7/filebeat/inputsource/common" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/filebeat/inputsource/tcp" "github.com/elastic/beats/v7/filebeat/inputsource/udp" "github.com/elastic/beats/v7/filebeat/inputsource/unix" @@ -59,16 +59,17 @@ var defaultTCP = syslogTCP{ } type syslogUnix struct { - unix.Config `config:",inline"` - LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + unix.Config `config:",inline"` } -var defaultUnix = syslogUnix{ - Config: unix.Config{ - Timeout: time.Minute * 5, - MaxMessageSize: 20 * humanize.MiByte, - }, - LineDelimiter: "\n", +func defaultUnix() syslogUnix { + return syslogUnix{ + Config: unix.Config{ + Timeout: time.Minute * 5, + MaxMessageSize: 20 * humanize.MiByte, + LineDelimiter: "\n", + }, + } } var defaultUDP = udp.Config{ @@ -89,32 +90,26 @@ func factory( return nil, err } - splitFunc := netcommon.SplitFunc([]byte(config.LineDelimiter)) + splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) if splitFunc == nil { return nil, fmt.Errorf("error creating splitFunc from delimiter %s", config.LineDelimiter) } logger := logp.NewLogger("input.syslog.tcp").With("address", config.Config.Host) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyTCP, logger, tcp.MetadataCallback, nf, splitFunc) + factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logger, tcp.MetadataCallback, nf, splitFunc) return tcp.New(&config.Config, factory) case unix.Name: cfgwarn.Beta("Syslog Unix socket support is beta.") - config := defaultUnix + config := defaultUnix() if err := cfg.Unpack(&config); err != nil { return nil, err } - splitFunc := netcommon.SplitFunc([]byte(config.LineDelimiter)) - if splitFunc == nil { - return nil, fmt.Errorf("error creating splitFunc from delimiter %s", config.LineDelimiter) - } - logger := logp.NewLogger("input.syslog.unix").With("path", config.Config.Path) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logger, unix.MetadataCallback, nf, splitFunc) - return unix.New(&config.Config, factory) + return unix.New(logger, &config.Config, nf) case udp.Name: config := defaultUDP diff --git a/filebeat/input/tcp/input.go b/filebeat/input/tcp/input.go index 96c6ce99022..598650d2a9c 100644 --- a/filebeat/input/tcp/input.go +++ b/filebeat/input/tcp/input.go @@ -26,7 +26,7 @@ import ( "github.com/elastic/beats/v7/filebeat/harvester" "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/inputsource" - netcommon "github.com/elastic/beats/v7/filebeat/inputsource/common" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/filebeat/inputsource/tcp" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -75,13 +75,13 @@ func NewInput( forwarder.Send(event) } - splitFunc := netcommon.SplitFunc([]byte(config.LineDelimiter)) + splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) if splitFunc == nil { return nil, fmt.Errorf("unable to create splitFunc for delimiter %s", config.LineDelimiter) } logger := logp.NewLogger("input.tcp").With("address", config.Config.Host) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyTCP, logger, tcp.MetadataCallback, cb, splitFunc) + factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logger, tcp.MetadataCallback, cb, splitFunc) server, err := tcp.New(&config.Config, factory) if err != nil { diff --git a/filebeat/input/unix/config.go b/filebeat/input/unix/config.go index 4d4400cb974..e1a24ff3f17 100644 --- a/filebeat/input/unix/config.go +++ b/filebeat/input/unix/config.go @@ -26,8 +26,7 @@ import ( ) type config struct { - unix.Config `config:",inline"` - LineDelimiter string `config:"line_delimiter" validate:"nonzero"` + unix.Config `config:",inline"` } func defaultConfig() config { @@ -35,7 +34,8 @@ func defaultConfig() config { Config: unix.Config{ Timeout: time.Minute * 5, MaxMessageSize: 20 * humanize.MiByte, + SocketType: unix.StreamSocket, + LineDelimiter: "\n", }, - LineDelimiter: "\n", } } diff --git a/filebeat/input/unix/input.go b/filebeat/input/unix/input.go index 3f5be8c8b87..7346521c6c8 100644 --- a/filebeat/input/unix/input.go +++ b/filebeat/input/unix/input.go @@ -18,15 +18,12 @@ package unix import ( - "bufio" - "fmt" "net" "time" input "github.com/elastic/beats/v7/filebeat/input/v2" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/filebeat/inputsource" - netcommon "github.com/elastic/beats/v7/filebeat/inputsource/common" "github.com/elastic/beats/v7/filebeat/inputsource/unix" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -35,8 +32,8 @@ import ( ) type server struct { + unix.Server config - splitFunc bufio.SplitFunc } func Plugin() input.Plugin { @@ -59,12 +56,7 @@ func configure(cfg *common.Config) (stateless.Input, error) { } func newServer(config config) (*server, error) { - splitFunc := netcommon.SplitFunc([]byte(config.LineDelimiter)) - if splitFunc == nil { - return nil, fmt.Errorf("unable to create splitFunc for delimiter %s", config.LineDelimiter) - } - - return &server{config: config, splitFunc: splitFunc}, nil + return &server{config: config}, nil } func (s *server) Name() string { return "unix" } @@ -83,17 +75,17 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { log.Info("Starting Unix socket input") defer log.Info("Unix socket input stopped") - cb := func(data []byte, metadata inputsource.NetworkMetadata) { + cb := inputsource.NetworkFunc(func(data []byte, metadata inputsource.NetworkMetadata) { event := createEvent(data, metadata) publisher.Publish(event) - } - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, log, unix.MetadataCallback, cb, s.splitFunc) - server, err := unix.New(&s.config.Config, factory) + }) + + server, err := unix.New(log, &s.config.Config, cb) if err != nil { return err } - log.Debugf("TCP Input '%v' initialized", ctx.ID) + log.Debugf("%s Input '%v' initialized", s.config.Config.SocketType, ctx.ID) err = server.Run(ctxtool.FromCanceller(ctx.Cancelation)) diff --git a/filebeat/inputsource/common/dgram/handler.go b/filebeat/inputsource/common/dgram/handler.go new file mode 100644 index 00000000000..a9fa50d8f19 --- /dev/null +++ b/filebeat/inputsource/common/dgram/handler.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dgram + +import ( + "context" + "fmt" + "net" + "runtime" + "strings" + + "github.com/elastic/beats/v7/filebeat/inputsource" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// HandlerFactory returns a ConnectionHandler func +type HandlerFactory func(config ListenerConfig) ConnectionHandler + +// ConnectionHandler is able to read from incoming connections. +type ConnectionHandler func(context.Context, net.PacketConn) error + +// MetadataFunc defines callback executed when a line is read from the split handler. +type MetadataFunc func(net.Conn) inputsource.NetworkMetadata + +// DatagramReaderFactory allows creation of a handler which can read packets from connections. +func DatagramReaderFactory( + family inputsource.Family, + logger *logp.Logger, + callback inputsource.NetworkFunc, +) HandlerFactory { + return func(config ListenerConfig) ConnectionHandler { + return ConnectionHandler(func(ctx context.Context, conn net.PacketConn) error { + for ctx.Err() == nil { + + buffer := make([]byte, config.MaxMessageSize) + //conn.SetDeadline(time.Now().Add(config.Timeout)) + + // If you are using Windows and you are using a fixed buffer and you get a datagram which + // is bigger than the specified size of the buffer, it will return an `err` and the buffer will + // contains a subset of the data. + // + // On Unix based system, the buffer will be truncated but no error will be returned. + length, addr, err := conn.ReadFrom(buffer) + if err != nil { + if family == inputsource.FamilyUnix { + fmt.Println("connection handler error", err) + } + // don't log any deadline events. + e, ok := err.(net.Error) + if ok && e.Timeout() { + continue + } + + // Closed network error string will never change in Go 1.X + // https://github.com/golang/go/issues/4373 + opErr, ok := err.(*net.OpError) + if ok && strings.Contains(opErr.Err.Error(), "use of closed network connection") { + logger.Info("Connection has been closed") + return nil + } + + logger.Errorf("Error reading from the socket %s", err) + + // On Windows send the current buffer and mark it as truncated. + // The buffer will have content but length will return 0, addr will be nil. + if family == inputsource.FamilyUDP && isLargerThanBuffer(err) { + callback(buffer, inputsource.NetworkMetadata{RemoteAddr: addr, Truncated: true}) + continue + } + } + + if length > 0 { + callback(buffer[:length], inputsource.NetworkMetadata{RemoteAddr: addr}) + } + } + fmt.Println("end of connection handling") + return nil + }) + } +} + +func isLargerThanBuffer(err error) bool { + if runtime.GOOS != "windows" { + return false + } + return strings.Contains(err.Error(), windowErrBuffer) +} diff --git a/filebeat/inputsource/common/dgram/server.go b/filebeat/inputsource/common/dgram/server.go new file mode 100644 index 00000000000..ecb4844ed67 --- /dev/null +++ b/filebeat/inputsource/common/dgram/server.go @@ -0,0 +1,131 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dgram + +import ( + "context" + "net" + "time" + + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" + + "github.com/elastic/beats/v7/filebeat/inputsource" + "github.com/elastic/beats/v7/libbeat/common/cfgtype" + "github.com/elastic/beats/v7/libbeat/logp" +) + +const windowErrBuffer = "A message sent on a datagram socket was larger than the internal message" + + " buffer or some other network limit, or the buffer used to receive a datagram into was smaller" + + " than the datagram itself." + +// ListenerFactory is used to craete connections based on the configuration. +type ListenerFactory func() (net.PacketConn, error) + +type ListenerConfig struct { + Timeout time.Duration + MaxMessageSize cfgtype.ByteSize +} + +type Listener struct { + log *logp.Logger + family inputsource.Family + config *ListenerConfig + listener ListenerFactory + connect HandlerFactory + tg unison.TaskGroup +} + +func NewListener( + f inputsource.Family, + path string, + connect HandlerFactory, + listenerFactory ListenerFactory, + config *ListenerConfig, +) *Listener { + return &Listener{ + log: logp.NewLogger(f.String()), + family: f, + config: config, + listener: listenerFactory, + connect: connect, + tg: unison.TaskGroup{}, + } +} + +func (l *Listener) Run(ctx context.Context) error { + l.log.Info("Started listening for " + l.family.String() + " connection") + + for ctx.Err() == nil { + conn, err := l.listener() + if err != nil { + l.log.Debugw("Cannot connect", "error", err) + continue + } + connCtx, connCancel := ctxtool.WithFunc(ctx, func() { + conn.Close() + }) + + err = l.run(connCtx, conn) + if err != nil { + l.log.Debugw("Error while processing input", "error", err) + connCancel() + continue + } + connCancel() + } + return nil +} + +func (l *Listener) Start() error { + l.log.Info("Started listening for " + l.family.String() + " connection") + + conn, err := l.listener() + if err != nil { + return err + } + + l.tg.Go(func(ctx unison.Canceler) error { + connCtx, connCancel := ctxtool.WithFunc(ctxtool.FromCanceller(ctx), func() { + conn.Close() + }) + defer connCancel() + + return l.run(ctxtool.FromCanceller(connCtx), conn) + }) + return nil +} + +func (l *Listener) run(ctx context.Context, conn net.PacketConn) error { + handler := l.connect(*l.config) + for ctx.Err() == nil { + err := handler(ctx, conn) + if err != nil { + return err + } + } + return nil +} + +func (l *Listener) Stop() { + l.log.Debug("Stopping datagram socket server for " + l.family.String()) + err := l.tg.Stop() + if err != nil { + l.log.Errorf("Error while stopping datagram socket server: %v", err) + } +} diff --git a/filebeat/inputsource/common/config.go b/filebeat/inputsource/common/streaming/config.go similarity index 98% rename from filebeat/inputsource/common/config.go rename to filebeat/inputsource/common/streaming/config.go index 2ae5bf52e35..5f48ce003cd 100644 --- a/filebeat/inputsource/common/config.go +++ b/filebeat/inputsource/common/streaming/config.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "time" diff --git a/filebeat/inputsource/common/conn.go b/filebeat/inputsource/common/streaming/conn.go similarity index 99% rename from filebeat/inputsource/common/conn.go rename to filebeat/inputsource/common/streaming/conn.go index c6cf86d2292..e0c2b3e9038 100644 --- a/filebeat/inputsource/common/conn.go +++ b/filebeat/inputsource/common/streaming/conn.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "io" diff --git a/filebeat/inputsource/common/conn_test.go b/filebeat/inputsource/common/streaming/conn_test.go similarity index 99% rename from filebeat/inputsource/common/conn_test.go rename to filebeat/inputsource/common/streaming/conn_test.go index f5e41a58c63..6e93e7a0653 100644 --- a/filebeat/inputsource/common/conn_test.go +++ b/filebeat/inputsource/common/streaming/conn_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "math/rand" diff --git a/filebeat/inputsource/common/handler.go b/filebeat/inputsource/common/streaming/handler.go similarity index 92% rename from filebeat/inputsource/common/handler.go rename to filebeat/inputsource/common/streaming/handler.go index a55ee1755d5..69ae5aedc9c 100644 --- a/filebeat/inputsource/common/handler.go +++ b/filebeat/inputsource/common/streaming/handler.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "bufio" @@ -38,14 +38,14 @@ type ConnectionHandler func(context.Context, net.Conn) error type MetadataFunc func(net.Conn) inputsource.NetworkMetadata // SplitHandlerFactory allows creation of a handler that has splitting capabilities. -func SplitHandlerFactory(family Family, logger *logp.Logger, metadataCallback MetadataFunc, callback inputsource.NetworkFunc, splitFunc bufio.SplitFunc) HandlerFactory { +func SplitHandlerFactory(family inputsource.Family, logger *logp.Logger, metadataCallback MetadataFunc, callback inputsource.NetworkFunc, splitFunc bufio.SplitFunc) HandlerFactory { return func(config ListenerConfig) ConnectionHandler { return ConnectionHandler(func(ctx context.Context, conn net.Conn) error { metadata := metadataCallback(conn) maxMessageSize := uint64(config.MaxMessageSize) var log *logp.Logger - if family == FamilyUnix { + if family == inputsource.FamilyUnix { // unix sockets have an empty `RemoteAddr` value, so no need to capture it log = logger.With("handler", "split_client") } else { diff --git a/filebeat/inputsource/common/listener.go b/filebeat/inputsource/common/streaming/listener.go similarity index 89% rename from filebeat/inputsource/common/listener.go rename to filebeat/inputsource/common/streaming/listener.go index f4890ccc767..cff66b28b6c 100644 --- a/filebeat/inputsource/common/listener.go +++ b/filebeat/inputsource/common/streaming/listener.go @@ -15,35 +15,21 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "bufio" "bytes" "context" "net" - "strings" "sync" + "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/go-concert/ctxtool" ) -// Family represents the type of connection we're handling -type Family string - -const ( - // FamilyUnix represents a unix socket listener - FamilyUnix Family = "unix" - // FamilyTCP represents a tcp socket listener - FamilyTCP Family = "tcp" -) - -func (f Family) String() string { - return strings.ToUpper(string(f)) -} - // ListenerFactory returns a net.Listener type ListenerFactory func() (net.Listener, error) @@ -51,7 +37,7 @@ type ListenerFactory func() (net.Listener, error) type Listener struct { Listener net.Listener config *ListenerConfig - family Family + family inputsource.Family wg sync.WaitGroup log *logp.Logger ctx context.Context @@ -62,7 +48,7 @@ type Listener struct { } // NewListener creates a new Listener -func NewListener(family Family, location string, handlerFactory HandlerFactory, listenerFactory ListenerFactory, config *ListenerConfig) *Listener { +func NewListener(family inputsource.Family, location string, handlerFactory HandlerFactory, listenerFactory ListenerFactory, config *ListenerConfig) *Listener { return &Listener{ config: config, family: family, @@ -141,7 +127,7 @@ func (l *Listener) run() { l.registerHandler() defer l.unregisterHandler() - if l.family == FamilyUnix { + if l.family == inputsource.FamilyUnix { // unix sockets have an empty `RemoteAddr` value, so no need to capture it l.log.Debugw("New client", "total", l.clientsCount.Load()) } else { @@ -155,7 +141,7 @@ func (l *Listener) run() { } defer func() { - if l.family == FamilyUnix { + if l.family == inputsource.FamilyUnix { // unix sockets have an empty `RemoteAddr` value, so no need to capture it l.log.Debugw("client disconnected", "total", l.clientsCount.Load()) } else { @@ -184,6 +170,10 @@ func (l *Listener) unregisterHandler() { // SplitFunc allows to create a `bufio.SplitFunc` based on a delimiter provided. func SplitFunc(lineDelimiter []byte) bufio.SplitFunc { + if len(lineDelimiter) == 0 { + return nil + } + ld := []byte(lineDelimiter) if bytes.Equal(ld, []byte("\n")) { // This will work for most usecases and will also strip \r if present. diff --git a/filebeat/inputsource/common/scan.go b/filebeat/inputsource/common/streaming/scan.go similarity index 98% rename from filebeat/inputsource/common/scan.go rename to filebeat/inputsource/common/streaming/scan.go index 63653006238..de2b342049a 100644 --- a/filebeat/inputsource/common/scan.go +++ b/filebeat/inputsource/common/streaming/scan.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "bufio" diff --git a/filebeat/inputsource/common/scan_test.go b/filebeat/inputsource/common/streaming/scan_test.go similarity index 99% rename from filebeat/inputsource/common/scan_test.go rename to filebeat/inputsource/common/streaming/scan_test.go index 5e266141193..eccd2b66300 100644 --- a/filebeat/inputsource/common/scan_test.go +++ b/filebeat/inputsource/common/streaming/scan_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package common +package streaming import ( "bufio" diff --git a/filebeat/inputsource/family.go b/filebeat/inputsource/family.go new file mode 100644 index 00000000000..025cb59ff08 --- /dev/null +++ b/filebeat/inputsource/family.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inputsource + +import "strings" + +// Family represents the type of connection we're handling +type Family string + +const ( + // FamilyUnix represents a unix socket listener + FamilyUnix Family = "unix" + // FamilyTCP represents a tcp socket listener + FamilyTCP Family = "tcp" + // FamilyUDP represents a udp socket listener + FamilyUDP Family = "udp" +) + +func (f Family) String() string { + return strings.ToUpper(string(f)) +} diff --git a/filebeat/inputsource/tcp/server.go b/filebeat/inputsource/tcp/server.go index eaf83c8526b..270ebc9c0c5 100644 --- a/filebeat/inputsource/tcp/server.go +++ b/filebeat/inputsource/tcp/server.go @@ -24,13 +24,14 @@ import ( "golang.org/x/net/netutil" - "github.com/elastic/beats/v7/filebeat/inputsource/common" + "github.com/elastic/beats/v7/filebeat/inputsource" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) // Server represent a TCP server type Server struct { - *common.Listener + *streaming.Listener config *Config tlsConfig *tlscommon.TLSConfig @@ -39,7 +40,7 @@ type Server struct { // New creates a new tcp server func New( config *Config, - factory common.HandlerFactory, + factory streaming.HandlerFactory, ) (*Server, error) { tlsConfig, err := tlscommon.LoadTLSServerConfig(config.TLS) if err != nil { @@ -54,7 +55,7 @@ func New( config: config, tlsConfig: tlsConfig, } - server.Listener = common.NewListener(common.FamilyTCP, config.Host, factory, server.createServer, &common.ListenerConfig{ + server.Listener = streaming.NewListener(inputsource.FamilyTCP, config.Host, factory, server.createServer, &streaming.ListenerConfig{ Timeout: config.Timeout, MaxMessageSize: config.MaxMessageSize, MaxConnections: config.MaxConnections, diff --git a/filebeat/inputsource/tcp/server_test.go b/filebeat/inputsource/tcp/server_test.go index 032f7d33e29..2d05ed77d57 100644 --- a/filebeat/inputsource/tcp/server_test.go +++ b/filebeat/inputsource/tcp/server_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/filebeat/inputsource" - netcommon "github.com/elastic/beats/v7/filebeat/inputsource/common" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -69,76 +69,76 @@ func TestReceiveEventsAndMetadata(t *testing.T) { { name: "NewLine", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\n"), }, { name: "NewLineWithCR", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\r\n")), + splitFunc: streaming.SplitFunc([]byte("\r\n")), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\r\n"), }, { name: "CustomDelimiter", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + splitFunc: streaming.SplitFunc([]byte(";")), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ";"), }, { name: "MultipleCharsCustomDelimiter", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("")), + splitFunc: streaming.SplitFunc([]byte("")), expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ""), }, { name: "SingleCharCustomDelimiterMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + splitFunc: streaming.SplitFunc([]byte(";")), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "MultipleCharCustomDelimiterMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("")), + splitFunc: streaming.SplitFunc([]byte("")), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineMessageWithoutBoundaries", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineLargeMessagePayload", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, "\n"), }, { name: "CustomLargeMessagePayload", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + splitFunc: streaming.SplitFunc([]byte(";")), expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, ";"), }, { name: "ReadRandomLargePayload", cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), expectedMessages: []string{randomGeneratedText}, messageSent: randomGeneratedText, }, { name: "MaxReadBufferReachedUserConfigured", - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), cfg: map[string]interface{}{ "max_message_size": 50000, }, @@ -147,7 +147,7 @@ func TestReceiveEventsAndMetadata(t *testing.T) { }, { name: "MaxBufferSizeSet", - splitFunc: netcommon.SplitFunc([]byte("\n")), + splitFunc: streaming.SplitFunc([]byte("\n")), cfg: map[string]interface{}{ "max_message_size": 66 * 1024, }, @@ -171,7 +171,7 @@ func TestReceiveEventsAndMetadata(t *testing.T) { return } - factory := netcommon.SplitHandlerFactory(netcommon.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, test.splitFunc) + factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, test.splitFunc) server, err := New(&config, factory) if !assert.NoError(t, err) { return @@ -223,7 +223,7 @@ func TestReceiveNewEventsConcurrently(t *testing.T) { return } - factory := netcommon.SplitHandlerFactory(netcommon.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, bufio.ScanLines) + factory := streaming.SplitHandlerFactory(inputsource.FamilyTCP, logp.NewLogger("test"), MetadataCallback, to, bufio.ScanLines) server, err := New(&config, factory) if !assert.NoError(t, err) { diff --git a/filebeat/inputsource/udp/server.go b/filebeat/inputsource/udp/server.go index 9df2e57904a..123c54ba67f 100644 --- a/filebeat/inputsource/udp/server.go +++ b/filebeat/inputsource/udp/server.go @@ -19,134 +19,55 @@ package udp import ( "net" - "runtime" - "strings" - "sync" - "time" "github.com/dustin/go-humanize" "github.com/elastic/beats/v7/filebeat/inputsource" + "github.com/elastic/beats/v7/filebeat/inputsource/common/dgram" "github.com/elastic/beats/v7/libbeat/logp" ) // Name is the human readable name and identifier. const Name = "udp" -const windowErrBuffer = "A message sent on a datagram socket was larger than the internal message" + - " buffer or some other network limit, or the buffer used to receive a datagram into was smaller" + - " than the datagram itself." - // Server creates a simple UDP Server and listen to a specific host:port and will send any // event received to the callback method. type Server struct { - config *Config - callback inputsource.NetworkFunc - Listener *net.UDPConn - log *logp.Logger - wg sync.WaitGroup - done chan struct{} + *dgram.Listener + config *Config + + localaddress string } // New returns a new UDPServer instance. func New(config *Config, callback inputsource.NetworkFunc) *Server { - return &Server{ - config: config, - callback: callback, - log: logp.NewLogger("udp").With("address", config.Host), - done: make(chan struct{}), - } + server := &Server{config: config} + log := logp.NewLogger("udp").With("address", config.Host) + factory := dgram.DatagramReaderFactory(inputsource.FamilyUDP, log, callback) + server.Listener = dgram.NewListener(inputsource.FamilyUDP, config.Host, factory, server.createConn, &dgram.ListenerConfig{ + Timeout: config.Timeout, + MaxMessageSize: config.MaxMessageSize, + }) + return server } -// Start starts the UDP Server and listen to incoming events. -func (u *Server) Start() error { +func (u *Server) createConn() (net.PacketConn, error) { var err error udpAdddr, err := net.ResolveUDPAddr("udp", u.config.Host) if err != nil { - return err + return nil, err } - u.Listener, err = net.ListenUDP("udp", udpAdddr) + listener, err := net.ListenUDP("udp", udpAdddr) if err != nil { - return err + return nil, err } socketSize := int(u.config.ReadBuffer) * humanize.KiByte if socketSize != 0 { - if err := u.Listener.SetReadBuffer(int(u.config.ReadBuffer)); err != nil { - return err + if err := listener.SetReadBuffer(int(u.config.ReadBuffer)); err != nil { + return nil, err } } - if err != nil { - return err - } - u.log.Info("Started listening for UDP connection") - u.wg.Add(1) - go func() { - defer u.wg.Done() - u.run() - }() - return nil -} - -func (u *Server) run() { - for { - select { - case <-u.done: - return - default: - } - - buffer := make([]byte, u.config.MaxMessageSize) - u.Listener.SetDeadline(time.Now().Add(u.config.Timeout)) - - // If you are using Windows and you are using a fixed buffer and you get a datagram which - // is bigger than the specified size of the buffer, it will return an `err` and the buffer will - // contains a subset of the data. - // - // On Unix based system, the buffer will be truncated but no error will be returned. - length, addr, err := u.Listener.ReadFrom(buffer) - if err != nil { - // don't log any deadline events. - e, ok := err.(net.Error) - if ok && e.Timeout() { - continue - } - - // Closed network error string will never change in Go 1.X - // https://github.com/golang/go/issues/4373 - opErr, ok := err.(*net.OpError) - if ok && strings.Contains(opErr.Err.Error(), "use of closed network connection") { - u.log.Info("Connection has been closed") - return - } + u.localaddress = listener.LocalAddr().String() - u.log.Errorf("Error reading from the socket %s", err) - - // On Windows send the current buffer and mark it as truncated. - // The buffer will have content but length will return 0, addr will be nil. - if isLargerThanBuffer(err) { - u.callback(buffer, inputsource.NetworkMetadata{RemoteAddr: addr, Truncated: true}) - continue - } - } - - if length > 0 { - u.callback(buffer[:length], inputsource.NetworkMetadata{RemoteAddr: addr}) - } - } -} - -// Stop stops the current udp server. -func (u *Server) Stop() { - u.log.Info("Stopping UDP server") - close(u.done) - u.Listener.Close() - u.wg.Wait() - u.log.Info("UDP server stopped") -} - -func isLargerThanBuffer(err error) bool { - if runtime.GOOS != "windows" { - return false - } - return strings.Contains(err.Error(), windowErrBuffer) + return listener, err } diff --git a/filebeat/inputsource/udp/server_test.go b/filebeat/inputsource/udp/server_test.go index 766f7d0b7aa..0434c836ae6 100644 --- a/filebeat/inputsource/udp/server_test.go +++ b/filebeat/inputsource/udp/server_test.go @@ -75,7 +75,7 @@ func TestReceiveEventFromUDP(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - conn, err := net.Dial("udp", s.Listener.LocalAddr().String()) + conn, err := net.Dial("udp", s.localaddress) if !assert.NoError(t, err) { return } diff --git a/filebeat/inputsource/unix/config.go b/filebeat/inputsource/unix/config.go index 5051ab86e75..f6a2e4fa83a 100644 --- a/filebeat/inputsource/unix/config.go +++ b/filebeat/inputsource/unix/config.go @@ -24,8 +24,24 @@ import ( "github.com/elastic/beats/v7/libbeat/common/cfgtype" ) -// Name is the human readable name and identifier. -const Name = "unix" +type SocketType uint8 + +const ( + // StreamSocket is used when reading from a Unix stream socket. + StreamSocket SocketType = iota + // DatagramSocket is used when reading from a Unix datagram socket. + DatagramSocket +) + +const ( + // Name is the human readable name and identifier. + Name = "unix" +) + +var socketTypes = map[string]SocketType{ + "stream": StreamSocket, + "datagram": DatagramSocket, +} // Config exposes the unix configuration. type Config struct { @@ -35,6 +51,8 @@ type Config struct { Timeout time.Duration `config:"timeout" validate:"nonzero,positive"` MaxMessageSize cfgtype.ByteSize `config:"max_message_size" validate:"nonzero,positive"` MaxConnections int `config:"max_connections"` + LineDelimiter string `config:"line_delimiter"` + SocketType SocketType `config:"socket_type"` } // Validate validates the Config option for the unix input. @@ -42,5 +60,26 @@ func (c *Config) Validate() error { if len(c.Path) == 0 { return fmt.Errorf("need to specify the path to the unix socket") } + + if c.SocketType == StreamSocket && c.LineDelimiter == "" { + return fmt.Errorf("line_delimiter cannot be empty when using stream socket") + } + return nil + +} + +func (s *SocketType) Unpack(value string) error { + setting, ok := socketTypes[value] + if !ok { + availableTypes := make([]string, len(socketTypes)) + i := 0 + for t := range socketTypes { + availableTypes[i] = t + i++ + } + return fmt.Errorf("unknown socket type: %s, supported types: %v", value, availableTypes) + } + + *s = setting return nil } diff --git a/filebeat/inputsource/unix/config_test.go b/filebeat/inputsource/unix/config_test.go new file mode 100644 index 00000000000..c57e654f812 --- /dev/null +++ b/filebeat/inputsource/unix/config_test.go @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package unix + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestErrorMissingPath(t *testing.T) { + c := common.MustNewConfigFrom(map[string]interface{}{ + "timeout": 1, + "max_message_size": 1, + }) + var config Config + err := c.Unpack(&config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "need to specify the path to the unix socket") +} + +func TestErrorOnEmptyLineDelimiterWhenStreamSocket(t *testing.T) { + c := common.MustNewConfigFrom(map[string]interface{}{ + "timeout": 1, + "max_message_size": 1, + "path": "my-path", + "socket_type": "stream", + }) + var config Config + err := c.Unpack(&config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "line_delimiter cannot be empty when using stream socket") +} + +func TestInvalidSocketType(t *testing.T) { + c := common.MustNewConfigFrom(map[string]interface{}{ + "timeout": 1, + "max_message_size": 1, + "path": "my-path", + "socket_type": "invalid_type", + }) + var config Config + err := c.Unpack(&config) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown socket type") +} diff --git a/filebeat/inputsource/unix/server.go b/filebeat/inputsource/unix/server.go index ee9a0f4564d..e85ced4cf49 100644 --- a/filebeat/inputsource/unix/server.go +++ b/filebeat/inputsource/unix/server.go @@ -18,50 +18,69 @@ package unix import ( + "context" "fmt" "net" - "os" - "os/user" - "runtime" - "strconv" - "github.com/pkg/errors" "golang.org/x/net/netutil" - "github.com/elastic/beats/v7/filebeat/inputsource/common" + "github.com/elastic/beats/v7/filebeat/inputsource" + "github.com/elastic/beats/v7/filebeat/inputsource/common/dgram" + "github.com/elastic/beats/v7/filebeat/inputsource/common/streaming" "github.com/elastic/beats/v7/libbeat/logp" ) -// Server represent a unix server -type Server struct { - *common.Listener +// Server is run by the input. +type Server interface { + inputsource.Network + Run(context.Context) error +} +// streamServer is a server for reading from Unix stream sockets. +type streamServer struct { + *streaming.Listener config *Config } -// New creates a new unix server -func New( - config *Config, - factory common.HandlerFactory, -) (*Server, error) { - if factory == nil { - return nil, fmt.Errorf("HandlerFactory can't be empty") - } +// datagramServer is a server for reading from Unix datagram sockets. +type datagramServer struct { + *dgram.Listener + config *Config +} - server := &Server{ - config: config, +// New creates a new unix server. +func New(log *logp.Logger, config *Config, nf inputsource.NetworkFunc) (Server, error) { + switch config.SocketType { + case StreamSocket: + splitFunc := streaming.SplitFunc([]byte(config.LineDelimiter)) + if splitFunc == nil { + return nil, fmt.Errorf("unable to create splitFunc for delimiter %s", config.LineDelimiter) + } + factory := streaming.SplitHandlerFactory(inputsource.FamilyUnix, log, MetadataCallback, nf, splitFunc) + server := &streamServer{config: config} + server.Listener = streaming.NewListener(inputsource.FamilyUnix, config.Path, factory, server.createServer, &streaming.ListenerConfig{ + Timeout: config.Timeout, + MaxMessageSize: config.MaxMessageSize, + MaxConnections: config.MaxConnections, + }) + return server, nil + + case DatagramSocket: + server := &datagramServer{config: config} + factory := dgram.DatagramReaderFactory(inputsource.FamilyUnix, log, nf) + server.Listener = dgram.NewListener(inputsource.FamilyUnix, config.Path, factory, server.createConn, &dgram.ListenerConfig{ + Timeout: config.Timeout, + MaxMessageSize: config.MaxMessageSize, + }) + return server, nil + + default: } - server.Listener = common.NewListener(common.FamilyUnix, config.Path, factory, server.createServer, &common.ListenerConfig{ - Timeout: config.Timeout, - MaxMessageSize: config.MaxMessageSize, - MaxConnections: config.MaxConnections, - }) - - return server, nil + return nil, fmt.Errorf("unknown unix server type") } -func (s *Server) createServer() (net.Listener, error) { - if err := s.cleanupStaleSocket(); err != nil { +func (s *streamServer) createServer() (net.Listener, error) { + if err := cleanupStaleSocket(s.config.Path); err != nil { return nil, err } @@ -70,11 +89,11 @@ func (s *Server) createServer() (net.Listener, error) { return nil, err } - if err := s.setSocketOwnership(); err != nil { + if err := setSocketOwnership(s.config.Path, s.config.Group); err != nil { return nil, err } - if err := s.setSocketMode(); err != nil { + if err := setSocketMode(s.config.Path, s.config.Mode); err != nil { return nil, err } @@ -84,68 +103,26 @@ func (s *Server) createServer() (net.Listener, error) { return l, nil } -func (s *Server) cleanupStaleSocket() error { - path := s.config.Path - info, err := os.Lstat(path) - if err != nil { - // If the file does not exist, then the cleanup can be considered successful. - if os.IsNotExist(err) { - return nil - } - return errors.Wrapf(err, "cannot lstat unix socket file at location %s", path) - } - - if runtime.GOOS != "windows" { - // see https://github.com/golang/go/issues/33357 for context on Windows socket file attributes bug - if info.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("refusing to remove file at location %s, it is not a socket", path) - } +func (s *datagramServer) createConn() (net.PacketConn, error) { + if err := cleanupStaleSocket(s.config.Path); err != nil { + return nil, err } - if err := os.Remove(path); err != nil { - return errors.Wrapf(err, "cannot remove existing unix socket file at location %s", path) + addr, err := net.ResolveUnixAddr("unixgram", s.config.Path) + if err != nil { + return nil, err } - - return nil -} - -func (s *Server) setSocketOwnership() error { - if s.config.Group != nil { - if runtime.GOOS == "windows" { - logp.NewLogger("unix").Warn("windows does not support the 'group' configuration option, ignoring") - return nil - } - g, err := user.LookupGroup(*s.config.Group) - if err != nil { - return err - } - gid, err := strconv.Atoi(g.Gid) - if err != nil { - return err - } - return os.Chown(s.config.Path, -1, gid) + conn, err := net.ListenUnixgram("unixgram", addr) + if err != nil { + return nil, err } - return nil -} -func (s *Server) setSocketMode() error { - if s.config.Mode != nil { - mode, err := parseFileMode(*s.config.Mode) - if err != nil { - return err - } - return os.Chmod(s.config.Path, mode) + if err := setSocketOwnership(s.config.Path, s.config.Group); err != nil { + return nil, err } - return nil -} -func parseFileMode(mode string) (os.FileMode, error) { - parsed, err := strconv.ParseUint(mode, 8, 32) - if err != nil { - return 0, err - } - if parsed > 0777 { - return 0, errors.New("invalid file mode") + if err := setSocketMode(s.config.Path, s.config.Mode); err != nil { + return nil, err } - return os.FileMode(parsed), nil + return conn, nil } diff --git a/filebeat/inputsource/unix/server_test.go b/filebeat/inputsource/unix/server_test.go index fc9545100d5..460b85b8b1e 100644 --- a/filebeat/inputsource/unix/server_test.go +++ b/filebeat/inputsource/unix/server_test.go @@ -18,7 +18,6 @@ package unix import ( - "bufio" "fmt" "math/rand" "net" @@ -36,15 +35,17 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/filebeat/inputsource" - netcommon "github.com/elastic/beats/v7/filebeat/inputsource/common" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/logp" ) -var defaultConfig = Config{ - Timeout: time.Minute * 5, - MaxMessageSize: 20 * humanize.MiByte, +func defaultConfig() Config { + return Config{ + Timeout: time.Minute * 5, + MaxMessageSize: 20 * humanize.MiByte, + SocketType: StreamSocket, + } } type info struct { @@ -52,10 +53,20 @@ type info struct { mt inputsource.NetworkMetadata } +func TestErrorOnInvalidSocketType(t *testing.T) { + config := &Config{ + SocketType: SocketType(7), + } + _, err := New(logp.L(), config, nil) + assert.Error(t, err) +} + func TestErrorOnEmptyLineDelimiter(t *testing.T) { - c := common.NewConfig() - config := defaultConfig - err := c.Unpack(&config) + config := &Config{ + SocketType: StreamSocket, + LineDelimiter: "", + } + _, err := New(logp.L(), config, nil) assert.Error(t, err) } @@ -72,93 +83,82 @@ func TestReceiveEventsAndMetadata(t *testing.T) { tests := []struct { name string cfg map[string]interface{} - splitFunc bufio.SplitFunc expectedMessages []string messageSent string }{ { name: "NewLine", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + cfg: map[string]interface{}{"line_delimiter": "\n"}, expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\n"), }, { name: "NewLineWithCR", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\r\n")), + cfg: map[string]interface{}{"line_delimiter": "\r\n"}, expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, "\r\n"), }, { name: "CustomDelimiter", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + cfg: map[string]interface{}{"line_delimiter": ";"}, expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ";"), }, { name: "MultipleCharsCustomDelimiter", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("")), + cfg: map[string]interface{}{"line_delimiter": ""}, expectedMessages: expectedMessages, messageSent: strings.Join(expectedMessages, ""), }, { name: "SingleCharCustomDelimiterMessageWithoutBoundaries", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + cfg: map[string]interface{}{"line_delimiter": ";"}, expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "MultipleCharCustomDelimiterMessageWithoutBoundaries", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("")), + cfg: map[string]interface{}{"line_delimiter": ""}, expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineMessageWithoutBoundaries", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + cfg: map[string]interface{}{"line_delimiter": "\n"}, expectedMessages: []string{"hello"}, messageSent: "hello", }, { name: "NewLineLargeMessagePayload", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + cfg: map[string]interface{}{"line_delimiter": "\n"}, expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, "\n"), }, { name: "CustomLargeMessagePayload", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte(";")), + cfg: map[string]interface{}{"line_delimiter": ";"}, expectedMessages: largeMessages, messageSent: strings.Join(largeMessages, ";"), }, { name: "ReadRandomLargePayload", - cfg: map[string]interface{}{}, - splitFunc: netcommon.SplitFunc([]byte("\n")), + cfg: map[string]interface{}{"line_delimiter": "\n"}, expectedMessages: []string{randomGeneratedText}, messageSent: randomGeneratedText, }, { - name: "MaxReadBufferReachedUserConfigured", - splitFunc: netcommon.SplitFunc([]byte("\n")), + name: "MaxReadBufferReachedUserConfigured", cfg: map[string]interface{}{ + "line_delimiter": "\n", "max_message_size": 50000, }, expectedMessages: []string{}, messageSent: randomGeneratedText, }, { - name: "MaxBufferSizeSet", - splitFunc: netcommon.SplitFunc([]byte("\n")), + name: "MaxBufferSizeSet", cfg: map[string]interface{}{ + "line_delimiter": "\n", "max_message_size": 66 * 1024, }, expectedMessages: extraLargeMessages, @@ -176,14 +176,13 @@ func TestReceiveEventsAndMetadata(t *testing.T) { path := filepath.Join(os.TempDir(), "test.sock") test.cfg["path"] = path cfg, _ := common.NewConfigFrom(test.cfg) - config := defaultConfig + config := defaultConfig() err := cfg.Unpack(&config) if !assert.NoError(t, err) { return } - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logp.NewLogger("test"), MetadataCallback, to, test.splitFunc) - server, err := New(&config, factory) + server, err := New(logp.L(), &config, to) if !assert.NoError(t, err) { return } @@ -234,16 +233,16 @@ func TestSocketOwnershipAndMode(t *testing.T) { path := filepath.Join(os.TempDir(), "test.sock") cfg, _ := common.NewConfigFrom(map[string]interface{}{ - "path": path, - "group": group.Name, - "mode": "0740", + "path": path, + "group": group.Name, + "mode": "0740", + "line_delimiter": "\n", }) - config := defaultConfig + config := defaultConfig() err = cfg.Unpack(&config) require.NoError(t, err) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logp.NewLogger("test"), MetadataCallback, nil, netcommon.SplitFunc([]byte("\n"))) - server, err := New(&config, factory) + server, err := New(logp.L(), &config, nil) require.NoError(t, err) err = server.Start() require.NoError(t, err) @@ -269,12 +268,12 @@ func TestSocketCleanup(t *testing.T) { defer mockStaleSocket.Close() cfg, _ := common.NewConfigFrom(map[string]interface{}{ - "path": path, + "path": path, + "line_delimiter": "\n", }) - config := defaultConfig + config := defaultConfig() require.NoError(t, cfg.Unpack(&config)) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logp.NewLogger("test"), MetadataCallback, nil, netcommon.SplitFunc([]byte("\n"))) - server, err := New(&config, factory) + server, err := New(logp.L(), &config, nil) require.NoError(t, err) err = server.Start() require.NoError(t, err) @@ -293,12 +292,12 @@ func TestSocketCleanupRefusal(t *testing.T) { defer os.Remove(path) cfg, _ := common.NewConfigFrom(map[string]interface{}{ - "path": path, + "path": path, + "line_delimiter": "\n", }) - config := defaultConfig + config := defaultConfig() require.NoError(t, cfg.Unpack(&config)) - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logp.NewLogger("test"), MetadataCallback, nil, netcommon.SplitFunc([]byte("\n"))) - server, err := New(&config, factory) + server, err := New(logp.L(), &config, nil) require.NoError(t, err) err = server.Start() require.Error(t, err) @@ -310,55 +309,87 @@ func TestReceiveNewEventsConcurrently(t *testing.T) { t.Skip("test is only supported on non-windows. See https://github.com/elastic/beats/issues/21757") return } - workers := 4 - eventsCount := 100 - path := filepath.Join(os.TempDir(), "test.sock") - ch := make(chan *info, eventsCount*workers) - defer close(ch) - to := func(message []byte, mt inputsource.NetworkMetadata) { - ch <- &info{message: string(message), mt: mt} - } - cfg, err := common.NewConfigFrom(map[string]interface{}{"path": path}) - if !assert.NoError(t, err) { - return - } - config := defaultConfig - err = cfg.Unpack(&config) - if !assert.NoError(t, err) { - return - } - factory := netcommon.SplitHandlerFactory(netcommon.FamilyUnix, logp.NewLogger("test"), MetadataCallback, to, bufio.ScanLines) + for socketType, _ := range socketTypes { + if runtime.GOOS == "darwin" && socketType == "datagram" { + t.Skip("test is only supported on linux. See https://github.com/elastic/beats/issues/22775") + return + } - server, err := New(&config, factory) - if !assert.NoError(t, err) { - return + t.Run("socket_type "+socketType, func(t *testing.T) { + workers := 1 + eventsCount := 100 + path := filepath.Join(os.TempDir(), "test.sock") + ch := make(chan *info, eventsCount*workers) + defer close(ch) + to := func(message []byte, mt inputsource.NetworkMetadata) { + ch <- &info{message: string(message), mt: mt} + } + cfg, err := common.NewConfigFrom(map[string]interface{}{ + "path": path, + "line_delimiter": "\n", + "socket_type": socketType, + }) + if !assert.NoError(t, err) { + return + } + config := defaultConfig() + err = cfg.Unpack(&config) + if !assert.NoError(t, err) { + return + } + + server, err := New(logp.L(), &config, to) + if !assert.NoError(t, err) { + return + } + err = server.Start() + if !assert.NoError(t, err) { + return + } + defer server.Stop() + + samples := generateMessages(eventsCount, 1024) + for w := 0; w < workers; w++ { + if socketType == "stream" { + go sendOverUnixStream(t, path, samples) + } else if socketType == "datagram" { + go sendOverUnixDatagram(t, path, samples) + } + } + + var events []*info + for len(events) < eventsCount*workers { + select { + case event := <-ch: + events = append(events, event) + default: + } + } + }) } - err = server.Start() +} + +func sendOverUnixStream(t *testing.T, path string, samples []string) { + conn, err := net.Dial("unix", path) if !assert.NoError(t, err) { return } - defer server.Stop() + defer conn.Close() - samples := generateMessages(eventsCount, 1024) - for w := 0; w < workers; w++ { - go func() { - conn, err := net.Dial("unix", path) - defer conn.Close() - assert.NoError(t, err) - for _, sample := range samples { - fmt.Fprintln(conn, sample) - } - }() + for _, sample := range samples { + fmt.Fprintln(conn, sample) } +} - var events []*info - for len(events) < eventsCount*workers { - select { - case event := <-ch: - events = append(events, event) - default: - } +func sendOverUnixDatagram(t *testing.T, path string, samples []string) { + conn, err := net.Dial("unixgram", path) + if !assert.NoError(t, err) { + return + } + defer conn.Close() + for _, sample := range samples { + fmt.Fprintln(conn, sample) } } @@ -374,7 +405,7 @@ func randomString(l int) string { func generateMessages(c int, l int) []string { messages := make([]string, c) for i := range messages { - messages[i] = randomString(l) + messages[i] = randomString(l) + "-" + strconv.Itoa(i) } return messages } diff --git a/filebeat/inputsource/unix/socket.go b/filebeat/inputsource/unix/socket.go new file mode 100644 index 00000000000..576dedad564 --- /dev/null +++ b/filebeat/inputsource/unix/socket.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package unix + +import ( + "fmt" + "os" + "os/user" + "runtime" + "strconv" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func cleanupStaleSocket(path string) error { + info, err := os.Lstat(path) + if err != nil { + // If the file does not exist, then the cleanup can be considered successful. + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "cannot lstat unix socket file at location %s", path) + } + + if runtime.GOOS != "windows" { + // see https://github.com/golang/go/issues/33357 for context on Windows socket file attributes bug + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("refusing to remove file at location %s, it is not a socket", path) + } + } + + if err := os.Remove(path); err != nil { + return errors.Wrapf(err, "cannot remove existing unix socket file at location %s", path) + } + + return nil +} + +func setSocketOwnership(path string, group *string) error { + if group != nil { + if runtime.GOOS == "windows" { + logp.NewLogger("unix").Warn("windows does not support the 'group' configuration option, ignoring") + return nil + } + g, err := user.LookupGroup(*group) + if err != nil { + return err + } + gid, err := strconv.Atoi(g.Gid) + if err != nil { + return err + } + return os.Chown(path, -1, gid) + } + return nil +} + +func setSocketMode(path string, mode *string) error { + if mode != nil { + m, err := parseFileMode(*mode) + if err != nil { + return err + } + return os.Chmod(path, m) + } + return nil +} + +func parseFileMode(mode string) (os.FileMode, error) { + parsed, err := strconv.ParseUint(mode, 8, 32) + if err != nil { + return 0, err + } + if parsed > 0777 { + return 0, errors.New("invalid file mode") + } + return os.FileMode(parsed), nil +} diff --git a/filebeat/module/nats/_meta/kibana/7/dashboard/Filebeat-nats-overview.json b/filebeat/module/nats/_meta/kibana/7/dashboard/Filebeat-nats-overview.json index 3d6311c6721..0b281987921 100644 --- a/filebeat/module/nats/_meta/kibana/7/dashboard/Filebeat-nats-overview.json +++ b/filebeat/module/nats/_meta/kibana/7/dashboard/Filebeat-nats-overview.json @@ -1,12 +1,259 @@ { "objects": [ + { + "attributes": { + "description": "Overview of NATS server statistics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Message Types Timeline" + }, + "gridData": { + "h": 11, + "i": "1", + "w": 17, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "title": "Message Types Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Communication Directions" + }, + "gridData": { + "h": 11, + "i": "2", + "w": 17, + "x": 31, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "title": "Communication Directions", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Topics Timeline" + }, + "gridData": { + "h": 12, + "i": "3", + "w": 25, + "x": 0, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "title": "Topics Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": " Bytes Timeline", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "4", + "w": 12, + "x": 11, + "y": 11 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "title": " Bytes Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Communication Directions Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "5", + "w": 11, + "x": 0, + "y": 11 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "title": "Communication Directions Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Log Level Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "6", + "w": 11, + "x": 37, + "y": 11 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "title": "Log Level Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Message Type Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 11, + "i": "7", + "w": 14, + "x": 17, + "y": 0 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "title": "Message Type Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Log Level Timeline" + }, + "gridData": { + "h": 9, + "i": "8", + "w": 14, + "x": 23, + "y": 11 + }, + "panelIndex": "8", + "panelRefName": "panel_7", + "title": "Log Level Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Client IP Count Timeline" + }, + "gridData": { + "h": 12, + "i": "9", + "w": 22, + "x": 25, + "y": 20 + }, + "panelIndex": "9", + "panelRefName": "panel_8", + "title": "Client IP Count Timeline", + "version": "7.10.0" + } + ], + "timeRestore": false, + "title": "[Filebeat NATS] Overview ECS", + "version": 1 + }, + "id": "Filebeat-nats-overview-ecs", + "migrationVersion": { + "dashboard": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "6987a800-41a8-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "0b2061d0-41ad-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "4a6d9ec0-41a8-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "c3d1ab80-41a8-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "7716c780-41ad-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "3f6cca40-41ae-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "7ed62870-41ae-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "04083600-41af-11e9-a4da-b1df688edbcd-ecs", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "c669ae20-41ed-11e9-ac5c-71ffa38a62e3-ecs", + "name": "panel_8", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2020-11-23T16:25:23.231Z", + "version": "WzYyNywxXQ==" + }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" @@ -118,9 +365,22 @@ } }, "id": "6987a800-41a8-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:47:49.627Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwMSwxXQ==" }, { "attributes": { @@ -128,7 +388,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "service.type: nats" @@ -237,9 +497,22 @@ } }, "id": "0b2061d0-41ad-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:54:53.381Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwMiwxXQ==" }, { "attributes": { @@ -247,7 +520,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" @@ -356,9 +629,22 @@ } }, "id": "4a6d9ec0-41a8-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:49:49.112Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwMywxXQ==" }, { "attributes": { @@ -366,7 +652,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" @@ -468,9 +754,22 @@ } }, "id": "c3d1ab80-41a8-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:38:11.578Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwNCwxXQ==" }, { "attributes": { @@ -478,7 +777,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "service.type: nats" @@ -526,9 +825,22 @@ } }, "id": "7716c780-41ad-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:39:03.899Z", - "version": 5 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwNSwxXQ==" }, { "attributes": { @@ -536,7 +848,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "service.type: nats" @@ -584,9 +896,22 @@ } }, "id": "3f6cca40-41ae-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:44:31.263Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwNiwxXQ==" }, { "attributes": { @@ -594,7 +919,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" @@ -642,9 +967,22 @@ } }, "id": "7ed62870-41ae-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:48:10.554Z", - "version": 3 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwNywxXQ==" }, { "attributes": { @@ -652,7 +990,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "service.type: nats" @@ -762,9 +1100,22 @@ } }, "id": "04083600-41af-11e9-a4da-b1df688edbcd-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-03-08T21:48:44.582Z", - "version": 2 + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwOCwxXQ==" }, { "attributes": { @@ -772,7 +1123,7 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "filebeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "service.type: nats" @@ -901,181 +1252,23 @@ } }, "id": "c669ae20-41ed-11e9-ac5c-71ffa38a62e3-ecs", - "type": "visualization", - "updated_at": "2019-03-08T22:01:50.337Z", - "version": 1 - }, - { - "attributes": { - "description": "Overview of NATS server statistics", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "1", - "w": 17, - "x": 0, - "y": 0 - }, - "id": "6987a800-41a8-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "1", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "2", - "w": 17, - "x": 31, - "y": 0 - }, - "id": "0b2061d0-41ad-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "2", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "3", - "w": 25, - "x": 0, - "y": 20 - }, - "id": "4a6d9ec0-41a8-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "3", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": false - } - }, - "gridData": { - "h": 9, - "i": "4", - "w": 12, - "x": 11, - "y": 11 - }, - "id": "c3d1ab80-41a8-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "4", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": false - } - }, - "gridData": { - "h": 9, - "i": "5", - "w": 11, - "x": 0, - "y": 11 - }, - "id": "7716c780-41ad-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "5", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": false - } - }, - "gridData": { - "h": 9, - "i": "6", - "w": 11, - "x": 37, - "y": 11 - }, - "id": "3f6cca40-41ae-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "6", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": false - } - }, - "gridData": { - "h": 11, - "i": "7", - "w": 14, - "x": 17, - "y": 0 - }, - "id": "7ed62870-41ae-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "7", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 9, - "i": "8", - "w": 14, - "x": 23, - "y": 11 - }, - "id": "04083600-41af-11e9-a4da-b1df688edbcd-ecs", - "panelIndex": "8", - "type": "visualization", - "version": "6.6.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "9", - "w": 22, - "x": 25, - "y": 20 - }, - "id": "c669ae20-41ed-11e9-ac5c-71ffa38a62e3-ecs", - "panelIndex": "9", - "type": "visualization", - "version": "6.6.0" - } - ], - "timeRestore": false, - "title": "[Filebeat NATS] Overview ECS", - "version": 1 + "migrationVersion": { + "visualization": "7.10.0" }, - "id": "Filebeat-nats-overview-ecs", - "type": "dashboard", - "updated_at": "2019-03-08T22:02:50.580Z", - "version": 5 + "namespaces": [ + "default" + ], + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-11-23T16:05:55.609Z", + "version": "WzEwOSwxXQ==" } ], - "version": "6.6.0" -} + "version": "7.10.0" +} \ No newline at end of file diff --git a/filebeat/tests/system/test_shutdown.py b/filebeat/tests/system/test_shutdown.py index eb4084c5205..8f18337435f 100644 --- a/filebeat/tests/system/test_shutdown.py +++ b/filebeat/tests/system/test_shutdown.py @@ -1,5 +1,6 @@ import gzip import os +import platform import time import unittest from filebeat import BaseTest @@ -11,6 +12,8 @@ class Test(BaseTest): + @unittest.skipIf(platform.platform().startswith("Windows-7"), + "Flaky test: https://github.com/elastic/beats/issues/22795") def test_shutdown(self): """ Test starting and stopping Filebeat under load. diff --git a/filebeat/tests/system/test_syslog.py b/filebeat/tests/system/test_syslog.py index ed8f5a58f2f..272f0c5b81d 100644 --- a/filebeat/tests/system/test_syslog.py +++ b/filebeat/tests/system/test_syslog.py @@ -127,6 +127,8 @@ def test_syslog_with_udp(self): filebeat.check_kill_and_wait() + sock.close() + output = self.read_output() assert len(output) == 2 @@ -135,13 +137,28 @@ def test_syslog_with_udp(self): # AF_UNIX support in python isn't available until # Python 3.9, see https://bugs.python.org/issue33408 @unittest.skipIf(not hasattr(socket, 'AF_UNIX'), "No Windows AF_UNIX support before Python 3.9") - def test_syslog_with_unix(self): + def test_syslog_with_unix_stream(self): + """ + Test syslog input with events from UNIX stream. + """ + + self.run_filebeat_and_send_using_socket("stream", send_stream_socket) + + # AF_UNIX support in python isn't available until + # Python 3.9, see https://bugs.python.org/issue33408 + @unittest.skipIf(not hasattr(socket, 'AF_UNIX'), "No Windows AF_UNIX support before Python 3.9") + def test_syslog_with_unix_datagram(self): """ - Test syslog input with events from UNIX. + Test syslog input with events from UNIX stream. """ + + self.run_filebeat_and_send_using_socket("datagram", send_datagram_socket) + + def run_filebeat_and_send_using_socket(self, socket_type, send_over_socket): # we create the socket in a temporary directory because # go will fail to create a unix socket if the path length # is longer than 108 characters. See https://github.com/golang/go/issues/6895 + with tempfile.TemporaryDirectory() as tempdir: path = os.path.join(tempdir, "filebeat.sock") input_raw = """ @@ -149,9 +166,10 @@ def test_syslog_with_unix(self): protocol: unix: path: {} + socket_type: {} """ - input_raw = input_raw.format(path) + input_raw = input_raw.format(path, socket_type) self.render_config_template( input_raw=input_raw, inputs=False, @@ -161,15 +179,9 @@ def test_syslog_with_unix(self): self.wait_until(lambda: self.log_contains("Started listening for UNIX connection")) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # UNIX - - sock.connect(path) - - for n in range(0, 2): - m = "<13>Oct 11 22:14:15 wopr.mymachine.co postfix/smtpd[2000]:" \ - " 'su root' failed for lonvick on /dev/pts/8 {}\n" - m = m.format(n) - sock.send(m.encode("utf-8")) + sock = send_over_socket(path, + "<13>Oct 11 22:14:15 wopr.mymachine.co postfix/smtpd[2000]:" + " 'su root' failed for lonvick on /dev/pts/8 {}\n") self.wait_until(lambda: self.output_count(lambda x: x >= 2)) @@ -180,13 +192,30 @@ def test_syslog_with_unix(self): assert len(output) == 2 self.assert_syslog(output[0], False) + sock.close() + # AF_UNIX support in python isn't available until # Python 3.9, see https://bugs.python.org/issue33408 + @unittest.skipIf(not hasattr(socket, 'AF_UNIX'), "No Windows AF_UNIX support before Python 3.9") - def test_syslog_with_unix_invalid_message(self): + def test_syslog_with_unix_stream_invalid_message(self): """ Test syslog input with invalid events from UNIX. """ + + self.run_filebeat_and_send_invalid_message_using_socket("stream", send_stream_socket) + + # AF_UNIX support in python isn't available until + # Python 3.9, see https://bugs.python.org/issue33408 + @unittest.skipIf(not hasattr(socket, 'AF_UNIX'), "No Windows AF_UNIX support before Python 3.9") + def test_syslog_with_unix_datagram_invalid_message(self): + """ + Test syslog input with invalid events from UNIX. + """ + + self.run_filebeat_and_send_invalid_message_using_socket("datagram", send_datagram_socket) + + def run_filebeat_and_send_invalid_message_using_socket(self, socket_type, send_over_socket): # we create the socket in a temporary directory because # go will fail to create a unix socket if the path length # is longer than 108 characters. See https://github.com/golang/go/issues/6895 @@ -197,9 +226,10 @@ def test_syslog_with_unix_invalid_message(self): protocol: unix: path: {} + socket_type: {} """ - input_raw = input_raw.format(path) + input_raw = input_raw.format(path, socket_type) self.render_config_template( input_raw=input_raw, inputs=False, @@ -209,11 +239,7 @@ def test_syslog_with_unix_invalid_message(self): self.wait_until(lambda: self.log_contains("Started listening for UNIX connection")) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # UNIX - - sock.connect(path) - for n in range(0, 2): - sock.send("invalid\n".encode("utf-8")) + sock = send_over_socket(path, "invalid\n") self.wait_until(lambda: self.output_count(lambda x: x >= 2)) @@ -222,7 +248,10 @@ def test_syslog_with_unix_invalid_message(self): output = self.read_output() assert len(output) == 2 - assert output[0]["message"] == "invalid" + expected_message = "invalid" + if socket_type == "datagram": + expected_message += "\n" + assert output[0]["message"] == expected_message sock.close() def assert_syslog(self, syslog, has_address=True): @@ -238,3 +267,25 @@ def assert_syslog(self, syslog, has_address=True): assert syslog["syslog.facility_label"] == "user-level" if has_address: assert len(syslog["log.source.address"]) > 0 + + +def send_stream_socket(path, message): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + sock.connect(path) + + for n in range(0, 2): + message = message.format(n) + sock.send(message.encode("utf-8")) + + return sock + + +def send_datagram_socket(path, message): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + + for n in range(0, 2): + message = message.format(n) + sock.sendto(message.encode("utf-8"), path) + + return sock diff --git a/filebeat/tests/system/test_udp.py b/filebeat/tests/system/test_udp.py index 844aa38a605..7a8b73489bd 100644 --- a/filebeat/tests/system/test_udp.py +++ b/filebeat/tests/system/test_udp.py @@ -1,9 +1,12 @@ from filebeat import BaseTest +import os import socket +import unittest class Test(BaseTest): + @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/22809') def test_udp(self): host = "127.0.0.1" diff --git a/filebeat/tests/system/test_unix.py b/filebeat/tests/system/test_unix.py index bb9b7f25bd5..bc506b47d7b 100644 --- a/filebeat/tests/system/test_unix.py +++ b/filebeat/tests/system/test_unix.py @@ -61,11 +61,44 @@ def send_events_with_delimiter(self, delimiter): self.wait_until(lambda: self.log_contains("Started listening for UNIX connection")) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # UNIX - sock.connect(path) + sock = send_stream_socket(path, delimiter) - for n in range(0, 2): - sock.send(bytes("Hello World: " + str(n) + delimiter, "utf-8")) + self.wait_until(lambda: self.output_count(lambda x: x >= 2)) + + filebeat.check_kill_and_wait() + + output = self.read_output() + + assert len(output) == 2 + assert output[0]["input.type"] == "unix" + + sock.close() + + def test_unix_datagram_socket(self): + # we create the socket in a temporary directory because + # go will fail to create a unix socket if the path length + # is longer than 108 characters. See https://github.com/golang/go/issues/6895 + with tempfile.TemporaryDirectory() as tempdir: + path = os.path.join(tempdir, "filebeat.sock") + input_raw = """ +- type: unix + path: {} + enabled: true + socket_type: datagram +""" + + input_raw = input_raw.format(path) + + self.render_config_template( + input_raw=input_raw, + inputs=False, + ) + + filebeat = self.start_beat() + + self.wait_until(lambda: self.log_contains("Started listening for UNIX connection")) + + sock = send_datagram_socket(path) self.wait_until(lambda: self.output_count(lambda x: x >= 2)) @@ -74,6 +107,27 @@ def send_events_with_delimiter(self, delimiter): output = self.read_output() assert len(output) == 2 + assert output[0]["message"] == "Hello World: 0;" assert output[0]["input.type"] == "unix" sock.close() + + +def send_stream_socket(path, delimiter): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + + for n in range(0, 2): + sock.send(bytes("Hello World: " + str(n) + delimiter, "utf-8")) + + return sock + + +def send_datagram_socket(path): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + sock.connect(path) + + for n in range(0, 2): + sock.sendto(bytes("Hello World: " + str(n) + ";", "utf-8"), path) + + return sock diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index c0d8abfa9c8..f0155d34274 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index de63e60253f..e944798887e 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -21,12 +21,12 @@ import ( "fmt" "time" - "github.com/elastic/beats/v7/heartbeat/hbregistry" - "github.com/pkg/errors" "github.com/elastic/beats/v7/heartbeat/config" + "github.com/elastic/beats/v7/heartbeat/hbregistry" "github.com/elastic/beats/v7/heartbeat/monitors" + "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" "github.com/elastic/beats/v7/heartbeat/scheduler" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/beat" @@ -127,8 +127,13 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) error { for _, cfg := range bt.config.Monitors { created, err := factory.Create(b.Publisher, cfg) if err != nil { + if err == stdfields.ErrPluginDisabled { + continue // don't stop loading monitors just because they're disabled + } + return errors.Wrap(err, "could not create monitor") } + created.Start() } return nil diff --git a/heartbeat/monitors/stdfields/stdfields.go b/heartbeat/monitors/stdfields/stdfields.go index 784b84cabd7..433f6223863 100644 --- a/heartbeat/monitors/stdfields/stdfields.go +++ b/heartbeat/monitors/stdfields/stdfields.go @@ -28,7 +28,7 @@ import ( ) // ErrPluginDisabled is returned when the monitor plugin is marked as disabled. -var ErrPluginDisabled = errors.New("Monitor not loaded, plugin is disabled") +var ErrPluginDisabled = errors.New("monitor not loaded, plugin is disabled") type ServiceFields struct { Name string `config:"name"` diff --git a/heartbeat/tests/system/test_base.py b/heartbeat/tests/system/test_base.py index 85453825329..643f9f31bf7 100644 --- a/heartbeat/tests/system/test_base.py +++ b/heartbeat/tests/system/test_base.py @@ -32,6 +32,30 @@ def test_base(self): self.wait_until(lambda: self.log_contains("heartbeat is running")) heartbeat_proc.check_kill_and_wait() + def test_disabled(self): + """ + Basic test against a disabled monitor + """ + + config = { + "monitors": [ + { + "type": "http", + "enabled": "false", + "urls": ["http://localhost:9200"], + } + ] + } + + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + **config + ) + + heartbeat_proc = self.start_beat() + self.wait_until(lambda: self.log_contains("heartbeat is running")) + heartbeat_proc.check_kill_and_wait() + def test_fields_under_root(self): """ Basic test with fields and tags in monitor diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile index dd1d377f88d..6df4d47d885 100644 --- a/journalbeat/Dockerfile +++ b/journalbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index b72fbaa58b5..b7dde2b92cf 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go index 95e23b33d2a..0f37ade6b5a 100644 --- a/libbeat/autodiscover/providers/kubernetes/node.go +++ b/libbeat/autodiscover/providers/kubernetes/node.go @@ -168,6 +168,11 @@ func (n *node) emit(node *kubernetes.Node, flag string) { return } + // If the node is not in ready state then dont monitor it unless its a stop event + if !isNodeReady(node) && flag != "stop" { + return + } + eventID := fmt.Sprint(node.GetObjectMeta().GetUID()) meta := n.metagen.Generate(node) @@ -237,6 +242,12 @@ func getAddress(node *kubernetes.Node) string { } } + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeHostName && address.Address != "" { + return address.Address + } + } + return "" } diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index 518a0121578..6eb22b185e1 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -156,6 +156,12 @@ func TestEmitEvent_Node(t *testing.T) { Address: "node1", }, }, + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + }, }, }, Expected: bus.Event{ @@ -183,6 +189,57 @@ func TestEmitEvent_Node(t *testing.T) { "config": []*common.Config{}, }, }, + { + Message: "Test node start with just node name", + Flag: "start", + Node: &kubernetes.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + TypeMeta: typeMeta, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "node1", + }, + }, + Conditions: []v1.NodeCondition{ + { + Type: v1.NodeReady, + Status: v1.ConditionTrue, + }, + }, + }, + }, + Expected: bus.Event{ + "start": true, + "host": "node1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "node": common.MapStr{ + "name": "metricbeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "hostname": "node1", + }, + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "node": common.MapStr{ + "name": "metricbeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "hostname": "node1", + }, + }, + }, + "config": []*common.Config{}, + }, + }, { Message: "Test service without host", Flag: "start", @@ -221,7 +278,7 @@ func TestEmitEvent_Node(t *testing.T) { }, Expected: bus.Event{ "stop": true, - "host": "", + "host": "node1", "id": uid, "provider": UUID, "kubernetes": common.MapStr{ diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 42545930884..6e96666161f 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -88,13 +88,13 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub } nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil) if err != nil { - return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err) + logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err) } namespaceWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Namespace{}, kubernetes.WatchOptions{ SyncTimeout: config.SyncPeriod, }, nil) if err != nil { - return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err) + logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err) } metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf) diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index 9cd63f03366..5d3a782f1ad 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -49,7 +49,13 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command { b.Config.Kibana = common.NewConfig() } - client, err := kibana.NewKibanaClient(b.Config.Kibana) + // Initialize kibana config. If username and password is set in + // elasticsearch output config but not in kibana, initKibanaConfig + // will attach the username and password into kibana config as a + // part of the initialization. + initConfig := instance.InitKibanaConfig(b.Config) + + client, err := kibana.NewKibanaClient(initConfig) if err != nil { fatalf("Error creating Kibana client: %+v.\n", err) } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index b873ddebf95..0ec0bacc769 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -774,10 +774,7 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { // Initialize kibana config. If username and password is set in elasticsearch output config but not in kibana, // initKibanaConfig will attach the username and password into kibana config as a part of the initialization. - kibanaConfig, err := initKibanaConfig(b.Config) - if err != nil { - return fmt.Errorf("error initKibanaConfig: %v", err) - } + kibanaConfig := InitKibanaConfig(b.Config) client, err := kibana.NewKibanaClient(kibanaConfig) if err != nil { @@ -1041,7 +1038,7 @@ func LoadKeystore(cfg *common.Config, name string) (keystore.Keystore, error) { return keystore.Factory(keystoreCfg, defaultPathConfig) } -func initKibanaConfig(beatConfig beatConfig) (*common.Config, error) { +func InitKibanaConfig(beatConfig beatConfig) *common.Config { var esConfig *common.Config if beatConfig.Output.Name() == "elasticsearch" { esConfig = beatConfig.Output.Config() @@ -1064,7 +1061,7 @@ func initKibanaConfig(beatConfig beatConfig) (*common.Config, error) { kibanaConfig.SetString("password", -1, password) } } - return kibanaConfig, nil + return kibanaConfig } func initPaths(cfg *common.Config) error { diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go index e302bed1711..a0db00c853c 100644 --- a/libbeat/cmd/instance/beat_test.go +++ b/libbeat/cmd/instance/beat_test.go @@ -82,8 +82,7 @@ func TestInitKibanaConfig(t *testing.T) { err = cfg.Unpack(&b.Config) assert.NoError(t, err) - kibanaConfig, err := initKibanaConfig(b.Config) - assert.NoError(t, err) + kibanaConfig := InitKibanaConfig(b.Config) username, err := kibanaConfig.String("username", -1) password, err := kibanaConfig.String("password", -1) protocol, err := kibanaConfig.String("protocol", -1) diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go index cf1ae945248..e1cbd0e8629 100644 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ b/libbeat/common/kubernetes/metadata/metadata.go @@ -60,8 +60,13 @@ func GetPodMetaGen( namespaceWatcher kubernetes.Watcher, metaConf *AddResourceMetadataConfig) MetaGen { - nodeMetaGen := NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store()) - namespaceMetaGen := NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store()) + var nodeMetaGen, namespaceMetaGen MetaGen + if nodeWatcher != nil { + nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store()) + } + if namespaceWatcher != nil { + namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store()) + } metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), nodeMetaGen, namespaceMetaGen) return metaGen diff --git a/libbeat/docs/http-endpoint.asciidoc b/libbeat/docs/http-endpoint.asciidoc index 153bd106db2..0f69fd1ba92 100644 --- a/libbeat/docs/http-endpoint.asciidoc +++ b/libbeat/docs/http-endpoint.asciidoc @@ -33,9 +33,9 @@ current user. `http.named_pipe.security_descriptor`:: (Optional) Windows Security descriptor string defined in the SDDL format. Default to read and write permission for the current user. -This is the list of paths you can access. For pretty JSON output append ?pretty to the URL. +This is the list of paths you can access. For pretty JSON output append `?pretty` to the URL. -You can query a unix socket using the `CURL` command and the `--unix-socket` flag. +You can query a unix socket using the `cURL` command and the `--unix-socket` flag. [source,js] ---- diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 9c88601125c..bae6b3fe9b0 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -60,8 +60,8 @@ relative path is set, it is considered relative to the config path. See the <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: awsfargate + period: 10s + metricsets: + - task_stats +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::awsfargate/task_stats.asciidoc[] + diff --git a/metricbeat/docs/modules/awsfargate/task_stats.asciidoc b/metricbeat/docs/modules/awsfargate/task_stats.asciidoc new file mode 100644 index 00000000000..9e43c8a2cea --- /dev/null +++ b/metricbeat/docs/modules/awsfargate/task_stats.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-awsfargate-task_stats]] +[role="xpack"] +=== AWS Fargate task_stats metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 68476e1acaa..847a13703ce 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -32,6 +32,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .11+| .11+| |<> beta[] |<> beta[] @@ -298,6 +300,7 @@ include::modules/aerospike.asciidoc[] include::modules/apache.asciidoc[] include::modules/appsearch.asciidoc[] include::modules/aws.asciidoc[] +include::modules/awsfargate.asciidoc[] include::modules/azure.asciidoc[] include::modules/beat.asciidoc[] include::modules/ceph.asciidoc[] diff --git a/metricbeat/module/docker/cpu/cpu_test.go b/metricbeat/module/docker/cpu/cpu_test.go index d14fda678d9..d599abd19ed 100644 --- a/metricbeat/module/docker/cpu/cpu_test.go +++ b/metricbeat/module/docker/cpu/cpu_test.go @@ -29,8 +29,8 @@ import ( var cpuService CPUService -func cpuUsageFor(stats types.StatsJSON) *cpuUsage { - u := cpuUsage{ +func cpuUsageFor(stats types.StatsJSON) *CPUUsage { + u := CPUUsage{ Stat: &docker.Stat{Stats: stats}, systemDelta: 1000000000, // Nanoseconds in a second } diff --git a/metricbeat/module/docker/cpu/helper.go b/metricbeat/module/docker/cpu/helper.go index 75527285f1e..fcb8dc2de55 100644 --- a/metricbeat/module/docker/cpu/helper.go +++ b/metricbeat/module/docker/cpu/helper.go @@ -62,7 +62,7 @@ func (c *CPUService) getCPUStatsList(rawStats []docker.Stat, dedot bool) []CPUSt } func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { - usage := cpuUsage{Stat: myRawStat} + usage := CPUUsage{Stat: myRawStat} stats := CPUStats{ Time: common.Time(myRawStat.Stats.Read), @@ -89,7 +89,7 @@ func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { // TODO: These helper should be merged with the cpu helper in system/cpu -type cpuUsage struct { +type CPUUsage struct { *docker.Stat cpus uint32 @@ -98,7 +98,7 @@ type cpuUsage struct { // CPUS returns the number of cpus. If number of cpus is equal to zero, the field will // be updated/initialized with the corresponding value retrieved from Docker API. -func (u *cpuUsage) CPUs() uint32 { +func (u *CPUUsage) CPUs() uint32 { if u.cpus == 0 { if u.Stats.CPUStats.OnlineCPUs != 0 { u.cpus = u.Stats.CPUStats.OnlineCPUs @@ -119,7 +119,7 @@ func (u *cpuUsage) CPUs() uint32 { } // SystemDelta calculates system delta. -func (u *cpuUsage) SystemDelta() uint64 { +func (u *CPUUsage) SystemDelta() uint64 { if u.systemDelta == 0 { u.systemDelta = u.Stats.CPUStats.SystemUsage - u.Stats.PreCPUStats.SystemUsage } @@ -127,7 +127,7 @@ func (u *cpuUsage) SystemDelta() uint64 { } // PerCPU calculates per CPU usage. -func (u *cpuUsage) PerCPU() common.MapStr { +func (u *CPUUsage) PerCPU() common.MapStr { var output common.MapStr if len(u.Stats.CPUStats.CPUUsage.PercpuUsage) == len(u.Stats.PreCPUStats.CPUUsage.PercpuUsage) { output = common.MapStr{} @@ -151,42 +151,42 @@ func (u *cpuUsage) PerCPU() common.MapStr { } // TotalNormalized calculates total CPU usage normalized. -func (u *cpuUsage) Total() float64 { +func (u *CPUUsage) Total() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.TotalUsage, u.Stats.PreCPUStats.CPUUsage.TotalUsage, u.CPUs()) } // TotalNormalized calculates total CPU usage normalized by the number of CPU cores. -func (u *cpuUsage) TotalNormalized() float64 { +func (u *CPUUsage) TotalNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.TotalUsage, u.Stats.PreCPUStats.CPUUsage.TotalUsage, 1) } // InKernelMode calculates percentage of time in kernel space. -func (u *cpuUsage) InKernelMode() float64 { +func (u *CPUUsage) InKernelMode() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInKernelmode, u.Stats.PreCPUStats.CPUUsage.UsageInKernelmode, u.CPUs()) } // InKernelModeNormalized calculates percentage of time in kernel space normalized by the number of CPU cores. -func (u *cpuUsage) InKernelModeNormalized() float64 { +func (u *CPUUsage) InKernelModeNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInKernelmode, u.Stats.PreCPUStats.CPUUsage.UsageInKernelmode, 1) } // InUserMode calculates percentage of time in user space. -func (u *cpuUsage) InUserMode() float64 { +func (u *CPUUsage) InUserMode() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInUsermode, u.Stats.PreCPUStats.CPUUsage.UsageInUsermode, u.CPUs()) } // InUserModeNormalized calculates percentage of time in user space normalized by the number of CPU cores. -func (u *cpuUsage) InUserModeNormalized() float64 { +func (u *CPUUsage) InUserModeNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInUsermode, u.Stats.PreCPUStats.CPUUsage.UsageInUsermode, 1) } // System calculates percentage of total CPU time in the system. -func (u *cpuUsage) System() float64 { +func (u *CPUUsage) System() float64 { return u.calculatePercentage(u.Stats.CPUStats.SystemUsage, u.Stats.PreCPUStats.SystemUsage, u.CPUs()) } // SystemNormalized calculates percentage of total CPU time in the system, normalized by the number of CPU cores. -func (u *cpuUsage) SystemNormalized() float64 { +func (u *CPUUsage) SystemNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.SystemUsage, u.Stats.PreCPUStats.SystemUsage, 1) } @@ -194,7 +194,7 @@ func (u *cpuUsage) SystemNormalized() float64 { // The "oldValue" refers to the CPU statistics of the last read. // Time here is expressed by second and not by nanoseconde. // The main goal is to expose the %, in the same way, it's displayed by docker Client. -func (u *cpuUsage) calculatePercentage(newValue uint64, oldValue uint64, numCPUS uint32) float64 { +func (u *CPUUsage) calculatePercentage(newValue uint64, oldValue uint64, numCPUS uint32) float64 { if newValue < oldValue { logp.Err("Error calculating CPU time change for docker module: new stats value (%v) is lower than the old one(%v)", newValue, oldValue) return -1 diff --git a/metricbeat/module/kibana/_meta/Dockerfile b/metricbeat/module/kibana/_meta/Dockerfile index 850f34e6391..1c34cfd0681 100644 --- a/metricbeat/module/kibana/_meta/Dockerfile +++ b/metricbeat/module/kibana/_meta/Dockerfile @@ -1,4 +1,3 @@ ARG KIBANA_VERSION FROM docker.elastic.co/kibana/kibana:${KIBANA_VERSION} -HEALTHCHECK --interval=1s --retries=300 --start-period=60s CMD python -c 'import urllib, json; response = urllib.urlopen("http://myelastic:changeme@localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);' - +HEALTHCHECK --interval=1s --retries=300 --start-period=60s CMD curl -u myelastic:changeme -f "http://localhost:5601/api/stats?extended=true&legacy=true&exclude_usage=false" | grep '"status":"green"' diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index a6e19d50f42..0335e814fd4 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -19,7 +19,6 @@ package stats import ( "fmt" - "strconv" "strings" "time" @@ -38,10 +37,8 @@ func init() { } const ( - statsPath = "api/stats" - settingsPath = "api/settings" - usageCollectionPeriod = 24 * time.Hour - usageCollectionBackoff = 1 * time.Hour + statsPath = "api/stats" + settingsPath = "api/settings" ) var ( @@ -55,11 +52,9 @@ var ( // MetricSet type defines all fields of the MetricSet type MetricSet struct { *kibana.MetricSet - statsHTTP *helper.HTTP - settingsHTTP *helper.HTTP - usageLastCollectedOn time.Time - usageNextCollectOn time.Time - isUsageExcludable bool + statsHTTP *helper.HTTP + settingsHTTP *helper.HTTP + isUsageExcludable bool } // New create a new instance of the MetricSet @@ -157,31 +152,17 @@ func (m *MetricSet) fetchStats(r mb.ReporterV2, now time.Time) error { var content []byte var err error - // Collect usage stats only once every usageCollectionPeriod + // Add exclude_usage=true if the Kibana Version supports it if m.isUsageExcludable { origURI := m.statsHTTP.GetURI() defer m.statsHTTP.SetURI(origURI) - shouldCollectUsage := m.shouldCollectUsage(now) - m.statsHTTP.SetURI(origURI + "&exclude_usage=" + strconv.FormatBool(!shouldCollectUsage)) - - content, err = m.statsHTTP.FetchContent() - if err != nil { - if shouldCollectUsage { - // When errored in collecting the usage stats it may be counterproductive to try again on the next poll, try to collect the stats again after usageCollectionBackoff - m.usageNextCollectOn = now.Add(usageCollectionBackoff) - } - return err - } + m.statsHTTP.SetURI(origURI + "&exclude_usage=true") + } - if shouldCollectUsage { - m.usageLastCollectedOn = now - } - } else { - content, err = m.statsHTTP.FetchContent() - if err != nil { - return err - } + content, err = m.statsHTTP.FetchContent() + if err != nil { + return err } if m.XPackEnabled { @@ -219,7 +200,3 @@ func (m *MetricSet) fetchSettings(r mb.ReporterV2, now time.Time) { func (m *MetricSet) calculateIntervalMs() int64 { return m.Module().Config().Period.Nanoseconds() / 1000 / 1000 } - -func (m *MetricSet) shouldCollectUsage(now time.Time) bool { - return now.Sub(m.usageLastCollectedOn) > usageCollectionPeriod && now.Sub(m.usageNextCollectOn) > 0 -} diff --git a/metricbeat/module/kibana/stats/stats_test.go b/metricbeat/module/kibana/stats/stats_test.go index 56cbfc17e1f..a7b76603352 100644 --- a/metricbeat/module/kibana/stats/stats_test.go +++ b/metricbeat/module/kibana/stats/stats_test.go @@ -23,7 +23,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" "github.com/stretchr/testify/require" @@ -31,7 +30,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/kibana/mtest" ) -func TestFetchUsage(t *testing.T) { +func TestFetchExcludeUsage(t *testing.T) { // Spin up mock Kibana server numStatsRequests := 0 kib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -45,17 +44,15 @@ func TestFetchUsage(t *testing.T) { // Make GET /api/stats return 503 for first call, 200 for subsequent calls switch numStatsRequests { case 0: // first call - require.Equal(t, "false", excludeUsage) + require.Equal(t, "true", excludeUsage) // exclude_usage is always true w.WriteHeader(503) case 1: // second call - // Make sure exclude_usage is true since first call failed and it should not try again until usageCollectionBackoff time has passed - require.Equal(t, "true", excludeUsage) + require.Equal(t, "true", excludeUsage) // exclude_usage is always true w.WriteHeader(200) case 2: // third call - // Make sure exclude_usage is still true - require.Equal(t, "true", excludeUsage) + require.Equal(t, "true", excludeUsage) // exclude_usage is always true w.WriteHeader(200) } @@ -78,39 +75,25 @@ func TestFetchUsage(t *testing.T) { mbtest.ReportingFetchV2Error(f) } -func TestShouldCollectUsage(t *testing.T) { - now := time.Now() - - cases := map[string]struct { - usageLastCollectedOn time.Time - usageNextCollectOn time.Time - expectedResult bool - }{ - "within_usage_collection_period": { - usageLastCollectedOn: now.Add(-1 * usageCollectionPeriod), - expectedResult: false, - }, - "after_usage_collection_period_but_before_next_scheduled_collection": { - usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod), - usageNextCollectOn: now.Add(3 * time.Hour), - expectedResult: false, - }, - "after_usage_collection_period_and_after_next_scheduled_collection": { - usageLastCollectedOn: now.Add(-2 * usageCollectionPeriod), - usageNextCollectOn: now.Add(-1 * time.Hour), - expectedResult: true, - }, - } - - for name, test := range cases { - t.Run(name, func(t *testing.T) { - m := MetricSet{ - usageLastCollectedOn: test.usageLastCollectedOn, - usageNextCollectOn: test.usageNextCollectOn, - } +func TestFetchNoExcludeUsage(t *testing.T) { + // Spin up mock Kibana server + kib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/status": + w.Write([]byte("{ \"version\": { \"number\": \"7.0.0\" }}")) // v7.0.0 does not support exclude_usage and should not be sent - actualResult := m.shouldCollectUsage(now) - require.Equal(t, test.expectedResult, actualResult) - }) - } + case "/api/stats": + excludeUsage := r.FormValue("exclude_usage") + require.Empty(t, excludeUsage) // exclude_usage should not be provided + w.WriteHeader(200) + } + })) + defer kib.Close() + + config := mtest.GetConfig("stats", kib.URL, true) + + f := mbtest.NewReportingMetricSetV2Error(t, config) + + // First fetch + mbtest.ReportingFetchV2Error(f) } diff --git a/metricbeat/module/logstash/docker-compose.yml b/metricbeat/module/logstash/docker-compose.yml index a776d6d4b66..26b6608c01c 100644 --- a/metricbeat/module/logstash/docker-compose.yml +++ b/metricbeat/module/logstash/docker-compose.yml @@ -2,22 +2,22 @@ version: '2.3' services: logstash: - image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.9.0}-1 + image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.10.0}-1 build: context: ./_meta args: - LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.9.0} + LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.10.0} ports: - 9600 depends_on: - elasticsearch elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.9.0}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.10.0}-1 build: context: ../elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.9.0} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.10.0} environment: - "network.host=" - "transport.host=127.0.0.1" diff --git a/metricbeat/module/logstash/logstash.go b/metricbeat/module/logstash/logstash.go index fbc030fa310..abd737f3ed4 100644 --- a/metricbeat/module/logstash/logstash.go +++ b/metricbeat/module/logstash/logstash.go @@ -57,13 +57,13 @@ type MetricSet struct { XPack bool } -type graph struct { +type Graph struct { Vertices []map[string]interface{} `json:"vertices"` Edges []map[string]interface{} `json:"edges"` } -type graphContainer struct { - Graph *graph `json:"graph,omitempty"` +type GraphContainer struct { + Graph *Graph `json:"graph,omitempty"` Type string `json:"type"` Version string `json:"version"` Hash string `json:"hash"` @@ -74,8 +74,8 @@ type PipelineState struct { ID string `json:"id"` Hash string `json:"hash"` EphemeralID string `json:"ephemeral_id"` - Graph *graphContainer `json:"graph,omitempty"` - Representation *graphContainer `json:"representation"` + Graph *GraphContainer `json:"graph,omitempty"` + Representation *GraphContainer `json:"representation"` BatchSize int `json:"batch_size"` Workers int `json:"workers"` } diff --git a/metricbeat/module/logstash/node/data_xpack.go b/metricbeat/module/logstash/node/data_xpack.go index 96fc252158c..66d3623c7de 100644 --- a/metricbeat/module/logstash/node/data_xpack.go +++ b/metricbeat/module/logstash/node/data_xpack.go @@ -66,21 +66,26 @@ func makeClusterToPipelinesMap(pipelines []logstash.PipelineState, overrideClust var clusterToPipelinesMap map[string][]logstash.PipelineState clusterToPipelinesMap = make(map[string][]logstash.PipelineState) + if overrideClusterUUID != "" { + clusterToPipelinesMap[overrideClusterUUID] = pipelines + return clusterToPipelinesMap + } + for _, pipeline := range pipelines { - var clusterUUIDs []string + clusterUUIDs := common.StringSet{} for _, vertex := range pipeline.Graph.Graph.Vertices { clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) if clusterUUID != "" { - clusterUUIDs = append(clusterUUIDs, clusterUUID) + clusterUUIDs.Add(clusterUUID) } } // If no cluster UUID was found in this pipeline, assign it a blank one if len(clusterUUIDs) == 0 { - clusterUUIDs = []string{""} + clusterUUIDs.Add("") } - for _, clusterUUID := range clusterUUIDs { + for clusterUUID := range clusterUUIDs { clusterPipelines := clusterToPipelinesMap[clusterUUID] if clusterPipelines == nil { clusterToPipelinesMap[clusterUUID] = []logstash.PipelineState{} diff --git a/metricbeat/module/logstash/node/data_xpack_test.go b/metricbeat/module/logstash/node/data_xpack_test.go new file mode 100644 index 00000000000..17ae0aaaf91 --- /dev/null +++ b/metricbeat/module/logstash/node/data_xpack_test.go @@ -0,0 +1,328 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package node + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/metricbeat/module/logstash" +) + +func TestMakeClusterToPipelinesMap(t *testing.T) { + tests := map[string]struct { + pipelines []logstash.PipelineState + overrideClusterUUID string + expectedMap map[string][]logstash.PipelineState + }{ + "no_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "one_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "two_pipelines": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + "no_override_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "", + expectedMap: map[string][]logstash.PipelineState{ + "es_1": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "es_2": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "": { + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) + require.Equal(t, test.expectedMap, actualMap) + }) + } +} diff --git a/metricbeat/module/logstash/node_stats/data_xpack.go b/metricbeat/module/logstash/node_stats/data_xpack.go index a6a9867b7cd..e5d82365b53 100644 --- a/metricbeat/module/logstash/node_stats/data_xpack.go +++ b/metricbeat/module/logstash/node_stats/data_xpack.go @@ -219,20 +219,20 @@ func makeClusterToPipelinesMap(pipelines []PipelineStats, overrideClusterUUID st } for _, pipeline := range pipelines { - var clusterUUIDs []string + clusterUUIDs := common.StringSet{} for _, vertex := range pipeline.Vertices { clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) if clusterUUID != "" { - clusterUUIDs = append(clusterUUIDs, clusterUUID) + clusterUUIDs.Add(clusterUUID) } } // If no cluster UUID was found in this pipeline, assign it a blank one if len(clusterUUIDs) == 0 { - clusterUUIDs = []string{""} + clusterUUIDs.Add("") } - for _, clusterUUID := range clusterUUIDs { + for clusterUUID := range clusterUUIDs { clusterPipelines := clusterToPipelinesMap[clusterUUID] if clusterPipelines == nil { clusterToPipelinesMap[clusterUUID] = []PipelineStats{} diff --git a/metricbeat/module/logstash/node_stats/data_xpack_test.go b/metricbeat/module/logstash/node_stats/data_xpack_test.go new file mode 100644 index 00000000000..6593be72534 --- /dev/null +++ b/metricbeat/module/logstash/node_stats/data_xpack_test.go @@ -0,0 +1,273 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package node_stats + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMakeClusterToPipelinesMap(t *testing.T) { + tests := map[string]struct { + pipelines []PipelineStats + overrideClusterUUID string + expectedMap map[string][]PipelineStats + }{ + "no_vertex_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + "one_vertex_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + "two_pipelines": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + "no_override_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + expectedMap: map[string][]PipelineStats{ + "es_1": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + "es_2": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + "": { + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) + require.Equal(t, test.expectedMap, actualMap) + }) + } +} diff --git a/metricbeat/module/nats/_meta/kibana/7/dashboard/Metricbeat-nats-overview.json b/metricbeat/module/nats/_meta/kibana/7/dashboard/Metricbeat-nats-overview.json index 7ff6121a2f6..99537cafeec 100644 --- a/metricbeat/module/nats/_meta/kibana/7/dashboard/Metricbeat-nats-overview.json +++ b/metricbeat/module/nats/_meta/kibana/7/dashboard/Metricbeat-nats-overview.json @@ -1,12 +1,368 @@ { "objects": [ + { + "attributes": { + "description": "Overview of NATS server status", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "Subscriptions Info" + }, + "gridData": { + "h": 11, + "i": "6", + "w": 24, + "x": 0, + "y": 38 + }, + "panelIndex": "6", + "panelRefName": "panel_0", + "title": "Subscriptions Info", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Server Uptime" + }, + "gridData": { + "h": 11, + "i": "8", + "w": 24, + "x": 24, + "y": 38 + }, + "panelIndex": "8", + "panelRefName": "panel_1", + "title": "Server Uptime", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Subscriptions Stats Timeline" + }, + "gridData": { + "h": 10, + "i": "12", + "w": 13, + "x": 11, + "y": 28 + }, + "panelIndex": "12", + "panelRefName": "panel_2", + "title": "Subscriptions Stats Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Cache Hit Rate Timeline" + }, + "gridData": { + "h": 10, + "i": "18", + "w": 12, + "x": 24, + "y": 28 + }, + "panelIndex": "18", + "panelRefName": "panel_3", + "title": "Cache Hit Rate Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Memory Usage" + }, + "gridData": { + "h": 8, + "i": "4ffa8ccd-bd36-4eaf-973e-688b3025e95c", + "w": 15, + "x": 0, + "y": 0 + }, + "panelIndex": "4ffa8ccd-bd36-4eaf-973e-688b3025e95c", + "panelRefName": "panel_4", + "title": "Memory Usage", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Incoming Bytes Rate" + }, + "gridData": { + "h": 8, + "i": "b07d6b97-c0b5-4663-8507-8d3cc2a63367", + "w": 16, + "x": 15, + "y": 0 + }, + "panelIndex": "b07d6b97-c0b5-4663-8507-8d3cc2a63367", + "panelRefName": "panel_5", + "title": "Incoming Bytes Rate", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Connection Incoming Bytes" + }, + "gridData": { + "h": 8, + "i": "eb420bb0-754e-4544-bc1f-027568db1c8c", + "w": 17, + "x": 31, + "y": 0 + }, + "panelIndex": "eb420bb0-754e-4544-bc1f-027568db1c8c", + "panelRefName": "panel_6", + "title": "Connection Incoming Bytes", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Incoming Messages Rate" + }, + "gridData": { + "h": 10, + "i": "1ed3e570-4ece-42a4-92b1-fdc19e3e1ad5", + "w": 15, + "x": 0, + "y": 8 + }, + "panelIndex": "1ed3e570-4ece-42a4-92b1-fdc19e3e1ad5", + "panelRefName": "panel_7", + "title": "Incoming Messages Rate", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Connections" + }, + "gridData": { + "h": 10, + "i": "ed6181bc-4274-400a-a9d8-a84a027a4a77", + "w": 16, + "x": 15, + "y": 8 + }, + "panelIndex": "ed6181bc-4274-400a-a9d8-a84a027a4a77", + "panelRefName": "panel_8", + "title": "Connections", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Connections Uptime" + }, + "gridData": { + "h": 10, + "i": "7862e4cd-22db-493b-a3be-247570eaaa8a", + "w": 17, + "x": 31, + "y": 8 + }, + "panelIndex": "7862e4cd-22db-493b-a3be-247570eaaa8a", + "panelRefName": "panel_9", + "title": "Connections Uptime", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Total Routes" + }, + "gridData": { + "h": 10, + "i": "5c6f0fdf-67e6-4a39-8543-d46c3f833ac6", + "w": 15, + "x": 0, + "y": 18 + }, + "panelIndex": "5c6f0fdf-67e6-4a39-8543-d46c3f833ac6", + "panelRefName": "panel_10", + "title": "Total Routes", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Total Connections" + }, + "gridData": { + "h": 10, + "i": "258a5243-55f6-4195-90cb-ef3ec94707db", + "w": 16, + "x": 15, + "y": 18 + }, + "panelIndex": "258a5243-55f6-4195-90cb-ef3ec94707db", + "panelRefName": "panel_11", + "title": "Total Connections", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "title": "Connection Pending Bytes" + }, + "gridData": { + "h": 10, + "i": "0e68fd8a-abd9-4391-b2d0-026e79714835", + "w": 17, + "x": 31, + "y": 18 + }, + "panelIndex": "0e68fd8a-abd9-4391-b2d0-026e79714835", + "panelRefName": "panel_12", + "title": "Connection Pending Bytes", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Slow Consumers" + }, + "gridData": { + "h": 10, + "i": "3141f1f6-d2d1-4b3f-8a7a-7d915bcb5d7c", + "w": 11, + "x": 0, + "y": 28 + }, + "panelIndex": "3141f1f6-d2d1-4b3f-8a7a-7d915bcb5d7c", + "panelRefName": "panel_13", + "title": "Slow Consumers", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Subscription Cache Actions" + }, + "gridData": { + "h": 10, + "i": "66f0d0ac-bf45-40e3-ba8c-32d6360e8584", + "w": 12, + "x": 36, + "y": 28 + }, + "panelIndex": "66f0d0ac-bf45-40e3-ba8c-32d6360e8584", + "panelRefName": "panel_14", + "title": "Subscription Cache Actions", + "version": "7.10.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat NATS] Overview ECS", + "version": 1 + }, + "id": "Metricbeat-Nats-Dashboard-ecs", + "migrationVersion": { + "dashboard": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "b129b220-1e44-11e9-a1b4-79a7ae42ab61-ecs", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "206f1bc0-1e45-11e9-a1b4-79a7ae42ab61-ecs", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "754215c0-1e46-11e9-a1b4-79a7ae42ab61-ecs", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "dff743a0-1f1c-11e9-a673-d9577e5e50eb-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "b877eb90-2988-11eb-8245-71f739a9f622", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "d3142a50-2987-11eb-8245-71f739a9f622", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "5146f2a0-2987-11eb-8245-71f739a9f622", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "eeb33da0-2987-11eb-8245-71f739a9f622", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "431edfc0-2988-11eb-8245-71f739a9f622", + "name": "panel_8", + "type": "visualization" + }, + { + "id": "898d2fe0-2986-11eb-8245-71f739a9f622", + "name": "panel_9", + "type": "visualization" + }, + { + "id": "3a670a80-2986-11eb-8245-71f739a9f622", + "name": "panel_10", + "type": "visualization" + }, + { + "id": "55c2d340-2986-11eb-8245-71f739a9f622", + "name": "panel_11", + "type": "visualization" + }, + { + "id": "68d40020-2987-11eb-8245-71f739a9f622", + "name": "panel_12", + "type": "visualization" + }, + { + "id": "84e60a90-2a79-11eb-952d-594e5c56d011", + "name": "panel_13", + "type": "visualization" + }, + { + "id": "d80d4c30-2a81-11eb-9625-31ed579c09b3", + "name": "panel_14", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2020-11-19T16:12:16.796Z", + "version": "WzM3MDAsMV0=" + }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "metricbeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" @@ -125,9 +481,22 @@ } }, "id": "b129b220-1e44-11e9-a1b4-79a7ae42ab61-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-24T07:54:30.301Z", - "version": 3 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1MywxXQ==" }, { "attributes": { @@ -135,14 +504,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "metricbeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" } } }, - "title": "Current Memory Usage [Metricbeat NATS] ECS", + "title": "Server Uptime [Metricbeat NATS] ECS", "uiStateJSON": {}, "version": 1, "visState": { @@ -151,77 +520,27 @@ "enabled": true, "id": "1", "params": { - "customLabel": "Memory (Bytes)", - "field": "nats.stats.mem.bytes" + "customLabel": "Server Uptime", + "field": "nats.stats.uptime" }, "schema": "metric", "type": "avg" - } - ], - "params": { - "addLegend": false, - "addTooltip": true, - "metric": { - "colorSchema": "Green to Red", - "colorsRange": [ - { - "from": 0, - "to": 10000 - } - ], - "invertColors": false, - "labels": { - "show": true - }, - "metricColorMode": "None", - "percentageMode": false, - "style": { - "bgColor": false, - "bgFill": "#000", - "fontSize": 42, - "labelColor": false, - "subText": "" - }, - "useRanges": false }, - "type": "metric" - }, - "title": "Current Memory Usage [Metricbeat NATS] ECS", - "type": "metric" - } - }, - "id": "30a61c00-1e45-11e9-a1b4-79a7ae42ab61-ecs", - "type": "visualization", - "updated_at": "2019-01-24T07:56:32.097Z", - "version": 4 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } - }, - "title": "Server Uptime [Metricbeat NATS] ECS", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ { "enabled": true, - "id": "1", + "id": "2", "params": { - "customLabel": "Server Uptime", - "field": "nats.stats.uptime" + "field": "nats.server.id", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 }, - "schema": "metric", - "type": "avg" + "schema": "group", + "type": "terms" } ], "params": { @@ -257,9 +576,22 @@ } }, "id": "206f1bc0-1e45-11e9-a1b4-79a7ae42ab61-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-24T07:57:04.084Z", - "version": 4 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1NCwxXQ==" }, { "attributes": { @@ -267,14 +599,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "metricbeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" } } }, - "title": "Total Connections [Metricbeat NATS] ECS", + "title": "Subscription Stats Timeline [Metricbeat NATS] ECS", "uiStateJSON": {}, "version": 1, "visState": { @@ -283,74 +615,8 @@ "enabled": true, "id": "1", "params": { - "customLabel": "Total Connections", - "field": "nats.stats.total_connections" - }, - "schema": "metric", - "type": "avg" - } - ], - "params": { - "addLegend": false, - "addTooltip": true, - "metric": { - "colorSchema": "Green to Red", - "colorsRange": [ - { - "from": 0, - "to": 10000 - } - ], - "invertColors": false, - "labels": { - "show": true - }, - "metricColorMode": "None", - "percentageMode": false, - "style": { - "bgColor": false, - "bgFill": "#000", - "fontSize": 42, - "labelColor": false, - "subText": "" - }, - "useRanges": false - }, - "type": "metric" - }, - "title": "Total Connections [Metricbeat NATS] ECS", - "type": "metric" - } - }, - "id": "4c380ff0-1e45-11e9-a1b4-79a7ae42ab61-ecs", - "type": "visualization", - "updated_at": "2019-01-24T07:57:32.006Z", - "version": 4 - }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } - }, - "title": "Remotes-Subsz-Connz-Routez Timeline [Metricbeat NATS] ECS", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Connections", - "field": "nats.connections.total" + "customLabel": "Cache Fanout Avg", + "field": "nats.subscriptions.cache.fanout.avg" }, "schema": "metric", "type": "avg" @@ -359,11 +625,17 @@ "enabled": true, "id": "2", "params": { - "customInterval": "2h", + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 0, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" @@ -372,28 +644,8 @@ "enabled": true, "id": "3", "params": { - "customLabel": "Routes", - "field": "nats.routes.total" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "4", - "params": { - "customLabel": "Remotes", - "field": "nats.stats.remotes" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "5", - "params": { - "customLabel": "Subscriptions", - "field": "nats.subscriptions.total" + "customLabel": "Cache Fanout Max", + "field": "nats.subscriptions.cache.fanout.max" }, "schema": "metric", "type": "avg" @@ -426,63 +678,43 @@ "color": "#eee" } }, + "labels": {}, "legendPosition": "right", "seriesParams": [ { "data": { "id": "1", - "label": "Connections" + "label": "Cache Fanout Avg" }, "drawLinesBetweenPoints": true, - "interpolate": "linear", - "mode": "stacked", + "mode": "normal", "show": "true", "showCircles": true, - "type": "area", + "type": "line", "valueAxis": "ValueAxis-1" }, { "data": { "id": "3", - "label": "Routes" - }, - "drawLinesBetweenPoints": true, - "interpolate": "linear", - "mode": "stacked", - "show": true, - "showCircles": true, - "type": "area", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "4", - "label": "Remotes" - }, - "drawLinesBetweenPoints": true, - "interpolate": "linear", - "mode": "stacked", - "show": true, - "showCircles": true, - "type": "area", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "5", - "label": "Subscriptions" + "label": "Cache Fanout Max" }, "drawLinesBetweenPoints": true, - "interpolate": "linear", - "mode": "stacked", + "mode": "normal", "show": true, "showCircles": true, - "type": "area", + "type": "line", "valueAxis": "ValueAxis-1" } ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], - "type": "area", + "type": "line", "valueAxes": [ { "id": "ValueAxis-1", @@ -501,20 +733,33 @@ "show": true, "style": {}, "title": { - "text": "Connections" + "text": "Cache Fanout Avg" }, "type": "value" } ] }, - "title": "Remotes-Subsz-Connz-Routez Timeline [Metricbeat NATS] ECS", - "type": "area" + "title": "Subscription Stats Timeline [Metricbeat NATS] ECS", + "type": "line" } }, - "id": "199d3d30-1e46-11e9-a1b4-79a7ae42ab61-ecs", + "id": "754215c0-1e46-11e9-a1b4-79a7ae42ab61-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-24T07:53:31.785Z", - "version": 3 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1NSwxXQ==" }, { "attributes": { @@ -522,14 +767,14 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "metricbeat-*", + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", "query": "" } } }, - "title": "Subscription Stats Timeline [Metricbeat NATS] ECS", + "title": "Cache Hit Rate Timeline [Metricbeat NATS] ECS", "uiStateJSON": {}, "version": 1, "visState": { @@ -538,8 +783,8 @@ "enabled": true, "id": "1", "params": { - "customLabel": "Cache Fanout Avg", - "field": "nats.subscriptions.cache.fanout.avg" + "customLabel": "Cache Hit Rate", + "field": "nats.subscriptions.cache.hit_rate" }, "schema": "metric", "type": "avg" @@ -548,7 +793,6 @@ "enabled": true, "id": "2", "params": { - "customInterval": "2h", "extended_bounds": {}, "field": "@timestamp", "interval": "auto", @@ -556,46 +800,6 @@ }, "schema": "segment", "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "customLabel": "Cache Fanout Max", - "field": "nats.subscriptions.cache.fanout.max" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "5", - "params": { - "customLabel": "Inserts", - "field": "nats.subscriptions.inserts" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "6", - "params": { - "customLabel": "Removes", - "field": "nats.subscriptions.removes" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "7", - "params": { - "customLabel": "Matches", - "field": "nats.subscriptions.matches" - }, - "schema": "metric", - "type": "avg" } ], "params": { @@ -630,7 +834,7 @@ { "data": { "id": "1", - "label": "Cache Fanout Avg" + "label": "Cache Hit Rate" }, "drawLinesBetweenPoints": true, "mode": "normal", @@ -638,54 +842,6 @@ "showCircles": true, "type": "line", "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "3", - "label": "Cache Fanout Max" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "5", - "label": "Inserts" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "6", - "label": "Removes" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "7", - "label": "Matches" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" } ], "times": [], @@ -708,659 +864,701 @@ "show": true, "style": {}, "title": { - "text": "Cache Fanout Avg" + "text": "Cache Hit Rate (%)" }, "type": "value" } ] }, - "title": "Subscription Stats Timeline [Metricbeat NATS] ECS", + "title": "Cache Hit Rate Timeline [Metricbeat NATS] ECS", "type": "line" } }, - "id": "754215c0-1e46-11e9-a1b4-79a7ae42ab61-ecs", + "id": "dff743a0-1f1c-11e9-a673-d9577e5e50eb-ecs", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], "type": "visualization", - "updated_at": "2019-01-23T14:55:04.899Z", - "version": 3 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1NiwxXQ==" }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, - "title": "Slow Consumers Timeline [Metricbeat NATS] ECS", + "title": "Memory Usage [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Slow Consumers", - "field": "nats.stats.slow_consumers" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "bytes", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Memory Usage", + "line_width": 2, + "metrics": [ + { + "field": "nats.stats.mem.bytes", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } - }, - "legendPosition": "right", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "Slow Consumers" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - } - ], - "times": [], - "type": "line", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "Slow Consumers" - }, - "type": "value" - } - ] + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "Slow Consumers Timeline [Metricbeat NATS] ECS", - "type": "line" + "title": "Memory Usage [Metricbeat NATS]", + "type": "metrics" } }, - "id": "94534190-1e97-11e9-b9e7-93b3bd2eec90-ecs", + "id": "b877eb90-2988-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-23T14:53:57.137Z", - "version": 2 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1NywxXQ==" }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, - "title": "IO Bytes Stats [Metricbeat NATS] ECS", + "title": "Incoming Bytes Rate [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "In Bytes", - "field": "nats.stats.in.bytes" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customLabel": "Out Bytes", - "field": "nats.stats.out.bytes" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "3", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" - } - ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "legendPosition": "right", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "In Bytes" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "data": { - "id": "2", - "label": "Out Bytes" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "bytes", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Incoming Bytes Rate", + "line_width": 2, + "metrics": [ + { + "field": "nats.stats.in.bytes", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "positive_rate" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "times": [], - "type": "line", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "IO Bytes" - }, - "type": "value" - } - ] + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "IO Bytes Stats [Metricbeat NATS] ECS", - "type": "line" + "title": "Incoming Bytes Rate [Metricbeat NATS]", + "type": "metrics" } }, - "id": "be1d8a20-1e98-11e9-b9e7-93b3bd2eec90-ecs", + "id": "d3142a50-2987-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-24T07:48:22.914Z", - "version": 4 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1OCwxXQ==" }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, - "title": "Memory Utilization Timeline [Metricbeat NATS] ECS", + "title": "Connection Incoming Bytes [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Memory Avg", - "field": "nats.stats.mem.bytes" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" - } - ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "legendPosition": "right", - "seriesParams": [ + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "data": { - "id": "1", - "label": "Memory Avg" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "bytes", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Connection Incoming Bytes", + "line_width": 2, + "metrics": [ + { + "field": "nats.connection.in.bytes", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.connection.name", + "type": "timeseries", + "value_template": "{{value}}" } ], - "times": [], - "type": "line", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "Memory Avg (Bytes)" - }, - "type": "value" - } - ] + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "Memory Utilization Timeline [Metricbeat NATS] ECS", - "type": "line" + "title": "Connection Incoming Bytes [Metricbeat NATS]", + "type": "metrics" } }, - "id": "8204e820-1e99-11e9-b9e7-93b3bd2eec90-ecs", + "id": "5146f2a0-2987-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-24T07:52:55.445Z", - "version": 5 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI1OSwxXQ==" }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, - "title": "IO Messages Stats [Metricbeat NATS] ECS", + "title": "Incoming Messages Rate [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "In Messages", - "field": "nats.stats.in.messages" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customLabel": "Out Messages", - "field": "nats.stats.out.messages" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "3", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Incoming Messages Rate", + "line_width": 2, + "metrics": [ + { + "field": "nats.stats.in.messages", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "positive_rate", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" + }, + "title": "Incoming Messages Rate [Metricbeat NATS]", + "type": "metrics" + } + }, + "id": "eeb33da0-2987-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2MCwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Connections [Metricbeat NATS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "legendPosition": "right", - "seriesParams": [ + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "data": { - "id": "1", - "label": "In Messages" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" - }, - { - "data": { - "id": "2", - "label": "Out Messages" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": true, - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Connections", + "line_width": 2, + "metrics": [ + { + "field": "nats.stats.total_connections", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "positive_rate", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "times": [], - "type": "line", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "IO Messages" - }, - "type": "value" - } - ] + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "IO Messages Stats [Metricbeat NATS] ECS", - "type": "line" + "title": "Connections [Metricbeat NATS]", + "type": "metrics" } }, - "id": "cdbf4110-1f0d-11e9-a673-d9577e5e50eb-ecs", + "id": "431edfc0-2988-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-24T07:47:25.774Z", - "version": 2 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2MSwxXQ==" }, { "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "index": "metricbeat-*", - "query": { + "searchSourceJSON": {} + }, + "title": "Connections Uptime [Metricbeat NATS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { "language": "kuery", "query": "" - } - } + }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "s,s,", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Connection Uptime", + "line_width": 2, + "metrics": [ + { + "field": "nats.connection.uptime", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.connection.name", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" + }, + "title": "Connections Uptime [Metricbeat NATS]", + "type": "metrics" + } + }, + "id": "898d2fe0-2986-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2MiwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} }, - "title": "CPU Utilization Timeline [Metricbeat NATS] ECS", + "title": "Total Routes [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "CPU Avg", - "field": "nats.stats.cpu" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Routes", + "line_width": 2, + "metrics": [ + { + "field": "nats.routes.total", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "value_template": "{{value}}" } ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" + }, + "title": "Total Routes [Metricbeat NATS]", + "type": "metrics" + } + }, + "id": "3a670a80-2986-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2MywxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Total Connections [Metricbeat NATS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "legendPosition": "right", - "seriesParams": [ + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "data": { - "id": "1", - "label": "CPU Avg" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Connections", + "line_width": 2, + "metrics": [ + { + "field": "nats.connections.total", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "times": [], - "type": "line", - "valueAxes": [ + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" + }, + "title": "Total Connections [Metricbeat NATS]", + "type": "metrics" + } + }, + "id": "55c2d340-2986-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2NCwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Connection Pending Bytes [Metricbeat NATS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "CPU Avg (%)" - }, - "type": "value" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "bytes", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Connection Pending Bytes", + "line_width": 2, + "metrics": [ + { + "field": "nats.connection.pending_bytes", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.connection.name", + "type": "timeseries", + "value_template": "{{value}}" } - ] + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "CPU Utilization Timeline [Metricbeat NATS] ECS", - "type": "line" + "title": "Connection Pending Bytes [Metricbeat NATS]", + "type": "metrics" } }, - "id": "138dc660-1f1a-11e9-a673-d9577e5e50eb-ecs", + "id": "68d40020-2987-11eb-8245-71f739a9f622", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-24T07:51:51.767Z", - "version": 2 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2NSwxXQ==" }, { "attributes": { @@ -1368,322 +1566,236 @@ "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "index": "metricbeat-*", "query": { "language": "kuery", "query": "" } } }, - "title": "Cache Hit Rate Timeline [Metricbeat NATS] ECS", + "title": "Slow Consumers [Metricbeat NATS]", "uiStateJSON": {}, "version": 1, "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "Cache Hit Rate", - "field": "nats.subscriptions.cache.hit_rate" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], + "aggs": [], "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" - } - ], - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "legendPosition": "right", - "seriesParams": [ + "id": "e4c53250-2985-11eb-9192-5db805fbad79", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ { - "data": { - "id": "1", - "label": "Cache Hit Rate" - }, - "drawLinesBetweenPoints": true, - "mode": "normal", - "show": "true", - "showCircles": true, - "type": "line", - "valueAxis": "ValueAxis-1" + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "e4c53251-2985-11eb-9192-5db805fbad79", + "label": "Slow Consumers", + "line_width": 2, + "metrics": [ + { + "field": "nats.stats.slow_consumers", + "id": "e4c53252-2985-11eb-9192-5db805fbad79", + "type": "avg", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "nats.server.id", + "type": "timeseries", + "value_template": "{{value}}" } ], - "times": [], - "type": "line", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "Cache Hit Rate (%)" - }, - "type": "value" - } - ] + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - "title": "Cache Hit Rate Timeline [Metricbeat NATS] ECS", - "type": "line" + "title": "Slow Consumers [Metricbeat NATS]", + "type": "metrics" } }, - "id": "dff743a0-1f1c-11e9-a673-d9577e5e50eb-ecs", + "id": "84e60a90-2a79-11eb-952d-594e5c56d011", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], "type": "visualization", - "updated_at": "2019-01-23T14:57:20.994Z", - "version": 2 + "updated_at": "2020-11-19T15:52:48.969Z", + "version": "WzI2NiwxXQ==" }, { "attributes": { - "description": "Overview of NATS server status", - "hits": 0, + "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { "filter": [], - "highlightAll": true, "query": { "language": "kuery", "query": "" - }, - "version": true + } } }, - "optionsJSON": { - "darkTheme": false, - "hidePanelTitles": false, - "useMargins": true - }, - "panelsJSON": [ - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "6", - "w": 24, - "x": 0, - "y": 45 - }, - "id": "b129b220-1e44-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "6", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 7, - "i": "7", - "w": 13, - "x": 24, - "y": 34 - }, - "id": "30a61c00-1e45-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "7", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 7, - "i": "8", - "w": 11, - "x": 37, - "y": 34 - }, - "id": "206f1bc0-1e45-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "8", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 8, - "i": "9", - "w": 8, - "x": 24, - "y": 41 - }, - "id": "4c380ff0-1e45-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "9", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 11, - "i": "11", - "w": 24, - "x": 0, - "y": 34 - }, - "id": "199d3d30-1e46-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "11", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 10, - "i": "12", - "w": 18, - "x": 15, - "y": 0 - }, - "id": "754215c0-1e46-11e9-a1b4-79a7ae42ab61-ecs", - "panelIndex": "12", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 10, - "i": "13", - "w": 15, - "x": 0, - "y": 0 - }, - "id": "94534190-1e97-11e9-b9e7-93b3bd2eec90-ecs", - "panelIndex": "13", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "14", - "w": 24, - "x": 24, - "y": 10 + "title": "Subscription Cache Actions [Metricbeat NATS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" }, - "id": "be1d8a20-1e98-11e9-b9e7-93b3bd2eec90-ecs", - "panelIndex": "14", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": { - "vis": { - "legendOpen": true + "id": "a9b96760-2a81-11eb-8cd4-770b42226f97", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "a9b96761-2a81-11eb-8cd4-770b42226f97", + "label": "rate(inserts)", + "line_width": 2, + "metrics": [ + { + "field": "nats.subscriptions.inserts", + "id": "a9b96762-2a81-11eb-8cd4-770b42226f97", + "type": "max" + }, + { + "field": "a9b96762-2a81-11eb-8cd4-770b42226f97", + "id": "a9b96764-2a81-11eb-8cd4-770b42226f97", + "type": "derivative", + "unit": "1s" + }, + { + "field": "a9b96764-2a81-11eb-8cd4-770b42226f97", + "id": "a9b96763-2a81-11eb-8cd4-770b42226f97", + "type": "positive_only" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}}/s" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#D36086", + "fill": 0, + "formatter": "number", + "id": "a9b96765-2a81-11eb-8cd4-770b42226f97", + "label": "rate(removes)", + "line_width": 2, + "metrics": [ + { + "field": "nats.subscriptions.removes", + "id": "a9b96766-2a81-11eb-8cd4-770b42226f97", + "type": "max" + }, + { + "field": "a9b96766-2a81-11eb-8cd4-770b42226f97", + "id": "a9b96768-2a81-11eb-8cd4-770b42226f97", + "type": "derivative", + "unit": "1s" + }, + { + "field": "a9b96768-2a81-11eb-8cd4-770b42226f97", + "id": "a9b96767-2a81-11eb-8cd4-770b42226f97", + "type": "positive_only" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}}/s" + }, + { + "axis_position": "right", + "chart_type": "line", + "color": "#54B399", + "fill": 0, + "formatter": "number", + "id": "a9b96769-2a81-11eb-8cd4-770b42226f97", + "label": "rate(matches)", + "line_width": 2, + "metrics": [ + { + "field": "nats.subscriptions.matches", + "id": "a9b9676a-2a81-11eb-8cd4-770b42226f97", + "type": "max" + }, + { + "field": "a9b9676a-2a81-11eb-8cd4-770b42226f97", + "id": "a9b9676c-2a81-11eb-8cd4-770b42226f97", + "type": "derivative", + "unit": "1s" + }, + { + "field": "a9b9676c-2a81-11eb-8cd4-770b42226f97", + "id": "a9b9676b-2a81-11eb-8cd4-770b42226f97", + "type": "positive_only" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "value_template": "{{value}}/s" } - }, - "gridData": { - "h": 12, - "i": "15", - "w": 24, - "x": 24, - "y": 22 - }, - "id": "8204e820-1e99-11e9-b9e7-93b3bd2eec90-ecs", - "panelIndex": "15", - "type": "visualization", - "version": "6.5.4" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "16", - "w": 24, - "x": 0, - "y": 10 - }, - "id": "cdbf4110-1f0d-11e9-a673-d9577e5e50eb-ecs", - "panelIndex": "16", - "type": "visualization", - "version": "6.3.2" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "17", - "w": 24, - "x": 0, - "y": 22 - }, - "id": "138dc660-1f1a-11e9-a673-d9577e5e50eb-ecs", - "panelIndex": "17", - "type": "visualization", - "version": "6.3.2" + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "tooltip_mode": "show_all", + "type": "timeseries" }, - { - "embeddableConfig": {}, - "gridData": { - "h": 10, - "i": "18", - "w": 15, - "x": 33, - "y": 0 - }, - "id": "dff743a0-1f1c-11e9-a673-d9577e5e50eb-ecs", - "panelIndex": "18", - "type": "visualization", - "version": "6.3.2" - } - ], - "timeRestore": false, - "title": "[Metricbeat NATS] Overview ECS", - "version": 1 + "title": "Subscription Cache Actions [Metricbeat NATS]", + "type": "metrics" + } }, - "id": "Metricbeat-Nats-Dashboard-ecs", - "type": "dashboard", - "updated_at": "2019-01-24T08:13:29.732Z", - "version": 4 + "id": "d80d4c30-2a81-11eb-9625-31ed579c09b3", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-11-19T16:11:12.882Z", + "version": "WzMyNjEsMV0=" } ], - "version": "6.3.2" + "version": "7.10.0" } diff --git a/metricbeat/module/system/diskio/_meta/fields.yml b/metricbeat/module/system/diskio/_meta/fields.yml index 0aadb9ed666..969af0b822b 100644 --- a/metricbeat/module/system/diskio/_meta/fields.yml +++ b/metricbeat/module/system/diskio/_meta/fields.yml @@ -55,6 +55,11 @@ type: long description: > The total number of of milliseconds spent doing I/Os. + + - name: io.ops + type: long + description: > + The total number of I/Os in progress. - name: iostat.read.request.merges_per_sec type: float diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 793d2d23682..30578f972ad 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -107,6 +107,11 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { }, } + // Add linux-only ops in progress + if runtime.GOOS == "linux" { + event.Put("io.ops", counters.IopsInProgress) + } + // accumulate values from all interfaces diskReadBytes += counters.ReadBytes diskWriteBytes += counters.WriteBytes diff --git a/metricbeat/module/system/fields.go b/metricbeat/module/system/fields.go index a4f53a7c489..e04c8bb3678 100644 --- a/metricbeat/module/system/fields.go +++ b/metricbeat/module/system/fields.go @@ -32,5 +32,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded gzipped contents of module/system. func AssetSystem() string { - return "eJzsff+PGzey5+/5KwgfFhm/m5E93mzevvnhAMfzcjeAEw9sB/uAw0GmuksS37DJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/9YXFqmKxeEOeYHdH9E4bSL4jxDDD4Y68+IS/ePEdITHoSLHUMCnuyP/6jhBC3B+JNtRkmiRgFIv0NeHsCci7x98IFTFJIJFqRzJNV3BNzJoaQhWQSHIOkYGYLJVMiFkDkSkoaphYeRSz7wjRa6nMPJJiyVZ3xKgMviNEAQeq4Y6s6HeELBnwWN8hoBsiaAIVMuyP2aX2s0pmqf9NgBT788V97QuJpDCUCU24jCj3o+X0zfznq/NW546kguKXodn3IKiguLHjVKBYfnoEZCkVoUQzseKA8xG5JJQkGTcMv1fhYP5TZ1r+0ySiSgiLa7/OSeFSrBp/2EON/bHQ31lUIksWoEpUtU/+D/IIKgJh6Ap0EFCmQc3SyARh6YhyiOdLLmnzA0upEmruSOrGHwf+8xryL9IVMtqSY1gCRKcgDGECgRGd0gg6aKtRYFj0pKdhrQVHE5kJcyQwry+XyNwnUAL4GComZHAvh0egEyyCy+OwFITL7U2qmFTM7EiqZARagx5Czdk4fShKFvML5DmiGgD8fIo8AJDcUmYukJeCWGDkSgoSM/30chgd57QR4/Cp3y+PyRrUhkXWNbMu3ZqKmNv/WFMVb603x4QBpbLU9K5H9fv5WD8Zai2X5luSi8V7GIXPLZsDkBug/PIkwwRhYiN5JgxVO2cCFjuMczZMmYxy/MZ2zTjgb9e71LJES9WabEt1jV/SrEHlW6BUs9YX3m4o43TBgUjBd3bz/E2wr4MYeU67eLkMKmK5NDsqlIvSrBVNWqpsxKyPi85smDeloFxslgsKRyepAu29L5SA1GbmPizFjbDrh7M/oBkmksrK0GTLOCdrugEboNKvLMkSsqE8w0Xz5fb167+Rf3PTfcGxW4OV89TGpVwBjXfE0CerH0z7UZkwktAoQrVztmXTHjSAxUL5U4em5INopwj0dWvYncxIRIUTWpXlRfJmpYAaUPYXwvGN/CwVga80STlcE7Ykf28N61TKfp0a8uPrv1lo11avnHL5tMcsSrNZzs0vTnsWQG7/2SmcP1cI++cKEr/d8OvPEu18Q17rX355gMK/vNtpvFsjzYUy0vqCoIkjG3fUh5gDKs7Dh39ZK9TllPxaekaD/BPrSV0kC8amqS+WkLEb/WUSctRuf5kkDd/yLxT/Afv+ZVIy+eb/TZF5qAdwmUR+q27ApXFziBdwnSdCNMQ5k8ucDQbXAdobHsPnVnbvWzmZvuQz3W/jFPQCDxMv+hDuuY9CDt8Rnxv5oZvcX2cPVZ5YPWXyuyYrxhw/2CEq5w/2P8nDh6KMbGANXv4z/ozC/jMozyfYbaVqHhz4/PEd0TG9HS9uJM9O2adsoBjlc7d5joA3EML32s+Ql7uRz2umSUJ3REhDFmCVY8Nit41Tzkumt8b0OfoeghTQeIYHHhMuHvSUKh6GncSqjJWQVRmdRVbDlxnnux58W8UMnBwgznIgQuTgYmeGn6jlrmDoSweAx2EQRh02+SDIeyayr+6IizWnIg0/UENkpPIj4WFPypnXNEGo1lliOYOfIpr9gX7oP27fDJLg8zPI4jAgpuFRPthANrVG7WcbqpXdd06o9gnjNiaIpIi13968WcEVO0iwzwbRrdleZ/HUAMMYY2n3wYdXH/oB2uhthtJW8HsG2swSUCvQ8xTUXEMUxB6KMHvAN4/qcZn7KTXBOfGUnDhK3IntFhSQ3zPIICZG4mKIYcN6YxtPllOR89KFc56asJq8ziqoEj3TuoW+QucBAjqvZKalBCXiCdiz20xAxk/lflv4vi3MTd89vKX1EkRtfDEtIXQDiq6gGtMspWpoWVAiRloP1AYsEI9Z/2eUilOxU4rFkXQ+uTQWzUSCyVc83azm1kc5DSno/Vwx4dj70orJoh5oAYZRgjb8xHTgHISDWJn1SYg45zKfVpFc+gLmnV7W8UrkZnCEWGWqulsvkaiHVx+mlcci07vpqHkMZ+7jTFkncbtm0bpOQvemeLWgIt6y2KxJZhhnf1A7LTKh/NTLGbl3H9fUZMp9REZRZgMXVzNXljxqEnGpUfT1KsacJSCMkunumGRSmbby1yHbY45PENF80PmCmUlTfwVaO7AVWRtuCeP5j4JKvB7nteUmNWwDufakUvIiZP/h9X/82JLyknGo3XwlB2UNy2Fatcvln6YoYS6IPlNOAROEeKpT4beRNuTPRKrYhnGwcQaeTeU73iwI3S3S+cgE56gkZrWk9o58eRXD5pX96+2XICI77wmg2DGaUOCr+SEMAhPu81SyjkzfwVhwYGtpcewWb8JoUFtPmDaw4xMhY9BWW+waxd+0M+cVSAqeVdv3a7VFN5+aaxV+KYBDmIZ8PxPXnIwrvNvPsUzDeRPHdsKR8J5/d2uA3lenUKiithvMUfuYVyk3UmUrq2xi+UkYXa0UrGhxFEY5dyancbml/OrRt3cOPQz5tW5+PBqylFkzMq4tnyOW9eeA2dMz8qvEgoR/MRHLbYf+uakDQd2+VRCWdJsRoFFeJRdILCPdyg4EpED2W+S9rOlD38IZiiaIZypaxMaaaAK0i+fZAOLK7QEYMs/nQ+jM4BUCTXmmkacv2yEQlzQ+xpzYkM+Okce0RxqAF7cvxhpl+ycmVvMljYxUdzbUG2eY31fgF+Emp9qQhInMQHgNv/jHJSH9h8faYXBe3F4U2tsA3DBuLEl8Lp0I6AKJWVGjMKzUsE3Oc4kirDBTUPRs2tWhVUfShB8KU3TaS8Qt6+y6hB3l7rkhWikL339sgnTF2cIQ3Ncc7v5+UucLP36zW+wgWGeMcl28Vlb6oUflZV7ERq4CyyVLYwkaC7GYiHgWFx+OpHBVH4td7k5GNFqDJlS0/a9FtlyC0uRKQxG7etbQyGSUzxpuyMWHZ4ME62g7zF9vI3mLo5UdAiHGOlLLuT4vfq+3HFwR5AweqSeows+KDj4YosAbQ+0y/cwqEYgIyALMFvxNeK/SWOVQzd14CQWbJNif5idJDCmIWOeW98MnlzdLpAISg6GM62uSohkk0RqipyJmrujwlw6VIM8fQ3l2h5f8g8FzEcqjjGNgv6BWLBVe1MvGnHXI+w38Akl54IEpgVepktGrBBImlvK6zQv7I1V1QvxaFRyGJ6VRKYwIW9ZHt8CthSoE2o697M8HQT58+i/CkFBKdJY0DWCuQ0zQCI8SchX6UMTt1/778Ht7YXspykIt/NeHqkWHeSNDTBzpNXNkYJTYPmtprdK+guEtbVq2bpuXKliyr3fkxf9Fsv5f072qp1as5uEopdtiPRWmDYu0OwIqzw8tjlo/1VybQ8nT/szHM8ftJTFDVem5zDo6PuPwPpdFLA9pR8GVmZmlrcvjAzDXMEW5E4ZDIYTUmtzM9CNg4nQAmOifXwGN6Rrrz46GgbwvBiT1AQcgwC1idM5vHwQckayrZ+zfmtXOhi3C4kifrmCOQd9h3mq+ZWP1SmGRR1rYdKUjKuZPFvaBipVzM3hbpYXa631EhXCRjJu6D2DMFEQHWoBjALp5ebNMp4oPg4GzAbOzFcmUVilFi3cGKD+jdMvsikOrIOKUJUMljWjPJ+putL1idx+Yw3LJIgYi2p3RHrm5c7SkxFCxRzPylnC5BVW1UUzELMJL3KXyWM9aG5WtVngx0shi3KYRa7LAifN5WODmPjsLgnZ8na0gpKxnd8AtEK/Jz+17D1t/4Y21PD6uEOQLMVIp+aX74r9UkkVMEMq5jFBEJTlTRKY9BBzl29RLSWt61VmxSyaKLaZRnTLTdLASKXAVys9LSI6CLDLjUi4BfRpJmc5UyrMTb659hMkNqEgmCRu9NGJY0oybUNHGYBqOWN/3bnpX6LqUahx4u3HNqvFmE3how2ghq4g+FMNW6O2w8qQRiIR4QfqY2YI1BFHFSlDOFzR6mmTqd3lgXWEN1ugnmcYr7TrlzP7LEjvLbmlahVe0AwCzlaqKaPwpnx+jcsznf1NtbFB7Hyf/OzajWDYqWc7X0wDMeuTBLx6oNsEPaXAgM3PWmsTmLW0NoqtpYSXd85wIFUTA+u/HuLRY9AST3k2oBkY49kCGnQ6JKpAMZAwTM1BKqtOwxQ3t2684REysBsjqXJg0iLgfEROzWElrrE+CiIlIJlgS72VXXpry0w7g2CkBysys5H6A1YN5pgnlW7prb5avbax1T9XWOvwiJj99uicLiGimwZ9eWddNQSqVKdM33a1sGvvRXGdJQgdUnxSbxQIMHbZf/eJ3JHeZx8W/Ky4XlBemHY/mmNkN3H9YOvu3oLjk4r+hFdH0COzh0eXMQYW7wploytk+v+uZLounnO63+/7p5pwZmHjO98zA/olZlEwqxXe/BCgtHFDXiuoor8uPUfG6/G+sy0Vjauh19YHC6+rLj41nE8m0XhfljDYtRkrNuqB7FvhqwlbuRmXxpGR7RmzIOMLRG1J143mGQzfuML1QmRBMrF6E61rTjscY+8lvf3MI9ekREx444+rwGdtfHTJjlMSciYllvMw4JzbypiK+scO7VJWRVurKuESCw33tS9BwWwjU9FC1yhIsFtKQUkX93hasxmcrIRXM6UJu4I68ef3DP8MWT4M6YCm57uGHraNoe6hY7e7IxMofWdTLQ4fODmIz3My6X86P1AAQG6aksJIjG6oYXXCf2wtqgXtQx5rQUOcqWmkWSH5WAD99ur92ZUvOyH74RP4rbDLqbxeR6XLm7x5/u9EpRGzJomqyPC37Ho5Nh3d2nyWjTr27z5IDrSBN1SLva0vbBOt6CKPTeiK0xZtEFqw7bdBMROC0x9uLLl43gV7eUX6jG6f31wtZIKVFsXuWxrhbPphKoKBZwjhVvjAqOO3f7CwFI6sTxEynnO7KSMHINDfZeTvOdufFMHM7Okl/UxyGTS39UB+5Gp5VXuJq3Tco6/0tF5khioquxCcWRr5ud6tosnhP62dyZrsQbgndBOx04pR4XW3wXvHu4ae1HqEuLyW6uO30jkFnMW3zF71yJmKHXDs1dFxQbV3+IMPrdNx54Nj9qG+/69uvnulwpNSAvE2xj7Gq7F5TXS3wddXNjcrzd3g0RN6tqVoBuTKBixTFyNS5K3k0RwVdgbKzEHfAhKXOmHD3IUyO5GXxIp/PurpLTEz3a6rS+tlOmC2TP4JmsV1an8CQT+wPmDWsRYDvMoqylLlj6YTaf7jPXH18+8vLXolEmVJ2Qu/0Eg3uDOy6UtL+a7UmvPPudZOPl7c7jWfeeMq3z6hBjuZ/eUlOp0K5/kjB91QObS9A5tNIfRi9ek3VcxljnDsOkZbp4Su2OLINR8oH3Kz6mXEoPiOVjyHyTJxz66yFQbG4DDXTFaTLQKN7d1fGenc+Dp16U5UpiKPcpka3mToPNI4/2FfiLGFmpuVydI3QUNWRS+NmySvJeqAXXndwyFo2oTJ2RAVZAInW1h2Pm5EANYSKHfptfaxY01YyZCpW2KFPxYrK2JYV+OTCAoii+Ts6SkrTkUAJLbyDl2R+EmQXEOLRZYtTNxO218VGguiPUf3kLnYlgM8ItEb03yp62Cgoz8BaTrj119xAes1SLJ1rDSikuLHs8CMjAzXUJkD+1SwamoWx+Z5Wvpb0ZF4HMJh4bXq4R8fUapLERj6OGk2o1jJimEbdMrN2bphlczj2fcCsATZxFN8bQvNRH+5dMs+3MM9Hx9GQ7vwSYXBUuthz2E9qdUNmfTom2dHza2Vej5r9Bv2vdbZwcfj32rVEch3YRrEMZzsH09o5TzKu9KubY1GalbwgOlpDnHHQGItTfI3AxTdUPxUVg34dBcd8676T22cpjJKce8u2lUXOv5hK6Wvy7udPaEA+fg4Pav+uDRWxA5O/hcF3ZEmZKofydiZV0toLJgXlgXJ85A42mPBRY552yG8r52IsrtZuga3WZkY+fq7ACI6rgHKfw2iA0mB05X32YIYm6LmS8j2sugCQyf5+f96xlZIV24CwPiqT+6puh1X5BQ0aGbBeSVMDH+7zfGVTe/YC6DAXB0Ho8ukJeTzEbHSOFjIne4mMlnrmBRYssCWjCxv3kIrzoCz8E30Ji5TMn4jA0lS5JQpWGafK7oqdQzmWfK9zO2Ek6rICLTMVgSZ6LTMeo18CRQXyCJ78nklDT8+Sz40UUSdj3EKmPHyhHCHlZpJW16jKRL4+pQC/NskV1SSGJXNuXzeXq8rR1XkjxD0M4k7Nu7cCazhXoHxWDBNrPm0J1uAVCwnxVA1e56C1Lra501hj66xynpRPFnvr2M3JNPNMce53XuT7hlilZ6t11Rvdy15lLni9Fuuym78d6xXTd2MXqjIzlQkMtS6BGXj8I8UKtEHvg4lMZtqvuc6BmWiEKPVFvKYb6OLaQDa5Rk0OxqnZVN6X8KYGy6w3lGs0OrUFYxdF3cR0Gze7tJEVwGm6/1JPm3SzVtIYDvHZmWB1RXdJdeHa03hs5AqJZPq6c9z8Os3WFT5Y254XbZo17DyDvq5phn088SG85V67VDF3VqtrEnL5AKYI7oVDzX+T4+LkW2hx0pI/H+A6+18xQQQVsvYkgl9phTx6HIyQnIbFTDTaky8eFDf5KCjv0h2o+ct//vKmi59n9qZ9BcN5vMbqW9cVRa+1XbMmoBo/9+j7qDXuStimobWgpQq+AI4lRImMu27phPH5WofzIbxyJQ0vx0BNQYUzLGR/YV3+UyuwO16zCipb1rMgWwoCNFrjRxsatmf7ZrpfxfYWL5Bx1tNfavZpYVc7/ZcBPZEBHW8oE0hmeILWeaJMhqzQvjPHEYRXW4T6w73FrjP9Vb7eNZrghH69HKLXUGQFqz0jp6bcnXddItVl6sVtMvWeh5bavJ7cxu3d2yeN1vDSl/a089V40FPx3DM9dH+w3FtSxrPT51PqZ70+cmnUKrljv6uGTF+SbavwvPxRgM23hhOst5dmG9aQ94LMi4pqhgK7UGIjLdc636+hzvGmXFsFsy7VrrTKc+xm3GZWjSl7HIljmXXxpijPJHmFazINhb0/TzK1AdLbSzJBzcWGAu0c8aoldTRWI43S06X6K76I+mRuy9Pl+y1NFvS6L52jjufMxRsTuWzwZ5+B6M4SHmI4ni7TdWnKbUrfxY49N1F6kaaiZiPsJvP53WPZL7v15tkYQi/VNFRtQpPigI3oSY4dZj2RTd+CnfDMavKpZTD6uHSwp1Fw6zKNRlOQ3YdV4/0Ll7B0HeXnVMhwK5/BDJhQV94KKXaJzHTpgbrOx1IQ3wGfA9XmRkEEwvDdDa62q/cff+tmEGfa1K5qJ+lSkyu9TiB5eT3WGNWYZ6P0MzPvZ8bhZkGjp7JsvWTO+4+/FeQeQBXy+sz0PNoNAieeWkZrBoqqaM0iyueOVfPLMo3VtHERieWwvfdUNOyo2Aln+7pPbidhl95eJrfKiGww3zqHrPPzML7lb3N8O5a0eE2kai5qK687wG2uyIM49Qxms5tTYYMa5NEB2pFgs8fLoviTf4PeUXvjIBL/f/g4bLcpntbmYJN/7Bd69iIZayNocbuiHp0axVYrUBDbT+xLgCH0kfrw31LNvwG6EWgP4eTFL/ZTL9x/arK2KiTKuys+GeDe6+E7vMNi5L7w1z13hM1U8HJNzKq3OwZqlJ53pl1OUHiGvVTtP7H6TFZe+WLuUl7e2esAOrp6xJ6aEJlVgrRjSdl3F3wQKefYFv3Rm10higqdUjx3KZrXv7wmQnbnfad1XJXWczvzxXDt10b3VbkktGBkkF/jSme2NL0YWj8Vxx4HSi8TsGGRwWffLoWoXyrp2IgKIY27q+Af9BhEaU7lgj+xkA0fUS7zE5dRtd/zX1UyU1fJHFAk46oJL0VjXRa52fYZbc0SlHLZPvfYqEywLGyBShXbxdc5OdlzWDOKTUyep+qyZMDDqw95R1wpsMzfcttVyFnyDycc67ChvFrva4+xEYzkLNq1G+/mN7QDjXeZ4XBHHr1/+WlgZ949TPFD1HrDVy5Ggy7a1gQfqD75Q9F9HQUbcixhI1ymW3grN04cYRMhqbyaUe/zMwgLi1sFSccDsYOOQqE5QHoKluQDj0Njpmy+XQHjxh2F5Q+ZLNj0EnLDjkISA52eJTE+xBhGQR7M95psQO1IJjh7Au5dHWbcrXQbllKFb8QwQbRM/F06yolmJvMmlRmS0J0PYsOkZeJJyG0zuDyeupKwyrWRtXu4EFtR81h87302oxhsrN1XNiTziNomWtGadzTa7Da+f/J3Nvp4RZOiD6Tb6rqWJDWt23lHzJs3d79xohiAgMMGwpvHwR1prSzcuHUAYQ7sRDS3sGVYTw9C8c4XItrBiRv8mjCH5ePbh3tClaI7d68yzkRMhQm/WRAz/ZQfn020jKrvWrmcrZtkz/yn3OBxhoqQ8C4D08aGzfswYQw9PUtw2LifJUvK+GRbWWV+N27//Li+9Jim+sd3eyYJxaY9im4RhbO34Zb/GF5MqzmVtAoOXlWateQxpuHJ7es3P9zY8CeHsA+eXZ8ncEg8Pu9ge4gulazwRpidtwdtYZ9ANWzXZI91VEOEvM+Lm02fYdPCCo/KNtUmtLJJSBrPj3qh4DNe/6YxqW1M++Y8erp8LxwxZbY4nkqdLW7GETnHBsnBOQP9cVsT4kmJoUmaT4hdlr0vhn3YZr5TUg4Fk+Nu76EizuOra+ej2v8ZTbK03aWt6HL/FaJ5JOOj+PTp4X+/+z/v74kdp2xN5hF+r12DxvZjIlXvlhl3Unm8zKrysuO2L7S0Z92AiKWaY1vVprEfNXucv0E4HEXR5SA4b2+7OG9tik5rrU5lYa3tb7QWpZkvewzeLB3eva/sRF+/a9k5sz/82V9cOnz+WkFoq3qgOTmWAM7wtOmoWSsVd1hUOFwwlT2h9ephHceglHf+WFSuZ503Kwblvc9dZ+Kyrx2nqhVU4RfuTocrf5isB1n3M4CDoQWn7XvCs+sdx8Nm9WdvlTnbvpW0wI4J+z+/e/Sj6NLBc1vbcTlV9whMV1Da/ZiMXzizru93PSITBLGkCWv1yRuKwH7umMm5jCifsXBD0tavi4elbv/jzez17M3slkhF3rx+fXv3+v6nf969/ek/7+/++Y+//3h3dzvOrX9vcZCHR0LjWPk+q6xoZEgFeXjc/GAne3jc/Fh8aAhtqVThfTug4gV9b94cAt9O1YNJQSINXADDPyKQiTnuqTsLyz0Bw3m+lnqMA1cA+/cfb97c3t7c3v77zd9/nIntzP9lFsnWC/09mB8/fyQKIqni4KavcpnMyAO+QCkXhmKHug2jRMEGlG5vzw+PhEv51HlY2GADGB7PU57puRz1TFn55vCh5OM7VsslRP6QOL1x6cNYYhRwBZ/f37/MPWPPCys0V10rBZBEtq9ocboAXnv37hoHsKP9z1sMu18spZwtqJqtJKdiNZNqNXth+fui+ovWgX/xhJYdIwYDKmEifyfJDk8imYDvuEwFgWQBcQwxiWS6K5Ki1LRaLOEX1sakd69epdmCs0hnyyX7ijgG6/IcX489NCRpK+d/2uH8hxY5ma61ViET1ECvbsRfUulB3P1kYN8eN/6xwb0A/GNMB4IIJGEOQzH1+4A/V94GJLWh9+KAr4c+fQlfIcqwlOgYfmDrpNEqEf7W+Ik7U2o9Uy8zzucjVKHuA3eXJnzCv5OhbwaPqEyQS/dEQe4/s7IewScIjvKg2+1YD+5l/xb1WAjnUTeF0JuT2BuW+y6pfQFxuPLFAkMedqOrvmugDQQKJCbEUkyBzk/nG8tTysU9snyobHq6WXUzZIIHVX6pX4qvhpJ5wue6bDVepmaKRqy+BhlLc11CLXVPJ/4BM/JOKgU6xaZzRua9tjTgmf4razFf6Z1+JcC8Yunmh1cmSucJJDPyoePJg+4Sx3Dj46O70PdLlwxMAEmVrun+GvduSQ9Ei4jdWvdC8tNCbFU+F203f/dS0GVDpiYgtyf9fB9mV06Az0LbZ2ea8EBbj4Dpdeug7wQAyzPAyrSjuBlxqWG+pZ1tU06CtoHQ2oh5iWQePAyr4zYsuQzYBZAhqPVOzHX42a+zgs5xDMWsIGo+9PwsmC2OIZiXTKBMmqmgs4MugIxB3cz/PBvqN0NQc6rNnEahE5izgs5xDMFsbc1ZdpB+k8fEKoS4CNLiSd3X3+7/JO6rJeQZ3dcsvkT3db90yUD39dzOXxfqPf9SrI608YDH6CzBFzfEl3onB3+VQ6xyVXGf8rmEI4/afFP6WRKuZggcDeTLJ/9q489MpJmZ5x9KGOcsXD4woJj1w6ecVnzVohyqXSqWaVC6l/cHFIq9l6sVxDd5K20NWjMpmgnkfTzuSKcdXOJbXkDzYIKzamg9nHXEvG9F9WiEyxWzlqs5xZ67bkfSfP9Tpn0Vp3thbgAHAoewR6KwXy9qhCra0CGAUK3IMTIolG9oaUr9eCKIZCElh1Z+oBeJ/RphImaRs0w0Pxnay5FjSsXCEsm73jaK/vZgiOTUWlGRhjPQcWCWsuSfxq3N6uB68jXgm6bkcZhNcDKajzxy7d1C39aOBf2ZdNkutgGo/Jf/HwAA//+KudXt" + return "eJzsff+PGzey5+/5KwgfFhm/m5E93mzevvnhAMfzcjeAEw9sB/uAw0GmuksS37DJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/9YXFqmKxeEOeYHdH9E4bSL4jxDDD4Y68+IS/ePEdITHoSLHUMCnuyP/6jhBC3B+JNtRkmiRgFIv0NeHsCci7x98IFTFJIJFqRzJNV3BNzJoaQhWQSHIOkYGYLJVMiFkDkSkoaphYeRSz7wjRa6nMPJJiyVZ3xKgMviNEAQeq4Y6s6HeELBnwWN8hoBsiaAIVMuyP2aX2s0pmqf9NgBT788V97QuJpDCUCU24jCj3o+X0zfznq/NW546kguKXodn3IKiguLHjVKBYfnoEZCkVoUQzseKA8xG5JJQkGTcMv1fhYP5TZ1r+0ySiSgiLa7/OSeFSrBp/2EON/bHQ31lUIksWoEpUtU/+D/IIKgJh6Ap0EFCmQc3SyARh6YhyiOdLLmnzA0upEmruSOrGHwf+8xryL9IVMtqSY1gCRKcgDGECgRGd0gg6aKtRYFj0pKdhrQVHE5kJcyQwry+XyNwnUAL4GComZHAvh0egEyyCy+OwFITL7U2qmFTM7EiqZARagx5Czdk4fShKFvML5DmiGgD8fIo8AJDcUmYukJeCWGDkSgoSM/30chgd57QR4/Cp3y+PyRrUhkXWNbMu3ZqKmNv/WFMVb603x4QBpbLU9K5H9fv5WD8Zai2X5luSi8V7GIXPLZsDkBug/PIkwwRhYiN5JgxVO2cCFjuMczZMmYxy/MZ2zTjgb9e71LJES9WabEt1jV/SrEHlW6BUs9YX3m4o43TBgUjBd3bz/E2wr4MYeU67eLkMKmK5NDsqlIvSrBVNWqpsxKyPi85smDeloFxslgsKRyepAu29L5SA1GbmPizFjbDrh7M/oBkmksrK0GTLOCdrugEboNKvLMkSsqE8w0Xz5fb167+Rf3PTfcGxW4OV89TGpVwBjXfE0CerH0z7UZkwktAoQrVztmXTHjSAxUL5U4em5INopwj0dWvYncxIRIUTWpXlRfJmpYAaUPYXwvGN/CwVga80STlcE7Ykf28N61TKfp0a8uPrv1lo11avnHL5tMcsSrNZzs0vTnsWQG7/2SmcP1cI++cKEr/d8OvPEu18Q17rX355gMK/vNtpvFsjzYUy0vqCoIkjG3fUh5gDKs7Dh39ZK9TllPxaekaD/BPrSV0kC8amqS+WkLEb/WUSctRuf5kkDd/yLxT/Afv+ZVIy+eb/TZF5qAdwmUR+q27ApXFziBdwnSdCNMQ5k8ucDQbXAdobHsPnVnbvWzmZvuQz3W/jFPQCDxMv+hDuuY9CDt8Rnxv5oZvcX2cPVZ5YPWXyuyYrxhw/2CEq5w/2P8nDh6KMbGANXv4z/ozC/jMozyfYbaVqHhz4/PEd0TG9HS9uJM9O2adsoBjlc7d5joA3EML32s+Ql7uRz2umSUJ3REhDFmCVY8Nit41Tzkumt8b0OfoeghTQeIYHHhMuHvSUKh6GncSqjJWQVRmdRVbDlxnnux58W8UMnBwgznIgQuTgYmeGn6jlrmDoSweAx2EQRh02+SDIeyayr+6IizWnIg0/UENkpPIj4WFPypnXNEGo1lliOYOfIpr9gX7oP27fDJLg8zPI4jAgpuFRPthANrVG7WcbqpXdd06o9gnjNiaIpIi13968WcEVO0iwzwbRrdleZ/HUAMMYY2n3wYdXH3R9E+8CKdMpnZcmRovDOi6pkisFup9pNqKcoQYq+D0DbWYJqBXoeQpqriEKYg1FvT1gm+UDaHr8lJrgnHhyTxx33SnyFhSQ3zPIICZG4gKNYcN64y1PllPb89KFc56asJq8ziqoEj3TuoW+QucBAjqvZKalBCXiCdizA05Axk+lD1D44y3MzXgivM32EkRtzDMtIXQDiq6gGmctpWpoWVAiRlqv2AZREI9Z/2eUilOxU4rFkXQ+uTQWzUSCyVc83azm1m86DSnokV0x4dj70orJoh5oAYZRgjb8xHTgHISDWJn1SYg45zKfVpFcSgXmnZ7f8UrkZnCEWGWquoAvkaiHVx+mlcci07vpqHkMnybEmbKO63bNonWdhO5N8WpBRbxlsVmTzDDO/qB2WmRC+amXM3LvPq6pyZT7iIyizAZTro6vLMPUJOJSo+jrlZU5S0AYJdMqO0YnuMpUmr+i2R5zfNKK5oPOF8xMmo4s0NqBrcjacEsYz388VeL1OK8tN6lhG8i1J5WSF2mEH17/x48tKS8Zh9ptXHJQJrMcplVPXf5pirLqgugz5TkwaYknTRV+G0moIJlIFdswDjbOwPOyfMebBaG7RTofmXQdlVitlvnekS+vYti8sn+9/RJEZOc9ARQ7RhMKfDU/hEHgIcA8lawj+3gwFhzYWlocu8WbMBrU1hOmCez4RMgYMFlg1yj+pp3Nr0BS8Kzavl+rLbr51Fyr8EsBHMI05PuZuOZkXOHdfo5lGs6bzLYTjoT3/LtbA/S+2olCFbXdYI7ax7xKuZEqW1llE8tP5+hqpWBFi+M5yrkzOY0LN+VXj75RdOgBza918+PRkKXMmpFxbfkcsaw/B8yenpFfJRZJ/IuJWG479M9NHQjq9q2CsKTbjACN8iq5QGIZ6VZ2ICAFst8i72VNH/oWzlA0QTxT0SI21kQToF08zwYQV24PwJB5Ph9CZwavEGjKM408fdkOgbik8THmxIZ8dow8pj3SALy4fTHWKNs/MbGaL2lkpLqzod44w/y+Ar8INznVhiRMZAbCa/jFPy4J6T881g6D8+L2otDeBuCGcWOZ5HPpREAXSMyKuolh5Y9tcp5LFGGFmYKiZ9OuDq06kib8UJii015sblln17nsKHfPDdFKWfieaBOkK84WhuC+5nD397g6X/jxm91iB8E6Y5Tr4rWy+hA9Ki/zIjZyVWEuWRpL0FgcxkTEs7j4cCSFq0RZ7HJ3MqLRGjShou1/LbLlEpQmVxqK2NWzhkYmo3zWcEMuPjwbJFhH22H+ehvJWxyt7FoIMda2Ws71efF7veXgiiBn8Eg9QRV+VnTwwRAF3hhql+lnVolAREAWYLbgb+d7lcYqh2ruxkso2LjB/jQ/SWJIQcQ6t7wfPrm8WSIVkBgMZVxfkxTNIInWED0VMXNFh790qAR5/hjKszu85B8MnotQHmUcA/sFtWKp8KJeyuasQ94D4RdIygMPTAm8SpWMXiWQMLGU121e2B+pqhPi16rgMDwpjUphRNiyProFbi1UIdB27GV/Pgjy4dN/EYaEUqKzpGkAcx1igkZ4lJCr0Icibr/234ff2wvbS1EWauG/PlQtOswbGWLiSK+ZIwOjxPZZS2uV9hUxb2nTsnXbvFTBkn29Iy/+L5L1/5ruVT21YjUPRyndFuupMG1YpN0RUHl+aHHUerzm2hxKnvZnPp45bi+JGapKz2XW0fEZh/e5LGJ5SDsKrszMLG1daB+AuYYpyp0wHAohpNbkZqYfAROnA8BE//wKaEzXWH92NAzkfTEgqQ84AAFuEaNzfvsg4IhkXT1j/9asdjZsERZH+nQFcwz6DvNW8y0bq1cKizzSwqYrHVExf7KwD1SsnJvBGzQt1F7vIyqEi2Tc1H0AY6YgOtACHAPQzcubZTpVfBgMnA2Yna1IprRKKVq8M0D5GaVbZlccWgURpywZKmlEez5Rd6PtFbv7wByWSxYxENHujPbIzZ2jJSWGij2akbeEyy2oqo1iImYRXiwvlcd61tqobLXCy5pGFuM2jViTBU6cz8MCN/fZWRC04+tsBSFlPbsDboF4TX5u33vY+gtvrOXxcYUgX4iRSskv3Rf/pZIsYoJQzmWEIirJmSIy7SHgKN+mXkpa06vOil0yUWwxjeqUmaaDlUiBq1B+XkJyFGSRGZdyCejTSMp0plKenXhz7SNMbkBFMknY6KURw5Jm3ISKNgbTcMT6vnfTu0LXpVTjwNuNa1aNN5vAQxtGC1lF9KEYtkJvh5UnjUAkxAvSx8wWrCGIKlaCcr6g0dMkU7/LA+sKa7BGP8k0XrPXKWf2X5bY7XZL0yq8okUBmK1UVUTjT/n8GJVjPv+barOF2ps9+d+xQcayUclyvj4LYNYjD37xQLUJfkjTBZmZs9YkNm+OaxBdjRQr6Z7nRKggAtZ/P8alxaInmPRuQjUwwrEHMux0SFSBZCBjmJiBUlKdhi1uaN8SxiFiYjVAVufCpEHE/YiYmMVKWmN9EkRMRDLBkngvu/LSlJ92AMdOCVBmZiX3A6wezDNNKN/SXXuzfG1jrXuqttbhFzH56dM9WUBEMw3+9Mq6bgpSqUyZvulur9PYj+Y6SxI6oPqk2CwWYOiw/eoXvyO5yzwu/l1xuaC8MO14NMfMbuD+w9LZvwXFJRf/Da2IpkdgD48uZw4q3KnORFPO9vldz3RZPOV0v933TzfnzMDEc75nBvZPzKJkUim++yVAaeGAuvZYR3ldfoyK1+V/Y10uGlNDr6uPJl5XX6NsPOVIpvW6KGe0aTFSatYF3bPAVxO2cjcqi2cu2zNik8gRjt6QqhvPMxy6cYfphcqEYGL1IlzXmnY8ENlPfvubQ6hPj5jwwBlXh8/Y/uqQGaMk5kxMLONlxjmxkTcV8Y0d3qWqjLRSV8YlEhzua1+ChttCoKaHqlWWYLGQhpQq6ve2YDU+WwmpYE4XcgN35M3rH/4Ztnga1AFLyXU0P2wdRdtDxWp3RyZW/siiXh46dHYQm+Fm1v1yfqQGgNgwJYWVHNlQxeiC+9xeUAvcIz/WhIa6adFKA0PyswL46dP9tStbckb2wyfyX2GTUX9PiUyXM3/3+NuNTiFiSxZVk+Vp2YtxbDq8syMuGXXq3X2WHGhPaaoWeV+r3CZY19cYndYToS3eSbJg3WmDZiICpz3eXnTxugn08o7yGx1Cvb9eyAIpLYrdszTG3fLBVAIFzRLGqfKFUcFp/2ZnKRhZnSBmOuV0V0YKRqa5yc5bhLa7QYaZ29Hd+pviMGxq6Yf6yNXwrPI6WOu+QVnvb7nIDFFUdCU+sTDydbtbRZPFe9pRkzPbhXCb6iZgpxOnxOtqg/eKdw8/rfUIdXkp0cVtp3cMOotpm78yljMRu/baqaHjgmrr8gcZXqfjzgPH7kd9+13ffvVMhyOlBuStk32MVWX3mupqga+rbm5Unr/DoyHybk3VCsiVCVykKEamzl3Jozkq6AqUnYW4AyYsdcaEuw9hciQvi1cCfdbVXWJiul9TldbPdsJsmfwRNIvt0voEhnxif8CsYS0CfJdRlKXMHUsn1P7Dfebq49tfXvZKJMqUshN6p5docGdg15WS9l+rNeGdd6+bfLy83Wk888ZTvn1GDXI0/8tLcjoVyvVHCr6ncmh7ATKfRurD6NVrqp7LGOPccYi0TA9fscWRbThSPuBm1c+MQ/EZqXwMkWfinFtnLQyKxWWoma4gXQaa77u7Mta783Ho1JuqTEEc5TY1us3UeaBx/MG+EmcJMzMtl6NrhIaqjlwaN0teSdYDvfC6g0PWsgmVsSMqyAJItLbueNyMBKghVOzQb+tjxZq2kiFTscIOfSpWVMa2rMBnIBZAFM3f9lFSmo4ESmjhHbwk85Mgu4AQjy5bnLqZsL0uNhJEf4zqJ3exKwF82qA1ov9W0cNGQXkG1nLCrb/mBtJrlmLpXGtAIcWNZYcfGRmooTYB8q9m0dAsjM33tPK1pCfzOoDBxGvTwz06plaTJDbycdRoQrWWEcM06paZtXPDLJvDse8DZg2wiaP43hCaj/pw75J5vq16PjqOhnTnlwiDo9LFnsN+UqsbMuvTMcmOnl8r83rU7Dfof62zhYvDv9euJZLrwDaKZTjbOZjWznmScaVf3RyL0qzkBdHRGuKMg8ZYnOILCS6+ofqpqBj06yg45lv3ndw+S2GU5Nxbtq0scv7FVEpfk3c/f0ID8vFzeFD7d22oiB2Y/H0OviNLylQ5lLczqZLWXjApKA+U4yN3sMGEjxrztEN+WzkXY3G1dgtstTYz8vFzBUZwXAWU+xxGA5QGoytvxgczNEHPlZRvdNUFgEz29/vzjq2UrNgGhPVRmdxXdTusyi9o0MiA9UqaGvhwn+crm9qzF0CHuTgIQpdPT8jjIWajc7SQOdlLZLTUMy+wYIEtGV3YuIdUnAdl4Z8NTFikZP5sBZamyi1RsMo4VXZX7BzKseR7ndsJI1GXFWiZqQg00WuZ8Rj9EigqkEfw5PdMGnp6lnxupIg6GeMWMuXhC+UIKTeTtLpGVSby9SkF+LVJrqgmMSyZc/u6uVxVjq7OGyHuYRB3at69FVjDuQLls2KYWPNpS7AGr1hIiKdq8DoHrXWxzZ3GGltnlfOkfLLYW8duTqaZZ4pzv/Mi3zfEKj1brave6F72KnPB67VYl9387VivmL4bu1CVmalMYKh1CczA4x8pVqANeh9MZDLTfs11DsxEI0SpL+I13UAX1wayyTVqcjBOzabyvoQ3NVhmvaFco9GpLRi7KOomptu42aWNrABO0/2Xetqkm7WSxnCIz84Eqyu6S6oL157GYyNXSCTT153j5tdptq7wwdr2vGjTrGHnGfR1TTPs44mP8y332qWKubNaXZOQywcwRXAvHGr+mxwXJ99Ci5OW/PkA19n/igkiqJC1JxH8Sivk0eNghOQ0LGai0Z588aC4yUdBeZfuQM1f/vOXN138PLM37SsYzuM1Vt/frih6re2aNQHV+LlH30etcVfCNg2tBS1V8AVwLCFKZNx1SyeMz9c6nA/hlStpeDkGagoqnGEh+wvr8p9agd3xmlVQ2bKeBdlSEKDRGj/a0LA92zfT/Sq2t3iBjLOe/lKzTwu72um/DOiJDOh4Q5lAMsMTtM4TZTJkhfadOY4gvNoi1B/uLXad6a/y9a7RBCf06+UQvYYiK1jtGTk15e686xKpLlMvbpOp9zy01Ob15DZu794+abSGl760p52vxoOeiuee6aH7g+XekjKenT6fUj/r9ZFLo1bJHftdNWT6kmxbhefljwJsvjWcYL29NNuwhrwXZF5UVDMU2IUSG2m51vl+DXWON+XaKph1qXalVZ5jN+M2s2pM2eNIHMusizdFeSbJK1yTaSjs/XmSqQ2Q3l6SCWouNhRo54hXLamjsRpplJ4u1V/xRdQnc1ueLt9vabKg133pHHU8Zy7emMhlgz/7DER3lvAQw/F0ma5LU25T+i527LmJ0os0FTUbYTeZz+8ey37ZrTfPxhB6qaahahOaFAdsRE9y7DDriWz6FuyEZ1aTTy2D0celgz2NgluXaTSaguw+rBrvX7iEpesoP6dChlv5DGbAhLryVkixS2SmSw/UdT6WgvgO+ByoNjcKIhCG725wtV29//hbN4M406Z2VTtJl5pc6XUCycvrscaoxjwbpZ+ZeT8zDjcLGj2VZeslc95//K0g9wCqkNdnpufRbhA48dQyWjNQVEVrFlE+d6yaX5ZprKaNi0gsh+29p6JhR8VOONvXfXI7Cbv09jK5VUZkg/nWOWSdn4fxLX+b49uxpMVrIlVzUVt53QFuc0UexKlnMJvdnAob1CCPDtCOBJs9XhbFn/wb9I7aGweR+P/Dx2G7TfG0Ngeb/GO/0LMXyVgbQYvbFfXo1Ci2WoGC2H5iXwIMoY/Uh/+Wav4N0I1AewgnL36xn3rh/lOTtVUhUd5d8ckA914P3+EdFiP3hb/uuSNspoKXa2JWvd0xUKP0vDPtcoLCM+ylav+J1Wey8soXc5fy8s5eB9DR1SP21ITIrBKkHUvKvrvgg0g5x7boj97sClFU6JTiuUvRvP7lNRGyO+87reOqtJ7bmS+Ga782uq/KJaEFI4P8Glc6s6XpxdD6qTj2OFB6mYANiww++3YpRP1SScdGVAhp3F0F/6DHIEpzKhf8iYVs+IhymZ+4jKr9nv+qkpm6SuaAIhlXTXgpGuuyyM22z2hrlqCUy/a5x0ZlgmVhC1Sq2C6+zsnJnsOaUWxi8jxVlyUDHl59yDviSoFl/pbbrkLOkn844ViHDeXVel97jI1gJGfRrt14N7+hHWi8ywyHO/Lo/ctPAzvz7mGKH6LWG75yMRp00bYm+ED1yR+K7uso2JBjCRvhMt3CW7lx4gibCEnl1Yx6n59BWFjcKkg6HogddBQKzQHSU7AkH3gcGjNl8+0KGDfuKCx/yGTBppeQG3YUkhjo9CyJ8SHGMAryYL7XZANqRzLB2RNw7+ow426l27CUKnwjhgmiZeLv0lFONDOZN6nMkITufBAbJi0TT0Jum8Hl8dSVhFWujazdw4XYiprH4nvvsxnFYGPtvrIhmUfUNtGK1ryj0Wa38f2Tv7PRxyuaFH0g3VbXtSSpad3OO2LevLn7jRPFAAQcNhDePA7uSGtl4catAwhzYCeiuYUtw3p6EIp3vhDRDk7c4NeEOSwf3z7cE6oU3bl7lXEmYipM+M2CmOmn/PhsomVUfdfK5WzdJHvmP+UGjzNUhIR3GZg2Nmzehwlj6OlZgsPG/SxZUsYn28oq87tx++fH9aXHNNU/vtszSSg27VF0iyicvQ23/MfwYlrNqaRVcPCq0qwljzENT25fv/nhxoY/OYR98Oz6PIFD4vF5B9tDdKlkhTfC7Lw9aAv7BKphuyZ7rKMaIuR9Xtxs+gybFlZ4VLapNqGVTULSeH7UCwWf8fo3jUltY9o359HT5XvhiCmzxfFU6mxxM47IOTZIDs4Z6I/bmhBPSgxN0nxC7LLsfTHswzbznZJyKJgcd3sPFXEeX107H9X+z2iSpe0ubUWX+68QzSMZH8WnTw//+93/eX9P7DhlazKP8HvtGjS2HxOperfMuJPK42VWlZcdt32hpT3rBkQs1RzbqjaN/ajZ4/wNwuEoii4HwXl728V5a1N0Wmt1KgtrbX+jtSjNfNlj8Gbp8O59ZSf6+l3Lzpn94c/+4tLh89cKQlvVA83JsQRwhqdNR81aqbjDosLhgqnsCa1XD+s4BqW888eicj3rvFkxKO997joTl33tOFWtoAq/cHc6XPnDZD3Iup8BHAwtOG3fE55d7zgeNqs/e6vM2fatpAV2TNj/+d2jH0WXDp7b2o7LqbpHYLqC0u7HZPzCmXV9v+sRmSCIJU1Yq0/eUAT2c8dMzmVE+YyFG5K2fl08LHX7H29mr2dvZrdEKvLm9evbu9f3P/3z7u1P/3l/989//P3Hu7vbcW79e4uDPDwSGsfK91llRSNDKsjD4+YHO9nD4+bH4kNDaEulCu/bARUv6Hvz5hD4dqoeTAoSaeACGP4RgUzMcU/dWVjuCRjO87XUYxy4Ati//3jz5vb25vb232/+/uNMbGf+L7NItl7o78H8+PkjURBJFQc3fZXLZEYe8AVKuTAUO9RtGCUKNqB0e3t+eCRcyqfOw8IGG8DweJ7yTM/lqGfKyjeHDyUf37FaLiHyh8TpjUsfxhKjgCv4/P7+Ze4Ze15YobnqWimAJLJ9RYvTBfDau3fXOIAd7X/eYtj9YinlbEHVbCU5FauZVKvZC8vfF9VftA78iye07BgxGFAJE/k7SXZ4EskEfMdlKggkC4hjiEkk012RFKWm1WIJv7A2Jr179SrNFpxFOlsu2VfEMViX5/h67KEhSVs5/9MO5z+0yMl0rbUKmaAGenUj/pJKD+LuJwP79rjxjw3uBeAfYzoQRCAJcxiKqd8H/LnyNiCpDb0XB3w99OlL+ApRhqVEx/ADWyeNVonwt8ZP3JlS65l6mXE+H6EKdR+4uzThE/6dDH0zeERlgly6Jwpy/5mV9Qg+QXCUB91ux3pwL/u3qMdCOI+6KYTenMTesNx3Se0LiMOVLxYY8rAbXfVdA20gUCAxIZZiCnR+Ot9YnlIu7pHlQ2XT082qmyETPKjyS/1SfDWUzBM+12Wr8TI1UzRi9TXIWJrrEmqpezrxD5iRd1Ip0Ck2nTMy77WlAc/0X1mL+Urv9CsB5hVLNz+8MlE6TyCZkQ8dTx50lziGGx8f3YW+X7pkYAJIqnRN99e4d0t6IFpE7Na6F5KfFmKr8rlou/m7l4IuGzI1Abk96ef7MLtyAnwW2j4704QH2noETK9bB30nAFieAVamHcXNiEsN8y3tbJtyErQNhNZGzEsk8+BhWB23YcllwC6ADEGtd2Kuw89+nRV0jmMoZgVR86HnZ8FscQzBvGQCZdJMBZ0ddAFkDOpm/ufZUL8ZgppTbeY0Cp3AnBV0jmMIZmtrzrKD9Js8JlYhxEWQFk/qvv52/ydxXy0hz+i+ZvEluq/7pUsGuq/ndv66UO/5l2J1pI0HPEZnCb64Ib7UOzn4qxxilauK+5TPJRx51Oab0s+ScDVD4GggXz75Vxt/ZiLNzDz/UMI4Z+HygQHFrB8+5bTiqxblUO1SsUyD0r28P6BQ7L1crSC+yVtpa9CaSdFMIO/jcUc67eAS3/ICmgcTnFVD6+GsI+Z9K6pHI1yumLVczSn23HU7kub7nzLtqzjdC3MDOBA4hD0Shf16USNU0YYOAYRqRY6RQaF8Q0tT6scTQSQLKTm08gO9SOzXCBMxi5xlovnJ0F6OHFMqFpZI3vW2UfS3B0Mkp9aKijScgY4Ds5Ql/zRubVYH15OvAd80JY/DbIKT0XzkkWvvFvq2dizoz6TLdrENQOW//P8AAAD//6eg+V4=" } diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 30a6770edc7..65f1d08ba80 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -39,7 +39,7 @@ "write.bytes", "read.time", "write.time"] SYSTEM_DISKIO_FIELDS_LINUX = ["name", "read.count", "write.count", "read.bytes", - "write.bytes", "read.time", "write.time", "io.time", + "write.bytes", "read.time", "write.time", "io.time", "io.ops", "iostat.read.request.merges_per_sec", "iostat.write.request.merges_per_sec", "iostat.read.request.per_sec", "iostat.write.request.per_sec", "iostat.read.per_sec.bytes", "iostat.write.per_sec.bytes" "iostat.request.avg_size", "iostat.queue.avg_size", "iostat.await", "iostat.service_time", "iostat.busy"] diff --git a/metricbeat/scripts/mage/docs_collector.go b/metricbeat/scripts/mage/docs_collector.go index b58bad3edae..b505db79a69 100644 --- a/metricbeat/scripts/mage/docs_collector.go +++ b/metricbeat/scripts/mage/docs_collector.go @@ -158,7 +158,7 @@ func getDefaultMetricsets() (map[string][]string, error) { return nil, err } for k, v := range msetMap { - masterMap[k] = v + masterMap[k] = append(masterMap[k], v...) } } diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 52a31a9e99d..2b0faecc26f 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 59dde477bed..d5d944f918b 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0 healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -14,9 +14,13 @@ services: - "transport.host=127.0.0.1" - "http.host=0.0.0.0" - "xpack.security.enabled=false" + - "script.context.template.max_compilations_rate=unlimited" + - "script.context.ingest.cache_max_size=2000" + - "script.context.processor_conditional.cache_max_size=2000" + - "script.context.template.cache_max_size=2000" logstash: - image: docker.elastic.co/logstash/logstash:7.9.0 + image: docker.elastic.co/logstash/logstash:7.10.0 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -26,7 +30,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.9.0 + image: docker.elastic.co/kibana/kibana:7.10.0 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300 diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 9d3555d7854..4f15ba5582f 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -15,6 +15,10 @@ services: - "http.host=0.0.0.0" - "xpack.security.enabled=false" - "indices.id_field_data.enabled=true" + - "script.context.template.max_compilations_rate=unlimited" + - "script.context.ingest.cache_max_size=2000" + - "script.context.processor_conditional.cache_max_size=2000" + - "script.context.template.cache_max_size=2000" logstash: image: docker.elastic.co/logstash/logstash@sha256:e01cf165142edf8d67485115b938c94deeda66153e9516aa2ce69ee417c5fc33 diff --git a/winlogbeat/tests/system/test_wineventlog.py b/winlogbeat/tests/system/test_wineventlog.py index 363e90edbd2..8b06841ff70 100644 --- a/winlogbeat/tests/system/test_wineventlog.py +++ b/winlogbeat/tests/system/test_wineventlog.py @@ -1,5 +1,6 @@ import codecs import os +import platform import sys import time import unittest @@ -250,6 +251,8 @@ def test_query_level_multiple(self): self.assertEqual(evts[0]["log.level"], "error") self.assertEqual(evts[1]["log.level"], "warning") + @unittest.skipIf(platform.platform().startswith("Windows-7"), + "Flaky test: https://github.com/elastic/beats/issues/22753") def test_query_ignore_older(self): """ wineventlog - Query by time (ignore_older than 2s) diff --git a/x-pack/auditbeat/module/system/socket/config.go b/x-pack/auditbeat/module/system/socket/config.go index 55ac5b4f907..79413f7649c 100644 --- a/x-pack/auditbeat/module/system/socket/config.go +++ b/x-pack/auditbeat/module/system/socket/config.go @@ -4,7 +4,10 @@ package socket -import "time" +import ( + "reflect" + "time" +) // Config defines this metricset's configuration options. type Config struct { @@ -64,11 +67,27 @@ type Config struct { EnableIPv6 *bool `config:"socket.enable_ipv6"` } -// Validate validates the host metricset config. +// Validate validates the socket metricset config. func (c *Config) Validate() error { return nil } +// Equals compares two Config objects +func (c *Config) Equals(other Config) bool { + // reflect.DeepEquals() doesn't compare pointed-to values, so strip + // all pointers and then compare them manually. + simpler := [2]Config{*c, other} + for idx := range simpler { + simpler[idx].EnableIPv6 = nil + simpler[idx].TraceFSPath = nil + } + return reflect.DeepEqual(simpler[0], simpler[1]) && + (c.EnableIPv6 == nil) == (other.EnableIPv6 == nil) && + (c.EnableIPv6 == nil || *c.EnableIPv6 == *other.EnableIPv6) && + (c.TraceFSPath == nil) == (other.TraceFSPath == nil) && + (c.TraceFSPath == nil || *c.TraceFSPath == *other.TraceFSPath) +} + var defaultConfig = Config{ PerfQueueSize: 4096, LostQueueSize: 128, diff --git a/x-pack/auditbeat/module/system/socket/guess/creds.go b/x-pack/auditbeat/module/system/socket/guess/creds.go index ad35ed06e93..7e3bd4791d9 100644 --- a/x-pack/auditbeat/module/system/socket/guess/creds.go +++ b/x-pack/auditbeat/module/system/socket/guess/creds.go @@ -51,7 +51,7 @@ const ( ) func init() { - if err := Registry.AddGuess(&guessStructCreds{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessStructCreds{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/cskxmit6.go b/x-pack/auditbeat/module/system/socket/guess/cskxmit6.go index b17efafabaa..aae0e2fd9b6 100644 --- a/x-pack/auditbeat/module/system/socket/guess/cskxmit6.go +++ b/x-pack/auditbeat/module/system/socket/guess/cskxmit6.go @@ -34,7 +34,7 @@ import ( */ func init() { - if err := Registry.AddGuess(&guessInet6CskXmit{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessInet6CskXmit{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/deref.go b/x-pack/auditbeat/module/system/socket/guess/deref.go index a019b7cf7b9..648580e5cc0 100644 --- a/x-pack/auditbeat/module/system/socket/guess/deref.go +++ b/x-pack/auditbeat/module/system/socket/guess/deref.go @@ -28,7 +28,7 @@ import ( */ func init() { - if err := Registry.AddGuess(&guessDeref{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessDeref{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/inetsock.go b/x-pack/auditbeat/module/system/socket/guess/inetsock.go index cc08fa79bf4..57a133d0056 100644 --- a/x-pack/auditbeat/module/system/socket/guess/inetsock.go +++ b/x-pack/auditbeat/module/system/socket/guess/inetsock.go @@ -35,7 +35,7 @@ import ( // matched the remote address. This is used by guess_inet_sock6. func init() { - if err := Registry.AddGuess(&guessInetSockIPv4{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessInetSockIPv4{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/inetsock6.go b/x-pack/auditbeat/module/system/socket/guess/inetsock6.go index c76b47e3d19..37cd56b06fb 100644 --- a/x-pack/auditbeat/module/system/socket/guess/inetsock6.go +++ b/x-pack/auditbeat/module/system/socket/guess/inetsock6.go @@ -103,7 +103,7 @@ import ( const inetSockDumpSize = 8 * 256 func init() { - if err := Registry.AddGuess(&guessInetSockIPv6{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessInetSockIPv6{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/inetsockaf.go b/x-pack/auditbeat/module/system/socket/guess/inetsockaf.go index ec8075060d4..2e66eb2a724 100644 --- a/x-pack/auditbeat/module/system/socket/guess/inetsockaf.go +++ b/x-pack/auditbeat/module/system/socket/guess/inetsockaf.go @@ -45,7 +45,7 @@ import ( const inetSockAfDumpSize = 8 * 16 func init() { - if err := Registry.AddGuess(&guessInetSockFamily{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessInetSockFamily{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/iplocalout.go b/x-pack/auditbeat/module/system/socket/guess/iplocalout.go index 170d2bf6885..d78140bda13 100644 --- a/x-pack/auditbeat/module/system/socket/guess/iplocalout.go +++ b/x-pack/auditbeat/module/system/socket/guess/iplocalout.go @@ -39,7 +39,7 @@ const ( ) func init() { - if err := Registry.AddGuess(&guessIPLocalOut{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessIPLocalOut{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/registry.go b/x-pack/auditbeat/module/system/socket/guess/registry.go index 16c583cbb9e..66971a1f1d5 100644 --- a/x-pack/auditbeat/module/system/socket/guess/registry.go +++ b/x-pack/auditbeat/module/system/socket/guess/registry.go @@ -8,29 +8,33 @@ package guess import "fmt" -// Registry serves as a registration point for guesses. -var Registry = Register{ - guesses: make(map[string]Guesser), -} +// GuesserFactory is a factory function for guesses. +type GuesserFactory func() Guesser // Register stores the registered guesses. type Register struct { - guesses map[string]Guesser + factories map[string]GuesserFactory +} + +// Registry serves as a registration point for guesses. +var Registry = Register{ + factories: make(map[string]GuesserFactory), } // AddGuess registers a new guess. -func (r *Register) AddGuess(guess Guesser) error { - if _, found := r.guesses[guess.Name()]; found { +func (r *Register) AddGuess(factory GuesserFactory) error { + guess := factory() + if _, found := r.factories[guess.Name()]; found { return fmt.Errorf("guess %s is duplicated", guess.Name()) } - r.guesses[guess.Name()] = guess + r.factories[guess.Name()] = factory return nil } // GetList returns a list of registered guesses. func (r *Register) GetList() (list []Guesser) { - for _, guess := range r.guesses { - list = append(list, guess) + for _, factory := range r.factories { + list = append(list, factory()) } return list } diff --git a/x-pack/auditbeat/module/system/socket/guess/skbuff.go b/x-pack/auditbeat/module/system/socket/guess/skbuff.go index 78a73de1187..9960c036465 100644 --- a/x-pack/auditbeat/module/system/socket/guess/skbuff.go +++ b/x-pack/auditbeat/module/system/socket/guess/skbuff.go @@ -45,13 +45,13 @@ import ( const maxSafePayload = 508 func init() { - if err := Registry.AddGuess(&guessSkBuffLen{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSkBuffLen{} }); err != nil { panic(err) } - if err := Registry.AddGuess(&guessSkBuffProto{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSkBuffProto{} }); err != nil { panic(err) } - if err := Registry.AddGuess(&guessSkBuffDataPtr{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSkBuffDataPtr{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/sockaddrin.go b/x-pack/auditbeat/module/system/socket/guess/sockaddrin.go index c9df8356472..356442b1d86 100644 --- a/x-pack/auditbeat/module/system/socket/guess/sockaddrin.go +++ b/x-pack/auditbeat/module/system/socket/guess/sockaddrin.go @@ -29,8 +29,7 @@ import ( */ func init() { - if err := Registry.AddGuess( - &guessSockaddrIn{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSockaddrIn{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/sockaddrin6.go b/x-pack/auditbeat/module/system/socket/guess/sockaddrin6.go index ffed4c577e3..8d1d0b15b25 100644 --- a/x-pack/auditbeat/module/system/socket/guess/sockaddrin6.go +++ b/x-pack/auditbeat/module/system/socket/guess/sockaddrin6.go @@ -29,8 +29,7 @@ import ( */ func init() { - if err := Registry.AddGuess( - &guessSockaddrIn6{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSockaddrIn6{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/socketsk.go b/x-pack/auditbeat/module/system/socket/guess/socketsk.go index e084c82f1a4..072eba6b66b 100644 --- a/x-pack/auditbeat/module/system/socket/guess/socketsk.go +++ b/x-pack/auditbeat/module/system/socket/guess/socketsk.go @@ -29,7 +29,7 @@ import ( // "SOCKET_SOCK": 32 func init() { - if err := Registry.AddGuess(&guessSocketSock{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessSocketSock{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/syscallargs.go b/x-pack/auditbeat/module/system/socket/guess/syscallargs.go index 563ec9e1355..44796b73fa9 100644 --- a/x-pack/auditbeat/module/system/socket/guess/syscallargs.go +++ b/x-pack/auditbeat/module/system/socket/guess/syscallargs.go @@ -31,8 +31,10 @@ import ( */ func init() { - if err := Registry.AddGuess(&guessSyscallArgs{ - expected: [2]uintptr{^uintptr(0x11111111), ^uintptr(0x22222222)}, + if err := Registry.AddGuess(func() Guesser { + return &guessSyscallArgs{ + expected: [2]uintptr{^uintptr(0x11111111), ^uintptr(0x22222222)}, + } }); err != nil { panic(err) } diff --git a/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgargs.go b/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgargs.go index f0382ad3ed2..60bc66f7f91 100644 --- a/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgargs.go +++ b/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgargs.go @@ -24,7 +24,7 @@ import ( // TCP_SENDMSG_LEN : +4(%sp) func init() { - if err := Registry.AddGuess(&guessTCPSendMsg{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessTCPSendMsg{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgsk.go b/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgsk.go index 0e3ede37b54..09004388ada 100644 --- a/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgsk.go +++ b/x-pack/auditbeat/module/system/socket/guess/tcpsendmsgsk.go @@ -28,7 +28,7 @@ import ( // TCP_SENDMSG_SOCK : %di func init() { - if err := Registry.AddGuess(&guessTcpSendmsgSock{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessTcpSendmsgSock{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/guess/udpsendmsg.go b/x-pack/auditbeat/module/system/socket/guess/udpsendmsg.go index f381b5ba95e..de5889d9312 100644 --- a/x-pack/auditbeat/module/system/socket/guess/udpsendmsg.go +++ b/x-pack/auditbeat/module/system/socket/guess/udpsendmsg.go @@ -28,7 +28,7 @@ import ( // UDP_SENDMSG_MSG: $stack3 func init() { - if err := Registry.AddGuess(&guessUDPSendMsg{}); err != nil { + if err := Registry.AddGuess(func() Guesser { return &guessUDPSendMsg{} }); err != nil { panic(err) } } diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index 11f8a22289e..36ae6276e14 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -14,6 +14,7 @@ import ( "sort" "strconv" "strings" + "sync" "sync/atomic" "syscall" "time" @@ -73,6 +74,7 @@ type MetricSet struct { mountedFS *mountPoint isDebug bool isDetailed bool + terminated sync.WaitGroup } func init() { @@ -86,20 +88,45 @@ func init() { } } +var ( + // Singleton to instantiate one socket dataset at a time. + instance *MetricSet + instanceMutex sync.Mutex +) + // New constructs a new MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The %s dataset is beta.", fullName) + instanceMutex.Lock() + defer instanceMutex.Unlock() config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.Wrapf(err, "failed to unpack the %s config", fullName) } + if instance != nil { + // Do not instantiate a new dataset if the config hasn't changed. + // This is necessary when run under config reloader even though the + // reloader itself already checks the config for changes, because + // the first time it runs it will allocate two consecutive instances + // (one for checking the config, one for running). This saves + // running the guesses twice on startup. + if config.Equals(instance.config) { + return instance, nil + } + instance.terminated.Wait() + } + var err error + instance, err = newSocketMetricset(config, base) + return instance, err +} + +func newSocketMetricset(config Config, base mb.BaseMetricSet) (*MetricSet, error) { + cfgwarn.Beta("The %s dataset is beta.", fullName) logger := logp.NewLogger(metricsetName) sniffer, err := dns.NewSniffer(base, logger) if err != nil { return nil, errors.Wrap(err, "unable to create DNS sniffer") } - ms := &MetricSet{ SystemMetricSet: system.NewSystemMetricSet(base), templateVars: make(common.MapStr), @@ -110,10 +137,9 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { isDetailed: logp.HasSelector(detailSelector), sniffer: sniffer, } - // Setup the metricset before Run() so that startup can be halted in case of // error. - if err := ms.Setup(); err != nil { + if err = ms.Setup(); err != nil { return nil, errors.Wrapf(err, "%s dataset setup failed", fullName) } return ms, nil @@ -121,7 +147,9 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Run the metricset. This will loop until the passed reporter is cancelled. func (m *MetricSet) Run(r mb.PushReporterV2) { + m.terminated.Add(1) defer m.log.Infof("%s terminated.", fullName) + defer m.terminated.Done() defer m.Cleanup() st := NewState(r, @@ -235,7 +263,6 @@ func (m *MetricSet) Setup() (err error) { // var traceFS *tracing.TraceFS if m.config.TraceFSPath == nil { - if err := tracing.IsTraceFSAvailable(); err != nil { m.log.Debugf("tracefs/debugfs not found. Attempting to mount") for _, mount := range defaultMounts { diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 9864728cc14..f2e70273436 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -51,3 +51,4 @@ - Removed `install-service.ps1` and `uninstall-service.ps1` from Windows .zip packaging {pull}21694[21694] - Add `priority` to `AddOrUpdate` on dynamic composable input providers communication channel {pull}22352[22352] - Ship `endpoint-security` logs to elasticsearch {pull}22526[22526] +- Log level reloadable from fleet {pull}22690[22690] diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index e88dea9534e..4a2fb93d10f 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -200,3 +200,5 @@ agent.logging.to_stderr: true # information. Recommended to use in combination with `logging.json=true` # Defaults to false. #agent.logging.ecs: false + +{{template "providers.yml.tmpl" .}} diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index 55ed22e65a3..53c90a7f783 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -200,3 +200,5 @@ agent.logging.to_stderr: true # information. Recommended to use in combination with `logging.json=true` # Defaults to false. #agent.logging.ecs: false + +{{template "providers.yml.tmpl" .}} diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 2f8187a1604..bb77ce3947c 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -200,3 +200,5 @@ agent.logging.to_stderr: true # information. Recommended to use in combination with `logging.json=true` # Defaults to false. #agent.logging.ecs: false + +{{template "providers.yml.tmpl" .}} diff --git a/x-pack/elastic-agent/_meta/config/providers.yml.tmpl b/x-pack/elastic-agent/_meta/config/providers.yml.tmpl new file mode 100644 index 00000000000..02d81408cb0 --- /dev/null +++ b/x-pack/elastic-agent/_meta/config/providers.yml.tmpl @@ -0,0 +1,42 @@ +# Providers + +# Providers supply the key/values pairs that are used for variable substitution +# and conditionals. Each provider's keys are automatically prefixed with the name +# of the provider. + +#providers: + +# Agent provides information about the running agent. +# agent: +# enabled: true + +# Docker provides inventory information from Docker. +# docker: +# enabled: true +# host: "unix:///var/run/docker.sock" +# cleanup_timeout: 60 + +# Env providers information about the running environment. +# env: +# enabled: true + +# Host provides information about the current host. +# host: +# enabled: true + +# Local provides custom keys to use as variable. +# local: +# enabled: true +# vars: +# foo: bar + +# Local dynamic allows you to define multiple key/values to generate multiple configurations. +# local_dynamic: +# enabled: true +# items: +# - vars: +# my_var: key1 +# - vars: +# my_var: key2 +# - vars: +# my_var: key3 diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc index 7d2531e65e6..09f2669cd06 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc @@ -8,7 +8,9 @@ The policy settings for {fleet}-managed agents are specified through the UI. You do not set them explicitly in a configuration file. For standalone agents, you need to configure settings in the `elastic-agent.yml` -file. +file. Prior to installation, edit the file located in the extracted {agent} +package. After installation, edit the file located in the directory +described in <>. TIP: To get started quickly, you can use {fleet} to generate a standalone configuration. For more information, see <>. diff --git a/x-pack/elastic-agent/docs/elastic-agent-providers.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-providers.asciidoc index 868c5b2fd2e..44125efe349 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-providers.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-providers.asciidoc @@ -172,13 +172,18 @@ defines 3 values for `item`: ---- inputs: - type: logfile - paths: "/var/${item}/app.log" + streams: + - paths: "/var/${local_dynamic.my_var}/app.log" providers: - vars: - - item: key1 - - item: key2 - - item: key3 + local_dynamic: + items: + - vars: + my_var: key1 + - vars: + my_var: key2 + - vars: + my_var: key3 ---- The configuration generated by this policy looks like: @@ -187,11 +192,14 @@ The configuration generated by this policy looks like: ---- inputs: - type: logfile - paths: "/var/key1/app.log" + streams: + - paths: "/var/key1/app.log" - type: logfile - paths: "/var/key2/app.log" + streams: + - paths: "/var/key2/app.log" - type: logfile - paths: "/var/key3/app.log" + streams: + - paths: "/var/key3/app.log" ---- [[docker-provider]] diff --git a/x-pack/elastic-agent/docs/elastic-agent.asciidoc b/x-pack/elastic-agent/docs/elastic-agent.asciidoc index 56df8eeca6a..bfe3c2b12a3 100644 --- a/x-pack/elastic-agent/docs/elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent.asciidoc @@ -1,5 +1,3 @@ -:release-state: released - [[elastic-agent-installation-configuration]] [role="xpack"] @@ -16,21 +14,25 @@ collect data and send it to the {stack}. Behind the scenes, {agent} runs the To learn how to install, configure, and run your {agent}s, see: * <> +* <> * <> * <> -* <> +* <> * <> * <> * <> * <> +* <> include::install-elastic-agent.asciidoc[leveloffset=+1] +include::run-elastic-agent-standalone.asciidoc[leveloffset=+1] + include::uninstall-elastic-agent.asciidoc[leveloffset=+1] include::upgrade-elastic-agent.asciidoc[leveloffset=+1] -include::run-elastic-agent-standalone.asciidoc[leveloffset=+1] +include::start-elastic-agent.asciidoc[leveloffset=+1] include::stop-elastic-agent.asciidoc[leveloffset=+1] diff --git a/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc index ac9f641be21..444bdebd7a1 100644 --- a/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc @@ -36,17 +36,18 @@ include::{beats-repo-dir}/x-pack/elastic-agent/docs/tab-widgets/download-widget. // end::install-elastic-agent[] -- -. Install {agent} as a managed service, enroll it in {fleet}, and start the -service: +. From the agent directory, run the appropriate command to install {agent} as +a managed service, enroll it in {fleet}, and start the service. Don't have a +{fleet} enrollment key? Read the +{ingest-guide}/fleet-quick-start.html[Quick start guide] to learn how to get one +from {fleet}. + -- include::{beats-repo-dir}/x-pack/elastic-agent/docs/tab-widgets/install-widget.asciidoc[] - -- - -Don't have a {fleet} enrollment key? Read the -{ingest-guide}/ingest-management-getting-started.html[Quick start guide] to -learn how to generate one. ++ +This step installs the {agent} files into the directory locations described +in <>. Because {agent} is installed as an auto-starting service, it will restart automatically if the system is rebooted. diff --git a/x-pack/elastic-agent/docs/run-elastic-agent-standalone.asciidoc b/x-pack/elastic-agent/docs/run-elastic-agent-standalone.asciidoc index 99b39788305..51b5e1c5a93 100644 --- a/x-pack/elastic-agent/docs/run-elastic-agent-standalone.asciidoc +++ b/x-pack/elastic-agent/docs/run-elastic-agent-standalone.asciidoc @@ -16,10 +16,10 @@ To save time, use {fleet} in {kib} to generate your standalone configuration: . Log in to {kib} and go to **Management > Fleet**. -. On the **Agents** tab, click **Add agent** and look at the deployment +. On the **Agents** tab, click **Add agent**, and look at the deployment instructions under **Run standalone**. -. If you haven't already, download the {agent} to your host: +. If you haven't already, download and extract the {agent} to your host: + -- include::{beats-repo-dir}/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc[tag=install-elastic-agent] @@ -28,9 +28,9 @@ See the https://www.elastic.co/downloads/elastic-agent[download page] for other installation options. -- -. Under **Choose an agent policy**, select a policy to use for the agent. The -default policy includes a system integration for collecting logs and metrics -from the host system. +. Back in {fleet}, under **Choose an agent policy**, select a policy to use for +the agent. The default policy includes a system integration for collecting logs +and metrics from the host system. . Under **Configure the agent**, copy or download the policy. Copy this policy to the `elastic-agent.yml` on the host where the {agent} is extracted. @@ -61,12 +61,17 @@ outputs: [...] ---- -. Install {agent} as a managed service and start the service: +. From the agent directory, run the appropriate command to install {agent} as a +managed service and start the service: + -- include::{beats-repo-dir}/x-pack/elastic-agent/docs/tab-widgets/run-standalone-widget.asciidoc[] -- ++ +This step installs the {agent} files, including the `elastic-agent.yml` file +you modified earlier, into the directory locations described in +<>. For additional configuration options, see <>. diff --git a/x-pack/elastic-agent/docs/start-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/start-elastic-agent.asciidoc new file mode 100644 index 00000000000..0a094c226c4 --- /dev/null +++ b/x-pack/elastic-agent/docs/start-elastic-agent.asciidoc @@ -0,0 +1,11 @@ +[[start-elastic-agent]] +[role="xpack"] += Start {agent} + +If you've stopped the {agent} service and want to restart it, use the commands +that work with your system: + +include::{beats-repo-dir}/x-pack/elastic-agent/docs/tab-widgets/start-widget.asciidoc[] + +// Add Javascript and CSS for tabbed panels +include::tab-widgets/code.asciidoc[] diff --git a/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc index 82078c5eb78..c9a05dd8428 100644 --- a/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc @@ -2,8 +2,8 @@ [role="xpack"] = Stop {agent} -To stop {agent} and its related executables, stop the {agent} process. Use the -commands that work for your system. +To stop {agent} and its related executables, stop the {agent} service. Use the +commands that work with your system: include::{beats-repo-dir}/x-pack/elastic-agent/docs/tab-widgets/stop-widget.asciidoc[] diff --git a/x-pack/elastic-agent/docs/tab-widgets/install-layout.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/install-layout.asciidoc index 22d48b201b6..63a8d5418a7 100644 --- a/x-pack/elastic-agent/docs/tab-widgets/install-layout.asciidoc +++ b/x-pack/elastic-agent/docs/tab-widgets/install-layout.asciidoc @@ -6,8 +6,6 @@ Main {agent} configuration `/Library/Elastic/Agent/fleet.yml`:: Main {agent} {fleet} configuration -`/Library/Elastic/Agent/elastic-agent.sock`:: -Running {agent} communication socket `/Library/Elastic/Agent/elastic-agent.log`:: Log files for {agent} `/usr/bin/elastic-agent`:: @@ -23,8 +21,6 @@ Shell wrapper installed into PATH Main {agent} configuration `/opt/Elastic/Agent/fleet.yml`:: Main {agent} {fleet} configuration -`/opt/Elastic/Agent/elastic-agent.sock`:: -Running {agent} communication socket `/opt/Elastic/Agent/elastic-agent.log`:: Log files for {agent} `/usr/bin/elastic-agent`:: @@ -40,8 +36,6 @@ Shell wrapper installed into PATH Main {agent} configuration `C:\Program/ Files\Elastic\Agent\fleet.yml`:: Main {agent} {fleet} configuration -`C:\Program/ Files\Elastic\Agent\elastic-agent.sock`:: -Running {agent} communication socket `C:\Program/ Files\Elastic\Agent\elastic-agent.log`:: Log files for {agent} @@ -49,15 +43,13 @@ Log files for {agent} // tag::deb[] -`/opt/Elastic/Agent/*`:: +`/usr/share/elastic-agent/*`:: {agent} program files -`/opt/Elastic/Agent/elastic-agent.yml`:: +`/etc/elastic-agent/elastic-agent.yml`:: Main {agent} configuration -`/opt/Elastic/Agent/fleet.yml`:: +`/etc/elastic-agent/fleet.yml`:: Main {agent} {fleet} configuration -`/opt/Elastic/Agent/elastic-agent.sock`:: -Running {agent} communication socket -`/opt/Elastic/Agent/elastic-agent.log`:: +`/var/lib/elastic-agent/data/elastic-agent-*/logs/*`:: Log files for {agent} `/usr/bin/elastic-agent`:: Shell wrapper installed into PATH @@ -66,15 +58,13 @@ Shell wrapper installed into PATH // tag::rpm[] -`/opt/Elastic/Agent/*`:: +`/usr/share/elastic-agent/*`:: {agent} program files -`/opt/Elastic/Agent/elastic-agent.yml`:: +`/etc/elastic-agent/elastic-agent.yml`:: Main {agent} configuration -`/opt/Elastic/Agent/fleet.yml`:: +`/etc/elastic-agent/fleet.yml`:: Main {agent} {fleet} configuration -`/opt/Elastic/Agent/elastic-agent.sock`:: -Running {agent} communication socket -`/opt/Elastic/Agent/elastic-agent.log`:: +`/var/lib/elastic-agent/data/elastic-agent-*/logs/*`:: Log files for {agent} `/usr/bin/elastic-agent`:: Shell wrapper installed into PATH diff --git a/x-pack/elastic-agent/docs/tab-widgets/install.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/install.asciidoc index 2661b4dacf8..d3d717a149a 100644 --- a/x-pack/elastic-agent/docs/tab-widgets/install.asciidoc +++ b/x-pack/elastic-agent/docs/tab-widgets/install.asciidoc @@ -1,23 +1,23 @@ // tag::deb[] // tag::install-tip[] -TIP: We recommend that you run this command as the root user because some +TIP: You must run this command as the root user because some integrations require root privileges to collect sensitive data. // end::install-tip[] [source,shell] ---- -elastic-agent enroll KIBANA_URL ENROLLMENT_KEY -systemctl enable elastic-agent <1> -systemctl start elastic-agent +sudo elastic-agent enroll <1> +sudo systemctl enable elastic-agent <2> +sudo systemctl start elastic-agent ---- -<1> The DEB package includes a service unit for Linux systems with systemd. On +<1> `kibana_url` is the {kib} URL where {fleet} is running, and +`enrollment_token` is the enrollment token acquired from {fleet}. +<2> The DEB package includes a service unit for Linux systems with systemd. On these systems, you can manage {agent} by using the usual systemd commands. If you don't have systemd, run `sudo service elastic-agent start`. -include::install.asciidoc[tag=where-description] - // end::deb[] // tag::rpm[] @@ -26,11 +26,13 @@ include::install.asciidoc[tag=install-tip] [source,shell] ---- -elastic-agent enroll KIBANA_URL ENROLLMENT_KEY -systemctl enable elastic-agent <1> -systemctl start elastic-agent +sudo elastic-agent enroll <1> +sudo systemctl enable elastic-agent <2> +sudo systemctl start elastic-agent ---- -<1> The RPM package includes a service unit for Linux systems with systemd. On +<1> `kibana_url` is the {kib} URL where {fleet} is running, and +`enrollment_token` is the enrollment token acquired from {fleet}. +<2> The RPM package includes a service unit for Linux systems with systemd. On these systems, you can manage {agent} by using the usual systemd commands. If you don't have systemd, run `sudo service elastic-agent start`. @@ -42,12 +44,11 @@ include::install.asciidoc[tag=install-tip] [source,shell] ---- -./elastic-agent install -f --kibana-url=KIBANA_URL --enrollment-token=ENROLLMENT_KEY +sudo ./elastic-agent install -f --kibana-url= --enrollment-token= <1> <2> ---- - -include::install.asciidoc[tag=where-description] - -Omit `-f` to run an interactive installation. +<1> `kibana_url` is the {kib} URL where {fleet} is running, and +`enrollment_token` is the enrollment token acquired from {fleet}. +<2> Omit `-f` to run an interactive installation. // end::mac[] @@ -57,13 +58,12 @@ include::install.asciidoc[tag=install-tip] [source,shell] ---- -./elastic-agent install -f --kibana-url=KIBANA_URL --enrollment-token=ENROLLMENT_KEY <1> +sudo ./elastic-agent install -f --kibana-url= --enrollment-token= <1> <2> <3> ---- -<1> This command requires a system and service manager like systemd. - -include::install.asciidoc[tag=where-description] - -Omit `-f` to run an interactive installation. +<1> `kibana_url` is the {kib} URL where {fleet} is running, and +`enrollment_token` is the enrollment token acquired from {fleet}. +<2> This command requires a system and service manager like systemd. +<3> Omit `-f` to run an interactive installation. // end::linux[] @@ -76,17 +76,10 @@ and run: [source,shell] ---- -.\elastic-agent.exe install -f --kibana-url=KIBANA_URL --enrollment-token=ENROLLMENT_KEY +.\elastic-agent.exe install -f --kibana-url= --enrollment-token= <1> <2> ---- - -include::install.asciidoc[tag=where-description] - -Omit `-f` to run an interactive installation. +<1> `kibana_url` is the {kib} URL where {fleet} is running, and +`enrollment_token` is the enrollment token acquired from {fleet}. +<2> Omit `-f` to run an interactive installation. // end::win[] - -// tag::where-description[] -`KIBANA_URL` is the {kib} URL where {fleet} is running, and -`ENROLLMENT_KEY` is the enrollment token acquired from {fleet}. - -// end::where-description[] \ No newline at end of file diff --git a/x-pack/elastic-agent/docs/tab-widgets/run-standalone.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/run-standalone.asciidoc index 45aa38e0d69..4d11b322fc5 100644 --- a/x-pack/elastic-agent/docs/tab-widgets/run-standalone.asciidoc +++ b/x-pack/elastic-agent/docs/tab-widgets/run-standalone.asciidoc @@ -1,61 +1,53 @@ // tag::deb[] -// tag::install-tip[] -TIP: We recommend that you run this command as the root user because some -integrations require root privileges to collect sensitive data. - -// end::install-tip[] +include::install.asciidoc[tag=install-tip] [source,shell] ---- -systemctl enable elastic-agent <1> -systemctl start elastic-agent +sudo systemctl enable elastic-agent <1> +sudo systemctl start elastic-agent ---- <1> The DEB package includes a service unit for Linux systems with systemd. On these systems, you can manage {agent} by using the usual systemd commands. If you don't have systemd, run `sudo service elastic-agent start`. -// tag::config-flag[] -Use the `-c` flag to specify the policy file. If no policy file is -specified, {agent} uses the default policy, `elastic-agent.yml`, which is -located in the same directory as {agent}. -// end::config-flag[] - // end::deb[] // tag::rpm[] +include::install.asciidoc[tag=install-tip] + [source,shell] ---- -systemctl enable elastic-agent <1> -systemctl start elastic-agent +sudo systemctl enable elastic-agent <1> +sudo systemctl start elastic-agent ---- <1> The RPM package includes a service unit for Linux systems with systemd. On these systems, you can manage {agent} by using the usual systemd commands. If you don't have systemd, run `sudo service elastic-agent start`. - -include::run-standalone.asciidoc[tag=config-flag] // end::rpm[] // tag::mac[] +include::install.asciidoc[tag=install-tip] + [source,shell] ---- -./elastic-agent install +sudo ./elastic-agent install ---- -include::run-standalone.asciidoc[tag=config-flag] // end::mac[] // tag::linux[] +include::install.asciidoc[tag=install-tip] + [source,shell] ---- -./elastic-agent install +sudo ./elastic-agent install ---- -include::run-standalone.asciidoc[tag=config-flag] // end::linux[] // tag::win[] @@ -70,5 +62,4 @@ and run: .\elastic-agent.exe install ---- -include::run-standalone.asciidoc[tag=config-flag] // end::win[] diff --git a/x-pack/elastic-agent/docs/tab-widgets/start-widget.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/start-widget.asciidoc new file mode 100644 index 00000000000..886c0f079f9 --- /dev/null +++ b/x-pack/elastic-agent/docs/tab-widgets/start-widget.asciidoc @@ -0,0 +1,94 @@ +++++ +
+
+ + + + + +
+
+++++ + +include::start.asciidoc[tag=mac] + +++++ +
+ + + + +
+++++ \ No newline at end of file diff --git a/x-pack/elastic-agent/docs/tab-widgets/start.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/start.asciidoc new file mode 100644 index 00000000000..d8394017d6b --- /dev/null +++ b/x-pack/elastic-agent/docs/tab-widgets/start.asciidoc @@ -0,0 +1,57 @@ +// tag::deb[] + +The DEB package includes a service unit for Linux systems with systemd. On these +systems, you can manage {agent} by using the usual systemd commands. + +// tag::start-command[] +Use `systemctl` to start the agent: + +[source,shell] +---- +sudo systemctl start elastic-agent +---- + +Otherwise, use: + +[source,shell] +---- +sudo service elastic-agent start +---- +// end::start-command[] + +// end::deb[] + +// tag::rpm[] +The RPM package includes a service unit for Linux systems with systemd. On these +systems, you can manage {agent} by using the usual systemd commands. + +include::start.asciidoc[tag=start-command] + +// end::rpm[] + +// tag::mac[] + +[source,shell] +---- +sudo launchctl load /Library/LaunchDaemons/co.elastic.elastic-agent.plist +---- + +// end::mac[] + +// tag::linux[] + +[source,shell] +---- +sudo service elastic-agent start +---- + +// end::linux[] + +// tag::win[] + +[source,shell] +---- +Start-Service elastic-agent +---- + +// end::win[] diff --git a/x-pack/elastic-agent/docs/tab-widgets/stop.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/stop.asciidoc index 5349985d121..cc441f9bdcf 100644 --- a/x-pack/elastic-agent/docs/tab-widgets/stop.asciidoc +++ b/x-pack/elastic-agent/docs/tab-widgets/stop.asciidoc @@ -8,7 +8,7 @@ Use `systemctl` to stop the agent: [source,shell] ---- -systemctl stop elastic-agent +sudo systemctl stop elastic-agent ---- Otherwise, use: @@ -33,35 +33,28 @@ include::stop.asciidoc[tag=stop-command] // end::rpm[] // tag::mac[] -// tag::kill-process[] -Get the process ID (PID) of the `elastic-agent` process: [source,shell] ---- -ps | grep elastic-agent <1> +sudo launchctl unload /Library/LaunchDaemons/co.elastic.elastic-agent.plist ---- -<1> Make sure you list processes as the root user. -Then kill the process, replacing the PID in this example with the PID from -the grep command: +NOTE: {agent} will restart automatically if the system is rebooted. + +// end::mac[] +// tag::linux[] [source,shell] ---- -kill -9 90682 +sudo service elastic-agent stop ---- NOTE: {agent} will restart automatically if the system is rebooted. -// end::kill-process[] -// end::mac[] - -// tag::linux[] -include::stop.asciidoc[tag=kill-process] // end::linux[] // tag::win[] -If you installed {agent} as a service, stop the service. [source,shell] ---- Stop-Service elastic-agent diff --git a/x-pack/elastic-agent/docs/tab-widgets/uninstall.asciidoc b/x-pack/elastic-agent/docs/tab-widgets/uninstall.asciidoc index 588a6218e8c..4b50b5b41ad 100644 --- a/x-pack/elastic-agent/docs/tab-widgets/uninstall.asciidoc +++ b/x-pack/elastic-agent/docs/tab-widgets/uninstall.asciidoc @@ -9,7 +9,7 @@ include::uninstall.asciidoc[tag=uninstall-tip] [source,shell] ---- -elastic-agent uninstall +sudo elastic-agent uninstall ---- // end::mac[] @@ -20,7 +20,7 @@ include::uninstall.asciidoc[tag=uninstall-tip] [source,shell] ---- -elastic-agent uninstall +sudo elastic-agent uninstall ---- // end::linux[] diff --git a/x-pack/elastic-agent/docs/uninstall-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/uninstall-elastic-agent.asciidoc index 0f884614709..a0a3081056f 100644 --- a/x-pack/elastic-agent/docs/uninstall-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/uninstall-elastic-agent.asciidoc @@ -4,8 +4,10 @@ beta[] +== Uninstall on macOS, Linux, and Windows + To uninstall {agent}, run the `uninstall` command from the directory where -{agent} is running. Not sure where the agent is running? See +{agent} is running. Not sure where the agent is running? See <>. -- @@ -19,5 +21,16 @@ stops and uninstalls any managed programs, such as {beats} and If you run into problems, see <>. +== Uninstall on DEB or RPM + +The `uninstall` command is not supported for DEB or RPM installations. To +uninstall {agent} on DEB or RPM: + +. If the agent is managed by {fleet}, <>. +. On your host, <>. +. Manually remove the agent files from your system. + +//TODO: Confirm this procedure. + // Add Javascript and CSS for tabbed panels include::tab-widgets/code.asciidoc[] diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index 2f8187a1604..6028291bfc6 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -200,3 +200,47 @@ agent.logging.to_stderr: true # information. Recommended to use in combination with `logging.json=true` # Defaults to false. #agent.logging.ecs: false + +# Providers + +# Providers supply the key/values pairs that are used for variable substitution +# and conditionals. Each provider's keys are automatically prefixed with the name +# of the provider. + +#providers: + +# Agent provides information about the running agent. +# agent: +# enabled: true + +# Docker provides inventory information from Docker. +# docker: +# enabled: true +# host: "unix:///var/run/docker.sock" +# cleanup_timeout: 60 + +# Env providers information about the running environment. +# env: +# enabled: true + +# Host provides information about the current host. +# host: +# enabled: true + +# Local provides custom keys to use as variable. +# local: +# enabled: true +# vars: +# foo: bar + +# Local dynamic allows you to define multiple key/values to generate multiple configurations. +# local_dynamic: +# enabled: true +# items: +# - vars: +# my_var: key1 +# - vars: +# my_var: key2 +# - vars: +# my_var: key3 + diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 08a12d7907a..022ce746647 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -207,3 +207,47 @@ agent.logging.to_stderr: true # Defaults to false. #agent.logging.ecs: false +# Providers + +# Providers supply the key/values pairs that are used for variable substitution +# and conditionals. Each provider's keys are automatically prefixed with the name +# of the provider. + +#providers: + +# Agent provides information about the running agent. +# agent: +# enabled: true + +# Docker provides inventory information from Docker. +# docker: +# enabled: true +# host: "unix:///var/run/docker.sock" +# cleanup_timeout: 60 + +# Env providers information about the running environment. +# env: +# enabled: true + +# Host provides information about the current host. +# host: +# enabled: true + +# Local provides custom keys to use as variable. +# local: +# enabled: true +# vars: +# foo: bar + +# Local dynamic allows you to define multiple key/values to generate multiple configurations. +# local_dynamic: +# enabled: true +# items: +# - vars: +# my_var: key1 +# - vars: +# my_var: key2 +# - vars: +# my_var: key3 + + diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index 232ff03c62e..6134c396036 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -207,3 +207,47 @@ agent.logging.to_stderr: true # Defaults to false. #agent.logging.ecs: false +# Providers + +# Providers supply the key/values pairs that are used for variable substitution +# and conditionals. Each provider's keys are automatically prefixed with the name +# of the provider. + +#providers: + +# Agent provides information about the running agent. +# agent: +# enabled: true + +# Docker provides inventory information from Docker. +# docker: +# enabled: true +# host: "unix:///var/run/docker.sock" +# cleanup_timeout: 60 + +# Env providers information about the running environment. +# env: +# enabled: true + +# Host provides information about the current host. +# host: +# enabled: true + +# Local provides custom keys to use as variable. +# local: +# enabled: true +# vars: +# foo: bar + +# Local dynamic allows you to define multiple key/values to generate multiple configurations. +# local_dynamic: +# enabled: true +# items: +# - vars: +# my_var: key1 +# - vars: +# my_var: key2 +# - vars: +# my_var: key3 + + diff --git a/x-pack/elastic-agent/pkg/agent/application/application.go b/x-pack/elastic-agent/pkg/agent/application/application.go index d721a8aa148..f87cad3a09c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/application.go +++ b/x-pack/elastic-agent/pkg/agent/application/application.go @@ -31,7 +31,7 @@ type upgraderControl interface { } // New creates a new Agent and bootstrap the required subsystem. -func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upgraderControl) (Application, error) { +func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upgraderControl, agentInfo *info.AgentInfo) (Application, error) { // Load configuration from disk to understand in which mode of operation // we must start the elastic-agent, the mode of operation cannot be changed without restarting the // elastic-agent. @@ -44,7 +44,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upg return nil, err } - return createApplication(log, pathConfigFile, rawConfig, reexec, uc) + return createApplication(log, pathConfigFile, rawConfig, reexec, uc, agentInfo) } func createApplication( @@ -53,6 +53,7 @@ func createApplication( rawConfig *config.Config, reexec reexecManager, uc upgraderControl, + agentInfo *info.AgentInfo, ) (Application, error) { warn.LogNotGA(log) log.Info("Detecting execution mode") @@ -63,16 +64,16 @@ func createApplication( return nil, err } - if isStandalone(cfg.Fleet) { + if IsStandalone(cfg.Fleet) { log.Info("Agent is managed locally") - return newLocal(ctx, log, pathConfigFile, rawConfig, reexec, uc) + return newLocal(ctx, log, pathConfigFile, rawConfig, reexec, uc, agentInfo) } log.Info("Agent is managed by Fleet") - return newManaged(ctx, log, rawConfig, reexec) + return newManaged(ctx, log, rawConfig, reexec, agentInfo) } -// missing of fleet.enabled: true or fleet.{access_token,kibana} will place Elastic Agent into standalone mode. -func isStandalone(cfg *configuration.FleetAgentConfig) bool { +// IsStandalone decides based on missing of fleet.enabled: true or fleet.{access_token,kibana} will place Elastic Agent into standalone mode. +func IsStandalone(cfg *configuration.FleetAgentConfig) bool { return cfg == nil || !cfg.Enabled } diff --git a/x-pack/elastic-agent/pkg/agent/application/config_test.go b/x-pack/elastic-agent/pkg/agent/application/config_test.go index 4d4527a1e60..09acd68dd83 100644 --- a/x-pack/elastic-agent/pkg/agent/application/config_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/config_test.go @@ -70,7 +70,7 @@ func testMgmtMode(t *testing.T) { err := c.Unpack(&m) require.NoError(t, err) assert.Equal(t, false, m.Fleet.Enabled) - assert.Equal(t, true, isStandalone(m.Fleet)) + assert.Equal(t, true, IsStandalone(m.Fleet)) }) @@ -80,7 +80,7 @@ func testMgmtMode(t *testing.T) { err := c.Unpack(&m) require.NoError(t, err) assert.Equal(t, true, m.Fleet.Enabled) - assert.Equal(t, false, isStandalone(m.Fleet)) + assert.Equal(t, false, IsStandalone(m.Fleet)) }) } diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_settings.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_settings.go new file mode 100644 index 00000000000..bb0e2def363 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_settings.go @@ -0,0 +1,52 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +// handlerSettings handles settings change coming from fleet and updates log level. +type handlerSettings struct { + log *logger.Logger + reexec reexecManager + agentInfo *info.AgentInfo +} + +// Handle handles SETTINGS action. +func (h *handlerSettings) Handle(ctx context.Context, a action, acker fleetAcker) error { + h.log.Debugf("handlerUpgrade: action '%+v' received", a) + action, ok := a.(*fleetapi.ActionSettings) + if !ok { + return fmt.Errorf("invalid type, expected ActionSettings and received %T", a) + } + + if !isSupportedLogLevel(action.LogLevel) { + return fmt.Errorf("invalid log level, expected debug|info|warning|error and received '%s'", action.LogLevel) + } + + if err := h.agentInfo.LogLevel(action.LogLevel); err != nil { + return errors.New("failed to update log level", err) + } + + if err := acker.Ack(ctx, a); err != nil { + h.log.Errorf("failed to acknowledge SETTINGS action with id '%s'", action.ActionID) + } else if err := acker.Commit(ctx); err != nil { + h.log.Errorf("failed to commit acker after acknowledging action with id '%s'", action.ActionID) + } + + h.reexec.ReExec() + return nil +} + +func isSupportedLogLevel(level string) bool { + return level == "error" || level == "debug" || level == "info" || level == "warning" +} diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go index a93483ca1cd..f18fa542a25 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go @@ -26,8 +26,11 @@ const agentInfoKey = "agent" // defaultAgentActionStoreFile is the file that will contains the action that can be replayed after restart. const defaultAgentActionStoreFile = "action_store.yml" +const defaultLogLevel = "info" + type persistentAgentInfo struct { - ID string `json:"id" yaml:"id" config:"id"` + ID string `json:"id" yaml:"id" config:"id"` + LogLevel string `json:"logging.level,omitempty" yaml:"logging.level,omitempty" config:"logging.level,omitempty"` } type ioStore interface { @@ -45,6 +48,25 @@ func AgentActionStoreFile() string { return filepath.Join(paths.Home(), defaultAgentActionStoreFile) } +// updateLogLevel updates log level and persists it to disk. +func updateLogLevel(level string) error { + ai, err := loadAgentInfo(false, defaultLogLevel) + if err != nil { + return err + } + + if ai.LogLevel == level { + // no action needed + return nil + } + + agentConfigFile := AgentConfigFile() + s := storage.NewDiskStore(agentConfigFile) + + ai.LogLevel = level + return updateAgentInfo(s, ai) +} + func generateAgentID() (string, error) { uid, err := uuid.NewV4() if err != nil { @@ -54,11 +76,11 @@ func generateAgentID() (string, error) { return uid.String(), nil } -func loadAgentInfo(forceUpdate bool) (*persistentAgentInfo, error) { +func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) { agentConfigFile := AgentConfigFile() s := storage.NewDiskStore(agentConfigFile) - agentinfo, err := getInfoFromStore(s) + agentinfo, err := getInfoFromStore(s, logLevel) if err != nil { return nil, err } @@ -79,7 +101,7 @@ func loadAgentInfo(forceUpdate bool) (*persistentAgentInfo, error) { return agentinfo, nil } -func getInfoFromStore(s ioStore) (*persistentAgentInfo, error) { +func getInfoFromStore(s ioStore, logLevel string) (*persistentAgentInfo, error) { agentConfigFile := AgentConfigFile() reader, err := s.Load() if err != nil { @@ -104,7 +126,9 @@ func getInfoFromStore(s ioStore) (*persistentAgentInfo, error) { agentInfoSubMap, found := configMap[agentInfoKey] if !found { - return &persistentAgentInfo{}, nil + return &persistentAgentInfo{ + LogLevel: logLevel, + }, nil } cc, err := config.NewConfigFrom(agentInfoSubMap) @@ -112,7 +136,9 @@ func getInfoFromStore(s ioStore) (*persistentAgentInfo, error) { return nil, errors.New(err, "failed to create config from agent info submap") } - pid := &persistentAgentInfo{} + pid := &persistentAgentInfo{ + LogLevel: logLevel, + } if err := cc.Unpack(&pid); err != nil { return nil, errors.New(err, "failed to unpack stored config to map") } diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go index b0abbe19e64..827ae6300b9 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go @@ -4,42 +4,51 @@ package info -import "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +import ( + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +) // AgentInfo is a collection of information about agent. type AgentInfo struct { - agentID string + agentID string + logLevel string } -// NewAgentInfo creates a new agent information. +// NewAgentInfoWithLog creates a new agent information. // In case when agent ID was already created it returns, // this created ID otherwise it generates // new unique identifier for agent. // If agent config file does not exist it gets created. -func NewAgentInfo() (*AgentInfo, error) { - agentInfo, err := loadAgentInfo(false) +// Initiates log level to predefined value. +func NewAgentInfoWithLog(level string) (*AgentInfo, error) { + agentInfo, err := loadAgentInfo(false, level) if err != nil { return nil, err } return &AgentInfo{ - agentID: agentInfo.ID, + agentID: agentInfo.ID, + logLevel: agentInfo.LogLevel, }, nil } -// ForceNewAgentInfo creates a new agent information. -// Generates new unique identifier for agent regardless -// of any existing ID. +// NewAgentInfo creates a new agent information. +// In case when agent ID was already created it returns, +// this created ID otherwise it generates +// new unique identifier for agent. // If agent config file does not exist it gets created. -func ForceNewAgentInfo() (*AgentInfo, error) { - agentInfo, err := loadAgentInfo(true) - if err != nil { - return nil, err +func NewAgentInfo() (*AgentInfo, error) { + return NewAgentInfoWithLog(defaultLogLevel) +} + +// LogLevel updates log level of agent. +func (i *AgentInfo) LogLevel(level string) error { + if err := updateLogLevel(level); err != nil { + return err } - return &AgentInfo{ - agentID: agentInfo.ID, - }, nil + i.logLevel = level + return nil } // AgentID returns an agent identifier. diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go index c5712646cfb..af35372ac2e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_metadata.go @@ -40,6 +40,9 @@ type AgentECSMeta struct { BuildOriginal string `json:"build.original"` // Upgradeable is a flag specifying if it is possible for agent to be upgraded. Upgradeable bool `json:"upgradeable"` + // LogLevel describes currently set log level. + // Possible values: "debug"|"info"|"warning"|"error" + LogLevel string `json:"log_level"` } // SystemECSMeta is a collection of operating system metadata in ECS compliant object form. @@ -140,6 +143,7 @@ func (i *AgentInfo) ECSMetadata() (*ECSMeta, error) { // only upgradeable if running from Agent installer and running under the // control of the system supervisor (or built specifically with upgrading enabled) Upgradeable: release.Upgradeable() || (install.RunningInstalled() && install.RunningUnderSupervisor()), + LogLevel: i.logLevel, }, }, Host: &HostECSMeta{ diff --git a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go index edf1ad8cdf2..f1bd2893bf0 100644 --- a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go @@ -46,7 +46,7 @@ func (c *InspectConfigCmd) inspectConfig() error { return err } - if isStandalone(cfg.Fleet) { + if IsStandalone(cfg.Fleet) { return printConfig(rawConfig) } diff --git a/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go index bb319ce1569..39c578344c4 100644 --- a/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go @@ -66,7 +66,7 @@ func (c *InspectOutputCmd) inspectOutputs(agentInfo *info.AgentInfo) error { return err } - if isStandalone(cfg.Fleet) { + if IsStandalone(cfg.Fleet) { return listOutputsFromConfig(l, agentInfo, rawConfig) } @@ -119,7 +119,7 @@ func (c *InspectOutputCmd) inspectOutput(agentInfo *info.AgentInfo) error { return err } - if isStandalone(cfg.Fleet) { + if IsStandalone(cfg.Fleet) { return printOutputFromConfig(l, agentInfo, c.output, c.program, rawConfig) } diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go index f0c4153f474..b1736485ceb 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go @@ -63,6 +63,7 @@ func newLocal( rawConfig *config.Config, reexec reexecManager, uc upgraderControl, + agentInfo *info.AgentInfo, ) (*Local, error) { cfg, err := configuration.NewFromConfig(rawConfig) if err != nil { @@ -75,10 +76,6 @@ func newLocal( return nil, err } } - agentInfo, err := info.NewAgentInfo() - if err != nil { - return nil, err - } logR := logreporter.NewReporter(log) diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index fa31215f75d..aab53374c12 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -62,12 +62,8 @@ func newManaged( log *logger.Logger, rawConfig *config.Config, reexec reexecManager, + agentInfo *info.AgentInfo, ) (*Managed, error) { - agentInfo, err := info.NewAgentInfo() - if err != nil { - return nil, err - } - path := info.AgentConfigFile() store := storage.NewDiskStore(path) @@ -241,6 +237,15 @@ func newManaged( }, ) + actionDispatcher.MustRegister( + &fleetapi.ActionSettings{}, + &handlerSettings{ + log: log, + reexec: reexec, + agentInfo: agentInfo, + }, + ) + actionDispatcher.MustRegister( &fleetapi.ActionUnknown{}, &handlerUnknown{log: log}, diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index b014cd69084..95df0700f72 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -17,12 +17,15 @@ import ( "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -75,6 +78,10 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se errors.M(errors.MetaKeyPath, pathConfigFile)) } + if err := getOverwrites(rawConfig); err != nil { + return errors.New(err, "could not read overwrites") + } + cfg, err := configuration.NewFromConfig(rawConfig) if err != nil { return errors.New(err, @@ -83,6 +90,14 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se errors.M(errors.MetaKeyPath, pathConfigFile)) } + agentInfo, err := info.NewAgentInfoWithLog(defaultLogLevel(cfg)) + if err != nil { + return errors.New(err, + "could not load agent info", + errors.TypeFilesystem, + errors.M(errors.MetaKeyPath, pathConfigFile)) + } + logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig) if err != nil { return err @@ -106,7 +121,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se } defer control.Stop() - app, err := application.New(logger, pathConfigFile, rex, control) + app, err := application.New(logger, pathConfigFile, rex, control, agentInfo) if err != nil { return err } @@ -164,3 +179,51 @@ func reexecPath() (string, error) { return potentialReexec, nil } + +func getOverwrites(rawConfig *config.Config) error { + path := info.AgentConfigFile() + + store := storage.NewDiskStore(path) + reader, err := store.Load() + if err != nil && errors.Is(err, os.ErrNotExist) { + // no fleet file ignore + return nil + } else if err != nil { + return errors.New(err, "could not initialize config store", + errors.TypeFilesystem, + errors.M(errors.MetaKeyPath, path)) + } + + config, err := config.NewConfigFrom(reader) + if err != nil { + return errors.New(err, + fmt.Sprintf("fail to read configuration %s for the elastic-agent", path), + errors.TypeFilesystem, + errors.M(errors.MetaKeyPath, path)) + } + + err = rawConfig.Merge(config) + if err != nil { + return errors.New(err, + fmt.Sprintf("fail to merge configuration with %s for the elastic-agent", path), + errors.TypeConfig, + errors.M(errors.MetaKeyPath, path)) + } + + return nil +} + +func defaultLogLevel(cfg *configuration.Configuration) string { + if application.IsStandalone(cfg.Fleet) { + // for standalone always take the one from config and don't override + return "" + } + + defaultLogLevel := logger.DefaultLoggingConfig().Level.String() + if configuredLevel := cfg.Settings.LoggingConfig.Level.String(); configuredLevel != "" && configuredLevel != defaultLogLevel { + // predefined log level + return configuredLevel + } + + return defaultLogLevel +} diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index ddbe0eda468..9e615b9c271 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -17,12 +17,13 @@ var SupportedMap map[string]Spec func init() { // Packed Files + // spec/apm-server.yml // spec/endpoint.yml // spec/filebeat.yml // spec/heartbeat.yml // spec/metricbeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzMWVuTo7p2fs/P2K8nlYBodw6pOg+GDre26TE9jYTekGQDtsDsMdiGVP57SgJzcXfPJZPsysOUbVpIS+vyre9b859/nMot/ddtwcpjVlT/0uT8j3//g+RWhb8ekwAuDtTWS1JskjcQ7hnySuYcniOgHl4yg5M8uBDAa2aqLYa+SnOubDdlSougxLm1Z0/HBI97VNgOgVn4nBa4jMDbo/sUaS9PyTOx9QJpRsrstH1JjolrByktWEn2x8TN9HUMvSZC3mKlhKcI+UoM1+eJbWeqBSmz385inxUIa+wY5xgulNXrsRbPXLnntSRF+PCSLbMV0C/M1C1iWy2z+X6lTN7RfCVCAV+B6xk3+uSOyp+rXDxzn13T0GK4OBCNtWK/TVtSZBkNAayJoJIEOT9h5FP0Hzd75ffhDGRdW2ZbCg6vVN7dun54TgRSHoFqF8OFWH8iT8fn1avBt3a4RwCXxH57NLNl4prGJULBUdgy9TcdY5b161Kas87H5jJZvS4zlodNDPHC7Z8xm1cY6qqI3bpdPlNbb5kl9vOVCF5PL8mxcu3wAUN/h8U9vx6fxXuuI+KXPLqm93FevLrdOttqsLbu7V5WrukNe7sTu1avqkpt4c+A3z1vMPLPDHl7jNbZZJ9Pzp2tr7c5v3x0V3+/vJiF0eBQV0nOa5FTxL48mpmSYJTySNXzGF75zXfUtpT46Zi4+SR3kM8jLWxiFAz+jJBX0Lb30c3ncPHuzu9t8VRih23nb1xuLb1ljscjqDy6TqWb/XPiBJxyHUTwqmJ086vRYnjlkRac6f6YxHBxYSho+799w+jw6DrBgtpvfexwSpyQj3Yq07p9lj7I+YnZYYO0u7WOz4kd7pmtNy+ZUZLCUJmz7mNd8e1XiQFplF85XvZ3za0Tg+EkDw2FFiGXd7rtJ3MuOA/+BuEJQ18hmte+ZAbBYj+0qSPo7zHyWwSsSxzq4m4n18YnDENllVdllFt1FCrzHB3+bl3ijaypKkLLu1oycmKHnPU20yI8jf5dVq7jcQJ1gLszb8/lvxiEi5fMSCPgc6r5uwgZJQIV326G+zYYqmeWhzu5tr/j1Gcx4FkEF+kszvv3Pp/FrPPJ8Hse92Xl2rrKHEO93UnagXBJAT+T5PjMQMoF5hI7rLEWHJ/N4N+6PQP9+XX5N/dpmURwcXBtgVdMic3ksAW8pk6oUE0p3aeHZG0aKck3SWxb7SsIF2IPooWKWLN7vSQeGDCxxdBqIpAUz5vjP/74564V7TK+Jdv4XSsSUAM9HqHNrf3IcozyMGXLsoO1zCBuplpudkncwufMCS+rnJ/I64KT3MqIHR6+QJG+Ppdr7tcWASfIOEn4z8M6gt4Jw42Oc+tEwVu2MpfZ6q37JNCqI8g4gWHNzEVFQMC/oKSitrWPG7VLHdM9uaZbBa/i06tEODEIKyygYrI/czwVv87WnghgRQwXxSq/cpaHpy8w4FERFi5XniPkKTHEaaRtHl1b+CRoV7IdhBmGlvJD6MhkavwpygkBXmM7fLilIHP4RfhbtCV6kaVRkrwUULKjWtBgaFVIE61OpvZ5SElbrxHwzyTHpxj6SgcFotUHuwhiBaMe/jvYeXTt6xlrawktBFqXe1i9g6yGwesMniKgX7ahnhL7umO2viM2b9nTCLOuaSikPSY3m+llWmLvbK0J0C/TEsYo3WNkKDKnCl+heZgStJaxj+FGfg6wJuPsXWiuSygSECXidGerQlT9FCNfmZc7V7CMy8Snxfp/eo/R53mYE83rIVW0RllHfaxwQ4Dy6Np96V5uLejv4zNtuPNz3wIVKiid1d0BAWG32n4Wt3t7YxRw8vX9PWZnXj6F4nlbcYb8HttHbtUUXAUNnEHxzS6Z15up79SUOsYIqcPz6xn31Ep+n/pb5gXmgnL21EfWyfQ81zREvdbM1FtmB5LCUi04xPDh7pwQSBzQgj0V9tn+5ZN9VOwsH10nPNDl3JaO7gbnCFTiHgm29X0MwuZunxMB9Ezz8BAjf0fB9cwEtRU5JZ+t39+/0dst8sV7j67jL8Q7Nz/8TOtiyOcIfNBqfvAeti0lCgesGuqH5mFFNMxlC/06q/GOytiC/lsDPq3yRUpg2Aosxr/Qcu/Or+Vv5AsKIPJS9BsFI293T19GauK+q6meCihbZPA+p++ommjLlkaBfyJaeGDAUiKQjNiBSpXmb1WXd8GRQXeCK9czg0FONEFBvcW4n38mRZDGcMHpWCMHAvxvAw4LugD8ywoZalT4ajSuOzInuKCJjBr3TRXmGH9SoNfjsyrFeZWOv2e5UlEUTN5fcGbjE9HoYANp18CHloptrkypFbb5SFuc9eS7r2Cb15Pf9/mpRNpy2J/B4DKuDesYjb5lgEt5+NtU2h56+ad0Wvb4zbyfSv5QBELe5QLjt5oisVb2i5/C6knP/yWaOHKpIQ49RZvElW+RpNvcLNgRw4fHOe0bz17lv08BV+ay6PGoWEnMYN8iiL9Fr/TkmkzwISGB29ikpZn8Y6CL6Tb+Vn3AF1/tMKVF0Pkm9yssetPsWSdDXes0cDoKQoWhZR3Da/Uj/ndby+yworbsO/XAB57UPILX9o7T3fE/9YztN31rqpcI+t9WsJMkM16aqynJrQJDVfSc6f5SLs3Xir7FSpLTmsjectGxHWYM0gzdjWQkZ3bW55k/imlPlTnygG4c4GuXm11e3NeJv2OAK7GlNxgyvnWWY/+99YNZ7zUaER9U+AuBZxgFZaStz6vsNOD653j6HRn6Axz+tFY/kKN3NTs7d1wzwYViPeMcssanfMX8WH4NMg2ECyr2yd8+lGA3zNm9HpIv2fLi2laNTeMYIX+F0eHoOdWZoUCs0UUdYXhNqSb86vMIefvYlDXUYBiUtKEnYasHOtz2GlGPgov7otcdvebwfKutfFt9y+gHxfUVhgrN+b4XX3sChRBROXO8MgK9SOvmHglshgJoMQpUai5KYis/KpbbWkEyL8S2FPwjEXdXLATqB/xVfVghocNPVa+Nvyfixv1R0DB4J/hsvcBCLDWLk2zGT+oBQ0/FjccEmDCb51FHwmVB0UavMAqaGPp9gRlnqgWzWV6XFB0pmc3SZvMl9YwdOXuosSmJmRAM9Raqw+xIiAThb4w2jwJoCAhkMa/yzZlqvBUgtSp4RcyFIIE30fI8zjQ+Lvhps4rh4oBRcmuKktC8ZMbtjm3XgHgd53J205ModUcd7xyBsKVAH4qHgMUuAnqN82vZiVheUxA2zNJTXAQDaRnEaJ9vvUBoRO4QOMw+c5rr1XuREJzHZ/7Nnt5ONaVPd3PLD4TPJ2JjTzRjgYB1ItYnoq47ezxzAg7v7744k4GwdMR/a/ucOhvZnAbB1Mi6KHuxOeRqN8CYiccMbe5s1YIzAteSapv5nOomyiYxmgnMX7rHEMMMQyzB7C8Wju8IOtJYyex0R/OwwCgdhgwfkPKuKWUP31agxzFtffguAfxrSeNviuLw8yb9PZHseKLGt89P+uZLN7j52yo7le991DdSccbTMfGms+VOwNURVPlcdPXEdrZ2HHgI/GbwykdCqqYxCHcR8proft7a58iAE2AgspNcudns87Ex/4y4nLz3K2L2bub91wpg+budzoL/KhF9J/7/lwXPnGOId+9zCxfeeZoLkqMIzvG+Nwx98lfE02zfMbY9AZvM3iek7mMBNbtLfcv33xJRnXAaiN7PiqgypoftRyrqzbb2MQiVGdFzBKGqOLPviR6tgu6gHxA9sebd2u8SPclSG9Xq2OpPET2pKFdvb/LzB0RvvvZTosc+I3pSwWH0qZL6PcVyB5CfqRXax8rszx0n/MJXi27Sn//99p+H/xeq5v+FepGJ/V//9N8BAAD//5HMGNA=") + unpacked := packer.MustUnpack("eJzMWUl3qz6W3/fH+G+rBxBxuuhzamFIM9khz+QFCe2QZAO2wPwDHqBPf/c+EhiDnbyhXtc7tXKMhYare3/Dzf/8UZVr+h9xmf9btX4/rt//vcn5H//1B8mtGn/dJ6vQ8Jehz2mBOU3KLYGrR9e2TuRVbTHyAEbuIkKeEkOcRtqHvxW03Sewcevg1a1c06sjOEsxCGsMZ8oyDw8R9CoMVzpzPBXLMZ+NVY/YftPXpnqKoP++hLjCMFTc7JS4mWrJz3yy/gHblhKFesscj0dQbe/X85hZeCqxw/Yl2SeuqSQ45w9I8xWahyn5uk/WmrJwzXnt2rgkTsAp10EEzypGz49mNk9cc564js+JHW6ZrTcvmVGSwlCZ87yQv5nzJAbh7CUzlDUy+PUdnBIn5LTdD+O6dawTWQ17qZgdNt3exO9Gi+GZR1pwpMVofXOeLF/v1+3m01XmGCqbl1psh4eXzKgwnBXMTvaeU/fv+Pridf4X92meRHC2c+00pUrN16/Jbg369x2lck3GiW21zOZbCsKU5v7ea3aLP/61S6R1wcp9VtQ3aRTA2Y7aekmKVfIGwi1DXsmc3SIC6u4lMzjJgxMB/MBMtcXQV2nOlfWqTGkRlDi3tuxpn+DrHDW2Q2AWMi3LCLw9uk+R9vKULIitF0gzUman3XXaQUoLVpLtPnEz/TmGXhMhb7ZUwipCvhLD5+Nob0eqBSmz345iniUID9gxjrFIvdf9QTxz5ZznkhThw0s2z5ZAPzFTty4hWSqjdzRfiVDAl+B8xI0+OqPy5zIXz9yFaxpaDGc7orFWzLdqS4osoyGANRFUkiDnFUY+Rf992a/8e1gDWeeW2ZaCwzOVZ7fOH64TgZRHoN7EcCbGV+Rpv1i+Gnxth1sEcEnstz6VjFOEgr3Yyzje9HpnWT8upTlrL2m5fJ1nLA+bGOKZ2z9jNq8x1FVxd8/tfEFtvWWWmM9XIniuXpJ97drhA4b+RqQ5/tqXgSPuL3l0Te/jvHh1u3G21WBtKIHaNb1hbne0r+WrqlJbxDPgN88bjPwjQ94Wo+dsNM8n607GH9Y5P310Vn87P5mF0eBQV0nODyKniH16NDMlwSjlkarnMTzzS+yobSnx0z5x81HuIJ9HWtjEKBji2UPp4lru84zB2d2Z7/cywJuEsLV1gUPl0XVq3eyffwxtI7jZ7pMYzk4MBW3/2ztGu0fXCWbUflt8DGnKuG4XMgY9pCHtZuy3INSu+fqrxIA0ys8cz/uz5lbFYDjKQ0OhRcjlmS7zyZwLjkO8QVhh6CtE89qXzCBYzIdWhwj6W4z8FgHrFIe6OFvl2h3FLPO6jHLrEIXKNEeH361TvJI1VUdoflNLRk7skLN+z7QIqxF11K7jcQJ1gLs1J5De00YaAZ9Tzd9EyCgRqPl6NZy3wVA9sjzcyLEDTVxjFgOeCRqd3PP2PuaTO+tiMnyf3vuYUkb7QLikgB9Jsl8wkHKBucQOD1gL9gsz+M9uzuCGZgReMSU2Bc3wA3VChWpK6T49JM+mkZJ8lcS21b6CcCbmIFqoiDGb11PigQETWwytJgJJsVjt/3ahok3G12Qd31GRgBro8QitLvQjyzHKw5TNyw7WMoMMiqLwOXPC0zLnFXmdcZJbGbHD3Rco0tfnE/VxGVsEnCCjkvA/Uhw4tyoK3rKlOc+Wb90ngdYhgowTGB6YOasJCPgXlNTUtrZxo3apY35LFX1TQVUEsCKGs2KZnznLw+oLDHhUhIXLlRvlJmIStEtJB2GGoaV8FzoymRp/inJCgB+wHT5cUpA5/CTiLWiJnmRplCQvBZRsqBY0GFo10gTVydQ+Dilp6wcE/CPJcRVDX+mgQFB9sIkgVoSq7FJXws6ja5+PWHuW0EKgdbqF1RvIahg8T+ApAvppHeopsc8bZusbYvOWPY0VmaGQdp9c9kxP4xK72+uBAP00LmGM0i1GhiJzquhVJXqWdx/DlfwcYE3es3eiuS6hSECUuKebvSpE1asY+cq03LmC5b2MYlo8/73nuMY8D3OieT2kCmqUddTfFW4IUB5duy/d04WC/np9pg1nXvQUqFAh6azuDAiIfavtZ/d2u98YBVyo8tvnkzVPn0LxlFacsfLv6SO3DhSchQycQPFlXzKvV+PYqSl1jCukDs/PR9xLK/n3ON4yLzAXkrOXPrJOxuu5piHq9cBMvWV2ICUs1YJdDB9u1gmBxAEt2FKxP9s/fTKPip35o+uEOzqf7qWTu8ExArU4R4JtfRuDsLmZpyKAHmke7mLkbyg4H5mQtiKn5LPn+/M3ertGvnjv0XX8mXjnEocfoS6GfI7AB1Tznfemru9aPzQPa6JhLin066TGOyljC/lvDfi0zGcpgWErsBj/BOXerN+5UOQLCSDyUvCNgpG3uZUvV2ni3tXUvYM0bl2ocI4aBX5FtHDHgKVEILliBypVmr/VXd4FewbdEa6cjwwGOdGEBPVm1/n8IymCNIYzTq81siPAfx9wWMgF4J+WyFCjwlej67g9c4ITGtmo67ypwhzjTwr0w/VZneK8Tq/fJ7lSUxSM3p9xZuOKaHTYA2mfgQ8tFdtcGUsrbPOrbHGeR3/7Crb5YfT9Nj+VSJsP8zMYnK5jw0OMrrFlgEt7+MtS2h64/FM5LTl+NeVTqR+KQNi7XGD8WlMk1kq++CGsHnH+T8nEq5Ya7qGXaKN75Wsk5TY3C7bH8OFxKvuuay/zX5eAS3Ne9HhULCVmsPcI4vfolVauyYQeEha4jU1amsnfBrmYruP3+gO9+GqHKS2CLja5X2PBTZNnnQ11rWrQdBSECkPzQwzP9ff032Uss8Oa2pJ3DoMeeFLzCJ7bX++KqSnJrQJDVXDOeH5pl6ZjBW+xkuT0QCS3nHRshxmDNEM3LRmpmZ3n4yQef3c3zd8wwJXY0hsMGV878yv/Xvhgwr1GI+4HFf5M4BlGQRlpz8dlVg24/jmefsOGfgeHP63VD+zoTc3+Azp603UER1AxT/72oQW7YM7mdZd8yeYn17YO2DT2EfKXGO32nlMfGQrEGF3UEYbnlGoirj6PkLeNTVlDDYZBSRtaib16oMNtrxH1KLS4L7hu0hXM1/V7Rj8orq8wVGjOt7356tvLKmeOV0agN2n3LeQWo0Cl5qwktvK9YrmMFSLzRGxLwd8zcTfFQqC+w1/VhyUSPryqe2/8LRN3nR8FDYM3hs/WCyzMUjOrJBk/qTsMPRU3HhNgwmyeR50IlwVFG73GKGhi6PcFZhypFkx6eV1SdKJk0kub9JfUI3Zk7+GATSnMhGE4rKE69I6ESRDxxmj1KICGgEAW8zJfHanGWwFSy4LXxJwJEXgxLYtrT+Pjgh+TVQxnO4ySCylKQfOSGZczth0B8UOcy97NpXW/oY53jEDYUqAPxUPAbBMB/YDzc9mZWH6gIGyYpae4CAbRMpjRPt96g9CI3CFw6H3mNNfre5MQHK/P/OFfCd0+1ZQ+3fQtPzA+n5iNLdGMGQJWRaxPTF239nXNETjcn312JINg6YT/2vY5dVaSnAbD1Mi6KHuzOeRq18CYmMcMrW72qgVHBM4l1VbTPtXFlI3uaGIwf+ocwx1mGGIJZr/ZON4JdKSxktnphuZhgVE6NBk+EOUdKWUP70vQ45j2vPumAPy9ovEXTXH4OUl/yyQ7nqjx9eJJX33pGjd/WWZVeR+jnkjFGk/7xBv3ljsDd4igyqemqxe2k7HXhofAbwbP/CpI1TQG4SZCXhPd9lv7HBlwAgxCdpQrlz37/ErMP2IuR+/9jJm96Xn/XgMsv7fjXvDvMtE35v//2fBMNYZ49za3cOEdx7kgNYrQHPfcMPDkz5inybzXu+0F2Kj3PhJ1HxuoyVkOl3z/JRPVGadB6P2oiSpjult/5KLebGsbg1CZCD1HCKqaM/tW6NE66Bb6jtATY+7GflPoSZXaqFanVn9I6ElHuXx7k5/fEXrTsZ8KPfaZ0JMODqNPndSvOZYbgPzMrdD+rsx+3WuHX8Rq1nX6879e/nn4j3A1/xTuRSb2//7L/wUAAP//S63auA==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index a2886ccf28e..3f36e2b540a 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -54,7 +54,7 @@ func new(name string, cfg *Config) (*Logger, error) { if err != nil { return nil, err } - internal, err := makeInternalFileOutput() + internal, err := makeInternalFileOutput(cfg) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func toCommonConfig(cfg *Config) (*common.Config, error) { func DefaultLoggingConfig() *Config { cfg := logp.DefaultConfig(logp.DefaultEnvironment) cfg.Beat = agentName - cfg.Level = logp.DebugLevel + cfg.Level = logp.InfoLevel cfg.Files.Path = paths.Logs() cfg.Files.Name = fmt.Sprintf("%s.log", agentName) @@ -96,7 +96,7 @@ func DefaultLoggingConfig() *Config { // makeInternalFileOutput creates a zapcore.Core logger that cannot be changed with configuration. // // This is the logger that the spawned filebeat expects to read the log file from and ship to ES. -func makeInternalFileOutput() (zapcore.Core, error) { +func makeInternalFileOutput(cfg *Config) (zapcore.Core, error) { // defaultCfg is used to set the defaults for the file rotation of the internal logging // these settings cannot be changed by a user configuration defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment) @@ -115,5 +115,5 @@ func makeInternalFileOutput() (zapcore.Core, error) { } encoder := zapcore.NewJSONEncoder(ecszap.ECSCompatibleEncoderConfig(logp.JSONEncoderConfig())) - return ecszap.WrapCore(zapcore.NewCore(encoder, rotator, zapcore.DebugLevel)), nil + return ecszap.WrapCore(zapcore.NewCore(encoder, rotator, cfg.Level.ZapLevel())), nil } diff --git a/x-pack/elastic-agent/pkg/fleetapi/action.go b/x-pack/elastic-agent/pkg/fleetapi/action.go index 2329546629c..211b9199f2f 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/action.go +++ b/x-pack/elastic-agent/pkg/fleetapi/action.go @@ -19,6 +19,8 @@ const ( ActionTypeUnenroll = "UNENROLL" // ActionTypePolicyChange specifies policy change action. ActionTypePolicyChange = "POLICY_CHANGE" + // ActionTypeSettings specifies change of agent settings. + ActionTypeSettings = "SETTINGS" ) // Action base interface for all the implemented action from the fleet API. @@ -145,6 +147,34 @@ func (a *ActionUnenroll) ID() string { return a.ActionID } +// ActionSettings is a request to change agent settings. +type ActionSettings struct { + ActionID string + ActionType string + LogLevel string `json:"log_level"` +} + +func (a *ActionSettings) String() string { + var s strings.Builder + s.WriteString("action_id: ") + s.WriteString(a.ActionID) + s.WriteString(", type: ") + s.WriteString(a.ActionType) + s.WriteString(", log_level: ") + s.WriteString(a.LogLevel) + return s.String() +} + +// Type returns the type of the Action. +func (a *ActionSettings) Type() string { + return a.ActionType +} + +// ID returns the ID of the Action. +func (a *ActionSettings) ID() string { + return a.ActionID +} + // Actions is a list of Actions to executes and allow to unmarshal heterogenous action type. type Actions []Action @@ -195,6 +225,17 @@ func (a *Actions) UnmarshalJSON(data []byte) error { "fail to decode UPGRADE_ACTION action", errors.TypeConfig) } + case ActionTypeSettings: + action = &ActionSettings{ + ActionID: response.ActionID, + ActionType: response.ActionType, + } + + if err := json.Unmarshal(response.Data, action); err != nil { + return errors.New(err, + "fail to decode SETTINGS_ACTION action", + errors.TypeConfig) + } default: action = &ActionUnknown{ ActionID: response.ActionID, diff --git a/x-pack/elastic-agent/spec/apm-server.yml.disabled b/x-pack/elastic-agent/spec/apm-server.yml similarity index 85% rename from x-pack/elastic-agent/spec/apm-server.yml.disabled rename to x-pack/elastic-agent/spec/apm-server.yml index c84405dfadd..d86e77fb0ff 100644 --- a/x-pack/elastic-agent/spec/apm-server.yml.disabled +++ b/x-pack/elastic-agent/spec/apm-server.yml @@ -1,7 +1,7 @@ name: APM-Server cmd: apm-server artifact: apm-server -args: ["-E", "management.enabled=true", "-E", "management.mode=x-pack-fleet"] +args: ["-E", "management.enabled=true", "-E", "management.mode=x-pack-fleet", "-E", "apm-server.data_streams.enabled=true"] rules: - fix_stream: {} - filter_values: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 42b7e32547f..ffcf422494b 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -365,6 +365,16 @@ filebeat.modules: # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: @@ -813,8 +823,65 @@ filebeat.modules: # "+02:00" for GMT+02:00 # var.tz_offset: local -#----------------------------- Google Cloud Module ----------------------------- -- module: googlecloud +#--------------------- Google Cloud Platform (GCP) Module --------------------- +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + +#----------------------------- Googlecloud Module ----------------------------- +# googlecloud module is deprecated, please use gcp instead +- module: gcp vpcflow: enabled: true @@ -823,11 +890,11 @@ filebeat.modules: # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be # configured to use this topic as a sink for VPC flow logs. - var.topic: googlecloud-vpc-flowlogs + var.topic: gcp-vpc-flowlogs # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -841,11 +908,11 @@ filebeat.modules: # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-firewall + var.topic: gcp-vpc-firewall # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-firewall-sub + var.subscription_name: filebeat-gcp-firewall-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -859,11 +926,11 @@ filebeat.modules: # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-audit + var.topic: gcp-vpc-audit # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-audit + var.subscription_name: filebeat-gcp-audit # Credentials file for the service account with authorization to read from # the subscription. diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index dd98b643c3f..48ff49e7aa9 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -29,7 +29,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/envoyproxy" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/f5" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/fortinet" - _ "github.com/elastic/beats/v7/x-pack/filebeat/module/googlecloud" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/gcp" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/gsuite" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/ibmmq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/imperva" diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index bd9b1d32769..a8a6e5ae726 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -28,6 +28,32 @@ processors: field: ["aws.vpcflow.start", "aws.vpcflow.end"] ignore_missing: true + - script: + lang: painless + ignore_failure: true + if: ctx?.aws != null + source: >- + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v instanceof String && v == "-"); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx.aws); + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json index d508bd63479..6b9e4382bb5 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json @@ -1,6 +1,5 @@ [ { - "aws.vpcflow.instance_id": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json index 22705d87101..e8224ee08b1 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.version": "2", @@ -27,7 +26,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.version": "2", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log new file mode 100644 index 00000000000..2ce24460ff9 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log @@ -0,0 +1,5 @@ +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA + +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json new file mode 100644 index 00000000000..b28207021b6 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "SKIPDATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 183, + "service.type": "aws", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "NODATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 526, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index ab7f477b8bb..fdea9b1f252 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -14,6 +14,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 485d6ddcab3..ee7c5961f85 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -19,6 +19,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -41,6 +44,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index d358caa2edd..f37b4bf9ee8 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW01z2zYQvftX7PiYady7Dp1RnaTjmSbOOM6ZA5MrGjVEsAAoj/LrO+D3BwBCFkil0+rUmsq+R2B3sfsWeg8veNwA+VEIvAJQVDHcwPVW///1FYBAhkTiBp5QkSuABGUsaK4ozzbw2xUAQPld+MyTgmkTO4oskZvy0XvIyB478/qjjjluIBW8yOu/GGwOzfRNyeKp/XZEk/Z5Y/gFj69c9P9uNF99Kup9k3D3YQIZcyGQkSCIt50tE5TCjGTqbJTKjAlAoOSFiHFiv78hM9YfpjbGu9WHHLyM64VmYPvQ/VfrY43fIhTc1G6DmAt+oAmKBUC1jV81iszJYLU7dPPTVeHDIX8ZW2vTR6GeuaA/qhAUVZ4JBLrt24aB7RY+VvRA1ZHxVM7GzThhdjR2pGAqKiNlAzvCJPqF26cytmDHRZ2vGj6gCd14RyFmiqqjceVMYTOzbndmeyYKfRoxI3QvI5pRRYnCJHo6RoWchI+bmgc9/bktsaDFgqcjWLBstMHt7EOqZv87gTCYgmBMI6UHzNbh8ocTqjuRxTp0vjmAGjK7grF12HxyIbVrEz/jnqywNGacYdTdvLNGGX/6C2NleFw9iOao9r4W7Ume0yyt/831u+tzotf6SoMzYYnssZ0B8EkZMub5GoFhhekfYca3CMxka8dpqKAul7JJxTIkY9u3E6h8dOO4tq9PV3CGEZGSptkeMxW5thQ8F/OEt9CfB84QOgpOrxoRT3BXHn/jzmUt1h3+uGK3UV6f5MkucJGV7O2/x0rmgmYxzQlbnezXBvk0mprJpYhOsBt6PEdR9RxhG537xu60puz16bpbMCzLea1dwZT9fWtQ7WhEdYpNOGSz5bZKIQpTLswtyptgb00Wu4NIB3N40I/arhs6F9q1FMVxffPmduyrzeJcQyZRHGiMkcC/C5SW5DYXfh5h963CgYcKB+6mdlpKiqhCRjFPTCkhBJcSAAYA066/SKjyavlP6t211UnjPiMOuPr68P77+IytVeA7UM/YZcIbgNtCCMwUO/4C2/JtqKy+k7EjyCLPudDN9oGwAm9WzauPfab23NrBH1DIab0ajIHJvJce8yZsoyLjEpbPfdeJyAw/z1kSPsPqrW2j125/Lt9W67JMXvuTpzb7Yy0zSqjMGTmaIjAQm20jU9ZQZgFt5C8CiXT082cReihtl8lY5ys2v1rWyU9AVpaJ0JgK42layaX1ib2YC6WVUmqDmWZRS+0egE1XKBsxrBOmQPCO7ejiiChU1BFD+gtnBpC2LxXZTxWRmSog0DpYrbenCxEp6uCtRkcupdGm7CygID6WrNqR51RL9BPxZjIlhFXQPriSJcw5fWAylha+3XZ70x6QxKMNpV2JPCJJIlCadjgwGaA5bB1gDadCoog6eWMd3/kuUXSayrwL7XlCdxSTqKtmjJELXrqs56gA3hDsJ6zB5/qd7BVa8/FVgDN8jcpmxu4US+hTX/AV3LAnJKiFODbJyo8nZ8klFvKeJRaCbQLpTaNPPrh86ohsx8W+vmGAKREJzdKyBq2R+Vtn0iS3RWOwOcrWCuEbQHUl2Qqu1hF3n3lQF2jEJ4/cCNPAWp/vbBEAQx8wiGmL8tvmuY+8P974tWlOt32moLHcDoGQ8fTdjuEbUJrmBaPJt8yAf10orT0s84ghmrvKzcWYGcpcn/9oI5+mGc2Cq+fa7HuahdTP/9ei/3vicA3fMxWcgM32Ree7K88cVp7nLiP5V5nsPM3/AvpkLLBsaIh91PB2YfK2Ml5aAKP4OZBBlh81lBWJ15jhJ1L0/SSiUKszU68NSC21MCURl6ie54thW1qWPvQKI7E893NTp6QZInPkYLPf3RmmmKlIL0whcalQKUH06oMRpAvbLCmvERIWkThGKaPqlshSEdzCQQUHFri2jhM0pZrd0vdo7msgEM1FGrtTUxnRTKEoh1ULufSdBBdGW2LyF8wiKmWBYsEAe9QwUMG4I2xAaLkJ5oCQc4iZC64drZTC6R4jw933htGOceO57sHna4tSHtxAM9hTxqhEHWJ2/xZUvkQJKkLZMgv1QOULWAAGJBgekEUkTQWmugpZkE4JBQ4oA7GkEHoLq7JxcW4VWlmk3k3RBvR0AlvIyUtCZvvj352ucMi2P2D0OmlnjpEzVP9vNss+wj4KwYXtCiSEHRt+1FhgxGrlOywvic4kgEstVs1ujYn4hxJqTkeuZRedB45S4X55XvcNIjgQG3pPgr/OKd1BWP3uAPrJr1coUUijPrMAqUeN5a4OmmvaMz/sCHY5un109U8AAAD//4AUZiI=" + return "eJzsXM9v2zoSvuevGORYbLP3HBbwpukiwLYpkvQsMNJY4Qst6pGUA/evfyBl/bJIio4pOQ+vPrWR830fh5zhcIbKZ3jF3TWQX5XACwBFFcNruFzp/19eAAhkSCRewzMqcgGQoUwFLRXlxTX85wIAwHwXvvGsYhpiTZFl8to8+gwF2WAHrz9qV+I15IJX5f4nFswhTB9KVs/ttxOatc8b4FfcvXHR/7kVvv7U0vuQcPdlRJlyIZCRKIw3HZaNSmFBCnUySw1jIxAoeSVSHOH3J2QC/WGMcThbfcrBYHwDmqDtU/eH1uc6HEUsujFuw1gKvqUZihlINca/NYssycDaHbv96aL08Zi/H6K14aNSL1zQX7ULijrORCJd9bFhgN3Sp4puqdoxnstJvzkMmJ2MNamYSoynXMOaMIlh7vbV+BasudjHq0YPaEFXwV6IhaJqZ7WczW0m7HZnx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyH380gLk6c+N4YKWC5534OByyQb/Yh9Kta+/IwSDzQkOZeR0i8UyWv7npep2ZLGMnEcPUSNmXTG2jJqvPqbWNukLbsgCprHzDL3u6pPTy/jzH5gqy+P6QTIltfe1ZEPKkhb5/ncuP12e4r3OIQ32hDmix2qCICRkyJSXSziGk6a/hVlHEVnJys3TSEGdLhWjjGUoxjVvR0i59fP4pq8vV3CGCZGS5sUGC5X4phQCjXnEKPTngTOEToJ3VR0Iz3Bttr/Dk8tSqjv+w4zdJXl5kUcvgbNYsjf/AZYsBS1SWhK2uNgfDfNxMrWScwkdcTfyeImiPnPEPejcN7jjnLJ3TtenBYtZTjvaVUy5x7sn1QuNqK5iE4/ZjtxmKURhzoX9iPIu2hsbYrcRaWeOT3qrcf3UpdBLS1E8zG/efRz74UKcOpBJFFuaYiLwzwqlI7hNuV+A2z3WPPBQ88DdGKeVpIiqZJLyzBYSYmgxBDAgGJ/6q4yqoCP/UWd3jTo6uE8UB3zn+vjr9+kFW1Tga1Av2EXCK4CbSggsFNv9C1ZmNFTW3ynYDmRVllzow/aWsAqvFo2rT32l7tja0W9RyHG+Gk2BDT6oHvMubmtFxldYPnWsoyIzfJy9JH6E1VPbeq8bfyre1naZJ679n+cu/MNaZpJRWTKys3lgJDWrpky5p7IX0A7Wi0AiPef5kwQ9GGwTjHW8YtPWcnZ+IqpydIQOpTCe53W5dL9jz7aE8rpS6qIZR1FH7h5BTZcoWzmcHaZI9J7p6PyIKFTU40P6Cyc6kMaXimzGFZGJLCCSHZzo7e5CRI7aeevWka/S6KrszFBBfDKq2pbnuJYYVsSbiJQQt4L2xRcsYWrRRxbjOMK30+4+tEcU8eRiaS1RJiTLBErbDEcWA7SElYes0VRJFElX3lhm7fyUKLqayvQS2vCMrilmSZfNWD0Xguqyga0CeIezH2GDb/sxuTO05hNaAS7wLTGHGfeimKM+9R3fwE97RICaSWMTrMJ0cpadw5D3LHMIbANIrxt99MYVkkcUay42+xsGmBOR0SI3Oeiemb+3J01KlzdG66OsnBShDrTPJNuCq7PF3VcedQk0xaeA2Ahjx1pe72QSAMM1YCmmzapvVZYh5f3DiV9a5njaJxIax+0QiOlPP90coQ6lZZ7Rm0LTDPjbudLSzbIAH6KlL92cTZklzQ35R1t+Y0TpPe+jXJlr9Bx1Ze53+y8y8+/2X7/BFY/ycYzXGjUtv8e/I+xEbWhvGK8ithrGcA3RbbGlgpt7ERGn0AHakuopfqIbfFSCFnnkteMAbk1LGIt53dyC11A9poThz4JGNK0dsiFsKp6jRPGUC952zHGjaFT68FyNDKh1HFPjOL65P9pxdWylRfR+tYb9TIuYHevf3d9/Xjt2T9+Dii7AhX3WlGrhLv/CKdQ8TfY6kp3WZT9DRzAVaEqIxN3cf38r8KYGNwhgbTcOGg/zN/dNDSCosf+BeuhhTZlY1pmokAxEzWUYI8TXxi7L2bgdRcI+9QKXUMoybJl6m4gxIkcJLvzuLR2qj6TaMJXEuVzFkGjrg5Wkc9siMxf3CUtImqKUifXYGktWRwc1nf2U3MvjBM2pVjf3zdX7PRGI5uqqe1FTmdBCoTDXQ2Za0ncSfBxtislfsUiolBWKGR3sSdNATeP3sIGg+e4MDQR5rw2VguuFZg5mdIOJ5W2zRtGaceu+HqDnR8tiNm6gBWwoY1SidjH3+hZUviYZKkLZPIZ6oPIVHAQDEQy3yBKS5wJznYXMKMdQgYfKIiyrhJ7COm2cXVvNZpLUuzHbQJ4OYDMtciPIjn/4lx4W2GTbPxkQtNNObCMn9NktddD6E9JKRyG4cL10AHEv6txqLrBytQ0zNK9lTASAcxlrr26JO2hfDNVU53ZfdtFxYCcVbubXdd8wgoexkfcs+NtUbzmKqv96iD74hUYlKmmtz8wg6klz+bOD5sWoiVcpo72O1D66+CsAAP//n/8+qg==" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml new file mode 100644 index 00000000000..ac03e0004f5 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -0,0 +1,66 @@ +- name: platformlogs + type: group + release: beta + default_field: false + description: > + Fields for Azure platform logs. + fields: + - name: operation_name + type: keyword + description: > + Operation name + - name: result_type + type: keyword + description: > + Result type + - name: result_signature + type: keyword + description: > + Result signature + - name: category + type: keyword + description: > + Category + - name: event_category + type: keyword + description: > + Event Category + - name: status + type: keyword + description: > + Status + - name: ccpNamespace + type: keyword + description: > + ccpNamespace + - name: Cloud + type: keyword + description: > + Cloud + - name: Environment + type: keyword + description: > + Environment + - name: EventTimeString + type: keyword + description: > + EventTimeString + - name: Caller + type: keyword + description: > + Caller + - name: ScaleUnit + type: keyword + description: > + ScaleUnit + - name: ActivityId + type: keyword + description: > + ActivityId + - name: properties.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Properties + diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml new file mode 100644 index 00000000000..496480aa1d0 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml @@ -0,0 +1,16 @@ +type: azure-eventhub +connection_string: {{ .connection_string }} +eventhub: {{ .eventhub }} +consumer_group: {{ .consumer_group }} +storage_account: {{ .storage_account }} +storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-platformlogs-{{ .eventhub }} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/config/file.yml b/x-pack/filebeat/module/azure/platformlogs/config/file.yml new file mode 100644 index 00000000000..e9470671e07 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/file.yml @@ -0,0 +1,14 @@ +type: log +paths: + {{ range $i, $path := .paths }} +- {{$path}} + {{ end }} +exclude_files: [".gz$"] +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml new file mode 100644 index 00000000000..8493ef886fe --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -0,0 +1,195 @@ +description: Pipeline for parsing azure platform logs. +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- script: + source: ctx.message = ctx.message.replace(params.empty_field_name, '') + params: + empty_field_name: '"":"",' + ignore_failure: true +- json: + field: message + target_field: azure.platformlogs +- date: + field: azure.platformlogs.time + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- date: + field: azure.platformlogs.EventTimeString + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" +- remove: + field: + - message + - azure.platformlogs.time + ignore_missing: true +- rename: + field: azure.platformlogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.platformlogs.Region + target_field: cloud.region + ignore_missing: true +- json: + field: azure.platformlogs.EventProperties + target_field: azure.platformlogs.properties + ignore_failure: true +- remove: + if: ctx.azure.platformlogs.properties != null + field: + - azure.platformlogs.EventProperties + ignore_missing: true +- rename: + field: azure.platformlogs.EventName + target_field: event.action + ignore_missing: true +- rename: + field: azure.platformlogs.properties.log + target_field: message + ignore_missing: true +- rename: + field: azure.platformlogs.callerIpAddress + target_field: source.ip + ignore_missing: true +- rename: + field: azure.platformlogs.level + target_field: log.level + ignore_missing: true +- rename: + field: azure.platformlogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration + * params.param_nano;} + params: + param_nano: 1000000 + ignore_failure: true +- rename: + field: azure.platformlogs.location + target_field: geo.name + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx?.azure?.platformlogs?.properties?.eventCategory != null) { + ctx.azure.platformlogs.event_category = ctx.azure.platformlogs.properties.eventCategory; + } + else if (ctx?.azure?.platformlogs?.properties?.policies != null) { + ctx.azure.platformlogs.event_category = 'Policy'; + } + else { + ctx.azure.platformlogs.event_category = 'Administrative'; + } + ignore_failure: true +- rename: + field: azure.platformlogs.resultType + target_field: azure.platformlogs.result_type + ignore_missing: true +- convert: + field: azure.platformlogs.result_type + target_field: event.outcome + type: string + if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')" +- convert: + field: azure.platformlogs.properties.result + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)" +- convert: + field: azure.platformlogs.Status + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)" +- rename: + field: azure.platformlogs.operationName + target_field: azure.platformlogs.operation_name + ignore_missing: true +- convert: + field: azure.platformlogs.operation_name + target_field: event.action + type: string + ignore_missing: true +- rename: + field: azure.platformlogs.resultSignature + target_field: azure.platformlogs.result_signature + ignore_missing: true +- rename: + field: azure.platformlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.platformlogs.properties.statusCode + target_field: azure.platformlogs.properties.status_code + ignore_missing: true +- rename: + field: azure.platformlogs.Status + target_field: azure.platformlogs.status + ignore_missing: true +- geoip: + field: source.ip + target_field: geo + ignore_missing: true +- script: + lang: painless + ignore_failure: true + params: + "write": + type: + - change + "read": + type: + - access + "delete": + type: + - deletion + "action": + type: + - change + source: >- + if (ctx?.azure?.platformlogs?.category == null) { + return; + } + def hm = new HashMap(params.get(ctx.azure.platformlogs.category.toLowerCase())); + hm.forEach((k, v) -> ctx.event[k] = v); +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/platformlogs/manifest.yml b/x-pack/filebeat/module/azure/platformlogs/manifest.yml new file mode 100644 index 00000000000..a67dc604dd2 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: input + default: azure-eventhub + - name: eventhub + - name: consumer_group + default: "$Default" + - name: connection_string + - name: storage_account + - name: storage_account_key + - name: resource_manager_endpoint + - name: tags + default: [forwarded] + +ingest_pipeline: + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log new file mode 100644 index 00000000000..13f18cfe2c2 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log @@ -0,0 +1 @@ +{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json new file mode 100644 index 00000000000..ca2c95be824 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -0,0 +1,36 @@ +[ + { + "@timestamp": "2020-11-03T09:06:42.000Z", + "azure.platformlogs.ActivityId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8", + "azure.platformlogs.Caller": "Portal", + "azure.platformlogs.Environment": "PROD", + "azure.platformlogs.EventTimeString": "11/3/2020 9:06:42 AM +00:00", + "azure.platformlogs.ScaleUnit": "PROD-AM3-AZ501", + "azure.platformlogs.category": "OperationalLogs", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.properties.Namespace": "obstesteventhubs", + "azure.platformlogs.properties.SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "azure.platformlogs.properties.TrackingId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2", + "azure.platformlogs.properties.Via": "sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04&$skip=0&$top=100", + "azure.platformlogs.status": "Succeeded", + "azure.resource.group": "OBS-TEST", + "azure.resource.id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "azure.resource.name": "OBSTESTEVENTHUBS", + "azure.resource.provider": "MICROSOFT.EVENTHUB/NAMESPACES", + "azure.subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "cloud.provider": "azure", + "cloud.region": "West Europe", + "event.action": "Retreive ConsumerGroup", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "event.outcome": "succeeded", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "service.type": "azure", + "tags": [ + "forwarded" + ] + } +] diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log new file mode 100644 index 00000000000..7b8930fb341 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log @@ -0,0 +1 @@ +{"Cloud":"AzureCloud","Environment":"prod","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json new file mode 100644 index 00000000000..fb95fe0ba80 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -0,0 +1,31 @@ +[ + { + "@timestamp": "2020-11-09T10:57:31.000Z", + "azure.platformlogs.Cloud": "AzureCloud", + "azure.platformlogs.Environment": "prod", + "azure.platformlogs.category": "kube-audit", + "azure.platformlogs.ccpNamespace": "5e4bf4baee195b00017cdbfa", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "azure.platformlogs.properties.pod": "kube-apiserver-666bd4b459-hjgdc", + "azure.platformlogs.properties.stream": "stdout", + "azure.resource.group": "OBS-INFRASTRUCTURE", + "azure.resource.id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "azure.resource.name": "OBSKUBE", + "azure.resource.provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "azure.subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "cloud.provider": "azure", + "event.action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", + "service.type": "azure", + "tags": [ + "forwarded" + ] + } +] diff --git a/x-pack/filebeat/module/gcp/_meta/config.yml b/x-pack/filebeat/module/gcp/_meta/config.yml new file mode 100644 index 00000000000..613f8b1b8d1 --- /dev/null +++ b/x-pack/filebeat/module/gcp/_meta/config.yml @@ -0,0 +1,54 @@ +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/x-pack/filebeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/filebeat/module/gcp/_meta/docs.asciidoc similarity index 87% rename from x-pack/filebeat/module/googlecloud/_meta/docs.asciidoc rename to x-pack/filebeat/module/gcp/_meta/docs.asciidoc index adda332e62f..17f989377f9 100644 --- a/x-pack/filebeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/gcp/_meta/docs.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] -:modulename: googlecloud +:modulename: gcp :has-dashboards: false == Google Cloud module @@ -24,18 +24,18 @@ include::../include/config-option-intro.asciidoc[] ==== `audit` fileset settings [role="screenshot"] -image::./images/filebeat-googlecloud-audit.png[] +image::./images/filebeat-gcp-audit.png[] Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp audit: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-audit - var.subscription_name: filebeat-googlecloud-audit-sub + var.topic: gcp-vpc-audit + var.subscription_name: filebeat-gcp-audit-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- @@ -75,12 +75,12 @@ Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp vpcflow: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-flowlogs - var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + var.topic: gcp-vpc-flowlogs + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- @@ -120,12 +120,12 @@ Example config: [source,yaml] ---- -- module: googlecloud +- module: gcp firewall: enabled: true var.project_id: my-gcp-project-id - var.topic: googlecloud-vpc-firewall - var.subscription_name: filebeat-googlecloud-vpc-firewall-sub + var.topic: gcp-vpc-firewall + var.subscription_name: filebeat-gcp-vpc-firewall-sub var.credentials_file: ${path.config}/gcp-service-account-xyz.json var.keep_original_message: false ---- diff --git a/x-pack/filebeat/module/googlecloud/_meta/fields.yml b/x-pack/filebeat/module/gcp/_meta/fields.yml similarity index 98% rename from x-pack/filebeat/module/googlecloud/_meta/fields.yml rename to x-pack/filebeat/module/gcp/_meta/fields.yml index 8f97f9b19c0..f574d666eb7 100644 --- a/x-pack/filebeat/module/googlecloud/_meta/fields.yml +++ b/x-pack/filebeat/module/gcp/_meta/fields.yml @@ -1,5 +1,5 @@ -- key: googlecloud - title: Google Cloud +- key: gcp + title: Google Cloud Platform (GCP) description: > Module for handling logs from Google Cloud. fields: diff --git a/x-pack/filebeat/module/googlecloud/_meta/kibana/7/dashboard/filebeat-googlecloud-audit.json b/x-pack/filebeat/module/gcp/_meta/kibana/7/dashboard/filebeat-gcp-audit.json similarity index 95% rename from x-pack/filebeat/module/googlecloud/_meta/kibana/7/dashboard/filebeat-googlecloud-audit.json rename to x-pack/filebeat/module/gcp/_meta/kibana/7/dashboard/filebeat-gcp-audit.json index b87e6793afb..0c6cc78c153 100644 --- a/x-pack/filebeat/module/googlecloud/_meta/kibana/7/dashboard/filebeat-googlecloud-audit.json +++ b/x-pack/filebeat/module/gcp/_meta/kibana/7/dashboard/filebeat-gcp-audit.json @@ -120,7 +120,7 @@ } ], "timeRestore": false, - "title": "[Filebeat GoogleCloud] Audit", + "title": "[Filebeat GCP] Audit", "version": 1 }, "id": "6576c480-73a2-11ea-a345-f985c61fe654", @@ -198,9 +198,9 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"866b5ce1-6ca0-47db-a6f2-54c5e0dcd2f0\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"79ec6461-7561-45e4-a6a2-9d6fbd4cf986\",\"geoField\":\"source.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"LIMIT\",\"topHitsSize\":1,\"type\":\"ES_SEARCH\",\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"279da950-e9a7-4287-ab37-25906e448455\",\"label\":\"Source Locations\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"joins\":[],\"query\":{\"query\":\"event.dataset:googlecloud.audit\",\"language\":\"kuery\"}}]", + "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"866b5ce1-6ca0-47db-a6f2-54c5e0dcd2f0\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"79ec6461-7561-45e4-a6a2-9d6fbd4cf986\",\"geoField\":\"source.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"LIMIT\",\"topHitsSize\":1,\"type\":\"ES_SEARCH\",\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"279da950-e9a7-4287-ab37-25906e448455\",\"label\":\"Source Locations\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"joins\":[],\"query\":{\"query\":\"event.dataset:gcp.audit\",\"language\":\"kuery\"}}]", "mapStateJSON": "{\"zoom\":1.97,\"center\":{\"lon\":0,\"lat\":19.94277},\"timeFilters\":{\"from\":\"now-7d\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", - "title": "Audit Source Locations [Filebeat GoogleCloud]", + "title": "Audit Source Locations [Filebeat GCP]", "uiStateJSON": { "isLayerTOCOpen": true, "openTOCDetails": [] @@ -231,7 +231,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Audit Events Outcome over time [Filebeat GoogleCloud]", + "title": "Audit Events Outcome over time [Filebeat GCP]", "uiStateJSON": {}, "version": 1, "visState": { @@ -356,7 +356,7 @@ } ] }, - "title": "Audit Event Outcome over time [Filebeat GoogleCloud]", + "title": "Audit Event Outcome over time [Filebeat GCP]", "type": "histogram" } }, @@ -388,7 +388,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Audit Event Action [Filebeat GoogleCloud]", + "title": "Audit Event Action [Filebeat GCP]", "uiStateJSON": {}, "version": 1, "visState": { @@ -430,7 +430,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Audit Event Action [Filebeat GoogleCloud]", + "title": "Audit Event Action [Filebeat GCP]", "type": "pie" } }, @@ -462,7 +462,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Audit Top User Email [Filebeat GoogleCloud]", + "title": "Audit Top User Email [Filebeat GCP]", "uiStateJSON": {}, "version": 1, "visState": { @@ -498,7 +498,7 @@ "scale": "linear", "showLabel": true }, - "title": "Audit Top User Email [Filebeat GoogleCloud]", + "title": "Audit Top User Email [Filebeat GCP]", "type": "tagcloud" } }, @@ -530,7 +530,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Audit User Agent [Filebeat GoogleCloud]", + "title": "Audit User Agent [Filebeat GCP]", "uiStateJSON": {}, "version": 1, "visState": { @@ -572,7 +572,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Audit User Agent [Filebeat GoogleCloud]", + "title": "Audit User Agent [Filebeat GCP]", "type": "pie" } }, @@ -604,7 +604,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Audit Resource Name [Filebeat GoogleCloud]", + "title": "Audit Resource Name [Filebeat GCP]", "uiStateJSON": {}, "version": 1, "visState": { @@ -620,7 +620,7 @@ "enabled": true, "id": "2", "params": { - "field": "googlecloud.audit.resource_name", + "field": "gcp.audit.resource_name", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", @@ -646,7 +646,7 @@ "legendPosition": "right", "type": "pie" }, - "title": "Audit Resource Name [Filebeat GoogleCloud]", + "title": "Audit Resource Name [Filebeat GCP]", "type": "pie" } }, @@ -670,7 +670,7 @@ "columns": [ "user.email", "service.name", - "googlecloud.audit.type", + "gcp.audit.type", "event.action", "event.outcome", "source.ip", @@ -692,13 +692,13 @@ "key": "event.dataset", "negate": false, "params": { - "query": "googlecloud.audit" + "query": "gcp.audit" }, "type": "phrase" }, "query": { "match_phrase": { - "event.dataset": "googlecloud.audit" + "event.dataset": "gcp.audit" } } } @@ -713,7 +713,7 @@ } }, "sort": [], - "title": "Audit [Filebeat GoogleCloud]", + "title": "Audit [Filebeat GCP]", "version": 1 }, "id": "d88364c0-73a1-11ea-a345-f985c61fe654", diff --git a/x-pack/filebeat/module/googlecloud/audit/_meta/fields.yml b/x-pack/filebeat/module/gcp/audit/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/audit/_meta/fields.yml rename to x-pack/filebeat/module/gcp/audit/_meta/fields.yml diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/gcp/audit/config/input.yml similarity index 87% rename from x-pack/filebeat/module/googlecloud/audit/config/input.yml rename to x-pack/filebeat/module/gcp/audit/config/input.yml index f1c71d4b84f..3b89f0f630e 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/gcp/audit/config/input.yml @@ -27,8 +27,8 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - script: lang: javascript - id: googlecloud_audit_script - file: ${path.home}/module/googlecloud/audit/config/pipeline.js + id: gcp_audit_script + file: ${path.home}/module/gcp/audit/config/pipeline.js params: keep_original_message: {{ .keep_original_message }} - add_fields: diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/gcp/audit/config/pipeline.js similarity index 100% rename from x-pack/filebeat/module/googlecloud/audit/config/pipeline.js rename to x-pack/filebeat/module/gcp/audit/config/pipeline.js diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml rename to x-pack/filebeat/module/gcp/audit/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/gcp/audit/manifest.yml similarity index 92% rename from x-pack/filebeat/module/googlecloud/audit/manifest.yml rename to x-pack/filebeat/module/gcp/audit/manifest.yml index 42b5c4880d6..ebe77788fe3 100644 --- a/x-pack/filebeat/module/googlecloud/audit/manifest.yml +++ b/x-pack/filebeat/module/gcp/audit/manifest.yml @@ -8,7 +8,7 @@ var: - name: topic default: stackdriver-audit - name: subscription_name - default: filebeat-googlecloud-audit + default: filebeat-gcp-audit - name: credentials_file - name: credentials_json - name: keep_original_message diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log similarity index 100% rename from x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log rename to x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json similarity index 95% rename from x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json rename to x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json index d8efe2892a5..8b4b2ed642d 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/gcp/audit/test/audit-log-entries.json.log-expected.json @@ -3,10 +3,10 @@ "@timestamp": "2019-12-19T00:49:36.086Z", "cloud.project.id": "elastic-beats", "event.action": "GetResourceBillingInfo", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "-uihnmjctwo", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", @@ -29,7 +29,7 @@ "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 0, "service.name": "cloudbilling.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.ip": "192.168.1.1", "tags": [ "forwarded" @@ -40,10 +40,10 @@ "@timestamp": "2019-12-19T00:45:51.228Z", "cloud.project.id": "elastic-beats", "event.action": "beta.compute.machineTypes.aggregatedList", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "-h6onuze1h7dg", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "failure", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", @@ -73,7 +73,7 @@ "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 945, "service.name": "compute.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.ip": "192.168.1.1", "tags": [ "forwarded" @@ -91,10 +91,10 @@ "@timestamp": "2019-12-19T00:44:25.051Z", "cloud.project.id": "elastic-beats", "event.action": "beta.compute.instances.aggregatedList", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "yonau2dg2zi", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", @@ -130,7 +130,7 @@ "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 2252, "service.name": "compute.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.ip": "192.168.1.1", "tags": [ "forwarded" @@ -148,10 +148,10 @@ "@timestamp": "2019-12-19T00:44:25.051Z", "cloud.project.id": "elastic-beats", "event.action": "beta.compute.instances.aggregatedList", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "yonau3dc2zi", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "failure", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "xxx@xxx.xxx", @@ -182,7 +182,7 @@ "log.logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 3776, "service.name": "compute.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.ip": "192.168.1.1", "tags": [ "forwarded" @@ -200,10 +200,10 @@ "@timestamp": "2020-08-05T21:07:30.974Z", "cloud.project.id": "elastic-siem", "event.action": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "87efd529-6349-45d2-b905-fc607e6c5d3b", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "system:serviceaccount:cert-manager:cert-manager-webhook", @@ -228,7 +228,7 @@ "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access", "log.offset": 5100, "service.name": "k8s.io", - "service.type": "googlecloud", + "service.type": "gcp", "source.ip": "10.11.12.13", "tags": [ "forwarded" @@ -243,10 +243,10 @@ "@timestamp": "2020-08-05T21:59:26.456Z", "cloud.project.id": "foo", "event.action": "v1.compute.images.insert", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "v2spcwdzmc2", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "success", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "user@mycompany.com", @@ -278,7 +278,7 @@ "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity", "log.offset": 7530, "service.name": "compute.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.geo.city_name": "Moscow", "source.geo.continent_name": "Europe", "source.geo.country_iso_code": "RU", @@ -304,10 +304,10 @@ "cloud.instance.id": "590261181", "cloud.project.id": "foo", "event.action": "beta.compute.instances.stop", - "event.dataset": "googlecloud.audit", + "event.dataset": "gcp.audit", "event.id": "-c7ctxmd2zab", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.outcome": "unknown", "fileset.name": "audit", "googlecloud.audit.authentication_info.principal_email": "user@mycompany.com", @@ -322,7 +322,7 @@ "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity", "log.offset": 9946, "service.name": "compute.googleapis.com", - "service.type": "googlecloud", + "service.type": "gcp", "source.as.number": 3215, "source.as.organization.name": "Orange", "source.geo.city_name": "Clermont-Ferrand", diff --git a/x-pack/filebeat/module/gcp/fields.go b/x-pack/filebeat/module/gcp/fields.go new file mode 100644 index 00000000000..0e5675483bb --- /dev/null +++ b/x-pack/filebeat/module/gcp/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package gcp + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "gcp", asset.ModuleFieldsPri, AssetGcp); err != nil { + panic(err) + } +} + +// AssetGcp returns asset data. +// This is the base64 encoded gzipped contents of module/gcp. +func AssetGcp() string { + return "eJzsWktv48gRvvtX1M27gIeLXH0IYMjjjZFx4LWVCZCL0GqWyI6b3Uw/pGh+fdAvik9ZtuhJBhidbD6++lhVXfV1kZ/gBffXUND6AsAww/Eafpey4AgLLm0Oj5yYjVQV/PL74vHXC4AcNVWsNkyKa/jzBQDAg8wtR9hIBSUROWeiAC4LDRslqw5cdgGwYchzfe3v/ASCVHgNhb+Gukv8cQCzr91xJW0dj4wYdr87Dzc05Rlk8bK2zbbdHLVhgjjMjAltiKDYXDRG4ggR97vfgCmxDQsyHKJSCKT+yI5oIPD1AbikxGAOUvhLNKkQvj4urjqQpmQ68AemoZa15f6mHTOlA0m0IUdDGNcZ3Asg8FwShbmD66BRKTassMpzu4JayX8hNSuWA5VKoa6lyDUY6QnFs2BKYkDuhHZHO3DJ+BVYbQnn+/AgqLaMNvdnrVv6gWgH40CmczqF4QX3O6n6544EwwfkNgUgPQyVwhAmXI66w18fsotRNgoLJsV8TJ48XmIzafabFDif0X9KgaMmxxbAtqY/VO4/LkCg2Un1Mnvuu3wP3EupzY+dyNuartxf83FxnnexLBkto20XH1mj860oJohou47xmpnPcwN8Kq2GkrSK4pylPyC+J/N9tncgf1b9n1X/Q6p+TPt5Cv53yfiftf5nrfe/k2t9nxGxOTPnpHvaaEjV3Wd44M5uA0ZzJhFxhi9O88ZRXyz3tU+QGpXZZyOGiDUlCsOoXwQrJjZyxG7fA69YvemAggNVVdCPMFzKk4uGCcpqwldYEcbny45lieAhgeS5Qq3TQmr5AnOwGhVU5CWtJ4X/tqhN7wnafpSKmf1KI0dqpJqXcIMPCR90jZRtGOaw3rcZSnUFbANE7DO4Ny7jhTRQWKKIMIg5DAz4+hZKSfR5qMucyx3mrgJajUFoNzw6fuh54dvRZCJKkf3bkqnBbOeSX2WOdVzQUmSnJxeqimk9axtfxhAw12vubx5aRiaSpvARGW8Kayk5kj69Vyj8o0RTogKpfMw74fDuUhg7MRF5i58Pd2QzwTXduSLGKLa2BvUo72GpOC2/G9S0GpPBrHf9WFhbPSUIzGGKH4nrqwQDRWcgkYtm+twSi5F2NjuFKf9MtpCZODiIYxyS/QpNKfNhZ39vJxuPQDTjMv5QBuBOKrh5vAdKONdBQvarni6l5Tms0aO1kd2NATXr3+Rw8T+kqjlewWUYSGY5McRVXcy2f8pum3+erPjDotpfjjlH2GoVFKbGFTNY6REfcSmKNzrIVmu3/DfgMUGhsUpgHiafBL4wbZyrPLHWg4aGUdecUbLmo+GMzeV8cbBsV+ymq75BFkgjZ9aLHZV0yO0RZonFhnGDM7b4O493kul5H/1vnYpyTOE0DeDDCRzKypjGiBetKjTELb3zM/IhIgFZS2umI3AsLV2lQbVi/dYXCA0On9AR7x/7+jTYmAhPJKCtW8WYr5x8XZEChZlX4XhZ7HH7tJauyrb1WVSeXVnthMcAuFWMFYbLKJUqZ6Lgo9uWVDvnL0cB9/+yHo1RO8zo/cBjTkkWIY/bf02PjdmeXQp5I1M66IWJoYkZCDjcvm+audP/VheexMUOplGzULHsRK80it0QY8cT9z2L59nDDZJ2vJNMdbQzNGqzw5IKqOQ8zjn9Fpv5MTkYogpsymdLtk5JXrfvBk1LrPATdw3h70/3V762MkG5zdOIwgm6pIvdja/oV10i36L+7fkvn7/cre5vf1tL+aJH9WrjKj+m7W+b3114E9rxPc3RvmuVQmEaXvMlEiwCdEPy6M60t//8sG1PO8Q1KtdsU/SnE6kT9sN2JexfSM10RmU1+jTDtfnuWOvOwpRbVITzSdJHYy7z8WbLhMFiIMtPaHWRmwO+ivPigyQhAlDYCraEWx+HuO9TNc0WMu+vssP+V2tSzKgKbiDHLXLnsU8bQl3YUSmpkqUhcybgsyg402UGN2LvtVu6dQDfwWqBuPTn7FuUbtqtCBbe0bT8EKpuliTDANyH88rVxANcnFpSzlC0Nx2HPZbCHeHtye9s8/i7iP2GkbyyHFdjMutdK+L2cDItifS83lJoGBUxtMQ8DDAOb81eL4+dMbqf1070+sGMYcB8GebCYficRq4drlMKg9BBp4C3iYybVvP0ldd5JpY9DVIEB03Zz5nCsyncJhAfJkU2G0bbwdEhOMf8oHCDCsV5U8mnBJJewZ8Ugti1FRGDSvQm62FwtEnvcT2ePsSlm7p+kOROy8nIHL7/mY9b+6OiSYKvc4suM2S4Ls6Ux4mo49F6KW5IcYYrg5j8HnSjbD2PLqtXtVRm+G4IjrwfehNdVoc9OZVc+5Z1mGuCM51yw78OsRwnN0gxEZKiI5RKO5imfFRSJI0XrZ6fIN/5MWKynPMYre8VNlzuPkIGfH1cgMN+iwxAl0Q9kXmGuNcsxyjcInQe/MPlLoMFEU6DIfOv9S6fnxaXTkRd3n5+XrY2amM8jclmeKvwhRgUdA9EQ4VEW4U5/OL8uFw8eo6uDfP9r5BblTYihrk9qzCotoSnueBgY5YuRE5q7eQgmh2icArTb2gJPH/+wy9ghRTZNhw7fJnj/r9Z/LUH665nzbcwYb+dvgV5Wi7dc+zQ9YFwKtaGOPsLHxPlyMk+u/hvAAAA///1JdgD" +} diff --git a/x-pack/filebeat/module/googlecloud/firewall/_meta/fields.yml b/x-pack/filebeat/module/gcp/firewall/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/firewall/_meta/fields.yml rename to x-pack/filebeat/module/gcp/firewall/_meta/fields.yml diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/gcp/firewall/config/input.yml similarity index 87% rename from x-pack/filebeat/module/googlecloud/firewall/config/input.yml rename to x-pack/filebeat/module/gcp/firewall/config/input.yml index 1ddda931c49..e2999de6ade 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/gcp/firewall/config/input.yml @@ -27,11 +27,11 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - script: lang: javascript - id: googlecloud_firewall_script + id: gcp_firewall_script params: debug: {{ .debug }} keep_original_message: {{ .keep_original_message }} - file: ${path.home}/module/googlecloud/firewall/config/pipeline.js + file: ${path.home}/module/gcp/firewall/config/pipeline.js - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/pipeline.js b/x-pack/filebeat/module/gcp/firewall/config/pipeline.js similarity index 100% rename from x-pack/filebeat/module/googlecloud/firewall/config/pipeline.js rename to x-pack/filebeat/module/gcp/firewall/config/pipeline.js diff --git a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/gcp/firewall/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml rename to x-pack/filebeat/module/gcp/firewall/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/gcp/firewall/manifest.yml similarity index 92% rename from x-pack/filebeat/module/googlecloud/firewall/manifest.yml rename to x-pack/filebeat/module/gcp/firewall/manifest.yml index 009ace59c23..9f2b2840df3 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml +++ b/x-pack/filebeat/module/gcp/firewall/manifest.yml @@ -8,7 +8,7 @@ var: - name: topic default: stackdriver-firewall - name: subscription_name - default: filebeat-googlecloud-firewall + default: filebeat-gcp-firewall - name: credentials_file - name: credentials_json - name: debug diff --git a/x-pack/filebeat/module/googlecloud/firewall/test/rare.log b/x-pack/filebeat/module/gcp/firewall/test/rare.log similarity index 100% rename from x-pack/filebeat/module/googlecloud/firewall/test/rare.log rename to x-pack/filebeat/module/gcp/firewall/test/rare.log diff --git a/x-pack/filebeat/module/googlecloud/firewall/test/rare.log-expected.json b/x-pack/filebeat/module/gcp/firewall/test/rare.log-expected.json similarity index 95% rename from x-pack/filebeat/module/googlecloud/firewall/test/rare.log-expected.json rename to x-pack/filebeat/module/gcp/firewall/test/rare.log-expected.json index fb34db02422..1d799e8edbc 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/test/rare.log-expected.json +++ b/x-pack/filebeat/module/gcp/firewall/test/rare.log-expected.json @@ -7,10 +7,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1dobeotg13df9f5", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -57,7 +57,7 @@ "10.128.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.142.0.10", "source.domain": "test-es", "source.ip": "10.142.0.10", @@ -74,10 +74,10 @@ "destination.port": 57794, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1dobeotg13df9f7", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -124,7 +124,7 @@ "10.128.0.10" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.142.0.16", "source.domain": "local-adrian-test", "source.ip": "10.142.0.16", diff --git a/x-pack/filebeat/module/googlecloud/firewall/test/test.log b/x-pack/filebeat/module/gcp/firewall/test/test.log similarity index 100% rename from x-pack/filebeat/module/googlecloud/firewall/test/test.log rename to x-pack/filebeat/module/gcp/firewall/test/test.log diff --git a/x-pack/filebeat/module/googlecloud/firewall/test/test.log-expected.json b/x-pack/filebeat/module/gcp/firewall/test/test.log-expected.json similarity index 95% rename from x-pack/filebeat/module/googlecloud/firewall/test/test.log-expected.json rename to x-pack/filebeat/module/gcp/firewall/test/test.log-expected.json index 73f9e79c29a..908b2436bd9 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/test/test.log-expected.json +++ b/x-pack/filebeat/module/gcp/firewall/test/test.log-expected.json @@ -13,10 +13,10 @@ "destination.port": 53, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "4zuj4nfn4llkb", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -56,7 +56,7 @@ "8.8.8.8" ], "rule.name": "network:default/firewall:adrian-test-1", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.128.0.16", "source.domain": "adrian-test", "source.ip": "10.128.0.16", @@ -73,10 +73,10 @@ "destination.port": 3389, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1f21ciqfpfssuo", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -119,7 +119,7 @@ "10.42.0.2" ], "rule.name": "network:windows-isolated/firewall:windows-isolated-allow-rdp", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.126", "source.geo.continent_name": "Asia", "source.geo.country_name": "omn", @@ -137,10 +137,10 @@ "destination.port": 8080, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "8vcfeailjd", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -184,7 +184,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.219", "source.geo.city_name": "Krasnodar", "source.geo.continent_name": "Europe", @@ -204,10 +204,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1bqgmw9feiabij", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -251,7 +251,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.14", "source.geo.continent_name": "Europe", "source.geo.country_name": "deu", @@ -269,10 +269,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1jrxaqbfe48bir", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -316,7 +316,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.14", "source.geo.continent_name": "Europe", "source.geo.country_name": "deu", @@ -334,10 +334,10 @@ "destination.port": 8080, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1fw7drlfe2ty27", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -381,7 +381,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.151", "source.geo.city_name": "Berdychiv", "source.geo.continent_name": "Europe", @@ -401,10 +401,10 @@ "destination.port": 8080, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1yre751fekaxzs", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -448,7 +448,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.241", "source.geo.city_name": "Vicenza", "source.geo.continent_name": "Europe", @@ -468,10 +468,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "5kanfzfiqepkh", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -515,7 +515,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.114", "source.geo.city_name": "Tula", "source.geo.continent_name": "Europe", @@ -535,10 +535,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "59z0t8fiow9vg", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -582,7 +582,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.251", "source.geo.city_name": "Stavropol", "source.geo.continent_name": "Europe", @@ -602,10 +602,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1y7e4yzff816cq", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -649,7 +649,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.189", "source.geo.city_name": "Viol\u00e8s", "source.geo.continent_name": "Europe", @@ -669,10 +669,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "lx5jlsfggpr0q", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -716,7 +716,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.189", "source.geo.city_name": "Viol\u00e8s", "source.geo.continent_name": "Europe", @@ -736,10 +736,10 @@ "destination.port": 8080, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "18ynfbufer19m1", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -783,7 +783,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.200", "source.geo.city_name": "\u0130zmir", "source.geo.continent_name": "Asia", @@ -809,10 +809,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "tzddthfsr6fv5", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -852,7 +852,7 @@ "8.8.8.8" ], "rule.name": "network:default/firewall:adrian-test-1", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.28.0.16", "source.domain": "adrian-test", "source.ip": "10.28.0.16", @@ -875,10 +875,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1k2b7kefsnhzq7", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -918,7 +918,7 @@ "8.8.8.8" ], "rule.name": "network:default/firewall:adrian-test-1", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.28.0.16", "source.domain": "adrian-test", "source.ip": "10.28.0.16", @@ -935,10 +935,10 @@ "destination.port": 9200, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1sdfuwxfk8hq1c", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -987,7 +987,7 @@ "10.42.0.10" ], "rule.name": "network:default/firewall:allow9200", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.114", "source.domain": "test-kibana", "source.geo.continent_name": "America", @@ -1006,10 +1006,10 @@ "destination.port": 9200, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1sdfuwxfk8hq1b", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -1058,7 +1058,7 @@ "10.42.0.10" ], "rule.name": "network:default/firewall:allow9200", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.114", "source.domain": "test-kibana", "source.geo.continent_name": "America", @@ -1077,10 +1077,10 @@ "destination.port": 3389, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "yot1ojetjdiw", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -1123,7 +1123,7 @@ "10.42.0.2" ], "rule.name": "network:windows-isolated/firewall:windows-isolated-allow-rdp", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.7", "source.geo.city_name": "Almelo", "source.geo.continent_name": "Europe", @@ -1143,10 +1143,10 @@ "destination.port": 9200, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "5a27u1g22jks9e", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -1195,7 +1195,7 @@ "10.42.0.10" ], "rule.name": "network:default/firewall:allow9200", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.114", "source.domain": "test-kibana", "source.geo.continent_name": "America", @@ -1214,10 +1214,10 @@ "destination.port": 9200, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "5a27u1g22jks8t", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "allowed" @@ -1266,7 +1266,7 @@ "10.42.0.10" ], "rule.name": "network:default/firewall:allow9200", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.114", "source.domain": "test-kibana", "source.geo.continent_name": "America", @@ -1285,10 +1285,10 @@ "destination.port": 80, "event.action": "firewall-rule", "event.category": "network", - "event.dataset": "googlecloud.firewall", + "event.dataset": "gcp.firewall", "event.id": "1dobeotg13df9f5", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.type": [ "connection", "denied" @@ -1338,7 +1338,7 @@ "10.28.0.16" ], "rule.name": "network:default/firewall:adrian-test-3", - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.42.0.10", "source.domain": "test-es", "source.ip": "10.42.0.10", diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/_meta/fields.yml b/x-pack/filebeat/module/gcp/vpcflow/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/vpcflow/_meta/fields.yml rename to x-pack/filebeat/module/gcp/vpcflow/_meta/fields.yml diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/gcp/vpcflow/config/input.yml similarity index 87% rename from x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml rename to x-pack/filebeat/module/gcp/vpcflow/config/input.yml index 2854b8ed332..499e13b3dc7 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/gcp/vpcflow/config/input.yml @@ -27,8 +27,8 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - script: lang: javascript - id: googlecloud_vpcflow_script - file: ${path.home}/module/googlecloud/vpcflow/config/pipeline.js + id: gcp_vpcflow_script + file: ${path.home}/module/gcp/vpcflow/config/pipeline.js params: keep_original_message: {{ .keep_original_message }} - add_fields: diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/pipeline.js b/x-pack/filebeat/module/gcp/vpcflow/config/pipeline.js similarity index 100% rename from x-pack/filebeat/module/googlecloud/vpcflow/config/pipeline.js rename to x-pack/filebeat/module/gcp/vpcflow/config/pipeline.js diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/gcp/vpcflow/ingest/pipeline.yml similarity index 100% rename from x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml rename to x-pack/filebeat/module/gcp/vpcflow/ingest/pipeline.yml diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml b/x-pack/filebeat/module/gcp/vpcflow/manifest.yml similarity index 91% rename from x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml rename to x-pack/filebeat/module/gcp/vpcflow/manifest.yml index 3ddb0800223..71048699be9 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/gcp/vpcflow/manifest.yml @@ -8,7 +8,7 @@ var: - name: topic default: stackdriver-vpcflow - name: subscription_name - default: filebeat-googlecloud-vpcflow + default: filebeat-gcp-vpcflow - name: credentials_file - name: credentials_json - name: keep_original_message diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/test/vpc-flow-log-entries.json.log b/x-pack/filebeat/module/gcp/vpcflow/test/vpc-flow-log-entries.json.log similarity index 100% rename from x-pack/filebeat/module/googlecloud/vpcflow/test/vpc-flow-log-entries.json.log rename to x-pack/filebeat/module/gcp/vpcflow/test/vpc-flow-log-entries.json.log diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/test/vpc-flow-log-entries.json.log-expected.json b/x-pack/filebeat/module/gcp/vpcflow/test/vpc-flow-log-entries.json.log-expected.json similarity index 94% rename from x-pack/filebeat/module/googlecloud/vpcflow/test/vpc-flow-log-entries.json.log-expected.json rename to x-pack/filebeat/module/gcp/vpcflow/test/vpc-flow-log-entries.json.log-expected.json index 9a71b1c35a6..b9d0250b9be 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/test/vpc-flow-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/gcp/vpcflow/test/vpc-flow-log-entries.json.log-expected.json @@ -11,11 +11,11 @@ "destination.ip": "203.0.113.12", "destination.port": 33478, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:45:37.301953198Z", "event.id": "ut8lbrffooxyw", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:45:37.186193305Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -42,7 +42,7 @@ "10.87.40.76", "203.0.113.12" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1776, "source.domain": "kibana", @@ -63,11 +63,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33970, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821302149Z", "event.id": "ut8lbrffooxzb", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.466657665Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -100,7 +100,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 173663, @@ -127,11 +127,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33576, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821143836Z", "event.id": "ut8lbrffooxze", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:20.510622432Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -164,7 +164,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 155707, "source.domain": "elasticsearch", @@ -189,11 +189,11 @@ "destination.ip": "192.0.2.23", "destination.port": 59679, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:46.031032701Z", "event.id": "ut8lbrffooxyz", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:45.860349247Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -219,7 +219,7 @@ "10.139.99.242", "192.0.2.23" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 0, "source.domain": "elasticsearch", @@ -242,11 +242,11 @@ "destination.ip": "192.0.2.117", "destination.port": 50646, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:37.048196137Z", "event.id": "ut8lbrffooxz6", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:36.895188084Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -273,7 +273,7 @@ "10.87.40.76", "192.0.2.117" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1784, "source.domain": "kibana", @@ -294,11 +294,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:37.048196137Z", "event.id": "ut8lbrffooxzf", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:36.895188084Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -325,7 +325,7 @@ "192.0.2.117", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.117", "source.as.number": 15169, "source.bytes": 1464, @@ -348,11 +348,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33692, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565287007Z", "event.id": "ut8lbrffooxz1", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500498059Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -385,7 +385,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 186151, @@ -412,11 +412,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821308944Z", "event.id": "ut8lbrffooxyp", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.469099728Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -449,7 +449,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 15169, "source.domain": "kibana", @@ -470,11 +470,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33554, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565311154Z", "event.id": "ut8lbrffooxzd", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500506974Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -507,7 +507,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 250864, @@ -531,11 +531,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33880, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821308944Z", "event.id": "ut8lbrffooxz8", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.469099728Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -568,7 +568,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 167939, @@ -592,11 +592,11 @@ "destination.ip": "10.139.99.242", "destination.port": 22, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:46.031032701Z", "event.id": "ut8lbrffooxyt", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:45.860349247Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -622,7 +622,7 @@ "192.0.2.23", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.23", "source.as.number": 49505, "source.bytes": 0, @@ -647,11 +647,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821056075Z", "event.id": "ut8lbrffooxz5", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:20.510622432Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -684,7 +684,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 11773, @@ -708,11 +708,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.393910944Z", "event.id": "ut8lbrffooxza", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:01.074897435Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -745,7 +745,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 65699, @@ -772,11 +772,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565287007Z", "event.id": "ut8lbrffooxyq", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500498059Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -809,7 +809,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 66029, "source.domain": "kibana", @@ -833,11 +833,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565272745Z", "event.id": "ut8lbrffooxz2", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.150720950Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -870,7 +870,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 65154, "source.domain": "kibana", @@ -894,11 +894,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821302149Z", "event.id": "ut8lbrffooxyo", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.466657665Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -931,7 +931,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 13643, "source.domain": "kibana", @@ -952,11 +952,11 @@ "destination.ip": "10.49.136.133", "destination.port": 46864, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:29.432367659Z", "event.id": "ut8lbrffooxzc", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:17.343890802Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -983,7 +983,7 @@ "203.0.113.93", "10.49.136.133" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.93", "source.bytes": 34509840, "source.ip": "203.0.113.93", @@ -1003,11 +1003,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:48:39.076420731Z", "event.id": "ut8lbrffooxz7", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:48:38.961050187Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1034,7 +1034,7 @@ "203.0.113.12", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.12", "source.as.number": 15169, "source.bytes": 1467, @@ -1060,11 +1060,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565311154Z", "event.id": "ut8lbrffooxyu", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500506974Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1097,7 +1097,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 63671, "source.domain": "kibana", @@ -1122,11 +1122,11 @@ "destination.ip": "203.0.113.58", "destination.port": 65320, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220714119Z", "event.id": "ut8lbrffooxyv", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.560917237Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1153,7 +1153,7 @@ "10.139.99.242", "203.0.113.58" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 51075, "source.domain": "elasticsearch", @@ -1177,11 +1177,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33562, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.393910944Z", "event.id": "ut8lbrffooxz0", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:01.074897435Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1214,7 +1214,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 197840, "source.domain": "elasticsearch", @@ -1234,11 +1234,11 @@ "destination.ip": "203.0.113.93", "destination.port": 9243, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:58.716492806Z", "event.id": "ut8lbrffooxys", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:17.306085222Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1265,7 +1265,7 @@ "10.49.136.133", "203.0.113.93" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.49.136.133", "source.bytes": 173805495, "source.domain": "simianhacker-demo", @@ -1286,11 +1286,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:45:37.301953198Z", "event.id": "ut8lbrffooxyx", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:45:37.186193305Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1317,7 +1317,7 @@ "203.0.113.12", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.12", "source.as.number": 15169, "source.bytes": 1468, @@ -1343,11 +1343,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33548, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.393651211Z", "event.id": "ut8lbrffooxz4", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147252064Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1380,7 +1380,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 159704, "source.domain": "elasticsearch", @@ -1401,11 +1401,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220714119Z", "event.id": "ut8lbrffooxz3", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.560917237Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1432,7 +1432,7 @@ "203.0.113.58", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.58", "source.as.number": 33652, "source.bytes": 70775, @@ -1457,11 +1457,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33542, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565272745Z", "event.id": "ut8lbrffooxz9", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.150720950Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1494,7 +1494,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 281147, @@ -1518,11 +1518,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:48.537763242Z", "event.id": "ut8lbrffooxyr", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147252064Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1555,7 +1555,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 63590, @@ -1581,11 +1581,11 @@ "destination.ip": "203.0.113.12", "destination.port": 34836, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:48:39.076420731Z", "event.id": "ut8lbrffooxyy", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:48:38.961050187Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1612,7 +1612,7 @@ "10.87.40.76", "203.0.113.12" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1780, "source.domain": "kibana", @@ -1633,11 +1633,11 @@ "destination.ip": "10.139.99.242", "destination.port": 22, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:52.361155668Z", "event.id": "1ulp77rfdvho4g", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:46.541094678Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1664,7 +1664,7 @@ "192.0.2.165", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.165", "source.as.number": 45899, "source.bytes": 1239, @@ -1692,11 +1692,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.213244028Z", "event.id": "1ulp77rfdvho5r", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075811571Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1729,7 +1729,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 63853, "source.domain": "kibana", @@ -1750,11 +1750,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:20.745658276Z", "event.id": "1ulp77rfdvho5k", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:20.634435179Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1781,7 +1781,7 @@ "198.51.100.107", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.107", "source.as.number": 15169, "source.bytes": 1458, @@ -1807,11 +1807,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33534, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.597088427Z", "event.id": "1ulp77rfdvho55", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075942176Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1844,7 +1844,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 252397, "source.domain": "elasticsearch", @@ -1868,11 +1868,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33694, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565117754Z", "event.id": "1ulp77rfdvho60", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.566551903Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1905,7 +1905,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 205787, "source.domain": "elasticsearch", @@ -1930,11 +1930,11 @@ "destination.ip": "203.0.113.58", "destination.port": 65263, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220748025Z", "event.id": "1ulp77rfdvho49", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:01.270990648Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -1961,7 +1961,7 @@ "10.139.99.242", "203.0.113.58" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 106409, "source.domain": "elasticsearch", @@ -1982,11 +1982,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.597088427Z", "event.id": "1ulp77rfdvho4t", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075942176Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2019,7 +2019,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 61242, @@ -2046,11 +2046,11 @@ "destination.ip": "203.0.113.101", "destination.port": 49680, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.705469925Z", "event.id": "1ulp77rfdvho68", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.711043814Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2083,7 +2083,7 @@ "10.139.99.242", "203.0.113.101" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 248826, "source.domain": "elasticsearch", @@ -2106,11 +2106,11 @@ "destination.ip": "192.0.2.117", "destination.port": 33862, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:11.779780615Z", "event.id": "1ulp77rfdvho5n", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:11.655143526Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2137,7 +2137,7 @@ "10.87.40.76", "192.0.2.117" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1777, "source.domain": "kibana", @@ -2162,11 +2162,11 @@ "destination.ip": "203.0.113.58", "destination.port": 65321, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.312105537Z", "event.id": "1ulp77rfdvho5l", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.843986502Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2193,7 +2193,7 @@ "10.139.99.242", "203.0.113.58" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 116845, "source.domain": "elasticsearch", @@ -2214,11 +2214,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.461087350Z", "event.id": "1ulp77rfdvho65", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:24.790136141Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2251,7 +2251,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 4614, @@ -2278,11 +2278,11 @@ "destination.ip": "192.0.2.177", "destination.port": 60112, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:18.224268993Z", "event.id": "1ulp77rfdvho4b", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:14.031541248Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2315,7 +2315,7 @@ "10.139.99.242", "192.0.2.177" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 50379, "source.domain": "elasticsearch", @@ -2336,11 +2336,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33552, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.213244028Z", "event.id": "1ulp77rfdvho4m", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075811571Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2373,7 +2373,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 200417, @@ -2400,11 +2400,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33524, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.461087350Z", "event.id": "1ulp77rfdvho5t", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:24.790136141Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2437,7 +2437,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 30233, "source.domain": "elasticsearch", @@ -2458,11 +2458,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33548, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565451051Z", "event.id": "1ulp77rfdvho50", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147072949Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2495,7 +2495,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 160693, @@ -2519,11 +2519,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565117754Z", "event.id": "1ulp77rfdvho63", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.566551903Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2556,7 +2556,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 59903, @@ -2582,11 +2582,11 @@ "destination.ip": "198.51.100.107", "destination.port": 33924, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:20.745658276Z", "event.id": "1ulp77rfdvho4r", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:20.634545217Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2613,7 +2613,7 @@ "10.87.40.76", "198.51.100.107" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1780, "source.domain": "kibana", @@ -2638,11 +2638,11 @@ "destination.ip": "203.0.113.58", "destination.port": 65271, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.318940798Z", "event.id": "1ulp77rfdvho4i", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.155378070Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2669,7 +2669,7 @@ "10.139.99.242", "203.0.113.58" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 129335, "source.domain": "elasticsearch", @@ -2690,11 +2690,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:11.779780615Z", "event.id": "1ulp77rfdvho5v", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:11.655143526Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2721,7 +2721,7 @@ "192.0.2.117", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.117", "source.as.number": 15169, "source.bytes": 1464, @@ -2744,11 +2744,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.312105537Z", "event.id": "1ulp77rfdvho5i", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.843986502Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2775,7 +2775,7 @@ "203.0.113.58", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.58", "source.as.number": 33652, "source.bytes": 75477, @@ -2804,11 +2804,11 @@ "destination.ip": "203.0.113.58", "destination.port": 65316, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220838853Z", "event.id": "1ulp77rfdvho5c", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.565831992Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2835,7 +2835,7 @@ "10.139.99.242", "203.0.113.58" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 102119, "source.domain": "elasticsearch", @@ -2856,11 +2856,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.705469925Z", "event.id": "1ulp77rfdvho5p", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.711043814Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2893,7 +2893,7 @@ "203.0.113.101", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.101", "source.as.number": 15169, "source.bytes": 1541638, @@ -2917,11 +2917,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:18.224268993Z", "event.id": "1ulp77rfdvho4y", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:14.031541248Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -2954,7 +2954,7 @@ "192.0.2.177", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.177", "source.as.number": 15169, "source.bytes": 755901, @@ -2981,11 +2981,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33558, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.394676451Z", "event.id": "1ulp77rfdvho4o", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:58.492572765Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3018,7 +3018,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 248715, "source.domain": "elasticsearch", @@ -3039,11 +3039,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220838853Z", "event.id": "1ulp77rfdvho5g", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.565831992Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3070,7 +3070,7 @@ "203.0.113.58", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.58", "source.as.number": 33652, "source.bytes": 69757, @@ -3095,11 +3095,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:56.220748025Z", "event.id": "1ulp77rfdvho59", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:01.270990648Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3126,7 +3126,7 @@ "203.0.113.58", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.58", "source.as.number": 33652, "source.bytes": 69440, @@ -3151,11 +3151,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:20.569744903Z", "event.id": "1ulp77rfdvho57", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:20.454046087Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3182,7 +3182,7 @@ "192.0.2.117", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.117", "source.as.number": 15169, "source.bytes": 1457, @@ -3207,11 +3207,11 @@ "destination.ip": "192.0.2.117", "destination.port": 50438, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:20.569744903Z", "event.id": "1ulp77rfdvho5e", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:20.454046087Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3238,7 +3238,7 @@ "10.87.40.76", "192.0.2.117" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1784, "source.domain": "kibana", @@ -3263,11 +3263,11 @@ "destination.ip": "192.0.2.165", "destination.port": 59623, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:40:52.361155668Z", "event.id": "1ulp77rfdvho4d", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:46.541094678Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3294,7 +3294,7 @@ "10.139.99.242", "192.0.2.165" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 2395, "source.domain": "elasticsearch", @@ -3315,11 +3315,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:48.538257098Z", "event.id": "1ulp77rfdvho5y", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:58.492572765Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3352,7 +3352,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 60335, @@ -3379,11 +3379,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565451051Z", "event.id": "1ulp77rfdvho6a", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147072949Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3416,7 +3416,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 65565, "source.domain": "kibana", @@ -3437,11 +3437,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:55.318940798Z", "event.id": "1ulp77rfdvho4v", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.155378070Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3468,7 +3468,7 @@ "203.0.113.58", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.58", "source.as.number": 33652, "source.bytes": 70174, @@ -3493,11 +3493,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:51.355687385Z", "event.id": "bnj3cofh3cdk1", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:51.237256499Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3524,7 +3524,7 @@ "203.0.113.12", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.12", "source.as.number": 15169, "source.bytes": 1461, @@ -3547,11 +3547,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:45:51.090104692Z", "event.id": "bnj3cofh3cdjx", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:45:50.954948790Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3578,7 +3578,7 @@ "198.51.100.107", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.107", "source.as.number": 15169, "source.bytes": 1460, @@ -3601,11 +3601,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565131125Z", "event.id": "bnj3cofh3cdju", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:02.143837873Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3638,7 +3638,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 66736, @@ -3664,11 +3664,11 @@ "destination.ip": "198.51.100.107", "destination.port": 33602, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:45:51.090104692Z", "event.id": "bnj3cofh3cdjz", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:45:50.954948790Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3695,7 +3695,7 @@ "10.87.40.76", "198.51.100.107" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1776, "source.domain": "kibana", @@ -3716,11 +3716,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:42:40.888804332Z", "event.id": "bnj3cofh3cdkk", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:42:40.779893091Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3747,7 +3747,7 @@ "203.0.113.27", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.27", "source.as.number": 15169, "source.bytes": 1464, @@ -3770,11 +3770,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33534, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.597279654Z", "event.id": "bnj3cofh3cdk0", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075756033Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3807,7 +3807,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 259510, @@ -3833,11 +3833,11 @@ "destination.ip": "203.0.113.27", "destination.port": 52260, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:42:11.183868408Z", "event.id": "bnj3cofh3cdk8", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:42:11.063146265Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3864,7 +3864,7 @@ "10.87.40.76", "203.0.113.27" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1781, "source.domain": "kibana", @@ -3888,11 +3888,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565300944Z", "event.id": "bnj3cofh3cdkp", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.140119099Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3925,7 +3925,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 65069, "source.domain": "kibana", @@ -3949,11 +3949,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565335113Z", "event.id": "bnj3cofh3cdkc", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500498059Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -3986,7 +3986,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 60530, "source.domain": "kibana", @@ -4007,11 +4007,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821047175Z", "event.id": "bnj3cofh3cdkm", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.469473010Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4044,7 +4044,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 11384, @@ -4071,11 +4071,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33554, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565131125Z", "event.id": "bnj3cofh3cdjy", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:02.143837873Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4108,7 +4108,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 272063, "source.domain": "elasticsearch", @@ -4131,11 +4131,11 @@ "destination.ip": "203.0.113.27", "destination.port": 53706, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:43:50.822333871Z", "event.id": "bnj3cofh3cdjv", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:43:50.703302550Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4162,7 +4162,7 @@ "10.87.40.76", "203.0.113.27" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1791, "source.domain": "kibana", @@ -4183,11 +4183,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.789039435Z", "event.id": "bnj3cofh3cdkh", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.458515996Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4220,7 +4220,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 18295, @@ -4244,11 +4244,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:44:40.243022993Z", "event.id": "bnj3cofh3cdkg", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:44:40.125336665Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4275,7 +4275,7 @@ "198.51.100.107", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.107", "source.as.number": 15169, "source.bytes": 1467, @@ -4298,11 +4298,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33556, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565335113Z", "event.id": "bnj3cofh3cdk7", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500498059Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4335,7 +4335,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 165290, @@ -4359,11 +4359,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:43:50.822333871Z", "event.id": "bnj3cofh3cdk9", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:43:50.703302550Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4390,7 +4390,7 @@ "203.0.113.27", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.27", "source.as.number": 15169, "source.bytes": 1458, @@ -4413,11 +4413,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:42:11.183868408Z", "event.id": "bnj3cofh3cdkj", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:42:11.063146265Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4444,7 +4444,7 @@ "203.0.113.27", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.27", "source.as.number": 15169, "source.bytes": 1464, @@ -4469,11 +4469,11 @@ "destination.ip": "203.0.113.27", "destination.port": 34090, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:37.827345444Z", "event.id": "bnj3cofh3cdki", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:37.712749588Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4500,7 +4500,7 @@ "10.87.40.76", "203.0.113.27" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1780, "source.domain": "kibana", @@ -4523,11 +4523,11 @@ "destination.ip": "203.0.113.12", "destination.port": 34178, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:51.355687385Z", "event.id": "bnj3cofh3cdkd", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:51.237256499Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4554,7 +4554,7 @@ "10.87.40.76", "203.0.113.12" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1780, "source.domain": "kibana", @@ -4577,11 +4577,11 @@ "destination.ip": "198.51.100.107", "destination.port": 33064, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:44:40.243022993Z", "event.id": "bnj3cofh3cdjw", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:44:40.125336665Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4608,7 +4608,7 @@ "10.87.40.76", "198.51.100.107" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1776, "source.domain": "kibana", @@ -4629,11 +4629,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:48:50.757255245Z", "event.id": "bnj3cofh3cdk3", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:48:50.642206049Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4660,7 +4660,7 @@ "198.51.100.107", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.107", "source.as.number": 15169, "source.bytes": 1461, @@ -4685,11 +4685,11 @@ "destination.ip": "203.0.113.12", "destination.port": 58216, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:36.982303071Z", "event.id": "bnj3cofh3cdkb", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:49:36.865198297Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4716,7 +4716,7 @@ "10.87.40.76", "203.0.113.12" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1781, "source.domain": "kibana", @@ -4740,11 +4740,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.597279654Z", "event.id": "bnj3cofh3cdk4", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075756033Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4777,7 +4777,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 60222, "source.domain": "kibana", @@ -4801,11 +4801,11 @@ "destination.ip": "198.51.100.248", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565335113Z", "event.id": "bnj3cofh3cdkf", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500418290Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4838,7 +4838,7 @@ "10.87.40.76", "198.51.100.248" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 61810, "source.domain": "kibana", @@ -4859,11 +4859,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:36.982303071Z", "event.id": "bnj3cofh3cdkl", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:49:36.865198297Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4890,7 +4890,7 @@ "203.0.113.12", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.12", "source.as.number": 15169, "source.bytes": 1467, @@ -4913,11 +4913,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33510, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565335113Z", "event.id": "bnj3cofh3cdk2", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:39:59.500418290Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -4950,7 +4950,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 136558, @@ -4976,11 +4976,11 @@ "destination.ip": "198.51.100.107", "destination.port": 34906, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:48:50.757255245Z", "event.id": "bnj3cofh3cdko", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:48:50.642206049Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5007,7 +5007,7 @@ "10.87.40.76", "198.51.100.107" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1781, "source.domain": "kibana", @@ -5030,11 +5030,11 @@ "destination.ip": "203.0.113.27", "destination.port": 52454, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:42:40.888804332Z", "event.id": "bnj3cofh3cdke", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:42:40.779893091Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5061,7 +5061,7 @@ "10.87.40.76", "203.0.113.27" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.87.40.76", "source.bytes": 1781, "source.domain": "kibana", @@ -5082,11 +5082,11 @@ "destination.ip": "10.87.40.76", "destination.port": 5601, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:46:37.827345444Z", "event.id": "bnj3cofh3cdka", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:46:37.712749588Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5113,7 +5113,7 @@ "203.0.113.27", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.27", "source.as.number": 15169, "source.bytes": 1467, @@ -5136,11 +5136,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33530, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565300944Z", "event.id": "bnj3cofh3cdkn", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.140119099Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5173,7 +5173,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 170396, @@ -5200,11 +5200,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33570, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:51.821129119Z", "event.id": "bnj3cofh3cdk5", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.469473010Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5237,7 +5237,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 171610, "source.domain": "elasticsearch", @@ -5261,11 +5261,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33858, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:37.933164456Z", "event.id": "bnj3cofh3cdk6", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.458515996Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5298,7 +5298,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 15186, "source.domain": "elasticsearch", @@ -5322,11 +5322,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33590, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565116665Z", "event.id": "y4wffpfk2ero3", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147151100Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5359,7 +5359,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 208416, "source.domain": "elasticsearch", @@ -5383,11 +5383,11 @@ "destination.ip": "192.0.2.177", "destination.port": 60108, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:54.108975753Z", "event.id": "y4wffpfk2eroh", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.762958327Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5420,7 +5420,7 @@ "10.139.99.242", "192.0.2.177" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 90977, "source.domain": "elasticsearch", @@ -5444,11 +5444,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33536, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565156020Z", "event.id": "y4wffpfk2erom", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.150481417Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5481,7 +5481,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 187301, "source.domain": "elasticsearch", @@ -5502,11 +5502,11 @@ "destination.ip": "10.87.40.76", "destination.port": 33560, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565287007Z", "event.id": "y4wffpfk2ero9", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:06.075859688Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5539,7 +5539,7 @@ "198.51.100.248", "10.87.40.76" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "198.51.100.248", "source.as.number": 15169, "source.bytes": 139106, @@ -5563,11 +5563,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:54.108975753Z", "event.id": "y4wffpfk2erog", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:00.762958327Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5600,7 +5600,7 @@ "192.0.2.177", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "192.0.2.177", "source.as.number": 15169, "source.bytes": 1733360, @@ -5627,11 +5627,11 @@ "destination.ip": "203.0.113.134", "destination.port": 33874, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:37.933099658Z", "event.id": "y4wffpfk2ero7", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:20.513551480Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5664,7 +5664,7 @@ "10.139.99.242", "203.0.113.134" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "10.139.99.242", "source.bytes": 149157, "source.domain": "elasticsearch", @@ -5685,11 +5685,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:37.965119632Z", "event.id": "y4wffpfk2eroe", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:08.480430427Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5722,7 +5722,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 11108, @@ -5746,11 +5746,11 @@ "destination.ip": "10.139.99.242", "destination.port": 9200, "event.category": "network", - "event.dataset": "googlecloud.vpcflow", + "event.dataset": "gcp.vpcflow", "event.end": "2019-06-14T03:49:59.565116665Z", "event.id": "y4wffpfk2eroa", "event.kind": "event", - "event.module": "googlecloud", + "event.module": "gcp", "event.start": "2019-06-14T03:40:05.147151100Z", "event.type": "connection", "fileset.name": "vpcflow", @@ -5783,7 +5783,7 @@ "203.0.113.134", "10.139.99.242" ], - "service.type": "googlecloud", + "service.type": "gcp", "source.address": "203.0.113.134", "source.as.number": 15169, "source.bytes": 67337, diff --git a/x-pack/filebeat/module/googlecloud/_meta/config.yml b/x-pack/filebeat/module/googlecloud/_meta/config.yml index 7ca54bd84c0..2c535fb4664 100644 --- a/x-pack/filebeat/module/googlecloud/_meta/config.yml +++ b/x-pack/filebeat/module/googlecloud/_meta/config.yml @@ -1,4 +1,5 @@ -- module: googlecloud +# googlecloud module is deprecated, please use gcp instead +- module: gcp vpcflow: enabled: true @@ -7,11 +8,11 @@ # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be # configured to use this topic as a sink for VPC flow logs. - var.topic: googlecloud-vpc-flowlogs + var.topic: gcp-vpc-flowlogs # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -25,11 +26,11 @@ # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-firewall + var.topic: gcp-vpc-firewall # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-firewall-sub + var.subscription_name: filebeat-gcp-firewall-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -43,11 +44,11 @@ # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-audit + var.topic: gcp-vpc-audit # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-audit + var.subscription_name: filebeat-gcp-audit # Credentials file for the service account with authorization to read from # the subscription. diff --git a/x-pack/filebeat/module/googlecloud/fields.go b/x-pack/filebeat/module/googlecloud/fields.go deleted file mode 100644 index 91fb012da25..00000000000 --- a/x-pack/filebeat/module/googlecloud/fields.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package googlecloud - -import ( - "github.com/elastic/beats/v7/libbeat/asset" -) - -func init() { - if err := asset.SetFields("filebeat", "googlecloud", asset.ModuleFieldsPri, AssetGooglecloud); err != nil { - panic(err) - } -} - -// AssetGooglecloud returns asset data. -// This is the base64 encoded gzipped contents of module/googlecloud. -func AssetGooglecloud() string { - return "eJzsWltv47oRfs+vmLe0QFYHfd2HAoGzaYNuipzE3QJ9MRhybLGhSJUXu95fX/Am6+o4sbI9B1g/Jbp882lmOPNxpE/wgvvPsFFqI5AK5dgFgOVW4Gf4SzgIi3SUoaGa15Yr+Rn+fAEAcK+YEwhrpaEkkgkuNyDUxsBaq6pzf3EBsOYomPkc7vwEklTYN+x/dl/741q5Oh0ZMex/twFuaCowKNJlbZttuwyN5ZJ4zIJLY4mk2Fw0RuIIEf+7W4MtsQ0LKh6iSkqk4ciOGCDw7R6EosQiAyXDJYZUCN8eFlcdSFtyE/kDN1Cr2olw047b0oNk2sDQEi5MAXcSCDyVRCPzcB00quSab5wO3K6g1urfSO2KM6BKazS1ksyAVYFQOgu2JBbUThp/tAOXjV+BM44IsY8PgnrLaXN/0bqlH4h2MA5kOqdzGF5wv1O6f+5IMEJAbnIA8sNQJS3h0ueoP/ztvrgYZaNxw5Wcj8ljwMtsJs1+VxLnM/ovJXHU5NgC2Nb0d5X7DwuQaHdKv8ye+z7fI/dSGfv7TuRtTVf+r/m4eM/7WJaclsm2j4+q0ftWbiaIGPec4jUzn6cG+FRaDSXlNMU5S39EfE/mh2zvQP6s+j+r/odU/ZT28xT8H5LxP2v9z1offifX+j4j4hi356R73mgo3d1nBODObgNGcyYT8YYvTvPGUV8s93VIkBq13RcjhoizJUrLaVgEKy7XasRu3wOvWL3ugIIH1VXUjzBcypOLhkvKayJWWBEu5suOZYkQIIEwptGYvJBavkAGzqCGirzk9aTxPw6N7T1B249Kc7tfGRRIrdLzEm7wIeODqZHyNUcGz/s2Q6WvgK+ByH0Bd9ZnvFQWNo5oIi0ig4GBUN9iKUk+j3VZCLVD5iugMxiFdsOj44eeF74fTSaiNdm/LZkazHYuhVXmWacFrWRxenKhrrgxs7bxZQoB973m7vq+ZWQiaTYhIuNN4VkpgaRP7xUK/yzRlqhB6RDzTjiCuzSmTkwka/EL4U5sJrjmO1fEWs2fnUUzyntYKk7L7wY1r8ZssOhdPxbWVk+JAnOY4kfi+irBSNEbyOSSmT63zGKknc1OYco/ky1kJg4e4hiHbL9CWyo27Ozv7WTjEUhmfMYfygDcKg3XD3dAiRAmSsh+1TOlcoLBMwa0NrK/MaIW/Zs8Lv6XVLXAK7iMA8mCEUt81cVi+6fipvnn0clfHer95ZhzpKtWUWEaXHGLlRnxkVBy80YHuerZL/81BEzQaJ2WyOLkk8BXbqx3VSDWetDYMOpacEqexWg4U3M5Xxws2xW76apvkAXKqpn1YkclHXJ7hFlmsebC4owt/jbgnWR63kf/e6eiHFM4TQP4cAKHsjKmMdJFqwot8Uvv/Iy8T0hAnpWz0xE4lpa+0qBe8X7ri4QGh0/oiHcPfX0abUyEJxEwzq9iZCsvX1dkg9LOq3CCLA64fVpLX2Xb+iwpz66s9sJjANwqxhrjZZQqzbjciNFtS66d85ejiPubrEdj1A4z+jDwmFOSJcjj9l/TY2O2Z5dCwciUDnrhcmhiBgIet++bZu70/9WFJ3Fxg2nULFQcP9ErjWK3xLrxxH3P4nkKcIOkHe8kUx3tDI3a7LCUBqqESHPOsMXmYUwOlugNNuWzJVunJK/fd4OhJVb4SfiG8I/Hu6tQW7mkwrE8ovCCLutif+Mr+tWUKLZofnn665evt6u7m1+elXoxo3q1cVUY0/a3ze8uvBnt+J7maN91WqO0Da/5EgkWEboheXRn2tt/fti2px3iGrVvtjn604nUCfthuxL3L6TmpqCqGn2a4dp8d6xNZ2GqLWoixCTpozFXbLzZcmlxM5DlJ7S6xM0DX6V58UGSEAkoXQVbIlyIQ9r36ZoWC8X6q+yw/zWGbGZUBdfAcIvCe+zTmlAfdtRa6WxpyJxL+CI3gpuygGu5D9ot3zqA72C1QHz6C/49STfjVwSP72hafohVt8iSYQAewnnla+IBLk0tqeAo25uOwx5L446I9uR3tnn8bcJ+w0heO4GrMZn1rhVxcziZl0R+3mApNoyKWFoiiwOMw1uz18tjZ4we5rUTvX4wYxgwX8a5cBw+55Frh+uUwiB00CngbSLjutU8Q+X1nkllz4CS0UFT9hnXeDaFmwwSwqTJes1pOzgmBueYHzSuUaM8byr5mEHyK/iTQpC6tiZyUIneZD0Ojtb5PW7AM4e4dFM3DJL8aTUZmcP3P/Nxa39UNEnwdW7JZZYM18WZ8jgT9TxaL8Ut2ZzhyigmfwTdJFvPo8vrVa20Hb4bgiPvh95El9dxT06VMKFlHeaa4E3n3AivQ5zAyQ1SSoSs6Ailyg2mKR+VFFnjJavnJ8gPfoyULOc8Rut7hbVQu4+QAd8eFuCx3yID0CdRT2SeIe4NZ5iEW4Jm0T9C7QpYEOk1GPLwWu/y6XFx6UXU5c2Xp2VrozbG09pihrcKX4lFSfdADFRIjNPI4A/ej8vFQ+Do27DY/xGY03kjYrnfs0qLektEngsONmb5QhSkNl4Oot0hSq8ww4aWwNOXX8MC1kiRb+Oxw5c5/v/rxd96sP563nwLE/fb+VuQx+XSP8cOfR+Ip1JtSLO/+DERQ0H2xcX/AgAA//8iHdat" -} diff --git a/x-pack/filebeat/module/googlecloud/module.yml b/x-pack/filebeat/module/googlecloud/module.yml new file mode 100644 index 00000000000..e5d6de04886 --- /dev/null +++ b/x-pack/filebeat/module/googlecloud/module.yml @@ -0,0 +1 @@ +movedTo: gcp diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index a958993a61c..8b2bd7e8324 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -447,6 +447,47 @@ processors: value: "{{panw.panos.ruleset}}" ignore_empty_value: true +# Set url and file values + - rename: + if: 'ctx?.panw?.panos?.sub_type != "url"' + field: url.original + target_field: file.name + ignore_missing: true + + - grok: + field: url.original + patterns: + - '(%{ANY:url.scheme}\:\/\/)?(%{USERNAME:url.username}(\:%{PASSWORD:url.password})?\@)?%{DOMAIN:url.domain}(\:%{POSINT:url.port})?(%{PATH:url.path})?(\?%{QUERY:url.query})?(\#%{ANY:url.fragment})?' + ignore_missing: true + pattern_definitions: + USERNAME: '[^\:]*' + PASSWORD: '[^@]*' + DOMAIN: '[^\/\?#\:]*' + PATH: '[^\?#]*' + QUERY: '[^#]*' + ANY: '.*' + if: 'ctx?.url?.original != null && ctx?.url?.original != "-/" && ctx?.url?.original != ""' + + - grok: + field: url.path + patterns: + - '%{FILENAME}((?:\.%{ANY})*(\.%{ANY:url.extension}))?' + ignore_missing: true + pattern_definitions: + FILENAME: '[^\.]+' + ANY: '.*' + if: 'ctx?.url?.path != null && ctx?.url?.path != ""' + + - grok: + field: file.name + patterns: + - '%{FILENAME}((?:\.%{ANY})*(\.%{ANY:file.extension}))?' + ignore_missing: true + pattern_definitions: + FILENAME: '[^\.]+' + ANY: '.*' + if: 'ctx?.file?.name != null && ctx?.file?.name != ""' + - append: field: related.user value: "{{client.user.name}}" @@ -467,6 +508,12 @@ processors: value: "{{destination.user.name}}" if: "ctx?.destination?.user?.name != null" + - append: + field: related.user + value: "{{url.username}}" + if: "ctx?.url?.username != null && ctx?.url?.username != ''" + allow_duplicates: false + - append: field: related.hash value: "{{panw.panos.file.hash}}" @@ -478,6 +525,12 @@ processors: if: "ctx?.observer?.hostname != null && ctx.observer?.hostname != ''" allow_duplicates: false + - append: + field: related.hosts + value: "{{url.domain}}" + if: "ctx?.url?.domain != null && ctx.url?.domain != ''" + allow_duplicates: false + # Remove temporary fields. - remove: field: diff --git a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json index 8e5df2e94e4..cf6c021da90 100644 --- a/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/pan_inc_threat.log-expected.json @@ -69,6 +69,9 @@ "panw.panos.threat.resource": "lorexx.cn/loader.exe", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lorexx.cn" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -92,7 +95,10 @@ "pan-os", "forwarded" ], - "url.original": "lorexx.cn/loader.exe" + "url.domain": "lorexx.cn", + "url.extension": "exe", + "url.original": "lorexx.cn/loader.exe", + "url.path": "/loader.exe" }, { "@timestamp": "2012-04-10T04:39:56.000-02:00", @@ -164,6 +170,9 @@ "panw.panos.threat.resource": "lsiu.info/evo/count.php?o=2", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lsiu.info" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -187,7 +196,11 @@ "pan-os", "forwarded" ], - "url.original": "lsiu.info/evo/count.php?o=2" + "url.domain": "lsiu.info", + "url.extension": "php", + "url.original": "lsiu.info/evo/count.php?o=2", + "url.path": "/evo/count.php", + "url.query": "o=2" }, { "@timestamp": "2012-04-10T04:39:56.000-02:00", @@ -259,6 +272,9 @@ "panw.panos.threat.resource": "lsiu.info/evo/count.php?o=5", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lsiu.info" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -282,7 +298,11 @@ "pan-os", "forwarded" ], - "url.original": "lsiu.info/evo/count.php?o=5" + "url.domain": "lsiu.info", + "url.extension": "php", + "url.original": "lsiu.info/evo/count.php?o=5", + "url.path": "/evo/count.php", + "url.query": "o=5" }, { "@timestamp": "2012-04-10T04:39:57.000-02:00", @@ -354,6 +374,9 @@ "panw.panos.threat.resource": "lsiu.info/evo/count.php?o=7", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lsiu.info" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -377,7 +400,11 @@ "pan-os", "forwarded" ], - "url.original": "lsiu.info/evo/count.php?o=7" + "url.domain": "lsiu.info", + "url.extension": "php", + "url.original": "lsiu.info/evo/count.php?o=7", + "url.path": "/evo/count.php", + "url.query": "o=7" }, { "@timestamp": "2012-04-10T04:39:57.000-02:00", @@ -449,6 +476,9 @@ "panw.panos.threat.resource": "lsiu.info/evo/exploits/x18.php?o=2&t=1241403746&i=1365814122", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lsiu.info" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -472,7 +502,11 @@ "pan-os", "forwarded" ], - "url.original": "lsiu.info/evo/exploits/x18.php?o=2&t=1241403746&i=1365814122" + "url.domain": "lsiu.info", + "url.extension": "php", + "url.original": "lsiu.info/evo/exploits/x18.php?o=2&t=1241403746&i=1365814122", + "url.path": "/evo/exploits/x18.php", + "url.query": "o=2&t=1241403746&i=1365814122" }, { "@timestamp": "2012-04-10T04:39:57.000-02:00", @@ -544,6 +578,9 @@ "panw.panos.threat.resource": "lsiu.info/evo/exploits/x19.php?o=2&t=1241403746&i=1365814122", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lsiu.info" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -567,7 +604,11 @@ "pan-os", "forwarded" ], - "url.original": "lsiu.info/evo/exploits/x19.php?o=2&t=1241403746&i=1365814122" + "url.domain": "lsiu.info", + "url.extension": "php", + "url.original": "lsiu.info/evo/exploits/x19.php?o=2&t=1241403746&i=1365814122", + "url.path": "/evo/exploits/x19.php", + "url.query": "o=2&t=1241403746&i=1365814122" }, { "@timestamp": "2012-04-10T04:39:54.000-02:00", @@ -639,6 +680,9 @@ "panw.panos.threat.resource": "liteautobestguide.cn/load.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "liteautobestguide.cn" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -662,7 +706,10 @@ "pan-os", "forwarded" ], - "url.original": "liteautobestguide.cn/load.php" + "url.domain": "liteautobestguide.cn", + "url.extension": "php", + "url.original": "liteautobestguide.cn/load.php", + "url.path": "/load.php" }, { "@timestamp": "2012-04-10T04:39:54.000-02:00", @@ -734,6 +781,9 @@ "panw.panos.threat.resource": "liteautobestguide.cn/index.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "liteautobestguide.cn" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -757,7 +807,10 @@ "pan-os", "forwarded" ], - "url.original": "liteautobestguide.cn/index.php" + "url.domain": "liteautobestguide.cn", + "url.extension": "php", + "url.original": "liteautobestguide.cn/index.php", + "url.path": "/index.php" }, { "@timestamp": "2012-04-10T04:39:55.000-02:00", @@ -829,6 +882,9 @@ "panw.panos.threat.resource": "litetopdetect.cn/index.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "litetopdetect.cn" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -852,7 +908,10 @@ "pan-os", "forwarded" ], - "url.original": "litetopdetect.cn/index.php" + "url.domain": "litetopdetect.cn", + "url.extension": "php", + "url.original": "litetopdetect.cn/index.php", + "url.path": "/index.php" }, { "@timestamp": "2012-04-10T04:39:55.000-02:00", @@ -924,6 +983,9 @@ "panw.panos.threat.resource": "lkmpmlm.com/fff9999.php?aid=0&uid=6cbbc5081e7548e276611ff5059df6ed30c8f8f1&os=513", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "lkmpmlm.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -947,7 +1009,11 @@ "pan-os", "forwarded" ], - "url.original": "lkmpmlm.com/fff9999.php?aid=0&uid=6cbbc5081e7548e276611ff5059df6ed30c8f8f1&os=513" + "url.domain": "lkmpmlm.com", + "url.extension": "php", + "url.original": "lkmpmlm.com/fff9999.php?aid=0&uid=6cbbc5081e7548e276611ff5059df6ed30c8f8f1&os=513", + "url.path": "/fff9999.php", + "url.query": "aid=0&uid=6cbbc5081e7548e276611ff5059df6ed30c8f8f1&os=513" }, { "@timestamp": "2012-04-10T04:39:52.000-02:00", @@ -1019,6 +1085,9 @@ "panw.panos.threat.resource": "girlteenxxxfreemov.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "girlteenxxxfreemov.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1042,7 +1111,9 @@ "pan-os", "forwarded" ], - "url.original": "girlteenxxxfreemov.com/" + "url.domain": "girlteenxxxfreemov.com", + "url.original": "girlteenxxxfreemov.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:39:53.000-02:00", @@ -1114,6 +1185,9 @@ "panw.panos.threat.resource": "imagesrepository.com/resolution.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "imagesrepository.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1137,7 +1211,10 @@ "pan-os", "forwarded" ], - "url.original": "imagesrepository.com/resolution.php" + "url.domain": "imagesrepository.com", + "url.extension": "php", + "url.original": "imagesrepository.com/resolution.php", + "url.path": "/resolution.php" }, { "@timestamp": "2012-04-10T04:39:53.000-02:00", @@ -1209,6 +1286,9 @@ "panw.panos.threat.resource": "hottestfiles.com/search/search.php?q=xxx", "panw.panos.type": "THREAT", "panw.panos.url.category": "search-engines", + "related.hosts": [ + "hottestfiles.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1232,7 +1312,11 @@ "pan-os", "forwarded" ], - "url.original": "hottestfiles.com/search/search.php?q=xxx" + "url.domain": "hottestfiles.com", + "url.extension": "php", + "url.original": "hottestfiles.com/search/search.php?q=xxx", + "url.path": "/search/search.php", + "url.query": "q=xxx" }, { "@timestamp": "2012-04-10T04:39:54.000-02:00", @@ -1303,6 +1387,9 @@ "panw.panos.threat.resource": "infodist1.com/in.cgi?11¶meter=404", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "infodist1.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1326,7 +1413,11 @@ "pan-os", "forwarded" ], - "url.original": "infodist1.com/in.cgi?11¶meter=404" + "url.domain": "infodist1.com", + "url.extension": "cgi", + "url.original": "infodist1.com/in.cgi?11¶meter=404", + "url.path": "/in.cgi", + "url.query": "11¶meter=404" }, { "@timestamp": "2012-04-10T04:39:51.000-02:00", @@ -1398,6 +1489,9 @@ "panw.panos.threat.resource": "cls-softwares.com/suc.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "cls-softwares.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1421,7 +1515,10 @@ "pan-os", "forwarded" ], - "url.original": "cls-softwares.com/suc.php" + "url.domain": "cls-softwares.com", + "url.extension": "php", + "url.original": "cls-softwares.com/suc.php", + "url.path": "/suc.php" }, { "@timestamp": "2012-04-10T04:39:51.000-02:00", @@ -1493,6 +1590,9 @@ "panw.panos.threat.resource": "cls-softwares.com/softwarefortubeview.40013.exe", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "cls-softwares.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1516,7 +1616,10 @@ "pan-os", "forwarded" ], - "url.original": "cls-softwares.com/softwarefortubeview.40013.exe" + "url.domain": "cls-softwares.com", + "url.extension": "exe", + "url.original": "cls-softwares.com/softwarefortubeview.40013.exe", + "url.path": "/softwarefortubeview.40013.exe" }, { "@timestamp": "2012-04-10T04:39:52.000-02:00", @@ -1584,6 +1687,9 @@ "panw.panos.threat.resource": "findmorepill.com/klik/search.php?q=xxx", "panw.panos.type": "THREAT", "panw.panos.url.category": "online-gambling", + "related.hosts": [ + "findmorepill.com" + ], "related.ip": [ "192.168.0.2", "78.159.99.224", @@ -1607,7 +1713,11 @@ "pan-os", "forwarded" ], - "url.original": "findmorepill.com/klik/search.php?q=xxx" + "url.domain": "findmorepill.com", + "url.extension": "php", + "url.original": "findmorepill.com/klik/search.php?q=xxx", + "url.path": "/klik/search.php", + "url.query": "q=xxx" }, { "@timestamp": "2012-04-10T04:39:48.000-02:00", @@ -1679,6 +1789,9 @@ "panw.panos.threat.resource": "allowedwebsurfing.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "allowedwebsurfing.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1702,7 +1815,9 @@ "pan-os", "forwarded" ], - "url.original": "allowedwebsurfing.com/" + "url.domain": "allowedwebsurfing.com", + "url.original": "allowedwebsurfing.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:39:49.000-02:00", @@ -1774,6 +1889,9 @@ "panw.panos.threat.resource": "antivirus-remote.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "antivirus-remote.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1797,7 +1915,9 @@ "pan-os", "forwarded" ], - "url.original": "antivirus-remote.com/" + "url.domain": "antivirus-remote.com", + "url.original": "antivirus-remote.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:39:49.000-02:00", @@ -1869,6 +1989,9 @@ "panw.panos.threat.resource": "bklinkov.ru/hi/start.cfg", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "bklinkov.ru" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1892,7 +2015,10 @@ "pan-os", "forwarded" ], - "url.original": "bklinkov.ru/hi/start.cfg" + "url.domain": "bklinkov.ru", + "url.extension": "cfg", + "url.original": "bklinkov.ru/hi/start.cfg", + "url.path": "/hi/start.cfg" }, { "@timestamp": "2012-04-10T04:39:50.000-02:00", @@ -1964,6 +2090,9 @@ "panw.panos.threat.resource": "blogsexnakedgirlxxx.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "blogsexnakedgirlxxx.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -1987,7 +2116,9 @@ "pan-os", "forwarded" ], - "url.original": "blogsexnakedgirlxxx.com/" + "url.domain": "blogsexnakedgirlxxx.com", + "url.original": "blogsexnakedgirlxxx.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:39:50.000-02:00", @@ -2059,6 +2190,9 @@ "panw.panos.threat.resource": "bklinkov.ru/hi/start.exe", "panw.panos.type": "THREAT", "panw.panos.url.category": "not-resolved", + "related.hosts": [ + "bklinkov.ru" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -2082,7 +2216,10 @@ "pan-os", "forwarded" ], - "url.original": "bklinkov.ru/hi/start.exe" + "url.domain": "bklinkov.ru", + "url.extension": "exe", + "url.original": "bklinkov.ru/hi/start.exe", + "url.path": "/hi/start.exe" }, { "@timestamp": "2012-04-10T04:39:47.000-02:00", @@ -3195,6 +3332,9 @@ "panw.panos.threat.resource": "wantfinest.com/tds/in.cgi?default", "panw.panos.type": "THREAT", "panw.panos.url.category": "unknown", + "related.hosts": [ + "wantfinest.com" + ], "related.ip": [ "192.168.0.2", "69.43.161.167", @@ -3218,7 +3358,11 @@ "pan-os", "forwarded" ], - "url.original": "wantfinest.com/tds/in.cgi?default" + "url.domain": "wantfinest.com", + "url.extension": "cgi", + "url.original": "wantfinest.com/tds/in.cgi?default", + "url.path": "/tds/in.cgi", + "url.query": "default" }, { "@timestamp": "2012-04-10T04:39:38.000-02:00", @@ -3286,6 +3430,9 @@ "panw.panos.threat.resource": "sameshitasiteverwas.com/traf/tds/in.cgi?2", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "sameshitasiteverwas.com" + ], "related.ip": [ "192.168.0.2", "202.31.187.154", @@ -3309,7 +3456,11 @@ "pan-os", "forwarded" ], - "url.original": "sameshitasiteverwas.com/traf/tds/in.cgi?2" + "url.domain": "sameshitasiteverwas.com", + "url.extension": "cgi", + "url.original": "sameshitasiteverwas.com/traf/tds/in.cgi?2", + "url.path": "/traf/tds/in.cgi", + "url.query": "2" }, { "@timestamp": "2012-04-10T04:39:39.000-02:00", @@ -3377,6 +3528,9 @@ "panw.panos.threat.resource": "svarkon.ru/update.exe", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "svarkon.ru" + ], "related.ip": [ "192.168.0.2", "89.111.176.67", @@ -3400,7 +3554,10 @@ "pan-os", "forwarded" ], - "url.original": "svarkon.ru/update.exe" + "url.domain": "svarkon.ru", + "url.extension": "exe", + "url.original": "svarkon.ru/update.exe", + "url.path": "/update.exe" }, { "@timestamp": "2012-04-10T04:39:36.000-02:00", @@ -3471,6 +3628,9 @@ "panw.panos.threat.resource": "onlinescanxpp.com/land/eurl/1.php?code=", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "onlinescanxpp.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -3494,7 +3654,11 @@ "pan-os", "forwarded" ], - "url.original": "onlinescanxpp.com/land/eurl/1.php?code=" + "url.domain": "onlinescanxpp.com", + "url.extension": "php", + "url.original": "onlinescanxpp.com/land/eurl/1.php?code=", + "url.path": "/land/eurl/1.php", + "url.query": "code=" }, { "@timestamp": "2012-04-10T04:39:34.000-02:00", @@ -3562,6 +3726,9 @@ "panw.panos.threat.resource": "nolagtime.com/conn/?JKV_1RWbUUdIfRUWUaITfdIfbREdYEYdfTTRI-6XBB_1WQR-6GF5_1AU-6LC6_1Y-gW-gEUQQ-gE-tsDF6K5D_rpX51_rR-t-66FC_1Q_fQ_fQ_fQ_fQ_fQ_fQ_fQ-62BG_1Q-672V_1YOR-6N8J_1Q-6252_1WQRR-69LV_1-65GZ_1W-6", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "nolagtime.com" + ], "related.ip": [ "192.168.0.2", "208.73.210.29", @@ -3585,7 +3752,10 @@ "pan-os", "forwarded" ], - "url.original": "nolagtime.com/conn/?JKV_1RWbUUdIfRUWUaITfdIfbREdYEYdfTTRI-6XBB_1WQR-6GF5_1AU-6LC6_1Y-gW-gEUQQ-gE-tsDF6K5D_rpX51_rR-t-66FC_1Q_fQ_fQ_fQ_fQ_fQ_fQ_fQ-62BG_1Q-672V_1YOR-6N8J_1Q-6252_1WQRR-69LV_1-65GZ_1W-6" + "url.domain": "nolagtime.com", + "url.original": "nolagtime.com/conn/?JKV_1RWbUUdIfRUWUaITfdIfbREdYEYdfTTRI-6XBB_1WQR-6GF5_1AU-6LC6_1Y-gW-gEUQQ-gE-tsDF6K5D_rpX51_rR-t-66FC_1Q_fQ_fQ_fQ_fQ_fQ_fQ_fQ-62BG_1Q-672V_1YOR-6N8J_1Q-6252_1WQRR-69LV_1-65GZ_1W-6", + "url.path": "/conn/", + "url.query": "JKV_1RWbUUdIfRUWUaITfdIfbREdYEYdfTTRI-6XBB_1WQR-6GF5_1AU-6LC6_1Y-gW-gEUQQ-gE-tsDF6K5D_rpX51_rR-t-66FC_1Q_fQ_fQ_fQ_fQ_fQ_fQ_fQ-62BG_1Q-672V_1YOR-6N8J_1Q-6252_1WQRR-69LV_1-65GZ_1W-6" }, { "@timestamp": "2012-04-10T04:39:35.000-02:00", @@ -3653,6 +3823,9 @@ "panw.panos.threat.resource": "nolagtime.com/gwc.txt", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "nolagtime.com" + ], "related.ip": [ "192.168.0.2", "208.73.210.29", @@ -3676,7 +3849,10 @@ "pan-os", "forwarded" ], - "url.original": "nolagtime.com/gwc.txt" + "url.domain": "nolagtime.com", + "url.extension": "txt", + "url.original": "nolagtime.com/gwc.txt", + "url.path": "/gwc.txt" }, { "@timestamp": "2012-04-10T04:38:19.000-02:00", @@ -3747,6 +3923,9 @@ "panw.panos.threat.resource": "karavan.us/bon/index.php", "panw.panos.type": "THREAT", "panw.panos.url.category": "unknown", + "related.hosts": [ + "karavan.us" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -3770,7 +3949,10 @@ "pan-os", "forwarded" ], - "url.original": "karavan.us/bon/index.php" + "url.domain": "karavan.us", + "url.extension": "php", + "url.original": "karavan.us/bon/index.php", + "url.path": "/bon/index.php" }, { "@timestamp": "2012-04-10T04:38:14.000-02:00", @@ -3838,6 +4020,9 @@ "panw.panos.threat.resource": "findnolimits.com/go.php?sid=1", "panw.panos.type": "THREAT", "panw.panos.url.category": "dead-sites", + "related.hosts": [ + "findnolimits.com" + ], "related.ip": [ "192.168.0.2", "208.73.210.29", @@ -3861,7 +4046,11 @@ "pan-os", "forwarded" ], - "url.original": "findnolimits.com/go.php?sid=1" + "url.domain": "findnolimits.com", + "url.extension": "php", + "url.original": "findnolimits.com/go.php?sid=1", + "url.path": "/go.php", + "url.query": "sid=1" }, { "@timestamp": "2012-04-10T04:38:12.000-02:00", @@ -3929,6 +4118,9 @@ "panw.panos.threat.resource": "bizoplata.ru/moun.html", "panw.panos.type": "THREAT", "panw.panos.url.category": "parked-domains", + "related.hosts": [ + "bizoplata.ru" + ], "related.ip": [ "192.168.0.2", "89.108.64.156", @@ -3952,7 +4144,10 @@ "pan-os", "forwarded" ], - "url.original": "bizoplata.ru/moun.html" + "url.domain": "bizoplata.ru", + "url.extension": "html", + "url.original": "bizoplata.ru/moun.html", + "url.path": "/moun.html" }, { "@timestamp": "2012-04-10T04:38:12.000-02:00", @@ -4020,6 +4215,9 @@ "panw.panos.threat.resource": "bizoplata.ru/palast.html", "panw.panos.type": "THREAT", "panw.panos.url.category": "parked-domains", + "related.hosts": [ + "bizoplata.ru" + ], "related.ip": [ "192.168.0.2", "89.108.64.156", @@ -4043,7 +4241,10 @@ "pan-os", "forwarded" ], - "url.original": "bizoplata.ru/palast.html" + "url.domain": "bizoplata.ru", + "url.extension": "html", + "url.original": "bizoplata.ru/palast.html", + "url.path": "/palast.html" }, { "@timestamp": "2012-04-10T04:37:28.000-02:00", @@ -4066,6 +4267,8 @@ "event.outcome": "success", "event.severity": 1, "event.timezone": "-02:00", + "file.extension": "php", + "file.name": "controller.php", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -4133,8 +4336,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "controller.php" + ] }, { "@timestamp": "2012-04-10T04:37:32.000-02:00", @@ -4205,6 +4407,9 @@ "panw.panos.threat.resource": "www.15min.it/", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "www.15min.it" + ], "related.ip": [ "192.168.0.2", "216.8.179.25", @@ -4228,7 +4433,9 @@ "pan-os", "forwarded" ], - "url.original": "www.15min.it/" + "url.domain": "www.15min.it", + "url.original": "www.15min.it/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:37:27.000-02:00", @@ -4296,6 +4503,9 @@ "panw.panos.threat.resource": "tubemov.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "adult-and-pornography", + "related.hosts": [ + "tubemov.com" + ], "related.ip": [ "192.168.0.2", "69.43.161.154", @@ -4319,7 +4529,9 @@ "pan-os", "forwarded" ], - "url.original": "tubemov.com/" + "url.domain": "tubemov.com", + "url.original": "tubemov.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:37:25.000-02:00", @@ -4387,6 +4599,9 @@ "panw.panos.threat.resource": "pagesinxt.com/?dn=teenstube.us&flrdr=yes&nxte=js", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "pagesinxt.com" + ], "related.ip": [ "192.168.0.2", "208.91.196.252", @@ -4410,7 +4625,10 @@ "pan-os", "forwarded" ], - "url.original": "pagesinxt.com/?dn=teenstube.us&flrdr=yes&nxte=js" + "url.domain": "pagesinxt.com", + "url.original": "pagesinxt.com/?dn=teenstube.us&flrdr=yes&nxte=js", + "url.path": "/", + "url.query": "dn=teenstube.us&flrdr=yes&nxte=js" }, { "@timestamp": "2012-04-10T04:37:05.000-02:00", @@ -4478,6 +4696,9 @@ "panw.panos.threat.resource": "movfree.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "spyware-and-adware", + "related.hosts": [ + "movfree.com" + ], "related.ip": [ "192.168.0.2", "208.73.210.29", @@ -4501,7 +4722,9 @@ "pan-os", "forwarded" ], - "url.original": "movfree.com/" + "url.domain": "movfree.com", + "url.original": "movfree.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:36:51.000-02:00", @@ -4572,6 +4795,9 @@ "panw.panos.threat.resource": "gometascan.com/", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "gometascan.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -4595,7 +4821,9 @@ "pan-os", "forwarded" ], - "url.original": "gometascan.com/" + "url.domain": "gometascan.com", + "url.original": "gometascan.com/", + "url.path": "/" }, { "@timestamp": "2012-04-10T04:36:39.000-02:00", @@ -4666,6 +4894,9 @@ "panw.panos.threat.resource": "antivirus-powerful-scannerv2.com/download/Install_11-1.exe", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "antivirus-powerful-scannerv2.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -4689,7 +4920,10 @@ "pan-os", "forwarded" ], - "url.original": "antivirus-powerful-scannerv2.com/download/Install_11-1.exe" + "url.domain": "antivirus-powerful-scannerv2.com", + "url.extension": "exe", + "url.original": "antivirus-powerful-scannerv2.com/download/Install_11-1.exe", + "url.path": "/download/Install_11-1.exe" }, { "@timestamp": "2012-04-10T04:36:38.000-02:00", @@ -4760,6 +4994,9 @@ "panw.panos.threat.resource": "antivirus-powerful-scannerv2.com/1/?id=11-1&back==TQzyDTyMUQNMI=N", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "antivirus-powerful-scannerv2.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -4783,7 +5020,10 @@ "pan-os", "forwarded" ], - "url.original": "antivirus-powerful-scannerv2.com/1/?id=11-1&back==TQzyDTyMUQNMI=N" + "url.domain": "antivirus-powerful-scannerv2.com", + "url.original": "antivirus-powerful-scannerv2.com/1/?id=11-1&back==TQzyDTyMUQNMI=N", + "url.path": "/1/", + "url.query": "id=11-1&back==TQzyDTyMUQNMI=N" }, { "@timestamp": "2012-04-10T04:36:27.000-02:00", @@ -4854,6 +5094,9 @@ "panw.panos.threat.resource": "basdzsdas.com/poker/config.bin", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "basdzsdas.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -4877,7 +5120,10 @@ "pan-os", "forwarded" ], - "url.original": "basdzsdas.com/poker/config.bin" + "url.domain": "basdzsdas.com", + "url.extension": "bin", + "url.original": "basdzsdas.com/poker/config.bin", + "url.path": "/poker/config.bin" }, { "@timestamp": "2012-04-10T04:36:27.000-02:00", @@ -4948,6 +5194,9 @@ "panw.panos.threat.resource": "basdzsdas.com/poker/config.bin", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "basdzsdas.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -4971,7 +5220,10 @@ "pan-os", "forwarded" ], - "url.original": "basdzsdas.com/poker/config.bin" + "url.domain": "basdzsdas.com", + "url.extension": "bin", + "url.original": "basdzsdas.com/poker/config.bin", + "url.path": "/poker/config.bin" }, { "@timestamp": "2012-04-10T04:19:59.000-02:00", @@ -4997,6 +5249,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "uLLGRaXP.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5064,8 +5318,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "uLLGRaXP.exe" + ] }, { "@timestamp": "2012-04-10T04:36:27.000-02:00", @@ -5136,6 +5389,9 @@ "panw.panos.threat.resource": "basdzsdas.com/poker/config.bin", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "basdzsdas.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -5159,7 +5415,10 @@ "pan-os", "forwarded" ], - "url.original": "basdzsdas.com/poker/config.bin" + "url.domain": "basdzsdas.com", + "url.extension": "bin", + "url.original": "basdzsdas.com/poker/config.bin", + "url.path": "/poker/config.bin" }, { "@timestamp": "2012-04-10T04:51:29.000-02:00", @@ -5185,6 +5444,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "FunkyEmoticons_setup.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5252,8 +5513,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "FunkyEmoticons_setup.exe" + ] }, { "@timestamp": "2012-04-10T04:54:33.000-02:00", @@ -5279,6 +5539,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "52hxw.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5345,8 +5607,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "52hxw.exe" + ] }, { "@timestamp": "2012-04-10T05:01:00.000-02:00", @@ -5417,6 +5678,9 @@ "panw.panos.threat.resource": "softsellfast.com/test/config.bin", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "softsellfast.com" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -5440,7 +5704,10 @@ "pan-os", "forwarded" ], - "url.original": "softsellfast.com/test/config.bin" + "url.domain": "softsellfast.com", + "url.extension": "bin", + "url.original": "softsellfast.com/test/config.bin", + "url.path": "/test/config.bin" }, { "@timestamp": "2012-04-10T04:45:17.000-02:00", @@ -5466,6 +5733,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "setup.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5530,8 +5799,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "setup.exe" + ] }, { "@timestamp": "2012-04-10T04:46:16.000-02:00", @@ -5557,6 +5825,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "Live-Player_setup.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5624,8 +5894,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "Live-Player_setup.exe" + ] }, { "@timestamp": "2012-04-10T04:42:39.000-02:00", @@ -5693,6 +5962,9 @@ "panw.panos.threat.resource": "boialex.narod.ru/config.txt", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "boialex.narod.ru" + ], "related.ip": [ "192.168.0.2", "213.180.199.61", @@ -5716,7 +5988,10 @@ "pan-os", "forwarded" ], - "url.original": "boialex.narod.ru/config.txt" + "url.domain": "boialex.narod.ru", + "url.extension": "txt", + "url.original": "boialex.narod.ru/config.txt", + "url.path": "/config.txt" }, { "@timestamp": "2012-04-10T04:42:42.000-02:00", @@ -5784,6 +6059,9 @@ "panw.panos.threat.resource": "edw-melon.narod.ru/config.txt", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "edw-melon.narod.ru" + ], "related.ip": [ "192.168.0.2", "213.180.199.61", @@ -5807,7 +6085,10 @@ "pan-os", "forwarded" ], - "url.original": "edw-melon.narod.ru/config.txt" + "url.domain": "edw-melon.narod.ru", + "url.extension": "txt", + "url.original": "edw-melon.narod.ru/config.txt", + "url.path": "/config.txt" }, { "@timestamp": "2012-04-10T04:42:51.000-02:00", @@ -5875,6 +6156,9 @@ "panw.panos.threat.resource": "maximtushin.narod.ru/config.txt", "panw.panos.type": "THREAT", "panw.panos.url.category": "malware-sites", + "related.hosts": [ + "maximtushin.narod.ru" + ], "related.ip": [ "192.168.0.2", "213.180.199.61", @@ -5898,7 +6182,10 @@ "pan-os", "forwarded" ], - "url.original": "maximtushin.narod.ru/config.txt" + "url.domain": "maximtushin.narod.ru", + "url.extension": "txt", + "url.original": "maximtushin.narod.ru/config.txt", + "url.path": "/config.txt" }, { "@timestamp": "2012-04-10T04:19:59.000-02:00", @@ -5924,6 +6211,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "uLLGRaXP.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -5991,8 +6280,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "uLLGRaXP.exe" + ] }, { "@timestamp": "2012-04-10T04:09:01.000-02:00", @@ -6063,6 +6351,9 @@ "panw.panos.threat.resource": "marketingsoluchion.biz/fkn/config.bin", "panw.panos.type": "THREAT", "panw.panos.url.category": "unknown", + "related.hosts": [ + "marketingsoluchion.biz" + ], "related.ip": [ "192.168.0.2", "204.232.231.46", @@ -6086,7 +6377,10 @@ "pan-os", "forwarded" ], - "url.original": "marketingsoluchion.biz/fkn/config.bin" + "url.domain": "marketingsoluchion.biz", + "url.extension": "bin", + "url.original": "marketingsoluchion.biz/fkn/config.bin", + "url.path": "/fkn/config.bin" }, { "@timestamp": "2012-04-09T08:18:27.000-02:00", @@ -6122,6 +6416,8 @@ "event.type": [ "allowed" ], + "file.extension": "aspx", + "file.name": "default.aspx", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6179,8 +6475,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "default.aspx" + ] }, { "@timestamp": "2012-04-09T08:18:29.000-02:00", @@ -6206,6 +6501,8 @@ "event.type": [ "allowed" ], + "file.extension": "aspx", + "file.name": "sck.aspx", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6273,8 +6570,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "sck.aspx" + ] }, { "@timestamp": "2012-04-09T08:18:32.000-02:00", @@ -6300,6 +6596,8 @@ "event.type": [ "allowed" ], + "file.extension": "dll", + "file.name": "ADSAdClient31.dll", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6367,8 +6665,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "ADSAdClient31.dll" + ] }, { "@timestamp": "2012-04-09T08:18:33.000-02:00", @@ -6404,6 +6701,8 @@ "event.type": [ "allowed" ], + "file.extension": "gif", + "file.name": "c.gif", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6461,8 +6760,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "c.gif" + ] }, { "@timestamp": "2012-04-09T08:18:37.000-02:00", @@ -6488,6 +6786,7 @@ "event.type": [ "allowed" ], + "file.name": "csi", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6552,8 +6851,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "csi" + ] }, { "@timestamp": "2012-04-09T08:50:12.000-02:00", @@ -6586,6 +6884,8 @@ "event.type": [ "allowed" ], + "file.extension": "com", + "file.name": "internal-tuner.pandora.com", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6643,8 +6943,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "internal-tuner.pandora.com" + ] }, { "@timestamp": "2012-04-09T08:58:18.000-02:00", @@ -6670,6 +6969,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6734,8 +7034,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T08:22:27.000-02:00", @@ -6761,6 +7060,8 @@ "event.type": [ "denied" ], + "file.extension": "exe", + "file.name": "about.exe", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6828,8 +7129,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "about.exe" + ] }, { "@timestamp": "2012-04-09T07:11:43.000-02:00", @@ -6855,6 +7155,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -6919,8 +7220,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T07:14:02.000-02:00", @@ -6946,6 +7246,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7010,8 +7311,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T07:14:39.000-02:00", @@ -7037,6 +7337,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7101,8 +7402,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T07:16:03.000-02:00", @@ -7128,6 +7428,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7192,8 +7493,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T07:18:14.000-02:00", @@ -7226,6 +7526,8 @@ "event.type": [ "allowed" ], + "file.extension": "gif", + "file.name": "__utm.gif", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7283,8 +7585,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "__utm.gif" + ] }, { "@timestamp": "2012-04-09T07:25:04.000-02:00", @@ -7310,6 +7611,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7374,8 +7676,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T07:36:04.000-02:00", @@ -7401,6 +7702,8 @@ "event.type": [ "allowed" ], + "file.extension": "png", + "file.name": "nav_logo107.png", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7465,8 +7768,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "nav_logo107.png" + ] }, { "@timestamp": "2012-04-09T08:08:08.000-02:00", @@ -7492,6 +7794,7 @@ "event.type": [ "allowed" ], + "file.name": "Eadweard_Muybridge", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7556,8 +7859,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "Eadweard_Muybridge" + ] }, { "@timestamp": "2012-04-09T08:08:44.000-02:00", @@ -7583,6 +7885,8 @@ "event.type": [ "allowed" ], + "file.extension": "php", + "file.name": "load.php", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7647,8 +7951,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "load.php" + ] }, { "@timestamp": "2012-04-09T08:16:57.000-02:00", @@ -7674,6 +7977,8 @@ "event.type": [ "denied" ], + "file.extension": "css", + "file.name": "8fe44cb728c0f40750c64ee906eb72.css", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7741,8 +8046,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "8fe44cb728c0f40750c64ee906eb72.css" + ] }, { "@timestamp": "2012-04-09T04:06:41.000-02:00", @@ -7768,6 +8072,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7832,8 +8137,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T04:12:52.000-02:00", @@ -7859,6 +8163,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -7923,8 +8228,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T06:07:49.000-02:00", @@ -7950,6 +8254,8 @@ "event.type": [ "allowed" ], + "file.extension": "xml", + "file.name": "appcast.xml", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8017,8 +8323,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "appcast.xml" + ] }, { "@timestamp": "2012-04-09T06:48:44.000-02:00", @@ -8044,6 +8349,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8108,8 +8414,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T06:48:59.000-02:00", @@ -8135,6 +8440,7 @@ "event.type": [ "allowed" ], + "file.name": "csi", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8199,8 +8505,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "csi" + ] }, { "@timestamp": "2012-04-09T06:50:14.000-02:00", @@ -8226,6 +8531,8 @@ "event.type": [ "allowed" ], + "file.extension": "php", + "file.name": "index.php", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8293,8 +8600,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "index.php" + ] }, { "@timestamp": "2012-04-09T06:51:34.000-02:00", @@ -8320,6 +8626,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8384,8 +8691,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T06:53:41.000-02:00", @@ -8418,6 +8724,8 @@ "event.type": [ "allowed" ], + "file.extension": "gif", + "file.name": "__utm.gif", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8475,8 +8783,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "__utm.gif" + ] }, { "@timestamp": "2012-04-09T06:54:35.000-02:00", @@ -8502,6 +8809,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8566,8 +8874,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T06:54:55.000-02:00", @@ -8593,6 +8900,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8657,8 +8965,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T03:44:49.000-02:00", @@ -8691,6 +8998,8 @@ "event.type": [ "allowed" ], + "file.extension": "com", + "file.name": "internal-tuner.pandora.com", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8748,8 +9057,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "internal-tuner.pandora.com" + ] }, { "@timestamp": "2012-04-09T03:45:45.000-02:00", @@ -8775,6 +9083,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8839,8 +9148,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T03:49:17.000-02:00", @@ -8866,6 +9174,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -8930,8 +9239,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T03:53:41.000-02:00", @@ -8957,6 +9265,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -9021,8 +9330,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T03:55:23.000-02:00", @@ -9048,6 +9356,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -9112,8 +9421,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] }, { "@timestamp": "2012-04-09T03:55:52.000-02:00", @@ -9139,6 +9447,8 @@ "event.type": [ "allowed" ], + "file.extension": "js", + "file.name": "ga.js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -9203,8 +9513,7 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "ga.js" + ] }, { "@timestamp": "2012-04-09T04:03:55.000-02:00", @@ -9230,6 +9539,7 @@ "event.type": [ "denied" ], + "file.name": "js", "fileset.name": "panos", "input.type": "log", "labels.captive_portal": true, @@ -9294,7 +9604,6 @@ "tags": [ "pan-os", "forwarded" - ], - "url.original": "js" + ] } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/panw/panos/test/threat.log-expected.json b/x-pack/filebeat/module/panw/panos/test/threat.log-expected.json index de6c83a2fa1..d03e24e00c7 100644 --- a/x-pack/filebeat/module/panw/panos/test/threat.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/threat.log-expected.json @@ -75,7 +75,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -99,7 +100,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -177,7 +180,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -201,7 +205,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -279,7 +285,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -303,7 +310,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -381,7 +390,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -405,7 +415,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -483,7 +495,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -507,7 +520,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -585,7 +600,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -609,7 +625,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -687,7 +705,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -711,7 +730,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -789,7 +810,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -813,7 +835,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -891,7 +915,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -915,7 +940,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -993,7 +1020,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1017,7 +1045,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -1095,7 +1125,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1119,7 +1150,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -1197,7 +1230,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1221,7 +1255,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -1299,7 +1335,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1323,7 +1360,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -1401,7 +1440,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1425,7 +1465,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:36.000-02:00", @@ -1503,7 +1545,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1527,7 +1570,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:37.000-02:00", @@ -1605,7 +1650,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1629,7 +1675,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:37.000-02:00", @@ -1707,7 +1755,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1731,7 +1780,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:37.000-02:00", @@ -1809,7 +1860,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1833,7 +1885,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:37.000-02:00", @@ -1911,7 +1965,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -1935,7 +1990,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:38.000-02:00", @@ -2013,7 +2070,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2037,7 +2095,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:38.000-02:00", @@ -2115,7 +2175,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "b.scorecardresearch.com" ], "related.ip": [ "192.168.15.224", @@ -2139,7 +2200,9 @@ "pan-os", "forwarded" ], - "url.original": "b.scorecardresearch.com/" + "url.domain": "b.scorecardresearch.com", + "url.original": "b.scorecardresearch.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:38.000-02:00", @@ -2217,7 +2280,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2241,7 +2305,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2319,7 +2385,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2343,7 +2410,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2421,7 +2490,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2445,7 +2515,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2523,7 +2595,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2547,7 +2620,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2625,7 +2700,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2649,7 +2725,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2727,7 +2805,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2751,7 +2830,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2829,7 +2910,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2853,7 +2935,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -2931,7 +3015,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -2955,7 +3040,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3033,7 +3120,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3057,7 +3145,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3135,7 +3225,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3159,7 +3250,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3237,7 +3330,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3261,7 +3355,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3339,7 +3435,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3363,7 +3460,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3441,7 +3540,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3465,7 +3565,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:46.000-02:00", @@ -3543,7 +3645,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "consent.cmp.oath.com" ], "related.ip": [ "192.168.15.224", @@ -3567,7 +3670,9 @@ "pan-os", "forwarded" ], - "url.original": "consent.cmp.oath.com/" + "url.domain": "consent.cmp.oath.com", + "url.original": "consent.cmp.oath.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:53.000-02:00", @@ -3645,7 +3750,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "cdn.taboola.com" ], "related.ip": [ "192.168.15.224", @@ -3669,7 +3775,9 @@ "pan-os", "forwarded" ], - "url.original": "cdn.taboola.com/" + "url.domain": "cdn.taboola.com", + "url.original": "cdn.taboola.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:54.000-02:00", @@ -3750,7 +3858,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "rules.quantcount.com" ], "related.ip": [ "192.168.15.224", @@ -3774,7 +3883,9 @@ "pan-os", "forwarded" ], - "url.original": "rules.quantcount.com/" + "url.domain": "rules.quantcount.com", + "url.original": "rules.quantcount.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:58.000-02:00", @@ -3855,7 +3966,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -3879,7 +3991,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:58.000-02:00", @@ -3960,7 +4074,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -3984,7 +4099,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:58.000-02:00", @@ -4065,7 +4182,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4089,7 +4207,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:58.000-02:00", @@ -4170,7 +4290,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4194,7 +4315,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:59.000-02:00", @@ -4275,7 +4398,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4299,7 +4423,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:59.000-02:00", @@ -4380,7 +4506,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4404,7 +4531,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:44:59.000-02:00", @@ -4485,7 +4614,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4509,7 +4639,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:00.000-02:00", @@ -4590,7 +4722,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4614,7 +4747,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:00.000-02:00", @@ -4695,7 +4830,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4719,7 +4855,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:00.000-02:00", @@ -4800,7 +4938,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4824,7 +4963,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:00.000-02:00", @@ -4905,7 +5046,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -4929,7 +5071,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:00.000-02:00", @@ -5010,7 +5154,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "srv-2018-11-30-22.config.parsely.com" ], "related.ip": [ "192.168.15.224", @@ -5034,7 +5179,9 @@ "pan-os", "forwarded" ], - "url.original": "srv-2018-11-30-22.config.parsely.com/" + "url.domain": "srv-2018-11-30-22.config.parsely.com", + "url.original": "srv-2018-11-30-22.config.parsely.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:13.000-02:00", @@ -5115,7 +5262,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "www.googleadservices.com" ], "related.ip": [ "192.168.15.224", @@ -5139,7 +5287,9 @@ "pan-os", "forwarded" ], - "url.original": "www.googleadservices.com/" + "url.domain": "www.googleadservices.com", + "url.original": "www.googleadservices.com/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:15.000-02:00", @@ -5217,7 +5367,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5241,7 +5392,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:15.000-02:00", @@ -5319,7 +5472,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5343,7 +5497,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:15.000-02:00", @@ -5421,7 +5577,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5445,7 +5602,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:15.000-02:00", @@ -5523,7 +5682,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5547,7 +5707,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -5625,7 +5787,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5649,7 +5812,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -5727,7 +5892,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5751,7 +5917,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -5829,7 +5997,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5853,7 +6022,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -5931,7 +6102,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -5955,7 +6127,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -6033,7 +6207,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -6057,7 +6232,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:16.000-02:00", @@ -6135,7 +6312,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "service.maxymiser.net" ], "related.ip": [ "192.168.15.224", @@ -6159,7 +6337,9 @@ "pan-os", "forwarded" ], - "url.original": "service.maxymiser.net/" + "url.domain": "service.maxymiser.net", + "url.original": "service.maxymiser.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:26.000-02:00", @@ -6240,7 +6420,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6264,7 +6445,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:26.000-02:00", @@ -6345,7 +6528,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6369,7 +6553,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:26.000-02:00", @@ -6450,7 +6636,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6474,7 +6661,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:26.000-02:00", @@ -6555,7 +6744,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6579,7 +6769,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:26.000-02:00", @@ -6660,7 +6852,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6684,7 +6877,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -6765,7 +6960,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6789,7 +6985,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -6870,7 +7068,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6894,7 +7093,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -6975,7 +7176,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -6999,7 +7201,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -7080,7 +7284,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7104,7 +7309,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -7185,7 +7392,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7209,7 +7417,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -7290,7 +7500,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7314,7 +7525,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:27.000-02:00", @@ -7395,7 +7608,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7419,7 +7633,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:28.000-02:00", @@ -7500,7 +7716,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7524,7 +7741,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:28.000-02:00", @@ -7605,7 +7824,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7629,7 +7849,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:28.000-02:00", @@ -7710,7 +7932,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7734,7 +7957,9 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" }, { "@timestamp": "2018-11-30T16:45:29.000-02:00", @@ -7815,7 +8040,8 @@ "panw.panos.type": "THREAT", "panw.panos.url.category": "business-and-economy", "related.hosts": [ - "PA-220" + "PA-220", + "segment-data.zqtk.net" ], "related.ip": [ "192.168.15.224", @@ -7839,6 +8065,8 @@ "pan-os", "forwarded" ], - "url.original": "segment-data.zqtk.net/" + "url.domain": "segment-data.zqtk.net", + "url.original": "segment-data.zqtk.net/", + "url.path": "/" } ] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 0c7eb3d6e01..dcf5b1764d7 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -17,6 +17,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/modules.d/gcp.yml.disabled b/x-pack/filebeat/modules.d/gcp.yml.disabled new file mode 100644 index 00000000000..330c7d375e1 --- /dev/null +++ b/x-pack/filebeat/modules.d/gcp.yml.disabled @@ -0,0 +1,57 @@ +# Module: gcp +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-gcp.html + +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/x-pack/filebeat/modules.d/googlecloud.yml.disabled b/x-pack/filebeat/modules.d/googlecloud.yml.disabled index 9bf81802677..6f3e6b53e21 100644 --- a/x-pack/filebeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/filebeat/modules.d/googlecloud.yml.disabled @@ -1,7 +1,8 @@ # Module: googlecloud # Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-googlecloud.html -- module: googlecloud +# googlecloud module is deprecated, please use gcp instead +- module: gcp vpcflow: enabled: true @@ -10,11 +11,11 @@ # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be # configured to use this topic as a sink for VPC flow logs. - var.topic: googlecloud-vpc-flowlogs + var.topic: gcp-vpc-flowlogs # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-vpc-flowlogs-sub + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -28,11 +29,11 @@ # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-firewall + var.topic: gcp-vpc-firewall # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-firewall-sub + var.subscription_name: filebeat-gcp-firewall-sub # Credentials file for the service account with authorization to read from # the subscription. @@ -46,11 +47,11 @@ # Google Pub/Sub topic containing firewall logs. Stackdriver must be # configured to use this topic as a sink for firewall logs. - var.topic: googlecloud-vpc-audit + var.topic: gcp-vpc-audit # Google Pub/Sub subscription for the topic. Filebeat will create this # subscription if it does not exist. - var.subscription_name: filebeat-googlecloud-audit + var.subscription_name: filebeat-gcp-audit # Credentials file for the service account with authorization to read from # the subscription. diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 3abd7b67c5e..907a989eb4d 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 40977aa6cf9..06ca7a1ffad 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.14.12 RUN \ apt-get update \ diff --git a/x-pack/libbeat/common/cloudfoundry/dopplerconsumer.go b/x-pack/libbeat/common/cloudfoundry/dopplerconsumer.go index 10ea50dd928..96349eeb7ea 100644 --- a/x-pack/libbeat/common/cloudfoundry/dopplerconsumer.go +++ b/x-pack/libbeat/common/cloudfoundry/dopplerconsumer.go @@ -111,7 +111,7 @@ func (c *DopplerConsumer) firehose(cb func(evt Event), filter consumer.EnvelopeF if !filterFn(env) { continue } - event := envelopeToEvent(env) + event := EnvelopeToEvent(env) if event == nil { c.log.Debugf("Envelope couldn't be converted to event: %+v", env) continue diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index adaa944773c..4d1a67e2b1b 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -461,7 +461,7 @@ func newEventError(env *events.Envelope) *EventError { } } -func envelopeToEvent(env *events.Envelope) Event { +func EnvelopeToEvent(env *events.Envelope) Event { switch *env.EventType { case events.Envelope_HttpStartStop: return newEventHttpAccess(env) diff --git a/x-pack/libbeat/common/cloudfoundry/rlplistener.go b/x-pack/libbeat/common/cloudfoundry/rlplistener.go index e80db747c8e..0c08b12a7fd 100644 --- a/x-pack/libbeat/common/cloudfoundry/rlplistener.go +++ b/x-pack/libbeat/common/cloudfoundry/rlplistener.go @@ -79,7 +79,7 @@ func (c *RlpListener) Start(ctx context.Context) { for i := range envelopes { v1s := conversion.ToV1(envelopes[i]) for _, v := range v1s { - evt := envelopeToEvent(v) + evt := EnvelopeToEvent(v) if evt.EventType() == EventTypeHttpAccess && c.callbacks.HttpAccess != nil { c.callbacks.HttpAccess(evt.(*EventHttpAccess)) } else if evt.EventType() == EventTypeLog && c.callbacks.Log != nil { diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc index e5123ae75a0..d54a567f3f5 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc @@ -25,6 +25,10 @@ metadata in all events from the firehose since version 2.8. In these cases the metadata in the events is used, and `add_cloudfoundry_metadata` processor doesn't modify these fields. +For efficient annotation, application metadata retrieved by the Cloud Foundry +client is stored in a persistent cache on the filesystem under the `path.data` +directory. This is done so the metadata can persist across restarts of {beatname_uc}. +For control over this cache, use the `cache_duration` and `cache_retry_delay` settings. [source,yaml] ------------------------------------------------------------------------------- diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index ad95961aada..f026d2cc7cf 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -24,11 +24,11 @@ services: kibana: # Copied configuration from OSS metricbeat because services with depends_on # cannot be extended with extends - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.9.0}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.10.0}-1 build: context: ../../metricbeat/module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-7.9.0} + KIBANA_VERSION: ${KIBANA_VERSION:-7.10.0} depends_on: - elasticsearch ports: diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index f4a2c1c6cfc..15a643ec929 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -17,6 +17,8 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/rds" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/sqs" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate/task_stats" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/billing" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 164dc564f2f..0a6a954ec6f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -237,6 +237,12 @@ metricbeat.modules: - usage - vpn +#----------------------------- AWS Fargate Module ----------------------------- +- module: awsfargate + period: 10s + metricsets: + - task_stats + #-------------------------------- Azure Module -------------------------------- - module: azure metricsets: diff --git a/x-pack/metricbeat/module/awsfargate/_meta/config.yml b/x-pack/metricbeat/module/awsfargate/_meta/config.yml new file mode 100644 index 00000000000..b83eee4cccf --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/config.yml @@ -0,0 +1,4 @@ +- module: awsfargate + period: 10s + metricsets: + - task_stats diff --git a/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc new file mode 100644 index 00000000000..0f307ec2b50 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc @@ -0,0 +1,61 @@ +Amazon ECS on Fargate provides a method to retrieve various metadata, network +metrics, and Docker stats about tasks and containers. This is referred to as the +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[task metadata endpoint] +and this endpoint is available per container. + +The environment variable is injected by default into the containers of Amazon +ECS tasks on Fargate that use platform version 1.4.0 or later and Amazon ECS +tasks on Amazon EC2 that are running at least version 1.39.0 of the Amazon ECS +container agent. + +The awsfargate module is a Metricbeat module which collects AWS fargate metrics +from task metadata endpoint. + +[float] +== Introduction to AWS ECS and Fargate +Amazon Elastic Container Service (Amazon ECS) is a highly scalable, fast, +container management service that makes it easy to run, stop, and manage +containers. ECS has two launch types that can define how compute resources will +be managed: ECS EC2 and ECS Fargate. + +* *ECS EC2* + +ECS EC2 launches containers that run on EC2 instances. Users have to manage EC2 +instances. Pricing depends on the number of EC2 instances running. + +One can monitor these containers by deploying Metricbeat on the corresponding EC2 instances with the +Metricbeat Docker module enabled. + +In order to achieve this one will need: +-- +. to ensure access to these EC2 instances using ssh keys +coupled with EC2 instances (attach ssh keys on cluster creation using `Key pair` option) +. to enable shh access for the instances with the +proper https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/authorizing-access-to-an-instance.html[inbound rules]. +-- + +* *ECS Fargate* + +ECS Fargate removes the responsibility of provisioning, configuring, and +managing the EC2 instances by allowing AWS to manage the EC2 instances. Users +only need to specify containers and tasks. Pricing based on the number of tasks. + +[float] +== Task Metadata Endpoint +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[Task metadata endpoint] +returns https://docs.docker.com/engine/api/v1.30/#operation/ContainerStats[Docker stats] +in JSON format for all the containers associated with the task. +This endpoint is only available from within the task definition itself, which +means Metricbeat needs to be run as a sidecar container within the task +definition. Since the metadata endpoint is only accessible from within the +Fargate Task, there is no authentication in place. + +[float] +== Metricsets +Currently, we have `task_stats` metricset in `awsfargate` module. + +[float] +=== `task_stats` +This metricset collects runtime CPU metrics, disk I/O metrics, memory metrics, +network metrics and container metadata from both endpoint +`${ECS_CONTAINER_METADATA_URI_V4}/task/stats` and `${ECS_CONTAINER_METADATA_URI_V4}/task`. diff --git a/x-pack/metricbeat/module/awsfargate/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/_meta/fields.yml new file mode 100644 index 00000000000..11242ff2de3 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/fields.yml @@ -0,0 +1,6 @@ +- key: awsfargate + title: "AWS Fargate" + description: > + `awsfargate` module collects AWS fargate metrics from task metadata endpoint. + release: beta + fields: diff --git a/x-pack/metricbeat/module/awsfargate/awsfargate.go b/x-pack/metricbeat/module/awsfargate/awsfargate.go new file mode 100644 index 00000000000..94a030b90ca --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/awsfargate.go @@ -0,0 +1,54 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awsfargate + +import ( + "time" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// Config defines all required and optional parameters for awsfargate metricsets +type Config struct { + Period time.Duration `config:"period" validate:"nonzero,required"` +} + +// MetricSet is the base metricset for all aws metricsets +type MetricSet struct { + mb.BaseMetricSet + Period time.Duration +} + +// ModuleName is the name of this module. +const ModuleName = "awsfargate" + +func init() { + if err := mb.Registry.AddModule(ModuleName, newModule); err != nil { + panic(err) + } +} + +func newModule(base mb.BaseModule) (mb.Module, error) { + var config Config + if err := base.UnpackConfig(&config); err != nil { + return nil, err + } + return &base, nil +} + +// NewMetricSet creates a base metricset for awsfargate metricsets +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + var config Config + err := base.Module().UnpackConfig(&config) + if err != nil { + return nil, err + } + + metricSet := MetricSet{ + BaseMetricSet: base, + Period: config.Period, + } + return &metricSet, nil +} diff --git a/x-pack/metricbeat/module/awsfargate/cloudformation.yml b/x-pack/metricbeat/module/awsfargate/cloudformation.yml new file mode 100644 index 00000000000..18e0759414a --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/cloudformation.yml @@ -0,0 +1,82 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + SubnetID: + Type: String +Resources: + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: metricbeat-cloudformation-fargate + ClusterSettings: + - Name: containerInsights + Value: enabled + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: metricbeat-fargate-log-group + ExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: ecsFargateTaskExecutionRole + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub 'EcsTaskExecutionRole-${AWS::StackName}' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - + - + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: deployment-task-metricbeat + Cpu: 256 + Memory: 512 + NetworkMode: awsvpc + ExecutionRoleArn: !Ref ExecutionRole + ContainerDefinitions: + - Name: deployment-task-metricbeat-container + Image: kaiyansheng/metricbeat-awsfargate:v1 + Secrets: + - Name: ELASTIC_CLOUD_ID + ValueFrom: + - Name: ELASTIC_CLOUD_AUTH + ValueFrom: + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: ecs + EntryPoint: + - sh + - -c + Command: + - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + RequiresCompatibilities: + - EC2 + - FARGATE + Service: + Type: AWS::ECS::Service + Properties: + ServiceName: deployment-metricbeat-service + Cluster: !Ref Cluster + TaskDefinition: !Ref TaskDefinition + DesiredCount: 1 + LaunchType: FARGATE + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + Subnets: + - !Ref SubnetID diff --git a/x-pack/metricbeat/module/awsfargate/doc.go b/x-pack/metricbeat/module/awsfargate/doc.go new file mode 100644 index 00000000000..516d3548dd2 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/doc.go @@ -0,0 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awsfargate diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go new file mode 100644 index 00000000000..9659165097f --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package awsfargate + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "awsfargate", asset.ModuleFieldsPri, AssetAwsfargate); err != nil { + panic(err) + } +} + +// AssetAwsfargate returns asset data. +// This is the base64 encoded gzipped contents of module/awsfargate. +func AssetAwsfargate() string { + return "eJzsWk1z2zYQvftX7Hh6yjTKJdODDp1RnWTGB9sZfzRHGQJXMioSYAAwitLpf+8ABCRKAimQsmTWDY8iwH1v9+0uPvQW5rgcAlmoKZEzovEMQDOd4hDOR1/u4FP56/kZQIKKSpZrJvgQfj8DAHhcz3uETCRFikBFmiLVCsx09xIy1JJRBVMpMtBEzc0vJCGaAPIkF4zrwRmAxBSJwiFMUJMzgCnDNFFDa8s8b4GTDIf2C2OliVarVwB6meMQZlIUeeXXAGr/PK4/8whUcE0YV6Cf1nD1E9GwQImgqCQ5JiWBKjNLxn4Dfvn748Xd+OLm+n50ef3xdnz18X70YXQ/Gj/cXo7/fP/POzP2XTk2SN8/227wz7Y7qi6hebHxe50/dnxyW3DNMoSLzw+e92BrfMhu1fYcJcd0kFO9M8TDUJSkmIynqSChQVMhM6KHkKOkyEMjGuLon8/lZDJDEFOwpBh34EDlhOI2sR0KXMjsP8ADDE6Ssh+YwGRpJcuLbILSTDCBpELiThh32GpG56qWair4rBsBg8B+O97/aqk09s71QpPUsSmDYDztoO6j0kcpNfA5VFKO9TEl5YBaE/VACoWyb3533jbQ9iWChd9H7exwOFQxlulJSlCM321uvLjP71cZWigy2wu3FzrZwtxFFp5VwtSciW7rmA9MzeHy3U3XRYxEktT6MWQ90j0jSousSInGxNpQkBSS8Zn1TMqmZXY9oV+BogwFqgZ8lYDIQ4kE+5IpkoZ5rldx7ExkDXey1NgZsBdu00ciSf1hPmEJHcZHlpunTnRa+58WUiLXLg65KW9IBQ8JuNKiUX5jFMemlB8ZaVkTbM/QwhuGyxuQ+LVApdWvpi5zwkWJOxxDD3xBmD4xao8TVG78bBAYaTAOXwssUBkFel6tuNjp4Tg9N5F12S2Nrkg169hIqhHfnqZS4swlUlPzhvDb4P2Lyn4h2Z7MrKvuLTBWq7w12LE6Npd6iCr3EKOkFtxgIwaHsYur/9EMYvpAS6ZlPzAsNfLDaTa0hWiWneLkc8XFa0+yQLs+cQzoz9MzoEXfODKLZ+8hENtHjkesW08pNdjzphKXKKskKbKMyOUp+0rZ9Qh/7R3GbOJEjpKYaa++09idhw/q62g5leD9bDsnZ/Gz7WwckvW06+wU85pM8VwyzMROu4k8Gruyc7sejNkLy8Gb2rMxMfkLg0eQ5YtxoyIqY8YZyXPGZ27C+Zvzbgdut2ThvOXuZctLZ1NCraeUezswb4FxjXJKaChZY07dqMgyVqedZ1sGXFgr2h6pGrEIDl8YT8SiLmNje3xTisAL9sYy7Zs/6DnkSOY9pPAZydzJIzpwK0qSfSMaxwsh5yYlFOrB/mp2mgNThw0cNlCoo3lNCUsHVBTBuwZoV5ajwH4iLAVrEOXubUoVWsr25fFp3OuKtYXTjFiq5i3VM9Sd27u7jVoaBvRayo1jKlGxxCydFGpQ7EfgGm6bV/gObpNVhKgh6k6uA7+rGmbeSvCucZukvd7bS7NJdS1RP9j7xBj9QYQGq1Qy8r1xXKQSoYUaoR1381yR7559zXVw9YnTIrTVI7TSJHRgWWEYpUdoUVCgJ6HcKDCBYHo+HLXpq93W+dfl5K4L/aalcIlgjsuFkKHtX4Q7PLqVGWu2/i8OjE9EETw3OPhO/pJTkZm1i3O32ygoxiluHraYN1JjElJjzPagL7fc29to5j1gv928zkikyPOT31k6q2ukOaFzrGsBHitKKeSx/wdRQi1NmZVvO4hu0InduR/j6ti30EdLvJtCz8T/OvGE90DvE2+FtL+J1w7iyyTeLsZ/AwAA//8SjLaw" +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json new file mode 100644 index 00000000000..98a48adecbb --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -0,0 +1,148 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "awsfargate": { + "task_stats": { + "cpu": { + "core": null, + "kernel": { + "norm": { + "pct": 0 + }, + "pct": 0, + "ticks": 1520000000 + }, + "system": { + "norm": { + "pct": 1 + }, + "pct": 2, + "ticks": 1420180000000 + }, + "total": { + "norm": { + "pct": 0.2 + }, + "pct": 0.4 + }, + "user": { + "norm": { + "pct": 0 + }, + "pct": 0, + "ticks": 490000000 + } + }, + "diskio": { + "read": { + "bytes": 3452928, + "ops": 118, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "reads": 0, + "summary": { + "bytes": 3452928, + "ops": 118, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "total": 0, + "write": { + "bytes": 0, + "ops": 0, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "writes": 0 + }, + "memory": { + "fail": { + "count": 0 + }, + "limit": 0, + "rss": { + "pct": 0.0010557805807105247, + "total": 4157440 + }, + "stats": { + "active_anon": 4157440, + "active_file": 4497408, + "cache": 6000640, + "dirty": 16384, + "hierarchical_memory_limit": 2147483648, + "hierarchical_memsw_limit": 9223372036854772000, + "inactive_anon": 0, + "inactive_file": 1503232, + "mapped_file": 2183168, + "pgfault": 6668, + "pgmajfault": 52, + "pgpgin": 5925, + "pgpgout": 3445, + "rss": 4157440, + "rss_huge": 0, + "total_active_anon": 4157440, + "total_active_file": 4497408, + "total_cache": 600064, + "total_dirty": 16384, + "total_inactive_anon": 0, + "total_inactive_file": 4497408, + "total_mapped_file": 2183168, + "total_pgfault": 6668, + "total_pgmajfault": 52, + "total_pgpgin": 5925, + "total_pgpgout": 3445, + "total_rss": 4157440, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "usage": { + "max": 15294464, + "pct": 0.003136136404770672, + "total": 12349440 + } + }, + "network": { + "eth0": { + "inbound": { + "bytes": 137315578, + "dropped": 0, + "errors": 0, + "packets": 94338 + }, + "outbound": { + "bytes": 1086811, + "dropped": 0, + "errors": 0, + "packets": 25857 + } + } + } + } + }, + "container": { + "id": "query-metadata-1", + "image": { + "name": "mreferre/eksutils" + }, + "labels": { + "com_amazonaws_ecs_cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com_amazonaws_ecs_container-name": "query-metadata", + "com_amazonaws_ecs_task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com_amazonaws_ecs_task-definition-family": "query-metadata", + "com_amazonaws_ecs_task-definition-version": "7" + }, + "name": "query-metadata" + }, + "service": { + "type": "awsfargate" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc new file mode 100644 index 00000000000..6d5983fe63d --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc @@ -0,0 +1,153 @@ +The `task_stats` metricset in `awsfargate` module allows users to monitor +containers inside the same AWS Fargate task. It fetches runtime CPU metrics, +disk I/O metrics, memory metrics, network metrics and container metadata from +both endpoint `${ECS_CONTAINER_METADATA_URI_V4}/task/stats` and +`${ECS_CONTAINER_METADATA_URI_V4}/task`. + +[float] +=== Configuration Example +This metricset should be ran as a sidecar inside the same AWS Fargate task +definition, and the default configuration file should work. + +[source,yaml] +---- +- module: awsfargate + period: 10s + metricsets: + - task_stats +---- + +[float] +=== Setup Metricbeat Using AWS Fargate +This section is to provide users an AWS native way of configuring Fargate task +definition to run application containers and Metricbeat container using AWS +CloudFormation. + +[float] +==== Store Elastic Cloud Credentials into AWS Secret Manager +If users are using Elastic Cloud, it's recommended to store cloud id and cloud +auth into AWS secret manager. Here are the AWS CLI example: + +Create secret ELASTIC_CLOUD_AUTH: +---- +aws --region us-east-1 secretsmanager create-secret --name ELASTIC_CLOUD_AUTH --secret-string XXX +---- + +Create secret ELASTIC_CLOUD_ID: +---- +aws --region us-east-1 secretsmanager create-secret --name ELASTIC_CLOUD_ID --secret-string YYYY +---- + +[float] +==== AWS CloudFormation Template Example +Here is an example of AWS CloudFormation template to create a new cluster, +create a task definition that runs Metricbeat container and start the service. +Please copy this section into a `cloud_formation.yml` file locally and replace +``, ``, and `` with +your own preferred names. Also you can find `` and +`` values from AWS secret manager. + +[source,yaml] +---- +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + SubnetID: + Type: String +Resources: + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: + ClusterSettings: + - Name: containerInsights + Value: enabled + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: + ExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: ecsFargateTaskExecutionRole + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub 'EcsTaskExecutionRole-${AWS::StackName}' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - + - + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: deployment-task-metricbeat + Cpu: 256 + Memory: 512 + NetworkMode: awsvpc + ExecutionRoleArn: !Ref ExecutionRole + ContainerDefinitions: + - Name: deployment-task-metricbeat-container + Image: kaiyansheng/metricbeat-awsfargate:v1 + Secrets: + - Name: ELASTIC_CLOUD_ID + ValueFrom: + - Name: ELASTIC_CLOUD_AUTH + ValueFrom: + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: ecs + EntryPoint: + - sh + - -c + Command: + - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + RequiresCompatibilities: + - EC2 + - FARGATE + Service: + Type: AWS::ECS::Service + Properties: + ServiceName: + Cluster: !Ref Cluster + TaskDefinition: !Ref TaskDefinition + DesiredCount: 1 + LaunchType: FARGATE + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + Subnets: + - !Ref SubnetID +---- + +[float] +==== Create CloudFormation Stack +Here is the AWS CLI to create a stack using the CloudFormation config file above: +---- +aws --region us-east-1 cloudformation create-stack --stack-name --template-body file://./cloudformation.yml --capabilities CAPABILITY_NAMED_IAM --parameters 'ParameterKey=SubnetID,ParameterValue=' +---- + +Make sure to replace `` with your own subnet in this command. Please go +to Services -> VPC -> Subnets to find subnet ID to use. You can also add several +more containers under the TaskDefinition section. + +[float] +==== Delete CloudFormation Stack +Here is the AWS CLI to delete a stack including the cluster, task definition and +all containers: +---- +aws cloudformation delete-stack --stack-name +---- diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml new file mode 100644 index 00000000000..e0099191f35 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -0,0 +1,298 @@ +- name: task_stats + type: group + description: > + `task_stats` contains the metrics that were scraped from AWS fargate task stats ${ECS_CONTAINER_METADATA_URI_V4}/task/stats metadata endpoint. + release: beta + fields: + - name: cpu + type: group + description: Runtime CPU metrics. + fields: + - name: kernel.pct + type: scaled_float + format: percent + description: > + Percentage of time in kernel space. + - name: kernel.norm.pct + type: scaled_float + format: percent + description: > + Percentage of time in kernel space normalized by the number of CPU cores. + - name: kernel.ticks + type: long + description: > + CPU ticks in kernel space. + - name: system.pct + type: scaled_float + format: percent + description: > + Percentage of total CPU time in the system. + - name: system.norm.pct + type: scaled_float + format: percent + description: > + Percentage of total CPU time in the system normalized by the number of CPU cores. + - name: system.ticks + type: long + description: > + CPU system ticks. + - name: user.pct + type: scaled_float + format: percent + description: > + Percentage of time in user space. + - name: user.norm.pct + type: scaled_float + format: percent + description: > + Percentage of time in user space normalized by the number of CPU cores. + - name: user.ticks + type: long + description: > + CPU ticks in user space. + - name: total.pct + type: scaled_float + format: percent + description: > + Total CPU usage. + - name: total.norm.pct + type: scaled_float + format: percent + description: > + Total CPU usage normalized by the number of CPU cores. + - name: diskio + type: group + description: Disk I/O metrics. + fields: + - name: read + type: group + description: > + Accumulated reads during the life of the container + fields: + - name: ops + type: long + description: > + Number of reads during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes read during the life of the container + - name: rate + type: long + description: > + Number of current reads per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: reads + type: scaled_float + deprecated: 6.4 + description: > + Number of current reads per second + - name: write + type: group + description: > + Accumulated writes during the life of the container + fields: + - name: ops + type: long + description: > + Number of writes during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes written during the life of the container + - name: rate + type: long + description: > + Number of current writes per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: writes + type: scaled_float + deprecated: 6.4 + description: > + Number of current writes per second + - name: summary + type: group + description: > + Accumulated reads and writes during the life of the container + fields: + - name: ops + type: long + description: > + Number of I/O operations during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes read and written during the life of the container + - name: rate + type: long + description: > + Number of current operations per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: total + type: scaled_float + deprecated: 6.4 + description: > + Number of reads and writes per second + - name: memory + type: group + description: Memory metrics. + fields: + - name: stats.* + type: object + object_type: long + object_type_mapping_type: "*" + description: > + Raw memory stats from the cgroups memory.stat interface + fields: + - name: commit + type: group + description: > + Committed bytes on Windows + fields: + - name: total + type: long + format: bytes + description: > + Total bytes + - name: peak + type: long + format: bytes + description: > + Peak committed bytes on Windows + - name: private_working_set.total + type: long + format: bytes + description: > + private working sets on Windows + - name: fail.count + type: scaled_float + description: > + Fail counter. + - name: limit + type: long + format: bytes + description: > + Memory limit. + - name: rss + type: group + description: > + RSS memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory resident set size. + - name: pct + type: scaled_float + format: percent + description: > + Memory resident set size percentage. + - name: usage + type: group + description: > + Usage memory stats. + fields: + - name: max + type: long + format: bytes + description: > + Max memory usage. + - name: pct + type: scaled_float + format: percent + description: > + Memory usage percentage. + - name: total + type: long + format: bytes + description: > + Total memory usage. + - name: network + type: group + description: Network metrics. + fields: + - name: interface + type: keyword + description: > + Network interface name. + - name: inbound + type: group + description: > + Incoming network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of incoming bytes. + - name: dropped + type: long + description: > + Total number of dropped incoming packets. + - name: errors + type: long + description: > + Total errors on incoming packets. + - name: packets + type: long + description: > + Total number of incoming packets. + - name: outbound + type: group + description: > + Outgoing network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of outgoing bytes. + - name: dropped + type: long + description: > + Total number of dropped outgoing packets. + - name: errors + type: long + description: > + Total errors on outgoing packets. + - name: packets + type: long + description: > + Total number of outgoing packets. diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json new file mode 100644 index 00000000000..65219722319 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -0,0 +1,17 @@ +{ + "Cluster": "arn:aws:ecs:us-west-2:123:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", + "Family": "query-metadata-1", + "Revision": "7", + "Containers": [{ + "DockerId": "query-metadata-1", + "Name": "query-metadata", + "Image": "mreferre/eksutils", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "query-metadata", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com.amazonaws.ecs.task-definition-family": "query-metadata", + "com.amazonaws.ecs.task-definition-version": "7"} + }] +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json new file mode 100644 index 00000000000..88ddceeb6b4 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json @@ -0,0 +1,119 @@ +{ + "query-metadata-1": { + "blkio_stats": { + "io_service_bytes_recursive": [ + {"major": 202, "minor": 26368, "op": "Read", "value": 3452928}, + {"major": 202, "minor": 26368, "op": "Write", "value": 0}, + {"major": 202, "minor": 26368, "op": "Sync", "value": 3452928}, + {"major": 202, "minor": 26368, "op": "Async", "value": 0}, + {"major": 202, "minor": 26368, "op": "Total", "value": 3452928} + ], + "io_serviced_recursive": [ + {"major": 202, "minor": 26368, "op": "Read", "value": 118}, + {"major": 202, "minor": 26368, "op": "Write", "value": 0}, + {"major": 202, "minor": 26368, "op": "Sync", "value": 118}, + {"major": 202, "minor": 26368, "op": "Async", "value": 0}, + {"major": 202, "minor": 26368, "op": "Total", "value": 118} + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": []}, + "cpu_stats": { + "cpu_usage": { + "percpu_usage": [1800000000, 500000000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "total_usage": 2300000000, + "usage_in_kernelmode": 1520000000, + "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1420180000000, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "id": "query-metadata-1", + "memory_stats": { + "limit": 3937787904, + "max_usage": 15294464, + "stats": { + "active_anon": 4157440, + "active_file": 4497408, + "cache": 6000640, + "dirty": 16384, + "hierarchical_memory_limit": 2147483648, + "hierarchical_memsw_limit": 9223372036854772000, + "inactive_anon": 0, + "inactive_file": 1503232, + "mapped_file": 2183168, + "pgfault": 6668, + "pgmajfault": 52, + "pgpgin": 5925, + "pgpgout": 3445, + "rss": 4157440, + "rss_huge": 0, + "total_active_anon": 4157440, + "total_active_file": 4497408, + "total_cache": 600064, + "total_dirty": 16384, + "total_inactive_anon": 0, + "total_inactive_file": 4497408, + "total_mapped_file": 2183168, + "total_pgfault": 6668, + "total_pgmajfault": 52, + "total_pgpgin": 5925, + "total_pgpgout": 3445, + "total_rss": 4157440, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "usage": 12349440 + }, + "name": "query-metadata-1", + "network_rate_stats": { + "rx_bytes_per_sec": 9.6001425, + "tx_bytes_per_sec": 8.7001295 + }, + "networks": { + "eth0": { + "rx_bytes": 137315578, + "rx_dropped": 0, + "rx_errors": 0, + "rx_packets": 94338, + "tx_bytes": 1086811, + "tx_dropped": 0, + "tx_errors": 0, + "tx_packets": 25857 + } + }, + "num_procs": 0, + "pids_stats": { + "current": 8 + }, + "precpu_stats": { + "cpu_usage": { + "percpu_usage": [1600000000, 300000000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "total_usage": 1900000000, + "usage_in_kernelmode": 1520000000, + "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1418180000000, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "preread": "2020-10-28T01:00:08.03754086Z", + "read": "2020-10-28T01:00:09.044989605Z", + "storage_stats": {} + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go new file mode 100644 index 00000000000..0a21b9c0744 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "github.com/elastic/beats/v7/libbeat/common" + helpers "github.com/elastic/beats/v7/libbeat/common/docker" +) + +// container is a struct representation of a container +type container struct { + DockerId string + Name string + Image string + Labels map[string]string +} + +// ContainerMetadata is an struct represents container metadata +type ContainerMetadata struct { + Cluster string + TaskARN string + Family string + Revision string + Container *container +} + +func getContainerStats(c *container) *container { + return &container{ + DockerId: c.DockerId, + Image: c.Image, + Name: helpers.ExtractContainerName([]string{c.Name}), + Labels: deDotLabels(c.Labels), + } +} + +func deDotLabels(labels map[string]string) map[string]string { + outputLabels := map[string]string{} + for k, v := range labels { + // This is necessary so that ES does not interpret '.' fields as new + // nested JSON objects, and also makes this compatible with ES 2.x. + label := common.DeDot(k) + outputLabels[label] = v + } + + return outputLabels +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go b/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go new file mode 100644 index 00000000000..e10560c47cf --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "github.com/docker/docker/api/types" + + "github.com/elastic/beats/v7/metricbeat/module/docker" + "github.com/elastic/beats/v7/metricbeat/module/docker/cpu" +) + +func getCPUStats(taskStats types.StatsJSON) cpu.CPUStats { + usage := cpu.CPUUsage{Stat: &docker.Stat{Stats: taskStats}} + + return cpu.CPUStats{ + TotalUsage: usage.Total(), + TotalUsageNormalized: usage.TotalNormalized(), + UsageInKernelmode: taskStats.Stats.CPUStats.CPUUsage.UsageInKernelmode, + UsageInKernelmodePercentage: usage.InKernelMode(), + UsageInKernelmodePercentageNormalized: usage.InKernelModeNormalized(), + UsageInUsermode: taskStats.Stats.CPUStats.CPUUsage.UsageInUsermode, + UsageInUsermodePercentage: usage.InUserMode(), + UsageInUsermodePercentageNormalized: usage.InUserModeNormalized(), + SystemUsage: taskStats.Stats.CPUStats.SystemUsage, + SystemUsagePercentage: usage.System(), + SystemUsagePercentageNormalized: usage.SystemNormalized(), + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go new file mode 100644 index 00000000000..ca8cca5dca8 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -0,0 +1,164 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func eventsMapping(r mb.ReporterV2, statsList []Stats) { + for _, stats := range statsList { + r.Event(createEvent(&stats)) + } +} + +func createEvent(stats *Stats) mb.Event { + return mb.Event{ + Timestamp: time.Time(stats.Time), + RootFields: createContainerFields(stats), + MetricSetFields: common.MapStr{ + "cpu": createCPUFields(stats), + "memory": createMemoryFields(stats), + "network": createNetworkFields(stats), + "diskio": createDiskIOFields(stats), + }, + } +} + +func createContainerFields(stats *Stats) common.MapStr { + return common.MapStr{ + "container": common.MapStr{ + "id": stats.Container.DockerId, + "image": common.MapStr{ + "name": stats.Container.Image, + }, + "name": stats.Container.Name, + "labels": stats.Container.Labels, + }, + } +} + +func createCPUFields(stats *Stats) common.MapStr { + return common.MapStr{ + "core": stats.cpuStats.PerCPUUsage, + "total": common.MapStr{ + "pct": stats.cpuStats.TotalUsage, + "norm": common.MapStr{ + "pct": stats.cpuStats.TotalUsageNormalized, + }, + }, + "kernel": common.MapStr{ + "ticks": stats.cpuStats.UsageInKernelmode, + "pct": stats.cpuStats.UsageInKernelmodePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.UsageInKernelmodePercentageNormalized, + }, + }, + "user": common.MapStr{ + "ticks": stats.cpuStats.UsageInUsermode, + "pct": stats.cpuStats.UsageInUsermodePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.UsageInUsermodePercentageNormalized, + }, + }, + "system": common.MapStr{ + "ticks": stats.cpuStats.SystemUsage, + "pct": stats.cpuStats.SystemUsagePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.SystemUsagePercentageNormalized, + }, + }, + } +} + +func createMemoryFields(stats *Stats) common.MapStr { + var memoryFields common.MapStr + if stats.memoryStats.Commit+stats.memoryStats.CommitPeak+stats.memoryStats.PrivateWorkingSet > 0 { + memoryFields = common.MapStr{ + "commit": common.MapStr{ + "total": stats.memoryStats.Commit, + "peak": stats.memoryStats.CommitPeak, + }, + "private_working_set": common.MapStr{ + "total": stats.memoryStats.PrivateWorkingSet, + }, + } + } else { + memoryFields = common.MapStr{ + "stats": stats.memoryStats.Stats, + "fail": common.MapStr{ + "count": stats.memoryStats.Failcnt, + }, + "limit": stats.memoryStats.Limit, + "rss": common.MapStr{ + "total": stats.memoryStats.TotalRss, + "pct": stats.memoryStats.TotalRssP, + }, + "usage": common.MapStr{ + "total": stats.memoryStats.Usage, + "pct": stats.memoryStats.UsageP, + "max": stats.memoryStats.MaxUsage, + }, + } + } + + return memoryFields +} + +func createNetworkFields(stats *Stats) common.MapStr { + networkFields := common.MapStr{} + for _, n := range stats.networkStats { + networkFields.Put(n.NameInterface, + common.MapStr{"inbound": common.MapStr{ + "bytes": n.Total.RxBytes, + "dropped": n.Total.RxDropped, + "errors": n.Total.RxErrors, + "packets": n.Total.RxPackets, + }, + "outbound": common.MapStr{ + "bytes": n.Total.TxBytes, + "dropped": n.Total.TxDropped, + "errors": n.Total.TxErrors, + "packets": n.Total.TxPackets, + }}) + } + return networkFields +} + +func createDiskIOFields(stats *Stats) common.MapStr { + return common.MapStr{ + "reads": stats.blkioStats.reads, + "writes": stats.blkioStats.writes, + "total": stats.blkioStats.totals, + "read": common.MapStr{ + "ops": stats.blkioStats.serviced.reads, + "bytes": stats.blkioStats.servicedBytes.reads, + "rate": stats.blkioStats.reads, + "service_time": stats.blkioStats.servicedTime.reads, + "wait_time": stats.blkioStats.waitTime.reads, + "queued": stats.blkioStats.queued.reads, + }, + "write": common.MapStr{ + "ops": stats.blkioStats.serviced.writes, + "bytes": stats.blkioStats.servicedBytes.writes, + "rate": stats.blkioStats.writes, + "service_time": stats.blkioStats.servicedTime.writes, + "wait_time": stats.blkioStats.waitTime.writes, + "queued": stats.blkioStats.queued.writes, + }, + "summary": common.MapStr{ + "ops": stats.blkioStats.serviced.totals, + "bytes": stats.blkioStats.servicedBytes.totals, + "rate": stats.blkioStats.totals, + "service_time": stats.blkioStats.servicedTime.totals, + "wait_time": stats.blkioStats.waitTime.totals, + "queued": stats.blkioStats.queued.totals, + }, + } + +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go b/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go new file mode 100644 index 00000000000..c7d4791aa48 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +// BlkioRaw sums raw Blkio stats +type BlkioRaw struct { + reads uint64 + writes uint64 + totals uint64 +} + +type blkioStats struct { + reads float64 + writes float64 + totals float64 + + serviced BlkioRaw + servicedBytes BlkioRaw + servicedTime BlkioRaw + waitTime BlkioRaw + queued BlkioRaw +} + +// getBlkioStats collects diskio metrics from BlkioStats structures(not populated in Windows) +func getBlkioStats(raw types.BlkioStats) blkioStats { + return blkioStats{ + serviced: getNewStats(raw.IoServicedRecursive), + servicedBytes: getNewStats(raw.IoServiceBytesRecursive), + servicedTime: getNewStats(raw.IoServiceTimeRecursive), + waitTime: getNewStats(raw.IoWaitTimeRecursive), + queued: getNewStats(raw.IoQueuedRecursive), + } +} + +func getNewStats(blkioEntry []types.BlkioStatEntry) BlkioRaw { + stats := BlkioRaw{ + reads: 0, + writes: 0, + totals: 0, + } + + for _, myEntry := range blkioEntry { + switch myEntry.Op { + case "Write": + stats.writes += myEntry.Value + case "Read": + stats.reads += myEntry.Value + case "Total": + stats.totals += myEntry.Value + } + } + return stats +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go new file mode 100644 index 00000000000..0dfb68d9317 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +type memoryStats struct { + Failcnt uint64 + Limit uint64 + MaxUsage uint64 + TotalRss uint64 + TotalRssP float64 + Usage uint64 + UsageP float64 + //Raw stats from the cgroup subsystem + Stats map[string]uint64 + //Windows-only memory stats + Commit uint64 + CommitPeak uint64 + PrivateWorkingSet uint64 +} + +func getMemoryStats(taskStats types.StatsJSON) memoryStats { + totalRSS := taskStats.Stats.MemoryStats.Stats["total_rss"] + + return memoryStats{ + TotalRss: totalRSS, + MaxUsage: taskStats.Stats.MemoryStats.MaxUsage, + TotalRssP: float64(totalRSS) / float64(taskStats.Stats.MemoryStats.Limit), + Usage: taskStats.Stats.MemoryStats.Usage, + UsageP: float64(taskStats.Stats.MemoryStats.Usage) / float64(taskStats.Stats.MemoryStats.Limit), + Stats: taskStats.Stats.MemoryStats.Stats, + //Windows memory statistics + Commit: taskStats.Stats.MemoryStats.Commit, + CommitPeak: taskStats.Stats.MemoryStats.CommitPeak, + PrivateWorkingSet: taskStats.Stats.MemoryStats.PrivateWorkingSet, + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/network.go b/x-pack/metricbeat/module/awsfargate/task_stats/network.go new file mode 100644 index 00000000000..80cec79748a --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/network.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +type networkStats struct { + NameInterface string + Total types.NetworkStats +} + +func getNetworkStats(taskStats types.StatsJSON) []networkStats { + var networks []networkStats + for nameInterface, rawNetStats := range taskStats.Networks { + networks = append(networks, networkStats{ + NameInterface: nameInterface, + Total: rawNetStats, + }) + } + return networks +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go new file mode 100644 index 00000000000..d4e8316611b --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -0,0 +1,193 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/docker/docker/api/types" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/docker/cpu" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate" +) + +var ( + metricsetName = "task_stats" + taskStatsPath = "task/stats" + taskPath = "task" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(awsfargate.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *awsfargate.MetricSet + logger *logp.Logger + taskStatsEndpoint string + taskEndpoint string +} + +// Stats is a struct represents information regarding a container +type Stats struct { + Time common.Time + Container *container + cpuStats cpu.CPUStats + memoryStats memoryStats + networkStats []networkStats + blkioStats blkioStats +} + +// TaskMetadata is an struct represents response body from ${ECS_CONTAINER_METADATA_URI_V4}/task +type TaskMetadata struct { + Cluster string `json:"Cluster"` + TaskARN string `json:"TaskARN"` + Family string `json:"Family"` + Revision string `json:"Revision"` + Containers []*container `json:"Containers"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logger := logp.NewLogger(metricsetName) + metricSet, err := awsfargate.NewMetricSet(base) + if err != nil { + return nil, fmt.Errorf("error creating %s metricset: %w", metricsetName, err) + } + + ecsURI, ok := os.LookupEnv("ECS_CONTAINER_METADATA_URI_V4") + if !ok { + return nil, fmt.Errorf("lookup $ECS_CONTAINER_METADATA_URI_V4 failed") + } + + return &MetricSet{ + MetricSet: metricSet, + logger: logger, + taskStatsEndpoint: fmt.Sprintf("%s/%s", ecsURI, taskStatsPath), + taskEndpoint: fmt.Sprintf("%s/%s", ecsURI, taskPath), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + formattedStats, err := m.queryTaskMetadataEndpoints() + if err != nil { + err := fmt.Errorf("queryTaskMetadataEndpoints failed: %w", err) + m.logger.Error(err) + return err + } + + eventsMapping(report, formattedStats) + return nil +} + +func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { + // Get response from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats + taskStatsResp, err := http.Get(m.taskStatsEndpoint) + if err != nil { + return nil, fmt.Errorf("http.Get failed: %w", err) + } + taskStatsOutput, err := getTaskStats(taskStatsResp) + if err != nil { + return nil, fmt.Errorf("getTaskStats failed: %w", err) + } + + // Collect container metadata information from ${ECS_CONTAINER_METADATA_URI_V4}/task + taskResp, err := http.Get(m.taskEndpoint) + if err != nil { + return nil, fmt.Errorf("http.Get failed: %w", err) + } + taskOutput, err := getTask(taskResp) + if err != nil { + return nil, fmt.Errorf("getTask failed: %w", err) + } + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + return formattedStats, nil +} + +func getTaskStats(taskStatsResp *http.Response) (map[string]types.StatsJSON, error) { + taskStatsBody, err := ioutil.ReadAll(taskStatsResp.Body) + if err != nil { + return nil, fmt.Errorf("ioutil.ReadAll failed: %w", err) + } + + var taskStatsOutput map[string]types.StatsJSON + err = json.Unmarshal(taskStatsBody, &taskStatsOutput) + if err != nil { + return nil, fmt.Errorf("json.Unmarshal failed: %w", err) + } + return taskStatsOutput, nil +} + +func getTask(taskResp *http.Response) (TaskMetadata, error) { + taskBody, err := ioutil.ReadAll(taskResp.Body) + if err != nil { + return TaskMetadata{}, fmt.Errorf("ioutil.ReadAll failed: %w", err) + } + + var taskOutput TaskMetadata + err = json.Unmarshal(taskBody, &taskOutput) + if err != nil { + return TaskMetadata{}, fmt.Errorf("json.Unmarshal failed: %w", err) + } + return taskOutput, nil +} + +func getStatsList(taskStatsOutput map[string]types.StatsJSON, taskOutput TaskMetadata) []Stats { + containersInfo := map[string]ContainerMetadata{} + for _, c := range taskOutput.Containers { + // Skip ~internal~ecs~pause container + if c.Name == "~internal~ecs~pause" { + continue + } + + containerMetadata := ContainerMetadata{ + Container: c, + Family: taskOutput.Family, + TaskARN: taskOutput.TaskARN, + Cluster: taskOutput.Cluster, + Revision: taskOutput.Revision, + } + containersInfo[c.DockerId] = containerMetadata + } + + var formattedStats []Stats + for id, taskStats := range taskStatsOutput { + if cInfo, ok := containersInfo[id]; ok { + statsPerContainer := Stats{ + Time: common.Time(taskStats.Stats.Read), + Container: getContainerStats(cInfo.Container), + cpuStats: getCPUStats(taskStats), + memoryStats: getMemoryStats(taskStats), + networkStats: getNetworkStats(taskStats), + blkioStats: getBlkioStats(taskStats.BlkioStats), + } + + formattedStats = append(formattedStats, statsPerContainer) + } + } + return formattedStats +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go new file mode 100644 index 00000000000..7193f640a63 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package task_stats + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/mb/testing/flags" +) + +func TestData(t *testing.T) { + if !*flags.DataFlag { + t.Skip("skip data generation tests") + } + + config := map[string]interface{}{ + "period": "10s", + "module": "awsfargate", + "metricsets": []string{"task_stats"}, + } + + m := mbtest.NewFetcher(t, config) + + taskStatsFile, err := os.Open("./_meta/testdata/task_stats.json") + assert.NoError(t, err) + defer taskStatsFile.Close() + + byteTaskStats, err := ioutil.ReadAll(taskStatsFile) + assert.NoError(t, err) + + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(byteTaskStats)), + } + + taskFile, err := os.Open("./_meta/testdata/task.json") + assert.NoError(t, err) + defer taskStatsFile.Close() + + byteTask, err := ioutil.ReadAll(taskFile) + assert.NoError(t, err) + + byteTaskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(byteTask)), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + + taskOutput, err := getTask(byteTaskResp) + assert.NoError(t, err) + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + assert.Equal(t, 1, len(formattedStats)) + event := createEvent(&formattedStats[0]) + standardizeEvent := m.StandardizeEvent(event) + + mbtest.WriteEventToDataJSON(t, standardizeEvent, "") +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go new file mode 100644 index 00000000000..c54c3d5efdb --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + taskStatsJson = `{ + "query-metadata-1": { + "read": "2020-04-06T16:12:01.090148907Z", + "preread": "2020-04-06T16:12:01.090148907Z", + "cpu_stats": { + "cpu_usage": { + "percpu_usage": [1800000000, 500000000, 0, 0, 0, 0, 0, 0], + "total_usage": 2300000000, "usage_in_kernelmode": 1520000000, "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1420180000000, + "throttling_data": {"periods": 0, "throttled_periods": 0, "throttled_time": 0}}, + "precpu_stats": { + "cpu_usage": { + "percpu_usage": [1600000000, 300000000, 0, 0, 0, 0, 0, 0], + "total_usage": 1900000000, "usage_in_kernelmode": 1520000000, "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1418180000000, + "throttling_data": {"periods": 0, "throttled_periods": 0, "throttled_time": 0}}, + "memory_stats": {"limit": 8362348544, "usage": 4390912, "max_usage": 6488064, "stats": {"total_rss": 278528}}, + "name": "query-metadata-1", + "id": "query-metadata-1", + "networks": {"eth0": {"rx_bytes": 1802, "rx_packets": 19, "rx_errors": 0, "rx_dropped": 0, + "tx_bytes": 567, "tx_packets": 7, "tx_errors": 0, "tx_dropped": 0}} + }}` + + taskRespJson = `{ + "Cluster": "arn:aws:ecs:us-west-2:123:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", + "Family": "query-metadata-1", + "Revision": "7", + "Containers": [{ + "DockerId": "query-metadata-1", + "Name": "query-metadata", + "Image": "mreferre/eksutils", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "query-metadata", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com.amazonaws.ecs.task-definition-family": "query-metadata", + "com.amazonaws.ecs.task-definition-version": "7"} + }] + }` +) + +func TestGetTaskStats(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + assert.Equal(t, uint64(2300000000), taskStatsOutput["query-metadata-1"].CPUStats.CPUUsage.TotalUsage) +} + +func TestGetTask(t *testing.T) { + taskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + } + + taskOutput, err := getTask(taskResp) + assert.NoError(t, err) + + assert.Equal(t, "arn:aws:ecs:us-west-2:123:cluster/default", taskOutput.Cluster) + assert.Equal(t, "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", taskOutput.TaskARN) + assert.Equal(t, "query-metadata-1", taskOutput.Family) + assert.Equal(t, "7", taskOutput.Revision) + + assert.Equal(t, 1, len(taskOutput.Containers)) + assert.Equal(t, "query-metadata-1", taskOutput.Containers[0].DockerId) + assert.Equal(t, "query-metadata", taskOutput.Containers[0].Name) + assert.Equal(t, "mreferre/eksutils", taskOutput.Containers[0].Image) + assert.Equal(t, 5, len(taskOutput.Containers[0].Labels)) +} + +func TestGetStatsList(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + + taskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + } + + taskOutput, err := getTask(taskResp) + assert.NoError(t, err) + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + assert.Equal(t, 1, len(formattedStats)) +} + +func TestGetCPUStats(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + assert.Equal(t, 1, len(taskStatsOutput)) + + cpuStats := getCPUStats(taskStatsOutput["query-metadata-1"]) + assert.Equal(t, 0.4, cpuStats.TotalUsage) + assert.Equal(t, 0.2, cpuStats.TotalUsageNormalized) +} diff --git a/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go b/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go index 961827469dd..1486c9b14c0 100644 --- a/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go +++ b/x-pack/metricbeat/module/cloudfoundry/cloudfoundry.go @@ -22,19 +22,30 @@ func init() { } type Module interface { + mb.Module RunCounterReporter(mb.PushReporterV2) RunContainerReporter(mb.PushReporterV2) RunValueReporter(mb.PushReporterV2) } func newModule(base mb.BaseModule) (mb.Module, error) { + factory := func(cfg *cfcommon.Config, name string, log *logp.Logger) CloudfoundryHub { + return &HubAdapter{cfcommon.NewHub(cfg, name, log)} + } + return NewModuleWithHubFactory(base, factory) +} + +type hubFactory func(cfg *cfcommon.Config, name string, log *logp.Logger) CloudfoundryHub + +// NewModuleWithHubFactory initializes a module with a hub created with a hub factory +func NewModuleWithHubFactory(base mb.BaseModule, hubFactory hubFactory) (mb.Module, error) { var cfg cfcommon.Config if err := base.UnpackConfig(&cfg); err != nil { return nil, err } log := logp.NewLogger("cloudfoundry") - hub := cfcommon.NewHub(&cfg, "metricbeat", log) + hub := hubFactory(&cfg, "metricbeat", log) switch cfg.Version { case cfcommon.ConsumerVersionV1: diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container.go b/x-pack/metricbeat/module/cloudfoundry/container/container.go index 4f8c6227103..50287ef8c49 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/container.go +++ b/x-pack/metricbeat/module/cloudfoundry/container/container.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -41,5 +42,23 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Run method provides the module with a reporter with which events can be reported. func (m *MetricSet) Run(reporter mb.PushReporterV2) { - m.mod.RunContainerReporter(reporter) + m.mod.RunContainerReporter(&containerReporter{reporter, m.Logger()}) +} + +type containerReporter struct { + mb.PushReporterV2 + + logger *logp.Logger +} + +func (r *containerReporter) Event(event mb.Event) bool { + cpuPctKey := "cloudfoundry.container.cpu.pct" + found, err := cloudfoundry.HasNonNumericFloat(event.RootFields, cpuPctKey) + if err != nil { + r.logger.Debugf("Unexpected failure while checking for non-numeric values: %v", err) + } + if found { + event.RootFields.Delete(cpuPctKey) + } + return r.PushReporterV2.Event(event) } diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_test.go new file mode 100644 index 00000000000..46b1206f1b3 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/container/container_test.go @@ -0,0 +1,163 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package container + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudfoundry/sonde-go/events" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func init() { + if err := mb.Registry.AddModule("cloudfoundrytest", mtest.NewModuleMock); err != nil { + panic(err) + } + mb.Registry.MustAddMetricSet("cloudfoundrytest", "test", newTestMetricSet, + mb.WithHostParser(parse.EmptyHostParser), + mb.DefaultMetricSet(), + ) +} + +func newTestMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { + return New(base) +} + +func TestMetricSet(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("cloudfoundry")) + + config := map[string]interface{}{ + "module": "cloudfoundrytest", + "client_id": "dummy", + "client_secret": "dummy", + "api_address": "dummy", + "shard_id": "dummy", + } + + ms := mbtest.NewPushMetricSetV2(t, config) + hub := ms.Module().(*mtest.ModuleMock).Hub + + go func() { + hub.SendEnvelope(containerMetricsEnvelope(containerMetrics{app: "1234", memory: 1024, cpupct: 12.34})) + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + require.NotEmpty(t, events) + + expectedFields := common.MapStr{ + "cloudfoundry.app.id": "1234", + "cloudfoundry.container.cpu.pct": float64(12.34), + "cloudfoundry.container.disk.bytes": uint64(0), + "cloudfoundry.container.disk.quota.bytes": uint64(0), + "cloudfoundry.container.instance_index": int32(0), + "cloudfoundry.container.memory.bytes": uint64(1024), + "cloudfoundry.container.memory.quota.bytes": uint64(0), + "cloudfoundry.envelope.deployment": "test", + "cloudfoundry.envelope.index": "index", + "cloudfoundry.envelope.ip": "127.0.0.1", + "cloudfoundry.envelope.job": "test", + "cloudfoundry.envelope.origin": "test", + "cloudfoundry.type": "container", + } + require.Equal(t, expectedFields, events[0].RootFields.Flatten()) +} + +func TestMetricValuesAreNumbers(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("cloudfoundry")) + + config := map[string]interface{}{ + "module": "cloudfoundrytest", + "client_id": "dummy", + "client_secret": "dummy", + "api_address": "dummy", + "shard_id": "dummy", + } + + ms := mbtest.NewPushMetricSetV2(t, config) + hub := ms.Module().(*mtest.ModuleMock).Hub + + go func() { + hub.SendEnvelope(containerMetricsEnvelope(containerMetrics{app: "0000", memory: 1024, cpupct: math.NaN()})) + hub.SendEnvelope(containerMetricsEnvelope(containerMetrics{app: "1234", memory: 1024, cpupct: 12.34})) + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 2, ms) + require.NotEmpty(t, events) + + for _, e := range events { + memory, err := e.RootFields.GetValue("cloudfoundry.container.memory.bytes") + if assert.NoError(t, err, "checking memory") { + assert.Equal(t, uint64(1024), memory.(uint64)) + } + + app, err := e.RootFields.GetValue("cloudfoundry.app.id") + require.NoError(t, err, "getting app id") + + cpuPctKey := "cloudfoundry.container.cpu.pct" + switch app { + case "0000": + _, err := e.RootFields.GetValue(cpuPctKey) + require.Error(t, err, "non-numeric metric shouldn't be there") + case "1234": + v, err := e.RootFields.GetValue(cpuPctKey) + if assert.NoError(t, err, "checking cpu pct") { + assert.Equal(t, 12.34, v.(float64)) + } + default: + t.Errorf("unexpected app: %s", app) + } + } +} + +type containerMetrics struct { + app string + instance int32 + cpupct float64 + memory uint64 + disk uint64 + memoryQuota uint64 + diskQuota uint64 +} + +func containerMetricsEnvelope(metrics containerMetrics) *events.Envelope { + eventType := events.Envelope_ContainerMetric + origin := "test" + deployment := "test" + job := "test" + ip := "127.0.0.1" + index := "index" + timestamp := time.Now().Unix() + return &events.Envelope{ + EventType: &eventType, + Timestamp: ×tamp, + Origin: &origin, + Deployment: &deployment, + Job: &job, + Ip: &ip, + Index: &index, + ContainerMetric: &events.ContainerMetric{ + ApplicationId: &metrics.app, + InstanceIndex: &metrics.instance, + CpuPercentage: &metrics.cpupct, + MemoryBytes: &metrics.memory, + DiskBytes: &metrics.disk, + MemoryBytesQuota: &metrics.memoryQuota, + DiskBytesQuota: &metrics.diskQuota, + }, + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_test.go new file mode 100644 index 00000000000..a315709878e --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/counter/counter_test.go @@ -0,0 +1,96 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package counter + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/cloudfoundry/sonde-go/events" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func init() { + if err := mb.Registry.AddModule("cloudfoundrytest", mtest.NewModuleMock); err != nil { + panic(err) + } + mb.Registry.MustAddMetricSet("cloudfoundrytest", "test", newTestMetricSet, + mb.WithHostParser(parse.EmptyHostParser), + mb.DefaultMetricSet(), + ) +} + +func newTestMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { + return New(base) +} + +func TestMetricSet(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("cloudfoundry")) + + config := map[string]interface{}{ + "module": "cloudfoundrytest", + "client_id": "dummy", + "client_secret": "dummy", + "api_address": "dummy", + "shard_id": "dummy", + } + + ms := mbtest.NewPushMetricSetV2(t, config) + hub := ms.Module().(*mtest.ModuleMock).Hub + + go func() { + hub.SendEnvelope(counterMetricEnvelope("requests", 1234, 123)) + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + require.NotEmpty(t, events) + + expectedFields := common.MapStr{ + "cloudfoundry.counter.delta": uint64(123), + "cloudfoundry.counter.name": "requests", + "cloudfoundry.counter.total": uint64(1234), + "cloudfoundry.envelope.deployment": "test", + "cloudfoundry.envelope.index": "index", + "cloudfoundry.envelope.ip": "127.0.0.1", + "cloudfoundry.envelope.job": "test", + "cloudfoundry.envelope.origin": "test", + "cloudfoundry.type": "counter", + } + require.Equal(t, expectedFields, events[0].RootFields.Flatten()) +} + +func counterMetricEnvelope(name string, total uint64, delta uint64) *events.Envelope { + eventType := events.Envelope_CounterEvent + origin := "test" + deployment := "test" + job := "test" + ip := "127.0.0.1" + index := "index" + timestamp := time.Now().Unix() + return &events.Envelope{ + EventType: &eventType, + Timestamp: ×tamp, + Origin: &origin, + Deployment: &deployment, + Job: &job, + Ip: &ip, + Index: &index, + CounterEvent: &events.CounterEvent{ + Name: &name, + Total: &total, + Delta: &delta, + }, + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/hub.go b/x-pack/metricbeat/module/cloudfoundry/hub.go new file mode 100644 index 00000000000..5057a7bdbc5 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/hub.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudfoundry + +import ( + "context" + + cfcommon "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" +) + +// DopplerConsumer is the interface that a Doppler Consumer must implement for the Cloud Foundry module. +type DopplerConsumer interface { + Run() + Stop() +} + +// RlpListener is the interface that a RLP listener must implement for the Cloud Foundry module. +type RlpListener interface { + Start(context.Context) + Stop() +} + +// CloudfoundryHub is the interface that a Hub must implement for the Cloud Foundry module. +type CloudfoundryHub interface { + DopplerConsumer(cfcommon.DopplerCallbacks) (DopplerConsumer, error) + RlpListener(cfcommon.RlpListenerCallbacks) (RlpListener, error) +} + +// HubAdapter adapt a cloudfoundry Hub to the hub expected by the metricbeat module. +// This adaptation is needed to return different but compatible types, so the Hub can be mocked. +type HubAdapter struct { + hub *cfcommon.Hub +} + +func (h *HubAdapter) DopplerConsumer(cbs cfcommon.DopplerCallbacks) (DopplerConsumer, error) { + return h.hub.DopplerConsumer(cbs) +} + +func (h *HubAdapter) RlpListener(cbs cfcommon.RlpListenerCallbacks) (RlpListener, error) { + return h.hub.RlpListener(cbs) +} diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/modulemock.go b/x-pack/metricbeat/module/cloudfoundry/mtest/modulemock.go new file mode 100644 index 00000000000..22b5260fd66 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/mtest/modulemock.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mtest + +import ( + "fmt" + + "github.com/cloudfoundry/sonde-go/events" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + cfcommon "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" +) + +// ModuleMock is a Module with a mocked hub +type ModuleMock struct { + cloudfoundry.Module + + Hub *HubMock +} + +// NewModuleMock creates a mocked module. It contains a mocked hub that can be used to +// send envelopes for testing. +func NewModuleMock(base mb.BaseModule) (mb.Module, error) { + module := ModuleMock{} + factory := func(*cfcommon.Config, string, *logp.Logger) cloudfoundry.CloudfoundryHub { + if module.Hub == nil { + module.Hub = NewHubMock() + } + return module.Hub + } + m, err := cloudfoundry.NewModuleWithHubFactory(base, factory) + if err != nil { + return nil, err + } + + module.Module = m.(cloudfoundry.Module) + return &module, nil +} + +// HubMock is a mocked hub, it can be used to send envelopes for testing. +type HubMock struct { + envelopes chan *events.Envelope +} + +// NewHubMock creates a mocked hub, it cannot be shared between metricsets. +func NewHubMock() *HubMock { + return &HubMock{ + envelopes: make(chan *events.Envelope), + } +} + +// SendEnvelope is the main method to be used on testing, it sends an envelope through the hub. +func (h *HubMock) SendEnvelope(envelope *events.Envelope) { + h.envelopes <- envelope +} + +// DopplerConsumer creates a doppler consumer for testing, this consumer receives the events +// sent with `SendEnvelope()`. +func (h *HubMock) DopplerConsumer(cbs cfcommon.DopplerCallbacks) (cloudfoundry.DopplerConsumer, error) { + return &MockedDopplerConsumer{h, cbs}, nil +} + +// RlpListener creates a RLP listener for testing, this consumer receives the events +// sent with `SendEnvelope()`. +func (h *HubMock) RlpListener(cbs cfcommon.RlpListenerCallbacks) (cloudfoundry.RlpListener, error) { + return nil, fmt.Errorf("mocked hub doesn't support RLP yet: not implemented") +} + +// MokedDopplerConsumer is a mocked doppler consumer, it receives events sent through a mocked hub. +// It only supports the "Metrics" callback. +type MockedDopplerConsumer struct { + hub *HubMock + cbs cfcommon.DopplerCallbacks +} + +// Run runs the doppler consumer. +// Only supports the metrics callback, what is enough for Metricbeat. +// To generalize it a dispatching mechanism should be implemented. +func (c *MockedDopplerConsumer) Run() { + go func() { + for envelope := range c.hub.envelopes { + c.cbs.Metric(cfcommon.EnvelopeToEvent(envelope)) + } + }() +} + +// Stop stops the doppler consumer and the hub it uses. +func (c *MockedDopplerConsumer) Stop() { + close(c.hub.envelopes) +} diff --git a/x-pack/metricbeat/module/cloudfoundry/util.go b/x-pack/metricbeat/module/cloudfoundry/util.go new file mode 100644 index 00000000000..4775ac539be --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/util.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudfoundry + +import ( + "fmt" + "math" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// HasNonNumericFloat checks if an event has a non-numeric float in the specific key. +// It returns false and an error if the key cannot be found in the event +func HasNonNumericFloat(event common.MapStr, key string) (bool, error) { + v, err := event.GetValue(key) + if err != nil { + return false, fmt.Errorf("getting value for key %s: %w", key, err) + } + + if v, ok := v.(float64); ok && (math.IsNaN(v) || math.IsInf(v, 0)) { + return true, nil + } + + return false, nil +} diff --git a/x-pack/metricbeat/module/cloudfoundry/util_test.go b/x-pack/metricbeat/module/cloudfoundry/util_test.go new file mode 100644 index 00000000000..3570607707f --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/util_test.go @@ -0,0 +1,70 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudfoundry + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func TestHasNonNumericFloat(t *testing.T) { + type caseKey struct { + key string + expectedFound bool + expectedErr bool + } + cases := []struct { + title string + event common.MapStr + keys []caseKey + }{ + { + title: "Empty event", + event: common.MapStr{}, + keys: []caseKey{ + {"", false, true}, + {"somekey", false, true}, + }, + }, + { + title: "Event with non-numeric values", + event: common.MapStr{ + "someobject": common.MapStr{ + "inf": math.Inf(1), + "nan": math.NaN(), + "number": int64(42), + "float": float64(42), + }, + }, + keys: []caseKey{ + {"", false, true}, + {"someobject", false, false}, + {"someobject.inf", true, false}, + {"someobject.nan", true, false}, + {"someobject.number", false, false}, + {"someobject.float", false, false}, + {"someobject.notexists", false, true}, + }, + }, + } + + for _, c := range cases { + for _, k := range c.keys { + t.Run(c.title+"/"+k.key, func(t *testing.T) { + found, err := HasNonNumericFloat(c.event, k.key) + assert.Equal(t, k.expectedFound, found, "key has numeric float") + if k.expectedErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/v1.go b/x-pack/metricbeat/module/cloudfoundry/v1.go index 7d9daf24673..db7f6b500fa 100644 --- a/x-pack/metricbeat/module/cloudfoundry/v1.go +++ b/x-pack/metricbeat/module/cloudfoundry/v1.go @@ -17,13 +17,13 @@ type ModuleV1 struct { log *logp.Logger running atomic.Bool - consumer *cfcommon.DopplerConsumer + consumer DopplerConsumer events chan cfcommon.Event subscriptions chan subscription } -func newModuleV1(base mb.BaseModule, hub *cfcommon.Hub, log *logp.Logger) (*ModuleV1, error) { +func newModuleV1(base mb.BaseModule, hub CloudfoundryHub, log *logp.Logger) (*ModuleV1, error) { m := ModuleV1{ BaseModule: base, log: log, diff --git a/x-pack/metricbeat/module/cloudfoundry/v2.go b/x-pack/metricbeat/module/cloudfoundry/v2.go index 5cf7de6c103..d2987f3c401 100644 --- a/x-pack/metricbeat/module/cloudfoundry/v2.go +++ b/x-pack/metricbeat/module/cloudfoundry/v2.go @@ -18,8 +18,8 @@ type ModuleV2 struct { log *logp.Logger - hub *cfcommon.Hub - listener *cfcommon.RlpListener + hub CloudfoundryHub + listener RlpListener listenerLock sync.Mutex counterReporter mb.PushReporterV2 @@ -27,7 +27,7 @@ type ModuleV2 struct { containerReporter mb.PushReporterV2 } -func newModuleV2(base mb.BaseModule, hub *cfcommon.Hub, log *logp.Logger) (mb.Module, error) { +func newModuleV2(base mb.BaseModule, hub CloudfoundryHub, log *logp.Logger) (mb.Module, error) { // early check that listener can be created _, err := hub.RlpListener(cfcommon.RlpListenerCallbacks{}) if err != nil { diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value.go b/x-pack/metricbeat/module/cloudfoundry/value/value.go index 55cb6ca689a..de2b3d60a69 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/value.go +++ b/x-pack/metricbeat/module/cloudfoundry/value/value.go @@ -7,9 +7,9 @@ package value import ( "fmt" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" - + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" ) // init registers the MetricSet with the central registry. @@ -41,5 +41,23 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Run method provides the module with a reporter with which events can be reported. func (m *MetricSet) Run(reporter mb.PushReporterV2) { - m.mod.RunValueReporter(reporter) + m.mod.RunValueReporter(&valueReporter{reporter, m.Logger()}) +} + +type valueReporter struct { + mb.PushReporterV2 + + logger *logp.Logger +} + +func (r *valueReporter) Event(event mb.Event) bool { + found, err := cloudfoundry.HasNonNumericFloat(event.RootFields, "cloudfoundry.value.value") + if err != nil { + r.logger.Debugf("Unexpected failure while checking for non-numeric values: %v", err) + } + if found { + r.logger.Debugf("Ignored event with float value that is not a number: %+v", event) + return true + } + return r.PushReporterV2.Event(event) } diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_test.go new file mode 100644 index 00000000000..2003388e228 --- /dev/null +++ b/x-pack/metricbeat/module/cloudfoundry/value/value_test.go @@ -0,0 +1,131 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package value + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudfoundry/sonde-go/events" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry/mtest" +) + +func init() { + if err := mb.Registry.AddModule("cloudfoundrytest", mtest.NewModuleMock); err != nil { + panic(err) + } + mb.Registry.MustAddMetricSet("cloudfoundrytest", "test", newTestMetricSet, + mb.WithHostParser(parse.EmptyHostParser), + mb.DefaultMetricSet(), + ) +} + +func newTestMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { + return New(base) +} + +func TestMetricSet(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("cloudfoundry")) + + config := map[string]interface{}{ + "module": "cloudfoundrytest", + "client_id": "dummy", + "client_secret": "dummy", + "api_address": "dummy", + "shard_id": "dummy", + } + + ms := mbtest.NewPushMetricSetV2(t, config) + hub := ms.Module().(*mtest.ModuleMock).Hub + + go func() { + hub.SendEnvelope(valueMetricEnvelope("duration", 12.34, "ms")) + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + require.NotEmpty(t, events) + + expectedFields := common.MapStr{ + "cloudfoundry.envelope.deployment": "test", + "cloudfoundry.envelope.index": "index", + "cloudfoundry.envelope.ip": "127.0.0.1", + "cloudfoundry.envelope.job": "test", + "cloudfoundry.envelope.origin": "test", + "cloudfoundry.type": "value", + "cloudfoundry.value.name": "duration", + "cloudfoundry.value.unit": "ms", + "cloudfoundry.value.value": float64(12.34), + } + require.Equal(t, expectedFields, events[0].RootFields.Flatten()) +} + +func TestValuesAreNumbers(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("cloudfoundry")) + + config := map[string]interface{}{ + "module": "cloudfoundrytest", + "client_id": "dummy", + "client_secret": "dummy", + "api_address": "dummy", + "shard_id": "dummy", + } + + ms := mbtest.NewPushMetricSetV2(t, config) + hub := ms.Module().(*mtest.ModuleMock).Hub + + go func() { + hub.SendEnvelope(valueMetricEnvelope("duration", math.NaN(), "ms")) + hub.SendEnvelope(valueMetricEnvelope("duration", 12.34, "ms")) + hub.SendEnvelope(valueMetricEnvelope("duration", math.Inf(1), "ms")) + hub.SendEnvelope(valueMetricEnvelope("duration", 34.56, "ms")) + }() + + events := mbtest.RunPushMetricSetV2(10*time.Second, 2, ms) + require.NotEmpty(t, events) + + for _, e := range events { + value, err := e.RootFields.GetValue("cloudfoundry.value.value") + if assert.NoError(t, err) { + assert.False(t, math.IsNaN(value.(float64))) + assert.False(t, math.IsInf(value.(float64), 0)) + } + } +} + +func valueMetricEnvelope(name string, value float64, unit string) *events.Envelope { + eventType := events.Envelope_ValueMetric + origin := "test" + deployment := "test" + job := "test" + ip := "127.0.0.1" + index := "index" + timestamp := time.Now().Unix() + return &events.Envelope{ + EventType: &eventType, + Timestamp: ×tamp, + Origin: &origin, + Deployment: &deployment, + Job: &job, + Ip: &ip, + Index: &index, + ValueMetric: &events.ValueMetric{ + Name: &name, + Value: &value, + Unit: &unit, + }, + } +} diff --git a/x-pack/metricbeat/modules.d/awsfargate.yml.disabled b/x-pack/metricbeat/modules.d/awsfargate.yml.disabled new file mode 100644 index 00000000000..b2b91f06ee4 --- /dev/null +++ b/x-pack/metricbeat/modules.d/awsfargate.yml.disabled @@ -0,0 +1,7 @@ +# Module: awsfargate +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-awsfargate.html + +- module: awsfargate + period: 10s + metricsets: + - task_stats