From 0fb4ae21a77cf4df08898d177218eeaadc864899 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 18 Feb 2021 17:43:30 +0000 Subject: [PATCH 01/12] [CI] enable x-pack/heartbeat in the CI (#23873) --- x-pack/heartbeat/Jenkinsfile.yml | 57 ++++++++++++++++++++++++++++++++ x-pack/heartbeat/magefile.go | 2 ++ 2 files changed, 59 insertions(+) diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml index b715d109a12..3cd24f2c2b7 100644 --- a/x-pack/heartbeat/Jenkinsfile.yml +++ b/x-pack/heartbeat/Jenkinsfile.yml @@ -13,6 +13,63 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-18" ## default label for all the stages stages: + Lint: + make: | + make -C x-pack/heartbeat check; + make -C x-pack/heartbeat update; + make -C heartbeat check; + make -C heartbeat update; + make check-no-changes; + build: + mage: "mage build test" + macos: + mage: "mage build test" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Override the top-level when. + comments: + - "/test x-pack/heartbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + branches: true ## for all the branches + tags: true ## for all the tags +# TODO: there are windows test failures already reported +# https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 +# waiting for being fixed. +# windows: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-2019" +# windows-2016: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-2016" +# windows-2012: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-2012-r2" +# windows-10: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-10" +# windows-2008: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-2008-r2" +# windows-8: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-8" +# windows-7: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-7" +# windows-7-32: +# mage: "mage build test" +# platforms: ## override default labels in this specific stage. +# - "windows-7-32-bit" packaging-linux: packaging-linux: "mage package" e2e: diff --git a/x-pack/heartbeat/magefile.go b/x-pack/heartbeat/magefile.go index 356d5e55750..f9c9c54b724 100644 --- a/x-pack/heartbeat/magefile.go +++ b/x-pack/heartbeat/magefile.go @@ -22,6 +22,8 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/build" + // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests" // mage:import From 96346ad67cae6305cdd2d7a80a0caa457999b9e9 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 19 Feb 2021 09:51:29 +0000 Subject: [PATCH 02/12] [CI] Run if manual UI (#24116) --- .ci/apm-beats-update.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index f8da89d6fa6..515e7aeb2bb 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -86,10 +86,9 @@ pipeline { branch "v\\d?" tag "v\\d+\\.\\d+\\.\\d+*" allOf { - expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } + expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() || isUserTrigger() } changeRequest() } - } } steps { From ea5d413d6e2a620e65311f4cfc0fec678e62d6f4 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 19 Feb 2021 13:56:36 +0200 Subject: [PATCH 03/12] Add logrotation section on Running Filebeat on k8s (#24120) --- filebeat/docs/running-on-kubernetes.asciidoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/filebeat/docs/running-on-kubernetes.asciidoc b/filebeat/docs/running-on-kubernetes.asciidoc index a04fde00ef1..1ea7cc6e6d3 100644 --- a/filebeat/docs/running-on-kubernetes.asciidoc +++ b/filebeat/docs/running-on-kubernetes.asciidoc @@ -233,3 +233,12 @@ annotations: co.elastic.logs.json-logging/json.add_error_key: "true" co.elastic.logs.json-logging/json.message_key: "message" ------------------------------------------------ + +[float] +==== Logrotation + +According to https://kubernetes.io/docs/concepts/cluster-administration/logging/#logging-at-the-node-level[kubernetes documentation] +_Kubernetes is not responsible for rotating logs, but rather a deployment tool should set up a solution to address that_. +Different logrotation strategies can cause issues that might make Filebeat losing events or even duplicating events. +Users can find more information about Filebeat's logrotation best practises at Filebeat's +<> From 0616c04c204d7daf729eaaf45b8640ade2119772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 19 Feb 2021 13:17:37 +0100 Subject: [PATCH 04/12] Set Elastic licence type for APM server Beats update job (#24122) ## What does this PR do? Adds a new licence override temporarily when testing if Beats can be updated in APM Server. ## Why is it important? We get the following error in the Beats update job: ``` Failed to detect licences: failed to detect licence type of /var/lib/jenkins/workspace/Beats_apm-beats-update_master/src/github.com/elastic/beats-local from /var/lib/jenkins/workspace/Beats_apm-beats-update_master/src/github.com/elastic/beats-local/LICENSE.txt: failed to detect licence type of /var/lib/jenkins/workspace/Beats_apm-beats-update_master/src/github.com/elastic/beats-local/LICENSE.txt ``` --- .ci/apm-beats-update.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index 515e7aeb2bb..1d1099ebb03 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -126,6 +126,7 @@ def beatsUpdate() { git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*" go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local + echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json make update git commit -a -m beats-update From a7f820f7a6a7ef158d76bf5c53efe4a00159a882 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 19 Feb 2021 16:45:07 +0100 Subject: [PATCH 05/12] [Ingest Manager] Fix: Successfully installed and enrolled agent running standalone (#24128) [Ingest Manager] Fix: Successfully installed and enrolled agent running standalone (#24128) --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go | 6 ------ x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 4 ---- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index b0b5066d27d..8f5efd70aa3 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -35,6 +35,7 @@ - Fixed make status readable in the log. {pull}23849[23849] - Windows agent doesn't uninstall with a lowercase `c:` drive in the path {pull}23998[23998] - Fix reloading of log level for services {pull}[24055]24055 +- Fix: Successfully installed and enrolled agent running standalone{pull}[24128]24128 ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go index 82d996bd620..22d3f8625d0 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go @@ -84,7 +84,6 @@ type EnrollCmdOption struct { Staging string FleetServerConnStr string FleetServerPolicyID string - NoRestart bool } func (e *EnrollCmdOption) kibanaConfig() (*kibana.Config, error) { @@ -178,7 +177,6 @@ func (c *EnrollCmd) Execute(ctx context.Context) error { // enroll should use localhost as fleet-server is now running // it must also restart c.options.URL = "http://localhost:8000" - c.options.NoRestart = false } err := c.enrollWithBackoff(ctx) @@ -186,10 +184,6 @@ func (c *EnrollCmd) Execute(ctx context.Context) error { return errors.New(err, "fail to enroll") } - if c.options.NoRestart { - return nil - } - if c.daemonReload(ctx) != nil { c.log.Info("Elastic Agent might not be running; unable to trigger restart") } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index 58c99306e71..cd4b12ef422 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -38,7 +38,6 @@ func newEnrollCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStr addEnrollFlags(cmd) cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") - cmd.Flags().Bool("no-restart", false, "Skip restarting the currently running daemon") // used by install command cmd.Flags().BoolP("from-install", "", false, "Set by install command to signal this was executed from install") @@ -141,11 +140,9 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args } } - noRestart, _ := cmd.Flags().GetBool("no-restart") force, _ := cmd.Flags().GetBool("force") if fromInstall { force = true - noRestart = true } // prompt only when it is not forced and is already enrolled @@ -192,7 +189,6 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args Staging: staging, FleetServerConnStr: fServer, FleetServerPolicyID: fPolicy, - NoRestart: noRestart, } c, err := application.NewEnrollCmd( From 3319f5b6c5bdbf65ea37ee9e4aa086a8a0aebee0 Mon Sep 17 00:00:00 2001 From: gitck Date: Fri, 19 Feb 2021 18:30:20 +0100 Subject: [PATCH 06/12] [Enhancement] Add RotateOnStartup feature flag for file output (#19347) * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add change again * add RotateOnStartup feature enhance file output to also be able to disable file rotation on startup, which is enabled by default (see rotator) * document change * add change again * add RotateOnStartup feature * update reference.yml files (make update) Co-authored-by: Claus Klammer --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 +++- filebeat/filebeat.reference.yml | 4 +++- heartbeat/heartbeat.reference.yml | 4 +++- journalbeat/journalbeat.reference.yml | 4 +++- .../config/output-file.reference.yml.tmpl | 3 +++ libbeat/outputs/fileout/config.go | 20 ++++++++++--------- libbeat/outputs/fileout/docs/fileout.asciidoc | 5 +++++ libbeat/outputs/fileout/file.go | 1 + metricbeat/metricbeat.reference.yml | 4 +++- packetbeat/packetbeat.reference.yml | 4 +++- winlogbeat/winlogbeat.reference.yml | 4 +++- x-pack/auditbeat/auditbeat.reference.yml | 4 +++- x-pack/filebeat/filebeat.reference.yml | 4 +++- x-pack/heartbeat/heartbeat.reference.yml | 4 +++- x-pack/metricbeat/metricbeat.reference.yml | 4 +++- x-pack/packetbeat/packetbeat.reference.yml | 4 +++- x-pack/winlogbeat/winlogbeat.reference.yml | 4 +++- 18 files changed, 60 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 641f7a769a9..c5dfe0cac9c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -572,6 +572,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add keystore support for autodiscover static configurations. {pull]16306[16306] - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607] - Add support for multiple sets of hints on autodiscover {pull}18883[18883] +- Add config option `rotate_on_startup` to file output {issue}19150[19150] {pull}19347[19347] - Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181] - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] - Add support for DNS over TLS for the dns_processor. {pull}19321[19321] diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index cc8cfdba2db..8ceb7914f04 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1103,7 +1103,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index e232640ffd0..38ecc9fb0b5 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1995,7 +1995,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 85e00f43342..d1373f73750 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1281,7 +1281,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 35c6dbb4c05..8114260a853 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -1046,7 +1046,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/libbeat/_meta/config/output-file.reference.yml.tmpl b/libbeat/_meta/config/output-file.reference.yml.tmpl index 2c383444107..7f7ff9998b3 100644 --- a/libbeat/_meta/config/output-file.reference.yml.tmpl +++ b/libbeat/_meta/config/output-file.reference.yml.tmpl @@ -31,3 +31,6 @@ # Permissions to use for file creation. The default is 0600. #permissions: 0600 + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true \ No newline at end of file diff --git a/libbeat/outputs/fileout/config.go b/libbeat/outputs/fileout/config.go index 4b83cdbab97..8f1061db184 100644 --- a/libbeat/outputs/fileout/config.go +++ b/libbeat/outputs/fileout/config.go @@ -25,19 +25,21 @@ import ( ) type config struct { - Path string `config:"path"` - Filename string `config:"filename"` - RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` - NumberOfFiles uint `config:"number_of_files"` - Codec codec.Config `config:"codec"` - Permissions uint32 `config:"permissions"` + Path string `config:"path"` + Filename string `config:"filename"` + RotateEveryKb uint `config:"rotate_every_kb" validate:"min=1"` + NumberOfFiles uint `config:"number_of_files"` + Codec codec.Config `config:"codec"` + Permissions uint32 `config:"permissions"` + RotateOnStartup bool `config:"rotate_on_startup"` } var ( defaultConfig = config{ - NumberOfFiles: 7, - RotateEveryKb: 10 * 1024, - Permissions: 0600, + NumberOfFiles: 7, + RotateEveryKb: 10 * 1024, + Permissions: 0600, + RotateOnStartup: true, } ) diff --git a/libbeat/outputs/fileout/docs/fileout.asciidoc b/libbeat/outputs/fileout/docs/fileout.asciidoc index 6922f8f7142..a0979f92ef5 100644 --- a/libbeat/outputs/fileout/docs/fileout.asciidoc +++ b/libbeat/outputs/fileout/docs/fileout.asciidoc @@ -22,6 +22,7 @@ output.file: #rotate_every_kb: 10000 #number_of_files: 7 #permissions: 0600 + #rotate_on_startup: true ------------------------------------------------------------------------------ ==== Configuration options @@ -61,6 +62,10 @@ The number of files must be between 2 and 1024. The default is 7. Permissions to use for file creation. The default is 0600. +===== `rotate_on_startup` + +If the output file already exists on startup, immediately rotate it and start writing to a new file instead of appending to the existing one. Defaults to true. + ===== `codec` Output codec configuration. If the `codec` section is missing, events will be json encoded. diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index 2c2f5216294..ed70797dcb6 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -87,6 +87,7 @@ func (out *fileOutput) init(beat beat.Info, c config) error { file.MaxSizeBytes(c.RotateEveryKb*1024), file.MaxBackups(c.NumberOfFiles), file.Permissions(os.FileMode(c.Permissions)), + file.RotateOnStartup(c.RotateOnStartup), file.WithLogger(logp.NewLogger("rotator").With(logp.Namespace("rotator"))), ) if err != nil { diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index b2cc7ce7c1b..685dac86452 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1892,7 +1892,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 9f25343877f..7a4eb765660 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1598,7 +1598,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 7b98270f0bf..d6a610c1292 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1026,7 +1026,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 44b58a736e1..274bc3f3b33 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1159,7 +1159,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index db79f9abb8c..b714d4a82ba 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3908,7 +3908,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 85e00f43342..d1373f73750 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1281,7 +1281,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 50127225c63..be76277068f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2393,7 +2393,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 9f25343877f..7a4eb765660 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1598,7 +1598,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index a9cb100ce33..dcb17bb6932 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1069,7 +1069,9 @@ output.elasticsearch: # Permissions to use for file creation. The default is 0600. #permissions: 0600 - + + # Configure automatic file rotation on every startup. The default is true. + #rotate_on_startup: true # ------------------------------- Console Output ------------------------------- #output.console: # Boolean flag to enable or disable the output module. From f394755de2980b24af45d138ac3e291550d5727e Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Fri, 19 Feb 2021 19:38:51 +0100 Subject: [PATCH 07/12] [Filebeat] Adding fixes to the TI module (#24133) * cleaning up TI module, adding safer config options, updating docs and fixing the MISP tag copy painless script * updating otx pipeline to remove specific null value * fixing grok pattern in MISP to fetch hash values --- filebeat/docs/modules/threatintel.asciidoc | 4 +- x-pack/filebeat/filebeat.reference.yml | 28 +-- .../module/threatintel/_meta/config.yml | 28 +-- .../module/threatintel/_meta/docs.asciidoc | 4 +- .../abusemalware/config/config.yml | 6 +- .../abusechmalware.ndjson.log-expected.json | 50 ++++++ .../threatintel/abuseurl/config/config.yml | 6 +- .../test/abusechurl.ndjson.log-expected.json | 100 +++++++++++ .../threatintel/anomali/config/config.yml | 8 +- .../threatintel/anomali/ingest/pipeline.yml | 12 +- .../anomali_limo.ndjson.log-expected.json | 100 +++++++++++ .../module/threatintel/misp/config/config.yml | 2 +- .../threatintel/misp/ingest/pipeline.yml | 50 ++++-- .../test/misp_sample.ndjson.log-expected.json | 106 +++++++---- .../module/threatintel/otx/config/config.yml | 9 +- .../threatintel/otx/ingest/pipeline.yml | 11 +- .../module/threatintel/otx/manifest.yml | 3 + .../test/otx_sample.ndjson.log-expected.json | 168 +++++------------- .../modules.d/threatintel.yml.disabled | 28 +-- 19 files changed, 500 insertions(+), 223 deletions(-) diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc index ef98c6344cd..9a228a73b77 100644 --- a/filebeat/docs/modules/threatintel.asciidoc +++ b/filebeat/docs/modules/threatintel.asciidoc @@ -12,8 +12,8 @@ This file is generated! See scripts/docs_collector.py == Threat Intel module beta[] -This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also -compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html. +This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also +compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors]. The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields. Currently supporting: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index b714d4a82ba..1d6778167d6 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1996,7 +1996,7 @@ filebeat.modules: var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m abusemalware: enabled: true @@ -2008,7 +2008,7 @@ filebeat.modules: var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m misp: enabled: true @@ -2022,6 +2022,10 @@ filebeat.modules: # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. var.api_token: API_KEY + # Configures the type of SSL verification done, if MISP is running on self signed certificates + # then the certificate would either need to be trusted, or verification_mode set to none. + #var.ssl.verification_mode: none + # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. # For examples please reference the filebeat module documentation. #var.filters: @@ -2030,10 +2034,10 @@ filebeat.modules: # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer # than the last event that was already ingested. - var.first_interval: 24h + var.first_interval: 300h # The interval to poll the API for updates. - var.interval: 60m + var.interval: 5m otx: enabled: true @@ -2050,14 +2054,17 @@ filebeat.modules: # Optional filters that can be applied to retrieve only specific indicators. #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" + # The timeout of the HTTP client connecting to the OTX API + #var.http_client_timeout: 120s + # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. - var.lookback_range: 2h + var.lookback_range: 1h # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m anomali: enabled: true @@ -2065,7 +2072,8 @@ filebeat.modules: # Input used for ingesting threat intel data var.input: httpjson - # The URL used for Threat Intel API calls. + # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending + # on the type of threat intel source that is needed. var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects # The Username used by anomali Limo, defaults to guest. @@ -2075,10 +2083,10 @@ filebeat.modules: #var.password: guest # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m #---------------------------- Apache Tomcat Module ---------------------------- - module: tomcat diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml index 9ee88db47ed..72a5df6377b 100644 --- a/x-pack/filebeat/module/threatintel/_meta/config.yml +++ b/x-pack/filebeat/module/threatintel/_meta/config.yml @@ -9,7 +9,7 @@ var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m abusemalware: enabled: true @@ -21,7 +21,7 @@ var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m misp: enabled: true @@ -35,6 +35,10 @@ # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. var.api_token: API_KEY + # Configures the type of SSL verification done, if MISP is running on self signed certificates + # then the certificate would either need to be trusted, or verification_mode set to none. + #var.ssl.verification_mode: none + # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. # For examples please reference the filebeat module documentation. #var.filters: @@ -43,10 +47,10 @@ # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer # than the last event that was already ingested. - var.first_interval: 24h + var.first_interval: 300h # The interval to poll the API for updates. - var.interval: 60m + var.interval: 5m otx: enabled: true @@ -63,14 +67,17 @@ # Optional filters that can be applied to retrieve only specific indicators. #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" + # The timeout of the HTTP client connecting to the OTX API + #var.http_client_timeout: 120s + # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. - var.lookback_range: 2h + var.lookback_range: 1h # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m anomali: enabled: true @@ -78,7 +85,8 @@ # Input used for ingesting threat intel data var.input: httpjson - # The URL used for Threat Intel API calls. + # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending + # on the type of threat intel source that is needed. var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects # The Username used by anomali Limo, defaults to guest. @@ -88,7 +96,7 @@ #var.password: guest # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc index b6711a419dc..997460dcd23 100644 --- a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc @@ -7,8 +7,8 @@ == Threat Intel module beta[] -This module is a collection of different threat intelligence sources. The ingested data is meant to be used with [Indicator Match rules]https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule, but is also -compatible with other features like [Enrich Processors]https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html. +This module is a collection of different threat intelligence sources. The ingested data is meant to be used with https://www.elastic.co/guide/en/security/7.11/rules-ui-create.html#create-indicator-rule[Indicator Match rules], but is also +compatible with other features like https://www.elastic.co/guide/en/elasticsearch/reference/current/enrich-processor.html[Enrich Processors]. The related threat intel attribute that is meant to be used for matching incoming source data is stored under the `threatintel.indicator.*` fields. Currently supporting: diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml index 5922dd8838a..145dfe246dd 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml @@ -6,7 +6,7 @@ interval: {{ .interval }} request.method: GET {{ if .ssl }} - - request.ssl: {{ .ssl | tojson }} +request.ssl: {{ .ssl | tojson }} {{ end }} request.url: {{ .url }} request.transforms: @@ -33,9 +33,11 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - decode_json_fields: - document_id: "md5_hash" fields: [message] target: json + - fingerprint: + fields: ["json.md5_hash"] + target_field: "@metadata._id" - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json index 3a511662725..c3d6c804d75 100644 --- a/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json +++ b/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json @@ -10,6 +10,7 @@ "input.type": "log", "log.offset": 0, "related.hash": [ + "7871286a8f1f68a14b18ae475683f724", "48a6aee18bcfe9058b35b1018832aef1c9efd8f50ac822f49abb484a5e2a4b1f", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG5:X5DpBw/KViMTB1MnEWk0115JW", "68aea345b134d576ccdef7f06db86088" @@ -19,6 +20,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "7871286a8f1f68a14b18ae475683f724", "threatintel.indicator.file.hash.sha256": "48a6aee18bcfe9058b35b1018832aef1c9efd8f50ac822f49abb484a5e2a4b1f", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG5:X5DpBw/KViMTB1MnEWk0115JW", "threatintel.indicator.file.hash.tlsh": "1344D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -39,6 +41,7 @@ "input.type": "log", "log.offset": 580, "related.hash": [ + "7b4c77dc293347b467fb860e34515163", "ec59538e8de8525b1674b3b8fe0c180ac822145350bcce054ad3fc6b95b1b5a4", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGY:X5DpBw/KViMTB1MnEWk0115Jr", "68aea345b134d576ccdef7f06db86088" @@ -48,6 +51,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "7b4c77dc293347b467fb860e34515163", "threatintel.indicator.file.hash.sha256": "ec59538e8de8525b1674b3b8fe0c180ac822145350bcce054ad3fc6b95b1b5a4", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGY:X5DpBw/KViMTB1MnEWk0115Jr", "threatintel.indicator.file.hash.tlsh": "4E44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -68,6 +72,7 @@ "input.type": "log", "log.offset": 1160, "related.hash": [ + "373d34874d7bc89fd4cefa6272ee80bf", "b0e914d1bbe19433cc9df64ea1ca07fe77f7b150b511b786e46e007941a62bd7", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGG:X5DpBw/KViMTB1MnEWk0115Jd", "68aea345b134d576ccdef7f06db86088" @@ -80,6 +85,7 @@ "threatintel.abusemalware.virustotal.link": "https://www.virustotal.com/gui/file/b0e914d1bbe19433cc9df64ea1ca07fe77f7b150b511b786e46e007941a62bd7/detection/f-b0e914d", "threatintel.abusemalware.virustotal.percent": "37.88", "threatintel.abusemalware.virustotal.result": "25 / 66", + "threatintel.indicator.file.hash.md5": "373d34874d7bc89fd4cefa6272ee80bf", "threatintel.indicator.file.hash.sha256": "b0e914d1bbe19433cc9df64ea1ca07fe77f7b150b511b786e46e007941a62bd7", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGG:X5DpBw/KViMTB1MnEWk0115Jd", "threatintel.indicator.file.hash.tlsh": "7544D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -100,6 +106,7 @@ "input.type": "log", "log.offset": 1904, "related.hash": [ + "e2e02aae857488dbdbe6631c29abf3f8", "7483e834a73fb6817769596fe4c0fa01d28639f52bbbdc2b8a56c36d466dd7f8", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ9:0h3eZgRQCcw+MN54dEq7kqRtoLZH", "68aea345b134d576ccdef7f06db86088" @@ -109,6 +116,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "e2e02aae857488dbdbe6631c29abf3f8", "threatintel.indicator.file.hash.sha256": "7483e834a73fb6817769596fe4c0fa01d28639f52bbbdc2b8a56c36d466dd7f8", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ9:0h3eZgRQCcw+MN54dEq7kqRtoLZH", "threatintel.indicator.file.hash.tlsh": "5554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -129,6 +137,7 @@ "input.type": "log", "log.offset": 2493, "related.hash": [ + "3e988e32b0c3c230d534e286665b89a5", "760e729426fb115b967a41e5a6f2f42d7a52a5cee74ed99065a6dc39bf89f59b", "6:TE6ll8uXi0jIAv6BHvPuA7RKTmOQamsQMGvMQgTYbtsWsQ72hCqPZG/:TTll8uTo5uA7RKtQamsS0QJfsQ7mCR" ], @@ -137,6 +146,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "3e988e32b0c3c230d534e286665b89a5", "threatintel.indicator.file.hash.sha256": "760e729426fb115b967a41e5a6f2f42d7a52a5cee74ed99065a6dc39bf89f59b", "threatintel.indicator.file.hash.ssdeep": "6:TE6ll8uXi0jIAv6BHvPuA7RKTmOQamsQMGvMQgTYbtsWsQ72hCqPZG/:TTll8uTo5uA7RKtQamsS0QJfsQ7mCR", "threatintel.indicator.file.hash.tlsh": "3CE0C002AB26C036500D154C221655B3B871911503CA14E6A6824BEA765D4A3290D190", @@ -156,6 +166,7 @@ "input.type": "log", "log.offset": 3054, "related.hash": [ + "dcc20d534cdf29eab03d8148bf728857", "86655c0bcf9b21b5efc682f58eb80f42811042ba152358e1bfbbb867315a60ac", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGI:X5DpBw/KViMTB1MnEWk0115JH", "68aea345b134d576ccdef7f06db86088" @@ -168,6 +179,7 @@ "threatintel.abusemalware.virustotal.link": "https://www.virustotal.com/gui/file/86655c0bcf9b21b5efc682f58eb80f42811042ba152358e1bfbbb867315a60ac/detection/f-86655c0", "threatintel.abusemalware.virustotal.percent": "39.13", "threatintel.abusemalware.virustotal.result": "27 / 69", + "threatintel.indicator.file.hash.md5": "dcc20d534cdf29eab03d8148bf728857", "threatintel.indicator.file.hash.sha256": "86655c0bcf9b21b5efc682f58eb80f42811042ba152358e1bfbbb867315a60ac", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGI:X5DpBw/KViMTB1MnEWk0115JH", "threatintel.indicator.file.hash.tlsh": "0D44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -188,6 +200,7 @@ "input.type": "log", "log.offset": 3798, "related.hash": [ + "f6facbf7a90b9e67a6de9f6634eb40ba", "e91c9e11d3ce4f55fabd7196279367482d2fabfa32df81e614b15fc53b4e26be", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ1:0h3eZgRQCcw+MN54dEq7kqRtoLZL", "68aea345b134d576ccdef7f06db86088" @@ -197,6 +210,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "f6facbf7a90b9e67a6de9f6634eb40ba", "threatintel.indicator.file.hash.sha256": "e91c9e11d3ce4f55fabd7196279367482d2fabfa32df81e614b15fc53b4e26be", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ1:0h3eZgRQCcw+MN54dEq7kqRtoLZL", "threatintel.indicator.file.hash.tlsh": "2554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -217,6 +231,7 @@ "input.type": "log", "log.offset": 4387, "related.hash": [ + "44325fd5bdda2e2cdea07c3a39953bb1", "beedbbcacfc34b5edd8c68e3e4acf364992ebbcd989548e09e38fa03c5659bac", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Jg", "68aea345b134d576ccdef7f06db86088" @@ -226,6 +241,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "44325fd5bdda2e2cdea07c3a39953bb1", "threatintel.indicator.file.hash.sha256": "beedbbcacfc34b5edd8c68e3e4acf364992ebbcd989548e09e38fa03c5659bac", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Jg", "threatintel.indicator.file.hash.tlsh": "A044D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -246,6 +262,7 @@ "input.type": "log", "log.offset": 4967, "related.hash": [ + "4c549051950522a3f1b0814aa9b1f6d1", "7cba55da723c0e020267a02e6ffc83e03a83701757fc4ec65ea398618ad881cf", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG4:X5DpBw/KViMTB1MnEWk0115Jv", "68aea345b134d576ccdef7f06db86088" @@ -256,6 +273,7 @@ "forwarded" ], "threatintel.abusemalware.signature": "Heodo", + "threatintel.indicator.file.hash.md5": "4c549051950522a3f1b0814aa9b1f6d1", "threatintel.indicator.file.hash.sha256": "7cba55da723c0e020267a02e6ffc83e03a83701757fc4ec65ea398618ad881cf", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG4:X5DpBw/KViMTB1MnEWk0115Jv", "threatintel.indicator.file.hash.tlsh": "4544D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -276,6 +294,7 @@ "input.type": "log", "log.offset": 5550, "related.hash": [ + "d7333113098d88b6a5dd5b8eb24f9b87", "426be5e085e6bbad8430223dc89d8d3ced497133f8d478fd00005bcbb73399d4", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJw:0h3eZgRQCcw+MN54dEq7kqRtoLZW", "68aea345b134d576ccdef7f06db86088" @@ -285,6 +304,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "d7333113098d88b6a5dd5b8eb24f9b87", "threatintel.indicator.file.hash.sha256": "426be5e085e6bbad8430223dc89d8d3ced497133f8d478fd00005bcbb73399d4", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJw:0h3eZgRQCcw+MN54dEq7kqRtoLZW", "threatintel.indicator.file.hash.tlsh": "9454CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -305,6 +325,7 @@ "input.type": "log", "log.offset": 6139, "related.hash": [ + "c8dbb261c1f450534c3693da2f4b479f", "25093afdaeb3ea000743ab843360a6b64f58c0a1ab950072ba6528056735deb9", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGe:X5DpBw/KViMTB1MnEWk0115JR", "68aea345b134d576ccdef7f06db86088" @@ -314,6 +335,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "c8dbb261c1f450534c3693da2f4b479f", "threatintel.indicator.file.hash.sha256": "25093afdaeb3ea000743ab843360a6b64f58c0a1ab950072ba6528056735deb9", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGe:X5DpBw/KViMTB1MnEWk0115JR", "threatintel.indicator.file.hash.tlsh": "F344D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -334,6 +356,7 @@ "input.type": "log", "log.offset": 6719, "related.hash": [ + "714953f1d0031a4bb2f0c44afd015931", "b3327a96280365e441057f490df6261c9a2400fd63719eb9a7a0c9db95beecc5", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115J7", "68aea345b134d576ccdef7f06db86088" @@ -343,6 +366,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "714953f1d0031a4bb2f0c44afd015931", "threatintel.indicator.file.hash.sha256": "b3327a96280365e441057f490df6261c9a2400fd63719eb9a7a0c9db95beecc5", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115J7", "threatintel.indicator.file.hash.tlsh": "F644D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -363,6 +387,7 @@ "input.type": "log", "log.offset": 7299, "related.hash": [ + "20fd22742500d4cec123398afc3d3672", "e92b54904391c171238863b584355197ba4508f73320a8e89afbb5425fc2dc4b", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115JP", "68aea345b134d576ccdef7f06db86088" @@ -372,6 +397,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "20fd22742500d4cec123398afc3d3672", "threatintel.indicator.file.hash.sha256": "e92b54904391c171238863b584355197ba4508f73320a8e89afbb5425fc2dc4b", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115JP", "threatintel.indicator.file.hash.tlsh": "BE44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -392,6 +418,7 @@ "input.type": "log", "log.offset": 7879, "related.hash": [ + "aa81ceea053797a6f8c38a0f2f9b80b0", "dd15e74b3cd3a4fdb5f47adefd6f90e27d5a20e01316cc791711f6dce7c0f52e", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGf:X5DpBw/KViMTB1MnEWk0115Jo", "68aea345b134d576ccdef7f06db86088" @@ -401,6 +428,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "aa81ceea053797a6f8c38a0f2f9b80b0", "threatintel.indicator.file.hash.sha256": "dd15e74b3cd3a4fdb5f47adefd6f90e27d5a20e01316cc791711f6dce7c0f52e", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGf:X5DpBw/KViMTB1MnEWk0115Jo", "threatintel.indicator.file.hash.tlsh": "CC44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -421,6 +449,7 @@ "input.type": "log", "log.offset": 8459, "related.hash": [ + "a2ce6795664c0fa93b07fa54ba868991", "0fae1eeabc4f5e07bd16f7851aec5ab6032d407c7ff0270f2b6e85c2a3efebd1", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGD:X5DpBw/KViMTB1MnEWk0115JY", "68aea345b134d576ccdef7f06db86088" @@ -431,6 +460,7 @@ "forwarded" ], "threatintel.abusemalware.signature": "Heodo", + "threatintel.indicator.file.hash.md5": "a2ce6795664c0fa93b07fa54ba868991", "threatintel.indicator.file.hash.sha256": "0fae1eeabc4f5e07bd16f7851aec5ab6032d407c7ff0270f2b6e85c2a3efebd1", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGD:X5DpBw/KViMTB1MnEWk0115JY", "threatintel.indicator.file.hash.tlsh": "8C44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -451,6 +481,7 @@ "input.type": "log", "log.offset": 9042, "related.hash": [ + "9b9bac158dacb9c2f5511e9c464a7de4", "07a9d84c0b2c8cf1fd90ab409b9399d06920ab4b6efb647b5a3b9bef1045ee7e", "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKk:W5MT4WNaHy9P1FjbrjlKk", "68aea345b134d576ccdef7f06db86088" @@ -460,6 +491,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "9b9bac158dacb9c2f5511e9c464a7de4", "threatintel.indicator.file.hash.sha256": "07a9d84c0b2c8cf1fd90ab409b9399d06920ab4b6efb647b5a3b9bef1045ee7e", "threatintel.indicator.file.hash.ssdeep": "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKk:W5MT4WNaHy9P1FjbrjlKk", "threatintel.indicator.file.hash.tlsh": "6B54CF217A53C826F5E800FCA6E9878914167F346F44A4C773D40F6AA8759E2EF2B317", @@ -480,6 +512,7 @@ "input.type": "log", "log.offset": 9611, "related.hash": [ + "e48e3fa5e0f7b21c1ecf1efc81ff91e8", "708c0193aec6354af6877f314d4b0e3864552bac77258bee9ee5bf886a116df5", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGo:X5DpBw/KViMTB1MnEWk0115Jj", "68aea345b134d576ccdef7f06db86088" @@ -489,6 +522,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "e48e3fa5e0f7b21c1ecf1efc81ff91e8", "threatintel.indicator.file.hash.sha256": "708c0193aec6354af6877f314d4b0e3864552bac77258bee9ee5bf886a116df5", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGo:X5DpBw/KViMTB1MnEWk0115Jj", "threatintel.indicator.file.hash.tlsh": "6644D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -509,6 +543,7 @@ "input.type": "log", "log.offset": 10191, "related.hash": [ + "8957f5347633ab4b10c2ae4fb92c8572", "f70a3c016fe791eb30959961f0bcaa08ba7b738491b9ae61cb4a667cd1de8b37", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJy:0h3eZgRQCcw+MN54dEq7kqRtoLZM", "68aea345b134d576ccdef7f06db86088" @@ -519,6 +554,7 @@ "forwarded" ], "threatintel.abusemalware.signature": "Heodo", + "threatintel.indicator.file.hash.md5": "8957f5347633ab4b10c2ae4fb92c8572", "threatintel.indicator.file.hash.sha256": "f70a3c016fe791eb30959961f0bcaa08ba7b738491b9ae61cb4a667cd1de8b37", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJy:0h3eZgRQCcw+MN54dEq7kqRtoLZM", "threatintel.indicator.file.hash.tlsh": "0754CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -539,6 +575,7 @@ "input.type": "log", "log.offset": 10783, "related.hash": [ + "09cc76b7077b4d5704e46e864575ff03", "94ca186561b13fa9b1bf15f7e66118debc686b40d2a62a5cf4b3c6ca6ee1c7a1", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Js", "68aea345b134d576ccdef7f06db86088" @@ -548,6 +585,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "09cc76b7077b4d5704e46e864575ff03", "threatintel.indicator.file.hash.sha256": "94ca186561b13fa9b1bf15f7e66118debc686b40d2a62a5cf4b3c6ca6ee1c7a1", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Js", "threatintel.indicator.file.hash.tlsh": "BB44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -568,6 +606,7 @@ "input.type": "log", "log.offset": 11363, "related.hash": [ + "98a1cdf7de4232363f1d1e0f33dbfd99", "909f890dbc5748845cf06d0fb0b73a5c0cb17761f37e9cd4810eea0d0eb8627f", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJQ:0h3eZgRQCcw+MN54dEq7kqRtoLZ+", "68aea345b134d576ccdef7f06db86088" @@ -577,6 +616,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "98a1cdf7de4232363f1d1e0f33dbfd99", "threatintel.indicator.file.hash.sha256": "909f890dbc5748845cf06d0fb0b73a5c0cb17761f37e9cd4810eea0d0eb8627f", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJQ:0h3eZgRQCcw+MN54dEq7kqRtoLZ+", "threatintel.indicator.file.hash.tlsh": "C554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -597,6 +637,7 @@ "input.type": "log", "log.offset": 11952, "related.hash": [ + "8a51830c1662513ba6bd44e2f7849547", "d1fa76346bef5bc8adaa615e109894a7c30f0bef07ab6272409c4056ea8d52aa", "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJh:0h3eZgRQCcw+MN54dEq7kqRtoLZ/", "68aea345b134d576ccdef7f06db86088" @@ -607,6 +648,7 @@ "forwarded" ], "threatintel.abusemalware.signature": "Heodo", + "threatintel.indicator.file.hash.md5": "8a51830c1662513ba6bd44e2f7849547", "threatintel.indicator.file.hash.sha256": "d1fa76346bef5bc8adaa615e109894a7c30f0bef07ab6272409c4056ea8d52aa", "threatintel.indicator.file.hash.ssdeep": "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJh:0h3eZgRQCcw+MN54dEq7kqRtoLZ/", "threatintel.indicator.file.hash.tlsh": "1654CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717", @@ -627,6 +669,7 @@ "input.type": "log", "log.offset": 12544, "related.hash": [ + "ae21d742a8118d6b86674aa5370bd6a7", "3b9698b6c18bcba15ee33378440dd3f42509730e6b1d2d5832c71a74b1920e51", "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKS:W5MT4WNaHy9P1FjbrjlKS", "68aea345b134d576ccdef7f06db86088" @@ -636,6 +679,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "ae21d742a8118d6b86674aa5370bd6a7", "threatintel.indicator.file.hash.sha256": "3b9698b6c18bcba15ee33378440dd3f42509730e6b1d2d5832c71a74b1920e51", "threatintel.indicator.file.hash.ssdeep": "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKS:W5MT4WNaHy9P1FjbrjlKS", "threatintel.indicator.file.hash.tlsh": "5454CF217A53C826F5E800FCA6E9878925167F346F44A4C373D40F6AA8759E2DF2B317", @@ -656,6 +700,7 @@ "input.type": "log", "log.offset": 13113, "related.hash": [ + "78c9d88d24ed1d982a83216eed1590f6", "d11edc90f0e879a175abc6e2ce5c94a263aa2a01cd3b6e8b9fdf93a51235ae99", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG8:X5DpBw/KViMTB1MnEWk0115Jr", "68aea345b134d576ccdef7f06db86088" @@ -665,6 +710,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "78c9d88d24ed1d982a83216eed1590f6", "threatintel.indicator.file.hash.sha256": "d11edc90f0e879a175abc6e2ce5c94a263aa2a01cd3b6e8b9fdf93a51235ae99", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG8:X5DpBw/KViMTB1MnEWk0115Jr", "threatintel.indicator.file.hash.tlsh": "6044D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", @@ -685,6 +731,7 @@ "input.type": "log", "log.offset": 13693, "related.hash": [ + "236577d5d83e2a8d08623a7a7f724188", "8cd28fed7ebdcd79ea2509dca84f0a727ca28d4eaaed5a92cd10b1279ff16afa", "6144:X1G3WVIOY6Bdjehj+qudd96ou/6mv5wdC:X1GmSafShjYdd96z/6cwdC", "ed2860c18f5483e3b5388bad75169dc1" @@ -694,6 +741,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "236577d5d83e2a8d08623a7a7f724188", "threatintel.indicator.file.hash.sha256": "8cd28fed7ebdcd79ea2509dca84f0a727ca28d4eaaed5a92cd10b1279ff16afa", "threatintel.indicator.file.hash.ssdeep": "6144:X1G3WVIOY6Bdjehj+qudd96ou/6mv5wdC:X1GmSafShjYdd96z/6cwdC", "threatintel.indicator.file.hash.tlsh": "8D34BE41B28B8B4BD163163C2976D1F8953CFC909761CE693B64B22F0F739D0892E7A5", @@ -714,6 +762,7 @@ "input.type": "log", "log.offset": 14256, "related.hash": [ + "ff60107d82dcda7e6726d214528758e7", "fb25d13188a5d0913bbcf5aeff6c7e3208ad92a7d10ab6bed2735f4d43310a27", "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGz:X5DpBw/KViMTB1MnEWk0115JU", "68aea345b134d576ccdef7f06db86088" @@ -723,6 +772,7 @@ "threatintel-abusemalware", "forwarded" ], + "threatintel.indicator.file.hash.md5": "ff60107d82dcda7e6726d214528758e7", "threatintel.indicator.file.hash.sha256": "fb25d13188a5d0913bbcf5aeff6c7e3208ad92a7d10ab6bed2735f4d43310a27", "threatintel.indicator.file.hash.ssdeep": "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGz:X5DpBw/KViMTB1MnEWk0115JU", "threatintel.indicator.file.hash.tlsh": "9244D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717", diff --git a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml index 0ac7ef6c143..96affa7da97 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml +++ b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml @@ -6,7 +6,7 @@ interval: {{ .interval }} request.method: GET {{ if .ssl }} - - request.ssl: {{ .ssl | tojson }} +request.ssl: {{ .ssl | tojson }} {{ end }} request.url: {{ .url }} request.transforms: @@ -33,9 +33,11 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - decode_json_fields: - document_id: "id" fields: [message] target: json + - fingerprint: + fields: ["json.id"] + target_field: "@metadata._id" - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/threatintel/abuseurl/test/abusechurl.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/abuseurl/test/abusechurl.ndjson.log-expected.json index 5f12181b2db..25ce780046f 100644 --- a/x-pack/filebeat/module/threatintel/abuseurl/test/abusechurl.ndjson.log-expected.json +++ b/x-pack/filebeat/module/threatintel/abuseurl/test/abusechurl.ndjson.log-expected.json @@ -16,6 +16,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961548", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "elf", @@ -51,6 +52,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961546", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "elf", @@ -86,6 +88,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961547", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "elf", @@ -121,6 +124,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961545", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "elf", @@ -156,6 +160,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961544", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -191,6 +196,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961543", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -226,6 +232,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961540", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -261,6 +268,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961541", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -296,6 +304,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961542", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -331,6 +340,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961539", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -366,6 +376,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961538", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -401,6 +412,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961537", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -436,6 +448,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961531", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -471,6 +484,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961532", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -506,6 +520,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961533", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -541,6 +556,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961534", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -576,6 +592,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961535", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -611,6 +628,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961536", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -646,6 +664,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961530", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "elf", @@ -681,6 +700,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961525", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -716,6 +736,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961526", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -751,6 +772,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961527", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -786,6 +808,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961528", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -821,6 +844,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961529", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -856,6 +880,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961524", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "Mozi" @@ -890,6 +915,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961523", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "Mozi" @@ -924,6 +950,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961520", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "Mozi" @@ -958,6 +985,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961521", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "Mozi" @@ -992,6 +1020,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961522", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "Mozi" @@ -1026,6 +1055,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961518", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1060,6 +1090,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961519", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1095,6 +1126,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961516", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "32-bit", @@ -1131,6 +1163,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961517", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1166,6 +1199,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961515", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1200,6 +1234,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961513", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1234,6 +1269,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961514", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1268,6 +1304,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961509", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1302,6 +1339,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961510", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1336,6 +1374,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961511", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "32-bit", @@ -1371,6 +1410,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961512", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "Mozi" @@ -1405,6 +1445,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961507", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1440,6 +1481,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961508", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1475,6 +1517,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961506", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1510,6 +1553,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961504", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1545,6 +1589,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961505", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1580,6 +1625,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961500", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1615,6 +1661,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961501", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1650,6 +1697,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961502", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1685,6 +1733,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961503", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1720,6 +1769,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961496", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1755,6 +1805,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961497", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1790,6 +1841,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961498", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1825,6 +1877,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961499", "threatintel.abuseurl.larted": true, "threatintel.abuseurl.tags": [ "elf", @@ -1860,6 +1913,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961494", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -1893,6 +1947,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961495", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -1926,6 +1981,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961492", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -1959,6 +2015,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961493", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -1992,6 +2049,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961490", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2025,6 +2083,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961491", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2058,6 +2117,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961489", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2091,6 +2151,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961488", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2124,6 +2185,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961487", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2157,6 +2219,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961485", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2190,6 +2253,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961486", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2223,6 +2287,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961482", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2256,6 +2321,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961483", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2289,6 +2355,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961484", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2322,6 +2389,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961480", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2355,6 +2423,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961481", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2388,6 +2457,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961478", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2421,6 +2491,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961479", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2454,6 +2525,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961476", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2487,6 +2559,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961477", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2520,6 +2593,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961470", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2553,6 +2627,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961471", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2586,6 +2661,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961472", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2619,6 +2695,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961473", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2652,6 +2729,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961474", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2685,6 +2763,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961475", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2718,6 +2797,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961468", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2751,6 +2831,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961469", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2784,6 +2865,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961467", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2817,6 +2899,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961464", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2850,6 +2933,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961465", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2883,6 +2967,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961466", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2916,6 +3001,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961461", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2949,6 +3035,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961462", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -2982,6 +3069,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961463", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3015,6 +3103,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961458", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3048,6 +3137,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961459", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3081,6 +3171,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961460", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3114,6 +3205,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961455", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3147,6 +3239,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961456", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3180,6 +3273,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961457", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3213,6 +3307,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961450", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3246,6 +3341,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961451", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3279,6 +3375,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961452", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3312,6 +3409,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961453", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3345,6 +3443,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961454", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" @@ -3378,6 +3477,7 @@ ], "threatintel.abuseurl.blacklists.spamhaus_dbl": "not listed", "threatintel.abuseurl.blacklists.surbl": "not listed", + "threatintel.abuseurl.id": "961448", "threatintel.abuseurl.larted": false, "threatintel.abuseurl.tags": [ "sLoad" diff --git a/x-pack/filebeat/module/threatintel/anomali/config/config.yml b/x-pack/filebeat/module/threatintel/anomali/config/config.yml index 19e58b4bc12..fd55b6e07c2 100644 --- a/x-pack/filebeat/module/threatintel/anomali/config/config.yml +++ b/x-pack/filebeat/module/threatintel/anomali/config/config.yml @@ -12,7 +12,7 @@ auth.basic.password: {{ .password }} {{ end }} request.method: GET {{ if .ssl }} - - request.ssl: {{ .ssl | tojson }} +request.ssl: {{ .ssl | tojson }} {{ end }} request.url: {{ .url }} request.redirect.forward_headers: true @@ -32,7 +32,7 @@ request.transforms: - set: target: url.params.added_after value: '[[.cursor.timestamp]]' - default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02T15:04:05.999Z" ]]' + default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02T15:04:05.000Z" ]]' response.split: target: body.objects @@ -58,8 +58,10 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - decode_json_fields: fields: [message] - document_id: id target: json + - fingerprint: + fields: ["json.id"] + target_field: "@metadata._id" - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml index 0f16b62643a..239cbc608f5 100644 --- a/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/anomali/ingest/pipeline.yml @@ -32,6 +32,10 @@ processors: - date: field: threatintel.anomali.created formats: + - "yyyy-MM-dd'T'HH:mm:ssz" + - "yyyy-MM-dd'T'HH:mm:ssZ" + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" - "yyyy-MM-dd'T'HH:mm:ss.SSz" - "yyyy-MM-dd'T'HH:mm:ss.SSZ" - "yyyy-MM-dd'T'HH:mm:ss.SSSz" @@ -41,20 +45,24 @@ processors: field: threatintel.anomali.modified target_field: threatintel.anomali.modified formats: + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" - "yyyy-MM-dd'T'HH:mm:ss.SSz" - "yyyy-MM-dd'T'HH:mm:ss.SSZ" - "yyyy-MM-dd'T'HH:mm:ss.SSSz" - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" - if: "ctx?.threatintel?.anomali?.created != null" + if: "ctx?.threatintel?.anomali?.modified != null" - date: field: threatintel.anomali.valid_from target_field: threatintel.anomali.valid_from formats: + - "yyyy-MM-dd'T'HH:mm:ss.Sz" + - "yyyy-MM-dd'T'HH:mm:ss.SZ" - "yyyy-MM-dd'T'HH:mm:ss.SSz" - "yyyy-MM-dd'T'HH:mm:ss.SSZ" - "yyyy-MM-dd'T'HH:mm:ss.SSSz" - "yyyy-MM-dd'T'HH:mm:ss.SSSZ" - if: "ctx?.threatintel?.anomali?.created != null" + if: "ctx?.threatintel?.anomali?.valid_from != null" - grok: field: threatintel.anomali.pattern patterns: diff --git a/x-pack/filebeat/module/threatintel/anomali/test/anomali_limo.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/anomali/test/anomali_limo.ndjson.log-expected.json index 69205da6d59..c40db227906 100644 --- a/x-pack/filebeat/module/threatintel/anomali/test/anomali_limo.ndjson.log-expected.json +++ b/x-pack/filebeat/module/threatintel/anomali/test/anomali_limo.ndjson.log-expected.json @@ -14,6 +14,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332361; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--44c85d4f-45ca-4977-b693-c810bbfb7a28", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -49,6 +50,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332307; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f9fe5c81-6869-4247-af81-62b7c8aba209", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -84,6 +86,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332302; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b0e14122-9005-4776-99fc-00872476c6d1", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -118,6 +121,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332312; iType: mal_url; State: active; Org: Digital Ocean; Source: CyberCrime", + "threatintel.anomali.id": "indicator--111ec76f-616d-4aa8-80fd-e11ef0066aba", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -152,6 +156,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332386; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--189ce776-6d7e-4e85-9222-de5876644988", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -187,6 +192,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332391; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--a4144d34-b86d-475e-8047-eb46b48ee325", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -222,6 +228,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332372; iType: mal_ip; State: active; Org: Unified Layer; Source: CyberCrime", + "threatintel.anomali.id": "indicator--983d9c3d-b7f8-4345-b643-b1d18e6ac6b2", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -253,6 +260,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332313; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f9c6386b-dba2-41f9-8160-d307671e5c8e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -288,6 +296,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332350; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--98fad53e-5389-47f7-a3ff-44d334af2d6b", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -323,6 +332,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332291; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--76c01735-fb76-463d-9609-9ea3aedf3f4f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -357,6 +367,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332343; iType: mal_ip; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e0a812dc-63c8-4949-b038-2241b2dbfcdc", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -388,6 +399,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332316; iType: mal_url; State: active; Org: Sksa Technology Sdn Bhd; Source: CyberCrime", + "threatintel.anomali.id": "indicator--6f0d8607-21cb-4738-9712-f4fd91a37f7d", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -423,6 +435,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332284; iType: mal_url; State: active; Org: Oltelecom Jsc; Source: CyberCrime", + "threatintel.anomali.id": "indicator--c649d6d4-87c4-4b76-bfc2-75a509ccb187", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -457,6 +470,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332337; iType: mal_ip; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--408ebd2d-063f-4646-b2e7-c00519869736", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -488,6 +502,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332324; iType: mal_ip; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e1d215cb-c7a5-40e0-bc53-8f92a2bcaba8", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -519,6 +534,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332296; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--6f3a4a2b-62e3-48ef-94ae-70103f09cf7e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -553,6 +569,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332400; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--213519c9-f511-4188-89c8-159f35f08008", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -588,6 +605,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332396; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--5a563c85-c528-4e33-babe-2dcff34f73c4", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -623,6 +641,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332363; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f3e33aab-e2af-4c15-8cb9-f008a37cf986", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -658,6 +677,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332320; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f03f098d-2fa9-49e1-a7dd-02518aa105fa", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -693,6 +713,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332367; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e72e3ba0-7de5-46bb-ab1e-efdf3e0a0b3b", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -728,6 +749,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332317; iType: mal_url; State: active; Org: SoftLayer Technologies; Source: CyberCrime", + "threatintel.anomali.id": "indicator--d6b59b66-5020-4368-85a7-196026856ea9", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -763,6 +785,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332309; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--aff7b07f-acc7-4bec-ab19-1fce972bfd09", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -798,6 +821,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332286; iType: mal_url; State: active; Org: Garanntor-Hosting; Source: CyberCrime", + "threatintel.anomali.id": "indicator--ba71ba3a-1efd-40da-ab0d-f4397d6fc337", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -833,6 +857,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332339; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--17777e7f-3e91-4446-a43d-79139de8a948", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -868,6 +893,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332319; iType: mal_ip; State: active; Org: SoftLayer Technologies; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f6be1804-cfe4-4f41-9338-2b65f5b1dda1", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -899,6 +925,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332305; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b4fd8489-9589-4f70-996c-84989245a21b", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -933,6 +960,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332346; iType: mal_url; State: active; Org: Ifx Networks Colombia; Source: CyberCrime", + "threatintel.anomali.id": "indicator--bc50c62f-a015-4460-87df-2137626877e3", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -968,6 +996,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332323; iType: mal_url; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--2765af4b-bfb7-4ac8-82d2-ab6ed8a52461", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1003,6 +1032,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332399; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--9c0e63a1-c32a-470a-bf09-51488e239c63", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1038,6 +1068,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332328; iType: mal_ip; State: active; Org: RUCloud; Source: CyberCrime", + "threatintel.anomali.id": "indicator--8047678e-20be-4116-9bc4-7bb7c26554e0", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1069,6 +1100,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332377; iType: mal_url; State: active; Org: A100 ROW GmbH; Source: CyberCrime", + "threatintel.anomali.id": "indicator--c57a880c-1ce0-45de-9bab-fb2910454a61", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1104,6 +1136,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332101; iType: mal_ip; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--6056152c-0fa5-4e34-871a-3c8990f1ee46", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1135,6 +1168,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332357; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--23215acb-4989-4434-ac6d-8f9367734f0f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1170,6 +1204,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332289; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--452ece92-9ff2-4f99-8a7f-fd614ebea8cf", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1204,6 +1239,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332334; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--10958d74-ec60-41af-a1ab-1613257e670f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1239,6 +1275,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332326; iType: mal_url; State: active; Org: RUCloud; Source: CyberCrime", + "threatintel.anomali.id": "indicator--19556daa-6293-400d-8706-d0baa6b16b7a", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1274,6 +1311,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332311; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b09d9be9-6703-4a7d-a066-2baebb6418fc", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1309,6 +1347,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332341; iType: mal_url; State: active; Org: Institute of Philosophy, Russian Academy of Scienc; Source: CyberCrime", + "threatintel.anomali.id": "indicator--43febf7d-4185-4a12-a868-e7be690b14aa", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1343,6 +1382,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332303; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--a34728e6-f91d-47e6-a4d8-a69176299e45", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1377,6 +1417,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55241332380; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--ac821704-5eb2-4f8f-a8b6-2a168dbd0e54", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1412,6 +1453,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868747; iType: mal_ip; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--0d3e1bd8-0f16-4c22-b8a1-663ec255ad79", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1443,6 +1485,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868770; iType: mal_url; State: active; Org: Mills College; Source: CyberCrime", + "threatintel.anomali.id": "indicator--2cdd130a-c884-402d-b63c-e03f9448f5d9", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1478,6 +1521,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868769; iType: mal_url; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--88e98e13-4bfd-4188-941a-f696a7b86b71", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1513,6 +1557,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868772; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--27323b7d-85d3-4e89-8249-b7696925a772", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1548,6 +1593,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868766; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b0639721-de55-48c6-b237-3859d61aecfb", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1582,6 +1628,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868749; iType: mal_url; State: active; Org: ColoCrossing; Source: CyberCrime", + "threatintel.anomali.id": "indicator--677e714d-c237-42a1-b6b7-9145acd13eee", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1617,6 +1664,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868767; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--5baa1dbd-d74e-408c-92b5-0a9f97e4b87a", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1652,6 +1700,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55245868768; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--4563241e-5d2f-41a7-adb9-3925a5eeb1b1", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1687,6 +1736,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078037; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--70cb5d42-91d3-4efe-8c47-995fc0ac4141", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1722,6 +1772,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078030; iType: mal_ip; State: active; Org: Best-Hoster Group Co. Ltd.; Source: CyberCrime", + "threatintel.anomali.id": "indicator--3aa712bb-b5d4-4632-bf50-48a4aeeaeb6d", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1753,6 +1804,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078019; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--64227c7d-86ea-4146-a868-3decb5aa5f1d", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1787,6 +1839,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078035; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--37fcf9a7-1a90-4d81-be0a-e824a4fa938e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1822,6 +1875,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078008; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--5a38786f-107e-4060-a7c9-ea8a5ded6aac", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1857,6 +1911,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078038; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--3eb79b31-1d6d-438c-a848-24a3407f6e32", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1892,6 +1947,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078026; iType: mal_url; State: active; Org: IT DeLuxe Ltd.; Source: CyberCrime", + "threatintel.anomali.id": "indicator--a050832c-db6e-49a0-8470-7a3cd8f17178", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1926,6 +1982,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078034; iType: mal_url; State: active; Org: Branch of BachKim Network solutions jsc; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e88008f4-76fc-428d-831a-4b389e48b712", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1961,6 +2018,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078032; iType: mal_url; State: active; Org: ColoCrossing; Source: CyberCrime", + "threatintel.anomali.id": "indicator--dafe91cf-787c-471c-9afe-f7bb20a1b93f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -1996,6 +2054,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078031; iType: mal_url; State: active; Org: IT House, Ltd; Source: CyberCrime", + "threatintel.anomali.id": "indicator--232bdc34-44cb-4f41-af52-f6f1cd28818e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2031,6 +2090,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078027; iType: mal_url; State: active; Org: Branch of BachKim Network solutions jsc; Source: CyberCrime", + "threatintel.anomali.id": "indicator--4adabe80-3be4-401a-948a-f9724c872374", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2066,6 +2126,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078013; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--1d7051c0-a42b-4801-bd7f-f0abf2cc125c", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2100,6 +2161,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078017; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--fb06856c-8aad-4fae-92fc-b73aae4f6dc7", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2134,6 +2196,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078012; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--33e674f5-a64a-48f4-9d8c-248348356135", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2168,6 +2231,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078018; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--6311f539-1d5d-423f-a238-d0c1dc167432", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2202,6 +2266,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078033; iType: mal_ip; State: active; Org: ColoCrossing; Source: CyberCrime", + "threatintel.anomali.id": "indicator--1c91f219-cfa6-44c7-a5ee-1c760489b43c", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2233,6 +2298,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078010; iType: mal_url; State: active; Org: QuadraNet; Source: CyberCrime", + "threatintel.anomali.id": "indicator--c58983e2-18fd-47b8-aab4-6c8a2e2dcb35", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2268,6 +2334,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078000; iType: mal_ip; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--1ab178a8-7991-4879-b9aa-8da49f40e92e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2299,6 +2366,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078020; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--d5bdff38-6939-4a47-8e11-b910520565c4", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2333,6 +2401,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078009; iType: mal_url; State: active; Org: ColoCrossing; Source: CyberCrime", + "threatintel.anomali.id": "indicator--1be74977-5aa6-4175-99dd-32b54863a06b", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2368,6 +2437,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078023; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--eacc25ce-584c-4b40-98ab-7935dabd5cb1", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2402,6 +2472,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078025; iType: mal_url; State: active; Org: Cloudflare; Source: CyberCrime", + "threatintel.anomali.id": "indicator--504f4011-eaea-4921-aad5-f102bef7c798", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2436,6 +2507,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078014; iType: mal_ip; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e3ffb953-6c59-461a-8242-0d26c2b5c358", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2467,6 +2539,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078036; iType: mal_ip; State: active; Org: Global Frag Networks; Source: CyberCrime", + "threatintel.anomali.id": "indicator--3a47ad46-930d-4ced-b0e7-dc9d0776153e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2498,6 +2571,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078011; iType: mal_url; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--0e10924c-745c-4a58-8e27-ab3a6bacd666", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2533,6 +2607,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078015; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--c3fb816a-cc3b-4442-be4d-d62113ae5168", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2567,6 +2642,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078029; iType: mal_url; State: active; Org: IT House, Ltd; Source: CyberCrime", + "threatintel.anomali.id": "indicator--9159e46d-f3a4-464b-ac68-8beaf87e1a8f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2602,6 +2678,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078016; iType: mal_url; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--fefa8e76-ae0f-41ab-84e7-ea43ab055573", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2636,6 +2713,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078024; iType: mal_url; State: active; Org: CyrusOne LLC; Source: CyberCrime", + "threatintel.anomali.id": "indicator--6a76fa89-4d5f-40d0-9b03-671bdb2d5b4b", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2671,6 +2749,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078022; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--21055dfd-d0cb-42ec-93bd-ffaeadd11d80", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2705,6 +2784,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078021; iType: mal_url; State: active; Org: MoreneHost; Source: CyberCrime", + "threatintel.anomali.id": "indicator--7471a595-e8b0-4c41-be4c-0a3e55675630", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2739,6 +2819,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55250078007; iType: mal_ip; State: active; Source: CyberCrime", + "threatintel.anomali.id": "indicator--ead1e7e5-fdb3-47c2-9476-aa82741c038e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2770,6 +2851,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484365; iType: mal_url; State: active; Org: Petersburg Internet Network ltd.; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b0aee6bf-32f4-4f65-8de6-f65e04e92b15", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2804,6 +2886,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484350; iType: mal_url; State: active; Org: ColoCrossing; Source: CyberCrime", + "threatintel.anomali.id": "indicator--54afbceb-72f3-484e-aee4-904f77beeff6", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2838,6 +2921,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484356; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--da030e10-af9f-462d-bda8-33abb223e950", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2873,6 +2957,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484343; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--d38e051a-bc5b-4723-884a-65e017d98299", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2907,6 +2992,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484367; iType: mal_url; State: active; Org: Petersburg Internet Network ltd.; Source: CyberCrime", + "threatintel.anomali.id": "indicator--46491826-6ba1-4217-a35e-1eb0081a9e6a", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2942,6 +3028,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484342; iType: mal_url; State: active; Org: SPRINTHOST.RU - shared/premium hosting, VDS, dedic; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b9715fd5-b89a-4859-b19f-55e052709227", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -2976,6 +3063,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484363; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--e3177515-f481-46c8-bad8-582ba0858ef3", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3011,6 +3099,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484339; iType: mal_url; State: active; Org: DDoS-GUARD GmbH; Source: CyberCrime", + "threatintel.anomali.id": "indicator--33cdeaeb-5201-4fbb-b9ae-9c23377e7533", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3045,6 +3134,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484351; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--2baaa5f0-c2f6-4bd1-b59d-3a75931da735", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3080,6 +3170,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484366; iType: mal_url; State: active; Org: World Hosting Farm Limited; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f1bdef49-666f-46b5-a323-efa1f1446b62", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3114,6 +3205,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484354; iType: mal_url; State: active; Org: McHost.Ru; Source: CyberCrime", + "threatintel.anomali.id": "indicator--a173f4b1-67ce-44f8-a6d0-bd8a24e8c593", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3149,6 +3241,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484362; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--b53dded1-d293-4cd1-9e63-b6e0cbd850f0", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3184,6 +3277,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484364; iType: mal_url; State: active; Org: World Hosting Farm Limited; Source: CyberCrime", + "threatintel.anomali.id": "indicator--2b30f8fe-13e8-4a7d-8eba-3e59c288bef7", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3218,6 +3312,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484357; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--f502199a-17a4-404b-a114-fb5eda28c32c", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3253,6 +3348,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484359; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--af7422eb-5d8e-4878-bdd1-395313434dae", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3288,6 +3384,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484358; iType: mal_url; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--71b36c05-86dd-4685-81c0-5a99e2e14c23", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3323,6 +3420,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484352; iType: mal_url; State: active; Org: Best-Hoster Group Co. Ltd.; Source: CyberCrime", + "threatintel.anomali.id": "indicator--9d948509-dfb4-45b6-b8bc-780df88a213f", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3358,6 +3456,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484224; iType: mal_ip; State: active; Org: Namecheap; Source: CyberCrime", + "threatintel.anomali.id": "indicator--9f613f8e-2040-4eee-8044-044023a8093e", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", @@ -3389,6 +3488,7 @@ "forwarded" ], "threatintel.anomali.description": "TS ID: 55253484361; iType: mal_url; State: active; Org: ServerMania; Source: CyberCrime", + "threatintel.anomali.id": "indicator--518c3959-6c26-413f-9a5f-c8f76d86185a", "threatintel.anomali.labels": [ "malicious-activity", "threatstream-severity-medium", diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index c0700f6b425..e28c6c1d9a7 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -6,7 +6,7 @@ interval: {{ .interval }} request.method: POST {{ if .ssl }} - - request.ssl: {{ .ssl | tojson }} +request.ssl: {{ .ssl | tojson }} {{ end }} request.url: {{ .url }} request.body: diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index e62a6e407d7..14868f968d3 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -38,16 +38,16 @@ processors: - remove: field: - threatintel.misp.ShadowAttribute - - message - threatintel.misp.RelatedEvent - threatintel.misp.Galaxy - threatintel.misp.Attribute.Galaxy - threatintel.misp.Attribute.ShadowAttribute - threatintel.misp.Object - threatintel.misp.EventReport + - message ignore_missing: true - date: - field: threatintel.misp.Attribute.timestamp + field: threatintel.misp.timestamp formats: - UNIX ignore_failure: true @@ -102,22 +102,22 @@ processors: field: threatintel.misp.attribute.value target_field: "threatintel.indicator.file.hash.{{threatintel.misp.attribute.type}}" ignore_missing: true - if: "ctx?.threatintel?.indicator?.type == 'file' && !ctx?.threatintel?.misp?.attribute?.type.startsWith('filename')" + if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type != null && !ctx?.threatintel?.misp?.attribute?.type.startsWith('filename')" - rename: field: threatintel.misp.attribute.value target_field: threatintel.indicator.file.name ignore_missing: true - if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type != 'filename'" + if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type == 'filename'" - grok: field: threatintel.misp.attribute.type patterns: - - "%{DATA}\\|%{DATA:_tmp.hashtype}" + - "%{WORD}\\|%{WORD:_tmp.hashtype}" ignore_missing: true if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') - grok: field: threatintel.misp.attribute.value patterns: - - "%{DATA:threatintel.indicator.file.name}\\|%{DATA:_tmp.hashvalue}" + - "%{DATA:threatintel.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}" ignore_missing: true if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') - set: @@ -129,7 +129,7 @@ processors: - set: field: threatintel.indicator.type value: url - if: "ctx?.threatintel?.indicator?.type == null && ['url', 'link', 'uri'].contains(ctx?.threatintel?.misp?.attribute?.type)" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx?.threatintel?.misp?.attribute?.type)" - uri_parts: field: threatintel.misp.attribute.value target_field: threatintel.indicator.url @@ -146,7 +146,7 @@ processors: - set: field: threatintel.indicator.type value: windows-registry-key - if: "ctx?.threatintel?.indicator?.type == null && ctx?.threatintel?.misp?.attribute?.type.startsWith('regkey')" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('regkey')" - rename: field: threatintel.misp.attribute.value target_field: threatintel.indicator.registry.key @@ -163,27 +163,33 @@ processors: - set: field: threatintel.indicator.type value: autonomous-system - if: "ctx?.threatintel?.indicator?.type == null && ctx?.threatintel?.misp?.attribute?.type == 'AS'" -- rename: + if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type == 'AS'" +- convert: field: threatintel.misp.attribute.value - target_field: threatintel.indicator.as + type: long + target_field: threatintel.indicator.as.number ignore_missing: true if: ctx?.threatintel?.indicator?.type == 'autonomous-system' ## Domain/IP/Port indicator operations -- append: +- set: field: threatintel.indicator.type value: domain-name - if: "ctx?.threatintel?.indicator?.type == null && ctx?.threatintel?.misp?.attribute?.type.startsWith('domain')" -- append: + if: "ctx?.threatintel?.misp?.attribute?.type != null && (ctx?.threatintel?.misp?.attribute?.type == 'hostname' || ctx?.threatintel?.misp?.attribute?.type.startsWith('domain'))" +- set: field: threatintel.indicator.type value: ipv4-addr - if: "ctx?.threatintel?.indicator?.type == null && ['domain|ip', 'ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)" - rename: field: threatintel.misp.attribute.value target_field: threatintel.indicator.domain ignore_missing: true if: "ctx?.threatintel?.indicator?.type == 'domain-name' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip'" +- rename: + field: threatintel.misp.attribute.value + target_field: threatintel.indicator.ip + ignore_missing: true + if: "ctx?.threatintel?.indicator?.type == 'ipv4-addr' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip' && !['ip-src|port', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)" - grok: field: threatintel.misp.attribute.value patterns: @@ -202,11 +208,11 @@ processors: - set: field: threatintel.indicator.type value: email-addr - if: "ctx?.threatintel?.indicator?.type == null && ['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)" - set: field: threatintel.indicator.type value: email-message - if: "ctx?.threatintel?.indicator?.type == null && ctx.threatintel?.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx.threatintel?.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)" - rename: field: threatintel.misp.attribute.value target_field: threatintel.indicator.email.address @@ -217,7 +223,7 @@ processors: - set: field: threatintel.indicator.type value: mac-addr - if: "ctx?.threatintel?.indicator?.type == null && ['mac-address', 'mac-eui-64'].contains(ctx.threatintel?.misp?.attribute?.type)" + if: "ctx?.threatintel?.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.threatintel?.misp?.attribute?.type)" - rename: field: threatintel.misp.attribute.value target_field: threatintel.indicator.mac @@ -241,7 +247,7 @@ processors: .collect(Collectors.toList()); ctx.tags = tags; - ctx.threatintel.indicator = ['marking' : [ 'tlp': tlpTags ]]; + ctx.threatintel.indicator.marking = [ 'tlp': tlpTags ]; # Setting indicator type to unknown if it does not match anything - set: @@ -277,6 +283,11 @@ processors: } handleMap(ctx); # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event +- remove: + field: + - threatintel.misp.attribute.value + ignore_missing: true + if: ctx?.threatintel?.indicator?.type != 'unknown' - remove: field: - threatintel.misp.Attribute.timestamp @@ -285,6 +296,7 @@ processors: - threatintel.misp.org - threatintel.misp.analysis - _tmp + - json ignore_missing: true on_failure: diff --git a/x-pack/filebeat/module/threatintel/misp/test/misp_sample.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/misp/test/misp_sample.ndjson.log-expected.json index 660df12cb76..27638c4be7b 100644 --- a/x-pack/filebeat/module/threatintel/misp/test/misp_sample.ndjson.log-expected.json +++ b/x-pack/filebeat/module/threatintel/misp/test/misp_sample.ndjson.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2017-08-28T14:24:32.000Z", + "@timestamp": "2017-08-28T14:24:36.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -17,10 +17,13 @@ "malware_classification:malware-category=Ransomware", "osint:source-type=blog - post" ], + "threatintel.indicator.file.hash.md5": "f2679bdabe46e10edc6352fff3c829bc", "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "file", "threatintel.misp.attribute.category": "Payload delivery", "threatintel.misp.attribute.comment": "- Xchecked via VT: a683494fc0d017fd3b4638f8b84caaaac145cc28bc211bd7361723368b4bb21e", "threatintel.misp.attribute.deleted": false, @@ -55,7 +58,7 @@ "threatintel.misp.uuid": "59a3d08d-5dc8-4153-bc7c-456d950d210f" }, { - "@timestamp": "2018-11-19T18:34:42.000Z", + "@timestamp": "2017-08-28T14:24:36.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -72,10 +75,14 @@ "malware_classification:malware-category=Ransomware", "osint:source-type=blog - post" ], + "threatintel.indicator.domain": "your-ip.getmyip.com", + "threatintel.indicator.ip": "178.128.103.74", "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "domain-name", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "1st stage", "threatintel.misp.attribute.deleted": false, @@ -88,7 +95,6 @@ "threatintel.misp.attribute.timestamp": "1542652482", "threatintel.misp.attribute.to_ids": false, "threatintel.misp.attribute.type": "domain|ip", - "threatintel.misp.attribute.value": "your-ip.getmyip.com|178.128.103.74", "threatintel.misp.attribute_count": "7", "threatintel.misp.date": "2017-08-25", "threatintel.misp.disable_correlation": false, @@ -111,7 +117,7 @@ "threatintel.misp.uuid": "59a3d08d-5dc8-4153-bc7c-456d950d210f" }, { - "@timestamp": "2017-03-30T12:55:50.000Z", + "@timestamp": "2017-04-28T18:23:44.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -128,7 +134,13 @@ "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "www.virustotal.com", + "threatintel.indicator.url.original": "https://www.virustotal.com/file/7fa4482bfbca550ce296d8e791b1091d60d733ea8042167fd0eb853530584452/analysis/1486030116/", + "threatintel.indicator.url.path": "/file/7fa4482bfbca550ce296d8e791b1091d60d733ea8042167fd0eb853530584452/analysis/1486030116/", + "threatintel.indicator.url.scheme": "https", "threatintel.misp.attribute.category": "External analysis", "threatintel.misp.attribute.comment": "Carbon sample - Xchecked via VT: a08b8371ead1919500a4759c2f46553620d5a9d9", "threatintel.misp.attribute.deleted": false, @@ -163,7 +175,7 @@ "threatintel.misp.uuid": "58dcfe62-ed84-4e5e-b293-4991950d210f" }, { - "@timestamp": "2014-10-06T07:09:54.000Z", + "@timestamp": "2014-10-06T07:12:57.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -177,10 +189,13 @@ "type:OSINT", "tlp:green" ], + "threatintel.indicator.file.hash.sha256": "0a1103bc90725d4665b932f88e81d39eafa5823b0de3ab146e2d4548b7da79a0", "threatintel.indicator.marking.tlp": [ "green" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "file", "threatintel.misp.attribute.category": "External analysis", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, @@ -215,7 +230,7 @@ "threatintel.misp.uuid": "54323f2c-e50c-4268-896c-4867950d210b" }, { - "@timestamp": "2014-10-06T07:10:57.000Z", + "@timestamp": "2014-10-06T07:12:57.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -229,10 +244,13 @@ "type:OSINT", "tlp:green" ], + "threatintel.indicator.ip": "223.25.233.248", "threatintel.indicator.marking.tlp": [ "green" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "ipv4-addr", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, @@ -245,7 +263,6 @@ "threatintel.misp.attribute.timestamp": "1412579457", "threatintel.misp.attribute.to_ids": true, "threatintel.misp.attribute.type": "ip-dst", - "threatintel.misp.attribute.value": "223.25.233.248", "threatintel.misp.attribute_count": "29", "threatintel.misp.date": "2014-10-03", "threatintel.misp.disable_correlation": false, @@ -268,7 +285,7 @@ "threatintel.misp.uuid": "54323f2c-e50c-4268-896c-4867950d210b" }, { - "@timestamp": "2014-10-06T07:12:28.000Z", + "@timestamp": "2014-10-06T07:12:57.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -282,10 +299,13 @@ "type:OSINT", "tlp:green" ], + "threatintel.indicator.domain": "xenserver.ddns.net", "threatintel.indicator.marking.tlp": [ "green" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "domain-name", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, @@ -298,7 +318,6 @@ "threatintel.misp.attribute.timestamp": "1412579548", "threatintel.misp.attribute.to_ids": true, "threatintel.misp.attribute.type": "hostname", - "threatintel.misp.attribute.value": "xenserver.ddns.net", "threatintel.misp.attribute_count": "29", "threatintel.misp.date": "2014-10-03", "threatintel.misp.disable_correlation": false, @@ -338,6 +357,8 @@ "threatintel.indicator.marking.tlp": [ "green" ], + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, "threatintel.indicator.type": "unknown", "threatintel.misp.attribute.category": "External analysis", "threatintel.misp.attribute.comment": "", @@ -374,7 +395,7 @@ "threatintel.misp.uuid": "54323f2c-e50c-4268-896c-4867950d210b" }, { - "@timestamp": "2016-02-18T20:12:23.000Z", + "@timestamp": "2014-10-06T07:12:57.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -388,10 +409,13 @@ "type:OSINT", "tlp:green" ], + "threatintel.indicator.file.hash.sha1": "0ea76f1586c008932d90c991dfdd5042f3aac8ea", "threatintel.indicator.marking.tlp": [ "green" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "file", "threatintel.misp.attribute.category": "External analysis", "threatintel.misp.attribute.comment": "Automatically added (via 7915aabb2e66ff14841e4ef0fbff7486)", "threatintel.misp.attribute.deleted": false, @@ -426,7 +450,7 @@ "threatintel.misp.uuid": "54323f2c-e50c-4268-896c-4867950d210b" }, { - "@timestamp": "2016-05-05T13:29:23.000Z", + "@timestamp": "2014-10-06T07:12:57.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -440,10 +464,13 @@ "type:OSINT", "tlp:green" ], + "threatintel.indicator.domain": "whatsapp.com", "threatintel.indicator.marking.tlp": [ "green" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "domain-name", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, @@ -456,7 +483,6 @@ "threatintel.misp.attribute.timestamp": "1462454963", "threatintel.misp.attribute.to_ids": false, "threatintel.misp.attribute.type": "domain", - "threatintel.misp.attribute.value": "whatsapp.com", "threatintel.misp.attribute_count": "29", "threatintel.misp.date": "2014-10-03", "threatintel.misp.disable_correlation": false, @@ -479,7 +505,7 @@ "threatintel.misp.uuid": "54323f2c-e50c-4268-896c-4867950d210b" }, { - "@timestamp": "2018-01-08T16:08:12.000Z", + "@timestamp": "2018-08-28T13:20:17.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -497,7 +523,14 @@ "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 0, + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "get.adobe.com", + "threatintel.indicator.url.original": "http://get.adobe.com/stats/AbfFcBebD/?q=", + "threatintel.indicator.url.path": "/stats/AbfFcBebD/", + "threatintel.indicator.url.query": "q=", + "threatintel.indicator.url.scheme": "http", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "Fake adobe URL", "threatintel.misp.attribute.deleted": false, @@ -532,7 +565,7 @@ "threatintel.misp.uuid": "5a5395d1-40a0-45fc-b692-334a0a016219" }, { - "@timestamp": "2018-01-08T16:31:29.000Z", + "@timestamp": "2018-08-28T13:20:17.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -550,7 +583,9 @@ "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 0, + "threatintel.indicator.type": "url", "threatintel.misp.attribute.category": "Network activity", "threatintel.misp.attribute.comment": "Win32 backdoor C&C URI", "threatintel.misp.attribute.deleted": false, @@ -563,7 +598,6 @@ "threatintel.misp.attribute.timestamp": "1515429089", "threatintel.misp.attribute.to_ids": false, "threatintel.misp.attribute.type": "uri", - "threatintel.misp.attribute.value": "/scripts/m/query.php?id=", "threatintel.misp.attribute_count": "61", "threatintel.misp.date": "2018-01-08", "threatintel.misp.disable_correlation": false, @@ -586,7 +620,7 @@ "threatintel.misp.uuid": "5a5395d1-40a0-45fc-b692-334a0a016219" }, { - "@timestamp": "2018-01-08T16:31:29.000Z", + "@timestamp": "2018-08-28T13:20:17.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -601,10 +635,14 @@ "Turla", "tlp:white" ], + "threatintel.indicator.file.hash.sha1": "c51d288469df9f25e2fb7ac491918b3e579282ea", + "threatintel.indicator.file.name": "google_update_checker.js", "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 0, + "threatintel.indicator.type": "file", "threatintel.misp.attribute.category": "Artifacts dropped", "threatintel.misp.attribute.comment": "JavaScript backdoor", "threatintel.misp.attribute.deleted": false, @@ -639,7 +677,7 @@ "threatintel.misp.uuid": "5a5395d1-40a0-45fc-b692-334a0a016219" }, { - "@timestamp": "2016-02-23T22:27:02.000Z", + "@timestamp": "2018-01-23T16:09:56.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -652,10 +690,13 @@ "tags": [ "tlp:white" ], + "threatintel.indicator.email.address": "claudiobonadio88@gmail.com", "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "email-addr", "threatintel.misp.attribute.category": "Payload delivery", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, @@ -690,7 +731,7 @@ "threatintel.misp.uuid": "56ccdcaf-f7e4-40d8-bca1-51299062e56a" }, { - "@timestamp": "2016-02-23T22:27:34.000Z", + "@timestamp": "2018-01-23T16:09:56.000Z", "event.category": "threat", "event.dataset": "threatintel.misp", "event.kind": "enrichment", @@ -706,7 +747,10 @@ "threatintel.indicator.marking.tlp": [ "white" ], - "threatintel.indicator.type": "unknown", + "threatintel.indicator.provider": "misp", + "threatintel.indicator.registry.key": "HKLM\\SOFTWARE\\Microsoft\\Active", + "threatintel.indicator.scanner_stats": 2, + "threatintel.indicator.type": "windows-registry-key", "threatintel.misp.attribute.category": "Artifacts dropped", "threatintel.misp.attribute.comment": "", "threatintel.misp.attribute.deleted": false, diff --git a/x-pack/filebeat/module/threatintel/otx/config/config.yml b/x-pack/filebeat/module/threatintel/otx/config/config.yml index 42af0a0c8e1..252c64a21f4 100644 --- a/x-pack/filebeat/module/threatintel/otx/config/config.yml +++ b/x-pack/filebeat/module/threatintel/otx/config/config.yml @@ -6,7 +6,10 @@ interval: {{ .interval }} request.method: GET {{ if .ssl }} - - request.ssl: {{ .ssl | tojson }} +request.ssl: {{ .ssl | tojson }} +{{ end }} +{{ if .http_client_timeout }} +request.timeout: {{ .http_client_timeout }} {{ end }} request.url: {{ .url }} request.transforms: @@ -56,8 +59,10 @@ publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - decode_json_fields: fields: [message] - document_id: id target: json + - fingerprint: + fields: ["json.id"] + target_field: "@metadata._id" - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml index 08ce44a43d7..ffd95787726 100644 --- a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml @@ -78,7 +78,7 @@ processors: - set: field: threatintel.indicator.type value: url - if: "ctx?.threatintel?.indicator?.type == null && ['url', 'uri'].contains(ctx.threatintel?.otx?.type)" + if: "ctx?.threatintel?.indicator?.type == null && ['URL', 'URI'].contains(ctx.threatintel?.otx?.type)" - uri_parts: field: threatintel.otx.indicator target_field: threatintel.indicator.url @@ -94,7 +94,7 @@ processors: field: threatintel.otx.indicator target_field: threatintel.indicator.url.path ignore_missing: true - if: "ctx?.threatintel?.otx?.type == 'uri'" + if: "ctx?.threatintel?.otx?.type == 'URI'" ## Email indicator operations - set: @@ -111,7 +111,7 @@ processors: - set: field: threatintel.indicator.type value: domain-name - if: ctx.threatintel?.otx?.type == 'domain' + if: "ctx?.threatintel?.indicator?.type == null && ['domain', 'hostname'].contains(ctx.threatintel?.otx?.type)" - rename: field: threatintel.otx.indicator target_field: threatintel.indicator.domain @@ -149,6 +149,11 @@ processors: } } handleMap(ctx); +- remove: + field: + - threatintel.otx.content + ignore_missing: true + if: ctx?.threatintel?.otx?.content == "" - remove: field: - threatintel.otx.type diff --git a/x-pack/filebeat/module/threatintel/otx/manifest.yml b/x-pack/filebeat/module/threatintel/otx/manifest.yml index 5bc84d42da3..c17efa499e9 100644 --- a/x-pack/filebeat/module/threatintel/otx/manifest.yml +++ b/x-pack/filebeat/module/threatintel/otx/manifest.yml @@ -9,7 +9,10 @@ var: default: 24h - name: api_token - name: ssl + - name: http_client_timeout + default: 120s - name: types + default: "domain,IPv4,hostname,url,FileHash-SHA256,FileHash-MD5" - name: lookback_range default: 2h - name: url diff --git a/x-pack/filebeat/module/threatintel/otx/test/otx_sample.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/otx/test/otx_sample.ndjson.log-expected.json index e49896b9dea..ca9e4425b46 100644 --- a/x-pack/filebeat/module/threatintel/otx/test/otx_sample.ndjson.log-expected.json +++ b/x-pack/filebeat/module/threatintel/otx/test/otx_sample.ndjson.log-expected.json @@ -14,8 +14,7 @@ "forwarded" ], "threatintel.indicator.ip": "86.104.194.30", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -33,7 +32,6 @@ ], "threatintel.indicator.file.hash.md5": "90421f8531f963d81cf54245b72cde80", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of a5725af4391d21a232dc6d4ad33d7d915bd190bdac9b1826b73f364dc5c1aa65", "threatintel.otx.title": "Win32:Hoblig-B" }, @@ -51,9 +49,8 @@ "threatintel-otx", "forwarded" ], - "threatintel.indicator.type": "unknown", - "threatintel.otx.content": "", - "threatintel.otx.indicator": "ip.anysrc.net" + "threatintel.indicator.domain": "ip.anysrc.net", + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -70,8 +67,7 @@ "forwarded" ], "threatintel.indicator.ip": "107.173.58.176", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -88,8 +84,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "d8c70ca70fd3555a0828fede6cc1f59e2c320ede80157039b6a2f09c336d5f7a", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -107,7 +102,6 @@ ], "threatintel.indicator.file.hash.md5": "f8e58af3ffefd4037fef246e93a55dc8", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of df9b37477a83189cd4541674e64ce29bf7bf98338ed0d635276660e0c6419d09" }, { @@ -125,8 +119,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "1c62f004d0c9b91d3467b1b8106772e667e7e2075470c2ec7982b63573c90c54", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -144,7 +137,6 @@ ], "threatintel.indicator.file.hash.sha256": "8d24a14f2600482d0231396b6350cf21773335ec2f0b8919763317fdab78baae", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -162,8 +154,7 @@ "forwarded" ], "threatintel.indicator.ip": "213.252.244.38", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -180,8 +171,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "c758ec922b173820374e552c2f015ac53cc5d9f99cc92080e608652aaa63695b", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -198,8 +188,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "0df586aa0334dcbe047d24ce859d00e537fdb5e0ca41886dab27479b6fc61ba6", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -217,7 +206,6 @@ ], "threatintel.indicator.file.hash.md5": "aeb08b0651bc8a13dcf5e5f6c0d482f8", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 0df586aa0334dcbe047d24ce859d00e537fdb5e0ca41886dab27479b6fc61ba6" }, { @@ -235,8 +223,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "6df5e1a017dff52020c7ff6ad92fdd37494e31769e1be242f6b23d1ea2d60140", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -253,8 +240,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "c72fef3835f65cb380f6920b22c3488554d1af6d298562ccee92284f265c9619", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -271,8 +257,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "e711fcd0f182b214c6ec74011a395f4c853068d59eb7c57f90c4a3e1de64434a", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -289,8 +274,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "d3ec8f4a46b21fb189fc3d58f3d87bf9897653ecdf90b7952dcc71f3b4023b4e", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -307,8 +291,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "70447996722e5c04514d20b7a429d162b46546002fb0c87f512b40f16bac99bb", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -326,7 +309,6 @@ ], "threatintel.indicator.file.hash.md5": "29340643ca2e6677c19e1d3bf351d654", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 113af75f13547be184822f1268f984b79f35965a1b1f963d23b50a09741b0aec", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -346,7 +328,6 @@ ], "threatintel.indicator.file.hash.md5": "86c314bc2dc37ba84f7364acd5108c2b", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 9b86a50b36aea5cc4cb60573a3660cf799a9ec1f69a3d4572d3dc277361a0ad2", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -366,7 +347,6 @@ ], "threatintel.indicator.file.hash.md5": "cb0c1248d3899358a375888bb4e8f3fe", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 1455091954ecf9ccd6fe60cb8e982d9cfb4b3dc8414443ccfdfc444079829d56", "threatintel.otx.title": "Trojan:Win32/Occamy.B" }, @@ -386,7 +366,6 @@ ], "threatintel.indicator.file.hash.md5": "d348f536e214a47655af387408b4fca5", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 3012f472969327d5f8c9dac63b8ea9c5cb0de002d16c120a6bba4685120f58b4", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -406,7 +385,6 @@ ], "threatintel.indicator.file.hash.sha256": "29ff1903832827e328ad9ec05fdf268eadd6db8b613597cf65f8740c211be413", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "vad_contains_network_strings" }, { @@ -424,8 +402,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "b105891f90b2a8730bbadf02b5adeccbba539883bf75dec2ff7a5a97625dd222", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -442,8 +419,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "e4db5405ac7ab517d43722e1ca8d653ea4a32802bc8a5410d032275eedc7b7ee", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -461,7 +437,6 @@ ], "threatintel.indicator.file.hash.sha256": "465e7c1e36899284da5c4425dfd687af2496f397fe60c85ea2b4d85dff5a08aa", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win.Malware.TrickbotSystemInfo-6335590-0" }, { @@ -479,8 +454,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "5051906d6ed1b2ae9c9a9f070ef73c9be8f591d2e41d144649a0dc96e28d0400", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -498,7 +472,6 @@ ], "threatintel.indicator.file.hash.md5": "14b74cb9be8cad8eb5fa8842d00bb692", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 465e7c1e36899284da5c4425dfd687af2496f397fe60c85ea2b4d85dff5a08aa", "threatintel.otx.title": "Win.Malware.TrickbotSystemInfo-6335590-0" }, @@ -518,7 +491,6 @@ ], "threatintel.indicator.file.hash.sha1": "a5b59f7d133e354dfc73f40517aab730f322f0ef", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "SHA1 of 465e7c1e36899284da5c4425dfd687af2496f397fe60c85ea2b4d85dff5a08aa", "threatintel.otx.title": "Win.Malware.TrickbotSystemInfo-6335590-0" }, @@ -537,8 +509,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "8d3f68b16f0710f858d8c1d2c699260e6f43161a5510abb0e7ba567bd72c965b", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -556,7 +527,6 @@ ], "threatintel.indicator.file.hash.md5": "ff2dcea4963e060a658f4dffbb119529", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 5cb822616d2c9435c9ddd060d6abdbc286ab57cfcf6dc64768c52976029a925b", "threatintel.otx.title": "vad_contains_network_strings" }, @@ -576,7 +546,6 @@ ], "threatintel.indicator.file.hash.md5": "0d73f1a1c4b2f8723fffc83eb3d00f31", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 29ff1903832827e328ad9ec05fdf268eadd6db8b613597cf65f8740c211be413", "threatintel.otx.title": "vad_contains_network_strings" }, @@ -595,8 +564,7 @@ "forwarded" ], "threatintel.indicator.ip": "185.25.50.167", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -613,8 +581,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "d35a30264c0698709ad554489004e0077e263d354ced0c54552a0b500f91ecc0", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -631,8 +598,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "5264b455f453820be629a324196131492ff03c80491e823ac06657c9387250dd", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -650,7 +616,6 @@ ], "threatintel.indicator.file.hash.sha256": "1455091954ecf9ccd6fe60cb8e982d9cfb4b3dc8414443ccfdfc444079829d56", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Trojan:Win32/Occamy.B" }, { @@ -669,7 +634,6 @@ ], "threatintel.indicator.file.hash.sha256": "3012f472969327d5f8c9dac63b8ea9c5cb0de002d16c120a6bba4685120f58b4", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -687,8 +651,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "b8e463789a076b16a90d1aae73cea9d3880ac0ead1fd16587b8cd79e37a1a3d8", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -706,7 +669,6 @@ ], "threatintel.indicator.file.hash.sha256": "113af75f13547be184822f1268f984b79f35965a1b1f963d23b50a09741b0aec", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -725,7 +687,6 @@ ], "threatintel.indicator.file.hash.sha256": "9b86a50b36aea5cc4cb60573a3660cf799a9ec1f69a3d4572d3dc277361a0ad2", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -743,8 +704,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "c51024bb119211c335f95e731cfa9a744fcdb645a57d35fb379d01b7dbdd098e", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -762,7 +722,6 @@ ], "threatintel.indicator.file.hash.sha1": "ad20c6fac565f901c82a21b70f9739037eb54818", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "SHA1 of 9b86a50b36aea5cc4cb60573a3660cf799a9ec1f69a3d4572d3dc277361a0ad2", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -782,7 +741,6 @@ ], "threatintel.indicator.file.hash.sha1": "13f11e273f9a4a56557f03821c3bfd591cca6ebc", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "SHA1 of 3012f472969327d5f8c9dac63b8ea9c5cb0de002d16c120a6bba4685120f58b4", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -802,7 +760,6 @@ ], "threatintel.indicator.file.hash.sha1": "1581fe76e3c96dc33182daafd09c8cf5c17004e0", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "SHA1 of 113af75f13547be184822f1268f984b79f35965a1b1f963d23b50a09741b0aec", "threatintel.otx.title": "Win64:Malware-gen" }, @@ -822,7 +779,6 @@ ], "threatintel.indicator.file.hash.sha1": "b72e75e9e901a44b655a5cf89cf0eadcaff46037", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "SHA1 of 1455091954ecf9ccd6fe60cb8e982d9cfb4b3dc8414443ccfdfc444079829d56", "threatintel.otx.title": "Trojan:Win32/Occamy.B" }, @@ -841,8 +797,7 @@ "forwarded" ], "threatintel.indicator.domain": "maper.info", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -859,8 +814,7 @@ "forwarded" ], "threatintel.indicator.ip": "213.252.244.126", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -877,8 +831,7 @@ "forwarded" ], "threatintel.indicator.ip": "78.129.139.131", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -896,7 +849,6 @@ ], "threatintel.indicator.file.hash.sha256": "9af8a93519d22ed04ffb9ccf6861c9df1b77dc5d22e0aeaff4a582dbf8660ba6", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -915,7 +867,6 @@ ], "threatintel.indicator.file.hash.sha256": "be9fb556a3c7aef0329e768d7f903e7dd42a821abc663e11fb637ce33b007087", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -934,7 +885,6 @@ ], "threatintel.indicator.file.hash.sha256": "3bfec096c4837d1e6485fe0ae0ea6f1c0b44edc611d4f2204cc9cf73c985cbc2", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -953,7 +903,6 @@ ], "threatintel.indicator.file.hash.sha256": "dff2e39b2e008ea89a3d6b36dcd9b8c927fb501d60c1ad5a52ed1ffe225da2e2", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -972,7 +921,6 @@ ], "threatintel.indicator.file.hash.sha256": "6b4d271a48d118843aee3dee4481fa2930732ed7075db3241a8991418f00d92b", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -991,7 +939,6 @@ ], "threatintel.indicator.file.hash.sha256": "26de4265303491bed1424d85b263481ac153c2b3513f9ee48ffb42c12312ac43", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -1010,7 +957,6 @@ ], "threatintel.indicator.file.hash.sha256": "02f54da6c6f2f87ff7b713d46e058dedac1cedabd693643bb7f6dfe994b2105d", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "xor_0x20_xord_javascript" }, { @@ -1028,8 +974,7 @@ "forwarded" ], "threatintel.indicator.ip": "103.13.67.4", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1046,8 +991,7 @@ "forwarded" ], "threatintel.indicator.ip": "80.90.87.201", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1064,8 +1008,7 @@ "forwarded" ], "threatintel.indicator.ip": "80.80.163.182", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1082,8 +1025,7 @@ "forwarded" ], "threatintel.indicator.ip": "91.187.114.210", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1100,8 +1042,7 @@ "forwarded" ], "threatintel.indicator.ip": "170.238.117.187", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1118,8 +1059,7 @@ "forwarded" ], "threatintel.indicator.file.hash.sha256": "e999b83629355ec7ff3b6fda465ef53ce6992c9327344fbf124f7eb37808389d", - "threatintel.indicator.type": "file", - "threatintel.otx.content": "" + "threatintel.indicator.type": "file" }, { "event.category": "threat", @@ -1136,8 +1076,7 @@ "forwarded" ], "threatintel.indicator.ip": "103.84.238.3", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1154,8 +1093,7 @@ "forwarded" ], "threatintel.indicator.ip": "179.43.158.171", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1172,8 +1110,7 @@ "forwarded" ], "threatintel.indicator.ip": "198.211.116.199", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1191,7 +1128,6 @@ ], "threatintel.indicator.ip": "203.176.135.102", "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "", "threatintel.otx.title": "Trickbot" }, { @@ -1209,8 +1145,7 @@ "forwarded" ], "threatintel.indicator.domain": "fotmailz.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1227,8 +1162,7 @@ "forwarded" ], "threatintel.indicator.domain": "pori89g5jqo3v8.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1245,8 +1179,7 @@ "forwarded" ], "threatintel.indicator.domain": "sebco.co.ke", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1264,7 +1197,6 @@ ], "threatintel.indicator.ip": "177.74.232.124", "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "", "threatintel.otx.title": "Trickbot" }, { @@ -1282,8 +1214,7 @@ "forwarded" ], "threatintel.indicator.domain": "chishir.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1300,8 +1231,7 @@ "forwarded" ], "threatintel.indicator.domain": "kostunivo.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1318,8 +1248,7 @@ "forwarded" ], "threatintel.indicator.domain": "mangoclone.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1336,8 +1265,7 @@ "forwarded" ], "threatintel.indicator.domain": "onixcellent.com", - "threatintel.indicator.type": "domain-name", - "threatintel.otx.content": "" + "threatintel.indicator.type": "domain-name" }, { "event.category": "threat", @@ -1355,7 +1283,6 @@ ], "threatintel.indicator.file.hash.sha1": "fc0efd612ad528795472e99cae5944b68b8e26dc", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -1374,7 +1301,6 @@ ], "threatintel.indicator.file.hash.sha1": "24d4bbc982a6a561f0426a683b9617de1a96a74a", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Sf:ShellCode-DZ\\ [Trj]" }, { @@ -1393,7 +1319,6 @@ ], "threatintel.indicator.file.hash.sha1": "fa98074dc18ad7e2d357b5d168c00a91256d87d1", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -1412,7 +1337,6 @@ ], "threatintel.indicator.file.hash.sha1": "e5dc7c8bfa285b61dda1618f0ade9c256be75d1a", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.title": "Win64:Malware-gen" }, { @@ -1431,7 +1355,6 @@ ], "threatintel.indicator.ip": "96.9.77.142", "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "", "threatintel.otx.title": "Trickbot" }, { @@ -1449,8 +1372,7 @@ "forwarded" ], "threatintel.indicator.ip": "36.89.106.69", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1467,8 +1389,7 @@ "forwarded" ], "threatintel.indicator.ip": "96.9.73.73", - "threatintel.indicator.type": "ipv4-addr", - "threatintel.otx.content": "" + "threatintel.indicator.type": "ipv4-addr" }, { "event.category": "threat", @@ -1486,7 +1407,6 @@ ], "threatintel.indicator.file.hash.md5": "10ec3571596c30b9993b89f12d29d23c", "threatintel.indicator.type": "file", - "threatintel.otx.content": "", "threatintel.otx.description": "MD5 of 9af8a93519d22ed04ffb9ccf6861c9df1b77dc5d22e0aeaff4a582dbf8660ba6", "threatintel.otx.title": "xor_0x20_xord_javascript" } diff --git a/x-pack/filebeat/modules.d/threatintel.yml.disabled b/x-pack/filebeat/modules.d/threatintel.yml.disabled index 3e03fee654f..14cffd52531 100644 --- a/x-pack/filebeat/modules.d/threatintel.yml.disabled +++ b/x-pack/filebeat/modules.d/threatintel.yml.disabled @@ -12,7 +12,7 @@ var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m abusemalware: enabled: true @@ -24,7 +24,7 @@ var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ # The interval to poll the API for updates. - var.interval: 60m + var.interval: 10m misp: enabled: true @@ -38,6 +38,10 @@ # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. var.api_token: API_KEY + # Configures the type of SSL verification done, if MISP is running on self signed certificates + # then the certificate would either need to be trusted, or verification_mode set to none. + #var.ssl.verification_mode: none + # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. # For examples please reference the filebeat module documentation. #var.filters: @@ -46,10 +50,10 @@ # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer # than the last event that was already ingested. - var.first_interval: 24h + var.first_interval: 300h # The interval to poll the API for updates. - var.interval: 60m + var.interval: 5m otx: enabled: true @@ -66,14 +70,17 @@ # Optional filters that can be applied to retrieve only specific indicators. #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" + # The timeout of the HTTP client connecting to the OTX API + #var.http_client_timeout: 120s + # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. - var.lookback_range: 2h + var.lookback_range: 1h # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m anomali: enabled: true @@ -81,7 +88,8 @@ # Input used for ingesting threat intel data var.input: httpjson - # The URL used for Threat Intel API calls. + # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending + # on the type of threat intel source that is needed. var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects # The Username used by anomali Limo, defaults to guest. @@ -91,7 +99,7 @@ #var.password: guest # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 24h + var.first_interval: 400h # The interval to poll the API for updates - var.interval: 60m + var.interval: 5m From 3ca53aa150b0a47fdea218628596d57200f6a096 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 19 Feb 2021 19:19:07 -0500 Subject: [PATCH 08/12] [Filebeat] Document netflow internal_networks and set default (#24110) Documentation for the `internal_networks` option of the Netflow input and module was missing. Also the module's manifest did not declare the option so if it was not set in the module config it would cause an error. I did not see where a default was set for the netflow input's internal_networks option so I set that to `private` to keep the old behavior before this was configurable. Fixes #24094 --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/modules/netflow.asciidoc | 7 +++++++ x-pack/filebeat/docs/inputs/input-netflow.asciidoc | 11 +++++++++++ x-pack/filebeat/input/netflow/config.go | 1 + x-pack/filebeat/module/netflow/_meta/docs.asciidoc | 7 +++++++ x-pack/filebeat/module/netflow/log/config/netflow.yml | 2 +- x-pack/filebeat/module/netflow/log/manifest.yml | 1 + 7 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c5dfe0cac9c..10b788744bc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -387,6 +387,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709] - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920] - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904] +- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110] *Heartbeat* diff --git a/filebeat/docs/modules/netflow.asciidoc b/filebeat/docs/modules/netflow.asciidoc index ebb40dfd5c9..c3ab408b24d 100644 --- a/filebeat/docs/modules/netflow.asciidoc +++ b/filebeat/docs/modules/netflow.asciidoc @@ -72,6 +72,13 @@ details. monitor sequence numbers in the Netflow packets to detect an Exporting Process reset. See <> for details. +`var.internal_networks`:: A list of CIDR ranges describing the IP addresses that +you consider internal. This is used in determining the values of +`source.locality`, `destination.locality`, and `flow.locality`. The values +can be either a CIDR value or one of the named ranges supported by the +<> condition. The default value is `[private]` +which classifies RFC 1918 (IPv4) and RFC 4193 (IPv6) addresses as internal. + *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/x-pack/filebeat/docs/inputs/input-netflow.asciidoc b/x-pack/filebeat/docs/inputs/input-netflow.asciidoc index 840ad70ec05..b53881cc961 100644 --- a/x-pack/filebeat/docs/inputs/input-netflow.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-netflow.asciidoc @@ -120,6 +120,17 @@ cause flow loss until the exporter provides new templates. If set to `false`, if the exporter process is reset. This option is only applicable to Netflow V9 and IPFIX. Default is `true`. +[float] +[[internal_networks]] +==== `internal_networks` + +A list of CIDR ranges describing the IP addresses that you consider internal. +This is used in determining the values of `source.locality`, +`destination.locality`, and `flow.locality`. The values can be either a CIDR +value or one of the named ranges supported by the +<> condition. The default value is `[private]` +which classifies RFC 1918 (IPv4) and RFC 4193 (IPv6) addresses as internal. + [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/netflow/config.go b/x-pack/filebeat/input/netflow/config.go index 4d795a44eec..b13b6722ab6 100644 --- a/x-pack/filebeat/input/netflow/config.go +++ b/x-pack/filebeat/input/netflow/config.go @@ -33,6 +33,7 @@ var defaultConfig = config{ ForwarderConfig: harvester.ForwarderConfig{ Type: inputName, }, + InternalNetworks: []string{"private"}, Protocols: []string{"v5", "v9", "ipfix"}, ExpirationTimeout: time.Minute * 30, PacketQueueSize: 8192, diff --git a/x-pack/filebeat/module/netflow/_meta/docs.asciidoc b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc index 830b397ec45..09ffda3d024 100644 --- a/x-pack/filebeat/module/netflow/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc @@ -67,6 +67,13 @@ details. monitor sequence numbers in the Netflow packets to detect an Exporting Process reset. See <> for details. +`var.internal_networks`:: A list of CIDR ranges describing the IP addresses that +you consider internal. This is used in determining the values of +`source.locality`, `destination.locality`, and `flow.locality`. The values +can be either a CIDR value or one of the named ranges supported by the +<> condition. The default value is `[private]` +which classifies RFC 1918 (IPv4) and RFC 4193 (IPv6) addresses as internal. + *`var.tags`*:: A list of tags to include in events. Including `forwarded` indicates that the diff --git a/x-pack/filebeat/module/netflow/log/config/netflow.yml b/x-pack/filebeat/module/netflow/log/config/netflow.yml index 65baa78eaac..dd111c35097 100644 --- a/x-pack/filebeat/module/netflow/log/config/netflow.yml +++ b/x-pack/filebeat/module/netflow/log/config/netflow.yml @@ -6,7 +6,7 @@ expiration_timeout: '{{.expiration_timeout}}' queue_size: {{.queue_size}} {{if .internal_networks}} -internal_hosts: +internal_networks: {{range .internal_networks}} - '{{ . }}' {{end}} diff --git a/x-pack/filebeat/module/netflow/log/manifest.yml b/x-pack/filebeat/module/netflow/log/manifest.yml index e46428b2fc0..250c2b414e9 100644 --- a/x-pack/filebeat/module/netflow/log/manifest.yml +++ b/x-pack/filebeat/module/netflow/log/manifest.yml @@ -17,6 +17,7 @@ var: - name: detect_sequence_reset - name: tags default: [forwarded] + - name: internal_networks ingest_pipeline: ingest/pipeline.yml input: config/netflow.yml From e315d66b518acc2aa82b028532912e4bc8f5f40e Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 22 Feb 2021 11:23:51 +0100 Subject: [PATCH 09/12] indicator type url is in upper case (#24152) --- x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml index ffd95787726..a4a16035111 100644 --- a/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/otx/ingest/pipeline.yml @@ -89,7 +89,7 @@ processors: field: threatintel.otx.indicator target_field: threatintel.indicator.url.full ignore_missing: true - if: "ctx?.threatintel?.otx?.type == 'url' && ctx?.threatintel?.indicator?.url?.original == null" + if: "ctx?.threatintel?.otx?.type == 'URL' && ctx?.threatintel?.indicator?.url?.original == null" - rename: field: threatintel.otx.indicator target_field: threatintel.indicator.url.path From 7f92834108090aeb1bf8c61e304fcb28b3ad3b11 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 22 Feb 2021 15:10:48 +0200 Subject: [PATCH 10/12] [Agent] Add agent standalone manifests for system module & Pod's log collection (#23938) --- .../elastic-agent-standalone-kubernetes.yml | 245 ++++++++++++------ ...-agent-standalone-daemonset-configmap.yaml | 231 +++++++++++++++++ .../elastic-agent-standalone-daemonset.yaml | 70 +++++ ...agent-standalone-deployment-configmap.yaml | 161 ++++++++++++ .../elastic-agent-standalone-deployment.yaml | 58 +++++ ...elastic-agent-standalone-role-binding.yaml | 12 + .../elastic-agent-standalone-role.yaml | 35 +++ ...stic-agent-standalone-service-account.yaml | 7 + 8 files changed, 739 insertions(+), 80 deletions(-) create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index f99281b6889..46b625fa0ce 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -23,10 +23,10 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME @@ -52,11 +52,35 @@ spec: mountPath: /etc/agent.yml readOnly: true subPath: agent.yml + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: datastreams configMap: defaultMode: 0640 name: agent-node-datastreams + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log --- apiVersion: v1 kind: ConfigMap @@ -67,8 +91,6 @@ metadata: k8s-app: elastic-agent data: agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -87,9 +109,120 @@ data: node: ${NODE_NAME} scope: node inputs: - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f8 - name: kubernetes-node-metrics - revision: 1 + - name: log-1 + type: logfile + use_output: default + meta: + package: + name: log + version: 0.4.6 + data_stream: + namespace: default + streams: + - data_stream: + dataset: generic + symlinks: true + paths: + - /var/log/containers/*${kubernetes.container.id}.log + - name: system-3 + type: system/metrics + use_output: default + meta: + package: + name: system + version: 0.10.9 + data_stream: + namespace: default + streams: + - data_stream: + dataset: system.core + type: metrics + metricsets: + - core + core.metrics: + - percentages + - data_stream: + dataset: system.cpu + type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages + metricsets: + - cpu + - data_stream: + dataset: system.diskio + type: metrics + period: 10s + diskio.include_devices: null + metricsets: + - diskio + - data_stream: + dataset: system.filesystem + type: metrics + period: 1m + metricsets: + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.fsstat + type: metrics + period: 1m + metricsets: + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.load + type: metrics + period: 10s + metricsets: + - load + - data_stream: + dataset: system.memory + type: metrics + period: 10s + metricsets: + - memory + - data_stream: + dataset: system.network + type: metrics + period: 10s + network.interfaces: null + metricsets: + - network + - data_stream: + dataset: system.process + type: metrics + process.include_top_n.by_memory: 5 + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.cgroups.enabled: false + process.cmdline.cache.enabled: true + metricsets: + - process + process.include_cpu_ticks: false + system.hostfs: /hostfs + - data_stream: + dataset: system.process_summary + type: metrics + period: 10s + metricsets: + - process_summary + system.hostfs: /hostfs + - data_stream: + dataset: system.socket_summary + type: metrics + period: 10s + metricsets: + - socket_summary + system.hostfs: /hostfs + - name: kubernetes-node-metrics type: kubernetes/metrics use_output: default meta: @@ -99,9 +232,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.controllermanager-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.controllermanager type: metrics metricsets: @@ -110,9 +241,7 @@ data: - '${kubernetes.pod.ip}:10252' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager' - - id: >- - kubernetes/metrics-kubernetes.scheduler-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.scheduler type: metrics metricsets: @@ -121,9 +250,7 @@ data: - '${kubernetes.pod.ip}:10251' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' - - id: >- - kubernetes/metrics-kubernetes.proxy-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.proxy type: metrics metricsets: @@ -131,9 +258,7 @@ data: hosts: - 'localhost:10249' period: 10s - - id: >- - kubernetes/metrics-kubernetes.container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.container type: metrics metricsets: @@ -144,9 +269,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.node type: metrics metricsets: @@ -157,8 +280,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: kubernetes/metrics-kubernetes.pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.pod type: metrics metricsets: @@ -169,9 +291,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.system-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.system type: metrics metricsets: @@ -182,9 +302,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.volume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.volume type: metrics metricsets: @@ -215,10 +333,10 @@ spec: serviceAccountName: elastic-agent containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME @@ -265,8 +383,6 @@ metadata: data: # This part requires `kube-state-metrics` up and running under `kube-system` namespace agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -282,9 +398,7 @@ data: logs: true metrics: true inputs: - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 - name: kubernetes-cluster-metrics - revision: 1 + - name: kubernetes-cluster-metrics type: kubernetes/metrics use_output: default meta: @@ -294,9 +408,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.apiserver type: metrics metricsets: @@ -307,18 +419,14 @@ data: period: 30s ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - id: >- - kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.event type: metrics metricsets: - event period: 10s add_metadata: true - - id: >- - kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_container type: metrics metricsets: @@ -327,9 +435,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_cronjob type: metrics metricsets: @@ -338,9 +444,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_deployment type: metrics metricsets: @@ -349,9 +453,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_node type: metrics metricsets: @@ -360,9 +462,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolume type: metrics metricsets: @@ -371,9 +471,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolumeclaim type: metrics metricsets: @@ -382,9 +480,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_pod type: metrics metricsets: @@ -393,9 +489,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_replicaset type: metrics metricsets: @@ -404,9 +498,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_resourcequota type: metrics metricsets: @@ -415,9 +507,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_service type: metrics metricsets: @@ -426,9 +516,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_statefulset type: metrics metricsets: @@ -437,9 +525,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_storageclass type: metrics metricsets: @@ -475,7 +561,6 @@ rules: - namespaces - events - pods - - secrets verbs: ["get", "list", "watch"] - apiGroups: ["extensions"] resources: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml new file mode 100644 index 00000000000..11c2992c93c --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -0,0 +1,231 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-node-datastreams + namespace: kube-system + labels: + k8s-app: elastic-agent +data: + agent.yml: |- + outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + providers.kubernetes: + node: ${NODE_NAME} + scope: node + inputs: + - name: log-1 + type: logfile + use_output: default + meta: + package: + name: log + version: 0.4.6 + data_stream: + namespace: default + streams: + - data_stream: + dataset: generic + symlinks: true + paths: + - /var/log/containers/*${kubernetes.container.id}.log + - name: system-3 + type: system/metrics + use_output: default + meta: + package: + name: system + version: 0.10.9 + data_stream: + namespace: default + streams: + - data_stream: + dataset: system.core + type: metrics + metricsets: + - core + core.metrics: + - percentages + - data_stream: + dataset: system.cpu + type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages + metricsets: + - cpu + - data_stream: + dataset: system.diskio + type: metrics + period: 10s + diskio.include_devices: null + metricsets: + - diskio + - data_stream: + dataset: system.filesystem + type: metrics + period: 1m + metricsets: + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.fsstat + type: metrics + period: 1m + metricsets: + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - data_stream: + dataset: system.load + type: metrics + period: 10s + metricsets: + - load + - data_stream: + dataset: system.memory + type: metrics + period: 10s + metricsets: + - memory + - data_stream: + dataset: system.network + type: metrics + period: 10s + network.interfaces: null + metricsets: + - network + - data_stream: + dataset: system.process + type: metrics + process.include_top_n.by_memory: 5 + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.cgroups.enabled: false + process.cmdline.cache.enabled: true + metricsets: + - process + process.include_cpu_ticks: false + system.hostfs: /hostfs + - data_stream: + dataset: system.process_summary + type: metrics + period: 10s + metricsets: + - process_summary + system.hostfs: /hostfs + - data_stream: + dataset: system.socket_summary + type: metrics + period: 10s + metricsets: + - socket_summary + system.hostfs: /hostfs + - name: kubernetes-node-metrics + type: kubernetes/metrics + use_output: default + meta: + package: + name: kubernetes + version: 0.2.8 + data_stream: + namespace: default + streams: + - data_stream: + dataset: kubernetes.controllermanager + type: metrics + metricsets: + - controllermanager + hosts: + - '${kubernetes.pod.ip}:10252' + period: 10s + condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager' + - data_stream: + dataset: kubernetes.scheduler + type: metrics + metricsets: + - scheduler + hosts: + - '${kubernetes.pod.ip}:10251' + period: 10s + condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' + - data_stream: + dataset: kubernetes.proxy + type: metrics + metricsets: + - proxy + hosts: + - 'localhost:10249' + period: 10s + - data_stream: + dataset: kubernetes.container + type: metrics + metricsets: + - container + add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.NODE_NAME}:10250' + period: 10s + ssl.verification_mode: none + - data_stream: + dataset: kubernetes.node + type: metrics + metricsets: + - node + add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.NODE_NAME}:10250' + period: 10s + ssl.verification_mode: none + - data_stream: + dataset: kubernetes.pod + type: metrics + metricsets: + - pod + add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.NODE_NAME}:10250' + period: 10s + ssl.verification_mode: none + - data_stream: + dataset: kubernetes.system + type: metrics + metricsets: + - system + add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.NODE_NAME}:10250' + period: 10s + ssl.verification_mode: none + - data_stream: + dataset: kubernetes.volume + type: metrics + metricsets: + - volume + add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.NODE_NAME}:10250' + period: 10s + ssl.verification_mode: none diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml new file mode 100644 index 00000000000..e97e0743926 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: elastic-agent + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: elastic-agent + image: docker.elastic.co/beats/elastic-agent:%VERSION% + args: [ + "-c", "/etc/agent.yml", + "-e", + ] + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + value: "" + - name: ES_HOST + value: "" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: datastreams + mountPath: /etc/agent.yml + readOnly: true + subPath: agent.yml + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: datastreams + configMap: + defaultMode: 0640 + name: agent-node-datastreams + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml new file mode 100644 index 00000000000..58a92665e4e --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml @@ -0,0 +1,161 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-deployment-datastreams + namespace: kube-system + labels: + k8s-app: elastic-agent +data: + # This part requires `kube-state-metrics` up and running under `kube-system` namespace + agent.yml: |- + outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + inputs: + - name: kubernetes-cluster-metrics + type: kubernetes/metrics + use_output: default + meta: + package: + name: kubernetes + version: 0.2.8 + data_stream: + namespace: default + streams: + - data_stream: + dataset: kubernetes.apiserver + type: metrics + metricsets: + - apiserver + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}' + period: 30s + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - data_stream: + dataset: kubernetes.event + type: metrics + metricsets: + - event + period: 10s + add_metadata: true + - data_stream: + dataset: kubernetes.state_container + type: metrics + metricsets: + - state_container + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_cronjob + type: metrics + metricsets: + - state_cronjob + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_deployment + type: metrics + metricsets: + - state_deployment + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_node + type: metrics + metricsets: + - state_node + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_persistentvolume + type: metrics + metricsets: + - state_persistentvolume + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_persistentvolumeclaim + type: metrics + metricsets: + - state_persistentvolumeclaim + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_pod + type: metrics + metricsets: + - state_pod + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_replicaset + type: metrics + metricsets: + - state_replicaset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_resourcequota + type: metrics + metricsets: + - state_resourcequota + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_service + type: metrics + metricsets: + - state_service + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_statefulset + type: metrics + metricsets: + - state_statefulset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - data_stream: + dataset: kubernetes.state_storageclass + type: metrics + metricsets: + - state_storageclass + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml new file mode 100644 index 00000000000..0def8b88571 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + serviceAccountName: elastic-agent + containers: + - name: elastic-agent + image: docker.elastic.co/beats/elastic-agent:%VERSION% + args: [ + "-c", "/etc/agent.yml", + "-e", + ] + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + value: "" + - name: ES_HOST + value: "" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # this is needed because we cannot use hostNetwork + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: datastreams + mountPath: /etc/agent.yml + readOnly: true + subPath: agent.yml + volumes: + - name: datastreams + configMap: + defaultMode: 0640 + name: agent-deployment-datastreams diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml new file mode 100644 index 00000000000..b352b2901d0 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: ClusterRole + name: elastic-agent + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml new file mode 100644 index 00000000000..dcf2b4a5ff2 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-agent + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + # required for apiserver + - nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml new file mode 100644 index 00000000000..43372b547d0 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent From daa447144a894612a3d603730ee5b40a7c302a91 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Feb 2021 14:54:25 +0000 Subject: [PATCH 11/12] [PACKAGING] Push docker images with the architecture in the version (#24121) --- .ci/packaging.groovy | 71 ++++++++++++++++++++++------------ .ci/scripts/docker-tag-push.sh | 12 ++++++ Jenkinsfile | 67 +++++++++++++++++++------------- 3 files changed, 98 insertions(+), 52 deletions(-) create mode 100755 .ci/scripts/docker-tag-push.sh diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 1fb61253808..ad0fef0575d 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -152,7 +152,9 @@ pipeline { withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { deleteDir() release() - pushCIDockerImages() + dir("${BASE_DIR}"){ + pushCIDockerImages(arch: 'amd64') + } } prepareE2ETestForPackage("${BEATS_FOLDER}") } @@ -234,7 +236,9 @@ pipeline { withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") { deleteWorkspace() release() - pushCIDockerImages() + dir("${BASE_DIR}"){ + pushCIDockerImages(arch: 'arm64') + } } } post { @@ -247,7 +251,6 @@ pipeline { } } } - /* stage('Run E2E Tests for Packages'){ agent { label 'ubuntu-18 && immutable' } options { skipDefaultCheckout() } @@ -255,7 +258,6 @@ pipeline { runE2ETests() } } - */ } post { success { @@ -272,27 +274,37 @@ pipeline { } } -def pushCIDockerImages(){ +/** +* @param arch what architecture +*/ +def pushCIDockerImages(Map args = [:]) { + def arch = args.get('arch', 'amd64') catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { if (env?.BEATS_FOLDER?.endsWith('auditbeat')) { - tagAndPush('auditbeat') + tagAndPush(beatName: 'auditbeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) { - tagAndPush('filebeat') + tagAndPush(beatName: 'filebeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) { - tagAndPush('heartbeat') + tagAndPush(beatName: 'heartbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "journalbeat"){ - tagAndPush('journalbeat') + tagAndPush(beatName: 'journalbeat', arch: arch) } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) { - tagAndPush('metricbeat') + tagAndPush(beatName: 'metricbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "packetbeat"){ - tagAndPush('packetbeat') + tagAndPush(beatName: 'packetbeat', arch: arch) } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") { - tagAndPush('elastic-agent') + tagAndPush(beatName: 'elastic-agent', arch: arch) } } } -def tagAndPush(beatName){ +/** +* @param beatName name of the Beat +* @param arch what architecture +*/ +def tagAndPush(Map args = [:]) { + def beatName = args.beatName + def arch = args.get('arch', 'amd64') def libbetaVer = env.BEAT_VERSION def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ @@ -309,14 +321,22 @@ def tagAndPush(beatName){ dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + // supported tags + def tags = [tagName, "${env.GIT_BASE_COMMIT}"] + if (!isPR() && aliasVersion != "") { + tags << aliasVersion + } // supported image flavours def variants = ["", "-oss", "-ubi8"] variants.each { variant -> - doTagAndPush(beatName, variant, libbetaVer, tagName) - doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - - if (!isPR() && aliasVersion != "") { - doTagAndPush(beatName, variant, libbetaVer, aliasVersion) + tags.each { tag -> + // TODO: + // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until + // they support the versioning with the architecture + if ("${arch}" == "amd64") { + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}") + } + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}") } } } @@ -327,18 +347,19 @@ def tagAndPush(beatName){ * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace */ -def doTagAndPush(beatName, variant, sourceTag, targetTag) { +def doTagAndPush(Map args = [:]) { + def beatName = args.beatName + def variant = args.variant + def sourceTag = args.sourceTag + def targetTag = args.targetTag def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}" def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}" - def iterations = 0 retryWithSleep(retries: 3, seconds: 5, backoff: true) { iterations++ - def status = sh(label: "Change tag and push ${targetName}", script: """ - docker tag ${sourceName} ${targetName} - docker push ${targetName} - """, returnStatus: true) - + def status = sh(label: "Change tag and push ${targetName}", + script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}", + returnStatus: true) if ( status > 0 && iterations < 3) { error("tag and push failed for ${beatName}, retry") } else if ( status > 0 ) { diff --git a/.ci/scripts/docker-tag-push.sh b/.ci/scripts/docker-tag-push.sh new file mode 100755 index 00000000000..49886422c34 --- /dev/null +++ b/.ci/scripts/docker-tag-push.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -exuo pipefail +MSG="parameter missing." +SOURCE_IMAGE=${1:?$MSG} +TARGET_IMAGE=${2:?$MSG} + +if docker image inspect "${SOURCE_IMAGE}" &> /dev/null ; then + docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}" + docker push "${TARGET_IMAGE}" +else + echo "docker image ${SOURCE_IMAGE} does not exist" +fi diff --git a/Jenkinsfile b/Jenkinsfile index 8e8d67c782b..b317251ce53 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -321,23 +321,26 @@ def uploadPackages(bucketUri, beatsFolder){ /** * Push the docker images for the given beat. * @param beatsFolder beats folder +* @param arch what architecture */ -def pushCIDockerImages(beatsFolder){ +def pushCIDockerImages(Map args = [:]) { + def arch = args.get('arch', 'amd64') + def beatsFolder = args.beatsFolder catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') { if (beatsFolder.endsWith('auditbeat')) { - tagAndPush('auditbeat') + tagAndPush(beatName: 'auditbeat', arch: arch) } else if (beatsFolder.endsWith('filebeat')) { - tagAndPush('filebeat') + tagAndPush(beatName: 'filebeat', arch: arch) } else if (beatsFolder.endsWith('heartbeat')) { - tagAndPush('heartbeat') + tagAndPush(beatName: 'heartbeat', arch: arch) } else if ("${beatsFolder}" == "journalbeat"){ - tagAndPush('journalbeat') + tagAndPush(beatName: 'journalbeat', arch: arch) } else if (beatsFolder.endsWith('metricbeat')) { - tagAndPush('metricbeat') + tagAndPush(beatName: 'metricbeat', arch: arch) } else if ("${beatsFolder}" == "packetbeat"){ - tagAndPush('packetbeat') + tagAndPush(beatName: 'packetbeat', arch: arch) } else if ("${beatsFolder}" == "x-pack/elastic-agent") { - tagAndPush('elastic-agent') + tagAndPush(beatName: 'elastic-agent', arch: arch) } } } @@ -346,7 +349,9 @@ def pushCIDockerImages(beatsFolder){ * Tag and push all the docker images for the given beat. * @param beatName name of the Beat */ -def tagAndPush(beatName){ +def tagAndPush(Map args = [:]) { + def beatName = args.beatName + def arch = args.get('arch', 'amd64') def libbetaVer = env.VERSION if("${env?.SNAPSHOT.trim()}" == "true"){ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version @@ -360,41 +365,40 @@ def tagAndPush(beatName){ tagName = "pr-${env.CHANGE_ID}" } + // supported tags + def tags = [tagName, "${env.GIT_BASE_COMMIT}"] + if (!isPR() && aliasVersion != "") { + tags << aliasVersion + } // supported image flavours def variants = ["", "-oss", "-ubi8"] variants.each { variant -> - doTagAndPush(beatName, variant, libbetaVer, tagName) - doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - - if (!isPR() && aliasVersion != "") { - doTagAndPush(beatName, variant, libbetaVer, aliasVersion) + tags.each { tag -> + doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}") } } } /** -* Tag and push the given sourceTag docker image with the tag name targetTag. * @param beatName name of the Beat * @param variant name of the variant used to build the docker image name * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace */ -def doTagAndPush(beatName, variant, sourceTag, targetTag) { +def doTagAndPush(Map args = [:]) { + def beatName = args.beatName + def variant = args.variant + def sourceTag = args.sourceTag + def targetTag = args.targetTag def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}" def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}" def iterations = 0 retryWithSleep(retries: 3, seconds: 5, backoff: true) { iterations++ - def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash - docker images - if docker image inspect "${sourceName}" &> /dev/null ; then - docker tag ${sourceName} ${targetName} - docker push ${targetName} - else - echo 'docker image ${sourceName} does not exist' - fi - """, returnStatus: true) + def status = sh(label: "Change tag and push ${targetName}", + script: ".ci/scripts/docker-tag-push.sh ${sourceName} ${targetName}", + returnStatus: true) if ( status > 0 && iterations < 3) { error("tag and push failed for ${beatName}, retry") } else if ( status > 0 ) { @@ -462,6 +466,7 @@ def target(Map args = [:]) { def isMage = args.get('isMage', false) def isE2E = args.e2e?.get('enabled', false) def isPackaging = args.get('package', false) + def dockerArch = args.get('dockerArch', 'amd64') withNode(args.label) { withGithubNotify(context: "${context}") { withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { @@ -482,7 +487,7 @@ def target(Map args = [:]) { // TODO: // push docker images should happen only after the e2e? if (isPackaging) { - pushCIDockerImages("${directory}") + pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch) } } } @@ -918,7 +923,15 @@ class RunCommand extends co.elastic.beats.BeatsFunction { steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) } if(args?.content?.containsKey('packaging-linux')) { - steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true) + steps.packagingLinux(context: args.context, + command: args.content.get('packaging-linux'), + directory: args.project, + label: args.label, + isMage: true, + id: args.id, + e2e: args.content.get('e2e'), + package: true, + dockerArch: 'amd64') } if(args?.content?.containsKey('k8sTest')) { steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) From 239f2438348c5550980ca2c50b164d18d9bba64b Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 22 Feb 2021 11:15:44 -0500 Subject: [PATCH 12/12] [Elastic Agent] Fix docker entrypoint for elastic-agent. (#24155) * Fix docker entrypoint for elastic-agent. * Add changelog entry. --- .../templates/docker/docker-entrypoint.elastic-agent.tmpl | 2 +- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 348a99dea4c..f1e6febfe8a 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -63,7 +63,7 @@ function enroll(){ insecure_flag="--insecure" fi - ./{{ .BeatName }} enroll ${insecure_flag} ${KIBANA_HOST:-http://localhost:5601} $apikey -f + ./{{ .BeatName }} enroll ${insecure_flag} -f --url=${KIBANA_HOST:-http://localhost:5601} --enrollment-token=$apikey } if [[ -n "${FLEET_SETUP}" ]] && [[ ${FLEET_SETUP} == 1 ]]; then setup; fi diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 3f0419ca38d..cb53ef11824 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -40,6 +40,7 @@ - Fix libbeat from reporting back degraded on config update {pull}23537[23537] - Fix issues with dynamic inputs and conditions {pull}23886[23886] - Fix bad substitution of API key. {pull}24036[24036] +- Fix docker enrollment issue related to Fleet Server change. {pull}24155[24155] ==== New features