diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 3162b9d8047..7af3ff6f60c 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -339,6 +339,7 @@ def triggerE2ETests(String suite) { booleanParam(name: 'forceSkipGitChecks', value: true), booleanParam(name: 'forceSkipPresubmit', value: true), booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()), + booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true), string(name: 'runTestsSuites', value: suite), string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME), string(name: 'GITHUB_CHECK_REPO', value: env.REPO), @@ -346,7 +347,6 @@ def triggerE2ETests(String suite) { ] if (isPR()) { def version = "pr-${env.CHANGE_ID}" - parameters.push(booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true)) parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}")) parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}")) } diff --git a/.ci/schedule-weekly.groovy b/.ci/schedule-weekly.groovy index df6511a3435..29cd9223537 100644 --- a/.ci/schedule-weekly.groovy +++ b/.ci/schedule-weekly.groovy @@ -20,8 +20,8 @@ pipeline { stages { stage('Nighly beats builds') { steps { - build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)]) - build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)]) + build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false) + build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false) } } } diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat index 839cb745fba..6892e38e5fe 100644 --- a/.ci/scripts/install-tools.bat +++ b/.ci/scripts/install-tools.bat @@ -79,4 +79,4 @@ gcc --version where gcc REM Reset the USERPROFILE -SET USERPROFILE=%PREVIOUS_USERPROFILE% \ No newline at end of file +SET USERPROFILE=%PREVIOUS_USERPROFILE% diff --git a/.go-version b/.go-version index 4ed70fac17d..545fd574d35 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.14.12 +1.15.7 diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 4d40f0c99b3..51a4565e4ff 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -106,3 +106,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827] - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267] - Add support for customized monitoring API. {pull}22605[22605] +- Update Go version to 1.15.7. {pull}22495[22495] diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cb98ddbb781..9da8d021506 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,22 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-7.10.2]] +=== Beats version 7.10.2 +https://github.com/elastic/beats/compare/v7.10.1\...v7.10.2[View commits] + +==== Bugfixes + +*Filebeat* + +- Add JSON body check for SQS message. {pull}21727[21727] +- Fix cisco umbrella module config by adding input variable. {pull}22892[22892] +- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] + +*Metricbeat* + +- Change `vsphere.datastore.capacity.used.pct` value to betweeen 0 and 1. {pull}23148[23148] + [[release-notes-7.10.1]] === Beats version 7.10.1 https://github.com/elastic/beats/compare/v7.10.0\...v7.10.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1382447376c..4ba723374fe 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571] - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] +- Rename `s3` input to `aws-s3` input. {pull}23469[23469] *Heartbeat* - Adds negative body match. {pull}20728[20728] @@ -230,6 +231,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183] - Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154] - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419] +- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484] *Auditbeat* @@ -267,6 +269,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523] - Fix various processing errors in the Suricata module. {pull}23236[23236] - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] +- Change the `event.created` in Netflow events to be the time the event was created by Filebeat + to be consistent with ECS. {pull}23094[23094] +- Update `filestream` reader offset when a line is skipped. {pull}23417[23417] *Filebeat* @@ -363,6 +368,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126] - Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204] - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273] +- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] *Heartbeat* @@ -479,9 +485,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove io.time from windows {pull}22237[22237] - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148] - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327] +- Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286] +- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] *Packetbeat* +- Fix SIP parser logic related to line length check. {pull}23411[23411] + *Winlogbeat* @@ -493,6 +503,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* +*Elastic Logging Plugin* +- Fix out of date CLI flags on docs. {pull}23628[23628] + ==== Added @@ -572,6 +585,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Honor kube event resysncs to handle missed watch events {pull}22668[22668] - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] +- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] +- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] +- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] +- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] *Auditbeat* @@ -796,8 +813,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Google Workspace module and mark Gsuite module as deprecated {pull}22950[22950] - Mark m365 defender, defender atp, okta and google workspace modules as GA {pull}23113[23113] - Added support for first_event context in filebeat httpjson input {pull}23437[23437] +- Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157] - Added `alternative_host` option to google pubsub input {pull}23215[23215] - Added username parsing from Cisco ASA message 302013. {pull}21196[21196] +- Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478] +- Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521] +- Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521] *Heartbeat* @@ -961,6 +982,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add additional event categorization for security and sysmon modules. {pull}22988[22988] - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999] - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046] +- Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684] *Elastic Log Driver* @@ -970,6 +992,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- Selecting `full` in `ssl.verification_mode` option will not treat CommonName field in x509 certificates as + a hostname when Subject Alternative Name is not present from v8.0. + Please update your certificates so it contains at least one DNSName instead of relying on CommonName in the new + major version of Beats. + *Filebeat* - The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed. diff --git a/Jenkinsfile b/Jenkinsfile index f062ee1bde4..072f2c3176a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,6 +80,7 @@ pipeline { whenTrue(env.ONLY_DOCS == 'false') { cmd(label: "make check-python", script: "make check-python") cmd(label: "make check-go", script: "make check-go") + cmd(label: "make notice", script: "make notice") cmd(label: "Check for changes", script: "make check-no-changes") } } diff --git a/NOTICE.txt b/NOTICE.txt index 27137916fa1..5a6c3df026e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2216,11 +2216,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/sarama -Version: v1.19.1-0.20200629123429-0e7b69039eec +Version: v1.19.1-0.20210120173147-5c8cb347d877 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20200629123429-0e7b69039eec/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20210120173147-5c8cb347d877/LICENSE: Copyright (c) 2013 Shopify @@ -5757,6 +5757,38 @@ SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/eapache/go-resiliency +Version: v1.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + -------------------------------------------------------------------------------- Dependency : github.com/eclipse/paho.mqtt.golang Version: v1.2.1-0.20200121105743-0d940dd29fd2 @@ -8478,11 +8510,11 @@ limitations under the License. -------------------------------------------------------------------------------- Dependency : github.com/fatih/color -Version: v1.5.0 +Version: v1.9.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/fatih/color@v1.5.0/LICENSE.md: +Contents of probable licence file $GOMODCACHE/github.com/fatih/color@v1.9.0/LICENSE.md: The MIT License (MIT) @@ -8581,191 +8613,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/garyburd/redigo -Version: v1.0.1-0.20160525165706-b8dc90050f24 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/garyburd/redigo@v1.0.1-0.20160525165706-b8dc90050f24/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - -------------------------------------------------------------------------------- Dependency : github.com/go-sql-driver/mysql Version: v1.4.1 @@ -9672,6 +9519,193 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/gomodule/redigo +Version: v1.8.3 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gomodule/redigo@v1.8.3/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + -------------------------------------------------------------------------------- Dependency : github.com/google/flatbuffers Version: v1.7.2-0.20170925184458-7a6b2bf521e9 @@ -9886,11 +9920,11 @@ Contents of probable licence file $GOMODCACHE/github.com/google/flatbuffers@v1.7 -------------------------------------------------------------------------------- Dependency : github.com/google/go-cmp -Version: v0.4.0 +Version: v0.5.2 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.4.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/google/go-cmp@v0.5.2/LICENSE: Copyright (c) 2017 The Go Authors. All rights reserved. @@ -11745,36 +11779,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/jstemmer/go-junit-report -Version: v0.9.1 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/jstemmer/go-junit-report@v0.9.1/LICENSE: - -Copyright (c) 2012 Joel Stemmer - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/blakerouse/service Version: v1.1.1-0.20200924160513-057808572ffa @@ -12036,11 +12040,11 @@ Contents of probable licence file $GOMODCACHE/github.com/magefile/mage@v1.10.0/L -------------------------------------------------------------------------------- Dependency : github.com/mattn/go-colorable -Version: v0.1.4 +Version: v0.1.6 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mattn/go-colorable@v0.1.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/mattn/go-colorable@v0.1.6/LICENSE: The MIT License (MIT) @@ -13704,11 +13708,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/prometheus@v -------------------------------------------------------------------------------- Dependency : github.com/rcrowley/go-metrics -Version: v0.0.0-20190826022208-cac0b30c2563 +Version: v0.0.0-20200313005456-10cdbea86bc0 Licence type (autodetected): BSD-2-Clause-FreeBSD -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20190826022208-cac0b30c2563/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20200313005456-10cdbea86bc0/LICENSE: Copyright 2012 Richard Crowley. All rights reserved. @@ -13809,11 +13813,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/shirou/gopsutil -Version: v2.19.11+incompatible +Version: v3.20.12+incompatible Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/shirou/gopsutil@v2.19.11+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/shirou/gopsutil@v3.20.12+incompatible/LICENSE: gopsutil is distributed under BSD license reproduced below. @@ -16075,11 +16079,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.0.0-20200622213623-75b288015ac9 +Version: v0.0.0-20200820211705-5c72a883971a Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200622213623-75b288015ac9/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200820211705-5c72a883971a/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -16149,11 +16153,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.0.0-20200707034311-ab3426394381 +Version: v0.0.0-20200904194848-62affa334b73 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20200707034311-ab3426394381/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20200904194848-62affa334b73/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -16260,11 +16264,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.0.0-20200625212154-ddb9806d33ae +Version: v0.0.0-20201009025420-dfb3f7c4e634 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20200625212154-ddb9806d33ae/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.0.0-20201009025420-dfb3f7c4e634/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -17422,6 +17426,218 @@ See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gotest.tools/gotestsum +Version: v0.6.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gotest.tools/gotestsum@v0.6.0/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : howett.net/plist Version: v0.0.0-20181124034731-591f970eefbb @@ -24752,6 +24968,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/creack/pty +Version: v1.1.9 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/creack/pty@v1.1.9/LICENSE: + +Copyright (c) 2011 Keith Rarick + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/cucumber/godog Version: v0.8.1 @@ -26023,38 +26272,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/eapache/go-resiliency -Version: v1.2.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Evan Huus - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - - -------------------------------------------------------------------------------- Dependency : github.com/eapache/go-xerial-snappy Version: v0.0.0-20180814174437-776d5712da21 @@ -26893,11 +27110,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.7.2 +Version: v1.10.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.10.2/LICENSE: MIT License @@ -30543,6 +30760,218 @@ Contents of probable licence file $GOMODCACHE/github.com/google/renameio@v0.1.0/ limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/google/shlex +Version: v0.0.0-20191202100458-e7afc7fbc510 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/google/shlex@v0.0.0-20191202100458-e7afc7fbc510/COPYING: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/google/subcommands Version: v1.0.1 @@ -34412,6 +34841,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/jstemmer/go-junit-report +Version: v0.9.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/jstemmer/go-junit-report@v0.9.1/LICENSE: + +Copyright (c) 2012 Joel Stemmer + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/jtolds/gls Version: v4.20.0+incompatible @@ -34614,11 +35073,11 @@ match.go, match_test.go: -------------------------------------------------------------------------------- Dependency : github.com/klauspost/compress -Version: v1.9.8 +Version: v1.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.9.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.11.0/LICENSE: Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2019 Klaus Post. All rights reserved. @@ -34693,11 +35152,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- Dependency : github.com/kr/pretty -Version: v0.2.0 +Version: v0.2.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.2.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.2.1/License: Copyright 2012 Keith Rarick @@ -34755,11 +35214,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/text -Version: v0.1.0 +Version: v0.2.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.1.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.2.0/License: Copyright 2012 Keith Rarick @@ -36236,6 +36695,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/niemeyer/pretty +Version: v0.0.0-20200227124842-a10e7caefd8e +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/niemeyer/pretty@v0.0.0-20200227124842-a10e7caefd8e/License: + +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/onsi/ginkgo Version: v1.11.0 @@ -37623,11 +38111,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/pierrec/lz4 -Version: v2.4.1+incompatible +Version: v2.5.2+incompatible Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.4.1+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.5.2+incompatible/LICENSE: Copyright (c) 2015, Pierre Curto All rights reserved. @@ -41381,11 +41869,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/xerrors -Version: v0.0.0-20191204190536-9bdfabe68543 +Version: v0.0.0-20200804184101-5ec99f83aff1 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20200804184101-5ec99f83aff1/LICENSE: Copyright (c) 2019 The Go Authors. All rights reserved. @@ -41696,11 +42184,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : gopkg.in/check.v1 -Version: v1.0.0-20190902080502-41f04d3bba15 +Version: v1.0.0-20200902074654-038fdea0a05b Licence type (autodetected): BSD-2-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20200902074654-038fdea0a05b/LICENSE: Gocheck - A rich testing framework for Go @@ -42719,11 +43207,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 -Version: v3.0.0-20200313102051-9f266ea9e77c +Version: v3.0.0-20200615113413-eeeca48fe776 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20200313102051-9f266ea9e77c/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776/LICENSE: This project is covered by two different licenses: MIT and Apache. @@ -42777,6 +43265,29 @@ See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Dependency : gotest.tools/v3 +Version: v3.0.3 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gotest.tools/v3@v3.0.3/LICENSE: + +Copyright 2018 gotest.tools authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + -------------------------------------------------------------------------------- Dependency : honnef.co/go/tools Version: v0.0.1-2019.2.3 diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index c6d4c0c0735..e5767ab1167 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index de0a4ae491f..3a022706d8c 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -36,10 +36,14 @@ stages: mage: "mage build test" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags crosscompile: make: "make -C auditbeat crosscompile" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -61,6 +65,8 @@ stages: #- "windows-2008-r2" https://github.com/elastic/beats/issues/19799 when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index ef82491a4dd..712ebf7ee67 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -529,6 +529,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -656,6 +662,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -844,6 +856,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -853,6 +869,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1009,6 +1031,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1305,6 +1333,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1502,6 +1536,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index 948d15c754f..a2e6582ca7e 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -18,7 +18,6 @@ package mage import ( - "bytes" "context" "fmt" "io" @@ -28,16 +27,14 @@ import ( "os/exec" "path" "path/filepath" - "runtime" - "sort" "strings" "time" - "github.com/jstemmer/go-junit-report/formatter" - "github.com/jstemmer/go-junit-report/parser" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // GoTestArgs are the arguments used for the "go*Test" targets and they define @@ -52,6 +49,7 @@ type GoTestArgs struct { OutputFile string // File to write verbose test output to. JUnitReportFile string // File to write a JUnit XML test report to. CoverageProfileFile string // Test coverage profile file (enables -cover). + Output io.Writer // Write stderr and stdout to Output if set } // TestBinaryArgs are the arguments used when building binary for testing. @@ -183,40 +181,80 @@ func GoTestIntegrationForModule(ctx context.Context) error { return nil } +// InstallGoTestTools installs additional tools that are required to run unit and integration tests. +func InstallGoTestTools() { + gotool.Install( + gotool.Install.Package("gotest.tools/gotestsum"), + ) +} + // GoTest invokes "go test" and reports the results to stdout. It returns an // error if there was any failure executing the tests or if there were any // test failures. func GoTest(ctx context.Context, params GoTestArgs) error { + mg.Deps(InstallGoTestTools) + fmt.Println(">> go test:", params.TestName, "Testing") - // Build args list to Go. - args := []string{"test"} - args = append(args, "-v") + // We use gotestsum to drive the tests and produce a junit report. + // The tool runs `go test -json` in order to produce a structured log which makes it easier + // to parse the actual test output. + // Of OutputFile is given the original JSON file will be written as well. + // + // The runner needs to set CLI flags for gotestsum and for "go test". We track the different + // CLI flags in the gotestsumArgs and testArgs variables, such that we can finally produce command like: + // $ gotestsum -- + // + // The additional arguments given via GoTestArgs are applied to `go test` only. Callers can not + // modify any of the gotestsum arguments. + + gotestsumArgs := []string{"--no-color"} + if mg.Verbose() { + gotestsumArgs = append(gotestsumArgs, "-f", "standard-verbose") + } else { + gotestsumArgs = append(gotestsumArgs, "-f", "standard-quiet") + } + if params.JUnitReportFile != "" { + CreateDir(params.JUnitReportFile) + gotestsumArgs = append(gotestsumArgs, "--junitfile", params.JUnitReportFile) + } + if params.OutputFile != "" { + CreateDir(params.OutputFile) + gotestsumArgs = append(gotestsumArgs, "--jsonfile", params.OutputFile+".json") + } + + var testArgs []string // -race is only supported on */amd64 if os.Getenv("DEV_ARCH") == "amd64" { if params.Race { - args = append(args, "-race") + testArgs = append(testArgs, "-race") } } if len(params.Tags) > 0 { - args = append(args, "-tags", strings.Join(params.Tags, " ")) + params := strings.Join(params.Tags, " ") + if params != "" { + testArgs = append(testArgs, "-tags", params) + } } if params.CoverageProfileFile != "" { params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile)) - args = append(args, + testArgs = append(testArgs, "-covermode=atomic", "-coverprofile="+params.CoverageProfileFile, ) } - args = append(args, params.ExtraFlags...) - args = append(args, params.Packages...) + testArgs = append(testArgs, params.ExtraFlags...) + testArgs = append(testArgs, params.Packages...) - goTest := makeCommand(ctx, params.Env, "go", args...) + args := append(gotestsumArgs, append([]string{"--"}, testArgs...)...) + goTest := makeCommand(ctx, params.Env, "gotestsum", args...) // Wire up the outputs. - bufferOutput := new(bytes.Buffer) - outputs := []io.Writer{bufferOutput} + var outputs []io.Writer + if params.Output != nil { + outputs = append(outputs, params.Output) + } if params.OutputFile != "" { fileOutput, err := os.Create(createDir(params.OutputFile)) @@ -227,18 +265,16 @@ func GoTest(ctx context.Context, params GoTestArgs) error { outputs = append(outputs, fileOutput) } output := io.MultiWriter(outputs...) - goTest.Stdout = output - goTest.Stderr = output - - if mg.Verbose() { + if params.Output == nil { goTest.Stdout = io.MultiWriter(output, os.Stdout) goTest.Stderr = io.MultiWriter(output, os.Stderr) + } else { + goTest.Stdout = output + goTest.Stderr = output } - // Execute 'go test' and measure duration. - start := time.Now() err := goTest.Run() - duration := time.Since(start) + var goTestErr *exec.ExitError if err != nil { // Command ran. @@ -251,30 +287,11 @@ func GoTest(ctx context.Context, params GoTestArgs) error { goTestErr = exitErr } - // Parse the verbose test output. - report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName) - if err != nil { - return errors.Wrap(err, "failed to parse go test output") - } - if goTestErr != nil && len(report.Packages) == 0 { + if goTestErr != nil { // No packages were tested. Probably the code didn't compile. - fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String()) return errors.Wrap(goTestErr, "go test returned a non-zero value") } - // Generate a JUnit XML report. - if params.JUnitReportFile != "" { - junitReport, err := os.Create(createDir(params.JUnitReportFile)) - if err != nil { - return errors.Wrap(err, "failed to create junit report") - } - defer junitReport.Close() - - if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil { - return errors.Wrap(err, "failed to write junit report") - } - } - // Generate a HTML code coverage report. var htmlCoverReport string if params.CoverageProfileFile != "" { @@ -288,27 +305,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error { } } - // Summarize the results and log to stdout. - summary, err := NewGoTestSummary(duration, report, map[string]string{ - "Output File": params.OutputFile, - "JUnit Report": params.JUnitReportFile, - "Coverage Report": htmlCoverReport, - }) - if err != nil { - return err - } - if !mg.Verbose() && summary.Fail > 0 { - fmt.Println(summary.Failures()) - } - fmt.Println(summary.String()) - // Return an error indicating that testing failed. - if summary.Fail > 0 || goTestErr != nil { + if goTestErr != nil { fmt.Println(">> go test:", params.TestName, "Test Failed") - if summary.Fail > 0 { - return errors.Errorf("go test failed: %d test failures", summary.Fail) - } - return errors.Wrap(goTestErr, "go test returned a non-zero value") } @@ -329,117 +328,10 @@ func makeCommand(ctx context.Context, env map[string]string, cmd string, args .. c.Stderr = os.Stderr c.Stdin = os.Stdin log.Println("exec:", cmd, strings.Join(args, " ")) + fmt.Println("exec:", cmd, strings.Join(args, " ")) return c } -// GoTestSummary is a summary of test results. -type GoTestSummary struct { - *parser.Report // Report generated by parsing test output. - Pass int // Number of passing tests. - Fail int // Number of failed tests. - Skip int // Number of skipped tests. - Packages int // Number of packages tested. - Duration time.Duration // Total go test running duration. - Files map[string]string -} - -// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot -// resolve the absolute paths to the given files. -func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) { - files := map[string]string{} - for name, file := range outputFiles { - if file == "" { - continue - } - absFile, err := filepath.Abs(file) - if err != nil { - return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file) - } - files[name+":"] = absFile - } - - summary := &GoTestSummary{ - Report: r, - Duration: d, - Packages: len(r.Packages), - Files: files, - } - - for _, pkg := range r.Packages { - for _, t := range pkg.Tests { - switch t.Result { - case parser.PASS: - summary.Pass++ - case parser.FAIL: - summary.Fail++ - case parser.SKIP: - summary.Skip++ - default: - return nil, errors.Errorf("Unknown test result value: %v", t.Result) - } - } - } - - return summary, nil -} - -// Failures returns a string containing the list of failed test cases and their -// output. -func (s *GoTestSummary) Failures() string { - b := new(strings.Builder) - - if s.Fail > 0 { - fmt.Fprintln(b, "FAILURES:") - for _, pkg := range s.Report.Packages { - for _, t := range pkg.Tests { - if t.Result != parser.FAIL { - continue - } - fmt.Fprintln(b, "Package:", pkg.Name) - fmt.Fprintln(b, "Test: ", t.Name) - for _, line := range t.Output { - if strings.TrimSpace(line) != "" { - fmt.Fprintln(b, line) - } - } - fmt.Fprintln(b, "----") - } - } - } - - return strings.TrimRight(b.String(), "\n") -} - -// String returns a summary of the testing results (number of fail/pass/skip, -// test duration, number packages, output files). -func (s *GoTestSummary) String() string { - b := new(strings.Builder) - - fmt.Fprintln(b, "SUMMARY:") - fmt.Fprintln(b, " Fail: ", s.Fail) - fmt.Fprintln(b, " Skip: ", s.Skip) - fmt.Fprintln(b, " Pass: ", s.Pass) - fmt.Fprintln(b, " Packages:", len(s.Report.Packages)) - fmt.Fprintln(b, " Duration:", s.Duration) - - // Sort the list of files and compute the column width. - var names []string - var nameWidth int - for name := range s.Files { - if len(name) > nameWidth { - nameWidth = len(name) - } - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - fmt.Fprintf(b, " %-*s %s\n", nameWidth, name, s.Files[name]) - } - - return strings.TrimRight(b.String(), "\n") -} - // BuildSystemTestBinary runs BuildSystemTestGoBinary with default values. func BuildSystemTestBinary() error { return BuildSystemTestGoBinary(DefaultTestBinaryArgs()) diff --git a/dev-tools/mage/gotest_test.go b/dev-tools/mage/gotest_test.go new file mode 100644 index 00000000000..24ce55614cb --- /dev/null +++ b/dev-tools/mage/gotest_test.go @@ -0,0 +1,382 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "context" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/magefile/mage/mg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const envGoTestHelper = "GOTEST_WANT_HELPER" + +var gotestHelperMode = os.Getenv(envGoTestHelper) == "1" + +// TestGoTest_CaptureOutput runs different `go test` scenarios via `GoTest` and +// captures the stderr and stdout output of the test run. The output is then +// validated using a regular expression. +// +// For each scenario a GoTest helper test is defined and a regular expression +// that the test output must match. The naming convention for scenario X is: +// - TestGoTest_Helper_: the test function to be executed +// - wantTest: regular expression the output must match. +// +// TestGoTest_CaptureOutput sets the `GOTEST_WANT_HELPER` environment variable when it executes the tests. +// each test helper must check if it is driven by this function or not: +// +// func TestGoTest_Helper_X(t *testing.T) { +// if !gotestHelperMode { +// return +// } +// +// // sample test +// } +// +func TestGoTest_CaptureOutput(t *testing.T) { + errNonZero := "go test returned a non-zero value" + makeArgs := func(test string) GoTestArgs { + return GoTestArgs{ + TestName: "asserts", + Packages: []string{"."}, + Env: map[string]string{envGoTestHelper: "1"}, + ExtraFlags: []string{"-test.run", test}, + } + } + + tests := map[string]struct { + args GoTestArgs + verbose bool + wantErr string + want string + }{ + "passing test without output": { + args: makeArgs("TestGoTest_Helper_OK"), + verbose: true, + want: wantTestOK, + }, + "capture output from assert failures": { + args: makeArgs("TestGoTest_Helper_AssertOutput"), + wantErr: errNonZero, + want: wantTestAssertOutput, + }, + "capture test log output": { + args: makeArgs("TestGoTest_Helper_LogOutput"), + wantErr: errNonZero, + want: wantTestLogOutput, + }, + "capture panic": { + args: makeArgs("TestGoTest_Helper_WithPanic"), + wantErr: errNonZero, + want: wantTestWithPanic, + }, + "capture wrong panic": { + args: makeArgs("TestGoTest_Helper_WithWrongPanic"), + wantErr: errNonZero, + want: wantTestWithWrongPanic, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + oldVerboseArg := os.Getenv(mg.VerboseEnv) + defer func() { + os.Setenv(mg.VerboseEnv, oldVerboseArg) + }() + + if test.verbose { + os.Setenv(mg.VerboseEnv, "true") + } else { + os.Setenv(mg.VerboseEnv, "false") + } + + var buf strings.Builder + args := test.args + args.Output = &buf + err := GoTest(context.TODO(), args) + + if test.wantErr == "" { + if err != nil { + t.Fatalf("GoTest did return an unexpected error: %v", err) + } + } else { + if err == nil { + t.Fatalf("GoTest was expected to return an error saying '%v'", test.wantErr) + } + + errString := err.Error() + if !strings.Contains(errString, test.wantErr) { + t.Fatalf("GoTest error does not match expected error message:\nwant: '%v'\ngot: '%v'", + test.wantErr, errString) + } + } + + re, err := regexp.Compile(test.want) + if err != nil { + t.Fatalf("Failed to compile test match regex: %v", err) + } + + output := buf.String() + if !re.MatchString(output) { + t.Fatalf("GoTest output missmatch:\nwant:\n%v\n\ngot:\n%v", test.want, output) + } + }) + } +} + +func TestGoTest_Helper_OK(t *testing.T) { + if !gotestHelperMode { + return + } + + // Succeeding test without any additional output or test logs. +} + +var wantTestOK = `--- PASS: TestGoTest_Helper_OK.*` + +func TestGoTest_Helper_AssertOutput(t *testing.T) { + if !gotestHelperMode { + return + } + + t.Run("assert fails", func(t *testing.T) { + assert.True(t, false) + }) + + t.Run("assert with message", func(t *testing.T) { + assert.True(t, false, "My message") + }) + + t.Run("assert with messagef", func(t *testing.T) { + assert.True(t, false, "My message with arguments: %v", 42) + }) + + t.Run("require fails", func(t *testing.T) { + require.True(t, false) + }) + + t.Run("require with message", func(t *testing.T) { + require.True(t, false, "My message") + }) + + t.Run("require with messagef", func(t *testing.T) { + require.True(t, false, "My message with arguments: %v", 42) + }) + + t.Run("equals map", func(t *testing.T) { + want := map[string]interface{}{ + "a": 1, + "b": true, + "c": "test", + "e": map[string]interface{}{ + "x": "y", + }, + } + + got := map[string]interface{}{ + "a": 42, + "b": false, + "c": "test", + } + + assert.Equal(t, want, got) + }) +} + +var wantTestAssertOutput = `(?sm: +=== Failed +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/assert_fails.* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/assert_fails.* + --- FAIL: TestGoTest_Helper_AssertOutput/assert_fails .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/assert_with_message .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/assert_with_message.* + Messages: My message.* + --- FAIL: TestGoTest_Helper_AssertOutput/assert_with_message .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/assert_with_messagef .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/assert_with_messagef.* + Messages: My message with arguments: 42.* + --- FAIL: TestGoTest_Helper_AssertOutput/assert_with_messagef .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/require_fails .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/require_fails.* + --- FAIL: TestGoTest_Helper_AssertOutput/require_fails .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/require_with_message .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/require_with_message.* + Messages: My message.* + --- FAIL: TestGoTest_Helper_AssertOutput/require_with_message .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/require_with_messagef .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Should be true.* + Test: TestGoTest_Helper_AssertOutput/require_with_messagef.* + Messages: My message with arguments: 42.* + --- FAIL: TestGoTest_Helper_AssertOutput/require_with_messagef .* +=== FAIL: dev-tools/mage TestGoTest_Helper_AssertOutput/equals_map .* + gotest_test.go:\d+:.* + Error Trace: gotest_test.go:\d+.* + Error: Not equal:.* +\s+expected: map\[string\]interface \{\}\{"a":1, "b":true, "c":"test", "e":map\[string\]interface \{\}\{"x":"y"\}\}.* +\s+actual : map\[string\]interface \{\}\{"a":42, "b":false, "c":"test"\}.* +\s+Diff:.* +\s+--- Expected.* +\s+\+\+\+ Actual.* +\s+-\(map\[string\]interface \{\}\) \(len=4\) \{.* +\s+- \(string\) \(len=1\) "a": \(int\) 1,.* +\s+- \(string\) \(len=1\) "b": \(bool\) true,.* +\s+- \(string\) \(len=1\) "c": \(string\) \(len=4\) "test",.* +\s+- \(string\) \(len=1\) "e": \(map\[string\]interface \{\}\) \(len=1\) \{.* +\s+- \(string\) \(len=1\) "x": \(string\) \(len=1\) "y".* +\s+- }.* +\s+\+\(map\[string\]interface \{\}\) \(len=3\) \{.* +\s+\+ \(string\) \(len=1\) "a": \(int\) 42,.* +\s+\+ \(string\) \(len=1\) "b": \(bool\) false,.* +\s+\+ \(string\) \(len=1\) "c": \(string\) \(len=4\) "test".* +\s+\}.* +)` + +func TestGoTest_Helper_LogOutput(t *testing.T) { + if !gotestHelperMode { + return + } + + t.Run("on error", func(t *testing.T) { + t.Log("Log message should be printed") + t.Logf("printf style log message: %v", 42) + t.Error("Log should fail") + t.Errorf("Log should fail with printf style log: %v", 23) + }) + + t.Run("on fatal", func(t *testing.T) { + t.Log("Log message should be printed") + t.Logf("printf style log message: %v", 42) + t.Fatal("Log should fail") + }) + + t.Run("on fatalf", func(t *testing.T) { + t.Log("Log message should be printed") + t.Logf("printf style log message: %v", 42) + t.Fatalf("Log should fail with printf style log: %v", 42) + }) + + t.Run("with newlines", func(t *testing.T) { + t.Log("Log\nmessage\nshould\nbe\nprinted") + t.Logf("printf\nstyle\nlog\nmessage:\n%v", 42) + t.Fatalf("Log\nshould\nfail\nwith\nprintf\nstyle\nlog:\n%v", 42) + }) +} + +var wantTestLogOutput = `(?sm: +=== Failed.* +=== FAIL: dev-tools/mage TestGoTest_Helper_LogOutput/on_error.* + gotest_test.go:\d+: Log message should be printed.* + gotest_test.go:\d+: printf style log message: 42.* + gotest_test.go:\d+: Log should fail.* + gotest_test.go:\d+: Log should fail with printf style log: 23.* + --- FAIL: TestGoTest_Helper_LogOutput/on_error.* +=== FAIL: dev-tools/mage TestGoTest_Helper_LogOutput/on_fatal.* + gotest_test.go:\d+: Log message should be printed.* + gotest_test.go:\d+: printf style log message: 42.* + gotest_test.go:\d+: Log should fail.* + --- FAIL: TestGoTest_Helper_LogOutput/on_fatal.* +=== FAIL: dev-tools/mage TestGoTest_Helper_LogOutput/on_fatalf.* + gotest_test.go:\d+: Log message should be printed.* + gotest_test.go:\d+: printf style log message: 42.* + gotest_test.go:\d+: Log should fail with printf style log: 42.* + --- FAIL: TestGoTest_Helper_LogOutput/on_fatalf.* +=== FAIL: dev-tools/mage TestGoTest_Helper_LogOutput/with_newlines.* + gotest_test.go:\d+: Log.* + message.* + should.* + be.* + printed.* + gotest_test.go:\d+: printf.* + style.* + log.* + message:.* + 42.* + gotest_test.go:\d+: Log.* + should.* + fail.* + with.* + printf.* + style.* + log:.* + 42.* + --- FAIL: TestGoTest_Helper_LogOutput/with_newlines.* +=== FAIL: dev-tools/mage TestGoTest_Helper_LogOutput.* +DONE 5 tests, 5 failures in.* +)` + +func TestGoTest_Helper_WithPanic(t *testing.T) { + if !gotestHelperMode { + return + } + + panic("Kaputt.") +} + +var wantTestWithPanic = `(?sm: +=== FAIL: dev-tools/mage TestGoTest_Helper_WithPanic.* +panic: Kaputt. \[recovered\].* + panic: Kaputt.* +)` + +func TestGoTest_Helper_WithWrongPanic(t *testing.T) { + if !gotestHelperMode { + return + } + + t.Run("setup failing go-routine", func(t *testing.T) { + go func() { + time.Sleep(1 * time.Second) + t.Fatal("oops") + }() + }) + + t.Run("false positive failure", func(t *testing.T) { + time.Sleep(10 * time.Second) + }) +} + +// The regular expression must very forgiving. Unfortunately the order of the +// tests and log lines can differ per run. +var wantTestWithWrongPanic = `(?sm: +=== FAIL: dev-tools/mage TestGoTest_Helper_WithWrongPanic.* +.* +panic: Fail in goroutine after TestGoTest_Helper_WithWrongPanic/setup_failing_go-routine has completed.* +)` diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ce48eb22098..7c1801f5fe3 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -96,6 +96,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true @@ -177,6 +189,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true - &agent_binary_files '{{.BeatName}}{{.BinaryExt}}': @@ -247,6 +271,19 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true + # Binary package spec (zip for windows) for community beats. - &agent_windows_binary_spec @@ -285,6 +322,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' + mode: 0644 + skip_on_missing: true - &agent_docker_spec <<: *agent_binary_spec diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index dd7a0080d6b..e42e525644c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -8,14 +8,6 @@ FROM {{ .buildFrom }} AS home COPY beat {{ $beatHome }} -{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} -RUN mkdir -p {{ $beatHome }}/.node \ - {{ $beatHome }}/.npm \ - {{ $beatHome }}/.cache \ - {{ $beatHome }}/.config \ - {{ $beatHome }}/suites -{{- end }} - RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ chown -R root:root {{ $beatHome }} && \ find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ @@ -48,6 +40,16 @@ RUN yum -y --setopt=tsflags=nodocs update \ # See https://access.redhat.com/discussions/3195102 for why rm is needed {{- end }} +{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} +ENV NODE_PATH={{ $beatHome }}/.node +RUN echo \ + $NODE_PATH \ + {{ $beatHome }}/.config \ + {{ $beatHome }}/suites \ + {{ $beatHome }}/.npm \ + {{ $beatHome }}/.cache \ + | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -98,28 +100,30 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses +{{- if ne .user "root" }} +RUN groupadd --gid 1000 {{ .BeatName }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} +{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} +RUN chown {{ .user }} $NODE_PATH +{{- end }} +{{- end }} +USER {{ .user }} + {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} # Setup synthetics env vars ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV SUITES_DIR={{ $beatHome }}/suites -ENV NODE_PATH={{ $beatHome }}/.node - -# Setup node -RUN cd /usr/share/heartbeat/.node \ - && mkdir node \ - && curl https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-x64.tar.xz | tar -xJ --strip 1 -C node -ENV PATH="/usr/share/heartbeat/.node/node/bin:$PATH" +ENV NODE_VERSION=12.18.4 +ENV PATH="$NODE_PATH/node/bin:$PATH" # Install the latest version of @elastic/synthetics forcefully ignoring the previously -# cached node_modules, hearbeat then calls the global executable to run test suites -RUN npm i -g -f @elastic/synthetics -{{- end }} - - -{{- if ne .user "root" }} -RUN groupadd --gid 1000 {{ .BeatName }} -RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} +# cached node_modules, heartbeat then calls the global executable to run test suites +# Setup node +RUN cd /usr/share/heartbeat/.node \ + && mkdir -p node \ + && curl https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-x64.tar.xz | tar -xJ --strip 1 -C node \ + && chmod ug+rwX -R $NODE_PATH \ + && npm i -g -f @elastic/synthetics && chmod ug+rwX -R $NODE_PATH {{- end }} -USER {{ .user }} {{- range $i, $port := .ExposePorts }} EXPOSE {{ $port }} diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index d9707991a5f..8b3983fa8da 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index aea966e10e7..403b1fcc291 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -37,6 +37,8 @@ stages: withModule: true ## run the ITs only if the changeset affects a specific module. when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -57,6 +59,8 @@ stages: #- "windows-2008-r2" https://github.com/elastic/beats/issues/19795 when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/filebeat/docs/aws-credentials-examples.asciidoc b/filebeat/docs/aws-credentials-examples.asciidoc index e306e2bb0a0..c8509bf826e 100644 --- a/filebeat/docs/aws-credentials-examples.asciidoc +++ b/filebeat/docs/aws-credentials-examples.asciidoc @@ -3,7 +3,7 @@ [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue access_key_id: '' secret_access_key: '' @@ -15,7 +15,7 @@ or [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' @@ -27,7 +27,7 @@ filebeat.inputs: [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue role_arn: arn:aws:iam::123456789012:role/test-mb ---- @@ -37,7 +37,7 @@ filebeat.inputs: [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue credential_profile_name: test-fb ---- diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 70156f74fee..f55a29a067e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -2250,6 +2250,16 @@ type: keyword The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST +type: keyword + +-- + +*`aws.vpcflow.tcp_flags_array`*:: ++ +-- +List of TCP flags: 'fin, syn, rst, psh, ack, urg' + + type: keyword -- diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index b8e230bfd0e..704d7ef99e4 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -63,6 +63,7 @@ subdirectories of a directory. You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-aws-cloudwatch>> +* <<{beatname_lc}-input-aws-s3>> * <<{beatname_lc}-input-azure-eventhub>> * <<{beatname_lc}-input-cloudfoundry>> * <<{beatname_lc}-input-container>> @@ -76,7 +77,6 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-netflow>> * <<{beatname_lc}-input-o365audit>> * <<{beatname_lc}-input-redis>> -* <<{beatname_lc}-input-s3>> * <<{beatname_lc}-input-stdin>> * <<{beatname_lc}-input-syslog>> * <<{beatname_lc}-input-tcp>> diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index d8c4b44df4d..8f55d5c16d8 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -388,7 +388,7 @@ will be found under `rsa.raw`. The default is false. The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. -To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-s3.html[S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. +To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-aws-s3.html[AWS S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. This fileset supports all 4 log types: - Proxy @@ -409,7 +409,7 @@ Example config: - module: cisco umbrella: enabled: true - var.input: s3 + var.input: aws-s3 var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue var.access_key_id: 123456 var.secret_access_key: PASSWORD diff --git a/filebeat/docs/modules/mssql.asciidoc b/filebeat/docs/modules/mssql.asciidoc index 7ecaa5e247a..8d07a4e2194 100644 --- a/filebeat/docs/modules/mssql.asciidoc +++ b/filebeat/docs/modules/mssql.asciidoc @@ -20,14 +20,14 @@ include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] The following example shows how to set paths in the +modules.d/{modulename}.yml+ -file to override the default paths for Træfik logs: +file to override the default paths for MSSQL logs: ["source","yaml",subs="attributes"] ----- - module: mssql log: enabled: true - var.paths: ["/var/opt/mssql/log/error*"] + var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] ----- @@ -35,7 +35,7 @@ To specify the same settings at the command line, you use: ["source","sh",subs="attributes"] ----- --M "mssql.log.var.paths=[/var/opt/mssql/log/error*]" +-M "mssql.log.var.paths=['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']" ----- //set the fileset name used in the included example diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 91b1dfa3190..858d307a57e 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1409,6 +1409,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1536,6 +1542,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1724,6 +1736,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1733,6 +1749,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1889,6 +1911,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2185,6 +2213,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2382,6 +2416,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 7e253bcc9ec..8d40284c366 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -318,13 +318,13 @@ func (inp *filestream) readFromSource( return nil } + s.Offset += int64(message.Bytes) + if message.IsEmpty() || inp.isDroppedLine(log, string(message.Content)) { continue } event := inp.eventFromMessage(message, path) - s.Offset += int64(message.Bytes) - if err := p.Publish(event, s); err != nil { return err } diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 0e4888b90c3..75a0957744a 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -177,7 +177,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { } if tls != nil { k.Net.TLS.Enable = true - k.Net.TLS.Config = tls.BuildModuleConfig("") + k.Net.TLS.Config = tls.BuildModuleClientConfig("") } if config.Kerberos.IsEnabled() { diff --git a/filebeat/input/mqtt/client.go b/filebeat/input/mqtt/client.go index 701498f3c56..068dca04386 100644 --- a/filebeat/input/mqtt/client.go +++ b/filebeat/input/mqtt/client.go @@ -40,7 +40,7 @@ func createClientOptions(config mqttInputConfig, onConnectHandler func(client li if err != nil { return nil, err } - clientOptions.SetTLSConfig(tlsConfig.BuildModuleConfig("")) + clientOptions.SetTLSConfig(tlsConfig.BuildModuleClientConfig("")) } return clientOptions, nil } diff --git a/filebeat/input/redis/harvester.go b/filebeat/input/redis/harvester.go index 49d153cf26c..9c575f83665 100644 --- a/filebeat/input/redis/harvester.go +++ b/filebeat/input/redis/harvester.go @@ -22,8 +22,8 @@ import ( "strings" "time" - rd "github.com/garyburd/redigo/redis" "github.com/gofrs/uuid" + rd "github.com/gomodule/redigo/redis" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" diff --git a/filebeat/input/redis/input.go b/filebeat/input/redis/input.go index 8b3b95d013e..215b855151f 100644 --- a/filebeat/input/redis/input.go +++ b/filebeat/input/redis/input.go @@ -20,7 +20,7 @@ package redis import ( "time" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/harvester" diff --git a/filebeat/inputsource/tcp/server.go b/filebeat/inputsource/tcp/server.go index 270ebc9c0c5..42c88693879 100644 --- a/filebeat/inputsource/tcp/server.go +++ b/filebeat/inputsource/tcp/server.go @@ -68,7 +68,7 @@ func (s *Server) createServer() (net.Listener, error) { var l net.Listener var err error if s.tlsConfig != nil { - t := s.tlsConfig.BuildModuleConfig(s.config.Host) + t := s.tlsConfig.BuildServerConfig(s.config.Host) l, err = tls.Listen("tcp", s.config.Host, t) if err != nil { return nil, err diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index 4001fd863c9..845ade6b288 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -127,6 +127,8 @@ def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): with pytest.raises(ssl.SSLError): tls.connect((config.get('host'), config.get('port'))) + sock.close() + def test_tcp_over_tls_mutual_auth_fails(self): """ Test filebeat TCP with TLS with default setting to enforce client auth, with bad client certificates @@ -171,6 +173,8 @@ def test_tcp_over_tls_mutual_auth_fails(self): # so that the failure can be reported as an exception when it arrives. tls.recv(1) + sock.close() + def test_tcp_over_tls_mutual_auth_succeed(self): """ Test filebeat TCP with TLS when enforcing client auth with good client certificates. @@ -275,6 +279,8 @@ def test_tcp_tls_with_a_plain_text_socket(self): assert path.isfile(path.join(self.working_dir, "output/" + self.beat_name)) is False + sock.close() + def assert_output(self, output): assert len(output) == 2 assert output[0]["input.type"] == "tcp" diff --git a/go.mod b/go.mod index 62ff0b55953..b0c1c974fc1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.14 +go 1.15 require ( 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b @@ -57,6 +57,7 @@ require ( github.com/dop251/goja v0.0.0-20200831102558-9af81ddcf0e1 github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/dustin/go-humanize v1.0.0 + github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 github.com/elastic/ecs v1.6.0 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200709172729-d43b7ad5833a @@ -73,10 +74,9 @@ require ( github.com/elastic/go-ucfg v0.8.3 github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.13.0 - github.com/fatih/color v1.5.0 + github.com/fatih/color v1.9.0 github.com/fsnotify/fsevents v0.1.1 github.com/fsnotify/fsnotify v1.4.9 - github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 // indirect github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-sql-driver/mysql v1.4.1 @@ -89,8 +89,9 @@ require ( github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.4.2 github.com/golang/snappy v0.0.1 + github.com/gomodule/redigo v1.8.3 github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 - github.com/google/go-cmp v0.4.0 + github.com/google/go-cmp v0.5.2 github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 @@ -109,15 +110,13 @@ require ( github.com/jonboulle/clockwork v0.2.2 github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/jpillora/backoff v1.0.0 // indirect - github.com/jstemmer/go-junit-report v0.9.1 github.com/kardianos/service v1.1.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 github.com/magefile/mage v1.10.0 github.com/mailru/easyjson v0.7.1 // indirect - github.com/mattn/go-colorable v0.1.4 + github.com/mattn/go-colorable v0.1.6 github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect - github.com/mattn/go-isatty v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/miekg/dns v1.1.15 github.com/mitchellh/gox v1.0.1 @@ -137,13 +136,13 @@ require ( github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.11 github.com/prometheus/prometheus v2.5.0+incompatible - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 github.com/reviewdog/reviewdog v0.9.17 github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 // indirect github.com/satori/go.uuid v1.2.0 // indirect - github.com/shirou/gopsutil v2.19.11+incompatible + github.com/shirou/gopsutil v3.20.12+incompatible github.com/shopspring/decimal v1.2.0 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 @@ -165,12 +164,12 @@ require ( go.uber.org/atomic v1.5.0 go.uber.org/multierr v1.3.0 go.uber.org/zap v1.14.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/lint v0.0.0-20200130185559-910be7a94367 - golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/net v0.0.0-20200904194848-62affa334b73 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a - golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae + golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 golang.org/x/text v0.3.3 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/tools v0.0.0-20200904185747-39188db58858 @@ -183,6 +182,7 @@ require ( gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 gopkg.in/yaml.v2 v2.3.0 gotest.tools v2.2.0+incompatible + gotest.tools/gotestsum v0.6.0 howett.net/plist v0.0.0-20181124034731-591f970eefbb k8s.io/api v0.19.4 k8s.io/apimachinery v0.19.4 @@ -192,7 +192,7 @@ require ( replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15 - github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec + github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f diff --git a/go.sum b/go.sum index 6ff61bcfea1..74cc1c71701 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,7 @@ github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1D github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= @@ -285,8 +286,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.13.0 h1:EIeuQcLPKia759s6mlVztlxUyKiKYHo6y6kOODOLO7A= github.com/elastic/gosigar v0.13.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI= -github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= +github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 h1:C9LsbipColsz04JKpKoLlp0pgMJRLq2uXVTeKRDcNcY= +github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -296,14 +297,12 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.5.0 h1:vBh+kQp8lg9XPr56u1CPrWjFXtdphMoGWVHr9/1c+A0= -github.com/fatih/color v1.5.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= -github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 h1:nREVDi4H8mwnNqfxFU9NMzZrDCg8TXbEatMvHozxKwU= -github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk= +github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -375,6 +374,8 @@ github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.8.3 h1:HR0kYDX2RJZvAup8CsiJwxB4dTCSC0AaUq6S4SiLwUc= +github.com/gomodule/redigo v1.8.3/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 h1:b4EyQBj8pgtcWOr7YCSxK6NUQzJr0n4hxJ3mc+dtKk4= @@ -384,6 +385,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= @@ -401,6 +404,8 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= @@ -477,6 +482,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd h1:KikNiFwUO3QLyeKyN4k9yBH9Pcu/gU/yficWi61cJIw= @@ -500,8 +506,8 @@ github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -510,9 +516,13 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -529,12 +539,14 @@ github.com/markbates/pkger v0.17.0 h1:RFfyBPufP2V6cddUyyEVSHBpaAnM1WzaMNyqomeT+i github.com/markbates/pkger v0.17.0/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 h1:YFh+sjyJTMQSYjKwM4dFKhJPJC/wfo98tPUc17HdoYw= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe h1:YioO2TiJyAHWHyCRQCP8jk5IzTqmsbGc5qQPIhHo6xs= github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-shellwords v1.0.7 h1:KqhVjVZomx2puPACkj9vrGFqnp42Htvo9SEAWePHKOs= @@ -571,6 +583,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -609,8 +623,8 @@ github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= -github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 h1:i5VIxp6QB8oWZ8IkK8zrDgeT6ORGIUeiN+61iETwJbI= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -646,8 +660,8 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4 github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd h1:fvaEkjpr2NJbtnFRCft7D6y/mQ5/2OQU0pKJLW8dwFA= github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd/go.mod h1:giYAXnpegRDPsXUO7TRpDKXJo1lFGYxyWRfEt5iQ+OA= github.com/reviewdog/reviewdog v0.9.17 h1:MKb3rlQZgkEXr3d85iqtYNITXn7gDJr2kT0IhgX/X9A= @@ -672,8 +686,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4= -github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.20.12+incompatible h1:6VEGkOXP/eP4o2Ilk8cSsX0PhOEfX6leqAnD+urrp9M= +github.com/shirou/gopsutil v3.20.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -794,9 +808,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -847,6 +862,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -887,6 +904,7 @@ golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -894,11 +912,12 @@ golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -915,6 +934,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -971,6 +992,8 @@ gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1001,8 +1024,14 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/gotestsum v0.6.0 h1:0zIxynXq9gkAcRpboAi3qOQIkZkCt/stfQzd7ab7Czs= +gotest.tools/gotestsum v0.6.0/go.mod h1:LEX+ioCVdeWhZc8GYfiBRag360eBhwixWJ62R9eDQtI= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index f0155d34274..51c2b06d485 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 7c30e7a8b04..ac2fc59cc64 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -36,6 +36,8 @@ stages: mage: "mage build test" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -55,6 +57,8 @@ stages: - "windows-2019" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 37e3e2ed122..63501b14e2d 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -707,6 +707,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -834,6 +840,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1022,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1031,6 +1047,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1187,6 +1209,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1483,6 +1511,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1680,6 +1714,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/heartbeat/monitors/active/fixtures/expired.cert b/heartbeat/monitors/active/fixtures/expired.cert index e39ad893bd6..959486cec37 100644 --- a/heartbeat/monitors/active/fixtures/expired.cert +++ b/heartbeat/monitors/active/fixtures/expired.cert @@ -1,23 +1,21 @@ -----BEGIN CERTIFICATE----- -MIID3zCCAsegAwIBAgIUS+ahW2wxDZ1bT/qYnenS8jrXUcAwDQYJKoZIhvcNAQEL -BQAwfzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk1OMRQwEgYDVQQHDAtNaW5uZWFw -b2xpczEVMBMGA1UECgwMRWxhc3RpYywgSW5jMRQwEgYDVQQLDAtFbmdpbmVlcmlu -ZzEgMB4GA1UEAwwXZXhwaXJlZHRlc3QuZXhhbXBsZS5uZXQwHhcNMjAwNDIxMTQw -MDE0WhcNMjAwNDIyMTQwMDE0WjB/MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTU4x -FDASBgNVBAcMC01pbm5lYXBvbGlzMRUwEwYDVQQKDAxFbGFzdGljLCBJbmMxFDAS -BgNVBAsMC0VuZ2luZWVyaW5nMSAwHgYDVQQDDBdleHBpcmVkdGVzdC5leGFtcGxl -Lm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKh1iS5EZ7bDSKgW -R3JXAepMIaEewMSdbaoBtuNQb48XJGwI0mudF983a7JxGCSfw9mhVYa4YsSv79UE -XomGrWVrS01Cmf1VRIOmxevWMPhvnE6UH+5VxKUBk5ooNSty4iHkDFy2i5WWjxiv -de6Xqnn/dVQhuT/sW+rU/grCsGcdUwqsWnC547ekqiYRTtyZrdh+U0KRKqy5iBlH -9Woua+CnXmsD7+4MgGekErg9XLRHYveLOmLucbNlAIlRyfMDZL1RlXufcGwhzItz -JNM9N0NJ5bwrpuP0RYlYbbMYal+b1Tn2e8qkMm88hniQkuu69kUpKeewIOr62vIK -tI273GECAwEAAaNTMFEwHQYDVR0OBBYEFKgd6wQcgIdUSjtJREObD+R3q3MPMB8G -A1UdIwQYMBaAFKgd6wQcgIdUSjtJREObD+R3q3MPMA8GA1UdEwEB/wQFMAMBAf8w -DQYJKoZIhvcNAQELBQADggEBADkBqmCUcvVTqu5IIZ5PLz40jdg2luaDHEA6I2Ga -1ioabETfQhXeaNJflojYm0Bzsy2aneVLGM2KaZ76wN0yvib3MZ4miu4C/mDsR3bB -wq7/CAK2AcJXv1jk0vIrK6DhZfA2HaelBkQ8UHwWK7AO+JmS6jozIt1vySwPI1E7 -lMFWbs3bmsSmunj3+66XS2XguUKzFwUIAEOfsPFqT2OMsPIa7weUWuCV/zMi7fuB -HbgVouYvMTve8wx7+ozDk6CyvlRlx20xwdOvXaH3JILw7gTQWcAEWZLcB2ct1Zks -UTtbIAjBV6s0Pm/2/6MxxkDCVVUpwXiiKBRkHxzkgoH7TQw= +MIIDazCCAlOgAwIBAgIUJGs/M/NeLac1U+H58kwZYgSUoMgwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTAxMTgxNTQxNDdaFw0yMTAx +MTkxNTQxNDdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDGo1zHFcEZT4gFe24in23O1C1AtCSP2wfZCHyA9Tvj +fak3dgI5BESPHHu8lqo8V4C2ViAnBBoQ9Uq3nd38CdVJyYK5vtrHpTVLK8OIasOD +eMHBwEOMQQN6js9in0lvixBTK2oZOOcONSSxQVFUKIgSfY8sBJViYftkolPmT/OZ +bipzBoikclKDQykM9GAh/IVSQpfC+PIJT31xOXmwvy9zL5eiLDRIsZpuauI6L7/f +RkapMXtDz0QuP3aZ/VZNydagmfrz0LiBfoX7B93ZRAPLmk9KzRb/RTcuxD28d45v +KtZXUr/cR6S5gwqzb1sEaOOR+kz9aeNzwGFl6LmcgpqfAgMBAAGjUzBRMB0GA1Ud +DgQWBBTKlcyCuhJ07eTv/y33LN6/SVB1FzAfBgNVHSMEGDAWgBTKlcyCuhJ07eTv +/y33LN6/SVB1FzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBo +mv85XCM+qNvn7nNXXvI8h7PFBYZAn6trD713liiD4fURjRmQt/EDXo1ZyJhmqm3n +UHqTbQ1srnhz2zTKbBD1m4rby3bu9l0IvoCihNnd3jEzcz6IkT5TqPy6FwQdKoD4 +U0miGvmSyKs7/IyJepy6k+XUI9KSsi4k/ECx4nAvvf05Yv7XtSvPpVPHW6uvLsTu +jv8C8CcND66LrUsqn2CRkiXXX68KJSDPvT4fdNDz1nIykN0AAvVs8rA3R35dgjiR +9aHsDqkqf6QIZ0jCKbZL4Z/j5tj26P1nEUsDeZ46FECipie2z9oZII897X+Yzrcg +d9RKMlVpBwQg/iFD9orw -----END CERTIFICATE----- diff --git a/heartbeat/monitors/active/fixtures/expired.key b/heartbeat/monitors/active/fixtures/expired.key index 2a11440f7aa..3f29313cc4e 100644 --- a/heartbeat/monitors/active/fixtures/expired.key +++ b/heartbeat/monitors/active/fixtures/expired.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCodYkuRGe2w0io -FkdyVwHqTCGhHsDEnW2qAbbjUG+PFyRsCNJrnRffN2uycRgkn8PZoVWGuGLEr+/V -BF6Jhq1la0tNQpn9VUSDpsXr1jD4b5xOlB/uVcSlAZOaKDUrcuIh5AxctouVlo8Y -r3Xul6p5/3VUIbk/7Fvq1P4KwrBnHVMKrFpwueO3pKomEU7cma3YflNCkSqsuYgZ -R/VqLmvgp15rA+/uDIBnpBK4PVy0R2L3izpi7nGzZQCJUcnzA2S9UZV7n3BsIcyL -cyTTPTdDSeW8K6bj9EWJWG2zGGpfm9U59nvKpDJvPIZ4kJLruvZFKSnnsCDq+try -CrSNu9xhAgMBAAECggEBAIc32QYvWESmWeK6B11rI5lqxK+snLT1XLpSp/esb++e -dtjU9/nzXd8JgEP6bZOwPiepTZpW1MjmJA+Lc0rWtMYsqoP4ityDHfzC2CmmgyZX -iFK2qS7I35BHRLA/x/X5QDRN9fJRgJdxA6mf5Xy/dtJ4UDhY3XbHBTzo/IWsoqYQ -4V3WBQYMGlhBArCoOx07pwc9NMTnXwpfe4rUdm3EaGGpe/9JT08JcTyFZfFUeFT1 -lfSYo5i+xPOCQ/FcC5GfWdciyY0c8ej8iwdxZb0kPI4hBu36+D6zD+YoNoC3CQTb -MecRFQ0MeTTuUMCdzFWtg+2FWnJucaLiaK9fKbVzi7UCgYEA0BAlfUdXdeDYMlW3 -2ReeOgH32bchPYwn2UvHYkIhhDp40STVw3BYQ0Zj9yJQXLFaoY1SFhwRJR1kpbSd -IfME/IzR/oMFvRUNQEPQZVH0Mg9FWIXLdXlV4qbU3AyA2r4x+VUCt3jp1n/5rG7g -cmoKBdCXNUAhK30bRGTdXB06Fp8CgYEAz0V+IlkGyDKcyCkja0ypA3AdSod/43az -7HMS3nf32hOFpgQuEtVYZc3NW/rdJFPksnRd6+RlD2nIoHZEa+adl2gESjGH2asw -nhxP/Pr4m8PGZF5BwdILRTVFukf5yrM6g63FgrgA9d+QdCsqoqrctItRyCgcfpL4 -XYXEKVWELP8CgYATxbUKVsFE/n0NK4AxLUFoGc/H7NNH2g3fZIgqGka9HiFlHq8B -x6dbnVDap3QjmucV+ywV1cz7TfPGm4djNoj+xxMdsK3W7i04MjmXp1Yhe7oHU4+m -NkWnKFuKHdYQ84okO6Pqc58lNzwu2sqRlOom60+zS8jbLSRuN3ehzVU72QKBgGm0 -qCo+Ou44maqfCFg9hWiicd3Dkt5feE0bNsFMb5PBJwTO1ux175ojxhqlqshPHLBC -FnAqT7v3mAD1r9lTiIVh3+YysnS5EJdiGw0KtWVDB9fCFkkRpPvLul7RPDw7AZmM -MtGCo8LBHHuSVDEXcG2HK9MnWbjXnWCcyrjFyx3jAoGAYsNGYm+OBr16NNsPtx3S -nRQJz9wqB2mIqNU8rRSjd5EUp03jhHiTEN9DT6iEnLGaTDBUgD2RlPvEVGk1N7FT -nh9tLtg2ytWIC/P+QrKwzdUUa00MSswTxRS3Cmy459UbLBiPgHBJ2h1G7gsiHPOt -erJWqYJ8DXvLzCPdMVzQxj8= +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDGo1zHFcEZT4gF +e24in23O1C1AtCSP2wfZCHyA9Tvjfak3dgI5BESPHHu8lqo8V4C2ViAnBBoQ9Uq3 +nd38CdVJyYK5vtrHpTVLK8OIasODeMHBwEOMQQN6js9in0lvixBTK2oZOOcONSSx +QVFUKIgSfY8sBJViYftkolPmT/OZbipzBoikclKDQykM9GAh/IVSQpfC+PIJT31x +OXmwvy9zL5eiLDRIsZpuauI6L7/fRkapMXtDz0QuP3aZ/VZNydagmfrz0LiBfoX7 +B93ZRAPLmk9KzRb/RTcuxD28d45vKtZXUr/cR6S5gwqzb1sEaOOR+kz9aeNzwGFl +6LmcgpqfAgMBAAECggEAdd93LMSiA80IIIiwQGdvF+8cs2qyz5LgQ1Af4b6kvUhS +ZBDpHMCFlo8GdlDJ0Gj0fj75cW2jsjB9GBzcH7YOFW0439R7q0FEMMlVjNweDSRZ +hgn7i30A/XdFZXa0czaIh3IZ0cHMisfKcE0cX951cNOFZE+L/ge60Grrcp47Je4H +r5F8fBaP2j729fQlpF3Kzl0uYCRqn9CD+KRMmz8nx9ifxmPM4ddVK8Xg8quVSU1x +HyeN7VfF0Hcvsv6+DRiZqgASu9pSggVffEmW4EBRNuJK4RPKFUriTTUTd8DUS39d +xxA4CjM/t+YjQOYcKlZGxxqu+ILPJnSTfyVe+lvp0QKBgQDvi5GRPuVGegO8nPqE +EHn9fh0CuVtRZjr+7YwJmlYY6WsEuVFOGlo+OdPQwk+iJuT187FYc83PwozRRoI+ +NfMC8lByMGXK0deTfCuK422U+PBuXP/U3xEK6rb1plFiNyw37wxivRTvSX/xZ8ZI +u+zS89LmWQn4oX2fbJUjcfol0wKBgQDUSHAv4PgYQutydgm2n2+asY/BeZupKqmR +FUINmcM8zZ6OC37xrT2TZoGT4WfCRJlvsj6Gpwzf6gM+MAluTQJ6ZxdAP8ifsUJT +69lyRL4o11qLvXsIexdHFUDRxv9E0pWQbEOUH7dZnWuZAXOc9sBVQLW7Hb9UJhzU +JNhsjWK8hQKBgESqO0XpQ4uaOiv8y4rDtlRFrEc1nsmMhmjA6x1tkjR95GFsRbQf +tbKUnPUAXhdbEtK10iZKu7pMSFVM4tS2Xjx//TNeUC99S1BJjam4W5cSbMkV+3en +oZkBgwSTcky3CAPYkDJVhQS6iuRGHP6Ib/BgWBoTd9o/YGUsmOuaJlxlAoGAUkH7 +juH64NPVjlXk76oaQfQ6AzuPMN8CM+Dr5n9FoyL+JIaZ2yZJLPvUjCwU27dfSWPl +TKIgoAd62DcfAGnA4xNddB0DLbYvQHusl9XBZewE/w8eRKwF2hqHMoKM2etL31nE +WfpsPsJ8Nh7U4ObQonhfszEUuZPHKgFZB79Fmk0CgYB0XEuZuSva3uKACbQQS2L2 +jeO23cmE+rSdfw9tL6O2OLZkKbDkHxMXu/G305ciyW+Hmw4CwC6B2IDJuCjzYGVr +7FL/XP0d/pksOIqjNi2VQvARO+7NStXqcgmB9p/vlyYkKLIcGKQ2w169QaNOLylc +L/DKQsIUqMqNsRcd4at/Kg== -----END PRIVATE KEY----- diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index 2e5a43656ad..cd2c03dee3f 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -626,7 +626,12 @@ func TestNewRoundTripper(t *testing.T) { require.NotNil(t, transp.Dial) require.NotNil(t, transport.TLSDialer) - require.Equal(t, (&tlscommon.TLSConfig{}).ToConfig(), transp.TLSClientConfig) + expected := (&tlscommon.TLSConfig{}).ToConfig() + require.Equal(t, expected.InsecureSkipVerify, transp.TLSClientConfig.InsecureSkipVerify) + // When we remove support for the legacy common name treatment + // this test has to be adjusted, as we will not depend on our + // VerifyConnection callback. + require.NotNil(t, transp.TLSClientConfig.VerifyConnection) require.True(t, transp.DisableKeepAlives) }) } diff --git a/heartbeat/monitors/stdfields/stdfields.go b/heartbeat/monitors/stdfields/stdfields.go index 433f6223863..26b28628fc3 100644 --- a/heartbeat/monitors/stdfields/stdfields.go +++ b/heartbeat/monitors/stdfields/stdfields.go @@ -18,7 +18,6 @@ package stdfields import ( - "fmt" "time" "github.com/pkg/errors" @@ -50,7 +49,6 @@ func ConfigToStdMonitorFields(config *common.Config) (StdMonitorFields, error) { mpi := StdMonitorFields{Enabled: true} if err := config.Unpack(&mpi); err != nil { - fmt.Printf("HIER %s", err) return mpi, errors.Wrap(err, "error unpacking monitor plugin config") } diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile index 6df4d47d885..36af746307c 100644 --- a/journalbeat/Dockerfile +++ b/journalbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 3f21f7dad4a..148db6aed52 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -36,3 +36,5 @@ stages: mage: "mage build unitTest" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 7e875edcf8e..7664a3edbd4 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -472,6 +472,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -599,6 +605,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -787,6 +799,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -796,6 +812,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -952,6 +974,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1248,6 +1276,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1445,6 +1479,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index b7dde2b92cf..cc9829656ed 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 57ec6319076..1f944143138 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -33,11 +33,17 @@ stages: mage: "mage build test" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags crosscompile: make: "make -C libbeat crosscompile" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags stress-tests: make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/libbeat/_meta/config/output-kafka.reference.yml.tmpl b/libbeat/_meta/config/output-kafka.reference.yml.tmpl index c1240f75867..366652fd23e 100644 --- a/libbeat/_meta/config/output-kafka.reference.yml.tmpl +++ b/libbeat/_meta/config/output-kafka.reference.yml.tmpl @@ -131,6 +131,10 @@ # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + {{include "ssl.reference.yml.tmpl" . | indent 2 }} # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. #kerberos.enabled: true diff --git a/libbeat/_meta/config/ssl.reference.yml.tmpl b/libbeat/_meta/config/ssl.reference.yml.tmpl index 88f638e21b0..69b666f9c97 100644 --- a/libbeat/_meta/config/ssl.reference.yml.tmpl +++ b/libbeat/_meta/config/ssl.reference.yml.tmpl @@ -7,6 +7,12 @@ # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. +# * strict, which verifies that the provided certificate is signed by a trusted +# authority (CA) and also verifies that the server's hostname (or IP address) +# matches the names identified within the certificate. If the Subject Alternative +# Name is empty, it returns an error. +# * certificate, which verifies that the provided certificate is signed by a +# trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/libbeat/common/cli/confirm.go b/libbeat/common/cli/confirm.go index 4beaa95bac9..5129071a7fa 100644 --- a/libbeat/common/cli/confirm.go +++ b/libbeat/common/cli/confirm.go @@ -32,10 +32,10 @@ import ( // returns true for yes, false for no func Confirm(prompt string, def bool) (bool, error) { reader := bufio.NewReader(os.Stdin) - return confirm(reader, prompt, def) + return confirm(reader, os.Stdout, prompt, def) } -func confirm(r io.Reader, prompt string, def bool) (bool, error) { +func confirm(r io.Reader, out io.Writer, prompt string, def bool) (bool, error) { options := " [Y/n]" if !def { options = " [y/N]" @@ -43,7 +43,7 @@ func confirm(r io.Reader, prompt string, def bool) (bool, error) { reader := bufio.NewScanner(r) for { - fmt.Print(prompt + options + ":") + fmt.Fprintf(out, prompt+options+":") if !reader.Scan() { break @@ -56,7 +56,7 @@ func confirm(r io.Reader, prompt string, def bool) (bool, error) { case "n", "no": return false, nil default: - fmt.Println("Please write 'y' or 'n'") + fmt.Fprintln(out, "Please write 'y' or 'n'") } } diff --git a/libbeat/common/cli/confirm_test.go b/libbeat/common/cli/confirm_test.go index 97a9475b332..dca2064aaa1 100644 --- a/libbeat/common/cli/confirm_test.go +++ b/libbeat/common/cli/confirm_test.go @@ -25,49 +25,51 @@ import ( ) func TestConfirm(t *testing.T) { - tests := []struct { - name string + tests := map[string]struct { input string + prompt string def bool result bool error bool }{ - { - name: "Test default yes", + "Test default yes": { input: "\n", + prompt: "> [Y/n]:", def: true, result: true, }, - { - name: "Test default no", + "Test default no": { input: "\n", + prompt: "> [y/N]:", def: false, result: false, }, - { - name: "Test YeS", + "Test YeS": { input: "YeS\n", + prompt: "> [y/N]:", def: false, result: true, }, - { - name: "Test Y", + "Test Y": { input: "Y\n", + prompt: "> [y/N]:", def: false, result: true, }, - { - name: "Test No", + "Test No": { input: "No\n", def: true, + prompt: "> [Y/n]:", result: false, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var buf strings.Builder r := strings.NewReader(test.input) - result, err := confirm(r, "prompt", test.def) + + result, err := confirm(r, &buf, ">", test.def) assert.Equal(t, test.result, result) if test.error { @@ -75,6 +77,8 @@ func TestConfirm(t *testing.T) { } else { assert.NoError(t, err) } + + assert.Equal(t, test.prompt, buf.String()) }) } } diff --git a/libbeat/common/cli/input.go b/libbeat/common/cli/input.go index a6a516fd3d4..7025f63f2aa 100644 --- a/libbeat/common/cli/input.go +++ b/libbeat/common/cli/input.go @@ -29,12 +29,12 @@ import ( // ReadInput shows the text and ask the user to provide input. func ReadInput(prompt string) (string, error) { reader := bufio.NewReader(os.Stdin) - return input(reader, prompt) + return input(reader, os.Stdout, prompt) } -func input(r io.Reader, prompt string) (string, error) { +func input(r io.Reader, out io.Writer, prompt string) (string, error) { reader := bufio.NewScanner(r) - fmt.Print(prompt + " ") + fmt.Fprintf(out, prompt+" ") if !reader.Scan() { return "", errors.New("error reading user input") diff --git a/libbeat/common/cli/input_test.go b/libbeat/common/cli/input_test.go index de87b5efe2a..94c1d26bb7c 100644 --- a/libbeat/common/cli/input_test.go +++ b/libbeat/common/cli/input_test.go @@ -25,39 +25,37 @@ import ( ) func TestReadInput(t *testing.T) { - tests := []struct { - name string + tests := map[string]struct { input string res string }{ - { - name: "Question 1?", + "Question 1?": { input: "\n", res: "", }, - { - name: "Question 2?", + "Question 2?": { input: "full string input\n", res: "full string input", }, - { - name: "Question 3?", + + "Question 3?": { input: "123456789\n", res: "123456789", }, - { - name: "Question 4?", + "Question 4?": { input: "false\n", res: "false", }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + for name, test := range tests { + t.Run(name, func(t *testing.T) { + var buf strings.Builder r := strings.NewReader(test.input) - result, err := input(r, test.name) + result, err := input(r, &buf, name) assert.NoError(t, err) assert.Equal(t, test.res, result) + assert.Equal(t, name+" ", buf.String()) }) } } diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index 0373297fd00..10ece84dc47 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -62,7 +62,7 @@ func TestTLSDialer( tlsConfig = lastTLSConfig } if tlsConfig == nil { - tlsConfig = config.BuildModuleConfig(host) + tlsConfig = config.BuildModuleClientConfig(host) lastNetwork = network lastAddress = address lastTLSConfig = tlsConfig diff --git a/libbeat/common/transport/tlscommon/ca_pinning_test.go b/libbeat/common/transport/tlscommon/ca_pinning_test.go index c188a20e63a..ac441225090 100644 --- a/libbeat/common/transport/tlscommon/ca_pinning_test.go +++ b/libbeat/common/transport/tlscommon/ca_pinning_test.go @@ -44,22 +44,22 @@ var ser int64 = 1 func TestCAPinning(t *testing.T) { host := "127.0.0.1" - t.Run("when the ca_sha256 field is not defined we use normal certificate validation", - func(t *testing.T) { - cfg := common.MustNewConfigFrom(map[string]interface{}{ - "certificate_authorities": []string{"ca_test.pem"}, - }) + t.Run("when the ca_sha256 field is not defined we use normal certificate validation", func(t *testing.T) { + cfg := common.MustNewConfigFrom(map[string]interface{}{ + "verification_mode": "strict", + "certificate_authorities": []string{"ca_test.pem"}, + }) - config := &Config{} - err := cfg.Unpack(config) - require.NoError(t, err) + config := &Config{} + err := cfg.Unpack(config) + require.NoError(t, err) - tlsCfg, err := LoadTLSConfig(config) - require.NoError(t, err) + tlsCfg, err := LoadTLSConfig(config) + require.NoError(t, err) - tls := tlsCfg.BuildModuleConfig(host) - require.Nil(t, tls.VerifyPeerCertificate) - }) + tls := tlsCfg.BuildModuleClientConfig(host) + require.Nil(t, tls.VerifyConnection) + }) t.Run("when the ca_sha256 field is defined we use CA cert pinning", func(t *testing.T) { cfg := common.MustNewConfigFrom(map[string]interface{}{ @@ -73,83 +73,93 @@ func TestCAPinning(t *testing.T) { tlsCfg, err := LoadTLSConfig(config) require.NoError(t, err) - tls := tlsCfg.BuildModuleConfig(host) - require.NotNil(t, tls.VerifyPeerCertificate) + tls := tlsCfg.BuildModuleClientConfig(host) + require.NotNil(t, tls.VerifyConnection) }) t.Run("CA Root -> Certificate and we have the CA root pin", func(t *testing.T) { - msg := []byte("OK received message") - - ca, err := genCA() - require.NoError(t, err) - - serverCert, err := genSignedCert(ca, x509.KeyUsageDigitalSignature, false) - require.NoError(t, err) - - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write(msg) - }) - - // Select a random available port from the OS. - addr := "localhost:0" - - l, err := net.Listen("tcp", addr) - - server := &http.Server{ - Handler: mux, - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{ - serverCert, - }, - }, - } - - // Start server and shut it down when the tests are over. - go server.ServeTLS(l, "", "") - defer l.Close() - - // Root CA Pool - require.NoError(t, err) - rootCAs := x509.NewCertPool() - rootCAs.AddCert(ca.Leaf) - - // Get the pin of the RootCA. - pin := Fingerprint(ca.Leaf) - - tlsC := &TLSConfig{ - RootCAs: rootCAs, - CASha256: []string{pin}, + verificationModes := []TLSVerificationMode{ + VerifyFull, + VerifyStrict, + VerifyCertificate, } - - config := tlsC.BuildModuleConfig("localhost") - hostToConnect := l.Addr().String() - - transport := &http.Transport{ - TLSClientConfig: config, + for _, mode := range verificationModes { + t.Run(mode.String(), func(t *testing.T) { + msg := []byte("OK received message") + + ca, err := genCA() + require.NoError(t, err) + + serverCert, err := genSignedCert(ca, x509.KeyUsageDigitalSignature, false) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(msg) + }) + + // Select a random available port from the OS. + addr := "localhost:0" + + l, err := net.Listen("tcp", addr) + + server := &http.Server{ + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + serverCert, + }, + }, + } + + // Start server and shut it down when the tests are over. + go server.ServeTLS(l, "", "") + defer l.Close() + + // Root CA Pool + require.NoError(t, err) + rootCAs := x509.NewCertPool() + rootCAs.AddCert(ca.Leaf) + + // Get the pin of the RootCA. + pin := Fingerprint(ca.Leaf) + + tlsC := &TLSConfig{ + Verification: mode, + RootCAs: rootCAs, + CASha256: []string{pin}, + } + + config := tlsC.BuildModuleClientConfig("localhost") + hostToConnect := l.Addr().String() + + transport := &http.Transport{ + TLSClientConfig: config, + } + + client := &http.Client{Transport: transport} + + port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") + + req, err := http.NewRequest("GET", "https://localhost:"+port, nil) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + content, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + assert.True(t, bytes.Equal(msg, content)) + + // 1. create key-pair + // 2. create pin + // 3. start server + // 4. Connect + // 5. Check wrong key do not work + // 6. Check good key work + // 7. check plain text fails to work. + }) } - - client := &http.Client{Transport: transport} - - port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") - - req, err := http.NewRequest("GET", "https://localhost:"+port, nil) - require.NoError(t, err) - resp, err := client.Do(req) - require.NoError(t, err) - content, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - assert.True(t, bytes.Equal(msg, content)) - - // 1. create key-pair - // 2. create pin - // 3. start server - // 4. Connect - // 5. Check wrong key do not work - // 6. Check good key work - // 7. check plain text fails to work. }) t.Run("CA Root -> Intermediate -> Certificate and we receive the CA Root Pin", func(t *testing.T) { @@ -205,7 +215,7 @@ func TestCAPinning(t *testing.T) { CASha256: []string{pin}, } - config := tlsC.BuildModuleConfig("localhost") + config := tlsC.BuildModuleClientConfig("localhost") hostToConnect := l.Addr().String() transport := &http.Transport{ @@ -279,7 +289,7 @@ func TestCAPinning(t *testing.T) { CASha256: []string{pin}, } - config := tlsC.BuildModuleConfig("localhost") + config := tlsC.BuildModuleClientConfig("localhost") hostToConnect := l.Addr().String() transport := &http.Transport{ @@ -343,6 +353,7 @@ func genCA() (tls.Certificate, error) { func genSignedCert(ca tls.Certificate, keyUsage x509.KeyUsage, isCA bool) (tls.Certificate, error) { // Create another Cert/key cert := &x509.Certificate{ + DNSNames: []string{"localhost"}, SerialNumber: big.NewInt(2000), Subject: pkix.Name{ CommonName: "localhost", diff --git a/libbeat/common/transport/tlscommon/config.go b/libbeat/common/transport/tlscommon/config.go index 8d7650eb5bf..30009c2b13b 100644 --- a/libbeat/common/transport/tlscommon/config.go +++ b/libbeat/common/transport/tlscommon/config.go @@ -21,6 +21,8 @@ import ( "crypto/tls" "github.com/joeshaw/multierror" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" ) // Config defines the user configurable options in the yaml file. @@ -96,6 +98,8 @@ func LoadTLSConfig(config *Config) (*TLSConfig, error) { // Validate values the TLSConfig struct making sure certificate sure we have both a certificate and // a key. func (c *Config) Validate() error { + cfgwarn.Deprecate("8.0.0", "Treating the CommonName field on X.509 certificates as a host name when no Subject Alternative Names are present is going to be removed. Please update your certificates if needed.") + return c.Certificate.Validate() } diff --git a/libbeat/common/transport/tlscommon/server_config.go b/libbeat/common/transport/tlscommon/server_config.go index 866d6e3c28c..e85a0c409c3 100644 --- a/libbeat/common/transport/tlscommon/server_config.go +++ b/libbeat/common/transport/tlscommon/server_config.go @@ -28,13 +28,14 @@ import ( // ServerConfig defines the user configurable tls options for any TCP based service. type ServerConfig struct { Enabled *bool `config:"enabled"` - VerificationMode TLSVerificationMode `config:"verification_mode"` // one of 'none', 'full' + VerificationMode TLSVerificationMode `config:"verification_mode"` // one of 'none', 'full', 'strict', 'certificate' Versions []TLSVersion `config:"supported_protocols"` CipherSuites []tlsCipherSuite `config:"cipher_suites"` CAs []string `config:"certificate_authorities"` Certificate CertificateConfig `config:",inline"` CurveTypes []tlsCurveType `config:"curve_types"` ClientAuth tlsClientAuth `config:"client_authentication"` //`none`, `optional` or `required` + CASha256 []string `config:"ca_sha256" yaml:"ca_sha256,omitempty"` } // LoadTLSServerConfig tranforms a ServerConfig into a `tls.Config` to be used directly with golang @@ -88,6 +89,7 @@ func LoadTLSServerConfig(config *ServerConfig) (*TLSConfig, error) { CipherSuites: cipherSuites, CurvePreferences: curves, ClientAuth: tls.ClientAuthType(config.ClientAuth), + CASha256: config.CASha256, }, nil } diff --git a/libbeat/common/transport/tlscommon/testdata/cacert.crt b/libbeat/common/transport/tlscommon/testdata/cacert.crt new file mode 100644 index 00000000000..debdf7e246e --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/cacert.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEBDCCAuygAwIBAgIUXwbLbwGjWWlQNrMUsdDpKzeGixEwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBQMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDTALBgNV +BAsMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtXsn+VCrW +ibutoByM5EeIK29XYffBwN78EeNjDdaZZqMF4wGZZ6z2xQXH6mFx+m1gjnf5R2qo +yfentYH5VRZz5AEtBGPsOqMffV9u5PkHSo/2ilCX40eBVp5u3qh6aFPZ5DKqexWu +5jUMYolTXpvAtML5YbMH9XvW6pn5WAqwHPLNe+fVuPg4tJN0u/ff0wKqSUBIhVOP +7EPhz3yLflACScgj+LPXz/5gtUXe9RR5RB8zyWGfNL91eoVVaApcdp4kIU+DHmgI +p+T4CpgdYWsYuOWH49F7RJyLpocUU4H+heeC4+zH0LIUcELa+n/M2DUDW3RE109a +tv9OEJKR8/YHAgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +fyEN1Qe7FlWa+2RBnl8Vd4ZCFkIwgY0GA1UdIwSBhTCBgoAUfyEN1Qe7FlWa+2RB +nl8Vd4ZCFkKhVKRSMFAxCzAJBgNVBAYTAkNBMQ8wDQYDVQQIDAZRdWViZWMxETAP +BgNVBAcMCE1vbnRyZWFsMQ4wDAYDVQQKDAViZWF0czENMAsGA1UECwwEcm9vdIIU +XwbLbwGjWWlQNrMUsdDpKzeGixEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +CwUAA4IBAQAANxJCfDMcNNnAVRlXLdh+loVx8Y5STf1gTgX2gtf9tHZGYE7/ix2P +dG1uQcEz/ETlcGSWRZcQSNR8dNeBi5YWK5dmDUD7reQr3FoyIDvPGHyIcF3clglg +blYhsQN0TVwx4G3kZDenjzKNSyVLR81opLq/PDIGW61ZCioJUQKs5q+IqsKj+okn +in6/b5YfQqyTDIWY3IPiXjvcysbKC0pYc0TkmwGUnidxDny7txrVCVJ1vwIedQug +B/UOjVxi0qsNwpWS08mwEOVvgvObi0mFoGQl8l427M0kM//86NM7vDc4Z0QYHOlq +A0ZjtnSbR3RqfhBGXV3BL+GHtXevn55Z +-----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/cacert.key b/libbeat/common/transport/tlscommon/testdata/cacert.key new file mode 100644 index 00000000000..e864b93ed66 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/cacert.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArV7J/lQq1om7raAcjORHiCtvV2H3wcDe/BHjYw3WmWajBeMB +mWes9sUFx+phcfptYI53+UdqqMn3p7WB+VUWc+QBLQRj7DqjH31fbuT5B0qP9opQ +l+NHgVaebt6oemhT2eQyqnsVruY1DGKJU16bwLTC+WGzB/V71uqZ+VgKsBzyzXvn +1bj4OLSTdLv339MCqklASIVTj+xD4c98i35QAknII/iz18/+YLVF3vUUeUQfM8lh +nzS/dXqFVWgKXHaeJCFPgx5oCKfk+AqYHWFrGLjlh+PRe0Sci6aHFFOB/oXnguPs +x9CyFHBC2vp/zNg1A1t0RNdPWrb/ThCSkfP2BwIDAQABAoIBAQCQmLJYENL5xD5n +/VZSnEKc670dYHRHgRl5m2HPR8doghYN3tuCmtnDp2e+6VkEux1mnuypWEs5I9oO +YnBZCAKF/fCNH1BHwlAy/1oNH6Qj1Khls86sH7+PvDK/va0/CqyE2rL3RVk8Wnx8 +K+LlSc8V1q2XWUj8pl33TgvFzwx6/QpmGa1ofK84GaeWNskRt8xyf2HECiRl6ZFm +zZr2Ror3nRbgZK9FYWpcp6HUgxAH/8GQ3+8vMvftfTsDGD5TmmEq6CFgAFCVj92L +d7AZmNWR1483NzZF0HWOQ6ew9qrWkqVpER7kKKp/kkfoh2qXgvtQBTrw4IcCRwwa +szaSsIEBAoGBANiqXhBzPQJszm1Ajln07ZeyvgRB8PgzZXcAHS9AfGqh/mGQw5/X +3vqHdGiEynphoYtNqK1YT7RH7pkjkpqDzdunZGz1xog7i4ys8kVtivkDGlhn6cXI +4wmFcmyCaf76VPPr1RX8PNjsEKDK3jq1d86lBjSLPgcHT7J16WZgOcJnAoGBAMzY +QVNpjk1WNT7gid3MUXciIIZAovej4AiVyn97XxxLSyByXmNds65f3dM8NOJkJUvT +iV7pAjKl9pd1lE+WTNQSjCgSxw7G+4u9cQfNE7p6klAh/Rek76Mani9rAmQ2PdJl +EFaEgLom3wbR5eOkYURjw2jfqzFYQ8T1YZkWBithAoGAa3EYkknDIFe6ifzwWnWV ++Jr/lXbpuvspvrhEwLDWwb4xOkqiZ7qR7WSMemQXUFbn1/+bvNJFPB5LmI9GXO8t +f1Zj+5BpchctHYaJ4Znvx4odX2ewSo9S3t7ZHiwRygpzZD43fd6Ggf+WQ1Y2m6Bv +l/7Hs/i0uqGKiPHl2wmuutMCgYABZN9c7/T19cY6/VAy4DcVtne+MiZpxQW7STmt +kGtfR+vk9qJJztNwNlrOGzTI7aGLWI8wxCktqw94jGZL/FvdfZrSkv4jzZrcopdo +VC70L+1a+kA8rvSqiX3WGMZVZEEbc3CfBhvSKH2QEFGeMPowevVTe2Iw3cboSjs1 +zX6RQQKBgFV7gOstMfvixCSUCD2s5j/skhNJsB3Wd/tVYRbl/vgA6hHW8UOy2oWv +UTE45vJNVzRv030G5katjOYhlxHf9rpeSAbeIyty54I3X9/vDJZLXwe8WilQjUr7 +Dw8yNwH44j/0s8xcQXG8yE0h1Aa9GxHHtJtYrRYdx7sSwNHtwpnp +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/testdata/client1.crt b/libbeat/common/transport/tlscommon/testdata/client1.crt new file mode 100644 index 00000000000..c3139a72a77 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/client1.crt @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIEFzCCAv+gAwIBAgIUeaB7uk2DjAM2cuRl0kaE9ly7Lj4wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBmMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDzANBgNV +BAsMBnNlcnZlcjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA3jXEj7vN+BDlj6cYblKSml0FWpO4yi9C58cubXXDWXI6 +hdpzNpDa0+n606Jg4eVZpFUZPTnnjQmFIcesO0+i85V4Etswr4T22uobDu1AWV7n +26nDMY/vlf+kDI8H/uFgxQg/Htuh12nHuYrjIS+ot/D6gThwIWVldu0TaBaFfvL5 +5qTPRJoteiBPo5y+VuWLhzPWg8cQYZ4KJ4XREk8H4d7PqFRHp+zATfn2YLBjUK7Z +zd0W3mxkdB2P7MnzZuH5n5zrgJ8OI9voopX8QadMYtUSeITP1INmNKhi4vLbpZjU +mt+N/u1G6xwbuyJiSlklBoXdRcWj5kSljpLtF1evvwIDAQABo4HQMIHNMAwGA1Ud +EwEB/wQCMAAwHQYDVR0OBBYEFAuDdHxE9/Zr7iVwfnUJ/lRtJnZkMIGNBgNVHSME +gYUwgYKAFH8hDdUHuxZVmvtkQZ5fFXeGQhZCoVSkUjBQMQswCQYDVQQGEwJDQTEP +MA0GA1UECAwGUXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVh +dHMxDTALBgNVBAsMBHJvb3SCFF8Gy28Bo1lpUDazFLHQ6Ss3hosRMA4GA1UdDwEB +/wQEAwIF4DANBgkqhkiG9w0BAQsFAAOCAQEACzuX6AiVHk5Igs/LdOW2sJ9lm95N +Su1PQCobM0Jo8wX3pDAEQlLmaWTDcr4bfrQPfI8pih1F89DQU9z0nzNCRfxiQaA7 +myF8ftvf8v5j3LpaPWlkdWgCRieCl58fgy5vtcKx73eTY4a6SRB4zbWpl0rX9H6w +En1kQbpCJDzh8W+xmr8AKvY77CSC1vt7TaKan6F+fGwbt8kIng6P6C7dvMGsDKQN +2Tiq/wtH16DB8mOeO+zfxJfa84TPWL4UcSbZJ8w5Fyz4GJormaymxJGtKv58RO7J +u63WF9vlEnKGyqY1FckTsp3P9ivGEb/Y75+NyRwmNq5VO5BPrRBMOF3VAg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEBDCCAuygAwIBAgIUXwbLbwGjWWlQNrMUsdDpKzeGixEwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBQMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDTALBgNV +BAsMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtXsn+VCrW +ibutoByM5EeIK29XYffBwN78EeNjDdaZZqMF4wGZZ6z2xQXH6mFx+m1gjnf5R2qo +yfentYH5VRZz5AEtBGPsOqMffV9u5PkHSo/2ilCX40eBVp5u3qh6aFPZ5DKqexWu +5jUMYolTXpvAtML5YbMH9XvW6pn5WAqwHPLNe+fVuPg4tJN0u/ff0wKqSUBIhVOP +7EPhz3yLflACScgj+LPXz/5gtUXe9RR5RB8zyWGfNL91eoVVaApcdp4kIU+DHmgI +p+T4CpgdYWsYuOWH49F7RJyLpocUU4H+heeC4+zH0LIUcELa+n/M2DUDW3RE109a +tv9OEJKR8/YHAgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +fyEN1Qe7FlWa+2RBnl8Vd4ZCFkIwgY0GA1UdIwSBhTCBgoAUfyEN1Qe7FlWa+2RB +nl8Vd4ZCFkKhVKRSMFAxCzAJBgNVBAYTAkNBMQ8wDQYDVQQIDAZRdWViZWMxETAP +BgNVBAcMCE1vbnRyZWFsMQ4wDAYDVQQKDAViZWF0czENMAsGA1UECwwEcm9vdIIU +XwbLbwGjWWlQNrMUsdDpKzeGixEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +CwUAA4IBAQAANxJCfDMcNNnAVRlXLdh+loVx8Y5STf1gTgX2gtf9tHZGYE7/ix2P +dG1uQcEz/ETlcGSWRZcQSNR8dNeBi5YWK5dmDUD7reQr3FoyIDvPGHyIcF3clglg +blYhsQN0TVwx4G3kZDenjzKNSyVLR81opLq/PDIGW61ZCioJUQKs5q+IqsKj+okn +in6/b5YfQqyTDIWY3IPiXjvcysbKC0pYc0TkmwGUnidxDny7txrVCVJ1vwIedQug +B/UOjVxi0qsNwpWS08mwEOVvgvObi0mFoGQl8l427M0kM//86NM7vDc4Z0QYHOlq +A0ZjtnSbR3RqfhBGXV3BL+GHtXevn55Z +-----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/client1.key b/libbeat/common/transport/tlscommon/testdata/client1.key new file mode 100644 index 00000000000..ce5274b758f --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/client1.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA3jXEj7vN+BDlj6cYblKSml0FWpO4yi9C58cubXXDWXI6hdpz +NpDa0+n606Jg4eVZpFUZPTnnjQmFIcesO0+i85V4Etswr4T22uobDu1AWV7n26nD +MY/vlf+kDI8H/uFgxQg/Htuh12nHuYrjIS+ot/D6gThwIWVldu0TaBaFfvL55qTP +RJoteiBPo5y+VuWLhzPWg8cQYZ4KJ4XREk8H4d7PqFRHp+zATfn2YLBjUK7Zzd0W +3mxkdB2P7MnzZuH5n5zrgJ8OI9voopX8QadMYtUSeITP1INmNKhi4vLbpZjUmt+N +/u1G6xwbuyJiSlklBoXdRcWj5kSljpLtF1evvwIDAQABAoIBABdTza7JKHZCT9ck +04vBX2KVIVrA50VScNOkNVuIYVmihEJJDI9N5asZhRtykHkmeqKlzGCBE63asf85 +1vrjAVhQ+KoCGLpUWxXgPbbzcS3wqKaGy9cIJT65957Z5Rz8zAvjMb0rkXHryOvR +iMaTGkM1KRcntZ3L5zr06HSk6J7K8QCEexKHl7Q7Ki1498tvBWdJGeGWRiUtI89j +wOUdcf3pVSVqI7J8gmmqVwNrVMbVxhlen7nkckXofWAackYVQDBD+hU1n3doNKLa +NP6mZkI02BOB29WLDXLuHtKDZtgnXex4JUz6zw53uV42FCDoQf3DUiVsMEL8xRCJ +27H6bwECgYEA/w53zS00mNdYdXO7dGhAw3UYPc3PDyg6Z823BQzfdOzsn5Yw0BIw +nPgstzwzOL0kw2p/PgwkG/7LOsF5CWs2xvU3LhUdOhgmw4B5IbMOYvbkVoYGz+22 +HJf4qyexAr7tKCITB+LCzUwoAgXp8uju1XdLVpk6xmJ3u+kIhMYTxkUCgYEA3wgx +71/uIUsoW6bVL5K00yXPWTTFtTBWM768VJ8Y++k2igPgcvKaBVaElr4AbvX5iCGz +1Ycc9xsGAYAo7+q4D+4cuOki/m0PMKD3DgXWpTtN0kJ+npWUBdE98NyDlTJYsa/w +xjeMQoDvC8tE2bAiwtVIOPQL2C/3emqkJcsVcDMCgYB8NeOJ/DXdKSJfMJldu1eu +2FuR3aS00PaAjuJOh1JbcvZZUZ879V/PUd0U7zBStWot8LM+2FLNf2whlQ8I0zm9 +8rWIr6eoHxLhqrNTAgxDjdDtgh/XKwDBNBFZ6N5/Y9PC87Uo5fnQWQIy2gZw0Zde +RdZeugixjEqbLIWFg6ElsQKBgHRy6O+c3M6RWU8ROnoOVU9xjGN9REUoKbn2uopM +T1UoHQvOnmAl/vkOhUfXiI5m65SCVE0GsL7sYyRhb/5kRRo8Ls71GwpQkv/G63ds +4PeAkU9Y3JecbZ7j8z1RRXqewOR1gndcBWWrwCQeS6KFboDfr0fdVFnaIZLPH0mE +UXs1AoGBAM3zpcyl5o99dO6x9N/8SSnyLT9TzzbJ6pU6d0F0ELn3OxTUBH1oA1dy +q1fADcRgN5vNuJljY4es/scK2BMeX1isFitXoIzk01F4R61xoXr8T33731eXFG6L +ehoECH2Yj9H4qNbVW531iYKheuSyaMaxCxaDoK9jBzcKaxMGbTlc +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/tls_config.go b/libbeat/common/transport/tlscommon/tls_config.go index 22cebb2bf8d..9e7eb4548db 100644 --- a/libbeat/common/transport/tlscommon/tls_config.go +++ b/libbeat/common/transport/tlscommon/tls_config.go @@ -20,8 +20,12 @@ package tlscommon import ( "crypto/tls" "crypto/x509" + "fmt" + "net" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/logp" ) @@ -75,8 +79,13 @@ type TLSConfig struct { time func() time.Time } -// ToConfig generates a tls.Config object. Note, you must use BuildModuleConfig to generate a config with +var ( + MissingPeerCertificate = errors.New("missing peer certificates") +) + +// ToConfig generates a tls.Config object. Note, you must use BuildModuleClientConfig to generate a config with // ServerName set, use that method for servers with SNI. +// By default VerifyConnection is set to client mode. func (c *TLSConfig) ToConfig() *tls.Config { if c == nil { return &tls.Config{} @@ -84,36 +93,37 @@ func (c *TLSConfig) ToConfig() *tls.Config { minVersion, maxVersion := extractMinMaxVersion(c.Versions) - // When we are using the CAsha256 pin to validate the CA used to validate the chain, - // or when we are using 'certificate' TLS verification mode, we add a custom callback - verifyPeerCertFn := makeVerifyPeerCertificate(c) - - insecure := c.Verification != VerifyFull + insecure := c.Verification != VerifyStrict if c.Verification == VerifyNone { logp.NewLogger("tls").Warn("SSL/TLS verifications disabled.") } - return &tls.Config{ - MinVersion: minVersion, - MaxVersion: maxVersion, - Certificates: c.Certificates, - RootCAs: c.RootCAs, - ClientCAs: c.ClientCAs, - InsecureSkipVerify: insecure, - CipherSuites: c.CipherSuites, - CurvePreferences: c.CurvePreferences, - Renegotiation: c.Renegotiation, - ClientAuth: c.ClientAuth, - VerifyPeerCertificate: verifyPeerCertFn, - Time: c.time, + MinVersion: minVersion, + MaxVersion: maxVersion, + Certificates: c.Certificates, + RootCAs: c.RootCAs, + ClientCAs: c.ClientCAs, + InsecureSkipVerify: insecure, + CipherSuites: c.CipherSuites, + CurvePreferences: c.CurvePreferences, + Renegotiation: c.Renegotiation, + ClientAuth: c.ClientAuth, + Time: c.time, + VerifyConnection: makeVerifyConnection(c), } } // BuildModuleConfig takes the TLSConfig and transform it into a `tls.Config`. -func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config { +func (c *TLSConfig) BuildModuleClientConfig(host string) *tls.Config { if c == nil { // use default TLS settings, if config is empty. - return &tls.Config{ServerName: host} + return &tls.Config{ + ServerName: host, + InsecureSkipVerify: true, + VerifyConnection: makeVerifyConnection(&TLSConfig{ + Verification: VerifyFull, + }), + } } config := c.ToConfig() @@ -121,33 +131,169 @@ func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config { return config } -// makeVerifyPeerCertificate creates the verification combination of checking certificate pins and skipping host name validation depending on the config -func makeVerifyPeerCertificate(cfg *TLSConfig) verifyPeerCertFunc { - pin := len(cfg.CASha256) > 0 - skipHostName := cfg.Verification == VerifyCertificate - - if pin && !skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - return verifyCAPin(cfg.CASha256, verifiedChains) +// BuildServerConfig takes the TLSConfig and transform it into a `tls.Config` for server side objects. +func (c *TLSConfig) BuildServerConfig(host string) *tls.Config { + if c == nil { + // use default TLS settings, if config is empty. + return &tls.Config{ + ServerName: host, + InsecureSkipVerify: true, + VerifyConnection: makeVerifyServerConnection(&TLSConfig{ + Verification: VerifyFull, + }), } } - if pin && skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - _, _, err := verifyCertificateExceptServerName(rawCerts, cfg) + config := c.ToConfig() + config.ServerName = host + config.VerifyConnection = makeVerifyServerConnection(c) + return config +} + +func makeVerifyConnection(cfg *TLSConfig) func(tls.ConnectionState) error { + switch cfg.Verification { + case VerifyFull: + return func(cs tls.ConnectionState) error { + // On the client side, PeerCertificates can't be empty. + if len(cs.PeerCertificates) == 0 { + return MissingPeerCertificate + } + + opts := x509.VerifyOptions{ + Roots: cfg.RootCAs, + Intermediates: x509.NewCertPool(), + } + err := verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) if err != nil { return err } - return verifyCAPin(cfg.CASha256, verifiedChains) + return verifyHostname(cs.PeerCertificates[0], cs.ServerName) + } + case VerifyCertificate: + return func(cs tls.ConnectionState) error { + // On the client side, PeerCertificates can't be empty. + if len(cs.PeerCertificates) == 0 { + return MissingPeerCertificate + } + + opts := x509.VerifyOptions{ + Roots: cfg.RootCAs, + Intermediates: x509.NewCertPool(), + } + return verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + } + case VerifyStrict: + if len(cfg.CASha256) > 0 { + return func(cs tls.ConnectionState) error { + return verifyCAPin(cfg.CASha256, cs.VerifiedChains) + } } + default: } - if !pin && skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - _, _, err := verifyCertificateExceptServerName(rawCerts, cfg) - return err + return nil + +} + +func makeVerifyServerConnection(cfg *TLSConfig) func(tls.ConnectionState) error { + switch cfg.Verification { + case VerifyFull: + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + if cfg.ClientAuth == tls.RequireAndVerifyClientCert { + return MissingPeerCertificate + } + return nil + } + + opts := x509.VerifyOptions{ + Roots: cfg.ClientCAs, + Intermediates: x509.NewCertPool(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + err := verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + if err != nil { + return err + } + return verifyHostname(cs.PeerCertificates[0], cs.ServerName) + } + case VerifyCertificate: + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + if cfg.ClientAuth == tls.RequireAndVerifyClientCert { + return MissingPeerCertificate + } + return nil + } + + opts := x509.VerifyOptions{ + Roots: cfg.ClientCAs, + Intermediates: x509.NewCertPool(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + return verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + } + case VerifyStrict: + if len(cfg.CASha256) > 0 { + return func(cs tls.ConnectionState) error { + return verifyCAPin(cfg.CASha256, cs.VerifiedChains) + } } + default: + } + + return nil + +} + +func verifyCertsWithOpts(certs []*x509.Certificate, casha256 []string, opts x509.VerifyOptions) error { + for _, cert := range certs[1:] { + opts.Intermediates.AddCert(cert) + } + verifiedChains, err := certs[0].Verify(opts) + if err != nil { + return err } + if len(casha256) > 0 { + return verifyCAPin(casha256, verifiedChains) + } return nil } + +func verifyHostname(cert *x509.Certificate, hostname string) error { + if hostname == "" { + return nil + } + // check if the server name is an IP + ip := hostname + if len(ip) >= 3 && ip[0] == '[' && ip[len(ip)-1] == ']' { + ip = ip[1 : len(ip)-1] + } + parsedIP := net.ParseIP(ip) + if parsedIP != nil { + for _, certIP := range cert.IPAddresses { + if parsedIP.Equal(certIP) { + return nil + } + } + return x509.HostnameError{Certificate: cert, Host: hostname} + } + + dnsnames := cert.DNSNames + if len(dnsnames) == 0 || len(dnsnames) == 1 && dnsnames[0] == "" { + if cert.Subject.CommonName != "" { + dnsnames = []string{cert.Subject.CommonName} + } + } + + for _, name := range dnsnames { + if len(name) > 0 && len(hostname) > 0 && name == hostname { + if !validHostname(name, true) { + return fmt.Errorf("invalid hostname in cert") + } + return nil + } + } + return x509.HostnameError{Certificate: cert, Host: hostname} +} diff --git a/libbeat/common/transport/tlscommon/tls_config_test.go b/libbeat/common/transport/tlscommon/tls_config_test.go new file mode 100644 index 00000000000..1490664d3d3 --- /dev/null +++ b/libbeat/common/transport/tlscommon/tls_config_test.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeVerifyServerConnection(t *testing.T) { + testCerts, err := openTestCerts() + if err != nil { + t.Fatalf("failed to open test certs: %+v", err) + } + + testCA, errs := LoadCertificateAuthorities([]string{filepath.Join("testdata", "cacert.crt")}) + if len(errs) > 0 { + t.Fatalf("failed to load test certificate authorities: %+v", errs) + } + + testcases := map[string]struct { + verificationMode TLSVerificationMode + clientAuth tls.ClientAuthType + certAuthorities *x509.CertPool + peerCerts []*x509.Certificate + serverName string + expectedCallback bool + expectedError error + }{ + "default verification without certificates when required": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + peerCerts: nil, + serverName: "", + expectedCallback: true, + expectedError: MissingPeerCertificate, + }, + "default verification with certificates when required with expired cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["expired"]}, + serverName: "", + expectedCallback: true, + expectedError: x509.CertificateInvalidError{Cert: testCerts["expired"], Reason: x509.Expired}, + }, + "default verification with certificates when required with incorrect server name in cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "bad.example.com", + expectedCallback: true, + expectedError: x509.HostnameError{Certificate: testCerts["correct"], Host: "bad.example.com"}, + }, + "default verification with certificates when required with correct cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: nil, + }, + "certificate verification with certificates when required with correct cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: nil, + }, + "certificate verification with certificates when required with expired cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["expired"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: x509.CertificateInvalidError{Cert: testCerts["expired"], Reason: x509.Expired}, + }, + "certificate verification with certificates when required with incorrect server name in cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "bad.example.com", + expectedCallback: true, + expectedError: nil, + }, + "strict verification with certificates when required with correct cert": { + verificationMode: VerifyStrict, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: false, + expectedError: nil, + }, + "default verification with certificates when required with cert signed by unkown authority": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["unknown authority"]}, + serverName: "", + expectedCallback: true, + expectedError: x509.UnknownAuthorityError{Cert: testCerts["unknown authority"]}, + }, + "default verification without certificates not required": { + verificationMode: VerifyFull, + clientAuth: tls.NoClientCert, + peerCerts: nil, + serverName: "", + expectedCallback: true, + expectedError: nil, + }, + "no verification without certificates not required": { + verificationMode: VerifyNone, + clientAuth: tls.NoClientCert, + peerCerts: nil, + serverName: "", + expectedError: nil, + }, + } + + for name, test := range testcases { + t.Run(name, func(t *testing.T) { + test := test + cfg := &TLSConfig{ + Verification: test.verificationMode, + ClientAuth: test.clientAuth, + ClientCAs: test.certAuthorities, + } + + verifier := makeVerifyServerConnection(cfg) + if !test.expectedCallback { + assert.Nil(t, verifier) + return + } + + err := verifier(tls.ConnectionState{ + PeerCertificates: test.peerCerts, + ServerName: test.serverName, + }) + if test.expectedError == nil { + assert.Nil(t, err) + } else { + assert.Error(t, test.expectedError, err) + } + }) + } + +} + +func openTestCerts() (map[string]*x509.Certificate, error) { + certs := make(map[string]*x509.Certificate, 0) + + for testcase, certname := range map[string]string{ + "expired": "tls.crt", + "unknown authority": "unsigned_tls.crt", + "correct": "client1.crt", + } { + + certBytes, err := ioutil.ReadFile(filepath.Join("testdata", certname)) + if err != nil { + return nil, err + } + block, _ := pem.Decode(certBytes) + testCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + certs[testcase] = testCert + } + + return certs, nil +} diff --git a/libbeat/common/transport/tlscommon/tls_test.go b/libbeat/common/transport/tlscommon/tls_test.go index 53e9da18db3..45c0ebf1f7f 100644 --- a/libbeat/common/transport/tlscommon/tls_test.go +++ b/libbeat/common/transport/tlscommon/tls_test.go @@ -132,12 +132,12 @@ func TestApplyEmptyConfig(t *testing.T) { t.Fatal(err) } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Len(t, cfg.Certificates, 0) assert.Nil(t, cfg.RootCAs) - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) assert.Equal(t, tls.RenegotiateNever, cfg.Renegotiation) @@ -159,7 +159,7 @@ func TestApplyWithConfig(t *testing.T) { t.Fatal(err) } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) assert.Len(t, cfg.Certificates, 1) assert.NotNil(t, cfg.RootCAs) @@ -184,7 +184,7 @@ key: mykey.pem tmp, err := LoadTLSServerConfig(&c) require.NoError(t, err) - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) // values not set by default @@ -193,7 +193,7 @@ key: mykey.pem assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) // values set by default - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Equal(t, tls.NoClientCert, cfg.ClientAuth) @@ -213,7 +213,7 @@ key: mykey.pem tmp, err := LoadTLSServerConfig(&c) require.NoError(t, err) - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) // values not set by default @@ -222,7 +222,7 @@ key: mykey.pem assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) // values set by default - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Equal(t, tls.RequireAndVerifyClientCert, cfg.ClientAuth) @@ -260,7 +260,7 @@ func TestApplyWithServerConfig(t *testing.T) { return } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) assert.Len(t, cfg.Certificates, 1) assert.NotNil(t, cfg.ClientCAs) diff --git a/libbeat/common/transport/tlscommon/types.go b/libbeat/common/transport/tlscommon/types.go index c130a57c71a..29b11c92010 100644 --- a/libbeat/common/transport/tlscommon/types.go +++ b/libbeat/common/transport/tlscommon/types.go @@ -127,11 +127,13 @@ const ( VerifyFull TLSVerificationMode = iota VerifyNone VerifyCertificate + VerifyStrict ) var tlsVerificationModes = map[string]TLSVerificationMode{ "": VerifyFull, "full": VerifyFull, + "strict": VerifyStrict, "none": VerifyNone, "certificate": VerifyCertificate, } diff --git a/libbeat/common/transport/tlscommon/verify.go b/libbeat/common/transport/tlscommon/validhostname.go similarity index 59% rename from libbeat/common/transport/tlscommon/verify.go rename to libbeat/common/transport/tlscommon/validhostname.go index 867e70874a9..15370b4d4f9 100644 --- a/libbeat/common/transport/tlscommon/verify.go +++ b/libbeat/common/transport/tlscommon/validhostname.go @@ -43,58 +43,55 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// This file contains code adapted from golang's crypto/tls/handshake_client.go +// This file contains code adapted from golang's crypto/x509/verify.go package tlscommon -import ( - "crypto/x509" - "time" +import "strings" - "github.com/pkg/errors" -) - -// verifyCertificateExceptServerName is a TLS Certificate verification utility method that verifies that the provided -// certificate chain is valid and is signed by one of the root CAs in the provided tls.Config. It is intended to be -// as similar as possible to the default verify, but does not verify that the provided certificate matches the -// ServerName in the tls.Config. -func verifyCertificateExceptServerName( - rawCerts [][]byte, - c *TLSConfig, -) ([]*x509.Certificate, [][]*x509.Certificate, error) { - // this is where we're a bit suboptimal, as we have to re-parse the certificates that have been presented - // during the handshake. - // the verification code here is taken from verifyServerCertificate in crypto/tls/handshake_client.go:824 - certs := make([]*x509.Certificate, len(rawCerts)) - for i, asn1Data := range rawCerts { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - return nil, nil, errors.Wrap(err, "tls: failed to parse certificate from server") - } - certs[i] = cert +// validHostname reports whether host is a valid hostname that can be matched or +// matched against according to RFC 6125 2.2, with some leniency to accommodate +// legacy values. +func validHostname(host string, isPattern bool) bool { + if !isPattern { + host = strings.TrimSuffix(host, ".") } - - var t time.Time - if c.time != nil { - t = c.time() - } else { - t = time.Now() - } - - // DNSName omitted in VerifyOptions in order to skip ServerName verification - opts := x509.VerifyOptions{ - Roots: c.RootCAs, - CurrentTime: t, - Intermediates: x509.NewCertPool(), + if len(host) == 0 { + return false } - for _, cert := range certs[1:] { - opts.Intermediates.AddCert(cert) + for i, part := range strings.Split(host, ".") { + if part == "" { + // Empty label. + return false + } + if isPattern && i == 0 && part == "*" { + // Only allow full left-most wildcards, as those are the only ones + // we match, and matching literal '*' characters is probably never + // the expected behavior. + continue + } + for j, c := range part { + if 'a' <= c && c <= 'z' { + continue + } + if '0' <= c && c <= '9' { + continue + } + if 'A' <= c && c <= 'Z' { + continue + } + if c == '-' && j != 0 { + continue + } + if c == '_' { + // Not a valid character in hostnames, but commonly + // found in deployments outside the WebPKI. + continue + } + return false + } } - headCert := certs[0] - - // defer to the default verification performed - chains, err := headCert.Verify(opts) - return certs, chains, err + return true } diff --git a/libbeat/common/transport/tlscommon/verify_test.go b/libbeat/common/transport/tlscommon/verify_test.go deleted file mode 100644 index c08e60642f5..00000000000 --- a/libbeat/common/transport/tlscommon/verify_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package tlscommon - -import ( - "crypto/x509" - "encoding/pem" - "io/ioutil" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// This does not actually test that it ignores the server name because no part of the func even consumes the server name -func Test_verifyCertificateExceptServerName(t *testing.T) { - - tests := []struct { - name string - ca string - chain string - cert string - time func() time.Time - wantErr bool - }{ - { - name: "happy path", - // a CA for morello.ovh valid from August 9 2019 to 2029 - ca: "ca.crt", - // a cert signed by morello.ovh that expired in nov 2019 - cert: "tls.crt", - time: func() time.Time { - layout := "2006-01-02" - t, _ := time.Parse(layout, "2019-10-01") - return t - }, - wantErr: false, - }, - { - name: "cert not signed by CA", - ca: "ca.crt", - // a self-signed cert for www.example.com valid from July 23 2020 to 2030 - cert: "unsigned_tls.crt", - time: func() time.Time { - layout := "2006-01-02" - t, _ := time.Parse(layout, "2020-07-24") - return t - }, - wantErr: true, - }, - { - name: "cert expired", - ca: "ca.crt", - cert: "tls.crt", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := &TLSConfig{time: tc.time} - // load the CA - if tc.ca != "" { - ca := loadFileBytes(tc.ca) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(ca) - cfg.RootCAs = caCertPool - } - - // load the cert - rawCerts := [][]byte{} - if tc.cert != "" { - pemCert := loadFileBytes(tc.cert) - block, _ := pem.Decode(pemCert) - rawCerts = append(rawCerts, block.Bytes) - } - - _, _, got := verifyCertificateExceptServerName(rawCerts, cfg) - if tc.wantErr { - assert.Error(t, got) - } else { - assert.NoError(t, got) - } - }) - } -} - -func loadFileBytes(fileName string) []byte { - contents, err := ioutil.ReadFile(filepath.Join("testdata", fileName)) - if err != nil { - panic(err) - } - return contents -} diff --git a/libbeat/common/transport/transptest/testing.go b/libbeat/common/transport/transptest/testing.go index 64763ba9b71..1649957c624 100644 --- a/libbeat/common/transport/transptest/testing.go +++ b/libbeat/common/transport/transptest/testing.go @@ -135,7 +135,7 @@ func NewMockServerTLS(t *testing.T, to time.Duration, cert string, proxy *transp t.Fatalf("failed to load certificate") } - listener := tls.NewListener(tcpListener, tlsConfig.BuildModuleConfig("")) + listener := tls.NewListener(tcpListener, tlsConfig.BuildServerConfig("")) server := &MockServer{Listener: listener, Timeout: to} server.Handshake = func(client net.Conn) { diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 767bd7c2f08..db9b8619a7a 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -164,10 +164,11 @@ endif::serverless[] [float] ==== `logging.metrics.enabled` -If enabled, {beatname_uc} periodically logs its internal metrics that have +By default, {beatname_uc} periodically logs its internal metrics that have changed in the last period. For each metric that changed, the delta from the value at the beginning of the period is logged. Also, the total values for all -non-zero internal metrics are logged on shutdown. The default is true. +non-zero internal metrics are logged on shutdown. Set this to false to disable +this behavior. The default is true. Here is an example log line: diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index ba3fee9eebe..b870aabf077 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 906aa673a14..89578a6f7a2 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.0.0 :doc-branch: master -:go-version: 1.14.12 +:go-version: 1.15.7 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index c785f9e729f..c505e7d1824 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -24,8 +24,10 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/Shopify/sarama" + "github.com/eapache/go-resiliency/breaker" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/common/transport" @@ -47,6 +49,7 @@ type client struct { codec codec.Codec config sarama.Config mux sync.Mutex + done chan struct{} producer sarama.AsyncProducer @@ -85,6 +88,7 @@ func newKafkaClient( index: strings.ToLower(index), codec: writer, config: *cfg, + done: make(chan struct{}), } return c, nil } @@ -121,6 +125,7 @@ func (c *client) Close() error { return nil } + close(c.done) c.producer.AsyncClose() c.wg.Wait() c.producer = nil @@ -237,12 +242,92 @@ func (c *client) successWorker(ch <-chan *sarama.ProducerMessage) { } func (c *client) errorWorker(ch <-chan *sarama.ProducerError) { + breakerOpen := false defer c.wg.Done() defer c.log.Debug("Stop kafka error handler") for errMsg := range ch { msg := errMsg.Msg.Metadata.(*message) msg.ref.fail(msg, errMsg.Err) + + if errMsg.Err == breaker.ErrBreakerOpen { + // ErrBreakerOpen is a very special case in Sarama. It happens only when + // there have been repeated critical (broker / topic-level) errors, and it + // puts Sarama into a state where it immediately rejects all input + // for 10 seconds, ignoring retry / backoff settings. + // With this output's current design (in which Publish passes through to + // Sarama's input channel with no further synchronization), retrying + // these failed values causes an infinite retry loop that degrades + // the entire system. + // "Nice" approaches and why we haven't used them: + // - Use exposed API to navigate this state and its effect on retries. + // * Unfortunately, Sarama's circuit breaker and its errors are + // hard-coded and undocumented. We'd like to address this in the + // future. + // - If a batch fails with a circuit breaker error, delay before + // retrying it. + // * This would fix the most urgent performance issues, but requires + // extra bookkeeping because the Kafka output handles each batch + // independently. It results in potentially many batches / 10s of + // thousands of events being loaded and attempted, even though we + // know there's a fatal error early in the first batch. It also + // makes it hard to know when each batch should be retried. + // - In the Kafka Publish method, add a blocking first-pass intake step + // that can gate on error conditions, rather than handing off data + // to Sarama immediately. + // * This would fix the issue but would require a lot of work and + // testing, and we need a fix for the release now. It's also a + // fairly elaborate workaround for something that might be + // easier to fix in the library itself. + // + // Instead, we have applied the following fix, which is not very "nice" + // but satisfies all other important constraints: + // - When we receive a circuit breaker error, sleep for 10 seconds + // (Sarama's hard-coded timeout) on the _error worker thread_. + // + // This works because connection-level errors that can trigger the + // circuit breaker are on the critical path for input processing, and + // thus blocking on the error channel applies back-pressure to the + // input channel. This means that if there are any more errors while the + // error worker is asleep, any call to Publish will block until we + // start reading again. + // + // Reasons this solution is preferred: + // - It responds immediately to Sarama's global error state, rather than + // trying to detect it independently in each batch or adding more + // cumbersome synchronization to the output + // - It gives the minimal delay that is consistent with Sarama's + // internal behavior + // - It requires only a few lines of code and no design changes + // + // That said, this is still relying on undocumented library internals + // for correct behavior, which isn't ideal, but the error itself is an + // undocumented library internal, so this is de facto necessary for now. + // We'd like to have a more official / permanent fix merged into Sarama + // itself in the future. + + // The "breakerOpen" flag keeps us from sleeping the first time we see + // a circuit breaker error, because it might be an old error still + // sitting in the channel from 10 seconds ago. So we only end up + // sleeping every _other_ reported breaker error. + if breakerOpen { + // Immediately log the error that presumably caused this state, + // since the error reporting on this batch will be delayed. + if msg.ref.err != nil { + c.log.Errorf("Kafka (topic=%v): %v", msg.topic, msg.ref.err) + } + select { + case <-time.After(10 * time.Second): + // Sarama's circuit breaker is hard-coded to reject all inputs + // for 10sec. + case <-msg.ref.client.done: + // Allow early bailout if the output itself is closing. + } + breakerOpen = false + } else { + breakerOpen = true + } + } } } @@ -262,9 +347,18 @@ func (r *msgRef) fail(msg *message, err error) { len(msg.key)+len(msg.value)) r.client.observer.Dropped(1) + case breaker.ErrBreakerOpen: + // Add this message to the failed list, but don't overwrite r.err since + // all the breaker error means is "there were a lot of other errors". + r.failed = append(r.failed, msg.data) + default: r.failed = append(r.failed, msg.data) - r.err = err + if r.err == nil { + // Don't overwrite an existing error. This way at tne end of the batch + // we report the first error that we saw, rather than the last one. + r.err = err + } } r.dec() } diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index b3c8e984fe9..3747a2fa63c 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -69,6 +69,7 @@ type kafkaConfig struct { Password string `config:"password"` Codec codec.Config `config:"codec"` Sasl saslConfig `config:"sasl"` + EnableFAST bool `config:"enable_krb5_fast"` } type saslConfig struct { @@ -227,7 +228,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err if tls != nil { k.Net.TLS.Enable = true - k.Net.TLS.Config = tls.BuildModuleConfig("") + k.Net.TLS.Config = tls.BuildModuleClientConfig("") } switch { @@ -244,6 +245,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err Username: config.Kerberos.Username, Password: config.Kerberos.Password, Realm: config.Kerberos.Realm, + DisablePAFXFAST: !config.EnableFAST, } case config.Username != "": diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc index f61d4c5d985..026d0431345 100644 --- a/libbeat/outputs/kafka/docs/kafka.asciidoc +++ b/libbeat/outputs/kafka/docs/kafka.asciidoc @@ -309,6 +309,12 @@ The ACK reliability level required from broker. 0=no response, 1=wait for local Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error. +===== `enable_krb5_fast` + +beta[] + +Enable Kerberos FAST authentication. This may conflict with some Active Directory installations. It is separate from the standard Kerberos settings because this flag only applies to the Kafka output. The default is `false`. + ===== `ssl` Configuration options for SSL parameters like the root CA for Kafka connections. diff --git a/libbeat/outputs/redis/backoff.go b/libbeat/outputs/redis/backoff.go index 41f448ca318..ef3dcd7cc48 100644 --- a/libbeat/outputs/redis/backoff.go +++ b/libbeat/outputs/redis/backoff.go @@ -21,7 +21,7 @@ import ( "context" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" b "github.com/elastic/beats/v7/libbeat/common/backoff" "github.com/elastic/beats/v7/libbeat/publisher" diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 70e316cba3f..914536bbb78 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" diff --git a/libbeat/outputs/redis/redis_integration_test.go b/libbeat/outputs/redis/redis_integration_test.go index 25189fa9008..3e146cad912 100644 --- a/libbeat/outputs/redis/redis_integration_test.go +++ b/libbeat/outputs/redis/redis_integration_test.go @@ -28,7 +28,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 324f98c8d6d..4b76f73bafb 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt update \ diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 4ef7caf47a7..3d624fbcaaa 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -23,20 +23,28 @@ stages: mage: "mage build unitTest" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags goIntegTest: mage: "mage goIntegTest" withModule: true when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags pythonIntegTest: mage: "mage pythonIntegTest" withModule: true when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags crosscompile: make: "make -C metricbeat crosscompile" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -54,10 +62,10 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - #- "windows-2008-r2" https://github.com/elastic/beats/issues/19800 - #- "windows-7-32-bit" https://github.com/elastic/beats/issues/19835 when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. @@ -102,6 +110,17 @@ stages: - "windows-8" branches: true ## for all the branches tags: true ## for all the tags + windows-2008: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2008-r2" + when: ## Override the top-level when. + comments: + - "/test metricbeat for windows-2008" + labels: + - "windows-2008" + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. @@ -113,3 +132,14 @@ stages: - "windows-7" branches: true ## for all the branches tags: true ## for all the tags + windows-7-32: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-7-32-bit" + when: ## Override the top-level when. + comments: + - "/test metricbeat for windows-7-32" + labels: + - "windows-7-32" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b47f7d09ca8..b430fffdae9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -30873,6 +30873,18 @@ format: bytes -- +*`kubernetes.volume.fs.used.pct`*:: ++ +-- +Percentage of used storage + + +type: scaled_float + +format: percent + +-- + *`kubernetes.volume.fs.inodes.used`*:: + diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index e02a7a81460..085436b05d9 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -31,7 +31,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -40,6 +40,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index 18f9004a99a..b0da11703c2 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -12,6 +12,14 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. +[float] +=== Usage for Stack Monitoring + +The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} +UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` +from the module's configuration. Alternatively, run `metricbeat modules disable elasticsearch` and +`metricbeat modules enable elasticsearch-xpack`. + [float] === Module-specific configuration notes @@ -24,11 +32,6 @@ the `hosts` list is interpreted by the module. * If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct {es} cluster (for example, a load-balancing proxy fronting the cluster). -The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} -UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` -from the module's configuration. Alternatively, run `metricbeat modules disable elasticsearch` and -`metricbeat modules enable elasticsearch-xpack`. - [float] === Example configuration diff --git a/metricbeat/helper/elastic/elastic_test.go b/metricbeat/helper/elastic/elastic_test.go index 8630488dfcd..0e1e62805f5 100644 --- a/metricbeat/helper/elastic/elastic_test.go +++ b/metricbeat/helper/elastic/elastic_test.go @@ -100,10 +100,10 @@ func TestFixTimestampField(t *testing.T) { { "converts float64s in scientific notation to ints", map[string]interface{}{ - "foo": 1.571284349e12, + "foo": 1.571284349e+09, }, map[string]interface{}{ - "foo": 1571284349000, + "foo": 1571284349, }, }, { diff --git a/metricbeat/helper/server/http/http.go b/metricbeat/helper/server/http/http.go index b4bd17477b6..3fe02001d7a 100644 --- a/metricbeat/helper/server/http/http.go +++ b/metricbeat/helper/server/http/http.go @@ -76,7 +76,7 @@ func getDefaultHttpServer(mb mb.BaseMetricSet) (*HttpServer, error) { Addr: net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port))), } if tlsConfig != nil { - httpServer.TLSConfig = tlsConfig.BuildModuleConfig(config.Host) + httpServer.TLSConfig = tlsConfig.BuildModuleClientConfig(config.Host) } h.server = httpServer return h, nil diff --git a/metricbeat/mb/testing/data/data_test.go b/metricbeat/mb/testing/data/data_test.go index 775f539b59a..b81b8142676 100644 --- a/metricbeat/mb/testing/data/data_test.go +++ b/metricbeat/mb/testing/data/data_test.go @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +// skipping tests on windows 32 bit versions, not supported +// +build !windows,!386 + package data import ( diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f7530d7e386..8250323ca04 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1306,6 +1306,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1433,6 +1439,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1621,6 +1633,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1630,6 +1646,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1786,6 +1808,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2082,6 +2110,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2279,6 +2313,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/metricbeat/module/ceph/cluster_status/cluster_status_test.go b/metricbeat/module/ceph/cluster_status/cluster_status_test.go index 3f6cd3ed7bd..c0c3858afd9 100644 --- a/metricbeat/module/ceph/cluster_status/cluster_status_test.go +++ b/metricbeat/module/ceph/cluster_status/cluster_status_test.go @@ -95,9 +95,9 @@ func TestFetchEventContents(t *testing.T) { //check pg info pgInfo := event["pg"].(common.MapStr) assert.EqualValues(t, 1054023794, pgInfo["data_bytes"]) - assert.EqualValues(t, 9965821952, pgInfo["avail_bytes"]) - assert.EqualValues(t, 12838682624, pgInfo["total_bytes"]) - assert.EqualValues(t, 2872860672, pgInfo["used_bytes"]) + assert.EqualValues(t, int64(9965821952), pgInfo["avail_bytes"]) + assert.EqualValues(t, int64(12838682624), pgInfo["total_bytes"]) + assert.EqualValues(t, int64(2872860672), pgInfo["used_bytes"]) //check pg_state info pgStateInfo := events[1].MetricSetFields["pg_state"].(common.MapStr) diff --git a/metricbeat/module/elasticsearch/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/_meta/docs.asciidoc index b2b1a585a69..db28ab50c74 100644 --- a/metricbeat/module/elasticsearch/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/_meta/docs.asciidoc @@ -5,6 +5,14 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. +[float] +=== Usage for Stack Monitoring + +The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} +UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` +from the module's configuration. Alternatively, run `metricbeat modules disable elasticsearch` and +`metricbeat modules enable elasticsearch-xpack`. + [float] === Module-specific configuration notes @@ -16,8 +24,3 @@ the `hosts` list is interpreted by the module. {es} cluster. * If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct {es} cluster (for example, a load-balancing proxy fronting the cluster). - -The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} -UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` -from the module's configuration. Alternatively, run `metricbeat modules disable elasticsearch` and -`metricbeat modules enable elasticsearch-xpack`. diff --git a/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain b/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain index 5b67f3fe431..0c75ae2b3d3 100644 --- a/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain +++ b/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain @@ -207,9 +207,9 @@ etcd_disk_backend_snapshot_duration_seconds_sum 0 etcd_disk_backend_snapshot_duration_seconds_count 0 # HELP etcd_disk_wal_fsync_duration_seconds The latency distributions of fsync called by wal. # TYPE etcd_disk_wal_fsync_duration_seconds histogram -etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 0 -etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 0 -etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 1 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 1 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 1 etcd_disk_wal_fsync_duration_seconds_bucket{le="0.008"} 2 etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} 2 etcd_disk_wal_fsync_duration_seconds_bucket{le="0.032"} 2 diff --git a/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json b/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json index fd9a62b86d8..ae9e6097773 100644 --- a/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json +++ b/metricbeat/module/etcd/metrics/_meta/testdata/metrics.plain-expected.json @@ -33,15 +33,15 @@ "ns": { "bucket": { "+Inf": 3, - "1000000": 0, + "1000000": 1, "1024000000": 3, "128000000": 3, "16000000": 2, - "2000000": 0, + "2000000": 1, "2048000000": 3, "256000000": 3, "32000000": 2, - "4000000": 0, + "4000000": 1, "4096000000": 3, "512000000": 3, "64000000": 3, @@ -102,4 +102,4 @@ "type": "etcd" } } -] \ No newline at end of file +] diff --git a/metricbeat/module/etcd/metrics/metrics_integration_test.go b/metricbeat/module/etcd/metrics/metrics_integration_test.go index 2faa820a744..89167b7c1ed 100644 --- a/metricbeat/module/etcd/metrics/metrics_integration_test.go +++ b/metricbeat/module/etcd/metrics/metrics_integration_test.go @@ -43,7 +43,7 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) } -func TestData(t *testing.T) { +func TestIntegrationData(t *testing.T) { service := compose.EnsureUp(t, "etcd") m := mbtest.NewFetcher(t, getConfig(service.Host())) diff --git a/metricbeat/module/etcd/metrics/metrics_test.go b/metricbeat/module/etcd/metrics/metrics_test.go index 9be6c8b95f3..c3ce0d490b0 100644 --- a/metricbeat/module/etcd/metrics/metrics_test.go +++ b/metricbeat/module/etcd/metrics/metrics_test.go @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package metrics diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 132ad0271df..9d104c32a7e 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.15.7 COPY test/main.go main.go diff --git a/metricbeat/module/kafka/metricset.go b/metricbeat/module/kafka/metricset.go index 73c1be02218..5ec46332b35 100644 --- a/metricbeat/module/kafka/metricset.go +++ b/metricbeat/module/kafka/metricset.go @@ -49,7 +49,7 @@ func NewMetricSet(base mb.BaseMetricSet, options MetricSetOptions) (*MetricSet, var tls *tls.Config if tlsCfg != nil { - tls = tlsCfg.BuildModuleConfig("") + tls = tlsCfg.BuildModuleClientConfig("") } timeout := base.Module().Config().Timeout diff --git a/metricbeat/module/kubernetes/apiserver/apiserver_test.go b/metricbeat/module/kubernetes/apiserver/apiserver_test.go index 42d8b223143..2674698b212 100644 --- a/metricbeat/module/kubernetes/apiserver/apiserver_test.go +++ b/metricbeat/module/kubernetes/apiserver/apiserver_test.go @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package apiserver diff --git a/metricbeat/module/kubernetes/container/container_test.go b/metricbeat/module/kubernetes/container/container_test.go index 42f1fe710f3..94b98863c31 100644 --- a/metricbeat/module/kubernetes/container/container_test.go +++ b/metricbeat/module/kubernetes/container/container_test.go @@ -53,8 +53,8 @@ func TestEventMapping(t *testing.T) { "cpu.usage.core.ns": 43959424, "cpu.usage.nanocores": 11263994, - "logs.available.bytes": 98727014400, - "logs.capacity.bytes": 101258067968, + "logs.available.bytes": int64(98727014400), + "logs.capacity.bytes": int64(101258067968), "logs.used.bytes": 28672, "logs.inodes.count": 6258720, "logs.inodes.free": 6120096, @@ -75,8 +75,8 @@ func TestEventMapping(t *testing.T) { "name": "nginx", - "rootfs.available.bytes": 98727014400, - "rootfs.capacity.bytes": 101258067968, + "rootfs.available.bytes": int64(98727014400), + "rootfs.capacity.bytes": int64(101258067968), "rootfs.used.bytes": 61440, "rootfs.inodes.used": 21, } diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index bc7f92ad298..aec239344fd 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKubernetes returns asset data. // This is the base64 encoded gzipped contents of module/kubernetes. func AssetKubernetes() string { - return "eJzsXU9z47aSv8+nQM1psuXosLW1hzlsVeK8t8+VzDyv7ckctrYUmGxJiEmAAUB79D79FsB/IAmAoAjJHls6pDK21f1Dd6PR6AYaP6IH2H9ED+U9cAoSxDuEJJEZfETvf21/+P4dQimIhJNCEkY/ov96hxBC3R+gHCQnifo2hwywgI9oi98hJEBKQrfiI/rf90Jk7y/Q+52Uxfv/U7/bMS7XCaMbsv2INjgT8A6hDYEsFR81gx8RxTkM4KmP3BeKA2dlUf/EAk99ruiG8RyrHyNMUyQklkRIkgjENqhgqUA5pngLKbrfG3xWNQUTjYkIF0QAfwTe/sYGygNsIL+frq9QRdAQZfPpi7T5DKGZ8Dj8VYKQqyQjQGXvTxqcD7B/Yjwd/M6DVn0uNT0E3yAplV4bRsKLgoNgJU8gHo6bijKkyEp7CECU98fE4CI/gpGwIj4ApMmiD0lWCgn8QjMVBU7gopXOD15cj8Dv48H6x93dNRqRHFkmSyOKQvMckRzzpBKoXCtG8dVQY9As0IjFEEvK92te0ngwvoLcAUdyBw0PVAoQKOV7NGQ0BPNA6JDbAiS/Epoq71pTn1BJXjAa10c1JNEO0zRTXsoQihfN0HcvRKKcuiaJNqzRTICbeAQuCItoGjXBFsV4mEMIWnK9xW0hhGaS2AgPmecgdyyiPeqJaSE6GjQTEc2wHfGQasO24CwBIawcbYZoW+9NeklRrgQko983NFNW3mdDvzcayOX1FyQgYTQdIus45ZAzvlfLOkmBytX9vovMxnwzRreWX1Zx2Ufk+nIP1c/qjxChqOFZY5iC+Ei4LHF2SoQ1yymAm1SsWAF0lbBy5P0mofVYfy7ze+DK4yqCaEMyaP+AcbcahcRcQhrBaG4rg0GC0AS0i6mNu+FhnQBqIxDN+tt1teQ62l+VYlUAT4BKksHq35wjZPd/QmJTQPWL9Rw5NHO+AYFyknBWTyfUwXHrxDYMUeYL9ePHlZR5mWFJHgHZWPmgLTfeBpqmpFeohv4kEEH+BdXMjqnpOaAVgllqNSD7tBrDIfUwzlSxAfMYGlbkPRhEwaiAZ1VvBWGOfsegj69gE2WwhsdAY6i4hmInNQ7649tUMzDrSlOlQVY+/k7ejqW2SXwgLJAlyzIYcrwgL2K0YM3dmMwyLIEm+0Ms2aYt0RC8UCaqEFT/JlXgZK5Jk5DimVCLic4XzH2ZPIA86ZJTs0Y7IiTbcpyjCoQbbGgoMQdFQ7PSZKjyjhM5dFioGQhXPwwD8wx67FCHazIpOVd+bLnsrugmI9udDDB1Rre8pJTQbdStSuc/E71oqW+jmpE/qwwySVeV3KN48i7pX2tTICw1Fyt7XKZEruDRpYi57DU9pOnZx1sx5KCgQRqRZ0NyyLxba6jEhC6rcRjSbelFKXHoneVaktyeyk2xHP5iImFzqwiiEUEjvRK8ik9lKK+/oFLgLVgE4Rq2CUV/1zkPbYB8VHuDZNxGeJr4FAOTicUpD9k4t7XNZ0LC5ueyNTsl90vGoRY+xdS5ZPXwYsqUYFywAyAHwq0MA9IJli0wlsKqsK5LHS6R4AzS9SZj2PWHzbaj3unEGIOSLxYINzTVv9lGp4YkkzjT2BHOMpZgie8zUN/zDjYjOZHf32hT2BAKaQW/zcB3rvCD+olTIohsUEn1dyG1F/Eytg3PIU+M6je2VaH4hs10SPgRkwzbzX+5U3LthlHY3JvaVKNwbWv5tINFCS5wQuReBcB26q1frf/yLcinsuZw2SiH9xbkoh17uFiI8gfumsWyVd4ex6Ooi9mdtoNutjgHZBRFOPjDj3i4FKsQSA7rPAYkbSAWSP2aVrRU0ltx2kM7nCjMHS+4fmkiqQThHPALjzM/GehnhpoOC0AvPtoMGfOCgLM2CHfMaUqIj44uoFc2S25ub/1zpIH8xPgDoVsB7uTY65DI12qgSIAMk0yBt7DBZWZJMM6rYNsxdRktxQg5OLXrJ/6T8ZMh0tycuNpZxJjcRDwD9Db2GTeMSX3OReyFhHz2luOthD52OZkh+XlvZpdRHYs/3x7tJPuOL5Ydh5n95yzLgFcXJBZVAS5bYvV1izg1gGc5pnrKk+unPgp74iOw6r/x2H3GOYSdtP4XoxH5XtENx0LyMpElhzHx84HfajjnA7/nA7/nA78Bwzgf+LUDOR/4DcZ4PvB7PvB7PvC7/MCvJcqcewT4ifGHv0oo7RHnIUufAg0q4KyO5S1fzn+rCLbn7+rF3BdLlHRDKBG7KOHEl5ZYCGucpjFs+GujF0VwwpBTKOQuKk9NcXL6SE6izNeOr3nKWVO3b8xYCqtEbdgTyez760MMFx5JoiOJmDGwLmM0lH0GuwOcyV2Ms+Md85YqsqeCjnFu38+pwuMoXYWzu+4VltyDbH0S4BT4ioh1joV05GTuGcsADwO9qYvtu+5mu9Y1EWjA490QjT7R+m7IfkbC6m4HZnuO6oRsk7MCtQ7pudH+Ru6wRJgD2gIFjmXVT6Q5T1z71R4HQtXGVgn312F3EzQjGeY2MIeuvdK+rJZXxQVxSBhPRSX31vgkyaH6WYG5JEmZYV4JAe2wQCzRh9RTC0L9TYnzwoJy7Ex8ab8N4UKua1bU0dNj/gHguwagGqfmgToe6mdDqzIvhBwdkGIxgafLhYhRVa7CIOGbDLeGTxWd2hIg7RoIkEegFnEkrNivJbMh6NY0LAZbPXfqzYvuRlMKBdda4bAxx4Hc7/ZFW3L3c7TkIV1G7+eoy/hNbwsOBeOyam5BhEUXvgl01K4bG85y9LQjyU4Lp/INRHSe0Z4bipp5/qzWCUUYMRqKxci54xRLvFxjn2pKCAvBEqJXhScid9455NOb3YXOj8haO+AwUgjyOayAylLPaWkGhFH/TOkANXpZx60M/HdNtjaJTWcM9ug3flkiiKdu2hSXsSaJSDMJqgnwhKdmY1M9WUfvRfN73YvGFIi/WFOSiAWwL5T8VQLSJQWyISqsZAYQS0qpdeOQbdYZoQ8Rwdz8pvw4B6HQ1H2KXMsIoY8se4R0bcF4LO/U8LTJxeencEHiW85P11dtJ6PaejzqitvSSvF+qNtaTTCO6zxMh+Vherz52lCeIfq4E/bL1S8TvM2kxZI9n3FVUe8zz7cUz7cUHZ/4txR1xPq9X1A831Sw/835psLoE++mwvlA+gjy+UC6C/r5QPrEgXQKUllPNN/Nv71yE7yBBMijzve7aLVVCc5tdc1g1KGIvrk4tXmk166UO46pyImUL0kvd1a9tEWN8y2Q5hMoz7+fL4DMFtH57of5GYnnbVz7MI4vOK6cD2GdpmNAh+ul9AroELn6BbRxTkmdWZ5DfDjJVVx4pP4P7vVhmsEUExQ401F4GiVkxqN56ZarXMfB81cQFLiKoLctyIB1Bs1xe29SiPbVqN3K9u5wLcl2Fyz9LpPd571q8znvVc3P96WU73Cv+iaqTC+mqjIC9hLb8cxp+/imWj2qxbXtviOG7XfqHo+MAmIc5YyD+cc1YUUCc5jqBBm56nYuL42Av8iZd26EFW86HtwN660kFHsTxj3oQSFy/forkZVgnkb1SPdm49WXqyuRtA0clEz0zdUJwRR4C+sjVkQrWMH12fVp8Lirs0b7kG/7JTt+4z6TprX8weH28L6lXc7BtzpcHXi6XHQa5QaHrfOOcQdg2CJnCZcROeeth6VS69MzTGfUeWbOxavezs9639c7DQL7zfTj2KluM56rjH4fdkifmQE0f5eZyMi8/WXaaoi/u4wH0oLOMr28n+MieLhhzOkp086fYUeZw6x6di8ZbwOKkD4yUbrI+ODbekLEQuRtPeEDtcw4gzvHDCGENhYJ12o42ImOIh6obg0udS3hnWI88GLr0tIjxrjdMK9DTFxFhvWG8YE9piqDe8L4AC5VpqcbzNCEItqNLTicavxyyK34wJYv7XK4p0nQouRl+lDeQxWm18H6nibWHPnE0lZmIAJXhmnx3+5pcq3g3Ciyg2cA2ab9wdSDjm50y8zDiS/gaUA3JufzgDH9jBP61PuAg2powfUf54Ruo6n9c0UaGbRnPQEZCHFh7OoFOcMAJlCexBr8g3GbxChrIJIdpGW2rH2vkTlo6Z3TBmMeryxtMLrKeiCbqca8RmRSZlEGdltbKcJSQl7IMemGZ+sNIrJVk9VG95yOOadjpiCd0zHndMxMROd0zDkdc07HnNMx53SMFYO3M2XF39aX0gthTk/K0V5s2AnysEUS/h1Ovy39G02RZAhoagzGviwFwl6SlpiBxjMBh4iWzQg7Jt9MLFi6KjiobYpCoBvZ5pP6nEZyzVLU0UU13XkglmjHzt+jCAeGZfpwoJhSSL1vjMG9IeXl2Rpgba+njHdvLXNnemEdIV4W4tpABK2fIxwLM8iuSftuyLg9RfhuyOWwWzjdE50x7uIc3INrJJ7LFhexNwsUEssy3u31YoeF+xilfQDDQfiOcbfD0YzQh7o58wV6wkTq/5HAc0Kx//lTwKn7gr290XUgyg6hZmKXby+AVBty9+E0QiVsRx25DwBT8ZlsWj9q8GuCWaS/r5WG0IcW1aVuMKqUdsmx2P3GWPEzTh7YZnOB/sa5vlh3XWbZBWr/t/79WLXqw3irfeWBPlyyvMhAQnrRSeISU8rkTUk1C8Yv0D//+elXkmWQ/lAPf2WdKHOuzUy+AaHPZbsui1R0XcexZ6n98vqLbrgmKpYevTcx/kkg1ewgRXaGfTn5LtZ4VgaFq+CQKFfwEf3n6j9iIG+xBArUh30a3sToDpb6SRvAVUo8/ptuUyKoz71XNwom20Q0Cnx+3J3amksNrkvFCWf0T3YfK6SpqEUJaEbFqPCQBl3WOEY0hlXSpQysdIyAse7eb58ZIXw6EqhgGRlQam+jJCpoXvDsUZdiqUipPZHoHpMfGYkRd4q1KEUBNB3d6PeFRj3uZnalMSGido42up3l6l7mlqqHZxPS37oXLNkhMap7NBCesLB2TG+9FBZy3VhANBxK6PrthwYGL6l9gsC3I7FXlCfZp4DTjFA35ymb+6Um0LLGGwm8nVIaScL0qylcBYEbTDJDEyH/4/+ne6uXYsgZ7V9iWnKI4hdN71ZfEjqxZ+wWpyIjCQ7ftk0sONbR1UwOvNo9fafu8LDmrveoTdetpBELKoB3A3FCTEEQ7mmTtQxgTb33ZtcseP5NaizpVbvUmdBKejr1GrymYHaKLTK2zxe+ImWEQh3BKHO+wJaWOcELrHf6GkgrLrYExAnciIGjVRqhGzbTi0xN0UXZkV86jJ25NdO2Rf1BFJAsuTscC+N4Giyan7Fg2eanG1iRWt+siQ6q4jMGZDYDiOQcYj43EDMV618/FqXydL97M7+JPkhewgXa4EzoZhglfaDsibrnTUnr2NBrpItSsRplj4/PGcbM7xmdCI6XUmsfHTD7HvjzaU2HvAlQhy/dLaa2F9/p3hkwZP5cSZvPrjYUU9mmVjHPirxG6++jaBRaj6I73dDjWKZp6qZglpc5Rgo5Khzdz2TYz7MVMHBBhAQqH1lW5rGWq44squg2a1f19p/6yx+Vm4Qfnzvx93sFT5FwlGR8syYsb1HzsDZV9dU05w6iKlfiJGE81Y+FMUMnjmiAcbyFdZLhUbORYO63FRGkibQpwJE9oZDEissukwyT/GjGmWT4BZvo9e+XHvushrDoOcSfCU0hbYThZlUXCda11SyYETddba6ZXvFnhZKbJmCnjZMEhFjnw1s/Mzj8pEkgRcLO44jz6/r3y5VrOtmXz0VzJlIDWGJ/mHT04/AkgEJ2dW1ltmNCro/DUZF2sZ25xZrHuN4KHZYvPeLBmQHM+uTMTXNy5hqoWpJWq9WhB2Ziolu2q2zqDe4MQ0ysLTcb3osx2mHebVlJwnABTapeRCpKHDExaEJ1ZyhfUn2hl3S+qf7xfGWFw3E9Wz0hABu71+3MjiW0+r1k/QppzQnd7/Va3YHTZzg5y4aXmlGvSnwPPu8SS4qbMsv2DbdJaRqHCfXt3L9K1nsBfZlrMWhGcS7Hq/Xf1Fj/R2OdqvgPpTQHQcWB0A3jOaToww7zVC9QAtIffLel42w7+gN1HoxR9A5lYY6wmjnqqxfoDzXUP9RY/1CD/cOxflgGfsD4NDktysr8cFFkBASSbLw99f/TvZ1V7oAksbIrNbVnPzR1W+PwJE+yUkjgriA8gMcVlcApztDVdWvy9fjtLOFb9YVFO+JmZA0x9MvnW/cUaFkePswRQ8feImM4Xd/jDNNkkVh/YzhFP9d0WoNyMF0yxZuBjWi0O0K65WofvsRENAUX+oaB2rItsYmGzT9sdAbrjt3jT7xu0ohK01DOsPcFc3cJmzKLF9g3FKNF9j4hTGWGxoHL3c4QiSQ5CInzAn0AtUBX6+BtPYJh9HeCrUZPeG0MddBu48jxqdHfqAlPezGfS4joGbYdowMSPoANuC4AP7aejVDfCF5elrpbJRtgX4aaG+UGABvkUYdp1GVez8yqPnso1UPjDqgKzh6JIIyO9o+zi0UdpS6wMlG4agC6FLO2nC2fFXtrKvUJ9ao1yJ7inCRY7UnrBaSuSNhLV3Xd457oxOKiNP4nllaHh9Pqne1ONoRuEaYpqrnEX/J7ap9Y+PXDb7Gsv3pFznjnIsrCb7nwOksTltef2isq7qsZJ35U7k28bZUwfvxHK0cNFsdsJh9bDLzUi2ppXzIOtcgppo5WFAOUL+WZryMdajo/49QD/fqfXbm5vQ0TRf1Yzet/m+fr6FWeCckUeAtHfPGlu10X/ArNyRBNv0MT9ShZ//zYswXoY6kYJ8es3EZPcx/uoT3Pcb+mWTh6bH/6se3wSu0rFVHAM9rdOu/Z0r9S8dgfyB4OwPJqv4l+iWwm3y2PETd+qUbpGIThkjj4g/oYYP7OAULA2JvExEZTBX41nP8PAAD//4Df+K8=" + return "eJzsXU9z47aSv8+nQM1psuXosLW1hzlsVeK8t8+VzDyv7ckctrYUmGxJiEmAAUB79D79FsB/IAmAoAjJHls6pDK21f1Dd6PR6AYaP6IH2H9ED+U9cAoSxDuEJJEZfETvf21/+P4dQimIhJNCEkY/ov96hxBC3R+gHCQnifo2hwywgI9oi98hJEBKQrfiI/rf90Jk7y/Q+52Uxfv/U7/bMS7XCaMbsv2INjgT8A6hDYEsFR81gx8RxTkM4KmP3BeKA2dlUf/EAk99ruiG8RyrHyNMUyQklkRIkgjENqhgqUA5pngLKbrfG3xWNQUTjYkIF0QAfwTe/sYGygNsIL+frq9QRdAQZfPpi7T5DKGZ8Dj8VYKQqyQjQGXvTxqcD7B/Yjwd/M6DVn0uNT0E3yAplV4bRsKLgoNgJU8gHo6bijKkyEp7CECU98fE4CI/gpGwIj4ApMmiD0lWCgn8QjMVBU7gopXOD15cj8Dv48H6x93dNRqRHFkmSyOKQvMckRzzpBKoXCtG8dVQY9As0IjFEEvK92te0ngwvoLcAUdyBw0PVAoQKOV7NGQ0BPNA6JDbAiS/Epoq71pTn1BJXjAa10c1JNEO0zRTXsoQihfN0HcvRKKcuiaJNqzRTICbeAQuCItoGjXBFsV4mEMIWnK9xW0hhGaS2AgPmecgdyyiPeqJaSE6GjQTEc2wHfGQasO24CwBIawcbYZoW+9NeklRrgQko983NFNW3mdDvzcayOX1FyQgYTQdIus45ZAzvlfLOkmBytX9vovMxnwzRreWX1Zx2Ufk+nIP1c/qjxChqOFZY5iC+Ei4LHF2SoQ1yymAm1SsWAF0lbBy5P0mofVYfy7ze+DK4yqCaEMyaP+AcbcahcRcQhrBaG4rg0GC0AS0i6mNu+FhnQBqIxDN+tt1teQ62l+VYlUAT4BKksHq35wjZPd/QmJTQPWL9Rw5NHO+AYFyknBWTyfUwXHrxDYMUeYL9ePHlZR5mWFJHgHZWPmgLTfeBpqmpFeohv4kEEH+BdXMjqnpOaAVgllqNSD7tBrDIfUwzlSxAfMYGlbkPRhEwaiAZ1VvBWGOfsegj69gE2WwhsdAY6i4hmInNQ7649tUMzDrSlOlQVY+/k7ejqW2SXwgLJAlyzIYcrwgL2K0YM3dmMwyLIEm+0Ms2aYt0RC8UCaqEFT/JlXgZK5Jk5DimVCLic4XzH2ZPIA86ZJTs0Y7IiTbcpyjCoQbbGgoMQdFQ7PSZKjyjhM5dFioGQhXPwwD8wx67FCHazIpOVd+bLnsrugmI9udDDB1Rre8pJTQbdStSuc/E71oqW+jmpE/qwwySVeV3KN48i7pX2tTICw1Fyt7XKZEruDRpYi57DU9pOnZx1sx5KCgQRqRZ0NyyLxba6jEhC6rcRjSbelFKXHoneVaktyeyk2xHP5iImFzqwiiEUEjvRK8ik9lKK+/oFLgLVgE4Rq2CUV/1zkPbYB8VHuDZNxGeJr4FAOTicUpD9k4t7XNZ0LC5ueyNTsl90vGoRY+xdS5ZPXwYsqUYFywAyAHwq0MA9IJli0wlsKqsK5LHS6R4AzS9SZj2PWHzbaj3unEGIOSLxYINzTVv9lGp4YkkzjT2BHOMpZgie8zUN/zDjYjOZHf32hT2BAKaQW/zcB3rvCD+olTIohsUEn1dyG1F/Eytg3PIU+M6je2VaH4hs10SPgRkwzbzX+5U3LthlHY3JvaVKNwbWv5tINFCS5wQuReBcB26q1frf/yLcinsuZw2SiH9xbkoh17uFiI8gfumsWyVd4ex6Ooi9mdtoNutjgHZBRFOPjDj3i4FKsQSA7rPAYkbSAWSP2aVrRU0ltx2kM7nCjMHS+4fmkiqQThHPALjzM/GehnhpoOC0AvPtoMGfOCgLM2CHfMaUqIj44uoFc2S25ub/1zpIH8xPgDoVsB7uTY65DI12qgSIAMk0yBt7DBZWZJMM6rYNsxdRktxQg5OLXrJ/6T8ZMh0tycuNpZxJjcRDwD9Db2GTeMSX3OReyFhHz2luOthD52OZkh+XlvZpdRHYs/3x7tJPuOL5Ydh5n95yzLgFcXJBZVAS5bYvV1izg1gGc5pnrKk+unPgp74iOw6r/x2H3GOYSdtP4XoxH5XtENx0LyMpElhzHx84HfajjnA7/nA7/nA78Bwzgf+LUDOR/4DcZ4PvB7PvB7PvC7/MCvJcqcewT4ifGHv0oo7RHnIUufAg0q4KyO5S1fzn+rCLbn7+rF3BdLlHRDKBG7KOHEl5ZYCGucpjFs+GujF0VwwpBTKOQuKk9NcXL6SE6izNeOr3nKWVO3b8xYCqtEbdgTyez760MMFx5JoiOJmDGwLmM0lH0GuwOcyV2Ms+Md85YqsqeCjnFu38+pwuMoXYWzu+4VltyDbH0S4BT4ioh1joV05GTuGcsADwO9qYvtu+5mu9Y1EWjA490QjT7R+m7IfkbC6m4HZnuO6oRsk7MCtQ7pudH+Ru6wRJgD2gIFjmXVT6Q5T1z71R4HQtXGVgn312F3EzQjGeY2MIeuvdK+rJZXxQVxSBhPRSX31vgkyaH6WYG5JEmZYV4JAe2wQCzRh9RTC0L9TYnzwoJy7Ex8ab8N4UKua1bU0dNj/gHguwagGqfmgToe6mdDqzIvhBwdkGIxgafLhYhRVa7CIOGbDLeGTxWd2hIg7RoIkEegFnEkrNivJbMh6NY0LAZbPXfqzYvuRlMKBdda4bAxx4Hc7/ZFW3L3c7TkIV1G7+eoy/hNbwsOBeOyam5BhEUXvgl01K4bG85y9LQjyU4Lp/INRHSe0Z4bipp5/qzWCUUYMRqKxci54xRLvFxjn2pKCAvBEqJXhScid9455NOb3YXOj8haO+AwUgjyOayAylLPaWkGhFH/TOkANXpZx60M/HdNtjaJTWcM9ug3flkiiKdu2hSXsSaJSDMJqgnwhKdmY1M9WUfvRfN73YvGFIi/WFOSiAWwL5T8VQLSJQWyISqsZAYQS0qpdeOQbdYZoQ8Rwdz8pvw4B6HQ1H2KXMsIoY8se4R0bcF4LO/U8LTJxeencEHiW85P11dtJ6PaejzqitvSSvF+qNtaTTCO6zxMh+Vherz52lCeIfq4E/bL1S8TvM2kxZI9n3FVUe8zz7cUz7cUHZ/4txR1xPq9X1A831Sw/835psLoE++mwvlA+gjy+UC6C/r5QPrEgXQKUllPNN/Nv71yE7yBBMijzve7aLVVCc5tdc1g1KGIvrk4tXmk166UO46pyImUL0kvd1a9tEWN8y2Q5hMoz7+fL4DMFtH57of5GYnnbVz7MI4vOK6cD2GdpmNAh+ul9AroELn6BbRxTkmdWZ5DfDjJVVx4pP4P7vVhmsEUExQ401F4GiVkxqN56ZarXMfB81cQFLiKoLctyIB1Bs1xe29SiPbVqN3K9u5wLcl2Fyz9LpPd571q8znvVc3P96WU73Cv+iaqTC+mqjIC9hLb8cxp+/imWj2qxbXtviOG7XfqHo+MAmIc5YyD+cc1YUUCc5jqBBm56nYuL42Av8iZd26EFW86HtwN660kFHsTxj3oQSFy/forkZVgnkb1SPdm49WXqyuRtA0clEz0zdUJwRR4C+sjVkQrWMH12fVp8Lirs0b7kG/7JTt+4z6TprX8weH28L6lXc7BtzpcHXi6XHQa5QaHrfOOcQdg2CJnCZcROeeth6VS69MzTGfUeWbOxavezs9639c7DQL7zfTj2KluM56rjH4fdkifmQE0f5eZyMi8/WXaaoi/u4wH0oLOMr28n+MieLhhzOkp086fYUeZw6x6di8ZbwOKkD4yUbrI+ODbekLEQuRtPeEDtcw4gzvHDCGENhYJ12o42ImOIh6obg0udS3hnWI88GLr0tIjxrjdMK9DTFxFhvWG8YE9piqDe8L4AC5VpqcbzNCEItqNLTicavxyyK34wJYv7XK4p0nQouRl+lDeQxWm18H6nibWHPnE0lZmIAJXhmnx3+5pcq3g3Ciyg2cA2ab9wdSDjm50y8zDiS/gaUA3JufzgDH9jBP61PuAg2powfUf54Ruo6n9c0UaGbRnPQEZCHFh7OoFOcMAJlCexBr8g3GbxChrIJIdpGW2rH2vkTlo6Z3TBmMeryxtMLrKeiCbqca8RmRSZlEGdltbKcJSQl7IMemGZ+sNIrJVk9VG95yOOadjpiCd0zHndMxMROd0zDkdc07HnNMx53SMFYO3M2XF39aX0gthTk/K0V5s2AnysEUS/h1Ovy39G02RZAhoagzGviwFwl6SlpiBxjMBh4iWzQg7Jt9MLFi6KjiobYpCoBvZ5pP6nEZyzVLU0UU13XkglmjHzt+jCAeGZfpwoJhSSL1vjMG9IeXl2Rpgba+njHdvLXNnemEdIV4W4tpABK2fIxwLM8iuSftuyLg9RfhuyOWwWzjdE50x7uIc3INrJJ7LFhexNwsUEssy3u31YoeF+xilfQDDQfiOcbfD0YzQh7o58wV6wkTq/5HAc0Kx//lTwKn7gr290XUgyg6hZmKXby+AVBty9+E0QiVsRx25DwBT8ZlsWj9q8GuCWaS/r5WG0IcW1aVuMKqUdsmx2P3GWPEzTh7YZnOB/sa5vlh3XWbZBWr/t/79WLXqw3irfeWBPlyyvMhAQnrRSeISU8rkTUk1C8Yv0D//+elXkmWQ/lAPf2WdKHOuzUy+AaHPZbsui1R0XcexZ6n98vqLbrgmKpYevTcx/kkg1ewgRXaGfTn5LtZ4VgaFq+CQKFfwEf3n6j9iIG+xBArUh30a3sToDpb6SRvAVUo8/ptuUyKoz71XNwom20Q0Cnx+3J3amksNrkvFCWf0T3YfK6SpqEUJaEbFqPCQBl3WOEY0hlXSpQysdIyAse7eb58ZIXw6EqhgGRlQam+jJCpoXvDsUZdiqUipPZHoHpMfGYkRd4q1KEUBNB3d6PeFRj3uZnalMSGido42up3l6l7mlqqHZxPS37oXLNkhMap7NBCesLB2TG+9FBZy3VhANBxK6PrthwYGL6l9gsC3I7FXlCfZp4DTjFA35ymb+6Um0LLGGwm8nVIaScL0qylcBYEbTDJDEyH/4/+ne6uXYsgZ7V9iWnKI4hdN71ZfEjqxZ+wWpyIjCQ7ftk0sONbR1UwOvNo9fafu8LDmrveoTdetpBELKoB3A3FCTEEQ7mmTtQxgTb33ZtcseP5NaizpVbvUmdBKejr1GrymYHaKLTK2zxe+ImWEQh3BKHO+wJaWOcELrHf6GkgrLrYExAnciIGjVRqhGzbTi0xN0UXZkV86jJ25NdO2Rf1BFJAsuTscC+N4Giyan7Fg2eanG1iRWt+siQ6q4jMGZDYDiOQcYj43EDMV618/FqXydL97M7+JPkhewgXa4EzoZhglfaDsibrnTUnr2NBrpItSsRplj4/PGcbM7xmdCI6XUmsfHTD7HvjzaU2HvAlQhy/dLaa2F9/p3hkwZP5cSZvPrjYUU9mmVjHPirxG6++jaBRaj6I73dDjWKZp6qZglpc5Rgo5Khzdz2TYz7MVMHBBhAQqH1lW5rGWq44squg2a1f19p/6yx+Vm4Qfnzvx93sFT5FwlGR8syYsb1HzsDZV9dU05w6iKlfiJGE81Y+FMUMnjmiAcbyFdZLhUbORYO63FRGkibQpwJE9oZDEissukwyT/GjGmWT4BZvo9e+XHvushrDoOcSfCU0hbYThZlUXCda11SyYETddba6ZXvFnhZKbJmCnjZMEhFjnw1s/Mzj8pEkgRcLO44jz6/r3y5VrOtmXz0VzJlIDWGJ/mHT04/AkgEJ2dW1ltmNCro/DUZF2sZ25xZrHuN4KHZYvPeLBmQHM+uTMTXNy5hqoWpJWq9WhB2Ziolu2q2zqDe4MQ0ysLTcb3osx2mHebVlJwnABTapeRCpKHDExaEJ1ZyhfUn2hl3S+qf7xfGWFw3E9Wz0hABu71+3MjiW0+r1k/QppzQnd7/Va3YHTZzg5y4aXmlGvSnwPPu8SS4qbMsv2DbdJaRqHCfXt3L9K1nsBfZlrMWhGcS7Hq/Xf1Fj/R2OdqvgPpTQHQcWB0A3jOaToww7zVC9QAtIffLel42w7+gN1HoxR9A5lYY6wmjnqqxfoDzXUP9RY/1CD/cOxflgGfsD4NDktysr8cFFkBASSbLw99f/TvZ1V7oAksbIrNbVnPzR1W+PwJE+yUkjgriA8gMcVlcApztDVdWvy9fjtLOFb9YVFO+JmZA0x9MvnW/cUaFkePswRQ8feImM4Xd/jDNNkkVh/YzhFP9d0WoNyMF0yxZuBjWi0O0K65WofvsRENAUX+oaB2rItsYmGzT9sdAbrjt3jT7xu0ohK01DOsPcFc3cJmzKLF9g3FKNF9j4hTGWGxoHL3c4QiSQ5CInzAn0AtUBX6+BtPYJh9HeCrUZPeG0MddBu48jxqdHfqAlPezGfS4joGbYdowMSPoANuC4AP7aejVDfCF5elrpbJRtgX4aaG+UGABvkUYdp1GVez8yqPnso1UPjDqgKzh6JIIyO9o+zi0UdpS6wMlG4agC6FLO2nC2fFXtrKvUJ9ao1yJ7inCRY7UnrBaSuSNhLV3Xd457oxOKiNP4nllaHh9Pqne1ONoRuEaYpqrnEX/J7ap9Y+PXDb7Gsv3pFznjnIsrCb7nwOksTltef2isq7qsZJ35U7k28bZUwfvxHK0cNFsdsJh9bDLzUi2ppXzIOtcgppo5WFAOUL+WZryMdajo/49QD/fqfXbm5vQ0TRf1Yzet/m+fr6FWeCckUeAtHfPGlu10X/ArNyRBNv0MT9ShZ//zYswXoY6kYJ8es3EZPcx/uoT3Pcb+mWTh6bH/6se3wSu0rFVHAM9rdOu/Z0r9S8dgfyLYN4qU9YHjde66wPsOgd4tOBRPKUs/B7iUqnnx+PUb4+6VSlmMQhmfl4N+bxADzdw4QAsbe6yY2mip+reH8fwAAAP//CPgmJQ==" } diff --git a/metricbeat/module/kubernetes/node/node_test.go b/metricbeat/module/kubernetes/node/node_test.go index 9951c42bcd8..311078f1a9b 100644 --- a/metricbeat/module/kubernetes/node/node_test.go +++ b/metricbeat/module/kubernetes/node/node_test.go @@ -42,11 +42,11 @@ func TestEventMapping(t *testing.T) { assert.NoError(t, err, "error mapping "+testFile) testCases := map[string]interface{}{ - "cpu.usage.core.ns": 4189523881380, + "cpu.usage.core.ns": int64(4189523881380), "cpu.usage.nanocores": 18691146, "memory.available.bytes": 1768316928, - "memory.usage.bytes": 2764943360, + "memory.usage.bytes": int64(2764943360), "memory.rss.bytes": 2150400, "memory.workingset.bytes": 2111090688, "memory.pagefaults": 131567, @@ -54,9 +54,9 @@ func TestEventMapping(t *testing.T) { "name": "gke-beats-default-pool-a5b33e2e-hdww", - "fs.available.bytes": 98727014400, - "fs.capacity.bytes": 101258067968, - "fs.used.bytes": 2514276352, + "fs.available.bytes": int64(98727014400), + "fs.capacity.bytes": int64(101258067968), + "fs.used.bytes": int64(2514276352), "fs.inodes.used": 138624, "fs.inodes.free": uint64(18446744073709551615), "fs.inodes.count": 6258720, @@ -66,8 +66,8 @@ func TestEventMapping(t *testing.T) { "network.tx.bytes": 812729002, "network.tx.errors": 0, - "runtime.imagefs.available.bytes": 98727014400, - "runtime.imagefs.capacity.bytes": 101258067968, + "runtime.imagefs.available.bytes": int64(98727014400), + "runtime.imagefs.capacity.bytes": int64(101258067968), "runtime.imagefs.used.bytes": 860204379, } diff --git a/metricbeat/module/kubernetes/proxy/proxy_test.go b/metricbeat/module/kubernetes/proxy/proxy_test.go index 79a940dced0..72505ba935b 100644 --- a/metricbeat/module/kubernetes/proxy/proxy_test.go +++ b/metricbeat/module/kubernetes/proxy/proxy_test.go @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package proxy diff --git a/metricbeat/module/kubernetes/scheduler/scheduler_test.go b/metricbeat/module/kubernetes/scheduler/scheduler_test.go index 6f7f4d43332..2356ddb8f98 100644 --- a/metricbeat/module/kubernetes/scheduler/scheduler_test.go +++ b/metricbeat/module/kubernetes/scheduler/scheduler_test.go @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package scheduler diff --git a/metricbeat/module/kubernetes/system/system_test.go b/metricbeat/module/kubernetes/system/system_test.go index 657c4c1903b..a38d8fd8760 100644 --- a/metricbeat/module/kubernetes/system/system_test.go +++ b/metricbeat/module/kubernetes/system/system_test.go @@ -52,7 +52,7 @@ func TestEventMapping(t *testing.T) { "memory.pagefaults": 100835242, "memory.majorpagefaults": 0, - "cpu.usage.core.ns": 2357800908948, + "cpu.usage.core.ns": int64(2357800908948), "cpu.usage.nanocores": 11263994, } diff --git a/metricbeat/module/kubernetes/volume/_meta/fields.yml b/metricbeat/module/kubernetes/volume/_meta/fields.yml index 93b6204d53e..4eedf6014ba 100644 --- a/metricbeat/module/kubernetes/volume/_meta/fields.yml +++ b/metricbeat/module/kubernetes/volume/_meta/fields.yml @@ -35,6 +35,11 @@ format: bytes description: > Filesystem total used in bytes + - name: pct + type: scaled_float + format: percent + description: > + Percentage of used storage - name: inodes type: group fields: diff --git a/metricbeat/module/kubernetes/volume/data.go b/metricbeat/module/kubernetes/volume/data.go index ae0f5d891f4..6170bbadbcc 100644 --- a/metricbeat/module/kubernetes/volume/data.go +++ b/metricbeat/module/kubernetes/volume/data.go @@ -67,6 +67,9 @@ func eventMapping(content []byte) ([]common.MapStr, error) { }, }, } + if volume.CapacityBytes > 0 { + volumeEvent.Put("fs.used.pct", float64(volume.UsedBytes)/float64(volume.CapacityBytes)) + } events = append(events, volumeEvent) } diff --git a/metricbeat/module/kubernetes/volume/volume_test.go b/metricbeat/module/kubernetes/volume/volume_test.go index e527a64d452..71ca971c5e5 100644 --- a/metricbeat/module/kubernetes/volume/volume_test.go +++ b/metricbeat/module/kubernetes/volume/volume_test.go @@ -49,6 +49,7 @@ func TestEventMapping(t *testing.T) { "fs.available.bytes": 1939689472, "fs.capacity.bytes": 1939701760, "fs.used.bytes": 12288, + "fs.used.pct": float64(12288) / float64(1939701760), "fs.inodes.used": 9, "fs.inodes.free": 473551, "fs.inodes.count": 473560, diff --git a/metricbeat/module/mongodb/metricset.go b/metricbeat/module/mongodb/metricset.go index b1b09a7ac4d..dc329b57f25 100644 --- a/metricbeat/module/mongodb/metricset.go +++ b/metricbeat/module/mongodb/metricset.go @@ -65,7 +65,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { logp.Warn("Failed to obtain hostname from `%s`: %s", hostname, err) hostname = "" } - return tls.Dial("tcp", addr.String(), tlsConfig.BuildModuleConfig(hostname)) + return tls.Dial("tcp", addr.String(), tlsConfig.BuildModuleClientConfig(hostname)) } } diff --git a/metricbeat/module/rabbitmq/node/node_test.go b/metricbeat/module/rabbitmq/node/node_test.go index bc68b824ebc..9f11c05de84 100644 --- a/metricbeat/module/rabbitmq/node/node_test.go +++ b/metricbeat/module/rabbitmq/node/node_test.go @@ -60,7 +60,7 @@ func testFetch(t *testing.T, collect string) { disk := event["disk"].(common.MapStr) free := disk["free"].(common.MapStr) - assert.EqualValues(t, 98317942784, free["bytes"]) + assert.EqualValues(t, int64(98317942784), free["bytes"]) limit := free["limit"].(common.MapStr) assert.EqualValues(t, 50000000, limit["bytes"]) @@ -73,7 +73,7 @@ func testFetch(t *testing.T, collect string) { num := gc["num"].(common.MapStr) assert.EqualValues(t, 1049055, num["count"]) reclaimed := gc["reclaimed"].(common.MapStr) - assert.EqualValues(t, 27352751800, reclaimed["bytes"]) + assert.EqualValues(t, int64(27352751800), reclaimed["bytes"]) io := event["io"].(common.MapStr) fileHandle := io["file_handle"].(common.MapStr) @@ -109,7 +109,7 @@ func testFetch(t *testing.T, collect string) { mem := event["mem"].(common.MapStr) limit = mem["limit"].(common.MapStr) - assert.EqualValues(t, 6628692787, limit["bytes"]) + assert.EqualValues(t, int64(6628692787), limit["bytes"]) used := mem["used"].(common.MapStr) assert.EqualValues(t, 105504768, used["bytes"]) diff --git a/metricbeat/module/redis/key/key_integration_test.go b/metricbeat/module/redis/key/key_integration_test.go index bbe7b4831dc..e90c88f922e 100644 --- a/metricbeat/module/redis/key/key_integration_test.go +++ b/metricbeat/module/redis/key/key_integration_test.go @@ -23,7 +23,7 @@ import ( "fmt" "testing" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go index 239283f0f68..65a6591e3ff 100644 --- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go +++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go @@ -26,7 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/tests/compose" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" ) diff --git a/metricbeat/module/redis/metricset.go b/metricbeat/module/redis/metricset.go index dbd600f4838..0fa950ed420 100644 --- a/metricbeat/module/redis/metricset.go +++ b/metricbeat/module/redis/metricset.go @@ -23,7 +23,7 @@ import ( "strings" "time" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/pkg/errors" "github.com/elastic/beats/v7/metricbeat/mb" diff --git a/metricbeat/module/redis/metricset_integration_test.go b/metricbeat/module/redis/metricset_integration_test.go index 6f59508c323..b6abbe6e07d 100644 --- a/metricbeat/module/redis/metricset_integration_test.go +++ b/metricbeat/module/redis/metricset_integration_test.go @@ -22,7 +22,7 @@ package redis import ( "testing" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/metricbeat/module/redis/redis.go b/metricbeat/module/redis/redis.go index 2167a95fd7a..e5f5c620fa5 100644 --- a/metricbeat/module/redis/redis.go +++ b/metricbeat/module/redis/redis.go @@ -24,7 +24,7 @@ import ( "strings" "time" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/elastic/beats/v7/libbeat/logp" ) diff --git a/metricbeat/module/redis/redis_integration_test.go b/metricbeat/module/redis/redis_integration_test.go index ccf6e59d46f..abe91d77e68 100644 --- a/metricbeat/module/redis/redis_integration_test.go +++ b/metricbeat/module/redis/redis_integration_test.go @@ -23,7 +23,7 @@ import ( "strings" "testing" - rd "github.com/garyburd/redigo/redis" + rd "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 65f1d08ba80..1d57c84467c 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -586,6 +586,7 @@ def test_socket_summary(self): assert isinstance(udp["all"]["count"], int) + @unittest.skipIf(sys.platform == "win32", "Flaky test") def check_username(self, observed, expected=None): if expected is None: expected = getpass.getuser() diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go index d3dfd780ee6..2a991a7c265 100644 --- a/metricbeat/module/vsphere/host/data_test.go +++ b/metricbeat/module/vsphere/host/data_test.go @@ -57,10 +57,10 @@ func TestEventMapping(t *testing.T) { assert.EqualValues(t, 4521, cpuFree) memoryUsed, _ := event.GetValue("memory.used.bytes") - assert.EqualValues(t, 2251799812636672, memoryUsed) + assert.EqualValues(t, int64(2251799812636672), memoryUsed) memoryTotal, _ := event.GetValue("memory.total.bytes") - assert.EqualValues(t, 2251799812636672, memoryTotal) + assert.EqualValues(t, int64(2251799812636672), memoryTotal) memoryFree, _ := event.GetValue("memory.free.bytes") assert.EqualValues(t, 0, memoryFree) diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index 6e4e9c6ef0a..3dad6dc729e 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -38,58 +38,65 @@ var processRegexp = regexp.MustCompile(`(.+?[^\s])(?:#\d+|$)`) func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { eventMap := make(map[string]*mb.Event) for counterPath, values := range counters { - if hasCounter, counter := re.getCounter(counterPath); hasCounter { - for ind, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err.Error != nil { - if !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err.Error, logp.Namespace("perfmon"), "query", counterPath) - continue - } - // The counter has a negative value or the counter was successfully found, but the data returned is not valid. - // This error can occur if the counter value is less than the previous value. (Because counter values always increment, the counter value rolls over to zero when it reaches its maximum value.) - // This is not an error that stops the application from running successfully and a positive counter value should be retrieved in the later calls. - if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { - re.log.Debugw("Counter value retrieval returned", - "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) - continue - } - } - var eventKey string - if re.config.GroupMeasurements && val.Err.Error == nil { - // Send measurements with the same instance label as part of the same event - eventKey = val.Instance - } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) + hasCounter, counter := re.getCounter(counterPath) + if !hasCounter { + continue + } + + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err.Error != nil { + if !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err.Error, logp.Namespace("perfmon"), "query", counterPath) + continue } - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err.Error, "failed on query=%v", counterPath), - } - if val.Instance != "" { - //will ignore instance index - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) - } else { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) - } - } + // The counter has a negative value or the counter was successfully found, but the data returned is not valid. + // This error can occur if the counter value is less than the previous value. (Because counter values always increment, the counter value rolls over to zero when it reaches its maximum value.) + // This is not an error that stops the application from running successfully and a positive counter value should be retrieved in the later calls. + if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { + re.log.Debugw("Counter value retrieval returned", + "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) + continue } - if val.Measurement != nil { - eventMap[eventKey].MetricSetFields.Put(counter.QueryField, val.Measurement) - } else { - eventMap[eventKey].MetricSetFields.Put(counter.QueryField, 0) + } + + var eventKey string + if re.config.GroupMeasurements && val.Err.Error == nil { + // Send measurements from the same object with the same instance label as part of the same event + eventKey = counter.ObjectName + "\\" + val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err.Error, "failed on query=%v", counterPath), } - if counter.ObjectField != "" { - eventMap[eventKey].MetricSetFields.Put(counter.ObjectField, counter.ObjectName) + if val.Instance != "" { + // will ignore instance index + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) + } } } + + if val.Measurement != nil { + eventMap[eventKey].MetricSetFields.Put(counter.QueryField, val.Measurement) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.QueryField, 0) + } + + if counter.ObjectField != "" { + eventMap[eventKey].MetricSetFields.Put(counter.ObjectField, counter.ObjectName) + } } } // Write the values into the map. diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index 7203963d2cc..2554a87a9ce 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -30,6 +30,9 @@ import ( func TestGroupToEvents(t *testing.T) { reader := Reader{ + config: Config{ + GroupMeasurements: true, + }, query: pdh.Query{}, executed: true, log: nil, @@ -42,6 +45,26 @@ func TestGroupToEvents(t *testing.T) { ObjectField: "object", ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, }, + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(_Total)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(_Total)\% Processor Time`}, + }, + { + QueryField: "current_disk_queue_length", + QueryName: `\PhysicalDisk(_Total)\Current Disk Queue Length`, + Format: "float", + ObjectName: "PhysicalDisk", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\PhysicalDisk(_Total)\Current Disk Queue Length`}, + }, }, } counters := map[string][]pdh.CounterValue{ @@ -52,23 +75,76 @@ func TestGroupToEvents(t *testing.T) { Err: pdh.CounterValueError{}, }, }, + `\Processor Information(_Total)\% Processor Time`: { + { + Instance: "_Total", + Measurement: 11, + }, + }, + `\PhysicalDisk(_Total)\Current Disk Queue Length`: { + { + Instance: "_Total", + Measurement: 20, + }, + }, } + events := reader.groupToEvents(counters) assert.NotNil(t, events) - assert.Equal(t, len(events), 1) - ok, err := events[0].MetricSetFields.HasKey("datagrams_sent_per_sec") - assert.NoError(t, err) - assert.True(t, ok) - ok, err = events[0].MetricSetFields.HasKey("object") - assert.NoError(t, err) - assert.True(t, ok) - val, err := events[0].MetricSetFields.GetValue("datagrams_sent_per_sec") - assert.NoError(t, err) - assert.Equal(t, val, 23) - val, err = events[0].MetricSetFields.GetValue("object") - assert.NoError(t, err) - assert.Equal(t, val, "UDPv4") + assert.Equal(t, 3, len(events)) + for _, event := range events { + ok, err := event.MetricSetFields.HasKey("datagrams_sent_per_sec") + if ok { + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.Equal(t, val, 23) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "UDPv4") + } else { + ok, err := event.MetricSetFields.HasKey("%_processor_time") + if ok { + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("%_processor_time") + assert.NoError(t, err) + assert.Equal(t, val, 11) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "Processor Information") + } else { + ok, err = event.MetricSetFields.HasKey("current_disk_queue_length") + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("current_disk_queue_length") + assert.NoError(t, err) + assert.Equal(t, val, 20) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "PhysicalDisk") + } + } + } } func TestGroupToSingleEvent(t *testing.T) { diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 2b0faecc26f..6c5abe6309a 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index e14c3546293..1d65795e022 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -36,6 +36,8 @@ stages: mage: "mage build test" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. @@ -55,6 +57,8 @@ stages: - "windows-2019" when: ## Override the top-level when. not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. diff --git a/packetbeat/_meta/config/beat.reference.yml.tmpl b/packetbeat/_meta/config/beat.reference.yml.tmpl index 77a1de3c876..5d8d8fa9c7a 100644 --- a/packetbeat/_meta/config/beat.reference.yml.tmpl +++ b/packetbeat/_meta/config/beat.reference.yml.tmpl @@ -80,7 +80,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: true + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. #enabled: true # Set to true to publish fields with null values in events. diff --git a/packetbeat/_meta/config/beat.yml.tmpl b/packetbeat/_meta/config/beat.yml.tmpl index c1ad53bf9e9..2a69df42517 100644 --- a/packetbeat/_meta/config/beat.yml.tmpl +++ b/packetbeat/_meta/config/beat.yml.tmpl @@ -38,7 +38,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: false + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. enabled: true - type: amqp @@ -47,7 +47,8 @@ packetbeat.protocols: ports: [5672] - type: cassandra - #Cassandra port for traffic monitoring. + # Configure the ports where to listen for Cassandra traffic. You can disable + # the Cassandra protocol by commenting out the list of ports. ports: [9042] - type: dhcpv4 @@ -112,7 +113,8 @@ packetbeat.protocols: - 9243 # Elasticsearch - type: sip - # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + # Configure the ports where to listen for SIP traffic. You can disable + # the SIP protocol by commenting out the list of ports. ports: [5060] {{header "Elasticsearch template setting"}} diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 57142c14263..073bec9c768 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -80,7 +80,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: true + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. #enabled: true # Set to true to publish fields with null values in events. @@ -1024,6 +1024,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1151,6 +1157,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1339,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1348,6 +1364,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1504,6 +1526,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1800,6 +1828,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1997,6 +2031,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 2ac9186d43e..15a0df9ebd1 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -38,7 +38,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: false + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. enabled: true - type: amqp @@ -47,7 +47,8 @@ packetbeat.protocols: ports: [5672] - type: cassandra - #Cassandra port for traffic monitoring. + # Configure the ports where to listen for Cassandra traffic. You can disable + # the Cassandra protocol by commenting out the list of ports. ports: [9042] - type: dhcpv4 @@ -112,7 +113,8 @@ packetbeat.protocols: - 9243 # Elasticsearch - type: sip - # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + # Configure the ports where to listen for SIP traffic. You can disable + # the SIP protocol by commenting out the list of ports. ports: [5060] # ======================= Elasticsearch template setting ======================= diff --git a/packetbeat/protos/sip/parser.go b/packetbeat/protos/sip/parser.go index 7ee5a10bb5b..8d49ad61742 100644 --- a/packetbeat/protos/sip/parser.go +++ b/packetbeat/protos/sip/parser.go @@ -164,8 +164,9 @@ func (*parser) parseSIPLine(pi *parsingInfo, m *message) error { err error ) + const minStatusLineLength = len("SIP/2.0 XXX OK") fline := pi.data[pi.parseOffset:i] - if len(fline) < 16 { // minimum line will be "SIP/2.0 XXX OK\r\n" + if len(fline) < minStatusLineLength { if isDebug { debugf("First line too small") } diff --git a/packetbeat/tests/system/golden/sip-expected.json b/packetbeat/tests/system/golden/sip-expected.json index 37d95d715a7..7b133413af3 100644 --- a/packetbeat/tests/system/golden/sip-expected.json +++ b/packetbeat/tests/system/golden/sip-expected.json @@ -157,6 +157,117 @@ "status": "OK", "type": "sip" }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0\r\nFrom: \"DVI4/8000\" ;tag=1\r\nTo: test ;tag=e2jv529vDZ3eQ\r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 1 INVITE\r\nContact: \r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nAccept: application/sdp\r\nAllow: INVITE, ACK, BYE, CANCEL, OPTIONS, MESSAGE, INFO, UPDATE, REGISTER, REFER, NOTIFY, PUBLISH, SUBSCRIBE\r\nSupported: timer, path, replaces\r\nAllow-Events: talk, hold, conference, presence, as-feature-event, dialog, line-seize, call-info, sla, include-session-description, presence.winfo, message-summary, refer\r\nContent-Type: application/sdp\r\nContent-Disposition: session\r\nContent-Length: 226\r\nRemote-Party-ID: \"test\" ;party=calling;privacy=off;screen=no\r\n\r\nv=0\r\no=FreeSWITCH 1480142495 1480142496 IN IP4 10.0.2.15\r\ns=FreeSWITCH\r\nc=IN IP4 10.0.2.15\r\nt=0 0\r\nm=audio 30490 RTP/AVP 5 101\r\na=rtpmap:5 DVI4/8000\r\na=rtpmap:101 telephone-event/8000\r\na=fmtp:101 0-16\r\na=sendonly\r\na=ptime:20\r\n", + "event.outcome": "success", + "event.reason": "OK", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test", + "FreeSWITCH" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.accept": "application/sdp", + "sip.allow": [ + "invite", + "ack", + "bye", + "cancel", + "options", + "message", + "info", + "update", + "register", + "refer", + "notify", + "publish", + "subscribe" + ], + "sip.call_id": "1-2187@10.0.2.20", + "sip.code": 200, + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.content_length": 226, + "sip.content_type": "application/sdp", + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/8000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.sdp.body.original": "v=0\r\no=FreeSWITCH 1480142495 1480142496 IN IP4 10.0.2.15\r\ns=FreeSWITCH\r\nc=IN IP4 10.0.2.15\r\nt=0 0\r\nm=audio 30490 RTP/AVP 5 101\r\na=rtpmap:5 DVI4/8000\r\na=rtpmap:101 telephone-event/8000\r\na=fmtp:101 0-16\r\na=sendonly\r\na=ptime:20\r\n", + "sip.sdp.connection.address": "10.0.2.15", + "sip.sdp.connection.info": "IN IP4 10.0.2.15", + "sip.sdp.owner.ip": "10.0.2.15", + "sip.sdp.owner.session_id": "1480142495", + "sip.sdp.owner.username": "FreeSWITCH", + "sip.sdp.owner.version": "1480142496", + "sip.sdp.session.name": "FreeSWITCH", + "sip.sdp.version": "0", + "sip.status": "OK", + "sip.supported": [ + "timer", + "path", + "replaces" + ], + "sip.to.display_info": "test", + "sip.to.tag": "e2jv529vDZ3eQ", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "response", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, { "@metadata.beat": "packetbeat", "@metadata.type": "_doc", @@ -332,6 +443,83 @@ "status": "OK", "type": "sip" }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-bye", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKDQ7XK6BBH57ya\r\nFrom: test ;tag=e2jv529vDZ3eQ\r\nTo: \"DVI4/8000\" ;tag=1\r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 99750433 BYE\r\nContact: \r\nContent-Length: 0\r\n\r\n", + "event.outcome": "success", + "event.reason": "OK", + "event.sequence": 99750433, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2187@10.0.2.20", + "sip.code": 200, + "sip.contact.display_info": "DVI4/8000", + "sip.contact.uri.host": "10.0.2.20", + "sip.contact.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "sipp", + "sip.cseq.code": 99750433, + "sip.cseq.method": "BYE", + "sip.from.display_info": "test", + "sip.from.tag": "e2jv529vDZ3eQ", + "sip.from.uri.host": "10.0.2.15", + "sip.from.uri.original": "sip:test@10.0.2.15:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "test", + "sip.status": "OK", + "sip.to.display_info": "DVI4/8000", + "sip.to.tag": "1", + "sip.to.uri.host": "10.0.2.20", + "sip.to.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "sipp", + "sip.type": "response", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKDQ7XK6BBH57ya" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, { "@metadata.beat": "packetbeat", "@metadata.type": "_doc", @@ -490,6 +678,117 @@ "status": "OK", "type": "sip" }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0\r\nFrom: \"DVI4/16000\" ;tag=1\r\nTo: test ;tag=FBcN7Xt0a8S1j\r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 1 INVITE\r\nContact: \r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nAccept: application/sdp\r\nAllow: INVITE, ACK, BYE, CANCEL, OPTIONS, MESSAGE, INFO, UPDATE, REGISTER, REFER, NOTIFY, PUBLISH, SUBSCRIBE\r\nSupported: timer, path, replaces\r\nAllow-Events: talk, hold, conference, presence, as-feature-event, dialog, line-seize, call-info, sla, include-session-description, presence.winfo, message-summary, refer\r\nContent-Type: application/sdp\r\nContent-Disposition: session\r\nContent-Length: 227\r\nRemote-Party-ID: \"test\" ;party=calling;privacy=off;screen=no\r\n\r\nv=0\r\no=FreeSWITCH 1480147848 1480147849 IN IP4 10.0.2.15\r\ns=FreeSWITCH\r\nc=IN IP4 10.0.2.15\r\nt=0 0\r\nm=audio 25146 RTP/AVP 6 101\r\na=rtpmap:6 DVI4/16000\r\na=rtpmap:101 telephone-event/8000\r\na=fmtp:101 0-16\r\na=sendonly\r\na=ptime:20\r\n", + "event.outcome": "success", + "event.reason": "OK", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test", + "FreeSWITCH" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.accept": "application/sdp", + "sip.allow": [ + "invite", + "ack", + "bye", + "cancel", + "options", + "message", + "info", + "update", + "register", + "refer", + "notify", + "publish", + "subscribe" + ], + "sip.call_id": "1-2189@10.0.2.20", + "sip.code": 200, + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.content_length": 227, + "sip.content_type": "application/sdp", + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/16000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.sdp.body.original": "v=0\r\no=FreeSWITCH 1480147848 1480147849 IN IP4 10.0.2.15\r\ns=FreeSWITCH\r\nc=IN IP4 10.0.2.15\r\nt=0 0\r\nm=audio 25146 RTP/AVP 6 101\r\na=rtpmap:6 DVI4/16000\r\na=rtpmap:101 telephone-event/8000\r\na=fmtp:101 0-16\r\na=sendonly\r\na=ptime:20\r\n", + "sip.sdp.connection.address": "10.0.2.15", + "sip.sdp.connection.info": "IN IP4 10.0.2.15", + "sip.sdp.owner.ip": "10.0.2.15", + "sip.sdp.owner.session_id": "1480147848", + "sip.sdp.owner.username": "FreeSWITCH", + "sip.sdp.owner.version": "1480147849", + "sip.sdp.session.name": "FreeSWITCH", + "sip.sdp.version": "0", + "sip.status": "OK", + "sip.supported": [ + "timer", + "path", + "replaces" + ], + "sip.to.display_info": "test", + "sip.to.tag": "FBcN7Xt0a8S1j", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "response", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, { "@metadata.beat": "packetbeat", "@metadata.type": "_doc", @@ -664,5 +963,82 @@ "source.port": 5060, "status": "OK", "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-bye", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 200 OK\r\nVia: SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKe00pN1veeeyHp\r\nFrom: test ;tag=FBcN7Xt0a8S1j\r\nTo: \"DVI4/16000\" ;tag=1\r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 99750437 BYE\r\nContact: \r\nContent-Length: 0\r\n\r\n", + "event.outcome": "success", + "event.reason": "OK", + "event.sequence": 99750437, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2189@10.0.2.20", + "sip.code": 200, + "sip.contact.display_info": "DVI4/16000", + "sip.contact.uri.host": "10.0.2.20", + "sip.contact.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "sipp", + "sip.cseq.code": 99750437, + "sip.cseq.method": "BYE", + "sip.from.display_info": "test", + "sip.from.tag": "FBcN7Xt0a8S1j", + "sip.from.uri.host": "10.0.2.15", + "sip.from.uri.original": "sip:test@10.0.2.15:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "test", + "sip.status": "OK", + "sip.to.display_info": "DVI4/16000", + "sip.to.tag": "1", + "sip.to.uri.host": "10.0.2.20", + "sip.to.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "sipp", + "sip.type": "response", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKe00pN1veeeyHp" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" } ] \ No newline at end of file diff --git a/packetbeat/tests/system/golden/sip_authenticated_register-expected.json b/packetbeat/tests/system/golden/sip_authenticated_register-expected.json index 133792cc157..0a582c42b35 100644 --- a/packetbeat/tests/system/golden/sip_authenticated_register-expected.json +++ b/packetbeat/tests/system/golden/sip_authenticated_register-expected.json @@ -138,5 +138,80 @@ "source.port": 5060, "status": "OK", "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "212.242.33.35", + "client.port": 5060, + "destination.ip": "192.168.1.2", + "destination.port": 5060, + "event.action": "sip-register", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 200 OK\r\nCall-ID: 578222729-4665d775@578222732-4665d772\r\nContact: ;q=0.500;expires=1200\r\nCSeq: 75 REGISTER\r\nFrom: ;tag=6bac55c\r\nP-Associated-URI: \r\nTo: ;tag=00-04081-1701b256-5586b7324\r\nVia: SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060;branch=z9hG4bKnp112903503-43a64480192.168.1.2\r\nContent-Length: 0\r\n\r\n", + "event.outcome": "success", + "event.reason": "OK", + "event.sequence": 75, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:dOa61R2NaaJsJlcFAiMIiyXX+Kk=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "sip.cybercity.dk>", + "sip.cybercity.dk" + ], + "related.ip": [ + "212.242.33.35", + "192.168.1.2" + ], + "related.user": [ + "35104723", + "voi18062" + ], + "server.ip": "192.168.1.2", + "server.port": 5060, + "sip.call_id": "578222729-4665d775@578222732-4665d772", + "sip.code": 200, + "sip.contact.uri.host": "sip.cybercity.dk", + "sip.contact.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "voi18062", + "sip.cseq.code": 75, + "sip.cseq.method": "REGISTER", + "sip.from.tag": "6bac55c", + "sip.from.uri.host": "sip.cybercity.dk", + "sip.from.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "voi18062", + "sip.private.uri.host": "sip.cybercity.dk>", + "sip.private.uri.original": "", + "sip.private.uri.scheme": " 0 && !managedApplication.wasUnenrolled() { // TODO(ph) We will need an improvement on fleet, if there is an error while dispatching a @@ -273,6 +280,7 @@ func newManaged( fleetR, actionAcker, statusController, + stateStore, ) if err != nil { return nil, err @@ -316,7 +324,7 @@ func (m *Managed) AgentInfo() *info.AgentInfo { } func (m *Managed) wasUnenrolled() bool { - actions := m.as.Actions() + actions := m.stateStore.Actions() for _, a := range actions { if a.Type() == "UNENROLL" { return true diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go index 920b1a4b5bf..ab9ff6bbc63 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go @@ -5,6 +5,7 @@ package application import ( + "crypto/md5" "fmt" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" @@ -15,6 +16,7 @@ import ( const ( monitoringName = "FLEET_MONITORING" programsKey = "programs" + monitoringChecksumKey = "monitoring_checksum" monitoringKey = "agent.monitoring" monitoringUseOutputKey = "agent.monitoring.use_output" monitoringOutputFormatKey = "outputs.%s" @@ -74,12 +76,15 @@ func injectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr } programList := make([]string, 0, len(programsToRun)) + cfgHash := md5.New() for _, p := range programsToRun { programList = append(programList, p.Spec.Cmd) + cfgHash.Write(p.Config.Hash()) } - // making program list part of the config + // making program list and their hashes part of the config // so it will get regenerated with every change config[programsKey] = programList + config[monitoringChecksumKey] = fmt.Sprintf("%x", cfgHash.Sum(nil)) monitoringProgram.Config, err = transpiler.NewAST(config) if err != nil { diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go index 6a3be4100be..e23027e62fc 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go @@ -5,6 +5,7 @@ package application import ( + "fmt" "testing" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" @@ -27,6 +28,10 @@ func TestMonitoringInjection(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -102,6 +107,10 @@ func TestMonitoringInjectionDefaults(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -177,6 +186,10 @@ func TestMonitoringInjectionDisabled(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 2 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 2)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -203,19 +216,19 @@ GROUPLOOP: } // is enabled set - settingsObj, found := cm["settings"] + agentObj, found := cm["agent"] if !found { t.Errorf("settings not found for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP } - settingsMap, ok := settingsObj.(map[string]interface{}) + agentMap, ok := agentObj.(map[string]interface{}) if !ok { t.Errorf("settings not a map for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP } - monitoringObj, found := settingsMap["monitoring"] + monitoringObj, found := agentMap["monitoring"] if !found { t.Errorf("agent.monitoring not found for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP @@ -247,6 +260,97 @@ GROUPLOOP: } } +func TestChangeInMonitoringWithChangeInInput(t *testing.T) { + agentInfo, err := info.NewAgentInfo() + if err != nil { + t.Fatal(err) + } + + astBefore, err := transpiler.NewAST(inputChange1) + if err != nil { + t.Fatal(err) + } + + programsToRunBefore, err := program.Programs(agentInfo, astBefore) + if err != nil { + t.Fatal(err) + } + + if len(programsToRunBefore) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + + astAfter, err := transpiler.NewAST(inputChange2) + if err != nil { + t.Fatal(err) + } + + programsToRunAfter, err := program.Programs(agentInfo, astAfter) + if err != nil { + t.Fatal(err) + } + + if len(programsToRunAfter) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + + // inject to both + var hashConfigBefore, hashConfigAfter string +GROUPLOOPBEFORE: + for group, ptr := range programsToRunBefore { + programsCount := len(ptr) + newPtr, err := injectMonitoring(agentInfo, group, astBefore, ptr) + if err != nil { + t.Error(err) + continue GROUPLOOPBEFORE + } + + if programsCount+1 != len(newPtr) { + t.Errorf("incorrect programs to run count, expected: %d, got %d", programsCount+1, len(newPtr)) + continue GROUPLOOPBEFORE + } + + for _, p := range newPtr { + if p.Spec.Name != monitoringName { + continue + } + + hashConfigBefore = p.Config.HashStr() + } + } + +GROUPLOOPAFTER: + for group, ptr := range programsToRunAfter { + programsCount := len(ptr) + newPtr, err := injectMonitoring(agentInfo, group, astAfter, ptr) + if err != nil { + t.Error(err) + continue GROUPLOOPAFTER + } + + if programsCount+1 != len(newPtr) { + t.Errorf("incorrect programs to run count, expected: %d, got %d", programsCount+1, len(newPtr)) + continue GROUPLOOPAFTER + } + + for _, p := range newPtr { + if p.Spec.Name != monitoringName { + continue + } + + hashConfigAfter = p.Config.HashStr() + } + } + + if hashConfigAfter == "" || hashConfigBefore == "" { + t.Fatal("hash configs uninitialized") + } + + if hashConfigAfter == hashConfigBefore { + t.Fatal("hash config equal, expected to be different") + } +} + var inputConfigMap = map[string]interface{}{ "agent.monitoring": map[string]interface{}{ "enabled": true, @@ -279,40 +383,33 @@ var inputConfigMap = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + "inputs": []map[string]interface{}{ + { + "type": "log", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, - }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ - map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, - }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, }, }, - "use_output": "infosec1", }, }, } @@ -343,40 +440,34 @@ var inputConfigMapDefaults = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + + "inputs": []map[string]interface{}{ + { + "type": "log", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, - }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ - map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, - }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, }, }, - "use_output": "infosec1", }, }, } @@ -410,40 +501,114 @@ var inputConfigMapDisabled = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + }, + }, + }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, + }, + }, + }, + }, +} + +var inputChange1 = map[string]interface{}{ + "agent.monitoring": map[string]interface{}{ + "enabled": true, + "logs": true, + "metrics": true, + "use_output": "monitoring", + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "index_name": "general", + "pass": "xxx", + "type": "elasticsearch", + "url": "xxxxx", + "username": "xxx", + }, + "monitoring": map[string]interface{}{ + "type": "elasticsearch", + "index_name": "general", + "pass": "xxx", + "url": "xxxxx", + "username": "monitoring-uname", + }, + }, + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ + map[string]interface{}{ + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ + }, +} + +var inputChange2 = map[string]interface{}{ + "agent.monitoring": map[string]interface{}{ + "enabled": true, + "logs": true, + "metrics": true, + "use_output": "monitoring", + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "index_name": "general", + "pass": "xxx", + "type": "elasticsearch", + "url": "xxxxx", + "username": "xxx", + }, + "monitoring": map[string]interface{}{ + "type": "elasticsearch", + "index_name": "general", + "pass": "xxx", + "url": "xxxxx", + "username": "monitoring-uname", + }, + }, + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + {"paths": "/yyyy"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, - "use_output": "infosec1", }, }, } diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store.go b/x-pack/elastic-agent/pkg/agent/application/state_store.go new file mode 100644 index 00000000000..283ab8e480d --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/state_store.go @@ -0,0 +1,303 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "fmt" + "io" + "sync" + + yaml "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +// stateStore is a combined agent state storage initially derived from the former actionStore +// and modified to allow persistence of additional agent specific state information. +// The following is the original actionStore implementation description: +// receives multiples actions to persist to disk, the implementation of the store only +// take care of action policy change every other action are discarded. The store will only keep the +// last good action on disk, we assume that the action is added to the store after it was ACK with +// Fleet. The store is not threadsafe. +type stateStore struct { + log *logger.Logger + store storeLoad + dirty bool + state stateT + + mx sync.RWMutex +} + +type stateT struct { + action action + ackToken string +} + +// Combined yml serializer for the ActionPolicyChange and ActionUnenroll +type actionSerializer struct { + ID string `yaml:"action_id"` + Type string `yaml:"action_type"` + Policy map[string]interface{} `yaml:"policy,omitempty"` + IsDetected *bool `yaml:"is_detected,omitempty"` +} + +type stateSerializer struct { + Action *actionSerializer `yaml:"action,omitempty"` + AckToken string `yaml:"ack_token,omitempty"` +} + +func migrateStateStore(log *logger.Logger, actionStorePath, stateStorePath string) (err error) { + log = log.Named("state_migration") + actionDiskStore := storage.NewDiskStore(actionStorePath) + stateDiskStore := storage.NewDiskStore(stateStorePath) + + stateStoreExits, err := stateDiskStore.Exists() + if err != nil { + log.With() + log.Errorf("failed to check if state store %s exists: %v", stateStorePath, err) + return err + } + + // do not migrate if the state store already exists + if stateStoreExits { + log.Debugf("state store %s already exists", stateStorePath) + return nil + } + + actionStoreExits, err := actionDiskStore.Exists() + if err != nil { + log.Errorf("failed to check if action store %s exists: %v", actionStorePath, err) + return err + } + + // delete the actions store file upon successful migration + defer func() { + if err == nil && actionStoreExits { + err = actionDiskStore.Delete() + if err != nil { + log.Errorf("failed to delete action store %s exists: %v", actionStorePath, err) + } + } + }() + + // nothing to migrate if the action store doesn't exists + if !actionStoreExits { + log.Debugf("action store %s doesn't exists, nothing to migrate", actionStorePath) + return nil + } + + actionStore, err := newActionStore(log, actionDiskStore) + if err != nil { + log.Errorf("failed to create action store %s: %v", actionStorePath, err) + return err + } + + // no actions stored nothing to migrate + if len(actionStore.Actions()) == 0 { + log.Debugf("no actions stored in the action store %s, nothing to migrate", actionStorePath) + return nil + } + + stateStore, err := newStateStore(log, stateDiskStore) + if err != nil { + return err + } + + // set actions from the action store to the state store + stateStore.Add(actionStore.Actions()[0]) + + err = stateStore.Save() + if err != nil { + log.Debugf("failed to save agent state store %s, err: %v", stateStorePath, err) + } + return err +} + +func newStateStoreWithMigration(log *logger.Logger, actionStorePath, stateStorePath string) (*stateStore, error) { + err := migrateStateStore(log, actionStorePath, stateStorePath) + if err != nil { + return nil, err + } + + return newStateStore(log, storage.NewDiskStore(stateStorePath)) +} + +func newStateStore(log *logger.Logger, store storeLoad) (*stateStore, error) { + // If the store exists we will read it, if any errors is returned we assume we do not have anything + // persisted and we return an empty store. + reader, err := store.Load() + if err != nil { + return &stateStore{log: log, store: store}, nil + } + defer reader.Close() + + var sr stateSerializer + + dec := yaml.NewDecoder(reader) + err = dec.Decode(&sr) + if err == io.EOF { + return &stateStore{ + log: log, + store: store, + }, nil + } + + if err != nil { + return nil, err + } + + state := stateT{ + ackToken: sr.AckToken, + } + + if sr.Action != nil { + if sr.Action.IsDetected != nil { + state.action = &fleetapi.ActionUnenroll{ + ActionID: sr.Action.ID, + ActionType: sr.Action.Type, + IsDetected: *sr.Action.IsDetected, + } + } else { + state.action = &fleetapi.ActionPolicyChange{ + ActionID: sr.Action.ID, + ActionType: sr.Action.Type, + Policy: sr.Action.Policy, + } + } + } + + return &stateStore{ + log: log, + store: store, + state: state, + }, nil +} + +// Add is only taking care of ActionPolicyChange for now and will only keep the last one it receive, +// any other type of action will be silently ignored. +func (s *stateStore) Add(a action) { + s.mx.Lock() + defer s.mx.Unlock() + + switch v := a.(type) { + case *fleetapi.ActionPolicyChange, *fleetapi.ActionUnenroll: + // Only persist the action if the action is different. + if s.state.action != nil && s.state.action.ID() == v.ID() { + return + } + s.dirty = true + s.state.action = a + } +} + +// SetAckToken set ack token to the agent state +func (s *stateStore) SetAckToken(ackToken string) { + s.mx.Lock() + defer s.mx.Unlock() + + if s.state.ackToken == ackToken { + return + } + s.dirty = true + s.state.ackToken = ackToken +} + +func (s *stateStore) Save() error { + s.mx.Lock() + defer s.mx.Unlock() + + defer func() { s.dirty = false }() + if !s.dirty { + return nil + } + + var reader io.Reader + serialize := stateSerializer{ + AckToken: s.state.ackToken, + } + + if s.state.action != nil { + if apc, ok := s.state.action.(*fleetapi.ActionPolicyChange); ok { + serialize.Action = &actionSerializer{apc.ActionID, apc.ActionType, apc.Policy, nil} + } else if aun, ok := s.state.action.(*fleetapi.ActionUnenroll); ok { + serialize.Action = &actionSerializer{aun.ActionID, aun.ActionType, nil, &aun.IsDetected} + } else { + return fmt.Errorf("incompatible type, expected ActionPolicyChange and received %T", s.state.action) + } + } + + reader, err := yamlToReader(&serialize) + if err != nil { + return err + } + + if err := s.store.Save(reader); err != nil { + return err + } + s.log.Debugf("save state on disk : %+v", s.state) + return nil +} + +// Actions returns a slice of action to execute in order, currently only a action policy change is +// persisted. +func (s *stateStore) Actions() []action { + s.mx.RLock() + defer s.mx.RUnlock() + + if s.state.action == nil { + return []action{} + } + + return []action{s.state.action} +} + +// AckToken return the agent state persisted ack_token +func (s *stateStore) AckToken() string { + s.mx.RLock() + defer s.mx.RUnlock() + return s.state.ackToken +} + +// actionStoreAcker wraps an existing acker and will send any acked event to the action store, +// its up to the action store to decide if we need to persist the event for future replay or just +// discard the event. +type stateStoreActionAcker struct { + acker fleetAcker + store *stateStore +} + +func (a *stateStoreActionAcker) Ack(ctx context.Context, action fleetapi.Action) error { + if err := a.acker.Ack(ctx, action); err != nil { + return err + } + a.store.Add(action) + return a.store.Save() +} + +func (a *stateStoreActionAcker) Commit(ctx context.Context) error { + return a.acker.Commit(ctx) +} + +func newStateStoreActionAcker(acker fleetAcker, store *stateStore) *stateStoreActionAcker { + return &stateStoreActionAcker{acker: acker, store: store} +} + +func replayActions( + log *logger.Logger, + dispatcher dispatcher, + acker fleetAcker, + actions ...action, +) error { + log.Info("restoring current policy from disk") + + if err := dispatcher.Dispatch(acker, actions...); err != nil { + return err + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store_test.go b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go new file mode 100644 index 00000000000..1c6a7bfd731 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +func TestStateStore(t *testing.T) { + t.Run("ack token", func(t *testing.T) { + runTestStateStore(t, "") + }) + + t.Run("no ack token", func(t *testing.T) { + runTestStateStore(t, "czlV93YBwdkt5lYhBY7S") + }) +} + +func runTestStateStore(t *testing.T, ackToken string) { + log, _ := logger.New("state_store") + withFile := func(fn func(t *testing.T, file string)) func(*testing.T) { + return func(t *testing.T) { + dir, err := ioutil.TempDir("", "state-store") + require.NoError(t, err) + defer os.RemoveAll(dir) + file := filepath.Join(dir, "state.yml") + fn(t, file) + } + } + + t.Run("action returns empty when no action is saved on disk", + withFile(func(t *testing.T, file string) { + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + require.Equal(t, 0, len(store.Actions())) + })) + + t.Run("will discard silently unknown action", + withFile(func(t *testing.T, file string) { + actionPolicyChange := &fleetapi.ActionUnknown{ + ActionID: "abc123", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(actionPolicyChange) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 0, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("can save to disk known action type", + withFile(func(t *testing.T, file string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ + "hello": "world", + }, + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(ActionPolicyChange) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + + s = storage.NewDiskStore(file) + store1, err := newStateStore(log, s) + require.NoError(t, err) + + actions := store1.Actions() + require.Equal(t, 1, len(actions)) + + require.Equal(t, ActionPolicyChange, actions[0]) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("can save to disk unenroll action type", + withFile(func(t *testing.T, file string) { + action := &fleetapi.ActionUnenroll{ + ActionID: "abc123", + ActionType: "UNENROLL", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(action) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + + s = storage.NewDiskStore(file) + store1, err := newStateStore(log, s) + require.NoError(t, err) + + actions := store1.Actions() + require.Equal(t, 1, len(actions)) + + require.Equal(t, action, actions[0]) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("when we ACK we save to disk", + withFile(func(t *testing.T, file string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + store.SetAckToken(ackToken) + + acker := newStateStoreActionAcker(&testAcker{}, store) + require.Equal(t, 0, len(store.Actions())) + + require.NoError(t, acker.Ack(context.Background(), ActionPolicyChange)) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("migrate actions file does not exists", + withFile(func(t *testing.T, actionStorePath string) { + withFile(func(t *testing.T, stateStorePath string) { + err := migrateStateStore(log, actionStorePath, stateStorePath) + require.NoError(t, err) + stateStore, err := newStateStore(log, storage.NewDiskStore(stateStorePath)) + require.NoError(t, err) + stateStore.SetAckToken(ackToken) + require.Equal(t, 0, len(stateStore.Actions())) + require.Equal(t, ackToken, stateStore.AckToken()) + }) + })) + + t.Run("migrate", + withFile(func(t *testing.T, actionStorePath string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ + "hello": "world", + }, + } + + actionStore, err := newActionStore(log, storage.NewDiskStore(actionStorePath)) + require.NoError(t, err) + + require.Equal(t, 0, len(actionStore.Actions())) + actionStore.Add(ActionPolicyChange) + err = actionStore.Save() + require.NoError(t, err) + require.Equal(t, 1, len(actionStore.Actions())) + + withFile(func(t *testing.T, stateStorePath string) { + err = migrateStateStore(log, actionStorePath, stateStorePath) + require.NoError(t, err) + + stateStore, err := newStateStore(log, storage.NewDiskStore(stateStorePath)) + require.NoError(t, err) + stateStore.SetAckToken(ackToken) + diff := cmp.Diff(actionStore.Actions(), stateStore.Actions()) + if diff != "" { + t.Error(diff) + } + require.Equal(t, ackToken, stateStore.AckToken()) + }) + })) + +} diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/service.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/service.go index 2c21d020f8b..097fabb6855 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/service.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/service.go @@ -171,7 +171,12 @@ func (p *dbusPidProvider) Close() { } func (p *dbusPidProvider) PID(ctx context.Context) (int, error) { - prop, err := p.dbusConn.GetServiceProperty(install.ServiceName, "MainPID") + sn := install.ServiceName + if !strings.HasSuffix(sn, ".service") { + sn += ".service" + } + + prop, err := p.dbusConn.GetServiceProperty(sn, "MainPID") if err != nil { return 0, errors.New("failed to read service", err) } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/install.go b/x-pack/elastic-agent/pkg/agent/cmd/install.go index 7475a366486..7fd5b23ea18 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/install.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/install.go @@ -51,7 +51,8 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, return fmt.Errorf("unable to perform install command, not executed with %s permissions", install.PermissionUser) } status, reason := install.Status() - if status == install.Installed { + force, _ := cmd.Flags().GetBool("force") + if status == install.Installed && !force { return fmt.Errorf("already installed at: %s", install.InstallPath) } @@ -66,7 +67,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, locker.Unlock() warn.PrintNotGA(streams.Out) - force, _ := cmd.Flags().GetBool("force") + if status == install.Broken { if !force { fmt.Fprintf(streams.Out, "Elastic Agent is installed but currently broken: %s\n", reason) diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 43cab2fd3b4..2ec6b531456 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -76,7 +76,7 @@ func getTestOperator(t *testing.T, downloadPath string, installPath string, p *a } operator.config.DownloadConfig.OperatingSystem = "darwin" - operator.config.DownloadConfig.Architecture = "32" + operator.config.DownloadConfig.Architecture = "64" // make the download path so the `operation_verify` can ensure the path exists downloadConfig := operator.config.DownloadConfig @@ -101,7 +101,7 @@ func getProgram(binary, version string) *app.Descriptor { downloadCfg := &artifact.Config{ InstallPath: installPath, OperatingSystem: "darwin", - Architecture: "32", + Architecture: "64", } return app.NewDescriptor(spec, version, downloadCfg, nil) } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index a7b835503be..882f9efdf14 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -129,8 +129,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() - // generate only on change - if watchLogs != o.isMonitoringLogs() { + // generate only when monitoring is running (for config refresh) or + // state changes (turning on/off) + if watchLogs != o.isMonitoringLogs() || watchLogs { fbConfig, any := o.getMonitoringFilebeatConfig(output) stepID := configrequest.StepRun if !watchLogs || !any { @@ -151,7 +152,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ steps = append(steps, filebeatStep) } - if watchMetrics != o.isMonitoringMetrics() { + if watchMetrics != o.isMonitoringMetrics() || watchMetrics { mbConfig, any := o.getMonitoringMetricbeatConfig(output) stepID := configrequest.StepRun if !watchMetrics || !any { @@ -188,7 +189,9 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i }, "paths": []string{ filepath.Join(paths.Home(), "logs", "elastic-agent-json.log"), + filepath.Join(paths.Home(), "logs", "elastic-agent-json.log*"), filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), + filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), }, "index": "logs-elastic_agent-default", "processors": []map[string]interface{}{ @@ -531,7 +534,10 @@ func (o *Operator) getLogFilePaths() map[string][]string { for _, a := range o.apps { logPath := a.Monitor().LogPath(a.Spec(), o.pipelineID) if logPath != "" { - paths[strings.ReplaceAll(a.Name(), "-", "_")] = append(paths[a.Name()], logPath) + paths[strings.ReplaceAll(a.Name(), "-", "_")] = []string{ + logPath, + fmt.Sprintf("%s*", logPath), + } } } @@ -547,7 +553,19 @@ func (o *Operator) getMetricbeatEndpoints() map[string][]string { for _, a := range o.apps { metricEndpoint := a.Monitor().MetricsPathPrefixed(a.Spec(), o.pipelineID) if metricEndpoint != "" { - endpoints[strings.ReplaceAll(a.Name(), "-", "_")] = append(endpoints[a.Name()], metricEndpoint) + safeName := strings.ReplaceAll(a.Name(), "-", "_") + // prevent duplicates + var found bool + for _, ep := range endpoints[safeName] { + if ep == metricEndpoint { + found = true + break + } + } + + if !found { + endpoints[safeName] = append(endpoints[safeName], metricEndpoint) + } } } diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go b/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go index f79eca617f8..6376492c2f2 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go @@ -82,7 +82,7 @@ func (o *retryableOperations) runOnce(application Application) func(context.Cont o.logger.Debugf("running operation '%s' of the block '%s'", op.Name(), o.Name()) if err := op.Run(ctx, application); err != nil { - o.logger.Errorf("operation %s failed", op.Name()) + o.logger.Errorf("operation %s failed, err: %v", op.Name(), err) return err } } diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go index 794dbdb5805..a7a3547fa88 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go @@ -100,6 +100,16 @@ func TestConfigurableRun(t *testing.T) { return nil }) + // wait to finish configuring + waitFor(t, func() error { + items := operator.State() + item, ok := items[p.ID()] + if ok && item.Status == state.Configuring { + return fmt.Errorf("process still configuring") + } + return nil + }) + items := operator.State() item0, ok := items[p.ID()] if !ok || item0.Status != state.Running { @@ -379,8 +389,7 @@ func TestConfigurableStartStop(t *testing.T) { } func TestConfigurableService(t *testing.T) { - t.Skipf("flaky see https://github.com/elastic/beats/issues/20836") - + t.Skip("Flaky test: https://github.com/elastic/beats/issues/23607") p := getProgram("serviceable", "1.0") operator := getTestOperator(t, downloadPath, installPath, p) @@ -427,6 +436,16 @@ func TestConfigurableService(t *testing.T) { return nil }) + // wait to finish configuring + waitFor(t, func() error { + items := operator.State() + item, ok := items[p.ID()] + if ok && item.Status == state.Configuring { + return fmt.Errorf("process still configuring") + } + return nil + }) + items := operator.State() item0, ok := items[p.ID()] if !ok || item0.Status != state.Running { diff --git a/x-pack/elastic-agent/pkg/agent/program/program.go b/x-pack/elastic-agent/pkg/agent/program/program.go index f3f17d06b9d..87d46bf07a0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program.go +++ b/x-pack/elastic-agent/pkg/agent/program/program.go @@ -69,6 +69,21 @@ func detectPrograms(agentInfo transpiler.AgentInfo, singleConfig *transpiler.AST programs := make([]Program, 0) for _, spec := range Supported { specificAST := singleConfig.Clone() + if len(spec.Constraints) > 0 { + constraints, err := eql.New(spec.Constraints) + if err != nil { + return nil, err + } + ok, err := constraints.Eval(specificAST) + if err != nil { + return nil, err + } + + if !ok { + continue + } + } + err := spec.Rules.Apply(agentInfo, specificAST) if err != nil { return nil, err diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 119574ebb6f..262c3e5cc8c 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -423,6 +423,9 @@ func TestConfiguration(t *testing.T) { "endpoint_unknown_output": { expected: 0, }, + "endpoint_arm": { + expected: 0, + }, } for name, test := range testcases { diff --git a/x-pack/elastic-agent/pkg/agent/program/spec.go b/x-pack/elastic-agent/pkg/agent/program/spec.go index 2ae1d698dba..f7d81b74a7e 100644 --- a/x-pack/elastic-agent/pkg/agent/program/spec.go +++ b/x-pack/elastic-agent/pkg/agent/program/spec.go @@ -39,6 +39,7 @@ type Spec struct { PostInstallSteps *transpiler.StepList `yaml:"post_install"` PreUninstallSteps *transpiler.StepList `yaml:"pre_uninstall"` When string `yaml:"when"` + Constraints string `yaml:"constraints"` } // ReadSpecs reads all the specs that match the provided globbing path. diff --git a/x-pack/elastic-agent/pkg/agent/program/spec_test.go b/x-pack/elastic-agent/pkg/agent/program/spec_test.go index df1fb082f7a..634f0bc70a1 100644 --- a/x-pack/elastic-agent/pkg/agent/program/spec_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/spec_test.go @@ -54,7 +54,8 @@ func TestSerialization(t *testing.T) { PreUninstallSteps: transpiler.NewStepList( transpiler.ExecFile(30, "app", "uninstall", "--force"), ), - When: "1 == 1", + When: "1 == 1", + Constraints: "2 == 2", } yml := `name: hello cmd: hellocmd @@ -118,6 +119,7 @@ pre_uninstall: - --force timeout: 30 when: 1 == 1 +constraints: 2 == 2 ` t.Run("serialization", func(t *testing.T) { b, err := yaml.Marshal(spec) diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 9e615b9c271..b49de1bce6f 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -23,7 +23,7 @@ func init() { // spec/heartbeat.yml // spec/metricbeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzMWUl3qz6W3/fH+G+rBxBxuuhzamFIM9khz+QFCe2QZAO2wPwDHqBPf/c+EhiDnbyhXtc7tXKMhYare3/Dzf/8UZVr+h9xmf9btX4/rt//vcn5H//1B8mtGn/dJ6vQ8Jehz2mBOU3KLYGrR9e2TuRVbTHyAEbuIkKeEkOcRtqHvxW03Sewcevg1a1c06sjOEsxCGsMZ8oyDw8R9CoMVzpzPBXLMZ+NVY/YftPXpnqKoP++hLjCMFTc7JS4mWrJz3yy/gHblhKFesscj0dQbe/X85hZeCqxw/Yl2SeuqSQ45w9I8xWahyn5uk/WmrJwzXnt2rgkTsAp10EEzypGz49mNk9cc564js+JHW6ZrTcvmVGSwlCZ87yQv5nzJAbh7CUzlDUy+PUdnBIn5LTdD+O6dawTWQ17qZgdNt3exO9Gi+GZR1pwpMVofXOeLF/v1+3m01XmGCqbl1psh4eXzKgwnBXMTvaeU/fv+Pridf4X92meRHC2c+00pUrN16/Jbg369x2lck3GiW21zOZbCsKU5v7ea3aLP/61S6R1wcp9VtQ3aRTA2Y7aekmKVfIGwi1DXsmc3SIC6u4lMzjJgxMB/MBMtcXQV2nOlfWqTGkRlDi3tuxpn+DrHDW2Q2AWMi3LCLw9uk+R9vKULIitF0gzUman3XXaQUoLVpLtPnEz/TmGXhMhb7ZUwipCvhLD5+Nob0eqBSmz345iniUID9gxjrFIvdf9QTxz5ZznkhThw0s2z5ZAPzFTty4hWSqjdzRfiVDAl+B8xI0+OqPy5zIXz9yFaxpaDGc7orFWzLdqS4osoyGANRFUkiDnFUY+Rf992a/8e1gDWeeW2ZaCwzOVZ7fOH64TgZRHoN7EcCbGV+Rpv1i+Gnxth1sEcEnstz6VjFOEgr3Yyzje9HpnWT8upTlrL2m5fJ1nLA+bGOKZ2z9jNq8x1FVxd8/tfEFtvWWWmM9XIniuXpJ97drhA4b+RqQ5/tqXgSPuL3l0Te/jvHh1u3G21WBtKIHaNb1hbne0r+WrqlJbxDPgN88bjPwjQ94Wo+dsNM8n607GH9Y5P310Vn87P5mF0eBQV0nODyKniH16NDMlwSjlkarnMTzzS+yobSnx0z5x81HuIJ9HWtjEKBji2UPp4lru84zB2d2Z7/cywJuEsLV1gUPl0XVq3eyffwxtI7jZ7pMYzk4MBW3/2ztGu0fXCWbUflt8DGnKuG4XMgY9pCHtZuy3INSu+fqrxIA0ys8cz/uz5lbFYDjKQ0OhRcjlmS7zyZwLjkO8QVhh6CtE89qXzCBYzIdWhwj6W4z8FgHrFIe6OFvl2h3FLPO6jHLrEIXKNEeH361TvJI1VUdoflNLRk7skLN+z7QIqxF11K7jcQJ1gLs1J5De00YaAZ9Tzd9EyCgRqPl6NZy3wVA9sjzcyLEDTVxjFgOeCRqd3PP2PuaTO+tiMnyf3vuYUkb7QLikgB9Jsl8wkHKBucQOD1gL9gsz+M9uzuCGZgReMSU2Bc3wA3VChWpK6T49JM+mkZJ8lcS21b6CcCbmIFqoiDGb11PigQETWwytJgJJsVjt/3ahok3G12Qd31GRgBro8QitLvQjyzHKw5TNyw7WMoMMiqLwOXPC0zLnFXmdcZJbGbHD3Rco0tfnE/VxGVsEnCCjkvA/Uhw4tyoK3rKlOc+Wb90ngdYhgowTGB6YOasJCPgXlNTUtrZxo3apY35LFX1TQVUEsCKGs2KZnznLw+oLDHhUhIXLlRvlJmIStEtJB2GGoaV8FzoymRp/inJCgB+wHT5cUpA5/CTiLWiJnmRplCQvBZRsqBY0GFo10gTVydQ+Dilp6wcE/CPJcRVDX+mgQFB9sIkgVoSq7FJXws6ja5+PWHuW0EKgdbqF1RvIahg8T+ApAvppHeopsc8bZusbYvOWPY0VmaGQdp9c9kxP4xK72+uBAP00LmGM0i1GhiJzquhVJXqWdx/DlfwcYE3es3eiuS6hSECUuKebvSpE1asY+cq03LmC5b2MYlo8/73nuMY8D3OieT2kCmqUddTfFW4IUB5duy/d04WC/np9pg1nXvQUqFAh6azuDAiIfavtZ/d2u98YBVyo8tvnkzVPn0LxlFacsfLv6SO3DhSchQycQPFlXzKvV+PYqSl1jCukDs/PR9xLK/n3ON4yLzAXkrOXPrJOxuu5piHq9cBMvWV2ICUs1YJdDB9u1gmBxAEt2FKxP9s/fTKPip35o+uEOzqf7qWTu8ExArU4R4JtfRuDsLmZpyKAHmke7mLkbyg4H5mQtiKn5LPn+/M3ertGvnjv0XX8mXjnEocfoS6GfI7AB1Tznfemru9aPzQPa6JhLin066TGOyljC/lvDfi0zGcpgWErsBj/BOXerN+5UOQLCSDyUvCNgpG3uZUvV2ni3tXUvYM0bl2ocI4aBX5FtHDHgKVEILliBypVmr/VXd4FewbdEa6cjwwGOdGEBPVm1/n8IymCNIYzTq81siPAfx9wWMgF4J+WyFCjwlej67g9c4ITGtmo67ypwhzjTwr0w/VZneK8Tq/fJ7lSUxSM3p9xZuOKaHTYA2mfgQ8tFdtcGUsrbPOrbHGeR3/7Crb5YfT9Nj+VSJsP8zMYnK5jw0OMrrFlgEt7+MtS2h64/FM5LTl+NeVTqR+KQNi7XGD8WlMk1kq++CGsHnH+T8nEq5Ya7qGXaKN75Wsk5TY3C7bH8OFxKvuuay/zX5eAS3Ne9HhULCVmsPcI4vfolVauyYQeEha4jU1amsnfBrmYruP3+gO9+GqHKS2CLja5X2PBTZNnnQ11rWrQdBSECkPzQwzP9ff032Uss8Oa2pJ3DoMeeFLzCJ7bX++KqSnJrQJDVXDOeH5pl6ZjBW+xkuT0QCS3nHRshxmDNEM3LRmpmZ3n4yQef3c3zd8wwJXY0hsMGV878yv/Xvhgwr1GI+4HFf5M4BlGQRlpz8dlVg24/jmefsOGfgeHP63VD+zoTc3+Azp603UER1AxT/72oQW7YM7mdZd8yeYn17YO2DT2EfKXGO32nlMfGQrEGF3UEYbnlGoirj6PkLeNTVlDDYZBSRtaib16oMNtrxH1KLS4L7hu0hXM1/V7Rj8orq8wVGjOt7356tvLKmeOV0agN2n3LeQWo0Cl5qwktvK9YrmMFSLzRGxLwd8zcTfFQqC+w1/VhyUSPryqe2/8LRN3nR8FDYM3hs/WCyzMUjOrJBk/qTsMPRU3HhNgwmyeR50IlwVFG73GKGhi6PcFZhypFkx6eV1SdKJk0kub9JfUI3Zk7+GATSnMhGE4rKE69I6ESRDxxmj1KICGgEAW8zJfHanGWwFSy4LXxJwJEXgxLYtrT+Pjgh+TVQxnO4ySCylKQfOSGZczth0B8UOcy97NpXW/oY53jEDYUqAPxUPAbBMB/YDzc9mZWH6gIGyYpae4CAbRMpjRPt96g9CI3CFw6H3mNNfre5MQHK/P/OFfCd0+1ZQ+3fQtPzA+n5iNLdGMGQJWRaxPTF239nXNETjcn312JINg6YT/2vY5dVaSnAbD1Mi6KHuzOeRq18CYmMcMrW72qgVHBM4l1VbTPtXFlI3uaGIwf+ocwx1mGGIJZr/ZON4JdKSxktnphuZhgVE6NBk+EOUdKWUP70vQ45j2vPumAPy9ovEXTXH4OUl/yyQ7nqjx9eJJX33pGjd/WWZVeR+jnkjFGk/7xBv3ljsDd4igyqemqxe2k7HXhofAbwbP/CpI1TQG4SZCXhPd9lv7HBlwAgxCdpQrlz37/ErMP2IuR+/9jJm96Xn/XgMsv7fjXvDvMtE35v//2fBMNYZ49za3cOEdx7kgNYrQHPfcMPDkz5inybzXu+0F2Kj3PhJ1HxuoyVkOl3z/JRPVGadB6P2oiSpjult/5KLebGsbg1CZCD1HCKqaM/tW6NE66Bb6jtATY+7GflPoSZXaqFanVn9I6ElHuXx7k5/fEXrTsZ8KPfaZ0JMODqNPndSvOZYbgPzMrdD+rsx+3WuHX8Rq1nX6879e/nn4j3A1/xTuRSb2//7L/wUAAP//S63auA==") + unpacked := packer.MustUnpack("eJzMWVt3qziWfp+fUa89Fy5xupm1+sGQQoAdckxOJKE3JNmALTAd4wvMmv8+S4Ax4OTkVNdUzTw5xkLakvb+Ljv/9cuhWLP/iIrs3w7r99P6/d+rTPzyn7/QzC7J9328gqa/hL5gOREsLrYUrR5dYJ/pq1oT7GkEu4sQe0qESBLqH/6Ws3ofo8otg1f34FpeGaJZQjRYEjRTlhk8hsg7ELQyuOOppBnz2Vj1RMCbsbbUc4j89yUiB4Kg4qbn2E1Vu/nMRusfCbCVEBo1dzwRIrW+X8/jVu6pFMD6Jd7HrqXEJBMPWPcVlsGEft/Ha11ZuNa8dAEpqBMIJgwtRBeV4OdHK53HrjWPXccXFMAtB0b1kpoFzU2VO8+L5jdrHkcanL2kprLGpri9QxLqQMHqfT+uXcc+01Ufy4EDWLWxyd/NmqCLCPXgxPLB+tY8Xr7er9vOZ6jcMVU+L/QIwONLah4ImuUcxHvPKbt3fGPxOv+L+zSPQzTbuSBJmFKK9Wu8W2vd+45ycC0uKLBrDsSWaTBhmb/3qt3il39tE2md82Kf5uUkjQI02zFgFDRfxW8a3HLsFdzZLUJN3b2kpqBZcKaaOHJLrQnyVZYJZb0qEpYHBcnsLX/ax+Q2R0kA1Ky8Scsi1N4e3adQf3mKFxQYOdbNhIOkvU4QJCznBd3uYzc1niPkVSH2ZksFHkLsKxF6Pg1iOzE9SDh4O8l5lho8Esc8RTL1XvdH+cxt5rwUNIcPL+k8XWrGmVuGfT2SpTJ4R/eVEAdiqV1OpDIGe1T+sczkM3fhWqYeodmO6ryW863qgmHbrKjGqxApcZCJA8E+w79e423+7tfA9qXmwFYIvLBm7/blw3VCLRGhVm4iNJPjD/Rpv1i+mmIN4BZrpKDgrUsl8xziYC9jGZ43u91Z2o1LWMbra1ouX+cpz2AVITJzu2cciJIgQ5V391zPFwwYNbflfL4SosvhJd6XLoAPBPkbmebke1cGjry/+NG1vI/z4tVtxwG7InpfAqVref3c7iCu5auqMiDPMxCT5xXB/oljb0vwczqY55N1R+OP60ycP9qrv52frdysCDRUmomjzCkKzo9WqsQEJyJUjSxCF3E9OwZsJXrax242yB3si1CHVYSD/jw7KF3cyn2ecjS72/N9LD28NRC2tq9wqDy6TmlY3fOPoW0AN9t9HKHZmeOg7n57J3j36DrBjIG3xceQpgzrdtGcQQdpWJ+M/RGEglKsvzcYkITZRZB5t9fMPnAEB3loKiyHotnTdb4m54JTf94aPBDkK1T36pfUpETOh1fHEPlbgv0aa/Y5gobc28EFLcUss7IIM/sYQmWco/3v9jlaNTVVhng+qSUzowAK3sXMcngYUEfpOp6gyNBIu+YI0jvaSELNF0z3NyE2C6yVYr3q91sRpJ54BjfN2J4mbmcWaSKVNDq65+39mY/urD2T/vv43oeUMogDk4Jp4kTj/YJriZCYSwE8Ej3YL6zgr+2cwYRmJF5xJbIkzYgjc6DCdKVwnx7iZ8tMaLaKI2DXrxqcyTmoDhU5ZvN6jj2tx8SaILsKtThfrIot1WYyvxNZR1J6eE5ZcTRTIqSKZSbHJYZr/Wq4Fk9Ypmi+xXr62qRiTdfRHX1JeEKeCPHqSllNCYcZTPi8aKEwNWmvQnJfcAeel5k40NeZoJmdUgB335BMeV+MFMt1bB4Iis1DQxkDlUIy+8C0t3RpzdPlW/tJkX0MERcUwSO3ZiXVAvENxyUD9jaq1DbdrB8pqR+qrgPVeB6hWb7MLoJn8PANBSLMYe4KZaL25JkE9bKhEJgSZCtfwk3apNM/ZAliTRwJgA/XtOWOOMvzllTGzk05FTQrJPxsmB5UBNkl1iU9NuVw6tMYGEes+SeakUOEfKWFDykPgk2IiCKVaJvuDVQ9uuByIvpzA0cU2ecpFE9gruLoMoK0UDPOa2gkFFw2HBgbCkTNn4YqzlRovY+vMbPzsCzvYj1SzTgPy57gZEuwqTQ5lXdKFD83dx+hVfPZQ2Fzz96ZZUYDXxLW5D1NYlWoahwi7CtjiBAKae5lcKb58z+7j9uZZzCjutfBsKTTpo66uyIV1ZRHF3Tlfr7S1t9uz/R+z4uONhUmZaDd7gFrMm61/uzepvFGOBBSyU+fj9Y8fwrfYypyhm6ho5zMPjLtIqXjCL6vcTV5vRqenZowx7zBcP/8ciKdHGv+Hp53kxdESJnayaWmTobruZYp6/XILaPmIGhkL9ODXYQeJutArcEBPdgyGR/wz5/MoxJn/ug6cMfm41haiRycQq2U+4gJMLaRBqvJPAeqsRPL4C7C/oZplxOXcljmVPPs+X7/lVGvsS/fe3QdfybfuZ7Dz9Adx77A2gf09MV7Y6d4qx+WwZLqRDS0+31U4638AdIy2D0+LbNZQhGsJRaT30DTk/Vb54p9KRtkXkq+UQj2NlPJc5Mz7l1N3btOc+pcpdvUWaVuKZB3zRMO/P3kt/r5dv7JOocVeVXb/ACJGt7qoJ0DkBOXki/fDTHoyIFd0KzPj9IFXe7c3s9DfV4yB6ZMhzc5DhKFO+ZmKCN7WeIECgNFTbWH2zPNziLt1/77oEZKF6gX7tzepxlUSHY58dv+Ts91qIZY5uZqmAODXFUmNSW/i3q0jhNs2S33ZA31Y6VkX99+k/ZP5vqfK42BOEZZI42lJjg2Ejkjp5fU/OumlbFbqpt9npLcO0n8nPBiww1koI2Ge/hQbt9iGein29lzHJz5UHZqcMak9cjePpGU/drHa2yb1138LZ2fXWAfiWXuQ+wvCd7tPafs5g+MpTXPCbokTA+KUPdFiL1tZLGDa/GKoKBgFTvIPXpamZCsTLxKyk2pGaSlH3c8knX0Xn6gGV8BTFgetJoo80si+Wn0rLWvrn3odR3ToMLx/BihS/mVBryO5QCWDDTcc+xz4EnNQnSpf383TU1oZucEqZJ3hvM3uTQeK7mLFzRjR9rwy9kgAKYcsRRPWjnNvTvPp9F5/NNdOH/DNaFEtlERxMXamd84+MoJI/41K3k/OPdnNA/2BMsceD4t00OP7Z9j6g9q9AssvvFFf6/9e9Na7bFCavM/pBM4XuerOvu/qq1sXb6n7IPi+o6gwjKx7QxY15ZWBXe8ItQ6o3bfeq4JDlRmzQoKlK+K5TpWCs0zBbZCvjJyk2KhyNiR7+rDEkv/fig7T/0jI3ebHwcVRxPTB4ycSMNUzQ6NEHxSdwR5Kqk8LsGEA5GFrRBvCopVRklwUEXI7wrMPDE9GPUAO1JshMmoBzfqS6kn4jQ9iyOxGnEmTcNxjdS+5yTJQJ43watHCTRUC5piXmarkyRHCVLLXJTUmkkheDUui1sv5OOCHxrACM12BMdXcd6ImpfUvO6xnhDbteW/YY53CjVYM83oi4dqs40UJSS7FK2RFUemwaohxDzoRUlvSLt860xCJXOHor5nmrHMKO+NQnC6PfP7f0G0caoJe5r0Oz8wP58YjoacsWYfqP2JsWvXvq05AIf7vc9OVJ8Pha1YA18wZ9WQU2+aqqYuis5w9rnaNjFGBjLFq0msenDC2qVg+mrc37oas8EdjUzmb9pHf4cpQaQBsz/ZPN6JdKzzgoNkwzKYE5z0jYYPhHlLSunD+1LrcEx/3v2wB/t7+7ZTIvqgd9sR0P+CMYafk/SPjLLjyRpfL56M1be2efOXZXoo7s+oI1K5xtM+9oY96VbAH0OkirHx6hoRo7E3YSvxm6OLGJiHJNLgJsReFU77tF2O9DihQWUUV5Mr15h9cSPmnzGYg/d+i6GdGII/1wQ332syNlF/ipGeNABGHNJwbd5wSSbvWHKMxIimSfZTDaqxxpDvTnPrapboUKNIzXHPDT1P/pbe/GjenzRPVs73BD08jvvto70cr/n+e/rvyyaem9BzLf4eIvIevjZ/H6jGpfaoI4sVVvz3v1+FXhGx3fojF/UG7G2kQWUk9BwpqErBwVTosTJoF/pC6Mkxd2N/KPQalVqpdqtWf0ro5VIJL9/ems8vhN547KdCj38m9BoHR/CnTur3OZYJQH7mVlh3V1a37q3LL89q1nb7s79d/+n4R7ia/xfupUns//6X/wkAAP//iTjyjQ==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_arm.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_arm.yml new file mode 100644 index 00000000000..30b760d9c97 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_arm.yml @@ -0,0 +1,115 @@ +revision: 5 +name: Endpoint Host +fleet: + agent: + id: fleet-agent-id + host: + id: host-agent-id + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + kibana: + protocol: https + host: localhost:5601 + timeout: 30s + +outputs: + default: + type: elasticsearch + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true +runtime: + arch: arm64 diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml index 9eee9b3a01d..728b4813a4e 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/endpoint_basic.yml @@ -111,3 +111,5 @@ inputs: process: true file: true network: true +runtime: + arch: x86_64 diff --git a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go index 61ba348f760..34bb76e551f 100644 --- a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go +++ b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go @@ -47,6 +47,10 @@ func (s *StateResolver) Resolve( s.l.Infof("New State ID is %s", newStateID) s.l.Infof("Converging state requires execution of %d step(s)", len(steps)) + for i, step := range steps { + // more detailed debug log + s.l.Debugf("step %d: %s", i, step.String()) + } // Allow the operator to ack the should state when applying the steps is done correctly. ack := func() { diff --git a/x-pack/elastic-agent/pkg/agent/storage/storage.go b/x-pack/elastic-agent/pkg/agent/storage/storage.go index 2435311486a..2ff2d7250f1 100644 --- a/x-pack/elastic-agent/pkg/agent/storage/storage.go +++ b/x-pack/elastic-agent/pkg/agent/storage/storage.go @@ -153,6 +153,23 @@ func NewDiskStore(target string) *DiskStore { return &DiskStore{target: target} } +// Exists check if the store file exists on the disk +func (d *DiskStore) Exists() (bool, error) { + _, err := os.Stat(d.target) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// Delete deletes the store file on the disk +func (d *DiskStore) Delete() error { + return os.Remove(d.target) +} + // Save accepts a persistedConfig and saved it to a target file, to do so we will // make a temporary files if the write is successful we are replacing the target file with the // original content. diff --git a/x-pack/elastic-agent/pkg/artifact/artifact.go b/x-pack/elastic-agent/pkg/artifact/artifact.go index 7f4001e77e0..5f8a099ed6a 100644 --- a/x-pack/elastic-agent/pkg/artifact/artifact.go +++ b/x-pack/elastic-agent/pkg/artifact/artifact.go @@ -13,12 +13,13 @@ import ( ) var packageArchMap = map[string]string{ - "linux-binary-32": "linux-x86.tar.gz", - "linux-binary-64": "linux-x86_64.tar.gz", - "windows-binary-32": "windows-x86.zip", - "windows-binary-64": "windows-x86_64.zip", - "darwin-binary-32": "darwin-x86_64.tar.gz", - "darwin-binary-64": "darwin-x86_64.tar.gz", + "linux-binary-32": "linux-x86.tar.gz", + "linux-binary-64": "linux-x86_64.tar.gz", + "linux-binary-arm64": "linux-arm64.tar.gz", + "windows-binary-32": "windows-x86.zip", + "windows-binary-64": "windows-x86_64.zip", + "darwin-binary-32": "darwin-x86_64.tar.gz", + "darwin-binary-64": "darwin-x86_64.tar.gz", } // GetArtifactName constructs a path to a downloaded artifact diff --git a/x-pack/elastic-agent/pkg/artifact/config.go b/x-pack/elastic-agent/pkg/artifact/config.go index 81ed3f856fc..78a0e62ad8a 100644 --- a/x-pack/elastic-agent/pkg/artifact/config.go +++ b/x-pack/elastic-agent/pkg/artifact/config.go @@ -77,7 +77,9 @@ func (c *Config) Arch() string { } arch := "32" - if strings.Contains(runtime.GOARCH, "64") { + if strings.Contains(runtime.GOARCH, "arm64") { + arch = "arm64" + } else if strings.Contains(runtime.GOARCH, "64") { arch = "64" } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go index e2097a9297f..a329b9b2f8e 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go @@ -137,6 +137,7 @@ func getTestCases() []testCase { return []testCase{ {"linux", "32"}, {"linux", "64"}, + {"linux", "arm64"}, {"darwin", "32"}, {"darwin", "64"}, {"windows", "32"}, @@ -164,6 +165,7 @@ func getElasticCoClient() http.Client { fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "i686.rpm"): struct{}{}, fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "x86_64.rpm"): struct{}{}, fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "linux-x86.tar.gz"): struct{}{}, + fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "linux-arm64.tar.gz"): struct{}{}, fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "linux-x86_64.tar.gz"): struct{}{}, fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "windows-x86.zip"): struct{}{}, fmt.Sprintf("%s-%s-%s", beatSpec.Cmd, version, "windows-x86_64.zip"): struct{}{}, diff --git a/x-pack/elastic-agent/pkg/core/authority/ca.go b/x-pack/elastic-agent/pkg/core/authority/ca.go index 2ddeae70410..f558e8a9eb3 100644 --- a/x-pack/elastic-agent/pkg/core/authority/ca.go +++ b/x-pack/elastic-agent/pkg/core/authority/ca.go @@ -38,10 +38,10 @@ type Pair struct { // NewCA creates a new certificate authority capable of generating child certificates func NewCA() (*CertificateAuthority, error) { ca := &x509.Certificate{ + DNSNames: []string{"localhost"}, SerialNumber: big.NewInt(1653), Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, - CommonName: "localhost", }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), @@ -101,13 +101,13 @@ func (c *CertificateAuthority) GeneratePair() (*Pair, error) { } // GeneratePairWithName generates child certificate with provided name as the common name. -func (c *CertificateAuthority) GeneratePairWithName(commonName string) (*Pair, error) { +func (c *CertificateAuthority) GeneratePairWithName(name string) (*Pair, error) { // Prepare certificate certTemplate := &x509.Certificate{ SerialNumber: big.NewInt(1658), + DNSNames: []string{name}, Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, - CommonName: commonName, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 19298aef69c..5944afa4f0c 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -144,7 +144,12 @@ func (b *Monitor) Cleanup(spec program.Spec, pipelineID string) error { // Prepare executes steps in order for monitoring to work correctly func (b *Monitor) Prepare(spec program.Spec, pipelineID string, uid, gid int) error { - takeOwnership := b.ownLoggingPath(spec) + if !b.ownLoggingPath(spec) { + // spec file passes a log path; so its up to the application to ensure the + // path exists and the write permissions are set so Elastic Agent can read it + return nil + } + drops := []string{b.generateLoggingPath(spec, pipelineID)} if drop := b.monitoringDrop(spec, pipelineID); drop != "" { drops = append(drops, drop) @@ -167,10 +172,8 @@ func (b *Monitor) Prepare(spec program.Spec, pipelineID string, uid, gid int) er } } - if takeOwnership { - if err := changeOwner(drop, uid, gid); err != nil { - return err - } + if err := changeOwner(drop, uid, gid); err != nil { + return err } } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/status.go b/x-pack/elastic-agent/pkg/core/plugin/process/status.go index eac8f2bc53e..473ae9a70c7 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/status.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/status.go @@ -35,9 +35,6 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St return } - // it was a crash, cleanup anything required - go a.cleanUp() - // kill the process if a.state.ProcessInfo != nil { _ = a.state.ProcessInfo.Process.Kill() diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index c7eef7040d4..f0b8ac73d8a 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -37,7 +37,7 @@ const ( InitialCheckinTimeout = 5 * time.Second // CheckinMinimumTimeoutGracePeriod is additional time added to the client.CheckinMinimumTimeout // to ensure the application is checking in correctly. - CheckinMinimumTimeoutGracePeriod = 2 * time.Second + CheckinMinimumTimeoutGracePeriod = 30 * time.Second // WatchdogCheckLoop is the amount of time that the watchdog will wait between checking for // applications that have not checked in the correct amount of time. WatchdogCheckLoop = 5 * time.Second diff --git a/x-pack/elastic-agent/pkg/fleetapi/action.go b/x-pack/elastic-agent/pkg/fleetapi/action.go index 211b9199f2f..d836aa801c2 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/action.go +++ b/x-pack/elastic-agent/pkg/fleetapi/action.go @@ -21,6 +21,8 @@ const ( ActionTypePolicyChange = "POLICY_CHANGE" // ActionTypeSettings specifies change of agent settings. ActionTypeSettings = "SETTINGS" + // ActionTypeApplication specifies agent action. + ActionTypeApplication = "APP_ACTION" ) // Action base interface for all the implemented action from the fleet API. @@ -154,6 +156,16 @@ type ActionSettings struct { LogLevel string `json:"log_level"` } +// ID returns the ID of the Action. +func (a *ActionSettings) ID() string { + return a.ActionID +} + +// Type returns the type of the Action. +func (a *ActionSettings) Type() string { + return a.ActionType +} + func (a *ActionSettings) String() string { var s strings.Builder s.WriteString("action_id: ") @@ -165,25 +177,45 @@ func (a *ActionSettings) String() string { return s.String() } -// Type returns the type of the Action. -func (a *ActionSettings) Type() string { - return a.ActionType +// ActionApp is the application action request. +type ActionApp struct { + ActionID string + ActionType string + Application string + Data json.RawMessage +} + +func (a *ActionApp) String() string { + var s strings.Builder + s.WriteString("action_id: ") + s.WriteString(a.ActionID) + s.WriteString(", type: ") + s.WriteString(a.ActionType) + s.WriteString(", application: ") + s.WriteString(a.Application) + return s.String() } // ID returns the ID of the Action. -func (a *ActionSettings) ID() string { +func (a *ActionApp) ID() string { return a.ActionID } +// Type returns the type of the Action. +func (a *ActionApp) Type() string { + return a.ActionType +} + // Actions is a list of Actions to executes and allow to unmarshal heterogenous action type. type Actions []Action // UnmarshalJSON takes every raw representation of an action and try to decode them. func (a *Actions) UnmarshalJSON(data []byte) error { type r struct { - ActionType string `json:"type"` - ActionID string `json:"id"` - Data json.RawMessage `json:"data"` + ActionType string `json:"type"` + Application string `json:"application"` + ActionID string `json:"id"` + Data json.RawMessage `json:"data"` } var responses []r @@ -209,6 +241,13 @@ func (a *Actions) UnmarshalJSON(data []byte) error { "fail to decode POLICY_CHANGE action", errors.TypeConfig) } + case ActionTypeApplication: + action = &ActionApp{ + ActionID: response.ActionID, + ActionType: response.ActionType, + Application: response.Application, + Data: response.Data, + } case ActionTypeUnenroll: action = &ActionUnenroll{ ActionID: response.ActionID, diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go index 19c936ed79c..79bcb39d40b 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go @@ -22,6 +22,7 @@ const checkingPath = "/api/fleet/agents/%s/checkin" // CheckinRequest consists of multiple events reported to fleet ui. type CheckinRequest struct { Status string `json:"status"` + AckToken string `json:"ack_token,omitempty"` Events []SerializableEvent `json:"events"` Metadata *info.ECSMeta `json:"local_metadata,omitempty"` } @@ -49,7 +50,8 @@ func (e *CheckinRequest) Validate() error { // CheckinResponse is the response send back from the server which contains all the action that // need to be executed or proxy to running processes. type CheckinResponse struct { - Actions Actions `json:"actions"` + AckToken string `json:"ack_token"` + Actions Actions `json:"actions"` } // Validate validates the response send from the server. diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml index 1bc5817f5c1..a435c2f9ed4 100644 --- a/x-pack/elastic-agent/spec/endpoint.yml +++ b/x-pack/elastic-agent/spec/endpoint.yml @@ -62,3 +62,4 @@ rules: - revision when: length(${fleet}) > 0 and length(${inputs}) > 0 and hasKey(${output}, 'elasticsearch') +constraints: ${runtime.arch} != 'arm64' diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 2472f2b5220..6aacf99ccf0 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -59,7 +59,8 @@ rules: selector: inputs key: type values: - - awscloudwatch + - aws-cloudwatch + - aws-s3 - azure-eventhub - cloudfoundry - container @@ -73,7 +74,6 @@ rules: - netflow - o365audit - redis - - s3 - stdin - syslog - tcp diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index c5351be3339..31c49fb19cd 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -54,21 +54,21 @@ # Path to a JSON file containing the credentials and key used to subscribe. credentials_file: ${path.config}/my-pubsub-subscriber-credentials.json -#------------------------------ S3 input -------------------------------- +#------------------------------ AWS S3 input -------------------------------- # Beta: Config options for AWS S3 input -#- type: s3 +#- type: aws-s3 #enabled: false # AWS Credentials # If access_key_id and secret_access_key are configured, then use them to make api calls. - # If not, s3 input will load default AWS config or load with given profile name. + # If not, aws-s3 input will load default AWS config or load with given profile name. #access_key_id: '${AWS_ACCESS_KEY_ID:""}' #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' #session_token: '${AWS_SESSION_TOKEN:"”}' - #credential_profile_name: test-s3-input + #credential_profile_name: test-aws-s3-input # Queue url (required) to receive queue messages from - #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-s3-logs-queue" + #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-aws-s3-logs-queue" # The duration (in seconds) that the received messages are hidden from subsequent # retrieve requests after being retrieved by a ReceiveMessage request. diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 3ea37b3c754..fde7f734d6c 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -2,18 +2,18 @@ :libbeat-xpack-dir: ../../../../x-pack/libbeat -:type: s3 +:type: aws-s3 [id="{beatname_lc}-input-{type}"] -=== S3 input +=== AWS S3 input ++++ -S3 +AWS S3 ++++ beta[] -Use the `s3` input to retrieve logs from S3 objects that are pointed by messages +Use the `aws-s3` input to retrieve logs from S3 objects that are pointed by messages from specific SQS queues. This input can, for example, be used to receive S3 server access logs to monitor detailed records for the requests that are made to a bucket. @@ -28,13 +28,13 @@ stopped and the sqs message will be returned back to the queue. ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.ap-southeast-1.amazonaws.com/1234/test-s3-queue credential_profile_name: elastic-beats expand_event_list_from_field: Records ---- -The `s3` input supports the following configuration options plus the +The `aws-s3` input supports the following configuration options plus the <<{beatname_lc}-input-{type}-common-options>> described later. [float] @@ -74,7 +74,7 @@ can be assigned the name of the field. This setting will be able to split the messages under the group value into separate events. For example, CloudTrail logs are in JSON format and events are found under the JSON object "Records". -Note: When `expand_event_list_from_field` parameter is given in the config, s3 +Note: When `expand_event_list_from_field` parameter is given in the config, aws-s3 input will assume the logs are in JSON format and decode them as JSON. Content type will not be checked. If a file has "application/json" content-type, `expand_event_list_from_field` @@ -132,7 +132,7 @@ is 0 seconds. The maximum is 12 hours. [float] ==== `aws credentials` -In order to make AWS API calls, `s3` input requires AWS credentials.Please see +In order to make AWS API calls, `aws-s3` input requires AWS credentials.Please see <> for more details. [float] @@ -170,7 +170,7 @@ During this time, Filebeat processes and deletes the message. However, if Filebeat fails before deleting the message and your system doesn't call the DeleteMessage action for that message before the visibility timeout expires, the message becomes visible to other {beatname_uc} instances, and the message is -received again. By default, the visibility timeout is set to 5 minutes for s3 +received again. By default, the visibility timeout is set to 5 minutes for aws-s3 input in {beatname_uc}. 5 minutes is sufficient time for {beatname_uc} to read SQS messages and process related s3 log files. diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index a5592ed783e..64247b517f8 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -34,9 +34,9 @@ filebeat.inputs: interval: 1m request.url: https://api.ipify.org/?format=json processors: - - decode_json_fields - fields: [message] - target: json + - decode_json_fields: + fields: ["message"] + target: "json" ---- ["source","yaml",subs="attributes"] @@ -346,6 +346,11 @@ The URL of the HTTP API. Required. HTTP method to use when making requests. `GET` or `POST` are the options. Default: `GET`. +[float] +==== `request.encode_as` + +ContentType used for encoding the request body. If set it will force the encoding in the specified format regardless of the `Content-Type` header value, otherwise it will honor it if possible or fallback to `application/json`. By default the requests are sent with `Content-Type: application/json`. Supported values: `application/json` and `application\x-www-form-urlencoded`. `application/x-www-form-encoded` will url encode the `url.params` and set them as the body. It is not set by default. + [float] ==== `request.body` @@ -449,6 +454,11 @@ filebeat.inputs: value: '[[now (parseDuration "-1h")]]' ---- +[float] +==== `response.decode_as` + +ContentType used for decoding the response body. If set it will force the decoding in the specified format regardless of the `Content-Type` header value, otherwise it will honor it if possible or fallback to `application/json`. Supported values: `application/json, application/x-ndjson`. It is not set by default. + [[response-transforms]] [float] ==== `response.transforms` @@ -820,9 +830,9 @@ filebeat.inputs: last_requested_at: value: '[[now]]' processors: - - decode_json_fields - fields: [message] - target: json + - decode_json_fields: + fields: ["message"] + target: "json" ---- [float] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d0d78781e93..9104290b7ce 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -604,7 +604,7 @@ filebeat.modules: umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input @@ -1410,7 +1410,7 @@ filebeat.modules: # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] #-------------------------------- MySQL Module -------------------------------- #- module: mysql @@ -2711,21 +2711,21 @@ filebeat.inputs: # Path to a JSON file containing the credentials and key used to subscribe. credentials_file: ${path.config}/my-pubsub-subscriber-credentials.json -#------------------------------ S3 input -------------------------------- +#------------------------------ AWS S3 input -------------------------------- # Beta: Config options for AWS S3 input -#- type: s3 +#- type: aws-s3 #enabled: false # AWS Credentials # If access_key_id and secret_access_key are configured, then use them to make api calls. - # If not, s3 input will load default AWS config or load with given profile name. + # If not, aws-s3 input will load default AWS config or load with given profile name. #access_key_id: '${AWS_ACCESS_KEY_ID:""}' #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' #session_token: '${AWS_SESSION_TOKEN:"”}' - #credential_profile_name: test-s3-input + #credential_profile_name: test-aws-s3-input # Queue url (required) to receive queue messages from - #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-s3-logs-queue" + #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-aws-s3-logs-queue" # The duration (in seconds) that the received messages are hidden from subsequent # retrieve requests after being retrieved by a ReceiveMessage request. @@ -3207,6 +3207,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3334,6 +3340,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3522,6 +3534,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -3531,6 +3547,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3687,6 +3709,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3983,6 +4011,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -4180,6 +4214,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index ec0c0b6b70d..f760be10844 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -9,10 +9,10 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" - _ "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/aws" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/azure" diff --git a/x-pack/filebeat/input/s3/_meta/fields.yml b/x-pack/filebeat/input/awss3/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/input/s3/_meta/fields.yml rename to x-pack/filebeat/input/awss3/_meta/fields.yml diff --git a/x-pack/filebeat/input/s3/_meta/s3-input.asciidoc b/x-pack/filebeat/input/awss3/_meta/s3-input.asciidoc similarity index 100% rename from x-pack/filebeat/input/s3/_meta/s3-input.asciidoc rename to x-pack/filebeat/input/awss3/_meta/s3-input.asciidoc diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/awss3/collector.go similarity index 99% rename from x-pack/filebeat/input/s3/collector.go rename to x-pack/filebeat/input/awss3/collector.go index 6767e6732a7..806bead57b2 100644 --- a/x-pack/filebeat/input/s3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "bufio" diff --git a/x-pack/filebeat/input/s3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go similarity index 99% rename from x-pack/filebeat/input/s3/collector_test.go rename to x-pack/filebeat/input/awss3/collector_test.go index b039e2d06b9..fa613e29df8 100644 --- a/x-pack/filebeat/input/s3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "bufio" diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/awss3/config.go similarity index 99% rename from x-pack/filebeat/input/s3/config.go rename to x-pack/filebeat/input/awss3/config.go index 6dc0746ce5f..c40a493b8c7 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "fmt" diff --git a/x-pack/filebeat/input/s3/fields.go b/x-pack/filebeat/input/awss3/fields.go similarity index 73% rename from x-pack/filebeat/input/s3/fields.go rename to x-pack/filebeat/input/awss3/fields.go index 3c373aeaa11..c507150f8e4 100644 --- a/x-pack/filebeat/input/s3/fields.go +++ b/x-pack/filebeat/input/awss3/fields.go @@ -4,20 +4,20 @@ // Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. -package s3 +package awss3 import ( "github.com/elastic/beats/v7/libbeat/asset" ) func init() { - if err := asset.SetFields("filebeat", "s3", asset.ModuleFieldsPri, AssetS3); err != nil { + if err := asset.SetFields("filebeat", "awss3", asset.ModuleFieldsPri, AssetAwss3); err != nil { panic(err) } } -// AssetS3 returns asset data. -// This is the base64 encoded gzipped contents of input/s3. -func AssetS3() string { +// AssetAwss3 returns asset data. +// This is the base64 encoded gzipped contents of input/awss3. +func AssetAwss3() string { return "eJykjjGugzAQRHufYkQPjTsX/wi/+QdABg8fB4ORvSTi9pGBJlKkFNlipZ3dnTc1Ju4GWStAvAQaVFlXCnDMffKr+LgY/CgA+NMYPIPLGFKckTX8sm7SKCAx0GYadBSrcN2Z463GYuey2vqJ0pbh0AHZV5qS4BGTu7Q33FK/dibiABlZcpxekNGW5jNC/EeiJM873ZGveYHH7sZe2on71+zT6gP7GQAA//+k2GkG" } diff --git a/x-pack/filebeat/input/s3/ftest/sample1.txt b/x-pack/filebeat/input/awss3/ftest/sample1.txt similarity index 100% rename from x-pack/filebeat/input/s3/ftest/sample1.txt rename to x-pack/filebeat/input/awss3/ftest/sample1.txt diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/awss3/input.go similarity index 97% rename from x-pack/filebeat/input/s3/input.go rename to x-pack/filebeat/input/awss3/input.go index a3f19f66327..98d8c60d77b 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "context" @@ -20,7 +20,7 @@ import ( "github.com/elastic/go-concert/ctxtool" ) -const inputName = "s3" +const inputName = "aws-s3" func Plugin() v2.Plugin { return v2.Plugin{ @@ -113,6 +113,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C log.Debug("s3 service name = ", s3Servicename) log.Debug("s3 input config max_number_of_messages = ", in.config.MaxNumberOfMessages) + log.Debug("s3 input config endpoint = ", in.config.AwsConfig.Endpoint) return &s3Collector{ cancellation: ctxtool.FromCanceller(ctx.Cancelation), logger: log, diff --git a/x-pack/filebeat/input/s3/s3_integration_test.go b/x-pack/filebeat/input/awss3/s3_integration_test.go similarity index 99% rename from x-pack/filebeat/input/s3/s3_integration_test.go rename to x-pack/filebeat/input/awss3/s3_integration_test.go index cadb5da035b..f2eca42787e 100644 --- a/x-pack/filebeat/input/s3/s3_integration_test.go +++ b/x-pack/filebeat/input/awss3/s3_integration_test.go @@ -5,7 +5,7 @@ // +build integration // +build aws -package s3 +package awss3 import ( "context" diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 4779b452f1d..a3381cb42d0 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -10,11 +10,11 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" - "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) func Init(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { @@ -30,6 +30,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 http_endpoint.Plugin(), httpjson.Plugin(log, store), o365audit.Plugin(log, store), - s3.Plugin(), + awss3.Plugin(), } } diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index bddf2be0a9e..2c799c1f14f 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -59,7 +59,7 @@ func newHTTPEndpoint(config config) (*httpEndpoint, error) { return nil, err } if tlsConfigBuilder != nil { - tlsConfig = tlsConfigBuilder.BuildModuleConfig(addr) + tlsConfig = tlsConfigBuilder.BuildModuleClientConfig(addr) } return &httpEndpoint{ diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go index a76b115cfca..d2c93363c87 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_request.go @@ -79,6 +79,7 @@ type requestConfig struct { URL *urlConfig `config:"url" validate:"required"` Method string `config:"method" validate:"required"` Body *common.MapStr `config:"body"` + EncodeAs string `config:"encode_as"` Timeout *time.Duration `config:"timeout"` SSL *tlscommon.Config `config:"ssl"` Retry retryConfig `config:"retry"` @@ -116,5 +117,11 @@ func (c *requestConfig) Validate() error { return err } + if c.EncodeAs != "" { + if _, found := registeredEncoders[c.EncodeAs]; !found { + return fmt.Errorf("encoder not found for contentType: %v", c.EncodeAs) + } + } + return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go index 844a2d346bb..be265409ba8 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/config_response.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/config_response.go @@ -15,6 +15,7 @@ const ( ) type responseConfig struct { + DecodeAs string `config:"decode_as"` RequestBodyOnPagination bool `config:"request_body_on_pagination"` Transforms transformsConfig `config:"transforms"` Pagination transformsConfig `config:"pagination"` @@ -37,6 +38,11 @@ func (c *responseConfig) Validate() error { if _, err := newBasicTransformsFromConfig(c.Pagination, paginationNamespace, nil); err != nil { return err } + if c.DecodeAs != "" { + if _, found := registeredDecoders[c.DecodeAs]; !found { + return fmt.Errorf("decoder not found for contentType: %v", c.DecodeAs) + } + } return nil } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/encoding.go b/x-pack/filebeat/input/httpjson/internal/v2/encoding.go new file mode 100644 index 00000000000..f57dec837ee --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/encoding.go @@ -0,0 +1,121 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +type encoderFunc func(trReq transformable) ([]byte, error) + +type decoderFunc func(p []byte, dst *response) error + +var ( + registeredEncoders = map[string]encoderFunc{} + registeredDecoders = map[string]decoderFunc{} + defaultEncoder encoderFunc = encodeAsJSON + defaultDecoder decoderFunc = decodeAsJSON +) + +func registerEncoder(contentType string, enc encoderFunc) error { + if contentType == "" { + return errors.New("content-type can't be empty") + } + + if enc == nil { + return errors.New("encoder can't be nil") + } + + if _, found := registeredEncoders[contentType]; found { + return errors.New("already registered") + } + + registeredEncoders[contentType] = enc + + return nil +} + +func registerDecoder(contentType string, dec decoderFunc) error { + if contentType == "" { + return errors.New("content-type can't be empty") + } + + if dec == nil { + return errors.New("decoder can't be nil") + } + + if _, found := registeredDecoders[contentType]; found { + return errors.New("already registered") + } + + registeredDecoders[contentType] = dec + + return nil +} + +func encode(contentType string, trReq transformable) ([]byte, error) { + enc, found := registeredEncoders[contentType] + if !found { + return defaultEncoder(trReq) + } + return enc(trReq) +} + +func decode(contentType string, p []byte, dst *response) error { + dec, found := registeredDecoders[contentType] + if !found { + return defaultDecoder(p, dst) + } + return dec(p, dst) +} + +func registerEncoders() { + log := logp.L().Named(logName) + log.Debug(registerEncoder("application/json", encodeAsJSON)) + log.Debug(registerEncoder("application/x-www-form-urlencoded", encodeAsForm)) +} + +func registerDecoders() { + log := logp.L().Named(logName) + log.Debug(registerDecoder("application/json", decodeAsJSON)) + log.Debug(registerDecoder("application/x-ndjson", decodeAsNdjson)) +} + +func encodeAsJSON(trReq transformable) ([]byte, error) { + if len(trReq.body()) == 0 { + return nil, nil + } + return json.Marshal(trReq.body()) +} + +func decodeAsJSON(p []byte, dst *response) error { + return json.Unmarshal(p, &dst.body) +} + +func encodeAsForm(trReq transformable) ([]byte, error) { + url := trReq.url() + body := []byte(url.RawQuery) + url.RawQuery = "" + trReq.setURL(url) + return body, nil +} + +func decodeAsNdjson(p []byte, dst *response) error { + var results []interface{} + dec := json.NewDecoder(bytes.NewReader(p)) + for dec.More() { + var o interface{} + if err := dec.Decode(&o); err != nil { + return err + } + results = append(results, o) + } + dst.body = results + return nil +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go b/x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go new file mode 100644 index 00000000000..4e3ba39f56b --- /dev/null +++ b/x-pack/filebeat/input/httpjson/internal/v2/encoding_test.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package v2 + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDecodeNdjson(t *testing.T) { + tests := []struct { + body string + result string + }{ + {"{}", "[{}]"}, + {"{\"a\":\"b\"}", "[{\"a\":\"b\"}]"}, + {"{\"a\":\"b\"}\n{\"c\":\"d\"}", "[{\"a\":\"b\"},{\"c\":\"d\"}]"}, + {"{\"a\":\"b\"}\r\n{\"c\":\"d\"}", "[{\"a\":\"b\"},{\"c\":\"d\"}]"}, + {"{\"a\":\"b\"}\r\n{\"c\":\"d\"}\n", "[{\"a\":\"b\"},{\"c\":\"d\"}]"}, + {"{\"a\":\"b\"}\r\n{\"c\":\"d\"}\r\n", "[{\"a\":\"b\"},{\"c\":\"d\"}]"}, + } + for _, test := range tests { + resp := &response{} + err := decodeAsNdjson([]byte(test.body), resp) + if err != nil { + t.Fatalf("decodeAsNdjson failed: %v", err) + } + j, err := json.Marshal(resp.body) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + assert.Equal(t, test.result, string(j)) + } +} + +func TestEncodeAsForm(t *testing.T) { + tests := []struct { + params map[string]string + body string + }{ + {map[string]string{"a": "b"}, "a=b"}, + {map[string]string{"a": "b", "c": "d"}, "a=b&c=d"}, + {nil, ""}, + } + for _, test := range tests { + u, err := url.Parse("http://localhost") + if err != nil { + t.Fatalf("url parse failed: %v", err) + } + q := u.Query() + for k, v := range test.params { + q.Set(k, v) + } + u.RawQuery = q.Encode() + trReq := transformable{} + trReq.setURL(*u) + res, err := encodeAsForm(trReq) + assert.Equal(t, test.body, string(res)) + } +} diff --git a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go index bec2991a71b..bdf04b108ed 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/input_manager.go @@ -44,6 +44,8 @@ func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { registerRequestTransforms() registerResponseTransforms() registerPaginationTransforms() + registerEncoders() + registerDecoders() return multierr.Append( m.stateless.Init(grp, mode), m.cursor.Init(grp, mode), diff --git a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go index 7c0016eb193..4796742d9bc 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/pagination.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/pagination.go @@ -6,7 +6,6 @@ package v2 import ( "context" - "encoding/json" "io/ioutil" "net/http" "net/url" @@ -27,11 +26,18 @@ type pagination struct { log *logp.Logger httpClient *httpClient requestFactory *requestFactory + decoder decoderFunc } func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pagination { pagination := &pagination{httpClient: httpClient, log: log} - if config.Response == nil || len(config.Response.Pagination) == 0 { + if config.Response == nil { + return pagination + } + + pagination.decoder = registeredDecoders[config.Response.DecodeAs] + + if len(config.Response.Pagination) == 0 { return pagination } @@ -47,6 +53,7 @@ func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pag requestFactory := newPaginationRequestFactory( config.Request.Method, + config.Request.EncodeAs, *config.Request.URL.URL, body, append(rts, pts...), @@ -57,7 +64,7 @@ func newPagination(config config, httpClient *httpClient, log *logp.Logger) *pag return pagination } -func newPaginationRequestFactory(method string, url url.URL, body *common.MapStr, ts []basicTransform, authConfig *authConfig, log *logp.Logger) *requestFactory { +func newPaginationRequestFactory(method, encodeAs string, url url.URL, body *common.MapStr, ts []basicTransform, authConfig *authConfig, log *logp.Logger) *requestFactory { // config validation already checked for errors here rf := &requestFactory{ url: url, @@ -65,6 +72,7 @@ func newPaginationRequestFactory(method string, url url.URL, body *common.MapStr body: body, transforms: ts, log: log, + encoder: registeredEncoders[encodeAs], } if authConfig != nil && authConfig.Basic.isEnabled() { rf.user = authConfig.Basic.User @@ -160,7 +168,12 @@ func (iter *pageIterator) getPage() (*response, error) { r.page = iter.n if len(bodyBytes) > 0 { - if err := json.Unmarshal(bodyBytes, &r.body); err != nil { + if iter.pagination.decoder != nil { + err = iter.pagination.decoder(bodyBytes, &r) + } else { + err = decode(iter.resp.Header.Get("Content-Type"), bodyBytes, &r) + } + if err != nil { return nil, err } } diff --git a/x-pack/filebeat/input/httpjson/internal/v2/request.go b/x-pack/filebeat/input/httpjson/internal/v2/request.go index 4568e54f87a..2ef92175494 100644 --- a/x-pack/filebeat/input/httpjson/internal/v2/request.go +++ b/x-pack/filebeat/input/httpjson/internal/v2/request.go @@ -7,7 +7,6 @@ package v2 import ( "bytes" "context" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -57,9 +56,6 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro header := http.Header{} header.Set("Accept", "application/json") header.Set("User-Agent", userAgent) - if rf.method == "POST" { - header.Set("Content-Type", "application/json") - } req.setHeader(header) var err error @@ -70,6 +66,14 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro } } + if rf.method == "POST" { + header = req.header() + if header.Get("Content-Type") == "" { + header.Set("Content-Type", "application/json") + req.setHeader(header) + } + } + rf.log.Debugf("new request: %#v", req) return req, nil @@ -83,6 +87,7 @@ type requestFactory struct { user string password string log *logp.Logger + encoder encoderFunc } func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp.Logger) *requestFactory { @@ -94,6 +99,7 @@ func newRequestFactory(config *requestConfig, authConfig *authConfig, log *logp. body: config.Body, transforms: ts, log: log, + encoder: registeredEncoders[config.EncodeAs], } if authConfig != nil && authConfig.Basic.isEnabled() { rf.user = authConfig.Basic.User @@ -109,15 +115,14 @@ func (rf *requestFactory) newHTTPRequest(stdCtx context.Context, trCtx *transfor } var body []byte - if len(trReq.body()) > 0 { - switch rf.method { - case "POST": - body, err = json.Marshal(trReq.body()) - if err != nil { - return nil, err - } - default: - rf.log.Errorf("A body is set, but method is not POST. The body will be ignored.") + if rf.method == "POST" { + if rf.encoder != nil { + body, err = rf.encoder(trReq) + } else { + body, err = encode(trReq.header().Get("Content-Type"), trReq) + } + if err != nil { + return nil, err } } diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go index 29d2609e5c8..67053b4e680 100644 --- a/x-pack/filebeat/input/netflow/convert.go +++ b/x-pack/filebeat/input/netflow/convert.go @@ -65,7 +65,7 @@ func toBeatEventCommon(flow record.Record) (event beat.Event) { // ECS Fields -- event ecsEvent := common.MapStr{ - "created": flow.Timestamp, + "created": time.Now().UTC(), "kind": "event", "category": []string{"network_traffic", "network"}, "action": flow.Fields["type"], diff --git a/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go b/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go index 9045108c449..0fba76aaab6 100644 --- a/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go +++ b/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go @@ -6,6 +6,7 @@ package template import ( "fmt" + "strconv" "sync" "testing" @@ -61,8 +62,8 @@ func ValidateTemplate(t testing.TB, template *Template) bool { func AssertFieldsEquals(t testing.TB, expected []FieldTemplate, actual []FieldTemplate) (succeeded bool) { if succeeded = assert.Len(t, actual, len(expected)); succeeded { for idx := range expected { - succeeded = assert.Equal(t, expected[idx].Length, actual[idx].Length, string(idx)) && succeeded - succeeded = assert.Equal(t, expected[idx].Info, actual[idx].Info, string(idx)) && succeeded + succeeded = assert.Equal(t, expected[idx].Length, actual[idx].Length, strconv.Itoa(idx)) && succeeded + succeeded = assert.Equal(t, expected[idx].Info, actual[idx].Info, strconv.Itoa(idx)) && succeeded } } return diff --git a/x-pack/filebeat/input/netflow/netflow_test.go b/x-pack/filebeat/input/netflow/netflow_test.go index 8a073df00df..f1c138989c0 100644 --- a/x-pack/filebeat/input/netflow/netflow_test.go +++ b/x-pack/filebeat/input/netflow/netflow_test.go @@ -197,7 +197,9 @@ func getFlowsFromDat(t testing.TB, name string, testCase TestCase) TestResult { } ev := make([]beat.Event, len(flows)) for i := range flows { - ev[i] = toBeatEvent(flows[i], []string{"private"}) + flow := toBeatEvent(flows[i], []string{"private"}) + flow.Fields.Delete("event.created") + ev[i] = flow } //return TestResult{Name: name, Error: err.Error(), Events: flowsToEvents(flows)} events = append(events, ev...) @@ -242,7 +244,9 @@ func getFlowsFromPCAP(t testing.TB, name, pcapFile string) TestResult { } ev := make([]beat.Event, len(flows)) for i := range flows { - ev[i] = toBeatEvent(flows[i], []string{"private"}) + flow := toBeatEvent(flows[i], []string{"private"}) + flow.Fields.Delete("event.created") + ev[i] = flow } events = append(events, ev...) } diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json index 3bdc7c5d159..5a806ce7b05 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-extended-uniflow-template-256.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-04-18T08:16:47Z", "duration": 0, "kind": "event", "type": [ @@ -109,7 +108,6 @@ "network_traffic", "network" ], - "created": "2018-04-18T08:16:47Z", "duration": 0, "kind": "event", "type": [ diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json index 3814fa8c843..7a4aa57c803 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Barracuda-firewall.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20269000000, "kind": "event", "type": [ @@ -97,7 +96,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20269000000, "kind": "event", "type": [ @@ -178,7 +176,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20306000000, "kind": "event", "type": [ @@ -259,7 +256,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20306000000, "kind": "event", "type": [ @@ -340,7 +336,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20317000000, "kind": "event", "type": [ @@ -421,7 +416,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20317000000, "kind": "event", "type": [ @@ -502,7 +496,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20368000000, "kind": "event", "type": [ @@ -583,7 +576,6 @@ "network_traffic", "network" ], - "created": "2017-06-29T13:58:28Z", "duration": 20368000000, "kind": "event", "type": [ diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Mikrotik-RouterOS-6.39.2.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Mikrotik-RouterOS-6.39.2.golden.json index 5b7004c43f3..81b36a93d3a 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Mikrotik-RouterOS-6.39.2.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Mikrotik-RouterOS-6.39.2.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -95,7 +94,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -174,7 +172,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -253,7 +250,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -332,7 +328,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -411,7 +406,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -490,7 +484,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -569,7 +562,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -648,7 +640,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -727,7 +718,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -806,7 +796,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -885,7 +874,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -964,7 +952,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1043,7 +1030,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1122,7 +1108,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1201,7 +1186,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1280,7 +1264,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1359,7 +1342,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1438,7 +1420,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1517,7 +1498,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1596,7 +1576,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1675,7 +1654,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1754,7 +1732,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1833,7 +1810,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1912,7 +1888,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -1991,7 +1966,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2070,7 +2044,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2149,7 +2122,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2226,7 +2198,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2293,7 +2264,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2360,7 +2330,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2427,7 +2396,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2494,7 +2462,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2561,7 +2528,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2628,7 +2594,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2695,7 +2660,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2762,7 +2726,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2829,7 +2792,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2896,7 +2858,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -2963,7 +2924,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3030,7 +2990,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3097,7 +3056,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3164,7 +3122,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3231,7 +3188,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3298,7 +3254,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" @@ -3365,7 +3320,6 @@ "network_traffic", "network" ], - "created": "2017-07-19T16:18:08Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Netscaler-with-variable-length-fields.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Netscaler-with-variable-length-fields.golden.json index d1ccaac0791..ebfee37989b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Netscaler-with-variable-length-fields.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Netscaler-with-variable-length-fields.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" @@ -118,7 +117,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" @@ -208,7 +206,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Nokia-BRAS.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Nokia-BRAS.golden.json index b77d79ff35c..54a8240bcd5 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Nokia-BRAS.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Nokia-BRAS.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-12-14T07:23:45Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-OpenBSD-pflow.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-OpenBSD-pflow.golden.json index 41fb9b9c48d..09b25e1256b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-OpenBSD-pflow.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-OpenBSD-pflow.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -91,7 +90,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -166,7 +164,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -241,7 +238,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -316,7 +312,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -391,7 +386,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -466,7 +460,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -541,7 +534,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -616,7 +608,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -691,7 +682,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -766,7 +756,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -841,7 +830,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -916,7 +904,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -991,7 +978,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1066,7 +1052,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1141,7 +1126,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1216,7 +1200,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1291,7 +1274,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1366,7 +1348,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1441,7 +1422,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1516,7 +1496,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1591,7 +1570,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1666,7 +1644,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1741,7 +1718,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1816,7 +1792,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" @@ -1891,7 +1866,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:30:37Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Procera.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Procera.golden.json index 1ec8673c346..47dcb8c3008 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Procera.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-Procera.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -98,7 +97,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -180,7 +178,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -262,7 +259,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -344,7 +340,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -426,7 +421,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -508,7 +502,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" @@ -590,7 +583,6 @@ "network_traffic", "network" ], - "created": "2018-04-15T03:30:00Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-VMware-virtual-distributed-switch.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-VMware-virtual-distributed-switch.golden.json index 7f97270bb03..c52420347bc 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-VMware-virtual-distributed-switch.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-VMware-virtual-distributed-switch.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-12-22T12:17:52Z", "kind": "event", "type": [ "connection" @@ -100,7 +99,6 @@ "network_traffic", "network" ], - "created": "2016-12-22T12:17:56Z", "kind": "event", "type": [ "connection" @@ -184,7 +182,6 @@ "network_traffic", "network" ], - "created": "2016-12-22T12:17:56Z", "kind": "event", "type": [ "connection" @@ -268,7 +265,6 @@ "network_traffic", "network" ], - "created": "2016-12-22T12:26:04Z", "kind": "event", "type": [ "connection" @@ -350,7 +346,6 @@ "network_traffic", "network" ], - "created": "2016-12-22T12:26:04Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-YAF-basic-with-applabel.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-YAF-basic-with-applabel.golden.json index fa7eed00986..1a3667d0659 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-YAF-basic-with-applabel.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-YAF-basic-with-applabel.golden.json @@ -18,7 +18,6 @@ "network_traffic", "network" ], - "created": "2016-12-25T13:03:38Z", "kind": "event", "type": [ "connection" @@ -103,7 +102,6 @@ "network_traffic", "network" ], - "created": "2016-12-25T12:58:38Z", "kind": "event", "type": [ "connection" @@ -187,7 +185,6 @@ "network_traffic", "network" ], - "created": "2016-12-25T13:03:33Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-configured-with-include_flowset_id.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-configured-with-include_flowset_id.golden.json index 1eda2ee228b..f996dac4c96 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-configured-with-include_flowset_id.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-configured-with-include_flowset_id.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" @@ -118,7 +117,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" @@ -208,7 +206,6 @@ "network_traffic", "network" ], - "created": "2016-11-11T12:09:19Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-options-template-from-Juniper-MX240-JunOS-15.1-R6-S3.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-options-template-from-Juniper-MX240-JunOS-15.1-R6-S3.golden.json index d4aa929699b..d46ebb089c0 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-options-template-from-Juniper-MX240-JunOS-15.1-R6-S3.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-options-template-from-Juniper-MX240-JunOS-15.1-R6-S3.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2018-06-01T15:11:53Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-vIPtela-with-VPN-id.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-vIPtela-with-VPN-id.golden.json index f477e8c3d37..9e949af09f4 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-vIPtela-with-VPN-id.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX-vIPtela-with-VPN-id.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-11-21T14:32:15Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX.golden.json index 1fdb6707c2e..cbab38ddc97 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/IPFIX.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/IPFIX.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event" }, "netflow": { @@ -55,7 +54,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -134,7 +132,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -213,7 +210,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -292,7 +288,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -371,7 +366,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -450,7 +444,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -529,7 +522,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:26Z", "kind": "event", "type": [ "connection" @@ -608,7 +600,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:28Z", "kind": "event", "type": [ "connection" @@ -687,7 +678,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:28Z", "kind": "event", "type": [ "connection" @@ -766,7 +756,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:28Z", "kind": "event", "type": [ "connection" @@ -845,7 +834,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:28Z", "kind": "event", "type": [ "connection" @@ -924,7 +912,6 @@ "network_traffic", "network" ], - "created": "2015-05-13T11:20:28Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json index ad5bbec160f..baf08b855fa 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-1941-K9-release-15.1.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -95,7 +94,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -174,7 +172,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -253,7 +250,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -332,7 +328,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -411,7 +406,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -490,7 +484,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -569,7 +562,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -648,7 +640,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -727,7 +718,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -806,7 +796,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -885,7 +874,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -964,7 +952,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1043,7 +1030,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1122,7 +1108,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1201,7 +1186,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1280,7 +1264,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1359,7 +1342,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1438,7 +1420,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1517,7 +1498,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1596,7 +1576,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1675,7 +1654,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1754,7 +1732,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1833,7 +1810,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1912,7 +1888,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -1991,7 +1966,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -2070,7 +2044,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -2149,7 +2122,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" @@ -2228,7 +2200,6 @@ "network_traffic", "network" ], - "created": "2017-10-03T17:03:39Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json index 68ca3bdb60e..5e8efb3e626 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json @@ -17,7 +17,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -99,7 +98,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -181,7 +179,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -263,7 +260,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -345,7 +341,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -427,7 +422,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -509,7 +503,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -590,7 +583,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -671,7 +663,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -753,7 +744,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -834,7 +824,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -915,7 +904,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -997,7 +985,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1078,7 +1065,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1159,7 +1145,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1241,7 +1226,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1322,7 +1306,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1403,7 +1386,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" @@ -1485,7 +1467,6 @@ "network_traffic", "network" ], - "created": "2016-07-21T13:50:37Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json index e250cce2afa..c586d597e75 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -99,7 +98,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -182,7 +180,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -265,7 +262,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -348,7 +344,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -431,7 +426,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -514,7 +508,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -597,7 +590,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -680,7 +672,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -763,7 +754,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -846,7 +836,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -929,7 +918,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -1012,7 +1000,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" @@ -1095,7 +1082,6 @@ "network_traffic", "network" ], - "created": "2015-10-09T09:47:51Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-options-template-256.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-options-template-256.golden.json index 625225c213b..1c6a2b45b9b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-options-template-256.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-options-template-256.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -48,7 +47,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -85,7 +83,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -122,7 +119,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -159,7 +155,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -196,7 +191,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -233,7 +227,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -270,7 +263,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -307,7 +299,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -344,7 +335,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -381,7 +371,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -418,7 +407,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -455,7 +443,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -492,7 +479,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -529,7 +515,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -566,7 +551,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -603,7 +587,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -640,7 +623,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { @@ -677,7 +659,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:48Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-template-260.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-template-260.golden.json index fa2e4920cce..961f90791b3 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-template-260.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR-9000-series-template-260.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.94Z", "kind": "event", @@ -104,7 +103,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 641000000, "end": "2016-12-06T10:08:54.583Z", "kind": "event", @@ -192,7 +190,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.945Z", "kind": "event", @@ -280,7 +277,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.947Z", "kind": "event", @@ -368,7 +364,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.948Z", "kind": "event", @@ -456,7 +451,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 83000000, "end": "2016-12-06T10:08:53.948Z", "kind": "event", @@ -544,7 +538,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.951Z", "kind": "event", @@ -632,7 +625,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.951Z", "kind": "event", @@ -720,7 +712,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 5418000000, "end": "2016-12-06T10:08:53.952Z", "kind": "event", @@ -808,7 +799,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 3317000000, "end": "2016-12-06T10:08:57.27Z", "kind": "event", @@ -896,7 +886,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 19894000000, "end": "2016-12-06T10:09:04.383Z", "kind": "event", @@ -984,7 +973,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.955Z", "kind": "event", @@ -1072,7 +1060,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.957Z", "kind": "event", @@ -1160,7 +1147,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 89000000, "end": "2016-12-06T10:08:53.959Z", "kind": "event", @@ -1248,7 +1234,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 17325000000, "end": "2016-12-06T10:09:05.882Z", "kind": "event", @@ -1336,7 +1321,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 2705000000, "end": "2016-12-06T10:08:56.186Z", "kind": "event", @@ -1424,7 +1408,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 361000000, "end": "2016-12-06T10:08:54.28Z", "kind": "event", @@ -1512,7 +1495,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 378000000, "end": "2016-12-06T10:08:54.037Z", "kind": "event", @@ -1600,7 +1582,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 11106000000, "end": "2016-12-06T10:09:03.759Z", "kind": "event", @@ -1688,7 +1669,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 0, "end": "2016-12-06T10:08:53.964Z", "kind": "event", @@ -1776,7 +1756,6 @@ "network_traffic", "network" ], - "created": "2016-12-06T10:09:24Z", "duration": 1587000000, "end": "2016-12-06T10:08:53.964Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR1001--X.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR1001--X.golden.json index 05ac80b86c9..7a3989121e3 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR1001--X.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASR1001--X.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -94,7 +93,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -172,7 +170,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -250,7 +247,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -328,7 +324,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -406,7 +401,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -484,7 +478,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -562,7 +555,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -640,7 +632,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -718,7 +709,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -796,7 +786,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -874,7 +863,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -952,7 +940,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1030,7 +1017,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1108,7 +1094,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1186,7 +1171,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1264,7 +1248,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1342,7 +1325,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1420,7 +1402,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1498,7 +1479,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1576,7 +1556,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1654,7 +1633,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1732,7 +1710,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1810,7 +1787,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" @@ -1888,7 +1864,6 @@ "network_traffic", "network" ], - "created": "2017-10-09T20:22:35Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json index 8f1a0a6b951..48dc5ef1c83 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-flowset-262.golden.json @@ -17,7 +17,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:10:36Z", "duration": 0, "end": "2017-02-14T11:10:19.368Z", "kind": "event", @@ -110,7 +109,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:10:36Z", "duration": 0, "end": "2017-02-14T11:10:19.368Z", "kind": "event", @@ -203,7 +201,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:10:36Z", "duration": 0, "end": "2017-02-14T11:10:19.924Z", "kind": "event", @@ -296,7 +293,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:10:36Z", "duration": 0, "end": "2017-02-14T11:10:19.996Z", "kind": "event", @@ -389,7 +385,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:10:36Z", "duration": 72000000, "end": "2017-02-14T11:10:20.008Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-options-template-260.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-options-template-260.golden.json index 8dd560a8d64..34101ef2e6c 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-options-template-260.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-NBAR-options-template-260.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -49,7 +48,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -87,7 +85,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -125,7 +122,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -163,7 +159,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -201,7 +196,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -239,7 +233,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -277,7 +270,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -315,7 +307,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -353,7 +344,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -391,7 +381,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -429,7 +418,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -467,7 +455,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -505,7 +492,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { @@ -543,7 +529,6 @@ "network_traffic", "network" ], - "created": "2017-02-14T11:09:59Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json index c55814ca02b..b4f6a597dbe 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-WLC.golden.json @@ -14,7 +14,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -80,7 +79,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -142,7 +140,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -208,7 +205,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -270,7 +266,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -334,7 +329,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -400,7 +394,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -462,7 +455,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -528,7 +520,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -590,7 +581,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -656,7 +646,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -718,7 +707,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -784,7 +772,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -846,7 +833,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -912,7 +898,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -974,7 +959,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -1040,7 +1024,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -1102,7 +1085,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" @@ -1168,7 +1150,6 @@ "network_traffic", "network" ], - "created": "2017-06-22T06:31:14Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-5.2.1.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-5.2.1.golden.json index 4fca04b8016..1ae09a26b58 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-5.2.1.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-5.2.1.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2017-07-18T05:42:14Z", "kind": "event" }, "netflow": { @@ -58,7 +57,6 @@ "network_traffic", "network" ], - "created": "2017-07-18T05:41:59Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-54x-appid.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-54x-appid.golden.json index bb4b82133d3..bb8678bcad1 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-54x-appid.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Fortigate-FortiOS-54x-appid.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 410000000, "end": "2018-05-11T00:54:09.99Z", "kind": "event", @@ -102,7 +101,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 1130000000, "end": "2018-05-11T00:54:09.74Z", "kind": "event", @@ -188,7 +186,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 1130000000, "end": "2018-05-11T00:54:09.74Z", "kind": "event", @@ -274,7 +271,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 1040000000, "end": "2018-05-11T00:54:09.74Z", "kind": "event", @@ -360,7 +356,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 1040000000, "end": "2018-05-11T00:54:09.74Z", "kind": "event", @@ -446,7 +441,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 410000000, "end": "2018-05-11T00:54:09.11Z", "kind": "event", @@ -532,7 +526,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 410000000, "end": "2018-05-11T00:54:09.11Z", "kind": "event", @@ -618,7 +611,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 370000000, "end": "2018-05-11T00:54:08.53Z", "kind": "event", @@ -704,7 +696,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 370000000, "end": "2018-05-11T00:54:08.53Z", "kind": "event", @@ -790,7 +781,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 80000000, "end": "2018-05-11T00:51:08.63Z", "kind": "event", @@ -872,7 +862,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 80000000, "end": "2018-05-11T00:51:08.63Z", "kind": "event", @@ -954,7 +943,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 80000000, "end": "2018-05-11T00:51:08.63Z", "kind": "event", @@ -1036,7 +1024,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 80000000, "end": "2018-05-11T00:51:08.63Z", "kind": "event", @@ -1118,7 +1105,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 2020000000, "end": "2018-05-11T00:54:06.21Z", "kind": "event", @@ -1200,7 +1186,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 2020000000, "end": "2018-05-11T00:54:06.21Z", "kind": "event", @@ -1282,7 +1267,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 4000000000, "end": "2018-05-11T00:54:00.19Z", "kind": "event", @@ -1364,7 +1348,6 @@ "network_traffic", "network" ], - "created": "2018-05-11T00:54:11Z", "duration": 4000000000, "end": "2018-05-11T00:54:00.19Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C-Netstream-with-varstring.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C-Netstream-with-varstring.golden.json index 93ea138cbac..a63ef85e3c4 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C-Netstream-with-varstring.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C-Netstream-with-varstring.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-07-18T01:35:35Z", "duration": 29695000000, "end": "2018-07-18T01:35:02.969Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C.golden.json index 5884e5c3a85..9ee961bb97f 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-H3C.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 89519000000, "end": "2018-05-21T09:25:03.677Z", "kind": "event", @@ -107,7 +106,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 60005000000, "end": "2018-05-21T09:25:03.662Z", "kind": "event", @@ -198,7 +196,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 60016000000, "end": "2018-05-21T09:25:03.656Z", "kind": "event", @@ -289,7 +286,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 90011000000, "end": "2018-05-21T09:25:03.643Z", "kind": "event", @@ -380,7 +376,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 30000000000, "end": "2018-05-21T09:24:03.629Z", "kind": "event", @@ -471,7 +466,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 29467000000, "end": "2018-05-21T09:24:03.669Z", "kind": "event", @@ -562,7 +556,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 29452000000, "end": "2018-05-21T09:24:03.67Z", "kind": "event", @@ -653,7 +646,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 29449000000, "end": "2018-05-21T09:24:03.684Z", "kind": "event", @@ -744,7 +736,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 30000000000, "end": "2018-05-21T09:24:03.685Z", "kind": "event", @@ -835,7 +826,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 29391000000, "end": "2018-05-21T09:24:03.691Z", "kind": "event", @@ -926,7 +916,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 29196000000, "end": "2018-05-21T09:24:03.699Z", "kind": "event", @@ -1017,7 +1006,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 30000000000, "end": "2018-05-21T09:24:03.753Z", "kind": "event", @@ -1108,7 +1096,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 89282000000, "end": "2018-05-21T09:25:03.971Z", "kind": "event", @@ -1199,7 +1186,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 90012000000, "end": "2018-05-21T09:25:03.95Z", "kind": "event", @@ -1290,7 +1276,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 60005000000, "end": "2018-05-21T09:25:03.938Z", "kind": "event", @@ -1381,7 +1366,6 @@ "network_traffic", "network" ], - "created": "2018-05-21T09:25:04Z", "duration": 60006000000, "end": "2018-05-21T09:25:03.928Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Huawei-Netstream.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Huawei-Netstream.golden.json index 231fd5e8797..0afe853c1ec 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Huawei-Netstream.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Huawei-Netstream.golden.json @@ -17,7 +17,6 @@ "network_traffic", "network" ], - "created": "2018-01-29T03:02:20Z", "duration": 327060000000, "end": "2018-01-29T03:02:19Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-IE150-IE151.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-IE150-IE151.golden.json index 326828e8304..58ebb30be4c 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-IE150-IE151.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-IE150-IE151.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-12-01T17:04:39Z", "kind": "event", "type": [ "connection" @@ -94,7 +93,6 @@ "network_traffic", "network" ], - "created": "2017-12-01T17:04:39Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-1-flowset-in-large-zero-filled-packet.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-1-flowset-in-large-zero-filled-packet.golden.json index c5b53c195be..5406a7ae845 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-1-flowset-in-large-zero-filled-packet.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-1-flowset-in-large-zero-filled-packet.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-06-06T13:20:17Z", "duration": 0, "end": "2018-06-06T13:20:02Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-PAN--OS-with-app--id.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-PAN--OS-with-app--id.golden.json index 754241fb7e0..931c4a8e276 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-PAN--OS-with-app--id.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Palo-Alto-PAN--OS-with-app--id.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -100,7 +99,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 339000000000, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -184,7 +182,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -268,7 +265,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -352,7 +348,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -436,7 +431,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -520,7 +514,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", @@ -604,7 +597,6 @@ "network_traffic", "network" ], - "created": "2017-11-13T14:39:31Z", "duration": 0, "end": "2017-11-13T14:39:31Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Streamcore.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Streamcore.golden.json index 0fd9b02e864..94d3ea85b2b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Streamcore.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Streamcore.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2017-01-11T11:48:15Z", "duration": 6012000000, "end": "2017-01-11T11:47:28.879Z", "kind": "event", @@ -95,7 +94,6 @@ "network_traffic", "network" ], - "created": "2017-01-11T11:48:15Z", "duration": 6020000000, "end": "2017-01-11T11:47:28.886Z", "kind": "event", @@ -174,7 +172,6 @@ "network_traffic", "network" ], - "created": "2017-01-11T11:23:51Z", "duration": 50997000000, "end": "2017-01-11T11:23:34.936Z", "kind": "event", @@ -253,7 +250,6 @@ "network_traffic", "network" ], - "created": "2017-01-11T11:23:51Z", "duration": 51015000000, "end": "2017-01-11T11:23:34.954Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json index cbfc95d8e22..07680d7cffa 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Ubiquiti-Edgerouter-with-MPLS-labels.golden.json @@ -17,7 +17,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 0, "end": "2016-09-10T16:17:25.825Z", "kind": "event", @@ -105,7 +104,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 0, "end": "2016-09-10T16:17:25.825Z", "kind": "event", @@ -193,7 +191,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 140227000000, "end": "2016-09-10T15:22:30.891Z", "kind": "event", @@ -281,7 +278,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 140227000000, "end": "2016-09-10T15:22:30.891Z", "kind": "event", @@ -369,7 +365,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 177102000000, "end": "2016-09-10T16:20:32.763Z", "kind": "event", @@ -457,7 +452,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 176903000000, "end": "2016-09-10T16:20:32.666Z", "kind": "event", @@ -545,7 +539,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 0, "end": "2016-09-10T15:22:36.207Z", "kind": "event", @@ -633,7 +626,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:23:30Z", "duration": 0, "end": "2016-09-10T16:17:35.661Z", "kind": "event", @@ -720,7 +712,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 116000000, "end": "2016-09-10T15:23:38.951Z", "kind": "event", @@ -806,7 +797,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -892,7 +882,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -978,7 +967,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -1064,7 +1052,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -1150,7 +1137,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -1236,7 +1222,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 0, "end": "2016-09-10T16:18:39.443Z", "kind": "event", @@ -1322,7 +1307,6 @@ "network_traffic", "network" ], - "created": "2016-09-10T16:24:08Z", "duration": 1250988000000, "end": "2016-09-10T15:23:44.363Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-field-layer2segmentid.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-field-layer2segmentid.golden.json index a3f5a4e3cc1..1311a75ee5c 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-field-layer2segmentid.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-field-layer2segmentid.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-01-16T09:45:02Z", "duration": 0, "end": "2018-01-16T09:44:47Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json index 106d624530a..7675fa91b41 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-ipt_netflow-reduced-size-encoding.golden.json @@ -17,7 +17,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 8996000000, "end": "2018-02-18T05:46:53.996Z", "kind": "event", @@ -105,7 +104,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 0, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -193,7 +191,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 7652000000, "end": "2018-02-18T05:46:53.988Z", "kind": "event", @@ -281,7 +278,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 16000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -369,7 +365,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 1168000000, "end": "2018-02-18T05:46:53.988Z", "kind": "event", @@ -457,7 +452,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 8992000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -545,7 +539,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 4432000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -633,7 +626,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 80000000, "end": "2018-02-18T05:46:53.996Z", "kind": "event", @@ -721,7 +713,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 400000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -809,7 +800,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 9024000000, "end": "2018-02-18T05:46:53.988Z", "kind": "event", @@ -897,7 +887,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 60000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", @@ -985,7 +974,6 @@ "network_traffic", "network" ], - "created": "2018-02-18T05:47:09Z", "duration": 192000000, "end": "2018-02-18T05:46:53.992Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json index c1857f85210..7f75a6ddf2a 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-macaddress.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:46:56Z", "kind": "event" }, "netflow": { @@ -54,7 +53,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -122,7 +120,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -190,7 +187,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -258,7 +254,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -326,7 +321,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -394,7 +388,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -462,7 +455,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -530,7 +522,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -598,7 +589,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -666,7 +656,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -734,7 +723,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -802,7 +790,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -870,7 +857,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -938,7 +924,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1006,7 +991,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1074,7 +1058,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1142,7 +1125,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1210,7 +1192,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1278,7 +1259,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1346,7 +1326,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1414,7 +1393,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1482,7 +1460,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1550,7 +1527,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1618,7 +1594,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1686,7 +1661,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1754,7 +1728,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1822,7 +1795,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1890,7 +1862,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" @@ -1958,7 +1929,6 @@ "network_traffic", "network" ], - "created": "2015-10-10T08:47:01Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-multiple-netflow-exporters.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-multiple-netflow-exporters.golden.json index cfec457a4f0..67a9901764b 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-multiple-netflow-exporters.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-multiple-netflow-exporters.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:06:29Z", "kind": "event" }, "netflow": { @@ -53,7 +52,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:46.141Z", "kind": "event", @@ -133,7 +131,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:46.141Z", "kind": "event", @@ -213,7 +210,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:51.814Z", "kind": "event", @@ -293,7 +289,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:51.814Z", "kind": "event", @@ -373,7 +368,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 0, "end": "2015-10-08T19:03:55.958Z", "kind": "event", @@ -453,7 +447,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 0, "end": "2015-10-08T19:03:55.958Z", "kind": "event", @@ -531,7 +524,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 38081000000, "end": "2015-10-08T19:04:25.9Z", "kind": "event", @@ -603,7 +595,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:06:29Z", "duration": 5000000, "end": "2015-10-08T19:05:55.015Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-nprobe-DPI-L7.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-nprobe-DPI-L7.golden.json index abf9608ce15..25307e840bd 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-nprobe-DPI-L7.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-nprobe-DPI-L7.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "1970-01-01T00:08:22Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-options-template-with-scope-fields.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-options-template-with-scope-fields.golden.json index ee3ffd12ab2..4cbb8be896f 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-options-template-with-scope-fields.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-options-template-with-scope-fields.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:06:29Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-template-with-0-length-fields.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-template-with-0-length-fields.golden.json index a1fa471443b..e0189eebddb 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-template-with-0-length-fields.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-template-with-0-length-fields.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:48.299Z", "kind": "event", @@ -100,7 +99,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:48.299Z", "kind": "event", @@ -184,7 +182,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.469Z", "kind": "event", @@ -268,7 +265,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.469Z", "kind": "event", @@ -352,7 +348,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.469Z", "kind": "event", @@ -436,7 +431,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.469Z", "kind": "event", @@ -520,7 +514,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.569Z", "kind": "event", @@ -604,7 +597,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.569Z", "kind": "event", @@ -688,7 +680,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.569Z", "kind": "event", @@ -772,7 +763,6 @@ "network_traffic", "network" ], - "created": "2016-12-23T01:35:31Z", "duration": 0, "end": "2016-12-23T01:34:51.569Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-valid-01.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-valid-01.golden.json index 2138b0b63fd..5d90c00cc64 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-valid-01.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-valid-01.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:46.141Z", "kind": "event", @@ -96,7 +95,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:46.141Z", "kind": "event", @@ -176,7 +174,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:51.814Z", "kind": "event", @@ -256,7 +253,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 1000000, "end": "2015-10-08T19:03:51.814Z", "kind": "event", @@ -336,7 +332,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 0, "end": "2015-10-08T19:03:55.958Z", "kind": "event", @@ -416,7 +411,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 0, "end": "2015-10-08T19:03:55.958Z", "kind": "event", @@ -494,7 +488,6 @@ "network_traffic", "network" ], - "created": "2015-10-08T19:04:30Z", "duration": 38081000000, "end": "2015-10-08T19:04:25.9Z", "kind": "event", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow9-Juniper-SRX-options-template-with-0-scope-field-length.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow9-Juniper-SRX-options-template-with-0-scope-field-length.golden.json index 1dc8829da2e..9ebcc615051 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow9-Juniper-SRX-options-template-with-0-scope-field-length.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow9-Juniper-SRX-options-template-with-0-scope-field-length.golden.json @@ -11,7 +11,6 @@ "network_traffic", "network" ], - "created": "2016-11-29T00:21:56Z", "kind": "event" }, "netflow": { diff --git a/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json index da28639ceae..b67858c32e4 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json @@ -16,7 +16,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -104,7 +103,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -192,7 +190,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -280,7 +277,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -368,7 +364,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -456,7 +451,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -544,7 +538,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -632,7 +625,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -720,7 +712,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -808,7 +799,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -896,7 +886,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -984,7 +973,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1072,7 +1060,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1160,7 +1147,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1248,7 +1234,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1336,7 +1321,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1424,7 +1408,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1512,7 +1495,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1600,7 +1582,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1688,7 +1669,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1776,7 +1756,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1864,7 +1843,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -1952,7 +1930,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2040,7 +2017,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2128,7 +2104,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2216,7 +2191,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2304,7 +2278,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2392,7 +2365,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" @@ -2480,7 +2452,6 @@ "event": { "action": "netflow_flow", "category": "network_session", - "created": "2018-07-03T10:47:00Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json index a71c8fb7ba8..6a8448a65c1 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_e10s_4_7byte_pad.pcap.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" @@ -92,7 +91,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" @@ -168,7 +166,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" @@ -244,7 +241,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" @@ -320,7 +316,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" @@ -396,7 +391,6 @@ "network_traffic", "network" ], - "created": "2020-04-16T23:22:51Z", "kind": "event", "type": [ "connection" diff --git a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json index 0f198716213..91df765da97 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/netflow9_ubiquiti_edgerouter.pcap.golden.json @@ -16,7 +16,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 287000000, "end": "2018-08-09T16:43:00.307Z", "kind": "event", @@ -102,7 +101,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 30209000000, "end": "2018-08-09T16:43:01.317Z", "kind": "event", @@ -188,7 +186,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 0, "end": "2018-08-09T16:43:01.41Z", "kind": "event", @@ -274,7 +271,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 59651000000, "end": "2018-08-09T16:43:02.334Z", "kind": "event", @@ -360,7 +356,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 40015000000, "end": "2018-08-09T16:43:02.876Z", "kind": "event", @@ -446,7 +441,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 37121000000, "end": "2018-08-09T16:43:02.43Z", "kind": "event", @@ -532,7 +526,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 31322000000, "end": "2018-08-09T16:43:02.43Z", "kind": "event", @@ -618,7 +611,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 31226000000, "end": "2018-08-09T16:43:02.334Z", "kind": "event", @@ -704,7 +696,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 30976000000, "end": "2018-08-09T16:43:02.334Z", "kind": "event", @@ -790,7 +781,6 @@ "network_traffic", "network" ], - "created": "2018-08-09T16:49:04Z", "duration": 0, "end": "2018-08-09T16:43:06.28Z", "kind": "event", diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml similarity index 99% rename from x-pack/filebeat/module/aws/cloudtrail/config/s3.yml rename to x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml index 16b8bc5eec2..4cc64e9e561 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} file_selectors: {{ if .process_cloudtrail_logs }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 03c7acf1336..bad63e1224b 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/cloudwatch/config/s3.yml rename to x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml index bec03a2090c..db50bdc4362 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index 5d9931b2e40..ca3a74dadd3 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/ec2/config/s3.yml rename to x-pack/filebeat/module/aws/ec2/config/aws-s3.yml index bec03a2090c..db50bdc4362 100644 --- a/x-pack/filebeat/module/aws/ec2/config/s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index 5d9931b2e40..ca3a74dadd3 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/elb/config/s3.yml rename to x-pack/filebeat/module/aws/elb/config/aws-s3.yml index bec03a2090c..db50bdc4362 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index dc95f6abb7e..54fa469d701 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 18b5652f804..51481178803 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG5/M1apyLa+mS9omByBWFGgIYAJROmf7xnQXAHxJBWbaoXKZ+SHwmCXx2sb+xwBiecHcDbGveAFhuBd7A5LfZGwCNApnBG1igZW8AMjSp5oXlSt7Av70BAPhZZaVAWCoNKyYzwWUOQuUGllqtaZjrNwBLjiIzN+6DMUi2xmo6+rG7Am8g16oswl8i89DPRzdMPbKb5zo8bU/RniYVqsysZlzUj2Iz0s8htdVPhktWCpu4KW5gyYTBvcdRsG3ASju8t4RlTlj2oMfgt0nADUqbbFAbruTeGxUlT7jbKp0dPDsCjH7mK2wjCuODWoJdIQH0ExP6NbPXUWilQZ3wDKXldheFdsjkLrBxFBmNPA0DAwpcE5RUScu4NJChZVwYYAtVWoeXZgO17Iw1nfwMFUCwK2ZhzTJ0n2j8o0RjR8BkBtsVT1eQanTvMmFgixo7w5UGs2uYLsHiulCa6V3nG/fOyM1Q4TYrtTWwUlv6a2fMzgBqQVRidn3wakxI2qtBPOg8PC4j3eWIvOBXJHDYEdaz5C3t1oeS+nIkXcGooEzW7E8l4RGNKnWK8ImtEa4mj5/eVgALzWXKCyYO1jxlQhyytYU6TdGY5Al3CY/hGwq/n4cGgukHj3DLjBMcsAoMz2VbQvsBGzSktAkpBn6xvZBjWngq4OmyjcUBdezccrtqqYHBtNQxkYB9ESd1qxXDkV5oteEZGuDS2xoyQ41mBxqj49asSzUyi5kztXalDLanjHzap0pt5q6XLGGlXdEoKY0efft5qTiV0RCkY8NEicANWE3/D+xXyjqjCEo7o+Z+3xKpvYNFLVNgUbOgTBjleLhHq19eFmc7/fz8cQIZbniK/wrKrlBvucGR945dgW3z1a0VSW3GbB94z9MjL7yEoTSMM/KWrxG2K/Ta1ZXdLse4MWXXEO/TUymhe1cfJahPD19C0RD6CK/Xyd7xgns73Z1VP8d0EZ53b9XPKWoIL+AzBOkJPiY4luNCMwJTpqujQzIDj0rZESnxZ4N6RAr9qESP0rQZUDu1uHe6NCO4tKglE+SzAjfacVXbg+XYLyewL3vPkx2PJS5N7eTxU0VlkIArlqaqlH7pnP11a6eVwLdHh4ux5xlBOoErHszXEYUwuadMbaW5nDRU9HK5UU+YJYuYRRsqMKOpqlWnjM2gJg/XlziQsgOLxRdQxah3t+9gUloFs5S55DjkgneCGctTeI9MGsvEUzzBQq2VTlKVHVq+0xO/eH7Vps5NUgcawa9otKWWxnkGen4M3xqNYfmQEKfHwfj0qjVIbZD6oYaxkoJptkaL+nDdzmVpM/CImMnkbhR0gbygId/qfXR/ZL8uheXJs3nesVC/5+EzRYw2m0yhpMEkxANDc6kav443KB5lKX1jKj17QkhXTOZo4MpH9qNuJl5QWOcscIYCKcLzg7z9GzKVZRkncEwkrqiSsb0K07l8ndTDU6zLWsmTL+HUFlkqS2Jqg7p0Bqr0jdxaWKe/ITcrTe64vnNl02dejlu+wrHkaPb01UeEwW4ukMu8W85hQmAGOUrUzLrvufFD99hQV9+LRLdnGdB9/FXxpiUPFcCsJSgaU6WzOExW8LOLkM/inDxM60okM0alvElG3fOtmRT8lgnRGclRMCc6j/B6zSTLnd3xijikEsJ7pQQy2SNG2xVSmtziNjdwaAWghdC/1efMWJYoKeJF17OXosHKDaiC5IRWhAC7qcc0dfOgD6MPnOMe5DWF4QkIbpzxqscOtTTMgMuGtS+tnF6uXFnXKCePn7qB4knB/BAwJiFgb1K3ioMUvkeqFoPWlJ9hjTNOtaboaiV94cHtRd3csK0ZB7s7dshuyNON6VP37x4JTHnBSdl7GXyOwjxioZHiOm+7WMNjp/saU+QbZ1+5OabMgS5vkZKwt3E5G1uH/TTdCLhMRZlRarIl1FbzPEft3ULcyPpampehUvwdg1izYhqzwNBB1/zfP08/tFznYtfeQ7MKSsn/KFHsKnluP49zM2xoupWh9JMyMx/KBhdifO5gFWR8uURN//D7s/s/Qf5MXMg2RZqgzArFh2bJgXj9+nAL1USkyn5nLQRQoSToUmlHdtcB0vdWAZOuqtxOVOuEu0quZ98di6hSZjFX+tUu8n+7pM7cRl7jwasp6pUqgye6V+qpLO78+lJE2FqVrmCP4aPSHc9vfJZbbwc0zyPVIT+Ei/6jH9OT3s+m0vB8ZU38U+6fxlmdKmmUwESonMfjwtc4+rBxbgpM+ZKnJA+3fqJ7mifgfKmTfz4JO466izzua5tszAkAfKBVoTT3eRqO0dGmZa0WXGBPPL5PycJHpT3vnLzr0A0O96ih7NIptEcGx5BVRDiZSaw6Cn+YfbXPj/edFTjO4CVLSJm/DmudC4UlS63S7X05cgrbnoIjhPYEyEpN3ryX1IrEpWDWouzQ+Hq1vZs1gzqL4yI74rta/A+m1hGofRHVlAsv6MA0ym8tPEm1lWQ7WbZhMu0UHgZV7z7au1Q+p+LnFVzgtKLLC6qYr6GxvyA+QE3z9NLisNCHKjTCi4qNL4vqh6N4wBgfTorzKzIznqOJ11HO8PzOldzWvXvwwc0C9yp/qdcXKk+WXHSqEA1MiSbe3HFSYl2XJO5V7uaputSakoRn0RFJsUzbxPJ1f6bd0wlxqi64GWj5P89vfReEJqn3atBABCIAUkXOO64Ilj3RMMwXglwyknp77katw32hcvDcWLFNXMQWiJK0iW+cJO5lUv28QpldlFMos/8XfDLfJYsyfYruvF5kt7TKyMBPSyG7J9H1hJRad6utYbYWS1fMHNB7lEIfVVyQwoYqP5Vr2bsKyd4oSnh0LKF88Fb3JwaGtInvp1Tillx+2Ce4oPDXUh9QrpXxZawqJQW2ViTWQhwLaEJZI1i/RsRPNIdKZF+HXD9xD6X9+Rq8mtJC44ar0iR/hbYe19AKyp469qUaJ2hnTdqKmVXCRK40t6v1X2SNaFKoJ+12pLjnfeIbY8URQsuF4KnrGV5ymaMuNI9auqHoXOEXlmHK10wAylRlmEFr5roD2uFyBovIjw63ZjZdhYix0HzDLLoPDhqR+YmsoNeZLTX+JevdLG+nb7oXbdNM5EpaR2vtfRH5c/h8jTDey1zKDLXYUWgQYnBD68VkXYKL14ibyqc3Tn7jZOSbgFnu37DMcmN5akZuG5Uo7QYn/qxDZCfSVmEiJTkos0qOjgNr2Lt34GZLgvXmkKNf58DNb4TlRQdujrUzRXYmnqkX7IOoBu8yDtN3X59jd7fv/PEtLlvAT2UcLxKWZRrN63evOuxrOk/RQhi92cN0FQfUEW6KxenczC/Dy/v3LxK7zjbm2Xxru0ShWAYLJphMsadD7qwWlCiA9pmhPQCOSZt3cE9/fB/+2LNzZZnO0SZu9a67O/VnQmz1+fqJvJg0pxl7S1t1hYEb8hOHe+hn4nLSE0bu7CdT8iklpv3NF4VWVqXqcM/xTFDVqPE1vVpZW5D7sGnx9pkmTK1SNIbL3EX41wbTHiesOrHLKXKnLBN1qm0wVTIzYHiVhDfc81VXX4DlpuFxKS0XwPe2XymVz2lJXB7O0ieUPb1S4eHfiMwWGfQkAATLhdj7gwsBTCieZlzmvb09vpT7lSmsi7zttav30/eo3F9Lxx7Be3syGtZ5utZxlyZUpwPwZVRV9YDWSklYcyF4IHYUqPXwVeH2VloEpUKZw0SstpyC8i+ZmRV7wsvSUZ0qm9/PoJ6SGJ2qdeEq5gd0gYpIaV3/QWPZQnCz6iOtUj9+WGA+08JNHw6DjEqIGkn3KdFzFrhCWCj9+jbDuBVW2lYlhnPRkcm+rredKIcozXnnDaKQ/cBAA3fV86r9tKug/gPyh6S/VmlfWf+9F/rvUYqNEUnKi9XQjno2uwc/bt39Mb+f/ZP7c70IPaENYbqMp6bpa2/9YlzpShmUSYraXjTi8vMAzcOX7uAphJY6n9G3hOC18A1qzgZmrh8TZLleoL4wLVymau3cqzAJEzi0MaHcKkcd2n3U0llwN0/Loy52sRC+0mNPldsgDhT3uyP6LUtiO4Nn0kGw3eA+fu9BoFmK53S/Rad2p++lrT3G7/81nqz/lOM5zTaeZr/DClnWl3X5IlyW6FJQOMVV5NTu2WG7H7VZZBc7laI+2eYg7B9u40sKeOmd8LjnWIAPkvALpmV3G/VM4KEPshrcn93ec3XNGbGrpdJbprMRLPkXzMaVZxjtHWy/vr5+ew1TCymT1U4tGNygZsKzp0cPNWZcY2qTUg9sTT4/3gcL7Tge5nGFw7Tq+KlZcOQY3rVGZoa+IcafDfQjV0cE6+UI+JaMi94Q1CfvwwZB92HLvQnW0LjKJk1j6ljUz23qE+3hoICLj45m9AH0RaKhCnsr5ml1AFdCa/ZrEn1+TjBjvLMZ/Gqg/bEdTzM0O5nCmlueHzn3sf9lcgmpPAAXxJN42a0Emu/8EZGvXQ6cfee6clBX17+8pDbot+yS2FmNc1nJpJI8ZcJvJjQHRNxcB0fgPYyejCK2gTlgAbPataxaMdoXYDTKzXLGZZ9ea1wri0lPztj58ykOqiiY9iHdySVq6NbD/rJFredzvWwMxk48S7l/90vVrH+8hjdkGDUBY13X6t7RimbLOnqyAllzsCAOtT6hNix7mxNxrkKbwcrfFQMZpoJRfsoMzH6ZPFzXb47g8W42v/5xPn9I1mhXKruuTmC5o58j+O3u/Ww6vzv2itLwfjK//fH6w9393fzu+pf3/3F3O4+T/oQDB5HfPOHum3YzaxMqUgQTtqIdyG/G31SxQsOqTKHvh7XsCYG5jea6vfS4pJWaD0vLox94/PlxukcR8b42LJ2duja0lbVFCBAGLKjJco2apx5HuyrSHNqL9AcPeFlEPGGv1fDORYO3KsP2OksVwkSVur6mvpLdzqJJTN+p21dzLOTgdSnYzePqSCPAL1Xjq2Nps3+xQU1JWZuMP1GrHjPimr8Sw/+Mc/ac6ikNWrtc32TGJTgJ7I2w3Jex1qhhOLlXpj10tVzCUvB8ZVvH1lxY862BArUpKHfZ9EioLbVMmFZlvIvzIvCZbQmwKchZtzLInSr1cR/i2jf10B56LxN3svkY5gnFgufLKe7yTdf8cWlonw3q8YRmOlrJCGd+Bq+vVBcCTD9UVe/a85zqbMIQ0+w5l7NSA0c2RMCXMVv/OebZ+J27/qeWRvxiUWZNwAXTDz114rqz6iLXwNbDV3wawYznvzq09Mv3o24LXTti3DMSr44rfRE9MSXvdHWeWwzHlIibKcogDNyzHWq4ms3u31aV++bADubK8vouSRL/WYw0etBTENs7rnXevR5xvxGaKuoLHPfPh/nrZyelXf3odNWfe9l/x2uxGcF/lqh3Mx9603t/0L+rWPyq0Dgm2cCMQry3r19ap1V+0oHLU9VB40osQz2dfn3mkLAV5jLaNNdMGrdH5wVtVt3vdjW/n72trVlL0kJ5/XA/unV0eynU9vQCxqW6w359uAVC8qLSxUV4TEg+EpJ7lZtqCncx806VJAzhgjBHeLhJIhxrDuznBt7VH/iDxjtgkJbGqnXfFz2iNMDtEvHA210FUN8qUdU1qyXo2zCyqJeX2OdoyggS7Vbpp2Yuh63pJreaLZc8DV0ZSmfHdw+GhXlweULsGqGAbwST29u7h7m7gvOuP5cWKj+W670aqVB5ToY2ZHqBudXyjuCXn0bw6ZcPk/nEeeKfpg/0e29Ls2XyoqteTeFY+22Xs6+QilEVutVjc+Mqj84o7lTZ0932ZBOjU5ZlcX/ymlJewSg6GAvcoIArpXnOJRNvq9JntzEkkNOPMDP2L0GYUa4ovWdvway3QY7h3BTpBSXG3eVBeljf3D+o9TDlQuLwZrfB7ye4JAk2LZKlYJ1jrWeSsOB2zcxTyOVqx6GEUFuyOPPbB3DT3sC7H2b//Wn0j3+h/40ntz+N/vHDx+mn0fc/PM7mcciXaxP2XLuB6cPm+xH9959dinf3cXL95v8CAAD//7hdU30=" + return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG5/M1apyLa+mS9omByBWFGgIYAJROmf7xnQXAHxJBWbaoXKZ+SHwmCXx2sb+xwBiecHcDbGveAFhuBd7A5LfZGwCNApnBG1igZW8AMjSp5oXlSt7Av70BAPhZZaVAWCoNKyYzwWUOQuUGllqtaZjrNwBLjiIzN+6DMUi2xmo6+rG7Am8g16oswl8i89DPRzdMPbKb5zo8bU/RniYVqsysZlzUj2Iz0s8htdVPhktWCpu4KW5gyYTBvcdRsG3ASju8t4RlTlj2oMfgt0nADUqbbFAbruTeGxUlT7jbKp0dPDsCjH7mK2wjCuODWoJdIQH0ExP6NbPXUWilQZ3wDKXldheFdsjkLrBxFBmNPA0DAwpcE5RUScu4NJChZVwYYAtVWoeXZgO17Iw1nfwMFUCwK2ZhzTJ0n2j8o0RjR8BkBtsVT1eQanTvMmFgixo7w5UGs2uYLsHiulCa6V3nG/fOyM1Q4TYrtTWwUlv6a2fMzgBqQVRidn3wakxI2qtBPOg8PC4j3eWIvOBXJHDYEdaz5C3t1oeS+nIkXcGooEzW7E8l4RGNKnWK8ImtEa4mj5/eVgALzWXKCyYO1jxlQhyytYU6TdGY5Al3CY/hGwq/n4cGgukHj3DLjBMcsAoMz2VbQvsBGzSktAkpBn6xvZBjWngq4OmyjcUBdezccrtqqYHBtNQxkYB9ESd1qxXDkV5oteEZGuDS2xoyQ41mBxqj49asSzUyi5kztXalDLanjHzap0pt5q6XLGGlXdEoKY0efft5qTiV0RCkY8NEicANWE3/D+xXyjqjCEo7o+Z+3xKpvYNFLVNgUbOgTBjleLhHq19eFmc7/fz8cQIZbniK/wrKrlBvucGR945dgW3z1a0VSW3GbB94z9MjL7yEoTSMM/KWrxG2K/Ta1ZXdLse4MWXXEO/TUymhe1cfJahPD19C0RD6CK/Xyd7xgns73Z1VP8d0EZ53b9XPKWoIL+AzBOkJPiY4luNCMwJTpqujQzIDj0rZESnxZ4N6RAr9qESP0rQZUDu1uHe6NCO4tKglE+SzAjfacVXbg+XYLyewL3vPkx2PJS5N7eTxU0VlkIArlqaqlH7pnP11a6eVwLdHh4ux5xlBOoErHszXEYUwuadMbaW5nDRU9HK5UU+YJYuYRRsqMKOpqlWnjM2gJg/XlziQsgOLxRdQxah3t+9gUloFs5S55DjkgneCGctTeI9MGsvEUzzBQq2VTlKVHVq+0xO/eH7Vps5NUgcawa9otKWWxnkGen4M3xqNYfmQEKfHwfj0qjVIbZD6oYaxkoJptkaL+nDdzmVpM/CImMnkbhR0gbygId/qfXR/ZL8uheXJs3nesVC/5+EzRYw2m0yhpMEkxANDc6kav443KB5lKX1jKj17QkhXTOZo4MpH9qNuJl5QWOcscIYCKcLzg7z9GzKVZRkncEwkrqiSsb0K07l8ndTDU6zLWsmTL+HUFlkqS2Jqg7p0Bqr0jdxaWKe/ITcrTe64vnNl02dejlu+wrHkaPb01UeEwW4ukMu8W85hQmAGOUrUzLrvufFD99hQV9+LRLdnGdB9/FXxpiUPFcCsJSgaU6WzOExW8LOLkM/inDxM60okM0alvElG3fOtmRT8lgnRGclRMCc6j/B6zSTLnd3xijikEsJ7pQQy2SNG2xVSmtziNjdwaAWghdC/1efMWJYoKeJF17OXosHKDaiC5IRWhAC7qcc0dfOgD6MPnOMe5DWF4QkIbpzxqscOtTTMgMuGtS+tnF6uXFnXKCePn7qB4knB/BAwJiFgb1K3ioMUvkeqFoPWlJ9hjTNOtaboaiV94cHtRd3csK0ZB7s7dshuyNON6VP37x4JTHnBSdl7GXyOwjxioZHiOm+7WMNjp/saU+QbZ1+5OabMgS5vkZKwt3E5G1uH/TTdCLhMRZlRarIl1FbzPEft3ULcyPpampehUvwdg1izYhqzwNBB1/zfP08/tFznYtfeQ7MKSsn/KFHsKnluP49zM2xoupWh9JMyMx/KBhdifO5gFWR8uURN//D7s/s/Qf5MXMg2RZqgzArFh2bJgXj9+nAL1USkyn5nLQRQoSToUmlHdtcB0vdWAZOuqtxOVOuEu0quZ98di6hSZjFX+tUu8n+7pM7cRl7jwasp6pUqgye6V+qpLO78+lJE2FqVrmCP4aPSHc9vfJZbbwc0zyPVIT+Ei/6jH9OT3s+m0vB8ZU38U+6fxlmdKmmUwESonMfjwtc4+rBxbgpM+ZKnJA+3fqJ7mifgfKmTfz4JO466izzua5tszAkAfKBVoTT3eRqO0dGmZa0WXGBPPL5PycJHpT3vnLzr0A0O96ih7NIptEcGx5BVRDiZSaw6Cn+YfbXPj/edFTjO4CVLSJm/DmudC4UlS63S7X05cgrbnoIjhPYEyEpN3ryX1IrEpWDWouzQ+Hq1vZs1gzqL4yI74rta/A+m1hGofRHVlAsv6MA0ym8tPEm1lWQ7WbZhMu0UHgZV7z7au1Q+p+LnFVzgtKLLC6qYr6GxvyA+QE3z9NLisNCHKjTCi4qNL4vqh6N4wBgfTorzKzIznqOJ11HO8PzOldzWvXvwwc0C9yp/qdcXKk+WXHSqEA1MiSbe3HFSYl2XJO5V7uaputSakoRn0RFJsUzbxPJ1f6bd0wlxqi64GWj5P89vfReEJqn3atBABCIAUkXOO64Ilj3RMMwXglwyknp77katw32hcvDcWLFNXMQWiJK0iW+cJO5lUv28QpldlFMos/8XfDLfJYsyfYruvF5kt7TKyMBPSyG7J9H1hJRad6utYbYWS1fMHNB7lEIfVVyQwoYqP5Vr2bsKyd4oSnh0LKF88Fb3JwaGtInvp1Tillx+2Ce4oPDXUh9QrpXxZawqJQW2ViTWQhwLaEJZI1i/RsRPNIdKZF+HXD9xD6X9+Rq8mtJC44ar0iR/hbYe19AKyp469qUaJ2hnTdqKmVXCRK40t6v1X2SNaFKoJ+12pLjnfeIbY8URQsuF4KnrGV5ymaMuNI9auqHoXOEXlmHK10wAylRlmEFr5roD2uFyBovIjw63ZjZdhYix0HzDLLoPDhqR+YmsoNeZLTX+JevdLG+nb7oXbdNM5EpaR2vtfRH5c/h8jTDey1zKDLXYUWgQYnBD68VkXYKL14ibyqc3Tn7jZOSbgFnu37DMcmN5akZuG5Uo7QYn/qxDZCfSVmEiJTkos0qOjgNr2Lt34GZLgvXmkKNf58DNb4TlRQdujrUzRXYmnqkX7IOoBu8yDtN3X59jd7fv/PEtLlvAT2UcLxKWZRrN63evOuxrOk/RQhi92cN0FQfUEW6KxenczC/Dy/v3LxK7zjbm2Xxru0ShWAYLJphMsadD7qwWlCiA9pmhPQCOSZt3cE9/fB/+2LNzZZnO0SZu9a67O/VnQmz1+fqJvJg0pxl7S1t1hYEb8hOHe+hn4nLSE0bu7CdT8iklpv3NF4VWVqXqcM/xTFDVqPE1vVpZW5D7sGnx9pkmTK1SNIbL3EX41wbTHiesOrHLKXKnLBN1qm0wVTIzYHiVhDfc81VXX4DlpuFxKS0XwPe2XymVz2lJXB7O0ieUPb1S4eHfiMwWGfQkAATLhdj7gwsBTCieZlzmvb09vpT7lSmsi7zttav30/eo3F9Lxx7Be3syGtZ5utZxlyZUpwPwZVRV9YDWSklYcyF4IHYUqPXwVeH2VloEpUKZw0SstpyC8i+ZmRV7wsvSUZ0qm9/PoJ6SGJ2qdeEq5gd0gYpIaV3/QWPZQnCz6iOtUj9+WGA+08JNHw6DjEqIGkn3KdFzFrhCWCj9+jbDuBVW2lYlhnPRkcm+rredKIcozXnnDaKQ/cBAA3fV86r9tKug/gPyh6S/VmlfWf+9F/rvUYqNEUnKi9XQjno2uwc/bt39Mb+f/ZP7c70IPaENYbqMp6bpa2/9YlzpShmUSYraXjTi8vMAzcOX7uAphJY6n9G3hOC18A1qzgZmrh8TZLleoL4wLVymau3cqzAJEzi0MaHcKkcd2n3U0llwN0/Loy52sRC+0mNPldsgDhT3uyP6LUtiO4Nn0kGw3eA+fu9BoFmK53S/Rad2p++lrT3G7/81nqz/lOM5zTaeZr/DClnWl3X5IlyW6FJQOMVV5NTu2WG7H7VZZBc7laI+2eYg7B9u40sKeOmd8LjnWIAPkvALpmV3G/VM4KEPshrcn93ec3XNGbGrpdJbprMRLPkXzMaVZxjtHWy/vr5+ew1TCymT1U4tGNygZsKzp0cPNWZcY2qTUg9sTT4/3gcL7Tge5nGFw7Tq+KlZcOQY3rVGZoa+IcafDfQjV0cE6+UI+JaMi94Q1CfvwwZB92HLvQnW0LjKJk1j6ljUz23qE+3hoICLj45m9AH0RaKhCnsr5ml1AFdCa/ZrEn1+TjBjvLMZ/Gqg/bEdTzM0O5nCmlueHzn3sf9lcgmpPAAXxJN42a0Emu/8EZGvXQ6cfee6clBX17+8pDbot+yS2FmNc1nJpJI8ZcJvJjQHRNxcB0fgPYyejCK2gTlgAbPataxaMdoXYDTKzXLGZZ9ea1wri0lPztj58ykOqiiY9iHdySVq6NbD/rJFredzvWwMxk48S7l/90vVrH+8hjdkGDUBY13X6t7RimbLOnqyAllzsCAOtT6hNix7mxNxrkKbwcrfFQMZpoJRfsoMzH6ZPFzXb47g8W42v/5xPn9I1mhXKruuTmC5o58j+O3u/Ww6vzv2itLwfjK//fH6w9393fzu+pf3/3F3O4+T/oQDB5HfPOHum3YzaxMqUgQTtqIdyG/G31SxQsOqTKHvh7XsCYG5jea6vfS4pJWaD0vLox94/PlxukcR8b42LJ2duja0lbVFCBAGLKjJco2apx5HuyrSHNqL9AcPeFlEPGGv1fDORYO3KsP2OksVwkSVur6mvpLdzqJJTN+p21dzLOTgdSnYzePqSCPAL1Xjq2Nps3+xQU1JWZuMP1GrHjPimr8Sw/+Mc/ac6ikNWrtc32TGJTgJ7I2w3Jex1qhhOLlXpj10tVzCUvB8ZVvH1lxY862BArUpKHfZ9EioLbVMmFZlvIvzIvCZbQmwKchZtzLInSr1cR/i2jf10B56LxN3svkY5gnFgufLKe7yTdf8cWlonw3q8YRmOlrJCGd+Bq+vVBcCTD9UVe/a85zqbMIQ0+w5l7NSA0c2RMCXMVv/OebZ+J27/qeWRvxiUWZNwAXTDz114rqz6iLXwNbDV3wawYznvzq09Mv3o24LXTti3DMSr44rfRE9MSXvdHWeWwzHlIibKcogDNyzHWq4ms3u31aV++bADubK8vouSRL/WYw0etBTENs7rnXevR5xvxGaKuoLHPfPh/nrZyelXf3odNWfe9l/x2uxGcF/lqh3Mx9603t/0L+rWPyq0Dgm2cCMQry3r19ap1V+0oHLU9VB40osQz2dfn3mkLAV5jLaNNdMGrdH5wVtVt3vdjW/n72trVlL0kJ5/XA/unV0eynU9vQCxqW6w359uAVC8qLSxUV4TEg+EpJ7lZtqCncx806VJAzhgjBHeLhJIhxrDuznBt7VH/iDxjtgkJbGqnXfFz2iNMDtEvHA210FUN8qUdU1qyXo2zCyqJeX2OdoyggS7Vbpp2Yuh63pJreaLZc8DV0ZSmfHdw+GhXlweULsGqGAbwST29u7h7m7gvOuP5cWKj+W670aqVB5ToY2ZHqBudXyjuCXn0bw6ZcPk/nEeeKfpg/0e29Ls2XyoqteTeFY+22Xs6+QilEVutVjc+Mqj84o7lTZ0932ZBOjU5ZlcX/ymlJewSg6GAvcoIArpXnOJRNvq9JntzEkkNOPMDP2L0GYUa4ovWdvway3QY7h3BTpBSXG3eVBeljf3D+o9TDlQuLwZrfB7ye4JAk2LZKlYJ1jrWeSsOB2zcxTyOVqx6GEUFuyOPPbB3DT3sC7H2b//Wn0j3+h/40ntz+N/vHDx+mn0fc/PM7mxyEnTGs2YJWx2hprgft2yeUIzE6OQFO+V5jVCFj6NIJS59/G4V2ui9kv6g1MHzbfj+i//+wy0LuPk+s3/xcAAP//jpR9HA==" } diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/s3access/config/s3.yml rename to x-pack/filebeat/module/aws/s3access/config/aws-s3.yml index bec03a2090c..db50bdc4362 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index 5d9931b2e40..ca3a74dadd3 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml index 22686069848..f1c658b8a45 100644 --- a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml @@ -49,6 +49,10 @@ type: keyword description: > The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST + - name: tcp_flags_array + type: keyword + description: > + List of TCP flags: 'fin, syn, rst, psh, ack, urg' - name: type type: keyword description: > diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 243496f9cc6..1752158d25e 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "s3" }} +{{ if eq .input "aws-s3" }} -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index a8a6e5ae726..0a87d6baade 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -128,6 +128,39 @@ processors: field: event.kind value: event + - script: + lang: painless + ignore_failure: true + source: | + if (ctx?.aws?.vpcflow?.tcp_flags == null) + return; + + if (ctx?.aws?.vpcflow?.tcp_flags_array == null) { + ArrayList al = new ArrayList(); + ctx.aws.vpcflow.put("tcp_flags_array", al); + } + + def flags = Integer.parseUnsignedInt(ctx.aws.vpcflow.tcp_flags); + + if ((flags & 0x01) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('fin'); + } + if ((flags & 0x02) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('syn'); + } + if ((flags & 0x04) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('rst'); + } + if ((flags & 0x08) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('psh'); + } + if ((flags & 0x10) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('ack'); + } + if ((flags & 0x20) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('urg'); + } + on_failure: - set: field: "error.message" diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index 19f40c7a3f7..d084692d5c4 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json index 0a8feef3be5..f8b8a3a3357 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json @@ -9,6 +9,10 @@ "aws.vpcflow.pkt_srcaddr": "10.20.33.164", "aws.vpcflow.subnet_id": "subnet-22222222bbbbbbbbb", "aws.vpcflow.tcp_flags": "3", + "aws.vpcflow.tcp_flags_array": [ + "fin", + "syn" + ], "aws.vpcflow.type": "IPv4", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log index 28ca1ca949f..32c4f31a9b6 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log @@ -1,2 +1,4 @@ version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json index ba0293752ca..7f79d489595 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json @@ -10,6 +10,9 @@ "aws.vpcflow.pkt_srcaddr": "52.213.180.42", "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", "aws.vpcflow.tcp_flags": "2", + "aws.vpcflow.tcp_flags_array": [ + "syn" + ], "aws.vpcflow.type": "IPv4", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", @@ -60,5 +63,138 @@ "tags": [ "forwarded" ] + }, + { + "@timestamp": "2019-08-27T19:13:13.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.action": "ACCEPT", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "OK", + "aws.vpcflow.pkt_dstaddr": "10.0.0.62", + "aws.vpcflow.pkt_srcaddr": "52.213.180.42", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.tcp_flags": "3", + "aws.vpcflow.tcp_flags_array": [ + "fin", + "syn" + ], + "aws.vpcflow.type": "IPv4", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "destination.address": "10.0.0.62", + "destination.ip": "10.0.0.62", + "destination.port": 5001, + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-27T19:13:13.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43638 5001 52.213.180.42 10.0.0.62 6 1260 17 1566933133 1566933193 ACCEPT 3 OK", + "event.outcome": "allow", + "event.start": "2019-08-27T19:12:13.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 393, + "network.bytes": 1260, + "network.community_id": "1:nOrJcppKxIxs557D2oKADkNCpno=", + "network.iana_number": "6", + "network.packets": 17, + "network.transport": "tcp", + "network.type": "ipv4", + "related.ip": [ + "52.213.180.42", + "10.0.0.62" + ], + "service.type": "aws", + "source.address": "52.213.180.42", + "source.as.number": 16509, + "source.as.organization.name": "Amazon.com, Inc.", + "source.bytes": 1260, + "source.geo.city_name": "Dublin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "IE", + "source.geo.country_name": "Ireland", + "source.geo.location.lat": 53.3338, + "source.geo.location.lon": -6.2488, + "source.geo.region_iso_code": "IE-L", + "source.geo.region_name": "Leinster", + "source.ip": "52.213.180.42", + "source.packets": 17, + "source.port": 43638, + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2019-08-27T19:13:13.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.action": "ACCEPT", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "OK", + "aws.vpcflow.pkt_dstaddr": "52.213.180.42", + "aws.vpcflow.pkt_srcaddr": "10.0.0.62", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.tcp_flags": "19", + "aws.vpcflow.tcp_flags_array": [ + "fin", + "syn", + "ack" + ], + "aws.vpcflow.type": "IPv4", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "destination.address": "52.213.180.42", + "destination.as.number": 16509, + "destination.as.organization.name": "Amazon.com, Inc.", + "destination.geo.city_name": "Dublin", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "IE", + "destination.geo.country_name": "Ireland", + "destination.geo.location.lat": 53.3338, + "destination.geo.location.lon": -6.2488, + "destination.geo.region_iso_code": "IE-L", + "destination.geo.region_name": "Leinster", + "destination.ip": "52.213.180.42", + "destination.port": 43638, + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-27T19:13:13.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 10.0.0.62 52.213.180.42 5001 43638 10.0.0.62 52.213.180.42 6 967 14 1566933133 1566933193 ACCEPT 19 OK", + "event.outcome": "allow", + "event.start": "2019-08-27T19:12:13.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 605, + "network.bytes": 967, + "network.community_id": "1:nOrJcppKxIxs557D2oKADkNCpno=", + "network.iana_number": "6", + "network.packets": 14, + "network.transport": "tcp", + "network.type": "ipv4", + "related.ip": [ + "10.0.0.62", + "52.213.180.42" + ], + "service.type": "aws", + "source.address": "10.0.0.62", + "source.bytes": 967, + "source.ip": "10.0.0.62", + "source.packets": 14, + "source.port": 5001, + "tags": [ + "forwarded" + ] } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/_meta/config.yml b/x-pack/filebeat/module/cisco/_meta/config.yml index b0fb55ed7cb..77b3658c42b 100644 --- a/x-pack/filebeat/module/cisco/_meta/config.yml +++ b/x-pack/filebeat/module/cisco/_meta/config.yml @@ -109,7 +109,7 @@ umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index d0d49ca2fed..c6bdd1854f8 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -383,7 +383,7 @@ will be found under `rsa.raw`. The default is false. The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. -To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-s3.html[S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. +To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-aws-s3.html[AWS S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. This fileset supports all 4 log types: - Proxy @@ -404,7 +404,7 @@ Example config: - module: cisco umbrella: enabled: true - var.input: s3 + var.input: aws-s3 var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue var.access_key_id: 123456 var.secret_access_key: PASSWORD diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 8b07b91acb4..1d225c42add 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -1604,17 +1604,26 @@ ] }, { + "cisco.asa.destination_interface": "net", "cisco.asa.message_id": "302023", + "cisco.asa.source_interface": "fw111", + "destination.address": "192.168.2.2", + "destination.ip": "192.168.2.2", + "destination.port": 10051, "event.action": "firewall-rule", "event.category": [ "network" ], "event.code": 302023, "event.dataset": "cisco.asa", + "event.duration": 0, + "event.end": "2021-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for fw111:10.10.10.10/39210 to net:192.168.2.2/10051 duration 0:00:00 forwarded bytes 0 Cluster flow with CLU closed on owner", + "event.reason": "Cluster flow with CLU closed on owner", "event.severity": 6, + "event.start": "2021-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -1624,31 +1633,52 @@ "input.type": "log", "log.level": "informational", "log.offset": 4949, + "network.bytes": "0", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "fw111", "observer.hostname": "dev01", + "observer.ingress.interface.name": "net", "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", "related.hosts": [ "dev01" ], + "related.ip": [ + "10.10.10.10", + "192.168.2.2" + ], "service.type": "cisco", + "source.address": "10.10.10.10", + "source.ip": "10.10.10.10", + "source.port": 39210, "tags": [ "cisco-asa", "forwarded" ] }, { + "cisco.asa.destination_interface": "unknown", "cisco.asa.message_id": "302023", + "cisco.asa.source_interface": "net", + "destination.address": "192.168.2.2", + "destination.ip": "192.168.2.2", + "destination.port": 39222, "event.action": "firewall-rule", "event.category": [ "network" ], "event.code": 302023, "event.dataset": "cisco.asa", + "event.duration": 0, + "event.end": "2021-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for net:10.10.10.10/10051 to unknown:192.168.2.2/39222 duration 0:00:00 forwarded bytes 0 Forwarding or redirect flow removed to create director or backup flow", + "event.reason": "Forwarding or redirect flow removed to create director or backup flow", "event.severity": 6, + "event.start": "2021-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -1658,14 +1688,26 @@ "input.type": "log", "log.level": "informational", "log.offset": 5142, + "network.bytes": "0", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "net", "observer.hostname": "dev01", + "observer.ingress.interface.name": "unknown", "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", "related.hosts": [ "dev01" ], + "related.ip": [ + "10.10.10.10", + "192.168.2.2" + ], "service.type": "cisco", + "source.address": "10.10.10.10", + "source.ip": "10.10.10.10", + "source.port": 10051, "tags": [ "cisco-asa", "forwarded" diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index bac9d9badc7..44bbb22f088 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -319,6 +319,10 @@ processors: if: "ctx._temp_.cisco.message_id == '302022'" field: "message" pattern: "Built %{} stub %{network.transport} connection for %{_temp_.cisco.source_interface}:%{source.address}/%{source.port} to %{_temp_.cisco.destination_interface}:%{destination.address}/%{destination.port}" + - dissect: + if: "ctx._temp_.cisco.message_id == '302023'" + field: "message" + pattern: "Teardown stub %{network.transport} connection for %{_temp_.cisco.source_interface}:%{source.address}/%{source.port} to %{_temp_.cisco.destination_interface}:%{destination.address}/%{destination.port} duration %{_temp_.duration_hms} forwarded bytes %{network.bytes} %{event.reason}" - grok: if: "ctx._temp_.cisco.message_id == '304001'" field: "message" diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml index a0dfd07c90a..d4b26c49ce8 100644 --- a/x-pack/filebeat/module/cisco/umbrella/config/input.yml +++ b/x-pack/filebeat/module/cisco/umbrella/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "s3" }} +{{ if eq .input "aws-s3" }} -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} access_key_id: {{ .access_key_id }} secret_access_key: {{ .secret_access_key }} diff --git a/x-pack/filebeat/module/mssql/_meta/config.yml b/x-pack/filebeat/module/mssql/_meta/config.yml index 652ca891056..a56e658f7b7 100644 --- a/x-pack/filebeat/module/mssql/_meta/config.yml +++ b/x-pack/filebeat/module/mssql/_meta/config.yml @@ -5,4 +5,4 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/x-pack/filebeat/module/mssql/_meta/docs.asciidoc b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc index 2861d2754ee..9defa42e5c3 100644 --- a/x-pack/filebeat/module/mssql/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc @@ -15,14 +15,14 @@ include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] The following example shows how to set paths in the +modules.d/{modulename}.yml+ -file to override the default paths for Træfik logs: +file to override the default paths for MSSQL logs: ["source","yaml",subs="attributes"] ----- - module: mssql log: enabled: true - var.paths: ["/var/opt/mssql/log/error*"] + var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] ----- @@ -30,7 +30,7 @@ To specify the same settings at the command line, you use: ["source","sh",subs="attributes"] ----- --M "mssql.log.var.paths=[/var/opt/mssql/log/error*]" +-M "mssql.log.var.paths=['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']" ----- //set the fileset name used in the included example diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index 9bfbeeecbc9..92bf9800d14 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -96,6 +96,8 @@ processors: } else { ctx.network.protocol = v; } + } else if (v instanceof List) { + ctx.event[k] = new ArrayList(v); } else { ctx.event[k] = v; } diff --git a/x-pack/filebeat/modules.d/cisco.yml.disabled b/x-pack/filebeat/modules.d/cisco.yml.disabled index d181f01abd3..fedb2c03d09 100644 --- a/x-pack/filebeat/modules.d/cisco.yml.disabled +++ b/x-pack/filebeat/modules.d/cisco.yml.disabled @@ -112,7 +112,7 @@ umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input diff --git a/x-pack/filebeat/modules.d/mssql.yml.disabled b/x-pack/filebeat/modules.d/mssql.yml.disabled index 5e03b661da8..3fdaac9e8a6 100644 --- a/x-pack/filebeat/modules.d/mssql.yml.disabled +++ b/x-pack/filebeat/modules.d/mssql.yml.disabled @@ -8,4 +8,4 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 907a989eb4d..77daddd7a86 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index b41a4035d68..d96ab60094e 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -815,6 +815,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -942,6 +948,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1221,6 +1233,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1418,6 +1436,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 37e3e2ed122..63501b14e2d 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -707,6 +707,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -834,6 +840,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1022,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1031,6 +1047,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1187,6 +1209,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1483,6 +1511,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1680,6 +1714,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 06ca7a1ffad..1a0c44db398 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 97d06a6a255..fafe8e7336b 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -24,7 +25,7 @@ type ConfigAWS struct { SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` RoleArn string `config:"role_arn"` - AWSPartition string `config:"aws_partition"` + AWSPartition string `config:"aws_partition"` // Deprecated. } // GetAWSCredentials function gets aws credentials from the config. @@ -104,3 +105,11 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName } return awsConfig } + +// Validate checks for deprecated config option +func (c ConfigAWS) Validate() error { + if c.AWSPartition != "" { + cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.") + } + return nil +} diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a9cb4ab8e88..a3a242554b4 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -9,9 +9,14 @@ To configure AWS credentials, either put the credentials into the {beatname_uc} * *session_token*: required when using temporary security credentials. * *credential_profile_name*: profile name in shared credentials file. * *shared_credential_file*: directory of the shared credentials file. -* *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. -* *aws_partition*: AWS region parttion name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`. +* *endpoint*: URL of the entry point for an AWS web service. +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. [float] ==== Supported Formats diff --git a/x-pack/libbeat/management/fleet/manager.go b/x-pack/libbeat/management/fleet/manager.go index dc0185a04ac..8aec0af1800 100644 --- a/x-pack/libbeat/management/fleet/manager.go +++ b/x-pack/libbeat/management/fleet/manager.go @@ -169,7 +169,7 @@ func (cm *Manager) OnConfig(s string) { if errs := cm.apply(blocks); !errs.IsEmpty() { // `cm.apply` already logs the errors; currently allow beat to run degraded - cm.UpdateStatus(management.Degraded, errs.Error()) + cm.UpdateStatus(management.Failed, errs.Error()) return } diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index ce7368f672d..19f9941ee47 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -49,7 +49,6 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" - #- "windows-7-32-bit" https://github.com/elastic/beats/issues/19835 windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. @@ -116,3 +115,14 @@ stages: - "windows-7" branches: true ## for all the branches tags: true ## for all the tags +# windows-7-32: +# mage: "mage build unitTest" +# platforms: ## override default labels in this specific stage. +# - "windows-7-32-bit" +# when: ## Override the top-level when. +# comments: +# - "/test x-pack/metricbeat for windows-7-32" +# labels: +# - "windows-7-32" +# branches: true ## for all the branches +# tags: true ## for all the tags diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 66c0e02eaa2..71eaa8f800f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1807,6 +1807,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1934,6 +1940,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2122,6 +2134,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -2131,6 +2147,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2287,6 +2309,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2583,6 +2611,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2780,6 +2814,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index e4e55e82136..df18966b2af 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -23,7 +23,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -32,6 +32,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 167e6a088a0..f3a7caf6cc8 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -6,6 +6,7 @@ package aws import ( "context" + "fmt" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -16,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" @@ -78,20 +78,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig) if err != nil { - return nil, errors.Wrap(err, "failed to get aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err) } _, err = awsConfig.Credentials.Retrieve() if err != nil { - return nil, errors.Wrap(err, "failed to retrieve aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err) } + base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint) metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, + Endpoint: config.AWSConfig.Endpoint, } base.Logger().Debug("Metricset level config for period: ", metricSet.Period) @@ -100,6 +102,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Get IAM account name, set region by aws_partition, default is aws global partition // refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html + if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" { + base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.") + } switch config.AWSConfig.AWSPartition { case "aws-cn": awsConfig.Region = "cn-north-1" @@ -109,6 +114,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig.Region = "us-east-1" } + // If regions in config is not empty, then overwrite the awsConfig.Region + if len(config.Regions) > 0 { + awsConfig.Region = config.Regions[0] + } + // Get IAM account id svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint( config.AWSConfig.Endpoint, "sts", "", awsConfig)) @@ -151,7 +161,7 @@ func getRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error req := svc.DescribeRegionsRequest(input) output, err := req.Send(context.TODO()) if err != nil { - err = errors.Wrap(err, "Failed DescribeRegions") + err = fmt.Errorf("failed DescribeRegions: %w", err) return } for _, region := range output.Regions { diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc index 6d5983fe63d..1bc226bdd53 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc @@ -40,12 +40,10 @@ aws --region us-east-1 secretsmanager create-secret --name ELASTIC_CLOUD_ID --se [float] ==== AWS CloudFormation Template Example -Here is an example of AWS CloudFormation template to create a new cluster, -create a task definition that runs Metricbeat container and start the service. -Please copy this section into a `cloud_formation.yml` file locally and replace -``, ``, and `` with -your own preferred names. Also you can find `` and -`` values from AWS secret manager. +Here is an example of AWS CloudFormation template for testing purpose only. +Please replace this with actual applications. This template shows how to define +a new cluster, how to create a task definition with multiple containers(including +Metricbeat), and how to start the service. [source,yaml] ---- @@ -53,22 +51,36 @@ AWSTemplateFormatVersion: "2010-09-09" Parameters: SubnetID: Type: String + CloudIDArn: + Type: String + CloudAuthArn: + Type: String + ClusterName: + Type: String + RoleName: + Type: String + TaskName: + Type: String + ServiceName: + Type: String + LogGroupName: + Type: String Resources: Cluster: Type: AWS::ECS::Cluster Properties: - ClusterName: + ClusterName: !Ref ClusterName ClusterSettings: - Name: containerInsights Value: enabled LogGroup: Type: AWS::Logs::LogGroup Properties: - LogGroupName: + LogGroupName: !Ref LogGroupName ExecutionRole: Type: AWS::IAM::Role Properties: - RoleName: ecsFargateTaskExecutionRole + RoleName: !Ref RoleName AssumeRolePolicyDocument: Statement: - Effect: Allow @@ -86,24 +98,24 @@ Resources: Action: - secretsmanager:GetSecretValue Resource: - - - - + - !Ref CloudIDArn + - !Ref CloudAuthArn TaskDefinition: Type: AWS::ECS::TaskDefinition Properties: - Family: deployment-task-metricbeat + Family: !Ref TaskName Cpu: 256 Memory: 512 NetworkMode: awsvpc ExecutionRoleArn: !Ref ExecutionRole ContainerDefinitions: - - Name: deployment-task-metricbeat-container - Image: kaiyansheng/metricbeat-awsfargate:v1 + - Name: metricbeat-container + Image: docker.elastic.co/beats/metricbeat:7.11.0-SNAPSHOT Secrets: - Name: ELASTIC_CLOUD_ID - ValueFrom: + ValueFrom: !Ref CloudIDArn - Name: ELASTIC_CLOUD_AUTH - ValueFrom: + ValueFrom: !Ref CloudAuthArn LogConfiguration: LogDriver: awslogs Options: @@ -114,14 +126,25 @@ Resources: - sh - -c Command: - - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + - ./metricbeat setup -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + - Name: stress-test + Image: containerstack/alpine-stress + Essential: false + DependsOn: + - ContainerName: metricbeat-container + Condition: START + EntryPoint: + - sh + - -c + Command: + - stress --cpu 8 --io 4 --vm 2 --vm-bytes 128M --timeout 6000s RequiresCompatibilities: - EC2 - FARGATE Service: Type: AWS::ECS::Service Properties: - ServiceName: + ServiceName: !Ref ServiceName Cluster: !Ref Cluster TaskDefinition: !Ref TaskDefinition DesiredCount: 1 @@ -135,12 +158,13 @@ Resources: [float] ==== Create CloudFormation Stack -Here is the AWS CLI to create a stack using the CloudFormation config file above: +Once the CloudFormation template is saved locally into `clouformation.yml`, AWS +CLI can be used to create a stack using one command: ---- -aws --region us-east-1 cloudformation create-stack --stack-name --template-body file://./cloudformation.yml --capabilities CAPABILITY_NAMED_IAM --parameters 'ParameterKey=SubnetID,ParameterValue=' +aws --region us-east-1 cloudformation create-stack --stack-name --template-body file://./cloudformation.yml --capabilities CAPABILITY_NAMED_IAM --parameters ParameterKey=SubnetID,ParameterValue= ParameterKey=CloudAuthArn,ParameterValue= ParameterKey=CloudIDArn,ParameterValue= ParameterKey=ClusterName,ParameterValue= ParameterKey=RoleName,ParameterValue= ParameterKey=TaskName,ParameterValue= ParameterKey=ServiceName,ParameterValue= ParameterKey=LogGroupName,ParameterValue= ---- -Make sure to replace `` with your own subnet in this command. Please go +Make sure to replace `` with your own subnet in this command. Please go to Services -> VPC -> Subnets to find subnet ID to use. You can also add several more containers under the TaskDefinition section. @@ -149,5 +173,12 @@ more containers under the TaskDefinition section. Here is the AWS CLI to delete a stack including the cluster, task definition and all containers: ---- -aws cloudformation delete-stack --stack-name +aws --region us-east-1 cloudformation delete-stack --stack-name ---- + +[float] +=== Dashboard + +The task_stats metricset comes with a predefined dashboard. For example: + +image::./images/metricbeat-awsfargate-overview.png[] diff --git a/x-pack/metricbeat/module/cloudfoundry/v1.go b/x-pack/metricbeat/module/cloudfoundry/v1.go index db7f6b500fa..8ad2e1e6e2a 100644 --- a/x-pack/metricbeat/module/cloudfoundry/v1.go +++ b/x-pack/metricbeat/module/cloudfoundry/v1.go @@ -61,11 +61,10 @@ func (m *ModuleV1) RunContainerReporter(reporter mb.PushReporterV2) { } func (m *ModuleV1) subscribe(eventType cfcommon.EventType, reporter mb.PushReporterV2) { - go m.run() - m.subscriptions <- subscription{ + go m.run(subscription{ eventType: eventType, reporter: reporter, - } + }) } func (m *ModuleV1) unsubscribe(eventType cfcommon.EventType, reporter mb.PushReporterV2) { @@ -80,8 +79,11 @@ func (m *ModuleV1) callback(event cfcommon.Event) { m.events <- event } -func (m *ModuleV1) run() { +// run ensures that the module is running with the passed subscription +func (m *ModuleV1) run(s subscription) { if !m.running.CAS(false, true) { + // Module is already running, queue subscription for current dispatcher. + m.subscriptions <- s return } defer func() { m.running.Store(false) }() @@ -91,6 +93,10 @@ func (m *ModuleV1) run() { dispatcher := newEventDispatcher(m.log) + // Ensure that the initial subscription is configured before starting the loop, + // this is specially relevant to make tests more deterministic. + dispatcher.handleSubscription(s) + for { // Handle subscriptions and events dispatching on the same // goroutine so locking is not needed. diff --git a/x-pack/metricbeat/module/cockroachdb/status/status_test.go b/x-pack/metricbeat/module/cockroachdb/status/status_test.go index fdfb099f4e5..bb6bab07097 100644 --- a/x-pack/metricbeat/module/cockroachdb/status/status_test.go +++ b/x-pack/metricbeat/module/cockroachdb/status/status_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package status diff --git a/x-pack/metricbeat/module/coredns/stats/stats_test.go b/x-pack/metricbeat/module/coredns/stats/stats_test.go index 6daaeb4881a..106e8e10480 100644 --- a/x-pack/metricbeat/module/coredns/stats/stats_test.go +++ b/x-pack/metricbeat/module/coredns/stats/stats_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package stats diff --git a/x-pack/metricbeat/module/istio/citadel/citadel_test.go b/x-pack/metricbeat/module/istio/citadel/citadel_test.go index 41bced9954b..239b34644ce 100644 --- a/x-pack/metricbeat/module/istio/citadel/citadel_test.go +++ b/x-pack/metricbeat/module/istio/citadel/citadel_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package citadel diff --git a/x-pack/metricbeat/module/istio/galley/galley_test.go b/x-pack/metricbeat/module/istio/galley/galley_test.go index debd92bac63..0dbd1b8832c 100644 --- a/x-pack/metricbeat/module/istio/galley/galley_test.go +++ b/x-pack/metricbeat/module/istio/galley/galley_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package galley diff --git a/x-pack/metricbeat/module/istio/mesh/mesh_test.go b/x-pack/metricbeat/module/istio/mesh/mesh_test.go index f27689ff60f..71d475d9ece 100644 --- a/x-pack/metricbeat/module/istio/mesh/mesh_test.go +++ b/x-pack/metricbeat/module/istio/mesh/mesh_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package mesh diff --git a/x-pack/metricbeat/module/istio/proxy/proxy_test.go b/x-pack/metricbeat/module/istio/proxy/proxy_test.go index eec0bfec301..48f3b104296 100644 --- a/x-pack/metricbeat/module/istio/proxy/proxy_test.go +++ b/x-pack/metricbeat/module/istio/proxy/proxy_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package proxy diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram_test.go b/x-pack/metricbeat/module/prometheus/collector/histogram_test.go index 460d0f3fffe..594a67de504 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram_test.go @@ -2,7 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// skipping tests on windows 32 bit versions, not supported +// +build !integration,!windows,!386 package collector diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 57142c14263..9aee618bed8 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1024,6 +1024,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1151,6 +1157,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1339,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -1348,6 +1364,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1504,6 +1526,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1800,6 +1828,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1997,6 +2031,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js index 56cdced6b51..44d0e8eb34d 100644 --- a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js +++ b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js @@ -180,6 +180,7 @@ var security = (function () { "4647": [["authentication"], ["end"], "logged-out"], "4648": [["authentication"], ["start"], "logged-in-explicit"], "4657": [["configuration"], ["change"], "registry-value-modified"], + "4670": [["iam", "configuration"],["admin", "change"],"permissions-changed"], "4672": [["iam"], ["admin"], "logged-in-special"], "4673": [["iam"], ["admin"], "privileged-service-called"], "4674": [["iam"], ["admin"], "privileged-operation"], @@ -197,6 +198,8 @@ var security = (function () { "4714": [["configuration"], ["change"], "encrypted-data-recovery-policy-changed"], "4715": [["configuration"], ["change"], "object-audit-policy-changed"], "4716": [["configuration"], ["change"], "trusted-domain-information-changed"], + "4717": [["iam", "configuration"],["admin", "change"],"system-security-access-granted"], + "4718": [["iam", "configuration"],["admin", "deletion"],"system-security-access-removed"], "4719": [["iam", "configuration"], ["admin", "change"], "changed-audit-config"], // remove iam and admin "4720": [["iam"], ["user", "creation"], "added-user-account"], "4722": [["iam"], ["user", "change"], "enabled-user-account"], @@ -252,7 +255,14 @@ var security = (function () { "4781": [["iam"], ["user", "change"], "renamed-user-account"], "4798": [["iam"], ["user", "info"], "group-membership-enumerated"], // process enumerates the local groups to which the specified user belongs "4799": [["iam"], ["group", "info"], "user-member-enumerated"], // a process enumerates the members of the specified local group - "4912": [["configuration"], ["change"], "per-user-audit-policy-changed"], + "4817": [["iam", "configuration"], ["admin", "change"],"object-audit-changed"], + "4902": [["iam", "configuration"], ["admin", "creation"],"user-audit-policy-created"], + "4904": [["iam", "configuration"], ["admin", "change"],"security-event-source-added"], + "4905": [["iam", "configuration"], ["admin", "deletion"], "security-event-source-removed"], + "4906": [["iam", "configuration"], ["admin", "change"], "crash-on-audit-changed"], + "4907": [["iam", "configuration"], ["admin", "change"], "audit-setting-changed"], + "4908": [["iam", "configuration"], ["admin", "change"], "special-group-table-changed"], + "4912": [["iam", "configuration"], ["admin", "change"], "per-user-audit-policy-changed"], "4950": [["configuration"], ["change"], "windows-firewall-setting-changed"], "4954": [["configuration"], ["change"], "windows-firewall-group-policy-changed"], "4964": [["iam"], ["admin", "group"], "logged-in-special"], @@ -263,16 +273,6 @@ var security = (function () { "5037": [["driver"], ["end"], "windows-firewall-driver-error"], }; - - // Audit Policy Changes Table - // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4719 - var auditActions = { - "8448": "Success Removed", - "8450": "Failure Removed", - "8449": "Success Added", - "8451": "Failure Added", - }; - // Services Types // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4697 var serviceTypes = { @@ -1209,7 +1209,7 @@ var security = (function () { "8448": "Success removed", "8449": "Success Added", "8450": "Failure removed", - "8451": "Failure added", + "8451": "Failure Added", "8452": "Success include removed", "8453": "Success include added", "8454": "Success exclude removed", @@ -1351,6 +1351,250 @@ var security = (function () { "16903": "Publish", }; + // Trust Types + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustTypes = { + "1": "TRUST_TYPE_DOWNLEVEL", + "2": "TRUST_TYPE_UPLEVEL", + "3": "TRUST_TYPE_MIT", + "4": "TRUST_TYPE_DCE" + } + + // Trust Direction + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustDirection = { + "0": "TRUST_DIRECTION_DISABLED", + "1": "TRUST_DIRECTION_INBOUND", + "2": "TRUST_DIRECTION_OUTBOUND", + "3": "TRUST_DIRECTION_BIDIRECTIONAL" + } + + // Trust Attributes + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustAttributes = { + "0": "UNDEFINED", + "1": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "2": "TRUST_ATTRIBUTE_UPLEVEL_ONLY", + "4": "TRUST_ATTRIBUTE_QUARANTINED_DOMAIN", + "8": "TRUST_ATTRIBUTE_FOREST_TRANSITIVE", + "16": "TRUST_ATTRIBUTE_CROSS_ORGANIZATION", + "32": "TRUST_ATTRIBUTE_WITHIN_FOREST", + "64": "TRUST_ATTRIBUTE_TREAT_AS_EXTERNAL", + "128": "TRUST_ATTRIBUTE_USES_RC4_ENCRYPTION", + "512": "TRUST_ATTRIBUTE_CROSS_ORGANIZATION_NO_TGT_DELEGATION", + "1024": "TRUST_ATTRIBUTE_PIM_TRUST" + } + + // SDDL Ace Types + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4715 + // https://docs.microsoft.com/en-us/windows/win32/secauthz/ace-strings + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/f4296d69-1c0f-491f-9587-a960b292d070 + var aceTypes = { + "A": "Access Allowed", + "D": "Access Denied", + "OA": "Object Access Allowed", + "OD": "Object Access Denied", + "AU": "System Audit", + "AL": "System Alarm", + "OU": "System Object Audit", + "OL": "System Object Alarm", + "ML": "System Mandatory Label", + "SP": "Central Policy ID" + } + + // SDDL Permissions + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4715 + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/f4296d69-1c0f-491f-9587-a960b292d070 + var permissionDescription = { + "GA": "Generic All", + "GR": "Generic Read", + "GW": "Generic Write", + "GX": "Generic Execute", + "RC": "Read Permissions", + "SD": "Delete", + "WD": "Modify Permissions", + "WO": "Modify Owner", + "RP": "Read All Properties", + "WP": "Write All Properties", + "CC": "Create All Child Objects", + "DC": "Delete All Child Objects", + "LC": "List Contents", + "SW": "All Validated", + "LO": "List Object", + "DT": "Delete Subtree", + "CR": "All Extended Rights", + "FA": "File All Access", + "FR": "File Generic Read", + "FX": "FILE GENERIC EXECUTE", + "FW": "FILE GENERIC WRITE", + "KA": "KEY ALL ACCESS", + "KR": "KEY READ", + "KW": "KEY WRITE", + "KX": "KEY EXECUTE" + } + + // Known SIDs + // https://support.microsoft.com/en-au/help/243330/well-known-security-identifier"S-in-window"S-operating-systems + // https://docs.microsoft.com/en-us/windows/win32/secauthz/sid-strings + var accountSIDDescription = { + "AO": "Account operators", + "RU": "Alias to allow previous Windows 2000", + "AN": "Anonymous logon", + "AU": "Authenticated users", + "BA": "Built-in administrators", + "BG": "Built-in guests", + "BO": "Backup operators", + "BU": "Built-in users", + "CA": "Certificate server administrators", + "CG": "Creator group", + "CO": "Creator owner", + "DA": "Domain administrators", + "DC": "Domain computers", + "DD": "Domain controllers", + "DG": "Domain guests", + "DU": "Domain users", + "EA": "Enterprise administrators", + "ED": "Enterprise domain controllers", + "WD": "Everyone", + "PA": "Group Policy administrators", + "IU": "Interactively logged-on user", + "LA": "Local administrator", + "LG": "Local guest", + "LS": "Local service account", + "SY": "Local system", + "NU": "Network logon user", + "NO": "Network configuration operators", + "NS": "Network service account", + "PO": "Printer operators", + "PS": "Personal self", + "PU": "Power users", + "RS": "RAS servers group", + "RD": "Terminal server users", + "RE": "Replicator", + "RC": "Restricted code", + "SA": "Schema administrators", + "SO": "Server operators", + "SU": "Service logon user", + "S-1-0": "Null Authority", + "S-1-0-0": "Nobody", + "S-1-1": "World Authority", + "S-1-1-0": "Everyone", + "S-1-16-0": "Untrusted Mandatory Level", + "S-1-16-12288": "High Mandatory Level", + "S-1-16-16384": "System Mandatory Level", + "S-1-16-20480": "Protected Process Mandatory Level", + "S-1-16-28672": "Secure Process Mandatory Level", + "S-1-16-4096": "Low Mandatory Level", + "S-1-16-8192": "Medium Mandatory Level", + "S-1-16-8448": "Medium Plus Mandatory Level", + "S-1-2": "Local Authority", + "S-1-2-0": "Local", + "S-1-2-1": "Console Logon", + "S-1-3": "Creator Authority", + "S-1-3-0": "Creator Owner", + "S-1-3-1": "Creator Group", + "S-1-3-2": "Creator Owner Server", + "S-1-3-3": "Creator Group Server", + "S-1-3-4": "Owner Rights", + "S-1-4": "Non-unique Authority", + "S-1-5": "NT Authority", + "S-1-5-1": "Dialup", + "S-1-5-10": "Principal Self", + "S-1-5-11": "Authenticated Users", + "S-1-5-12": "Restricted Code", + "S-1-5-13": "Terminal Server Users", + "S-1-5-14": "Remote Interactive Logon", + "S-1-5-15": "This Organization", + "S-1-5-17": "This Organization", + "S-1-5-18": "Local System", + "S-1-5-19": "NT Authority", + "S-1-5-2": "Network", + "S-1-5-20": "NT Authority", + "S-1-5-3": "Batch", + "S-1-5-32-544": "Administrators", + "S-1-5-32-545": "Users", + "S-1-5-32-546": "Guests", + "S-1-5-32-547": "Power Users", + "S-1-5-32-548": "Account Operators", + "S-1-5-32-549": "Server Operators", + "S-1-5-32-550": "Print Operators", + "S-1-5-32-551": "Backup Operators", + "S-1-5-32-552": "Replicators", + "S-1-5-32-554": "Builtin\Pre-Windows 2000 Compatible Access", + "S-1-5-32-555": "Builtin\Remote Desktop Users", + "S-1-5-32-556": "Builtin\Network Configuration Operators", + "S-1-5-32-557": "Builtin\Incoming Forest Trust Builders", + "S-1-5-32-558": "Builtin\Performance Monitor Users", + "S-1-5-32-559": "Builtin\Performance Log Users", + "S-1-5-32-560": "Builtin\Windows Authorization Access Group", + "S-1-5-32-561": "Builtin\Terminal Server License Servers", + "S-1-5-32-562": "Builtin\Distributed COM Users", + "S-1-5-32-569": "Builtin\Cryptographic Operators", + "S-1-5-32-573": "Builtin\Event Log Readers", + "S-1-5-32-574": "Builtin\Certificate Service DCOM Access", + "S-1-5-32-575": "Builtin\RDS Remote Access Servers", + "S-1-5-32-576": "Builtin\RDS Endpoint Servers", + "S-1-5-32-577": "Builtin\RDS Management Servers", + "S-1-5-32-578": "Builtin\Hyper-V Administrators", + "S-1-5-32-579": "Builtin\Access Control Assistance Operators", + "S-1-5-32-580": "Builtin\Remote Management Users", + "S-1-5-32-582": "Storage Replica Administrators", + "S-1-5-4": "Interactive", + "S-1-5-5-X-Y": "Logon Session", + "S-1-5-6": "Service", + "S-1-5-64-10": "NTLM Authentication", + "S-1-5-64-14": "SChannel Authentication", + "S-1-5-64-21": "Digest Authentication", + "S-1-5-7": "Anonymous", + "S-1-5-8": "Proxy", + "S-1-5-80": "NT Service", + "S-1-5-80-0": "All Services", + "S-1-5-83-0": "NT Virtual Machine\Virtual Machines", + "S-1-5-9": "Enterprise Domain Controllers", + "S-1-5-90-0": "Windows Manager\Windows Manager Group" + } + + // Domain-specific SIDs + // https://support.microsoft.com/en-au/help/243330/well-known-security-identifiers-in-windows-operating-systems + var domainSpecificSID = { + "498": "Enterprise Read-only Domain Controllers", + "500": "Administrator", + "501": "Guest", + "502": "KRBTGT", + "512": "Domain Admins", + "513": "Domain Users", + "514": "Domain Guests", + "515": "Domain Computers", + "516": "Domain Controllers", + "517": "Cert Publishers", + "518": "Schema Admins", + "519": "Enterprise Admins", + "520": "Group Policy Creator Owners", + "521": "Read-only Domain Controllers", + "522": "Cloneable Domain Controllers", + "526": "Key Admins", + "527": "Enterprise Key Admins", + "553": "RAS and IAS Servers", + "571": "Allowed RODC Password Replication Group", + "572": "Denied RODC Password Replication Group" + } + + // Object Permission Flags + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/7a53f60e-e730-4dfe-bbe9-b21b62eb790b + var permsFlags = [ + [0x80000000, 'Generic Read'], + [0x4000000, 'Generic Write'], + [0x20000000, 'Generic Execute'], + [0x10000000, 'Generic All'], + [0x02000000, 'Maximun Allowed'], + [0x01000000, 'Access System Security'], + [0x00100000, 'Syncronize'], + [0x00080000, 'Write Owner'], + [0x00040000, 'Write DACL'], + [0x00020000, 'Read Control'], + [0x00010000, 'Delete'] + ]; + // lookupMessageCode returns the string associated with the code. key should // be the name of the field in evt containing the code (e.g. %%2313). var lookupMessageCode = function (evt, key) { @@ -1455,7 +1699,7 @@ var security = (function () { var actionResults = []; for (var j = 0; j < codedActions.length; j++) { var actionCode = codedActions[j].replace("%%", '').replace(' ', ''); - actionResults.push(auditActions[actionCode]); + actionResults.push(msobjsMessageTable[actionCode]); } evt.Put("winlog.event_data.AuditPolicyChangesDescription", actionResults); }; @@ -1495,12 +1739,118 @@ var security = (function () { evt.Put("winlog.event_data.StatusDescription", kerberosTktStatusCodes[code]); }; + var translateSID = function(sid){ + var translatedSID = accountSIDDescription[sid]; + if (translatedSID == undefined) { + if (/^S\-1\-5\-21/.test(sid)) { + var uid = sid.match(/[0-9]{1,5}$/g); + if (uid) { + translatedSID = domainSpecificSID[uid]; + } + } + } + if (translatedSID == undefined) { + translatedSID = sid; + } + return translatedSID; + } + + var translatePermissionMask = function(mask) { + if (!mask) { + return; + } + var permCode = parseInt(mask); + var permResult = []; + for (var i = 0; i < permsFlags.length; i++) { + if ((permCode | permsFlags[i][0]) === permCode) { + permResult.push(permsFlags[i][1]); + } + } + if (permResult) { + return permResult; + } else { + return mask; + } + }; + + var translateACL = function(dacl) { + var aceArray = dacl.split(";"); + var aceResult = []; + var aceType = aceArray[0]; + var acePerm = aceArray[2]; + var aceTrustedSid = aceArray[5]; + if (aceTrustedSid) { + aceResult['grantee'] = translateSID(aceTrustedSid); + } + if (aceType) { + aceResult['type'] = aceTypes[aceType]; + } + if (acePerm) { + if (/^0x/.test(acePerm)) { + var perms = translatePermissionMask(acePerm); + } + else { + var perms = [] + var permPairs = acePerm.match(/.{1,2}/g); + for ( var i = 0; i < permPairs.length; i ++) { + perms.push(permissionDescription[permPairs[i]]) + } + } + aceResult['perms'] = perms; + } + return aceResult; + }; + + var enrichSDDL = function(evt, sddl) { + var sddlStr = evt.Get(sddl); + if (!sddlStr) { + return; + } + var sdOwner = sddlStr.match(/^O\:[A-Z]{2}/g); + var sdGroup = sddlStr.match(/^G\:[A-Z]{2}/g); + var sdDacl = sddlStr.match(/(D:([A-Z]*(\(.*\))*))/g); + var sdSacl = sddlStr.match(/(S:([A-Z]*(\(.*\))*))?$/g); + if (sdOwner) { + evt.Put(sddl+"Owner", translateSID(sdOwner)); + } + if (sdGroup) { + evt.Put(sddl+"Group", translateSID(sdGroup)); + } + if (sdDacl) { + // Split each entry of the DACL + var daclList = (sdDacl[0]).match(/\([^*\)]*\)/g); + if (daclList) { + for (var i = 0; i < daclList.length; i++) { + var newDacl = translateACL(daclList[i].replace("(", '').replace(")", '')); + evt.Put(sddl+"Dacl"+i, newDacl['grantee']+" :"+newDacl['type']+" ("+newDacl['perms']+")"); + if ( newDacl['grantee'] === "Administrator" || newDacl['grantee'] === "Guest" || newDacl['grantee'] === "KRBTGT" ) { + evt.AppendTo('related.user', newDacl['grantee']); + } + } + } + } + if (sdSacl) { + // Split each entry of the SACL + var saclList = (sdSacl[0]).match(/\([^*\)]*\)/g); + if (saclList) { + for (var i = 0; i < saclList.length; i++) { + var newSacl = translateACL(saclList[i].replace("(", '').replace(")", '')); + evt.Put(sddl+"Sacl"+i, newSacl['grantee']+" :"+newSacl['type']+" ("+newSacl['perms']+")"); + if ( newSacl['grantee'] === "Administrator" || newSacl['grantee'] === "Guest" || newSacl['grantee'] === "KRBTGT" ) { + evt.AppendTo('related.user', newSacl['grantee']); + } + } + } + } + }; + var addSessionData = new processor.Chain() .Convert({ fields: [ {from: "winlog.event_data.AccountName", to: "user.name"}, {from: "winlog.event_data.AccountDomain", to: "user.domain"}, {from: "winlog.event_data.ClientAddress", to: "source.ip", type: "ip"}, + {from: "winlog.event_data.ClientAddress", to: "related.ip", type: "ip"}, {from: "winlog.event_data.ClientName", to: "source.domain"}, {from: "winlog.event_data.LogonID", to: "winlog.logon.id"}, ], @@ -1529,6 +1879,26 @@ var security = (function () { }) .Build(); + var addTrustInformation = new processor.Chain() + .Add(function(evt) { + var code = evt.Get("winlog.event_data.TdoType"); + if (!code) { + return; + } + evt.Put("winlog.trustType", trustTypes[code]); + code = evt.Get("winlog.event_data.TdoDirection"); + if (!code) { + return; + } + evt.Put("winlog.trustDirection", trustDirection[code]); + code = evt.Get("winlog.event_data.TdoAttributes"); + if (!code) { + return; + } + evt.Put("winlog.trustAttribute", trustAttributes[code]); + + }) + .Build(); var copyTargetUser = new processor.Chain() .Convert({ fields: [ @@ -1635,6 +2005,7 @@ var security = (function () { {from: "winlog.event_data.ProcessId", to: "process.pid", type: "long"}, {from: "winlog.event_data.ProcessName", to: "process.executable"}, {from: "winlog.event_data.IpAddress", to: "source.ip", type: "ip"}, + {from: "winlog.event_data.ClientAddress", to: "related.ip", type: "ip"}, {from: "winlog.event_data.IpPort", to: "source.port", type: "long"}, {from: "winlog.event_data.WorkstationName", to: "source.domain"}, ], @@ -1772,10 +2143,12 @@ var security = (function () { .Add(addEventFields) .Add(function(evt) { var user = evt.Get("winlog.event_data.TargetUserName"); - var res = /^-$/.test(user); - if (!res) { - evt.AppendTo('related.user', user); - } + if (user) { + var res = /^-$/.test(user); + if (!res) { + evt.AppendTo('related.user', user); + } + } }) .Build(); @@ -1831,7 +2204,6 @@ var security = (function () { } evt.AppendTo("related.user", member.split(',')[0].replace('CN=', '').replace('cn=', '')); }) - .Build(); var auditLogCleared = new processor.Chain() @@ -1890,8 +2262,11 @@ var security = (function () { .Add(addEventFields) .Add(function(evt) { var ip = evt.Get("source.ip"); - if (/::ffff:/.test(ip)) { - evt.Put("source.ip", ip.replace("::ffff:", "")); + if (ip) { + if (/::ffff:/.test(ip)) { + evt.Put("source.ip", ip.replace("::ffff:", "")); + evt.Put("related.ip", ip.replace("::ffff:", "")); + } } }) .Build(); @@ -1939,6 +2314,67 @@ var security = (function () { }) .Build(); + var trustDomainMgmtEvts = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(addEventFields) + .Add(addTrustInformation) + .Build(); + + var policyChange = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(addEventFields) + .Build(); + + var objectPolicyChange = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(renameCommonAuthFields) + .Add(addEventFields) + .Add(function(evt) { + var oldSd = evt.Get("winlog.event_data.OldSd"); + var newSd = evt.Get("winlog.event_data.NewSd"); + if (oldSd) { + enrichSDDL(evt, "winlog.event_data.OldSd"); + } + if (newSd) { + enrichSDDL(evt, "winlog.event_data.NewSd"); + } + }) + .Build(); + + var genericAuditChange = new processor.Chain() + .Add(addEventFields) + .Build(); + + var event4908 = new processor.Chain() + .Add(addEventFields) + .Add(function(evt) { + var sids = evt.Get("winlog.event_data.SidList"); + if (!sids) { + return; + } + var sidList = sids.split(/\s+/); + evt.Put("winlog.event_data.SidList", sids.split(/\s+/)); + var sidListDesc = []; + for (var i = 0; i < sidList.length; i++) { + var sidTemp = sidList[i].replace("%", "").replace("{", "").replace("}", "").replace(" ",""); + if (sidTemp) { + sidListDesc.push(translateSID(sidTemp)); + } + } + evt.Put("winlog.event_data.SidListDesc", sidListDesc); + }) + .Build(); + + var securityEventSource = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(renameCommonAuthFields) + .Add(addEventFields) + .Build(); + return { // 1100 - The event logging service has shut down. @@ -1971,6 +2407,9 @@ var security = (function () { // 4648 - A logon was attempted using explicit credentials. 4648: event4648.Run, + // 4670 - Permissions on an object were changed. + 4670: objectPolicyChange.Run, + // 4672 - Special privileges assigned to new logon. 4672: event4672.Run, @@ -2004,6 +2443,24 @@ var security = (function () { // 4702 - A scheduled task was updated. 4702: scheduledTask.Run, + // 4706 - A new trust was created to a domain. + 4706: trustDomainMgmtEvts.Run, + + // 4707 - A trust to a domain was removed. + 4707: trustDomainMgmtEvts.Run, + + // 4713 - Kerberos policy was changed. + 4713: policyChange.Run, + + // 4716 - Trusted domain information was modified. + 4716: trustDomainMgmtEvts.Run, + + // 4717 - System security access was granted to an account. + 4717: policyChange.Run, + + // 4718 - System security access was removed from an account. + 4718: policyChange.Run, + // 4719 - System audit policy was changed. 4719: auditChanged.Run, @@ -2055,6 +2512,9 @@ var security = (function () { // 4737 - A security-enabled global group was changed. 4737: groupMgmtEvts.Run, + // 4739 - A security-enabled global group was changed. + 4739: policyChange.Run, + // 4738 - An user account was changed. 4738: userMgmtEvts.Run, @@ -2166,6 +2626,30 @@ var security = (function () { // 4799 - A security-enabled local group membership was enumerated. 4799: groupMgmtEvts.Run, + // 4817 - Auditing settings on object were changed. + 4817: objectPolicyChange.Run, + + // 4902 - The Per-user audit policy table was created. + 4902: genericAuditChange.Run, + + // 4904 - An attempt was made to register a security event source. + 4904: securityEventSource.Run, + + // 4905 - An attempt was made to unregister a security event source. + 4905: securityEventSource.Run, + + // 4906 - The CrashOnAuditFail value has changed. + 4906: genericAuditChange.Run, + + // 4907 - Auditing settings on object were changed. + 4907: objectPolicyChange.Run, + + // 4908 - Special Groups Logon table modified. + 4908: event4908.Run, + + // 4912 - Per User Audit Policy was changed. + 4912: auditChanged.Run, + // 4964 - Special groups have been assigned to a new logon. 4964: event4964.Run, diff --git a/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx new file mode 100644 index 00000000000..30c2adc8426 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..0666a8b5ac8 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,80 @@ +[ + { + "@timestamp": "2020-07-28T13:22:18.7993488Z", + "event": { + "action": "permissions-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4670, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "process": { + "executable": "C:\\Windows\\System32\\services.exe", + "name": "services.exe", + "pid": 764 + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "HandleId": "0x56c", + "NewSd": "D:(A;;GA;;;SY)(A;;RC;;;OW)(A;;GA;;;S-1-5-80-123231216-2592883651-3715271367-3753151631-4175906628)", + "NewSdDacl0": "Local system :Access Allowed (Generic All)", + "NewSdDacl1": "OW :Access Allowed (Read Permissions)", + "NewSdDacl2": "S-1-5-80-123231216-2592883651-3715271367-3753151631-4175906628 :Access Allowed (Generic All)", + "ObjectName": "-", + "ObjectServer": "Security", + "ObjectType": "Token", + "OldSd": "D:(A;;GA;;;SY)(A;;GA;;;NS)", + "OldSdDacl0": "Local system :Access Allowed (Generic All)", + "OldSdDacl1": "Network service account :Access Allowed (Generic All)", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4670, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 4, + "thread": { + "id": 4604 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 31932, + "task": "Authorization Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx new file mode 100644 index 00000000000..e87e18d5441 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..7cdf639ce48 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-07-27T09:42:48.3690009Z", + "event": { + "action": "domain-trust-added", + "category": [ + "configuration" + ], + "code": 4706, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "creation" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{be129571-63f8-0000-a795-12bef863d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "192.168.230.153", + "DomainSid": "S-1-0-0", + "SidFilteringEnabled": "%%1796", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TdoAttributes": "1", + "TdoDirection": "3", + "TdoType": "3" + }, + "event_id": 4706, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3056 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 6017, + "task": "Authentication Policy Change", + "trustAttribute": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "trustDirection": "TRUST_DIRECTION_BIDIRECTIONAL", + "trustType": "TRUST_TYPE_MIT" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx new file mode 100644 index 00000000000..3f9f51c4f98 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..d16ff334435 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2020-07-28T06:18:04.600444Z", + "event": { + "action": "domain-trust-removed", + "category": [ + "configuration" + ], + "code": 4707, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "deletion" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "192.168.230.153", + "DomainSid": "S-1-0-0", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" + }, + "event_id": 4707, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 2012 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 13679, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx new file mode 100644 index 00000000000..5d4bb4d159a Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..9dcfe4ddb59 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2020-07-28T10:15:43.4951882Z", + "event": { + "action": "kerberos-policy-changed", + "category": [ + "configuration" + ], + "code": 4713, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{be129571-63f8-0000-a795-12bef863d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "KerberosPolicyChange": "KerMinT: 0x53d1ac1000 (0x53ade8ca00); KerMaxR: 0x649534e0000 (0x58028e44000); KerProxy: 0xd693a400 (0xb2d05e00); ", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4713, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 2012 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 21265, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx new file mode 100644 index 00000000000..39053c8797e Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..6e43b04c6f3 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-07-28T08:17:00.4706442Z", + "event": { + "action": "trusted-domain-information-changed", + "category": [ + "configuration" + ], + "code": 4716, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{be129571-63f8-0000-a795-12bef863d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "-", + "DomainSid": "S-1-0-0", + "SidFilteringEnabled": "-", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TdoAttributes": "1", + "TdoDirection": "3", + "TdoType": "3" + }, + "event_id": 4716, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3776 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 14929, + "task": "Authentication Policy Change", + "trustAttribute": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "trustDirection": "TRUST_DIRECTION_BIDIRECTIONAL", + "trustType": "TRUST_TYPE_MIT" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx new file mode 100644 index 00000000000..f27db52c536 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..fe3d49133e0 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,67 @@ +[ + { + "@timestamp": "2020-07-27T09:30:41.9034803Z", + "event": { + "action": "system-security-access-granted", + "category": [ + "iam", + "configuration" + ], + "code": 4717, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "WORKGROUP", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{b69bb9ff-63f5-0000-35ba-9bb6f563d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6", + "event_data": { + "AccessGranted": "SeNetworkLogonRight", + "SubjectDomainName": "WORKGROUP", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18", + "TargetSid": "S-1-5-9" + }, + "event_id": 4717, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 820 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 1571, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx new file mode 100644 index 00000000000..43ef6f5a787 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..6e5fc0f6d54 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,67 @@ +[ + { + "@timestamp": "2020-07-27T09:30:41.8778082Z", + "event": { + "action": "system-security-access-removed", + "category": [ + "iam", + "configuration" + ], + "code": 4718, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "deletion" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "WORKGROUP", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{b69bb9ff-63f5-0000-35ba-9bb6f563d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6", + "event_data": { + "AccessRemoved": "SeNetworkLogonRight", + "SubjectDomainName": "WORKGROUP", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18", + "TargetSid": "S-1-5-32-545" + }, + "event_id": 4718, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 820 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 1565, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx new file mode 100644 index 00000000000..c30822f456f Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..4bc9323ce3f --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-08-18T13:45:57.4803543Z", + "event": { + "action": "changed-audit-config", + "category": [ + "iam", + "configuration" + ], + "code": 4719, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{65461d39-753f-0000-731d-46653f75d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "AuditPolicyChanges": "%%8448", + "AuditPolicyChangesDescription": [ + "Success removed" + ], + "Category": "Object Access", + "CategoryId": "%%8274", + "SubCategory": "Other Object Access Events", + "SubcategoryGuid": "{0cce9227-69ae-11d9-bed3-505054503030}", + "SubcategoryId": "%%12804", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x44d7d", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" + }, + "event_id": 4719, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x44d7d" + }, + "opcode": "Info", + "process": { + "pid": 780, + "thread": { + "id": 2764 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 123879, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx new file mode 100644 index 00000000000..60063744d4c Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..4035618ea1d --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,71 @@ +[ + { + "@timestamp": "2020-07-27T09:34:50.1578005Z", + "event": { + "action": "domain-policy-changed", + "category": [ + "configuration" + ], + "code": 4739, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainBehaviorVersion": "-", + "DomainName": "TEST", + "DomainPolicyChanged": "Password Policy", + "DomainSid": "S-1-5-21-2024912787-2692429404-2351956786", + "MachineAccountQuota": "-", + "MixedDomainMode": "-", + "OemInformation": "-", + "PasswordHistoryLength": "-", + "PrivilegeList": "-", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4739, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 812 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 3532, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx new file mode 100644 index 00000000000..7dda9113651 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..71607b7242c --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-08-17T12:49:09.4942066Z", + "event": { + "action": "object-audit-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4817, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": [ + "WIN-BVM4LI1L1Q6$", + "Administrator" + ] + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{dfcd2c2a-7481-0000-682c-cddf8174d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "NewSd": "S:(AU;SA;CCDCLCSWRPWPDTLOCRSDRCWDWO;;;S-1-5-21-2024912787-2692429404-2351956786-500)(AU;SA;CCDCLCSWRPWPDTLOCRSDRCWDWO;;;S-1-5-21-2024912787-2692429404-2351956786-1000)", + "NewSdSacl0": "Administrator :System Audit (Create All Child Objects,Delete All Child Objects,List Contents,All Validated,Read All Properties,Write All Properties,Delete Subtree,List Object,All Extended Rights,Delete,Read Permissions,Modify Permissions,Modify Owner)", + "NewSdSacl1": "S-1-5-21-2024912787-2692429404-2351956786-1000 :System Audit (Create All Child Objects,Delete All Child Objects,List Contents,All Validated,Read All Properties,Write All Properties,Delete Subtree,List Object,All Extended Rights,Delete,Read Permissions,Modify Permissions,Modify Owner)", + "ObjectName": "File", + "ObjectServer": "LSA", + "ObjectType": "Global SACL", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4817, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3052 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 114278, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx new file mode 100644 index 00000000000..695eae2bb3b Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..0c21de310ab --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,51 @@ +[ + { + "@timestamp": "2020-08-19T06:07:08.801981Z", + "event": { + "action": "user-audit-policy-created", + "category": [ + "iam", + "configuration" + ], + "code": 4902, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "creation" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "PuaCount": "0", + "PuaPolicyId": "0x9fd2" + }, + "event_id": 4902, + "keywords": [ + "Audit Success" + ], + "opcode": "Info", + "process": { + "pid": 784, + "thread": { + "id": 832 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 140273, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx new file mode 100644 index 00000000000..0fde0144e53 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..cb92cffa1b2 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-08-19T07:56:52.019802Z", + "event": { + "action": "security-event-source-added", + "category": [ + "iam", + "configuration" + ], + "code": 4904, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "process": { + "executable": "C:\\Windows\\System32\\inetsrv\\inetinfo.exe", + "name": "inetinfo.exe", + "pid": 3608 + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{dab46f85-75ee-0000-c36f-b4daee75d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "AuditSourceName": "IIS-METABASE", + "EventSourceId": "0x460422", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4904, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 784, + "thread": { + "id": 824 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 146939, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx new file mode 100644 index 00000000000..7e43d61f913 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..7b5c2e7c0c7 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-08-19T07:56:51.5792901Z", + "event": { + "action": "security-event-source-removed", + "category": [ + "iam", + "configuration" + ], + "code": 4905, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "deletion" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "process": { + "executable": "-", + "name": "-", + "pid": 4964 + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{dab46f85-75ee-0000-c36f-b4daee75d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "AuditSourceName": "IIS-METABASE", + "EventSourceId": "0x457b22", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4905, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 784, + "thread": { + "id": 824 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 146938, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx new file mode 100644 index 00000000000..43a47a29d32 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..9711989c89e --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,50 @@ +[ + { + "@timestamp": "2020-08-18T09:19:00.2372249Z", + "event": { + "action": "crash-on-audit-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4906, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "CrashOnAuditFailValue": "1" + }, + "event_id": 4906, + "keywords": [ + "Audit Success" + ], + "opcode": "Info", + "process": { + "pid": 780, + "thread": { + "id": 804 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 123786, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx new file mode 100644 index 00000000000..6a9530c298e Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..32dd648fc2a --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,75 @@ +[ + { + "@timestamp": "2020-08-19T07:56:17.1121901Z", + "event": { + "action": "audit-setting-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4907, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "process": { + "executable": "C:\\Windows\\WinSxS\\amd64_microsoft-windows-servicingstack_31bf3856ad364e35_10.0.14393.1883_none_7ed84bd822106081\\TiWorker.exe", + "name": "TiWorker.exe", + "pid": 4300 + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "HandleId": "0x93c", + "NewSd": "S:ARAI(AU;SAFA;DCLCRPCRSDWDWO;;;WD)", + "NewSdSacl0": "Everyone :System Audit (Delete All Child Objects,List Contents,Read All Properties,All Extended Rights,Delete,Modify Permissions,Modify Owner)", + "ObjectName": "C:\\Windows\\SysWOW64\\WindowsPowerShell\\v1.0\\Modules\\RemoteAccess\\RemoteAccess.psd1", + "ObjectServer": "Security", + "ObjectType": "File", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4907, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 4, + "thread": { + "id": 408 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 146933, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx new file mode 100644 index 00000000000..e319bcf9a0b Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..fcbdbce1d3d --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2020-08-19T06:07:25.0461779Z", + "event": { + "action": "special-group-table-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4908, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "SidList": [ + "", + "%{S-1-5-32-544}", + "%{S-1-5-32-123-54-65}" + ], + "SidListDesc": [ + "Administrators", + "S-1-5-32-123-54-65" + ] + }, + "event_id": 4908, + "keywords": [ + "Audit Success" + ], + "opcode": "Info", + "process": { + "pid": 784, + "thread": { + "id": 808 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 140274, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx new file mode 100644 index 00000000000..15a93a947a2 Binary files /dev/null and b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx differ diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json new file mode 100644 index 00000000000..5e9a933c7bb --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,70 @@ +[ + { + "@timestamp": "2020-08-18T14:36:41.2936839Z", + "event": { + "action": "per-user-audit-policy-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4912, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{65461d39-753f-0000-731d-46653f75d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "AuditPolicyChanges": "%%8452", + "CategoryId": "%%8276", + "SubcategoryGuid": "{0cce924a-69ae-11d9-bed3-505054503030}", + "SubcategoryId": "%%13317", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x44d7d", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TargetUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" + }, + "event_id": 4912, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x44d7d" + }, + "opcode": "Info", + "process": { + "pid": 780, + "thread": { + "id": 3300 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 123917, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json index 6519408002c..977ea0fe116 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "192.168.5.44", "user": "MPUIG" }, "source": { diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json index 649db8b0e23..f7944a0c686 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "10.100.150.9", "user": "at_adm" }, "source": { diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json index 12c23f0a09d..93f89a592a6 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "10.100.150.17", "user": "at_adm" }, "source": { diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 03652ce2788..636325f5571 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -495,6 +495,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -622,6 +628,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -810,6 +822,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true @@ -819,6 +835,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -975,6 +997,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1271,6 +1299,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1468,6 +1502,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary