From c7f855a871246f7df6674aaf427c20cbfe01d2b1 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 14 Jul 2022 17:04:50 +0930 Subject: [PATCH 1/2] windows: enrich user details from MemberSid where possible --- packages/windows/changelog.yml | 5 +++++ .../forwarded/agent/stream/winlog.yml.hbs | 8 +++++++- .../elasticsearch/ingest_pipeline/default.yml | 17 +++++++++++++++++ .../powershell/agent/stream/winlog.yml.hbs | 8 +++++++- .../elasticsearch/ingest_pipeline/default.yml | 13 +++++++++++++ .../agent/stream/winlog.yml.hbs | 8 +++++++- .../elasticsearch/ingest_pipeline/default.yml | 13 +++++++++++++ .../agent/stream/winlog.yml.hbs | 8 +++++++- .../elasticsearch/ingest_pipeline/default.yml | 13 +++++++++++++ packages/windows/manifest.yml | 2 +- 10 files changed, 90 insertions(+), 5 deletions(-) diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml index 0f6214979de..79e83d16692 100644 --- a/packages/windows/changelog.yml +++ b/packages/windows/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.14.0" + changes: + - description: Use MemberSid to enrich for user name and domain where possible. + type: enhancement + link: https://github.com/elastic/integrations/pull/3707 - version: "1.13.0" changes: - description: Added Processors for service datatstream. diff --git a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs index 965be31d600..90f155b50ea 100644 --- a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs @@ -21,7 +21,13 @@ include_xml: true {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true {{/contains}} -{{#if processors.length}} processors: + - translate_sid: + field: winlog.event_data.MemberSid + account_name_target: _user.name + domain_target: _user.domain + ignore_missing: true + ignore_failure: true +{{#if processors.length}} {{processors}} {{/if}} diff --git a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml index 6a274d1d5a7..b7c8ec038be 100644 --- a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml @@ -13,6 +13,23 @@ processors: - pipeline: name: '{{ IngestPipeline "sysmon_operational" }}' if: ctx?.winlog?.channel != null && ctx?.winlog?.channel == "Microsoft-Windows-Sysmon/Operational" + + # Get user details from the translate_sid processor enrichment + # if they are available and we don't already have them. + - rename: + field: _user.name + target_field: user.name + ignore_failure: true + ignore_missing: true + - rename: + field: _user.domain + target_field: user.domain + ignore_failure: true + ignore_missing: true + - remove: + field: _user + ignore_missing: true + on_failure: - set: field: "error.message" diff --git a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs index 8695fa23005..afbfc336859 100644 --- a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs @@ -18,7 +18,13 @@ tags: {{#if preserve_original_event}} include_xml: true {{/if}} -{{#if processors.length}} processors: + - translate_sid: + field: winlog.event_data.MemberSid + account_name_target: _user.name + domain_target: _user.domain + ignore_missing: true + ignore_failure: true +{{#if processors.length}} {{processors}} {{/if}} diff --git a/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml index fc3c153fe15..95d99c803d9 100644 --- a/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml @@ -117,6 +117,18 @@ processors: ignore_failure: true allow_duplicates: false if: ctx?.user?.name != null + # Get user details from the translate_sid processor enrichment + # if they are available and we don't already have them. + - rename: + field: _user.name + target_field: user.name + ignore_failure: true + ignore_missing: true + - rename: + field: _user.domain + target_field: user.domain + ignore_failure: true + ignore_missing: true ## PowerShell fields. @@ -402,6 +414,7 @@ processors: - remove: field: - _temp + - _user - winlog.event_data.param1 - winlog.event_data.param2 - winlog.event_data.param3 diff --git a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs index 55799473ece..e90683175ca 100644 --- a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs @@ -18,7 +18,13 @@ tags: {{#if preserve_original_event}} include_xml: true {{/if}} -{{#if processors.length}} processors: + - translate_sid: + field: winlog.event_data.MemberSid + account_name_target: _user.name + domain_target: _user.domain + ignore_missing: true + ignore_failure: true +{{#if processors.length}} {{processors}} {{/if}} diff --git a/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml index 16d21d8fe82..871f3301f8d 100644 --- a/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml @@ -171,6 +171,18 @@ processors: ignore_failure: true ignore_empty_value: true if: ctx?.source?.user != null + # Get user details from the translate_sid processor enrichment + # if they are available and we don't already have them. + - rename: + field: _user.name + target_field: user.name + ignore_failure: true + ignore_missing: true + - rename: + field: _user.domain + target_field: user.domain + ignore_failure: true + ignore_missing: true ## PowerShell fields. @@ -460,6 +472,7 @@ processors: - remove: field: - _temp + - _user - winlog.event_data.SequenceNumber - winlog.event_data.User - winlog.event_data.ConnectedUser diff --git a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs index 7795afb123d..2843775b09f 100644 --- a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs @@ -18,7 +18,13 @@ tags: {{#if preserve_original_event}} include_xml: true {{/if}} -{{#if processors.length}} processors: + - translate_sid: + field: winlog.event_data.MemberSid + account_name_target: _user.name + domain_target: _user.domain + ignore_missing: true + ignore_failure: true +{{#if processors.length}} {{processors}} {{/if}} diff --git a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml index 843d73b827c..a4311bc04bd 100644 --- a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml @@ -851,6 +851,18 @@ processors: ignore_failure: true ignore_empty_value: true if: ctx?._temp?.user_parts != null && ctx._temp.user_parts.size() == 2 + # Get user details from the translate_sid processor enrichment + # if they are available and we don't already have them. + - rename: + field: _user.name + target_field: user.name + ignore_failure: true + ignore_missing: true + - rename: + field: _user.domain + target_field: user.domain + ignore_failure: true + ignore_missing: true ## Sysmon fields @@ -1199,6 +1211,7 @@ processors: - remove: field: - _temp + - _user - winlog.event_data.ProcessId - winlog.event_data.ParentProcessId - winlog.event_data.SourceProcessId diff --git a/packages/windows/manifest.yml b/packages/windows/manifest.yml index 8c592bda67a..086739c989c 100644 --- a/packages/windows/manifest.yml +++ b/packages/windows/manifest.yml @@ -1,6 +1,6 @@ name: windows title: Windows -version: 1.13.0 +version: 1.14.0 description: Collect logs and metrics from Windows OS and services with Elastic Agent. type: integration categories: From 12c1d8fe5bed28c72a5e98e3733f5a3895de80ba Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Fri, 15 Jul 2022 08:20:07 +0930 Subject: [PATCH 2/2] address pr comment --- .../forwarded/agent/stream/winlog.yml.hbs | 5 +++-- .../elasticsearch/ingest_pipeline/default.yml | 14 +++++++++++--- .../powershell/agent/stream/winlog.yml.hbs | 5 +++-- .../elasticsearch/ingest_pipeline/default.yml | 16 +++++++++++++--- .../agent/stream/winlog.yml.hbs | 6 +++--- .../elasticsearch/ingest_pipeline/default.yml | 16 +++++++++++++--- .../agent/stream/winlog.yml.hbs | 5 +++-- .../elasticsearch/ingest_pipeline/default.yml | 16 +++++++++++++--- 8 files changed, 62 insertions(+), 21 deletions(-) diff --git a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs index 90f155b50ea..3d1241f4091 100644 --- a/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/forwarded/agent/stream/winlog.yml.hbs @@ -24,8 +24,9 @@ publisher_pipeline.disable_host: true processors: - translate_sid: field: winlog.event_data.MemberSid - account_name_target: _user.name - domain_target: _user.domain + account_name_target: winlog.event_data._MemberUserName + domain_target: winlog.event_data._MemberDomain + account_type_target: winlog.event_data._MemberAccountType ignore_missing: true ignore_failure: true {{#if processors.length}} diff --git a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml index b7c8ec038be..f63261eb8e2 100644 --- a/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/default.yml @@ -17,18 +17,26 @@ processors: # Get user details from the translate_sid processor enrichment # if they are available and we don't already have them. - rename: - field: _user.name + field: winlog.event_data._MemberUserName target_field: user.name ignore_failure: true ignore_missing: true - rename: - field: _user.domain + field: winlog.event_data._MemberDomain target_field: user.domain ignore_failure: true ignore_missing: true + - append: + value: '{{{winlog.event_data._MemberAccountType}}}' + field: user.roles + ignore_failure: true + allow_duplicates: false + if: ctx.winlog?.event_data?._MemberAccountType != null - remove: - field: _user + field: winlog.event_data._MemberAccountType ignore_missing: true + ignore_failure: true + if: ctx.user?.roles != null && ctx.winlog?.event_data?._MemberAccountType != null && ctx.user.roles.contains(ctx.winlog.event_data._MemberAccountType) on_failure: - set: diff --git a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs index afbfc336859..04490e72ae4 100644 --- a/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell/agent/stream/winlog.yml.hbs @@ -21,8 +21,9 @@ include_xml: true processors: - translate_sid: field: winlog.event_data.MemberSid - account_name_target: _user.name - domain_target: _user.domain + account_name_target: winlog.event_data._MemberUserName + domain_target: winlog.event_data._MemberDomain + account_type_target: winlog.event_data._MemberAccountType ignore_missing: true ignore_failure: true {{#if processors.length}} diff --git a/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml index 95d99c803d9..576141f63eb 100644 --- a/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/powershell/elasticsearch/ingest_pipeline/default.yml @@ -120,15 +120,26 @@ processors: # Get user details from the translate_sid processor enrichment # if they are available and we don't already have them. - rename: - field: _user.name + field: winlog.event_data._MemberUserName target_field: user.name ignore_failure: true ignore_missing: true - rename: - field: _user.domain + field: winlog.event_data._MemberDomain target_field: user.domain ignore_failure: true ignore_missing: true + - append: + value: '{{{winlog.event_data._MemberAccountType}}}' + field: user.roles + ignore_failure: true + allow_duplicates: false + if: ctx.winlog?.event_data?._MemberAccountType != null + - remove: + field: winlog.event_data._MemberAccountType + ignore_missing: true + ignore_failure: true + if: ctx.user?.roles != null && ctx.winlog?.event_data?._MemberAccountType != null && ctx.user.roles.contains(ctx.winlog.event_data._MemberAccountType) ## PowerShell fields. @@ -414,7 +425,6 @@ processors: - remove: field: - _temp - - _user - winlog.event_data.param1 - winlog.event_data.param2 - winlog.event_data.param3 diff --git a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs index e90683175ca..5628975e703 100644 --- a/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/powershell_operational/agent/stream/winlog.yml.hbs @@ -20,9 +20,9 @@ include_xml: true {{/if}} processors: - translate_sid: - field: winlog.event_data.MemberSid - account_name_target: _user.name - domain_target: _user.domain + account_name_target: winlog.event_data._MemberUserName + domain_target: winlog.event_data._MemberDomain + account_type_target: winlog.event_data._MemberAccountType ignore_missing: true ignore_failure: true {{#if processors.length}} diff --git a/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml index 871f3301f8d..79c2948a4aa 100644 --- a/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/powershell_operational/elasticsearch/ingest_pipeline/default.yml @@ -174,15 +174,26 @@ processors: # Get user details from the translate_sid processor enrichment # if they are available and we don't already have them. - rename: - field: _user.name + field: winlog.event_data._MemberUserName target_field: user.name ignore_failure: true ignore_missing: true - rename: - field: _user.domain + field: winlog.event_data._MemberDomain target_field: user.domain ignore_failure: true ignore_missing: true + - append: + value: '{{{winlog.event_data._MemberAccountType}}}' + field: user.roles + ignore_failure: true + allow_duplicates: false + if: ctx.winlog?.event_data?._MemberAccountType != null + - remove: + field: winlog.event_data._MemberAccountType + ignore_missing: true + ignore_failure: true + if: ctx.user?.roles != null && ctx.winlog?.event_data?._MemberAccountType != null && ctx.user.roles.contains(ctx.winlog.event_data._MemberAccountType) ## PowerShell fields. @@ -472,7 +483,6 @@ processors: - remove: field: - _temp - - _user - winlog.event_data.SequenceNumber - winlog.event_data.User - winlog.event_data.ConnectedUser diff --git a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs index 2843775b09f..0e0301e0cae 100644 --- a/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs +++ b/packages/windows/data_stream/sysmon_operational/agent/stream/winlog.yml.hbs @@ -21,8 +21,9 @@ include_xml: true processors: - translate_sid: field: winlog.event_data.MemberSid - account_name_target: _user.name - domain_target: _user.domain + account_name_target: winlog.event_data._MemberUserName + domain_target: winlog.event_data._MemberDomain + account_type_target: winlog.event_data._MemberAccountType ignore_missing: true ignore_failure: true {{#if processors.length}} diff --git a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml index a4311bc04bd..5eef7930485 100644 --- a/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml +++ b/packages/windows/data_stream/sysmon_operational/elasticsearch/ingest_pipeline/default.yml @@ -854,15 +854,26 @@ processors: # Get user details from the translate_sid processor enrichment # if they are available and we don't already have them. - rename: - field: _user.name + field: winlog.event_data._MemberUserName target_field: user.name ignore_failure: true ignore_missing: true - rename: - field: _user.domain + field: winlog.event_data._MemberDomain target_field: user.domain ignore_failure: true ignore_missing: true + - append: + value: '{{{winlog.event_data._MemberAccountType}}}' + field: user.roles + ignore_failure: true + allow_duplicates: false + if: ctx.winlog?.event_data?._MemberAccountType != null + - remove: + field: winlog.event_data._MemberAccountType + ignore_missing: true + ignore_failure: true + if: ctx.user?.roles != null && ctx.winlog?.event_data?._MemberAccountType != null && ctx.user.roles.contains(ctx.winlog.event_data._MemberAccountType) ## Sysmon fields @@ -1211,7 +1222,6 @@ processors: - remove: field: - _temp - - _user - winlog.event_data.ProcessId - winlog.event_data.ParentProcessId - winlog.event_data.SourceProcessId