From c76d49450f198565408fc97dbeb792fc48fa23e1 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 19 Oct 2023 17:24:31 +0200 Subject: [PATCH 01/31] First version of ETW input --- .github/CODEOWNERS | 1 + x-pack/filebeat/input/etw/config.go | 97 ++++++++++++++++++ x-pack/filebeat/input/etw/input.go | 146 ++++++++++++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 x-pack/filebeat/input/etw/config.go create mode 100644 x-pack/filebeat/input/etw/input.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ea38e969a3d..6b46f276786 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -112,6 +112,7 @@ CHANGELOG* /x-pack/filebeat/input/cel/ @elastic/security-service-integrations /x-pack/filebeat/input/cometd/ @elastic/obs-infraobs-integrations /x-pack/filebeat/input/entityanalytics/ @elastic/security-service-integrations +/x-pack/filebeat/input/etw/ @elastic/sec-windows-platform /x-pack/filebeat/input/gcppubsub/ @elastic/security-service-integrations /x-pack/filebeat/input/gcs/ @elastic/security-service-integrations /x-pack/filebeat/input/http_endpoint/ @elastic/security-service-integrations diff --git a/x-pack/filebeat/input/etw/config.go b/x-pack/filebeat/input/etw/config.go new file mode 100644 index 00000000000..22e260453d2 --- /dev/null +++ b/x-pack/filebeat/input/etw/config.go @@ -0,0 +1,97 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package etw_input + +import ( + "fmt" + + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" +) + +var validTraceLevel = map[string]bool{ + "critical": true, + "error": true, + "warning": true, + "information": true, + "verbose": true, +} + +type config struct { + Logfile string `config:"file"` + ProviderGUID string `config:"provider.guid"` + ProviderName string `config:"provider.name"` + SessionName string `config:"session_name"` // Tag for the new session + TraceLevel string `config:"trace_level"` + MatchAnyKeyword uint64 `config:"match_any_keyword"` + MatchAllKeyword uint64 `config:"match_all_keyword"` + Session string `config:"session"` +} + +// Create a conversion function to convert config to etw.Config +func convertConfig(cfg config) etw.Config { + return etw.Config{ + Logfile: cfg.Logfile, + ProviderGUID: cfg.ProviderGUID, + ProviderName: cfg.ProviderName, + SessionName: cfg.SessionName, + TraceLevel: cfg.TraceLevel, + MatchAnyKeyword: cfg.MatchAnyKeyword, + MatchAllKeyword: cfg.MatchAllKeyword, + Session: cfg.Session, + } +} + +func defaultConfig() config { + return config{ + Logfile: "", + ProviderName: "", + ProviderGUID: "", + SessionName: "", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + Session: "", + } +} + +// Config validation +func (c *config) validate() error { + if c.ProviderName == "" && c.ProviderGUID == "" && c.Logfile == "" && c.Session == "" { + return fmt.Errorf("provider, existing logfile or running session must be set") + } + + if !validTraceLevel[c.TraceLevel] { + return fmt.Errorf("invalid Trace Level value '%s'", c.TraceLevel) + } + + if c.ProviderGUID != "" { + if c.ProviderName != "" { + return fmt.Errorf("configuration constraint error: provider GUID and provider name cannot be defined together") + } + if c.Logfile != "" { + return fmt.Errorf("configuration constraint error: provider GUID and file cannot be defined together") + } + if c.Session != "" { + return fmt.Errorf("configuration constraint error: provider GUID and existing session cannot be defined together") + } + } + + if c.ProviderName != "" { + if c.Logfile != "" { + return fmt.Errorf("configuration constraint error: provider name and file cannot be defined together") + } + if c.Session != "" { + return fmt.Errorf("configuration constraint error: provider name and existing session cannot be defined together") + } + } + + if c.Logfile != "" { + if c.Session != "" { + return fmt.Errorf("configuration constraint error: file and existing session cannot be defined together") + } + } + + return nil +} diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go new file mode 100644 index 00000000000..5cb31370448 --- /dev/null +++ b/x-pack/filebeat/input/etw/input.go @@ -0,0 +1,146 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package etw_input + +import ( + "fmt" + "sync" + "syscall" + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const ( + inputName = "etw" +) + +type etw_input struct { + log *logp.Logger + config config + etwSession etw.Session +} + +func Plugin() input.Plugin { + return input.Plugin{ + Name: inputName, + Stability: feature.Beta, + Info: "Collect ETW logs.", + Manager: stateless.NewInputManager(configure), + } +} + +func configure(cfg *conf.C) (stateless.Input, error) { + conf := defaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, err + } + + return newETW(conf) +} + +func newETW(config config) (*etw_input, error) { + if err := config.validate(); err != nil { + return nil, err + } + + return &etw_input{ + config: config, + }, nil +} + +func (e *etw_input) Name() string { return inputName } + +func (e *etw_input) Test(_ input.TestContext) error { + // ToDo + return nil +} + +func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error { + var err error + e.etwSession, err = etw.NewSession(convertConfig(e.config)) + if err != nil { + return fmt.Errorf("error when inicializing '%s' session: %v", e.etwSession.Name, err) + } + + e.log := ctx.Logger.With("session", e.etwSession.Name) + e.log.Info("Starting " + inputName + " input") + + var wg sync.WaitGroup + + if e.etwSession.Realtime { + err = e.etwSession.CreateRealtimeSession() + if err != nil { + return fmt.Errorf("realtime session could not be created: %v", e.etwSession.Name, err) + } + e.log.Debug("created session") + } + defer func() { + wg.Wait() // Wait for the goroutine to finish + e.close() + e.log.Info(inputName + " input stopped") + }() + + // Define callback that will process ETW events + // Callback which receives every ETW event from the reading source + eventReceivedCallback := func(er *etw.EventRecord) { + if er == nil { + e.log.Error("received null event record") + return + } + + e.log.Debugf("received event %d with length %d", er.EventHeader.EventDescriptor.Id, er.UserDataLength) + + event := make(map[string]interface{}) + event["Header"] = er.EventHeader + + if data, err := etw.GetEventProperties(er); err == nil { + event["EventProperties"] = data + } else { + e.log.Errorf("failed to read event properties: %s", err) + return + } + + evt := beat.Event{ + Timestamp: time.Now(), + Fields: mapstr.M{ + "header": event["Header"], + "winlog": event["EventProperties"], + }, + } + publisher.Publish(evt) + + return + } + + e.etwSession.Callback = syscall.NewCallback(eventReceivedCallback) + + wg.Add(1) + go func() { + defer wg.Done() + e.log.Debug("starting to listen ETW events") + if err = e.etwSession.StartConsumer(); err != nil { + e.log.Warnf("events could not be read: %v", err) + } + e.log.Debug("stopped to read ETW events") + }() + + return nil +} + +// Closes all the opened handlers and resources +func (e *etw_input) close() { + if err := e.etwSession.StopSession(); err != nil { + e.log.Error("failed to shutdown ETW session") + } + e.log.Info("successfully shutdown") +} From 0f46bd9b787bd3cf9f9b7e2b481097c88a34d94c Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 26 Oct 2023 19:06:12 +0200 Subject: [PATCH 02/31] Minor fixes for ETW input --- x-pack/filebeat/input/etw/config.go | 2 ++ x-pack/filebeat/input/etw/input.go | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/filebeat/input/etw/config.go b/x-pack/filebeat/input/etw/config.go index 22e260453d2..002abea265b 100644 --- a/x-pack/filebeat/input/etw/config.go +++ b/x-pack/filebeat/input/etw/config.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build windows + package etw_input import ( diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 5cb31370448..ede576666c1 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -2,6 +2,8 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +//go:build windows + package etw_input import ( @@ -92,10 +94,10 @@ func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error // Define callback that will process ETW events // Callback which receives every ETW event from the reading source - eventReceivedCallback := func(er *etw.EventRecord) { + eventReceivedCallback := func(er *etw.EventRecord) uintptr { if er == nil { e.log.Error("received null event record") - return + return 1 } e.log.Debugf("received event %d with length %d", er.EventHeader.EventDescriptor.Id, er.UserDataLength) @@ -107,7 +109,7 @@ func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error event["EventProperties"] = data } else { e.log.Errorf("failed to read event properties: %s", err) - return + return 1 } evt := beat.Event{ @@ -119,7 +121,7 @@ func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error } publisher.Publish(evt) - return + return 0 } e.etwSession.Callback = syscall.NewCallback(eventReceivedCallback) From d3635150e899a15099bb3e78c8e2a3a3ba05d401 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Mon, 20 Nov 2023 11:36:04 +0100 Subject: [PATCH 03/31] More fixes and requested changes for ETW input --- x-pack/filebeat/input/etw/config.go | 47 ++++++++++++------- x-pack/filebeat/input/etw/input.go | 71 ++++++++++++++++------------- 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/x-pack/filebeat/input/etw/config.go b/x-pack/filebeat/input/etw/config.go index 002abea265b..2f3925884f3 100644 --- a/x-pack/filebeat/input/etw/config.go +++ b/x-pack/filebeat/input/etw/config.go @@ -4,7 +4,7 @@ //go:build windows -package etw_input +package etw import ( "fmt" @@ -21,17 +21,39 @@ var validTraceLevel = map[string]bool{ } type config struct { - Logfile string `config:"file"` - ProviderGUID string `config:"provider.guid"` - ProviderName string `config:"provider.name"` - SessionName string `config:"session_name"` // Tag for the new session - TraceLevel string `config:"trace_level"` + // Logfile is the path to an .etl file to read from. + Logfile string `config:"file"` + // ProviderGUID is the GUID of an ETW provider. + // Run 'logman query providers' to list the available providers. + ProviderGUID string `config:"provider.guid"` + // ProviderName is the name of an ETW provider. + // Run 'logman query providers' to list the available providers. + ProviderName string `config:"provider.name"` + // SessionName is the name used to create a new session for the + // defined provider. If missing, its default value is the provider ID + // prefixed by 'Elastic-' + SessionName string `config:"session_name"` + // TraceLevel filters all provider events with a level value + // that is less than or equal to this level. + // Allowed values are critical, error, warning, informational, and verbose. + TraceLevel string `config:"trace_level"` + // MatchAnyKeyword is an 8-byte bitmask that enables the filtering of + // events from specific provider subcomponents. The provider will write + // a particular event if the event's keyword bits match any of the bits + // in this bitmask. + // See https://learn.microsoft.com/en-us/message-analyzer/system-etw-provider-event-keyword-level-settings for more details. + // Use logman query providers "" to list the available keywords. MatchAnyKeyword uint64 `config:"match_any_keyword"` + // An 8-byte bitmask that enables the filtering of events from + // specific provider subcomponents. The provider will write a particular + // event if the event's keyword bits match all of the bits in this bitmask. + // See https://learn.microsoft.com/en-us/message-analyzer/system-etw-provider-event-keyword-level-settings for more details. MatchAllKeyword uint64 `config:"match_all_keyword"` - Session string `config:"session"` + // Session is the name of an existing session to read from. + // Run 'logman query -ets' to list existing sessions. + Session string `config:"session"` } -// Create a conversion function to convert config to etw.Config func convertConfig(cfg config) etw.Config { return etw.Config{ Logfile: cfg.Logfile, @@ -47,19 +69,12 @@ func convertConfig(cfg config) etw.Config { func defaultConfig() config { return config{ - Logfile: "", - ProviderName: "", - ProviderGUID: "", - SessionName: "", TraceLevel: "verbose", MatchAnyKeyword: 0xffffffffffffffff, - MatchAllKeyword: 0, - Session: "", } } -// Config validation -func (c *config) validate() error { +func (c *config) Validate() error { if c.ProviderName == "" && c.ProviderGUID == "" && c.Logfile == "" && c.Session == "" { return fmt.Errorf("provider, existing logfile or running session must be set") } diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index ede576666c1..4eaed482ec5 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -4,7 +4,7 @@ //go:build windows -package etw_input +package etw import ( "fmt" @@ -26,7 +26,8 @@ const ( inputName = "etw" ) -type etw_input struct { +// etwInput struct holds the configuration and state for the ETW input +type etwInput struct { log *logp.Logger config config etwSession etw.Session @@ -47,53 +48,59 @@ func configure(cfg *conf.C) (stateless.Input, error) { return nil, err } - return newETW(conf) -} - -func newETW(config config) (*etw_input, error) { - if err := config.validate(); err != nil { - return nil, err - } - - return &etw_input{ - config: config, + return &etwInput{ + config: conf, }, nil } -func (e *etw_input) Name() string { return inputName } +func (e *etwInput) Name() string { return inputName } -func (e *etw_input) Test(_ input.TestContext) error { +func (e *etwInput) Test(_ input.TestContext) error { // ToDo return nil } -func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error { +// Run starts the ETW session and processes incoming events. +func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { var err error + // Initialize a new ETW session with the provided configuration. e.etwSession, err = etw.NewSession(convertConfig(e.config)) if err != nil { - return fmt.Errorf("error when inicializing '%s' session: %v", e.etwSession.Name, err) + return fmt.Errorf("error when initializing '%s' session: %w", e.etwSession.Name, err) } - e.log := ctx.Logger.With("session", e.etwSession.Name) + // Set up logger with session information. + e.log = ctx.Logger.With("session", e.etwSession.Name) e.log.Info("Starting " + inputName + " input") var wg sync.WaitGroup + // Handle realtime session creation or attachment. if e.etwSession.Realtime { - err = e.etwSession.CreateRealtimeSession() - if err != nil { - return fmt.Errorf("realtime session could not be created: %v", e.etwSession.Name, err) + if !e.etwSession.NewSession { + // Attach to an existing session. + err = e.etwSession.GetHandler() + if err != nil { + return fmt.Errorf("unable to retrieve handler for session '%s': %w", e.etwSession.Name, err) + } + e.log.Debug("attached to existing session '%s'", e.etwSession.Name) + } else { + // Create a new realtime session. + err = e.etwSession.CreateRealtimeSession() + if err != nil { + return fmt.Errorf("realtime session '%s' could not be created: %w", e.etwSession.Name, err) + } + e.log.Debug("created session '%s'", e.etwSession.Name) } - e.log.Debug("created session") } + // Defer the cleanup and closing of resources. defer func() { - wg.Wait() // Wait for the goroutine to finish + wg.Wait() // Ensure all goroutines have finished before closing. e.close() e.log.Info(inputName + " input stopped") }() - // Define callback that will process ETW events - // Callback which receives every ETW event from the reading source + // eventReceivedCallback processes each ETW event. eventReceivedCallback := func(er *etw.EventRecord) uintptr { if er == nil { e.log.Error("received null event record") @@ -108,7 +115,7 @@ func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error if data, err := etw.GetEventProperties(er); err == nil { event["EventProperties"] = data } else { - e.log.Errorf("failed to read event properties: %s", err) + e.log.Errorf("failed to read event properties: %w", err) return 1 } @@ -124,25 +131,27 @@ func (e *etw_input) Run(ctx input.Context, publisher stateless.Publisher) error return 0 } + // Set the callback function for the ETW session. e.etwSession.Callback = syscall.NewCallback(eventReceivedCallback) + // Start a goroutine to consume ETW events. wg.Add(1) go func() { defer wg.Done() e.log.Debug("starting to listen ETW events") if err = e.etwSession.StartConsumer(); err != nil { - e.log.Warnf("events could not be read: %v", err) + e.log.Warnf("events could not be read from '%s': %w", e.etwSession.Name, err) } - e.log.Debug("stopped to read ETW events") + e.log.Debug("stopped to read ETW events from '%s'", e.etwSession.Name) }() return nil } -// Closes all the opened handlers and resources -func (e *etw_input) close() { +// close stops the ETW session and logs the outcome. +func (e *etwInput) close() { if err := e.etwSession.StopSession(); err != nil { - e.log.Error("failed to shutdown ETW session") + e.log.Error("failed to shutdown ETW session '%s'", e.etwSession.Name) } - e.log.Info("successfully shutdown") + e.log.Info("successfully shutdown for '%s'", e.etwSession.Name) } From 619492c7fac2ceeff5b844ab78d8f982471e67a0 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 28 Nov 2023 11:17:11 +0100 Subject: [PATCH 04/31] Include ETW in the default input list for Windows --- .../input/default-inputs/inputs_other.go | 2 +- .../input/default-inputs/inputs_windows.go | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/input/default-inputs/inputs_windows.go diff --git a/x-pack/filebeat/input/default-inputs/inputs_other.go b/x-pack/filebeat/input/default-inputs/inputs_other.go index d396d4635a1..0e8cd516536 100644 --- a/x-pack/filebeat/input/default-inputs/inputs_other.go +++ b/x-pack/filebeat/input/default-inputs/inputs_other.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build !aix +//go:build !aix && !windows package inputs diff --git a/x-pack/filebeat/input/default-inputs/inputs_windows.go b/x-pack/filebeat/input/default-inputs/inputs_windows.go new file mode 100644 index 00000000000..361883f39ad --- /dev/null +++ b/x-pack/filebeat/input/default-inputs/inputs_windows.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package inputs + +import ( + "github.com/elastic/beats/v7/filebeat/beater" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" + "github.com/elastic/beats/v7/x-pack/filebeat/input/azureblobstorage" + "github.com/elastic/beats/v7/x-pack/filebeat/input/cel" + "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" + "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics" + "github.com/elastic/beats/v7/x-pack/filebeat/input/etw" + "github.com/elastic/beats/v7/x-pack/filebeat/input/gcs" + "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" + "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" + "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" + "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" + "github.com/elastic/beats/v7/x-pack/filebeat/input/shipper" + "github.com/elastic/elastic-agent-libs/logp" +) + +func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { + return []v2.Plugin{ + azureblobstorage.Plugin(log, store), + cel.Plugin(log, store), + cloudfoundry.Plugin(), + entityanalytics.Plugin(log), + gcs.Plugin(log, store), + http_endpoint.Plugin(), + httpjson.Plugin(log, store), + o365audit.Plugin(log, store), + awss3.Plugin(store), + awscloudwatch.Plugin(), + lumberjack.Plugin(), + shipper.Plugin(log, store), + etw.Plugin(), + } +} From 2fdddf0f23482de01f57e21a02510bc0f4c88613 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 28 Nov 2023 12:58:25 +0100 Subject: [PATCH 05/31] Tests for config input --- x-pack/filebeat/input/etw/config_test.go | 138 +++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 x-pack/filebeat/input/etw/config_test.go diff --git a/x-pack/filebeat/input/etw/config_test.go b/x-pack/filebeat/input/etw/config_test.go new file mode 100644 index 00000000000..c2b88b5bb68 --- /dev/null +++ b/x-pack/filebeat/input/etw/config_test.go @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + confpkg "github.com/elastic/elastic-agent-libs/config" +) + +func Test_validateConfig(t *testing.T) { + testCases := []struct { + name string // Sub-test name. + config config // Load config parameters. + wantError string // Expected error + }{ + { + name: "valid config", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + SessionName: "MySession-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + }, + { + name: "minimal config", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + }, + }, + { + name: "missing source config", + config: config{ + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "provider, existing logfile or running session must be set", + }, + { + name: "invalid trace level", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "failed", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "invalid Trace Level value 'failed'", + }, + { + name: "conflict provider GUID and name", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and provider name cannot be defined together", + }, + { + name: "conflict provider GUID and logfile", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and file cannot be defined together", + }, + { + name: "conflict provider GUID and session", + config: config{ + ProviderGUID: "{eb79061a-a566-4698-1234-3ed2807033a0}", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider GUID and existing session cannot be defined together", + }, + { + name: "conflict provider name and logfile", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider name and file cannot be defined together", + }, + { + name: "conflict provider name and session", + config: config{ + ProviderName: "Microsoft-Windows-DNSServer", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: provider name and existing session cannot be defined together", + }, + { + name: "conflict logfile and session", + config: config{ + Logfile: "C:\\Windows\\System32\\winevt\\File.etl", + Session: "EventLog-Application", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + }, + wantError: "configuration constraint error: file and existing session cannot be defined together", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c := confpkg.MustNewConfigFrom(tc.config) + config := defaultConfig() + err := c.Unpack(&config) + + // Validate responses + if tc.wantError != "" { + if assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.wantError) + } else { + t.Fatalf("Configuration validation failed. No returned error while expecting '%s'", tc.wantError) + } + } else { + if err != nil { + t.Fatalf("Configuration validation failed. no error expected but got '%w'", err) + } + } + }) + } +} From df644f4664582ef67deef9cdd0d85953c69b6490 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 29 Nov 2023 08:16:05 +0100 Subject: [PATCH 06/31] Sync input close calls --- x-pack/filebeat/input/etw/input.go | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 4eaed482ec5..8253bfc5cd0 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -74,6 +74,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { e.log.Info("Starting " + inputName + " input") var wg sync.WaitGroup + var once sync.Once // Handle realtime session creation or attachment. if e.etwSession.Realtime { @@ -81,22 +82,22 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { // Attach to an existing session. err = e.etwSession.GetHandler() if err != nil { - return fmt.Errorf("unable to retrieve handler for session '%s': %w", e.etwSession.Name, err) + return fmt.Errorf("unable to retrieve handler: %w", err) } - e.log.Debug("attached to existing session '%s'", e.etwSession.Name) + e.log.Debug("attached to existing session") } else { // Create a new realtime session. err = e.etwSession.CreateRealtimeSession() if err != nil { - return fmt.Errorf("realtime session '%s' could not be created: %w", e.etwSession.Name, err) + return fmt.Errorf("realtime session could not be created: %w", err) } - e.log.Debug("created session '%s'", e.etwSession.Name) + e.log.Debug("created session") } } // Defer the cleanup and closing of resources. defer func() { wg.Wait() // Ensure all goroutines have finished before closing. - e.close() + once.Do(e.Close) e.log.Info(inputName + " input stopped") }() @@ -140,18 +141,24 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { defer wg.Done() e.log.Debug("starting to listen ETW events") if err = e.etwSession.StartConsumer(); err != nil { - e.log.Warnf("events could not be read from '%s': %w", e.etwSession.Name, err) + e.log.Warnf("events could not be read from session: %w", err) } - e.log.Debug("stopped to read ETW events from '%s'", e.etwSession.Name) + e.log.Debug("stopped to read ETW events from session") + }() + + // We ensure resources are closed when receiving a cancelation signal + go func() { + <-ctx.Cancelation.Done() + once.Do(e.Close) }() return nil } // close stops the ETW session and logs the outcome. -func (e *etwInput) close() { +func (e *etwInput) Close() { if err := e.etwSession.StopSession(); err != nil { - e.log.Error("failed to shutdown ETW session '%s'", e.etwSession.Name) + e.log.Error("failed to shutdown ETW session") } - e.log.Info("successfully shutdown for '%s'", e.etwSession.Name) + e.log.Info("successfully shutdown") } From 7ba80fdf099e958c2b0847507609f0c1ce4f6dd3 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 29 Nov 2023 08:16:48 +0100 Subject: [PATCH 07/31] Update config file and docs --- filebeat/docs/filebeat-options.asciidoc | 3 + .../filebeat/docs/inputs/input-etw.asciidoc | 140 ++++++++ x-pack/filebeat/filebeat.reference.yml | 299 ++++++++++-------- 3 files changed, 313 insertions(+), 129 deletions(-) create mode 100644 x-pack/filebeat/docs/inputs/input-etw.asciidoc diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index faff00e7e3d..8bbecb8b4d8 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -91,6 +91,7 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-tcp>> * <<{beatname_lc}-input-udp>> * <<{beatname_lc}-input-gcs>> +* <<{beatname_lc}-input-etw>> include::multiline.asciidoc[] @@ -145,3 +146,5 @@ include::inputs/input-udp.asciidoc[] include::inputs/input-unix.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[] + +include::../../x-pack/filebeat/docs/inputs/input-etw.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-etw.asciidoc b/x-pack/filebeat/docs/inputs/input-etw.asciidoc new file mode 100644 index 00000000000..ce0a138e0ba --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-etw.asciidoc @@ -0,0 +1,140 @@ +[role="xpack"] + +:type: etw + +[id="{beatname_lc}-input-{type}"] +=== ETW input + +++++ +ETW +++++ + +beta[] + +`https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal`[Event Tracing for Windows]) is a powerful logging and tracing mechanism built into the Windows operating system. It provides a detailed view of application and system behavior, performance issues, and runtime diagnostics. Trace events contain an event header and provider-defined data that describes the current state of an application or operation. You can use the events to debug an application and perform capacity and performance analysis. + +The ETW input can interact with ETW in three distinct ways: it can create a new session to capture events from user-mode providers, attach to an already existing session to collect ongoing event data, or read events from a pre-recorded .etl file. This functionality enables the module to adapt to different scenarios, such as real-time event monitoring or analyzing historical data. + +Example configurations: + +Read from a provider by name: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.name: Microsoft-Windows-DNSServer + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0x8000000000000000 + match_all_keyword: 0 +---- + +Same provider can be defined by its GUID: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0x8000000000000000 + match_all_keyword: 0 +---- + +Read from a current session: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + enabled: true + id: etw-dnsserver-session + session: UAL_Usermode_Provider +---- + +Read from a .etl file: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + enabled: true + id: etw-dnsserver-session + file: "C\Windows\System32\Winevt\Logs\Logfile.etl" +---- + +NOTE: Examples shown above are mutually exclusive, since the options `provider.name`, `provider.guid`, `session` and `file` cannot be present at the same time. Nevertheless, it is a requirement that one of them appears. + +Multiple providers example: +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: etw + id: etw-dnsserver + enabled: true + provider.name: Microsoft-Windows-DNSServer + session_name: DNSServer-Analytical + trace_level: verbose + match_any_keyword: 0xfffffffffffffffff + match_all_keyword: 0 +- type: etw + id: etw-security + enabled: true + provider.name: Microsoft-Windows-Security-Auditing + session_name: Security-Auditing + trace_level: warning + match_any_keyword: 0xffffffffffffffff + match_all_keyword: 0 +---- + +==== Configuration options + +The `ETW` input supports the following configuration options. + +[float] +==== `file` + +Specifies the path to an .etl file for reading ETW events. This file format is commonly used for storing ETW event logs. + +[float] +==== `provider.guid` + +Identifies the GUID of an ETW provider. To see available providers, use the command `logman query providers`. + +[float] +==== `provider.name` + +Specifies the name of the ETW provider. Available providers can be listed using `logman query providers`. + +[float] +==== `session_name` + +When specified a provider, a new session is created. It sets the name for a new ETW session associated with the provider. If not provided, the default is the provider ID prefixed with 'Elastic-'. + +[float] +==== `trace_level` + +Defines the filtering level for events based on severity. Valid options include critical, error, warning, informational, and verbose. + +[float] +==== `match_any_keyword` + +An 8-byte bitmask used for filtering events from specific provider subcomponents based on keyword matching. Any matching keyword will enable the event to be written. Default value is `0xfffffffffffffffff` so it matches every available keyword. + +Run `logman query providers ""` to list the available keywords for a specific provider. + +[float] +==== `match_all_keyword` + +Similar to MatchAnyKeyword, this 8-byte bitmask filters events that match all specified keyword bits. Default value is `0` to let every event pass. + +Run `logman query providers ""` to list the available keywords for a specific provider. + +[float] +==== `session` + +Names an existing ETW session to read from. Existing sessions can be listed using `logman query -ets`. + +:type!: diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 14308c2cce1..6c9d818b32e 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1878,135 +1878,135 @@ filebeat.modules: #var.password: #------------------------------ Salesforce Module ------------------------------ -- module: salesforce - - apex-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - login-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - login-stream: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - logout-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - logout-stream: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - setupaudittrail-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - # Interval, should include the time interval +- module: salesforce + + apex-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + login-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + login-stream: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + logout-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + logout-stream: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + setupaudittrail-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + # Interval, should include the time interval #var.interval: 1h #----------------------------- Google Santa Module ----------------------------- - module: santa @@ -3579,6 +3579,47 @@ filebeat.inputs: # collect logs when there is a delay in CloudWatch. #latency: 1m +#------------------------------ ETW input -------------------------------- +# Beta: Config options for ETW (Event Trace for Windows) input (Only available for Windows) +#- type: etw + #enabled: false + #id: etw-dnsserver + + # Path to an .etl file to read from. + #file: "C:\Windows\System32\Winevt\Logs\Logfile.etl" + + # GUID of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + + # Name of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.name: Microsoft-Windows-DNSServer + + # Tag to identify created sessions. + # If missing, its default value is the provider ID prefixed by 'Elastic-'. + #session_name: DNSServer-Analytical-Trace + + # Filter collected events with a level value that is less than or equal to this level. + # Allowed values are critical, error, warning, informational, and verbose. + #trace_level: verbose + + # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. + # The provider will write a particular event if the event's keyword bits match any of the bits + # in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_any_keyword: 0x8000000000000000 + + # 8-byte bitmask that enables the filtering of events from + # specific provider subcomponents. The provider will write a particular + # event if the event's keyword bits match all of the bits in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_all_keyword: 0 + + # An existing session to read from. + # Run 'logman query -ets' to list existing sessions. + #session: UAL_Usermode_Provider + # =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules From e67262e073767b2abc2c4787b6c02c6ea4f851f4 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 30 Nov 2023 23:45:25 +0100 Subject: [PATCH 08/31] Fix some tabs in reference file --- x-pack/filebeat/filebeat.reference.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6c9d818b32e..45cd1994eeb 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3589,11 +3589,11 @@ filebeat.inputs: #file: "C:\Windows\System32\Winevt\Logs\Logfile.etl" # GUID of an ETW provider. - # Run 'logman query providers' to list the available providers. + # Run 'logman query providers' to list the available providers. #provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} # Name of an ETW provider. - # Run 'logman query providers' to list the available providers. + # Run 'logman query providers' to list the available providers. #provider.name: Microsoft-Windows-DNSServer # Tag to identify created sessions. @@ -3606,18 +3606,18 @@ filebeat.inputs: # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. # The provider will write a particular event if the event's keyword bits match any of the bits - # in this bitmask. + # in this bitmask. # Run 'logman query providers ""' to list available keywords. #match_any_keyword: 0x8000000000000000 # 8-byte bitmask that enables the filtering of events from - # specific provider subcomponents. The provider will write a particular - # event if the event's keyword bits match all of the bits in this bitmask. + # specific provider subcomponents. The provider will write a particular + # event if the event's keyword bits match all of the bits in this bitmask. # Run 'logman query providers ""' to list available keywords. #match_all_keyword: 0 # An existing session to read from. - # Run 'logman query -ets' to list existing sessions. + # Run 'logman query -ets' to list existing sessions. #session: UAL_Usermode_Provider # =========================== Filebeat autodiscover ============================ From 18fa38034e2d49ac264040779cf84f70d4df1231 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 30 Nov 2023 23:45:39 +0100 Subject: [PATCH 09/31] Add metadata to ETW events --- x-pack/filebeat/input/etw/config_test.go | 2 +- x-pack/filebeat/input/etw/input.go | 59 ++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/input/etw/config_test.go b/x-pack/filebeat/input/etw/config_test.go index c2b88b5bb68..7be2acbd56b 100644 --- a/x-pack/filebeat/input/etw/config_test.go +++ b/x-pack/filebeat/input/etw/config_test.go @@ -130,7 +130,7 @@ func Test_validateConfig(t *testing.T) { } } else { if err != nil { - t.Fatalf("Configuration validation failed. no error expected but got '%w'", err) + t.Fatalf("Configuration validation failed. no error expected but got '%v'", err) } } }) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 8253bfc5cd0..2a41532bc33 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -111,10 +111,13 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { e.log.Debugf("received event %d with length %d", er.EventHeader.EventDescriptor.Id, er.UserDataLength) event := make(map[string]interface{}) - event["Header"] = er.EventHeader if data, err := etw.GetEventProperties(er); err == nil { - event["EventProperties"] = data + event = map[string]interface{}{ + "Header": er.EventHeader, + "EventProperties": data, + "Metadata": fillEventMetadata(er, e.etwSession, e.config), + } } else { e.log.Errorf("failed to read event properties: %w", err) return 1 @@ -123,8 +126,9 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { evt := beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "header": event["Header"], - "winlog": event["EventProperties"], + "metadata": event["Metadata"], + "header": event["Header"], + "winlog": event["EventProperties"], }, } publisher.Publish(evt) @@ -155,6 +159,53 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return nil } +// fillEventMetadata constructs a metadata map for an event record. +func fillEventMetadata(er *etw.EventRecord, session etw.Session, cfg config) map[string]interface{} { + // Mapping from Level to Severity + levelToSeverity := map[uint8]string{ + 1: "critical", + 2: "error", + 3: "warning", + 4: "information", + 5: "verbose", + } + + metadata := make(map[string]interface{}) + + // Get the severity level, with a default value if not found + severity, ok := levelToSeverity[er.EventHeader.EventDescriptor.Level] + if !ok { + severity = "unknown" // Default severity level + } + metadata["Severity"] = severity + + // Include provider name and GUID in metadata if available + if cfg.ProviderName != "" { + metadata["ProviderName"] = cfg.ProviderName + } + if cfg.ProviderGUID != "" { + metadata["ProviderGUID"] = cfg.ProviderGUID + } else if etw.IsGUIDValid(session.GUID) { + metadata["ProviderGUID"] = etw.GUIDToString(session.GUID) + } + + // Include logfile path if available + if cfg.Logfile != "" { + metadata["Logfile"] = cfg.Logfile + } + + // Include session name if available + if cfg.Session != "" { + metadata["Session"] = cfg.Session + } else if cfg.SessionName != "" { + metadata["Session"] = cfg.SessionName + } else if cfg.ProviderGUID != "" || cfg.ProviderName != "" { + metadata["Session"] = session.Name + } + + return metadata +} + // close stops the ETW session and logs the outcome. func (e *etwInput) Close() { if err := e.etwSession.StopSession(); err != nil { From 78e37c2bcafebf84594c2c81a07adad1ba873806 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Mon, 4 Dec 2023 10:51:41 +0100 Subject: [PATCH 10/31] Fix PR checks (docs and tests) --- .../filebeat.inputs.reference.xpack.yml.tmpl | 41 +++ x-pack/filebeat/filebeat.reference.yml | 264 +++++++++--------- x-pack/filebeat/input/etw/config_test.go | 6 +- 3 files changed, 177 insertions(+), 134 deletions(-) diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index a35c0af5dc4..c5861174636 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -180,3 +180,44 @@ # This is used to shift collection start time and end time back in order to # collect logs when there is a delay in CloudWatch. #latency: 1m + +#------------------------------ ETW input -------------------------------- +# Beta: Config options for ETW (Event Trace for Windows) input (Only available for Windows) +#- type: etw + #enabled: false + #id: etw-dnsserver + + # Path to an .etl file to read from. + #file: "C:\Windows\System32\Winevt\Logs\Logfile.etl" + + # GUID of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.guid: {EB79061A-A566-4698-9119-3ED2807060E7} + + # Name of an ETW provider. + # Run 'logman query providers' to list the available providers. + #provider.name: Microsoft-Windows-DNSServer + + # Tag to identify created sessions. + # If missing, its default value is the provider ID prefixed by 'Elastic-'. + #session_name: DNSServer-Analytical-Trace + + # Filter collected events with a level value that is less than or equal to this level. + # Allowed values are critical, error, warning, informational, and verbose. + #trace_level: verbose + + # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. + # The provider will write a particular event if the event's keyword bits match any of the bits + # in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_any_keyword: 0x8000000000000000 + + # 8-byte bitmask that enables the filtering of events from + # specific provider subcomponents. The provider will write a particular + # event if the event's keyword bits match all of the bits in this bitmask. + # Run 'logman query providers ""' to list available keywords. + #match_all_keyword: 0 + + # An existing session to read from. + # Run 'logman query -ets' to list existing sessions. + #session: UAL_Usermode_Provider diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 45cd1994eeb..419278ce85d 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1878,135 +1878,135 @@ filebeat.modules: #var.password: #------------------------------ Salesforce Module ------------------------------ -- module: salesforce - - apex-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - login-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - login-stream: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - logout-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - logout-stream: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - setupaudittrail-rest: - enabled: false - - # Oauth Client ID - #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Client Secret - #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - - # Oauth Token URL - #var.token_url: "https://login.salesforce.com/services/oauth2/token" - - # Oauth User, should include the User mail - #var.user: "abc.xyz@mail.com" - - # Oauth password, should include the User password - #var.password: "P@$$W0₹D" - - # URL, should include the instance_url - #var.url: "https://instance_id.my.salesforce.com" - - # Interval, should include the time interval +- module: salesforce + + apex-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + login-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + login-stream: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + logout-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + logout-stream: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + setupaudittrail-rest: + enabled: false + + # Oauth Client ID + #var.client_id: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Client Secret + #var.client_secret: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + + # Oauth Token URL + #var.token_url: "https://login.salesforce.com/services/oauth2/token" + + # Oauth User, should include the User mail + #var.user: "abc.xyz@mail.com" + + # Oauth password, should include the User password + #var.password: "P@$$W0₹D" + + # URL, should include the instance_url + #var.url: "https://instance_id.my.salesforce.com" + + # Interval, should include the time interval #var.interval: 1h #----------------------------- Google Santa Module ----------------------------- - module: santa @@ -3601,16 +3601,16 @@ filebeat.inputs: #session_name: DNSServer-Analytical-Trace # Filter collected events with a level value that is less than or equal to this level. - # Allowed values are critical, error, warning, informational, and verbose. + # Allowed values are critical, error, warning, informational, and verbose. #trace_level: verbose # 8-byte bitmask that enables the filtering of events from specific provider subcomponents. - # The provider will write a particular event if the event's keyword bits match any of the bits + # The provider will write a particular event if the event's keyword bits match any of the bits # in this bitmask. # Run 'logman query providers ""' to list available keywords. #match_any_keyword: 0x8000000000000000 - # 8-byte bitmask that enables the filtering of events from + # 8-byte bitmask that enables the filtering of events from # specific provider subcomponents. The provider will write a particular # event if the event's keyword bits match all of the bits in this bitmask. # Run 'logman query providers ""' to list available keywords. diff --git a/x-pack/filebeat/input/etw/config_test.go b/x-pack/filebeat/input/etw/config_test.go index 7be2acbd56b..d18af17bd64 100644 --- a/x-pack/filebeat/input/etw/config_test.go +++ b/x-pack/filebeat/input/etw/config_test.go @@ -33,7 +33,9 @@ func Test_validateConfig(t *testing.T) { { name: "minimal config", config: config{ - ProviderName: "Microsoft-Windows-DNSServer", + ProviderName: "Microsoft-Windows-DNSServer", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, }, }, { @@ -130,7 +132,7 @@ func Test_validateConfig(t *testing.T) { } } else { if err != nil { - t.Fatalf("Configuration validation failed. no error expected but got '%v'", err) + t.Fatalf("Configuration validation failed. No error expected but got '%v'", err) } } }) From a860810bc38314ed1a97e729050d01ba12b36340 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Mon, 4 Dec 2023 11:49:37 +0100 Subject: [PATCH 11/31] Fix lint error in input --- x-pack/filebeat/input/etw/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 2a41532bc33..c50c4d13bbf 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -110,7 +110,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { e.log.Debugf("received event %d with length %d", er.EventHeader.EventDescriptor.Id, er.UserDataLength) - event := make(map[string]interface{}) + var event map[string]interface{} if data, err := etw.GetEventProperties(er); err == nil { event = map[string]interface{}{ From 87779acd8b4875a612f03e77a0f1cda660e0a590 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 20 Dec 2023 08:35:21 +0100 Subject: [PATCH 12/31] Improve docs with supported providers and platforms --- x-pack/filebeat/docs/inputs/input-etw.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/docs/inputs/input-etw.asciidoc b/x-pack/filebeat/docs/inputs/input-etw.asciidoc index ce0a138e0ba..9ace3fdcc1b 100644 --- a/x-pack/filebeat/docs/inputs/input-etw.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-etw.asciidoc @@ -11,10 +11,14 @@ beta[] -`https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal`[Event Tracing for Windows]) is a powerful logging and tracing mechanism built into the Windows operating system. It provides a detailed view of application and system behavior, performance issues, and runtime diagnostics. Trace events contain an event header and provider-defined data that describes the current state of an application or operation. You can use the events to debug an application and perform capacity and performance analysis. +https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal[Event Tracing for Windows] is a powerful logging and tracing mechanism built into the Windows operating system. It provides a detailed view of application and system behavior, performance issues, and runtime diagnostics. Trace events contain an event header and provider-defined data that describes the current state of an application or operation. You can use the events to debug an application and perform capacity and performance analysis. The ETW input can interact with ETW in three distinct ways: it can create a new session to capture events from user-mode providers, attach to an already existing session to collect ongoing event data, or read events from a pre-recorded .etl file. This functionality enables the module to adapt to different scenarios, such as real-time event monitoring or analyzing historical data. +This input currently supports manifest-based, MOF (classic) and TraceLogging providers while WPP providers are not supported. https://learn.microsoft.com/en-us/windows/win32/etw/about-event-tracing#types-of-providers[Here] you can find more information about the available types of providers. + +It has been tested in every Windows versions supported by Filebeat, starting from Windows 8.1 and Windows Server 2016. In addition, administrative privileges are required in order to control event tracing sessions. + Example configurations: Read from a provider by name: From e73c27e8d67f9b0e889204dd6a2cf228d8bc9095 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 6 Feb 2024 00:39:43 +0100 Subject: [PATCH 13/31] Fix requested changes for ETW input --- filebeat/docs/filebeat-options.asciidoc | 12 +-- x-pack/filebeat/input/etw/input.go | 100 +++++++++++++++--------- x-pack/libbeat/reader/etw/session.go | 8 +- 3 files changed, 75 insertions(+), 45 deletions(-) diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 8bbecb8b4d8..100a984a80b 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -75,8 +75,10 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-cometd>> * <<{beatname_lc}-input-container>> * <<{beatname_lc}-input-entity-analytics>> +* <<{beatname_lc}-input-etw>> * <<{beatname_lc}-input-filestream>> * <<{beatname_lc}-input-gcp-pubsub>> +* <<{beatname_lc}-input-gcs>> * <<{beatname_lc}-input-http_endpoint>> * <<{beatname_lc}-input-httpjson>> * <<{beatname_lc}-input-journald>> @@ -90,8 +92,6 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-syslog>> * <<{beatname_lc}-input-tcp>> * <<{beatname_lc}-input-udp>> -* <<{beatname_lc}-input-gcs>> -* <<{beatname_lc}-input-etw>> include::multiline.asciidoc[] @@ -113,10 +113,14 @@ include::inputs/input-container.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-etw.asciidoc[] + include::inputs/input-filestream.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[] + include::../../x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-httpjson.asciidoc[] @@ -144,7 +148,3 @@ include::inputs/input-tcp.asciidoc[] include::inputs/input-udp.asciidoc[] include::inputs/input-unix.asciidoc[] - -include::../../x-pack/filebeat/docs/inputs/input-gcs.asciidoc[] - -include::../../x-pack/filebeat/docs/inputs/input-etw.asciidoc[] diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index c50c4d13bbf..7f9b92eff02 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -8,8 +8,8 @@ package etw import ( "fmt" + "math" "sync" - "syscall" "time" input "github.com/elastic/beats/v7/filebeat/input/v2" @@ -20,6 +20,8 @@ import ( conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + + "golang.org/x/sys/windows" ) const ( @@ -30,7 +32,7 @@ const ( type etwInput struct { log *logp.Logger config config - etwSession etw.Session + etwSession *etw.Session } func Plugin() input.Plugin { @@ -66,16 +68,13 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { // Initialize a new ETW session with the provided configuration. e.etwSession, err = etw.NewSession(convertConfig(e.config)) if err != nil { - return fmt.Errorf("error when initializing '%s' session: %w", e.etwSession.Name, err) + return fmt.Errorf("error initializing ETW session: %w", err) } // Set up logger with session information. e.log = ctx.Logger.With("session", e.etwSession.Name) e.log.Info("Starting " + inputName + " input") - var wg sync.WaitGroup - var once sync.Once - // Handle realtime session creation or attachment. if e.etwSession.Realtime { if !e.etwSession.NewSession { @@ -95,6 +94,9 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { } } // Defer the cleanup and closing of resources. + var wg sync.WaitGroup + var once sync.Once + defer func() { wg.Wait() // Ensure all goroutines have finished before closing. once.Do(e.Close) @@ -102,23 +104,16 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { }() // eventReceivedCallback processes each ETW event. - eventReceivedCallback := func(er *etw.EventRecord) uintptr { - if er == nil { + eventReceivedCallback := func(record *etw.EventRecord) uintptr { + if record == nil { e.log.Error("received null event record") return 1 } - e.log.Debugf("received event %d with length %d", er.EventHeader.EventDescriptor.Id, er.UserDataLength) - - var event map[string]interface{} + e.log.Debugf("received event %d with length %d", record.EventHeader.EventDescriptor.Id, record.UserDataLength) - if data, err := etw.GetEventProperties(er); err == nil { - event = map[string]interface{}{ - "Header": er.EventHeader, - "EventProperties": data, - "Metadata": fillEventMetadata(er, e.etwSession, e.config), - } - } else { + data, err := etw.GetEventProperties(record) + if err != nil { e.log.Errorf("failed to read event properties: %w", err) return 1 } @@ -126,18 +121,19 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { evt := beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "metadata": event["Metadata"], - "header": event["Header"], - "winlog": event["EventProperties"], + "metadata": fillEventMetadata(record, e.etwSession, e.config), + "header": fillEventHeader(record.EventHeader), + "winlog": data, }, } + publisher.Publish(evt) return 0 } // Set the callback function for the ETW session. - e.etwSession.Callback = syscall.NewCallback(eventReceivedCallback) + e.etwSession.Callback = eventReceivedCallback // Start a goroutine to consume ETW events. wg.Add(1) @@ -159,8 +155,8 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return nil } -// fillEventMetadata constructs a metadata map for an event record. -func fillEventMetadata(er *etw.EventRecord, session etw.Session, cfg config) map[string]interface{} { +// fillEventHeader constructs a header map for an event record header. +func fillEventHeader(h etw.EventHeader) map[string]interface{} { // Mapping from Level to Severity levelToSeverity := map[uint8]string{ 1: "critical", @@ -170,37 +166,71 @@ func fillEventMetadata(er *etw.EventRecord, session etw.Session, cfg config) map 5: "verbose", } - metadata := make(map[string]interface{}) - + header := make(map[string]interface{}) + + header["size"] = h.Size + header["type"] = h.HeaderType + header["flags"] = h.Flags + header["event_property"] = h.EventProperty + header["thread_id"] = h.ThreadId + header["process_id"] = h.ProcessId + header["timestamp"] = convertFileTimeToGoTime(uint64(h.TimeStamp)) + header["provider_guid"] = h.ProviderId.String() + header["event_id"] = h.EventDescriptor.Id + header["event_version"] = h.EventDescriptor.Version + header["channel"] = h.EventDescriptor.Channel + header["level"] = h.EventDescriptor.Level // Get the severity level, with a default value if not found - severity, ok := levelToSeverity[er.EventHeader.EventDescriptor.Level] + severity, ok := levelToSeverity[h.EventDescriptor.Level] if !ok { severity = "unknown" // Default severity level } - metadata["Severity"] = severity + header["severity"] = severity + header["opcode"] = h.EventDescriptor.Opcode + header["task"] = h.EventDescriptor.Task + header["keyword"] = h.EventDescriptor.Keyword + header["time"] = h.Time + header["activity_guid"] = h.ActivityId.String() + + return header +} + +// convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure. +func convertFileTimeToGoTime(fileTime64 uint64) time.Time { + fileTime := windows.Filetime{ + HighDateTime: uint32(fileTime64 >> 32), + LowDateTime: uint32(fileTime64 & math.MaxUint32), + } + + return time.Unix(0, fileTime.Nanoseconds()) +} + +// fillEventMetadata constructs a metadata map for an event record. +func fillEventMetadata(record *etw.EventRecord, session *etw.Session, cfg config) map[string]interface{} { + metadata := make(map[string]interface{}) // Include provider name and GUID in metadata if available if cfg.ProviderName != "" { - metadata["ProviderName"] = cfg.ProviderName + metadata["provider_name"] = cfg.ProviderName } if cfg.ProviderGUID != "" { - metadata["ProviderGUID"] = cfg.ProviderGUID + metadata["provider_guid"] = cfg.ProviderGUID } else if etw.IsGUIDValid(session.GUID) { - metadata["ProviderGUID"] = etw.GUIDToString(session.GUID) + metadata["provider_guid"] = session.GUID.String() } // Include logfile path if available if cfg.Logfile != "" { - metadata["Logfile"] = cfg.Logfile + metadata["logfile"] = cfg.Logfile } // Include session name if available if cfg.Session != "" { - metadata["Session"] = cfg.Session + metadata["session"] = cfg.Session } else if cfg.SessionName != "" { - metadata["Session"] = cfg.SessionName + metadata["session"] = cfg.SessionName } else if cfg.ProviderGUID != "" || cfg.ProviderName != "" { - metadata["Session"] = session.Name + metadata["session"] = session.Name } return metadata diff --git a/x-pack/libbeat/reader/etw/session.go b/x-pack/libbeat/reader/etw/session.go index 3a8e7be51d7..3216ff3af05 100644 --- a/x-pack/libbeat/reader/etw/session.go +++ b/x-pack/libbeat/reader/etw/session.go @@ -156,9 +156,8 @@ func newSessionProperties(sessionName string) *EventTraceProperties { } // NewSession initializes and returns a new ETW Session based on the provided configuration. -func NewSession(conf Config) (Session, error) { - var session Session - var err error +func NewSession(conf Config) (*Session, error) { + session := &Session{} // Assign ETW Windows API functions session.startTrace = _StartTrace @@ -183,9 +182,10 @@ func NewSession(conf Config) (Session, error) { session.NewSession = true // Indicate this is a new session + var err error session.GUID, err = setSessionGUIDFunc(conf) if err != nil { - return Session{}, err + return nil, fmt.Errorf("error when initializing session '%s': %w", session.Name, err) } // Initialize additional session properties. From aea30593a010d442d86bed47269ad716e921347a Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 6 Feb 2024 12:07:18 +0100 Subject: [PATCH 14/31] Add ETW input to changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9e27962b7e5..690951ad144 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -175,6 +175,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] - Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] - Add support for PEM-based Okta auth in CEL. {pull}37813[37813] +- Add ETW input. {pull}36915[36915] *Auditbeat* From 6d518d4a53a66f40dc2eb07b7538750efbfcae40 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Tue, 6 Feb 2024 12:09:29 +0100 Subject: [PATCH 15/31] Rename GetHandler to AttachToExistingSession in ETW input --- x-pack/filebeat/input/etw/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 7f9b92eff02..ef0b9e63300 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -79,7 +79,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { if e.etwSession.Realtime { if !e.etwSession.NewSession { // Attach to an existing session. - err = e.etwSession.GetHandler() + err = e.etwSession.AttachToExistingSession() if err != nil { return fmt.Errorf("unable to retrieve handler: %w", err) } From 81ce1717c9774a6e674b1620c4b273acd9940454 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 7 Feb 2024 10:57:05 +0100 Subject: [PATCH 16/31] Fix NewSession unit test --- x-pack/libbeat/reader/etw/session_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/libbeat/reader/etw/session_test.go b/x-pack/libbeat/reader/etw/session_test.go index 005b9839d5c..f79c9473f3e 100644 --- a/x-pack/libbeat/reader/etw/session_test.go +++ b/x-pack/libbeat/reader/etw/session_test.go @@ -209,9 +209,8 @@ func TestNewSession_GUIDError(t *testing.T) { } session, err := NewSession(conf) - assert.EqualError(t, err, "mock error") - expectedSession := Session{} - assert.Equal(t, expectedSession, session, "Session should be its zero value when an error occurs") + assert.EqualError(t, err, "error when initializing session 'Session1': mock error") + assert.Nil(t, session) } From 4b52f5461eaf9ebc67c16ff7a51795ea00e9bdac Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 7 Feb 2024 20:29:23 +0100 Subject: [PATCH 17/31] Add tests for input helpers --- x-pack/filebeat/input/etw/input_test.go | 202 ++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 x-pack/filebeat/input/etw/input_test.go diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go new file mode 100644 index 00000000000..1f89f4b7adf --- /dev/null +++ b/x-pack/filebeat/input/etw/input_test.go @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build windows + +package etw + +import ( + "testing" + "time" + + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" + "gotest.tools/assert" + + "golang.org/x/sys/windows" +) + +func Test_fillEventHeader(t *testing.T) { + tests := []struct { + name string + header etw.EventHeader + expected map[string]interface{} + }{ + { + name: "Test with Level 1 (Critical)", + header: etw.EventHeader{ + Size: 100, + HeaderType: 10, + Flags: 20, + EventProperty: 30, + ThreadId: 40, + ProcessId: 50, + TimeStamp: 133516441890350000, + ProviderId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + EventDescriptor: etw.EventDescriptor{ + Id: 60, + Version: 70, + Channel: 80, + Level: 1, // Critical + Opcode: 90, + Task: 100, + Keyword: 110, + }, + Time: 120, + ActivityId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + }, + expected: map[string]interface{}{ + "size": 100, + "type": 10, + "flags": 20, + "event_property": 30, + "thread_id": 40, + "process_id": 50, + "timestamp": "2024-02-05T22:03:09.035Z", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "event_id": 60, + "event_version": 70, + "channel": 80, + "level": 1, + "severity": "critical", + "opcode": 90, + "task": 100, + "keyword": 110, + "time": 120, + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + header := fillEventHeader(tt.header) + assert.Equal(t, tt.expected["size"], header["size"]) + + }) + } +} + +func Test_convertFileTimeToGoTime(t *testing.T) { + tests := []struct { + name string + fileTime uint64 + want time.Time + }{ + { + name: "Windows epoch", + fileTime: 0, // January 1, 1601 (Windows epoch) + want: time.Date(1601, 01, 01, 0, 0, 0, 0, time.UTC), + }, + { + name: "Unix epoch", + fileTime: 116444736000000000, // January 1, 1970 (Unix epoch) + want: time.Unix(0, 0), + }, + { + name: "Actual date", + fileTime: 133515900000000000, // February 05, 2023, 7:00:00 AM + want: time.Date(2023, 02, 05, 7, 0, 0, 0, time.UTC), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := convertFileTimeToGoTime(tt.fileTime) + if !got.Equal(tt.want) { + t.Errorf("convertFileTimeToGoTime() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_fillEventMetadata(t *testing.T) { + tests := []struct { + name string + record *etw.EventRecord + session *etw.Session + cfg config + expected map[string]interface{} + }{ + // Test Provider Name and GUID from config + { + name: "TestProviderNameAndGUIDFromConfig", + record: &etw.EventRecord{}, + session: &etw.Session{ + GUID: windows.GUID{}, + Name: "SessionName", + }, + cfg: config{ + ProviderName: "TestProvider", + ProviderGUID: "{12345678-1234-1234-1234-123456789abc}", + }, + expected: map[string]interface{}{ + "ProviderName": "TestProvider", + "ProviderGUID": "{12345678-1234-1234-1234-123456789abc}", + }, + }, + // Test Provider GUID from session if not available in config + { + name: "TestProviderGUIDFromSession", + record: &etw.EventRecord{}, + session: &etw.Session{ + GUID: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + }, + cfg: config{ + ProviderName: "TestProvider", + }, + expected: map[string]interface{}{ + "ProviderName": "TestProvider", + "ProviderGUID": "{12345678-1234-1234-1234-123456789abc}", + }, + }, + // Test Logfile and Session Information + { + name: "TestLogfileAndSessionInfo", + record: &etw.EventRecord{}, + session: &etw.Session{ + GUID: windows.GUID{}, + Name: "SessionName", + }, + cfg: config{ + Logfile: "C:\\Logs\\test.log", + Session: "TestSession", + SessionName: "DifferentSessionName", + }, + expected: map[string]interface{}{ + "Logfile": "C:\\Logs\\test.log", + "Session": "TestSession", + }, + }, + // Test with nil EventRecord + { + name: "TestWithNilEventRecord", + record: nil, + session: nil, + cfg: config{}, + expected: map[string]interface{}{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := fillEventMetadata(tt.record, tt.session, tt.cfg) + assert.Equal(t, tt.expected, result, "fillEventMetadata() should match the expected output") + }) + } +} From 1a8b6359abe121abbb7d55b6979d277d94e6fde9 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 7 Feb 2024 21:14:17 +0100 Subject: [PATCH 18/31] Fix linting error in input_test.go --- x-pack/filebeat/input/etw/input_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index 1f89f4b7adf..0d1ec077653 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -10,8 +10,9 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" - "gotest.tools/assert" "golang.org/x/sys/windows" ) From 755717919a68e116a9a0395098ea852e698b1106 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Wed, 7 Feb 2024 23:10:35 +0100 Subject: [PATCH 19/31] Fix some unit tests --- x-pack/filebeat/input/etw/input.go | 13 +++- x-pack/filebeat/input/etw/input_test.go | 95 +++++++++++++------------ 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index ef0b9e63300..5bfa1c2dacf 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -121,7 +121,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { evt := beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "metadata": fillEventMetadata(record, e.etwSession, e.config), + "metadata": fillEventMetadata(e.etwSession, e.config), "header": fillEventHeader(record.EventHeader), "winlog": data, }, @@ -197,6 +197,13 @@ func fillEventHeader(h etw.EventHeader) map[string]interface{} { // convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure. func convertFileTimeToGoTime(fileTime64 uint64) time.Time { + // Define the offset between Windows epoch (1601) and Unix epoch (1970) + const epochDifference = 116444736000000000 + if fileTime64 < epochDifference { + // Time is before the Unix epoch, adjust accordingly + return time.Time{} + } + fileTime := windows.Filetime{ HighDateTime: uint32(fileTime64 >> 32), LowDateTime: uint32(fileTime64 & math.MaxUint32), @@ -205,8 +212,8 @@ func convertFileTimeToGoTime(fileTime64 uint64) time.Time { return time.Unix(0, fileTime.Nanoseconds()) } -// fillEventMetadata constructs a metadata map for an event record. -func fillEventMetadata(record *etw.EventRecord, session *etw.Session, cfg config) map[string]interface{} { +// fillEventMetadata constructs a metadata map with session information. +func fillEventMetadata(session *etw.Session, cfg config) map[string]interface{} { metadata := make(map[string]interface{}) // Include provider name and GUID in metadata if available diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index 0d1ec077653..765d96404be 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -24,7 +24,7 @@ func Test_fillEventHeader(t *testing.T) { expected map[string]interface{} }{ { - name: "Test with Level 1 (Critical)", + name: "TestStandardHeader", header: etw.EventHeader{ Size: 100, HeaderType: 10, @@ -57,23 +57,23 @@ func Test_fillEventHeader(t *testing.T) { }, }, expected: map[string]interface{}{ - "size": 100, - "type": 10, - "flags": 20, - "event_property": 30, - "thread_id": 40, - "process_id": 50, + "size": uint16(100), + "type": uint16(10), + "flags": uint16(20), + "event_property": uint16(30), + "thread_id": uint32(40), + "process_id": uint32(50), "timestamp": "2024-02-05T22:03:09.035Z", "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "event_id": 60, - "event_version": 70, - "channel": 80, - "level": 1, + "event_id": uint16(60), + "event_version": uint8(70), + "channel": uint8(80), + "level": uint8(1), "severity": "critical", - "opcode": 90, - "task": 100, - "keyword": 110, - "time": 120, + "opcode": uint8(90), + "task": uint16(100), + "keyword": uint64(110), + "time": uint64(120), "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", }, }, @@ -83,7 +83,22 @@ func Test_fillEventHeader(t *testing.T) { t.Run(tt.name, func(t *testing.T) { header := fillEventHeader(tt.header) assert.Equal(t, tt.expected["size"], header["size"]) - + assert.Equal(t, tt.expected["type"], header["type"]) + assert.Equal(t, tt.expected["flags"], header["flags"]) + assert.Equal(t, tt.expected["event_property"], header["event_property"]) + assert.Equal(t, tt.expected["thread_id"], header["thread_id"]) + assert.Equal(t, tt.expected["process_id"], header["process_id"]) + assert.Equal(t, tt.expected["provider_guid"], header["provider_guid"]) + assert.Equal(t, tt.expected["event_id"], header["event_id"]) + assert.Equal(t, tt.expected["event_version"], header["event_version"]) + assert.Equal(t, tt.expected["channel"], header["channel"]) + assert.Equal(t, tt.expected["level"], header["level"]) + assert.Equal(t, tt.expected["severity"], header["severity"]) + assert.Equal(t, tt.expected["opcode"], header["opcode"]) + assert.Equal(t, tt.expected["task"], header["task"]) + assert.Equal(t, tt.expected["keyword"], header["keyword"]) + assert.Equal(t, tt.expected["time"], header["time"]) + assert.Equal(t, tt.expected["activity_guid"], header["activity_guid"]) }) } } @@ -95,19 +110,19 @@ func Test_convertFileTimeToGoTime(t *testing.T) { want time.Time }{ { - name: "Windows epoch", - fileTime: 0, // January 1, 1601 (Windows epoch) - want: time.Date(1601, 01, 01, 0, 0, 0, 0, time.UTC), + name: "TestZeroValue", + fileTime: 0, + want: time.Time{}, }, { - name: "Unix epoch", + name: "TestUnixEpoch", fileTime: 116444736000000000, // January 1, 1970 (Unix epoch) want: time.Unix(0, 0), }, { - name: "Actual date", - fileTime: 133515900000000000, // February 05, 2023, 7:00:00 AM - want: time.Date(2023, 02, 05, 7, 0, 0, 0, time.UTC), + name: "TestActualDate", + fileTime: 133515900000000000, // February 05, 2024, 7:00:00 AM + want: time.Date(2024, 02, 05, 7, 0, 0, 0, time.UTC), }, } @@ -124,32 +139,29 @@ func Test_convertFileTimeToGoTime(t *testing.T) { func Test_fillEventMetadata(t *testing.T) { tests := []struct { name string - record *etw.EventRecord session *etw.Session cfg config expected map[string]interface{} }{ // Test Provider Name and GUID from config { - name: "TestProviderNameAndGUIDFromConfig", - record: &etw.EventRecord{}, + name: "TestProviderNameAndGUIDFromConfig", session: &etw.Session{ GUID: windows.GUID{}, Name: "SessionName", }, cfg: config{ ProviderName: "TestProvider", - ProviderGUID: "{12345678-1234-1234-1234-123456789abc}", + ProviderGUID: "{12345678-1234-1234-1234-123456789ABC}", }, expected: map[string]interface{}{ - "ProviderName": "TestProvider", - "ProviderGUID": "{12345678-1234-1234-1234-123456789abc}", + "provider_name": "TestProvider", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", }, }, // Test Provider GUID from session if not available in config { - name: "TestProviderGUIDFromSession", - record: &etw.EventRecord{}, + name: "TestProviderGUIDFromSession", session: &etw.Session{ GUID: windows.GUID{ Data1: 0x12345678, @@ -162,14 +174,13 @@ func Test_fillEventMetadata(t *testing.T) { ProviderName: "TestProvider", }, expected: map[string]interface{}{ - "ProviderName": "TestProvider", - "ProviderGUID": "{12345678-1234-1234-1234-123456789abc}", + "provider_name": "TestProvider", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", }, }, // Test Logfile and Session Information { - name: "TestLogfileAndSessionInfo", - record: &etw.EventRecord{}, + name: "TestLogfileAndSessionInfo", session: &etw.Session{ GUID: windows.GUID{}, Name: "SessionName", @@ -180,23 +191,15 @@ func Test_fillEventMetadata(t *testing.T) { SessionName: "DifferentSessionName", }, expected: map[string]interface{}{ - "Logfile": "C:\\Logs\\test.log", - "Session": "TestSession", + "logfile": "C:\\Logs\\test.log", + "session": "TestSession", }, }, - // Test with nil EventRecord - { - name: "TestWithNilEventRecord", - record: nil, - session: nil, - cfg: config{}, - expected: map[string]interface{}{}, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := fillEventMetadata(tt.record, tt.session, tt.cfg) + result := fillEventMetadata(tt.session, tt.cfg) assert.Equal(t, tt.expected, result, "fillEventMetadata() should match the expected output") }) } From 3692a2a7884afb365999f1543099bd71ac9238ba Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 13:36:43 +0100 Subject: [PATCH 20/31] Unit tests for ETW input --- x-pack/filebeat/input/etw/input.go | 78 +++++-- x-pack/filebeat/input/etw/input_test.go | 276 +++++++++++++++++++++++- 2 files changed, 336 insertions(+), 18 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 5bfa1c2dacf..2985c2a1502 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -28,11 +28,44 @@ const ( inputName = "etw" ) +// It abstracts the underlying operations needed to work with ETW, allowing for easier +// testing and decoupling from the Windows-specific ETW API. +type ETWSessionOperator interface { + NewSession(config config) (*etw.Session, error) + AttachToExistingSession(session *etw.Session) error + CreateRealtimeSession(session *etw.Session) error + StartConsumer(session *etw.Session) error + StopSession(session *etw.Session) error +} + +type RealETWSessionOperator struct{} + +func (op *RealETWSessionOperator) NewSession(config config) (*etw.Session, error) { + return etw.NewSession(convertConfig(config)) +} + +func (op *RealETWSessionOperator) AttachToExistingSession(session *etw.Session) error { + return session.AttachToExistingSession() +} + +func (op *RealETWSessionOperator) CreateRealtimeSession(session *etw.Session) error { + return session.CreateRealtimeSession() +} + +func (op *RealETWSessionOperator) StartConsumer(session *etw.Session) error { + return session.StartConsumer() +} + +func (op *RealETWSessionOperator) StopSession(session *etw.Session) error { + return session.StopSession() +} + // etwInput struct holds the configuration and state for the ETW input type etwInput struct { log *logp.Logger config config etwSession *etw.Session + operator ETWSessionOperator } func Plugin() input.Plugin { @@ -51,59 +84,62 @@ func configure(cfg *conf.C) (stateless.Input, error) { } return &etwInput{ - config: conf, + config: conf, + operator: &RealETWSessionOperator{}, }, nil } func (e *etwInput) Name() string { return inputName } func (e *etwInput) Test(_ input.TestContext) error { - // ToDo return nil } // Run starts the ETW session and processes incoming events. func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { var err error - // Initialize a new ETW session with the provided configuration. - e.etwSession, err = etw.NewSession(convertConfig(e.config)) + + // Initialize a new ETW session with the provided configuration + e.etwSession, err = e.operator.NewSession(e.config) if err != nil { return fmt.Errorf("error initializing ETW session: %w", err) } - // Set up logger with session information. + // Set up logger with session information e.log = ctx.Logger.With("session", e.etwSession.Name) e.log.Info("Starting " + inputName + " input") - // Handle realtime session creation or attachment. + // Handle realtime session creation or attachment if e.etwSession.Realtime { if !e.etwSession.NewSession { - // Attach to an existing session. - err = e.etwSession.AttachToExistingSession() + // Attach to an existing session + err = e.operator.AttachToExistingSession(e.etwSession) if err != nil { return fmt.Errorf("unable to retrieve handler: %w", err) } e.log.Debug("attached to existing session") } else { - // Create a new realtime session. - err = e.etwSession.CreateRealtimeSession() + // Create a new realtime session + err = e.operator.CreateRealtimeSession(e.etwSession) if err != nil { return fmt.Errorf("realtime session could not be created: %w", err) } e.log.Debug("created session") } } - // Defer the cleanup and closing of resources. + // Defer the cleanup and closing of resources var wg sync.WaitGroup var once sync.Once + // Create an error channel to communicate errors from the goroutine + errChan := make(chan error, 1) + defer func() { - wg.Wait() // Ensure all goroutines have finished before closing. once.Do(e.Close) e.log.Info(inputName + " input stopped") }() - // eventReceivedCallback processes each ETW event. + // eventReceivedCallback processes each ETW event eventReceivedCallback := func(record *etw.EventRecord) uintptr { if record == nil { e.log.Error("received null event record") @@ -132,18 +168,20 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return 0 } - // Set the callback function for the ETW session. + // Set the callback function for the ETW session e.etwSession.Callback = eventReceivedCallback - // Start a goroutine to consume ETW events. + // Start a goroutine to consume ETW events wg.Add(1) go func() { defer wg.Done() e.log.Debug("starting to listen ETW events") - if err = e.etwSession.StartConsumer(); err != nil { - e.log.Warnf("events could not be read from session: %w", err) + if err = e.operator.StartConsumer(e.etwSession); err != nil { + errChan <- fmt.Errorf("failed to start consumer: %w", err) // Send error to channel + return } e.log.Debug("stopped to read ETW events from session") + errChan <- nil }() // We ensure resources are closed when receiving a cancelation signal @@ -152,6 +190,12 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { once.Do(e.Close) }() + wg.Wait() // Ensure all goroutines have finished before closing + close(errChan) + if err, ok := <-errChan; ok && err != nil { + return err + } + return nil } diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index 765d96404be..f91688cd7f0 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -7,16 +7,287 @@ package etw import ( + "context" + "fmt" "testing" "time" "github.com/stretchr/testify/assert" + input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" + "github.com/elastic/elastic-agent-libs/logp" "golang.org/x/sys/windows" ) +type MockETWSessionOperator struct { + // Fields to store function implementations that tests can customize + NewSessionFunc func(config config) (*etw.Session, error) + AttachToExistingSessionFunc func(session *etw.Session) error + CreateRealtimeSessionFunc func(session *etw.Session) error + StartConsumerFunc func(session *etw.Session) error + StopSessionFunc func(session *etw.Session) error +} + +func (m *MockETWSessionOperator) NewSession(config config) (*etw.Session, error) { + if m.NewSessionFunc != nil { + return m.NewSessionFunc(config) + } + return nil, nil +} + +func (m *MockETWSessionOperator) AttachToExistingSession(session *etw.Session) error { + if m.AttachToExistingSessionFunc != nil { + return m.AttachToExistingSessionFunc(session) + } + return nil +} + +func (m *MockETWSessionOperator) CreateRealtimeSession(session *etw.Session) error { + if m.CreateRealtimeSessionFunc != nil { + return m.CreateRealtimeSessionFunc(session) + } + return nil +} + +func (m *MockETWSessionOperator) StartConsumer(session *etw.Session) error { + if m.StartConsumerFunc != nil { + return m.StartConsumerFunc(session) + } + return nil +} + +func (m *MockETWSessionOperator) StopSession(session *etw.Session) error { + if m.StopSessionFunc != nil { + return m.StopSessionFunc(session) + } + return nil +} + +func Test_RunEtwInput_NewSessionError(t *testing.T) { + // Mocks + mockOperator := &MockETWSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + return nil, fmt.Errorf("failed creating session '%s'", config.SessionName) + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "error initializing ETW session: failed creating session 'MySession'") +} + +func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { + // Mocks + mockOperator := &MockETWSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: false} + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "unable to retrieve handler: mock error") +} + +func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { + // Mocks + mockOperator := &MockETWSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true} + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + + // Setup input + inputCtx := input.Context{ + Cancelation: nil, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "realtime session could not be created: mock error") +} + +func Test_RunEtwInput_StartConsumerError(t *testing.T) { + // Mocks + mockOperator := &MockETWSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true} + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for StartConsumer + mockOperator.StartConsumerFunc = func(session *etw.Session) error { + return fmt.Errorf("mock error") + } + + // Setup cancellation + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + // Setup input + inputCtx := input.Context{ + Cancelation: ctx, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + err := etwInput.Run(inputCtx, nil) + assert.EqualError(t, err, "failed to start consumer: mock error") +} + +func Test_RunEtwInput_Success(t *testing.T) { + // Mocks + mockOperator := &MockETWSessionOperator{} + + // Setup the mock behavior for NewSession + mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockSession := &etw.Session{ + Name: "MySession", + Realtime: true, + NewSession: true} + return mockSession, nil + } + // Setup the mock behavior for AttachToExistingSession + mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for CreateRealtimeSession + mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + return nil + } + // Setup the mock behavior for StartConsumer + mockOperator.StartConsumerFunc = func(session *etw.Session) error { + return nil + } + + // Setup cancellation + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + // Setup input + inputCtx := input.Context{ + Cancelation: ctx, + Logger: logp.NewLogger("test"), + } + + etwInput := &etwInput{ + config: config{ + ProviderName: "Microsoft-Windows-Provider", + SessionName: "MySession", + TraceLevel: "verbose", + MatchAnyKeyword: 0xffffffffffffffff, + MatchAllKeyword: 0, + }, + operator: mockOperator, + } + + // Run test + go func() { + err := etwInput.Run(inputCtx, nil) + if err != nil { + t.Errorf("Run() error = %v, wantErr %v", err, false) + } + }() + + // Simulate waiting for a condition + time.Sleep(time.Millisecond * 100) + cancelFunc() // Trigger cancellation to test cleanup and goroutine exit +} + func Test_fillEventHeader(t *testing.T) { tests := []struct { name string @@ -73,7 +344,7 @@ func Test_fillEventHeader(t *testing.T) { "opcode": uint8(90), "task": uint16(100), "keyword": uint64(110), - "time": uint64(120), + "time": int64(120), "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", }, }, @@ -157,6 +428,7 @@ func Test_fillEventMetadata(t *testing.T) { expected: map[string]interface{}{ "provider_name": "TestProvider", "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "SessionName", }, }, // Test Provider GUID from session if not available in config @@ -169,6 +441,7 @@ func Test_fillEventMetadata(t *testing.T) { Data3: 0x1234, Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, }, + Name: "Elastic-TestProvider", }, cfg: config{ ProviderName: "TestProvider", @@ -176,6 +449,7 @@ func Test_fillEventMetadata(t *testing.T) { expected: map[string]interface{}{ "provider_name": "TestProvider", "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "Elastic-TestProvider", }, }, // Test Logfile and Session Information From a2c138fc74b8a0509aa4b36e141a9162e3e42612 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 14:27:13 +0100 Subject: [PATCH 21/31] Fix CloseSession call in tests --- x-pack/filebeat/input/etw/input.go | 2 +- x-pack/filebeat/input/etw/input_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 2985c2a1502..b28c944f209 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -289,7 +289,7 @@ func fillEventMetadata(session *etw.Session, cfg config) map[string]interface{} // close stops the ETW session and logs the outcome. func (e *etwInput) Close() { - if err := e.etwSession.StopSession(); err != nil { + if err := e.operator.StopSession(e.etwSession); err != nil { e.log.Error("failed to shutdown ETW session") } e.log.Info("successfully shutdown") diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index f91688cd7f0..b764fc63e33 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -202,6 +202,10 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { mockOperator.StartConsumerFunc = func(session *etw.Session) error { return fmt.Errorf("mock error") } + // Setup the mock behavior for StopSession + mockOperator.StopSessionFunc = func(session *etw.Session) error { + return nil + } // Setup cancellation ctx, cancelFunc := context.WithCancel(context.Background()) @@ -253,6 +257,10 @@ func Test_RunEtwInput_Success(t *testing.T) { mockOperator.StartConsumerFunc = func(session *etw.Session) error { return nil } + // Setup the mock behavior for StopSession + mockOperator.StopSessionFunc = func(session *etw.Session) error { + return nil + } // Setup cancellation ctx, cancelFunc := context.WithCancel(context.Background()) From e4df38f909153b38cd4ccb06bd14a3e2b3711da5 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 21:41:03 +0100 Subject: [PATCH 22/31] Fix building of event and some refactors --- x-pack/filebeat/input/etw/input.go | 133 ++++----- x-pack/filebeat/input/etw/input_test.go | 353 +++++++++++++----------- 2 files changed, 243 insertions(+), 243 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index b28c944f209..ae8c6bb2c6b 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -30,33 +30,33 @@ const ( // It abstracts the underlying operations needed to work with ETW, allowing for easier // testing and decoupling from the Windows-specific ETW API. -type ETWSessionOperator interface { - NewSession(config config) (*etw.Session, error) - AttachToExistingSession(session *etw.Session) error - CreateRealtimeSession(session *etw.Session) error - StartConsumer(session *etw.Session) error - StopSession(session *etw.Session) error +type sessionOperator interface { + newSession(config config) (*etw.Session, error) + attachToExistingSession(session *etw.Session) error + createRealtimeSession(session *etw.Session) error + startConsumer(session *etw.Session) error + stopSession(session *etw.Session) error } -type RealETWSessionOperator struct{} +type realSessionOperator struct{} -func (op *RealETWSessionOperator) NewSession(config config) (*etw.Session, error) { +func (op *realSessionOperator) newSession(config config) (*etw.Session, error) { return etw.NewSession(convertConfig(config)) } -func (op *RealETWSessionOperator) AttachToExistingSession(session *etw.Session) error { +func (op *realSessionOperator) attachToExistingSession(session *etw.Session) error { return session.AttachToExistingSession() } -func (op *RealETWSessionOperator) CreateRealtimeSession(session *etw.Session) error { +func (op *realSessionOperator) createRealtimeSession(session *etw.Session) error { return session.CreateRealtimeSession() } -func (op *RealETWSessionOperator) StartConsumer(session *etw.Session) error { +func (op *realSessionOperator) startConsumer(session *etw.Session) error { return session.StartConsumer() } -func (op *RealETWSessionOperator) StopSession(session *etw.Session) error { +func (op *realSessionOperator) stopSession(session *etw.Session) error { return session.StopSession() } @@ -65,7 +65,7 @@ type etwInput struct { log *logp.Logger config config etwSession *etw.Session - operator ETWSessionOperator + operator sessionOperator } func Plugin() input.Plugin { @@ -85,7 +85,7 @@ func configure(cfg *conf.C) (stateless.Input, error) { return &etwInput{ config: conf, - operator: &RealETWSessionOperator{}, + operator: &realSessionOperator{}, }, nil } @@ -100,7 +100,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { var err error // Initialize a new ETW session with the provided configuration - e.etwSession, err = e.operator.NewSession(e.config) + e.etwSession, err = e.operator.newSession(e.config) if err != nil { return fmt.Errorf("error initializing ETW session: %w", err) } @@ -113,14 +113,14 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { if e.etwSession.Realtime { if !e.etwSession.NewSession { // Attach to an existing session - err = e.operator.AttachToExistingSession(e.etwSession) + err = e.operator.attachToExistingSession(e.etwSession) if err != nil { return fmt.Errorf("unable to retrieve handler: %w", err) } e.log.Debug("attached to existing session") } else { // Create a new realtime session - err = e.operator.CreateRealtimeSession(e.etwSession) + err = e.operator.createRealtimeSession(e.etwSession) if err != nil { return fmt.Errorf("realtime session could not be created: %w", err) } @@ -157,9 +157,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { evt := beat.Event{ Timestamp: time.Now(), Fields: mapstr.M{ - "metadata": fillEventMetadata(e.etwSession, e.config), - "header": fillEventHeader(record.EventHeader), - "winlog": data, + "winlog": buildEvent(data, record.EventHeader, e.etwSession, e.config), }, } @@ -176,7 +174,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { go func() { defer wg.Done() e.log.Debug("starting to listen ETW events") - if err = e.operator.StartConsumer(e.etwSession); err != nil { + if err = e.operator.startConsumer(e.etwSession); err != nil { errChan <- fmt.Errorf("failed to start consumer: %w", err) // Send error to channel return } @@ -199,8 +197,8 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return nil } -// fillEventHeader constructs a header map for an event record header. -func fillEventHeader(h etw.EventHeader) map[string]interface{} { +// buildEvent builds the winlog object. +func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) map[string]any { // Mapping from Level to Severity levelToSeverity := map[uint8]string{ 1: "critical", @@ -210,33 +208,45 @@ func fillEventHeader(h etw.EventHeader) map[string]interface{} { 5: "verbose", } - header := make(map[string]interface{}) - - header["size"] = h.Size - header["type"] = h.HeaderType - header["flags"] = h.Flags - header["event_property"] = h.EventProperty - header["thread_id"] = h.ThreadId - header["process_id"] = h.ProcessId - header["timestamp"] = convertFileTimeToGoTime(uint64(h.TimeStamp)) - header["provider_guid"] = h.ProviderId.String() - header["event_id"] = h.EventDescriptor.Id - header["event_version"] = h.EventDescriptor.Version - header["channel"] = h.EventDescriptor.Channel - header["level"] = h.EventDescriptor.Level // Get the severity level, with a default value if not found severity, ok := levelToSeverity[h.EventDescriptor.Level] if !ok { severity = "unknown" // Default severity level } - header["severity"] = severity - header["opcode"] = h.EventDescriptor.Opcode - header["task"] = h.EventDescriptor.Task - header["keyword"] = h.EventDescriptor.Keyword - header["time"] = h.Time - header["activity_guid"] = h.ActivityId.String() - - return header + + winlog := map[string]any{ + "activity_guid": h.ActivityId.String(), + "channel": h.EventDescriptor.Channel, + "event_data": data, + "event_id": h.EventDescriptor.Id, + "flags": h.Flags, + "keywords": h.EventDescriptor.Keyword, + "level": h.EventDescriptor.Level, + "opcode": h.EventDescriptor.Opcode, + "process_id": h.ProcessId, + "provider_guid": h.ProviderId.String(), + "session": session.Name, + "severity": severity, + "task": h.EventDescriptor.Task, + "thread_id": h.ThreadId, + "timestamp": convertFileTimeToGoTime(uint64(h.TimeStamp)), + "version": h.EventDescriptor.Version, + } + + // Include provider name and GUID if available + if cfg.ProviderName != "" { + winlog["provider_name"] = cfg.ProviderName + } + if winlog["provider_guid"] == "" && etw.IsGUIDValid(session.GUID) { + winlog["provider_guid"] = session.GUID.String() + } + + // Include logfile path if available + if cfg.Logfile != "" { + winlog["logfile"] = cfg.Logfile + } + + return winlog } // convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure. @@ -256,40 +266,9 @@ func convertFileTimeToGoTime(fileTime64 uint64) time.Time { return time.Unix(0, fileTime.Nanoseconds()) } -// fillEventMetadata constructs a metadata map with session information. -func fillEventMetadata(session *etw.Session, cfg config) map[string]interface{} { - metadata := make(map[string]interface{}) - - // Include provider name and GUID in metadata if available - if cfg.ProviderName != "" { - metadata["provider_name"] = cfg.ProviderName - } - if cfg.ProviderGUID != "" { - metadata["provider_guid"] = cfg.ProviderGUID - } else if etw.IsGUIDValid(session.GUID) { - metadata["provider_guid"] = session.GUID.String() - } - - // Include logfile path if available - if cfg.Logfile != "" { - metadata["logfile"] = cfg.Logfile - } - - // Include session name if available - if cfg.Session != "" { - metadata["session"] = cfg.Session - } else if cfg.SessionName != "" { - metadata["session"] = cfg.SessionName - } else if cfg.ProviderGUID != "" || cfg.ProviderName != "" { - metadata["session"] = session.Name - } - - return metadata -} - // close stops the ETW session and logs the outcome. func (e *etwInput) Close() { - if err := e.operator.StopSession(e.etwSession); err != nil { + if err := e.operator.stopSession(e.etwSession); err != nil { e.log.Error("failed to shutdown ETW session") } e.log.Info("successfully shutdown") diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index b764fc63e33..c84c02dbe7b 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -21,56 +21,56 @@ import ( "golang.org/x/sys/windows" ) -type MockETWSessionOperator struct { +type mockSessionOperator struct { // Fields to store function implementations that tests can customize - NewSessionFunc func(config config) (*etw.Session, error) - AttachToExistingSessionFunc func(session *etw.Session) error - CreateRealtimeSessionFunc func(session *etw.Session) error - StartConsumerFunc func(session *etw.Session) error - StopSessionFunc func(session *etw.Session) error + newSessionFunc func(config config) (*etw.Session, error) + attachToExistingSessionFunc func(session *etw.Session) error + createRealtimeSessionFunc func(session *etw.Session) error + startConsumerFunc func(session *etw.Session) error + stopSessionFunc func(session *etw.Session) error } -func (m *MockETWSessionOperator) NewSession(config config) (*etw.Session, error) { - if m.NewSessionFunc != nil { - return m.NewSessionFunc(config) +func (m *mockSessionOperator) newSession(config config) (*etw.Session, error) { + if m.newSessionFunc != nil { + return m.newSessionFunc(config) } return nil, nil } -func (m *MockETWSessionOperator) AttachToExistingSession(session *etw.Session) error { - if m.AttachToExistingSessionFunc != nil { - return m.AttachToExistingSessionFunc(session) +func (m *mockSessionOperator) attachToExistingSession(session *etw.Session) error { + if m.attachToExistingSessionFunc != nil { + return m.attachToExistingSessionFunc(session) } return nil } -func (m *MockETWSessionOperator) CreateRealtimeSession(session *etw.Session) error { - if m.CreateRealtimeSessionFunc != nil { - return m.CreateRealtimeSessionFunc(session) +func (m *mockSessionOperator) createRealtimeSession(session *etw.Session) error { + if m.createRealtimeSessionFunc != nil { + return m.createRealtimeSessionFunc(session) } return nil } -func (m *MockETWSessionOperator) StartConsumer(session *etw.Session) error { - if m.StartConsumerFunc != nil { - return m.StartConsumerFunc(session) +func (m *mockSessionOperator) startConsumer(session *etw.Session) error { + if m.startConsumerFunc != nil { + return m.startConsumerFunc(session) } return nil } -func (m *MockETWSessionOperator) StopSession(session *etw.Session) error { - if m.StopSessionFunc != nil { - return m.StopSessionFunc(session) +func (m *mockSessionOperator) stopSession(session *etw.Session) error { + if m.stopSessionFunc != nil { + return m.stopSessionFunc(session) } return nil } func Test_RunEtwInput_NewSessionError(t *testing.T) { // Mocks - mockOperator := &MockETWSessionOperator{} + mockOperator := &mockSessionOperator{} // Setup the mock behavior for NewSession - mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { return nil, fmt.Errorf("failed creating session '%s'", config.SessionName) } @@ -98,10 +98,10 @@ func Test_RunEtwInput_NewSessionError(t *testing.T) { func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { // Mocks - mockOperator := &MockETWSessionOperator{} + mockOperator := &mockSessionOperator{} // Setup the mock behavior for NewSession - mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, @@ -109,7 +109,7 @@ func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { return mockSession, nil } // Setup the mock behavior for AttachToExistingSession - mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { return fmt.Errorf("mock error") } @@ -137,10 +137,10 @@ func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { // Mocks - mockOperator := &MockETWSessionOperator{} + mockOperator := &mockSessionOperator{} // Setup the mock behavior for NewSession - mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, @@ -148,11 +148,11 @@ func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { return mockSession, nil } // Setup the mock behavior for AttachToExistingSession - mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for CreateRealtimeSession - mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { return fmt.Errorf("mock error") } @@ -180,10 +180,10 @@ func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { func Test_RunEtwInput_StartConsumerError(t *testing.T) { // Mocks - mockOperator := &MockETWSessionOperator{} + mockOperator := &mockSessionOperator{} // Setup the mock behavior for NewSession - mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, @@ -191,19 +191,19 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { return mockSession, nil } // Setup the mock behavior for AttachToExistingSession - mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for CreateRealtimeSession - mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for StartConsumer - mockOperator.StartConsumerFunc = func(session *etw.Session) error { + mockOperator.startConsumerFunc = func(session *etw.Session) error { return fmt.Errorf("mock error") } // Setup the mock behavior for StopSession - mockOperator.StopSessionFunc = func(session *etw.Session) error { + mockOperator.stopSessionFunc = func(session *etw.Session) error { return nil } @@ -235,10 +235,10 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { func Test_RunEtwInput_Success(t *testing.T) { // Mocks - mockOperator := &MockETWSessionOperator{} + mockOperator := &mockSessionOperator{} // Setup the mock behavior for NewSession - mockOperator.NewSessionFunc = func(config config) (*etw.Session, error) { + mockOperator.newSessionFunc = func(config config) (*etw.Session, error) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, @@ -246,19 +246,19 @@ func Test_RunEtwInput_Success(t *testing.T) { return mockSession, nil } // Setup the mock behavior for AttachToExistingSession - mockOperator.AttachToExistingSessionFunc = func(session *etw.Session) error { + mockOperator.attachToExistingSessionFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for CreateRealtimeSession - mockOperator.CreateRealtimeSessionFunc = func(session *etw.Session) error { + mockOperator.createRealtimeSessionFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for StartConsumer - mockOperator.StartConsumerFunc = func(session *etw.Session) error { + mockOperator.startConsumerFunc = func(session *etw.Session) error { return nil } // Setup the mock behavior for StopSession - mockOperator.StopSessionFunc = func(session *etw.Session) error { + mockOperator.stopSessionFunc = func(session *etw.Session) error { return nil } @@ -296,21 +296,27 @@ func Test_RunEtwInput_Success(t *testing.T) { cancelFunc() // Trigger cancellation to test cleanup and goroutine exit } -func Test_fillEventHeader(t *testing.T) { +func Test_buildEvent(t *testing.T) { tests := []struct { name string + data map[string]any header etw.EventHeader - expected map[string]interface{} + session *etw.Session + cfg config + expected map[string]any }{ { - name: "TestStandardHeader", + name: "TestStandardData", + data: map[string]any{ + "key": "value", + }, header: etw.EventHeader{ - Size: 100, - HeaderType: 10, - Flags: 20, + Size: 0, + HeaderType: 0, + Flags: 30, EventProperty: 30, - ThreadId: 40, - ProcessId: 50, + ThreadId: 80, + ProcessId: 60, TimeStamp: 133516441890350000, ProviderId: windows.GUID{ Data1: 0x12345678, @@ -319,15 +325,15 @@ func Test_fillEventHeader(t *testing.T) { Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, }, EventDescriptor: etw.EventDescriptor{ - Id: 60, - Version: 70, - Channel: 80, + Id: 20, + Version: 90, + Channel: 10, Level: 1, // Critical - Opcode: 90, - Task: 100, - Keyword: 110, + Opcode: 50, + Task: 70, + Keyword: 40, }, - Time: 120, + Time: 0, ActivityId: windows.GUID{ Data1: 0x12345678, Data2: 0x1234, @@ -335,49 +341,136 @@ func Test_fillEventHeader(t *testing.T) { Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, }, }, - expected: map[string]interface{}{ - "size": uint16(100), - "type": uint16(10), - "flags": uint16(20), - "event_property": uint16(30), - "thread_id": uint32(40), - "process_id": uint32(50), - "timestamp": "2024-02-05T22:03:09.035Z", - "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "event_id": uint16(60), - "event_version": uint8(70), - "channel": uint8(80), - "level": uint8(1), - "severity": "critical", - "opcode": uint8(90), - "task": uint16(100), - "keyword": uint64(110), - "time": int64(120), - "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + session: &etw.Session{ + GUID: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + Name: "Elastic-TestProvider", + }, + cfg: config{ + ProviderName: "TestProvider", + }, + + expected: map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": uint8(10), + "event_data": map[string]any{ + "key": "value", + }, + "event_id": uint16(20), + "flags": uint16(30), + "keywords": uint64(40), + "level": uint8(1), + "logfile": "", + "opcode": uint8(50), + "process_id": uint32(60), + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "provider_name": "TestProvider", + "session": "Elastic-TestProvider", + "severity": "critical", + "task": uint16(70), + "thread_id": uint32(80), + "timestamp": "2024-02-05T22:03:09.035Z", + "version": uint8(90), + }, + }, + { + // This case tests an unmapped severity, empty provider GUID and including logfile + name: "TestAlternativeMetadata", + data: map[string]any{ + "key": "value", + }, + header: etw.EventHeader{ + Size: 0, + HeaderType: 0, + Flags: 30, + EventProperty: 30, + ThreadId: 80, + ProcessId: 60, + TimeStamp: 133516441890350000, + ProviderId: windows.GUID{}, + EventDescriptor: etw.EventDescriptor{ + Id: 20, + Version: 90, + Channel: 10, + Level: 17, // Unknown + Opcode: 50, + Task: 70, + Keyword: 40, + }, + Time: 0, + ActivityId: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + }, + session: &etw.Session{ + GUID: windows.GUID{ + Data1: 0x12345678, + Data2: 0x1234, + Data3: 0x1234, + Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, + }, + Name: "Elastic-TestProvider", + }, + cfg: config{ + ProviderName: "TestProvider", + Logfile: "C:\\TestFile", + }, + + expected: map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": uint8(10), + "event_data": map[string]any{ + "key": "value", + }, + "event_id": uint16(20), + "flags": uint16(30), + "keywords": uint64(40), + "level": uint8(17), + "logfile": "C:\\TestFile", + "opcode": uint8(50), + "process_id": uint32(60), + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "provider_name": "TestProvider", + "session": "Elastic-TestProvider", + "severity": "unknown", + "task": uint16(70), + "thread_id": uint32(80), + "timestamp": "2024-02-05T22:03:09.035Z", + "version": uint8(90), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - header := fillEventHeader(tt.header) - assert.Equal(t, tt.expected["size"], header["size"]) - assert.Equal(t, tt.expected["type"], header["type"]) - assert.Equal(t, tt.expected["flags"], header["flags"]) - assert.Equal(t, tt.expected["event_property"], header["event_property"]) - assert.Equal(t, tt.expected["thread_id"], header["thread_id"]) - assert.Equal(t, tt.expected["process_id"], header["process_id"]) - assert.Equal(t, tt.expected["provider_guid"], header["provider_guid"]) - assert.Equal(t, tt.expected["event_id"], header["event_id"]) - assert.Equal(t, tt.expected["event_version"], header["event_version"]) - assert.Equal(t, tt.expected["channel"], header["channel"]) - assert.Equal(t, tt.expected["level"], header["level"]) - assert.Equal(t, tt.expected["severity"], header["severity"]) - assert.Equal(t, tt.expected["opcode"], header["opcode"]) - assert.Equal(t, tt.expected["task"], header["task"]) - assert.Equal(t, tt.expected["keyword"], header["keyword"]) - assert.Equal(t, tt.expected["time"], header["time"]) - assert.Equal(t, tt.expected["activity_guid"], header["activity_guid"]) + winlog := buildEvent(tt.data, tt.header, tt.session, tt.cfg) + + assert.Equal(t, tt.expected["activity_guid"], winlog["activity_guid"]) + assert.Equal(t, tt.expected["channel"], winlog["channel"]) + assert.Equal(t, tt.expected["event_data"], winlog["event_data"]) + assert.Equal(t, tt.expected["event_id"], winlog["event_id"]) + assert.Equal(t, tt.expected["flags"], winlog["flags"]) + assert.Equal(t, tt.expected["keywords"], winlog["keywords"]) + assert.Equal(t, tt.expected["level"], winlog["level"]) + assert.Equal(t, tt.expected["logfile"], winlog["logfile"]) + assert.Equal(t, tt.expected["opcode"], winlog["opcode"]) + assert.Equal(t, tt.expected["process_id"], winlog["process_id"]) + assert.Equal(t, tt.expected["provider_guid"], winlog["provider_guid"]) + assert.Equal(t, tt.expected["provider_name"], winlog["provider_name"]) + assert.Equal(t, tt.expected["session"], winlog["session"]) + assert.Equal(t, tt.expected["severity"], winlog["severity"]) + assert.Equal(t, tt.expected["task"], winlog["task"]) + assert.Equal(t, tt.expected["thread_id"], winlog["thread_id"]) + assert.Equal(t, tt.expected["timestamp"], winlog["timestamp"]) + assert.Equal(t, tt.expected["version"], winlog["version"]) + }) } } @@ -414,75 +507,3 @@ func Test_convertFileTimeToGoTime(t *testing.T) { }) } } - -func Test_fillEventMetadata(t *testing.T) { - tests := []struct { - name string - session *etw.Session - cfg config - expected map[string]interface{} - }{ - // Test Provider Name and GUID from config - { - name: "TestProviderNameAndGUIDFromConfig", - session: &etw.Session{ - GUID: windows.GUID{}, - Name: "SessionName", - }, - cfg: config{ - ProviderName: "TestProvider", - ProviderGUID: "{12345678-1234-1234-1234-123456789ABC}", - }, - expected: map[string]interface{}{ - "provider_name": "TestProvider", - "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "session": "SessionName", - }, - }, - // Test Provider GUID from session if not available in config - { - name: "TestProviderGUIDFromSession", - session: &etw.Session{ - GUID: windows.GUID{ - Data1: 0x12345678, - Data2: 0x1234, - Data3: 0x1234, - Data4: [8]byte{0x12, 0x34, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc}, - }, - Name: "Elastic-TestProvider", - }, - cfg: config{ - ProviderName: "TestProvider", - }, - expected: map[string]interface{}{ - "provider_name": "TestProvider", - "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "session": "Elastic-TestProvider", - }, - }, - // Test Logfile and Session Information - { - name: "TestLogfileAndSessionInfo", - session: &etw.Session{ - GUID: windows.GUID{}, - Name: "SessionName", - }, - cfg: config{ - Logfile: "C:\\Logs\\test.log", - Session: "TestSession", - SessionName: "DifferentSessionName", - }, - expected: map[string]interface{}{ - "logfile": "C:\\Logs\\test.log", - "session": "TestSession", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := fillEventMetadata(tt.session, tt.cfg) - assert.Equal(t, tt.expected, result, "fillEventMetadata() should match the expected output") - }) - } -} From 0fe3b02eaf420ce262561b6eabe6f4912a6e1c74 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 21:41:18 +0100 Subject: [PATCH 23/31] Add field mapping to ETW input --- x-pack/filebeat/input/etw/_meta/fields.yml | 129 +++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 x-pack/filebeat/input/etw/_meta/fields.yml diff --git a/x-pack/filebeat/input/etw/_meta/fields.yml b/x-pack/filebeat/input/etw/_meta/fields.yml new file mode 100644 index 00000000000..4863bdcd8fd --- /dev/null +++ b/x-pack/filebeat/input/etw/_meta/fields.yml @@ -0,0 +1,129 @@ +- key: winlog + title: "Windows ETW" + description: > + Fields from the ETW input (Event Tracing for Windows). + fields: + + - name: winlog + type: group + description: > + All fields specific to the Windows Event Tracing are defined here. + fields: + + - name: activity_id + type: keyword + required: false + description: > + A globally unique identifier that identifies the current activity. The + events that are published with this identifier are part of the same + activity. + + - name: channel + type: short + required: false + description: > + Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. + + - name: event_data + type: object + object_type: keyword + required: false + description: > + The event-specific data. The content of this object is specific to + any provider and event. + + - name: event_id + type: short + required: true + description: > + The event identifier. The value is specific to the source of the event. + + - name: flags + type: short + required: false + description: > + Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. + + - name: keywords + type: long + required: false + description: > + The keywords are used to indicate an event's membership in a set of event categories. + + - name: level + type: short + required: false + description: > + Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. + + - name: logfile + type: keyword + required: false + description: > + The source file from which events are logged. Only available for non real-time sessions. + + - name: opcode + type: short + required: false + description: > + The opcode defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. + + - name: process_id + type: integer + required: false + description: > + Identifies the process that generated the event. + + - name: provider_guid + type: keyword + required: false + description: > + A globally unique identifier that identifies the provider that logged + the event. + + - name: provider_name + type: keyword + required: false + description: > + The source of the event log record (the application or service that + logged the record). + + - name: session + type: keyword + required: false + description: > + Configured session to forward ETW events from providers to consumers. + + - name: severity + type: keyword + required: false + description: > + Human-readable level of severity. + + - name: task + type: short + required: false + description: > + The task defined in the event. Task and opcode are typically used to + identify the location in the application from where the event was + logged. + + - name: thread_id + type: integer + required: false + description: > + Identifies the thread that generated the event. + + - name: timestamp + type: date + required: false + description: > + Contains the time that the event occurred. + + - name: version + type: short + required: false + description: > + Specify the version of a manifest-based event. From 29d2896f80ac9f218784596ba42e8eeee6c9d3f1 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 22:02:57 +0100 Subject: [PATCH 24/31] Added files after make update --- x-pack/filebeat/include/list.go | 1 + x-pack/filebeat/input/etw/fields.go | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 x-pack/filebeat/input/etw/fields.go diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index f7c35308ed3..43b6758766e 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -12,6 +12,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cometd" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/etw" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" diff --git a/x-pack/filebeat/input/etw/fields.go b/x-pack/filebeat/input/etw/fields.go new file mode 100644 index 00000000000..9e4bc2dd62a --- /dev/null +++ b/x-pack/filebeat/input/etw/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package etw + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "etw", asset.ModuleFieldsPri, AssetEtw); err != nil { + panic(err) + } +} + +// AssetEtw returns asset data. +// This is the base64 encoded zlib format compressed contents of input/etw. +func AssetEtw() string { + return "eJzUV8Fu3DYQvfsrBrk0PqwRF3AOeyhQBAlaoEUP3SJHY0SOpKkpUiEpbffviyGpXcmSgQbxFu3eTJEz7828eaR38ESnPRzZGtfcAESOhvbw5jNb7Y4BPh4+v7kB0BSU5z6ys3v44QYA4BOT0QFq7zqILclOYNsPEd5+HMlGOHhUbBuonYcS7vbuBqBOB/c3KcoOLHY0AyC/eOppD413Q19WNvLL70djSjgIPSmuWUF0Cc6ZwAILegJNNVvS0JKnuxJqgWmOC1XkkePpkfX52wTwiU5H5+frnr4M7EnvoUYTaPblBQKJBDTGVWjMCQbLXwYC1mQj10weYovx8ndI1NTgvZCasN3BoaVFSBLSIR8Wyv1QGQ4taThybCG2HOZJ0hb0EVydEgTslvHOmdYFUi1aS2ZVnNA6H7+1NH8E0tJQslgZyj1Gk+lB752iENg2d/Aho4ARzUABKjLuCPfvEzNPgfxIOglxCATVCX5l5V1wddyIHj1h7CRDdTor2w82ckerTJLDasDKjQRHNgYqAm6s86Q3zsPbFF1YlToDhqmG8O42BVNoJUrDI1kB7HeTZAN1aCOrsNGIVJRHjRFXvXDVn6TmzcgLj68q40NLGcPuPIoCJokTlLNRKpoExqEAAF7M7VJy9iQdHlmLQK3OsV/kvTGeLysw+uFrSc3GJTNK/X9GIDfVDV7RNEovoa4NNuE6Q/NJQufZLwUEtrXzHcpukeoQL9ggDKoVEcqKoBDkQebKWeAIRwxgXNPkQZRG8IxZaiyyDUB/RbKadG76mnBR2ZqzcbZ5De1NCdLMD8U42GpWGAnQZsDfBeioq8iHlntgCwiBki4LH4zUOM+0NWKGxms53S8SOpd+JJ9MPS8Vn3kHsfVuaFp4WFxicy97duT9+cj9w8IIn+27v2z8/uFhcp9Zgku3p4Hcqo1rajZ0rUvycJksSZOfHceWVTtddsIwC/UOfrPmBDgim+TsYvzWWfCEZpdsuAh8q8muV06vebxKl4VFjn+uL9uZT8ABw1MasrJLSMVTzyo/D7KqFyGLL+UuGafykJeo2PeGy1KpGEnEc0OPGBbRSgHXVSmX7ZbPso3UkP/W2vy8fOSUhNnHGrLkz9fmS546qfOxGf5Dr7XzJZY+5gIvIv4jSnb5JLvWbM1vLcEKnpTzGt4+V5PzIGbCihKvDRGlQPn47QazMoHX4vTB2ZqbwadHU77MohMjOKLX6UVWbCONxVTmIJuUs2HoyG+Zw+TO14L909Ch3XlCnYzLrG6FNaSI4el6biXR/29eFVsp4L9nVTnf1ziV3EEhYtevEGqM9Araz2+yBE6uuwTtUkmn0r+QW7UbyW+O5asI6vf0VM7tL4lE3AgdWq4pxF2FIptStr8DAAD//2Iu8Dk=" +} From 027e5a45973192a0c7b48d155ab8617b6863f71f Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Thu, 8 Feb 2024 22:14:19 +0100 Subject: [PATCH 25/31] Export fields mapping to docs --- filebeat/docs/fields.asciidoc | 231 ++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 1c76bb70919..0a664cb4882 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -92,6 +92,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -158960,6 +158961,236 @@ alias to: source.geo.region_iso_code -- +[[exported-fields-winlog]] +== Windows ETW fields + +Fields from the ETW input (Event Tracing for Windows). + + + +[float] +=== winlog + +All fields specific to the Windows Event Tracing are defined here. + + + +*`winlog.activity_id`*:: ++ +-- +A globally unique identifier that identifies the current activity. The events that are published with this identifier are part of the same activity. + + +type: keyword + +required: False + +-- + +*`winlog.channel`*:: ++ +-- +Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. + + +type: short + +required: False + +-- + +*`winlog.event_data`*:: ++ +-- +The event-specific data. The content of this object is specific to any provider and event. + + +type: object + +required: False + +-- + +*`winlog.event_id`*:: ++ +-- +The event identifier. The value is specific to the source of the event. + + +type: short + +required: True + +-- + +*`winlog.flags`*:: ++ +-- +Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. + + +type: short + +required: False + +-- + +*`winlog.keywords`*:: ++ +-- +The keywords are used to indicate an event's membership in a set of event categories. + + +type: long + +required: False + +-- + +*`winlog.level`*:: ++ +-- +Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. + + +type: short + +required: False + +-- + +*`winlog.logfile`*:: ++ +-- +The source file from which events are logged. Only available for non real-time sessions. + + +type: keyword + +required: False + +-- + +*`winlog.opcode`*:: ++ +-- +The opcode defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. + + +type: short + +required: False + +-- + +*`winlog.process_id`*:: ++ +-- +Identifies the process that generated the event. + + +type: integer + +required: False + +-- + +*`winlog.provider_guid`*:: ++ +-- +A globally unique identifier that identifies the provider that logged the event. + + +type: keyword + +required: False + +-- + +*`winlog.provider_name`*:: ++ +-- +The source of the event log record (the application or service that logged the record). + + +type: keyword + +required: False + +-- + +*`winlog.session`*:: ++ +-- +Configured session to forward ETW events from providers to consumers. + + +type: keyword + +required: False + +-- + +*`winlog.severity`*:: ++ +-- +Human-readable level of severity. + + +type: keyword + +required: False + +-- + +*`winlog.task`*:: ++ +-- +The task defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. + + +type: short + +required: False + +-- + +*`winlog.thread_id`*:: ++ +-- +Identifies the thread that generated the event. + + +type: integer + +required: False + +-- + +*`winlog.timestamp`*:: ++ +-- +Contains the time that the event occurred. + + +type: date + +required: False + +-- + +*`winlog.version`*:: ++ +-- +Specify the version of a manifest-based event. + + +type: short + +required: False + +-- + [[exported-fields-zeek]] == Zeek fields From 83d60c67570a3c2fc3601853844826ae299ec021 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Fri, 9 Feb 2024 00:18:27 +0100 Subject: [PATCH 26/31] Fix timestamp and GUID for buildEvent tests --- x-pack/filebeat/input/etw/input.go | 4 +++- x-pack/filebeat/input/etw/input_test.go | 16 +++++++++++++--- x-pack/libbeat/reader/etw/provider.go | 6 ------ x-pack/libbeat/reader/etw/provider_test.go | 21 --------------------- 4 files changed, 16 insertions(+), 31 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index ae8c6bb2c6b..69448a73192 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -237,7 +237,9 @@ func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cf if cfg.ProviderName != "" { winlog["provider_name"] = cfg.ProviderName } - if winlog["provider_guid"] == "" && etw.IsGUIDValid(session.GUID) { + + zeroGUID := "{00000000-0000-0000-0000-000000000000}" + if winlog["provider_guid"] == zeroGUID { winlog["provider_guid"] = session.GUID.String() } diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index c84c02dbe7b..59f3f81b68f 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -364,7 +364,7 @@ func Test_buildEvent(t *testing.T) { "flags": uint16(30), "keywords": uint64(40), "level": uint8(1), - "logfile": "", + "logfile": nil, "opcode": uint8(50), "process_id": uint32(60), "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", @@ -391,7 +391,6 @@ func Test_buildEvent(t *testing.T) { ThreadId: 80, ProcessId: 60, TimeStamp: 133516441890350000, - ProviderId: windows.GUID{}, EventDescriptor: etw.EventDescriptor{ Id: 20, Version: 90, @@ -452,6 +451,17 @@ func Test_buildEvent(t *testing.T) { t.Run(tt.name, func(t *testing.T) { winlog := buildEvent(tt.data, tt.header, tt.session, tt.cfg) + // Parse the expected time string to time.Time + expectedTime, err := time.Parse(time.RFC3339, tt.expected["timestamp"].(string)) + if err != nil { + t.Fatalf("Failed to parse expected time string: %v", err) + } + + winlogTime := winlog["timestamp"].(time.Time) + + // Convert the expected time to the same time zone before comparison + expectedTime = expectedTime.In(winlogTime.Location()) + assert.Equal(t, tt.expected["activity_guid"], winlog["activity_guid"]) assert.Equal(t, tt.expected["channel"], winlog["channel"]) assert.Equal(t, tt.expected["event_data"], winlog["event_data"]) @@ -468,7 +478,7 @@ func Test_buildEvent(t *testing.T) { assert.Equal(t, tt.expected["severity"], winlog["severity"]) assert.Equal(t, tt.expected["task"], winlog["task"]) assert.Equal(t, tt.expected["thread_id"], winlog["thread_id"]) - assert.Equal(t, tt.expected["timestamp"], winlog["timestamp"]) + assert.Equal(t, expectedTime, winlogTime) assert.Equal(t, tt.expected["version"], winlog["version"]) }) diff --git a/x-pack/libbeat/reader/etw/provider.go b/x-pack/libbeat/reader/etw/provider.go index e0a20c3facd..63042d0f772 100644 --- a/x-pack/libbeat/reader/etw/provider.go +++ b/x-pack/libbeat/reader/etw/provider.go @@ -73,9 +73,3 @@ func guidFromProviderName(providerName string) (windows.GUID, error) { // No matching provider is found. return windows.GUID{}, fmt.Errorf("unable to find GUID from provider name") } - -// IsGUIDValid checks if GUID contains valid data -// (any of the fields in the GUID are non-zero) -func IsGUIDValid(guid windows.GUID) bool { - return guid.Data1 != 0 || guid.Data2 != 0 || guid.Data3 != 0 || guid.Data4 != [8]byte{} -} diff --git a/x-pack/libbeat/reader/etw/provider_test.go b/x-pack/libbeat/reader/etw/provider_test.go index d8c561ef3e4..0a10e0b495b 100644 --- a/x-pack/libbeat/reader/etw/provider_test.go +++ b/x-pack/libbeat/reader/etw/provider_test.go @@ -176,24 +176,3 @@ func TestGUIDFromProviderName_Success(t *testing.T) { assert.NoError(t, err) assert.Equal(t, mockGUID, guid, "GUID should match the mock GUID") } - -func TestIsGUIDValid_True(t *testing.T) { - // Valid GUID - validGUID := windows.GUID{ - Data1: 0xeb79061a, - Data2: 0xa566, - Data3: 0x4698, - Data4: [8]byte{0x12, 0x34, 0x3e, 0xd2, 0x80, 0x70, 0x33, 0xa0}, - } - - valid := IsGUIDValid(validGUID) - assert.True(t, valid, "IsGUIDValid should return true for a valid GUID") -} - -func TestIsGUIDValid_False(t *testing.T) { - // Invalid GUID (all zeros) - invalidGUID := windows.GUID{} - - valid := IsGUIDValid(invalidGUID) - assert.False(t, valid, "IsGUIDValid should return false for an invalid GUID") -} From c0349cc31058150f0a66ac9daa4427fc1589a3ee Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Sat, 10 Feb 2024 03:12:39 +0100 Subject: [PATCH 27/31] Adjust ETW mapping to fit ECS --- x-pack/filebeat/input/etw/_meta/fields.yml | 36 ++---- x-pack/filebeat/input/etw/input.go | 82 +++++++------ x-pack/filebeat/input/etw/input_test.go | 134 ++++++++++----------- 3 files changed, 114 insertions(+), 138 deletions(-) diff --git a/x-pack/filebeat/input/etw/_meta/fields.yml b/x-pack/filebeat/input/etw/_meta/fields.yml index 4863bdcd8fd..4dc9f758117 100644 --- a/x-pack/filebeat/input/etw/_meta/fields.yml +++ b/x-pack/filebeat/input/etw/_meta/fields.yml @@ -19,7 +19,7 @@ activity. - name: channel - type: short + type: keyword required: false description: > Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. @@ -32,38 +32,26 @@ The event-specific data. The content of this object is specific to any provider and event. - - name: event_id - type: short - required: true - description: > - The event identifier. The value is specific to the source of the event. - - name: flags - type: short + type: keyword required: false description: > Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. - name: keywords - type: long + type: keyword required: false description: > The keywords are used to indicate an event's membership in a set of event categories. - name: level - type: short - required: false - description: > - Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. - - - name: logfile type: keyword required: false description: > - The source file from which events are logged. Only available for non real-time sessions. + Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. - name: opcode - type: short + type: keyword required: false description: > The opcode defined in the event. Task and opcode are typically used to @@ -71,7 +59,7 @@ logged. - name: process_id - type: integer + type: keyword required: false description: > Identifies the process that generated the event. @@ -103,7 +91,7 @@ Human-readable level of severity. - name: task - type: short + type: keyword required: false description: > The task defined in the event. Task and opcode are typically used to @@ -111,19 +99,13 @@ logged. - name: thread_id - type: integer + type: keyword required: false description: > Identifies the thread that generated the event. - - name: timestamp - type: date - required: false - description: > - Contains the time that the event occurred. - - name: version - type: short + type: keyword required: false description: > Specify the version of a manifest-based event. diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 69448a73192..739c6241042 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -154,13 +154,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return 1 } - evt := beat.Event{ - Timestamp: time.Now(), - Fields: mapstr.M{ - "winlog": buildEvent(data, record.EventHeader, e.etwSession, e.config), - }, - } - + evt := buildEvent(data, record.EventHeader, e.etwSession, e.config) publisher.Publish(evt) return 0 @@ -198,7 +192,38 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { } // buildEvent builds the winlog object. -func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) map[string]any { +func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) beat.Event { + + winlog := map[string]any{ + "activity_guid": h.ActivityId.String(), + "channel": fmt.Sprintf("%d", h.EventDescriptor.Channel), + "event_data": data, + "flags": fmt.Sprintf("%d", h.Flags), + "keywords": fmt.Sprintf("%d", h.EventDescriptor.Keyword), + "opcode": fmt.Sprintf("%d", h.EventDescriptor.Opcode), + "process_id": fmt.Sprintf("%d", h.ProcessId), + "provider_guid": h.ProviderId.String(), + "session": session.Name, + "task": fmt.Sprintf("%d", h.EventDescriptor.Task), + "thread_id": fmt.Sprintf("%d", h.ThreadId), + "version": fmt.Sprintf("%d", h.EventDescriptor.Version), + } + + // Include provider GUID if not available in event header + zeroGUID := "{00000000-0000-0000-0000-000000000000}" + if winlog["provider_guid"] == zeroGUID { + winlog["provider_guid"] = session.GUID.String() + } + + // Define fields map with Windows data and ECS mapping + fields := mapstr.M{ + "winlog": winlog, + "event.code": fmt.Sprintf("%d", h.EventDescriptor.Id), + "event.created": time.Now(), + "event.kind": "event", + "event.severity": h.EventDescriptor.Level, + } + // Mapping from Level to Severity levelToSeverity := map[uint8]string{ 1: "critical", @@ -209,46 +234,25 @@ func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cf } // Get the severity level, with a default value if not found - severity, ok := levelToSeverity[h.EventDescriptor.Level] - if !ok { - severity = "unknown" // Default severity level + _, ok := levelToSeverity[h.EventDescriptor.Level] + if ok { + fields["log.level"] = levelToSeverity[h.EventDescriptor.Level] } - winlog := map[string]any{ - "activity_guid": h.ActivityId.String(), - "channel": h.EventDescriptor.Channel, - "event_data": data, - "event_id": h.EventDescriptor.Id, - "flags": h.Flags, - "keywords": h.EventDescriptor.Keyword, - "level": h.EventDescriptor.Level, - "opcode": h.EventDescriptor.Opcode, - "process_id": h.ProcessId, - "provider_guid": h.ProviderId.String(), - "session": session.Name, - "severity": severity, - "task": h.EventDescriptor.Task, - "thread_id": h.ThreadId, - "timestamp": convertFileTimeToGoTime(uint64(h.TimeStamp)), - "version": h.EventDescriptor.Version, - } - - // Include provider name and GUID if available + // Include provider name if available if cfg.ProviderName != "" { - winlog["provider_name"] = cfg.ProviderName - } - - zeroGUID := "{00000000-0000-0000-0000-000000000000}" - if winlog["provider_guid"] == zeroGUID { - winlog["provider_guid"] = session.GUID.String() + fields["event.provider"] = cfg.ProviderName } // Include logfile path if available if cfg.Logfile != "" { - winlog["logfile"] = cfg.Logfile + fields["log.file.path"] = cfg.Logfile } - return winlog + return beat.Event{ + Timestamp: convertFileTimeToGoTime(uint64(h.TimeStamp)), + Fields: fields, + } } // convertFileTimeToGoTime converts a Windows FileTime to a Go time.Time structure. diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index 59f3f81b68f..1663dd637b3 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -17,6 +17,7 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" "golang.org/x/sys/windows" ) @@ -303,7 +304,7 @@ func Test_buildEvent(t *testing.T) { header etw.EventHeader session *etw.Session cfg config - expected map[string]any + expected mapstr.M }{ { name: "TestStandardData", @@ -354,27 +355,27 @@ func Test_buildEvent(t *testing.T) { ProviderName: "TestProvider", }, - expected: map[string]any{ - "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", - "channel": uint8(10), - "event_data": map[string]any{ - "key": "value", + expected: mapstr.M{ + "winlog": map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": "10", + "event_data": map[string]any{ + "key": "value", + }, + "flags": "30", + "keywords": "40", + "opcode": "50", + "process_id": "60", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "Elastic-TestProvider", + "task": "70", + "thread_id": "80", + "version": "90", }, - "event_id": uint16(20), - "flags": uint16(30), - "keywords": uint64(40), - "level": uint8(1), - "logfile": nil, - "opcode": uint8(50), - "process_id": uint32(60), - "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "provider_name": "TestProvider", - "session": "Elastic-TestProvider", - "severity": "critical", - "task": uint16(70), - "thread_id": uint32(80), - "timestamp": "2024-02-05T22:03:09.035Z", - "version": uint8(90), + "event.code": "20", + "event.provider": "TestProvider", + "event.severity": uint8(1), + "log.level": "critical", }, }, { @@ -422,64 +423,53 @@ func Test_buildEvent(t *testing.T) { Logfile: "C:\\TestFile", }, - expected: map[string]any{ - "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", - "channel": uint8(10), - "event_data": map[string]any{ - "key": "value", + expected: mapstr.M{ + "winlog": map[string]any{ + "activity_guid": "{12345678-1234-1234-1234-123456789ABC}", + "channel": "10", + "event_data": map[string]any{ + "key": "value", + }, + "flags": "30", + "keywords": "40", + "opcode": "50", + "process_id": "60", + "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", + "session": "Elastic-TestProvider", + "task": "70", + "thread_id": "80", + "version": "90", }, - "event_id": uint16(20), - "flags": uint16(30), - "keywords": uint64(40), - "level": uint8(17), - "logfile": "C:\\TestFile", - "opcode": uint8(50), - "process_id": uint32(60), - "provider_guid": "{12345678-1234-1234-1234-123456789ABC}", - "provider_name": "TestProvider", - "session": "Elastic-TestProvider", - "severity": "unknown", - "task": uint16(70), - "thread_id": uint32(80), - "timestamp": "2024-02-05T22:03:09.035Z", - "version": uint8(90), + "event.code": "20", + "event.provider": "TestProvider", + "event.severity": uint8(17), + "log.file.path": "C:\\TestFile", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - winlog := buildEvent(tt.data, tt.header, tt.session, tt.cfg) - - // Parse the expected time string to time.Time - expectedTime, err := time.Parse(time.RFC3339, tt.expected["timestamp"].(string)) - if err != nil { - t.Fatalf("Failed to parse expected time string: %v", err) - } - - winlogTime := winlog["timestamp"].(time.Time) - - // Convert the expected time to the same time zone before comparison - expectedTime = expectedTime.In(winlogTime.Location()) - - assert.Equal(t, tt.expected["activity_guid"], winlog["activity_guid"]) - assert.Equal(t, tt.expected["channel"], winlog["channel"]) - assert.Equal(t, tt.expected["event_data"], winlog["event_data"]) - assert.Equal(t, tt.expected["event_id"], winlog["event_id"]) - assert.Equal(t, tt.expected["flags"], winlog["flags"]) - assert.Equal(t, tt.expected["keywords"], winlog["keywords"]) - assert.Equal(t, tt.expected["level"], winlog["level"]) - assert.Equal(t, tt.expected["logfile"], winlog["logfile"]) - assert.Equal(t, tt.expected["opcode"], winlog["opcode"]) - assert.Equal(t, tt.expected["process_id"], winlog["process_id"]) - assert.Equal(t, tt.expected["provider_guid"], winlog["provider_guid"]) - assert.Equal(t, tt.expected["provider_name"], winlog["provider_name"]) - assert.Equal(t, tt.expected["session"], winlog["session"]) - assert.Equal(t, tt.expected["severity"], winlog["severity"]) - assert.Equal(t, tt.expected["task"], winlog["task"]) - assert.Equal(t, tt.expected["thread_id"], winlog["thread_id"]) - assert.Equal(t, expectedTime, winlogTime) - assert.Equal(t, tt.expected["version"], winlog["version"]) + evt := buildEvent(tt.data, tt.header, tt.session, tt.cfg) + + assert.Equal(t, tt.expected["winlog"].(map[string]any)["activity_guid"], evt.Fields["winlog"].(map[string]any)["activity_guid"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["channel"], evt.Fields["winlog"].(map[string]any)["channel"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["event_data"], evt.Fields["winlog"].(map[string]any)["event_data"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["flags"], evt.Fields["winlog"].(map[string]any)["flags"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["keywords"], evt.Fields["winlog"].(map[string]any)["keywords"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["opcode"], evt.Fields["winlog"].(map[string]any)["opcode"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["process_id"], evt.Fields["winlog"].(map[string]any)["process_id"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["provider_guid"], evt.Fields["winlog"].(map[string]any)["provider_guid"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["session"], evt.Fields["winlog"].(map[string]any)["session"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["task"], evt.Fields["winlog"].(map[string]any)["task"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["thread_id"], evt.Fields["winlog"].(map[string]any)["thread_id"]) + assert.Equal(t, tt.expected["winlog"].(map[string]any)["version"], evt.Fields["winlog"].(map[string]any)["version"]) + + assert.Equal(t, tt.expected["event.code"], evt.Fields["event.code"]) + assert.Equal(t, tt.expected["event.provider"], evt.Fields["event.provider"]) + assert.Equal(t, tt.expected["event.severity"], evt.Fields["event.severity"]) + assert.Equal(t, tt.expected["log.file.path"], evt.Fields["log.file.path"]) + assert.Equal(t, tt.expected["log.level"], evt.Fields["log.level"]) }) } From 2aa117e6b897d446a971f0e976705b8a1597f4e9 Mon Sep 17 00:00:00 2001 From: chemamartinez Date: Sat, 10 Feb 2024 08:57:23 +0100 Subject: [PATCH 28/31] Update fields built files --- filebeat/docs/fields.asciidoc | 52 +++++------------------------ x-pack/filebeat/input/etw/fields.go | 2 +- 2 files changed, 9 insertions(+), 45 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 0a664cb4882..6b1ea670c79 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -158993,7 +158993,7 @@ required: False Used to enable special event processing. Channel values below 16 are reserved for use by Microsoft to enable special treatment by the ETW runtime. Channel values 16 and above will be ignored by the ETW runtime (treated the same as channel 0) and can be given user-defined semantics. -type: short +type: keyword required: False @@ -159011,25 +159011,13 @@ required: False -- -*`winlog.event_id`*:: -+ --- -The event identifier. The value is specific to the source of the event. - - -type: short - -required: True - --- - *`winlog.flags`*:: + -- Flags that provide information about the event such as the type of session it was logged to and if the event contains extended data. -type: short +type: keyword required: False @@ -159041,7 +159029,7 @@ required: False The keywords are used to indicate an event's membership in a set of event categories. -type: long +type: keyword required: False @@ -159053,18 +159041,6 @@ required: False Level of severity. Level values 0 through 5 are defined by Microsoft. Level values 6 through 15 are reserved. Level values 16 through 255 can be defined by the event provider. -type: short - -required: False - --- - -*`winlog.logfile`*:: -+ --- -The source file from which events are logged. Only available for non real-time sessions. - - type: keyword required: False @@ -159077,7 +159053,7 @@ required: False The opcode defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. -type: short +type: keyword required: False @@ -159089,7 +159065,7 @@ required: False Identifies the process that generated the event. -type: integer +type: keyword required: False @@ -159149,7 +159125,7 @@ required: False The task defined in the event. Task and opcode are typically used to identify the location in the application from where the event was logged. -type: short +type: keyword required: False @@ -159161,19 +159137,7 @@ required: False Identifies the thread that generated the event. -type: integer - -required: False - --- - -*`winlog.timestamp`*:: -+ --- -Contains the time that the event occurred. - - -type: date +type: keyword required: False @@ -159185,7 +159149,7 @@ required: False Specify the version of a manifest-based event. -type: short +type: keyword required: False diff --git a/x-pack/filebeat/input/etw/fields.go b/x-pack/filebeat/input/etw/fields.go index 9e4bc2dd62a..8464c2ab02d 100644 --- a/x-pack/filebeat/input/etw/fields.go +++ b/x-pack/filebeat/input/etw/fields.go @@ -19,5 +19,5 @@ func init() { // AssetEtw returns asset data. // This is the base64 encoded zlib format compressed contents of input/etw. func AssetEtw() string { - return "eJzUV8Fu3DYQvfsrBrk0PqwRF3AOeyhQBAlaoEUP3SJHY0SOpKkpUiEpbffviyGpXcmSgQbxFu3eTJEz7828eaR38ESnPRzZGtfcAESOhvbw5jNb7Y4BPh4+v7kB0BSU5z6ys3v44QYA4BOT0QFq7zqILclOYNsPEd5+HMlGOHhUbBuonYcS7vbuBqBOB/c3KcoOLHY0AyC/eOppD413Q19WNvLL70djSjgIPSmuWUF0Cc6ZwAILegJNNVvS0JKnuxJqgWmOC1XkkePpkfX52wTwiU5H5+frnr4M7EnvoUYTaPblBQKJBDTGVWjMCQbLXwYC1mQj10weYovx8ndI1NTgvZCasN3BoaVFSBLSIR8Wyv1QGQ4taThybCG2HOZJ0hb0EVydEgTslvHOmdYFUi1aS2ZVnNA6H7+1NH8E0tJQslgZyj1Gk+lB752iENg2d/Aho4ARzUABKjLuCPfvEzNPgfxIOglxCATVCX5l5V1wddyIHj1h7CRDdTor2w82ckerTJLDasDKjQRHNgYqAm6s86Q3zsPbFF1YlToDhqmG8O42BVNoJUrDI1kB7HeTZAN1aCOrsNGIVJRHjRFXvXDVn6TmzcgLj68q40NLGcPuPIoCJokTlLNRKpoExqEAAF7M7VJy9iQdHlmLQK3OsV/kvTGeLysw+uFrSc3GJTNK/X9GIDfVDV7RNEovoa4NNuE6Q/NJQufZLwUEtrXzHcpukeoQL9ggDKoVEcqKoBDkQebKWeAIRwxgXNPkQZRG8IxZaiyyDUB/RbKadG76mnBR2ZqzcbZ5De1NCdLMD8U42GpWGAnQZsDfBeioq8iHlntgCwiBki4LH4zUOM+0NWKGxms53S8SOpd+JJ9MPS8Vn3kHsfVuaFp4WFxicy97duT9+cj9w8IIn+27v2z8/uFhcp9Zgku3p4Hcqo1rajZ0rUvycJksSZOfHceWVTtddsIwC/UOfrPmBDgim+TsYvzWWfCEZpdsuAh8q8muV06vebxKl4VFjn+uL9uZT8ABw1MasrJLSMVTzyo/D7KqFyGLL+UuGafykJeo2PeGy1KpGEnEc0OPGBbRSgHXVSmX7ZbPso3UkP/W2vy8fOSUhNnHGrLkz9fmS546qfOxGf5Dr7XzJZY+5gIvIv4jSnb5JLvWbM1vLcEKnpTzGt4+V5PzIGbCihKvDRGlQPn47QazMoHX4vTB2ZqbwadHU77MohMjOKLX6UVWbCONxVTmIJuUs2HoyG+Zw+TO14L909Ch3XlCnYzLrG6FNaSI4el6biXR/29eFVsp4L9nVTnf1ziV3EEhYtevEGqM9Araz2+yBE6uuwTtUkmn0r+QW7UbyW+O5asI6vf0VM7tL4lE3AgdWq4pxF2FIptStr8DAAD//2Iu8Dk=" + return "eJzUVk2L5DYQvfevKPaSmUM3O4HZQx8CYdklgeSUDnscylLZrowsefVhp/990Ic99rQXEhiHpG8tS6/eq3pV0hGe6XqGkbUyzQHAs1d0hndfWEszOvh0+fLuACDJCcu9Z6PP8MMBAOAzk5IOams68C3FncC6Dx7uPg2kPVwsCtYN1MZCgbs/HQDqdPB8SChH0NjRgkD8+WtPZ2isCX1Z2Ygffz8qVeDA9SS4ZgHeJDqzgBUXtASSatYkoSVLpwK14rTkhcLzwP76xHL+NhF8puto7HLd0tfAluQZalSOFl++ISCJgEaZCpW6QtD8NRCwJO25ZrLgW/Qv/12SJoK1UdTE7QSXllaQFEW7fDhK7kOl2LUkYWTfgm/ZLYOkLWg9mDoFcNit8eZItwkSLWpNaq/k/O5IxpKSxkpRrjKqLBB6awQ5x7o5wcfMAwZUgRxUpMwIDx+SNkuO7EAyWTE4guoKv7Kwxpnab6B7S+i7GKG6zt62QXvu6CZSjKElYGUGgpGVgoqAG20syY3zcJfQo6qSaUA3ZRHe3ycwgTqiNDyQjoTtcTKtow61Z+E2SpGS8iTR4001TPUHCb9YzgtPb1qrS0uZw3Fuxkgm2ROE0T5mNFmMXSEAvOrcten0NVZ4YBktqmXG3tBdK2zcXgb8HMFzJxUywLo2tsO4O5Y9+FTKbEkXRBsLGlcij6jXRY8aDexhRAfKNE02dRTF9eJ0TBKydkB/etKSZE7greQibjfVsWJTjNRCofQha8kCPQHqzPk7Bx11FVnXcg+sAcFRKnORhJ4aY5m2HKto2G90/BLBc/4HsmlO5qXSuO/Bt9aEpoXH1b2wHA6vjnyYjzw8ribLq30PLxu/f3yc2nkR4KXkk8M3smN6YSTtWeIcYSbG+oXYCS7onpNFy66o1197FvmqyoZYQZYrJctTRuQWKajY94rLUno0jPECXmRiRLdCy32ykZcy9ne8kn9eX7glYJ4CDWmy8wD/1kyayvrUhP/Qy2Eep+ljTvAK8W9J0uvnwZu70plgBU2PkWwOZRqwJIyVcPfaTcZC7EIWlHRtmCgB5eP3G8rKgN5L00eja26CTdd3vgq8iW+REa1Mb4PyXkttMaXZxU3CaBc6slvDcxpre9H+KXSoj5ZQpseRuhmnt5Q8uuc9nRHx/2/Tyrcxhf/esMrx/smsGsjuaf/f0vsup7mEijZC6FBzTc4fK4zlKfT+CgAA//9BpltR" } From c8df9cbff50cd0d99e81423f0565fc4e99cdd008 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 12 Feb 2024 10:01:39 -0500 Subject: [PATCH 29/31] Address review comments --- x-pack/filebeat/input/etw/_meta/fields.yml | 2 +- x-pack/filebeat/input/etw/fields.go | 2 +- x-pack/filebeat/input/etw/input.go | 88 +++++++++++----------- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/x-pack/filebeat/input/etw/_meta/fields.yml b/x-pack/filebeat/input/etw/_meta/fields.yml index 4dc9f758117..9a732f73e3b 100644 --- a/x-pack/filebeat/input/etw/_meta/fields.yml +++ b/x-pack/filebeat/input/etw/_meta/fields.yml @@ -105,7 +105,7 @@ Identifies the thread that generated the event. - name: version - type: keyword + type: long required: false description: > Specify the version of a manifest-based event. diff --git a/x-pack/filebeat/input/etw/fields.go b/x-pack/filebeat/input/etw/fields.go index 8464c2ab02d..4ae281b363b 100644 --- a/x-pack/filebeat/input/etw/fields.go +++ b/x-pack/filebeat/input/etw/fields.go @@ -19,5 +19,5 @@ func init() { // AssetEtw returns asset data. // This is the base64 encoded zlib format compressed contents of input/etw. func AssetEtw() string { - return "eJzUVk2L5DYQvfevKPaSmUM3O4HZQx8CYdklgeSUDnscylLZrowsefVhp/990Ic99rQXEhiHpG8tS6/eq3pV0hGe6XqGkbUyzQHAs1d0hndfWEszOvh0+fLuACDJCcu9Z6PP8MMBAOAzk5IOams68C3FncC6Dx7uPg2kPVwsCtYN1MZCgbs/HQDqdPB8SChH0NjRgkD8+WtPZ2isCX1Z2Ygffz8qVeDA9SS4ZgHeJDqzgBUXtASSatYkoSVLpwK14rTkhcLzwP76xHL+NhF8puto7HLd0tfAluQZalSOFl++ISCJgEaZCpW6QtD8NRCwJO25ZrLgW/Qv/12SJoK1UdTE7QSXllaQFEW7fDhK7kOl2LUkYWTfgm/ZLYOkLWg9mDoFcNit8eZItwkSLWpNaq/k/O5IxpKSxkpRrjKqLBB6awQ5x7o5wcfMAwZUgRxUpMwIDx+SNkuO7EAyWTE4guoKv7Kwxpnab6B7S+i7GKG6zt62QXvu6CZSjKElYGUGgpGVgoqAG20syY3zcJfQo6qSaUA3ZRHe3ycwgTqiNDyQjoTtcTKtow61Z+E2SpGS8iTR4001TPUHCb9YzgtPb1qrS0uZw3Fuxkgm2ROE0T5mNFmMXSEAvOrcten0NVZ4YBktqmXG3tBdK2zcXgb8HMFzJxUywLo2tsO4O5Y9+FTKbEkXRBsLGlcij6jXRY8aDexhRAfKNE02dRTF9eJ0TBKydkB/etKSZE7greQibjfVsWJTjNRCofQha8kCPQHqzPk7Bx11FVnXcg+sAcFRKnORhJ4aY5m2HKto2G90/BLBc/4HsmlO5qXSuO/Bt9aEpoXH1b2wHA6vjnyYjzw8ribLq30PLxu/f3yc2nkR4KXkk8M3smN6YSTtWeIcYSbG+oXYCS7onpNFy66o1197FvmqyoZYQZYrJctTRuQWKajY94rLUno0jPECXmRiRLdCy32ykZcy9ne8kn9eX7glYJ4CDWmy8wD/1kyayvrUhP/Qy2Eep+ljTvAK8W9J0uvnwZu70plgBU2PkWwOZRqwJIyVcPfaTcZC7EIWlHRtmCgB5eP3G8rKgN5L00eja26CTdd3vgq8iW+REa1Mb4PyXkttMaXZxU3CaBc6slvDcxpre9H+KXSoj5ZQpseRuhmnt5Q8uuc9nRHx/2/Tyrcxhf/esMrx/smsGsjuaf/f0vsup7mEijZC6FBzTc4fK4zlKfT+CgAA//9BpltR" + return "eJzUVk2P2zYQvftXDHLp7sFGtsDm4EOBIkjQAu2pLnJcjMiRNF1qqJCUVP/7gh/ySmsFaNG4aHwzRb55b+bNkHt4pvMRJhZjmx1A4GDoCG8+sWg7efhw+vRmB6DJK8d9YCtH+GEHAPCRyWgPtbMdhJbiTmDphwB3H0aSACeHiqWB2joocPeHHUCdDh53CWUPgh0tCMRfOPd0hMbZoS8rG/Hj70djChz4nhTXrCDYROciYMUFHYGmmoU0tOToUKBWnJa8UAUeOZyfWF++zQSf6TxZt1x39HlgR/oINRpPiy9fEJBEQGNshcacYRD+PBCwJglcMzkILYaX/z5JU4NzUdTM7QCnllaQFEX7fDhK7ofKsG9Jw8ShhdCyXwZJW9AFsHUK4LFb410iXSdItShC5lbJ+d2TjiUlwcpQrjKaLBB6ZxV5z9Ic4H3mASOagTxUZOwED++SNkee3Eg6WXHwBNUZfmXlrLd12EAPjjB0MUJ1vnjbDRK4o6tIMYZowMqOBBMbAxUBN2Id6Y3zcJfQo6qSaUA/ZxHe3icwhRJRGh5JImG3n03rqUMJrPxGKVJSnjQGvKqGrf4gFRbLeeHpq9bq1FLmsL80YyST7AnKSogZTRZjXwgArzp3bTo5xwqPrKNFRWfsDd21wcbfyoAfI3jupEIGWGrrOoy7Y9mHkEqZLekH1caCxpXII+r10aNWgANM6MHYpsmmjqK4XpyOSUIWD/RnINGkcwKvJRdxN1MdKzbHSC00lD5k0awwEKBkzt956KiryPmWe2ABBE+pzEUSBmqsY9pyrKHxdqPjlwie8z+SS3MyL5XGfQuhdXZoWnhc3QvL4fDqyLvLkYfH1WR5te/hZeP3j49zOy8CvJR8dvhGdmyvrKZbljhHuBBjeSF2gBP652TRsivqDeeeVb6qsiFWkOVKyfKMVblFCir2veGylB4NU7yAF5mY0K/Qcp9s5KWM/RteyT+vL9wSME+BhoTcZYB/aSbNZX1qhv/Ry+EyTtPHnOAV4t+SJOvnwVd3pbeDUzQ/RrI5jG3AkbJOw91rN1kHsQtZUdK1YaIElI/fbygrA/pWmt5bqbkZXLq+81UQbHyLTOh0ehuU91pqiznNPm5SVvzQkdsanvNYuxXtn4YOZe8IdXocmatxek0poH++pTMi/rc2rUIbU/jfDasc75/MqpHcpv2NlebfkvstPe5yjkuc6CGEDoVr8mFfYaxN4fZXAAAA///9F1n8" } diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 739c6241042..050fcf6ddf9 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -9,6 +9,7 @@ package etw import ( "fmt" "math" + "strconv" "sync" "time" @@ -150,7 +151,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { data, err := etw.GetEventProperties(record) if err != nil { - e.log.Errorf("failed to read event properties: %w", err) + e.log.Errorw("failed to read event properties", "error", err) return 1 } @@ -176,7 +177,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { errChan <- nil }() - // We ensure resources are closed when receiving a cancelation signal + // We ensure resources are closed when receiving a cancellation signal go func() { <-ctx.Cancelation.Done() once.Do(e.Close) @@ -191,62 +192,60 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { return nil } -// buildEvent builds the winlog object. -func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) beat.Event { +var ( + // levelToSeverity maps ETW trace levels to names for use in ECS log.level. + levelToSeverity = map[uint8]string{ + 1: "critical", // Abnormal exit or termination events + 2: "error", // Severe error events + 3: "warning", // Warning events such as allocation failures + 4: "information", // Non-error events such as entry or exit events + 5: "verbose", // Detailed trace events + } + + // zeroGUID is the zero-value for a windows.GUID. + zeroGUID = windows.GUID{} +) +// buildEvent builds the final beat.Event emitted by this input. +func buildEvent(data map[string]any, h etw.EventHeader, session *etw.Session, cfg config) beat.Event { winlog := map[string]any{ "activity_guid": h.ActivityId.String(), - "channel": fmt.Sprintf("%d", h.EventDescriptor.Channel), + "channel": strconv.FormatUint(uint64(h.EventDescriptor.Channel), 10), "event_data": data, - "flags": fmt.Sprintf("%d", h.Flags), - "keywords": fmt.Sprintf("%d", h.EventDescriptor.Keyword), - "opcode": fmt.Sprintf("%d", h.EventDescriptor.Opcode), - "process_id": fmt.Sprintf("%d", h.ProcessId), + "flags": strconv.FormatUint(uint64(h.Flags), 10), + "keywords": strconv.FormatUint(h.EventDescriptor.Keyword, 10), + "opcode": strconv.FormatUint(uint64(h.EventDescriptor.Opcode), 10), + "process_id": strconv.FormatUint(uint64(h.ProcessId), 10), "provider_guid": h.ProviderId.String(), "session": session.Name, - "task": fmt.Sprintf("%d", h.EventDescriptor.Task), - "thread_id": fmt.Sprintf("%d", h.ThreadId), - "version": fmt.Sprintf("%d", h.EventDescriptor.Version), + "task": strconv.FormatUint(uint64(h.EventDescriptor.Task), 10), + "thread_id": strconv.FormatUint(uint64(h.ThreadId), 10), + "version": h.EventDescriptor.Version, } - - // Include provider GUID if not available in event header - zeroGUID := "{00000000-0000-0000-0000-000000000000}" - if winlog["provider_guid"] == zeroGUID { + // Fallback to the session GUID if there is no provider GUID. + if h.ProviderId == zeroGUID { winlog["provider_guid"] = session.GUID.String() } - // Define fields map with Windows data and ECS mapping - fields := mapstr.M{ - "winlog": winlog, - "event.code": fmt.Sprintf("%d", h.EventDescriptor.Id), - "event.created": time.Now(), - "event.kind": "event", - "event.severity": h.EventDescriptor.Level, + event := mapstr.M{ + "code": strconv.FormatUint(uint64(h.EventDescriptor.Id), 10), + "created": time.Now().UTC(), + "kind": "event", + "severity": h.EventDescriptor.Level, } - - // Mapping from Level to Severity - levelToSeverity := map[uint8]string{ - 1: "critical", - 2: "error", - 3: "warning", - 4: "information", - 5: "verbose", + if cfg.ProviderName != "" { + event["provider"] = cfg.ProviderName } - // Get the severity level, with a default value if not found - _, ok := levelToSeverity[h.EventDescriptor.Level] - if ok { - fields["log.level"] = levelToSeverity[h.EventDescriptor.Level] + fields := mapstr.M{ + "event": event, + "winlog": winlog, } - - // Include provider name if available - if cfg.ProviderName != "" { - fields["event.provider"] = cfg.ProviderName + if level, found := levelToSeverity[h.EventDescriptor.Level]; found { + fields.Put("log.level", level) } - - // Include logfile path if available if cfg.Logfile != "" { - fields["log.file.path"] = cfg.Logfile + fields.Put("log.file.path", cfg.Logfile) } return beat.Event{ @@ -269,13 +268,14 @@ func convertFileTimeToGoTime(fileTime64 uint64) time.Time { LowDateTime: uint32(fileTime64 & math.MaxUint32), } - return time.Unix(0, fileTime.Nanoseconds()) + return time.Unix(0, fileTime.Nanoseconds()).UTC() } -// close stops the ETW session and logs the outcome. +// Close stops the ETW session and logs the outcome. func (e *etwInput) Close() { if err := e.operator.stopSession(e.etwSession); err != nil { e.log.Error("failed to shutdown ETW session") + return } e.log.Info("successfully shutdown") } From 49404005f6899334becdadb5072d776e8434822a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 12 Feb 2024 10:38:32 -0500 Subject: [PATCH 30/31] filebeat/docs - rebuild with field changes --- filebeat/docs/fields.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 6b1ea670c79..ddc887d246f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -159149,7 +159149,7 @@ required: False Specify the version of a manifest-based event. -type: keyword +type: long required: False From 30b3f3aa55d1966a026c45867824a3fbeda4b3f8 Mon Sep 17 00:00:00 2001 From: narph Date: Tue, 13 Feb 2024 12:32:54 +0100 Subject: [PATCH 31/31] fix tests --- x-pack/filebeat/input/etw/input_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index 1663dd637b3..af1fa36d4bd 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -9,6 +9,7 @@ package etw import ( "context" "fmt" + "strconv" "testing" "time" @@ -451,7 +452,6 @@ func Test_buildEvent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { evt := buildEvent(tt.data, tt.header, tt.session, tt.cfg) - assert.Equal(t, tt.expected["winlog"].(map[string]any)["activity_guid"], evt.Fields["winlog"].(map[string]any)["activity_guid"]) assert.Equal(t, tt.expected["winlog"].(map[string]any)["channel"], evt.Fields["winlog"].(map[string]any)["channel"]) assert.Equal(t, tt.expected["winlog"].(map[string]any)["event_data"], evt.Fields["winlog"].(map[string]any)["event_data"]) @@ -463,13 +463,14 @@ func Test_buildEvent(t *testing.T) { assert.Equal(t, tt.expected["winlog"].(map[string]any)["session"], evt.Fields["winlog"].(map[string]any)["session"]) assert.Equal(t, tt.expected["winlog"].(map[string]any)["task"], evt.Fields["winlog"].(map[string]any)["task"]) assert.Equal(t, tt.expected["winlog"].(map[string]any)["thread_id"], evt.Fields["winlog"].(map[string]any)["thread_id"]) - assert.Equal(t, tt.expected["winlog"].(map[string]any)["version"], evt.Fields["winlog"].(map[string]any)["version"]) - - assert.Equal(t, tt.expected["event.code"], evt.Fields["event.code"]) - assert.Equal(t, tt.expected["event.provider"], evt.Fields["event.provider"]) - assert.Equal(t, tt.expected["event.severity"], evt.Fields["event.severity"]) - assert.Equal(t, tt.expected["log.file.path"], evt.Fields["log.file.path"]) - assert.Equal(t, tt.expected["log.level"], evt.Fields["log.level"]) + mapEv := evt.Fields.Flatten() + + assert.Equal(t, tt.expected["winlog"].(map[string]any)["version"], strconv.Itoa(int(mapEv["winlog.version"].(uint8)))) + assert.Equal(t, tt.expected["event.code"], mapEv["event.code"]) + assert.Equal(t, tt.expected["event.provider"], mapEv["event.provider"]) + assert.Equal(t, tt.expected["event.severity"], mapEv["event.severity"]) + assert.Equal(t, tt.expected["log.file.path"], mapEv["log.file.path"]) + assert.Equal(t, tt.expected["log.level"], mapEv["log.level"]) }) }