From 1471f9778807bc7ce460cdd0e2ac0b024eb6c176 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 19 Jan 2021 07:27:03 -0700 Subject: [PATCH 01/35] Fix typo in input-httpjson.asciidoc (#23536) --- x-pack/filebeat/docs/inputs/input-httpjson.asciidoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index c6b40991f976..64247b517f82 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -34,9 +34,9 @@ filebeat.inputs: interval: 1m request.url: https://api.ipify.org/?format=json processors: - - decode_json_fields - fields: [message] - target: json + - decode_json_fields: + fields: ["message"] + target: "json" ---- ["source","yaml",subs="attributes"] @@ -830,9 +830,9 @@ filebeat.inputs: last_requested_at: value: '[[now]]' processors: - - decode_json_fields - fields: [message] - target: json + - decode_json_fields: + fields: ["message"] + target: "json" ---- [float] From a233f034fcb1dc09e4a5dd948bdc2d433785709f Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 19 Jan 2021 11:08:32 -0500 Subject: [PATCH 02/35] Agent actions token support (#23452) * Agent actions token support * Make check happy * Consolidate action store and the ack token store into state.yml store * Make state storage thread safe --- .../pkg/agent/application/action_store.go | 41 +-- .../agent/application/action_store_test.go | 18 -- .../pkg/agent/application/fleet_gateway.go | 21 ++ .../agent/application/fleet_gateway_test.go | 15 + .../application/handler_action_application.go | 31 ++ .../application/handler_action_unenroll.go | 16 +- .../pkg/agent/application/info/agent_id.go | 8 +- .../agent/application/inspect_config_cmd.go | 4 +- .../pkg/agent/application/managed_mode.go | 32 +- .../pkg/agent/application/state_store.go | 303 ++++++++++++++++++ .../pkg/agent/application/state_store_test.go | 170 ++++++++++ .../agent/operation/operation_retryable.go | 2 +- .../pkg/agent/storage/storage.go | 17 + x-pack/elastic-agent/pkg/fleetapi/action.go | 53 ++- .../elastic-agent/pkg/fleetapi/checkin_cmd.go | 4 +- 15 files changed, 645 insertions(+), 90 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/agent/application/handler_action_application.go create mode 100644 x-pack/elastic-agent/pkg/agent/application/state_store.go create mode 100644 x-pack/elastic-agent/pkg/agent/application/state_store_test.go diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store.go b/x-pack/elastic-agent/pkg/agent/application/action_store.go index ce4ea785cf71..646ab828ef5e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store.go @@ -5,7 +5,6 @@ package application import ( - "context" "fmt" "io" @@ -19,6 +18,7 @@ import ( // take care of action policy change every other action are discarded. The store will only keep the // last good action on disk, we assume that the action is added to the store after it was ACK with // Fleet. The store is not threadsafe. +// ATTN!!!: THE actionStore is deprecated, please use and extend the stateStore instead. The actionStore will be eventually removed. type actionStore struct { log *logger.Logger store storeLoad @@ -148,42 +148,3 @@ type actionUnenrollSerializer struct { // Add a guards between the serializer structs and the original struct. var _ actionUnenrollSerializer = actionUnenrollSerializer(fleetapi.ActionUnenroll{}) - -// actionStoreAcker wraps an existing acker and will send any acked event to the action store, -// its up to the action store to decide if we need to persist the event for future replay or just -// discard the event. -type actionStoreAcker struct { - acker fleetAcker - store *actionStore -} - -func (a *actionStoreAcker) Ack(ctx context.Context, action fleetapi.Action) error { - if err := a.acker.Ack(ctx, action); err != nil { - return err - } - a.store.Add(action) - return a.store.Save() -} - -func (a *actionStoreAcker) Commit(ctx context.Context) error { - return a.acker.Commit(ctx) -} - -func newActionStoreAcker(acker fleetAcker, store *actionStore) *actionStoreAcker { - return &actionStoreAcker{acker: acker, store: store} -} - -func replayActions( - log *logger.Logger, - dispatcher dispatcher, - acker fleetAcker, - actions ...action, -) error { - log.Info("restoring current policy from disk") - - if err := dispatcher.Dispatch(acker, actions...); err != nil { - return err - } - - return nil -} diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go index f2691d66db68..cc5aa47ebca6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go @@ -5,7 +5,6 @@ package application import ( - "context" "io/ioutil" "os" "path/filepath" @@ -84,21 +83,4 @@ func TestActionStore(t *testing.T) { require.Equal(t, ActionPolicyChange, actions[0]) })) - - t.Run("when we ACK we save to disk", - withFile(func(t *testing.T, file string) { - ActionPolicyChange := &fleetapi.ActionPolicyChange{ - ActionID: "abc123", - } - - s := storage.NewDiskStore(file) - store, err := newActionStore(log, s) - require.NoError(t, err) - - acker := newActionStoreAcker(&testAcker{}, store) - require.Equal(t, 0, len(store.Actions())) - - require.NoError(t, acker.Ack(context.Background(), ActionPolicyChange)) - require.Equal(t, 1, len(store.Actions())) - })) } diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go index 8facdfc4ed15..0ec71d7a5fa7 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go @@ -77,6 +77,7 @@ type fleetGateway struct { unauthCounter int statusController status.Controller statusReporter status.Reporter + stateStore *stateStore } func newFleetGateway( @@ -88,6 +89,7 @@ func newFleetGateway( r fleetReporter, acker fleetAcker, statusController status.Controller, + stateStore *stateStore, ) (*fleetGateway, error) { scheduler := scheduler.NewPeriodicJitter(defaultGatewaySettings.Duration, defaultGatewaySettings.Jitter) @@ -102,6 +104,7 @@ func newFleetGateway( r, acker, statusController, + stateStore, ) } @@ -116,6 +119,7 @@ func newFleetGatewayWithScheduler( r fleetReporter, acker fleetAcker, statusController status.Controller, + stateStore *stateStore, ) (*fleetGateway, error) { // Backoff implementation doesn't support the using context as the shutdown mechanism. @@ -140,6 +144,7 @@ func newFleetGatewayWithScheduler( acker: acker, statusReporter: statusController.Register("gateway"), statusController: statusController, + stateStore: stateStore, }, nil } @@ -209,9 +214,16 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, f.log.Error(errors.New("failed to load metadata", err)) } + // retrieve ack token from the store + ackToken := f.stateStore.AckToken() + if ackToken != "" { + f.log.Debug("using previously saved ack token: %v", ackToken) + } + // checkin cmd := fleetapi.NewCheckinCmd(f.agentInfo, f.client) req := &fleetapi.CheckinRequest{ + AckToken: ackToken, Events: ee, Metadata: ecsMeta, Status: f.statusController.StatusString(), @@ -236,6 +248,15 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, return nil, err } + // Save the latest ackToken + if resp.AckToken != "" { + f.stateStore.SetAckToken(resp.AckToken) + serr := f.stateStore.Save() + if serr != nil { + f.log.Errorf("failed to save the ack token, err: %v", serr) + } + } + // ack events so they are dropped from queue ack() return resp, nil diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go index 7feb72424db5..a31f6a343a2c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go @@ -20,6 +20,8 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" repo "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter" fleetreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/fleet" @@ -117,6 +119,9 @@ func withGateway(agentInfo agentInfo, settings *fleetGatewaySettings, fn withGat ctx, cancel := context.WithCancel(context.Background()) defer cancel() + stateStore, err := newStateStore(log, storage.NewDiskStore(info.AgentStateStoreFile())) + require.NoError(t, err) + gateway, err := newFleetGatewayWithScheduler( ctx, log, @@ -128,6 +133,7 @@ func withGateway(agentInfo agentInfo, settings *fleetGatewaySettings, fn withGat rep, newNoopAcker(), &noopController{}, + stateStore, ) require.NoError(t, err) @@ -242,6 +248,9 @@ func TestFleetGateway(t *testing.T) { defer cancel() log, _ := logger.New("tst") + stateStore, err := newStateStore(log, storage.NewDiskStore(info.AgentStateStoreFile())) + require.NoError(t, err) + gateway, err := newFleetGatewayWithScheduler( ctx, log, @@ -253,6 +262,7 @@ func TestFleetGateway(t *testing.T) { getReporter(agentInfo, log, t), newNoopAcker(), &noopController{}, + stateStore, ) require.NoError(t, err) @@ -328,6 +338,10 @@ func TestFleetGateway(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) log, _ := logger.New("tst") + + stateStore, err := newStateStore(log, storage.NewDiskStore(info.AgentStateStoreFile())) + require.NoError(t, err) + gateway, err := newFleetGatewayWithScheduler( ctx, log, @@ -342,6 +356,7 @@ func TestFleetGateway(t *testing.T) { getReporter(agentInfo, log, t), newNoopAcker(), &noopController{}, + stateStore, ) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_application.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_application.go new file mode 100644 index 000000000000..56b5ee3499c6 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_application.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +type handlerAppAction struct { + log *logger.Logger +} + +func (h *handlerAppAction) Handle(ctx context.Context, a action, acker fleetAcker) error { + h.log.Debugf("handlerAppAction: action '%+v' received", a) + action, ok := a.(*fleetapi.ActionApp) + if !ok { + return fmt.Errorf("invalid type, expected ActionApp and received %T", a) + } + + _ = action + + // TODO: handle app action + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_unenroll.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_unenroll.go index da33f2001ff8..83dca329342e 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_unenroll.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_unenroll.go @@ -16,11 +16,11 @@ import ( // After running Unenroll agent is in idle state, non managed non standalone. // For it to be operational again it needs to be either enrolled or reconfigured. type handlerUnenroll struct { - log *logger.Logger - emitter emitterFunc - dispatcher programsDispatcher - closers []context.CancelFunc - actionStore *actionStore + log *logger.Logger + emitter emitterFunc + dispatcher programsDispatcher + closers []context.CancelFunc + stateStore *stateStore } func (h *handlerUnenroll) Handle(ctx context.Context, a action, acker fleetAcker) error { @@ -44,10 +44,10 @@ func (h *handlerUnenroll) Handle(ctx context.Context, a action, acker fleetAcker if err := acker.Commit(ctx); err != nil { return err } - } else if h.actionStore != nil { + } else if h.stateStore != nil { // backup action for future start to avoid starting fleet gateway loop - h.actionStore.Add(a) - h.actionStore.Save() + h.stateStore.Add(a) + h.stateStore.Save() } // close fleet gateway loop diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go index f18fa542a251..a6551b685803 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_id.go @@ -25,6 +25,7 @@ const agentInfoKey = "agent" // defaultAgentActionStoreFile is the file that will contains the action that can be replayed after restart. const defaultAgentActionStoreFile = "action_store.yml" +const defaultAgentStateStoreFile = "state.yml" const defaultLogLevel = "info" @@ -43,11 +44,16 @@ func AgentConfigFile() string { return filepath.Join(paths.Config(), defaultAgentConfigFile) } -// AgentActionStoreFile is the file that will contains the action that can be replayed after restart. +// AgentActionStoreFile is the file that contains the action that can be replayed after restart. func AgentActionStoreFile() string { return filepath.Join(paths.Home(), defaultAgentActionStoreFile) } +// AgentStateStoreFile is the file that contains the persisted state of the agent including the action that can be replayed after restart. +func AgentStateStoreFile() string { + return filepath.Join(paths.Home(), defaultAgentStateStoreFile) +} + // updateLogLevel updates log level and persists it to disk. func updateLogLevel(level string) error { ai, err := loadAgentInfo(false, defaultLogLevel) diff --git a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go index f1bd2893bf00..4ea725898f27 100644 --- a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go @@ -100,12 +100,12 @@ func loadFleetConfig(cfg *config.Config) (map[string]interface{}, error) { return nil, err } - as, err := newActionStore(log, storage.NewDiskStore(info.AgentActionStoreFile())) + stateStore, err := newStateStoreWithMigration(log, info.AgentActionStoreFile(), info.AgentStateStoreFile()) if err != nil { return nil, err } - for _, c := range as.Actions() { + for _, c := range stateStore.Actions() { cfgChange, ok := c.(*fleetapi.ActionPolicyChange) if !ok { continue diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 496424c18300..63e8611354db 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -54,7 +54,7 @@ type Managed struct { gateway *fleetGateway router *router srv *server.Server - as *actionStore + stateStore *stateStore upgrader *upgrade.Upgrader } @@ -185,13 +185,13 @@ func newManaged( batchedAcker := newLazyAcker(acker) - // Create the action store that will persist the last good policy change on disk. - actionStore, err := newActionStore(log, storage.NewDiskStore(info.AgentActionStoreFile())) + // Create the state store that will persist the last good policy change on disk. + stateStore, err := newStateStoreWithMigration(log, info.AgentActionStoreFile(), info.AgentStateStoreFile()) if err != nil { return nil, errors.New(err, fmt.Sprintf("fail to read action store '%s'", info.AgentActionStoreFile())) } - managedApplication.as = actionStore - actionAcker := newActionStoreAcker(batchedAcker, actionStore) + managedApplication.stateStore = stateStore + actionAcker := newStateStoreActionAcker(batchedAcker, stateStore) actionDispatcher, err := newActionDispatcher(managedApplication.bgContext, log, &handlerDefault{log: log}) if err != nil { @@ -223,11 +223,11 @@ func newManaged( actionDispatcher.MustRegister( &fleetapi.ActionUnenroll{}, &handlerUnenroll{ - log: log, - emitter: emit, - dispatcher: router, - closers: []context.CancelFunc{managedApplication.cancelCtxFn}, - actionStore: actionStore, + log: log, + emitter: emit, + dispatcher: router, + closers: []context.CancelFunc{managedApplication.cancelCtxFn}, + stateStore: stateStore, }, ) @@ -248,12 +248,19 @@ func newManaged( }, ) + actionDispatcher.MustRegister( + &fleetapi.ActionApp{}, + &handlerAppAction{ + log: log, + }, + ) + actionDispatcher.MustRegister( &fleetapi.ActionUnknown{}, &handlerUnknown{log: log}, ) - actions := actionStore.Actions() + actions := stateStore.Actions() if len(actions) > 0 && !managedApplication.wasUnenrolled() { // TODO(ph) We will need an improvement on fleet, if there is an error while dispatching a @@ -273,6 +280,7 @@ func newManaged( fleetR, actionAcker, statusController, + stateStore, ) if err != nil { return nil, err @@ -316,7 +324,7 @@ func (m *Managed) AgentInfo() *info.AgentInfo { } func (m *Managed) wasUnenrolled() bool { - actions := m.as.Actions() + actions := m.stateStore.Actions() for _, a := range actions { if a.Type() == "UNENROLL" { return true diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store.go b/x-pack/elastic-agent/pkg/agent/application/state_store.go new file mode 100644 index 000000000000..81d3f901469c --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/state_store.go @@ -0,0 +1,303 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "fmt" + "io" + "sync" + + yaml "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +// stateStore is a combined agent state storage initially derived from the former actionStore +// and modified to allow persistence of additional agent specific state information. +// The following is the original actionStore implementation description: +// receives multiples actions to persist to disk, the implementation of the store only +// take care of action policy change every other action are discarded. The store will only keep the +// last good action on disk, we assume that the action is added to the store after it was ACK with +// Fleet. The store is not threadsafe. +type stateStore struct { + log *logger.Logger + store storeLoad + dirty bool + state stateT + + mx sync.RWMutex +} + +type stateT struct { + action action + ackToken string +} + +// Combined yml serializer for the ActionPolicyChange and ActionUnenroll +type actionSerializer struct { + ID string `yaml:"action_id"` + Type string `yaml:"action_type"` + Policy map[string]interface{} `yaml:"policy,omitempty"` + IsDetected *bool `yaml:"is_detected,omitempty"` +} + +type stateSerializer struct { + Action *actionSerializer `yaml:"action,omitempty"` + AckToken string `yaml:"ack_token,omitempty"` +} + +func migrateStateStore(log *logger.Logger, actionStorePath, stateStorePath string) (err error) { + log = log.Named("state_migration") + actionDiskStore := storage.NewDiskStore(actionStorePath) + stateDiskStore := storage.NewDiskStore(stateStorePath) + + stateStoreExits, err := stateDiskStore.Exists() + if err != nil { + log.With() + log.Errorf("failed to check if state store %s exists: %v", stateStorePath, err) + return err + } + + // do not migrate if the state store already exists + if stateStoreExits { + log.Debugf("state store %s already exists", stateStorePath) + return nil + } + + actionStoreExits, err := actionDiskStore.Exists() + if err != nil { + log.Errorf("failed to check if action store %s exists: %v", actionStorePath, err) + return err + } + + // delete the actions store file upon successful migration + defer func() { + if err == nil && actionStoreExits { + err = actionDiskStore.Delete() + if err != nil { + log.Errorf("failed to delete action store %s exists: %v", actionStorePath, err) + } + } + }() + + // nothing to migrate if the action store doesn't exists + if !actionStoreExits { + log.Debugf("action store %s doesn't exists, nothing to migrate", actionStorePath) + return nil + } + + actionStore, err := newActionStore(log, actionDiskStore) + if err != nil { + log.Errorf("failed to create action store %s: %v", actionStorePath, err) + return err + } + + // no actions stored nothing to migrate + if len(actionStore.Actions()) == 0 { + log.Debugf("no actions stored in the action store %s, nothing to migrate", actionStorePath) + return nil + } + + stateStore, err := newStateStore(log, stateDiskStore) + if err != nil { + return err + } + + // set actions from the action store to the state store + stateStore.Add(actionStore.Actions()[0]) + + err = stateStore.Save() + if err != nil { + log.Debugf("failed to save agent state store %s, err: %v", stateStorePath, err) + } + return err +} + +func newStateStoreWithMigration(log *logger.Logger, actionStorePath, stateStorePath string) (*stateStore, error) { + err := migrateStateStore(log, actionStorePath, stateStorePath) + if err != nil { + return nil, err + } + + return newStateStore(log, storage.NewDiskStore(stateStorePath)) +} + +func newStateStore(log *logger.Logger, store storeLoad) (*stateStore, error) { + // If the store exists we will read it, if any errors is returned we assume we do not have anything + // persisted and we return an empty store. + reader, err := store.Load() + if err != nil { + return &stateStore{log: log, store: store}, nil + } + defer reader.Close() + + var sr stateSerializer + + dec := yaml.NewDecoder(reader) + err = dec.Decode(&sr) + if err == io.EOF { + return &stateStore{ + log: log, + store: store, + }, nil + } + + if err != nil { + return nil, err + } + + state := stateT{ + ackToken: sr.AckToken, + } + + if sr.Action != nil { + if sr.Action.IsDetected != nil { + state.action = &fleetapi.ActionUnenroll{ + ActionID: sr.Action.ID, + ActionType: sr.Action.Type, + IsDetected: *sr.Action.IsDetected, + } + } else { + state.action = &fleetapi.ActionPolicyChange{ + ActionID: sr.Action.ID, + ActionType: sr.Action.Type, + Policy: sr.Action.Policy, + } + } + } + + return &stateStore{ + log: log, + store: store, + state: state, + }, nil +} + +// Add is only taking care of ActionPolicyChange for now and will only keep the last one it receive, +// any other type of action will be silently ignored. +func (s *stateStore) Add(a action) { + s.mx.Lock() + defer s.mx.Unlock() + + switch v := a.(type) { + case *fleetapi.ActionPolicyChange, *fleetapi.ActionUnenroll: + // Only persist the action if the action is different. + if s.state.action != nil && s.state.action.ID() == v.ID() { + return + } + s.dirty = true + s.state.action = a + } +} + +// SetAckToken set ack token to the agent state +func (s *stateStore) SetAckToken(ackToken string) { + s.mx.Lock() + defer s.mx.Unlock() + + if s.state.ackToken == ackToken { + return + } + s.dirty = true + s.state.ackToken = ackToken +} + +func (s *stateStore) Save() error { + s.mx.Lock() + defer s.mx.Unlock() + + defer func() { s.dirty = false }() + if !s.dirty { + return nil + } + + var reader io.Reader + serialize := stateSerializer{ + AckToken: s.state.ackToken, + } + + if s.state.action != nil { + if apc, ok := s.state.action.(*fleetapi.ActionPolicyChange); ok { + serialize.Action = &actionSerializer{apc.ActionID, apc.ActionType, apc.Policy, nil} + } else if aun, ok := s.state.action.(*fleetapi.ActionUnenroll); ok { + serialize.Action = &actionSerializer{apc.ActionID, apc.ActionType, nil, &aun.IsDetected} + } else { + return fmt.Errorf("incompatible type, expected ActionPolicyChange and received %T", s.state.action) + } + } + + reader, err := yamlToReader(&serialize) + if err != nil { + return err + } + + if err := s.store.Save(reader); err != nil { + return err + } + s.log.Debugf("save state on disk : %+v", s.state) + return nil +} + +// Actions returns a slice of action to execute in order, currently only a action policy change is +// persisted. +func (s *stateStore) Actions() []action { + s.mx.RLock() + defer s.mx.RUnlock() + + if s.state.action == nil { + return []action{} + } + + return []action{s.state.action} +} + +// AckToken return the agent state persisted ack_token +func (s *stateStore) AckToken() string { + s.mx.RLock() + defer s.mx.RUnlock() + return s.state.ackToken +} + +// actionStoreAcker wraps an existing acker and will send any acked event to the action store, +// its up to the action store to decide if we need to persist the event for future replay or just +// discard the event. +type stateStoreActionAcker struct { + acker fleetAcker + store *stateStore +} + +func (a *stateStoreActionAcker) Ack(ctx context.Context, action fleetapi.Action) error { + if err := a.acker.Ack(ctx, action); err != nil { + return err + } + a.store.Add(action) + return a.store.Save() +} + +func (a *stateStoreActionAcker) Commit(ctx context.Context) error { + return a.acker.Commit(ctx) +} + +func newStateStoreActionAcker(acker fleetAcker, store *stateStore) *stateStoreActionAcker { + return &stateStoreActionAcker{acker: acker, store: store} +} + +func replayActions( + log *logger.Logger, + dispatcher dispatcher, + acker fleetAcker, + actions ...action, +) error { + log.Info("restoring current policy from disk") + + if err := dispatcher.Dispatch(acker, actions...); err != nil { + return err + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store_test.go b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go new file mode 100644 index 000000000000..26ea1eaca683 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go @@ -0,0 +1,170 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package application + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" +) + +func TestStateStore(t *testing.T) { + t.Run("ack token", func(t *testing.T) { + runTestStateStore(t, "") + }) + + t.Run("no ack token", func(t *testing.T) { + runTestStateStore(t, "czlV93YBwdkt5lYhBY7S") + }) +} + +func runTestStateStore(t *testing.T, ackToken string) { + log, _ := logger.New("state_store") + withFile := func(fn func(t *testing.T, file string)) func(*testing.T) { + return func(t *testing.T) { + dir, err := ioutil.TempDir("", "state-store") + require.NoError(t, err) + defer os.RemoveAll(dir) + file := filepath.Join(dir, "state.yml") + fn(t, file) + } + } + + t.Run("action returns empty when no action is saved on disk", + withFile(func(t *testing.T, file string) { + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + require.Equal(t, 0, len(store.Actions())) + })) + + t.Run("will discard silently unknown action", + withFile(func(t *testing.T, file string) { + actionPolicyChange := &fleetapi.ActionUnknown{ + ActionID: "abc123", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(actionPolicyChange) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 0, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("can save to disk known action type", + withFile(func(t *testing.T, file string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ + "hello": "world", + }, + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(ActionPolicyChange) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + + s = storage.NewDiskStore(file) + store1, err := newStateStore(log, s) + require.NoError(t, err) + + actions := store1.Actions() + require.Equal(t, 1, len(actions)) + + require.Equal(t, ActionPolicyChange, actions[0]) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("when we ACK we save to disk", + withFile(func(t *testing.T, file string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + store.SetAckToken(ackToken) + + acker := newStateStoreActionAcker(&testAcker{}, store) + require.Equal(t, 0, len(store.Actions())) + + require.NoError(t, acker.Ack(context.Background(), ActionPolicyChange)) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + })) + + t.Run("migrate actions file does not exists", + withFile(func(t *testing.T, actionStorePath string) { + withFile(func(t *testing.T, stateStorePath string) { + err := migrateStateStore(log, actionStorePath, stateStorePath) + require.NoError(t, err) + stateStore, err := newStateStore(log, storage.NewDiskStore(stateStorePath)) + require.NoError(t, err) + stateStore.SetAckToken(ackToken) + require.Equal(t, 0, len(stateStore.Actions())) + require.Equal(t, ackToken, stateStore.AckToken()) + }) + })) + + t.Run("migrate", + withFile(func(t *testing.T, actionStorePath string) { + ActionPolicyChange := &fleetapi.ActionPolicyChange{ + ActionID: "abc123", + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ + "hello": "world", + }, + } + + actionStore, err := newActionStore(log, storage.NewDiskStore(actionStorePath)) + require.NoError(t, err) + + require.Equal(t, 0, len(actionStore.Actions())) + actionStore.Add(ActionPolicyChange) + err = actionStore.Save() + require.NoError(t, err) + require.Equal(t, 1, len(actionStore.Actions())) + + withFile(func(t *testing.T, stateStorePath string) { + err = migrateStateStore(log, actionStorePath, stateStorePath) + require.NoError(t, err) + + stateStore, err := newStateStore(log, storage.NewDiskStore(stateStorePath)) + require.NoError(t, err) + stateStore.SetAckToken(ackToken) + diff := cmp.Diff(actionStore.Actions(), stateStore.Actions()) + if diff != "" { + t.Error(diff) + } + require.Equal(t, ackToken, stateStore.AckToken()) + }) + })) + +} diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go b/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go index f79eca617f82..6376492c2f29 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_retryable.go @@ -82,7 +82,7 @@ func (o *retryableOperations) runOnce(application Application) func(context.Cont o.logger.Debugf("running operation '%s' of the block '%s'", op.Name(), o.Name()) if err := op.Run(ctx, application); err != nil { - o.logger.Errorf("operation %s failed", op.Name()) + o.logger.Errorf("operation %s failed, err: %v", op.Name(), err) return err } } diff --git a/x-pack/elastic-agent/pkg/agent/storage/storage.go b/x-pack/elastic-agent/pkg/agent/storage/storage.go index 2435311486a3..2ff2d7250f13 100644 --- a/x-pack/elastic-agent/pkg/agent/storage/storage.go +++ b/x-pack/elastic-agent/pkg/agent/storage/storage.go @@ -153,6 +153,23 @@ func NewDiskStore(target string) *DiskStore { return &DiskStore{target: target} } +// Exists check if the store file exists on the disk +func (d *DiskStore) Exists() (bool, error) { + _, err := os.Stat(d.target) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// Delete deletes the store file on the disk +func (d *DiskStore) Delete() error { + return os.Remove(d.target) +} + // Save accepts a persistedConfig and saved it to a target file, to do so we will // make a temporary files if the write is successful we are replacing the target file with the // original content. diff --git a/x-pack/elastic-agent/pkg/fleetapi/action.go b/x-pack/elastic-agent/pkg/fleetapi/action.go index 211b9199f2f9..d836aa801c23 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/action.go +++ b/x-pack/elastic-agent/pkg/fleetapi/action.go @@ -21,6 +21,8 @@ const ( ActionTypePolicyChange = "POLICY_CHANGE" // ActionTypeSettings specifies change of agent settings. ActionTypeSettings = "SETTINGS" + // ActionTypeApplication specifies agent action. + ActionTypeApplication = "APP_ACTION" ) // Action base interface for all the implemented action from the fleet API. @@ -154,6 +156,16 @@ type ActionSettings struct { LogLevel string `json:"log_level"` } +// ID returns the ID of the Action. +func (a *ActionSettings) ID() string { + return a.ActionID +} + +// Type returns the type of the Action. +func (a *ActionSettings) Type() string { + return a.ActionType +} + func (a *ActionSettings) String() string { var s strings.Builder s.WriteString("action_id: ") @@ -165,25 +177,45 @@ func (a *ActionSettings) String() string { return s.String() } -// Type returns the type of the Action. -func (a *ActionSettings) Type() string { - return a.ActionType +// ActionApp is the application action request. +type ActionApp struct { + ActionID string + ActionType string + Application string + Data json.RawMessage +} + +func (a *ActionApp) String() string { + var s strings.Builder + s.WriteString("action_id: ") + s.WriteString(a.ActionID) + s.WriteString(", type: ") + s.WriteString(a.ActionType) + s.WriteString(", application: ") + s.WriteString(a.Application) + return s.String() } // ID returns the ID of the Action. -func (a *ActionSettings) ID() string { +func (a *ActionApp) ID() string { return a.ActionID } +// Type returns the type of the Action. +func (a *ActionApp) Type() string { + return a.ActionType +} + // Actions is a list of Actions to executes and allow to unmarshal heterogenous action type. type Actions []Action // UnmarshalJSON takes every raw representation of an action and try to decode them. func (a *Actions) UnmarshalJSON(data []byte) error { type r struct { - ActionType string `json:"type"` - ActionID string `json:"id"` - Data json.RawMessage `json:"data"` + ActionType string `json:"type"` + Application string `json:"application"` + ActionID string `json:"id"` + Data json.RawMessage `json:"data"` } var responses []r @@ -209,6 +241,13 @@ func (a *Actions) UnmarshalJSON(data []byte) error { "fail to decode POLICY_CHANGE action", errors.TypeConfig) } + case ActionTypeApplication: + action = &ActionApp{ + ActionID: response.ActionID, + ActionType: response.ActionType, + Application: response.Application, + Data: response.Data, + } case ActionTypeUnenroll: action = &ActionUnenroll{ ActionID: response.ActionID, diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go index 19c936ed79c5..79bcb39d40b0 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go @@ -22,6 +22,7 @@ const checkingPath = "/api/fleet/agents/%s/checkin" // CheckinRequest consists of multiple events reported to fleet ui. type CheckinRequest struct { Status string `json:"status"` + AckToken string `json:"ack_token,omitempty"` Events []SerializableEvent `json:"events"` Metadata *info.ECSMeta `json:"local_metadata,omitempty"` } @@ -49,7 +50,8 @@ func (e *CheckinRequest) Validate() error { // CheckinResponse is the response send back from the server which contains all the action that // need to be executed or proxy to running processes. type CheckinResponse struct { - Actions Actions `json:"actions"` + AckToken string `json:"ack_token"` + Actions Actions `json:"actions"` } // Validate validates the response send from the server. From 739f820da1475783be90f23467fb09ef4af4fa22 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Tue, 19 Jan 2021 17:14:53 +0100 Subject: [PATCH 03/35] Fix docs of MSSQL filebeat module referencing Traefik plus change paths from Linux to MS Windows (#23360) --- filebeat/docs/modules/mssql.asciidoc | 6 +++--- x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/module/mssql/_meta/config.yml | 2 +- x-pack/filebeat/module/mssql/_meta/docs.asciidoc | 6 +++--- x-pack/filebeat/modules.d/mssql.yml.disabled | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/filebeat/docs/modules/mssql.asciidoc b/filebeat/docs/modules/mssql.asciidoc index 7ecaa5e247a4..8d07a4e21946 100644 --- a/filebeat/docs/modules/mssql.asciidoc +++ b/filebeat/docs/modules/mssql.asciidoc @@ -20,14 +20,14 @@ include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] The following example shows how to set paths in the +modules.d/{modulename}.yml+ -file to override the default paths for Træfik logs: +file to override the default paths for MSSQL logs: ["source","yaml",subs="attributes"] ----- - module: mssql log: enabled: true - var.paths: ["/var/opt/mssql/log/error*"] + var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] ----- @@ -35,7 +35,7 @@ To specify the same settings at the command line, you use: ["source","sh",subs="attributes"] ----- --M "mssql.log.var.paths=[/var/opt/mssql/log/error*]" +-M "mssql.log.var.paths=['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']" ----- //set the fileset name used in the included example diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d0d78781e934..cec925422c7d 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1410,7 +1410,7 @@ filebeat.modules: # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] #-------------------------------- MySQL Module -------------------------------- #- module: mysql diff --git a/x-pack/filebeat/module/mssql/_meta/config.yml b/x-pack/filebeat/module/mssql/_meta/config.yml index 652ca8910568..a56e658f7b71 100644 --- a/x-pack/filebeat/module/mssql/_meta/config.yml +++ b/x-pack/filebeat/module/mssql/_meta/config.yml @@ -5,4 +5,4 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/x-pack/filebeat/module/mssql/_meta/docs.asciidoc b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc index 2861d2754ee7..9defa42e5c33 100644 --- a/x-pack/filebeat/module/mssql/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/mssql/_meta/docs.asciidoc @@ -15,14 +15,14 @@ include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] The following example shows how to set paths in the +modules.d/{modulename}.yml+ -file to override the default paths for Træfik logs: +file to override the default paths for MSSQL logs: ["source","yaml",subs="attributes"] ----- - module: mssql log: enabled: true - var.paths: ["/var/opt/mssql/log/error*"] + var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] ----- @@ -30,7 +30,7 @@ To specify the same settings at the command line, you use: ["source","sh",subs="attributes"] ----- --M "mssql.log.var.paths=[/var/opt/mssql/log/error*]" +-M "mssql.log.var.paths=['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']" ----- //set the fileset name used in the included example diff --git a/x-pack/filebeat/modules.d/mssql.yml.disabled b/x-pack/filebeat/modules.d/mssql.yml.disabled index 5e03b661da8c..3fdaac9e8a66 100644 --- a/x-pack/filebeat/modules.d/mssql.yml.disabled +++ b/x-pack/filebeat/modules.d/mssql.yml.disabled @@ -8,4 +8,4 @@ # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. - #var.paths: + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] From 2f50f9e673761f88e62fd4c2e476c70cd714caff Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 19 Jan 2021 11:53:37 -0500 Subject: [PATCH 04/35] [Elastic Agent] Increase checkin grace period. (#23568) * Include checkin grace period. * Add changelog entry. --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + x-pack/elastic-agent/pkg/core/server/server.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 00983d96c152..a68ae2d19caf 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -35,6 +35,7 @@ - Do not take ownership of Endpoint log path {pull}23444[23444] - Fixed fetching DBus service PID {pull}23496[23496] - Fix issue of missing log messages from filebeat monitor {pull}23514[23514] +- Increase checkin grace period to 30 seconds {pull}23568[23568] ==== New features diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index c7eef7040d46..f0b8ac73d8a5 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -37,7 +37,7 @@ const ( InitialCheckinTimeout = 5 * time.Second // CheckinMinimumTimeoutGracePeriod is additional time added to the client.CheckinMinimumTimeout // to ensure the application is checking in correctly. - CheckinMinimumTimeoutGracePeriod = 2 * time.Second + CheckinMinimumTimeoutGracePeriod = 30 * time.Second // WatchdogCheckLoop is the amount of time that the watchdog will wait between checking for // applications that have not checked in the correct amount of time. WatchdogCheckLoop = 5 * time.Second From 616266fa286f0ac4408348fcd4da7da2358182b1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 19 Jan 2021 10:05:46 -0700 Subject: [PATCH 05/35] Rename s3 input to aws-s3 input (#23469) --- CHANGELOG.next.asciidoc | 1 + .../docs/aws-credentials-examples.asciidoc | 8 ++++---- filebeat/docs/filebeat-options.asciidoc | 2 +- filebeat/docs/modules/cisco.asciidoc | 4 ++-- .../pkg/agent/program/supported.go | 2 +- x-pack/elastic-agent/spec/filebeat.yml | 4 ++-- .../filebeat.inputs.reference.xpack.yml.tmpl | 10 +++++----- .../filebeat/docs/inputs/input-aws-s3.asciidoc | 18 +++++++++--------- x-pack/filebeat/filebeat.reference.yml | 12 ++++++------ x-pack/filebeat/include/list.go | 2 +- .../input/{s3 => awss3}/_meta/fields.yml | 0 .../{s3 => awss3}/_meta/s3-input.asciidoc | 0 .../filebeat/input/{s3 => awss3}/collector.go | 2 +- .../input/{s3 => awss3}/collector_test.go | 2 +- x-pack/filebeat/input/{s3 => awss3}/config.go | 2 +- x-pack/filebeat/input/{s3 => awss3}/fields.go | 10 +++++----- .../input/{s3 => awss3}/ftest/sample1.txt | 0 x-pack/filebeat/input/{s3 => awss3}/input.go | 4 ++-- .../input/{s3 => awss3}/s3_integration_test.go | 2 +- x-pack/filebeat/input/default-inputs/inputs.go | 4 ++-- .../cloudtrail/config/{s3.yml => aws-s3.yml} | 2 +- .../module/aws/cloudtrail/manifest.yml | 2 +- .../s3.yml => cloudwatch/config/aws-s3.yml} | 2 +- .../module/aws/cloudwatch/manifest.yml | 2 +- .../config/s3.yml => ec2/config/aws-s3.yml} | 2 +- x-pack/filebeat/module/aws/ec2/manifest.yml | 2 +- .../config/s3.yml => elb/config/aws-s3.yml} | 2 +- x-pack/filebeat/module/aws/elb/manifest.yml | 2 +- .../s3.yml => s3access/config/aws-s3.yml} | 2 +- .../filebeat/module/aws/s3access/manifest.yml | 2 +- .../module/aws/vpcflow/config/input.yml | 4 ++-- .../filebeat/module/aws/vpcflow/manifest.yml | 2 +- x-pack/filebeat/module/cisco/_meta/config.yml | 2 +- .../filebeat/module/cisco/_meta/docs.asciidoc | 4 ++-- .../module/cisco/umbrella/config/input.yml | 4 ++-- x-pack/filebeat/modules.d/cisco.yml.disabled | 2 +- 36 files changed, 64 insertions(+), 63 deletions(-) rename x-pack/filebeat/input/{s3 => awss3}/_meta/fields.yml (100%) rename x-pack/filebeat/input/{s3 => awss3}/_meta/s3-input.asciidoc (100%) rename x-pack/filebeat/input/{s3 => awss3}/collector.go (99%) rename x-pack/filebeat/input/{s3 => awss3}/collector_test.go (99%) rename x-pack/filebeat/input/{s3 => awss3}/config.go (99%) rename x-pack/filebeat/input/{s3 => awss3}/fields.go (73%) rename x-pack/filebeat/input/{s3 => awss3}/ftest/sample1.txt (100%) rename x-pack/filebeat/input/{s3 => awss3}/input.go (98%) rename x-pack/filebeat/input/{s3 => awss3}/s3_integration_test.go (99%) rename x-pack/filebeat/module/aws/cloudtrail/config/{s3.yml => aws-s3.yml} (99%) rename x-pack/filebeat/module/aws/{elb/config/s3.yml => cloudwatch/config/aws-s3.yml} (98%) rename x-pack/filebeat/module/aws/{s3access/config/s3.yml => ec2/config/aws-s3.yml} (98%) rename x-pack/filebeat/module/aws/{cloudwatch/config/s3.yml => elb/config/aws-s3.yml} (98%) rename x-pack/filebeat/module/aws/{ec2/config/s3.yml => s3access/config/aws-s3.yml} (98%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6dece54c4d62..8026c0a6ac92 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571] - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] +- Rename `s3` input to `aws-s3` input. {pull}23469[23469] *Heartbeat* - Adds negative body match. {pull}20728[20728] diff --git a/filebeat/docs/aws-credentials-examples.asciidoc b/filebeat/docs/aws-credentials-examples.asciidoc index e306e2bb0a0a..c8509bf826e7 100644 --- a/filebeat/docs/aws-credentials-examples.asciidoc +++ b/filebeat/docs/aws-credentials-examples.asciidoc @@ -3,7 +3,7 @@ [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue access_key_id: '' secret_access_key: '' @@ -15,7 +15,7 @@ or [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' @@ -27,7 +27,7 @@ filebeat.inputs: [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue role_arn: arn:aws:iam::123456789012:role/test-mb ---- @@ -37,7 +37,7 @@ filebeat.inputs: [source,yaml] ---- filebeat.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue credential_profile_name: test-fb ---- diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index b8e230bfd0eb..704d7ef99e44 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -63,6 +63,7 @@ subdirectories of a directory. You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-aws-cloudwatch>> +* <<{beatname_lc}-input-aws-s3>> * <<{beatname_lc}-input-azure-eventhub>> * <<{beatname_lc}-input-cloudfoundry>> * <<{beatname_lc}-input-container>> @@ -76,7 +77,6 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-netflow>> * <<{beatname_lc}-input-o365audit>> * <<{beatname_lc}-input-redis>> -* <<{beatname_lc}-input-s3>> * <<{beatname_lc}-input-stdin>> * <<{beatname_lc}-input-syslog>> * <<{beatname_lc}-input-tcp>> diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index d8c4b44df4d0..8f55d5c16d86 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -388,7 +388,7 @@ will be found under `rsa.raw`. The default is false. The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. -To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-s3.html[S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. +To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-aws-s3.html[AWS S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. This fileset supports all 4 log types: - Proxy @@ -409,7 +409,7 @@ Example config: - module: cisco umbrella: enabled: true - var.input: s3 + var.input: aws-s3 var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue var.access_key_id: 123456 var.secret_access_key: PASSWORD diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 626466dd94c2..b49de1bce6f0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -23,7 +23,7 @@ func init() { // spec/heartbeat.yml // spec/metricbeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzMWUl3qz6W3/fH+G+rBxBxuuhzamFIMdkhz+QFCe2QZAO2wPwDHqBPf/c+EhiDnbyhXtc7vXKMhYare3/DzX//UZVr+h9xmf9btX4/rt//vcn5H//1B8mtGn/dJ6vQ8Jehz2mBOU3KLYGrR9e2TuRVbTHyAEbuIkKeEkOcRtqHvxW03Sewcevg1a1c06sjOEsxCGsMZ8oyDw8R9CoMVzpzPBXLMZ+NVY/YftPXpnqKoP++hLjCMFTc7JS4mWrJz3yy/gHblhKFesscj0dQbe/X85hZeCqxw/Yl2SeuqSQ45w9I8xWahyn5uk/WmrJwzXnt2rgkTsAp10EEzypGz49mNk9cc564js+JHW6ZrTcvmVGSwlCZ87yQv5nzJAbh7CUzlDUy+PUdnBIn5LTdD+O6dawTWQ17qZgdNt3exO9Gi+GZR1pwpMVofXOeLF/v1+3m01XmGCqbl1psh4eXzKgwnBXMTvaeU/fv+Pridf4X92meRHC2c+00pUrN16/Jbg369x2lck3GiW21zOZbCsKU5v7ea3aLP/61S6R1wcp9VtQ3aRTA2Y7aekmKVfIGwi1DXsmc3SIC6u4lMzjJgxMB/MBMtcXQV2nOlfWqTGkRlDi3tuxpn+DrHDW2Q2AWMi3LCLw9uk+R9vKULIitF0gzUman3XXaQUoLVpLtPnEz/TmGXhMhb7ZUwipCvhLD5+Nob0eqBSmz345iniUID9gxjrFIvdf9QTxz5ZznkhThw0s2z5ZAPzFTty4hWSqjdzRfiVDAl+B8xI0+OqPy5zIXz9yFaxpaDGc7orFWzLdqS4osoyGANRFUkiDnFUY+RX+/7Ff+PayBrHPLbEvB4ZnKs1vnD9eJQMojUG9iOBPjK/K0XyxfDb62wy0CuCT2W59KxilCwV7sZRxver2zrB+X0py1l7Rcvs4zlodNDPHM7Z8xm9cY6qq4u+d2vqC23jJLzOcrETxXL8m+du3wAUN/I9Icf+3LwBH3lzy6pvdxXry63TjbarA2lEDtmt4wtzva1/JVVakt4hnwm+cNRv6RIW+L0XM2mueTdSfjD+ucnz46q7+dn8zCaHCoqyTnB5FTxD49mpmSYJTySNXzGJ75JXbUtpT4aZ+4+Sh3kM8jLWxiFAzx7KF0cS33ecbg7O7M93sZ4E1C2Nq6wKHy6Dq1bvbPP4a2Edxs90kMZyeGgrb/7R2j3aPrBDNqvy0+hjRlXLcLGYMe0pB2M/ZbEGrXfP1VYkAa5WeO5/1Zc6tiMBzloaHQIuTyTJf5ZM4FxyHeIKww9BWiee1LZhAs5kOrQwT9LUZ+i4B1ikNdnK1y7Y5ilnldRrl1iEJlmqPD79YpXsmaqiM0v6klIyd2yFm/Z1qE1Yg6atfxOIE6wN2aE0jvaSONgM+p5m8iZJQI1Hy9Gs7bYKgeWR5u5NiBJq4xiwHPBI1O7nl7H/PJnXUxGb5P731MKaN9IFxSwI8k2S8YSLnAXGKHB6wF+4UZ/Gc3Z3BDMwKvmBKbgmb4gTqhQjWldJ8ekmfTSEm+SmLbal9BOBNzEC1UxJjN6ynxwICJLYZWE4GkWKzKLQEzkd+pqCMhPTynbhicKTFU+TIX41LdNf+uuyZLaa4A36QDfW0yvibr+I6+BDxBj0dodaEsWcJRHqZsXnZQmBlkUCGFz5kTnpY5r8jrjJPcyogd7r5AkfI+nyiWy9gi4AQZlaSMkUrBuVVR8JYtzXm2fOs+CbQOEWScwPDAzFlNQMC/oKSmtrWNG7VLN/NbSuqbqqsigBUxnBXL/MxZHlZfYMCjIixcrtyoPRGToF1KCgkzDC3lu3CTyXT6U5QgAvyA7fDhkrbM4ScRb0Fl9CTLqSR5KeBnQ7WgwdCqkSboUZbDcUhjWz8g4B9JjqsY+koHH0IeBJsIYkUo0S7dJVQ9uvb5iLVnCUcEWqdbKL6BuYbB8wTSIqCf1qGeEvu8Yba+ITZv2dNYxRkKaffJZc/0NC7Lu70eCNBP47LHKN1iZCgyp4peiaJnefcxXMnPAQrlPXsnmusSvgSsiXu62atCVL2Kka9MIYIrWN7LKKbF8z96jmvM8zAnmtfDsKBTWUf9XeGGAOXRtftyP11o66/XZ9pw5kVPmwoVMtDqzoCA2LfafnZvt/uNUcCFkr99Plnz9Cl8T6nIGbuFnnJy60DBWUjHCXxf9iXzejWOnZpSx7jC8PD8fMS9HJN/j+Mt8wJzIVN7uSTrZLyeaxqiXg/M1FtmB1L2Ui3YxfDhZp0QSBzQgi0V+7P90yfzqNiZP7pOuKPz6V46iRwcI1CLcyTY1rcxCJubeSoC6JHm4S5G/oaC85EJOSxySj57vj9/o7dr5Iv3Hl3Hn4l3LnH4EbpjyOcIfEBP33lv6hSv9UPzsCYa5pJ2v05qvJM/trAM1oBPy3yWEhi2AovxT9D0zfqdc0W+kA0iLwXfKBh5m1vJc5Uz7l1N3btO49a5CrepUeBXRAt3DFhKBJIrdqBSpflb3eVdsGfQHeHK+chgkBNNyFZvdp3PP5IiEPTL6bVGdgT47wMOC4kB/NMSGWpU+Gp0HbdnTnBCI+t1nTdVmGP8SYF+uD6rU5zX6fX7JFdqioLR+zPObFwRjQ57IO0z8KGlYpsrYzmGbX6VOs7z6G9fwTY/jL7f5qcSafNhfgaD03VseIjRNbYMcGkpf1l+2wOXfyrBJcevpnwq9UMRCEuYC4xfa4rEWskXP4TVI87/KWl51VLDPfSybnSvfI2kROdmwfYYPjxOpeJ17WX+67Jxac6LHo+KpcQM9h5B/B690so1mdBDwja3sUlLM/nb3y5yMV3H7/UHevHVDlNaBF1scr/Ggpsmzzrr6lrVoOkoCBWG5ocYnuvv6b/LWGaHNbUl7xwGPfCk5hE8t7/eSVNTklsFhqrgnPH80mJNxwreYiXJ6YFIbjnp2A4zBmmGbto4UjM7z8dJPP7hDpy/YYArsaU3GDK+duZX/r3wwYR7jUbcDyr8mcAzjIIy0p6Py6wacP1zPP2Gdf0ODn9aqx9Y2Jua/Sd0AafrCI6gYp787UPbdsGczesu+ZLNT65tHbBp7CPkLzHa7T2nPjIUiDG6qCMMzynVRFx9HiFvG5uyhhoMg5I2tBJ79UCH214j6lFocV9w3aSTmK/r94x+UFxfYajQnG9789W3pFXOHK+MQG/S7tvOLUaBSs1ZSWzle8VyGStE5onYloK/Z+JuioVAfYe/qg9LJLx7Vfd++lsm7jo/ChoGbwyfrRdYmKVmVkkyflJ3GHoqbjwmwITZPI86ES4LijZ6jVHQxNDvC8w4Ui2Y9P+6pOhEyaT/NulJqUfsyH7FAZtSmAnDcFhDdeg3CZMg4o3R6lEADQGBLOZlvjpSjbcCpJYFr4k5EyLwYloW1z7IxwU/JqsYznYYJRdSlILmJTMuZ2w7AuKHOJf9nku7f0Md7xiBsKVAH4qHgNkmAvoB5+eyM7H8QEHYMEtPcREMomUwo32+9QahEblD4NAvzWmu1/cmIThen/nDvx+6faopfbrpdX5gfD4xG1uiGTMErIpYn5i6bu3rmiNwuD/77EgGwdIJ/7Xtc+qsJDkNhqmRdVH2ZnPI1a6BMTGPGVrd7FULjgicS6qtpr2tiykb3dHEYP7UOYY7zDDEEsx+s3G8E+hIYyWz0w3NwwKjdGgyfCDKO1LKHt6XoMcx7Xn3TQH4e0XjL5ri8HOS/pZJdjxR4+vFk7760jVu/rLMqvI+Rj2RijWe9ok37kd3Bu4QQZVPTVcvbCdjrw0Pgd8MnvlVkKppDMJNhLwmuu3R9jky4AQYhOwoVy579vmVmH/EXI7e+xkze9Mn/70GWH5vx/3j32Wib8z//7HhmWoM8e5tbuHCO45zQWoUoTnuuWHgyZ8xT5N5r3fbC7BRv34k6j42UJOzHC75/ksmqjNOg9D7URNVxnS3/shFvdnWNgahMhF6jhBUNWf2rdCjddAt9B2hJ8bcjf2m0JMqtVGtTq3+kNCTjnL59iY/vyP0pmM/FXrsM6EnHRxGnzqpX3MsNwD5mVuh/V2Z/brXDr+I1azr9Od/vfzD8Z/hav5fuBeZ2P/zL/8bAAD//+ql68I=") + unpacked := packer.MustUnpack("eJzMWVt3qziWfp+fUa89Fy5xupm1+sGQQoAdckxOJKE3JNmALTAd4wvMmv8+S4Ax4OTkVNdUzTw5xkLakvb+Ljv/9cuhWLP/iIrs3w7r99P6/d+rTPzyn7/QzC7J9328gqa/hL5gOREsLrYUrR5dYJ/pq1oT7GkEu4sQe0qESBLqH/6Ws3ofo8otg1f34FpeGaJZQjRYEjRTlhk8hsg7ELQyuOOppBnz2Vj1RMCbsbbUc4j89yUiB4Kg4qbn2E1Vu/nMRusfCbCVEBo1dzwRIrW+X8/jVu6pFMD6Jd7HrqXEJBMPWPcVlsGEft/Ha11ZuNa8dAEpqBMIJgwtRBeV4OdHK53HrjWPXccXFMAtB0b1kpoFzU2VO8+L5jdrHkcanL2kprLGpri9QxLqQMHqfT+uXcc+01Ufy4EDWLWxyd/NmqCLCPXgxPLB+tY8Xr7er9vOZ6jcMVU+L/QIwONLah4ImuUcxHvPKbt3fGPxOv+L+zSPQzTbuSBJmFKK9Wu8W2vd+45ycC0uKLBrDsSWaTBhmb/3qt3il39tE2md82Kf5uUkjQI02zFgFDRfxW8a3HLsFdzZLUJN3b2kpqBZcKaaOHJLrQnyVZYJZb0qEpYHBcnsLX/ax+Q2R0kA1Ky8Scsi1N4e3adQf3mKFxQYOdbNhIOkvU4QJCznBd3uYzc1niPkVSH2ZksFHkLsKxF6Pg1iOzE9SDh4O8l5lho8Esc8RTL1XvdH+cxt5rwUNIcPL+k8XWrGmVuGfT2SpTJ4R/eVEAdiqV1OpDIGe1T+sczkM3fhWqYeodmO6ryW863qgmHbrKjGqxApcZCJA8E+w79e423+7tfA9qXmwFYIvLBm7/blw3VCLRGhVm4iNJPjD/Rpv1i+mmIN4BZrpKDgrUsl8xziYC9jGZ43u91Z2o1LWMbra1ouX+cpz2AVITJzu2cciJIgQ5V391zPFwwYNbflfL4SosvhJd6XLoAPBPkbmebke1cGjry/+NG1vI/z4tVtxwG7InpfAqVref3c7iCu5auqMiDPMxCT5xXB/oljb0vwczqY55N1R+OP60ycP9qrv52frdysCDRUmomjzCkKzo9WqsQEJyJUjSxCF3E9OwZsJXrax242yB3si1CHVYSD/jw7KF3cyn2ecjS72/N9LD28NRC2tq9wqDy6TmlY3fOPoW0AN9t9HKHZmeOg7n57J3j36DrBjIG3xceQpgzrdtGcQQdpWJ+M/RGEglKsvzcYkITZRZB5t9fMPnAEB3loKiyHotnTdb4m54JTf94aPBDkK1T36pfUpETOh1fHEPlbgv0aa/Y5gobc28EFLcUss7IIM/sYQmWco/3v9jlaNTVVhng+qSUzowAK3sXMcngYUEfpOp6gyNBIu+YI0jvaSELNF0z3NyE2C6yVYr3q91sRpJ54BjfN2J4mbmcWaSKVNDq65+39mY/urD2T/vv43oeUMogDk4Jp4kTj/YJriZCYSwE8Ej3YL6zgr+2cwYRmJF5xJbIkzYgjc6DCdKVwnx7iZ8tMaLaKI2DXrxqcyTmoDhU5ZvN6jj2tx8SaILsKtThfrIot1WYyvxNZR1J6eE5ZcTRTIqSKZSbHJYZr/Wq4Fk9Ypmi+xXr62qRiTdfRHX1JeEKeCPHqSllNCYcZTPi8aKEwNWmvQnJfcAeel5k40NeZoJmdUgB335BMeV+MFMt1bB4Iis1DQxkDlUIy+8C0t3RpzdPlW/tJkX0MERcUwSO3ZiXVAvENxyUD9jaq1DbdrB8pqR+qrgPVeB6hWb7MLoJn8PANBSLMYe4KZaL25JkE9bKhEJgSZCtfwk3apNM/ZAliTRwJgA/XtOWOOMvzllTGzk05FTQrJPxsmB5UBNkl1iU9NuVw6tMYGEes+SeakUOEfKWFDykPgk2IiCKVaJvuDVQ9uuByIvpzA0cU2ecpFE9gruLoMoK0UDPOa2gkFFw2HBgbCkTNn4YqzlRovY+vMbPzsCzvYj1SzTgPy57gZEuwqTQ5lXdKFD83dx+hVfPZQ2Fzz96ZZUYDXxLW5D1NYlWoahwi7CtjiBAKae5lcKb58z+7j9uZZzCjutfBsKTTpo66uyIV1ZRHF3Tlfr7S1t9uz/R+z4uONhUmZaDd7gFrMm61/uzepvFGOBBSyU+fj9Y8fwrfYypyhm6ho5zMPjLtIqXjCL6vcTV5vRqenZowx7zBcP/8ciKdHGv+Hp53kxdESJnayaWmTobruZYp6/XILaPmIGhkL9ODXYQeJutArcEBPdgyGR/wz5/MoxJn/ug6cMfm41haiRycQq2U+4gJMLaRBqvJPAeqsRPL4C7C/oZplxOXcljmVPPs+X7/lVGvsS/fe3QdfybfuZ7Dz9Adx77A2gf09MV7Y6d4qx+WwZLqRDS0+31U4638AdIy2D0+LbNZQhGsJRaT30DTk/Vb54p9KRtkXkq+UQj2NlPJc5Mz7l1N3btOc+pcpdvUWaVuKZB3zRMO/P3kt/r5dv7JOocVeVXb/ACJGt7qoJ0DkBOXki/fDTHoyIFd0KzPj9IFXe7c3s9DfV4yB6ZMhzc5DhKFO+ZmKCN7WeIECgNFTbWH2zPNziLt1/77oEZKF6gX7tzepxlUSHY58dv+Ts91qIZY5uZqmAODXFUmNSW/i3q0jhNs2S33ZA31Y6VkX99+k/ZP5vqfK42BOEZZI42lJjg2Ejkjp5fU/OumlbFbqpt9npLcO0n8nPBiww1koI2Ge/hQbt9iGein29lzHJz5UHZqcMak9cjePpGU/drHa2yb1138LZ2fXWAfiWXuQ+wvCd7tPafs5g+MpTXPCbokTA+KUPdFiL1tZLGDa/GKoKBgFTvIPXpamZCsTLxKyk2pGaSlH3c8knX0Xn6gGV8BTFgetJoo80si+Wn0rLWvrn3odR3ToMLx/BihS/mVBryO5QCWDDTcc+xz4EnNQnSpf383TU1oZucEqZJ3hvM3uTQeK7mLFzRjR9rwy9kgAKYcsRRPWjnNvTvPp9F5/NNdOH/DNaFEtlERxMXamd84+MoJI/41K3k/OPdnNA/2BMsceD4t00OP7Z9j6g9q9AssvvFFf6/9e9Na7bFCavM/pBM4XuerOvu/qq1sXb6n7IPi+o6gwjKx7QxY15ZWBXe8ItQ6o3bfeq4JDlRmzQoKlK+K5TpWCs0zBbZCvjJyk2KhyNiR7+rDEkv/fig7T/0jI3ebHwcVRxPTB4ycSMNUzQ6NEHxSdwR5Kqk8LsGEA5GFrRBvCopVRklwUEXI7wrMPDE9GPUAO1JshMmoBzfqS6kn4jQ9iyOxGnEmTcNxjdS+5yTJQJ43watHCTRUC5piXmarkyRHCVLLXJTUmkkheDUui1sv5OOCHxrACM12BMdXcd6ImpfUvO6xnhDbteW/YY53CjVYM83oi4dqs40UJSS7FK2RFUemwaohxDzoRUlvSLt860xCJXOHor5nmrHMKO+NQnC6PfP7f0G0caoJe5r0Oz8wP58YjoacsWYfqP2JsWvXvq05AIf7vc9OVJ8Pha1YA18wZ9WQU2+aqqYuis5w9rnaNjFGBjLFq0msenDC2qVg+mrc37oas8EdjUzmb9pHf4cpQaQBsz/ZPN6JdKzzgoNkwzKYE5z0jYYPhHlLSunD+1LrcEx/3v2wB/t7+7ZTIvqgd9sR0P+CMYafk/SPjLLjyRpfL56M1be2efOXZXoo7s+oI1K5xtM+9oY96VbAH0OkirHx6hoRo7E3YSvxm6OLGJiHJNLgJsReFU77tF2O9DihQWUUV5Mr15h9cSPmnzGYg/d+i6GdGII/1wQ332syNlF/ipGeNABGHNJwbd5wSSbvWHKMxIimSfZTDaqxxpDvTnPrapboUKNIzXHPDT1P/pbe/GjenzRPVs73BD08jvvto70cr/n+e/rvyyaem9BzLf4eIvIevjZ/H6jGpfaoI4sVVvz3v1+FXhGx3fojF/UG7G2kQWUk9BwpqErBwVTosTJoF/pC6Mkxd2N/KPQalVqpdqtWf0ro5VIJL9/ems8vhN547KdCj38m9BoHR/CnTur3OZYJQH7mVlh3V1a37q3LL89q1nb7s79d/+n4R7ia/xfupUns//6X/wkAAP//iTjyjQ==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 2472f2b52208..6aacf99ccf0d 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -59,7 +59,8 @@ rules: selector: inputs key: type values: - - awscloudwatch + - aws-cloudwatch + - aws-s3 - azure-eventhub - cloudfoundry - container @@ -73,7 +74,6 @@ rules: - netflow - o365audit - redis - - s3 - stdin - syslog - tcp diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index c5351be3339a..31c49fb19cd5 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -54,21 +54,21 @@ # Path to a JSON file containing the credentials and key used to subscribe. credentials_file: ${path.config}/my-pubsub-subscriber-credentials.json -#------------------------------ S3 input -------------------------------- +#------------------------------ AWS S3 input -------------------------------- # Beta: Config options for AWS S3 input -#- type: s3 +#- type: aws-s3 #enabled: false # AWS Credentials # If access_key_id and secret_access_key are configured, then use them to make api calls. - # If not, s3 input will load default AWS config or load with given profile name. + # If not, aws-s3 input will load default AWS config or load with given profile name. #access_key_id: '${AWS_ACCESS_KEY_ID:""}' #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' #session_token: '${AWS_SESSION_TOKEN:"”}' - #credential_profile_name: test-s3-input + #credential_profile_name: test-aws-s3-input # Queue url (required) to receive queue messages from - #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-s3-logs-queue" + #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-aws-s3-logs-queue" # The duration (in seconds) that the received messages are hidden from subsequent # retrieve requests after being retrieved by a ReceiveMessage request. diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 3ea37b3c754b..fde7f734d6cd 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -2,18 +2,18 @@ :libbeat-xpack-dir: ../../../../x-pack/libbeat -:type: s3 +:type: aws-s3 [id="{beatname_lc}-input-{type}"] -=== S3 input +=== AWS S3 input ++++ -S3 +AWS S3 ++++ beta[] -Use the `s3` input to retrieve logs from S3 objects that are pointed by messages +Use the `aws-s3` input to retrieve logs from S3 objects that are pointed by messages from specific SQS queues. This input can, for example, be used to receive S3 server access logs to monitor detailed records for the requests that are made to a bucket. @@ -28,13 +28,13 @@ stopped and the sqs message will be returned back to the queue. ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: -- type: s3 +- type: aws-s3 queue_url: https://sqs.ap-southeast-1.amazonaws.com/1234/test-s3-queue credential_profile_name: elastic-beats expand_event_list_from_field: Records ---- -The `s3` input supports the following configuration options plus the +The `aws-s3` input supports the following configuration options plus the <<{beatname_lc}-input-{type}-common-options>> described later. [float] @@ -74,7 +74,7 @@ can be assigned the name of the field. This setting will be able to split the messages under the group value into separate events. For example, CloudTrail logs are in JSON format and events are found under the JSON object "Records". -Note: When `expand_event_list_from_field` parameter is given in the config, s3 +Note: When `expand_event_list_from_field` parameter is given in the config, aws-s3 input will assume the logs are in JSON format and decode them as JSON. Content type will not be checked. If a file has "application/json" content-type, `expand_event_list_from_field` @@ -132,7 +132,7 @@ is 0 seconds. The maximum is 12 hours. [float] ==== `aws credentials` -In order to make AWS API calls, `s3` input requires AWS credentials.Please see +In order to make AWS API calls, `aws-s3` input requires AWS credentials.Please see <> for more details. [float] @@ -170,7 +170,7 @@ During this time, Filebeat processes and deletes the message. However, if Filebeat fails before deleting the message and your system doesn't call the DeleteMessage action for that message before the visibility timeout expires, the message becomes visible to other {beatname_uc} instances, and the message is -received again. By default, the visibility timeout is set to 5 minutes for s3 +received again. By default, the visibility timeout is set to 5 minutes for aws-s3 input in {beatname_uc}. 5 minutes is sufficient time for {beatname_uc} to read SQS messages and process related s3 log files. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index cec925422c7d..0fd068e19637 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -604,7 +604,7 @@ filebeat.modules: umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input @@ -2711,21 +2711,21 @@ filebeat.inputs: # Path to a JSON file containing the credentials and key used to subscribe. credentials_file: ${path.config}/my-pubsub-subscriber-credentials.json -#------------------------------ S3 input -------------------------------- +#------------------------------ AWS S3 input -------------------------------- # Beta: Config options for AWS S3 input -#- type: s3 +#- type: aws-s3 #enabled: false # AWS Credentials # If access_key_id and secret_access_key are configured, then use them to make api calls. - # If not, s3 input will load default AWS config or load with given profile name. + # If not, aws-s3 input will load default AWS config or load with given profile name. #access_key_id: '${AWS_ACCESS_KEY_ID:""}' #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' #session_token: '${AWS_SESSION_TOKEN:"”}' - #credential_profile_name: test-s3-input + #credential_profile_name: test-aws-s3-input # Queue url (required) to receive queue messages from - #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-s3-logs-queue" + #queue_url: "https://sqs.us-east-1.amazonaws.com/1234/test-aws-s3-logs-queue" # The duration (in seconds) that the received messages are hidden from subsequent # retrieve requests after being retrieved by a ReceiveMessage request. diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index ec0c0b6b70dc..f760be10844a 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -9,10 +9,10 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" - _ "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/aws" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/azure" diff --git a/x-pack/filebeat/input/s3/_meta/fields.yml b/x-pack/filebeat/input/awss3/_meta/fields.yml similarity index 100% rename from x-pack/filebeat/input/s3/_meta/fields.yml rename to x-pack/filebeat/input/awss3/_meta/fields.yml diff --git a/x-pack/filebeat/input/s3/_meta/s3-input.asciidoc b/x-pack/filebeat/input/awss3/_meta/s3-input.asciidoc similarity index 100% rename from x-pack/filebeat/input/s3/_meta/s3-input.asciidoc rename to x-pack/filebeat/input/awss3/_meta/s3-input.asciidoc diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/awss3/collector.go similarity index 99% rename from x-pack/filebeat/input/s3/collector.go rename to x-pack/filebeat/input/awss3/collector.go index 6767e6732a72..806bead57b26 100644 --- a/x-pack/filebeat/input/s3/collector.go +++ b/x-pack/filebeat/input/awss3/collector.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "bufio" diff --git a/x-pack/filebeat/input/s3/collector_test.go b/x-pack/filebeat/input/awss3/collector_test.go similarity index 99% rename from x-pack/filebeat/input/s3/collector_test.go rename to x-pack/filebeat/input/awss3/collector_test.go index b039e2d06b94..fa613e29df89 100644 --- a/x-pack/filebeat/input/s3/collector_test.go +++ b/x-pack/filebeat/input/awss3/collector_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "bufio" diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/awss3/config.go similarity index 99% rename from x-pack/filebeat/input/s3/config.go rename to x-pack/filebeat/input/awss3/config.go index 6dc0746ce5ff..c40a493b8c79 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "fmt" diff --git a/x-pack/filebeat/input/s3/fields.go b/x-pack/filebeat/input/awss3/fields.go similarity index 73% rename from x-pack/filebeat/input/s3/fields.go rename to x-pack/filebeat/input/awss3/fields.go index 3c373aeaa118..c507150f8e4b 100644 --- a/x-pack/filebeat/input/s3/fields.go +++ b/x-pack/filebeat/input/awss3/fields.go @@ -4,20 +4,20 @@ // Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. -package s3 +package awss3 import ( "github.com/elastic/beats/v7/libbeat/asset" ) func init() { - if err := asset.SetFields("filebeat", "s3", asset.ModuleFieldsPri, AssetS3); err != nil { + if err := asset.SetFields("filebeat", "awss3", asset.ModuleFieldsPri, AssetAwss3); err != nil { panic(err) } } -// AssetS3 returns asset data. -// This is the base64 encoded gzipped contents of input/s3. -func AssetS3() string { +// AssetAwss3 returns asset data. +// This is the base64 encoded gzipped contents of input/awss3. +func AssetAwss3() string { return "eJykjjGugzAQRHufYkQPjTsX/wi/+QdABg8fB4ORvSTi9pGBJlKkFNlipZ3dnTc1Ju4GWStAvAQaVFlXCnDMffKr+LgY/CgA+NMYPIPLGFKckTX8sm7SKCAx0GYadBSrcN2Z463GYuey2vqJ0pbh0AHZV5qS4BGTu7Q33FK/dibiABlZcpxekNGW5jNC/EeiJM873ZGveYHH7sZe2on71+zT6gP7GQAA//+k2GkG" } diff --git a/x-pack/filebeat/input/s3/ftest/sample1.txt b/x-pack/filebeat/input/awss3/ftest/sample1.txt similarity index 100% rename from x-pack/filebeat/input/s3/ftest/sample1.txt rename to x-pack/filebeat/input/awss3/ftest/sample1.txt diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/awss3/input.go similarity index 98% rename from x-pack/filebeat/input/s3/input.go rename to x-pack/filebeat/input/awss3/input.go index a3f19f66327c..584d306b1c77 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package s3 +package awss3 import ( "context" @@ -20,7 +20,7 @@ import ( "github.com/elastic/go-concert/ctxtool" ) -const inputName = "s3" +const inputName = "aws-s3" func Plugin() v2.Plugin { return v2.Plugin{ diff --git a/x-pack/filebeat/input/s3/s3_integration_test.go b/x-pack/filebeat/input/awss3/s3_integration_test.go similarity index 99% rename from x-pack/filebeat/input/s3/s3_integration_test.go rename to x-pack/filebeat/input/awss3/s3_integration_test.go index cadb5da035bf..f2eca42787e0 100644 --- a/x-pack/filebeat/input/s3/s3_integration_test.go +++ b/x-pack/filebeat/input/awss3/s3_integration_test.go @@ -5,7 +5,7 @@ // +build integration // +build aws -package s3 +package awss3 import ( "context" diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 4779b452f1d8..a3381cb42d0b 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -10,11 +10,11 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/filebeat/input/awss3" "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" - "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) func Init(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { @@ -30,6 +30,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 http_endpoint.Plugin(), httpjson.Plugin(log, store), o365audit.Plugin(log, store), - s3.Plugin(), + awss3.Plugin(), } } diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml similarity index 99% rename from x-pack/filebeat/module/aws/cloudtrail/config/s3.yml rename to x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml index 16b8bc5eec2c..4cc64e9e561f 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} file_selectors: {{ if .process_cloudtrail_logs }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 03c7acf1336d..bad63e1224b6 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/elb/config/s3.yml rename to x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml index bec03a2090c6..db50bdc4362c 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index 5d9931b2e40a..ca3a74dadd37 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/s3access/config/s3.yml rename to x-pack/filebeat/module/aws/ec2/config/aws-s3.yml index bec03a2090c6..db50bdc4362c 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index 5d9931b2e40a..ca3a74dadd37 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/cloudwatch/config/s3.yml rename to x-pack/filebeat/module/aws/elb/config/aws-s3.yml index bec03a2090c6..db50bdc4362c 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index dc95f6abb7ec..54fa469d7015 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml similarity index 98% rename from x-pack/filebeat/module/aws/ec2/config/s3.yml rename to x-pack/filebeat/module/aws/s3access/config/aws-s3.yml index bec03a2090c6..db50bdc4362c 100644 --- a/x-pack/filebeat/module/aws/ec2/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml @@ -1,4 +1,4 @@ -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index 5d9931b2e40a..ca3a74dadd37 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 243496f9cc6e..1752158d25e6 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "s3" }} +{{ if eq .input "aws-s3" }} -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} {{ if .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index 19f40c7a3f71..d084692d5c45 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -2,7 +2,7 @@ module_version: 1.0 var: - name: input - default: s3 + default: aws-s3 - name: queue_url - name: shared_credential_file - name: credential_profile_name diff --git a/x-pack/filebeat/module/cisco/_meta/config.yml b/x-pack/filebeat/module/cisco/_meta/config.yml index b0fb55ed7cb6..77b3658c42b1 100644 --- a/x-pack/filebeat/module/cisco/_meta/config.yml +++ b/x-pack/filebeat/module/cisco/_meta/config.yml @@ -109,7 +109,7 @@ umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index d0d49ca2fed6..c6bdd1854f83 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -383,7 +383,7 @@ will be found under `rsa.raw`. The default is false. The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. -To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-s3.html[S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. +To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-aws-s3.html[AWS S3 input documentation] to setup the necessary Amazon SQS queue. Retrieving logs from a Cisco-managed S3 bucket is not currently supported. This fileset supports all 4 log types: - Proxy @@ -404,7 +404,7 @@ Example config: - module: cisco umbrella: enabled: true - var.input: s3 + var.input: aws-s3 var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue var.access_key_id: 123456 var.secret_access_key: PASSWORD diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml index a0dfd07c90ab..d4b26c49ce8a 100644 --- a/x-pack/filebeat/module/cisco/umbrella/config/input.yml +++ b/x-pack/filebeat/module/cisco/umbrella/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "s3" }} +{{ if eq .input "aws-s3" }} -type: s3 +type: aws-s3 queue_url: {{ .queue_url }} access_key_id: {{ .access_key_id }} secret_access_key: {{ .secret_access_key }} diff --git a/x-pack/filebeat/modules.d/cisco.yml.disabled b/x-pack/filebeat/modules.d/cisco.yml.disabled index d181f01abd30..fedb2c03d093 100644 --- a/x-pack/filebeat/modules.d/cisco.yml.disabled +++ b/x-pack/filebeat/modules.d/cisco.yml.disabled @@ -112,7 +112,7 @@ umbrella: enabled: true - #var.input: s3 + #var.input: aws-s3 # AWS SQS queue url #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue # Access ID to authenticate with the S3 input From 29b440f7580653b4042ea26b3847288d31733e71 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 20 Jan 2021 13:49:18 +0200 Subject: [PATCH 06/35] Add kubernetes.volume.fs.used.pct field (#23564) * Add kubernetes.volume.fs.used.pct field Signed-off-by: chrismark * Add changelog entry Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 12 ++++++++++++ metricbeat/module/kubernetes/fields.go | 2 +- metricbeat/module/kubernetes/volume/_meta/fields.yml | 5 +++++ metricbeat/module/kubernetes/volume/data.go | 3 +++ metricbeat/module/kubernetes/volume/volume_test.go | 1 + 6 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8026c0a6ac92..1d9bacfa6bb1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -580,6 +580,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Honor kube event resysncs to handle missed watch events {pull}22668[22668] - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] +- Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] *Auditbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b47f7d09ca80..b430fffdae98 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -30873,6 +30873,18 @@ format: bytes -- +*`kubernetes.volume.fs.used.pct`*:: ++ +-- +Percentage of used storage + + +type: scaled_float + +format: percent + +-- + *`kubernetes.volume.fs.inodes.used`*:: + diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index bc7f92ad298a..aec239344fdc 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKubernetes returns asset data. // This is the base64 encoded gzipped contents of module/kubernetes. func AssetKubernetes() string { - return "eJzsXU9z47aSv8+nQM1psuXosLW1hzlsVeK8t8+VzDyv7ckctrYUmGxJiEmAAUB79D79FsB/IAmAoAjJHls6pDK21f1Dd6PR6AYaP6IH2H9ED+U9cAoSxDuEJJEZfETvf21/+P4dQimIhJNCEkY/ov96hxBC3R+gHCQnifo2hwywgI9oi98hJEBKQrfiI/rf90Jk7y/Q+52Uxfv/U7/bMS7XCaMbsv2INjgT8A6hDYEsFR81gx8RxTkM4KmP3BeKA2dlUf/EAk99ruiG8RyrHyNMUyQklkRIkgjENqhgqUA5pngLKbrfG3xWNQUTjYkIF0QAfwTe/sYGygNsIL+frq9QRdAQZfPpi7T5DKGZ8Dj8VYKQqyQjQGXvTxqcD7B/Yjwd/M6DVn0uNT0E3yAplV4bRsKLgoNgJU8gHo6bijKkyEp7CECU98fE4CI/gpGwIj4ApMmiD0lWCgn8QjMVBU7gopXOD15cj8Dv48H6x93dNRqRHFkmSyOKQvMckRzzpBKoXCtG8dVQY9As0IjFEEvK92te0ngwvoLcAUdyBw0PVAoQKOV7NGQ0BPNA6JDbAiS/Epoq71pTn1BJXjAa10c1JNEO0zRTXsoQihfN0HcvRKKcuiaJNqzRTICbeAQuCItoGjXBFsV4mEMIWnK9xW0hhGaS2AgPmecgdyyiPeqJaSE6GjQTEc2wHfGQasO24CwBIawcbYZoW+9NeklRrgQko983NFNW3mdDvzcayOX1FyQgYTQdIus45ZAzvlfLOkmBytX9vovMxnwzRreWX1Zx2Ufk+nIP1c/qjxChqOFZY5iC+Ei4LHF2SoQ1yymAm1SsWAF0lbBy5P0mofVYfy7ze+DK4yqCaEMyaP+AcbcahcRcQhrBaG4rg0GC0AS0i6mNu+FhnQBqIxDN+tt1teQ62l+VYlUAT4BKksHq35wjZPd/QmJTQPWL9Rw5NHO+AYFyknBWTyfUwXHrxDYMUeYL9ePHlZR5mWFJHgHZWPmgLTfeBpqmpFeohv4kEEH+BdXMjqnpOaAVgllqNSD7tBrDIfUwzlSxAfMYGlbkPRhEwaiAZ1VvBWGOfsegj69gE2WwhsdAY6i4hmInNQ7649tUMzDrSlOlQVY+/k7ejqW2SXwgLJAlyzIYcrwgL2K0YM3dmMwyLIEm+0Ms2aYt0RC8UCaqEFT/JlXgZK5Jk5DimVCLic4XzH2ZPIA86ZJTs0Y7IiTbcpyjCoQbbGgoMQdFQ7PSZKjyjhM5dFioGQhXPwwD8wx67FCHazIpOVd+bLnsrugmI9udDDB1Rre8pJTQbdStSuc/E71oqW+jmpE/qwwySVeV3KN48i7pX2tTICw1Fyt7XKZEruDRpYi57DU9pOnZx1sx5KCgQRqRZ0NyyLxba6jEhC6rcRjSbelFKXHoneVaktyeyk2xHP5iImFzqwiiEUEjvRK8ik9lKK+/oFLgLVgE4Rq2CUV/1zkPbYB8VHuDZNxGeJr4FAOTicUpD9k4t7XNZ0LC5ueyNTsl90vGoRY+xdS5ZPXwYsqUYFywAyAHwq0MA9IJli0wlsKqsK5LHS6R4AzS9SZj2PWHzbaj3unEGIOSLxYINzTVv9lGp4YkkzjT2BHOMpZgie8zUN/zDjYjOZHf32hT2BAKaQW/zcB3rvCD+olTIohsUEn1dyG1F/Eytg3PIU+M6je2VaH4hs10SPgRkwzbzX+5U3LthlHY3JvaVKNwbWv5tINFCS5wQuReBcB26q1frf/yLcinsuZw2SiH9xbkoh17uFiI8gfumsWyVd4ex6Ooi9mdtoNutjgHZBRFOPjDj3i4FKsQSA7rPAYkbSAWSP2aVrRU0ltx2kM7nCjMHS+4fmkiqQThHPALjzM/GehnhpoOC0AvPtoMGfOCgLM2CHfMaUqIj44uoFc2S25ub/1zpIH8xPgDoVsB7uTY65DI12qgSIAMk0yBt7DBZWZJMM6rYNsxdRktxQg5OLXrJ/6T8ZMh0tycuNpZxJjcRDwD9Db2GTeMSX3OReyFhHz2luOthD52OZkh+XlvZpdRHYs/3x7tJPuOL5Ydh5n95yzLgFcXJBZVAS5bYvV1izg1gGc5pnrKk+unPgp74iOw6r/x2H3GOYSdtP4XoxH5XtENx0LyMpElhzHx84HfajjnA7/nA7/nA78Bwzgf+LUDOR/4DcZ4PvB7PvB7PvC7/MCvJcqcewT4ifGHv0oo7RHnIUufAg0q4KyO5S1fzn+rCLbn7+rF3BdLlHRDKBG7KOHEl5ZYCGucpjFs+GujF0VwwpBTKOQuKk9NcXL6SE6izNeOr3nKWVO3b8xYCqtEbdgTyez760MMFx5JoiOJmDGwLmM0lH0GuwOcyV2Ms+Md85YqsqeCjnFu38+pwuMoXYWzu+4VltyDbH0S4BT4ioh1joV05GTuGcsADwO9qYvtu+5mu9Y1EWjA490QjT7R+m7IfkbC6m4HZnuO6oRsk7MCtQ7pudH+Ru6wRJgD2gIFjmXVT6Q5T1z71R4HQtXGVgn312F3EzQjGeY2MIeuvdK+rJZXxQVxSBhPRSX31vgkyaH6WYG5JEmZYV4JAe2wQCzRh9RTC0L9TYnzwoJy7Ex8ab8N4UKua1bU0dNj/gHguwagGqfmgToe6mdDqzIvhBwdkGIxgafLhYhRVa7CIOGbDLeGTxWd2hIg7RoIkEegFnEkrNivJbMh6NY0LAZbPXfqzYvuRlMKBdda4bAxx4Hc7/ZFW3L3c7TkIV1G7+eoy/hNbwsOBeOyam5BhEUXvgl01K4bG85y9LQjyU4Lp/INRHSe0Z4bipp5/qzWCUUYMRqKxci54xRLvFxjn2pKCAvBEqJXhScid9455NOb3YXOj8haO+AwUgjyOayAylLPaWkGhFH/TOkANXpZx60M/HdNtjaJTWcM9ug3flkiiKdu2hSXsSaJSDMJqgnwhKdmY1M9WUfvRfN73YvGFIi/WFOSiAWwL5T8VQLSJQWyISqsZAYQS0qpdeOQbdYZoQ8Rwdz8pvw4B6HQ1H2KXMsIoY8se4R0bcF4LO/U8LTJxeencEHiW85P11dtJ6PaejzqitvSSvF+qNtaTTCO6zxMh+Vherz52lCeIfq4E/bL1S8TvM2kxZI9n3FVUe8zz7cUz7cUHZ/4txR1xPq9X1A831Sw/835psLoE++mwvlA+gjy+UC6C/r5QPrEgXQKUllPNN/Nv71yE7yBBMijzve7aLVVCc5tdc1g1KGIvrk4tXmk166UO46pyImUL0kvd1a9tEWN8y2Q5hMoz7+fL4DMFtH57of5GYnnbVz7MI4vOK6cD2GdpmNAh+ul9AroELn6BbRxTkmdWZ5DfDjJVVx4pP4P7vVhmsEUExQ401F4GiVkxqN56ZarXMfB81cQFLiKoLctyIB1Bs1xe29SiPbVqN3K9u5wLcl2Fyz9LpPd571q8znvVc3P96WU73Cv+iaqTC+mqjIC9hLb8cxp+/imWj2qxbXtviOG7XfqHo+MAmIc5YyD+cc1YUUCc5jqBBm56nYuL42Av8iZd26EFW86HtwN660kFHsTxj3oQSFy/forkZVgnkb1SPdm49WXqyuRtA0clEz0zdUJwRR4C+sjVkQrWMH12fVp8Lirs0b7kG/7JTt+4z6TprX8weH28L6lXc7BtzpcHXi6XHQa5QaHrfOOcQdg2CJnCZcROeeth6VS69MzTGfUeWbOxavezs9639c7DQL7zfTj2KluM56rjH4fdkifmQE0f5eZyMi8/WXaaoi/u4wH0oLOMr28n+MieLhhzOkp086fYUeZw6x6di8ZbwOKkD4yUbrI+ODbekLEQuRtPeEDtcw4gzvHDCGENhYJ12o42ImOIh6obg0udS3hnWI88GLr0tIjxrjdMK9DTFxFhvWG8YE9piqDe8L4AC5VpqcbzNCEItqNLTicavxyyK34wJYv7XK4p0nQouRl+lDeQxWm18H6nibWHPnE0lZmIAJXhmnx3+5pcq3g3Ciyg2cA2ab9wdSDjm50y8zDiS/gaUA3JufzgDH9jBP61PuAg2powfUf54Ruo6n9c0UaGbRnPQEZCHFh7OoFOcMAJlCexBr8g3GbxChrIJIdpGW2rH2vkTlo6Z3TBmMeryxtMLrKeiCbqca8RmRSZlEGdltbKcJSQl7IMemGZ+sNIrJVk9VG95yOOadjpiCd0zHndMxMROd0zDkdc07HnNMx53SMFYO3M2XF39aX0gthTk/K0V5s2AnysEUS/h1Ovy39G02RZAhoagzGviwFwl6SlpiBxjMBh4iWzQg7Jt9MLFi6KjiobYpCoBvZ5pP6nEZyzVLU0UU13XkglmjHzt+jCAeGZfpwoJhSSL1vjMG9IeXl2Rpgba+njHdvLXNnemEdIV4W4tpABK2fIxwLM8iuSftuyLg9RfhuyOWwWzjdE50x7uIc3INrJJ7LFhexNwsUEssy3u31YoeF+xilfQDDQfiOcbfD0YzQh7o58wV6wkTq/5HAc0Kx//lTwKn7gr290XUgyg6hZmKXby+AVBty9+E0QiVsRx25DwBT8ZlsWj9q8GuCWaS/r5WG0IcW1aVuMKqUdsmx2P3GWPEzTh7YZnOB/sa5vlh3XWbZBWr/t/79WLXqw3irfeWBPlyyvMhAQnrRSeISU8rkTUk1C8Yv0D//+elXkmWQ/lAPf2WdKHOuzUy+AaHPZbsui1R0XcexZ6n98vqLbrgmKpYevTcx/kkg1ewgRXaGfTn5LtZ4VgaFq+CQKFfwEf3n6j9iIG+xBArUh30a3sToDpb6SRvAVUo8/ptuUyKoz71XNwom20Q0Cnx+3J3amksNrkvFCWf0T3YfK6SpqEUJaEbFqPCQBl3WOEY0hlXSpQysdIyAse7eb58ZIXw6EqhgGRlQam+jJCpoXvDsUZdiqUipPZHoHpMfGYkRd4q1KEUBNB3d6PeFRj3uZnalMSGido42up3l6l7mlqqHZxPS37oXLNkhMap7NBCesLB2TG+9FBZy3VhANBxK6PrthwYGL6l9gsC3I7FXlCfZp4DTjFA35ymb+6Um0LLGGwm8nVIaScL0qylcBYEbTDJDEyH/4/+ne6uXYsgZ7V9iWnKI4hdN71ZfEjqxZ+wWpyIjCQ7ftk0sONbR1UwOvNo9fafu8LDmrveoTdetpBELKoB3A3FCTEEQ7mmTtQxgTb33ZtcseP5NaizpVbvUmdBKejr1GrymYHaKLTK2zxe+ImWEQh3BKHO+wJaWOcELrHf6GkgrLrYExAnciIGjVRqhGzbTi0xN0UXZkV86jJ25NdO2Rf1BFJAsuTscC+N4Giyan7Fg2eanG1iRWt+siQ6q4jMGZDYDiOQcYj43EDMV618/FqXydL97M7+JPkhewgXa4EzoZhglfaDsibrnTUnr2NBrpItSsRplj4/PGcbM7xmdCI6XUmsfHTD7HvjzaU2HvAlQhy/dLaa2F9/p3hkwZP5cSZvPrjYUU9mmVjHPirxG6++jaBRaj6I73dDjWKZp6qZglpc5Rgo5Khzdz2TYz7MVMHBBhAQqH1lW5rGWq44squg2a1f19p/6yx+Vm4Qfnzvx93sFT5FwlGR8syYsb1HzsDZV9dU05w6iKlfiJGE81Y+FMUMnjmiAcbyFdZLhUbORYO63FRGkibQpwJE9oZDEissukwyT/GjGmWT4BZvo9e+XHvushrDoOcSfCU0hbYThZlUXCda11SyYETddba6ZXvFnhZKbJmCnjZMEhFjnw1s/Mzj8pEkgRcLO44jz6/r3y5VrOtmXz0VzJlIDWGJ/mHT04/AkgEJ2dW1ltmNCro/DUZF2sZ25xZrHuN4KHZYvPeLBmQHM+uTMTXNy5hqoWpJWq9WhB2Ziolu2q2zqDe4MQ0ysLTcb3osx2mHebVlJwnABTapeRCpKHDExaEJ1ZyhfUn2hl3S+qf7xfGWFw3E9Wz0hABu71+3MjiW0+r1k/QppzQnd7/Va3YHTZzg5y4aXmlGvSnwPPu8SS4qbMsv2DbdJaRqHCfXt3L9K1nsBfZlrMWhGcS7Hq/Xf1Fj/R2OdqvgPpTQHQcWB0A3jOaToww7zVC9QAtIffLel42w7+gN1HoxR9A5lYY6wmjnqqxfoDzXUP9RY/1CD/cOxflgGfsD4NDktysr8cFFkBASSbLw99f/TvZ1V7oAksbIrNbVnPzR1W+PwJE+yUkjgriA8gMcVlcApztDVdWvy9fjtLOFb9YVFO+JmZA0x9MvnW/cUaFkePswRQ8feImM4Xd/jDNNkkVh/YzhFP9d0WoNyMF0yxZuBjWi0O0K65WofvsRENAUX+oaB2rItsYmGzT9sdAbrjt3jT7xu0ohK01DOsPcFc3cJmzKLF9g3FKNF9j4hTGWGxoHL3c4QiSQ5CInzAn0AtUBX6+BtPYJh9HeCrUZPeG0MddBu48jxqdHfqAlPezGfS4joGbYdowMSPoANuC4AP7aejVDfCF5elrpbJRtgX4aaG+UGABvkUYdp1GVez8yqPnso1UPjDqgKzh6JIIyO9o+zi0UdpS6wMlG4agC6FLO2nC2fFXtrKvUJ9ao1yJ7inCRY7UnrBaSuSNhLV3Xd457oxOKiNP4nllaHh9Pqne1ONoRuEaYpqrnEX/J7ap9Y+PXDb7Gsv3pFznjnIsrCb7nwOksTltef2isq7qsZJ35U7k28bZUwfvxHK0cNFsdsJh9bDLzUi2ppXzIOtcgppo5WFAOUL+WZryMdajo/49QD/fqfXbm5vQ0TRf1Yzet/m+fr6FWeCckUeAtHfPGlu10X/ArNyRBNv0MT9ShZ//zYswXoY6kYJ8es3EZPcx/uoT3Pcb+mWTh6bH/6se3wSu0rFVHAM9rdOu/Z0r9S8dgfyB4OwPJqv4l+iWwm3y2PETd+qUbpGIThkjj4g/oYYP7OAULA2JvExEZTBX41nP8PAAD//4Df+K8=" + return "eJzsXU9z47aSv8+nQM1psuXosLW1hzlsVeK8t8+VzDyv7ckctrYUmGxJiEmAAUB79D79FsB/IAmAoAjJHls6pDK21f1Dd6PR6AYaP6IH2H9ED+U9cAoSxDuEJJEZfETvf21/+P4dQimIhJNCEkY/ov96hxBC3R+gHCQnifo2hwywgI9oi98hJEBKQrfiI/rf90Jk7y/Q+52Uxfv/U7/bMS7XCaMbsv2INjgT8A6hDYEsFR81gx8RxTkM4KmP3BeKA2dlUf/EAk99ruiG8RyrHyNMUyQklkRIkgjENqhgqUA5pngLKbrfG3xWNQUTjYkIF0QAfwTe/sYGygNsIL+frq9QRdAQZfPpi7T5DKGZ8Dj8VYKQqyQjQGXvTxqcD7B/Yjwd/M6DVn0uNT0E3yAplV4bRsKLgoNgJU8gHo6bijKkyEp7CECU98fE4CI/gpGwIj4ApMmiD0lWCgn8QjMVBU7gopXOD15cj8Dv48H6x93dNRqRHFkmSyOKQvMckRzzpBKoXCtG8dVQY9As0IjFEEvK92te0ngwvoLcAUdyBw0PVAoQKOV7NGQ0BPNA6JDbAiS/Epoq71pTn1BJXjAa10c1JNEO0zRTXsoQihfN0HcvRKKcuiaJNqzRTICbeAQuCItoGjXBFsV4mEMIWnK9xW0hhGaS2AgPmecgdyyiPeqJaSE6GjQTEc2wHfGQasO24CwBIawcbYZoW+9NeklRrgQko983NFNW3mdDvzcayOX1FyQgYTQdIus45ZAzvlfLOkmBytX9vovMxnwzRreWX1Zx2Ufk+nIP1c/qjxChqOFZY5iC+Ei4LHF2SoQ1yymAm1SsWAF0lbBy5P0mofVYfy7ze+DK4yqCaEMyaP+AcbcahcRcQhrBaG4rg0GC0AS0i6mNu+FhnQBqIxDN+tt1teQ62l+VYlUAT4BKksHq35wjZPd/QmJTQPWL9Rw5NHO+AYFyknBWTyfUwXHrxDYMUeYL9ePHlZR5mWFJHgHZWPmgLTfeBpqmpFeohv4kEEH+BdXMjqnpOaAVgllqNSD7tBrDIfUwzlSxAfMYGlbkPRhEwaiAZ1VvBWGOfsegj69gE2WwhsdAY6i4hmInNQ7649tUMzDrSlOlQVY+/k7ejqW2SXwgLJAlyzIYcrwgL2K0YM3dmMwyLIEm+0Ms2aYt0RC8UCaqEFT/JlXgZK5Jk5DimVCLic4XzH2ZPIA86ZJTs0Y7IiTbcpyjCoQbbGgoMQdFQ7PSZKjyjhM5dFioGQhXPwwD8wx67FCHazIpOVd+bLnsrugmI9udDDB1Rre8pJTQbdStSuc/E71oqW+jmpE/qwwySVeV3KN48i7pX2tTICw1Fyt7XKZEruDRpYi57DU9pOnZx1sx5KCgQRqRZ0NyyLxba6jEhC6rcRjSbelFKXHoneVaktyeyk2xHP5iImFzqwiiEUEjvRK8ik9lKK+/oFLgLVgE4Rq2CUV/1zkPbYB8VHuDZNxGeJr4FAOTicUpD9k4t7XNZ0LC5ueyNTsl90vGoRY+xdS5ZPXwYsqUYFywAyAHwq0MA9IJli0wlsKqsK5LHS6R4AzS9SZj2PWHzbaj3unEGIOSLxYINzTVv9lGp4YkkzjT2BHOMpZgie8zUN/zDjYjOZHf32hT2BAKaQW/zcB3rvCD+olTIohsUEn1dyG1F/Eytg3PIU+M6je2VaH4hs10SPgRkwzbzX+5U3LthlHY3JvaVKNwbWv5tINFCS5wQuReBcB26q1frf/yLcinsuZw2SiH9xbkoh17uFiI8gfumsWyVd4ex6Ooi9mdtoNutjgHZBRFOPjDj3i4FKsQSA7rPAYkbSAWSP2aVrRU0ltx2kM7nCjMHS+4fmkiqQThHPALjzM/GehnhpoOC0AvPtoMGfOCgLM2CHfMaUqIj44uoFc2S25ub/1zpIH8xPgDoVsB7uTY65DI12qgSIAMk0yBt7DBZWZJMM6rYNsxdRktxQg5OLXrJ/6T8ZMh0tycuNpZxJjcRDwD9Db2GTeMSX3OReyFhHz2luOthD52OZkh+XlvZpdRHYs/3x7tJPuOL5Ydh5n95yzLgFcXJBZVAS5bYvV1izg1gGc5pnrKk+unPgp74iOw6r/x2H3GOYSdtP4XoxH5XtENx0LyMpElhzHx84HfajjnA7/nA7/nA78Bwzgf+LUDOR/4DcZ4PvB7PvB7PvC7/MCvJcqcewT4ifGHv0oo7RHnIUufAg0q4KyO5S1fzn+rCLbn7+rF3BdLlHRDKBG7KOHEl5ZYCGucpjFs+GujF0VwwpBTKOQuKk9NcXL6SE6izNeOr3nKWVO3b8xYCqtEbdgTyez760MMFx5JoiOJmDGwLmM0lH0GuwOcyV2Ms+Md85YqsqeCjnFu38+pwuMoXYWzu+4VltyDbH0S4BT4ioh1joV05GTuGcsADwO9qYvtu+5mu9Y1EWjA490QjT7R+m7IfkbC6m4HZnuO6oRsk7MCtQ7pudH+Ru6wRJgD2gIFjmXVT6Q5T1z71R4HQtXGVgn312F3EzQjGeY2MIeuvdK+rJZXxQVxSBhPRSX31vgkyaH6WYG5JEmZYV4JAe2wQCzRh9RTC0L9TYnzwoJy7Ex8ab8N4UKua1bU0dNj/gHguwagGqfmgToe6mdDqzIvhBwdkGIxgafLhYhRVa7CIOGbDLeGTxWd2hIg7RoIkEegFnEkrNivJbMh6NY0LAZbPXfqzYvuRlMKBdda4bAxx4Hc7/ZFW3L3c7TkIV1G7+eoy/hNbwsOBeOyam5BhEUXvgl01K4bG85y9LQjyU4Lp/INRHSe0Z4bipp5/qzWCUUYMRqKxci54xRLvFxjn2pKCAvBEqJXhScid9455NOb3YXOj8haO+AwUgjyOayAylLPaWkGhFH/TOkANXpZx60M/HdNtjaJTWcM9ug3flkiiKdu2hSXsSaJSDMJqgnwhKdmY1M9WUfvRfN73YvGFIi/WFOSiAWwL5T8VQLSJQWyISqsZAYQS0qpdeOQbdYZoQ8Rwdz8pvw4B6HQ1H2KXMsIoY8se4R0bcF4LO/U8LTJxeencEHiW85P11dtJ6PaejzqitvSSvF+qNtaTTCO6zxMh+Vherz52lCeIfq4E/bL1S8TvM2kxZI9n3FVUe8zz7cUz7cUHZ/4txR1xPq9X1A831Sw/835psLoE++mwvlA+gjy+UC6C/r5QPrEgXQKUllPNN/Nv71yE7yBBMijzve7aLVVCc5tdc1g1KGIvrk4tXmk166UO46pyImUL0kvd1a9tEWN8y2Q5hMoz7+fL4DMFtH57of5GYnnbVz7MI4vOK6cD2GdpmNAh+ul9AroELn6BbRxTkmdWZ5DfDjJVVx4pP4P7vVhmsEUExQ401F4GiVkxqN56ZarXMfB81cQFLiKoLctyIB1Bs1xe29SiPbVqN3K9u5wLcl2Fyz9LpPd571q8znvVc3P96WU73Cv+iaqTC+mqjIC9hLb8cxp+/imWj2qxbXtviOG7XfqHo+MAmIc5YyD+cc1YUUCc5jqBBm56nYuL42Av8iZd26EFW86HtwN660kFHsTxj3oQSFy/forkZVgnkb1SPdm49WXqyuRtA0clEz0zdUJwRR4C+sjVkQrWMH12fVp8Lirs0b7kG/7JTt+4z6TprX8weH28L6lXc7BtzpcHXi6XHQa5QaHrfOOcQdg2CJnCZcROeeth6VS69MzTGfUeWbOxavezs9639c7DQL7zfTj2KluM56rjH4fdkifmQE0f5eZyMi8/WXaaoi/u4wH0oLOMr28n+MieLhhzOkp086fYUeZw6x6di8ZbwOKkD4yUbrI+ODbekLEQuRtPeEDtcw4gzvHDCGENhYJ12o42ImOIh6obg0udS3hnWI88GLr0tIjxrjdMK9DTFxFhvWG8YE9piqDe8L4AC5VpqcbzNCEItqNLTicavxyyK34wJYv7XK4p0nQouRl+lDeQxWm18H6nibWHPnE0lZmIAJXhmnx3+5pcq3g3Ciyg2cA2ab9wdSDjm50y8zDiS/gaUA3JufzgDH9jBP61PuAg2powfUf54Ruo6n9c0UaGbRnPQEZCHFh7OoFOcMAJlCexBr8g3GbxChrIJIdpGW2rH2vkTlo6Z3TBmMeryxtMLrKeiCbqca8RmRSZlEGdltbKcJSQl7IMemGZ+sNIrJVk9VG95yOOadjpiCd0zHndMxMROd0zDkdc07HnNMx53SMFYO3M2XF39aX0gthTk/K0V5s2AnysEUS/h1Ovy39G02RZAhoagzGviwFwl6SlpiBxjMBh4iWzQg7Jt9MLFi6KjiobYpCoBvZ5pP6nEZyzVLU0UU13XkglmjHzt+jCAeGZfpwoJhSSL1vjMG9IeXl2Rpgba+njHdvLXNnemEdIV4W4tpABK2fIxwLM8iuSftuyLg9RfhuyOWwWzjdE50x7uIc3INrJJ7LFhexNwsUEssy3u31YoeF+xilfQDDQfiOcbfD0YzQh7o58wV6wkTq/5HAc0Kx//lTwKn7gr290XUgyg6hZmKXby+AVBty9+E0QiVsRx25DwBT8ZlsWj9q8GuCWaS/r5WG0IcW1aVuMKqUdsmx2P3GWPEzTh7YZnOB/sa5vlh3XWbZBWr/t/79WLXqw3irfeWBPlyyvMhAQnrRSeISU8rkTUk1C8Yv0D//+elXkmWQ/lAPf2WdKHOuzUy+AaHPZbsui1R0XcexZ6n98vqLbrgmKpYevTcx/kkg1ewgRXaGfTn5LtZ4VgaFq+CQKFfwEf3n6j9iIG+xBArUh30a3sToDpb6SRvAVUo8/ptuUyKoz71XNwom20Q0Cnx+3J3amksNrkvFCWf0T3YfK6SpqEUJaEbFqPCQBl3WOEY0hlXSpQysdIyAse7eb58ZIXw6EqhgGRlQam+jJCpoXvDsUZdiqUipPZHoHpMfGYkRd4q1KEUBNB3d6PeFRj3uZnalMSGido42up3l6l7mlqqHZxPS37oXLNkhMap7NBCesLB2TG+9FBZy3VhANBxK6PrthwYGL6l9gsC3I7FXlCfZp4DTjFA35ymb+6Um0LLGGwm8nVIaScL0qylcBYEbTDJDEyH/4/+ne6uXYsgZ7V9iWnKI4hdN71ZfEjqxZ+wWpyIjCQ7ftk0sONbR1UwOvNo9fafu8LDmrveoTdetpBELKoB3A3FCTEEQ7mmTtQxgTb33ZtcseP5NaizpVbvUmdBKejr1GrymYHaKLTK2zxe+ImWEQh3BKHO+wJaWOcELrHf6GkgrLrYExAnciIGjVRqhGzbTi0xN0UXZkV86jJ25NdO2Rf1BFJAsuTscC+N4Giyan7Fg2eanG1iRWt+siQ6q4jMGZDYDiOQcYj43EDMV618/FqXydL97M7+JPkhewgXa4EzoZhglfaDsibrnTUnr2NBrpItSsRplj4/PGcbM7xmdCI6XUmsfHTD7HvjzaU2HvAlQhy/dLaa2F9/p3hkwZP5cSZvPrjYUU9mmVjHPirxG6++jaBRaj6I73dDjWKZp6qZglpc5Rgo5Khzdz2TYz7MVMHBBhAQqH1lW5rGWq44squg2a1f19p/6yx+Vm4Qfnzvx93sFT5FwlGR8syYsb1HzsDZV9dU05w6iKlfiJGE81Y+FMUMnjmiAcbyFdZLhUbORYO63FRGkibQpwJE9oZDEissukwyT/GjGmWT4BZvo9e+XHvushrDoOcSfCU0hbYThZlUXCda11SyYETddba6ZXvFnhZKbJmCnjZMEhFjnw1s/Mzj8pEkgRcLO44jz6/r3y5VrOtmXz0VzJlIDWGJ/mHT04/AkgEJ2dW1ltmNCro/DUZF2sZ25xZrHuN4KHZYvPeLBmQHM+uTMTXNy5hqoWpJWq9WhB2Ziolu2q2zqDe4MQ0ysLTcb3osx2mHebVlJwnABTapeRCpKHDExaEJ1ZyhfUn2hl3S+qf7xfGWFw3E9Wz0hABu71+3MjiW0+r1k/QppzQnd7/Va3YHTZzg5y4aXmlGvSnwPPu8SS4qbMsv2DbdJaRqHCfXt3L9K1nsBfZlrMWhGcS7Hq/Xf1Fj/R2OdqvgPpTQHQcWB0A3jOaToww7zVC9QAtIffLel42w7+gN1HoxR9A5lYY6wmjnqqxfoDzXUP9RY/1CD/cOxflgGfsD4NDktysr8cFFkBASSbLw99f/TvZ1V7oAksbIrNbVnPzR1W+PwJE+yUkjgriA8gMcVlcApztDVdWvy9fjtLOFb9YVFO+JmZA0x9MvnW/cUaFkePswRQ8feImM4Xd/jDNNkkVh/YzhFP9d0WoNyMF0yxZuBjWi0O0K65WofvsRENAUX+oaB2rItsYmGzT9sdAbrjt3jT7xu0ohK01DOsPcFc3cJmzKLF9g3FKNF9j4hTGWGxoHL3c4QiSQ5CInzAn0AtUBX6+BtPYJh9HeCrUZPeG0MddBu48jxqdHfqAlPezGfS4joGbYdowMSPoANuC4AP7aejVDfCF5elrpbJRtgX4aaG+UGABvkUYdp1GVez8yqPnso1UPjDqgKzh6JIIyO9o+zi0UdpS6wMlG4agC6FLO2nC2fFXtrKvUJ9ao1yJ7inCRY7UnrBaSuSNhLV3Xd457oxOKiNP4nllaHh9Pqne1ONoRuEaYpqrnEX/J7ap9Y+PXDb7Gsv3pFznjnIsrCb7nwOksTltef2isq7qsZJ35U7k28bZUwfvxHK0cNFsdsJh9bDLzUi2ppXzIOtcgppo5WFAOUL+WZryMdajo/49QD/fqfXbm5vQ0TRf1Yzet/m+fr6FWeCckUeAtHfPGlu10X/ArNyRBNv0MT9ShZ//zYswXoY6kYJ8es3EZPcx/uoT3Pcb+mWTh6bH/6se3wSu0rFVHAM9rdOu/Z0r9S8dgfyLYN4qU9YHjde66wPsOgd4tOBRPKUs/B7iUqnnx+PUb4+6VSlmMQhmfl4N+bxADzdw4QAsbe6yY2mip+reH8fwAAAP//CPgmJQ==" } diff --git a/metricbeat/module/kubernetes/volume/_meta/fields.yml b/metricbeat/module/kubernetes/volume/_meta/fields.yml index 93b6204d53eb..4eedf6014ba1 100644 --- a/metricbeat/module/kubernetes/volume/_meta/fields.yml +++ b/metricbeat/module/kubernetes/volume/_meta/fields.yml @@ -35,6 +35,11 @@ format: bytes description: > Filesystem total used in bytes + - name: pct + type: scaled_float + format: percent + description: > + Percentage of used storage - name: inodes type: group fields: diff --git a/metricbeat/module/kubernetes/volume/data.go b/metricbeat/module/kubernetes/volume/data.go index ae0f5d891f49..6170bbadbccc 100644 --- a/metricbeat/module/kubernetes/volume/data.go +++ b/metricbeat/module/kubernetes/volume/data.go @@ -67,6 +67,9 @@ func eventMapping(content []byte) ([]common.MapStr, error) { }, }, } + if volume.CapacityBytes > 0 { + volumeEvent.Put("fs.used.pct", float64(volume.UsedBytes)/float64(volume.CapacityBytes)) + } events = append(events, volumeEvent) } diff --git a/metricbeat/module/kubernetes/volume/volume_test.go b/metricbeat/module/kubernetes/volume/volume_test.go index e527a64d452c..71ca971c5e5e 100644 --- a/metricbeat/module/kubernetes/volume/volume_test.go +++ b/metricbeat/module/kubernetes/volume/volume_test.go @@ -49,6 +49,7 @@ func TestEventMapping(t *testing.T) { "fs.available.bytes": 1939689472, "fs.capacity.bytes": 1939701760, "fs.used.bytes": 12288, + "fs.used.pct": float64(12288) / float64(1939701760), "fs.inodes.used": 9, "fs.inodes.free": 473551, "fs.inodes.count": 473560, From 7149e5aecec288feac3c88422eac124b309968f9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 20 Jan 2021 13:36:31 +0100 Subject: [PATCH 07/35] Start doppler consumer loop only after a subscription is configured (#23584) Subscriptions and events are managed in the same loop in the same goroutine, so when doing black box testing of the metricset it is not possible to know if the subscription is configured before starting to dispatch events. If an event is handled before its subscription is configured, the event is dropped. This change ensures that the subscription is configured before starting the event loop. --- x-pack/metricbeat/module/cloudfoundry/v1.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/cloudfoundry/v1.go b/x-pack/metricbeat/module/cloudfoundry/v1.go index db7f6b500fa5..8ad2e1e6e2ac 100644 --- a/x-pack/metricbeat/module/cloudfoundry/v1.go +++ b/x-pack/metricbeat/module/cloudfoundry/v1.go @@ -61,11 +61,10 @@ func (m *ModuleV1) RunContainerReporter(reporter mb.PushReporterV2) { } func (m *ModuleV1) subscribe(eventType cfcommon.EventType, reporter mb.PushReporterV2) { - go m.run() - m.subscriptions <- subscription{ + go m.run(subscription{ eventType: eventType, reporter: reporter, - } + }) } func (m *ModuleV1) unsubscribe(eventType cfcommon.EventType, reporter mb.PushReporterV2) { @@ -80,8 +79,11 @@ func (m *ModuleV1) callback(event cfcommon.Event) { m.events <- event } -func (m *ModuleV1) run() { +// run ensures that the module is running with the passed subscription +func (m *ModuleV1) run(s subscription) { if !m.running.CAS(false, true) { + // Module is already running, queue subscription for current dispatcher. + m.subscriptions <- s return } defer func() { m.running.Store(false) }() @@ -91,6 +93,10 @@ func (m *ModuleV1) run() { dispatcher := newEventDispatcher(m.log) + // Ensure that the initial subscription is configured before starting the loop, + // this is specially relevant to make tests more deterministic. + dispatcher.handleSubscription(s) + for { // Handle subscriptions and events dispatching on the same // goroutine so locking is not needed. From c6af0496569b2ea95e7325e2711496b29e831405 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 20 Jan 2021 16:36:58 +0100 Subject: [PATCH 08/35] [Ingest Manager] Fix flaky TestConfigurable* unit tests (#23579) * wait to get out of configurable state * reenable serviceable test --- .../pkg/agent/operation/operator_test.go | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go index 794dbdb58050..5eb195302501 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go @@ -100,6 +100,16 @@ func TestConfigurableRun(t *testing.T) { return nil }) + // wait to finish configuring + waitFor(t, func() error { + items := operator.State() + item, ok := items[p.ID()] + if ok && item.Status == state.Configuring { + return fmt.Errorf("process still configuring") + } + return nil + }) + items := operator.State() item0, ok := items[p.ID()] if !ok || item0.Status != state.Running { @@ -379,8 +389,6 @@ func TestConfigurableStartStop(t *testing.T) { } func TestConfigurableService(t *testing.T) { - t.Skipf("flaky see https://github.com/elastic/beats/issues/20836") - p := getProgram("serviceable", "1.0") operator := getTestOperator(t, downloadPath, installPath, p) @@ -427,6 +435,16 @@ func TestConfigurableService(t *testing.T) { return nil }) + // wait to finish configuring + waitFor(t, func() error { + items := operator.State() + item, ok := items[p.ID()] + if ok && item.Status == state.Configuring { + return fmt.Errorf("process still configuring") + } + return nil + }) + items := operator.State() item0, ok := items[p.ID()] if !ok || item0.Status != state.Running { From e0881de6b633920138c799b7b5703a745eedac83 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 20 Jan 2021 16:18:23 -0500 Subject: [PATCH 09/35] [Elastic Agent] Set status Failed if configuration applying fails (#23537) * Set status to Failed if configuration applying fails. * Add changelog. * Don't cleanup paths on crash, as it will be restart. Fix ownership. --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + .../pkg/core/monitoring/beats/beats_monitor.go | 13 ++++++++----- .../elastic-agent/pkg/core/plugin/process/status.go | 3 --- x-pack/libbeat/management/fleet/manager.go | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index a68ae2d19caf..569341de1b4b 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -36,6 +36,7 @@ - Fixed fetching DBus service PID {pull}23496[23496] - Fix issue of missing log messages from filebeat monitor {pull}23514[23514] - Increase checkin grace period to 30 seconds {pull}23568[23568] +- Fix libbeat from reporting back degraded on config update {pull}23537[23537] ==== New features diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 19298aef69ca..5944afa4f0cc 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -144,7 +144,12 @@ func (b *Monitor) Cleanup(spec program.Spec, pipelineID string) error { // Prepare executes steps in order for monitoring to work correctly func (b *Monitor) Prepare(spec program.Spec, pipelineID string, uid, gid int) error { - takeOwnership := b.ownLoggingPath(spec) + if !b.ownLoggingPath(spec) { + // spec file passes a log path; so its up to the application to ensure the + // path exists and the write permissions are set so Elastic Agent can read it + return nil + } + drops := []string{b.generateLoggingPath(spec, pipelineID)} if drop := b.monitoringDrop(spec, pipelineID); drop != "" { drops = append(drops, drop) @@ -167,10 +172,8 @@ func (b *Monitor) Prepare(spec program.Spec, pipelineID string, uid, gid int) er } } - if takeOwnership { - if err := changeOwner(drop, uid, gid); err != nil { - return err - } + if err := changeOwner(drop, uid, gid); err != nil { + return err } } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/status.go b/x-pack/elastic-agent/pkg/core/plugin/process/status.go index eac8f2bc53e5..473ae9a70c77 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/status.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/status.go @@ -35,9 +35,6 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St return } - // it was a crash, cleanup anything required - go a.cleanUp() - // kill the process if a.state.ProcessInfo != nil { _ = a.state.ProcessInfo.Process.Kill() diff --git a/x-pack/libbeat/management/fleet/manager.go b/x-pack/libbeat/management/fleet/manager.go index dc0185a04ace..8aec0af1800b 100644 --- a/x-pack/libbeat/management/fleet/manager.go +++ b/x-pack/libbeat/management/fleet/manager.go @@ -169,7 +169,7 @@ func (cm *Manager) OnConfig(s string) { if errs := cm.apply(blocks); !errs.IsEmpty() { // `cm.apply` already logs the errors; currently allow beat to run degraded - cm.UpdateStatus(management.Degraded, errs.Error()) + cm.UpdateStatus(management.Failed, errs.Error()) return } From 9908acd2f1d16fe91b33726200c89fc561bf9c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 21 Jan 2021 10:17:38 +0100 Subject: [PATCH 10/35] bundle apm-server by default with Elastic Agent (#23548) --- dev-tools/packaging/packages.yml | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index ce48eb22098d..7c1801f5fe32 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -96,6 +96,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true @@ -177,6 +189,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc: + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true - &agent_binary_files '{{.BeatName}}{{.BinaryExt}}': @@ -247,6 +271,19 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc' + mode: 0644 + skip_on_missing: true + # Binary package spec (zip for windows) for community beats. - &agent_windows_binary_spec @@ -285,6 +322,18 @@ shared: source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' mode: 0644 skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512' + mode: 0644 + skip_on_missing: true + 'data/{{.BeatName}}-{{ commit_short }}/downloads/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc': + source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc' + mode: 0644 + skip_on_missing: true - &agent_docker_spec <<: *agent_binary_spec From 22fc53aba0e110ef6091d6cdb8cfe33aa2a36f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 21 Jan 2021 12:02:13 +0100 Subject: [PATCH 11/35] Skip flaky TestConfigurableService in agent (#23608) --- x-pack/elastic-agent/pkg/agent/operation/operator_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go index 5eb195302501..a7a3547fa88a 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator_test.go @@ -389,6 +389,7 @@ func TestConfigurableStartStop(t *testing.T) { } func TestConfigurableService(t *testing.T) { + t.Skip("Flaky test: https://github.com/elastic/beats/issues/23607") p := getProgram("serviceable", "1.0") operator := getTestOperator(t, downloadPath, installPath, p) From b04a4b13e141ad23b2fc957ef786e512707c4472 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 21 Jan 2021 12:40:21 +0100 Subject: [PATCH 12/35] Fix metricbeat/perfmon measurement grouping (#23505) --- CHANGELOG.next.asciidoc | 1 + metricbeat/module/windows/perfmon/data.go | 101 ++++++++++-------- .../module/windows/perfmon/data_test.go | 74 ++++++++++++- 3 files changed, 128 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1d9bacfa6bb1..a15022e7fe09 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -485,6 +485,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148] - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327] - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286] +- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] *Packetbeat* diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index 6e4e9c6ef0aa..3dad6dc729ef 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -38,58 +38,65 @@ var processRegexp = regexp.MustCompile(`(.+?[^\s])(?:#\d+|$)`) func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { eventMap := make(map[string]*mb.Event) for counterPath, values := range counters { - if hasCounter, counter := re.getCounter(counterPath); hasCounter { - for ind, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err.Error != nil { - if !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err.Error, logp.Namespace("perfmon"), "query", counterPath) - continue - } - // The counter has a negative value or the counter was successfully found, but the data returned is not valid. - // This error can occur if the counter value is less than the previous value. (Because counter values always increment, the counter value rolls over to zero when it reaches its maximum value.) - // This is not an error that stops the application from running successfully and a positive counter value should be retrieved in the later calls. - if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { - re.log.Debugw("Counter value retrieval returned", - "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) - continue - } - } - var eventKey string - if re.config.GroupMeasurements && val.Err.Error == nil { - // Send measurements with the same instance label as part of the same event - eventKey = val.Instance - } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) + hasCounter, counter := re.getCounter(counterPath) + if !hasCounter { + continue + } + + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err.Error != nil { + if !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err.Error, logp.Namespace("perfmon"), "query", counterPath) + continue } - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err.Error, "failed on query=%v", counterPath), - } - if val.Instance != "" { - //will ignore instance index - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) - } else { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) - } - } + // The counter has a negative value or the counter was successfully found, but the data returned is not valid. + // This error can occur if the counter value is less than the previous value. (Because counter values always increment, the counter value rolls over to zero when it reaches its maximum value.) + // This is not an error that stops the application from running successfully and a positive counter value should be retrieved in the later calls. + if val.Err.Error == pdh.PDH_CALC_NEGATIVE_VALUE || val.Err.Error == pdh.PDH_INVALID_DATA { + re.log.Debugw("Counter value retrieval returned", + "error", val.Err.Error, "cstatus", pdh.PdhErrno(val.Err.CStatus), logp.Namespace("perfmon"), "query", counterPath) + continue } - if val.Measurement != nil { - eventMap[eventKey].MetricSetFields.Put(counter.QueryField, val.Measurement) - } else { - eventMap[eventKey].MetricSetFields.Put(counter.QueryField, 0) + } + + var eventKey string + if re.config.GroupMeasurements && val.Err.Error == nil { + // Send measurements from the same object with the same instance label as part of the same event + eventKey = counter.ObjectName + "\\" + val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err.Error, "failed on query=%v", counterPath), } - if counter.ObjectField != "" { - eventMap[eventKey].MetricSetFields.Put(counter.ObjectField, counter.ObjectName) + if val.Instance != "" { + // will ignore instance index + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) + } } } + + if val.Measurement != nil { + eventMap[eventKey].MetricSetFields.Put(counter.QueryField, val.Measurement) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.QueryField, 0) + } + + if counter.ObjectField != "" { + eventMap[eventKey].MetricSetFields.Put(counter.ObjectField, counter.ObjectName) + } } } // Write the values into the map. diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index 7203963d2cce..e9f409ad98c2 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -30,6 +30,9 @@ import ( func TestGroupToEvents(t *testing.T) { reader := Reader{ + config: Config{ + GroupMeasurements: true, + }, query: pdh.Query{}, executed: true, log: nil, @@ -42,6 +45,26 @@ func TestGroupToEvents(t *testing.T) { ObjectField: "object", ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, }, + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(_Total)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(_Total)\% Processor Time`}, + }, + { + QueryField: "current_disk_queue_length", + QueryName: `\PhysicalDisk(_Total)\Current Disk Queue Length`, + Format: "float", + ObjectName: "PhysicalDisk", + ObjectField: "object", + InstanceName: "_Total", + InstanceField: "instance", + ChildQueries: []string{`\PhysicalDisk(_Total)\Current Disk Queue Length`}, + }, }, } counters := map[string][]pdh.CounterValue{ @@ -52,23 +75,72 @@ func TestGroupToEvents(t *testing.T) { Err: pdh.CounterValueError{}, }, }, + `\Processor Information(_Total)\% Processor Time`: { + { + Instance: "_Total", + Measurement: 11, + }, + }, + `\PhysicalDisk(_Total)\Current Disk Queue Length`: { + { + Instance: "_Total", + Measurement: 20, + }, + }, } + events := reader.groupToEvents(counters) assert.NotNil(t, events) - assert.Equal(t, len(events), 1) + assert.Equal(t, 3, len(events)) + ok, err := events[0].MetricSetFields.HasKey("datagrams_sent_per_sec") assert.NoError(t, err) assert.True(t, ok) + ok, err = events[0].MetricSetFields.HasKey("object") assert.NoError(t, err) assert.True(t, ok) + val, err := events[0].MetricSetFields.GetValue("datagrams_sent_per_sec") assert.NoError(t, err) assert.Equal(t, val, 23) + val, err = events[0].MetricSetFields.GetValue("object") assert.NoError(t, err) assert.Equal(t, val, "UDPv4") + ok, err = events[1].MetricSetFields.HasKey("%_processor_time") + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = events[1].MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err = events[1].MetricSetFields.GetValue("%_processor_time") + assert.NoError(t, err) + assert.Equal(t, val, 11) + + val, err = events[1].MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "Processor Information") + + ok, err = events[2].MetricSetFields.HasKey("current_disk_queue_length") + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = events[2].MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err = events[2].MetricSetFields.GetValue("current_disk_queue_length") + assert.NoError(t, err) + assert.Equal(t, val, 20) + + val, err = events[2].MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "PhysicalDisk") + } func TestGroupToSingleEvent(t *testing.T) { From dc1f9873b09d29d2fed6bfce48ff7e2f36e9d211 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Thu, 21 Jan 2021 16:02:22 +0000 Subject: [PATCH 13/35] fix for test (#23617) --- .../module/windows/perfmon/data_test.go | 100 +++++++++--------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index e9f409ad98c2..2554a87a9ce8 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -93,54 +93,58 @@ func TestGroupToEvents(t *testing.T) { assert.NotNil(t, events) assert.Equal(t, 3, len(events)) - ok, err := events[0].MetricSetFields.HasKey("datagrams_sent_per_sec") - assert.NoError(t, err) - assert.True(t, ok) - - ok, err = events[0].MetricSetFields.HasKey("object") - assert.NoError(t, err) - assert.True(t, ok) - - val, err := events[0].MetricSetFields.GetValue("datagrams_sent_per_sec") - assert.NoError(t, err) - assert.Equal(t, val, 23) - - val, err = events[0].MetricSetFields.GetValue("object") - assert.NoError(t, err) - assert.Equal(t, val, "UDPv4") - - ok, err = events[1].MetricSetFields.HasKey("%_processor_time") - assert.NoError(t, err) - assert.True(t, ok) - - ok, err = events[1].MetricSetFields.HasKey("object") - assert.NoError(t, err) - assert.True(t, ok) - - val, err = events[1].MetricSetFields.GetValue("%_processor_time") - assert.NoError(t, err) - assert.Equal(t, val, 11) - - val, err = events[1].MetricSetFields.GetValue("object") - assert.NoError(t, err) - assert.Equal(t, val, "Processor Information") - - ok, err = events[2].MetricSetFields.HasKey("current_disk_queue_length") - assert.NoError(t, err) - assert.True(t, ok) - - ok, err = events[2].MetricSetFields.HasKey("object") - assert.NoError(t, err) - assert.True(t, ok) - - val, err = events[2].MetricSetFields.GetValue("current_disk_queue_length") - assert.NoError(t, err) - assert.Equal(t, val, 20) - - val, err = events[2].MetricSetFields.GetValue("object") - assert.NoError(t, err) - assert.Equal(t, val, "PhysicalDisk") - + for _, event := range events { + ok, err := event.MetricSetFields.HasKey("datagrams_sent_per_sec") + if ok { + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.Equal(t, val, 23) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "UDPv4") + } else { + ok, err := event.MetricSetFields.HasKey("%_processor_time") + if ok { + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("%_processor_time") + assert.NoError(t, err) + assert.Equal(t, val, 11) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "Processor Information") + } else { + ok, err = event.MetricSetFields.HasKey("current_disk_queue_length") + assert.NoError(t, err) + assert.True(t, ok) + + ok, err = event.MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + + val, err := event.MetricSetFields.GetValue("current_disk_queue_length") + assert.NoError(t, err) + assert.Equal(t, val, 20) + + val, err = event.MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "PhysicalDisk") + } + } + } } func TestGroupToSingleEvent(t *testing.T) { From 5f195657c9d396918c07cf8c5c7002f2ac4dc545 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 21 Jan 2021 11:54:27 -0500 Subject: [PATCH 14/35] [libbeat] Update Sarama to version 1.27.2 (#23595) --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 110 +++++++++++++++++++++++++++++++--------- go.mod | 8 +-- go.sum | 38 ++++++++++---- 4 files changed, 118 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a15022e7fe09..eb905626e3d4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -581,6 +581,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Honor kube event resysncs to handle missed watch events {pull}22668[22668] - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] +- Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 1616d3b8a75d..5a6c3df026e9 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2216,11 +2216,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/sarama -Version: v1.19.1-0.20200629123429-0e7b69039eec +Version: v1.19.1-0.20210120173147-5c8cb347d877 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20200629123429-0e7b69039eec/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20210120173147-5c8cb347d877/LICENSE: Copyright (c) 2013 Shopify @@ -13708,11 +13708,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/prometheus@v -------------------------------------------------------------------------------- Dependency : github.com/rcrowley/go-metrics -Version: v0.0.0-20190826022208-cac0b30c2563 +Version: v0.0.0-20200313005456-10cdbea86bc0 Licence type (autodetected): BSD-2-Clause-FreeBSD -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20190826022208-cac0b30c2563/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rcrowley/go-metrics@v0.0.0-20200313005456-10cdbea86bc0/LICENSE: Copyright 2012 Richard Crowley. All rights reserved. @@ -16079,11 +16079,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.0.0-20200622213623-75b288015ac9 +Version: v0.0.0-20200820211705-5c72a883971a Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200622213623-75b288015ac9/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.0.0-20200820211705-5c72a883971a/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -16153,11 +16153,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.0.0-20200707034311-ab3426394381 +Version: v0.0.0-20200904194848-62affa334b73 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20200707034311-ab3426394381/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20200904194848-62affa334b73/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24968,6 +24968,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/creack/pty +Version: v1.1.9 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/creack/pty@v1.1.9/LICENSE: + +Copyright (c) 2011 Keith Rarick + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/cucumber/godog Version: v0.8.1 @@ -27077,11 +27110,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.7.2 +Version: v1.10.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.10.2/LICENSE: MIT License @@ -35040,11 +35073,11 @@ match.go, match_test.go: -------------------------------------------------------------------------------- Dependency : github.com/klauspost/compress -Version: v1.9.8 +Version: v1.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.9.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/klauspost/compress@v1.11.0/LICENSE: Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2019 Klaus Post. All rights reserved. @@ -35119,11 +35152,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------------------- Dependency : github.com/kr/pretty -Version: v0.2.0 +Version: v0.2.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.2.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/pretty@v0.2.1/License: Copyright 2012 Keith Rarick @@ -35181,11 +35214,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/text -Version: v0.1.0 +Version: v0.2.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.1.0/License: +Contents of probable licence file $GOMODCACHE/github.com/kr/text@v0.2.0/License: Copyright 2012 Keith Rarick @@ -36662,6 +36695,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/niemeyer/pretty +Version: v0.0.0-20200227124842-a10e7caefd8e +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/niemeyer/pretty@v0.0.0-20200227124842-a10e7caefd8e/License: + +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/onsi/ginkgo Version: v1.11.0 @@ -38049,11 +38111,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/pierrec/lz4 -Version: v2.4.1+incompatible +Version: v2.5.2+incompatible Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.4.1+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.5.2+incompatible/LICENSE: Copyright (c) 2015, Pierre Curto All rights reserved. @@ -41807,11 +41869,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/xerrors -Version: v0.0.0-20191204190536-9bdfabe68543 +Version: v0.0.0-20200804184101-5ec99f83aff1 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20200804184101-5ec99f83aff1/LICENSE: Copyright (c) 2019 The Go Authors. All rights reserved. @@ -42122,11 +42184,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : gopkg.in/check.v1 -Version: v1.0.0-20190902080502-41f04d3bba15 +Version: v1.0.0-20200902074654-038fdea0a05b Licence type (autodetected): BSD-2-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20200902074654-038fdea0a05b/LICENSE: Gocheck - A rich testing framework for Go @@ -43145,11 +43207,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 -Version: v3.0.0-20200313102051-9f266ea9e77c +Version: v3.0.0-20200615113413-eeeca48fe776 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20200313102051-9f266ea9e77c/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776/LICENSE: This project is covered by two different licenses: MIT and Apache. diff --git a/go.mod b/go.mod index 8e00d8d35174..1b6117118408 100644 --- a/go.mod +++ b/go.mod @@ -136,7 +136,7 @@ require ( github.com/prometheus/common v0.7.0 github.com/prometheus/procfs v0.0.11 github.com/prometheus/prometheus v2.5.0+incompatible - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 github.com/reviewdog/reviewdog v0.9.17 github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e // indirect github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 @@ -164,9 +164,9 @@ require ( go.uber.org/atomic v1.5.0 go.uber.org/multierr v1.3.0 go.uber.org/zap v1.14.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/lint v0.0.0-20200130185559-910be7a94367 - golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/net v0.0.0-20200904194848-62affa334b73 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 @@ -192,7 +192,7 @@ require ( replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15 - github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec + github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f diff --git a/go.sum b/go.sum index 732b2bb4cef8..74cc1c71701d 100644 --- a/go.sum +++ b/go.sum @@ -190,6 +190,7 @@ github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1D github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= @@ -285,8 +286,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.13.0 h1:EIeuQcLPKia759s6mlVztlxUyKiKYHo6y6kOODOLO7A= github.com/elastic/gosigar v0.13.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI= -github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= +github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877 h1:C9LsbipColsz04JKpKoLlp0pgMJRLq2uXVTeKRDcNcY= +github.com/elastic/sarama v1.19.1-0.20210120173147-5c8cb347d877/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -300,8 +301,8 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk= +github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -505,8 +506,8 @@ github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.0 h1:wJbzvpYMVGG9iTI9VxpnNZfd4DzMPoCWze3GgSqz8yg= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -515,9 +516,13 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -578,6 +583,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -616,8 +623,8 @@ github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= -github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 h1:i5VIxp6QB8oWZ8IkK8zrDgeT6ORGIUeiN+61iETwJbI= github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0/go.mod h1:4xpMLz7RBWyB+ElzHu8Llua96TRCB3YwX+l5EP1wmHk= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -653,8 +660,8 @@ github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4 github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd h1:fvaEkjpr2NJbtnFRCft7D6y/mQ5/2OQU0pKJLW8dwFA= github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd/go.mod h1:giYAXnpegRDPsXUO7TRpDKXJo1lFGYxyWRfEt5iQ+OA= github.com/reviewdog/reviewdog v0.9.17 h1:MKb3rlQZgkEXr3d85iqtYNITXn7gDJr2kT0IhgX/X9A= @@ -801,9 +808,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -854,6 +862,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -924,6 +934,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -980,6 +992,8 @@ gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1010,6 +1024,8 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/gotestsum v0.6.0 h1:0zIxynXq9gkAcRpboAi3qOQIkZkCt/stfQzd7ab7Czs= From f527935c507215780c4665e83099cc81fa84fdf6 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 21 Jan 2021 11:00:06 -0600 Subject: [PATCH 15/35] Fix permissions for synthetics docker image (#23576) The permissions were incorrect, and mostly set for the wrong UID. This fixes that, performing most tasks as a regular user. --- .../templates/docker/Dockerfile.tmpl | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index dd7a0080d6ba..e42e525644c0 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -8,14 +8,6 @@ FROM {{ .buildFrom }} AS home COPY beat {{ $beatHome }} -{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} -RUN mkdir -p {{ $beatHome }}/.node \ - {{ $beatHome }}/.npm \ - {{ $beatHome }}/.cache \ - {{ $beatHome }}/.config \ - {{ $beatHome }}/suites -{{- end }} - RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \ chown -R root:root {{ $beatHome }} && \ find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ @@ -48,6 +40,16 @@ RUN yum -y --setopt=tsflags=nodocs update \ # See https://access.redhat.com/discussions/3195102 for why rm is needed {{- end }} +{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} +ENV NODE_PATH={{ $beatHome }}/.node +RUN echo \ + $NODE_PATH \ + {{ $beatHome }}/.config \ + {{ $beatHome }}/suites \ + {{ $beatHome }}/.npm \ + {{ $beatHome }}/.cache \ + | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' +{{- end }} LABEL \ org.label-schema.build-date="{{ date }}" \ @@ -98,28 +100,30 @@ RUN mkdir /licenses COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses +{{- if ne .user "root" }} +RUN groupadd --gid 1000 {{ .BeatName }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} +{{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} +RUN chown {{ .user }} $NODE_PATH +{{- end }} +{{- end }} +USER {{ .user }} + {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal"))) }} # Setup synthetics env vars ENV ELASTIC_SYNTHETICS_CAPABLE=true ENV SUITES_DIR={{ $beatHome }}/suites -ENV NODE_PATH={{ $beatHome }}/.node - -# Setup node -RUN cd /usr/share/heartbeat/.node \ - && mkdir node \ - && curl https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-x64.tar.xz | tar -xJ --strip 1 -C node -ENV PATH="/usr/share/heartbeat/.node/node/bin:$PATH" +ENV NODE_VERSION=12.18.4 +ENV PATH="$NODE_PATH/node/bin:$PATH" # Install the latest version of @elastic/synthetics forcefully ignoring the previously -# cached node_modules, hearbeat then calls the global executable to run test suites -RUN npm i -g -f @elastic/synthetics -{{- end }} - - -{{- if ne .user "root" }} -RUN groupadd --gid 1000 {{ .BeatName }} -RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} +# cached node_modules, heartbeat then calls the global executable to run test suites +# Setup node +RUN cd /usr/share/heartbeat/.node \ + && mkdir -p node \ + && curl https://nodejs.org/dist/v12.18.4/node-v12.18.4-linux-x64.tar.xz | tar -xJ --strip 1 -C node \ + && chmod ug+rwX -R $NODE_PATH \ + && npm i -g -f @elastic/synthetics && chmod ug+rwX -R $NODE_PATH {{- end }} -USER {{ .user }} {{- range $i, $port := .ExposePorts }} EXPOSE {{ $port }} From 7bdbf28368ea7621f31b20a54a66d379c0379ae9 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 Jan 2021 19:30:48 +0100 Subject: [PATCH 16/35] [Ingest Manager] Fixed nil pointer during unenroll (#23609) [Ingest Manager] Fixed nil pointer during unenroll (#23609) --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + .../pkg/agent/application/fleet_acker.go | 8 ++++- .../pkg/agent/application/lazy_acker.go | 6 +++- .../pkg/agent/application/lazy_acker_test.go | 2 +- .../pkg/agent/application/managed_mode.go | 2 +- .../pkg/agent/application/state_store.go | 2 +- .../pkg/agent/application/state_store_test.go | 30 +++++++++++++++++++ 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index aa433f81d70f..d3fd1fa65d5d 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ - Fix Windows service installation script {pull}20203[20203] - Fix timeout issue stopping service applications {pull}20256[20256] - Fix incorrect hash when upgrading agent {pull}22322[22322] +- Fixed nil pointer during unenroll {pull}23609[23609] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go b/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go index 4544fa8a772f..dac05d0c3a06 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go @@ -7,6 +7,7 @@ package application import ( "context" "fmt" + "strings" "time" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" @@ -58,6 +59,8 @@ func (f *actionAcker) Ack(ctx context.Context, action fleetapi.Action) error { return errors.New(err, fmt.Sprintf("acknowledge action '%s' for elastic-agent '%s' failed", action.ID(), agentID), errors.TypeNetwork) } + f.log.Debugf("action with id '%s' was just acknowledged", action.ID()) + return nil } @@ -65,8 +68,10 @@ func (f *actionAcker) AckBatch(ctx context.Context, actions []fleetapi.Action) e // checkin agentID := f.agentInfo.AgentID() events := make([]fleetapi.AckEvent, 0, len(actions)) + ids := make([]string, 0, len(actions)) for _, action := range actions { events = append(events, constructEvent(action, agentID)) + ids = append(ids, action.ID()) } cmd := fleetapi.NewAckCmd(f.agentInfo, f.client) @@ -74,11 +79,12 @@ func (f *actionAcker) AckBatch(ctx context.Context, actions []fleetapi.Action) e Events: events, } + f.log.Debugf("%d actions with ids '%s' acknowledging", len(ids), strings.Join(ids, ",")) + _, err := cmd.Execute(ctx, req) if err != nil { return errors.New(err, fmt.Sprintf("acknowledge %d actions '%v' for elastic-agent '%s' failed", len(actions), actions, agentID), errors.TypeNetwork) } - return nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/lazy_acker.go b/x-pack/elastic-agent/pkg/agent/application/lazy_acker.go index 58b212ab8e41..4a4004e028fe 100644 --- a/x-pack/elastic-agent/pkg/agent/application/lazy_acker.go +++ b/x-pack/elastic-agent/pkg/agent/application/lazy_acker.go @@ -7,6 +7,7 @@ package application import ( "context" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" ) @@ -19,19 +20,22 @@ type ackForcer interface { } type lazyAcker struct { + log *logger.Logger acker batchAcker queue []fleetapi.Action } -func newLazyAcker(baseAcker batchAcker) *lazyAcker { +func newLazyAcker(baseAcker batchAcker, log *logger.Logger) *lazyAcker { return &lazyAcker{ acker: baseAcker, queue: make([]fleetapi.Action, 0), + log: log, } } func (f *lazyAcker) Ack(ctx context.Context, action fleetapi.Action) error { f.queue = append(f.queue, action) + f.log.Debugf("appending action with id '%s' to the queue", action.ID()) if _, isAckForced := action.(ackForcer); isAckForced { return f.Commit(ctx) diff --git a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go index 24c708c0d91a..b3d872d49460 100644 --- a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go @@ -32,7 +32,7 @@ func TestLazyAcker(t *testing.T) { t.Fatal(err) } - lacker := newLazyAcker(acker) + lacker := newLazyAcker(acker, log) if acker == nil { t.Fatal("acker not initialized") diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 63e8611354db..9ad1f24a3d0d 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -183,7 +183,7 @@ func newManaged( return nil, err } - batchedAcker := newLazyAcker(acker) + batchedAcker := newLazyAcker(acker, log) // Create the state store that will persist the last good policy change on disk. stateStore, err := newStateStoreWithMigration(log, info.AgentActionStoreFile(), info.AgentStateStoreFile()) diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store.go b/x-pack/elastic-agent/pkg/agent/application/state_store.go index 81d3f901469c..283ab8e480dc 100644 --- a/x-pack/elastic-agent/pkg/agent/application/state_store.go +++ b/x-pack/elastic-agent/pkg/agent/application/state_store.go @@ -225,7 +225,7 @@ func (s *stateStore) Save() error { if apc, ok := s.state.action.(*fleetapi.ActionPolicyChange); ok { serialize.Action = &actionSerializer{apc.ActionID, apc.ActionType, apc.Policy, nil} } else if aun, ok := s.state.action.(*fleetapi.ActionUnenroll); ok { - serialize.Action = &actionSerializer{apc.ActionID, apc.ActionType, nil, &aun.IsDetected} + serialize.Action = &actionSerializer{aun.ActionID, aun.ActionType, nil, &aun.IsDetected} } else { return fmt.Errorf("incompatible type, expected ActionPolicyChange and received %T", s.state.action) } diff --git a/x-pack/elastic-agent/pkg/agent/application/state_store_test.go b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go index 26ea1eaca683..1c6a7bfd7319 100644 --- a/x-pack/elastic-agent/pkg/agent/application/state_store_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/state_store_test.go @@ -101,6 +101,36 @@ func runTestStateStore(t *testing.T, ackToken string) { require.Equal(t, ackToken, store.AckToken()) })) + t.Run("can save to disk unenroll action type", + withFile(func(t *testing.T, file string) { + action := &fleetapi.ActionUnenroll{ + ActionID: "abc123", + ActionType: "UNENROLL", + } + + s := storage.NewDiskStore(file) + store, err := newStateStore(log, s) + require.NoError(t, err) + + require.Equal(t, 0, len(store.Actions())) + store.Add(action) + store.SetAckToken(ackToken) + err = store.Save() + require.NoError(t, err) + require.Equal(t, 1, len(store.Actions())) + require.Equal(t, ackToken, store.AckToken()) + + s = storage.NewDiskStore(file) + store1, err := newStateStore(log, s) + require.NoError(t, err) + + actions := store1.Actions() + require.Equal(t, 1, len(actions)) + + require.Equal(t, action, actions[0]) + require.Equal(t, ackToken, store.AckToken()) + })) + t.Run("when we ACK we save to disk", withFile(func(t *testing.T, file string) { ActionPolicyChange := &fleetapi.ActionPolicyChange{ From e5cd64f6229536f8ab54597624c543e9d16bcdc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 21 Jan 2021 19:37:35 +0100 Subject: [PATCH 17/35] Update filestream reader offset when line is skipped (#23417) ## What does this PR do? This PR adds two previously missing offset updates to the `filestream` reader when a line is skipped. ## Why is it important? The offset could be incorrect if Filebeat skips the line if the line should not be published because of user configuration in `export_line` or `import_line` If the offset is not updated in the reader, the state information of newer published events become incorrect. This might lead to duplicated events if Filebeat is restarted. --- CHANGELOG.next.asciidoc | 1 + filebeat/input/filestream/input.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eb905626e3d4..0e7edfce6f2c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -271,6 +271,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] - Change the `event.created` in Netflow events to be the time the event was created by Filebeat to be consistent with ECS. {pull}23094[23094] +- Update `filestream` reader offset when a line is skipped. {pull}23417[23417] *Filebeat* diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 7e253bcc9ec3..8d40284c3661 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -318,13 +318,13 @@ func (inp *filestream) readFromSource( return nil } + s.Offset += int64(message.Bytes) + if message.IsEmpty() || inp.isDroppedLine(log, string(message.Content)) { continue } event := inp.eventFromMessage(message, path) - s.Offset += int64(message.Bytes) - if err := p.Publish(event, s); err != nil { return err } From ba552f67d59034550ff6c6ac26993b99725a8f0b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 21 Jan 2021 12:35:48 -0700 Subject: [PATCH 18/35] Deprecate aws_partition and use endpoint,regions instead (#23539) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/aws.asciidoc | 23 ++++++++++++++++++- x-pack/filebeat/input/awss3/input.go | 1 + x-pack/libbeat/common/aws/credentials.go | 11 ++++++++- .../docs/aws-credentials-config.asciidoc | 9 ++++++-- .../metricbeat/module/aws/_meta/docs.asciidoc | 23 ++++++++++++++++++- x-pack/metricbeat/module/aws/aws.go | 18 +++++++++++---- 7 files changed, 77 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e7edfce6f2c..2b75b0e90055 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -582,6 +582,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Honor kube event resysncs to handle missed watch events {pull}22668[22668] - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] +- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index e02a7a814600..085436b05d98 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -31,7 +31,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -40,6 +40,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 584d306b1c77..98d8c60d77b4 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -113,6 +113,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C log.Debug("s3 service name = ", s3Servicename) log.Debug("s3 input config max_number_of_messages = ", in.config.MaxNumberOfMessages) + log.Debug("s3 input config endpoint = ", in.config.AwsConfig.Endpoint) return &s3Collector{ cancellation: ctxtool.FromCanceller(ctx.Cancelation), logger: log, diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 97d06a6a255b..fafe8e7336bf 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -24,7 +25,7 @@ type ConfigAWS struct { SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` RoleArn string `config:"role_arn"` - AWSPartition string `config:"aws_partition"` + AWSPartition string `config:"aws_partition"` // Deprecated. } // GetAWSCredentials function gets aws credentials from the config. @@ -104,3 +105,11 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName } return awsConfig } + +// Validate checks for deprecated config option +func (c ConfigAWS) Validate() error { + if c.AWSPartition != "" { + cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.") + } + return nil +} diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a9cb4ab8e880..a3a242554b40 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -9,9 +9,14 @@ To configure AWS credentials, either put the credentials into the {beatname_uc} * *session_token*: required when using temporary security credentials. * *credential_profile_name*: profile name in shared credentials file. * *shared_credential_file*: directory of the shared credentials file. -* *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. -* *aws_partition*: AWS region parttion name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`. +* *endpoint*: URL of the entry point for an AWS web service. +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. [float] ==== Supported Formats diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index e4e55e82136a..df18966b2af6 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -23,7 +23,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -32,6 +32,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 167e6a088a0e..f3a7caf6cc8b 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -6,6 +6,7 @@ package aws import ( "context" + "fmt" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -16,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" @@ -78,20 +78,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig) if err != nil { - return nil, errors.Wrap(err, "failed to get aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err) } _, err = awsConfig.Credentials.Retrieve() if err != nil { - return nil, errors.Wrap(err, "failed to retrieve aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err) } + base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint) metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, + Endpoint: config.AWSConfig.Endpoint, } base.Logger().Debug("Metricset level config for period: ", metricSet.Period) @@ -100,6 +102,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Get IAM account name, set region by aws_partition, default is aws global partition // refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html + if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" { + base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.") + } switch config.AWSConfig.AWSPartition { case "aws-cn": awsConfig.Region = "cn-north-1" @@ -109,6 +114,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig.Region = "us-east-1" } + // If regions in config is not empty, then overwrite the awsConfig.Region + if len(config.Regions) > 0 { + awsConfig.Region = config.Regions[0] + } + // Get IAM account id svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint( config.AWSConfig.Endpoint, "sts", "", awsConfig)) @@ -151,7 +161,7 @@ func getRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error req := svc.DescribeRegionsRequest(input) output, err := req.Send(context.TODO()) if err != nil { - err = errors.Wrap(err, "Failed DescribeRegions") + err = fmt.Errorf("failed DescribeRegions: %w", err) return } for _, region := range output.Regions { From e4ef5ccb06fe7e47954de05158e7e92bf280b414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 21 Jan 2021 21:03:11 +0100 Subject: [PATCH 19/35] Update Golang to 1.15.7 (#22495) ## What does this PR do? This PR updates the Go version in the repository to 1.15.7. ## Why is it important? Numerous improvements are coming with the new version of Golang. The most noteworthy is the binary sizes are reduced by 5-8%. You can check out the [release notes of Golang 1.15](https://golang.org/doc/go1.15) for more information. Another change is that the deprecated, legacy behaviour of treating the CommonName field on X.509 certificates as a hostname when no Subject Alternative Names are present is now disabled by default. We do not expect users to update their certificates immediately. Thus, the default behaviour is kept for all existing verification modes. Furthermore, a new verification mode is added named `strict`. If this mode is selected Beats expects to have a SAN in the certificate. `verification_mode` | behaviour ------- | ----- `""` | CommonName is checked if no DNSName is available, then full check (previous behaviour) `"strict"` | full check with new strict SAN requirements `"full"` | CommonName is checked if no DNSName is available, then full check (previous behaviour, default value) `"certificate"` | certificate check without hostname (previous behaviour) `"none"` | no check (previous behaviour) The legacy behaviour is going to be removed in the next major version 8.0. We expect users to update their certificates so it does not depend on CommonName. --- .go-version | 2 +- CHANGELOG-developer.next.asciidoc | 1 + CHANGELOG.next.asciidoc | 5 + auditbeat/Dockerfile | 2 +- auditbeat/auditbeat.reference.yml | 36 +++ filebeat/Dockerfile | 2 +- filebeat/filebeat.reference.yml | 36 +++ filebeat/input/kafka/config.go | 2 +- filebeat/input/mqtt/client.go | 2 +- filebeat/inputsource/tcp/server.go | 2 +- filebeat/tests/system/test_tcp_tls.py | 6 + go.mod | 2 +- heartbeat/Dockerfile | 2 +- heartbeat/heartbeat.reference.yml | 36 +++ .../monitors/active/fixtures/expired.cert | 40 ++-- .../monitors/active/fixtures/expired.key | 52 ++--- heartbeat/monitors/active/http/http_test.go | 7 +- journalbeat/Dockerfile | 2 +- journalbeat/journalbeat.reference.yml | 36 +++ libbeat/Dockerfile | 2 +- libbeat/_meta/config/ssl.reference.yml.tmpl | 6 + libbeat/common/transport/tls.go | 2 +- .../transport/tlscommon/ca_pinning_test.go | 185 ++++++++------- libbeat/common/transport/tlscommon/config.go | 4 + .../transport/tlscommon/server_config.go | 4 +- .../transport/tlscommon/testdata/cacert.crt | 24 ++ .../transport/tlscommon/testdata/cacert.key | 27 +++ .../transport/tlscommon/testdata/client1.crt | 48 ++++ .../transport/tlscommon/testdata/client1.key | 27 +++ .../common/transport/tlscommon/tls_config.go | 220 +++++++++++++++--- .../transport/tlscommon/tls_config_test.go | 199 ++++++++++++++++ .../common/transport/tlscommon/tls_test.go | 16 +- libbeat/common/transport/tlscommon/types.go | 2 + .../tlscommon/{verify.go => validhostname.go} | 87 ++++--- .../common/transport/tlscommon/verify_test.go | 110 --------- .../common/transport/transptest/testing.go | 2 +- libbeat/docs/version.asciidoc | 2 +- libbeat/outputs/kafka/config.go | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/helper/server/http/http.go | 2 +- metricbeat/metricbeat.reference.yml | 36 +++ metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/kafka/metricset.go | 2 +- metricbeat/module/mongodb/metricset.go | 2 +- packetbeat/Dockerfile | 2 +- packetbeat/packetbeat.reference.yml | 36 +++ .../environments/docker/logstash/gencerts.sh | 2 +- .../logstash/pki/tls/certs/logstash.crt | 32 +-- .../logstash/pki/tls/private/logstash.key | 52 ++--- testing/environments/docker/logstash/ssl.conf | 18 ++ winlogbeat/eventlog/bench_test.go | 2 +- winlogbeat/tests/system/test_config.py | 2 +- winlogbeat/winlogbeat.reference.yml | 36 +++ x-pack/auditbeat/auditbeat.reference.yml | 36 +++ .../pipelinemanager/clientLogReader_test.go | 3 +- .../pkg/agent/operation/common_test.go | 4 +- x-pack/elastic-agent/pkg/core/authority/ca.go | 6 +- x-pack/filebeat/filebeat.reference.yml | 36 +++ x-pack/filebeat/input/http_endpoint/input.go | 2 +- .../netflow/decoder/template/test_helpers.go | 5 +- x-pack/functionbeat/Dockerfile | 2 +- .../functionbeat/functionbeat.reference.yml | 24 ++ x-pack/heartbeat/heartbeat.reference.yml | 36 +++ x-pack/libbeat/Dockerfile | 2 +- x-pack/metricbeat/Jenkinsfile.yml | 22 +- x-pack/metricbeat/metricbeat.reference.yml | 36 +++ x-pack/packetbeat/packetbeat.reference.yml | 36 +++ x-pack/winlogbeat/winlogbeat.reference.yml | 36 +++ 68 files changed, 1333 insertions(+), 423 deletions(-) create mode 100644 libbeat/common/transport/tlscommon/testdata/cacert.crt create mode 100644 libbeat/common/transport/tlscommon/testdata/cacert.key create mode 100644 libbeat/common/transport/tlscommon/testdata/client1.crt create mode 100644 libbeat/common/transport/tlscommon/testdata/client1.key create mode 100644 libbeat/common/transport/tlscommon/tls_config_test.go rename libbeat/common/transport/tlscommon/{verify.go => validhostname.go} (59%) delete mode 100644 libbeat/common/transport/tlscommon/verify_test.go create mode 100644 testing/environments/docker/logstash/ssl.conf diff --git a/.go-version b/.go-version index 4ed70fac17d7..545fd574d35b 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.14.12 +1.15.7 diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 4d40f0c99b35..51a4565e4ff2 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -106,3 +106,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827] - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267] - Add support for customized monitoring API. {pull}22605[22605] +- Update Go version to 1.15.7. {pull}22495[22495] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2b75b0e90055..c24284950fa2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -986,6 +986,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- Selecting `full` in `ssl.verification_mode` option will not treat CommonName field in x509 certificates as + a hostname when Subject Alternative Name is not present from v8.0. + Please update your certificates so it contains at least one DNSName instead of relying on CommonName in the new + major version of Beats. + *Filebeat* - The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed. diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index c6d4c0c0735c..e5767ab11670 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index ef82491a4dd3..7a053b62884e 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -529,6 +529,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -656,6 +662,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -853,6 +865,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1009,6 +1027,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1305,6 +1329,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1502,6 +1532,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index d9707991a5fe..8b3983fa8da3 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 91b1dfa31901..6cf5f8d33fa7 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1409,6 +1409,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1536,6 +1542,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1733,6 +1745,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1889,6 +1907,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2185,6 +2209,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2382,6 +2412,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/filebeat/input/kafka/config.go b/filebeat/input/kafka/config.go index 0e4888b90c30..75a0957744ae 100644 --- a/filebeat/input/kafka/config.go +++ b/filebeat/input/kafka/config.go @@ -177,7 +177,7 @@ func newSaramaConfig(config kafkaInputConfig) (*sarama.Config, error) { } if tls != nil { k.Net.TLS.Enable = true - k.Net.TLS.Config = tls.BuildModuleConfig("") + k.Net.TLS.Config = tls.BuildModuleClientConfig("") } if config.Kerberos.IsEnabled() { diff --git a/filebeat/input/mqtt/client.go b/filebeat/input/mqtt/client.go index 701498f3c561..068dca043869 100644 --- a/filebeat/input/mqtt/client.go +++ b/filebeat/input/mqtt/client.go @@ -40,7 +40,7 @@ func createClientOptions(config mqttInputConfig, onConnectHandler func(client li if err != nil { return nil, err } - clientOptions.SetTLSConfig(tlsConfig.BuildModuleConfig("")) + clientOptions.SetTLSConfig(tlsConfig.BuildModuleClientConfig("")) } return clientOptions, nil } diff --git a/filebeat/inputsource/tcp/server.go b/filebeat/inputsource/tcp/server.go index 270ebc9c0c57..42c88693879e 100644 --- a/filebeat/inputsource/tcp/server.go +++ b/filebeat/inputsource/tcp/server.go @@ -68,7 +68,7 @@ func (s *Server) createServer() (net.Listener, error) { var l net.Listener var err error if s.tlsConfig != nil { - t := s.tlsConfig.BuildModuleConfig(s.config.Host) + t := s.tlsConfig.BuildServerConfig(s.config.Host) l, err = tls.Listen("tcp", s.config.Host, t) if err != nil { return nil, err diff --git a/filebeat/tests/system/test_tcp_tls.py b/filebeat/tests/system/test_tcp_tls.py index 4001fd863c95..845ade6b2880 100644 --- a/filebeat/tests/system/test_tcp_tls.py +++ b/filebeat/tests/system/test_tcp_tls.py @@ -127,6 +127,8 @@ def test_tcp_over_tls_and_verify_invalid_server_without_mutual_auth(self): with pytest.raises(ssl.SSLError): tls.connect((config.get('host'), config.get('port'))) + sock.close() + def test_tcp_over_tls_mutual_auth_fails(self): """ Test filebeat TCP with TLS with default setting to enforce client auth, with bad client certificates @@ -171,6 +173,8 @@ def test_tcp_over_tls_mutual_auth_fails(self): # so that the failure can be reported as an exception when it arrives. tls.recv(1) + sock.close() + def test_tcp_over_tls_mutual_auth_succeed(self): """ Test filebeat TCP with TLS when enforcing client auth with good client certificates. @@ -275,6 +279,8 @@ def test_tcp_tls_with_a_plain_text_socket(self): assert path.isfile(path.join(self.working_dir, "output/" + self.beat_name)) is False + sock.close() + def assert_output(self, output): assert len(output) == 2 assert output[0]["input.type"] == "tcp" diff --git a/go.mod b/go.mod index 1b6117118408..b0c1c974fc1d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.14 +go 1.15 require ( 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index f0155d342742..51c2b06d485f 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 37e3e2ed1226..e39dfa16d69a 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -707,6 +707,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -834,6 +840,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1031,6 +1043,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1187,6 +1205,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1483,6 +1507,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1680,6 +1710,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/heartbeat/monitors/active/fixtures/expired.cert b/heartbeat/monitors/active/fixtures/expired.cert index e39ad893bd6c..959486cec377 100644 --- a/heartbeat/monitors/active/fixtures/expired.cert +++ b/heartbeat/monitors/active/fixtures/expired.cert @@ -1,23 +1,21 @@ -----BEGIN CERTIFICATE----- -MIID3zCCAsegAwIBAgIUS+ahW2wxDZ1bT/qYnenS8jrXUcAwDQYJKoZIhvcNAQEL -BQAwfzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk1OMRQwEgYDVQQHDAtNaW5uZWFw -b2xpczEVMBMGA1UECgwMRWxhc3RpYywgSW5jMRQwEgYDVQQLDAtFbmdpbmVlcmlu -ZzEgMB4GA1UEAwwXZXhwaXJlZHRlc3QuZXhhbXBsZS5uZXQwHhcNMjAwNDIxMTQw -MDE0WhcNMjAwNDIyMTQwMDE0WjB/MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTU4x -FDASBgNVBAcMC01pbm5lYXBvbGlzMRUwEwYDVQQKDAxFbGFzdGljLCBJbmMxFDAS -BgNVBAsMC0VuZ2luZWVyaW5nMSAwHgYDVQQDDBdleHBpcmVkdGVzdC5leGFtcGxl -Lm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKh1iS5EZ7bDSKgW -R3JXAepMIaEewMSdbaoBtuNQb48XJGwI0mudF983a7JxGCSfw9mhVYa4YsSv79UE -XomGrWVrS01Cmf1VRIOmxevWMPhvnE6UH+5VxKUBk5ooNSty4iHkDFy2i5WWjxiv -de6Xqnn/dVQhuT/sW+rU/grCsGcdUwqsWnC547ekqiYRTtyZrdh+U0KRKqy5iBlH -9Woua+CnXmsD7+4MgGekErg9XLRHYveLOmLucbNlAIlRyfMDZL1RlXufcGwhzItz -JNM9N0NJ5bwrpuP0RYlYbbMYal+b1Tn2e8qkMm88hniQkuu69kUpKeewIOr62vIK -tI273GECAwEAAaNTMFEwHQYDVR0OBBYEFKgd6wQcgIdUSjtJREObD+R3q3MPMB8G -A1UdIwQYMBaAFKgd6wQcgIdUSjtJREObD+R3q3MPMA8GA1UdEwEB/wQFMAMBAf8w -DQYJKoZIhvcNAQELBQADggEBADkBqmCUcvVTqu5IIZ5PLz40jdg2luaDHEA6I2Ga -1ioabETfQhXeaNJflojYm0Bzsy2aneVLGM2KaZ76wN0yvib3MZ4miu4C/mDsR3bB -wq7/CAK2AcJXv1jk0vIrK6DhZfA2HaelBkQ8UHwWK7AO+JmS6jozIt1vySwPI1E7 -lMFWbs3bmsSmunj3+66XS2XguUKzFwUIAEOfsPFqT2OMsPIa7weUWuCV/zMi7fuB -HbgVouYvMTve8wx7+ozDk6CyvlRlx20xwdOvXaH3JILw7gTQWcAEWZLcB2ct1Zks -UTtbIAjBV6s0Pm/2/6MxxkDCVVUpwXiiKBRkHxzkgoH7TQw= +MIIDazCCAlOgAwIBAgIUJGs/M/NeLac1U+H58kwZYgSUoMgwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTAxMTgxNTQxNDdaFw0yMTAx +MTkxNTQxNDdaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDGo1zHFcEZT4gFe24in23O1C1AtCSP2wfZCHyA9Tvj +fak3dgI5BESPHHu8lqo8V4C2ViAnBBoQ9Uq3nd38CdVJyYK5vtrHpTVLK8OIasOD +eMHBwEOMQQN6js9in0lvixBTK2oZOOcONSSxQVFUKIgSfY8sBJViYftkolPmT/OZ +bipzBoikclKDQykM9GAh/IVSQpfC+PIJT31xOXmwvy9zL5eiLDRIsZpuauI6L7/f +RkapMXtDz0QuP3aZ/VZNydagmfrz0LiBfoX7B93ZRAPLmk9KzRb/RTcuxD28d45v +KtZXUr/cR6S5gwqzb1sEaOOR+kz9aeNzwGFl6LmcgpqfAgMBAAGjUzBRMB0GA1Ud +DgQWBBTKlcyCuhJ07eTv/y33LN6/SVB1FzAfBgNVHSMEGDAWgBTKlcyCuhJ07eTv +/y33LN6/SVB1FzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBo +mv85XCM+qNvn7nNXXvI8h7PFBYZAn6trD713liiD4fURjRmQt/EDXo1ZyJhmqm3n +UHqTbQ1srnhz2zTKbBD1m4rby3bu9l0IvoCihNnd3jEzcz6IkT5TqPy6FwQdKoD4 +U0miGvmSyKs7/IyJepy6k+XUI9KSsi4k/ECx4nAvvf05Yv7XtSvPpVPHW6uvLsTu +jv8C8CcND66LrUsqn2CRkiXXX68KJSDPvT4fdNDz1nIykN0AAvVs8rA3R35dgjiR +9aHsDqkqf6QIZ0jCKbZL4Z/j5tj26P1nEUsDeZ46FECipie2z9oZII897X+Yzrcg +d9RKMlVpBwQg/iFD9orw -----END CERTIFICATE----- diff --git a/heartbeat/monitors/active/fixtures/expired.key b/heartbeat/monitors/active/fixtures/expired.key index 2a11440f7aac..3f29313cc4eb 100644 --- a/heartbeat/monitors/active/fixtures/expired.key +++ b/heartbeat/monitors/active/fixtures/expired.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCodYkuRGe2w0io -FkdyVwHqTCGhHsDEnW2qAbbjUG+PFyRsCNJrnRffN2uycRgkn8PZoVWGuGLEr+/V -BF6Jhq1la0tNQpn9VUSDpsXr1jD4b5xOlB/uVcSlAZOaKDUrcuIh5AxctouVlo8Y -r3Xul6p5/3VUIbk/7Fvq1P4KwrBnHVMKrFpwueO3pKomEU7cma3YflNCkSqsuYgZ -R/VqLmvgp15rA+/uDIBnpBK4PVy0R2L3izpi7nGzZQCJUcnzA2S9UZV7n3BsIcyL -cyTTPTdDSeW8K6bj9EWJWG2zGGpfm9U59nvKpDJvPIZ4kJLruvZFKSnnsCDq+try -CrSNu9xhAgMBAAECggEBAIc32QYvWESmWeK6B11rI5lqxK+snLT1XLpSp/esb++e -dtjU9/nzXd8JgEP6bZOwPiepTZpW1MjmJA+Lc0rWtMYsqoP4ityDHfzC2CmmgyZX -iFK2qS7I35BHRLA/x/X5QDRN9fJRgJdxA6mf5Xy/dtJ4UDhY3XbHBTzo/IWsoqYQ -4V3WBQYMGlhBArCoOx07pwc9NMTnXwpfe4rUdm3EaGGpe/9JT08JcTyFZfFUeFT1 -lfSYo5i+xPOCQ/FcC5GfWdciyY0c8ej8iwdxZb0kPI4hBu36+D6zD+YoNoC3CQTb -MecRFQ0MeTTuUMCdzFWtg+2FWnJucaLiaK9fKbVzi7UCgYEA0BAlfUdXdeDYMlW3 -2ReeOgH32bchPYwn2UvHYkIhhDp40STVw3BYQ0Zj9yJQXLFaoY1SFhwRJR1kpbSd -IfME/IzR/oMFvRUNQEPQZVH0Mg9FWIXLdXlV4qbU3AyA2r4x+VUCt3jp1n/5rG7g -cmoKBdCXNUAhK30bRGTdXB06Fp8CgYEAz0V+IlkGyDKcyCkja0ypA3AdSod/43az -7HMS3nf32hOFpgQuEtVYZc3NW/rdJFPksnRd6+RlD2nIoHZEa+adl2gESjGH2asw -nhxP/Pr4m8PGZF5BwdILRTVFukf5yrM6g63FgrgA9d+QdCsqoqrctItRyCgcfpL4 -XYXEKVWELP8CgYATxbUKVsFE/n0NK4AxLUFoGc/H7NNH2g3fZIgqGka9HiFlHq8B -x6dbnVDap3QjmucV+ywV1cz7TfPGm4djNoj+xxMdsK3W7i04MjmXp1Yhe7oHU4+m -NkWnKFuKHdYQ84okO6Pqc58lNzwu2sqRlOom60+zS8jbLSRuN3ehzVU72QKBgGm0 -qCo+Ou44maqfCFg9hWiicd3Dkt5feE0bNsFMb5PBJwTO1ux175ojxhqlqshPHLBC -FnAqT7v3mAD1r9lTiIVh3+YysnS5EJdiGw0KtWVDB9fCFkkRpPvLul7RPDw7AZmM -MtGCo8LBHHuSVDEXcG2HK9MnWbjXnWCcyrjFyx3jAoGAYsNGYm+OBr16NNsPtx3S -nRQJz9wqB2mIqNU8rRSjd5EUp03jhHiTEN9DT6iEnLGaTDBUgD2RlPvEVGk1N7FT -nh9tLtg2ytWIC/P+QrKwzdUUa00MSswTxRS3Cmy459UbLBiPgHBJ2h1G7gsiHPOt -erJWqYJ8DXvLzCPdMVzQxj8= +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDGo1zHFcEZT4gF +e24in23O1C1AtCSP2wfZCHyA9Tvjfak3dgI5BESPHHu8lqo8V4C2ViAnBBoQ9Uq3 +nd38CdVJyYK5vtrHpTVLK8OIasODeMHBwEOMQQN6js9in0lvixBTK2oZOOcONSSx +QVFUKIgSfY8sBJViYftkolPmT/OZbipzBoikclKDQykM9GAh/IVSQpfC+PIJT31x +OXmwvy9zL5eiLDRIsZpuauI6L7/fRkapMXtDz0QuP3aZ/VZNydagmfrz0LiBfoX7 +B93ZRAPLmk9KzRb/RTcuxD28d45vKtZXUr/cR6S5gwqzb1sEaOOR+kz9aeNzwGFl +6LmcgpqfAgMBAAECggEAdd93LMSiA80IIIiwQGdvF+8cs2qyz5LgQ1Af4b6kvUhS +ZBDpHMCFlo8GdlDJ0Gj0fj75cW2jsjB9GBzcH7YOFW0439R7q0FEMMlVjNweDSRZ +hgn7i30A/XdFZXa0czaIh3IZ0cHMisfKcE0cX951cNOFZE+L/ge60Grrcp47Je4H +r5F8fBaP2j729fQlpF3Kzl0uYCRqn9CD+KRMmz8nx9ifxmPM4ddVK8Xg8quVSU1x +HyeN7VfF0Hcvsv6+DRiZqgASu9pSggVffEmW4EBRNuJK4RPKFUriTTUTd8DUS39d +xxA4CjM/t+YjQOYcKlZGxxqu+ILPJnSTfyVe+lvp0QKBgQDvi5GRPuVGegO8nPqE +EHn9fh0CuVtRZjr+7YwJmlYY6WsEuVFOGlo+OdPQwk+iJuT187FYc83PwozRRoI+ +NfMC8lByMGXK0deTfCuK422U+PBuXP/U3xEK6rb1plFiNyw37wxivRTvSX/xZ8ZI +u+zS89LmWQn4oX2fbJUjcfol0wKBgQDUSHAv4PgYQutydgm2n2+asY/BeZupKqmR +FUINmcM8zZ6OC37xrT2TZoGT4WfCRJlvsj6Gpwzf6gM+MAluTQJ6ZxdAP8ifsUJT +69lyRL4o11qLvXsIexdHFUDRxv9E0pWQbEOUH7dZnWuZAXOc9sBVQLW7Hb9UJhzU +JNhsjWK8hQKBgESqO0XpQ4uaOiv8y4rDtlRFrEc1nsmMhmjA6x1tkjR95GFsRbQf +tbKUnPUAXhdbEtK10iZKu7pMSFVM4tS2Xjx//TNeUC99S1BJjam4W5cSbMkV+3en +oZkBgwSTcky3CAPYkDJVhQS6iuRGHP6Ib/BgWBoTd9o/YGUsmOuaJlxlAoGAUkH7 +juH64NPVjlXk76oaQfQ6AzuPMN8CM+Dr5n9FoyL+JIaZ2yZJLPvUjCwU27dfSWPl +TKIgoAd62DcfAGnA4xNddB0DLbYvQHusl9XBZewE/w8eRKwF2hqHMoKM2etL31nE +WfpsPsJ8Nh7U4ObQonhfszEUuZPHKgFZB79Fmk0CgYB0XEuZuSva3uKACbQQS2L2 +jeO23cmE+rSdfw9tL6O2OLZkKbDkHxMXu/G305ciyW+Hmw4CwC6B2IDJuCjzYGVr +7FL/XP0d/pksOIqjNi2VQvARO+7NStXqcgmB9p/vlyYkKLIcGKQ2w169QaNOLylc +L/DKQsIUqMqNsRcd4at/Kg== -----END PRIVATE KEY----- diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go index 2e5a43656adf..cd2c03dee3f1 100644 --- a/heartbeat/monitors/active/http/http_test.go +++ b/heartbeat/monitors/active/http/http_test.go @@ -626,7 +626,12 @@ func TestNewRoundTripper(t *testing.T) { require.NotNil(t, transp.Dial) require.NotNil(t, transport.TLSDialer) - require.Equal(t, (&tlscommon.TLSConfig{}).ToConfig(), transp.TLSClientConfig) + expected := (&tlscommon.TLSConfig{}).ToConfig() + require.Equal(t, expected.InsecureSkipVerify, transp.TLSClientConfig.InsecureSkipVerify) + // When we remove support for the legacy common name treatment + // this test has to be adjusted, as we will not depend on our + // VerifyConnection callback. + require.NotNil(t, transp.TLSClientConfig.VerifyConnection) require.True(t, transp.DisableKeepAlives) }) } diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile index 6df4d47d8857..36af746307c1 100644 --- a/journalbeat/Dockerfile +++ b/journalbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 7e875edcf8e4..1eb7814f8344 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -472,6 +472,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -599,6 +605,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -796,6 +808,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -952,6 +970,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1248,6 +1272,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1445,6 +1475,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index b7dde2b92cfa..cc9829656ed7 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/libbeat/_meta/config/ssl.reference.yml.tmpl b/libbeat/_meta/config/ssl.reference.yml.tmpl index 88f638e21b0f..69b666f9c97e 100644 --- a/libbeat/_meta/config/ssl.reference.yml.tmpl +++ b/libbeat/_meta/config/ssl.reference.yml.tmpl @@ -7,6 +7,12 @@ # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. +# * strict, which verifies that the provided certificate is signed by a trusted +# authority (CA) and also verifies that the server's hostname (or IP address) +# matches the names identified within the certificate. If the Subject Alternative +# Name is empty, it returns an error. +# * certificate, which verifies that the provided certificate is signed by a +# trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index 0373297fd000..10ece84dc47f 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -62,7 +62,7 @@ func TestTLSDialer( tlsConfig = lastTLSConfig } if tlsConfig == nil { - tlsConfig = config.BuildModuleConfig(host) + tlsConfig = config.BuildModuleClientConfig(host) lastNetwork = network lastAddress = address lastTLSConfig = tlsConfig diff --git a/libbeat/common/transport/tlscommon/ca_pinning_test.go b/libbeat/common/transport/tlscommon/ca_pinning_test.go index c188a20e63a6..ac441225090d 100644 --- a/libbeat/common/transport/tlscommon/ca_pinning_test.go +++ b/libbeat/common/transport/tlscommon/ca_pinning_test.go @@ -44,22 +44,22 @@ var ser int64 = 1 func TestCAPinning(t *testing.T) { host := "127.0.0.1" - t.Run("when the ca_sha256 field is not defined we use normal certificate validation", - func(t *testing.T) { - cfg := common.MustNewConfigFrom(map[string]interface{}{ - "certificate_authorities": []string{"ca_test.pem"}, - }) + t.Run("when the ca_sha256 field is not defined we use normal certificate validation", func(t *testing.T) { + cfg := common.MustNewConfigFrom(map[string]interface{}{ + "verification_mode": "strict", + "certificate_authorities": []string{"ca_test.pem"}, + }) - config := &Config{} - err := cfg.Unpack(config) - require.NoError(t, err) + config := &Config{} + err := cfg.Unpack(config) + require.NoError(t, err) - tlsCfg, err := LoadTLSConfig(config) - require.NoError(t, err) + tlsCfg, err := LoadTLSConfig(config) + require.NoError(t, err) - tls := tlsCfg.BuildModuleConfig(host) - require.Nil(t, tls.VerifyPeerCertificate) - }) + tls := tlsCfg.BuildModuleClientConfig(host) + require.Nil(t, tls.VerifyConnection) + }) t.Run("when the ca_sha256 field is defined we use CA cert pinning", func(t *testing.T) { cfg := common.MustNewConfigFrom(map[string]interface{}{ @@ -73,83 +73,93 @@ func TestCAPinning(t *testing.T) { tlsCfg, err := LoadTLSConfig(config) require.NoError(t, err) - tls := tlsCfg.BuildModuleConfig(host) - require.NotNil(t, tls.VerifyPeerCertificate) + tls := tlsCfg.BuildModuleClientConfig(host) + require.NotNil(t, tls.VerifyConnection) }) t.Run("CA Root -> Certificate and we have the CA root pin", func(t *testing.T) { - msg := []byte("OK received message") - - ca, err := genCA() - require.NoError(t, err) - - serverCert, err := genSignedCert(ca, x509.KeyUsageDigitalSignature, false) - require.NoError(t, err) - - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write(msg) - }) - - // Select a random available port from the OS. - addr := "localhost:0" - - l, err := net.Listen("tcp", addr) - - server := &http.Server{ - Handler: mux, - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{ - serverCert, - }, - }, - } - - // Start server and shut it down when the tests are over. - go server.ServeTLS(l, "", "") - defer l.Close() - - // Root CA Pool - require.NoError(t, err) - rootCAs := x509.NewCertPool() - rootCAs.AddCert(ca.Leaf) - - // Get the pin of the RootCA. - pin := Fingerprint(ca.Leaf) - - tlsC := &TLSConfig{ - RootCAs: rootCAs, - CASha256: []string{pin}, + verificationModes := []TLSVerificationMode{ + VerifyFull, + VerifyStrict, + VerifyCertificate, } - - config := tlsC.BuildModuleConfig("localhost") - hostToConnect := l.Addr().String() - - transport := &http.Transport{ - TLSClientConfig: config, + for _, mode := range verificationModes { + t.Run(mode.String(), func(t *testing.T) { + msg := []byte("OK received message") + + ca, err := genCA() + require.NoError(t, err) + + serverCert, err := genSignedCert(ca, x509.KeyUsageDigitalSignature, false) + require.NoError(t, err) + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(msg) + }) + + // Select a random available port from the OS. + addr := "localhost:0" + + l, err := net.Listen("tcp", addr) + + server := &http.Server{ + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{ + serverCert, + }, + }, + } + + // Start server and shut it down when the tests are over. + go server.ServeTLS(l, "", "") + defer l.Close() + + // Root CA Pool + require.NoError(t, err) + rootCAs := x509.NewCertPool() + rootCAs.AddCert(ca.Leaf) + + // Get the pin of the RootCA. + pin := Fingerprint(ca.Leaf) + + tlsC := &TLSConfig{ + Verification: mode, + RootCAs: rootCAs, + CASha256: []string{pin}, + } + + config := tlsC.BuildModuleClientConfig("localhost") + hostToConnect := l.Addr().String() + + transport := &http.Transport{ + TLSClientConfig: config, + } + + client := &http.Client{Transport: transport} + + port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") + + req, err := http.NewRequest("GET", "https://localhost:"+port, nil) + require.NoError(t, err) + resp, err := client.Do(req) + require.NoError(t, err) + content, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + assert.True(t, bytes.Equal(msg, content)) + + // 1. create key-pair + // 2. create pin + // 3. start server + // 4. Connect + // 5. Check wrong key do not work + // 6. Check good key work + // 7. check plain text fails to work. + }) } - - client := &http.Client{Transport: transport} - - port := strings.TrimPrefix(hostToConnect, "127.0.0.1:") - - req, err := http.NewRequest("GET", "https://localhost:"+port, nil) - require.NoError(t, err) - resp, err := client.Do(req) - require.NoError(t, err) - content, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - - assert.True(t, bytes.Equal(msg, content)) - - // 1. create key-pair - // 2. create pin - // 3. start server - // 4. Connect - // 5. Check wrong key do not work - // 6. Check good key work - // 7. check plain text fails to work. }) t.Run("CA Root -> Intermediate -> Certificate and we receive the CA Root Pin", func(t *testing.T) { @@ -205,7 +215,7 @@ func TestCAPinning(t *testing.T) { CASha256: []string{pin}, } - config := tlsC.BuildModuleConfig("localhost") + config := tlsC.BuildModuleClientConfig("localhost") hostToConnect := l.Addr().String() transport := &http.Transport{ @@ -279,7 +289,7 @@ func TestCAPinning(t *testing.T) { CASha256: []string{pin}, } - config := tlsC.BuildModuleConfig("localhost") + config := tlsC.BuildModuleClientConfig("localhost") hostToConnect := l.Addr().String() transport := &http.Transport{ @@ -343,6 +353,7 @@ func genCA() (tls.Certificate, error) { func genSignedCert(ca tls.Certificate, keyUsage x509.KeyUsage, isCA bool) (tls.Certificate, error) { // Create another Cert/key cert := &x509.Certificate{ + DNSNames: []string{"localhost"}, SerialNumber: big.NewInt(2000), Subject: pkix.Name{ CommonName: "localhost", diff --git a/libbeat/common/transport/tlscommon/config.go b/libbeat/common/transport/tlscommon/config.go index 8d7650eb5bfd..30009c2b13be 100644 --- a/libbeat/common/transport/tlscommon/config.go +++ b/libbeat/common/transport/tlscommon/config.go @@ -21,6 +21,8 @@ import ( "crypto/tls" "github.com/joeshaw/multierror" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" ) // Config defines the user configurable options in the yaml file. @@ -96,6 +98,8 @@ func LoadTLSConfig(config *Config) (*TLSConfig, error) { // Validate values the TLSConfig struct making sure certificate sure we have both a certificate and // a key. func (c *Config) Validate() error { + cfgwarn.Deprecate("8.0.0", "Treating the CommonName field on X.509 certificates as a host name when no Subject Alternative Names are present is going to be removed. Please update your certificates if needed.") + return c.Certificate.Validate() } diff --git a/libbeat/common/transport/tlscommon/server_config.go b/libbeat/common/transport/tlscommon/server_config.go index 866d6e3c28c7..e85a0c409c3a 100644 --- a/libbeat/common/transport/tlscommon/server_config.go +++ b/libbeat/common/transport/tlscommon/server_config.go @@ -28,13 +28,14 @@ import ( // ServerConfig defines the user configurable tls options for any TCP based service. type ServerConfig struct { Enabled *bool `config:"enabled"` - VerificationMode TLSVerificationMode `config:"verification_mode"` // one of 'none', 'full' + VerificationMode TLSVerificationMode `config:"verification_mode"` // one of 'none', 'full', 'strict', 'certificate' Versions []TLSVersion `config:"supported_protocols"` CipherSuites []tlsCipherSuite `config:"cipher_suites"` CAs []string `config:"certificate_authorities"` Certificate CertificateConfig `config:",inline"` CurveTypes []tlsCurveType `config:"curve_types"` ClientAuth tlsClientAuth `config:"client_authentication"` //`none`, `optional` or `required` + CASha256 []string `config:"ca_sha256" yaml:"ca_sha256,omitempty"` } // LoadTLSServerConfig tranforms a ServerConfig into a `tls.Config` to be used directly with golang @@ -88,6 +89,7 @@ func LoadTLSServerConfig(config *ServerConfig) (*TLSConfig, error) { CipherSuites: cipherSuites, CurvePreferences: curves, ClientAuth: tls.ClientAuthType(config.ClientAuth), + CASha256: config.CASha256, }, nil } diff --git a/libbeat/common/transport/tlscommon/testdata/cacert.crt b/libbeat/common/transport/tlscommon/testdata/cacert.crt new file mode 100644 index 000000000000..debdf7e246e0 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/cacert.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIIEBDCCAuygAwIBAgIUXwbLbwGjWWlQNrMUsdDpKzeGixEwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBQMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDTALBgNV +BAsMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtXsn+VCrW +ibutoByM5EeIK29XYffBwN78EeNjDdaZZqMF4wGZZ6z2xQXH6mFx+m1gjnf5R2qo +yfentYH5VRZz5AEtBGPsOqMffV9u5PkHSo/2ilCX40eBVp5u3qh6aFPZ5DKqexWu +5jUMYolTXpvAtML5YbMH9XvW6pn5WAqwHPLNe+fVuPg4tJN0u/ff0wKqSUBIhVOP +7EPhz3yLflACScgj+LPXz/5gtUXe9RR5RB8zyWGfNL91eoVVaApcdp4kIU+DHmgI +p+T4CpgdYWsYuOWH49F7RJyLpocUU4H+heeC4+zH0LIUcELa+n/M2DUDW3RE109a +tv9OEJKR8/YHAgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +fyEN1Qe7FlWa+2RBnl8Vd4ZCFkIwgY0GA1UdIwSBhTCBgoAUfyEN1Qe7FlWa+2RB +nl8Vd4ZCFkKhVKRSMFAxCzAJBgNVBAYTAkNBMQ8wDQYDVQQIDAZRdWViZWMxETAP +BgNVBAcMCE1vbnRyZWFsMQ4wDAYDVQQKDAViZWF0czENMAsGA1UECwwEcm9vdIIU +XwbLbwGjWWlQNrMUsdDpKzeGixEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +CwUAA4IBAQAANxJCfDMcNNnAVRlXLdh+loVx8Y5STf1gTgX2gtf9tHZGYE7/ix2P +dG1uQcEz/ETlcGSWRZcQSNR8dNeBi5YWK5dmDUD7reQr3FoyIDvPGHyIcF3clglg +blYhsQN0TVwx4G3kZDenjzKNSyVLR81opLq/PDIGW61ZCioJUQKs5q+IqsKj+okn +in6/b5YfQqyTDIWY3IPiXjvcysbKC0pYc0TkmwGUnidxDny7txrVCVJ1vwIedQug +B/UOjVxi0qsNwpWS08mwEOVvgvObi0mFoGQl8l427M0kM//86NM7vDc4Z0QYHOlq +A0ZjtnSbR3RqfhBGXV3BL+GHtXevn55Z +-----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/cacert.key b/libbeat/common/transport/tlscommon/testdata/cacert.key new file mode 100644 index 000000000000..e864b93ed663 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/cacert.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArV7J/lQq1om7raAcjORHiCtvV2H3wcDe/BHjYw3WmWajBeMB +mWes9sUFx+phcfptYI53+UdqqMn3p7WB+VUWc+QBLQRj7DqjH31fbuT5B0qP9opQ +l+NHgVaebt6oemhT2eQyqnsVruY1DGKJU16bwLTC+WGzB/V71uqZ+VgKsBzyzXvn +1bj4OLSTdLv339MCqklASIVTj+xD4c98i35QAknII/iz18/+YLVF3vUUeUQfM8lh +nzS/dXqFVWgKXHaeJCFPgx5oCKfk+AqYHWFrGLjlh+PRe0Sci6aHFFOB/oXnguPs +x9CyFHBC2vp/zNg1A1t0RNdPWrb/ThCSkfP2BwIDAQABAoIBAQCQmLJYENL5xD5n +/VZSnEKc670dYHRHgRl5m2HPR8doghYN3tuCmtnDp2e+6VkEux1mnuypWEs5I9oO +YnBZCAKF/fCNH1BHwlAy/1oNH6Qj1Khls86sH7+PvDK/va0/CqyE2rL3RVk8Wnx8 +K+LlSc8V1q2XWUj8pl33TgvFzwx6/QpmGa1ofK84GaeWNskRt8xyf2HECiRl6ZFm +zZr2Ror3nRbgZK9FYWpcp6HUgxAH/8GQ3+8vMvftfTsDGD5TmmEq6CFgAFCVj92L +d7AZmNWR1483NzZF0HWOQ6ew9qrWkqVpER7kKKp/kkfoh2qXgvtQBTrw4IcCRwwa +szaSsIEBAoGBANiqXhBzPQJszm1Ajln07ZeyvgRB8PgzZXcAHS9AfGqh/mGQw5/X +3vqHdGiEynphoYtNqK1YT7RH7pkjkpqDzdunZGz1xog7i4ys8kVtivkDGlhn6cXI +4wmFcmyCaf76VPPr1RX8PNjsEKDK3jq1d86lBjSLPgcHT7J16WZgOcJnAoGBAMzY +QVNpjk1WNT7gid3MUXciIIZAovej4AiVyn97XxxLSyByXmNds65f3dM8NOJkJUvT +iV7pAjKl9pd1lE+WTNQSjCgSxw7G+4u9cQfNE7p6klAh/Rek76Mani9rAmQ2PdJl +EFaEgLom3wbR5eOkYURjw2jfqzFYQ8T1YZkWBithAoGAa3EYkknDIFe6ifzwWnWV ++Jr/lXbpuvspvrhEwLDWwb4xOkqiZ7qR7WSMemQXUFbn1/+bvNJFPB5LmI9GXO8t +f1Zj+5BpchctHYaJ4Znvx4odX2ewSo9S3t7ZHiwRygpzZD43fd6Ggf+WQ1Y2m6Bv +l/7Hs/i0uqGKiPHl2wmuutMCgYABZN9c7/T19cY6/VAy4DcVtne+MiZpxQW7STmt +kGtfR+vk9qJJztNwNlrOGzTI7aGLWI8wxCktqw94jGZL/FvdfZrSkv4jzZrcopdo +VC70L+1a+kA8rvSqiX3WGMZVZEEbc3CfBhvSKH2QEFGeMPowevVTe2Iw3cboSjs1 +zX6RQQKBgFV7gOstMfvixCSUCD2s5j/skhNJsB3Wd/tVYRbl/vgA6hHW8UOy2oWv +UTE45vJNVzRv030G5katjOYhlxHf9rpeSAbeIyty54I3X9/vDJZLXwe8WilQjUr7 +Dw8yNwH44j/0s8xcQXG8yE0h1Aa9GxHHtJtYrRYdx7sSwNHtwpnp +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/testdata/client1.crt b/libbeat/common/transport/tlscommon/testdata/client1.crt new file mode 100644 index 000000000000..c3139a72a77d --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/client1.crt @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIEFzCCAv+gAwIBAgIUeaB7uk2DjAM2cuRl0kaE9ly7Lj4wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBmMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDzANBgNV +BAsMBnNlcnZlcjESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA3jXEj7vN+BDlj6cYblKSml0FWpO4yi9C58cubXXDWXI6 +hdpzNpDa0+n606Jg4eVZpFUZPTnnjQmFIcesO0+i85V4Etswr4T22uobDu1AWV7n +26nDMY/vlf+kDI8H/uFgxQg/Htuh12nHuYrjIS+ot/D6gThwIWVldu0TaBaFfvL5 +5qTPRJoteiBPo5y+VuWLhzPWg8cQYZ4KJ4XREk8H4d7PqFRHp+zATfn2YLBjUK7Z +zd0W3mxkdB2P7MnzZuH5n5zrgJ8OI9voopX8QadMYtUSeITP1INmNKhi4vLbpZjU +mt+N/u1G6xwbuyJiSlklBoXdRcWj5kSljpLtF1evvwIDAQABo4HQMIHNMAwGA1Ud +EwEB/wQCMAAwHQYDVR0OBBYEFAuDdHxE9/Zr7iVwfnUJ/lRtJnZkMIGNBgNVHSME +gYUwgYKAFH8hDdUHuxZVmvtkQZ5fFXeGQhZCoVSkUjBQMQswCQYDVQQGEwJDQTEP +MA0GA1UECAwGUXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVh +dHMxDTALBgNVBAsMBHJvb3SCFF8Gy28Bo1lpUDazFLHQ6Ss3hosRMA4GA1UdDwEB +/wQEAwIF4DANBgkqhkiG9w0BAQsFAAOCAQEACzuX6AiVHk5Igs/LdOW2sJ9lm95N +Su1PQCobM0Jo8wX3pDAEQlLmaWTDcr4bfrQPfI8pih1F89DQU9z0nzNCRfxiQaA7 +myF8ftvf8v5j3LpaPWlkdWgCRieCl58fgy5vtcKx73eTY4a6SRB4zbWpl0rX9H6w +En1kQbpCJDzh8W+xmr8AKvY77CSC1vt7TaKan6F+fGwbt8kIng6P6C7dvMGsDKQN +2Tiq/wtH16DB8mOeO+zfxJfa84TPWL4UcSbZJ8w5Fyz4GJormaymxJGtKv58RO7J +u63WF9vlEnKGyqY1FckTsp3P9ivGEb/Y75+NyRwmNq5VO5BPrRBMOF3VAg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEBDCCAuygAwIBAgIUXwbLbwGjWWlQNrMUsdDpKzeGixEwDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCQ0ExDzANBgNVBAgMBlF1ZWJlYzERMA8GA1UEBwwITW9u +dHJlYWwxDjAMBgNVBAoMBWJlYXRzMQ0wCwYDVQQLDARyb290MCAXDTE5MDcyMjE5 +MjkwNVoYDzIxMTkwNjI4MTkyOTA1WjBQMQswCQYDVQQGEwJDQTEPMA0GA1UECAwG +UXVlYmVjMREwDwYDVQQHDAhNb250cmVhbDEOMAwGA1UECgwFYmVhdHMxDTALBgNV +BAsMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtXsn+VCrW +ibutoByM5EeIK29XYffBwN78EeNjDdaZZqMF4wGZZ6z2xQXH6mFx+m1gjnf5R2qo +yfentYH5VRZz5AEtBGPsOqMffV9u5PkHSo/2ilCX40eBVp5u3qh6aFPZ5DKqexWu +5jUMYolTXpvAtML5YbMH9XvW6pn5WAqwHPLNe+fVuPg4tJN0u/ff0wKqSUBIhVOP +7EPhz3yLflACScgj+LPXz/5gtUXe9RR5RB8zyWGfNL91eoVVaApcdp4kIU+DHmgI +p+T4CpgdYWsYuOWH49F7RJyLpocUU4H+heeC4+zH0LIUcELa+n/M2DUDW3RE109a +tv9OEJKR8/YHAgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU +fyEN1Qe7FlWa+2RBnl8Vd4ZCFkIwgY0GA1UdIwSBhTCBgoAUfyEN1Qe7FlWa+2RB +nl8Vd4ZCFkKhVKRSMFAxCzAJBgNVBAYTAkNBMQ8wDQYDVQQIDAZRdWViZWMxETAP +BgNVBAcMCE1vbnRyZWFsMQ4wDAYDVQQKDAViZWF0czENMAsGA1UECwwEcm9vdIIU +XwbLbwGjWWlQNrMUsdDpKzeGixEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB +CwUAA4IBAQAANxJCfDMcNNnAVRlXLdh+loVx8Y5STf1gTgX2gtf9tHZGYE7/ix2P +dG1uQcEz/ETlcGSWRZcQSNR8dNeBi5YWK5dmDUD7reQr3FoyIDvPGHyIcF3clglg +blYhsQN0TVwx4G3kZDenjzKNSyVLR81opLq/PDIGW61ZCioJUQKs5q+IqsKj+okn +in6/b5YfQqyTDIWY3IPiXjvcysbKC0pYc0TkmwGUnidxDny7txrVCVJ1vwIedQug +B/UOjVxi0qsNwpWS08mwEOVvgvObi0mFoGQl8l427M0kM//86NM7vDc4Z0QYHOlq +A0ZjtnSbR3RqfhBGXV3BL+GHtXevn55Z +-----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/client1.key b/libbeat/common/transport/tlscommon/testdata/client1.key new file mode 100644 index 000000000000..ce5274b758f4 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/client1.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA3jXEj7vN+BDlj6cYblKSml0FWpO4yi9C58cubXXDWXI6hdpz +NpDa0+n606Jg4eVZpFUZPTnnjQmFIcesO0+i85V4Etswr4T22uobDu1AWV7n26nD +MY/vlf+kDI8H/uFgxQg/Htuh12nHuYrjIS+ot/D6gThwIWVldu0TaBaFfvL55qTP +RJoteiBPo5y+VuWLhzPWg8cQYZ4KJ4XREk8H4d7PqFRHp+zATfn2YLBjUK7Zzd0W +3mxkdB2P7MnzZuH5n5zrgJ8OI9voopX8QadMYtUSeITP1INmNKhi4vLbpZjUmt+N +/u1G6xwbuyJiSlklBoXdRcWj5kSljpLtF1evvwIDAQABAoIBABdTza7JKHZCT9ck +04vBX2KVIVrA50VScNOkNVuIYVmihEJJDI9N5asZhRtykHkmeqKlzGCBE63asf85 +1vrjAVhQ+KoCGLpUWxXgPbbzcS3wqKaGy9cIJT65957Z5Rz8zAvjMb0rkXHryOvR +iMaTGkM1KRcntZ3L5zr06HSk6J7K8QCEexKHl7Q7Ki1498tvBWdJGeGWRiUtI89j +wOUdcf3pVSVqI7J8gmmqVwNrVMbVxhlen7nkckXofWAackYVQDBD+hU1n3doNKLa +NP6mZkI02BOB29WLDXLuHtKDZtgnXex4JUz6zw53uV42FCDoQf3DUiVsMEL8xRCJ +27H6bwECgYEA/w53zS00mNdYdXO7dGhAw3UYPc3PDyg6Z823BQzfdOzsn5Yw0BIw +nPgstzwzOL0kw2p/PgwkG/7LOsF5CWs2xvU3LhUdOhgmw4B5IbMOYvbkVoYGz+22 +HJf4qyexAr7tKCITB+LCzUwoAgXp8uju1XdLVpk6xmJ3u+kIhMYTxkUCgYEA3wgx +71/uIUsoW6bVL5K00yXPWTTFtTBWM768VJ8Y++k2igPgcvKaBVaElr4AbvX5iCGz +1Ycc9xsGAYAo7+q4D+4cuOki/m0PMKD3DgXWpTtN0kJ+npWUBdE98NyDlTJYsa/w +xjeMQoDvC8tE2bAiwtVIOPQL2C/3emqkJcsVcDMCgYB8NeOJ/DXdKSJfMJldu1eu +2FuR3aS00PaAjuJOh1JbcvZZUZ879V/PUd0U7zBStWot8LM+2FLNf2whlQ8I0zm9 +8rWIr6eoHxLhqrNTAgxDjdDtgh/XKwDBNBFZ6N5/Y9PC87Uo5fnQWQIy2gZw0Zde +RdZeugixjEqbLIWFg6ElsQKBgHRy6O+c3M6RWU8ROnoOVU9xjGN9REUoKbn2uopM +T1UoHQvOnmAl/vkOhUfXiI5m65SCVE0GsL7sYyRhb/5kRRo8Ls71GwpQkv/G63ds +4PeAkU9Y3JecbZ7j8z1RRXqewOR1gndcBWWrwCQeS6KFboDfr0fdVFnaIZLPH0mE +UXs1AoGBAM3zpcyl5o99dO6x9N/8SSnyLT9TzzbJ6pU6d0F0ELn3OxTUBH1oA1dy +q1fADcRgN5vNuJljY4es/scK2BMeX1isFitXoIzk01F4R61xoXr8T33731eXFG6L +ehoECH2Yj9H4qNbVW531iYKheuSyaMaxCxaDoK9jBzcKaxMGbTlc +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/tls_config.go b/libbeat/common/transport/tlscommon/tls_config.go index 22cebb2bf8d3..9e7eb4548dbf 100644 --- a/libbeat/common/transport/tlscommon/tls_config.go +++ b/libbeat/common/transport/tlscommon/tls_config.go @@ -20,8 +20,12 @@ package tlscommon import ( "crypto/tls" "crypto/x509" + "fmt" + "net" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/logp" ) @@ -75,8 +79,13 @@ type TLSConfig struct { time func() time.Time } -// ToConfig generates a tls.Config object. Note, you must use BuildModuleConfig to generate a config with +var ( + MissingPeerCertificate = errors.New("missing peer certificates") +) + +// ToConfig generates a tls.Config object. Note, you must use BuildModuleClientConfig to generate a config with // ServerName set, use that method for servers with SNI. +// By default VerifyConnection is set to client mode. func (c *TLSConfig) ToConfig() *tls.Config { if c == nil { return &tls.Config{} @@ -84,36 +93,37 @@ func (c *TLSConfig) ToConfig() *tls.Config { minVersion, maxVersion := extractMinMaxVersion(c.Versions) - // When we are using the CAsha256 pin to validate the CA used to validate the chain, - // or when we are using 'certificate' TLS verification mode, we add a custom callback - verifyPeerCertFn := makeVerifyPeerCertificate(c) - - insecure := c.Verification != VerifyFull + insecure := c.Verification != VerifyStrict if c.Verification == VerifyNone { logp.NewLogger("tls").Warn("SSL/TLS verifications disabled.") } - return &tls.Config{ - MinVersion: minVersion, - MaxVersion: maxVersion, - Certificates: c.Certificates, - RootCAs: c.RootCAs, - ClientCAs: c.ClientCAs, - InsecureSkipVerify: insecure, - CipherSuites: c.CipherSuites, - CurvePreferences: c.CurvePreferences, - Renegotiation: c.Renegotiation, - ClientAuth: c.ClientAuth, - VerifyPeerCertificate: verifyPeerCertFn, - Time: c.time, + MinVersion: minVersion, + MaxVersion: maxVersion, + Certificates: c.Certificates, + RootCAs: c.RootCAs, + ClientCAs: c.ClientCAs, + InsecureSkipVerify: insecure, + CipherSuites: c.CipherSuites, + CurvePreferences: c.CurvePreferences, + Renegotiation: c.Renegotiation, + ClientAuth: c.ClientAuth, + Time: c.time, + VerifyConnection: makeVerifyConnection(c), } } // BuildModuleConfig takes the TLSConfig and transform it into a `tls.Config`. -func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config { +func (c *TLSConfig) BuildModuleClientConfig(host string) *tls.Config { if c == nil { // use default TLS settings, if config is empty. - return &tls.Config{ServerName: host} + return &tls.Config{ + ServerName: host, + InsecureSkipVerify: true, + VerifyConnection: makeVerifyConnection(&TLSConfig{ + Verification: VerifyFull, + }), + } } config := c.ToConfig() @@ -121,33 +131,169 @@ func (c *TLSConfig) BuildModuleConfig(host string) *tls.Config { return config } -// makeVerifyPeerCertificate creates the verification combination of checking certificate pins and skipping host name validation depending on the config -func makeVerifyPeerCertificate(cfg *TLSConfig) verifyPeerCertFunc { - pin := len(cfg.CASha256) > 0 - skipHostName := cfg.Verification == VerifyCertificate - - if pin && !skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - return verifyCAPin(cfg.CASha256, verifiedChains) +// BuildServerConfig takes the TLSConfig and transform it into a `tls.Config` for server side objects. +func (c *TLSConfig) BuildServerConfig(host string) *tls.Config { + if c == nil { + // use default TLS settings, if config is empty. + return &tls.Config{ + ServerName: host, + InsecureSkipVerify: true, + VerifyConnection: makeVerifyServerConnection(&TLSConfig{ + Verification: VerifyFull, + }), } } - if pin && skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - _, _, err := verifyCertificateExceptServerName(rawCerts, cfg) + config := c.ToConfig() + config.ServerName = host + config.VerifyConnection = makeVerifyServerConnection(c) + return config +} + +func makeVerifyConnection(cfg *TLSConfig) func(tls.ConnectionState) error { + switch cfg.Verification { + case VerifyFull: + return func(cs tls.ConnectionState) error { + // On the client side, PeerCertificates can't be empty. + if len(cs.PeerCertificates) == 0 { + return MissingPeerCertificate + } + + opts := x509.VerifyOptions{ + Roots: cfg.RootCAs, + Intermediates: x509.NewCertPool(), + } + err := verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) if err != nil { return err } - return verifyCAPin(cfg.CASha256, verifiedChains) + return verifyHostname(cs.PeerCertificates[0], cs.ServerName) + } + case VerifyCertificate: + return func(cs tls.ConnectionState) error { + // On the client side, PeerCertificates can't be empty. + if len(cs.PeerCertificates) == 0 { + return MissingPeerCertificate + } + + opts := x509.VerifyOptions{ + Roots: cfg.RootCAs, + Intermediates: x509.NewCertPool(), + } + return verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + } + case VerifyStrict: + if len(cfg.CASha256) > 0 { + return func(cs tls.ConnectionState) error { + return verifyCAPin(cfg.CASha256, cs.VerifiedChains) + } } + default: } - if !pin && skipHostName { - return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { - _, _, err := verifyCertificateExceptServerName(rawCerts, cfg) - return err + return nil + +} + +func makeVerifyServerConnection(cfg *TLSConfig) func(tls.ConnectionState) error { + switch cfg.Verification { + case VerifyFull: + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + if cfg.ClientAuth == tls.RequireAndVerifyClientCert { + return MissingPeerCertificate + } + return nil + } + + opts := x509.VerifyOptions{ + Roots: cfg.ClientCAs, + Intermediates: x509.NewCertPool(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + err := verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + if err != nil { + return err + } + return verifyHostname(cs.PeerCertificates[0], cs.ServerName) + } + case VerifyCertificate: + return func(cs tls.ConnectionState) error { + if len(cs.PeerCertificates) == 0 { + if cfg.ClientAuth == tls.RequireAndVerifyClientCert { + return MissingPeerCertificate + } + return nil + } + + opts := x509.VerifyOptions{ + Roots: cfg.ClientCAs, + Intermediates: x509.NewCertPool(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + } + return verifyCertsWithOpts(cs.PeerCertificates, cfg.CASha256, opts) + } + case VerifyStrict: + if len(cfg.CASha256) > 0 { + return func(cs tls.ConnectionState) error { + return verifyCAPin(cfg.CASha256, cs.VerifiedChains) + } } + default: + } + + return nil + +} + +func verifyCertsWithOpts(certs []*x509.Certificate, casha256 []string, opts x509.VerifyOptions) error { + for _, cert := range certs[1:] { + opts.Intermediates.AddCert(cert) + } + verifiedChains, err := certs[0].Verify(opts) + if err != nil { + return err } + if len(casha256) > 0 { + return verifyCAPin(casha256, verifiedChains) + } return nil } + +func verifyHostname(cert *x509.Certificate, hostname string) error { + if hostname == "" { + return nil + } + // check if the server name is an IP + ip := hostname + if len(ip) >= 3 && ip[0] == '[' && ip[len(ip)-1] == ']' { + ip = ip[1 : len(ip)-1] + } + parsedIP := net.ParseIP(ip) + if parsedIP != nil { + for _, certIP := range cert.IPAddresses { + if parsedIP.Equal(certIP) { + return nil + } + } + return x509.HostnameError{Certificate: cert, Host: hostname} + } + + dnsnames := cert.DNSNames + if len(dnsnames) == 0 || len(dnsnames) == 1 && dnsnames[0] == "" { + if cert.Subject.CommonName != "" { + dnsnames = []string{cert.Subject.CommonName} + } + } + + for _, name := range dnsnames { + if len(name) > 0 && len(hostname) > 0 && name == hostname { + if !validHostname(name, true) { + return fmt.Errorf("invalid hostname in cert") + } + return nil + } + } + return x509.HostnameError{Certificate: cert, Host: hostname} +} diff --git a/libbeat/common/transport/tlscommon/tls_config_test.go b/libbeat/common/transport/tlscommon/tls_config_test.go new file mode 100644 index 000000000000..1490664d3d35 --- /dev/null +++ b/libbeat/common/transport/tlscommon/tls_config_test.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tlscommon + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMakeVerifyServerConnection(t *testing.T) { + testCerts, err := openTestCerts() + if err != nil { + t.Fatalf("failed to open test certs: %+v", err) + } + + testCA, errs := LoadCertificateAuthorities([]string{filepath.Join("testdata", "cacert.crt")}) + if len(errs) > 0 { + t.Fatalf("failed to load test certificate authorities: %+v", errs) + } + + testcases := map[string]struct { + verificationMode TLSVerificationMode + clientAuth tls.ClientAuthType + certAuthorities *x509.CertPool + peerCerts []*x509.Certificate + serverName string + expectedCallback bool + expectedError error + }{ + "default verification without certificates when required": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + peerCerts: nil, + serverName: "", + expectedCallback: true, + expectedError: MissingPeerCertificate, + }, + "default verification with certificates when required with expired cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["expired"]}, + serverName: "", + expectedCallback: true, + expectedError: x509.CertificateInvalidError{Cert: testCerts["expired"], Reason: x509.Expired}, + }, + "default verification with certificates when required with incorrect server name in cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "bad.example.com", + expectedCallback: true, + expectedError: x509.HostnameError{Certificate: testCerts["correct"], Host: "bad.example.com"}, + }, + "default verification with certificates when required with correct cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: nil, + }, + "certificate verification with certificates when required with correct cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: nil, + }, + "certificate verification with certificates when required with expired cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["expired"]}, + serverName: "localhost", + expectedCallback: true, + expectedError: x509.CertificateInvalidError{Cert: testCerts["expired"], Reason: x509.Expired}, + }, + "certificate verification with certificates when required with incorrect server name in cert": { + verificationMode: VerifyCertificate, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "bad.example.com", + expectedCallback: true, + expectedError: nil, + }, + "strict verification with certificates when required with correct cert": { + verificationMode: VerifyStrict, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["correct"]}, + serverName: "localhost", + expectedCallback: false, + expectedError: nil, + }, + "default verification with certificates when required with cert signed by unkown authority": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["unknown authority"]}, + serverName: "", + expectedCallback: true, + expectedError: x509.UnknownAuthorityError{Cert: testCerts["unknown authority"]}, + }, + "default verification without certificates not required": { + verificationMode: VerifyFull, + clientAuth: tls.NoClientCert, + peerCerts: nil, + serverName: "", + expectedCallback: true, + expectedError: nil, + }, + "no verification without certificates not required": { + verificationMode: VerifyNone, + clientAuth: tls.NoClientCert, + peerCerts: nil, + serverName: "", + expectedError: nil, + }, + } + + for name, test := range testcases { + t.Run(name, func(t *testing.T) { + test := test + cfg := &TLSConfig{ + Verification: test.verificationMode, + ClientAuth: test.clientAuth, + ClientCAs: test.certAuthorities, + } + + verifier := makeVerifyServerConnection(cfg) + if !test.expectedCallback { + assert.Nil(t, verifier) + return + } + + err := verifier(tls.ConnectionState{ + PeerCertificates: test.peerCerts, + ServerName: test.serverName, + }) + if test.expectedError == nil { + assert.Nil(t, err) + } else { + assert.Error(t, test.expectedError, err) + } + }) + } + +} + +func openTestCerts() (map[string]*x509.Certificate, error) { + certs := make(map[string]*x509.Certificate, 0) + + for testcase, certname := range map[string]string{ + "expired": "tls.crt", + "unknown authority": "unsigned_tls.crt", + "correct": "client1.crt", + } { + + certBytes, err := ioutil.ReadFile(filepath.Join("testdata", certname)) + if err != nil { + return nil, err + } + block, _ := pem.Decode(certBytes) + testCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + certs[testcase] = testCert + } + + return certs, nil +} diff --git a/libbeat/common/transport/tlscommon/tls_test.go b/libbeat/common/transport/tlscommon/tls_test.go index 53e9da18db3d..45c0ebf1f7fd 100644 --- a/libbeat/common/transport/tlscommon/tls_test.go +++ b/libbeat/common/transport/tlscommon/tls_test.go @@ -132,12 +132,12 @@ func TestApplyEmptyConfig(t *testing.T) { t.Fatal(err) } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Len(t, cfg.Certificates, 0) assert.Nil(t, cfg.RootCAs) - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) assert.Equal(t, tls.RenegotiateNever, cfg.Renegotiation) @@ -159,7 +159,7 @@ func TestApplyWithConfig(t *testing.T) { t.Fatal(err) } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) assert.Len(t, cfg.Certificates, 1) assert.NotNil(t, cfg.RootCAs) @@ -184,7 +184,7 @@ key: mykey.pem tmp, err := LoadTLSServerConfig(&c) require.NoError(t, err) - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) // values not set by default @@ -193,7 +193,7 @@ key: mykey.pem assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) // values set by default - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Equal(t, tls.NoClientCert, cfg.ClientAuth) @@ -213,7 +213,7 @@ key: mykey.pem tmp, err := LoadTLSServerConfig(&c) require.NoError(t, err) - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) // values not set by default @@ -222,7 +222,7 @@ key: mykey.pem assert.Len(t, cfg.CipherSuites, 0) assert.Len(t, cfg.CurvePreferences, 0) // values set by default - assert.Equal(t, false, cfg.InsecureSkipVerify) + assert.Equal(t, true, cfg.InsecureSkipVerify) assert.Equal(t, int(TLSVersionDefaultMin), int(cfg.MinVersion)) assert.Equal(t, int(TLSVersionDefaultMax), int(cfg.MaxVersion)) assert.Equal(t, tls.RequireAndVerifyClientCert, cfg.ClientAuth) @@ -260,7 +260,7 @@ func TestApplyWithServerConfig(t *testing.T) { return } - cfg := tmp.BuildModuleConfig("") + cfg := tmp.BuildModuleClientConfig("") assert.NotNil(t, cfg) assert.Len(t, cfg.Certificates, 1) assert.NotNil(t, cfg.ClientCAs) diff --git a/libbeat/common/transport/tlscommon/types.go b/libbeat/common/transport/tlscommon/types.go index c130a57c71a0..29b11c920100 100644 --- a/libbeat/common/transport/tlscommon/types.go +++ b/libbeat/common/transport/tlscommon/types.go @@ -127,11 +127,13 @@ const ( VerifyFull TLSVerificationMode = iota VerifyNone VerifyCertificate + VerifyStrict ) var tlsVerificationModes = map[string]TLSVerificationMode{ "": VerifyFull, "full": VerifyFull, + "strict": VerifyStrict, "none": VerifyNone, "certificate": VerifyCertificate, } diff --git a/libbeat/common/transport/tlscommon/verify.go b/libbeat/common/transport/tlscommon/validhostname.go similarity index 59% rename from libbeat/common/transport/tlscommon/verify.go rename to libbeat/common/transport/tlscommon/validhostname.go index 867e70874a9f..15370b4d4f99 100644 --- a/libbeat/common/transport/tlscommon/verify.go +++ b/libbeat/common/transport/tlscommon/validhostname.go @@ -43,58 +43,55 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// This file contains code adapted from golang's crypto/tls/handshake_client.go +// This file contains code adapted from golang's crypto/x509/verify.go package tlscommon -import ( - "crypto/x509" - "time" +import "strings" - "github.com/pkg/errors" -) - -// verifyCertificateExceptServerName is a TLS Certificate verification utility method that verifies that the provided -// certificate chain is valid and is signed by one of the root CAs in the provided tls.Config. It is intended to be -// as similar as possible to the default verify, but does not verify that the provided certificate matches the -// ServerName in the tls.Config. -func verifyCertificateExceptServerName( - rawCerts [][]byte, - c *TLSConfig, -) ([]*x509.Certificate, [][]*x509.Certificate, error) { - // this is where we're a bit suboptimal, as we have to re-parse the certificates that have been presented - // during the handshake. - // the verification code here is taken from verifyServerCertificate in crypto/tls/handshake_client.go:824 - certs := make([]*x509.Certificate, len(rawCerts)) - for i, asn1Data := range rawCerts { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - return nil, nil, errors.Wrap(err, "tls: failed to parse certificate from server") - } - certs[i] = cert +// validHostname reports whether host is a valid hostname that can be matched or +// matched against according to RFC 6125 2.2, with some leniency to accommodate +// legacy values. +func validHostname(host string, isPattern bool) bool { + if !isPattern { + host = strings.TrimSuffix(host, ".") } - - var t time.Time - if c.time != nil { - t = c.time() - } else { - t = time.Now() - } - - // DNSName omitted in VerifyOptions in order to skip ServerName verification - opts := x509.VerifyOptions{ - Roots: c.RootCAs, - CurrentTime: t, - Intermediates: x509.NewCertPool(), + if len(host) == 0 { + return false } - for _, cert := range certs[1:] { - opts.Intermediates.AddCert(cert) + for i, part := range strings.Split(host, ".") { + if part == "" { + // Empty label. + return false + } + if isPattern && i == 0 && part == "*" { + // Only allow full left-most wildcards, as those are the only ones + // we match, and matching literal '*' characters is probably never + // the expected behavior. + continue + } + for j, c := range part { + if 'a' <= c && c <= 'z' { + continue + } + if '0' <= c && c <= '9' { + continue + } + if 'A' <= c && c <= 'Z' { + continue + } + if c == '-' && j != 0 { + continue + } + if c == '_' { + // Not a valid character in hostnames, but commonly + // found in deployments outside the WebPKI. + continue + } + return false + } } - headCert := certs[0] - - // defer to the default verification performed - chains, err := headCert.Verify(opts) - return certs, chains, err + return true } diff --git a/libbeat/common/transport/tlscommon/verify_test.go b/libbeat/common/transport/tlscommon/verify_test.go deleted file mode 100644 index c08e60642f52..000000000000 --- a/libbeat/common/transport/tlscommon/verify_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package tlscommon - -import ( - "crypto/x509" - "encoding/pem" - "io/ioutil" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// This does not actually test that it ignores the server name because no part of the func even consumes the server name -func Test_verifyCertificateExceptServerName(t *testing.T) { - - tests := []struct { - name string - ca string - chain string - cert string - time func() time.Time - wantErr bool - }{ - { - name: "happy path", - // a CA for morello.ovh valid from August 9 2019 to 2029 - ca: "ca.crt", - // a cert signed by morello.ovh that expired in nov 2019 - cert: "tls.crt", - time: func() time.Time { - layout := "2006-01-02" - t, _ := time.Parse(layout, "2019-10-01") - return t - }, - wantErr: false, - }, - { - name: "cert not signed by CA", - ca: "ca.crt", - // a self-signed cert for www.example.com valid from July 23 2020 to 2030 - cert: "unsigned_tls.crt", - time: func() time.Time { - layout := "2006-01-02" - t, _ := time.Parse(layout, "2020-07-24") - return t - }, - wantErr: true, - }, - { - name: "cert expired", - ca: "ca.crt", - cert: "tls.crt", - wantErr: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := &TLSConfig{time: tc.time} - // load the CA - if tc.ca != "" { - ca := loadFileBytes(tc.ca) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(ca) - cfg.RootCAs = caCertPool - } - - // load the cert - rawCerts := [][]byte{} - if tc.cert != "" { - pemCert := loadFileBytes(tc.cert) - block, _ := pem.Decode(pemCert) - rawCerts = append(rawCerts, block.Bytes) - } - - _, _, got := verifyCertificateExceptServerName(rawCerts, cfg) - if tc.wantErr { - assert.Error(t, got) - } else { - assert.NoError(t, got) - } - }) - } -} - -func loadFileBytes(fileName string) []byte { - contents, err := ioutil.ReadFile(filepath.Join("testdata", fileName)) - if err != nil { - panic(err) - } - return contents -} diff --git a/libbeat/common/transport/transptest/testing.go b/libbeat/common/transport/transptest/testing.go index 64763ba9b711..1649957c6245 100644 --- a/libbeat/common/transport/transptest/testing.go +++ b/libbeat/common/transport/transptest/testing.go @@ -135,7 +135,7 @@ func NewMockServerTLS(t *testing.T, to time.Duration, cert string, proxy *transp t.Fatalf("failed to load certificate") } - listener := tls.NewListener(tcpListener, tlsConfig.BuildModuleConfig("")) + listener := tls.NewListener(tcpListener, tlsConfig.BuildServerConfig("")) server := &MockServer{Listener: listener, Timeout: to} server.Handshake = func(client net.Conn) { diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 906aa673a14a..89578a6f7a27 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.0.0 :doc-branch: master -:go-version: 1.14.12 +:go-version: 1.15.7 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index b3c8e984fe9b..567088417509 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -227,7 +227,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err if tls != nil { k.Net.TLS.Enable = true - k.Net.TLS.Config = tls.BuildModuleConfig("") + k.Net.TLS.Config = tls.BuildModuleClientConfig("") } switch { diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 324f98c8d6de..4b76f73bafb3 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt update \ diff --git a/metricbeat/helper/server/http/http.go b/metricbeat/helper/server/http/http.go index b4bd17477b6c..3fe02001d7ae 100644 --- a/metricbeat/helper/server/http/http.go +++ b/metricbeat/helper/server/http/http.go @@ -76,7 +76,7 @@ func getDefaultHttpServer(mb mb.BaseMetricSet) (*HttpServer, error) { Addr: net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port))), } if tlsConfig != nil { - httpServer.TLSConfig = tlsConfig.BuildModuleConfig(config.Host) + httpServer.TLSConfig = tlsConfig.BuildModuleClientConfig(config.Host) } h.server = httpServer return h, nil diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f7530d7e386b..f48de895ab6d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1306,6 +1306,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1433,6 +1439,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1630,6 +1642,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1786,6 +1804,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2082,6 +2106,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2279,6 +2309,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 132ad0271df2..9d104c32a7e5 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.15.7 COPY test/main.go main.go diff --git a/metricbeat/module/kafka/metricset.go b/metricbeat/module/kafka/metricset.go index 73c1be02218c..5ec46332b350 100644 --- a/metricbeat/module/kafka/metricset.go +++ b/metricbeat/module/kafka/metricset.go @@ -49,7 +49,7 @@ func NewMetricSet(base mb.BaseMetricSet, options MetricSetOptions) (*MetricSet, var tls *tls.Config if tlsCfg != nil { - tls = tlsCfg.BuildModuleConfig("") + tls = tlsCfg.BuildModuleClientConfig("") } timeout := base.Module().Config().Timeout diff --git a/metricbeat/module/mongodb/metricset.go b/metricbeat/module/mongodb/metricset.go index b1b09a7ac4da..dc329b57f259 100644 --- a/metricbeat/module/mongodb/metricset.go +++ b/metricbeat/module/mongodb/metricset.go @@ -65,7 +65,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { logp.Warn("Failed to obtain hostname from `%s`: %s", hostname, err) hostname = "" } - return tls.Dial("tcp", addr.String(), tlsConfig.BuildModuleConfig(hostname)) + return tls.Dial("tcp", addr.String(), tlsConfig.BuildModuleClientConfig(hostname)) } } diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 2b0faecc26fb..6c5abe6309a8 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 57142c142630..9737223098bd 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1024,6 +1024,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1151,6 +1157,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1348,6 +1360,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1504,6 +1522,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1800,6 +1824,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1997,6 +2027,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/testing/environments/docker/logstash/gencerts.sh b/testing/environments/docker/logstash/gencerts.sh index a04742a7672f..fa53523e9790 100755 --- a/testing/environments/docker/logstash/gencerts.sh +++ b/testing/environments/docker/logstash/gencerts.sh @@ -2,4 +2,4 @@ mkdir -p pki/tls/certs mkdir -p pki/tls/private -openssl req -subj '/CN=logstash/' -x509 -days $((100 * 365)) -batch -nodes -newkey rsa:2048 -keyout pki/tls/private/logstash.key -out pki/tls/certs/logstash.crt +openssl req -subj '/CN=logstash/' -x509 -days $((100 * 365)) -batch -nodes -newkey rsa:2048 -keyout pki/tls/private/logstash.key -out pki/tls/certs/logstash.crt -config ssl.conf diff --git a/testing/environments/docker/logstash/pki/tls/certs/logstash.crt b/testing/environments/docker/logstash/pki/tls/certs/logstash.crt index 1b18ba84a201..08d2903c7d7e 100644 --- a/testing/environments/docker/logstash/pki/tls/certs/logstash.crt +++ b/testing/environments/docker/logstash/pki/tls/certs/logstash.crt @@ -1,18 +1,18 @@ -----BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJALOvd7vXvRrFMA0GCSqGSIb3DQEBCwUAMBMxETAPBgNV -BAMMCGxvZ3N0YXNoMCAXDTE2MDgyNjEyMzMyNFoYDzIxMTYwODAyMTIzMzI0WjAT -MREwDwYDVQQDDAhsb2dzdGFzaDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAKw5gLdWfqG9eraHrAAfSn2NbemYq32YZgbwJGaM9SlY3DDHB1MgBKBjpzPW -FavMO4xaDcfFhZbBJXwCVjPJe3ORQeoHgm3hG2er6JtCXlt3vto8FVbs9H4jd3+U -gH4cNdomgtYh3lBobZFKOa/+mZvjQxsK71KM2Gwk4b5gnV9iLaXzAGRWmY1dlHkE -Gki4WGNg0FlGf7aDJXZK2Yyq8MmiMfUEIZ2sDRjO3f/rCLdz3amG4gJtDllekz5l -lUTLccvtTWstJiKIx1zIAUEvTqaqInjMiJkjQtwazlc9w5ofmauxI6bb9L3L1ZJX -rrt+u5mg8Mc/w63+GuS8ZETbAacCAwEAAaNQME4wHQYDVR0OBBYEFA9Ug44w4XmN -r0z225Zt1zjjrKtoMB8GA1UdIwQYMBaAFA9Ug44w4XmNr0z225Zt1zjjrKtoMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABwl9KCqg69dv2XNZ95VCdk7 -mAT0LcUbd0FyrzDibNolzx9OlymWYiIxe86KdZsWzgBUcm9Q3Gg+TzAs7UyyfqSp -LR5fgGGIz9PCuuoFBdZCppPL9Y3Dryi91lPXveDUh5zIemOU9Jf6Ni0XVrRsO9C8 -aoY7SLtl1W7du3Nm+ZFH8T0wCcBFaYttmHejyu311ZDyAF0suu6Qu8NAWFrr5QGe -hA8VcImc335VQntT9EcztHhoyt1aW96BxLU9L4kdSZLJ6FVZrGij7IpZNipUQB8p -bPEL9KuQUDHKjoCx2YaNZqmuZ73m6u84TiTxgDYgChSfYASRXyCq90rQrQHVF74= +MIIC+jCCAeKgAwIBAgIUImV3iegTZ0b1zTQna2L4aVKmq1owDQYJKoZIhvcNAQEL +BQAwEzERMA8GA1UEAwwIbG9nc3Rhc2gwIBcNMjAxMTEwMDkwMDQ4WhgPMjEyMDEw +MTcwOTAwNDhaMBMxETAPBgNVBAMMCGxvZ3N0YXNoMIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAxYKH925nceZVxHZZskBaMuUIlI8tLRMY6EeXBvsDzrqi +4+pRFSCJU45wZKhOFi/EigWxk8TqfrHqYbpe9Cbxmngup2xvhgDC2Kmr3R/SKARW +zKCAbwLiDcf0yiJyT98AVOdUXuQ7HHC9m8D3Ohp1knYEmV8dJGtiFE1vW3FtYsUW +p0MOu5WG2iOitaWTIdXmqxwxuK6Jo4I3znReS0PSBwLFXKwWzjEcM9yvXPtubIc6 +1mbbF1Stf0GvGxmOs9u3JCNJXQvTuyJ+O7OrUbnk9vN8nmS/w9GSlM1PqwUNrWIB +X0uHazTU8mSFk3QI8M3kBFfFIN5dL9zIXLGFdJYvfwIDAQABo0QwQjALBgNVHQ8E +BAMCBDAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHgYDVR0RBBcwFYIIbG9nc3Rhc2iC +CWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEATBSnhUXJuf5whNmBE9OdoMJK +tgSaBx/FGq1tJp6jHkI1i4Oii3Wizs6K4tmWEqepu8MB0IVXJUkxGgh70DD3svKV +1En1zNOkUoI/lAwPBMHOl9oq2Z/u4E1dOydzyLQLDBg0fLC1Ui06NfdJRONOovoX +g3DD4IR6DODVtlGqRnON24H04OvZ3VWfbumkurp2XMvv8cooKQOLcMZ4dLVEyJxm +AEyC6pxuMsY32p/vtVjKarElqOnNAJ9xxS6IPczMgAXUMaxr5cZFMQSrdju9lVh4 +kpY5UglFiIJ/yHdlD5c4O8tK93qJ0Xgo7I7ujm38S38itrShpclXiAfW1rJ/2w== -----END CERTIFICATE----- diff --git a/testing/environments/docker/logstash/pki/tls/private/logstash.key b/testing/environments/docker/logstash/pki/tls/private/logstash.key index 9d3234d202e7..83d148412860 100644 --- a/testing/environments/docker/logstash/pki/tls/private/logstash.key +++ b/testing/environments/docker/logstash/pki/tls/private/logstash.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCsOYC3Vn6hvXq2 -h6wAH0p9jW3pmKt9mGYG8CRmjPUpWNwwxwdTIASgY6cz1hWrzDuMWg3HxYWWwSV8 -AlYzyXtzkUHqB4Jt4Rtnq+ibQl5bd77aPBVW7PR+I3d/lIB+HDXaJoLWId5QaG2R -Sjmv/pmb40MbCu9SjNhsJOG+YJ1fYi2l8wBkVpmNXZR5BBpIuFhjYNBZRn+2gyV2 -StmMqvDJojH1BCGdrA0Yzt3/6wi3c92phuICbQ5ZXpM+ZZVEy3HL7U1rLSYiiMdc -yAFBL06mqiJ4zIiZI0LcGs5XPcOaH5mrsSOm2/S9y9WSV667fruZoPDHP8Ot/hrk -vGRE2wGnAgMBAAECggEAArbxUXJ6koATFBd1XZcgrHPzPJBce6FQUmGsoTUtlBZD -ej1Y3zWM/R40/3srYkbY1XCB8Rkq7uJifd7nju9pE7xBZrfxlVvL+8lY5EGajSSJ -DJWP3Ivlmqticc9cayB0tNiQjWGBSJEs0PJzkFOaBjwBzcZRWWLA8otuR3rsYBl8 -cb7dV3HV4Z50Qto1ABoUWH2DGz7nX9HCr/SR1ayR1hWHCwv2Q4KQ5wJkmTKmaSNZ -I2464JXvufM9XiV9Fjy4RdiCN3sVXQcUIJ1hY+qGXsR0DUc5lOmw9Eu4SbJgdExR -EWoX4BqJuHrjCeKRF6rsDf5ocAS2cxATbQr1mEbW0QKBgQDWmZMO9TtL+pmJNmoP -g+HzgopBnMLxctcjVOEysuWgZyWYz9sFbCj6Udp2Q/9hjoVYRba3IXEHSsA2mdcY -KKcWbjEOYE+xL6oDXiZRkiJ+Poix9dOnTBg+lt2SKjphuNWnLe3jqfQhZxfV40Nf -60Wx6NGC7Dzlf+pAmkOA12BX2QKBgQDNcyTwnShbVrWzNRpzRe7RSHaAq6jSu1Yi -6dY/8bWTInVhPjB3xUGL0ckiMpDMoi0mxtnBmahvK59GPj3jhz+9HZqG6dSS6Fok -eS104GM7pCWyf66Rd9k8xu5IdrMM9Sveu24s21jgOJDtZtAplP1hsDMxxuaAEDVv -c0RwoKu1fwKBgDL0SheuIMM8oIIU+n/ul5LjNwK3Pw5nby/DcqlAEwfQFfw/tkiG -UwCEuPOF17iJR54bB3RaK2VI2XTdeFYTKQFJbrp0Idf3ck3UaBLMOQZywLBIp1W8 -2rDZz4hqIGydn5VPcYGyE/ZubRlrGc9HpMfGeSC2CQuRIMTwHAEWopiZAoGAZpKm -Trsn+vI/pUlN+19e7H4RLAyILS36w1Ob9DDpRpxdnj8+U43YO8ZxdPFp+cC+ai29 -ajsdLOPKkXdhzscnu3OcQt9bkj0PREZ7u26MHKrHZ2b38Qi1HPL05JjerAl77agG -Sb75kHitYtmB9EC+gJdH+AIl8qolA4+5C8Ir+GECgYBJgkoapPrGgSzCWHH6WpVZ -wrpZJ/rj6685J9K/ji3nHXj7gS4MzEzrtVK2K3aFjIjQe/zXJDI+dxrSCNfYFuQD -YJh+siGPF7Gosyht3ec0vUg7gY7Do1FzwL2H/OxvnaNEO+PPidXGOu/wHV5fMZJ1 -O2aLO5ZIygL6YcVr+vdm1A== +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDFgof3bmdx5lXE +dlmyQFoy5QiUjy0tExjoR5cG+wPOuqLj6lEVIIlTjnBkqE4WL8SKBbGTxOp+seph +ul70JvGaeC6nbG+GAMLYqavdH9IoBFbMoIBvAuINx/TKInJP3wBU51Re5DsccL2b +wPc6GnWSdgSZXx0ka2IUTW9bcW1ixRanQw67lYbaI6K1pZMh1earHDG4romjgjfO +dF5LQ9IHAsVcrBbOMRwz3K9c+25shzrWZtsXVK1/Qa8bGY6z27ckI0ldC9O7In47 +s6tRueT283yeZL/D0ZKUzU+rBQ2tYgFfS4drNNTyZIWTdAjwzeQEV8Ug3l0v3Mhc +sYV0li9/AgMBAAECggEAXnrge7YuecfLQ12x7pjmDO6OujH7VFKMWaDVWBt/aMbS +4N/XxZl416WNkjkIkYhsJfIvThambdDB49n5TiXK8S+IccJnXtzCWE5hzjdImqfo +tZ4ZkgD5DcqXCJKNyDNOv06hh3r549LygK1AFmVN1K/r50oecKuFkVyxZjbOMq5o +pKuNOiYWYki1FXd0kVd2yLy4ZKejrgMqwKk06xpeY4o9UfWSeMHaG+rlYsxxO/Fe +3o1FueAFNdJv553xzzmjCDI6YVq43izoF11/Q6K0HHvqwPkpNEFo9ChzPuLmvwOd +3Pyif53aVyOWg01sIp7NXzrUMrBoku3QcDtvGvrm8QKBgQD5vyP6LqbaXZ33DX2N +NpxgEpp2H44KCSfuxc1+mu4IpwOKIdX2bqjkcfGqbU5uBammdwYB78ro++YP0qR/ +6MTVyJwbMxlutOHJY/FLDnh+KZdPSBmJmce62khU/+eCATwQJeHCiktK9GxLisq6 +nBlkWLfkLVtGMduq1JwO4lWf4wKBgQDKdI5yfj1GKkAR4DPUp+OBZo1RsSJj5A0H +qMS/eudKTGj9Gi6Xiw200+x6fpzRCWCzUmS3c+QfTXMgNJEBjhhIYSb3IRo1p7Gf +M3chbN53SZyYf6msQ4b7oRiGZFDtoYNm0v4lqIxuoNi98CGxGtmz1W6gejpFyb1Y +A+EkRbSMtQKBgQDb7WZROiPUx/wDQu39HMo4ECnVO5RpYga4TZfYlbZoCCslyQYS +LQCtq2mVGwyl9nafENFJg1C8Opct9+DEgsZTPIW7rhQHWWI7Zrdl0ShqcVW9i1Bx +y+oGsZJgx7mm0k+CKNnV5tLG/tce7un3yt7Rbw8A8LAf8Gfw16lVshqU4QKBgHzZ +WgrzHJhLb81WRMBMdHkVI+sP4FRXi02A3yvx//YKnugOoFLl9qLf2cJEmDI0pUSQ +d/nF5xUCrw9aO14JIaJo/x2BdWdHLbsugrXDLIHFjGNivuCzl+dPFg+yh1Gzu5PK +Y94XTdrfKCohjrVoCH7lDN674XmuCizf35R9w/TNAoGBAKqaP7TTSafEs9ugsAgt +u3RXBbd9OA8u9tLKEBys/f9XoDPYWZ5Ar8a0LjpubcC5V14S7KwBYdiTN0ynaPGK +NI89jze9Y4ByiBWIalXjR6CWh3VlBiUGYONUjxOHaoMGrbYxYACNvJYtqjlQS1yN +LJN6nFfCs4U6TgQ4XBCdVAFO -----END PRIVATE KEY----- diff --git a/testing/environments/docker/logstash/ssl.conf b/testing/environments/docker/logstash/ssl.conf new file mode 100644 index 000000000000..13fbe0dd2bf4 --- /dev/null +++ b/testing/environments/docker/logstash/ssl.conf @@ -0,0 +1,18 @@ +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_req +prompt = no +[req_distinguished_name] +C = US +ST = VA +L = SomeCity +O = Elastic +OU = Observability +CN = elastic.co +[v3_req] +keyUsage = keyEncipherment, dataEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = logstash +DNS.2 = localhost diff --git a/winlogbeat/eventlog/bench_test.go b/winlogbeat/eventlog/bench_test.go index df3417eb611d..ffecb69672fb 100644 --- a/winlogbeat/eventlog/bench_test.go +++ b/winlogbeat/eventlog/bench_test.go @@ -37,7 +37,7 @@ const gigabyte = 1 << 30 var ( benchTest = flag.Bool("benchtest", false, "Run benchmarks for the eventlog package.") - injectAmount = flag.Int("inject", 1E6, "Number of events to inject before running benchmarks.") + injectAmount = flag.Int("inject", 1e6, "Number of events to inject before running benchmarks.") ) // TestBenchmarkRead benchmarks each event log reader implementation with diff --git a/winlogbeat/tests/system/test_config.py b/winlogbeat/tests/system/test_config.py index 18df0594fca6..065de09fc53a 100644 --- a/winlogbeat/tests/system/test_config.py +++ b/winlogbeat/tests/system/test_config.py @@ -36,7 +36,7 @@ def test_invalid_ignore_older(self): ) self.run_config_tst(exit_code=1) assert self.log_contains( - "unknown unit hour in duration 1 hour " + "unknown unit \" hour\" in duration \"1 hour\" " "accessing 'winlogbeat.event_logs.0.ignore_older'") def test_invalid_level(self): diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 316df4ae3dd8..5eafb6ae1cb3 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -452,6 +452,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -579,6 +585,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -776,6 +788,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -932,6 +950,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1228,6 +1252,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1425,6 +1455,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index ec3ef722b0cc..9395908a0dec 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -585,6 +585,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -712,6 +718,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -909,6 +921,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1065,6 +1083,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1361,6 +1385,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1558,6 +1588,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go index fbb790479c76..4158ba554ef3 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader_test.go @@ -7,6 +7,7 @@ package pipelinemanager import ( "os" "path/filepath" + "strconv" "sync" "testing" "time" @@ -84,7 +85,7 @@ func createNewClient(t *testing.T, logString string, mockConnector *pipelinemock info := logger.Info{ ContainerID: "b87d3b0379f816a5f2f7070f28cc05e2f564a3fb549a67c64ec30fc5b04142ed", - LogPath: filepath.Join("/tmp/dockerbeattest/", string(time.Now().Unix())), + LogPath: filepath.Join("/tmp/dockerbeattest/", strconv.FormatInt(time.Now().Unix(), 10)), } err = os.MkdirAll(filepath.Dir(info.LogPath), 0755) diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 43cab2fd3b45..2ec6b531456b 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -76,7 +76,7 @@ func getTestOperator(t *testing.T, downloadPath string, installPath string, p *a } operator.config.DownloadConfig.OperatingSystem = "darwin" - operator.config.DownloadConfig.Architecture = "32" + operator.config.DownloadConfig.Architecture = "64" // make the download path so the `operation_verify` can ensure the path exists downloadConfig := operator.config.DownloadConfig @@ -101,7 +101,7 @@ func getProgram(binary, version string) *app.Descriptor { downloadCfg := &artifact.Config{ InstallPath: installPath, OperatingSystem: "darwin", - Architecture: "32", + Architecture: "64", } return app.NewDescriptor(spec, version, downloadCfg, nil) } diff --git a/x-pack/elastic-agent/pkg/core/authority/ca.go b/x-pack/elastic-agent/pkg/core/authority/ca.go index 2ddeae70410c..f558e8a9eb30 100644 --- a/x-pack/elastic-agent/pkg/core/authority/ca.go +++ b/x-pack/elastic-agent/pkg/core/authority/ca.go @@ -38,10 +38,10 @@ type Pair struct { // NewCA creates a new certificate authority capable of generating child certificates func NewCA() (*CertificateAuthority, error) { ca := &x509.Certificate{ + DNSNames: []string{"localhost"}, SerialNumber: big.NewInt(1653), Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, - CommonName: "localhost", }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), @@ -101,13 +101,13 @@ func (c *CertificateAuthority) GeneratePair() (*Pair, error) { } // GeneratePairWithName generates child certificate with provided name as the common name. -func (c *CertificateAuthority) GeneratePairWithName(commonName string) (*Pair, error) { +func (c *CertificateAuthority) GeneratePairWithName(name string) (*Pair, error) { // Prepare certificate certTemplate := &x509.Certificate{ SerialNumber: big.NewInt(1658), + DNSNames: []string{name}, Subject: pkix.Name{ Organization: []string{"elastic-fleet"}, - CommonName: commonName, }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 0fd068e19637..242b551c510a 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3207,6 +3207,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3334,6 +3340,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3531,6 +3543,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3687,6 +3705,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -3983,6 +4007,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -4180,6 +4210,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/filebeat/input/http_endpoint/input.go b/x-pack/filebeat/input/http_endpoint/input.go index bddf2be0a9e7..2c799c1f14fa 100644 --- a/x-pack/filebeat/input/http_endpoint/input.go +++ b/x-pack/filebeat/input/http_endpoint/input.go @@ -59,7 +59,7 @@ func newHTTPEndpoint(config config) (*httpEndpoint, error) { return nil, err } if tlsConfigBuilder != nil { - tlsConfig = tlsConfigBuilder.BuildModuleConfig(addr) + tlsConfig = tlsConfigBuilder.BuildModuleClientConfig(addr) } return &httpEndpoint{ diff --git a/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go b/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go index 9045108c4490..0fba76aaab6e 100644 --- a/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go +++ b/x-pack/filebeat/input/netflow/decoder/template/test_helpers.go @@ -6,6 +6,7 @@ package template import ( "fmt" + "strconv" "sync" "testing" @@ -61,8 +62,8 @@ func ValidateTemplate(t testing.TB, template *Template) bool { func AssertFieldsEquals(t testing.TB, expected []FieldTemplate, actual []FieldTemplate) (succeeded bool) { if succeeded = assert.Len(t, actual, len(expected)); succeeded { for idx := range expected { - succeeded = assert.Equal(t, expected[idx].Length, actual[idx].Length, string(idx)) && succeeded - succeeded = assert.Equal(t, expected[idx].Info, actual[idx].Info, string(idx)) && succeeded + succeeded = assert.Equal(t, expected[idx].Length, actual[idx].Length, strconv.Itoa(idx)) && succeeded + succeeded = assert.Equal(t, expected[idx].Info, actual[idx].Info, strconv.Itoa(idx)) && succeeded } } return diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index 907a989eb4d6..77daddd7a86c 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index b41a4035d682..d96ab60094e9 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -815,6 +815,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -942,6 +948,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1221,6 +1233,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1418,6 +1436,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 37e3e2ed1226..e39dfa16d69a 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -707,6 +707,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -834,6 +840,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1031,6 +1043,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1187,6 +1205,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1483,6 +1507,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1680,6 +1710,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/libbeat/Dockerfile b/x-pack/libbeat/Dockerfile index 06ca7a1ffad7..1a0c44db3986 100644 --- a/x-pack/libbeat/Dockerfile +++ b/x-pack/libbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.14.12 +FROM golang:1.15.7 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 8406f20f5ac9..19f9941ee47f 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -115,14 +115,14 @@ stages: - "windows-7" branches: true ## for all the branches tags: true ## for all the tags - windows-7-32: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-7-32-bit" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags +# windows-7-32: +# mage: "mage build unitTest" +# platforms: ## override default labels in this specific stage. +# - "windows-7-32-bit" +# when: ## Override the top-level when. +# comments: +# - "/test x-pack/metricbeat for windows-7-32" +# labels: +# - "windows-7-32" +# branches: true ## for all the branches +# tags: true ## for all the tags diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 66c0e02eaa2c..c003205f76c0 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1807,6 +1807,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1934,6 +1940,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2131,6 +2143,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2287,6 +2305,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2583,6 +2607,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -2780,6 +2810,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 57142c142630..9737223098bd 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1024,6 +1024,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1151,6 +1157,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1348,6 +1360,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1504,6 +1522,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1800,6 +1824,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1997,6 +2027,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 03652ce2788b..43427ba20f63 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -495,6 +495,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -622,6 +628,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -819,6 +831,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -975,6 +993,12 @@ output.elasticsearch: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1271,6 +1295,12 @@ setup.kibana: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary @@ -1468,6 +1498,12 @@ logging.files: # matches the names identified within the certificate. # * certificate, which verifies that the provided certificate is signed by a # trusted authority (CA), but does not perform any hostname verification. + # * strict, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. If the Subject Alternative + # Name is empty, it returns an error. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. # * none, which performs no verification of the server's certificate. This # mode disables many of the security benefits of SSL/TLS and should only be used # after very careful consideration. It is primarily intended as a temporary From 717cb229664092950ceea9c94cd2093bdc89c88d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 21 Jan 2021 22:01:49 +0100 Subject: [PATCH 20/35] [Ingest Manager] Fix refresh of monitoring configuration (#23619) * fix refresh of monitoring beats * more descriptive comment * changelog * newline --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + .../agent/application/monitoring_decorator.go | 7 +- .../application/monitoring_decorator_test.go | 327 +++++++++++++----- .../pkg/agent/operation/monitoring.go | 21 +- .../pkg/agent/stateresolver/stateresolver.go | 4 + 5 files changed, 274 insertions(+), 86 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index d3fd1fa65d5d..619850f4bc82 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ - Fix Windows service installation script {pull}20203[20203] - Fix timeout issue stopping service applications {pull}20256[20256] - Fix incorrect hash when upgrading agent {pull}22322[22322] +- Fix refresh of monitoring configuration {pull}23619[23619] - Fixed nil pointer during unenroll {pull}23609[23609] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go index 920b1a4b5bff..ab9ff6bbc63c 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go @@ -5,6 +5,7 @@ package application import ( + "crypto/md5" "fmt" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" @@ -15,6 +16,7 @@ import ( const ( monitoringName = "FLEET_MONITORING" programsKey = "programs" + monitoringChecksumKey = "monitoring_checksum" monitoringKey = "agent.monitoring" monitoringUseOutputKey = "agent.monitoring.use_output" monitoringOutputFormatKey = "outputs.%s" @@ -74,12 +76,15 @@ func injectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr } programList := make([]string, 0, len(programsToRun)) + cfgHash := md5.New() for _, p := range programsToRun { programList = append(programList, p.Spec.Cmd) + cfgHash.Write(p.Config.Hash()) } - // making program list part of the config + // making program list and their hashes part of the config // so it will get regenerated with every change config[programsKey] = programList + config[monitoringChecksumKey] = fmt.Sprintf("%x", cfgHash.Sum(nil)) monitoringProgram.Config, err = transpiler.NewAST(config) if err != nil { diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go index 6a3be4100be2..e23027e62fc2 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go @@ -5,6 +5,7 @@ package application import ( + "fmt" "testing" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" @@ -27,6 +28,10 @@ func TestMonitoringInjection(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -102,6 +107,10 @@ func TestMonitoringInjectionDefaults(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -177,6 +186,10 @@ func TestMonitoringInjectionDisabled(t *testing.T) { t.Fatal(err) } + if len(programsToRun) != 2 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 2)) + } + GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) @@ -203,19 +216,19 @@ GROUPLOOP: } // is enabled set - settingsObj, found := cm["settings"] + agentObj, found := cm["agent"] if !found { t.Errorf("settings not found for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP } - settingsMap, ok := settingsObj.(map[string]interface{}) + agentMap, ok := agentObj.(map[string]interface{}) if !ok { t.Errorf("settings not a map for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP } - monitoringObj, found := settingsMap["monitoring"] + monitoringObj, found := agentMap["monitoring"] if !found { t.Errorf("agent.monitoring not found for '%s(%s)': %v", group, p.Spec.Name, cm) continue GROUPLOOP @@ -247,6 +260,97 @@ GROUPLOOP: } } +func TestChangeInMonitoringWithChangeInInput(t *testing.T) { + agentInfo, err := info.NewAgentInfo() + if err != nil { + t.Fatal(err) + } + + astBefore, err := transpiler.NewAST(inputChange1) + if err != nil { + t.Fatal(err) + } + + programsToRunBefore, err := program.Programs(agentInfo, astBefore) + if err != nil { + t.Fatal(err) + } + + if len(programsToRunBefore) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + + astAfter, err := transpiler.NewAST(inputChange2) + if err != nil { + t.Fatal(err) + } + + programsToRunAfter, err := program.Programs(agentInfo, astAfter) + if err != nil { + t.Fatal(err) + } + + if len(programsToRunAfter) != 1 { + t.Fatal(fmt.Errorf("programsToRun expected to have %d entries", 1)) + } + + // inject to both + var hashConfigBefore, hashConfigAfter string +GROUPLOOPBEFORE: + for group, ptr := range programsToRunBefore { + programsCount := len(ptr) + newPtr, err := injectMonitoring(agentInfo, group, astBefore, ptr) + if err != nil { + t.Error(err) + continue GROUPLOOPBEFORE + } + + if programsCount+1 != len(newPtr) { + t.Errorf("incorrect programs to run count, expected: %d, got %d", programsCount+1, len(newPtr)) + continue GROUPLOOPBEFORE + } + + for _, p := range newPtr { + if p.Spec.Name != monitoringName { + continue + } + + hashConfigBefore = p.Config.HashStr() + } + } + +GROUPLOOPAFTER: + for group, ptr := range programsToRunAfter { + programsCount := len(ptr) + newPtr, err := injectMonitoring(agentInfo, group, astAfter, ptr) + if err != nil { + t.Error(err) + continue GROUPLOOPAFTER + } + + if programsCount+1 != len(newPtr) { + t.Errorf("incorrect programs to run count, expected: %d, got %d", programsCount+1, len(newPtr)) + continue GROUPLOOPAFTER + } + + for _, p := range newPtr { + if p.Spec.Name != monitoringName { + continue + } + + hashConfigAfter = p.Config.HashStr() + } + } + + if hashConfigAfter == "" || hashConfigBefore == "" { + t.Fatal("hash configs uninitialized") + } + + if hashConfigAfter == hashConfigBefore { + t.Fatal("hash config equal, expected to be different") + } +} + var inputConfigMap = map[string]interface{}{ "agent.monitoring": map[string]interface{}{ "enabled": true, @@ -279,40 +383,33 @@ var inputConfigMap = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + "inputs": []map[string]interface{}{ + { + "type": "log", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, - }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ - map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, - }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, }, }, - "use_output": "infosec1", }, }, } @@ -343,40 +440,34 @@ var inputConfigMapDefaults = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + + "inputs": []map[string]interface{}{ + { + "type": "log", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, - }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ - map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, - }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, }, }, - "use_output": "infosec1", }, }, } @@ -410,40 +501,114 @@ var inputConfigMapDisabled = map[string]interface{}{ "username": "monitoring-uname", }, }, - "datasources": []map[string]interface{}{ - map[string]interface{}{ - "inputs": []map[string]interface{}{ + + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "log", - "streams": []map[string]interface{}{ - map[string]interface{}{"paths": "/xxxx"}, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, - "processors": []interface{}{ - map[string]interface{}{ - "dissect": map[string]interface{}{ - "tokenizer": "---", - }, - }, + }, + }, + }, + { + "type": "system/metrics", + "use_output": "infosec1", + "streams": []map[string]interface{}{ + { + "id": "system/metrics-system.core", + "enabled": true, + "dataset": "system.core", + "period": "10s", + "metrics": []string{"percentages"}, + }, + }, + }, + }, +} + +var inputChange1 = map[string]interface{}{ + "agent.monitoring": map[string]interface{}{ + "enabled": true, + "logs": true, + "metrics": true, + "use_output": "monitoring", + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "index_name": "general", + "pass": "xxx", + "type": "elasticsearch", + "url": "xxxxx", + "username": "xxx", + }, + "monitoring": map[string]interface{}{ + "type": "elasticsearch", + "index_name": "general", + "pass": "xxx", + "url": "xxxxx", + "username": "monitoring-uname", + }, + }, + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + }, + "processors": []interface{}{ + map[string]interface{}{ + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, }, - map[string]interface{}{ - "inputs": []map[string]interface{}{ + }, +} + +var inputChange2 = map[string]interface{}{ + "agent.monitoring": map[string]interface{}{ + "enabled": true, + "logs": true, + "metrics": true, + "use_output": "monitoring", + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "index_name": "general", + "pass": "xxx", + "type": "elasticsearch", + "url": "xxxxx", + "username": "xxx", + }, + "monitoring": map[string]interface{}{ + "type": "elasticsearch", + "index_name": "general", + "pass": "xxx", + "url": "xxxxx", + "username": "monitoring-uname", + }, + }, + "inputs": []map[string]interface{}{ + { + "type": "log", + "streams": []map[string]interface{}{ + {"paths": "/xxxx"}, + {"paths": "/yyyy"}, + }, + "processors": []interface{}{ map[string]interface{}{ - "type": "system/metrics", - "streams": []map[string]interface{}{ - map[string]interface{}{ - "id": "system/metrics-system.core", - "enabled": true, - "dataset": "system.core", - "period": "10s", - "metrics": []string{"percentages"}, - }, + "dissect": map[string]interface{}{ + "tokenizer": "---", }, }, }, - "use_output": "infosec1", }, }, } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index c6a5492133fc..882f9efdf14b 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -129,8 +129,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() - // generate only on change - if watchLogs != o.isMonitoringLogs() { + // generate only when monitoring is running (for config refresh) or + // state changes (turning on/off) + if watchLogs != o.isMonitoringLogs() || watchLogs { fbConfig, any := o.getMonitoringFilebeatConfig(output) stepID := configrequest.StepRun if !watchLogs || !any { @@ -151,7 +152,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ steps = append(steps, filebeatStep) } - if watchMetrics != o.isMonitoringMetrics() { + if watchMetrics != o.isMonitoringMetrics() || watchMetrics { mbConfig, any := o.getMonitoringMetricbeatConfig(output) stepID := configrequest.StepRun if !watchMetrics || !any { @@ -552,7 +553,19 @@ func (o *Operator) getMetricbeatEndpoints() map[string][]string { for _, a := range o.apps { metricEndpoint := a.Monitor().MetricsPathPrefixed(a.Spec(), o.pipelineID) if metricEndpoint != "" { - endpoints[strings.ReplaceAll(a.Name(), "-", "_")] = append(endpoints[a.Name()], metricEndpoint) + safeName := strings.ReplaceAll(a.Name(), "-", "_") + // prevent duplicates + var found bool + for _, ep := range endpoints[safeName] { + if ep == metricEndpoint { + found = true + break + } + } + + if !found { + endpoints[safeName] = append(endpoints[safeName], metricEndpoint) + } } } diff --git a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go index 61ba348f760b..34bb76e551f4 100644 --- a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go +++ b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver.go @@ -47,6 +47,10 @@ func (s *StateResolver) Resolve( s.l.Infof("New State ID is %s", newStateID) s.l.Infof("Converging state requires execution of %d step(s)", len(steps)) + for i, step := range steps { + // more detailed debug log + s.l.Debugf("step %d: %s", i, step.String()) + } // Allow the operator to ack the should state when applying the steps is done correctly. ack := func() { From 621c9e2cf614235a104a170229cb324ebfbea811 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Thu, 21 Jan 2021 21:25:03 -0500 Subject: [PATCH 21/35] [libbeat] Expose the new Sarama flag 'DisablePAFXFAST' in the Kafka output (#23629) --- CHANGELOG.next.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 4 ++++ filebeat/filebeat.reference.yml | 4 ++++ heartbeat/heartbeat.reference.yml | 4 ++++ journalbeat/journalbeat.reference.yml | 4 ++++ libbeat/_meta/config/output-kafka.reference.yml.tmpl | 4 ++++ libbeat/outputs/kafka/config.go | 2 ++ libbeat/outputs/kafka/docs/kafka.asciidoc | 6 ++++++ metricbeat/metricbeat.reference.yml | 4 ++++ packetbeat/packetbeat.reference.yml | 4 ++++ winlogbeat/winlogbeat.reference.yml | 4 ++++ x-pack/auditbeat/auditbeat.reference.yml | 4 ++++ x-pack/filebeat/filebeat.reference.yml | 4 ++++ x-pack/heartbeat/heartbeat.reference.yml | 4 ++++ x-pack/metricbeat/metricbeat.reference.yml | 4 ++++ x-pack/packetbeat/packetbeat.reference.yml | 4 ++++ x-pack/winlogbeat/winlogbeat.reference.yml | 4 ++++ 17 files changed, 65 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c24284950fa2..3fa0e65f2d4f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -585,6 +585,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] +- Add the `enable_krb5_fast` flag to the Kafka output to explicitly opt-in to FAST authentication. {pull}23629[23629] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 7a053b62884e..712ebf7ee674 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -856,6 +856,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 6cf5f8d33fa7..858d307a57e1 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1736,6 +1736,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e39dfa16d69a..63501b14e2df 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1034,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 1eb7814f8344..7664a3edbd45 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -799,6 +799,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/libbeat/_meta/config/output-kafka.reference.yml.tmpl b/libbeat/_meta/config/output-kafka.reference.yml.tmpl index c1240f758673..366652fd23e4 100644 --- a/libbeat/_meta/config/output-kafka.reference.yml.tmpl +++ b/libbeat/_meta/config/output-kafka.reference.yml.tmpl @@ -131,6 +131,10 @@ # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + {{include "ssl.reference.yml.tmpl" . | indent 2 }} # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. #kerberos.enabled: true diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 567088417509..3747a2fa63ce 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -69,6 +69,7 @@ type kafkaConfig struct { Password string `config:"password"` Codec codec.Config `config:"codec"` Sasl saslConfig `config:"sasl"` + EnableFAST bool `config:"enable_krb5_fast"` } type saslConfig struct { @@ -244,6 +245,7 @@ func newSaramaConfig(log *logp.Logger, config *kafkaConfig) (*sarama.Config, err Username: config.Kerberos.Username, Password: config.Kerberos.Password, Realm: config.Kerberos.Realm, + DisablePAFXFAST: !config.EnableFAST, } case config.Username != "": diff --git a/libbeat/outputs/kafka/docs/kafka.asciidoc b/libbeat/outputs/kafka/docs/kafka.asciidoc index f61d4c5d9855..026d04313456 100644 --- a/libbeat/outputs/kafka/docs/kafka.asciidoc +++ b/libbeat/outputs/kafka/docs/kafka.asciidoc @@ -309,6 +309,12 @@ The ACK reliability level required from broker. 0=no response, 1=wait for local Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error. +===== `enable_krb5_fast` + +beta[] + +Enable Kerberos FAST authentication. This may conflict with some Active Directory installations. It is separate from the standard Kerberos settings because this flag only applies to the Kafka output. The default is `false`. + ===== `ssl` Configuration options for SSL parameters like the root CA for Kafka connections. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f48de895ab6d..8250323ca045 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1633,6 +1633,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 9737223098bd..9aee618bed83 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1351,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 5eafb6ae1cb3..1ab1796a809a 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -779,6 +779,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 9395908a0dec..ac1347037946 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -912,6 +912,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 242b551c510a..9104290b7ce9 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3534,6 +3534,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index e39dfa16d69a..63501b14e2df 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1034,6 +1034,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index c003205f76c0..71eaa8f800fe 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -2134,6 +2134,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 9737223098bd..9aee618bed83 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1351,6 +1351,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 43427ba20f63..636325f55715 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -822,6 +822,10 @@ output.elasticsearch: # purposes. The default is "beats". #client_id: beats + # Enables Kerberos FAST authentication in the Kafka output. This may + # conflict with certain Active Directory configurations. + #enable_krb5_fast: false + # Use SSL settings for HTTPS. #ssl.enabled: true From 6cad2813b38ed2ee634f1979aac80b65bd3a78b3 Mon Sep 17 00:00:00 2001 From: Yazdan <22779039+yzdann@users.noreply.github.com> Date: Mon, 25 Jan 2021 01:51:06 +0330 Subject: [PATCH 22/35] Fix cassandra and icmp comments in packetbeat.yml (#22584) * Uniform cassandra comment in packetbeat.yml * Fix icmp comment in packetbeat.yml * Uniform SIP comment in packetbeat.yml * Update the config templates Co-authored-by: Andrew Kroh --- packetbeat/_meta/config/beat.reference.yml.tmpl | 2 +- packetbeat/_meta/config/beat.yml.tmpl | 8 +++++--- packetbeat/packetbeat.reference.yml | 2 +- packetbeat/packetbeat.yml | 8 +++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packetbeat/_meta/config/beat.reference.yml.tmpl b/packetbeat/_meta/config/beat.reference.yml.tmpl index 77a1de3c8768..5d8d8fa9c7a7 100644 --- a/packetbeat/_meta/config/beat.reference.yml.tmpl +++ b/packetbeat/_meta/config/beat.reference.yml.tmpl @@ -80,7 +80,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: true + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. #enabled: true # Set to true to publish fields with null values in events. diff --git a/packetbeat/_meta/config/beat.yml.tmpl b/packetbeat/_meta/config/beat.yml.tmpl index c1ad53bf9e9e..2a69df425175 100644 --- a/packetbeat/_meta/config/beat.yml.tmpl +++ b/packetbeat/_meta/config/beat.yml.tmpl @@ -38,7 +38,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: false + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. enabled: true - type: amqp @@ -47,7 +47,8 @@ packetbeat.protocols: ports: [5672] - type: cassandra - #Cassandra port for traffic monitoring. + # Configure the ports where to listen for Cassandra traffic. You can disable + # the Cassandra protocol by commenting out the list of ports. ports: [9042] - type: dhcpv4 @@ -112,7 +113,8 @@ packetbeat.protocols: - 9243 # Elasticsearch - type: sip - # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + # Configure the ports where to listen for SIP traffic. You can disable + # the SIP protocol by commenting out the list of ports. ports: [5060] {{header "Elasticsearch template setting"}} diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 9aee618bed83..073bec9c7682 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -80,7 +80,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: true + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. #enabled: true # Set to true to publish fields with null values in events. diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 2ac9186d43ec..15a0df9ebd13 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -38,7 +38,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: false + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. enabled: true - type: amqp @@ -47,7 +47,8 @@ packetbeat.protocols: ports: [5672] - type: cassandra - #Cassandra port for traffic monitoring. + # Configure the ports where to listen for Cassandra traffic. You can disable + # the Cassandra protocol by commenting out the list of ports. ports: [9042] - type: dhcpv4 @@ -112,7 +113,8 @@ packetbeat.protocols: - 9243 # Elasticsearch - type: sip - # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + # Configure the ports where to listen for SIP traffic. You can disable + # the SIP protocol by commenting out the list of ports. ports: [5060] # ======================= Elasticsearch template setting ======================= From 4b7b267522768dd4eff189d431cfe8f5f5e9738a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 25 Jan 2021 11:39:06 +0100 Subject: [PATCH 23/35] chore: always use CI snapshots (#23633) We want to use them for PRs and for merges to branches --- .ci/packaging.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 3162b9d80475..7af3ff6f60c6 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -339,6 +339,7 @@ def triggerE2ETests(String suite) { booleanParam(name: 'forceSkipGitChecks', value: true), booleanParam(name: 'forceSkipPresubmit', value: true), booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()), + booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true), string(name: 'runTestsSuites', value: suite), string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME), string(name: 'GITHUB_CHECK_REPO', value: env.REPO), @@ -346,7 +347,6 @@ def triggerE2ETests(String suite) { ] if (isPR()) { def version = "pr-${env.CHANGE_ID}" - parameters.push(booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true)) parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}")) parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}")) } From dd7a1b3808eb98e77fb49b268cd3764cc17eff5b Mon Sep 17 00:00:00 2001 From: Anabella Cristaldi <33020901+janniten@users.noreply.github.com> Date: Mon, 25 Jan 2021 16:30:29 +0100 Subject: [PATCH 24/35] Audit and Authentication Policy Change Events (#20684) * [Winlogbeat] Audit and Authentication Policy Change Events Co-authored-by: Lee E. Hinman --- CHANGELOG.next.asciidoc | 1 + .../security/config/winlogbeat-security.js | 524 +++++++++++++++++- .../test/testdata/4670_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4670_WindowsSrv2016.evtx.golden.json | 80 +++ .../test/testdata/4706_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4706_WindowsSrv2016.evtx.golden.json | 72 +++ .../test/testdata/4707_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4707_WindowsSrv2016.evtx.golden.json | 64 +++ .../test/testdata/4713_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4713_WindowsSrv2016.evtx.golden.json | 64 +++ .../test/testdata/4716_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4716_WindowsSrv2016.evtx.golden.json | 72 +++ .../test/testdata/4717_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4717_WindowsSrv2016.evtx.golden.json | 67 +++ .../test/testdata/4718_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4718_WindowsSrv2016.evtx.golden.json | 67 +++ .../test/testdata/4719_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4719_WindowsSrv2016.evtx.golden.json | 74 +++ .../test/testdata/4739_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4739_WindowsSrv2016.evtx.golden.json | 71 +++ .../test/testdata/4817_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4817_WindowsSrv2016.evtx.golden.json | 74 +++ .../test/testdata/4902_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4902_WindowsSrv2016.evtx.golden.json | 51 ++ .../test/testdata/4904_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4904_WindowsSrv2016.evtx.golden.json | 72 +++ .../test/testdata/4905_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4905_WindowsSrv2016.evtx.golden.json | 72 +++ .../test/testdata/4906_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4906_WindowsSrv2016.evtx.golden.json | 50 ++ .../test/testdata/4907_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4907_WindowsSrv2016.evtx.golden.json | 75 +++ .../test/testdata/4908_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4908_WindowsSrv2016.evtx.golden.json | 58 ++ .../test/testdata/4912_WindowsSrv2016.evtx | Bin 0 -> 69632 bytes .../4912_WindowsSrv2016.evtx.golden.json | 70 +++ ...security-windows2012_4771.evtx.golden.json | 1 + ...security-windows2012_4778.evtx.golden.json | 1 + ...security-windows2012_4779.evtx.golden.json | 1 + 39 files changed, 1661 insertions(+), 20 deletions(-) create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4719_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4904_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4905_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4906_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4907_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx create mode 100644 x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3fa0e65f2d4f..acfdbc957c8b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -978,6 +978,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add additional event categorization for security and sysmon modules. {pull}22988[22988] - Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999] - Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046] +- Add Audit and Authentication Polixy Change Events and related.ip information {pull}20684[20684] *Elastic Log Driver* diff --git a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js index 56cdced6b512..44d0e8eb34db 100644 --- a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js +++ b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js @@ -180,6 +180,7 @@ var security = (function () { "4647": [["authentication"], ["end"], "logged-out"], "4648": [["authentication"], ["start"], "logged-in-explicit"], "4657": [["configuration"], ["change"], "registry-value-modified"], + "4670": [["iam", "configuration"],["admin", "change"],"permissions-changed"], "4672": [["iam"], ["admin"], "logged-in-special"], "4673": [["iam"], ["admin"], "privileged-service-called"], "4674": [["iam"], ["admin"], "privileged-operation"], @@ -197,6 +198,8 @@ var security = (function () { "4714": [["configuration"], ["change"], "encrypted-data-recovery-policy-changed"], "4715": [["configuration"], ["change"], "object-audit-policy-changed"], "4716": [["configuration"], ["change"], "trusted-domain-information-changed"], + "4717": [["iam", "configuration"],["admin", "change"],"system-security-access-granted"], + "4718": [["iam", "configuration"],["admin", "deletion"],"system-security-access-removed"], "4719": [["iam", "configuration"], ["admin", "change"], "changed-audit-config"], // remove iam and admin "4720": [["iam"], ["user", "creation"], "added-user-account"], "4722": [["iam"], ["user", "change"], "enabled-user-account"], @@ -252,7 +255,14 @@ var security = (function () { "4781": [["iam"], ["user", "change"], "renamed-user-account"], "4798": [["iam"], ["user", "info"], "group-membership-enumerated"], // process enumerates the local groups to which the specified user belongs "4799": [["iam"], ["group", "info"], "user-member-enumerated"], // a process enumerates the members of the specified local group - "4912": [["configuration"], ["change"], "per-user-audit-policy-changed"], + "4817": [["iam", "configuration"], ["admin", "change"],"object-audit-changed"], + "4902": [["iam", "configuration"], ["admin", "creation"],"user-audit-policy-created"], + "4904": [["iam", "configuration"], ["admin", "change"],"security-event-source-added"], + "4905": [["iam", "configuration"], ["admin", "deletion"], "security-event-source-removed"], + "4906": [["iam", "configuration"], ["admin", "change"], "crash-on-audit-changed"], + "4907": [["iam", "configuration"], ["admin", "change"], "audit-setting-changed"], + "4908": [["iam", "configuration"], ["admin", "change"], "special-group-table-changed"], + "4912": [["iam", "configuration"], ["admin", "change"], "per-user-audit-policy-changed"], "4950": [["configuration"], ["change"], "windows-firewall-setting-changed"], "4954": [["configuration"], ["change"], "windows-firewall-group-policy-changed"], "4964": [["iam"], ["admin", "group"], "logged-in-special"], @@ -263,16 +273,6 @@ var security = (function () { "5037": [["driver"], ["end"], "windows-firewall-driver-error"], }; - - // Audit Policy Changes Table - // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4719 - var auditActions = { - "8448": "Success Removed", - "8450": "Failure Removed", - "8449": "Success Added", - "8451": "Failure Added", - }; - // Services Types // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4697 var serviceTypes = { @@ -1209,7 +1209,7 @@ var security = (function () { "8448": "Success removed", "8449": "Success Added", "8450": "Failure removed", - "8451": "Failure added", + "8451": "Failure Added", "8452": "Success include removed", "8453": "Success include added", "8454": "Success exclude removed", @@ -1351,6 +1351,250 @@ var security = (function () { "16903": "Publish", }; + // Trust Types + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustTypes = { + "1": "TRUST_TYPE_DOWNLEVEL", + "2": "TRUST_TYPE_UPLEVEL", + "3": "TRUST_TYPE_MIT", + "4": "TRUST_TYPE_DCE" + } + + // Trust Direction + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustDirection = { + "0": "TRUST_DIRECTION_DISABLED", + "1": "TRUST_DIRECTION_INBOUND", + "2": "TRUST_DIRECTION_OUTBOUND", + "3": "TRUST_DIRECTION_BIDIRECTIONAL" + } + + // Trust Attributes + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4706 + var trustAttributes = { + "0": "UNDEFINED", + "1": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "2": "TRUST_ATTRIBUTE_UPLEVEL_ONLY", + "4": "TRUST_ATTRIBUTE_QUARANTINED_DOMAIN", + "8": "TRUST_ATTRIBUTE_FOREST_TRANSITIVE", + "16": "TRUST_ATTRIBUTE_CROSS_ORGANIZATION", + "32": "TRUST_ATTRIBUTE_WITHIN_FOREST", + "64": "TRUST_ATTRIBUTE_TREAT_AS_EXTERNAL", + "128": "TRUST_ATTRIBUTE_USES_RC4_ENCRYPTION", + "512": "TRUST_ATTRIBUTE_CROSS_ORGANIZATION_NO_TGT_DELEGATION", + "1024": "TRUST_ATTRIBUTE_PIM_TRUST" + } + + // SDDL Ace Types + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4715 + // https://docs.microsoft.com/en-us/windows/win32/secauthz/ace-strings + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/f4296d69-1c0f-491f-9587-a960b292d070 + var aceTypes = { + "A": "Access Allowed", + "D": "Access Denied", + "OA": "Object Access Allowed", + "OD": "Object Access Denied", + "AU": "System Audit", + "AL": "System Alarm", + "OU": "System Object Audit", + "OL": "System Object Alarm", + "ML": "System Mandatory Label", + "SP": "Central Policy ID" + } + + // SDDL Permissions + // https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4715 + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/f4296d69-1c0f-491f-9587-a960b292d070 + var permissionDescription = { + "GA": "Generic All", + "GR": "Generic Read", + "GW": "Generic Write", + "GX": "Generic Execute", + "RC": "Read Permissions", + "SD": "Delete", + "WD": "Modify Permissions", + "WO": "Modify Owner", + "RP": "Read All Properties", + "WP": "Write All Properties", + "CC": "Create All Child Objects", + "DC": "Delete All Child Objects", + "LC": "List Contents", + "SW": "All Validated", + "LO": "List Object", + "DT": "Delete Subtree", + "CR": "All Extended Rights", + "FA": "File All Access", + "FR": "File Generic Read", + "FX": "FILE GENERIC EXECUTE", + "FW": "FILE GENERIC WRITE", + "KA": "KEY ALL ACCESS", + "KR": "KEY READ", + "KW": "KEY WRITE", + "KX": "KEY EXECUTE" + } + + // Known SIDs + // https://support.microsoft.com/en-au/help/243330/well-known-security-identifier"S-in-window"S-operating-systems + // https://docs.microsoft.com/en-us/windows/win32/secauthz/sid-strings + var accountSIDDescription = { + "AO": "Account operators", + "RU": "Alias to allow previous Windows 2000", + "AN": "Anonymous logon", + "AU": "Authenticated users", + "BA": "Built-in administrators", + "BG": "Built-in guests", + "BO": "Backup operators", + "BU": "Built-in users", + "CA": "Certificate server administrators", + "CG": "Creator group", + "CO": "Creator owner", + "DA": "Domain administrators", + "DC": "Domain computers", + "DD": "Domain controllers", + "DG": "Domain guests", + "DU": "Domain users", + "EA": "Enterprise administrators", + "ED": "Enterprise domain controllers", + "WD": "Everyone", + "PA": "Group Policy administrators", + "IU": "Interactively logged-on user", + "LA": "Local administrator", + "LG": "Local guest", + "LS": "Local service account", + "SY": "Local system", + "NU": "Network logon user", + "NO": "Network configuration operators", + "NS": "Network service account", + "PO": "Printer operators", + "PS": "Personal self", + "PU": "Power users", + "RS": "RAS servers group", + "RD": "Terminal server users", + "RE": "Replicator", + "RC": "Restricted code", + "SA": "Schema administrators", + "SO": "Server operators", + "SU": "Service logon user", + "S-1-0": "Null Authority", + "S-1-0-0": "Nobody", + "S-1-1": "World Authority", + "S-1-1-0": "Everyone", + "S-1-16-0": "Untrusted Mandatory Level", + "S-1-16-12288": "High Mandatory Level", + "S-1-16-16384": "System Mandatory Level", + "S-1-16-20480": "Protected Process Mandatory Level", + "S-1-16-28672": "Secure Process Mandatory Level", + "S-1-16-4096": "Low Mandatory Level", + "S-1-16-8192": "Medium Mandatory Level", + "S-1-16-8448": "Medium Plus Mandatory Level", + "S-1-2": "Local Authority", + "S-1-2-0": "Local", + "S-1-2-1": "Console Logon", + "S-1-3": "Creator Authority", + "S-1-3-0": "Creator Owner", + "S-1-3-1": "Creator Group", + "S-1-3-2": "Creator Owner Server", + "S-1-3-3": "Creator Group Server", + "S-1-3-4": "Owner Rights", + "S-1-4": "Non-unique Authority", + "S-1-5": "NT Authority", + "S-1-5-1": "Dialup", + "S-1-5-10": "Principal Self", + "S-1-5-11": "Authenticated Users", + "S-1-5-12": "Restricted Code", + "S-1-5-13": "Terminal Server Users", + "S-1-5-14": "Remote Interactive Logon", + "S-1-5-15": "This Organization", + "S-1-5-17": "This Organization", + "S-1-5-18": "Local System", + "S-1-5-19": "NT Authority", + "S-1-5-2": "Network", + "S-1-5-20": "NT Authority", + "S-1-5-3": "Batch", + "S-1-5-32-544": "Administrators", + "S-1-5-32-545": "Users", + "S-1-5-32-546": "Guests", + "S-1-5-32-547": "Power Users", + "S-1-5-32-548": "Account Operators", + "S-1-5-32-549": "Server Operators", + "S-1-5-32-550": "Print Operators", + "S-1-5-32-551": "Backup Operators", + "S-1-5-32-552": "Replicators", + "S-1-5-32-554": "Builtin\Pre-Windows 2000 Compatible Access", + "S-1-5-32-555": "Builtin\Remote Desktop Users", + "S-1-5-32-556": "Builtin\Network Configuration Operators", + "S-1-5-32-557": "Builtin\Incoming Forest Trust Builders", + "S-1-5-32-558": "Builtin\Performance Monitor Users", + "S-1-5-32-559": "Builtin\Performance Log Users", + "S-1-5-32-560": "Builtin\Windows Authorization Access Group", + "S-1-5-32-561": "Builtin\Terminal Server License Servers", + "S-1-5-32-562": "Builtin\Distributed COM Users", + "S-1-5-32-569": "Builtin\Cryptographic Operators", + "S-1-5-32-573": "Builtin\Event Log Readers", + "S-1-5-32-574": "Builtin\Certificate Service DCOM Access", + "S-1-5-32-575": "Builtin\RDS Remote Access Servers", + "S-1-5-32-576": "Builtin\RDS Endpoint Servers", + "S-1-5-32-577": "Builtin\RDS Management Servers", + "S-1-5-32-578": "Builtin\Hyper-V Administrators", + "S-1-5-32-579": "Builtin\Access Control Assistance Operators", + "S-1-5-32-580": "Builtin\Remote Management Users", + "S-1-5-32-582": "Storage Replica Administrators", + "S-1-5-4": "Interactive", + "S-1-5-5-X-Y": "Logon Session", + "S-1-5-6": "Service", + "S-1-5-64-10": "NTLM Authentication", + "S-1-5-64-14": "SChannel Authentication", + "S-1-5-64-21": "Digest Authentication", + "S-1-5-7": "Anonymous", + "S-1-5-8": "Proxy", + "S-1-5-80": "NT Service", + "S-1-5-80-0": "All Services", + "S-1-5-83-0": "NT Virtual Machine\Virtual Machines", + "S-1-5-9": "Enterprise Domain Controllers", + "S-1-5-90-0": "Windows Manager\Windows Manager Group" + } + + // Domain-specific SIDs + // https://support.microsoft.com/en-au/help/243330/well-known-security-identifiers-in-windows-operating-systems + var domainSpecificSID = { + "498": "Enterprise Read-only Domain Controllers", + "500": "Administrator", + "501": "Guest", + "502": "KRBTGT", + "512": "Domain Admins", + "513": "Domain Users", + "514": "Domain Guests", + "515": "Domain Computers", + "516": "Domain Controllers", + "517": "Cert Publishers", + "518": "Schema Admins", + "519": "Enterprise Admins", + "520": "Group Policy Creator Owners", + "521": "Read-only Domain Controllers", + "522": "Cloneable Domain Controllers", + "526": "Key Admins", + "527": "Enterprise Key Admins", + "553": "RAS and IAS Servers", + "571": "Allowed RODC Password Replication Group", + "572": "Denied RODC Password Replication Group" + } + + // Object Permission Flags + // https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dtyp/7a53f60e-e730-4dfe-bbe9-b21b62eb790b + var permsFlags = [ + [0x80000000, 'Generic Read'], + [0x4000000, 'Generic Write'], + [0x20000000, 'Generic Execute'], + [0x10000000, 'Generic All'], + [0x02000000, 'Maximun Allowed'], + [0x01000000, 'Access System Security'], + [0x00100000, 'Syncronize'], + [0x00080000, 'Write Owner'], + [0x00040000, 'Write DACL'], + [0x00020000, 'Read Control'], + [0x00010000, 'Delete'] + ]; + // lookupMessageCode returns the string associated with the code. key should // be the name of the field in evt containing the code (e.g. %%2313). var lookupMessageCode = function (evt, key) { @@ -1455,7 +1699,7 @@ var security = (function () { var actionResults = []; for (var j = 0; j < codedActions.length; j++) { var actionCode = codedActions[j].replace("%%", '').replace(' ', ''); - actionResults.push(auditActions[actionCode]); + actionResults.push(msobjsMessageTable[actionCode]); } evt.Put("winlog.event_data.AuditPolicyChangesDescription", actionResults); }; @@ -1495,12 +1739,118 @@ var security = (function () { evt.Put("winlog.event_data.StatusDescription", kerberosTktStatusCodes[code]); }; + var translateSID = function(sid){ + var translatedSID = accountSIDDescription[sid]; + if (translatedSID == undefined) { + if (/^S\-1\-5\-21/.test(sid)) { + var uid = sid.match(/[0-9]{1,5}$/g); + if (uid) { + translatedSID = domainSpecificSID[uid]; + } + } + } + if (translatedSID == undefined) { + translatedSID = sid; + } + return translatedSID; + } + + var translatePermissionMask = function(mask) { + if (!mask) { + return; + } + var permCode = parseInt(mask); + var permResult = []; + for (var i = 0; i < permsFlags.length; i++) { + if ((permCode | permsFlags[i][0]) === permCode) { + permResult.push(permsFlags[i][1]); + } + } + if (permResult) { + return permResult; + } else { + return mask; + } + }; + + var translateACL = function(dacl) { + var aceArray = dacl.split(";"); + var aceResult = []; + var aceType = aceArray[0]; + var acePerm = aceArray[2]; + var aceTrustedSid = aceArray[5]; + if (aceTrustedSid) { + aceResult['grantee'] = translateSID(aceTrustedSid); + } + if (aceType) { + aceResult['type'] = aceTypes[aceType]; + } + if (acePerm) { + if (/^0x/.test(acePerm)) { + var perms = translatePermissionMask(acePerm); + } + else { + var perms = [] + var permPairs = acePerm.match(/.{1,2}/g); + for ( var i = 0; i < permPairs.length; i ++) { + perms.push(permissionDescription[permPairs[i]]) + } + } + aceResult['perms'] = perms; + } + return aceResult; + }; + + var enrichSDDL = function(evt, sddl) { + var sddlStr = evt.Get(sddl); + if (!sddlStr) { + return; + } + var sdOwner = sddlStr.match(/^O\:[A-Z]{2}/g); + var sdGroup = sddlStr.match(/^G\:[A-Z]{2}/g); + var sdDacl = sddlStr.match(/(D:([A-Z]*(\(.*\))*))/g); + var sdSacl = sddlStr.match(/(S:([A-Z]*(\(.*\))*))?$/g); + if (sdOwner) { + evt.Put(sddl+"Owner", translateSID(sdOwner)); + } + if (sdGroup) { + evt.Put(sddl+"Group", translateSID(sdGroup)); + } + if (sdDacl) { + // Split each entry of the DACL + var daclList = (sdDacl[0]).match(/\([^*\)]*\)/g); + if (daclList) { + for (var i = 0; i < daclList.length; i++) { + var newDacl = translateACL(daclList[i].replace("(", '').replace(")", '')); + evt.Put(sddl+"Dacl"+i, newDacl['grantee']+" :"+newDacl['type']+" ("+newDacl['perms']+")"); + if ( newDacl['grantee'] === "Administrator" || newDacl['grantee'] === "Guest" || newDacl['grantee'] === "KRBTGT" ) { + evt.AppendTo('related.user', newDacl['grantee']); + } + } + } + } + if (sdSacl) { + // Split each entry of the SACL + var saclList = (sdSacl[0]).match(/\([^*\)]*\)/g); + if (saclList) { + for (var i = 0; i < saclList.length; i++) { + var newSacl = translateACL(saclList[i].replace("(", '').replace(")", '')); + evt.Put(sddl+"Sacl"+i, newSacl['grantee']+" :"+newSacl['type']+" ("+newSacl['perms']+")"); + if ( newSacl['grantee'] === "Administrator" || newSacl['grantee'] === "Guest" || newSacl['grantee'] === "KRBTGT" ) { + evt.AppendTo('related.user', newSacl['grantee']); + } + } + } + } + }; + var addSessionData = new processor.Chain() .Convert({ fields: [ {from: "winlog.event_data.AccountName", to: "user.name"}, {from: "winlog.event_data.AccountDomain", to: "user.domain"}, {from: "winlog.event_data.ClientAddress", to: "source.ip", type: "ip"}, + {from: "winlog.event_data.ClientAddress", to: "related.ip", type: "ip"}, {from: "winlog.event_data.ClientName", to: "source.domain"}, {from: "winlog.event_data.LogonID", to: "winlog.logon.id"}, ], @@ -1529,6 +1879,26 @@ var security = (function () { }) .Build(); + var addTrustInformation = new processor.Chain() + .Add(function(evt) { + var code = evt.Get("winlog.event_data.TdoType"); + if (!code) { + return; + } + evt.Put("winlog.trustType", trustTypes[code]); + code = evt.Get("winlog.event_data.TdoDirection"); + if (!code) { + return; + } + evt.Put("winlog.trustDirection", trustDirection[code]); + code = evt.Get("winlog.event_data.TdoAttributes"); + if (!code) { + return; + } + evt.Put("winlog.trustAttribute", trustAttributes[code]); + + }) + .Build(); var copyTargetUser = new processor.Chain() .Convert({ fields: [ @@ -1635,6 +2005,7 @@ var security = (function () { {from: "winlog.event_data.ProcessId", to: "process.pid", type: "long"}, {from: "winlog.event_data.ProcessName", to: "process.executable"}, {from: "winlog.event_data.IpAddress", to: "source.ip", type: "ip"}, + {from: "winlog.event_data.ClientAddress", to: "related.ip", type: "ip"}, {from: "winlog.event_data.IpPort", to: "source.port", type: "long"}, {from: "winlog.event_data.WorkstationName", to: "source.domain"}, ], @@ -1772,10 +2143,12 @@ var security = (function () { .Add(addEventFields) .Add(function(evt) { var user = evt.Get("winlog.event_data.TargetUserName"); - var res = /^-$/.test(user); - if (!res) { - evt.AppendTo('related.user', user); - } + if (user) { + var res = /^-$/.test(user); + if (!res) { + evt.AppendTo('related.user', user); + } + } }) .Build(); @@ -1831,7 +2204,6 @@ var security = (function () { } evt.AppendTo("related.user", member.split(',')[0].replace('CN=', '').replace('cn=', '')); }) - .Build(); var auditLogCleared = new processor.Chain() @@ -1890,8 +2262,11 @@ var security = (function () { .Add(addEventFields) .Add(function(evt) { var ip = evt.Get("source.ip"); - if (/::ffff:/.test(ip)) { - evt.Put("source.ip", ip.replace("::ffff:", "")); + if (ip) { + if (/::ffff:/.test(ip)) { + evt.Put("source.ip", ip.replace("::ffff:", "")); + evt.Put("related.ip", ip.replace("::ffff:", "")); + } } }) .Build(); @@ -1939,6 +2314,67 @@ var security = (function () { }) .Build(); + var trustDomainMgmtEvts = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(addEventFields) + .Add(addTrustInformation) + .Build(); + + var policyChange = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(addEventFields) + .Build(); + + var objectPolicyChange = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(renameCommonAuthFields) + .Add(addEventFields) + .Add(function(evt) { + var oldSd = evt.Get("winlog.event_data.OldSd"); + var newSd = evt.Get("winlog.event_data.NewSd"); + if (oldSd) { + enrichSDDL(evt, "winlog.event_data.OldSd"); + } + if (newSd) { + enrichSDDL(evt, "winlog.event_data.NewSd"); + } + }) + .Build(); + + var genericAuditChange = new processor.Chain() + .Add(addEventFields) + .Build(); + + var event4908 = new processor.Chain() + .Add(addEventFields) + .Add(function(evt) { + var sids = evt.Get("winlog.event_data.SidList"); + if (!sids) { + return; + } + var sidList = sids.split(/\s+/); + evt.Put("winlog.event_data.SidList", sids.split(/\s+/)); + var sidListDesc = []; + for (var i = 0; i < sidList.length; i++) { + var sidTemp = sidList[i].replace("%", "").replace("{", "").replace("}", "").replace(" ",""); + if (sidTemp) { + sidListDesc.push(translateSID(sidTemp)); + } + } + evt.Put("winlog.event_data.SidListDesc", sidListDesc); + }) + .Build(); + + var securityEventSource = new processor.Chain() + .Add(copySubjectUser) + .Add(copySubjectUserLogonId) + .Add(renameCommonAuthFields) + .Add(addEventFields) + .Build(); + return { // 1100 - The event logging service has shut down. @@ -1971,6 +2407,9 @@ var security = (function () { // 4648 - A logon was attempted using explicit credentials. 4648: event4648.Run, + // 4670 - Permissions on an object were changed. + 4670: objectPolicyChange.Run, + // 4672 - Special privileges assigned to new logon. 4672: event4672.Run, @@ -2004,6 +2443,24 @@ var security = (function () { // 4702 - A scheduled task was updated. 4702: scheduledTask.Run, + // 4706 - A new trust was created to a domain. + 4706: trustDomainMgmtEvts.Run, + + // 4707 - A trust to a domain was removed. + 4707: trustDomainMgmtEvts.Run, + + // 4713 - Kerberos policy was changed. + 4713: policyChange.Run, + + // 4716 - Trusted domain information was modified. + 4716: trustDomainMgmtEvts.Run, + + // 4717 - System security access was granted to an account. + 4717: policyChange.Run, + + // 4718 - System security access was removed from an account. + 4718: policyChange.Run, + // 4719 - System audit policy was changed. 4719: auditChanged.Run, @@ -2055,6 +2512,9 @@ var security = (function () { // 4737 - A security-enabled global group was changed. 4737: groupMgmtEvts.Run, + // 4739 - A security-enabled global group was changed. + 4739: policyChange.Run, + // 4738 - An user account was changed. 4738: userMgmtEvts.Run, @@ -2166,6 +2626,30 @@ var security = (function () { // 4799 - A security-enabled local group membership was enumerated. 4799: groupMgmtEvts.Run, + // 4817 - Auditing settings on object were changed. + 4817: objectPolicyChange.Run, + + // 4902 - The Per-user audit policy table was created. + 4902: genericAuditChange.Run, + + // 4904 - An attempt was made to register a security event source. + 4904: securityEventSource.Run, + + // 4905 - An attempt was made to unregister a security event source. + 4905: securityEventSource.Run, + + // 4906 - The CrashOnAuditFail value has changed. + 4906: genericAuditChange.Run, + + // 4907 - Auditing settings on object were changed. + 4907: objectPolicyChange.Run, + + // 4908 - Special Groups Logon table modified. + 4908: event4908.Run, + + // 4912 - Per User Audit Policy was changed. + 4912: auditChanged.Run, + // 4964 - Special groups have been assigned to a new logon. 4964: event4964.Run, diff --git a/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4670_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..30c2adc84263bcfff0b595fa11920631f55738d8 GIT binary patch literal 69632 zcmeI5du$xV9mjunzPnufHX$SpN$NBKV!-h$eql)BIB|&bjtxmjT54kF=|_m2R}(1@ zKt=fjfp|!${UfaswH1g~sw$xn5G}Mop_B-fwt(8Itw^P|q5-u~6!rT3&g}Xgdv?x; zy;$(u)86gQ&dkov&dz-9J2Sss+P$rHXLpyREJok_|%bT z1eyby1DXSx1DXSx1DXSx1DXSx1DXSx1DXSx1DDN#rQO?FcJ%JXe|)bb0+UVb0*3TxQ^$ zTt0IAzwEfv?OSl$mXe-@*#FoONv5ToqsNJhVSnIkuY%uJ#CbL)9{R~CN%*R%xG~l~y@bvvjd%-{~ z_AlHk_rCSDN2E;-A?5*dMilVoSZov0<=)YYhoiFAoP9se)d^pFakLI=t6LaXJQ|YK zNNf)diWC+~AhOXZ4`q`>vP|}wkT_H%MLEqmvI^e5;rRG>$ApB;S}sd)>^iIw7CA;! z_I9qULlhmzvYoj6ohAv)y26k|f`vH}2?g@xD{yd2`qYUNF#j$Qs;0 z-WFyNPZwZJVVQ#+_sQAy>!_^2;X}A@eaOd7gB;oPJ<=mtXeM5H7qkc-k1uS;5U3KDXAJtAb1nG&1x!NOe0 zWm@h-Ah$yJ?gx3&V!SaO-6kb&wr+vH)>h{U&%Kx{&4_UT31?{;z?Ir~3nxfXgp~qs zd+pClgr~MK@tTGRRKVgAi}m0fZYA>cPbT654w{7TGbggQ_g1C@cT<>_+}}=g%IsEn z?8nDRMO2~@R;|3X_ccb5!#kY3bu*xFqC|57#ils1GUjp*if9XbcAy~jnsoD~6-yu# zh{ZO-18X(jxTfBy$*3L_1t!iuE5UGJnruK4R!g}oM!{T#2X!U(tMFZgvR#K;)Q+oa z!}mPoN*}V2rJeU-F`{=X=u`8e@WWfS52?1=jE9)_{C3Gz%`l+69ad zOX;x(A75+gOk{#wLPY0|m;qZ*_gqss%ELY!=M~cDcDd%U zd)p7TKeGJH*#%8Gb6+S(%>`Rk*Uf(WHMddV_VEb|Hy?TPO4OOTgrE|^>K@If2C z#tA=6s#hZXcC5W9G)#4lt&l35ki6T@_2#3Vi(f78!(Nkpdl3o+i|J)LnQt@Vp>mT# zwWC_tgOI_3aE2k`*A%fKn2$Wim*VG$%B{!&maA^WZLed36f;FkKYdS%`wk;T-SFFH zt~Y99P3vzpVmsjYn`r%&Iq_W`kI!4-FduX6`O6*u=25kGKQE<%ieF(K&JhwYCe}$( zDgi<2uue9Q9v@2FSF=AKzb`u5i|~u$%_YpIa`^gY&R~6W0i~Er>J!zD(y$v>-izn6 z8xH`NB><}0OXI#8N6In^vmTWu<7~n`*^oA_Hl(5oYc19WAP)Dc zrN*q>szM!Fj^A2o#CHQeHE^jlVQQovzqN4ZFWu|mm#($=<`IVGRypeBD&yXWbJXF( zxXR=bDhw_yxc)6EwbvG0WgMTFqAgDK%LdRkKl6}}Y_YJAGlyvX7vb=5t-dKdmv8N_ zMKl?=@@!d$Yu#5evdgsUBW=xZL!3jk;tl3}!?oVSx7yPXe%RKUh&2YSbTy()-#Smh zk%&L7^7b(6uf$e)>iM4@`R~@Z0&JBZMCI`8RoLL-W2-EwE%6D|I8IBvADhFqz{`-- z0-;cU0&shkFC5Xbt7u4v%KOV+V2)m3JpV~=3fSF`a(}k zhGbG>x=cHtRV?wA(P{geJZYJoF5idY0p6$Na%EU0^=i^GLdId!aKkO0^BN{gJ@2qA z(d>zXG~y|Bqs15J9KKjEhI}!d|F*G=7rr1%)EP2p1Wz=A3P=KUBSTagmS{8-QH`Tj z_)vbShre3mN(!Mb5=gxXN$RKrOVna*G;~rAu6Wwvs?51!KZs%n$c%DL0#zXIO0iDG zidx<*{N=t|<;M#0Fn|dTT>l9Q`3+X{w@Fe}t#_RYy17K&z-FqNbrcaqK6@3b5=l$1 zUhxS@uUZ*4N`BgmT9sa<;?tT=oiZSnp(;fn8CH#Ai#((XbtMiCp+5264x>8piMov1 zv~R#H;lbaW%-O%#}LKzY+@ zW63R17@Y-aG^sBTZ{7@gSql-1TGv?WS*J&!XHlD4CL6J?cQmTSP`<`e*CKC{E~_1$ zq~=Afs{wB0O1hIL$)nV+s8o?SYv5u@ltfCci_FQ;H8@Jus|J1>u*HtTbc&rT+}CP$5pT3aoGErasreW%;T*-)}NUE%sa;c({HrYc< zA|_Ybo?mh=>6ffbzuf0H37Rx&dwDHMSHhQH?0MX&{+As}V9!n1ZdUeG_EaTSmDsGu zi!$HA!o=TphNS^fz?{9X=lVbp8DtRh-DtVe2Ksx4h+{|ey zwlm){QazPChn>>`hA73Fjy&DUf6dVZ&GSxdFH_A^HBTi^CC{wM^MFV5Ja%N-qD1nX zq2#ILspP5Tnb~Y=_P77chyM1t<9hTd=P0qSoIU8A(Zq3_9IeSc_N%i8p8e|d!K0i3 zW&7yav(8bS+-Dpdv&vDQ9H&}|U&g@^pOyIatIyqS0(z7`RryS^wX+ESSd>4NKSz;2 zi^`r^n8=^gl|Pj~l|Pj~#}|LzJdVeEa&FP*i9ZLM0PT5K{#5=nmJ4aq&)RHk0E6r7 z`z&oRddGbJ`1-2``IBZjuU7t4{#5={{>+R&{jw(c*AeKSe)YU_X1r@R(rTQIM&l%p zQn73_R5~A;XSvxXqf!aAX1A>xN?p^=i?|sI{5y*eq{GnN1$AI%al8+4KGjA9~(jw3SjuV?O27DKTgy(gjKuzT zawVCQtZ8jH%?FR#UiNxy(T)>o)y|41U)pe->CT>cJ6E1opcszUm`J1kG3a13a6ExQ zJFvZ68B`fm8B`fGYX&{!VbF(muZ?}|*t-8d81y-1P*p#@)U7k#b;f(T8Si9JKi;J? ze+25M-waApIF zo64KYo8yZ&@1LZ+xkWZ$-fgdRV(#q$ygz=6w8 zk2k#*VI6`zneUv{-iTwIokU*^*rNqcy3pOP8SI0NfGVC! zph}=hprc2iUd8j1Lv`;bDxP0g0#yQ40#yRdj6nTTJfC-7t)?^BM}J0BHyh*F^cPeixZz^vpZ)V1uR6PB7bJ6s1Ji?P!IBB<&x+XPD zZqw!_-<-*BdxUm9t&T}Qv?kw()xPD3V?=42lfLcyj%eYNcH*gqQthOU%5T3kPq!JU zJDgy(b2GME)M}?%?NqCsYPBg{Ld~;K`G8XKy>-_s%_QG2Xu4oqg(j z3q0SNNsIC1R`#~D=bpCY{bEnI`mfg!pQ?YV{;B%s=;@!{5uXoNx4)Ojo?lh=RQ6Q% zRQAk_J^dQ-`MYsE;?uqpvjL%=pRjQAkvC6HX?tqjq!Vawcq9;u?Z@Vl6BA`Aj3aj; z$^pE#qyyg_NM^7(7F&kjlS#~20hh>%M?ZsT zkj_pC0=p-L5ZiK3Y*p!ED;gp;_l&^;q@D4t^2Ar0F20hK_?95YdysnOVLZ;DjWf$D zUWqujW9^mY=8AJ;%N#PCYQmF53*Y~y$_1<+fh_AOEAbaE{slcZnjmZvF%dGnPWgdQ0o1l5{!K$$WM)X-ZRv;U# zlw5(|2JCkYwiEqHd%-{~R*9tbwMS6d#!c_rR-_r}xp+7#YfXc<-!uwr9+W?oKMmoK zXV`WM3Yqr4XWjcgd%`SaU$Az8W3r_5*uw8r{?X;n)98JFSNSuEW|cotV8Q|APvy^{ z__H?{5FT5Eo_PFe^tnt7KV*WlulVRq`;;Nw(#gNyTK}dPBq< zTWZ~SobEO7$9U@ROFkV$H+&dW0}HCFqzEY7oV{6Zb# zWh&^(E1+NIhs_7_Jaf^8<{tgg7u9BRJX4lwT6j0e^bBLI4=^^SgySLT&e0Lnuw9Ls s{{tF~8*`ZdFzA=}l=;A%sdl ziS1ml5Ft<@RZyr(kqR#nQq`*}A#qbwp-NRHo(g>_ZN*DpfG5;)QLDTR|G&>ZGh<_` zKqbV(zejs!U)J7h?QgBU&WSTsU6@|1R%~NQ_<>9K4J>Ka7i<*y$o;^4<#^FUD$u3b_)N-kIGiRmpF|-3f$IZc6s3|Pe#)J z{WtZzb>z*__8qoo%)-#@Z)vkXr`l)ChB3Yme}?!^%WEI9<|*i8*$$m7?|b>*_;HT? z7-I_??D+!vANW6#Y}=q?$?wIr&-(Tu$WCL=s~h&A%>S}{+S~k2N2K-k+u_qr{*5Op zFFZQ@{I&Tvg5Dso?%+>{Z+$sl!S*A2KYQPqk^b=Y?DIQcz)2`I(QjAK^2a zN3U)RkSw5I$M`NgkGUFt@33KfcB3_Hd+=Rps$X8$O9YL^yYN-)xB6_xj@U))yy7$> zz$Y4wFT<|bqwlSSc8}A34!X*aI*Zvc)af`F4?dBy!!ULUlV-V`1(^qZJnS4y*%xiq zt;AHX^(H5hb_il$^%DP>>u7^@z$P(wD{6)z$8C1Lowa+h#T<|=;`ocs1ft8OER)D3 zEt3km>}Cwak$!z=hdtoM7D1|pxkRqZQYrZOMs(s3FTa+y83@u>RVQQ=O4tqP+~DJM zYAt26&KIYgNP-Fj%cH|9!1Cga8M_+}q+5uH>H_YRwp-ANZ_dfzhISCs7vZ-$IF`Yk zqG=mU0sB4ZJmz=Y`)C)GE;^?BAzFa}bWs^H4i9e;W9oBj*>xU_7XeSk`jbHx(bCE# z)5)GT+V^2YnoLU~@`+s5vb5!4EQ%AtPyX6v`<)o=s5(nxtj8cXJ?(FJ_Pwl4V8<00 z&S+V|Nuzyp9oCy+QlQ(5ul5^NM?3LMTP*5fxHK9|&=Dt*U;Vrj2e{x2Uv-Ki_kM|X zz?()}vcK>2F*^^zWqiC9p@kWyR=W1#@eqVhc(yTsT&IP}pwER9lQGLT5u*DcIfp>1 zIlJk!J_}MoqwydFnAPaGy|vJRT0$t$xah7V(!n-63KI_7kllsAJcNrngZ>DH; zrH?=Niv!G=XP&t3*B{KAJ?(zq2tv5~T|6(p@v%q}XWy92hQNkNtc-J?hv`HAa>gOV z$13J|VoE+_JAU}B*$cBjIPlcf&+kp%_Dau2DpBgbcK^@bf5#_zw(ouMM@RQvtEQ5F z_+H^Ot`4Q(!7`eB=CVv)W_ZP^d(wsa9LAXy_u-e%U`E*^^%OD{uOw%;qh^>5)3|Pv zI0sWVEse8cq6fI>&YgabPqbEau3K>g>&lks32cD!d^w=b9WD7B_^jG-JLw`}rRnZ? zWg=Qx^ebJmh-~ZKVr9$9WMt*IU;CMs*8UA^H$Npp>)*6`29zE`tszW#UY`PaPM(+H z;)!2plqNgKTHR4x&s7w|Yg2p-csERS3t`l5RxKMoh;HAO_ z@PHYxZD@zC4mPhHuU|LDMr<4p^#Xp!@IWu1J&H#=`$e=6xXprR3`=?tUyuLtrx%Wm ztzEezIsBc!LuLZn`T9^po5J~kGabe$Z}JbcZo5|pY%!| z`|if~4EDU}$PGgS%kZTDq&mHPeCz4Myh~|!Vr~uRDnVxjc2@9PLe|&=Yj@)_gi8C0 zXpN&i>SoKBA4Z=sQNdfpJlZ9!{+iQu$Z_Fo7wuy1U4;}iP1zZ=Ig*Ff^(`%1zbXsq z=)>2t_76vk(4Kdrd1rI0U2Kk*FcXarp*7wdC+*GT+xN`dF+@rkwSsyMbrE&m_c^z@ zeIu}m_A%mGI!2o95~8p*PVez#v@dRRZOhiuI$Mu7*XF^w0Hy^*Q)DM?rHy>uJ%zft zeSFEKeJwWBx6s^otc`04lw0%;2-+KGb!w&MQp^_bK z`eOOfHvf*c#YW!Qanakuf3A=n8IGbdt8MPzqPMU8tYn}f0wN#+A|L`HAOa#F0wN#+ zA|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`H zAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F z0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+ zA|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`H zAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F z0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+ jA|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L|)2L%2Jdqw{L literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..7cdf639ce487 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4706_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-07-27T09:42:48.3690009Z", + "event": { + "action": "domain-trust-added", + "category": [ + "configuration" + ], + "code": 4706, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "creation" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{be129571-63f8-0000-a795-12bef863d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "192.168.230.153", + "DomainSid": "S-1-0-0", + "SidFilteringEnabled": "%%1796", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TdoAttributes": "1", + "TdoDirection": "3", + "TdoType": "3" + }, + "event_id": 4706, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3056 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 6017, + "task": "Authentication Policy Change", + "trustAttribute": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "trustDirection": "TRUST_DIRECTION_BIDIRECTIONAL", + "trustType": "TRUST_TYPE_MIT" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..3f9f51c4f98aac5aed5e44b3716a7c4460ec1922 GIT binary patch literal 69632 zcmeI%TWl2983*uh_U841*LH~|M7V6KQc8+%nDt^PZ5(5_D29;W79#OtylZT{-UYli z5UVYsrIn&2N^g0|3qqn7HL6rq6&1=uB~+t;KQptt-r9l) z9`f)XYj$VOd~?ose&@^?V@8Y9W3$DAHE)ri;4;2Fi<)(K%^?qc-m$Uz=ZCID6-Yn= z5|DrdBp?9^NI(J-kbndva9@GZ;`GQ&X%7F!_qVmaFY_|~DA18I`_`_jPx16&Hn#J@ zmQ=E_jx7nZdo5-k#m(M}HC{2xAo2kI4Dm^~x9@Y$4YbK^3)NsK9M!*T!3 z^^gDc)BIn&cF(i*-n&P3{64&c$1m;Q_58X1&g7lcFL%C)o-k^-)2`u=9prXvecVQE z$qH7o3eLZeJ#i+u_*Q$|F50{mad80`;$GHf5LU6Xw$FO;vtUywDIjJN$36I-Mag+9 zj<~-IEx6k1ud~(kGDQVB(@+FL0 z8IF0lGc;~pF|Z%S$q#}W_rJOhjm|pL5rh_y0EZ}#7{|k>h;!;o%c&J37~TY)gmp%} z6lP0(Eg6fpHE2JN3Yju9@#Q1wl%<%O9&A7m?rnwiO+?=n`La z4f(_SHRgfQH0F}}YilrPrx3V+pCF5*B@)mp2Ek!@QVRt)4&tS2YgY{!-FG&0{JsIsaLS#Jx>+94Q>JZhION^DrLN zah&(#vmbMN5JNP9ULC_{4^+w^+B&TPuup_zdLbp;>U+>-1_qV=ofCcH7oc_^_!h@bKy&X$N#vw?YTny{oa3`t@Qp7 zPd=kCV&B6d-?7{dUg2~Km+FIVYEPn^tKb2AXYgL&ta_;&!^*@*kXN^0PjK~%+xy$EZQkM<3_$x8{Y8F zh<_)+nXOwK{M_2txU-&R)X#D#xc7_oRy&*SUA>o>N)MRzIFwFfFJW4;zg+ULe51jw zm{6FbSaLVmCLAZ--i0S)2(Pssd=KK4)`Q~#yxe%6#Swp3~ zYrCU;FMo`fVYJ2)yyxS5{^k2zv1{5=4Xr-H|K0h!`aU>`Xf`f#-u z-@({#fDuP;44=i|arfuzDB7DuEk%qOd)o@97#=9BJWaif1icR*JASk5(e$Z%g%c($ z{~HUn8=g8tj(Kav_XrltOkgwZ-mp_RpT=mN$M=%+^M56oL`enx^4~_=ZMz$t0gTUH zOSQ0GvpdIMx&5>F!1!y=e*3NW-$`E?#ws_8diVzM*H6A6SnnQuj>GAaTdw-t-9WzV zL8>)~4_5)+n{i9pK3rQy<~g)kK>G!J=g^Z5>8@1!XfXFRs0dh+<2lb`HlZp-_3v%$8hCcvlM1{ttgGYy4qp7A<95 zX~yMTKhD<5bGYJ{cjIWNRz8UStQOx`XU0zAImu%$U|+&Mi+w6M=e5=I>qkz^kI!4Z zjsDtk4$ok{oTJB{F~9Jb_ttaGt;qFM?cNNWd*IcB+2rTRT$v+(-JQd}dVc&Rm-*Fg z*xq!l-oXaf9F*(%E+7wjm9W!}xw4FYYv;G8mS24i<<=OlTy0D_zIC%fv=`<#&@jFO z4Ra&o^0*Y_!+);Oj)nu+xu&l^zS$tZ#=g=(n*<~v0SQPz0uqpb1SB8<2}nQ!5|Drd zBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J- zkbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(T zKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU z0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb z1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ! y5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}s~mAn-pd!uHbu literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..d16ff334435e --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4707_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2020-07-28T06:18:04.600444Z", + "event": { + "action": "domain-trust-removed", + "category": [ + "configuration" + ], + "code": 4707, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "deletion" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "192.168.230.153", + "DomainSid": "S-1-0-0", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" + }, + "event_id": 4707, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 2012 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 13679, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4713_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..5d4bb4d159ae89680ba533a7704a81149e9dcd6d GIT binary patch literal 69632 zcmeI$No-Yh6vy#%-^A^MzP2Cgcu`nXF`Zk5@SMKpl&c4HmqPN8X~TqzjKGS)Bq}R_r1*g+F)YHeV^CPsQhr>jnDuB2q1s}0tg_000IagfB*sr z+*6>nw|nEh{sa0yetg{N^SBr5M}c!ivny>2&c@OI{oVUny>P?L@h!|Qn1!L)?|HN9 zx$z}tjk^AXeuntB$EADJIio)Pn5{nj_#=tGm|XX3AL?57gz>D?`K{zfl9?0w=u6u2 zx}PTBw<)$wZJwXdj>p{Zadpx^Z{d;jhn=L#k|kPG*{19MSbcBx#q#B#CJ1awaBcg- z3uz7iezN7!od=uh!XJyDEk3K6c+~njJFTxiK|jt<$y=)(wl3?pA)UXiS7Cor_+?Gr zj#{7fs_>8s@A;L*EsDlg&IfdTnH^DC zzdn~+qyARvTcfSkd)H|9hi*H|1eMCKTB~&32HFd@%Z{n#p{PeC`1(raMU5+6(HkS7 z?TC6mtiH;M8dT{jz1q@b+;S>s+cnre6`GYw78G_T`LK9;&YrU2sHTc)tR}lYYugm| zS`u-yW70UR&9+Hp3-u~^%6T)3e<|9tYNA7AJ(_+`Gz6zB<*bk?Wv!44Cfh^0kY@Uf zs!8^I64oQBewAfPlP#Ci8qd*5n&Rb4dE2T$U#s3I$UG=xvve{m$Ofyv^51dgtu_KC~?=KBjdWkYicy9-6o5IkE51 z$-7C5E$>WLr#+GBMum210KPz4Ly6QM zjoTNk)`DD6sq9vO?`pnqE&X8x)u&YO!NvDVCLheS7c_+JR&N`WnA>!tZq<2{-kX%$ ztF%Pzn$<47H%Mhbgr4ob4ja^bxW?Hmp&{yk-w7qLA*@-CGOxoaXC2Cv%=+ z)3Uizoz2VHRD~)nqnEN+53_iNp-s2Y>a5mkR2oLF(27x*JTb*W~M4y*~P|JNqdvo2cI8t+;xS7?<#y7FD=6ys%um9T| z>y;nFD)$+CJW4CLUtj&1>&K=yJiXGbdsCF^9lGwjVyiy=8SGhhav-Oj%5O<8U7%ON zcf)*8(&&e6kL{1vX=rpYc4_2g%T$|9^JQ&L+KO)FxMs~AYNk7Drdp%*x@(fUOUHVx zn^1R?EdBA}T4VJyas3rZ{kx2MtLpF7t6xdvqt3<~)Vic*(^$_BL>t^H-CE9>{V%|T8Wp-ne*lGVf31A*Y&Bz@FVXzb@#hv{^;FuP3C_1raBnq zR=>*Ds_jPY5zV$jqi>Q!r{4Vm*{G65qh%c`?b74Y?=~-6Y%MCEJK9=@+SobT+A7_( zYoaz+s#e_E`1V%o+H$qlrMQ(z`Ry&nkI|!YNk>@Ly}w4I>_}>)quZmF%4(xI=_zd| zo~_>f-zyvlAbDVvE$gA3kA8h7os|8(imE#2+8rTYbR@O zQm>r^tHLT&ND&|^C?NHL2q`Lv3W*0)6cwmcA;C+aAQcJm(ih+bL<^{WKw7@<%$c3F zv8hmm#KSkM*_}CaF8}j8m)$s1wfXX5t!i73luy{eXJBcw-e7B%+rIC;v-zvru0$0` zKmrnwfCMBU0SQPz0uqpb1SIf|0#mj5iG}40_&0uZ?DV<8$NZx}m)YPq51i%g?Hp|9 zgmAYO|N8y?@gH`4L-*g0i%$NH zCpv!r=ZVudE3XATL14RrKc2kr<+uc|uO0i)BbSEz!dLU(+y4T3!d&Bhb{&`OAdj27 zvNmOFR<&ho;Qk9ZbLaiTmwK{x#g?pwhbwrH4MuGN*&24yj@kjdR%{L_Rpgn)^&mbM zk@AGqaaXr_pJ)X7`$o3hZ;~%pfZCJ-`63_0#nd6jWHT!>;x6`O%7G#U){-O(k zbcKxNQiZhTGC`NU7YXr9zq7N$KIyY9LTVY$QiU$dWH82e<0hWsm0MYxMn;ZR&1EuX zO4(hwxy#Sfsr8J_xUsn4vZUBxU`5<;3Gn#--8p+01IS?^5t|EGQ`X*xn|S4%|7B>h=>)P?5dep%f;^6gtB7;z!|VA?BN!I~ zPtN+%K_0Uu*-Ogl?l$d5P$5HRB$0fokheTzc^ZY{4dL5wcG-l>#yDy&l9-jdV7cs9 zc>ayNjibf}0_SXLpr=v2g%0bYH%SRDfnB&&Up?&&D2;l{98v_(NEldZ!ZaQ%@ z=J7d9(Fx?7#Y9?madXgmEyx7T=3~gfU5x{GY&mpLOPC4_E?O(8Y_P{3MF=Ntz#hcJ zJb{Hejr(D|4`XhRVTfkXt21~Xgi0NRob4Qk2T^^zL0=yXk%vRJil`%Av-coJF5au> zr*rlR^p`FBKJT*K=}e)|?#b94$cky{PRwF_nMFMet=~fHvtH}L)6kuv7 z>;6lH*H6Cn&rjd@={IIJetl}sMJyfW!U$Gy$?seqlTUGa#as8Jo9eSj=dL(~Pksk; zmR+l7uv2kK^65UDIqrtppn&LC?VO!=1cyuixk+Kw44XS zc5PvjG1mRQ#JiIcotsL;F4@S1F|mbrCHXtBDmXW0u$f$hYN|VDA?rTi^KveF{arrq zBzn%hD%y(Jk5qR@RBG6lx;H7+?MsbgI2&$do%73g1?8}3u(bz#IkxIs+YNP&Y8me} zl;om|=I0bzoW(4tA&q{W(8ypxVdt^kwg*cnbZ0-7Abw;ZEnar~WE#FXcx3zW_SF}E zaOLdS`t^h9p)dYDFsz0OMwcxuV!be8PDF0w@h-CPooHbX7^ZmZzy#%gW1)7#Dl?R? z@OKZktW#F>^~}3(*g4$KyBdpF4ABn|{v?@28x8a;`Zn5!i9d|pYXmYUrcfWhL8ASWqr!R*;e8r)UUhOqh=j+`MgXZ!pFdvp zEL(J2+EF}PN8}~6Sw;I*e3sCk!-(V%UIRE8-zct*;CjS8t>F0(?l~u_=-V8wODO%M zYwHB^H;}gA;^N-Bid<}I$}ZuWDHN*j7}=W36PL;(cl6@Vvi28KN6}Kzr50Vx$@OR} zy@V%G`T(wuw9=`)UH@BNd!goNG82anAXBKHEONVZ_Av#$c7SG2FT?;b&%& z&e3Df7+-wNvPrC^O|hP9l`X<~5MG0rO;MbTl`-f7_U-mOgX-Z*%;c3;~Qxk-;uVtQFL)!_3`kZ zE3~8G2+o5zw;$h8A76W4X`n*_5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz z0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8< z2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|Drd zBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J- zkbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(T zKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU o0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndv@P8oiFYAl~8~^|S literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..6e43b04c6f36 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4716_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,72 @@ +[ + { + "@timestamp": "2020-07-28T08:17:00.4706442Z", + "event": { + "action": "trusted-domain-information-changed", + "category": [ + "configuration" + ], + "code": 4716, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{be129571-63f8-0000-a795-12bef863d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainName": "-", + "DomainSid": "S-1-0-0", + "SidFilteringEnabled": "-", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x6a868", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TdoAttributes": "1", + "TdoDirection": "3", + "TdoType": "3" + }, + "event_id": 4716, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x6a868" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3776 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 14929, + "task": "Authentication Policy Change", + "trustAttribute": "TRUST_ATTRIBUTE_NON_TRANSITIVE", + "trustDirection": "TRUST_DIRECTION_BIDIRECTIONAL", + "trustType": "TRUST_TYPE_MIT" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..f27db52c5362e79c6a22584388e2f2ae14f7543c GIT binary patch literal 69632 zcmeI$U2Ggz83y2YW@k6+O?KB#l@dZ*9GbRCi_?TAg%D_Q{gYab-6VFPK_D>R^-sJ% zk$0WOq!&>jLW(Nd0xgwN35rxzl~9E!MIqt}gaiT=#6^*UxGEC8;DWSN^ahsqJ2MmS zt~W6yx9>#bojLQJzvnx1*6~njW_Z3-MEl`K1!wr_ApvxI?IAaO-@Uo+ zxgWmuRC5crrhvEC0q-P%w}R#;z?L-dY5p1F{T{EcQRhkaX~(VX(~f^d@js5rt?UBJ zX4=N{An)Ic{z$T=t&eilmeqYZx*lWfFx$M;){ezo@Axe}ZsfQ#{zJ|+8oB#!=9IQ^ z-ea47>Pm53hFN6Z~1c+I_9%&i5LP%gj}rx9VgH6If=; zb=M;ocz>gDjN^(kdgXK)hg|O`*;j#4i+p-N&*6A6jy@A$oP#a%A&|?#OFbH`haH;& z9Ke#R$wztQ6a5K{G3={R#Jf}Lny^MN$Y*!)OxcpNW_Dc8;4qt*BH28rKktTMbh!X2 zKbJr%@YdraEQnY7=B{=4Y!o(6sS2O@x%CJF=J+<=#7n&JdJ+d2Xt^r6AXt!(TX=Iz zv`&Xk2bgrZSa2b}b?Bj&H}(qH@ycx}Ji-K8T8zlL^Q@&Lw(%x@bB_N$jZr>cX5OlF zEYRITlh_oH{Sa@Sj#}LR&Gqbb-kAR4vmEsNI2XC*(fD5y6l-x*S)-Nz_csP2)GgmU` zXNz?X+*V7SGY#9#twTO#n}Vf%@2`4bU572%oUTzRYs;n4D6@}v6S??W7bkGa4SdOU z6sGrK8waxq8%yR-UD2AIW?+r~qEV!gPTAIKsXezZP2su7HU`LbA)WBL-Ey+c*p9Dp zi4HJwiVLaY#%)RKh8K8^#zPFSUCk1=zmj%PWiAC9T==f|NiUDjaR}q+!$Vw{V|-B$ z@_sje?&jLQA0}=M#ar?Bccz(COU@F<7eWFbW<)C9m1hp7aFjE%KBC!ez)guD*MqGA zHZqh8(4Abb{MriVlEx;a(SvT}`84g$G%_xZRopSmJA2LvV#43bGH-9$8syP?=G8y` zhy_wc+uLgOskx2--Z*mhOmT6&B_?^`v;KCXl%KDQZL z(G^VNWRlD&NP^bVdW+xL=Qj-y|~ zS8&|rseUyX6C8N~eipdN(z-Wl%e}N=&HOcL#(SurnAd!adB-COJZG-wfxfMK71yMxM4j?fy<)Gi7!pZRK76?b*iXk8npBvRM}P-VPmuHxa`V`YmUzPZI53)x^R5$ah*nJb8U}n z9bx+%`ngmK)Cl)>R&VnPzhlQZp8}oj`^|n0>*d+pPkSrv^Xl9RzdYEEWpKb{&VEx{ zq;!ePCme@88;O?sET4tjcO1bW9TJd$1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(T zKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU z0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb z1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ! z5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^ zNI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndv zAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp`wRL0}wHD5J>8q?gASuclDL z5*9I!3T82ZI!o#-slm_sNC*9m#xaJUuLc-&kEVFn&oedpOn~P0KET@6UMXs;pDh+J zz>;aqb!Q}1*B#3K&a&;S9rs#vBdFp8mRsg#9ekAM z2%j(D0LNY7x#%W1k1Fq)C4g;=y}lN(hcWL@0GJ#aJ2aQ%{8`t|N4*3*elG9X_t(Sc z|MU5ae|hTh@vT>~dC!B_Xaosh@N5@`aMBIA-joxcnHpu))=b^Ul2w`NUem|Vay*v% zFRp5Fs8fp{#8H~nTJpbE4*aYa_VJ}veH`f2#}qq*YLmQ3r_C< literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..fe3d49133e01 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4717_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,67 @@ +[ + { + "@timestamp": "2020-07-27T09:30:41.9034803Z", + "event": { + "action": "system-security-access-granted", + "category": [ + "iam", + "configuration" + ], + "code": 4717, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "WORKGROUP", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{b69bb9ff-63f5-0000-35ba-9bb6f563d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6", + "event_data": { + "AccessGranted": "SeNetworkLogonRight", + "SubjectDomainName": "WORKGROUP", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18", + "TargetSid": "S-1-5-9" + }, + "event_id": 4717, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 820 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 1571, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4718_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..43ef6f5a7877fefaf1892f54bc1fd04ba7c6f87b GIT binary patch literal 69632 zcmeI$S!`5Q90u_Jovkwiowg8+!EI1c!L+n=84#D$(kh{VKoL!dN$E_XbUM{8EH*}h z8WZ9gmzeOPCT=fAO;l`rGd@XBUo_D-P1FZtT%$gi@%zrbcV=2!*B2kYJ2^9VIm`e2 z&bjB_wzoLkH&QIv{4ElkIF4V>qGpZW{4D?ZxbfWb2miSeRUiQgNI(J-kbndvAOQ(T zKmrnwz+VdV7KeKdmPYY6zP~N@dz_c?Pl0UO?8I-+obhx2`){`MZEEkK+8QoMn0;So z_Cwt4t61$7vz1A+Yw&M~f4ZIhi!u+RO>P&UO>RF2`Ckjp^V}(%8=e=>bvXVd_$SHA zd2NgZb+PPA!TvVn?L(cj^Xg%qf4aQ~qrbad7XBk=kpgSiVoYHhj{EH7t55wh^pV%# zdA8L1_Q#!{hBa{i^;5Z>r++y8*p|<)J@X`b!d%@=b`rbnAh+42aqG3iR@80b_CZ-_+4+U_;g^e)i&U}P-%X0&QZiGmp{Q+h2u6pkg(l$3^h-=7BRuQ%jH`U zS2&{I%_Qw6*ZN_!l}D}#TY3n$~&Ct|i8!H(ggStes%;+9}OtXUDW8*JK@#H9vn zh;~P98?xOQF1{E?B*cb2&7857RfBJSPWx)Ar8VRo~11sF%iUsjek` z(fS(gU8s;DGZJ4ul1W>dvD}A3;ezn&_e-qDWn&yg7fHyv5Bd7~f;&9@UD~=);}int zY?(q&{dzNX){x+$z+vxvx5?PFUlaSZNTGU8mvVUwZG?-+>DN=}z)=_Yv}?#8-WwSQ zM$;He`r}kEXNQn+5}zQ7q$LwvS~;}$ZA?P=U|<^pWKx!ldW~*6aWUq052k1jat>l5 zm0a8$v_|t{Ub%cdGH_Mnz-=le9n=`60)zAKl}Oxcu)PRjyEWVOn3&seqxR#t4c~2; z+g(O+RunG9Pq!p7s2rSW#OFhrU4$Hoa8(}ePuL#xjBNzHTWZUru}qV#irF$`#RPN- z^A(RRzb{E!VM%MUMr**;q)SOlyL>3(wmuy3Iyz{3Bdc)E>nyQCY}w zdgG@d4CX9FmoVZ#?HI-1E$Zv3dZze7bVm ze1a1xT%pCU%cm{p)5-A|)+FA1yt*1G!F4b0Wf1+e?YBd2oTe&+u^W-+ zEmCi`oWt4~)a7nkzhuKACBro|Qjcg^c4JWX%4(}k^U7|9B_9;js+N!V<=cbue-d>+ z${#=~VG=RwsJ|4g2}+(QsJWZdGMd4GQQT#D;Eu%c|mqh=TO_Tc9K5@W;-YdN=g z7GJBd&-t_6^}t_(M{y5^0=uAiz&(sDxP2%2USLU%#WmPz#nC$a1~XuJpwNeM#o%`L zXKgP$2QfE_L2py8w|H;FgJrp`bUpQZi;;2y;mycjp2L%aTYLq+9((iEqx-sMo?ahq zec(r^bVHCY1tlatxF_87#nWFOS99}c4=Q(=p8RHB0S+*K$ZceQG zD`m5AZh>hFM%|B-u`))!?v5fY93NkD8DG_g?G0Dz?W%FjLAe@Vz9vt2K8xt8eVgk? zvvH&%(lpW-5-r=17WZqf#FwjSuj%ufU>TO?lF7R^N`b1|T(t-sQMD0%yGX&#rVq*->7Hg7RITn(D2!r_Ac!1%xe^T7{g?#f~sd~Vg zqPoQXS!sQC69mWcvs!rKMK!dlX?L&mKzNH~8vsaDiWhcL#+@`6Yq+#5h82ZZYFYKJQ@eGs7cH+YaylTAQtAFs$ckcbAcdT3o#uE&0LWxMCN%nG}S zv@CGt42Z%l%>^$;GV^!nNsV}tmXsvX>zT`wn32N}Q#w%7er#L*6I!5cEjgVKW&W-W zHfFAz+c9|Agvhvge&EDm?sqg_{?|s=PUu}4%<92LgV+G}y#uP(AGgPP_{RkLb;l~X zS!(y*+dJ9&_}ZrrUD_Hx@2%=`Dt_pGB>qtNBb!w8{hOo9Hq3jzD|LAH9huoUIw}PV z#__5-SIe5a)xCnyy#{8;`FWKg5s*_vaxHxQqzahDvl5NvlnIoX@i0oP-HMl+Y%8QeI%cu4)3pd{XE&RL<%rt3dacN>5Gdoo}!70C^ z6VU>t7QYop-WMSv=zA01SC~Y;UrLEn9zZ@ne%eeTOzZD`5L6Uc0e&3ER^A z6r@W3qag^udVti~pTOWV@wCT9L$H433{P`Y#3(hR#x<2r7o}Z9b#cMBj&3H~CG3XcGu!-C^BetmPY%?ixC$y+Vt@xFuk~wgwmA;bpW*AZR zn?P%qk?=p5{3GM1>;|l$=~HBW9JkAHLAzTJy!1Obh3CcdwN2#izbvTv{d}r3{@4s*u zTCJ)oY9??-EfH1bcyk<|Q35CHv4$7`q7j4ov&B>YK0ID|)xMiL6AO6GX&9>Bh z1#0qD)BC_RlgFJ~y6=2tG-0$BeZO(hzKyFM{`RH!7C!U$chPLm*IRhi^Hma?i`wTg z?-*t-rl?An3Mx8{k=`Q2XBFnuOI2| zN7x&nz@kE;Up8xDJ@oW{`8%iI?%fklVTqUM-CUw~JVbA}<9&#NKB9GiA*OnqpCryB z_$D7q@J&Ac#|Zy|@t)s3gm=BR^n3)*?;0SDihc-0$zPJ%Cvwn9{upX_0-D$?S4rs(ljd2UT%N~%1<8!i3nJ>2if1FlEADhbd%Rrm_uH-mQNqR3$vy#%FCr2j9`1fsX_R;3iHYB zMj0=~gIUFU&U>g00lBRDRUqO(F1ivAR~q%S<)E9oR9W1jg1GpD8U^vdO@NQbmU^iJ z1<1w1EaJ~;OvytZ#Dm##ZvL5%w!r%U%54aI>;d07sE113VEZ;ae955La8Euyx<~P} z2|=Ss02fgY!YKA|E8;r!;e-B!5;P-$Jzgrw(fp__mMm$_3B+-GI|y=;IVC#hU4?$~ zb6Soek|_}W_U(LXQo%SK{VF9UTW>fm1``_|YBPfm{;aporkyD*+VJ;PUStxL^ z*Z#PK_|rNi&QqF50j@3+69f2$DMX%rG7lNpuM$3{KBO1#CQb)RlQ=E@m-CD|+l_!D zIE_^J$meCz%0+vu-UklvHh60?pfHboIa;x*PArW1IEpITgpd(bq(PN#F0^9O+}gy% z#}I&}8W&u{piePs09An#r?(ZCM_Wugkc4&$(FRn^t!Suic&@^473y{vMbw2{b>epw zxH1G5a&6~w*Z}gTfIhLxhcH}ZV@S2mUZVFSgx8eSy=`8)9{J@j8+k6E`8n>w5-M}k zJOo9xR2OP7Ix{_oJ}M<2mB1I(A1=OpFrshEmzY1H8@!yB#kC=F3er<28%oh)U@;xLHbdqvD%2rVoYD zLoP5x;{yGVL3N2m!=FIIl%%czQp+(aj2Kr~G4c{(bmEKr1sBcd@YP26aw~ixcsxeC z=w?-WMq^dPjG3dytTkdP0i)-yC{0W&m8eeq)p{c?=g;6xi7jqCr>yuMJ$?qMZUcAw za1EkrbAG+L*sLeLv+v8iQD^Q?QOIbrW)mB=Xm2~2! zqSJ8&0(#sABPGlOBPH{a_@xguZUrxf6$KXGt^kG>QZwpnGl05RR@E_fWJp_9;P5zQy`F%&eOxifLW4SDByaTxn$Z#_YjjwMmeK2k$h zi^IdIxbN@2yZ@%}!4qq8D*p=NUA^aC$+^G%V64A=9a5Z+P~1s0#`*wqG>WhLDNl9k z++*oI5=)>Uc*}kC7`Sx}LR6yMcnEHt)rgrcE_clw>mX?CqH4sd#@S8vNJTfEqIe2n zlu$>t>Rmma>+t;$p84HcgbL%{gD?>l`gZj#u7jhh7TVL-LVJ-`PVuG5RWEW;gH%Uw zk09Ng%Nmt~8q}+`NPP%7tHad|7w4RLP>1nQ1pZ;=sgai1dlP1^(|JF$~z^Q6}*Mnv{jc={`o|C5Mtl@F-I%;;?5MT~8 zx-}r1T9aD{AMeaGIM&cI(cr4`%kVKz?MCiHaRiU7UG~b-*Lq&k_C7H0d+&A=omBUi zG}h?ckT(}hVRQ*KI{NN#`7IZq(cV7x{0shz&-MK$dVWKTdlgP6?TsZsqSmJLp+pJd z(UO4zU03s&1L3BFITi+us&es$*bonRYZtFFjiJG>a5M^UnU&{9|<59mC| zxGWfrOdOBLX^9!9!9*RIR!m2nLK=89&1fBK;AY)~0G?92Z9-{URIKt~El~&TSe&zd zRfBOMi}$N>hd!*_VZ+T}9s8}ovkBha3$WV^$9mRj2uxE$nvY^?hYl3P*!OzeH(^A) z9XekZ?WCJ9GVVa@ZNxo@v2Y7~Zh&7CT(xzjTQ(OTX+Cv&{F&#^)DEUeSAt3i7wg=dlK3+dox28k@Ky=El7eO38|3;bMWr| zK1TBRQ`d1g?!+ZXg8n~iK@uR}?Eh<4!#6<^9RFARXN@GBOhOWFD04;6L} zF6?~O2}xk9p<(F#{2K(u!MM#Xh2Joilqt^eV*_Vss2{%Wc-3>wcbt8;>y;;-I-MS8 z2!pnTGcf-D2aSN~2+ojQID_@S`{}rJOd=kOn2uZU3$cXAW=n`QDJ>r6+7cqBGW5ma z7aQSTE%*iZ|H|fG@C(5&_@BTjY~IBZLj7Y5Ekzv(w(Lpx#q#$%;}iS3kJ0_ZVH#bkAzrc-;e(@u*lK>u*Xk#FD z66Oq#AS`{%xd~z?Au}%QEoFK(EmNOyad~hCb7(*KlQ^8=+i>3}ID_B}GUI}0c*u+k zg&~-8<};mf@kA2Nux^Pn&hXEIGYHOrxf@fkNY~e>VB1=2hJwtvux60Rj0>4@F*7@f z?KV3Jo`GS`xX5NZi6^hV+a*j ztxtdY{D1uDX!&8&N#N$r90>q!v>JgU*%ZRBDBXL+pC)^E!WeAobScYI3N$dY* zie=oGgk`*Og)^4%Pl9C#mLXV%KF8g{X2fPfY$h(B&4d${k*wx8 z&IrZ0AQXa7hz*6

2l$bAWfXjo+_RcKsZ}Ty0*lVe(x+YiQJSVYKJ>|Gn|?v$u`c z;657&MUR0{%zW3+BZ5$56+*EeRuqYr5<2Q2yM89Fl_0x*%C4WOcm0eF-UUezBteh_ zY>dJGRnpg+5hOvO2rlwMza?xjUy7?lZ6eRhnVCzP@#1FY5x;9jGLlfU&>2a1QIG^M zQfwW>)Z-uy1HKkc948 zLlQnINJ5q&3Ei+Zh~R%mQ3@+e!9o(EwpkitlOQ$;Vv}IENr(;J1xXMjL68JN65KQo z+AO03d@+hJdq_fesWX!BGeHsrNf4U^u}Kh{1hGjtb$Wb8NW#(Rp}f})U%c+AvNz8y zJ^IH}vwWpUoJ?9F(!0PJNjNV^f*=WkBnXlqNJ91?30;7c3vv0N`|^fFD-^_x zwTJMW%_~H92G{i+8gjLrdho%#g@GqN>Vzb8;y=;DYUj&g%$?{^c8OEpV>2>qt24;@ zk=e8pg{&WG-~XV6_M(mO%&QF|@90C=Uq4ew%eiHDXS_e{2N%-3oru zP4x=H;oWq3N6xUao(a?Kii2D6ZZ`Mrc(3em5mOlizYt3Zv4jvyh}pA*NX9RAlsMxT zKNb8!@C(5&1iujcLhy^3T0*?Gyx`ZXuR4G2Jry@rUHtj_6LXJWd~(+C3t3$>rqh%}=R>LJ6eQ8HQiSP4Oi9*+0C#2o3)%l78&;Qe zvcI_3t=nd>T?V8HoyO`wnub;nku@Jb?Wn~;YJ}|RqD`=C*iM__+DSKI_tg%p5!r}) zkUFUaJ~zOx39j0@(k+{dk2KE|zu3Oy_jUiUu6e`bo8Kw_M(OT}xyLU)F>CWYWbMi9 z!Y^d)NwJd~+vZshCv zs81@FG&SC$A^U7zzI`^|t2o0a+PdoA_ej}wk3P~i?te9SZtii0kIx#;AS+X58_p2F zY6S0gVXY@2D^oVBSsVw^6U%%Iv1dq)GsK4Wf-?xtAUK2I40DMy{A<7&XZT0K8K8ru b(Yj1~1hHo@Z9D{LNQImD5^$4CfHV9*Hi+ZS literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..4035618ea1de --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4739_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,71 @@ +[ + { + "@timestamp": "2020-07-27T09:34:50.1578005Z", + "event": { + "action": "domain-policy-changed", + "category": [ + "configuration" + ], + "code": 4739, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "WIN-BVM4LI1L1Q6$" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "DomainBehaviorVersion": "-", + "DomainName": "TEST", + "DomainPolicyChanged": "Password Policy", + "DomainSid": "S-1-5-21-2024912787-2692429404-2351956786", + "MachineAccountQuota": "-", + "MixedDomainMode": "-", + "OemInformation": "-", + "PasswordHistoryLength": "-", + "PrivilegeList": "-", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4739, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 812 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 3532, + "task": "Authentication Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..7dda9113651d5fa4f43e4274230708bf48f58be2 GIT binary patch literal 69632 zcmeI0U2I&%701ury}Q0%+q-rn^AQnn(j+7SJF$ZU!QsRBqog>F8=E8*L13^=oUFZJ zZIeJG6aokdiHb^qka$6r5TdOrs)8s{)uKkN@<4$gK`V;d(mteWK_Yx8h?nL6pL=iC zUX!G0B*epiR(J2t%sDe>e&@`Yv3v4kz2o^&t1OAA;Q-#ACC%!+N|AG}*I!V2>zoJC z03sj)A|L`HAOa#F0wN#+A|L`HAOdG3(32nQ+Bvx!f8&ScRPO`q#yb?;D-UQiRjCrVH9Lk(c`Jn1-r02^nk8F;6R;_|h ztNA~ku=M=%FP}a!{jQhuJgfDN^wMm;s{SiFrT~X(6aw*$q6P8Ek6gpB~yX}P7wC%C0Z8bhqb|+d!Av28M zEqITkrDz4z3N{AG7Ss!9Uu}EQH;H$ft-v1O%S^;lK6bMY8I@G){VYP@uV4Y9A@Qv8T$c-7zVO&EPvdY zKy+E((ur)+(!N(?OVAKU`UTZhwlxwP2dPQ)C9*Z<`|$C_sKg;Y@3osdx| zVHcrtQ52^=`+Xa7zS!+V5}eSpCRDfuSblMF+BU<1bPEx2I*%izY$+=7&e`>5U^k)r zUihs5jw9es(Udj%z&?P=kE0PgepG`=#~sryh>pSlx@ZJ44i9$`V`{_x%p4EKO2Cu0 zhNPE4w3Ks6Z?bL{?JXFPCexCTd?K5%3~hNBN^wH?$tN||<-}-5-dPf3-2u7Y-e`ws zj%TbBBTmC`M$0r-8jhQ-vRs-;fo^aALxXYZa3rqj6iRgrmr`j0bHqvH*DqIN0sEZc zGj5{Ly*JVhc++T0=Finp%-#vXDSRR;0t?bit#s|f>jMzp8L^E4WUDPmdi5@xn2cE- zM2L1lau|U$>FlP{>do`LQfWH`nAPaGj>*7*nm{PfxNxr|Qr<$_1{3t%&V)aM2J}HHhyPP$>W*qn&=Z0mH`$`q`EMGIZGttPb&-T>^=8oU6zB(snb} z%bBA!*V=-lpKY+kzI_Lx2ut@MigB2QD-5jB0&B2(%b_=LPhc4*2NhR(QDZ-rZRh)o z+17rP1EMq?D8BmF*O@a{UH+%PKY*OK)4kvI0=WA_oR?$2jk4<|*gZFyHG>V4*a+6W z7p6DQTVpfgV+Q?PF(vP^Wludcv~TE%jn5srwj;UXa9t&pFm?Zg|2lYL z>-!C#FMIj5OZLNm^+i8@$I-Sv^~W#w5L zDs;QSlE(*WmGu*${@O_YG*Sy$L5QN*g4^F?sm!90=#}FqheVDNmha>m<9q?5i zv$3TMJ-zT;9##$FO0WWLH(Q76>p*W80*^9XXxWB5I)Lv1v<;zcGg`Nxy&K&i z2Q$nR-tjGsk_C*l3}ei0Ukf+Uz8SRkJBjeZ#N&jilN-7PGw}E{YpTW9^>|?2?FdTV zgX{6Wx`}OO2Y%!82#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y z2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0P zh=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%) zfCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y z2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0P zh=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%) zfCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph=2%)fCz|y2#A0Ph`^Z%j9bAb gZP@Zw>pk|u^1Zj#nE6n9$UM6dPrc<(58{Eo0{#d_LjV8( literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..71607b7242c0 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4817_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-08-17T12:49:09.4942066Z", + "event": { + "action": "object-audit-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4817, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": [ + "WIN-BVM4LI1L1Q6$", + "Administrator" + ] + }, + "user": { + "domain": "TEST", + "id": "S-1-5-18", + "name": "WIN-BVM4LI1L1Q6$" + }, + "winlog": { + "activity_id": "{dfcd2c2a-7481-0000-682c-cddf8174d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "NewSd": "S:(AU;SA;CCDCLCSWRPWPDTLOCRSDRCWDWO;;;S-1-5-21-2024912787-2692429404-2351956786-500)(AU;SA;CCDCLCSWRPWPDTLOCRSDRCWDWO;;;S-1-5-21-2024912787-2692429404-2351956786-1000)", + "NewSdSacl0": "Administrator :System Audit (Create All Child Objects,Delete All Child Objects,List Contents,All Validated,Read All Properties,Write All Properties,Delete Subtree,List Object,All Extended Rights,Delete,Read Permissions,Modify Permissions,Modify Owner)", + "NewSdSacl1": "S-1-5-21-2024912787-2692429404-2351956786-1000 :System Audit (Create All Child Objects,Delete All Child Objects,List Contents,All Validated,Read All Properties,Write All Properties,Delete Subtree,List Object,All Extended Rights,Delete,Read Permissions,Modify Permissions,Modify Owner)", + "ObjectName": "File", + "ObjectServer": "LSA", + "ObjectType": "Global SACL", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x3e7", + "SubjectUserName": "WIN-BVM4LI1L1Q6$", + "SubjectUserSid": "S-1-5-18" + }, + "event_id": 4817, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x3e7" + }, + "opcode": "Info", + "process": { + "pid": 776, + "thread": { + "id": 3052 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 114278, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4902_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..695eae2bb3bd58c67f3612dcb789524337ef1370 GIT binary patch literal 69632 zcmeI0X>3$g6vxk-c{80(+nE-uB7&BkqAi;cSyZ-`y;fQQV~tj(h0=BgX$w^nQ&BWA zQE?+Ags2!q#06uFgqTPqCVU_u#3d49;wSe7l|V3Y{Ga>YW!j;LNHj6|-^qLLF6Z5Q z?(Z!3wG}Oml}#-Tl3fCRM;D%s1Vsv+Y>`WD7xqhkbjg!&0umqr5+DH*AOR8}0TLhq z5+DH*Ab|@KsAy@Nxgyz&zwy0tzGs&X7CD_G z@@q)sYq$G|ND0=x5&wp`*mCFs?`+4KS_WcGEvJ0>1Af2uZ9!jSmOeM5ecJygN&l=h zTK&27-JO1YA!I8t=iaP&RK}Jg*!#a-hTDJa_zR611)D-IYxkO^)q$TnK6T=bBl*r( zRU`Jb5R!mV3WKGz9lTks@#asyoVnGSJPM!=#hxPP+xjC}f z7i$8kB*p@ss{IRP)Pxy zdb`?T2Bx)=j^Er{?LJO-Jwb@mtmV8uI;RScJB7QH%$qS z++UDQ!AsT0b3U1G4-l;%oc&_W(aY9sefrJ)-*1DaDg}#eLRHULEwk3EJK0n{%eZzu z`ZWtq!m|Wtu6osDDHoYZLr_Pr#1+=;7jiT?YHrf_=zy;65-1RfsOif5_ltR*oZK;EEW*e+7%V2eI(kf7^MGCBhpLvPj%V^D*;jczZ@aAm1TZMPmAPmZJ zUX{yu+{Ysnlz5HYZUUhKnrP8VlqLuD zCohLPp8g-ZS)>&fia>Ph$s;^$n`ka4hw0KLYx|_q(4@ zb}!yXgH6C1P`!Gk70-C3@5354>cg6r{H0ukk1D>x1dQ?YQeWz-j8t#AKTr_xg0fc~ zR8@sxlWL$K2x%Ag7kH@yPuHyJ<;A}jA-CZKNElrkfPA%YR{hHI(Cb=t$ka6$y$jDy z91Qw}*KCY_7jjJ#QsXL^rXDFmWEfhRuQZiRVXk4AqkI2&dTRk8@H#ILmM^#*+Xy9qvoGRbz)t}`1^g87 zQ@~FFKLz|0@KeA~0Y3%&6!25PPXRv#E|>xX@^kx7Ex3j5@k5Jgy@YpDqrkCvrHaQ~ z{&48%i~rAdk{|owYx@-%#wisHDs?7Ssgp7GJxaBw&wEod#7|ugU*McAWRsVMWRsWg z@V=iVnUgcuWQtB!+ z?8xb@S`kx9TX1i1@6sFoG5eiB^*}%+1WpWVaS&(l_VQk>N8Z|@M)2+Ur(15PZ^B3Q zt)Vv3CVxR*_9n!t0cwuQQ3a}q-rqy7xM{NU#p-mY3D535gX{DrqW?WYKH2m z+R@)EHJLuip<^b}c3WEK(I+!iA-yY9xpZV(dS6JNw^MUzUjeN* zip)446R4`1(g(=tbuX);M$2J& zlu`w>FDkLBiish{$@B(UJo|908cGK;t9m3 zKOH%dGE#x1o1rDCKun;dWGo%PQH{a%DhOH$HJ!2oiwpTmRBWK08bcC>sWz$~Wy~~k z)S>jg1O4wndAlni%AjwRPXF6dsuWTZa&Bi1{pj)t&>PzZ=@>>fn^cFASE)vHL>yvu z?$9_jioP#fmhYUPYDLE+)=*7iR82aRvZeK+Tug;o=oaCfKIkG&=Q zrIGIP!K@9X4F|Do`rdO%dYkjUV;jmJvuQtnV?J_5T(-P0h9VWaB=2obuQ-l|v4KQVKU+;w(=1L!r7+S-dA5qGrc^D)+C{Eq zxpL^t>V`8BS);0vT0XajoO?}~)fyGfO`){p;)7GG?EI+E`JLqaf1-LQoj-+M1(b6EY2C~IXRJtVQj|Abs%Q-9@E;$_g>|Dlj za?z4}Zu5kjT;&3bN@aa?mb7PCIznHKxj^a(6i2Q(8VoSJh*zlvragK}5Ka#~_sbk0nxImewxtRQeZw2Pq{UcL2 ze9&OY=gx7^`3EeTI6#@FIfUyUR=vYsevB6`cYLwpv~yf^{sD`oxCLdN=3?>>i-X-i zZWazt_FC~mrmOWUSTu2fGEZ{|*FUUw0=s`q5H8;w-n8F2E;|2!MH3e&^E8*R{Ub{_ zM4wvpk#ih$n6C$0TxZ=2b5J#4&28B4(t~> z!r|4yx!auMpo>$mXyO26p5_p4oU+Cj*yB{LaM>Bt>g_C7{R0;FjoWwKfTPX(zF1@Y zm~IEMQxT`2%+p*mO58V>QC=A5(?P zk-NLTIK@@}fJIaN3d%gq#pEB{nHdHD;B}sGSTbqWU-De#0E;FLP*y29SpAORzwzwh*P}2Mf~Hy;^9|KbId<fBqhTpLvW-WQ zO{49Gw3cyf@uefUczi66xhtX(kUV}Zjz?MX=qf&=M%fuBV(UJC|=dY4Qy$>in68H$RK5wH1dZ23J%rbV!ePweM`rDH%zt$pU{UA4 zduVVSlvP>|xpodSgu|qQ`{ujL0Ty-sa5d!+D2ogZw+e^iBlDNP=&C=!qK*Um1C&Ju zhnd3R*6r(7yUPIimH@Kv`sPm@ORsyCl@GX@OP!<^+<_L#QtNVQEu0Oz{i35~H28X%Ap~sg8 zUjD+>@&Oig9EUX4zOtA0A-QE zVS#YC>CBsNxa$wFh`2O^+R=F7{&NEuuDg{$+9s+2Cm%c%)1w^$&2Xd_Vdn-UND0rYR5d=rutt zq?Y;~G{ZqRl}fJ;^lC$|{&aRn+UiU{{%ucZ@o)P3(RLTw>O|X}X@6%rBedOx_H1)dTE1d7p-oIG#-|QVu=09CKGc20&AC!3_|Dm5K{PzBov>u29yuM91#Q)K44zOtA z0A-bwgY{cZbE$`L2OKHBSPa$1htyXXaFp5)L> zdLnC5T$?3Reat8~KMT)Tf>}0V>FZL5ANsR-%`uCYK}BwJfkhJ+D2qHU9JlTeE{n!w zH*wBibma#u>bPvDoC0N@M> zGWf&js%@O(Fj-T6EE0<*4p8Pv4*jJ3XhOAPuA%chXN9UUy_-!VT0--Pg=Vv&@y*th zT|CECA=QU@bWCXSuv{v4ta0gFj?lij%!|u?Uf(I)PGq{x4Hiw@pv>o{<93&Ddw$U$ zT=2J>=9P_egGCcJDD%1LxZN$>niadj%|1R67ERos%;%=#c8_pdz0z%NuxR22Wj;3@ zw_@QoaK#Z9;-uZ*V9~@4%6x7*ZubheItx#@%ncTG+_-%h%6x7*Zp($+{7ENW<_3!< zZcyfP({Z~`xb zr{TQ$kRK2ZYwC2I?;Ho+@6cgUlK(soQ08e4;m!|Waak+v_v|0P5-z!`A7A^OWB!4D zNPE2<7Hbn27Lg@@*gsCvI9n+51ec#aj-^PoLqFs(;_#qwXsI?k$YG`Qe~q74)*cUw zIu7e?$~?gViBZ~0m?kVx1yi@tno4I ze`|zGqq^NUpL0C^+3R(%hU!c@Fmb7+5rMfU?TUA*z%E`^5&~aC%uwddzLb=R`aC zTtrv|hvGArUyu71Zr6k|PjE1`YlhpNruN*K(Uy6H9`=gobYH6U*Nsc)nlx9lc^DXLut0x%}qmQsVbtmwvo5rOO^lC5%^&lhmo7Kgvo) z_qf5Li5ryp+$a;%QpSz_tBP>Td~M+Z&i&4C)Ab*~qK?~#qzcMB!HwhHyEG{z2aAh& z4(xf)Qw$21YVTk1*Y%EZ(eGpcrWy0aXn3LbT z%K;Wm{sCo`p99CKc;WEXu-Z3naLgZcaS9eq9H7k89Kx=bRebJiRpGKfX6F6QanZ#o zSTu2gGEZ|c#wq52IK}>vARIO~*|_63uKEWo);V(JjDdgsX!zEL-+1?U`n^q1UO<_r zIfU&Witg)~2iJ?456ysWE$MG=+U*<{oqxch$v>dX(_D=Hv4GMZ{=w@+;jlILlXH(d z<{vr^uvn8i&RIlOC_`9PMXpalS>@!g(8huNqPlQcQ*YSfO^$KUmE*8D`1XVz=}){e zWMB5DKdgM>+bW1tQ08e4;l?TJJ`Zt<*Gan;~qG{q?>i%c#xgv)_vm#y0D zs=vUZi3^lP9v3c8Y6_QaH#cv8$T2P$-)HYr@U~#$;|TO zTWSdhwPVn5cR9eK&Oew#sj>iN7g}Pu@jA_#VQnVO_Fz3xie`eK=j705{KOQR0fIiI zaU*n8Kbli5pFX#qZ=_xsXFU&kz?M-7lUfvXTpZq2rmIQiuF{U)70T=s)_7HGzBih; zMI{7&ItS8guV4K_I2`JEBEOld=`4Ej!sYrQnz;$eDycuvxO_W@+QQ*T zx4pNz%K;X3{=gidtdepdF9rv$Pt_3)FZ6rnp%#w$gRY(ni?zuESVR`RctJfE$~?j0 z=eDa`anoMUyXFFHq)bF2?pN71Q( z5H2e|?OCh4t6X5wRGvUtL~t3_;^0+L;KJjy8w!_K$F+{{?HCu`_Ff6YgTe*3 z1Mqy*Q07Svmwr4ev0`2r&)P(|1zWq#4Hiw@pv>n+nOO1DMZk^g8BK-Tj>YR5I_Gb? z-+#d(J*md>w=7Fu$lrK=LMZbjw~P5bcxkh@v(b8OM1Nvmrv|3Q07T4m%enMDgc7z>J?WBw>2Z(<_3!?joUk*%;%=#c8zd*X0O}aV9~@4%6x7*Zmoq|tpjdwv)4;t(ZmhPd~P~! zZG_vtqi%D9MH4qD^SSA`wH0nlHoWaZ`DXVwSk!Uj{!1wHxmD!G{gds4+m3|c_g&{| zzcVaOQECbe3qP;l8Ol7t?ccWFxxMh6ndl^6UB5Fdn)pJQC-`1OzjN(!{mvbP!?^lx zbAUw?2Pmto9Nt=l{(o*C?kF5SioSW1bMa2sUImLhFEY2~lMqV_+V!E#lN>I6yMCUU zY;V`^B-~cka+@10nz%uk&n;BH$ST*a-&weg8q;Zu^ZsUU_kcwcHz@PDS={J%cXog4 zBHRW|a)X<_J`al~ZcyfPv$!>gf8Nfmt8m+v<2E-~G;xD6pPR+a_PnRHR4?f!+#daE zl!I|nH$DdzP28Z&liV(5d`=$oz;iUZ3zvRsw98yz(ZmJHB8JQJ;KK2)hj2MGaFK&t zboF>xq!x>h&XSTW%bi1YdTwumGEZ{3^!4}(kESk7Mna@qf?ONe>ZQGJ3-M#;U#e?hTS#k2c zQ%}9QlO(dgah!xQpPPMHzQa$B-`DXVwSk!Ujar#i^bF0XW`(yeFx9RPsKHAr@ zerMhM8L+71_93Z)GEZ!hbKf{_|4gFz!Kiy9_B>y=3b>}$f=4pV%+A5&pD2vRn zvPr0Z$Q+=|(;R-n{29*%(7wUz{=(&%R;QYzIOZQZF0dG{qOA_ftO{J9EJC=%{NHvv zmlWah>($lj40M$XESlmLltl;^+k7(|&jtvWt*f>sJI6&=p1`8GN$HC3b2*MdnWwp! z$`kAPEcWt*W;d{w1x@}V-8l}rxCM))0AvFkpv==8!Y)sCkWV0PF`t3L<^AO^k9CfV z&Ocx=@mGHunUVQ;+OnA=KHh&vg8;ZdnWwoJ{bMJkJ^X{$gM>r9p)DSAmjf)C$`2^3 zoE&!9IIv#~77pE7FMG^64!Sr6izW_G=4lRL$EoLt1>zL1e<@s+Tyf;pL9W)XU{Tkv z$K^PbMFyAWD{vViTpsDPbmL%Gxxk{yU!W`^xa26BN6GFlLxs!oN0uJ=rK?X-?sORa~yQ_VpueB zfHF^W2)ABbZhn5|bAxahT)6vt=eX$n0~Sr?36y!7i^)IA^_MV*k%&|OtCkIOto+b% zNE3@D4p3GpIZ&z>e~UQ9>vZ9;_w}NE!(HV7i=RK%{@4%6i&7phtbbtLvYeJ^&w{c_ s$$^4%*V6VZE-x~KL&^ABe{_z6uDpOn69*{sG>34@3#-s%9^o$k4JT{->m~mnBee+zorTFer|+kx;iB&MuD~QN^M&7 z)q&8_m;aybwC=W{x&0gJ$0`*Ur_|XPrA`Iydz5NVpZBI_h)Z3LUgVsuWRsWrWRsU4 z5a)O}5kM*i`z?{pG%gkGY)XM5W6)OR45) z*pbs)wIZgJw%~qm@v_1HnDuU;Y9OEz0w;&GID#{HdwH+c!|&`+E$*H8=UVQdU&2TA zsirp4CjUTQ_9w)s6g69At9(^R?;oI7>{Qu#plXbotEQO`=b->6i($-Imt5^vMiWK<^4v4jtK+-WSm4?bICFmrv`Csy+R6rmgm>JN?ft zF+a1+yQn~M@fl(TZUq^!DqYQ^Z!WYfQU>o+Ts($!f#|mj;?yw9`fRe5MMuq`y* zip)4@O;8OXvD0X$Qi+Kw5Ib7(L(7^$)nCoB&ZJ#cRn_P|(JGA&n=eP4n^4J?SOZlu z?Q21=Sk7|%H7$?Dt6S(RCQy>)($CMeByiFt232fSVzi141}dv2^Z{~u?W-!Ok#blr zrBpuci%P7lf3fZiaBXB~}EgXutKm1iBK_@F3NkKWXie44T#s4^@TcUXr+vB7|9 zL~qyyczK~gtQtxH84D*7n+xza#i*wA2A(tZ4{>S`?Vd;63Md`3DBbzc7*#V!$v%wU zJR;xNYgJ`3nro$LKRPs<1Tdm3I>t&K_9D(xH!O%RLl7rX^2DlY(Sdl%Ev2y}HM)u| zwTIIevSd~gO8Kb7coolD?xT|s5H^2TS@p9HV;y;x5+v(bIxaO;JUsr}c-4o#v5Hwd1d+WjUPV7(t|feyWhjJq z0P7%{inYYQca`MXNp#>$`V$p#DlV2oE2F)yM;xW_6iHhoK;l&@E;^88WhV~Cyey?G z+K-N$Kp83D(#_D4R3I2AE*?z>a8zS(z4GI%gqlWKfyISN~iyADOCz62|2ejhraan2+$kb#?dj1Y!;~wC9hHq>4;dw z>YTx`Y6SgWwk*FnLDh&3CRS68gQ_|mO4-tSQ7)#!Ec6TGR81A9s;MMZmG;J2uQ(NN z9Y?2FODetN{q(cdzNp6hS(lbILi-vmjwwp9V&;7XyN{iorPM3d`rSYh;qFeazxI~! zmqxnF2D3JlHXOvV=y%T{>1{6fjcq7@%%c7Li}}dOs@dcBX3WiaY~a@YH};CY?v*O# zQzaC-qkc1J!U-uPc>BR$``mcU3SHNv`%BM`k zs-r^zN~^k*O*);yIg_2P)GB2+R^36YUR$c#9*XSehHOeuY$@l}Ej{l-=VpzuDv9m1 zk>f452&HW0a>1ZfTP4!Q8D1&!64i*ZcAWL9scOrbU4zn1QC8t4hF$2N`*PC~Up`4W zf|r5xmrnUQjj~n-<>kKgN~i563ph(Dl_hS((!~$BK0?P#qny-_A_13qT$gBPg_lqi zDjQ}K0@*h{^Ics!jp9&=6SIT!FJbY`qSTQD;*vqXvk&FV3^jnttl^|BLvevmOHi$) zBdz)ntw)K=cC4|Yary=IAzdut6G~%AsSLytPGnxXk|2&H!)V<{HRejS8a6keZswhI z8Z0h*zrm6%TRyyb*_M{~cKd3`-Tc!&qJyEV43+RAOB$uVJw;y~vtn(%_?J?)j!2l; zBERFJ@V>=OF4-LW>|923a?#S@{Fd>zxyl6=mCF3&9OaNqO9yifae=Z3;X)-AxGba8 z7$aP=$J{#EIW8SE{sN1FqhtjxQ08ea9GNMVtvJRN@=$$AQE5Z90#1L#ty_gdqn$-B zILAR3w_x$e`ni^Wynp)Hw|0?aaDXyTa|pXUp~`kidBS{dL)`lA&wHKYqVtaiVzImOFV!(M)j6E63Bwc?C( zTy*{ci>9~*WuE3@@(+uH-9K&@4$t&jv2U!a^($C3aey*Ua|qWztabvse~cF{-yPk2 z$T==L|A0jk7bx>Im$3aKQ#eGQUigV~9CYOeESfk#nWs5~>mL`C9}|SjnU~j|a*m76 zKVZ>xT?l2K=3?>>t6RoieoPb&QR5DscaDS3KVZ?s0m?khAzc3;3U37L<;Nu9(xu7E zQR5sd*LD5@iz(MGZ`$md-=}TfxG8zue~r8o*B?;kX)Y%Jp!SNOf`9NjOE_$PGJ4&3 zS2@6DhJ*5^RAaih8&&Pqj`f-+BYF~uqFwF=t(W3q6V`QpG2o#UYM4_GvD zfHF^W2-iO*t43;=-9M%Xmt*&LJuul-|A0kP{R+xF&Bf#&+?g2#|KN45a9BKX<{xui zg(nd5BZIzC--u@S>s3 zr#j{zIxetis$W4_L~yZ2PT0BR373I~-l;#$RW7h-;sRw6!KIK!#Eb%$?!;u8a9Mq3 z>G9=`aXDdKk1BQR+|uzTu&9e;*O4bfnJ2jH-S*7f_jt@FExg@x*o;xH57_hIq8c&3 zd7M6zQE$s>vo8%7;*o7Ul58q%*Qd3NV~Z~t!NucadCXlQjez9wYq31aibq%R8P!V9 zC>;Y_GE$94nDJ3pmmXD##vA3(i^n--(>TtkDp4wq#$Z{s3mUCdo%;BBL>3RjV~gB- z$zv`HX<&0Ge^}!lIlpj!KbM2~l5d_{RO^g$`KFU5-@u~IpJ{kgi84>+oA61{cumwp zcwHbIMjd@(=6=WgSH}Ssb^g0Iu~b=w<&b0NFkLuI%ztQ}yBuIq=MUFW4uP`B;BcpK zSaxjQ@&m5=11#z|us=XqWN?@v9PZq)Zne7{V9~?@$|8fqOyQ6oO#Ji%SN#DNb>%|_ z4M~Qw$ly>Y9HvYf^trnnU{U7}%mKzEO`tU1XxmrHJqON@4N#CF>GC0f?4qvv)C~}trEb9D$%Lgcn z3=Z>zLzfP_UFHCbCJs;*864&dhuhA+^`^W20E>uA)2SVe7w(VexCCX9!C`@Lxc}2v z?)%!+@&Oig{=gidtb%f&yI#@0w3rwy6b`3;U8UMLj&W!a;BjM0jkUGY!{W;Ub%c_= z%>9RZ={=Nrg2Nx4eerL1r;=(~c)Mud7h^u&H|kc35KkjVSw%Kc2`E_T9+V6u_dpqAjTu5#5J=Ivc;hks7;Booc^lk|K zFQmI^Gt_u`O`<6ea_Kc*EufbA9yG&2Hzi zwmQ*vXWHMH&IoOHp*>yIBKny!{|e_jwD&KP{5NZ-llf2A&J2sD{0C*8$baZ33ctNS zIjslc0I%;74)MQpn*%JGI6zqi!;j(a4R%7Sk9?c`Xu zxxu1|8%Zcd3vtr!Z?I_M24y}s9k&OCTdf5rUFHUhI&R!P z3}rqy9k=DeZQjIFE^~uL6E`UHx#_q)B;0zm-r3Ii>oo{vJi+DC$Fb0J6wnX(F>!cQ zIJ8t-9OST4`oG4_Ep3m7MIDFrHf5gRfbndcn*ZuiV^Pc%<6|BZzBSftbC55_$JqT0 z7Il2N{s3j3;9K6$e$x0D_P;g4rD5&vTh2Qk|LpZTSVX;U^!JBv#JDAne^BO0E*Bq% zK@X8b{NwfG!r@%+>QB4N0Tw?TzkTzM=X(uawP8nA(*CoIRRTuy@v$HgZR z54W%0>Tdm@NG$5O+(4?J%#&O$UQeT2U{EgcdcAPC>yg4|oabP_j)6rJ2Pmtk9HL4% zuwQHt4ri9Oq{rNre@?WM&qahqa9DQs>YH)h!tI(+<_Qj_cFl0x)70L3Gukqb(8FHw zobD@?{<`tXY;j59yMc!rHA#PjET1~P=Z^7o9naS*zou7=+t^ zQ@ZS>RKj@WSV^6_c_XY;bdMV>nz%uk&y6xMEv4MpzbXm0v9B+f&$-_jZo2*hSk!U* zh*UwDC%AFEdygiC{t;|CIGG&8*8gqd% zPjfN#uUXGZx7W+A5)O|BPyOgF2Us-u2b5KO4jiZAg~K;PYTmlRF@Mm-DOfadfHF^W z2)kZZK2B8@E{B3M9(In4E>6Lsi3^l@nu{?`F$csc_KyVNu%+?Fo&Vvgf52j`W7kga z|EEueZhQRA_fDig*cjynlzEy%*#4pDx}JG(y_osX4A|C^{?_I_&T-NC2P~TW1Ij$j z#poaNDed7OyiODj+j2fTzsWKG&~bpp>eO-0A~Hi6!m28AeG1AdCWi$!4(u0Ig~OUU zLl$jzjDxNmhsA++C-g{v`qe=Pvp)aJ%BR1tgg6Cdp5_p4oU*R-5T|&ZBwQZ){GM~} za)CutoPx5*xoh{69hdchd$#crqB!!^eK%Sp`-fJoN9UWx%GS__39YwdC)0aM7Dqc;7Nj zO)T>*?dV;B%uZpASGDGQqj_6YLg3OlkY0QJ>Sw~CsOQPNrml|ng2gzhQE^IV(u)@^ z*NbT8CMc_*{y^jM?Hphk_WJeOnUKxdM=cCg2PX3SGVG(y`FouaCzWt+cDR>$^{lfG%n-`H4mkTWF;uc?@Ls>*{q0AXmGH#3G*)_uD>c5Zb*UmA2(Y2evqRC&N%+p+q z?N=(;4KoN`%t(U5;OMh086)@Bh84WB#Gz z0*ktOFUKt?iwrI`sXxSCp41aAD?aa8qr0nIV9`{bKv_g^8PeiN^C)oPaoY8T%WGp= z$M<%Oi*EcqET%@OWD-)QUd;DvL76AG@c8@j)S$qr`LDhoPhtE$msbsh@BZ7I%RwmRJc->Gq%+Gb311y?2Kv@OlU~8Y;OB@;rhu>a1xsG%3PB)$v z7L%54A32g#OgVGYp?^@g;C29>j~dE6$>GY6XC+q53*%WE3%9t|ZgYc06E`UHxltxo z{B#j;<9bFD;kI+py86!fo9_NESfnS_SpJr2$qV@#&rb+tp5%5p_k)+z`vXa|;y!ab@Zi&4t^V;cjz-MH4qD z^SSA`wGeJkcT8&FT)hHu((Z4tXyOKCJ~th=>x5g|M}OurH&`@rgEF6+j$2FN*6eAw zxxu1|8A1BOZi_d(<3jmn_cvJ7apV3= zDD%0M=f?e$?S$LTgrN`JGc20; zLYXJ{UPixj%`*MY9fZS}x^8oTMH2@otEe2_UWopGZXfO_96pJ@eY11%PS;)qi##tf zx8;)%OAFfdq0EyUu6(czig=hjuYZO?X_8!Vc* zL7C6Z;%0l^(^{&RbQ5k*elxeK_`Tpr=-`YhI+21%$LYdD^$E~+;o73HGZm?+L24y}s9k&~W z+gp9y;AX#WhD8%MDD%1LxZNb&cBQ(_4Hiw@pv>o{<94%fTeDI<>wNiU_cvJ7apQ6N zQ08+h&yD+I`Utma?WR20$FY8A-TWD_sN?n#se&?3a5K#VWST$2nqLk5VSGK^S2#5I z+pIU76(L&T-NC2P`K3;`hTd#%@YmI%C+UhwiBt02e6pG#8_P?4qMX%1n>sTYX_ z;uNocE?gF0bL_PNuGX($QP;1><3Lxrz@o`tpe!P|WGk9S z$?h+Mh0F3MmK^@Mt6X5w#0APCgbU3|Z|9OGTs~aa{LMkGa)Ct?7buGeF4lMkJC`BC z<@xo$dLzwMF0g1SU!W`^xLD6w02i(o4;3zHjb{JRIWD^E6Ie7|H$s`GxtOj`tfy?? z`hC4f$~?`* z35t-McAq0YMBBH<|5+K1Qw3`%l+XXBs6{1k6`Obas#rDK? zTOwF}cXZ#of6l$<+}}C(yu<^g$$YU?Kz&I>1B-lnNCK^1y~thnTemE|cGsI|0}&7b z5fA|p5CIVo0TB=Z5fA|p5P=URFi@K8J6oRS|MA1}LGMMI<{t%~4uGD0Ki?Qi|MySR zb89;Do0U6kPXq660NzOffA&|-06FUYJpT;wam&pQS@Q~YTG>vWR{kEz|1KK0u@^Wt zS!d6~?7tiRk>uVw9W&8hw)V$S`zU4e-1BajI-U%cab9%%(6F&NlJFJ zKg;pGxX8IO-=D&R{OxD!L41bK!fo|)t9l7DhLb=x3om^liibVzJ|0EQt>jb-T9Um< z9HrQkk;LD|H>`kl82y~v&6>85vu*agnZXyi#W;}_dHkX?!O&%Wq!ZaB(!SS-9UO=w zeM`dzJRXS^Nvh1bM79yWPaogMP8{MBH&Zx5LGx9~3BiOC*v8JbC{711`51G)n06uw zQ|KYbj$HvOKf5oDV|1Xo#Sob~&vr`T0e0e>bM6lT9^>=^{WeRE6Xb59DYW~ahS;)L+>UmMZq#LSM8vn0lPigNjUbi*@mXVA+Xt2ErArOK0r`(`(w zC2dK;+An0JQPoTAXZgJuW439}{hMne>{ zXDB$wUt~pqAZ@ADTzmPm0STXt*v0^|4G5B6s|zPf##X+@5bdMnI0LEd>^7&h!t=eQ zr4tmetY(huDF+VJ3`4<;3-3xI<=u-XXu=S>@CXC*C@<;}_V@9*kFkA_E*j&hM)~X} zm02RRXg5DR!tLV({Ze;88FN{UR)=_j`zeu*bM@qrG>-GUra3xu6E-FNY#Ta!*e8o& z=_;d`AG7d;0ooCujk64YwyXxoI5{`SN1nY_w6J>kB|2>QKH3lGX&zd6A}e-EpQa#zT$sc0!Y!!p`q~eQbX0JBMGn{)L|8 zCw|>rPa#az^H)#ru;gy(+S<=->o~dh7vJ*#{oeSAr)jB4(PK}sY2R2Y{a>|kic@vJ z3+-`^TNWJUJI8OXMb(vApP9+7pv``oHEr21<+an{6%NiZK914WWy|v%Ez*mR^IhV3 z3P>~qfp7Rj%poFYlzo5j43$|~&!~qvf0c4S;)2yixi|=oYIH~|?he`wh?BD!;8|4d ziFVn_cXW^j#*2M%rYJwmca;Z9G!vM~cSZ8+=osT&Kkp8iSVzaGBUs^cSw6Xr4oeJ{ zp=v~1U2zUVBuF|2gTz^8-NCwr^&snVIw>GWzZKmjx4dMzZh>QYS{i=K?4)5b%hgev z!_Mg1_LI#Bd#AW_Ki7`3b!T9AQ_A)v{cIofud??Hy)2rU`INm@C?71ii#GhL?_{5+rm zuz&yj(F6XqnK)T~u&W{e{_TJNJU3PBvI}(&XXs1DLY|&clH6{WgAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+ zA|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`H zAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F z0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+ zA|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`H zAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F z0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+A|L`HAOa#F0wN#+ zA|L`HAOa#F0&5Xy_d4Kn;__A;3g;IW&)xsSf%j{-JU?E8CqQCtj$%~uJCz;@^+FEH7~evlLOm(u^)KN13Ei_8+`vWqPOv_kZK6- zxv65;t^{xo7kTu6)dbSFYk+nHXyYt66Sx{65BiY|12)*@8eDHomKboy^nSu2Wju^@)Ykk_w8lW+U>jG xV!fZO3R^a2JnRYnN^_YjRC#l6ASM=`^Eexi@Lj zn|ot^w3!v{+{U1YD5wl3SO@z0?lC=>ZdSme*44(H^l#Xy?&WFFJMo-HegS^{%PgU z+J5bO4t?V#`aFR41^Z8ubtQXD+qHD=4{g04vPrCYreqzJxsvNmaJrJ~cK;tcc5-7Q zd|ILZ(`ZKhCqVul7FSaJ!H>0N-PQwzF>PpB-RMkMe z&Qyk_M~+BFax#zhGboWMJNQaGEQ>NNSq#o$AndfuA&AY(jNB_N_|3@}dNPn1MZFo% zN%YK09<98LL$Vp|Jo;PYD8_Pl-Ywhk+m6~c*@1gzY5Tbqt$DR;32SN2!)(D9T1!O%M z>usF&91F>i@x@^y63`7DsYgSPfL2{8;6qe0s_$O!Tx328P@Z<1X z9vsu)uA*V74FUT;G`?k5?EFRrHk~v~cSAG-1=K}p$QV3&iZrHn9gD8;pg#$CB9aI? zQAA5Imn4H#%V-Z`K~-5b@yG{aQHiQ9Ph*mw5T1X(LUtQ5)gx;(@v)9TE}66^Jo;W# zy0GFr6xV2(ho#=Su`-EAG%2Xt+kc%9-P&78ZCYbem4-{9FpWL@B=XYF%3;8wQGCH{ za=QcLQbJ@00NrT)N!3T*MOQvD5!GYSqX%lbutVU2Bb+IL}2#gp!T8N zihCkNpn?WTHgc|MYhX=8|pP)}RyO2?rEkJ7zugGnXi1@jBye}gAVXy9N zZLXA>U?`T5O(D4nq6kY<5XE?zc^0}-E3PCkittBGiz`thXF739qE#(%6i4=kH)(hK zoi*NQ{qes&b><$;nSdbA;*g!c&!)S6=(#d^HGzdDtu*XB3RRmfvZ)F2uz+#3P$eg1 z%Xgm|S{(ZJo)^yC-x<8)S5>7Hyj0!ssdrxUUkX}(qU?dsfBCHwH>X~FxaKtUQ7L%r zB&zz3)vNn!8cu$y?lz%4ihj+4J$TmRovTrGA|FC#(j%y&TTmjJ{lZQR+AqjsGG+WS zzXU%8&C?RVr$UXovE8m^j;=Q|zQ#;HlLo4wtHH5S< zgPDP<@JF0cv?=1vP>#CCEyX=t=>-9et1Q;`>?o6J)kyVMX$@CvM;IDqA)7Mxy56d* z$_-$(qgJlmlWVZ+-CA7l;)J6dqu%~aR^A*{&-)c6394ve1@;I@C8pryO1(G&?NDmX z(bM|#wylR*kEM=Xz!TxES6GkvrSr|^rF?S^QLI<#iAo}tl;duEl#G;|*bzwUC&Mrp z4kLSQzer-AIolc=Df20l#abhXlUC%nZK(BQJdJasJItZ2Z-Z&{Y0A`hfuFY59%)7d z&0&pJ*;Vdaa4+T#qU@17aNjST*u4vcx=`zeC4DG^;I8)+dWKDT2&4MbR)f4&?X9-D z-CpKO;r)|>*80VhHufJx?czM=eVi5ZoR^mM#W-e8>I5v%gfb?F<)E~|mdEgY(}oYY zcC0#zwcE_<89dt|(F%6TuNk%N=-Y{+^&R-NVLWF}a0az|z%h;9?O0KtcjKqCJ5Y9@ zHUj@o!eV{(ABOBK*2$Q+s%O3V9=26JNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@ z1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L zfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@ z1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14c zNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-LfCNZ@1W14cNPq-L z;D#X(kpOIQr4}V2)u@M1)8a~09MmeM##q~nR<*=s2+z1|k|272gSx#w{pi5>#t#KUbm5$;Ki9B_S<^GF`36rfqzC7jcxE6O`vm1}Q#{D|WeX-$Ma$hY^ z`8%$~-Pi0o=Kj(1)EAzudZqP^za)PC*WaKVc8SJACdDulLUQiLs%S^EQ8+Ix|< zP$fppY#6sJnwn$ZPyO1ha^zzO%A!_^V->^w=TI7 zRUiQgNI(J-kbndvAOQ(TKmrnwz>ETGhx*s-DD33l_~mW7&;3@+e+rxkfmL0DXUfr^ z{oUGmJ!dMxHjY;V)mw#DurJLOX}G%%;^ zm&^NaUV9|*_px`ptmmQH`=E2l33rDbA820oz@v@1@a^=G`lB3)MYZH`h=(>n+m2VK zuoipJhXO`<{tWkEFfM#Go5DDTF~q_V7Noo;>|od^cHthZ;A;duywb;*ZXPe^=K!zl zMv-Sl^fPif&x^di0()6j;OA|)m9N`*bSv)Sd*AuyM=m%^c%{-mm@AjJVpo7oc#JiV zMlEuJx0Fg>W4~NRKiU^!W7PT{w$;n1-7LL>dz+h#8-C!Ulbs!AA&|+y3%17dp}x+? z{TPexWKkB`WJ?kq40}9|__(`b0@ix8vTO7u-yrU0Hz#6MF<~due`vuP}j@79+CeJgX^%MLcm&&fp(HY+&(Y%v+I; zy>z$G6zY7kZ{*4M;u@R3Q^iIHBGWYt?PCWlMZJuPc%IkjqE`hpC)o5&L&m-Nz{ zE&o|d+LARBv>#xF)@AESSw4|TBW-9p-z!n8dd(-PVSkvX2Y)sSKRAbYnR2pU*ZV@^5i%Je)JnHyZ z)KEFS>#QHlCaf>%KUKzawugZue8s&85eBxjT56xZJEZWA*wz7LDiJ2V+Gsl2Vr<(t zIYrknvYQjB5cS)VRtwMfN~Ns~uvN_x*IWoAs9{b8>sqmj z+j-u|_eRd`JD8#_j%pL%ms6!kLYwWDht;g!EzpOShm5h5jj`8dUf?E11a7VFZx67U z2O|mg!x}{q)W8a@WCrUbuMWX<>_2FUPr;UCsa3nDU$tV=T83 zTI6=z^22X+jdy*2{o&*HH7A$;wq~+Id8zKX_v}2k$lGz%-GAIN^3v}X)jvPWx2M~p zn6Za>Xy386wLWFj$t~5b(bVqdbz24RXIG*Lrh--x+-@x6wOm3-ZTeT>XA-+44~U&$rgj&A4ci5a3A8M`u$&U3aj@s4=2 z?=5W-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndv zAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnw zfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz z0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8< z2}nQ!5|DrdBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|Drd zBp?9^NI(J-kbndvAOQ(TKmrnwfCMBU0SQPz0uqpb1SB8<2}nQ!5|DrdBp?9^NI(J- zkbndvAOQ(TKmr#R2#{bvh&t{$)biNpk?kSU@OV^>Sy2`1cvg!ny7-yJLL_o}T@y>2Wbz?iP?8Ym>E9V*p-)}qe;-aCS|0Vp{ z=T+|tw6G_`QID#<;MY~{jbS_t6vsJ`h6cbsua8H!?Hgr+)qw)k6WD1-SrIDocl4l-w?ace)4j2Ta5S8e}X0-DKoIALxv~uvht>6CFPnVr)%C9It;w(77@j17Ax#wke zoVS)wFK6DSTBlr#Q{}Cd531|gRh+krtW#fz-G5cJJzA}=+&bl2oX*|#?Amj;c|Rw1 zAFsch?o+K(uEpuxUC*v!_y2vJdiX~*uQtB**W54vw!cs;4D`&nPTd&it^3E9GjCI^ zQ?A9S^7e8La6P+<^LF7n<=O8j_x&(8c5gUxCEO?eo^mZt=k9uT&A3OuUoZZ9|NOW3 Hj0gV)w~Z5L literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..fcbdbce1d3db --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4908_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2020-08-19T06:07:25.0461779Z", + "event": { + "action": "special-group-table-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4908, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "winlog": { + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "SidList": [ + "", + "%{S-1-5-32-544}", + "%{S-1-5-32-123-54-65}" + ], + "SidListDesc": [ + "Administrators", + "S-1-5-32-123-54-65" + ] + }, + "event_id": 4908, + "keywords": [ + "Audit Success" + ], + "opcode": "Info", + "process": { + "pid": 784, + "thread": { + "id": 808 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 140274, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx new file mode 100644 index 0000000000000000000000000000000000000000..15a93a947a21e60b5c5de31a57cd5c14625ff795 GIT binary patch literal 69632 zcmeI1X>3$g6vxk-c{99DXX((?1w{d=QnaN7i(ql7v{jVCXe(kYN~JB7PG@QtC^j0R z7!$YP8WQ|KOpL)T8e<^DsF9fP!Qd7l7-M2$;zCRmF&H%&|L4B zIrrXkf9KqDZ>P=ao|fKpw-l#y(-QrS$4&P{#-f># z16IPaBx?Pz?6MF29-4ow;fmYtt4lhcM_#LX9V=0(#-!{(Ru`ygPc$UWvO&5fBf}^^ ziYL6@YkV&el2PfGG#ZD{5VGoJ9mIxZK(3ZK_zcM!)O16p6Zsmvdr>neS(LKU1IZec zv#6gV8_|}*d#=>tb0u=MvH;)R$EP1ECmv*6-cjQ~Mol&XtzEn#7egZK$LgM?VYwaa)tS9DN2NSy$CEPM zmdOx>Tbc)6jKj=bp(7RINK(oqfmX*njzo+cx|r00lD1>na#e7;*45XRyKU1pKYHZg zKWW6wg;QI)uNx5=GVcRc6z2XuDiZtNi>|HEyD*qlgN+8UF06YaRIi?}#%lP-2-}^6<~CyAB`uC55e{ zQm|kXvU=ufYI;cB%Maa6#??DfuTilXZ#{$6%O1hYnp^8EvD z|9M{jQ)s;u{nz5jz=>4rpc|m-yq=fjXO6*VJK=0=vEGTRwcXXKrb=#VzF!0z#f@k> z&!7nuI#bBi@q{zEWOZ-o)++25cKe-zYjHpSSPH}to0MyVr)70s%gN(BF9kjI1X=@K z{b-=89)i{mX14`*2pk;cG6h=%HFzpaIf^_EMYDZ0k5fTYr~+Tl?s|K)WB%BVxxv~e zj$)5Ag0~*{89aI@51Q2~AwNPq(27WdbgEG*;gSn*DYy!;Me}ts51%@eZ+>Qf^v&Kw z$@jlZrP@=qwXWoqQV(e)y$8v>LqmN#XMEUEHFw4Cna_^?{RirFuYl1{$fjV)0!D@B zlYdmYmj*uo>jqM1-;U1b;||ID~Z+g|H56`vDlzb=2h; z*S*md8dRm)aBa8;?@=5Kdc*63Lk*&)$(Vn@9Fd!#MI{*JZ)qx-1&2!MD5#$YBdUHQ zs9k0x+zXTI`tFfkgBkSr6q$y&EZ5_KcB>J*%sm+ItEXw5$V0DAuQ)d09(^kQI2pWla~`>H;NL(zMEEf5tmL`s?jq_RLuF%HhN(ZeM2QTZ(DWO*WdiL_1v=?ZH~DG zstMKfE^s}`qt@!wgS-0UMr+ahU6*WIxoGRrcRru-$`_xa+S^}yk=6ZG5S#Pb4`bXx zjGRBB{`d|^F(rxY@6#)cO zTmM^bcFru=RxSR2?ix(ZL;@s00wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr uKmsH{0wh2Jrzg-WS;q^BKV2OF literal 0 HcmV?d00001 diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json new file mode 100644 index 000000000000..5e9a933c7bb8 --- /dev/null +++ b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json @@ -0,0 +1,70 @@ +[ + { + "@timestamp": "2020-08-18T14:36:41.2936839Z", + "event": { + "action": "per-user-audit-policy-changed", + "category": [ + "iam", + "configuration" + ], + "code": 4912, + "kind": "event", + "module": "security", + "outcome": "success", + "provider": "Microsoft-Windows-Security-Auditing", + "type": [ + "admin", + "change" + ] + }, + "host": { + "name": "WIN-BVM4LI1L1Q6.TEST.local" + }, + "log": { + "level": "information" + }, + "related": { + "user": "Administrator" + }, + "user": { + "domain": "TEST", + "id": "S-1-5-21-2024912787-2692429404-2351956786-500", + "name": "Administrator" + }, + "winlog": { + "activity_id": "{65461d39-753f-0000-731d-46653f75d601}", + "api": "wineventlog", + "channel": "Security", + "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", + "event_data": { + "AuditPolicyChanges": "%%8452", + "CategoryId": "%%8276", + "SubcategoryGuid": "{0cce924a-69ae-11d9-bed3-505054503030}", + "SubcategoryId": "%%13317", + "SubjectDomainName": "TEST", + "SubjectLogonId": "0x44d7d", + "SubjectUserName": "Administrator", + "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", + "TargetUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" + }, + "event_id": 4912, + "keywords": [ + "Audit Success" + ], + "logon": { + "id": "0x44d7d" + }, + "opcode": "Info", + "process": { + "pid": 780, + "thread": { + "id": 3300 + } + }, + "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", + "provider_name": "Microsoft-Windows-Security-Auditing", + "record_id": 123917, + "task": "Audit Policy Change" + } + } +] \ No newline at end of file diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json index 6519408002cb..977ea0fe1168 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4771.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "192.168.5.44", "user": "MPUIG" }, "source": { diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json index 649db8b0e232..f7944a0c686d 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4778.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "10.100.150.9", "user": "at_adm" }, "source": { diff --git a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json index 12c23f0a09d4..93f89a592a6d 100644 --- a/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json +++ b/x-pack/winlogbeat/module/security/test/testdata/security-windows2012_4779.evtx.golden.json @@ -22,6 +22,7 @@ "level": "information" }, "related": { + "ip": "10.100.150.17", "user": "at_adm" }, "source": { From 72880cabdc237e06e21365f276ff0ffe9decbb45 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Mon, 25 Jan 2021 07:30:59 -0800 Subject: [PATCH 25/35] [Elastic Log Driver] Change hosts config flag (#23628) * change hosts config flag * add changelog now --- CHANGELOG.next.asciidoc | 3 +++ x-pack/dockerlogbeat/docs/configuration.asciidoc | 2 +- x-pack/dockerlogbeat/docs/install.asciidoc | 4 ++-- x-pack/dockerlogbeat/docs/usage.asciidoc | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index acfdbc957c8b..95638ca4e716 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -503,6 +503,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* +*Elastic Logging Plugin* +- Fix out of date CLI flags on docs. {pull}23628[23628] + ==== Added diff --git a/x-pack/dockerlogbeat/docs/configuration.asciidoc b/x-pack/dockerlogbeat/docs/configuration.asciidoc index 7be2a1fe1f1c..e8d398b0b875 100644 --- a/x-pack/dockerlogbeat/docs/configuration.asciidoc +++ b/x-pack/dockerlogbeat/docs/configuration.asciidoc @@ -152,7 +152,7 @@ The local log also supports the `max-file`, `max-size` and `compress` options th ["source","sh",subs="attributes"] ---- docker run --log-driver=elastic/{log-driver-alias}:{version} \ - --log-opt endpoint="myhost:9200" \ + --log-opt hosts="myhost:9200" \ --log-opt user="myusername" \ --log-opt password="mypassword" \ --log-opt max-file=10 \ diff --git a/x-pack/dockerlogbeat/docs/install.asciidoc b/x-pack/dockerlogbeat/docs/install.asciidoc index ab36f67877d5..6aa51aba0b19 100644 --- a/x-pack/dockerlogbeat/docs/install.asciidoc +++ b/x-pack/dockerlogbeat/docs/install.asciidoc @@ -80,7 +80,7 @@ example: ["source","sh",subs="attributes"] ---- docker run --log-driver=elastic/{log-driver-alias}:{version} \ - --log-opt endpoint="https://myhost:9200" \ + --log-opt hosts="https://myhost:9200" \ --log-opt user="myusername" \ --log-opt password="mypassword" \ -it debian:jessie /bin/bash @@ -98,7 +98,7 @@ example: { "log-driver" : "elastic/{log-driver-alias}:{version}", "log-opts" : { - "endpoint" : "https://myhost:9200", + "hosts" : "https://myhost:9200", "user" : "myusername", "password" : "mypassword" } diff --git a/x-pack/dockerlogbeat/docs/usage.asciidoc b/x-pack/dockerlogbeat/docs/usage.asciidoc index bf23049c1999..446206e35449 100644 --- a/x-pack/dockerlogbeat/docs/usage.asciidoc +++ b/x-pack/dockerlogbeat/docs/usage.asciidoc @@ -16,7 +16,7 @@ The following examples show common configurations for the {log-driver}. ["source","sh",subs="attributes"] ---- docker run --log-driver=elastic/{log-driver-alias}:{version} \ - --log-opt endpoint="myhost:9200" \ + --log-opt hosts="myhost:9200" \ --log-opt user="myusername" \ --log-opt password="mypassword" \ -it debian:jessie /bin/bash @@ -29,7 +29,7 @@ docker run --log-driver=elastic/{log-driver-alias}:{version} \ { "log-driver" : "elastic/{log-driver-alias}:{version}", "log-opts" : { - "endpoint" : "myhost:9200", + "hosts" : "myhost:9200", "user" : "myusername", "password" : "mypassword", } @@ -71,7 +71,7 @@ docker run --log-driver=elastic/{log-driver-alias}:{version} \ ["source","sh",subs="attributes"] ---- docker run --log-driver=elastic/{log-driver-alias}:{version} \ - --log-opt endpoint="myhost:9200" \ + --log-opt hosts="myhost:9200" \ --log-opt user="myusername" \ --log-opt password="mypassword" \ --log-opt index="eld-%{[agent.version]}-%{+yyyy.MM.dd}" \ @@ -85,7 +85,7 @@ docker run --log-driver=elastic/{log-driver-alias}:{version} \ { "log-driver" : "elastic/{log-driver-alias}:{version}", "log-opts" : { - "endpoint" : "myhost:9200", + "hosts" : "myhost:9200", "user" : "myusername", "index" : "eld-%{[agent.version]}-%{+yyyy.MM.dd}", "password" : "mypassword", From 47889ebdcb2f427d16b4b35f0414f8ffb662ea11 Mon Sep 17 00:00:00 2001 From: Kevin Klopfenstein Date: Mon, 25 Jan 2021 16:33:34 +0100 Subject: [PATCH 26/35] [Filebeat] Add Cisco ASA message '302023' parsing (#23092) Enhance message parsing to Cisco ASA message 302023. Signed-off-by: Kevin Klopfenstein Signed-off-by: kevin --- .../additional_messages.log-expected.json | 42 +++++++++++++++++++ .../cisco/shared/ingest/asa-ftd-pipeline.yml | 4 ++ 2 files changed, 46 insertions(+) diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 8b07b91acb40..1d225c42addf 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -1604,17 +1604,26 @@ ] }, { + "cisco.asa.destination_interface": "net", "cisco.asa.message_id": "302023", + "cisco.asa.source_interface": "fw111", + "destination.address": "192.168.2.2", + "destination.ip": "192.168.2.2", + "destination.port": 10051, "event.action": "firewall-rule", "event.category": [ "network" ], "event.code": 302023, "event.dataset": "cisco.asa", + "event.duration": 0, + "event.end": "2021-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for fw111:10.10.10.10/39210 to net:192.168.2.2/10051 duration 0:00:00 forwarded bytes 0 Cluster flow with CLU closed on owner", + "event.reason": "Cluster flow with CLU closed on owner", "event.severity": 6, + "event.start": "2021-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -1624,31 +1633,52 @@ "input.type": "log", "log.level": "informational", "log.offset": 4949, + "network.bytes": "0", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "fw111", "observer.hostname": "dev01", + "observer.ingress.interface.name": "net", "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", "related.hosts": [ "dev01" ], + "related.ip": [ + "10.10.10.10", + "192.168.2.2" + ], "service.type": "cisco", + "source.address": "10.10.10.10", + "source.ip": "10.10.10.10", + "source.port": 39210, "tags": [ "cisco-asa", "forwarded" ] }, { + "cisco.asa.destination_interface": "unknown", "cisco.asa.message_id": "302023", + "cisco.asa.source_interface": "net", + "destination.address": "192.168.2.2", + "destination.ip": "192.168.2.2", + "destination.port": 39222, "event.action": "firewall-rule", "event.category": [ "network" ], "event.code": 302023, "event.dataset": "cisco.asa", + "event.duration": 0, + "event.end": "2021-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for net:10.10.10.10/10051 to unknown:192.168.2.2/39222 duration 0:00:00 forwarded bytes 0 Forwarding or redirect flow removed to create director or backup flow", + "event.reason": "Forwarding or redirect flow removed to create director or backup flow", "event.severity": 6, + "event.start": "2021-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -1658,14 +1688,26 @@ "input.type": "log", "log.level": "informational", "log.offset": 5142, + "network.bytes": "0", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "net", "observer.hostname": "dev01", + "observer.ingress.interface.name": "unknown", "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", "related.hosts": [ "dev01" ], + "related.ip": [ + "10.10.10.10", + "192.168.2.2" + ], "service.type": "cisco", + "source.address": "10.10.10.10", + "source.ip": "10.10.10.10", + "source.port": 10051, "tags": [ "cisco-asa", "forwarded" diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index 72920d75a0e4..c46227b79a13 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -318,6 +318,10 @@ processors: if: "ctx._temp_.cisco.message_id == '302022'" field: "message" pattern: "Built %{} stub %{network.transport} connection for %{_temp_.cisco.source_interface}:%{source.address}/%{source.port} to %{_temp_.cisco.destination_interface}:%{destination.address}/%{destination.port}" + - dissect: + if: "ctx._temp_.cisco.message_id == '302023'" + field: "message" + pattern: "Teardown stub %{network.transport} connection for %{_temp_.cisco.source_interface}:%{source.address}/%{source.port} to %{_temp_.cisco.destination_interface}:%{destination.address}/%{destination.port} duration %{_temp_.duration_hms} forwarded bytes %{network.bytes} %{event.reason}" - grok: if: "ctx._temp_.cisco.message_id == '304001'" field: "message" From bf46572ce1c2e1864d39966f6958ad484e686fce Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Jan 2021 10:58:19 -0500 Subject: [PATCH 27/35] Sync fixes from Integration Package Testing (#23424) * Sync changes to AWS CloudTrail https://github.com/elastic/integrations/pull/408 * Sync changes to CheckPoint Firewall Change type of event.severity. https://github.com/elastic/integrations/pull/409 * Sync changes from Cisco ASA / FTD https://github.com/elastic/integrations/pull/414 * Sync changes from Cisco IOS Make icmp and igmp fields strings because they are keywords. https://github.com/elastic/integrations/pull/416 * Sync changes to CrowdStrike Falcon Fix some field types. https://github.com/elastic/integrations/issues/377 * Sync changes to Fortinet Firewall Drop assignip if the value is "N/A". https://github.com/elastic/integrations/pull/437 * Sync changes to Juniper SRX Convert event.risk values to float Protect against missing event.timezone Convert event.severity to long. https://github.com/elastic/integrations/pull/443 * Sync changes to Suricata EVE Convert suricata.eve.flow_id to string because the field is a keyword in the mapping. https://github.com/elastic/integrations/pull/457 * Sync changes to Zeek DNS Fix usages of ignore_failure with convert processor. Make DNS transaction ID a string. https://github.com/elastic/integrations/pull/448 * Add changelog --- CHANGELOG.next.asciidoc | 7 +++ .../module/aws/cloudtrail/ingest/pipeline.yml | 8 +-- .../checkpoint/firewall/ingest/pipeline.yml | 5 +- .../module/cisco/ios/config/pipeline.js | 3 - .../module/cisco/ios/pipeline_test.go | 6 +- .../test/cisco-ios-syslog.log-expected.json | 10 +-- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 4 +- .../crowdstrike/falcon/config/pipeline.js | 46 +++++++++++++- .../test/falcon-events.log-expected.json | 1 - .../test/falcon-sample.log-expected.json | 12 ++-- .../fortinet/firewall/ingest/pipeline.yml | 45 +++++++++----- .../module/juniper/srx/ingest/flow.yml | 6 +- .../module/juniper/srx/ingest/pipeline.yml | 17 ++--- .../module/juniper/srx/ingest/utm.yml | 6 +- .../juniper/srx/test/atp.log-expected.json | 8 +-- .../juniper/srx/test/flow.log-expected.json | 62 +++++++++---------- .../juniper/srx/test/idp.log-expected.json | 14 ++--- .../juniper/srx/test/ids.log-expected.json | 24 +++---- .../srx/test/secintel.log-expected.json | 4 +- .../juniper/srx/test/utm.log-expected.json | 28 ++++----- .../module/suricata/eve/config/eve.yml | 1 + .../eve/test/eve-alerts.log-expected.json | 44 ++++++------- .../eve/test/eve-dns-4.1.4.log-expected.json | 48 +++++++------- .../eve/test/eve-small.log-expected.json | 22 +++---- .../filebeat/module/zeek/dns/config/dns.yml | 11 +++- .../zeek/dns/test/dns-json.log-expected.json | 12 ++-- 26 files changed, 266 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 95638ca4e716..d99315bd2a16 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -269,6 +269,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523] - Fix various processing errors in the Suricata module. {pull}23236[23236] - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] +- CheckPoint Firewall module: Change event.severity JSON data type to a number because the field mapping is a `long`. {pull}23424[23424] +- Cisco IOS: Change icmp.type/code and igmp.type JSON data types to strings because the fields mappings are `keyword`. {pull}23424[23424] +- CrowdStrike Falcon: Change JSON field types to match the field mappings. {pull}23424[23424] +- Fortinet Firewall: Drop `fortinet.firewall.assignip` when the value is "N/A". {pull}23424[23424] +- Juniper SRX: Change JSON field types to match the field mappings. {pull}23424[23424] +- Suricata EVE: Convert `suricata.eve.flow_id` to string because the field is a keyword in the mapping. {pull}23424[23424] +- Zeek DNS: Ignore failures in data type conversions. And change `dns.id` JSON field to a string to match its `keyword` mapping. {pull}23424[23424] - Change the `event.created` in Netflow events to be the time the event was created by Filebeat to be consistent with ECS. {pull}23094[23094] - Update `filestream` reader offset when a line is skipped. {pull}23417[23417] diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 3dd78f82c6db..76cf0f936b66 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -152,25 +152,25 @@ processors: Map map = new HashMap(); ctx.aws.cloudtrail.put("flattened", map); } - if (ctx.json.requestParameters != null) { + if (ctx.json?.requestParameters != null) { ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString(); if (ctx.aws.cloudtrail.request_parameters.length() < 32766) { ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters); } } - if (ctx.json.responseElements != null) { + if (ctx.json?.responseElements != null) { ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString(); if (ctx.aws.cloudtrail.response_elements.length() < 32766) { ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements); } } - if (ctx.json.additionalEventData != null) { + if (ctx.json?.additionalEventData != null) { ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString(); if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) { ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData); } } - if (ctx.json.serviceEventDetails != null) { + if (ctx.json?.serviceEventDetails != null) { ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString(); if (ctx.aws.cloudtrail.service_event_details.length() < 32766) { ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails); diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 975a0e76104a..b92624e2f6c4 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -309,9 +309,11 @@ processors: type: long ignore_failure: true ignore_missing: true -- rename: +- convert: field: checkpoint.severity target_field: event.severity + type: long + ignore_failure: true ignore_missing: true - rename: field: checkpoint.action @@ -859,6 +861,7 @@ processors: - checkpoint.xlatedst - checkpoint.uid - checkpoint.time + - checkpoint.severity - syslog5424_ts - _temp_ ignore_missing: true diff --git a/x-pack/filebeat/module/cisco/ios/config/pipeline.js b/x-pack/filebeat/module/cisco/ios/config/pipeline.js index 4506f67ccb34..42e10c6969d5 100644 --- a/x-pack/filebeat/module/cisco/ios/config/pipeline.js +++ b/x-pack/filebeat/module/cisco/ios/config/pipeline.js @@ -166,9 +166,6 @@ var ciscoIOS = (function() { {from: "source.port", type: "long"}, {from: "source.packets", type: "long"}, {from: "source.packets", to: "network.packets", type: "long"}, - {from: "icmp.type", type: "long"}, - {from: "icmp.code", type: "long"}, - {from: "igmp.type", type: "long"}, ], ignore_missing: true, }).Run; diff --git a/x-pack/filebeat/module/cisco/ios/pipeline_test.go b/x-pack/filebeat/module/cisco/ios/pipeline_test.go index 6104c25a3064..7f5e4a99120a 100644 --- a/x-pack/filebeat/module/cisco/ios/pipeline_test.go +++ b/x-pack/filebeat/module/cisco/ios/pipeline_test.go @@ -71,8 +71,8 @@ var testCases = []testCase{ "event.outcome": "deny", "event.severity": int64(6), "event.type": []string{"connection", "firewall"}, - "icmp.code": int64(5), - "icmp.type": int64(3), + "icmp.code": "5", + "icmp.type": "3", "log.level": "informational", "log.original": isdef.IsNonEmptyString, "message": "list 100 denied icmp 198.51.100.1 -> 198.51.100.2 (3/5), 1 packet", @@ -117,7 +117,7 @@ var testCases = []testCase{ "event.outcome": "deny", "event.severity": int64(6), "event.type": []string{"connection", "firewall"}, - "igmp.type": int64(20), + "igmp.type": "20", "log.level": "informational", "log.original": isdef.IsNonEmptyString, "message": "list INBOUND-ON-AP denied igmp 198.51.100.1 -> 224.0.0.2 (20), 1 packet", diff --git a/x-pack/filebeat/module/cisco/ios/test/cisco-ios-syslog.log-expected.json b/x-pack/filebeat/module/cisco/ios/test/cisco-ios-syslog.log-expected.json index 0695d3730aac..5841793ceb8b 100644 --- a/x-pack/filebeat/module/cisco/ios/test/cisco-ios-syslog.log-expected.json +++ b/x-pack/filebeat/module/cisco/ios/test/cisco-ios-syslog.log-expected.json @@ -66,7 +66,7 @@ "firewall" ], "fileset.name": "ios", - "igmp.type": 20, + "igmp.type": "20", "input.type": "log", "log.level": "informational", "log.offset": 140, @@ -251,8 +251,8 @@ "firewall" ], "fileset.name": "ios", - "icmp.code": 4, - "icmp.type": 3, + "icmp.code": "4", + "icmp.type": "3", "input.type": "log", "log.level": "informational", "log.offset": 760, @@ -1268,8 +1268,8 @@ "firewall" ], "fileset.name": "ios", - "icmp.code": 3, - "icmp.type": 3, + "icmp.code": "3", + "icmp.type": "3", "input.type": "log", "log.level": "informational", "log.offset": 4125, diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index c46227b79a13..8d0e1b24c63e 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -71,7 +71,7 @@ processors: # Parse the date included in FTD logs # - date: - if: "ctx.event.timezone == null" + if: "ctx.event?.timezone == null && ctx._temp_?.raw_date != null" field: "_temp_.raw_date" target_field: "@timestamp" formats: @@ -103,7 +103,7 @@ processors: }, ] - date: - if: "ctx.event.timezone != null" + if: "ctx.event?.timezone != null && ctx._temp_?.raw_date != null" timezone: "{{ event.timezone }}" field: "_temp_.raw_date" target_field: "@timestamp" diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js b/x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js index a447a25d15ed..46bbf671518e 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/pipeline.js @@ -12,6 +12,10 @@ var crowdstrikeFalconProcessor = (function () { function convertToMSEpoch(evt, field) { var timestamp = evt.Get(field); + if (timestamp == 0) { + evt.Delete(field) + return + } if (timestamp) { if (timestamp < 100000000000) { // check if we have a seconds timestamp, this is roughly 1973 in MS evt.Put(field, timestamp * 1000); @@ -103,7 +107,8 @@ var crowdstrikeFalconProcessor = (function () { type: "ip" }, { from: "crowdstrike.event.ProcessId", - to: "process.pid" + to: "process.pid", + type: "long" }, { from: "crowdstrike.event.ParentImageFileName", to: "process.parent.executable" @@ -284,6 +289,7 @@ var crowdstrikeFalconProcessor = (function () { }, { from: "crowdstrike.event.PID", to: "process.pid", + type: "long" }, { from: "crowdstrike.event.RuleId", @@ -421,6 +427,44 @@ var crowdstrikeFalconProcessor = (function () { ignore_missing: false, fail_on_error: true }) + .Convert({ + fields: [ + { + from: "crowdstrike.event.LateralMovement", + type: "long", + }, + { + from: "crowdstrike.event.LocalPort", + type: "long", + }, + { + from: "crowdstrike.event.MatchCount", + type: "long", + }, + { + from: "crowdstrike.event.MatchCountSinceLastReport", + type: "long", + }, + { + from: "crowdstrike.event.PID", + type: "long", + }, + { + from: "crowdstrike.event.RemotePort", + type: "long", + }, + { + from: "source.port", + type: "long", + }, + { + from: "destination.port", + type: "long", + } + ], + ignore_missing: true, + fail_on_error: false + }) .Build() .Run })(); diff --git a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json index 47c0e10f47ab..eab6fb1db0eb 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json +++ b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-events.log-expected.json @@ -29,7 +29,6 @@ "crowdstrike.event.PatternDispositionFlags.Rooting": false, "crowdstrike.event.PatternDispositionFlags.SensorOnly": false, "crowdstrike.event.PatternDispositionValue": 16, - "crowdstrike.event.ProcessEndTime": 0, "crowdstrike.event.ProcessId": 38684386611, "crowdstrike.event.ProcessStartTime": "2018-09-13T13:45:39.000Z", "crowdstrike.event.SHA256String": "6a671b92a69755de6fd063fcbe4ba926d83b49f78c42dbaeed8cdb6bbc57576a", diff --git a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-sample.log-expected.json b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-sample.log-expected.json index a122e788b08f..becdbecc7c8c 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-sample.log-expected.json +++ b/x-pack/filebeat/module/crowdstrike/falcon/test/falcon-sample.log-expected.json @@ -10,16 +10,16 @@ "crowdstrike.event.HostName": "TESTDEVICE01", "crowdstrike.event.Ipv": "ipv4", "crowdstrike.event.LocalAddress": "10.37.60.194", - "crowdstrike.event.LocalPort": "445", + "crowdstrike.event.LocalPort": 445, "crowdstrike.event.MatchCount": 1, "crowdstrike.event.MatchCountSinceLastReport": 1, "crowdstrike.event.NetworkProfile": "2", - "crowdstrike.event.PID": "206158879910", + "crowdstrike.event.PID": 206158879910, "crowdstrike.event.PolicyID": "74e7f1552a3a4d90a6d65578642c8584", "crowdstrike.event.PolicyName": "PROD-FW-Workstations-General", "crowdstrike.event.Protocol": "6", "crowdstrike.event.RemoteAddress": "10.37.60.21", - "crowdstrike.event.RemotePort": "54952", + "crowdstrike.event.RemotePort": 54952, "crowdstrike.event.RuleAction": "2", "crowdstrike.event.RuleFamilyID": "fec73e96a1bf4481be582c3f89b234fa", "crowdstrike.event.RuleGroupName": "SMB Rules", @@ -32,7 +32,7 @@ "crowdstrike.metadata.offset": 70689, "crowdstrike.metadata.version": "1.0", "destination.ip": "10.37.60.194", - "destination.port": "445", + "destination.port": 445, "event.action": "firewall_match_event", "event.category": [ "network" @@ -58,7 +58,7 @@ "message": "Firewall Rule 'Inbound SMB Block & Log Private' triggered", "network.direction": "ingress", "network.type": "ipv4", - "process.pid": "206158879910", + "process.pid": 206158879910, "related.ip": [ "10.37.60.21", "10.37.60.194" @@ -70,7 +70,7 @@ "rule.ruleset": "SMB Rules", "service.type": "crowdstrike", "source.ip": "10.37.60.21", - "source.port": "54952", + "source.port": 54952, "tags": [ "forwarded" ] diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 22d44e5664ab..a227d7700829 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -80,26 +80,26 @@ processors: formats: - UNIX_MS timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS - if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - script: lang: painless source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000" @@ -134,6 +134,9 @@ processors: field: fortinet.firewall.level target_field: log.level ignore_missing: true +- remove: + field: fortinet.firewall.assignip + if: "ctx.fortinet?.firewall?.assignip == 'N/A'" - remove: field: fortinet.firewall.dstip if: "ctx.fortinet?.firewall?.dstip == 'N/A'" @@ -222,16 +225,18 @@ processors: ) - remove: field: - - _temp - - message - - syslog5424_sd - - syslog5424_pri - - fortinet.firewall.tz - - fortinet.firewall.date - - fortinet.firewall.eventtime - - fortinet.firewall.time - - fortinet.firewall.duration - - host + - _temp.time + - _temp + - message + - syslog5424_sd + - syslog5424_pri + - fortinet.firewall.tz + - fortinet.firewall.date + - fortinet.firewall.devid + - fortinet.firewall.eventtime + - fortinet.firewall.time + - fortinet.firewall.duration + - host ignore_missing: true - pipeline: name: '{< IngestPipeline "event" >}' @@ -242,6 +247,18 @@ processors: - pipeline: name: '{< IngestPipeline "utm" >}' if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'" +- convert: + field: fortinet.firewall.quotamax + type: long + ignore_missing: true +- convert: + field: fortinet.firewall.quotaused + type: long + ignore_missing: true +- convert: + field: fortinet.firewall.size + type: long + ignore_missing: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/juniper/srx/ingest/flow.yml b/x-pack/filebeat/module/juniper/srx/ingest/flow.yml index 1a488a57bd8a..0671bff51b2d 100644 --- a/x-pack/filebeat/module/juniper/srx/ingest/flow.yml +++ b/x-pack/filebeat/module/juniper/srx/ingest/flow.yml @@ -13,11 +13,12 @@ processors: - append: field: event.category value: network -- rename: +- convert: field: juniper.srx.application_risk + type: float target_field: event.risk_score ignore_missing: true - if: "ctx.juniper?.srx?.application_risk != null" + ignore_failure: true - append: field: event.type value: @@ -344,6 +345,7 @@ processors: ############# - remove: field: + - juniper.srx.application_risk - juniper.srx.destination_port - juniper.srx.nat_destination_port - juniper.srx.bytes_from_client diff --git a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml index 5bc4d45e82e5..9fb9057b8fa8 100644 --- a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml +++ b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml @@ -27,7 +27,7 @@ processors: # Parse the date # - date: - if: "ctx.event.timezone == null" + if: "ctx?.event?.timezone == null" field: _temp_.raw_date target_field: "@timestamp" formats: @@ -36,7 +36,7 @@ processors: - yyyy-MM-dd HH:mm:ss Z - ISO8601 - date: - if: "ctx.event.timezone != null" + if: "ctx?.event?.timezone != null" timezone: "{{ event.timezone }}" field: _temp_.raw_date target_field: "@timestamp" @@ -55,7 +55,7 @@ processors: - rename: field: juniper.srx.elapsed_time target_field: juniper.srx.duration - if: "ctx.juniper?.srx?.elapsed_time != null" + if: "ctx?.juniper?.srx?.elapsed_time != null" # Sets starts, end and duration when start and duration is known - script: @@ -88,9 +88,11 @@ processors: - set: field: event.dataset value: juniper.srx -- set: - field: event.severity - value: '{{syslog_pri}}' +- convert: + field: syslog_pri + type: long + target_field: event.severity + ignore_failure: true - rename: field: log.original target_field: event.original @@ -197,8 +199,7 @@ processors: - remove: field: - message - - _temp_ - - _temp + - _temp_.raw_date - juniper.srx.duration - juniper.srx.dir_disp - juniper.srx.srczone diff --git a/x-pack/filebeat/module/juniper/srx/ingest/utm.yml b/x-pack/filebeat/module/juniper/srx/ingest/utm.yml index a80e5a94d970..794fe15883ba 100644 --- a/x-pack/filebeat/module/juniper/srx/ingest/utm.yml +++ b/x-pack/filebeat/module/juniper/srx/ingest/utm.yml @@ -13,11 +13,12 @@ processors: - append: field: event.category value: network -- rename: +- convert: field: juniper.srx.urlcategory_risk + type: float target_field: event.risk_score ignore_missing: true - if: "ctx.juniper?.srx?.urlcategory_risk != null" + ignore_failure: true - set: field: event.kind value: alert @@ -380,6 +381,7 @@ processors: - juniper.srx.nat_source_port - juniper.srx.bytes_from_server - juniper.srx.packets_from_server + - juniper.srx.urlcategory_risk ignore_missing: true on_failure: diff --git a/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json index 4187866594ed..69639938252b 100644 --- a/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json @@ -25,7 +25,7 @@ "event.module": "juniper", "event.original": "http-host=\"www.mytest.com\" file-category=\"executable\" action=\"BLOCK\" verdict-number=\"8\" verdict-source=\u201dcloud/blacklist/whitelist\u201d source-address=\"10.10.10.1\" source-port=\"57116\" destination-address=\"187.19.188.200\" destination-port=\"80\" protocol-id=\"6\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" policy-name=\"argon_policy\" username=\"user1\" session-id-32=\"50000002\" source-zone-name=\"untrust\" destination-zone-name=\"trust\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", @@ -82,7 +82,7 @@ "event.module": "juniper", "event.original": "timestamp=\"Thu Jun 23 09:55:38 2016\" tenant-id=\"ABC123456\" sample-sha256=\"ABC123\" client-ip=\"192.0.2.0\" verdict-number=\"9\" malware-info=\"Eicar:TestVirus\" username=\"admin\" hostname=\"host.example.com\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", @@ -130,7 +130,7 @@ "event.module": "juniper", "event.original": "timestamp=\"Thu Jun 23 09:55:38 2016\" tenant-id=\"ABC123456\" client-ip=\"192.0.2.0\" hostname=\"host.example.com\" status=\"in_progress\" policy-name=\"default\" th=\"7\" state=\"added\" reason=\"malware\" message=\"malware analysis detected host downloaded a malicious_file with score 9, sha256 ABC123\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "allowed", @@ -182,7 +182,7 @@ "event.module": "juniper", "event.original": "hostname=\"dummy_host\" file-category=\"executable\" verdict-number=\"10\" malware-info=\"Testfile\" action=\"PERMIT\" list-hit=\"N/A\" file-hash-lookup=\"FALSE\" source-address=\"1.1.1.1\" source-port=\"60148\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" policy-name=\"test-policy\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502156\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" sample-sha256=\"e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494\" file-name=\"dummy_file\" url=\"dummy_url\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.timezone": "-02:00", "event.type": [ "allowed", diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json index b597ed2afc52..9eb70c83a645 100644 --- a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json @@ -17,8 +17,8 @@ "event.module": "juniper", "event.original": "source-address=\"10.0.0.1\" source-port=\"594\" destination-address=\"10.128.0.1\" destination-port=\"10400\" connection-tag=\"0\" service-name=\"icmp\" nat-source-address=\"10.0.0.1\" nat-source-port=\"594\" nat-destination-address=\"10.128.0.1\" nat-destination-port=\"10400\" nat-connection-tag=\"0\" src-nat-rule-type=\"N/A\" src-nat-rule-name=\"N/A\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"1\" policy-name=\"vpn_trust_permit-all\" source-zone-name=\"vpn\" destination-zone-name=\"trust\" session-id-32=\"6093\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"st0.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", "event.outcome": "success", - "event.risk_score": "1", - "event.severity": "14", + "event.risk_score": 1.0, + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -78,8 +78,8 @@ "event.module": "juniper", "event.original": "source-address=\"10.0.0.26\" source-port=\"37233\" destination-address=\"10.128.0.1\" destination-port=\"161\" connection-tag=\"0\" service-name=\"None\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"MgmtAccess-trust-cleanup\" source-zone-name=\"trust\" destination-zone-name=\"junos-host\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\".local..0\" encrypted=\"No\" reason=\"Denied by policy\" session-id-32=\"7087\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", "event.outcome": "success", - "event.risk_score": "1", - "event.severity": "14", + "event.risk_score": 1.0, + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "denied", @@ -141,7 +141,7 @@ "event.module": "juniper", "event.original": "source-address=\"1.2.3.4\" source-port=\"56639\" destination-address=\"5.6.7.8\" destination-port=\"2003\" service-name=\"None\" protocol-id=\"6\" icmp-type=\"0\" policy-name=\"log-all-else\" source-zone-name=\"campus\" destination-zone-name=\"mngmt\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth6.0\" encrypted=\"No \"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "denied", @@ -217,7 +217,7 @@ "event.module": "juniper", "event.original": "reason=\"unset\" source-address=\"1.2.3.4\" source-port=\"63456\" destination-address=\"5.6.7.8\" destination-port=\"902\" service-name=\"None\" nat-source-address=\"1.2.3.4\" nat-source-port=\"63456\" nat-destination-address=\"5.6.7.8\" nat-destination-port=\"902\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"17\" policy-name=\"mngmt-to-vcenter\" source-zone-name=\"mngmt\" destination-zone-name=\"intra\" session-id-32=\"15353\" packets-from-client=\"1\" bytes-from-client=\"94\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"60\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth3.5\" encrypted=\"No \"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2014-05-01T06:28:10.933-02:00", "event.timezone": "-02:00", "event.type": [ @@ -299,7 +299,7 @@ "event.module": "juniper", "event.original": "source-address=\"50.0.0.100\" source-port=\"24065\" destination-address=\"30.0.0.100\" destination-port=\"768\" service-name=\"icmp\" nat-source-address=\"50.0.0.100\" nat-source-port=\"24065\" nat-destination-address=\"30.0.0.100\" nat-destination-port=\"768\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"alg-policy\" source-zone-name=\"untrust\" destination-zone-name=\"trust\" session-id-32=\"100000165\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth2.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -370,7 +370,7 @@ "event.module": "juniper", "event.original": "source-address=\"192.0.2.1\" source-port=\"1\" destination-address=\"198.51.100.12\" destination-port=\"46384\" service-name=\"icmp\" nat-source-address=\"192.0.2.1\" nat-source-port=\"1\" nat-destination-address=\"18.51.100.12\" nat-destination-port=\"46384\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"policy1\" source-zone-name=\"trustZone\" destination-zone-name=\"untrustZone\" session-id-32=\"41\" packet-incoming-interface=\"ge-0/0/1.0\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -442,7 +442,7 @@ "event.module": "juniper", "event.original": "reason=\"response received\" source-address=\"192.0.2.1\" source-port=\"1\" destination-address=\"198.51.100.12\" destination-port=\"46384\" service-name=\"icmp\" nat-source-address=\"192.0.2.1\" nat-source-port=\"1\" nat-destination-address=\"18.51.100.12\" nat-destination-port=\"46384\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"policy1\" source-zone-name=\"trustZone\" destination-zone-name=\"untrustZone\" session-id-32=\"41\" packets-from-client=\"1\" bytes-from-client=\"84\" packets-from-server=\"1\" bytes-from-server=\"84\" elapsed-time=\"0\" packet-incoming-interface=\"ge-0/0/1.0\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2010-09-30T04:55:07.188-02:00", "event.timezone": "-02:00", "event.type": [ @@ -524,8 +524,8 @@ "event.module": "juniper", "event.original": "reason=\"TCP FIN\" source-address=\"10.3.255.203\" source-port=\"47776\" destination-address=\"8.23.224.110\" destination-port=\"80\" connection-tag=\"0\" service-name=\"junos-http\" nat-source-address=\"10.3.136.49\" nat-source-port=\"19162\" nat-destination-address=\"8.23.224.110\" nat-destination-port=\"80\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"nat1\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit_all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"5\" packets-from-client=\"6\" bytes-from-client=\"337\" packets-from-server=\"4\" bytes-from-server=\"535\" elapsed-time=\"1\" application=\"HTTP\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/0.0\" encrypted=\"No\" application-category=\"Web\" application-sub-category=\"N/A\" application-risk=\"4\" application-characteristics=\"Can Leak Information;Supports File Transfer;Prone to Misuse;Known Vulnerabilities;Carrier of Malware;Capable of Tunneling;\"", "event.outcome": "success", - "event.risk_score": "4", - "event.severity": "14", + "event.risk_score": 4.0, + "event.severity": 14, "event.start": "2019-04-12T12:29:06.576-02:00", "event.timezone": "-02:00", "event.type": [ @@ -608,7 +608,7 @@ "event.module": "juniper", "event.original": "reason=\"TCP RST\" source-address=\"192.168.2.164\" source-port=\"53232\" destination-address=\"172.16.1.19\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"192.168.2.164\" nat-source-port=\"53232\" nat-destination-address=\"172.16.1.19\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"35\" source-zone-name=\"Trust\" destination-zone-name=\"Trust\" session-id-32=\"206\" packets-from-client=\"13\" bytes-from-client=\"4274\" packets-from-server=\"9\" bytes-from-server=\"1575\" elapsed-time=\"16\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/2.0\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2019-04-13T12:33:06.576-02:00", "event.timezone": "-02:00", "event.type": [ @@ -690,7 +690,7 @@ "event.module": "juniper", "event.original": "reason=\"idle Timeout\" source-address=\"100.73.10.92\" source-port=\"52890\" destination-address=\"58.68.126.198\" destination-port=\"53\" service-name=\"junos-dns-udp\" nat-source-address=\"58.78.140.131\" nat-source-port=\"11152\" nat-destination-address=\"58.68.126.198\" nat-destination-port=\"53\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"NAT_S\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"NAT\" source-zone-name=\"Gi_nat\" destination-zone-name=\"Internet\" session-id-32=\"220368889\" packets-from-client=\"1\" bytes-from-client=\"72\" packets-from-server=\"1\" bytes-from-server=\"136\" elapsed-time=\"8\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.108\" encrypted=\"UNKNOWN\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2018-10-06T23:32:20.898-02:00", "event.timezone": "-02:00", "event.type": [ @@ -784,7 +784,7 @@ "event.module": "juniper", "event.original": "reason=\"idle Timeout\" source-address=\"192.168.255.2\" source-port=\"62047\" destination-address=\"8.8.8.8\" destination-port=\"53\" service-name=\"junos-dns-udp\" nat-source-address=\"192.168.0.47\" nat-source-port=\"20215\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"rule001\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"trust-to-untrust-001\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"9621\" packets-from-client=\"1\" bytes-from-client=\"67\" packets-from-server=\"1\" bytes-from-server=\"116\" elapsed-time=\"3\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"fe-0/0/1.0\" encrypted=\"UNKNOWN\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2018-06-30T00:17:22.753-02:00", "event.timezone": "-02:00", "event.type": [ @@ -861,7 +861,7 @@ "event.module": "juniper", "event.original": "reason=\"application failure or action\" source-address=\"10.164.110.223\" source-port=\"9057\" destination-address=\"10.104.12.161\" destination-port=\"21\" service-name=\"junos-ftp\" nat-source-address=\"10.9.1.150\" nat-source-port=\"58020\" nat-destination-address=\"10.12.70.1\" nat-destination-port=\"21\" src-nat-rule-name=\"SNAT-Policy5\" dst-nat-rule-name=\"NAT-Policy10\" protocol-id=\"6\" policy-name=\"FW-FTP\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"24311\" packets-from-client=\"0\" bytes-from-client=\"0\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"1\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.0\" encrypted=\"No \"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2015-09-25T12:19:53.846-02:00", "event.timezone": "-02:00", "event.type": [ @@ -940,7 +940,7 @@ "event.module": "juniper", "event.original": "source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -1024,7 +1024,7 @@ "event.module": "juniper", "event.original": "source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" packets-from-client=\"1\" bytes-from-client=\"48\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"0\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2013-01-19T15:18:17.040-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1115,7 +1115,7 @@ "event.module": "juniper", "event.original": "reason=\"application failure or action\" source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"FTP\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" packets-from-client=\"3\" bytes-from-client=\"144\" packets-from-server=\"2\" bytes-from-server=\"104\" elapsed-time=\"1\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2013-01-19T15:18:17.040-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1208,7 +1208,7 @@ "event.module": "juniper", "event.original": "source-address=\"4.0.0.1\" source-port=\"33040\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"HTTP\" nested-application=\"FACEBOOK-SOCIALRSS\" nat-source-address=\"4.0.0.1\" nat-source-port=\"33040\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"28\" packets-from-client=\"371\" bytes-from-client=\"19592\" packets-from-server=\"584\" bytes-from-server=\"686432\" elapsed-time=\"60\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2013-01-19T15:18:18.040-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1296,7 +1296,7 @@ "event.module": "juniper", "event.original": "source-address=\"4.0.0.1\" source-port=\"33040\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"HTTP\" nested-application=\"FACEBOOK-SOCIALRSS\" nat-source-address=\"4.0.0.1\" nat-source-port=\"33040\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"28\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" profile-name=\u201dpf1\u201d rule-name=\u201dfacebook1\u201d routing-instance=\u201dinstance1\u201d destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -1386,7 +1386,7 @@ "event.module": "juniper", "event.original": "reason=\"TCP CLIENT RST\" source-address=\"4.0.0.1\" source-port=\"48873\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"4.0.0.1\" nat-source-port=\"48873\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"32\" packets-from-client=\"5\" bytes-from-client=\"392\" packets-from-server=\"3\" bytes-from-server=\"646\" elapsed-time=\"3\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2013-01-19T15:18:20.040-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1471,7 +1471,7 @@ "event.module": "juniper", "event.original": "source-address=\"50.0.0.100\" source-port=\"24065\" destination-address=\"30.0.0.100\" destination-port=\"768\" service-name=\"icmp\" nat-source-address=\"50.0.0.100\" nat-source-port=\"24065\" nat-destination-address=\"30.0.0.100\" nat-destination-port=\"768\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"alg-policy\" source-zone-name=\"untrust\" destination-zone-name=\"trust\" session-id-32=\"100000165\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth2.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -1534,8 +1534,8 @@ "event.module": "juniper", "event.original": "source-address=\"10.0.0.26\" source-port=\"37233\" destination-address=\"10.128.0.1\" destination-port=\"161\" connection-tag=\"0\" service-name=\"None\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"MgmtAccess-trust-cleanup\" source-zone-name=\"trust\" destination-zone-name=\"junos-host\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\".local..0\" encrypted=\"No\" reason=\"Denied by policy\" session-id-32=\"7087\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", "event.outcome": "success", - "event.risk_score": "1", - "event.severity": "14", + "event.risk_score": 1.0, + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "denied", @@ -1606,7 +1606,7 @@ "event.module": "juniper", "event.original": "reason=\"TCP CLIENT RST\" source-address=\"4.0.0.1\" source-port=\"48873\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"4.0.0.1\" nat-source-port=\"48873\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"32\" packets-from-client=\"5\" bytes-from-client=\"392\" packets-from-server=\"3\" bytes-from-server=\"646\" elapsed-time=\"3\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2020-01-19T15:18:20.040-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1702,7 +1702,7 @@ "event.module": "juniper", "event.original": "source-address=\"10.1.1.100\" source-port=\"58943\" destination-address=\"46.165.154.241\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"172.19.34.100\" nat-source-port=\"6018\" nat-destination-address=\"46.165.154.241\" nat-destination-port=\"80\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"16118\" packets-from-client=\"42\" bytes-from-client=\"2322\" packets-from-server=\"34\" bytes-from-server=\"2132\" elapsed-time=\"60\" username=\"N/A\" roles=\"N/A\" encrypted=\"No\" destination-interface-name=\"ge-0/0/0.0\" category=\"N/A\" sub-category=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2020-07-14T12:17:11.928-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1788,8 +1788,8 @@ "event.module": "juniper", "event.original": "reason=\"idle Timeout\" source-address=\"10.1.1.100\" source-port=\"64720\" destination-address=\"91.228.167.172\" destination-port=\"8883\" connection-tag=\"0\" service-name=\"None\" nat-source-address=\"172.19.34.100\" nat-source-port=\"24519\" nat-destination-address=\"91.228.167.172\" nat-destination-port=\"8883\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"3851\" packets-from-client=\"161\" bytes-from-client=\"9530\" packets-from-server=\"96\" bytes-from-server=\"9670\" elapsed-time=\"23755\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/1.0\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\" secure-web-proxy-session-type=\"NA\" peer-session-id=\"0\" peer-source-address=\"0.0.0.0\" peer-source-port=\"0\" peer-destination-address=\"0.0.0.0\" peer-destination-port=\"0\" hostname=\"NA NA\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", "event.outcome": "success", - "event.risk_score": "1", - "event.severity": "14", + "event.risk_score": 1.0, + "event.severity": 14, "event.start": "2020-07-13T14:43:05.041-02:00", "event.timezone": "-02:00", "event.type": [ @@ -1875,8 +1875,8 @@ "event.module": "juniper", "event.original": "source-address=\"10.1.1.100\" source-port=\"49583\" destination-address=\"8.8.8.8\" destination-port=\"53\" connection-tag=\"0\" service-name=\"junos-dns-udp\" nat-source-address=\"172.19.34.100\" nat-source-port=\"30838\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"15399\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/1.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", "event.outcome": "success", - "event.risk_score": "1", - "event.severity": "14", + "event.risk_score": 1.0, + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "start", @@ -1954,7 +1954,7 @@ "event.module": "juniper", "event.original": "reason=\"Closed by junos-alg\" source-address=\"10.1.1.100\" source-port=\"63381\" destination-address=\"8.8.8.8\" destination-port=\"53\" service-name=\"junos-dns-udp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"172.19.34.100\" nat-source-port=\"26764\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"15361\" packets-from-client=\"1\" bytes-from-client=\"66\" packets-from-server=\"1\" bytes-from-server=\"82\" elapsed-time=\"3\" username=\"N/A\" roles=\"N/A\" encrypted=\"No\" profile-name=\"N/A\" rule-name=\"N/A\" routing-instance=\"default\" destination-interface-name=\"ge-0/0/0.0\" uplink-incoming-interface-name=\"N/A\" uplink-tx-bytes=\"0\" uplink-rx-bytes=\"0\" category=\"N/A\" sub-category=\"N/A\" apbr-policy-name=\"N/A\" multipath-rule-name=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.start": "2020-07-13T14:12:05.530-02:00", "event.timezone": "-02:00", "event.type": [ diff --git a/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json index 7704c88fac07..8a5a73073558 100644 --- a/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json @@ -24,7 +24,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1583190783\" message-type=\"SIG\" source-address=\"10.11.11.1\" source-port=\"12345\" destination-address=\"187.188.188.10\" destination-port=\"123\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"3\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"20175\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"HTTP:MISC:GENERIC-DIR-TRAVERSAL\" nat-source-address=\"0.0.0.0\" nat-source-port=\"13312\" nat-destination-address=\"3.3.10.11\" nat-destination-port=\"9757\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"UNTRUST\" source-interface-name=\"reth1.24\" destination-zone-name=\"DMZ\" destination-interface-name=\"reth2.21\" packet-log-id=\"0\" alert=\"no\" username=\"unknown-user\" roles=\"N/A\" index=\"cnm\" type=\"idp\" message=\"-\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.start": "2020-03-02T21:13:03.193-02:00", "event.timezone": "-02:00", "event.type": [ @@ -112,7 +112,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1583190783\" message-type=\"SIG\" source-address=\"10.11.11.1\" source-port=\"12345\" destination-address=\"187.188.188.10\" destination-port=\"123\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"3\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"20175\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"CRITICAL\" attack-name=\"TCP:C2S:AMBIG:C2S-SYN-DATA\" nat-source-address=\"0.0.0.0\" nat-source-port=\"13312\" nat-destination-address=\"3.3.10.11\" nat-destination-port=\"9757\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"UNTRUST\" source-interface-name=\"reth1.24\" destination-zone-name=\"DMZ\" destination-interface-name=\"reth2.21\" packet-log-id=\"0\" alert=\"no\" username=\"unknown-user\" roles=\"N/A\" index=\"cnm\" type=\"idp\" message=\"-\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.start": "2020-03-02T21:13:03.197-02:00", "event.timezone": "-02:00", "event.type": [ @@ -200,7 +200,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.111.1\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.19.13.11\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.11\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.1\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.start": "2007-02-15T07:17:15.719-02:00", "event.timezone": "-02:00", "event.type": [ @@ -285,7 +285,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.30.11\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.16.1.10\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.11\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.1\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.start": "2017-10-12T19:55:55.792-02:00", "event.timezone": "-02:00", "event.type": [ @@ -359,7 +359,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1319367986\" ddos-application-name=\"Webserver\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.0\" destination-address=\"172.27.14.203\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" rulebase-name=\"DDOS\" policy-name=\"A DoS-Webserver\" repeat-count=\"0\" message=\"Connection rate exceeded limit 60\" context-value=\"N/A\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.timezone": "-02:00", "event.type": [ "info", @@ -414,7 +414,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1319419711\" ddos-application-name=\"Webserver\" source-zone-name=\"trust\" source-interface-name=\"reth1.O\" source-address=\"192.168.14.214\" source-port=\"50825\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.0\" destination-address=\"172.27.14.203\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" ruleebase-name=\"DDOS\" policy-name=\"AppDoS-Webserver\" repeat-count=\"0\" action=\"NONE\" threat-severity=\"INFO\" connection-hit-rate=\"30\" context-name=\"http-get-url\" context-hit-rate=\"123\" context-value-hit-rate=\"0\" time-scope=\"PEER\" time-count=\"3\" time-period=\"60\" context-value=\"N/A\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.timezone": "-02:00", "event.type": [ "info", @@ -482,7 +482,7 @@ "event.module": "juniper", "event.original": "epoch-time=\"1419419711\" ddos-application-name=\"Webserver\" source-zone-name=\"trust\" source-interface-name=\"reth3.0\" source-address=\"193.168.14.214\" source-port=\"50825\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.1\" destination-address=\"172.30.20.201\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" ruleebase-name=\"DDOS02\" policy-name=\"AppDoS-Webserver\" repeat-count=\"0\" action=\"NONE\" threat-severity=\"INFO\" connection-hit-rate=\"30\" context-name=\"http-get-url\" context-hit-rate=\"123\" context-value-hit-rate=\"0\" time-scope=\"PEER\" time-count=\"3\" time-period=\"60\" context-value=\"N/A\"", "event.outcome": "success", - "event.severity": "165", + "event.severity": 165, "event.timezone": "-02:00", "event.type": [ "info", diff --git a/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json index 10abae2fa6d8..e92c17e6a4c0 100644 --- a/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json @@ -22,7 +22,7 @@ "event.module": "juniper", "event.original": "attack-name=\"TCP sweep!\" source-address=\"113.113.17.17\" source-port=\"6000\" destination-address=\"40.177.177.1\" destination-port=\"1433\" source-zone-name=\"untrust\" interface-name=\"fe-0/0/2.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -82,7 +82,7 @@ "event.module": "juniper", "event.original": "attack-name=\"WinNuke attack!\" source-address=\"2000:0000:0000:0000:0000:0000:0000:0002\" source-port=\"3240\" destination-address=\"2001:0000:0000:0000:0000:0000:0000:0002\" destination-port=\"139\" source-zone-name=\"untrust\" interface-name=\"fe-0/0/2.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -140,7 +140,7 @@ "event.module": "juniper", "event.original": "attack-name=\"SYN flood!\" source-address=\"1.1.1.2\" source-port=\"40001\" destination-address=\"2.2.2.2\" destination-port=\"50010\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -206,7 +206,7 @@ "event.module": "juniper", "event.original": "attack-name=\"UDP flood!\" source-address=\"111.1.1.3\" source-port=\"40001\" destination-address=\"3.4.2.2\" destination-port=\"53\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -273,7 +273,7 @@ "event.module": "juniper", "event.original": "attack-name=\"ICMP fragment!\" source-address=\"111.1.1.3\" destination-address=\"3.4.2.2\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -337,7 +337,7 @@ "event.module": "juniper", "event.original": "attack-name=\"Record Route IP option!\" source-address=\"111.1.1.3\" destination-address=\"3.4.2.2\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -395,7 +395,7 @@ "event.module": "juniper", "event.original": "attack-name=\"Tunnel GRE 6in6!\" source-address=\"1212::12\" destination-address=\"1111::11\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -448,7 +448,7 @@ "event.module": "juniper", "event.original": "attack-name=\"Tunnel GRE 4in4!\" source-address=\"12.12.12.1\" destination-address=\"11.11.11.1\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -509,7 +509,7 @@ "event.module": "juniper", "event.original": "attack-name=\"SYN flood!\" destination-address=\"2.2.2.2\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"alarm-without-drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -553,7 +553,7 @@ "event.module": "juniper", "event.original": "attack-name=\"SYN flood!\" source-address=\"111.1.1.3\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"alarm-without-drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -610,7 +610,7 @@ "event.module": "juniper", "event.original": "attack-name=\"TCP port scan!\" source-address=\"10.1.1.100\" source-port=\"50630\" destination-address=\"10.1.1.1\" destination-port=\"10778\" source-zone-name=\"trust\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", @@ -661,7 +661,7 @@ "event.module": "juniper", "event.original": "attack-name=\"FIN but no ACK bit!\" source-address=\"10.1.1.100\" source-port=\"42799\" destination-address=\"10.1.1.1\" destination-port=\"7\" source-zone-name=\"trust\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", "event.outcome": "success", - "event.severity": "11", + "event.severity": 11, "event.timezone": "-02:00", "event.type": [ "info", diff --git a/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json index 49667e85897a..9385beef0b08 100644 --- a/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json @@ -15,7 +15,7 @@ "event.module": "juniper", "event.original": "category=\"secintel\" sub-category=\"Blacklist\" action=\"BLOCK\" action-detail=\"DROP\" http-host=\"N/A\" threat-severity=\"0\" source-address=\"5.196.121.161\" source-port=\"1\" destination-address=\"10.10.0.10\" destination-port=\"24039\" protocol-id=\"1\" application=\"N/A\" nested-application=\"N/A\" feed-name=\"Tor_Exit_Nodes\" policy-name=\"cc_policy\" profile-name=\"Blacklist\" username=\"N/A\" roles=\"N/A\" session-id-32=\"572564\" source-zone-name=\"Outside\" destination-zone-name=\"DMZ\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", @@ -81,7 +81,7 @@ "event.module": "juniper", "event.original": "category=\"secintel\" sub-category=\"CC\" action=\"BLOCK\" action-detail=\"CLOSE REDIRECT MSG\" http-host=\"dummy_host\" threat-severity=\"10\" source-address=\"1.1.1.1\" source-port=\"36612\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" feed-name=\"cc_url_data\" policy-name=\"test\" profile-name=\"test-profile\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502362\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" occur-count=\"0\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", diff --git a/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json index f9890a6ca0f2..6b0aa31072fa 100644 --- a/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json +++ b/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json @@ -22,7 +22,7 @@ "event.module": "juniper", "event.original": "source-address=\"192.168.1.100\" source-port=\"58071\" destination-address=\"103.235.46.39\" destination-port=\"80\" category=\"cat1\" reason=\"BY_BLACK_LIST\" profile=\"uf1\" url=\"www.baidu.com\" obj=\"/\" username=\"user01\" roles=\"N/A\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "info", @@ -83,7 +83,7 @@ "event.module": "juniper", "event.original": "source-address=\"10.10.10.50\" source-port=\"1402\" destination-address=\"216.200.241.66\" destination-port=\"80\" category=\"N/A\" reason=\"BY_OTHER\" profile=\"wf-profile\" url=\"www.checkpoint.com\" obj=\"/css/homepage2012.css\" username=\"user02\" roles=\"N/A\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "allowed", @@ -137,7 +137,7 @@ "event.module": "juniper", "event.original": "source-address=\"188.40.238.250\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"47095\" source-zone-name=\"untrust\" filename=\"www.eicar.org/download/eicar.com\" temporary-filename=\"www.eicar.org/download/eicar.com\" name=\"EICAR-Test-File\" url=\"EICAR-Test-File\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "info", @@ -197,7 +197,7 @@ "event.module": "juniper", "event.original": "source-address=\"74.125.155.147\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"33578\" filename=\"www.google.com/\" error-code=\"14\" error-message=\"scan engine is not ready\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "allowed", @@ -251,7 +251,7 @@ "event.module": "juniper", "event.original": "source-address=\"10.2.1.101\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"51727\" filename=\"10.2.1.101/images/junos- srxsme-10.2-20100106.0-domestic.tgz\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "allowed", @@ -295,7 +295,7 @@ "event.module": "juniper", "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-name=\"N/A\" source-address=\"10.10.10.1\" profile-name=\"antispam01\" action=\"drop\" reason=\"Match local blacklist\" username=\"user01\" roles=\"N/A\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", @@ -344,7 +344,7 @@ "event.module": "juniper", "event.original": "source-zone=\"untrust\" destination-zone=\"trust\" protocol=\"http\" source-address=\"192.0.2.3\" source-port=\"58071\" destination-address=\"198.51.100.2\" destination-port=\"80\" profile-name=\"content02\" action=\"drop\" reason=\"blocked due to file extension block list\" username=\"user01@testuser.com\" roles=\"N/A\" filename=\"test.cmd\"", "event.outcome": "success", - "event.severity": "14", + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "info", @@ -406,7 +406,7 @@ "event.module": "juniper", "event.original": "source-address=\"192.168.1.100\" source-port=\"58071\" destination-address=\"103.235.46.39\" destination-port=\"80\" category=\"cat1\" reason=\"BY_BLACK_LIST\" profile=\"uf1\" url=\"www.baidu.com\" obj=\"/\" username=\"user01\" roles=\"N/A\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "info", @@ -462,7 +462,7 @@ "event.module": "juniper", "event.original": "source-address=\"188.40.238.250\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"47095\" source-zone-name=\"untrust\" filename=\"www.eicar.org/download/eicar.com\" temporary-filename=\"www.eicar.org/download/eicar.com\" name=\"EICAR-Test-File\" url=\"EICAR-Test-File\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "info", @@ -529,8 +529,8 @@ "event.module": "juniper", "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"10.1.1.100\" source-port=\"58974\" destination-address=\"104.26.15.142\" destination-port=\"443\" session-id=\"16297\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" category=\"Enhanced_Information_Technology\" reason=\"BY_SITE_REPUTATION_MODERATELY_SAFE\" profile=\"WCF1\" url=\"datawrapper.dwcdn.net\" obj=\"/\" username=\"N/A\" roles=\"N/A\" application-sub-category=\"N/A\" urlcategory-risk=\"0\"", "event.outcome": "success", - "event.risk_score": "0", - "event.severity": "14", + "event.risk_score": 0.0, + "event.severity": 14, "event.timezone": "-02:00", "event.type": [ "allowed", @@ -594,8 +594,8 @@ "event.module": "juniper", "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"10.1.1.100\" source-port=\"59075\" destination-address=\"85.114.159.93\" destination-port=\"443\" session-id=\"16490\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" category=\"Enhanced_Advertisements\" reason=\"BY_SITE_REPUTATION_SUSPICIOUS\" profile=\"WCF1\" url=\"dsp.adfarm1.adition.com\" obj=\"/\" username=\"N/A\" roles=\"N/A\" application-sub-category=\"N/A\" urlcategory-risk=\"3\"", "event.outcome": "success", - "event.risk_score": "3", - "event.severity": "12", + "event.risk_score": 3.0, + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "info", @@ -651,7 +651,7 @@ "event.module": "juniper", "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"23.209.86.45\" source-port=\"80\" destination-address=\"10.1.1.100\" destination-port=\"58954\" profile-name=\"Custom-Sophos-Profile\" filename=\"download.cdn.mozilla.net/pub/firefox/releases/78.0.2/update/win64/de/firefox-78.0.2.complete.mar\" action=\"BLOCKED\" reason=\"exceeding maximum content size\" error-code=\"7\" username=\"N/A\" roles=\"N/A\"", "event.outcome": "success", - "event.severity": "12", + "event.severity": 12, "event.timezone": "-02:00", "event.type": [ "allowed", diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml index 5b6d1c821e0c..8ce699299833 100644 --- a/x-pack/filebeat/module/suricata/eve/config/eve.yml +++ b/x-pack/filebeat/module/suricata/eve/config/eve.yml @@ -24,6 +24,7 @@ processors: - {from: suricata.eve.dest_ip, to: destination.address} - {from: suricata.eve.dest_port, to: destination.port, type: long} - {from: suricata.eve.proto, to: network.transport} + - {from: suricata.eve.flow_id, type: string} - convert: ignore_missing: true fail_on_error: false diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index ecccab3a10f9..457a16da86fb 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -64,7 +64,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 2191386088856669, + "suricata.eve.flow_id": "2191386088856669", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -145,7 +145,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 678269478904081, + "suricata.eve.flow_id": "678269478904081", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -226,7 +226,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 1170030461115650, + "suricata.eve.flow_id": "1170030461115650", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -307,7 +307,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 49628113637132, + "suricata.eve.flow_id": "49628113637132", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -388,7 +388,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 116307482565223, + "suricata.eve.flow_id": "116307482565223", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -469,7 +469,7 @@ "suricata.eve.alert.signature": "ET POLICY curl User-Agent Outbound", "suricata.eve.alert.signature_id": 2013028, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 1205867738178946, + "suricata.eve.flow_id": "1205867738178946", "suricata.eve.http.http_content_type": "text/html", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", @@ -550,7 +550,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 764842923400056, + "suricata.eve.flow_id": "764842923400056", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 0, @@ -631,7 +631,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 0, @@ -712,7 +712,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 1, @@ -793,7 +793,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 764842923400056, + "suricata.eve.flow_id": "764842923400056", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 1, @@ -874,7 +874,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 764842923400056, + "suricata.eve.flow_id": "764842923400056", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 2, @@ -955,7 +955,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 764842923400056, + "suricata.eve.flow_id": "764842923400056", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 3, @@ -1036,7 +1036,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 2, @@ -1117,7 +1117,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 3, @@ -1198,7 +1198,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 4, @@ -1279,7 +1279,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 5, @@ -1360,7 +1360,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 6, @@ -1441,7 +1441,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 7, @@ -1521,7 +1521,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 8, @@ -1601,7 +1601,7 @@ "suricata.eve.alert.signature": "ET POLICY GNU/Linux APT User-Agent Outbound likely related to package management", "suricata.eve.alert.signature_id": 2013504, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 112424506237238, + "suricata.eve.flow_id": "112424506237238", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "enp0s3", "suricata.eve.tx_id": 9, @@ -1651,7 +1651,7 @@ "source.ip": "10.126.2.140", "source.port": 45884, "suricata.eve.event_type": "tls", - "suricata.eve.flow_id": 1091813059495729, + "suricata.eve.flow_id": "1091813059495729", "suricata.eve.in_iface": "enp5s0", "suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33", "suricata.eve.tls.issuerdn": "C=US, O=Google Inc, CN=Google Internet Authority G2", @@ -1741,7 +1741,7 @@ "suricata.eve.alert.signature": "SURICATA TLS on unusual port", "suricata.eve.alert.signature_id": 2610003, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 991192778198299, + "suricata.eve.flow_id": "991192778198299", "suricata.eve.in_iface": "enp0s31f6", "suricata.eve.tls.fingerprint": "36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc", "suricata.eve.tls.issuerdn": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json index c61e3f6d12aa..cdcf57030e69 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json @@ -40,7 +40,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 885455453886936, + "suricata.eve.flow_id": "885455453886936", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -87,7 +87,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1418448010418810, + "suricata.eve.flow_id": "1418448010418810", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -151,7 +151,7 @@ "suricata.eve.dns.rrtype": "AAAA", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1418448010418810, + "suricata.eve.flow_id": "1418448010418810", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -215,7 +215,7 @@ "suricata.eve.dns.rrtype": "A", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 885455453886936, + "suricata.eve.flow_id": "885455453886936", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -263,7 +263,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 40074894954311, + "suricata.eve.flow_id": "40074894954311", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -311,7 +311,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 2130691028471842, + "suricata.eve.flow_id": "2130691028471842", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -406,7 +406,7 @@ "suricata.eve.dns.rrtype": "A", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 40074894954311, + "suricata.eve.flow_id": "40074894954311", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -501,7 +501,7 @@ "suricata.eve.dns.rrtype": "AAAA", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 2130691028471842, + "suricata.eve.flow_id": "2130691028471842", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -549,7 +549,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -597,7 +597,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -655,7 +655,7 @@ "suricata.eve.dns.ttl": 1315, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -717,7 +717,7 @@ "suricata.eve.dns.ttl": 15, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -779,7 +779,7 @@ "suricata.eve.dns.ttl": 15, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -841,7 +841,7 @@ "suricata.eve.dns.ttl": 15, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -903,7 +903,7 @@ "suricata.eve.dns.ttl": 15, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 814378410010223, + "suricata.eve.flow_id": "814378410010223", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -961,7 +961,7 @@ "suricata.eve.dns.ttl": 1268, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1023,7 +1023,7 @@ "suricata.eve.dns.ttl": 53, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1085,7 +1085,7 @@ "suricata.eve.dns.ttl": 53, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1147,7 +1147,7 @@ "suricata.eve.dns.ttl": 53, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1209,7 +1209,7 @@ "suricata.eve.dns.ttl": 53, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1887239765714716, + "suricata.eve.flow_id": "1887239765714716", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1257,7 +1257,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 2181951993205289, + "suricata.eve.flow_id": "2181951993205289", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1305,7 +1305,7 @@ "suricata.eve.dns.tx_id": 0, "suricata.eve.dns.type": "query", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 928596784370390, + "suricata.eve.flow_id": "928596784370390", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1400,7 +1400,7 @@ "suricata.eve.dns.rrtype": "A", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 2181951993205289, + "suricata.eve.flow_id": "2181951993205289", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" @@ -1495,7 +1495,7 @@ "suricata.eve.dns.rrtype": "AAAA", "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 928596784370390, + "suricata.eve.flow_id": "928596784370390", "suricata.eve.in_iface": "enp0s3", "tags": [ "suricata" diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 66204334c439..50125bc3f3c2 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -29,7 +29,7 @@ "source.ip": "192.168.86.85", "source.port": 55406, "suricata.eve.event_type": "ssh", - "suricata.eve.flow_id": 298824096901438, + "suricata.eve.flow_id": "298824096901438", "suricata.eve.in_iface": "en0", "suricata.eve.ssh.client.proto_version": "2.0", "suricata.eve.ssh.client.software_version": "OpenSSH_7.6", @@ -88,7 +88,7 @@ "suricata.eve.alert.signature": "ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)", "suricata.eve.alert.signature_id": 2024833, "suricata.eve.event_type": "alert", - "suricata.eve.flow_id": 904992230150281, + "suricata.eve.flow_id": "904992230150281", "suricata.eve.in_iface": "en0", "suricata.eve.tls.session_resumed": true, "suricata.eve.tls.sni": "l2.io", @@ -142,7 +142,7 @@ "source.ip": "192.168.86.85", "source.port": 56119, "suricata.eve.event_type": "http", - "suricata.eve.flow_id": 2115002772430095, + "suricata.eve.flow_id": "2115002772430095", "suricata.eve.http.http_content_type": "text/xml", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "en0", @@ -204,7 +204,7 @@ "suricata.eve.fileinfo.state": "CLOSED", "suricata.eve.fileinfo.stored": false, "suricata.eve.fileinfo.tx_id": 0, - "suricata.eve.flow_id": 2211411903323127, + "suricata.eve.flow_id": "2211411903323127", "suricata.eve.http.http_content_type": "application/xml", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "en0", @@ -274,7 +274,7 @@ "suricata.eve.dns.ttl": 299, "suricata.eve.dns.type": "answer", "suricata.eve.event_type": "dns", - "suricata.eve.flow_id": 1684780223079543, + "suricata.eve.flow_id": "1684780223079543", "suricata.eve.in_iface": "en0", "tags": [ "suricata" @@ -455,7 +455,7 @@ "source.ip": "192.168.86.85", "source.port": 56187, "suricata.eve.event_type": "tls", - "suricata.eve.flow_id": 89751777876473, + "suricata.eve.flow_id": "89751777876473", "suricata.eve.in_iface": "en0", "suricata.eve.tls.fingerprint": "6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47", "suricata.eve.tls.issuerdn": "CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US", @@ -532,7 +532,7 @@ "suricata.eve.flow.alerted": false, "suricata.eve.flow.reason": "timeout", "suricata.eve.flow.state": "new", - "suricata.eve.flow_id": 1828507008887644, + "suricata.eve.flow_id": "1828507008887644", "tags": [ "suricata" ] @@ -574,7 +574,7 @@ "source.ip": "192.168.50.1", "source.port": 57134, "suricata.eve.event_type": "http", - "suricata.eve.flow_id": 913701662641234, + "suricata.eve.flow_id": "913701662641234", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "eno6", "suricata.eve.tx_id": 0, @@ -622,7 +622,7 @@ "source.ip": "192.168.50.1", "source.port": 60614, "suricata.eve.event_type": "tls", - "suricata.eve.flow_id": 1298574590709840, + "suricata.eve.flow_id": "1298574590709840", "suricata.eve.in_iface": "eno6", "suricata.eve.tls.fingerprint": "18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25", "suricata.eve.tls.issuerdn": "C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018", @@ -693,7 +693,7 @@ "source.ip": "192.168.50.1", "source.port": 50898, "suricata.eve.event_type": "http", - "suricata.eve.flow_id": 1097935193623328, + "suricata.eve.flow_id": "1097935193623328", "suricata.eve.http.protocol": "HTTP/1.1", "suricata.eve.in_iface": "eno6", "suricata.eve.tx_id": 0, @@ -742,7 +742,7 @@ "source.ip": "192.168.50.1", "source.port": 12509, "suricata.eve.event_type": "tls", - "suricata.eve.flow_id": 289459143040794, + "suricata.eve.flow_id": "289459143040794", "suricata.eve.in_iface": "eno6", "suricata.eve.tls.ja3.hash": "44d502d471cfdb99c59bdfb0f220e5a8", "suricata.eve.tls.ja3.string": "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0", diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index 091cacf2a18f..9381f616b899 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -164,7 +164,7 @@ processors: } - convert: ignore_missing: true - ignore_failure: true + fail_on_error: false mode: rename fields: - {from: zeek.dns.id.orig_h, to: source.address} @@ -175,16 +175,21 @@ processors: - {from: zeek.dns.proto, to: network.transport} - convert: ignore_missing: true - ignore_failure: true + fail_on_error: false mode: copy fields: - {from: source.address, to: source.ip, type: ip} - {from: destination.address, to: destination.ip, type: ip} - {from: zeek.session_id, to: event.id} - - {from: zeek.dns.trans_id, to: dns.id} + - {from: zeek.dns.trans_id, to: dns.id, type: string} - {from: zeek.dns.query, to: dns.question.name} - {from: zeek.dns.qtype_name, to: dns.question.type} - {from: zeek.dns.rcode_name, to: dns.response_code} + - convert: + ignore_missing: true + fail_on_error: false + fields: + - {from: zeek.dns.trans_id, type: string} - add_fields: target: event fields: diff --git a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json index 61a57b55bd8e..5be6888c0c95 100644 --- a/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/dns/test/dns-json.log-expected.json @@ -22,7 +22,7 @@ "RD", "RA" ], - "dns.id": 15209, + "dns.id": "15209", "dns.question.class": "IN", "dns.question.name": "dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io", "dns.question.registered_domain": "es.io", @@ -88,7 +88,7 @@ "zeek.dns.rcode_name": "NOERROR", "zeek.dns.rejected": false, "zeek.dns.rtt": 0.076967, - "zeek.dns.trans_id": 15209, + "zeek.dns.trans_id": "15209", "zeek.session_id": "CAcJw21BbVedgFnYH3" }, { @@ -96,7 +96,7 @@ "destination.address": "ff02::fb", "destination.ip": "ff02::fb", "destination.port": 5353, - "dns.id": 0, + "dns.id": "0", "dns.question.class": "IN", "dns.question.name": "_googlecast._tcp.local", "dns.question.registered_domain": "_tcp.local", @@ -143,7 +143,7 @@ "zeek.dns.qtype_name": "PTR", "zeek.dns.query": "_googlecast._tcp.local", "zeek.dns.rejected": false, - "zeek.dns.trans_id": 0, + "zeek.dns.trans_id": "0", "zeek.session_id": "C19a1k4lTv46YMbeOk" }, { @@ -158,7 +158,7 @@ } ], "dns.header_flags": "AA", - "dns.id": 0, + "dns.id": "0", "dns.question.name": "_googlecast._tcp.local", "dns.question.registered_domain": "_tcp.local", "dns.question.subdomain": "_googlecast", @@ -209,7 +209,7 @@ "zeek.dns.rcode": 0, "zeek.dns.rcode_name": "NOERROR", "zeek.dns.rejected": false, - "zeek.dns.trans_id": 0, + "zeek.dns.trans_id": "0", "zeek.session_id": "CdiVAw7jJw6gsX5H" } ] \ No newline at end of file From 364da7c4d5b547e55c765dcccceea4ecb5717bc6 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Mon, 25 Jan 2021 08:03:21 -0800 Subject: [PATCH 28/35] Add FAQ entry for MADV settings in older versions (#23429) * add FAQ entry for MADV settings in older versions * add changelog * finish sentence * Update libbeat/docs/shared-faq.asciidoc Co-authored-by: DeDe Morton * Update libbeat/docs/shared-faq.asciidoc Co-authored-by: DeDe Morton * Update libbeat/docs/shared-faq.asciidoc Co-authored-by: DeDe Morton Co-authored-by: DeDe Morton --- CHANGELOG.next.asciidoc | 1 + libbeat/docs/shared-faq.asciidoc | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d99315bd2a16..39af1c2bb221 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -230,6 +230,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix typo in config docs {pull}23185[23185] - Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183] - Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154] +- Add FAQ entry for madvdontneed variable {pull}23429[23429] - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419] - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484] diff --git a/libbeat/docs/shared-faq.asciidoc b/libbeat/docs/shared-faq.asciidoc index d6c48b73aa97..a3508a8d57bb 100644 --- a/libbeat/docs/shared-faq.asciidoc +++ b/libbeat/docs/shared-faq.asciidoc @@ -206,3 +206,10 @@ to +{beatname_lc}-customname-*+. For more information, see {kibana-ref}/index-patterns.html[Creating an index pattern] in the {kib} docs. endif::no_dashboards[] + +[[madvdontneed-rss]] +=== High RSS memory usage due to MADV settings + +In versions of {beatname_uc} prior to 7.10.2, the go runtime defaults to `MADV_FREE` by default. +In some cases, this can lead to high RSS memory usage while the kernel waits to reclaim any pages assigned to {beatname_uc}. +On versions prior to 7.10.2, set the `GODEBUG="madvdontneed=1"` environment variable if you run into RSS usage issues. From d6a5f17d65a073348ccab8b634abae50fd0364ea Mon Sep 17 00:00:00 2001 From: Michael Koch Date: Mon, 25 Jan 2021 19:21:51 +0200 Subject: [PATCH 29/35] Fix: Dissect Cisco ASA 302013 message usernames (#21196) - Add test log containing the AAA user field - Set destination.user.name - Copy destination.user.name to user.name. - Set related.user to user.name + destination.user.name. This allows logs like this to parse %ASA-6-302013: Built outbound TCP connection 447236 for outside:192.0.2.222/1234 (192.0.2.222/1234) to dmz:OCSP_Server/5678 (OCSP_Server/5678) Co-authored-by: Andrew Kroh --- CHANGELOG.next.asciidoc | 1 + .../filebeat/module/cisco/asa/test/sample.log | 2 +- .../cisco/asa/test/sample.log-expected.json | 130 +++++++++++++----- .../filebeat/module/cisco/ftd/test/sample.log | 1 - .../cisco/ftd/test/sample.log-expected.json | 62 ++++----- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 17 ++- 6 files changed, 147 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39af1c2bb221..c4269a4ec666 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -823,6 +823,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added support for first_event context in filebeat httpjson input {pull}23437[23437] - Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157] - Added `alternative_host` option to google pubsub input {pull}23215[23215] +- Added username parsing from Cisco ASA message 302013. {pull}21196[21196] - Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478] - Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521] - Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521] diff --git a/x-pack/filebeat/module/cisco/asa/test/sample.log b/x-pack/filebeat/module/cisco/asa/test/sample.log index 699d191e3775..73ea89341b00 100644 --- a/x-pack/filebeat/module/cisco/asa/test/sample.log +++ b/x-pack/filebeat/module/cisco/asa/test/sample.log @@ -38,7 +38,6 @@ Apr 30 2013 09:23:43: %ASA-5-106100: access-list acl_in est-allowed tcp inside/1 Apr 30 2013 09:23:43: %ASA-5-106100: access-list acl_in est-allowed tcp inside/10.0.0.16(2013) -> outside/192.0.0.89(2000) hit-cnt 1 first hit [0x71a87d94, 0x0] Apr 15 2018 09:34:34 EDT: %ASA-session-5-106100: access-list acl_in permitted tcp inside/10.0.0.16(2241) -> outside/192.0.0.99(2000) hit-cnt 1 first hit [0x71a87d94, 0x0] Dec 11 2018 08:01:24 : %ASA-6-302015: Built outbound UDP connection 447235 for outside:192.168.77.12/11180 (192.168.77.12/11180) to identity:10.0.13.13/80 (10.0.13.13/80) -Dec 11 2018 08:01:24 : %ASA-6-302015: Built outbound UDP connection 447235 for outside:192.168.77.12/11180 (192.168.77.12/11180) to identity:10.0.13.13/80port> (10.0.13.13/80) Dec 11 2018 08:01:24 : %ASA-4-106023: Deny udp src dmz:192.168.1.33/5555 dst outside:192.0.0.12/53 by access-group "dmz" [0x123a465e, 0x4c7bf613] Dec 11 2018 08:01:24 : %ASA-4-106023: Deny udp src dmz:192.168.1.33/5555 dst outside:192.0.0.12/53 by access-group "dmz" [0x123a465e, 0x4c7bf613] Dec 11 2018 08:01:31 : %ASA-6-302013: Built outbound TCP connection 447236 for outside:192.0.2.222/1234 (192.0.2.222/1234) to dmz:OCSP_Server/5678 (OCSP_Server/5678) @@ -70,3 +69,4 @@ Jan 14 2015 13:16:14: %ASA-4-338008: Dynamic Filter dropped blacklisted TCP traf Nov 16 2009 14:12:35: %ASA-5-304001: 10.30.30.30 Accessed URL 192.0.2.1:/app Nov 16 2009 14:12:36: %ASA-5-304001: 10.5.111.32 Accessed URL 192.0.2.32:http://example.com Nov 16 2009 14:12:37: %ASA-5-304002: Access denied URL http://www.example.net/images/favicon.ico SRC 10.69.6.39 DEST 192.0.0.19 on interface inside +Jan 13 2021 19:12:37: %ASA-6-302013: Built inbound TCP connection 27215708 for internet:10.2.3.4/49926 (1.2.3.4/49926)(LOCAL\username) to vlan-42:1.2.3.4/80 (1.2.3.4/80) (username) diff --git a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json index c990e91c7c33..b2c1d4cb8767 100644 --- a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json @@ -2049,7 +2049,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 6318, + "log.offset": 6138, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2100,7 +2100,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 6468, + "log.offset": 6288, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2153,7 +2153,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 6618, + "log.offset": 6438, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2209,7 +2209,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 6788, + "log.offset": 6608, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2265,7 +2265,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 6958, + "log.offset": 6778, "network.bytes": 14804, "network.iana_number": 6, "network.transport": "tcp", @@ -2319,7 +2319,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 7123, + "log.offset": 6943, "network.bytes": 134781, "network.iana_number": 6, "network.transport": "tcp", @@ -2373,7 +2373,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 7289, + "log.offset": 7109, "network.bytes": 134781, "network.iana_number": 6, "network.transport": "tcp", @@ -2422,7 +2422,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 7455, + "log.offset": 7275, "network.transport": "(no", "observer.egress.interface.name": "outside", "observer.product": "asa", @@ -2468,7 +2468,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 7597, + "log.offset": 7417, "network.transport": "(no", "observer.egress.interface.name": "outside", "observer.product": "asa", @@ -2517,7 +2517,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 7739, + "log.offset": 7559, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2570,7 +2570,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 7890, + "log.offset": 7710, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2624,7 +2624,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 8064, + "log.offset": 7884, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2678,7 +2678,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 8238, + "log.offset": 8058, "network.bytes": 11420, "network.iana_number": 6, "network.transport": "tcp", @@ -2732,7 +2732,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "informational", - "log.offset": 8403, + "log.offset": 8223, "network.bytes": 1416, "network.iana_number": 17, "network.transport": "udp", @@ -2781,7 +2781,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 8545, + "log.offset": 8365, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -2829,7 +2829,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 8666, + "log.offset": 8486, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -2877,7 +2877,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 8787, + "log.offset": 8607, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -2925,7 +2925,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 8908, + "log.offset": 8728, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -2973,7 +2973,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 9029, + "log.offset": 8849, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -3021,7 +3021,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 9150, + "log.offset": 8970, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -3069,7 +3069,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 9271, + "log.offset": 9091, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -3117,7 +3117,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "critical", - "log.offset": 9393, + "log.offset": 9213, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "asa", @@ -3168,7 +3168,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 9515, + "log.offset": 9335, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "outside", @@ -3220,7 +3220,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "error", - "log.offset": 9669, + "log.offset": 9489, "network.iana_number": 1, "network.transport": "icmp", "observer.egress.interface.name": "Outside", @@ -3269,7 +3269,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 9779, + "log.offset": 9599, "network.iana_number": 1, "network.transport": "icmp", "observer.egress.interface.name": "inside", @@ -3322,7 +3322,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 9915, + "log.offset": 9735, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3383,7 +3383,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 10166, + "log.offset": 9986, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3440,7 +3440,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "warning", - "log.offset": 10465, + "log.offset": 10285, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3487,7 +3487,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "notification", - "log.offset": 10762, + "log.offset": 10582, "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", @@ -3529,7 +3529,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "notification", - "log.offset": 10839, + "log.offset": 10659, "observer.product": "asa", "observer.type": "firewall", "observer.vendor": "Cisco", @@ -3572,7 +3572,7 @@ "input.type": "log", "log.file.path": "sample.log", "log.level": "notification", - "log.offset": 10931, + "log.offset": 10751, "observer.egress.interface.name": "inside", "observer.product": "asa", "observer.type": "firewall", @@ -3589,5 +3589,73 @@ "forwarded" ], "url.original": "http://www.example.net/images/favicon.ico" + }, + { + "@timestamp": "2021-01-13T19:12:37.000-02:00", + "cisco.asa.connection_id": "27215708", + "cisco.asa.destination_interface": "vlan-42", + "cisco.asa.mapped_destination_ip": "1.2.3.4", + "cisco.asa.mapped_destination_port": 80, + "cisco.asa.mapped_source_ip": "1.2.3.4", + "cisco.asa.mapped_source_port": 49926, + "cisco.asa.message_id": "302013", + "cisco.asa.source_interface": "internet", + "cisco.asa.source_username": "LOCAL\\username", + "destination.address": "1.2.3.4", + "destination.geo.city_name": "Moscow", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "RU", + "destination.geo.country_name": "Russia", + "destination.geo.location.lat": 55.7527, + "destination.geo.location.lon": 37.6172, + "destination.geo.region_iso_code": "RU-MOW", + "destination.geo.region_name": "Moscow", + "destination.ip": "1.2.3.4", + "destination.port": 80, + "destination.user.name": "username", + "event.action": "firewall-rule", + "event.category": [ + "network" + ], + "event.code": 302013, + "event.dataset": "cisco.asa", + "event.kind": "event", + "event.module": "cisco", + "event.original": "%ASA-6-302013: Built inbound TCP connection 27215708 for internet:10.2.3.4/49926 (1.2.3.4/49926)(LOCAL\\username) to vlan-42:1.2.3.4/80 (1.2.3.4/80) (username)", + "event.severity": 6, + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "asa", + "input.type": "log", + "log.file.path": "sample.log", + "log.level": "informational", + "log.offset": 10899, + "network.direction": "inbound", + "network.iana_number": 6, + "network.transport": "tcp", + "observer.egress.interface.name": "internet", + "observer.ingress.interface.name": "vlan-42", + "observer.product": "asa", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "10.2.3.4", + "1.2.3.4" + ], + "related.user": [ + "username" + ], + "service.type": "cisco", + "source.address": "10.2.3.4", + "source.ip": "10.2.3.4", + "source.nat.ip": "1.2.3.4", + "source.port": 49926, + "tags": [ + "cisco-asa", + "forwarded" + ], + "user.name": "username" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/ftd/test/sample.log b/x-pack/filebeat/module/cisco/ftd/test/sample.log index df85fe9a0966..09da866b4888 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/sample.log +++ b/x-pack/filebeat/module/cisco/ftd/test/sample.log @@ -38,7 +38,6 @@ Apr 30 2013 09:23:43: %FTD-5-106100: access-list acl_in est-allowed tcp inside/1 Apr 30 2013 09:23:43: %FTD-5-106100: access-list acl_in est-allowed tcp inside/10.0.0.16(2013) -> outside/192.0.0.89(2000) hit-cnt 1 first hit [0x71a87d94, 0x0] Apr 15 2018 09:34:34 EDT: %FTD-session-5-106100: access-list acl_in permitted tcp inside/10.0.0.16(2241) -> outside/192.0.0.99(2000) hit-cnt 1 first hit [0x71a87d94, 0x0] Dec 11 2018 08:01:24 127.0.0.1: %FTD-6-302015: Built outbound UDP connection 447235 for outside:192.168.77.12/11180 (192.168.77.12/11180) to identity:10.0.13.13/80 (10.0.13.13/80) -Dec 11 2018 08:01:24 127.0.0.1: %FTD-6-302015: Built outbound UDP connection 447235 for outside:192.168.77.12/11180 (192.168.77.12/11180) to identity:10.0.13.13/80port> (10.0.13.13/80) Dec 11 2018 08:01:24 127.0.0.1: %FTD-4-106023: Deny udp src dmz:192.168.1.33/5555 dst outside:192.0.0.12/53 by access-group "dmz" [0x123a465e, 0x4c7bf613] Dec 11 2018 08:01:24 127.0.0.1: %FTD-4-106023: Deny udp src dmz:192.168.1.33/5555 dst outside:192.0.0.12/53 by access-group "dmz" [0x123a465e, 0x4c7bf613] Dec 11 2018 08:01:31 127.0.0.1: %FTD-6-302013: Built outbound TCP connection 447236 for outside:192.0.2.222/1234 (192.0.2.222/1234) to dmz:OCSP_Server/5678 (OCSP_Server/5678) diff --git a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json index 2da682477423..d416dcb068c8 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json @@ -2013,7 +2013,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "warning", - "log.offset": 6328, + "log.offset": 6143, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2067,7 +2067,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "warning", - "log.offset": 6483, + "log.offset": 6298, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2123,7 +2123,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 6638, + "log.offset": 6453, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2180,7 +2180,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 6813, + "log.offset": 6628, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2237,7 +2237,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 6988, + "log.offset": 6803, "network.bytes": 14804, "network.iana_number": 6, "network.transport": "tcp", @@ -2294,7 +2294,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 7158, + "log.offset": 6973, "network.bytes": 134781, "network.iana_number": 6, "network.transport": "tcp", @@ -2351,7 +2351,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 7329, + "log.offset": 7144, "network.bytes": 134781, "network.iana_number": 6, "network.transport": "tcp", @@ -2403,7 +2403,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 7500, + "log.offset": 7315, "network.transport": "(no", "observer.egress.interface.name": "outside", "observer.hostname": "127.0.0.1", @@ -2452,7 +2452,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 7647, + "log.offset": 7462, "network.transport": "(no", "observer.egress.interface.name": "outside", "observer.hostname": "127.0.0.1", @@ -2504,7 +2504,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "warning", - "log.offset": 7794, + "log.offset": 7609, "network.iana_number": 17, "network.transport": "udp", "observer.egress.interface.name": "dmz", @@ -2560,7 +2560,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 7950, + "log.offset": 7765, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2617,7 +2617,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 8129, + "log.offset": 7944, "network.direction": "outbound", "network.iana_number": 6, "network.transport": "tcp", @@ -2674,7 +2674,7 @@ "host.hostname": "127.0.0.1", "input.type": "log", "log.level": "informational", - "log.offset": 8308, + "log.offset": 8123, "network.bytes": 11420, "network.iana_number": 6, "network.transport": "tcp", @@ -2730,7 +2730,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "informational", - "log.offset": 8478, + "log.offset": 8293, "network.bytes": 1416, "network.iana_number": 17, "network.transport": "udp", @@ -2778,7 +2778,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 8620, + "log.offset": 8435, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -2825,7 +2825,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 8741, + "log.offset": 8556, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -2872,7 +2872,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 8862, + "log.offset": 8677, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -2919,7 +2919,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 8983, + "log.offset": 8798, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -2966,7 +2966,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 9104, + "log.offset": 8919, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -3013,7 +3013,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 9225, + "log.offset": 9040, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -3060,7 +3060,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 9346, + "log.offset": 9161, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -3107,7 +3107,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "critical", - "log.offset": 9468, + "log.offset": 9283, "observer.egress.interface.name": "Mobile_Traffic", "observer.hostname": "GIFRCHN01", "observer.product": "ftd", @@ -3157,7 +3157,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "warning", - "log.offset": 9590, + "log.offset": 9405, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "outside", @@ -3208,7 +3208,7 @@ "host.hostname": "GIFRCHN01", "input.type": "log", "log.level": "error", - "log.offset": 9744, + "log.offset": 9559, "network.iana_number": 1, "network.transport": "icmp", "observer.egress.interface.name": "Outside", @@ -3256,7 +3256,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "warning", - "log.offset": 9854, + "log.offset": 9669, "network.iana_number": 1, "network.transport": "icmp", "observer.egress.interface.name": "inside", @@ -3308,7 +3308,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "warning", - "log.offset": 9990, + "log.offset": 9805, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3369,7 +3369,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "warning", - "log.offset": 10241, + "log.offset": 10056, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3426,7 +3426,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "warning", - "log.offset": 10540, + "log.offset": 10355, "network.iana_number": 6, "network.transport": "tcp", "observer.egress.interface.name": "inside", @@ -3472,7 +3472,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "notification", - "log.offset": 10839, + "log.offset": 10654, "observer.product": "ftd", "observer.type": "firewall", "observer.vendor": "Cisco", @@ -3513,7 +3513,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "notification", - "log.offset": 10916, + "log.offset": 10731, "observer.product": "ftd", "observer.type": "firewall", "observer.vendor": "Cisco", @@ -3555,7 +3555,7 @@ "fileset.name": "ftd", "input.type": "log", "log.level": "notification", - "log.offset": 11008, + "log.offset": 10823, "observer.egress.interface.name": "inside", "observer.product": "ftd", "observer.type": "firewall", diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index 8d0e1b24c63e..581691ebcf91 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -297,10 +297,11 @@ processors: if: "ctx._temp_.cisco.message_id == '113019'" field: "message" pattern: "Group = %{}, Username = %{source.user.name}, IP = %{destination.address}, Session disconnected. Session Type: %{}, Duration: %{_temp_.duration_hms}, Bytes xmt: %{source.bytes}, Bytes rcv: %{destination.bytes}, Reason: %{message}" - - dissect: + - grok: if: '["302013", "302015"].contains(ctx._temp_.cisco.message_id)' field: "message" - pattern: "Built %{network.direction} %{network.transport} connection %{_temp_.cisco.connection_id} for %{_temp_.cisco.source_interface}:%{source.address}/%{source.port} (%{_temp_.natsrcip}/%{_temp_.cisco.mapped_source_port}) to %{_temp_.cisco.destination_interface}:%{destination.address}/%{destination.port} (%{_temp_.natdstip}/%{_temp_.cisco.mapped_destination_port})" + patterns: + - "Built %{NOTSPACE:network.direction} %{NOTSPACE:network.transport} connection %{NUMBER:_temp_.cisco.connection_id} for %{NOTSPACE:_temp_.cisco.source_interface}:%{IP:source.address}/%{NUMBER:source.port} \\(%{IP:_temp_.natsrcip}/%{NUMBER:_temp_.cisco.mapped_source_port}\\)(\\(%{NOTSPACE:_temp_.cisco.source_username}\\))? to %{NOTSPACE:_temp_.cisco.destination_interface}:%{NOTSPACE:destination.address}/%{NUMBER:destination.port} \\(%{NOTSPACE:_temp_.natdstip}/%{NUMBER:_temp_.cisco.mapped_destination_port}\\)( \\(%{NOTSPACE:destination.user.name}\\))?%{GREEDYDATA}" - dissect: if: "ctx._temp_.cisco.message_id == '303002'" field: "message" @@ -1576,6 +1577,13 @@ processors: } } + - set: + description: copy destination.user.name to user.name if it is not set + field: user.name + value: "{{destination.user.name}}" + ignore_empty_value: true + if: ctx?.user?.name == null + # Configures observer fields with a copy from cisco and host fields. Later on these might replace host.hostname. - set: field: observer.hostname @@ -1613,6 +1621,11 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" + - append: + field: related.user + value: "{{destination.user.name}}" + allow_duplicates: false + if: "ctx?.destination?.user?.name != null" - append: field: related.hash value: "{{file.hash.sha256}}" From 629141994afe07f0b26ba299eef8031bf204edbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 25 Jan 2021 20:16:38 +0100 Subject: [PATCH 30/35] Use hostname check from verify.go to handle patterns in TLS certs (#23661) Previously, DNSNames in x509 certs with wildcards were not accepted. The function from Golang's `verify.go` is taken, so the check remains the same between Golang versions. --- .../transport/tlscommon/testdata/ca.crt | 47 +++++++++----- .../transport/tlscommon/testdata/ca.key | 51 +++++++++++++++ .../transport/tlscommon/testdata/server.crt | 22 +++++++ .../transport/tlscommon/testdata/server.key | 15 +++++ .../common/transport/tlscommon/tls_config.go | 2 +- .../transport/tlscommon/tls_config_test.go | 15 ++++- .../transport/tlscommon/validhostname.go | 64 ++++++++++++++++++- 7 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 libbeat/common/transport/tlscommon/testdata/ca.key create mode 100644 libbeat/common/transport/tlscommon/testdata/server.crt create mode 100644 libbeat/common/transport/tlscommon/testdata/server.key diff --git a/libbeat/common/transport/tlscommon/testdata/ca.crt b/libbeat/common/transport/tlscommon/testdata/ca.crt index da2bce043f72..f08fd34367e9 100644 --- a/libbeat/common/transport/tlscommon/testdata/ca.crt +++ b/libbeat/common/transport/tlscommon/testdata/ca.crt @@ -1,19 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIC/zCCAeegAwIBAgIJAIVZ8xw3LMNkMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV -BAMMC21vcmVsbG8ub3ZoMB4XDTE5MDgwOTA5MzQwMFoXDTI5MDgwNjA5MzQwMFow -FjEUMBIGA1UEAwwLbW9yZWxsby5vdmgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQCoM2HYyuTTlu41SlgVO0Hdx7eUQevGSKO6pjPjN49/KKY1z/3DoKzr -seWaGOjiWUAqx/GHX8AsR9ToVoKGBbSNeDxT33pt3I9aCnnOPTt3yDIOlr4ZWnKq -NnNHwfydsMBfBAYgdU/L506KuNHJQ18Zey5+A0roTWyHUT48mQBsjetXg77RfDMB -MYVOWETfl70GKAaAlVGZfJHCkfBzYnPcEjqtcuU/7d27WZrSMhXifzHAEmm0KPER -EWdo4UHTK23wLY6dvkp2O5i0bKHv+PuLpqYrm7R7SWGhhwD651n5S5W20FHDow+d -js0yW2gqYsZZN6S1uAsJ8rdYAEPhK9J9AgMBAAGjUDBOMB0GA1UdDgQWBBQ6Lsen -0HbE+7M6iV9r8n5rZrbl4jAfBgNVHSMEGDAWgBQ6Lsen0HbE+7M6iV9r8n5rZrbl -4jAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAgrLJnK4s/OVnh8CRk -GmikP+ZxhDs4k1nlr7+rTYkU0huoHK8p802w4zd74szYsHpo8kON/zSmFD7JpU4L -o2kseENqMsgrCPhF3+TDwf/Li43pbK162iAq8ZEpYnSXbQsRyP+Tz0lzoEoli6o7 -6KVn4VNookLMyhGIAOmhfbNm0jG+B2zz+bvoTAe9CiDfvq1k0fnuKFzRtRsj09NJ -FNMhSc02N4EDrGpL5CYmEXjPZS3lUsoYPwbYlmUt3Bzuf5hI0mDHCt3BYKH1vFI4 -W8/h9wwGn/yytsH21dkj41KEQK6N65gT9i0fBBiubuS2H1SVMMJ/J7PUqol278Ar -zGpS +MIIFhzCCA2+gAwIBAgIUL0vc8AdVKIcjap/RSpH21trR70swDQYJKoZIhvcNAQEL +BQAwUzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xFzAVBgNVBAMMDmNhQGV4YW1wbGUuY29tMB4XDTIxMDEy +NTE2MzQ0OFoXDTMxMDEyMzE2MzQ0OFowUzELMAkGA1UEBhMCVVMxEzARBgNVBAgM +CkNhbGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xFzAVBgNVBAMMDmNh +QGV4YW1wbGUuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAx0rP +p+sMWe3RehThE5Mh1s8uKsujG0q+Q62s4G4mBE5tQnmSS0LoezWuGMKNyjWQR4dt +IvicPZQfEhqOvdYAIA5fsQE8CMoXW50Q43kQlBUbvZH0yldUFtFtRLPD4RRtwB26 +sUhWLUUCdk4mZBUmAuhMbIoov+TZ8/EZBdqjRBqM9p+k/C9xfitqXKmBWvWOmc0i +NUpxMjJ0C18vVcoAneiMQbB4iBNFviSLxrhnH9sno6IKG/WSCmOaPirmGzMr/PYQ +Wa4j69xQfGd4VBwolShI+fkoCmMQMk06XENUXo9V75sgbV0U0PAjBv4Kqye/r6s2 +1wJKNnS8Ib4rBJAeh5PqebVmpgJUc8lAeC/4SE3Edw6yGILwuGnfZjZJeRgX+OMd +u5K29gvx4Kf0ZZ5F34vzsDwa8CGTTvdth8aNDhO4ETThxUtjqXSA91ewf93Tf3X5 +Rzbg1K5hSHFVcd53Hec6/5Aqiw5PBARa2Ekj1ZW9PHHrSf/x+axyOyK+akUOoI8X +FlgImdr21pKZPSFNpvrYURRYDz8/ftFlcbsx32D3/uQZJW6FpvyguFWnVrGFm7He +ptWvYP2wM0XSOsHQXhogv09sgZhxgViHbc7/PZXOpTFlQt1MXygXVuf0eBUTiJI4 +a595gF4F6Kx/ppBjWge+ZUUsnFjhHVhHvhzvncUCAwEAAaNTMFEwHQYDVR0OBBYE +FHg4mXfbBjMpE8mJUh/yPrfuD2yBMB8GA1UdIwQYMBaAFHg4mXfbBjMpE8mJUh/y +PrfuD2yBMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAA+yu1mF +QoMeL5MwWr7O8q41Fu1n6BpRMm6KD0JOVWCJezW7anOJmcuySk6j2FRMPl3Z2fMH +p1I4420LlxN9H7QD5TVUJWCcujb2W9vhH9/R0qj9G9gkixfI0H/cGWd+Pe71ub5b +wxBTIe7U20uQ9imje8rShiZvgg3EocbWgPZcDnfHFjXVw/A1ocyIwpqjxooU8jiN +n1479sYR+R5TMc0zgZrTOKspcbNq5TEK138sFt79VB2d4oJNV/D0p0GktKpwisiZ ++xjr6iD2gZ9GGi0l0nQmtmLs+QAMuj+yOZX8CPwJlg7JuJYJ/nu0I5tBB1kOBml6 +Jk2o5o3gU6FbfLc3j7aQ/kRP14ByfXqXPTVNbPxrVzFEsAx/NVWaVqbH9iwSye1G +M4kpvZ9RvEHHegNxoN3spKaJkpM056gTBJhWQIHGCOAqv7Izm68NqjSX6+wx92iZ +ujR1PR9pJdOYtjhdmQrWGLK7a06AaOo1v5iQOJ9SN48ucyN2hY2wIZ5IMdQC2I9P +IhIRTSX28cT0WRnH9Sdv9fWQLSfNwrcYWiTDd5+0ImspCC3HzwcTjqTCoT6utrmU +eHAzLPjoUu9FvnrZJW3eMOffvHSh3lK8yW3dv2HKFoXaBD5dL2irk4yacSAIIo2f +4T44UqQSs2U1ip1CHbP64vI1FRNfhDdZRU8w -----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/ca.key b/libbeat/common/transport/tlscommon/testdata/ca.key new file mode 100644 index 000000000000..9061f4479dc7 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAx0rPp+sMWe3RehThE5Mh1s8uKsujG0q+Q62s4G4mBE5tQnmS +S0LoezWuGMKNyjWQR4dtIvicPZQfEhqOvdYAIA5fsQE8CMoXW50Q43kQlBUbvZH0 +yldUFtFtRLPD4RRtwB26sUhWLUUCdk4mZBUmAuhMbIoov+TZ8/EZBdqjRBqM9p+k +/C9xfitqXKmBWvWOmc0iNUpxMjJ0C18vVcoAneiMQbB4iBNFviSLxrhnH9sno6IK +G/WSCmOaPirmGzMr/PYQWa4j69xQfGd4VBwolShI+fkoCmMQMk06XENUXo9V75sg +bV0U0PAjBv4Kqye/r6s21wJKNnS8Ib4rBJAeh5PqebVmpgJUc8lAeC/4SE3Edw6y +GILwuGnfZjZJeRgX+OMdu5K29gvx4Kf0ZZ5F34vzsDwa8CGTTvdth8aNDhO4ETTh +xUtjqXSA91ewf93Tf3X5Rzbg1K5hSHFVcd53Hec6/5Aqiw5PBARa2Ekj1ZW9PHHr +Sf/x+axyOyK+akUOoI8XFlgImdr21pKZPSFNpvrYURRYDz8/ftFlcbsx32D3/uQZ +JW6FpvyguFWnVrGFm7HeptWvYP2wM0XSOsHQXhogv09sgZhxgViHbc7/PZXOpTFl +Qt1MXygXVuf0eBUTiJI4a595gF4F6Kx/ppBjWge+ZUUsnFjhHVhHvhzvncUCAwEA +AQKCAgBD0xIY88WgVW+VPMXdA5XgpWHw7pz0DNvz7IeJWfNWQ9qrZPSE6KB2Ti6R +/qSCzPftYAmkfTevPVnVr3Qk50/jmQC4HNNxqlWIuEunHuyleQmX2rSUqGPV4DBv +0T44u9seJwAClbu+bF4KJU6rgQcOtkBMMDjuFdSXUNZTR7WI2ABhbrOXoA3gAqaQ +IqADdM6zSTz7JfqgOsuDk0Fo0Pakxx/0uzpKFUUHESkA2IFANjWnWt5V5Z1uK7ey +sKbfWur9jEDERJ+1BaXesNgix/KH0M4FROZ7ontAo0fZXpC4HdABW6oNjkAnW5KQ +LqUy2rRB9OMVCZJ4NgJQ+YdqZiH9J63IIObGDyV5MDNmu3N88aGaT/29LrOTX2+A +g1FZZxdEBapRonk1KRvoX2PfPWReB02ThSQePVb7LEQ4ETMP/Qdc4ACynTrkhRot +ehcHpuiSkpu0sZgZh+7l5+PxTIHspmeg/Dws6f0m9yJjpzS4ATOfo5AX6ZjFy6vf +IEJZ9Rj7F4CB15jj5jQ1Bd6lYgqqaHbrUOupLLN8fIe/S+IK6uETczAgvYaNazhf +8sS2xxFRNWKwImHrhg8yq/oA5Zfpyu4ypEck/XM/cXh49b3HjxhyKA1Xa2tBaXjX +JPgIw+lMRBMLkxCcp3B5s0pPqw+FB2JNW9xOE3aaRcmPmRm0AQKCAQEA+i6/U4kW +gVkDIqykj2OmBzIyWFnJSrnnDWKUmwf5F8X3oT6F5+v9ltqBeHuwfH8FaOQ4kf7e +5G2BT9mujay4yUtFX0/ok+/UdfWBli718JYp6TnqGQHn/ABwvngDKdDkvx4UJ9bV +jUZiQnKbMCHuuXlpiV6ByzyCPGdvU+ND5m5WBln82B184FicnsUQWcU4XnLVu3Rs +WBeLR42/mP91byQc/ZUjEUWnYPyiK/SFcHP2B5B88aJYNboMGwfupePTijl278Oh +Y0U8zcPaY9fp/tjEJtDvv299psY42wfv+kUl9qPWv9wY9DB47Itc+ecdmUeL2w76 +thi1ZeTL7Of8LQKCAQEAy+0ha/p/F3AkX4vZs2Du+u04BcU+Lnjn6jpXFVudeJl/ +o8X3ctcv5C8Lf2Zb3cl5xgDV17/W0sH8oVcyOje78mAC180mMrTSbGECMdgy5y0/ +kVu+qaksPL/PuVz1rdFGSJGh588DTGVceXGqEnvZwPO+zwbGExgXKDfZQirq95Fp +7ocvQqRHOj/jVyNhSzSMJEcjG6fWCzBLIvQGYnT/pmVZCV3zJhKyRf8LwDu01nZf +b62YYuzrrHm5xUXs/GtySdfibDgDDCAu0WBJzDlayaVttQHIXV98/1zdEMo3WQmz +QfhN6q1iXNP0TDZdixvSY7qPPkwpuAgoFIAVarBu+QKCAQAlkTl3oME5YRVgco4v +XeZDdF0s+SaJywqP7vqjoPndgQOjOdE/tycYLO1+GwywVR6J1qNMAPqVUIA3bleQ +vJvu1hevrT8eX8gGgnwYAis6GkJm3CRz5t5f7+z+HOVUtSJ0NF8QcGkW0rfUhIMb +Ii1HupyHXSKeUxK3YUzNSvYaNv1B7OdTyHHE+mliSfMfl8bUH+hKQrw2Kirm7rkT +j2Ch9MlJpshiQpRUsvrjIM+cyDzse9zXJ+qY/rvsny9Nx70vJ1vduwGT94Se3UcA +8R4Y/5HMxlkJ5QL4NRG6iiGV8iY2N/n6S2GP4Wt3EaI/gF3oc17j4QbyqxkqGMYq +Z9dtAoIBABHAdb1ZlACtUW1D1lKoKIWNaugNsIkJG18nbvc7/2xFKHhQPmcv1kpt +0floJq0a4c/buMHMQF4eZuiAl627tk+2DelNQXr+hKbMlw1RvbSkGrmDnAhW4rPz +GpkPP4++/PhKPSbZvXbECBXGUEDFqUIDteN9unZlDXcBzZbV95hPf0I1YGbTuCOY +9ihauxh53Y026BZ7OMXGkXPcfEXL24lXnzWSiR0SWZSATDyStf5JVat6GB7ccvOT +sRk8KhIbJFrLQTmccp92mby+Pg5aG84b6X3tRsziCBaOtevFPqYwkvs2K0o3h61/ +AfA0d1YpuNnXnqqUpLkRdLO8JqEf9LkCggEBALDwHe1SSdKc7BDws8zxr6bJyVxL +h2nvA43SWhhSA3jsQtxi4xS0f1GRzWoczjAJFNcLRCdnKkEBfMpBL03s9rexy1Pa +3Dd1WaiyDfvGpbTr8l0M+R5kEcr5MfCfsAKcnoN4/egaqkNLKMi7eFfrEloiRO3y +rX7DmH2p8Oe0cY5bhSBJrwMS2fVh/SYfOiCVLhymmuH4hVmtJ6YCBWAY4Fhk16Iu +wsiTfvbibKSvFA9MJe+25UQugdVgHLzG84OSK4lMDMd37BTex84dH1P/r5shvuLx +nBSUOKXETOQMAA71hoMveHxox9fwGSmh2oLeaEVvdpADpWfpsCZzt05iXjM= +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/testdata/server.crt b/libbeat/common/transport/tlscommon/testdata/server.crt new file mode 100644 index 000000000000..50ca5ce8b321 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAX0CAQEwDQYJKoZIhvcNAQELBQAwUzELMAkGA1UEBhMCVVMxEzARBgNV +BAgMCkNhbGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xFzAVBgNVBAMM +DmNhQGV4YW1wbGUuY29tMB4XDTIxMDEyNTE2MzQ0OVoXDTMxMDEyMzE2MzQ0OVow +UjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcMDVNh +biBGcmFuY2lzY28xFjAUBgNVBAMMDSouZXhhbXBsZS5jb20wgZ8wDQYJKoZIhvcN +AQEBBQADgY0AMIGJAoGBALFuNygrGLLSnD//JRfU6xMDqgizeVdQqDlLaP/HxQ84 +9RPWnjfbyx2M25JYcLvewPqKQ80lOYnMRhpvujmuKP7gQHNDWOsyXH5JljTX78Wb +I+nuVMeYjbUOh+6EgYNY59G5rH7xqgeu3y1YERfNdchEG8xjSxYeIZ7Ev6VMFF8r +AgMBAAEwDQYJKoZIhvcNAQELBQADggIBALyHDjVcY6Po1eHWTUCLLOW1ZzzkX4qu +gsfJM6qTIZIqh/O6tROGqH9kRw8SarIIZvtztfzuYtmQBE0qkBMzPzdN3x+3C4pz +jf2vsEKRqva9mf9y+JM0Mv0WUuPfusHxPKOCl1on71kP1GL1bYylKqazgVa2tAVa +78xs35YIuCM5apt0X+QO+Tnz/qfqJ7t3F7mP1aeCjYm8J20S8vKTYgkRkFX/8VJB +1zRPl0CAMyoHOMcrmb7wX8V1CIER7VBQ7h580B7/7okrw+Hr3xyMOA0w1DiRUQJE +biHBuDTRDmRg6W5nAwNLFLp/RfHttny0nEEcnzcjEStEKyDGbNg1W2ieWuIhgUza +L3W3ld9LDD9pMnQ8yYTMcL+J2Ir6ErhpGL3Hks42W2c/qYhvo3we6B2ADfsS7P+m +ku5W7/G2fDIlj6rtzaAeur+LSgsjU6kc1et2SJxjcJMPrS4xHxpAhJzD7h7f5N/B +RBc5cT2sE2vuUBRGkz0wC9AC2/kxmv4RwjsrYTY8rEOqHRkxDF18lfFocAoq7Hvr +lO6ft9/knzTQzKiizc6unXsLhUCvBzt50bA/gVLXmUmr1sncATKHWOLbvfRWat4I +0m52jlowgqnJPsXtl+wwNYHaw9gF71RTx/Ov2vZ8xm5SeBNkO8cpdAftETAEqpgp +fDlIVeywLvoN +-----END CERTIFICATE----- diff --git a/libbeat/common/transport/tlscommon/testdata/server.key b/libbeat/common/transport/tlscommon/testdata/server.key new file mode 100644 index 000000000000..8bb153a90069 --- /dev/null +++ b/libbeat/common/transport/tlscommon/testdata/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQCxbjcoKxiy0pw//yUX1OsTA6oIs3lXUKg5S2j/x8UPOPUT1p43 +28sdjNuSWHC73sD6ikPNJTmJzEYab7o5rij+4EBzQ1jrMlx+SZY01+/FmyPp7lTH +mI21DofuhIGDWOfRuax+8aoHrt8tWBEXzXXIRBvMY0sWHiGexL+lTBRfKwIDAQAB +AoGAaBKW5cfJl/JzVhJphn4MWL3YeXwUW4Pi+KBj+UwLKW+mSTmk2mzgyfd6P3AC +yB/Tn+GD/YutIUehgxYv7G9ceZC85EsPM6+1s887olgKNKbCiZZvrLBcBCzEhzkN +QpC2/cuOOVYdYYQJZp9RX7herAJ5aqxZHUUtCrudgfCiAckCQQDo37NhBBfUlLc4 +LW3ryxydsh7MrTMU63+5IVtXosV3TFdWN9LC6CCarkILcOG5tmEmM6v1UQRAgCkm +lb+/3SrXAkEAwwz9+mcAU1lTTiy+dCJkKepviT4Ex+BFl0yJPfSN5+/Wg15DjwsN +vdE0H5nAT65aECiYy8V9DKNwHNcTIaZXzQJBAMvoPOBhPiCVC410MgC6e9cVRWTA +766Muuy26Y1l6HQac4r6HGEv8oSeuxPbhrsfmBdkPVjz1L5Juj6f9yOgHEcCQHMH +pHkaaay+D00ZQjDHX38AzUqJEtS1xRTXhFDPeyj/3uiWnQ0tHauGR1EjobDcSC0j +ZAk4rOjZMnMvvA6qRTkCQQCT6B0edwnMc9q/4XcdF+LptWRiYNbSKkrisb304N+d +lqbB76fGQY22onWcZEvcOmifmzmgj56QXSUot+fkNlVK +-----END RSA PRIVATE KEY----- diff --git a/libbeat/common/transport/tlscommon/tls_config.go b/libbeat/common/transport/tlscommon/tls_config.go index 9e7eb4548dbf..718dbe42db98 100644 --- a/libbeat/common/transport/tlscommon/tls_config.go +++ b/libbeat/common/transport/tlscommon/tls_config.go @@ -288,7 +288,7 @@ func verifyHostname(cert *x509.Certificate, hostname string) error { } for _, name := range dnsnames { - if len(name) > 0 && len(hostname) > 0 && name == hostname { + if matchHostnames(name, hostname) { if !validHostname(name, true) { return fmt.Errorf("invalid hostname in cert") } diff --git a/libbeat/common/transport/tlscommon/tls_config_test.go b/libbeat/common/transport/tlscommon/tls_config_test.go index 1490664d3d35..76dfa61497f0 100644 --- a/libbeat/common/transport/tlscommon/tls_config_test.go +++ b/libbeat/common/transport/tlscommon/tls_config_test.go @@ -34,7 +34,10 @@ func TestMakeVerifyServerConnection(t *testing.T) { t.Fatalf("failed to open test certs: %+v", err) } - testCA, errs := LoadCertificateAuthorities([]string{filepath.Join("testdata", "cacert.crt")}) + testCA, errs := LoadCertificateAuthorities([]string{ + filepath.Join("testdata", "ca.crt"), + filepath.Join("testdata", "cacert.crt"), + }) if len(errs) > 0 { t.Fatalf("failed to load test certificate authorities: %+v", errs) } @@ -83,6 +86,15 @@ func TestMakeVerifyServerConnection(t *testing.T) { expectedCallback: true, expectedError: nil, }, + "default verification with certificates when required with correct wildcard cert": { + verificationMode: VerifyFull, + clientAuth: tls.RequireAndVerifyClientCert, + certAuthorities: testCA, + peerCerts: []*x509.Certificate{testCerts["wildcard"]}, + serverName: "hello.example.com", + expectedCallback: true, + expectedError: nil, + }, "certificate verification with certificates when required with correct cert": { verificationMode: VerifyCertificate, clientAuth: tls.RequireAndVerifyClientCert, @@ -181,6 +193,7 @@ func openTestCerts() (map[string]*x509.Certificate, error) { "expired": "tls.crt", "unknown authority": "unsigned_tls.crt", "correct": "client1.crt", + "wildcard": "server.crt", } { certBytes, err := ioutil.ReadFile(filepath.Join("testdata", certname)) diff --git a/libbeat/common/transport/tlscommon/validhostname.go b/libbeat/common/transport/tlscommon/validhostname.go index 15370b4d4f99..a6b2af7fb7c3 100644 --- a/libbeat/common/transport/tlscommon/validhostname.go +++ b/libbeat/common/transport/tlscommon/validhostname.go @@ -47,7 +47,69 @@ package tlscommon -import "strings" +import ( + "strings" + "unicode/utf8" +) + +func matchHostnames(pattern, host string) bool { + pattern = toLowerCaseASCII(pattern) + host = toLowerCaseASCII(strings.TrimSuffix(host, ".")) + + if len(pattern) == 0 || len(host) == 0 { + return false + } + + patternParts := strings.Split(pattern, ".") + hostParts := strings.Split(host, ".") + + if len(patternParts) != len(hostParts) { + return false + } + + for i, patternPart := range patternParts { + if i == 0 && patternPart == "*" { + continue + } + if patternPart != hostParts[i] { + return false + } + } + + return true +} + +// toLowerCaseASCII returns a lower-case version of in. See RFC 6125 6.4.1. We use +// an explicitly ASCII function to avoid any sharp corners resulting from +// performing Unicode operations on DNS labels. +func toLowerCaseASCII(in string) string { + // If the string is already lower-case then there's nothing to do. + isAlreadyLowerCase := true + for _, c := range in { + if c == utf8.RuneError { + // If we get a UTF-8 error then there might be + // upper-case ASCII bytes in the invalid sequence. + isAlreadyLowerCase = false + break + } + if 'A' <= c && c <= 'Z' { + isAlreadyLowerCase = false + break + } + } + + if isAlreadyLowerCase { + return in + } + + out := []byte(in) + for i, c := range out { + if 'A' <= c && c <= 'Z' { + out[i] += 'a' - 'A' + } + } + return string(out) +} // validHostname reports whether host is a valid hostname that can be matched or // matched against according to RFC 6125 2.2, with some leniency to accommodate From 294297996260b278f4d6e074fc2b2944497212fb Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Jan 2021 14:23:28 -0500 Subject: [PATCH 31/35] Update X-Pack Packetbeat config (#23666) --- x-pack/packetbeat/packetbeat.reference.yml | 2 +- x-pack/packetbeat/packetbeat.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 9aee618bed83..073bec9c7682 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -80,7 +80,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: true + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. #enabled: true # Set to true to publish fields with null values in events. diff --git a/x-pack/packetbeat/packetbeat.yml b/x-pack/packetbeat/packetbeat.yml index 2ac9186d43ec..15a0df9ebd13 100644 --- a/x-pack/packetbeat/packetbeat.yml +++ b/x-pack/packetbeat/packetbeat.yml @@ -38,7 +38,7 @@ packetbeat.flows: packetbeat.protocols: - type: icmp - # Enable ICMPv4 and ICMPv6 monitoring. Default: false + # Enable ICMPv4 and ICMPv6 monitoring. The default is true. enabled: true - type: amqp @@ -47,7 +47,8 @@ packetbeat.protocols: ports: [5672] - type: cassandra - #Cassandra port for traffic monitoring. + # Configure the ports where to listen for Cassandra traffic. You can disable + # the Cassandra protocol by commenting out the list of ports. ports: [9042] - type: dhcpv4 @@ -112,7 +113,8 @@ packetbeat.protocols: - 9243 # Elasticsearch - type: sip - # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + # Configure the ports where to listen for SIP traffic. You can disable + # the SIP protocol by commenting out the list of ports. ports: [5060] # ======================= Elasticsearch template setting ======================= From 26a84cbe2f29c78018cb115b8057f1a363ca9eeb Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 25 Jan 2021 11:40:13 -0800 Subject: [PATCH 32/35] Add missing SSL settings (#23632) --- auditbeat/auditbeat.reference.yml | 12 ------------ filebeat/filebeat.reference.yml | 12 ------------ heartbeat/heartbeat.reference.yml | 12 ------------ journalbeat/journalbeat.reference.yml | 12 ------------ libbeat/_meta/config/ssl.reference.yml.tmpl | 2 -- libbeat/docs/shared-faq.asciidoc | 2 +- libbeat/docs/shared-ssl-config.asciidoc | 6 +++++- metricbeat/metricbeat.reference.yml | 12 ------------ packetbeat/packetbeat.reference.yml | 12 ------------ winlogbeat/winlogbeat.reference.yml | 12 ------------ x-pack/auditbeat/auditbeat.reference.yml | 12 ------------ x-pack/filebeat/filebeat.reference.yml | 12 ------------ x-pack/functionbeat/functionbeat.reference.yml | 8 -------- x-pack/heartbeat/heartbeat.reference.yml | 12 ------------ x-pack/metricbeat/metricbeat.reference.yml | 12 ------------ x-pack/packetbeat/packetbeat.reference.yml | 12 ------------ x-pack/winlogbeat/winlogbeat.reference.yml | 12 ------------ 17 files changed, 6 insertions(+), 168 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 712ebf7ee674..cc8cfdba2db5 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -527,8 +527,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -660,8 +658,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -867,8 +863,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1029,8 +1023,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1331,8 +1323,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1534,8 +1524,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 858d307a57e1..2a5533bb6363 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1407,8 +1407,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1540,8 +1538,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1747,8 +1743,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1909,8 +1903,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2211,8 +2203,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2414,8 +2404,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 63501b14e2df..85e00f433427 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -705,8 +705,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -838,8 +836,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1045,8 +1041,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1207,8 +1201,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1509,8 +1501,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1712,8 +1702,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 7664a3edbd45..35c6dbb4c054 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -470,8 +470,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -603,8 +601,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -810,8 +806,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -972,8 +966,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1274,8 +1266,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1477,8 +1467,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/libbeat/_meta/config/ssl.reference.yml.tmpl b/libbeat/_meta/config/ssl.reference.yml.tmpl index 69b666f9c97e..65920fb646f0 100644 --- a/libbeat/_meta/config/ssl.reference.yml.tmpl +++ b/libbeat/_meta/config/ssl.reference.yml.tmpl @@ -5,8 +5,6 @@ # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. -# * certificate, which verifies that the provided certificate is signed by a -# trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/libbeat/docs/shared-faq.asciidoc b/libbeat/docs/shared-faq.asciidoc index a3508a8d57bb..d1c35e4710c3 100644 --- a/libbeat/docs/shared-faq.asciidoc +++ b/libbeat/docs/shared-faq.asciidoc @@ -154,7 +154,7 @@ To resolve this problem, try one of these solutions: * Create a DNS entry for the hostname mapping it to the server's IP. * Create an entry in `/etc/hosts` for the hostname. Or on Windows add an entry to `C:\Windows\System32\drivers\etc\hosts`. -* Re-create the server certificate and add a SubjectAltName (SAN) for the IP address of the server. This make the +* Re-create the server certificate and add a SubjectAltName (SAN) for the IP address of the server. This makes the server's certificate valid for both the hostname and the IP address. [[getsockopt-no-route-to-host]] diff --git a/libbeat/docs/shared-ssl-config.asciidoc b/libbeat/docs/shared-ssl-config.asciidoc index ce573aae38d1..31eedd1e19a8 100644 --- a/libbeat/docs/shared-ssl-config.asciidoc +++ b/libbeat/docs/shared-ssl-config.asciidoc @@ -105,7 +105,7 @@ NOTE: SSL settings are disabled if either `enabled` is set to `false` or the ==== `certificate_authorities` The list of root certificates for server verifications. If `certificate_authorities` is empty or not set, the trusted certificate authorities of the host system are used. If `certificate_authorities` is self-signed, the host system needs to trust that CA cert as well. -By default you can specify a list of file that +{beatname_lc} will read, but you can also embed a certificate directly in the `YAML` configuration: +By default you can specify a list of files that +{beatname_lc} will read, but you can also embed a certificate directly in the `YAML` configuration: [source,yaml] ---- @@ -234,6 +234,10 @@ Controls the verification of certificates. Valid values are: * `full`, which verifies that the provided certificate is signed by a trusted authority (CA) and also verifies that the server's hostname (or IP address) matches the names identified within the certificate. + * `strict`, which verifies that the provided certificate is signed by a trusted +authority (CA) and also verifies that the server's hostname (or IP address) +matches the names identified within the certificate. If the Subject Alternative +Name is empty, it returns an error. * `certificate`, which verifies that the provided certificate is signed by a trusted authority (CA), but does not perform any hostname verification. * `none`, which performs _no verification_ of the server's certificate. This diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 8250323ca045..bb2e7cdf8fb1 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1304,8 +1304,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1437,8 +1435,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1644,8 +1640,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1806,8 +1800,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2108,8 +2100,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2311,8 +2301,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 073bec9c7682..9f25343877fe 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1022,8 +1022,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1155,8 +1153,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1362,8 +1358,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1524,8 +1518,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1826,8 +1818,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2029,8 +2019,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 1ab1796a809a..7b98270f0bf9 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -450,8 +450,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -583,8 +581,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -790,8 +786,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -952,8 +946,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1254,8 +1246,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1457,8 +1447,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index ac1347037946..44b58a736e12 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -583,8 +583,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -716,8 +714,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -923,8 +919,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1085,8 +1079,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1387,8 +1379,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1590,8 +1580,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9104290b7ce9..01f65a4c9101 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3205,8 +3205,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -3338,8 +3336,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -3545,8 +3541,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -3707,8 +3701,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -4009,8 +4001,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -4212,8 +4202,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index d96ab60094e9..d0d5ecf487a4 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -813,8 +813,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -946,8 +944,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1231,8 +1227,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1434,8 +1428,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 63501b14e2df..85e00f433427 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -705,8 +705,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -838,8 +836,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1045,8 +1041,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1207,8 +1201,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1509,8 +1501,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1712,8 +1702,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 71eaa8f800fe..dfa898fbe80d 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1805,8 +1805,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1938,8 +1936,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2145,8 +2141,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2307,8 +2301,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2609,8 +2601,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2812,8 +2802,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 073bec9c7682..9f25343877fe 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1022,8 +1022,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1155,8 +1153,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1362,8 +1358,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1524,8 +1518,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1826,8 +1818,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -2029,8 +2019,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 636325f55715..a9cb100ce330 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -493,8 +493,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -626,8 +624,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -833,8 +829,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -995,8 +989,6 @@ output.elasticsearch: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1297,8 +1289,6 @@ setup.kibana: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative @@ -1500,8 +1490,6 @@ logging.files: # * full, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. - # * certificate, which verifies that the provided certificate is signed by a - # trusted authority (CA), but does not perform any hostname verification. # * strict, which verifies that the provided certificate is signed by a trusted # authority (CA) and also verifies that the server's hostname (or IP address) # matches the names identified within the certificate. If the Subject Alternative From d4e193df3be412c0c39d7558f5e0ca7418ad25f7 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Mon, 25 Jan 2021 16:02:40 -0600 Subject: [PATCH 33/35] Remove 4912 evtx from testing (#23669) - causing failures on Win 7,8, 2008R2 & 2012R2 --- .../test/testdata/4912_WindowsSrv2016.evtx | Bin 69632 -> 0 bytes .../4912_WindowsSrv2016.evtx.golden.json | 70 ------------------ 2 files changed, 70 deletions(-) delete mode 100644 x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx delete mode 100644 x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx deleted file mode 100644 index 15a93a947a21e60b5c5de31a57cd5c14625ff795..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 69632 zcmeI1X>3$g6vxk-c{99DXX((?1w{d=QnaN7i(ql7v{jVCXe(kYN~JB7PG@QtC^j0R z7!$YP8WQ|KOpL)T8e<^DsF9fP!Qd7l7-M2$;zCRmF&H%&|L4B zIrrXkf9KqDZ>P=ao|fKpw-l#y(-QrS$4&P{#-f># z16IPaBx?Pz?6MF29-4ow;fmYtt4lhcM_#LX9V=0(#-!{(Ru`ygPc$UWvO&5fBf}^^ ziYL6@YkV&el2PfGG#ZD{5VGoJ9mIxZK(3ZK_zcM!)O16p6Zsmvdr>neS(LKU1IZec zv#6gV8_|}*d#=>tb0u=MvH;)R$EP1ECmv*6-cjQ~Mol&XtzEn#7egZK$LgM?VYwaa)tS9DN2NSy$CEPM zmdOx>Tbc)6jKj=bp(7RINK(oqfmX*njzo+cx|r00lD1>na#e7;*45XRyKU1pKYHZg zKWW6wg;QI)uNx5=GVcRc6z2XuDiZtNi>|HEyD*qlgN+8UF06YaRIi?}#%lP-2-}^6<~CyAB`uC55e{ zQm|kXvU=ufYI;cB%Maa6#??DfuTilXZ#{$6%O1hYnp^8EvD z|9M{jQ)s;u{nz5jz=>4rpc|m-yq=fjXO6*VJK=0=vEGTRwcXXKrb=#VzF!0z#f@k> z&!7nuI#bBi@q{zEWOZ-o)++25cKe-zYjHpSSPH}to0MyVr)70s%gN(BF9kjI1X=@K z{b-=89)i{mX14`*2pk;cG6h=%HFzpaIf^_EMYDZ0k5fTYr~+Tl?s|K)WB%BVxxv~e zj$)5Ag0~*{89aI@51Q2~AwNPq(27WdbgEG*;gSn*DYy!;Me}ts51%@eZ+>Qf^v&Kw z$@jlZrP@=qwXWoqQV(e)y$8v>LqmN#XMEUEHFw4Cna_^?{RirFuYl1{$fjV)0!D@B zlYdmYmj*uo>jqM1-;U1b;||ID~Z+g|H56`vDlzb=2h; z*S*md8dRm)aBa8;?@=5Kdc*63Lk*&)$(Vn@9Fd!#MI{*JZ)qx-1&2!MD5#$YBdUHQ zs9k0x+zXTI`tFfkgBkSr6q$y&EZ5_KcB>J*%sm+ItEXw5$V0DAuQ)d09(^kQI2pWla~`>H;NL(zMEEf5tmL`s?jq_RLuF%HhN(ZeM2QTZ(DWO*WdiL_1v=?ZH~DG zstMKfE^s}`qt@!wgS-0UMr+ahU6*WIxoGRrcRru-$`_xa+S^}yk=6ZG5S#Pb4`bXx zjGRBB{`d|^F(rxY@6#)cO zTmM^bcFru=RxSR2?ix(ZL;@s00wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr zKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{ z0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2J zBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZrKmsH{0wh2JBtQZr uKmsH{0wh2Jrzg-WS;q^BKV2OF diff --git a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json b/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json deleted file mode 100644 index 5e9a933c7bb8..000000000000 --- a/x-pack/winlogbeat/module/security/test/testdata/4912_WindowsSrv2016.evtx.golden.json +++ /dev/null @@ -1,70 +0,0 @@ -[ - { - "@timestamp": "2020-08-18T14:36:41.2936839Z", - "event": { - "action": "per-user-audit-policy-changed", - "category": [ - "iam", - "configuration" - ], - "code": 4912, - "kind": "event", - "module": "security", - "outcome": "success", - "provider": "Microsoft-Windows-Security-Auditing", - "type": [ - "admin", - "change" - ] - }, - "host": { - "name": "WIN-BVM4LI1L1Q6.TEST.local" - }, - "log": { - "level": "information" - }, - "related": { - "user": "Administrator" - }, - "user": { - "domain": "TEST", - "id": "S-1-5-21-2024912787-2692429404-2351956786-500", - "name": "Administrator" - }, - "winlog": { - "activity_id": "{65461d39-753f-0000-731d-46653f75d601}", - "api": "wineventlog", - "channel": "Security", - "computer_name": "WIN-BVM4LI1L1Q6.TEST.local", - "event_data": { - "AuditPolicyChanges": "%%8452", - "CategoryId": "%%8276", - "SubcategoryGuid": "{0cce924a-69ae-11d9-bed3-505054503030}", - "SubcategoryId": "%%13317", - "SubjectDomainName": "TEST", - "SubjectLogonId": "0x44d7d", - "SubjectUserName": "Administrator", - "SubjectUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500", - "TargetUserSid": "S-1-5-21-2024912787-2692429404-2351956786-500" - }, - "event_id": 4912, - "keywords": [ - "Audit Success" - ], - "logon": { - "id": "0x44d7d" - }, - "opcode": "Info", - "process": { - "pid": 780, - "thread": { - "id": 3300 - } - }, - "provider_guid": "{54849625-5478-4994-a5ba-3e3b0328c30d}", - "provider_name": "Microsoft-Windows-Security-Auditing", - "record_id": 123917, - "task": "Audit Policy Change" - } - } -] \ No newline at end of file From 3220d463ac619102b01ca13001d3565b7537a9c5 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 26 Jan 2021 09:13:49 +0000 Subject: [PATCH 34/35] Add check when retrieving the worker process id using performance counters (#23647) * fix for test * add check * changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/iis/application_pool/reader.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c4269a4ec666..0480d59e246f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -495,6 +495,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327] - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286] - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] +- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647] *Packetbeat* diff --git a/x-pack/metricbeat/module/iis/application_pool/reader.go b/x-pack/metricbeat/module/iis/application_pool/reader.go index 32c0c2d7ec39..d15b0a477476 100644 --- a/x-pack/metricbeat/module/iis/application_pool/reader.go +++ b/x-pack/metricbeat/module/iis/application_pool/reader.go @@ -263,7 +263,7 @@ func getw3wpProceses() (map[int]string, error) { func getProcessIds(counterValues map[string][]pdh.CounterValue) []WorkerProcess { var workers []WorkerProcess for key, values := range counterValues { - if strings.Contains(key, "\\ID Process") { + if strings.Contains(key, "\\ID Process") && values[0].Measurement != nil { workers = append(workers, WorkerProcess{instanceName: values[0].Instance, processId: int(values[0].Measurement.(float64))}) } } From 3894f0cf47ebf5257a7207ba3596026e111e0a26 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 26 Jan 2021 10:41:52 +0000 Subject: [PATCH 35/35] [CI] Mandatory windows support for all the versions (#23615) --- .ci/scripts/generate_build_table.py | 3 +- auditbeat/Jenkinsfile.yml | 36 ++++++---------- filebeat/Jenkinsfile.yml | 36 ++++++---------- heartbeat/Jenkinsfile.yml | 63 ++++++++++------------------ metricbeat/Jenkinsfile.yml | 63 ++++++++++------------------ packetbeat/Jenkinsfile.yml | 63 ++++++++++------------------ winlogbeat/Jenkinsfile.yml | 52 ++++++++--------------- x-pack/auditbeat/Jenkinsfile.yml | 49 ---------------------- x-pack/elastic-agent/Jenkinsfile.yml | 42 ------------------- x-pack/filebeat/Jenkinsfile.yml | 42 ------------------- x-pack/functionbeat/Jenkinsfile.yml | 49 ---------------------- x-pack/metricbeat/Jenkinsfile.yml | 49 ---------------------- x-pack/packetbeat/Jenkinsfile.yml | 49 ---------------------- x-pack/winlogbeat/Jenkinsfile.yml | 49 ---------------------- 14 files changed, 107 insertions(+), 538 deletions(-) diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py index 350eb3e83e3c..47e26cc17559 100755 --- a/.ci/scripts/generate_build_table.py +++ b/.ci/scripts/generate_build_table.py @@ -31,6 +31,7 @@ if "withModule" in doc["stages"][stage]: withModule = doc["stages"][stage]["withModule"] if "when" in doc["stages"][stage]: - when = f"optional" + if "not_changeset_full_match" not in doc["stages"][stage]["when"]: + when = "optional" print("| {} | {} | `{}` | {} | `{}` | {} |".format( module, stage, command, withModule, platforms, when)) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 3a022706d8cd..755449b552c7 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -72,45 +72,33 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags #windows-7: See https://github.com/elastic/beats/issues/19831 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 403b1fcc2910..1680583d1bcb 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -66,34 +66,25 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test filebeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test filebeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test filebeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags #windows-7: See https://github.com/elastic/beats/issues/22317 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. @@ -110,9 +101,6 @@ stages: platforms: ## override default labels in this specific stage. - "windows-7-32-bit" when: ## Override the top-level when. - comments: - - "/test filebeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index ac2fc59cc647..ce0488be4e33 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -64,75 +64,54 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tag + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tag + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" when: ## Override the top-level when. - comments: - - "/test heartbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 3d624fbcaaa0..588c6bbbdb80 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -71,75 +71,54 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" when: ## Override the top-level when. - comments: - - "/test metricbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 1d65795e0224..b79b87b6d883 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -64,75 +64,54 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" when: ## Override the top-level when. - comments: - - "/test packetbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index aad1d6558e9b..bcef5172022d 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -39,62 +39,46 @@ stages: platforms: ## override default labels in this specific stage. - "windows-2016" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-7" - labels: - - "windows-7" + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" when: ## Override the top-level when. - comments: - - "/test winlogbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags + not_changeset_full_match: "^x-pack/.*" ## Disable the stage if ONLY changes for the x-pack + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 99321a98cca7..98feb4541911 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -55,76 +55,27 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/auditbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/auditbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/auditbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/auditbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/auditbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" - when: ## Override the top-level when. - comments: - - "/test auditbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index a237c483aada..0e6ecef9e4cd 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -54,68 +54,26 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/elastic-agent for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags #windows-7-32: See https://github.com/elastic/beats/issues/22316 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 672d07ad2e0f..2b72ef8ec6f1 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -55,68 +55,26 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/filebeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 59d3ddf22ddd..ec9a4ec57f09 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -52,76 +52,27 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" - when: ## Override the top-level when. - comments: - - "/test x-pack/functionbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 19f9941ee47f..c070fe47c1ee 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -53,76 +53,27 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/metricbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags # windows-7-32: # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. # - "windows-7-32-bit" -# when: ## Override the top-level when. -# comments: -# - "/test x-pack/metricbeat for windows-7-32" -# labels: -# - "windows-7-32" -# branches: true ## for all the branches -# tags: true ## for all the tags diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index a668ba809ea5..9846a3f67e2a 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -55,76 +55,27 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" - when: ## Override the top-level when. - comments: - - "/test x-pack/packetbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 4153e0587e2c..5ca64db11f53 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -29,76 +29,27 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2016" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-2016" - labels: - - "windows-2016" - branches: true ## for all the branches - tags: true ## for all the tags windows-2012: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2012-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-2012" - labels: - - "windows-2012" - branches: true ## for all the branches - tags: true ## for all the tags windows-10: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-10" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-10" - labels: - - "windows-10" - branches: true ## for all the branches - tags: true ## for all the tags windows-2008: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2008-r2" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-2008" - labels: - - "windows-2008" - branches: true ## for all the branches - tags: true ## for all the tags windows-8: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-8" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-8" - labels: - - "windows-8" - branches: true ## for all the branches - tags: true ## for all the tags windows-7: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-7" - labels: - - "windows-7" - branches: true ## for all the branches - tags: true ## for all the tags windows-7-32: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-7-32-bit" - when: ## Override the top-level when. - comments: - - "/test x-pack/winlogbeat for windows-7-32" - labels: - - "windows-7-32" - branches: true ## for all the branches - tags: true ## for all the tags