From 2cdb408751d60c0206489fb02939ff0300e64501 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 5 Apr 2019 14:42:27 +0100 Subject: [PATCH 01/13] First working version --- auditbeat/helper/file/hash.go | 186 ++++++++++++++++++ .../auditbeat/module/system/process/config.go | 13 +- .../module/system/process/process.go | 28 +++ 3 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 auditbeat/helper/file/hash.go diff --git a/auditbeat/helper/file/hash.go b/auditbeat/helper/file/hash.go new file mode 100644 index 00000000000..55e137698f0 --- /dev/null +++ b/auditbeat/helper/file/hash.go @@ -0,0 +1,186 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "io" + "strings" + + "github.com/OneOfOne/xxhash" + "github.com/pkg/errors" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/sha3" + + "github.com/elastic/beats/libbeat/common/file" +) + +// HashType identifies a cryptographic algorithm. +type HashType string + +// Unpack unpacks a string to a HashType for config parsing. +func (t *HashType) Unpack(v string) error { + *t = HashType(v) + return nil +} + +var validHashes = map[HashType]struct{}{ + BLAKE2B_256: {}, + BLAKE2B_384: {}, + BLAKE2B_512: {}, + MD5: {}, + SHA1: {}, + SHA224: {}, + SHA256: {}, + SHA384: {}, + SHA512: {}, + SHA512_224: {}, + SHA512_256: {}, + SHA3_224: {}, + SHA3_256: {}, + SHA3_384: {}, + SHA3_512: {}, + XXH64: {}, +} + +// Enum of hash types. +const ( + BLAKE2B_256 HashType = "blake2b_256" + BLAKE2B_384 HashType = "blake2b_384" + BLAKE2B_512 HashType = "blake2b_512" + MD5 HashType = "md5" + SHA1 HashType = "sha1" + SHA224 HashType = "sha224" + SHA256 HashType = "sha256" + SHA384 HashType = "sha384" + SHA3_224 HashType = "sha3_224" + SHA3_256 HashType = "sha3_256" + SHA3_384 HashType = "sha3_384" + SHA3_512 HashType = "sha3_512" + SHA512 HashType = "sha512" + SHA512_224 HashType = "sha512_224" + SHA512_256 HashType = "sha512_256" + XXH64 HashType = "xxh64" +) + +// Digest is a output of a hash function. +type Digest []byte + +// String returns the digest value in lower-case hexadecimal form. +func (d Digest) String() string { + return hex.EncodeToString(d) +} + +// MarshalText encodes the digest to a hexadecimal representation of itself. +func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } + +type FileHasher struct { + hashTypes []HashType +} + +func NewFileHasher(hashTypes []HashType) (*FileHasher, error) { + hasher := FileHasher{} + + // Check hash types are valid + for _, hashType := range hashTypes { + ht := HashType(strings.ToLower(string(hashType))) + if _, valid := validHashes[ht]; !valid { + return nil, errors.Errorf("invalid hash type '%v'", ht) + } + + hasher.hashTypes = append(hasher.hashTypes, ht) + } + + return &hasher, nil +} + +func (hasher *FileHasher) HashFile(name string) (map[HashType]Digest, error) { + var hashes []hash.Hash + for _, name := range hasher.hashTypes { + switch name { + case BLAKE2B_256: + h, _ := blake2b.New256(nil) + hashes = append(hashes, h) + case BLAKE2B_384: + h, _ := blake2b.New384(nil) + hashes = append(hashes, h) + case BLAKE2B_512: + h, _ := blake2b.New512(nil) + hashes = append(hashes, h) + case MD5: + hashes = append(hashes, md5.New()) + case SHA1: + hashes = append(hashes, sha1.New()) + case SHA224: + hashes = append(hashes, sha256.New224()) + case SHA256: + hashes = append(hashes, sha256.New()) + case SHA384: + hashes = append(hashes, sha512.New384()) + case SHA3_224: + hashes = append(hashes, sha3.New224()) + case SHA3_256: + hashes = append(hashes, sha3.New256()) + case SHA3_384: + hashes = append(hashes, sha3.New384()) + case SHA3_512: + hashes = append(hashes, sha3.New512()) + case SHA512: + hashes = append(hashes, sha512.New()) + case SHA512_224: + hashes = append(hashes, sha512.New512_224()) + case SHA512_256: + hashes = append(hashes, sha512.New512_256()) + case XXH64: + hashes = append(hashes, xxhash.New64()) + default: + return nil, errors.Errorf("unknown hash type '%v'", name) + } + } + + f, err := file.ReadOpen(name) + if err != nil { + return nil, errors.Wrap(err, "failed to open file for hashing") + } + defer f.Close() + + hashWriter := multiWriter(hashes) + if _, err := io.Copy(hashWriter, f); err != nil { + return nil, errors.Wrap(err, "failed to calculate file hashes") + } + + nameToHash := make(map[HashType]Digest, len(hashes)) + for i, h := range hashes { + nameToHash[hasher.hashTypes[i]] = h.Sum(nil) + } + + return nameToHash, nil +} + +func multiWriter(hash []hash.Hash) io.Writer { + writers := make([]io.Writer, 0, len(hash)) + for _, h := range hash { + writers = append(writers, h) + } + return io.MultiWriter(writers...) +} diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index 0cdeb737ad7..68b72a13058 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -6,17 +6,15 @@ package process import ( "time" + + "github.com/elastic/beats/auditbeat/helper/file" ) // Config defines the host metricset's configuration options. type Config struct { - StatePeriod time.Duration `config:"state.period"` - ProcessStatePeriod time.Duration `config:"process.state.period"` -} - -// Validate validates the host metricset config. -func (c *Config) Validate() error { - return nil + StatePeriod time.Duration `config:"state.period"` + ProcessStatePeriod time.Duration `config:"process.state.period"` + HashTypes []file.HashType `config:"hash_types"` } func (c *Config) effectiveStatePeriod() time.Duration { @@ -28,4 +26,5 @@ func (c *Config) effectiveStatePeriod() time.Duration { var defaultConfig = Config{ StatePeriod: 12 * time.Hour, + HashTypes: []file.HashType{file.SHA1}, } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index e4a2b6a6d8d..be12b114ab1 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/auditbeat/datastore" + "github.com/elastic/beats/auditbeat/helper/file" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" @@ -80,6 +81,7 @@ type MetricSet struct { log *logp.Logger bucket datastore.Bucket lastState time.Time + hasher *file.FileHasher suppressPermissionWarnings bool } @@ -90,6 +92,7 @@ type Process struct { UserInfo *types.UserInfo User *user.User Group *user.Group + Hashes map[file.HashType]file.Digest Error error } @@ -137,12 +140,18 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "failed to open persistent datastore") } + hasher, err := file.NewFileHasher(config.HashTypes) + if err != nil { + return nil, err + } + ms := &MetricSet{ SystemMetricSet: system.NewSystemMetricSet(base), config: config, log: logp.NewLogger(metricsetName), cache: cache.New(), bucket: bucket, + hasher: hasher, } // Load from disk: Time when state was last sent @@ -310,6 +319,13 @@ func (ms *MetricSet) processEvent(process *Process, eventType string, action eve event.RootFields.Put("user.group.name", process.Group.Name) } + if process.Hashes != nil { + for hashType, digest := range process.Hashes { + fieldName := "process.hash." + string(hashType) + event.RootFields.Put(fieldName, digest) + } + } + if process.Error != nil { event.RootFields.Put("error.message", process.Error.Error()) } @@ -423,6 +439,18 @@ func (ms *MetricSet) getProcesses() ([]*Process, error) { } } + if process.Info.Exe != "" { + hashes, err := ms.hasher.HashFile(process.Info.Exe) + if err != nil { + if process.Error == nil { + process.Error = errors.Wrapf(err, "failed to hash executable %v for PID %v", process.Info.Exe, + sysinfoProc.PID()) + } + } else { + process.Hashes = hashes + } + } + // Exclude Linux kernel processes, they are not very interesting. if runtime.GOOS == "linux" && userInfo.UID == "0" && process.Info.Exe == "" { continue From 73348fc6d78a093ee89d2439a064d68663b05067 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Sat, 6 Apr 2019 11:16:15 +0100 Subject: [PATCH 02/13] Rename to hasher --- auditbeat/helper/{file/hash.go => hasher/hasher.go} | 2 +- x-pack/auditbeat/module/system/process/config.go | 10 +++++----- x-pack/auditbeat/module/system/process/process.go | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) rename auditbeat/helper/{file/hash.go => hasher/hasher.go} (99%) diff --git a/auditbeat/helper/file/hash.go b/auditbeat/helper/hasher/hasher.go similarity index 99% rename from auditbeat/helper/file/hash.go rename to auditbeat/helper/hasher/hasher.go index 55e137698f0..a8ac5d9ca73 100644 --- a/auditbeat/helper/file/hash.go +++ b/auditbeat/helper/hasher/hasher.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package file +package hasher import ( "crypto/md5" diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index 68b72a13058..b9b70aea4f2 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -7,14 +7,14 @@ package process import ( "time" - "github.com/elastic/beats/auditbeat/helper/file" + "github.com/elastic/beats/auditbeat/helper/hasher" ) // Config defines the host metricset's configuration options. type Config struct { - StatePeriod time.Duration `config:"state.period"` - ProcessStatePeriod time.Duration `config:"process.state.period"` - HashTypes []file.HashType `config:"hash_types"` + StatePeriod time.Duration `config:"state.period"` + ProcessStatePeriod time.Duration `config:"process.state.period"` + HashTypes []hasher.HashType `config:"hash_types"` } func (c *Config) effectiveStatePeriod() time.Duration { @@ -26,5 +26,5 @@ func (c *Config) effectiveStatePeriod() time.Duration { var defaultConfig = Config{ StatePeriod: 12 * time.Hour, - HashTypes: []file.HashType{file.SHA1}, + HashTypes: []hasher.HashType{hasher.SHA1}, } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index be12b114ab1..6132fb679fc 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -18,7 +18,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/auditbeat/datastore" - "github.com/elastic/beats/auditbeat/helper/file" + "github.com/elastic/beats/auditbeat/helper/hasher" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" @@ -81,7 +81,7 @@ type MetricSet struct { log *logp.Logger bucket datastore.Bucket lastState time.Time - hasher *file.FileHasher + hasher *hasher.FileHasher suppressPermissionWarnings bool } @@ -92,7 +92,7 @@ type Process struct { UserInfo *types.UserInfo User *user.User Group *user.Group - Hashes map[file.HashType]file.Digest + Hashes map[hasher.HashType]hasher.Digest Error error } @@ -140,7 +140,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "failed to open persistent datastore") } - hasher, err := file.NewFileHasher(config.HashTypes) + hasher, err := hasher.NewFileHasher(config.HashTypes) if err != nil { return nil, err } From 2bdea61b936bfbf5e006bdca122e41de79b62f5a Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Sat, 6 Apr 2019 11:53:22 +0100 Subject: [PATCH 03/13] Introduce HasherConfig --- auditbeat/helper/hasher/hasher.go | 70 +++++++++++++++---- .../auditbeat/module/system/process/config.go | 21 ++++-- .../module/system/process/process.go | 2 +- 3 files changed, 74 insertions(+), 19 deletions(-) diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go index a8ac5d9ca73..c1830212909 100644 --- a/auditbeat/helper/hasher/hasher.go +++ b/auditbeat/helper/hasher/hasher.go @@ -25,12 +25,14 @@ import ( "encoding/hex" "hash" "io" - "strings" "github.com/OneOfOne/xxhash" + "github.com/dustin/go-humanize" + "github.com/joeshaw/multierror" "github.com/pkg/errors" "golang.org/x/crypto/blake2b" "golang.org/x/crypto/sha3" + "golang.org/x/time/rate" "github.com/elastic/beats/libbeat/common/file" ) @@ -44,6 +46,12 @@ func (t *HashType) Unpack(v string) error { return nil } +// IsValid checks if the hash type is valid. +func (t *HashType) IsValid() bool { + _, valid := validHashes[*t] + return valid +} + var validHashes = map[HashType]struct{}{ BLAKE2B_256: {}, BLAKE2B_384: {}, @@ -94,29 +102,63 @@ func (d Digest) String() string { // MarshalText encodes the digest to a hexadecimal representation of itself. func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } -type FileHasher struct { - hashTypes []HashType +// Config contains the configuration of a FileHasher. +type Config struct { + HashTypes []HashType `config:"hash_types"` + MaxFileSize string `config:"max_file_size"` + MaxFileSizeBytes uint64 `config:",ignore"` + ScanRatePerSec string `config:"scan_rate_per_sec"` + ScanRateBytesPerSec uint64 `config:",ignore"` } -func NewFileHasher(hashTypes []HashType) (*FileHasher, error) { - hasher := FileHasher{} +// Validate validates the config. +func (c *Config) Validate() error { + var errs multierror.Errors - // Check hash types are valid - for _, hashType := range hashTypes { - ht := HashType(strings.ToLower(string(hashType))) - if _, valid := validHashes[ht]; !valid { - return nil, errors.Errorf("invalid hash type '%v'", ht) + for _, ht := range c.HashTypes { + if !ht.IsValid() { + errs = append(errs, errors.Errorf("invalid hash_types value '%v'", ht)) } + } + + var err error + + c.MaxFileSizeBytes, err = humanize.ParseBytes(c.MaxFileSize) + if err != nil { + errs = append(errs, errors.Wrap(err, "invalid max_file_size value")) + } else if c.MaxFileSizeBytes <= 0 { + errs = append(errs, errors.Errorf("max_file_size value (%v) must be positive", c.MaxFileSize)) + } - hasher.hashTypes = append(hasher.hashTypes, ht) + c.ScanRateBytesPerSec, err = humanize.ParseBytes(c.ScanRatePerSec) + if err != nil { + errs = append(errs, errors.Wrap(err, "invalid scan_rate_per_sec value")) } - return &hasher, nil + return errs.Err() +} + +// FileHasher hashes the contents of files. +type FileHasher struct { + config Config + limiter *rate.Limiter +} + +// NewFileHasher creates a new FileHasher. +func NewFileHasher(c Config) (*FileHasher, error) { + return &FileHasher{ + config: c, + limiter: rate.NewLimiter( + rate.Limit(c.ScanRateBytesPerSec), // Fill Rate + int(c.MaxFileSizeBytes), // Max Capacity + ), + }, nil } +// HashFile hashes the contents of a file. func (hasher *FileHasher) HashFile(name string) (map[HashType]Digest, error) { var hashes []hash.Hash - for _, name := range hasher.hashTypes { + for _, name := range hasher.config.HashTypes { switch name { case BLAKE2B_256: h, _ := blake2b.New256(nil) @@ -171,7 +213,7 @@ func (hasher *FileHasher) HashFile(name string) (map[HashType]Digest, error) { nameToHash := make(map[HashType]Digest, len(hashes)) for i, h := range hashes { - nameToHash[hasher.hashTypes[i]] = h.Sum(nil) + nameToHash[hasher.config.HashTypes[i]] = h.Sum(nil) } return nameToHash, nil diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index b9b70aea4f2..09357686acc 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -12,9 +12,15 @@ import ( // Config defines the host metricset's configuration options. type Config struct { - StatePeriod time.Duration `config:"state.period"` - ProcessStatePeriod time.Duration `config:"process.state.period"` - HashTypes []hasher.HashType `config:"hash_types"` + StatePeriod time.Duration `config:"state.period"` + ProcessStatePeriod time.Duration `config:"process.state.period"` + + HasherConfig hasher.Config `config:",inline"` +} + +// Validate validates the config. +func (c *Config) Validate() error { + return c.HasherConfig.Validate() } func (c *Config) effectiveStatePeriod() time.Duration { @@ -26,5 +32,12 @@ func (c *Config) effectiveStatePeriod() time.Duration { var defaultConfig = Config{ StatePeriod: 12 * time.Hour, - HashTypes: []hasher.HashType{hasher.SHA1}, + + HasherConfig: hasher.Config{ + HashTypes: []hasher.HashType{hasher.SHA1}, + MaxFileSize: "100 MiB", + MaxFileSizeBytes: 100 * 1024 * 1024, + ScanRatePerSec: "50 MiB", + ScanRateBytesPerSec: 50 * 1024 * 1024, + }, } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index 6132fb679fc..820291547e0 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -140,7 +140,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "failed to open persistent datastore") } - hasher, err := hasher.NewFileHasher(config.HashTypes) + hasher, err := hasher.NewFileHasher(config.HasherConfig) if err != nil { return nil, err } From 4dd25882c61ad43565f255ed54dbd0611bc0f530 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Mon, 8 Apr 2019 15:57:19 +0100 Subject: [PATCH 04/13] Add to configs --- auditbeat/helper/hasher/hasher.go | 2 +- x-pack/auditbeat/auditbeat.reference.yml | 12 ++++++++++++ x-pack/auditbeat/auditbeat.yml | 12 ++++++++++++ x-pack/auditbeat/docs/modules/system.asciidoc | 12 ++++++++++++ .../auditbeat/module/system/_meta/config.yml.tmpl | 13 +++++++++++++ x-pack/auditbeat/module/system/process/config.go | 2 +- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go index c1830212909..feae0a0aec1 100644 --- a/auditbeat/helper/hasher/hasher.go +++ b/auditbeat/helper/hasher/hasher.go @@ -104,7 +104,7 @@ func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } // Config contains the configuration of a FileHasher. type Config struct { - HashTypes []HashType `config:"hash_types"` + HashTypes []HashType `config:"hash_types,replace"` MaxFileSize string `config:"max_file_size"` MaxFileSizeBytes uint64 `config:",ignore"` ScanRatePerSec string `config:"scan_rate_per_sec"` diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index cebdd6016d1..104acf540cc 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -144,6 +144,18 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] + # File patterns of the login record files. # wtmp: History of successful logins, logouts, and system shutdowns and boots. # btmp: Failed login attempts. diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index 5a40fa8888d..013edb4b2b1 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -66,6 +66,18 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] + # File patterns of the login record files. login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp* diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc index 92e8e4f663c..fc74b7fa9f5 100644 --- a/x-pack/auditbeat/docs/modules/system.asciidoc +++ b/x-pack/auditbeat/docs/modules/system.asciidoc @@ -147,6 +147,18 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] + # File patterns of the login record files. login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp* diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index cd50d80b963..3bf603f3386 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -51,6 +51,19 @@ # detect any changes. user.detect_password_changes: true + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] + + {{ if eq .GOOS "linux" -}} # File patterns of the login record files. {{- if .Reference }} # wtmp: History of successful logins, logouts, and system shutdowns and boots. diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index 09357686acc..49def8412a5 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -15,7 +15,7 @@ type Config struct { StatePeriod time.Duration `config:"state.period"` ProcessStatePeriod time.Duration `config:"process.state.period"` - HasherConfig hasher.Config `config:",inline"` + HasherConfig hasher.Config `config:"process.hash"` } // Validate validates the config. From 1032bd9171a452d6efeb5475e39c366f9ad0cba7 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Mon, 8 Apr 2019 15:58:09 +0100 Subject: [PATCH 05/13] Changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 99e9a2f4c49..5d45c5c6640 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Auditd module: Add `event.outcome` and `event.type` for ECS. {pull}11432[11432] - Package: Enable suse. {pull}11634[11634] - Add support to the system package dataset for the SUSE OS family. {pull}11634[11634] +- Process: Hash executable. *Filebeat* From 86e99ffe0bd606a55402f6b83bd4b854af07902a Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 9 Apr 2019 13:56:38 +0100 Subject: [PATCH 06/13] Add throttling and tests --- auditbeat/helper/hasher/hasher.go | 93 +++++++++++++++---- auditbeat/helper/hasher/hasher_test.go | 92 ++++++++++++++++++ .../module/system/process/_meta/data.json | 3 + .../module/system/process/process.go | 55 ++++++----- .../module/system/process/process_test.go | 15 ++- 5 files changed, 215 insertions(+), 43 deletions(-) create mode 100644 auditbeat/helper/hasher/hasher_test.go diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go index feae0a0aec1..7a666ac2d68 100644 --- a/auditbeat/helper/hasher/hasher.go +++ b/auditbeat/helper/hasher/hasher.go @@ -23,8 +23,11 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/hex" + "fmt" "hash" "io" + "os" + "time" "github.com/OneOfOne/xxhash" "github.com/dustin/go-humanize" @@ -102,6 +105,17 @@ func (d Digest) String() string { // MarshalText encodes the digest to a hexadecimal representation of itself. func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } +// FileTooLargeError is the error that occurs when a file that +// exceeds the max file size is attempting to be hashed. +type FileTooLargeError struct { + fileSize int64 +} + +// Error returns the error message for FileTooLargeError. +func (e FileTooLargeError) Error() string { + return fmt.Sprintf("hasher: file size %d exceeds max file size", e.fileSize) +} + // Config contains the configuration of a FileHasher. type Config struct { HashTypes []HashType `config:"hash_types,replace"` @@ -142,24 +156,41 @@ func (c *Config) Validate() error { type FileHasher struct { config Config limiter *rate.Limiter + + // To cancel hashing + done <-chan struct{} } // NewFileHasher creates a new FileHasher. -func NewFileHasher(c Config) (*FileHasher, error) { +func NewFileHasher(c Config, done <-chan struct{}) (*FileHasher, error) { return &FileHasher{ config: c, limiter: rate.NewLimiter( - rate.Limit(c.ScanRateBytesPerSec), // Fill Rate - int(c.MaxFileSizeBytes), // Max Capacity + rate.Limit(c.ScanRateBytesPerSec), // Rate + int(c.MaxFileSizeBytes), // Burst ), + done: done, }, nil } // HashFile hashes the contents of a file. -func (hasher *FileHasher) HashFile(name string) (map[HashType]Digest, error) { +func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) { + info, err := os.Stat(path) + if err != nil { + return nil, errors.Wrapf(err, "failed to stat file %v", path) + } + + // Throttle reading and hashing rate. + if len(hasher.config.HashTypes) > 0 { + err = hasher.throttle(info.Size()) + if err != nil { + return nil, errors.Wrapf(err, "failed to hash file %v", path) + } + } + var hashes []hash.Hash - for _, name := range hasher.config.HashTypes { - switch name { + for _, hashType := range hasher.config.HashTypes { + switch hashType { case BLAKE2B_256: h, _ := blake2b.New256(nil) hashes = append(hashes, h) @@ -196,27 +227,53 @@ func (hasher *FileHasher) HashFile(name string) (map[HashType]Digest, error) { case XXH64: hashes = append(hashes, xxhash.New64()) default: - return nil, errors.Errorf("unknown hash type '%v'", name) + return nil, errors.Errorf("unknown hash type '%v'", hashType) } } - f, err := file.ReadOpen(name) - if err != nil { - return nil, errors.Wrap(err, "failed to open file for hashing") + if len(hashes) > 0 { + f, err := file.ReadOpen(path) + if err != nil { + return nil, errors.Wrap(err, "failed to open file for hashing") + } + defer f.Close() + + hashWriter := multiWriter(hashes) + if _, err := io.Copy(hashWriter, f); err != nil { + return nil, errors.Wrap(err, "failed to calculate file hashes") + } + + nameToHash := make(map[HashType]Digest, len(hashes)) + for i, h := range hashes { + nameToHash[hasher.config.HashTypes[i]] = h.Sum(nil) + } + + return nameToHash, nil } - defer f.Close() - hashWriter := multiWriter(hashes) - if _, err := io.Copy(hashWriter, f); err != nil { - return nil, errors.Wrap(err, "failed to calculate file hashes") + return nil, nil +} + +func (hasher *FileHasher) throttle(fileSize int64) error { + reservation := hasher.limiter.ReserveN(time.Now(), int(fileSize)) + if !reservation.OK() { + // File is bigger than the max file size + return FileTooLargeError{fileSize} } - nameToHash := make(map[HashType]Digest, len(hashes)) - for i, h := range hashes { - nameToHash[hasher.config.HashTypes[i]] = h.Sum(nil) + delay := reservation.Delay() + if delay == 0 { + return nil } - return nameToHash, nil + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-hasher.done: + case <-timer.C: + } + + return nil } func multiWriter(hash []hash.Hash) io.Writer { diff --git a/auditbeat/helper/hasher/hasher_test.go b/auditbeat/helper/hasher/hasher_test.go new file mode 100644 index 00000000000..c9d781b35a4 --- /dev/null +++ b/auditbeat/helper/hasher/hasher_test.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package hasher + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestHasher(t *testing.T) { + dir, err := ioutil.TempDir("", "auditbeat-hasher-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + file := filepath.Join(dir, "exe") + if err = ioutil.WriteFile(file, []byte("test exe\n"), 0600); err != nil { + t.Fatal(err) + } + + config := Config{ + HashTypes: []HashType{SHA1, MD5}, + MaxFileSize: "100 MiB", + MaxFileSizeBytes: 100 * 1024 * 1024, + ScanRatePerSec: "50 MiB", + ScanRateBytesPerSec: 50 * 1024 * 1024, + } + hasher, err := NewFileHasher(config, nil) + if err != nil { + t.Fatal(err) + } + + hashes, err := hasher.HashFile(file) + if err != nil { + t.Fatal(err) + } + + assert.Len(t, hashes, 2) + assert.Equal(t, "44a36f2cd27e56794cd405ad8d44e82dba4c54fa", hashes["sha1"].String()) + assert.Equal(t, "1d7572082f6b0d18a393d618285d7100", hashes["md5"].String()) +} + +func TestHasherLimits(t *testing.T) { + dir, err := ioutil.TempDir("", "auditbeat-hasher-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + file := filepath.Join(dir, "exe") + if err = ioutil.WriteFile(file, []byte("test exe\n"), 0600); err != nil { + t.Fatal(err) + } + + configZeroSize := Config{ + HashTypes: []HashType{SHA1}, + MaxFileSize: "0 MiB", + MaxFileSizeBytes: 0, + ScanRatePerSec: "0 MiB", + ScanRateBytesPerSec: 0, + } + hasher, err := NewFileHasher(configZeroSize, nil) + if err != nil { + t.Fatal(err) + } + + hashes, err := hasher.HashFile(file) + assert.Empty(t, hashes) + assert.Error(t, err) + assert.IsType(t, FileTooLargeError{}, errors.Cause(err)) +} diff --git a/x-pack/auditbeat/module/system/process/_meta/data.json b/x-pack/auditbeat/module/system/process/_meta/data.json index abd1e7d5d68..a1f1d3fb51c 100644 --- a/x-pack/auditbeat/module/system/process/_meta/data.json +++ b/x-pack/auditbeat/module/system/process/_meta/data.json @@ -13,6 +13,9 @@ ], "entity_id": "+fYshazplsMYlr0y", "executable": "/bin/zsh", + "hash": { + "sha1": "33646536613061316366353134643135613631643363383733653261373130393737633131303364" + }, "name": "zsh", "pid": 9086, "ppid": 9085, diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index 820291547e0..aff75b33bab 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -140,7 +140,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "failed to open persistent datastore") } - hasher, err := hasher.NewFileHasher(config.HasherConfig) + hasher, err := hasher.NewFileHasher(config.HasherConfig, nil) if err != nil { return nil, err } @@ -224,6 +224,8 @@ func (ms *MetricSet) reportState(report mb.ReporterV2) error { return errors.Wrap(err, "error generating state ID") } for _, p := range processes { + ms.enrichProcess(p) + if p.Error == nil { event := ms.processEvent(p, eventTypeState, eventActionExistingProcess) event.RootFields.Put("event.id", stateID.String()) @@ -264,6 +266,7 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { for _, cacheValue := range started { p := cacheValue.(*Process) + ms.enrichProcess(p) if p.Error == nil { report.Event(ms.processEvent(p, eventTypeEvent, eventActionProcessStarted)) @@ -284,6 +287,34 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { return nil } +// enrichProcess enriches a process with user lookup information +// and executable file hash. +func (ms *MetricSet) enrichProcess(process *Process) { + if process.UserInfo != nil { + goUser, err := user.LookupId(process.UserInfo.UID) + if err == nil { + process.User = goUser + } + + group, err := user.LookupGroupId(process.UserInfo.GID) + if err == nil { + process.Group = group + } + } + + if process.Info.Exe != "" { + hashes, err := ms.hasher.HashFile(process.Info.Exe) + if err != nil { + if process.Error == nil { + process.Error = errors.Wrapf(err, "failed to hash executable %v for PID %v", process.Info.Exe, + process.Info.PID) + } + } else { + process.Hashes = hashes + } + } +} + func (ms *MetricSet) processEvent(process *Process, eventType string, action eventAction) mb.Event { event := mb.Event{ RootFields: common.MapStr{ @@ -427,28 +458,6 @@ func (ms *MetricSet) getProcesses() ([]*Process, error) { } } else { process.UserInfo = &userInfo - - goUser, err := user.LookupId(userInfo.UID) - if err == nil { - process.User = goUser - } - - group, err := user.LookupGroupId(userInfo.GID) - if err == nil { - process.Group = group - } - } - - if process.Info.Exe != "" { - hashes, err := ms.hasher.HashFile(process.Info.Exe) - if err != nil { - if process.Error == nil { - process.Error = errors.Wrapf(err, "failed to hash executable %v for PID %v", process.Info.Exe, - sysinfoProc.PID()) - } - } else { - process.Hashes = hashes - } } // Exclude Linux kernel processes, they are not very interesting. diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index a0999947a11..5fa6e16a722 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/auditbeat/core" + "github.com/elastic/beats/auditbeat/helper/hasher" abtest "github.com/elastic/beats/auditbeat/testing" "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -43,8 +44,12 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", - "metricsets": []string{"process"}, + "module": "system", + "datasets": []string{"process"}, + + // To speed things up during testing, we effectively + // disable hashing. + "process.hash.max_file_size": 1, } } @@ -71,6 +76,7 @@ func TestProcessEvent(t *testing.T) { "process.executable": "/bin/zsh", "process.args": []string{"zsh"}, "process.start": "2019-01-01 00:00:01 +0000 UTC", + "process.hash.sha1": "3de6a0a1cf514d15a61d3c873e2a710977c1103d", "user.id": "1000", "user.name": "elastic", @@ -87,6 +93,8 @@ func TestProcessEvent(t *testing.T) { switch v := value.(type) { case time.Time: assert.Equalf(t, expFieldValue, v.String(), "Unexpected value for field %v.", expFieldName) + case hasher.Digest: + assert.Equalf(t, expFieldValue, string(v), "Unexpected value for field %v.", expFieldName) default: assert.Equalf(t, expFieldValue, value, "Unexpected value for field %v.", expFieldName) } @@ -121,6 +129,9 @@ func testProcess() *Process { Gid: "1000", Name: "elastic", }, + Hashes: map[hasher.HashType]hasher.Digest{ + hasher.SHA1: []byte("3de6a0a1cf514d15a61d3c873e2a710977c1103d"), + }, } } From b5e28082489028e5c785dd3b660da076798c3fc6 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 9 Apr 2019 16:31:32 +0100 Subject: [PATCH 07/13] Add fields --- auditbeat/docs/fields.asciidoc | 151 ++++++++++++++++++ .../auditbeat/module/system/_meta/fields.yml | 70 ++++++++ x-pack/auditbeat/module/system/fields.go | 2 +- .../auditbeat/tests/system/test_metricsets.py | 7 +- 4 files changed, 227 insertions(+), 3 deletions(-) diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index 085afd25158..963372f995d 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -6540,6 +6540,157 @@ type: keyword ID uniquely identifying the process. It is computed as a SHA-256 hash of the host ID, PID, and process start time. +-- + +[float] +== hash fields + +Hashes of the executable. The keys are algorithm names and the values are the hex encoded digest values. + + + +*`process.hash.blake2b_256`*:: ++ +-- +type: keyword + +BLAKE2b-256 hash of the executable. + +-- + +*`process.hash.blake2b_384`*:: ++ +-- +type: keyword + +BLAKE2b-384 hash of the executable. + +-- + +*`process.hash.blake2b_512`*:: ++ +-- +type: keyword + +BLAKE2b-512 hash of the executable. + +-- + +*`process.hash.md5`*:: ++ +-- +type: keyword + +MD5 hash of the executable. + +-- + +*`process.hash.sha1`*:: ++ +-- +type: keyword + +SHA1 hash of the executable. + +-- + +*`process.hash.sha224`*:: ++ +-- +type: keyword + +SHA224 hash of the executable. + +-- + +*`process.hash.sha256`*:: ++ +-- +type: keyword + +SHA256 hash of the executable. + +-- + +*`process.hash.sha384`*:: ++ +-- +type: keyword + +SHA384 hash of the executable. + +-- + +*`process.hash.sha3_224`*:: ++ +-- +type: keyword + +SHA3_224 hash of the executable. + +-- + +*`process.hash.sha3_256`*:: ++ +-- +type: keyword + +SHA3_256 hash of the executable. + +-- + +*`process.hash.sha3_384`*:: ++ +-- +type: keyword + +SHA3_384 hash of the executable. + +-- + +*`process.hash.sha3_512`*:: ++ +-- +type: keyword + +SHA3_512 hash of the executable. + +-- + +*`process.hash.sha512`*:: ++ +-- +type: keyword + +SHA512 hash of the executable. + +-- + +*`process.hash.sha512_224`*:: ++ +-- +type: keyword + +SHA512/224 hash of the executable. + +-- + +*`process.hash.sha512_256`*:: ++ +-- +type: keyword + +SHA512/256 hash of the executable. + +-- + +*`process.hash.xxh64`*:: ++ +-- +type: keyword + +XX64 hash of the executable. + -- diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index 4e1e82d75c1..809f9a06f6e 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -35,6 +35,76 @@ description: > ID uniquely identifying the process. It is computed as a SHA-256 hash of the host ID, PID, and process start time. + - name: hash + type: group + description: > + Hashes of the executable. The keys are algorithm names and the values are + the hex encoded digest values. + + fields: + - name: blake2b_256 + type: keyword + description: BLAKE2b-256 hash of the executable. + + - name: blake2b_384 + type: keyword + description: BLAKE2b-384 hash of the executable. + + - name: blake2b_512 + type: keyword + description: BLAKE2b-512 hash of the executable. + + - name: md5 + type: keyword + description: MD5 hash of the executable. + + - name: sha1 + type: keyword + description: SHA1 hash of the executable. + + - name: sha224 + type: keyword + description: SHA224 hash of the executable. + + - name: sha256 + type: keyword + description: SHA256 hash of the executable. + + - name: sha384 + type: keyword + description: SHA384 hash of the executable. + + - name: sha3_224 + type: keyword + description: SHA3_224 hash of the executable. + + - name: sha3_256 + type: keyword + description: SHA3_256 hash of the executable. + + - name: sha3_384 + type: keyword + description: SHA3_384 hash of the executable. + + - name: sha3_512 + type: keyword + description: SHA3_512 hash of the executable. + + - name: sha512 + type: keyword + description: SHA512 hash of the executable. + + - name: sha512_224 + type: keyword + description: SHA512/224 hash of the executable. + + - name: sha512_256 + type: keyword + description: SHA512/256 hash of the executable. + + - name: xxh64 + type: keyword + description: XX64 hash of the executable. - name: socket type: group diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index a4cbb83c01e..d64c87cdc6b 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded gzipped contents of module/system. func AssetSystem() string { - return "eJzEWd9v2zYQfvdfcehLE8BV0GErBj8MaBdgDdCuweICe7Np8SxxoXgaSSVV//qBFGVLNuUfsYoJMGBR5Pd9d8cjT9QbeMR6BqY2FosJgBVW4gxePfiGVxMAjibVorSC1Ax+mwAAzHM0CEwj2BxhLVByAxkq1Mwih1Xt2xtMKIhXEpMJgEaJzOAMVmjZBMLA2WQC8AYUK3AG+ITKeg5blziDTFNV+vu2s/vf9iYtMqF8UzvgEetn0jy0RbS764sfB7T2Oj1nAvNcGEiZghUCg7WQCCWzOVxhkiWwvHli+kZS5n7J2+X1dING2sM4SS1kMD2loiSFyoLNmQVTlaUUyH0XzixrsRVaKdTj8jrp+qIyqE92BSorbL0Q/Hxv3N1CpcS/FcoaBHdA61qozKt0GoAUMMjJ2ATuLDgvUVFWLtLMAIOHj+/f/PTLO8iZybdOaRzhRsHd7bQBcn+Y4s2N0530bLCoC6GYPN+EeRjZ0jqCni9LTSka83+7M8g47sdgyAZ048f71ocBCoxl2oIVzpkdgw2lj3h6Jv0gexsVF5jbAIBQxHEKklIm4e6+/VeStlPQWJBF3+wcE27ds75HfE4mrOIi7peIfV1Xdd3l9G0a95EO+stdSwewhJSUZUK1a6ds7BZqTbpgblzSGbW7erbXrsbOAlK6adEjbpRKUlmvuSGcAa+05+09FKqs7KLtopgigykpbnq9qLLdbszcsjrao9SYCuOd8rb3/IC/3PXVWwNCdSUkEbNXRHbAcM4snsP5gajNrX2eED3U4jvyCNmKSCJT5/A9uLm+DtPAJcmGIybACftOChN3GxHQT9+TBPzZ2cRa+O5aPgW/Y314mB8UROu1QZsYTE+ZfUc0zbc6HKqbAQei71SO54+PAS3GJGJBfyEH3N3GKJhOc2ExtZUe0aAebKhBvv36bvHu5+uYiILFovgC7s/vfwfGuUZjMBo7UUaIdhqPcNzdH6YgE6HYXbmPsCzJdNbuznINbEWV9clCpSuG3T4Y9p3+eru3ZndKFcmsQ9whHfb6UZ98ediAhminqCyZKVSrStlqCs9CcXo210lU0V46XarGl8qNks8sdS1/D1CvWSFkPSp5AxnoNfKc2SlwXAmmprDWiCvDj3nkCbXZ3SYv1RUw44SPqBXK8fjmkSn62gSafSmbucnSR5bhRaVPwDiYQUyBUMYyKZG79ytX0z0hb/kvK4t2a91jzjzoyoPVflB7dvnbXpsyOCB5A0L5H1ois2YwbV9o4n2HPMYTS4YLqQ5YFeI9JluAHNqAx6Tq7rwxPilSVONaFyCj+22TY6MUyy1dwBysmo34ftIryUlkDixKUhUF0/ULAJuBMcxKyzHD8vWvT/vr6+bIp0txzuLqAI7WJq6TaU519ouT09fTH1UgAHztnw/teUnsIl7O1q+/t1zZuFx/uFgOknGhxzbstYGcCnTQmFrqT+1OwuQoRywvAO41ZZoVYAl0pYBZkJSJgerGTchFZ66O6vFwtOLPHLtHK/BFwSehqm9TsLkwbod2yZFhSqaZ7QMzYu9loVVIq38wtecJXHq4I8VQ3ZCa7ZGsMFAybV3hcLXCmhTfPHttoNTCrWLNqJ0SNp7JcDibj0XhpEjAZv7vpzYcTLktvVAWM9zNkjPph9KvZMZEjBt6Rzwe2xbwcHg3UQu94UqRDQVkaBHWoFyfHUmn/EdF8v2ebAebwD0ZI1YS4YnJCo3/SrQ0OeP0vNj4YwDzqme0r4xdYqrmLNlj+A8z19OtbxdcGLaSyJfTAdSloi2z42iSnTOVoabK+Hpc1aTQf/6RlIFQ177MHkJMdV3aLuhzjqofMh8bp/0GbXrjmzkYxMIMgFpqZ4l7/UHlOfw7T4O4F/1O1ciMXaS5M2g4dfbKueY6Kdhz/8Gq7q0xraHPzHgBEAQkk/8CAAD//2oPi7g=" + return "eJzEWl1v2zgWffevuOhLE8BVELcJCj8skG4Wm2DbbTBOgb7ZlHgtcUKRGpJKov76ASlKlmTJthwXIyBATJPnnPtFXUr+AE9YzEEX2mA6ATDMcJzDu4UbeDcBoKgjxTLDpJjDvyYAAI8JagSiEEyCsGbIqYYYBSpikEJYuPESE1JJc47BBEAhR6JxDiEaMgG/cD6ZAHwAQVKcAz6jMI7DFBnOIVYyz9znarL9v5otFYuZcEPVgicsXqSifqxHu72+u3Ug106n4wzgMWEaIiIgRCCwZhwhIyaBMwziAFYXz0RdcBnbv+BydT6t0aRyMFZSBelNj2SaSYHCgEmIAZ1nGWdI3RRKDKmwBRrOxNPqPGj6IteoDnYFCsNMsWR0vDfubyEX7K8ceQGMWqB1wUTsVFoNIAUQSKQ2AdwbsF6SaZbbSBMNBBZ3Nx9mV9eQEJ1snFI6wq6C+9tpCWT/IYKWH6zuoGWDQZUyQfh4Ex79yorWErR8mSkZodb/tDu9jP1+9IbUoLUfHyofeijQhigDhnWd2YhG1+BB6XdEJ6jrunjFKDck5GiLA6312hU94bFUzCSpo9JOjl3wTHiObkqN6HIAXwFFJClSoCxGbfxMF6FuBDYWhJw84Sxczq6uN3g9ceiY8+Xrzf/+Mwu7rmyaMxlg+vj50zFMHz9/Gst0dTk7hunqcnYoU0qvxjB8u706FFkn5HIM9OLu5nIE9mw2KgiLu5vZ7GD/W/xx6WTxD88knZCRSbS4uxmRPxZ/Od5Dbs04jtFecmtGcRzhqeVYX40sNMcxosp0QsYzjMY/IuJXl7OLcTF3PKOj7ngOj/vra3I9ypSfP693GlEbIKMnPLxz/E3391LFG27vJQAwISlOgcuIcLh/qP7LpDJTUJhKg27Y3nn9R/td2yOuBw1ITlm/X3rsa9+IG82E1KYe7GsodvjLXisLsIJICkOYqM4KvLSbibVUKbHrgsaq7mmhuroaGw1zZtugFnGplEsRt4ZLwjnQXDne1pdMZLlZVlMEEVJjJAXVrVkyN81pRN+SondGpjBi2jnlsvX9Dn/Z64ezBphoSgh6zA6lNAOGU2JwDOcXKZu9ZJvHRw8V+4W0hyyUkiMRY/gWNtfXPg1skdQcfQKssF9SYGA/9gjobiQHCPh/49BWwTfPLlNwJ7Qvi8edguR6rdEEGqNDsm+PpseNDotqM2BH9K3K0/njzqP1MbG+oB/JAfe3fRRERQkzGJlcndCgFqw/c79+vl5efzrvE5GSvigewf3t5t9AKFWoNfbGjmU9RJ3BPRz3D7sppO6h6O7ce1hWUjf27sZ2DSSUuXHFIjO0W6mIq/tOe7/d2rMbR3NOjEXskA57fa9Pvi9qUB/tCIWRegp5mAuTT+GFCSpf9HnQq2irnN6qxj0aKpV8I5Ed+TlAvSYp48VJyUtIT6+QJsRMgWLIiJjCWiGGmu7zyDMq3b1NvlWXx+wnfEIlkJ+O77EnRd9rT7Mtpc5NEj2RGN/U+niMnRVEBDChDeEcKUjlerpnpBX/29qibq+7z5k7Xbnz6ZZXO7r9ra66DfZIzgD/uMuP9GTNYNkeaeJDg7yPp68Y3ki1wyof71OyecihG/ApqZp33j4+ziIUp7XOQ/beb8saO0mzXNF5zMGuWbNfBx1JDiKzYL0keZoSVRwBWC7sw8wVP2VYfvzxdXt/rV9xNCnGbK4WYG9vYifp8i3GdnNy+H76uxoEgB/t9yFbXmJdxLeztfvvDVd8Wq7/2lgOklGmTm3Yew2JTNFCY2RkO7Wbz7uQn7C9AHhQMlYkBSNB5QKIAS5jNtDd2IRcNnL1pB73j1bcO7bmoxX4LuArE/nrFEzCtL1D2+KIMZK6zPaBjNg6LFQKZfgnRmacwJWD29MMFSWp3ryCZBoyooxtHM5CLKR/45SXEc8Us7tYuarTwvZXMuyu5n1ROCgSUOf/dmnDzpLb0DNhMMZulYykHyq/jGjdY9zQGXF/bCvA3eGto+Znw5mQxjeQfoQZjXw9OpJW+e+K5M2WbAsbwIPUmoW8+fYTVjohVL4sa38MYJ61jHadsS1MUT5Ldhjuhwjn041vl5RpEnKkq+kA6krIDbPlKIudEhGjkrl2/bgopED3cwcuY2Di3LXZQ4iRKjLTBH1JULRD5mJjtV+giS7cMAWNmOoBUCOrLLHHHxSOw515SsSt6De6RqLNMkqsQcOls9XOlddBwX50P9AoWntMZegL0U4AeAHB5O8AAAD//1pphx4=" } diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index a4956f77aa6..f4eaec2bcd1 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -67,8 +67,11 @@ def test_metricset_process(self): fields.extend(["user.effective.id", "user.saved.id", "user.effective.group.id", "user.saved.group.id", "user.name", "user.group.name"]) - # Metricset is beta and that generates a warning, TODO: remove later - self.check_metricset("system", "process", COMMON_FIELDS + fields, warnings_allowed=True) + # process.hash.max_file_size: 1 - To speed things up during testing, we effectively disable hashing. + # errors_allowed|warnings_allowed=True - Disabling hashing causes the dataset to add an error to the event + # and log a warning. That should not fail the test. + self.check_metricset("system", "process", COMMON_FIELDS + fields, {"process.hash.max_file_size": 1}, + errors_allowed=True, warnings_allowed=True) @unittest.skipUnless(sys.platform == "linux2", "Only implemented for Linux") def test_metricset_socket(self): From 29c67221e86317df4075c730f8e6b010f93393d7 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 9 Apr 2019 16:32:09 +0100 Subject: [PATCH 08/13] Changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5d45c5c6640..c224db19417 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,7 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Auditd module: Add `event.outcome` and `event.type` for ECS. {pull}11432[11432] - Package: Enable suse. {pull}11634[11634] - Add support to the system package dataset for the SUSE OS family. {pull}11634[11634] -- Process: Hash executable. +- Process: Hash executable. {pull}11722[11722] *Filebeat* From 0773318a9c692a8255370eb06c9ae65e2b12e8ee Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 1 May 2019 17:07:28 +0100 Subject: [PATCH 09/13] Changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c224db19417..19f95c2aed6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,7 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Auditd module: Add `event.outcome` and `event.type` for ECS. {pull}11432[11432] - Package: Enable suse. {pull}11634[11634] - Add support to the system package dataset for the SUSE OS family. {pull}11634[11634] -- Process: Hash executable. {pull}11722[11722] +- Process: Add file hash of process executable. {pull}11722[11722] *Filebeat* From f4890e6bda50a230c812044a5266466cf9928a00 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Wed, 1 May 2019 17:33:15 +0100 Subject: [PATCH 10/13] Lowercase hash type config user input. --- auditbeat/helper/hasher/hasher.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go index 7a666ac2d68..e7080db52a2 100644 --- a/auditbeat/helper/hasher/hasher.go +++ b/auditbeat/helper/hasher/hasher.go @@ -27,6 +27,7 @@ import ( "hash" "io" "os" + "strings" "time" "github.com/OneOfOne/xxhash" @@ -45,7 +46,7 @@ type HashType string // Unpack unpacks a string to a HashType for config parsing. func (t *HashType) Unpack(v string) error { - *t = HashType(v) + *t = HashType(strings.ToLower(v)) return nil } From 5b5b25a4194b504c023d7ad7eabb8be1e1678ea3 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Thu, 2 May 2019 11:57:57 +0100 Subject: [PATCH 11/13] Simplify hash construction. --- auditbeat/helper/hasher/hasher.go | 86 ++++++++++++------------------- 1 file changed, 32 insertions(+), 54 deletions(-) diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go index e7080db52a2..8f9cc25ab11 100644 --- a/auditbeat/helper/hasher/hasher.go +++ b/auditbeat/helper/hasher/hasher.go @@ -56,23 +56,34 @@ func (t *HashType) IsValid() bool { return valid } -var validHashes = map[HashType]struct{}{ - BLAKE2B_256: {}, - BLAKE2B_384: {}, - BLAKE2B_512: {}, - MD5: {}, - SHA1: {}, - SHA224: {}, - SHA256: {}, - SHA384: {}, - SHA512: {}, - SHA512_224: {}, - SHA512_256: {}, - SHA3_224: {}, - SHA3_256: {}, - SHA3_384: {}, - SHA3_512: {}, - XXH64: {}, +var validHashes = map[HashType](func() hash.Hash){ + BLAKE2B_256: func() hash.Hash { + h, _ := blake2b.New256(nil) + return h + }, + BLAKE2B_384: func() hash.Hash { + h, _ := blake2b.New384(nil) + return h + }, + BLAKE2B_512: func() hash.Hash { + h, _ := blake2b.New512(nil) + return h + }, + MD5: md5.New, + SHA1: sha1.New, + SHA224: sha256.New224, + SHA256: sha256.New, + SHA384: sha512.New384, + SHA512: sha512.New, + SHA512_224: sha512.New512_224, + SHA512_256: sha512.New512_256, + SHA3_224: sha3.New224, + SHA3_256: sha3.New256, + SHA3_384: sha3.New384, + SHA3_512: sha3.New512, + XXH64: func() hash.Hash { + return xxhash.New64() + }, } // Enum of hash types. @@ -191,45 +202,12 @@ func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) { var hashes []hash.Hash for _, hashType := range hasher.config.HashTypes { - switch hashType { - case BLAKE2B_256: - h, _ := blake2b.New256(nil) - hashes = append(hashes, h) - case BLAKE2B_384: - h, _ := blake2b.New384(nil) - hashes = append(hashes, h) - case BLAKE2B_512: - h, _ := blake2b.New512(nil) - hashes = append(hashes, h) - case MD5: - hashes = append(hashes, md5.New()) - case SHA1: - hashes = append(hashes, sha1.New()) - case SHA224: - hashes = append(hashes, sha256.New224()) - case SHA256: - hashes = append(hashes, sha256.New()) - case SHA384: - hashes = append(hashes, sha512.New384()) - case SHA3_224: - hashes = append(hashes, sha3.New224()) - case SHA3_256: - hashes = append(hashes, sha3.New256()) - case SHA3_384: - hashes = append(hashes, sha3.New384()) - case SHA3_512: - hashes = append(hashes, sha3.New512()) - case SHA512: - hashes = append(hashes, sha512.New()) - case SHA512_224: - hashes = append(hashes, sha512.New512_224()) - case SHA512_256: - hashes = append(hashes, sha512.New512_256()) - case XXH64: - hashes = append(hashes, xxhash.New64()) - default: + h, valid := validHashes[hashType] + if !valid { return nil, errors.Errorf("unknown hash type '%v'", hashType) } + + hashes = append(hashes, h()) } if len(hashes) > 0 { From 05f4c5678a0a19826d90b1068dc2f7985962c6db Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Thu, 2 May 2019 12:10:49 +0100 Subject: [PATCH 12/13] Move hash config to reference yml. --- x-pack/auditbeat/auditbeat.reference.yml | 18 ++++++------- x-pack/auditbeat/auditbeat.yml | 12 --------- .../module/system/_meta/config.yml.tmpl | 27 +++++++++---------- 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 104acf540cc..cefda6f5f7c 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -135,15 +135,6 @@ auditbeat.modules: # socket.state.period: 12h # user.state.period: 12h - # Disabled by default. If enabled, the socket dataset will - # report sockets to and from localhost. - # socket.include_localhost: false - - # Enabled by default. Auditbeat will read password fields in - # /etc/passwd and /etc/shadow and store a hash locally to - # detect any changes. - user.detect_password_changes: true - # Average file read rate for hashing of the process executable. Default is "50 MiB". process.hash.scan_rate_per_sec: 50 MiB @@ -156,6 +147,15 @@ auditbeat.modules: # Default is sha1. process.hash.hash_types: [sha1] + # Disabled by default. If enabled, the socket dataset will + # report sockets to and from localhost. + # socket.include_localhost: false + + # Enabled by default. Auditbeat will read password fields in + # /etc/passwd and /etc/shadow and store a hash locally to + # detect any changes. + user.detect_password_changes: true + # File patterns of the login record files. # wtmp: History of successful logins, logouts, and system shutdowns and boots. # btmp: Failed login attempts. diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index 013edb4b2b1..5a40fa8888d 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -66,18 +66,6 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true - # Average file read rate for hashing of the process executable. Default is "50 MiB". - process.hash.scan_rate_per_sec: 50 MiB - - # Limit on the size of the process executable that will be hashed. Default is "100 MiB". - process.hash.max_file_size: 100 MiB - - # Hash types to compute of the process executable. Supported types are - # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, - # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. - # Default is sha1. - process.hash.hash_types: [sha1] - # File patterns of the login record files. login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp* diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index 3bf603f3386..47583a5bd52 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -35,7 +35,19 @@ {{ if eq .GOOS "linux" -}} # socket.state.period: 12h # user.state.period: 12h - {{- end -}} + {{- end }} + + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] {{- end -}} {{- if eq .GOOS "linux" -}} @@ -51,19 +63,6 @@ # detect any changes. user.detect_password_changes: true - # Average file read rate for hashing of the process executable. Default is "50 MiB". - process.hash.scan_rate_per_sec: 50 MiB - - # Limit on the size of the process executable that will be hashed. Default is "100 MiB". - process.hash.max_file_size: 100 MiB - - # Hash types to compute of the process executable. Supported types are - # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, - # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. - # Default is sha1. - process.hash.hash_types: [sha1] - - {{ if eq .GOOS "linux" -}} # File patterns of the login record files. {{- if .Reference }} # wtmp: History of successful logins, logouts, and system shutdowns and boots. From f37995b787646801b2e152ab737fcc0f0447d0c0 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Thu, 2 May 2019 12:16:57 +0100 Subject: [PATCH 13/13] mage update --- x-pack/auditbeat/docs/modules/system.asciidoc | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc index fc74b7fa9f5..92e8e4f663c 100644 --- a/x-pack/auditbeat/docs/modules/system.asciidoc +++ b/x-pack/auditbeat/docs/modules/system.asciidoc @@ -147,18 +147,6 @@ auditbeat.modules: # detect any changes. user.detect_password_changes: true - # Average file read rate for hashing of the process executable. Default is "50 MiB". - process.hash.scan_rate_per_sec: 50 MiB - - # Limit on the size of the process executable that will be hashed. Default is "100 MiB". - process.hash.max_file_size: 100 MiB - - # Hash types to compute of the process executable. Supported types are - # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, - # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. - # Default is sha1. - process.hash.hash_types: [sha1] - # File patterns of the login record files. login.wtmp_file_pattern: /var/log/wtmp* login.btmp_file_pattern: /var/log/btmp*