diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 99e9a2f4c49b..19f95c2aed61 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Auditd module: Add `event.outcome` and `event.type` for ECS. {pull}11432[11432] - Package: Enable suse. {pull}11634[11634] - Add support to the system package dataset for the SUSE OS family. {pull}11634[11634] +- Process: Add file hash of process executable. {pull}11722[11722] *Filebeat* diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index 085afd251589..963372f995dd 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -6540,6 +6540,157 @@ type: keyword ID uniquely identifying the process. It is computed as a SHA-256 hash of the host ID, PID, and process start time. +-- + +[float] +== hash fields + +Hashes of the executable. The keys are algorithm names and the values are the hex encoded digest values. + + + +*`process.hash.blake2b_256`*:: ++ +-- +type: keyword + +BLAKE2b-256 hash of the executable. + +-- + +*`process.hash.blake2b_384`*:: ++ +-- +type: keyword + +BLAKE2b-384 hash of the executable. + +-- + +*`process.hash.blake2b_512`*:: ++ +-- +type: keyword + +BLAKE2b-512 hash of the executable. + +-- + +*`process.hash.md5`*:: ++ +-- +type: keyword + +MD5 hash of the executable. + +-- + +*`process.hash.sha1`*:: ++ +-- +type: keyword + +SHA1 hash of the executable. + +-- + +*`process.hash.sha224`*:: ++ +-- +type: keyword + +SHA224 hash of the executable. + +-- + +*`process.hash.sha256`*:: ++ +-- +type: keyword + +SHA256 hash of the executable. + +-- + +*`process.hash.sha384`*:: ++ +-- +type: keyword + +SHA384 hash of the executable. + +-- + +*`process.hash.sha3_224`*:: ++ +-- +type: keyword + +SHA3_224 hash of the executable. + +-- + +*`process.hash.sha3_256`*:: ++ +-- +type: keyword + +SHA3_256 hash of the executable. + +-- + +*`process.hash.sha3_384`*:: ++ +-- +type: keyword + +SHA3_384 hash of the executable. + +-- + +*`process.hash.sha3_512`*:: ++ +-- +type: keyword + +SHA3_512 hash of the executable. + +-- + +*`process.hash.sha512`*:: ++ +-- +type: keyword + +SHA512 hash of the executable. + +-- + +*`process.hash.sha512_224`*:: ++ +-- +type: keyword + +SHA512/224 hash of the executable. + +-- + +*`process.hash.sha512_256`*:: ++ +-- +type: keyword + +SHA512/256 hash of the executable. + +-- + +*`process.hash.xxh64`*:: ++ +-- +type: keyword + +XX64 hash of the executable. + -- diff --git a/auditbeat/helper/hasher/hasher.go b/auditbeat/helper/hasher/hasher.go new file mode 100644 index 000000000000..8f9cc25ab113 --- /dev/null +++ b/auditbeat/helper/hasher/hasher.go @@ -0,0 +1,264 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package hasher + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "fmt" + "hash" + "io" + "os" + "strings" + "time" + + "github.com/OneOfOne/xxhash" + "github.com/dustin/go-humanize" + "github.com/joeshaw/multierror" + "github.com/pkg/errors" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/sha3" + "golang.org/x/time/rate" + + "github.com/elastic/beats/libbeat/common/file" +) + +// HashType identifies a cryptographic algorithm. +type HashType string + +// Unpack unpacks a string to a HashType for config parsing. +func (t *HashType) Unpack(v string) error { + *t = HashType(strings.ToLower(v)) + return nil +} + +// IsValid checks if the hash type is valid. +func (t *HashType) IsValid() bool { + _, valid := validHashes[*t] + return valid +} + +var validHashes = map[HashType](func() hash.Hash){ + BLAKE2B_256: func() hash.Hash { + h, _ := blake2b.New256(nil) + return h + }, + BLAKE2B_384: func() hash.Hash { + h, _ := blake2b.New384(nil) + return h + }, + BLAKE2B_512: func() hash.Hash { + h, _ := blake2b.New512(nil) + return h + }, + MD5: md5.New, + SHA1: sha1.New, + SHA224: sha256.New224, + SHA256: sha256.New, + SHA384: sha512.New384, + SHA512: sha512.New, + SHA512_224: sha512.New512_224, + SHA512_256: sha512.New512_256, + SHA3_224: sha3.New224, + SHA3_256: sha3.New256, + SHA3_384: sha3.New384, + SHA3_512: sha3.New512, + XXH64: func() hash.Hash { + return xxhash.New64() + }, +} + +// Enum of hash types. +const ( + BLAKE2B_256 HashType = "blake2b_256" + BLAKE2B_384 HashType = "blake2b_384" + BLAKE2B_512 HashType = "blake2b_512" + MD5 HashType = "md5" + SHA1 HashType = "sha1" + SHA224 HashType = "sha224" + SHA256 HashType = "sha256" + SHA384 HashType = "sha384" + SHA3_224 HashType = "sha3_224" + SHA3_256 HashType = "sha3_256" + SHA3_384 HashType = "sha3_384" + SHA3_512 HashType = "sha3_512" + SHA512 HashType = "sha512" + SHA512_224 HashType = "sha512_224" + SHA512_256 HashType = "sha512_256" + XXH64 HashType = "xxh64" +) + +// Digest is a output of a hash function. +type Digest []byte + +// String returns the digest value in lower-case hexadecimal form. +func (d Digest) String() string { + return hex.EncodeToString(d) +} + +// MarshalText encodes the digest to a hexadecimal representation of itself. +func (d Digest) MarshalText() ([]byte, error) { return []byte(d.String()), nil } + +// FileTooLargeError is the error that occurs when a file that +// exceeds the max file size is attempting to be hashed. +type FileTooLargeError struct { + fileSize int64 +} + +// Error returns the error message for FileTooLargeError. +func (e FileTooLargeError) Error() string { + return fmt.Sprintf("hasher: file size %d exceeds max file size", e.fileSize) +} + +// Config contains the configuration of a FileHasher. +type Config struct { + HashTypes []HashType `config:"hash_types,replace"` + MaxFileSize string `config:"max_file_size"` + MaxFileSizeBytes uint64 `config:",ignore"` + ScanRatePerSec string `config:"scan_rate_per_sec"` + ScanRateBytesPerSec uint64 `config:",ignore"` +} + +// Validate validates the config. +func (c *Config) Validate() error { + var errs multierror.Errors + + for _, ht := range c.HashTypes { + if !ht.IsValid() { + errs = append(errs, errors.Errorf("invalid hash_types value '%v'", ht)) + } + } + + var err error + + c.MaxFileSizeBytes, err = humanize.ParseBytes(c.MaxFileSize) + if err != nil { + errs = append(errs, errors.Wrap(err, "invalid max_file_size value")) + } else if c.MaxFileSizeBytes <= 0 { + errs = append(errs, errors.Errorf("max_file_size value (%v) must be positive", c.MaxFileSize)) + } + + c.ScanRateBytesPerSec, err = humanize.ParseBytes(c.ScanRatePerSec) + if err != nil { + errs = append(errs, errors.Wrap(err, "invalid scan_rate_per_sec value")) + } + + return errs.Err() +} + +// FileHasher hashes the contents of files. +type FileHasher struct { + config Config + limiter *rate.Limiter + + // To cancel hashing + done <-chan struct{} +} + +// NewFileHasher creates a new FileHasher. +func NewFileHasher(c Config, done <-chan struct{}) (*FileHasher, error) { + return &FileHasher{ + config: c, + limiter: rate.NewLimiter( + rate.Limit(c.ScanRateBytesPerSec), // Rate + int(c.MaxFileSizeBytes), // Burst + ), + done: done, + }, nil +} + +// HashFile hashes the contents of a file. +func (hasher *FileHasher) HashFile(path string) (map[HashType]Digest, error) { + info, err := os.Stat(path) + if err != nil { + return nil, errors.Wrapf(err, "failed to stat file %v", path) + } + + // Throttle reading and hashing rate. + if len(hasher.config.HashTypes) > 0 { + err = hasher.throttle(info.Size()) + if err != nil { + return nil, errors.Wrapf(err, "failed to hash file %v", path) + } + } + + var hashes []hash.Hash + for _, hashType := range hasher.config.HashTypes { + h, valid := validHashes[hashType] + if !valid { + return nil, errors.Errorf("unknown hash type '%v'", hashType) + } + + hashes = append(hashes, h()) + } + + if len(hashes) > 0 { + f, err := file.ReadOpen(path) + if err != nil { + return nil, errors.Wrap(err, "failed to open file for hashing") + } + defer f.Close() + + hashWriter := multiWriter(hashes) + if _, err := io.Copy(hashWriter, f); err != nil { + return nil, errors.Wrap(err, "failed to calculate file hashes") + } + + nameToHash := make(map[HashType]Digest, len(hashes)) + for i, h := range hashes { + nameToHash[hasher.config.HashTypes[i]] = h.Sum(nil) + } + + return nameToHash, nil + } + + return nil, nil +} + +func (hasher *FileHasher) throttle(fileSize int64) error { + reservation := hasher.limiter.ReserveN(time.Now(), int(fileSize)) + if !reservation.OK() { + // File is bigger than the max file size + return FileTooLargeError{fileSize} + } + + delay := reservation.Delay() + if delay == 0 { + return nil + } + + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-hasher.done: + case <-timer.C: + } + + return nil +} + +func multiWriter(hash []hash.Hash) io.Writer { + writers := make([]io.Writer, 0, len(hash)) + for _, h := range hash { + writers = append(writers, h) + } + return io.MultiWriter(writers...) +} diff --git a/auditbeat/helper/hasher/hasher_test.go b/auditbeat/helper/hasher/hasher_test.go new file mode 100644 index 000000000000..c9d781b35a42 --- /dev/null +++ b/auditbeat/helper/hasher/hasher_test.go @@ -0,0 +1,92 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package hasher + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestHasher(t *testing.T) { + dir, err := ioutil.TempDir("", "auditbeat-hasher-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + file := filepath.Join(dir, "exe") + if err = ioutil.WriteFile(file, []byte("test exe\n"), 0600); err != nil { + t.Fatal(err) + } + + config := Config{ + HashTypes: []HashType{SHA1, MD5}, + MaxFileSize: "100 MiB", + MaxFileSizeBytes: 100 * 1024 * 1024, + ScanRatePerSec: "50 MiB", + ScanRateBytesPerSec: 50 * 1024 * 1024, + } + hasher, err := NewFileHasher(config, nil) + if err != nil { + t.Fatal(err) + } + + hashes, err := hasher.HashFile(file) + if err != nil { + t.Fatal(err) + } + + assert.Len(t, hashes, 2) + assert.Equal(t, "44a36f2cd27e56794cd405ad8d44e82dba4c54fa", hashes["sha1"].String()) + assert.Equal(t, "1d7572082f6b0d18a393d618285d7100", hashes["md5"].String()) +} + +func TestHasherLimits(t *testing.T) { + dir, err := ioutil.TempDir("", "auditbeat-hasher-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + file := filepath.Join(dir, "exe") + if err = ioutil.WriteFile(file, []byte("test exe\n"), 0600); err != nil { + t.Fatal(err) + } + + configZeroSize := Config{ + HashTypes: []HashType{SHA1}, + MaxFileSize: "0 MiB", + MaxFileSizeBytes: 0, + ScanRatePerSec: "0 MiB", + ScanRateBytesPerSec: 0, + } + hasher, err := NewFileHasher(configZeroSize, nil) + if err != nil { + t.Fatal(err) + } + + hashes, err := hasher.HashFile(file) + assert.Empty(t, hashes) + assert.Error(t, err) + assert.IsType(t, FileTooLargeError{}, errors.Cause(err)) +} diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index cebdd6016d1b..cefda6f5f7c4 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -135,6 +135,18 @@ auditbeat.modules: # socket.state.period: 12h # user.state.period: 12h + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] + # Disabled by default. If enabled, the socket dataset will # report sockets to and from localhost. # socket.include_localhost: false diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index cd50d80b963a..47583a5bd52d 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -35,7 +35,19 @@ {{ if eq .GOOS "linux" -}} # socket.state.period: 12h # user.state.period: 12h - {{- end -}} + {{- end }} + + # Average file read rate for hashing of the process executable. Default is "50 MiB". + process.hash.scan_rate_per_sec: 50 MiB + + # Limit on the size of the process executable that will be hashed. Default is "100 MiB". + process.hash.max_file_size: 100 MiB + + # Hash types to compute of the process executable. Supported types are + # blake2b_256, blake2b_384, blake2b_512, md5, sha1, sha224, sha256, sha384, + # sha512, sha512_224, sha512_256, sha3_224, sha3_256, sha3_384, sha3_512, and xxh64. + # Default is sha1. + process.hash.hash_types: [sha1] {{- end -}} {{- if eq .GOOS "linux" -}} diff --git a/x-pack/auditbeat/module/system/_meta/fields.yml b/x-pack/auditbeat/module/system/_meta/fields.yml index 4e1e82d75c19..809f9a06f6ee 100644 --- a/x-pack/auditbeat/module/system/_meta/fields.yml +++ b/x-pack/auditbeat/module/system/_meta/fields.yml @@ -35,6 +35,76 @@ description: > ID uniquely identifying the process. It is computed as a SHA-256 hash of the host ID, PID, and process start time. + - name: hash + type: group + description: > + Hashes of the executable. The keys are algorithm names and the values are + the hex encoded digest values. + + fields: + - name: blake2b_256 + type: keyword + description: BLAKE2b-256 hash of the executable. + + - name: blake2b_384 + type: keyword + description: BLAKE2b-384 hash of the executable. + + - name: blake2b_512 + type: keyword + description: BLAKE2b-512 hash of the executable. + + - name: md5 + type: keyword + description: MD5 hash of the executable. + + - name: sha1 + type: keyword + description: SHA1 hash of the executable. + + - name: sha224 + type: keyword + description: SHA224 hash of the executable. + + - name: sha256 + type: keyword + description: SHA256 hash of the executable. + + - name: sha384 + type: keyword + description: SHA384 hash of the executable. + + - name: sha3_224 + type: keyword + description: SHA3_224 hash of the executable. + + - name: sha3_256 + type: keyword + description: SHA3_256 hash of the executable. + + - name: sha3_384 + type: keyword + description: SHA3_384 hash of the executable. + + - name: sha3_512 + type: keyword + description: SHA3_512 hash of the executable. + + - name: sha512 + type: keyword + description: SHA512 hash of the executable. + + - name: sha512_224 + type: keyword + description: SHA512/224 hash of the executable. + + - name: sha512_256 + type: keyword + description: SHA512/256 hash of the executable. + + - name: xxh64 + type: keyword + description: XX64 hash of the executable. - name: socket type: group diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index a4cbb83c01e9..d64c87cdc6bc 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded gzipped contents of module/system. func AssetSystem() string { - return "eJzEWd9v2zYQfvdfcehLE8BV0GErBj8MaBdgDdCuweICe7Np8SxxoXgaSSVV//qBFGVLNuUfsYoJMGBR5Pd9d8cjT9QbeMR6BqY2FosJgBVW4gxePfiGVxMAjibVorSC1Ax+mwAAzHM0CEwj2BxhLVByAxkq1Mwih1Xt2xtMKIhXEpMJgEaJzOAMVmjZBMLA2WQC8AYUK3AG+ITKeg5blziDTFNV+vu2s/vf9iYtMqF8UzvgEetn0jy0RbS764sfB7T2Oj1nAvNcGEiZghUCg7WQCCWzOVxhkiWwvHli+kZS5n7J2+X1dING2sM4SS1kMD2loiSFyoLNmQVTlaUUyH0XzixrsRVaKdTj8jrp+qIyqE92BSorbL0Q/Hxv3N1CpcS/FcoaBHdA61qozKt0GoAUMMjJ2ATuLDgvUVFWLtLMAIOHj+/f/PTLO8iZybdOaRzhRsHd7bQBcn+Y4s2N0530bLCoC6GYPN+EeRjZ0jqCni9LTSka83+7M8g47sdgyAZ048f71ocBCoxl2oIVzpkdgw2lj3h6Jv0gexsVF5jbAIBQxHEKklIm4e6+/VeStlPQWJBF3+wcE27ds75HfE4mrOIi7peIfV1Xdd3l9G0a95EO+stdSwewhJSUZUK1a6ds7BZqTbpgblzSGbW7erbXrsbOAlK6adEjbpRKUlmvuSGcAa+05+09FKqs7KLtopgigykpbnq9qLLdbszcsjrao9SYCuOd8rb3/IC/3PXVWwNCdSUkEbNXRHbAcM4snsP5gajNrX2eED3U4jvyCNmKSCJT5/A9uLm+DtPAJcmGIybACftOChN3GxHQT9+TBPzZ2cRa+O5aPgW/Y314mB8UROu1QZsYTE+ZfUc0zbc6HKqbAQei71SO54+PAS3GJGJBfyEH3N3GKJhOc2ExtZUe0aAebKhBvv36bvHu5+uYiILFovgC7s/vfwfGuUZjMBo7UUaIdhqPcNzdH6YgE6HYXbmPsCzJdNbuznINbEWV9clCpSuG3T4Y9p3+eru3ZndKFcmsQ9whHfb6UZ98ediAhminqCyZKVSrStlqCs9CcXo210lU0V46XarGl8qNks8sdS1/D1CvWSFkPSp5AxnoNfKc2SlwXAmmprDWiCvDj3nkCbXZ3SYv1RUw44SPqBXK8fjmkSn62gSafSmbucnSR5bhRaVPwDiYQUyBUMYyKZG79ytX0z0hb/kvK4t2a91jzjzoyoPVflB7dvnbXpsyOCB5A0L5H1ois2YwbV9o4n2HPMYTS4YLqQ5YFeI9JluAHNqAx6Tq7rwxPilSVONaFyCj+22TY6MUyy1dwBysmo34ftIryUlkDixKUhUF0/ULAJuBMcxKyzHD8vWvT/vr6+bIp0txzuLqAI7WJq6TaU519ouT09fTH1UgAHztnw/teUnsIl7O1q+/t1zZuFx/uFgOknGhxzbstYGcCnTQmFrqT+1OwuQoRywvAO41ZZoVYAl0pYBZkJSJgerGTchFZ66O6vFwtOLPHLtHK/BFwSehqm9TsLkwbod2yZFhSqaZ7QMzYu9loVVIq38wtecJXHq4I8VQ3ZCa7ZGsMFAybV3hcLXCmhTfPHttoNTCrWLNqJ0SNp7JcDibj0XhpEjAZv7vpzYcTLktvVAWM9zNkjPph9KvZMZEjBt6Rzwe2xbwcHg3UQu94UqRDQVkaBHWoFyfHUmn/EdF8v2ebAebwD0ZI1YS4YnJCo3/SrQ0OeP0vNj4YwDzqme0r4xdYqrmLNlj+A8z19OtbxdcGLaSyJfTAdSloi2z42iSnTOVoabK+Hpc1aTQf/6RlIFQ177MHkJMdV3aLuhzjqofMh8bp/0GbXrjmzkYxMIMgFpqZ4l7/UHlOfw7T4O4F/1O1ciMXaS5M2g4dfbKueY6Kdhz/8Gq7q0xraHPzHgBEAQkk/8CAAD//2oPi7g=" + return "eJzEWl1v2zgWffevuOhLE8BVELcJCj8skG4Wm2DbbTBOgb7ZlHgtcUKRGpJKov76ASlKlmTJthwXIyBATJPnnPtFXUr+AE9YzEEX2mA6ATDMcJzDu4UbeDcBoKgjxTLDpJjDvyYAAI8JagSiEEyCsGbIqYYYBSpikEJYuPESE1JJc47BBEAhR6JxDiEaMgG/cD6ZAHwAQVKcAz6jMI7DFBnOIVYyz9znarL9v5otFYuZcEPVgicsXqSifqxHu72+u3Ug106n4wzgMWEaIiIgRCCwZhwhIyaBMwziAFYXz0RdcBnbv+BydT6t0aRyMFZSBelNj2SaSYHCgEmIAZ1nGWdI3RRKDKmwBRrOxNPqPGj6IteoDnYFCsNMsWR0vDfubyEX7K8ceQGMWqB1wUTsVFoNIAUQSKQ2AdwbsF6SaZbbSBMNBBZ3Nx9mV9eQEJ1snFI6wq6C+9tpCWT/IYKWH6zuoGWDQZUyQfh4Ex79yorWErR8mSkZodb/tDu9jP1+9IbUoLUfHyofeijQhigDhnWd2YhG1+BB6XdEJ6jrunjFKDck5GiLA6312hU94bFUzCSpo9JOjl3wTHiObkqN6HIAXwFFJClSoCxGbfxMF6FuBDYWhJw84Sxczq6uN3g9ceiY8+Xrzf/+Mwu7rmyaMxlg+vj50zFMHz9/Gst0dTk7hunqcnYoU0qvxjB8u706FFkn5HIM9OLu5nIE9mw2KgiLu5vZ7GD/W/xx6WTxD88knZCRSbS4uxmRPxZ/Od5Dbs04jtFecmtGcRzhqeVYX40sNMcxosp0QsYzjMY/IuJXl7OLcTF3PKOj7ngOj/vra3I9ypSfP693GlEbIKMnPLxz/E3391LFG27vJQAwISlOgcuIcLh/qP7LpDJTUJhKg27Y3nn9R/td2yOuBw1ITlm/X3rsa9+IG82E1KYe7GsodvjLXisLsIJICkOYqM4KvLSbibVUKbHrgsaq7mmhuroaGw1zZtugFnGplEsRt4ZLwjnQXDne1pdMZLlZVlMEEVJjJAXVrVkyN81pRN+SondGpjBi2jnlsvX9Dn/Z64ezBphoSgh6zA6lNAOGU2JwDOcXKZu9ZJvHRw8V+4W0hyyUkiMRY/gWNtfXPg1skdQcfQKssF9SYGA/9gjobiQHCPh/49BWwTfPLlNwJ7Qvi8edguR6rdEEGqNDsm+PpseNDotqM2BH9K3K0/njzqP1MbG+oB/JAfe3fRRERQkzGJlcndCgFqw/c79+vl5efzrvE5GSvigewf3t5t9AKFWoNfbGjmU9RJ3BPRz3D7sppO6h6O7ce1hWUjf27sZ2DSSUuXHFIjO0W6mIq/tOe7/d2rMbR3NOjEXskA57fa9Pvi9qUB/tCIWRegp5mAuTT+GFCSpf9HnQq2irnN6qxj0aKpV8I5Ed+TlAvSYp48VJyUtIT6+QJsRMgWLIiJjCWiGGmu7zyDMq3b1NvlWXx+wnfEIlkJ+O77EnRd9rT7Mtpc5NEj2RGN/U+niMnRVEBDChDeEcKUjlerpnpBX/29qibq+7z5k7Xbnz6ZZXO7r9ra66DfZIzgD/uMuP9GTNYNkeaeJDg7yPp68Y3ki1wyof71OyecihG/ApqZp33j4+ziIUp7XOQ/beb8saO0mzXNF5zMGuWbNfBx1JDiKzYL0keZoSVRwBWC7sw8wVP2VYfvzxdXt/rV9xNCnGbK4WYG9vYifp8i3GdnNy+H76uxoEgB/t9yFbXmJdxLeztfvvDVd8Wq7/2lgOklGmTm3Yew2JTNFCY2RkO7Wbz7uQn7C9AHhQMlYkBSNB5QKIAS5jNtDd2IRcNnL1pB73j1bcO7bmoxX4LuArE/nrFEzCtL1D2+KIMZK6zPaBjNg6LFQKZfgnRmacwJWD29MMFSWp3ryCZBoyooxtHM5CLKR/45SXEc8Us7tYuarTwvZXMuyu5n1ROCgSUOf/dmnDzpLb0DNhMMZulYykHyq/jGjdY9zQGXF/bCvA3eGto+Znw5mQxjeQfoQZjXw9OpJW+e+K5M2WbAsbwIPUmoW8+fYTVjohVL4sa38MYJ61jHadsS1MUT5Ldhjuhwjn041vl5RpEnKkq+kA6krIDbPlKIudEhGjkrl2/bgopED3cwcuY2Di3LXZQ4iRKjLTBH1JULRD5mJjtV+giS7cMAWNmOoBUCOrLLHHHxSOw515SsSt6De6RqLNMkqsQcOls9XOlddBwX50P9AoWntMZegL0U4AeAHB5O8AAAD//1pphx4=" } diff --git a/x-pack/auditbeat/module/system/process/_meta/data.json b/x-pack/auditbeat/module/system/process/_meta/data.json index abd1e7d5d68b..a1f1d3fb51c1 100644 --- a/x-pack/auditbeat/module/system/process/_meta/data.json +++ b/x-pack/auditbeat/module/system/process/_meta/data.json @@ -13,6 +13,9 @@ ], "entity_id": "+fYshazplsMYlr0y", "executable": "/bin/zsh", + "hash": { + "sha1": "33646536613061316366353134643135613631643363383733653261373130393737633131303364" + }, "name": "zsh", "pid": 9086, "ppid": 9085, diff --git a/x-pack/auditbeat/module/system/process/config.go b/x-pack/auditbeat/module/system/process/config.go index 0cdeb737ad73..49def8412a5a 100644 --- a/x-pack/auditbeat/module/system/process/config.go +++ b/x-pack/auditbeat/module/system/process/config.go @@ -6,17 +6,21 @@ package process import ( "time" + + "github.com/elastic/beats/auditbeat/helper/hasher" ) // Config defines the host metricset's configuration options. type Config struct { StatePeriod time.Duration `config:"state.period"` ProcessStatePeriod time.Duration `config:"process.state.period"` + + HasherConfig hasher.Config `config:"process.hash"` } -// Validate validates the host metricset config. +// Validate validates the config. func (c *Config) Validate() error { - return nil + return c.HasherConfig.Validate() } func (c *Config) effectiveStatePeriod() time.Duration { @@ -28,4 +32,12 @@ func (c *Config) effectiveStatePeriod() time.Duration { var defaultConfig = Config{ StatePeriod: 12 * time.Hour, + + HasherConfig: hasher.Config{ + HashTypes: []hasher.HashType{hasher.SHA1}, + MaxFileSize: "100 MiB", + MaxFileSizeBytes: 100 * 1024 * 1024, + ScanRatePerSec: "50 MiB", + ScanRateBytesPerSec: 50 * 1024 * 1024, + }, } diff --git a/x-pack/auditbeat/module/system/process/process.go b/x-pack/auditbeat/module/system/process/process.go index e4a2b6a6d8d0..aff75b33babf 100644 --- a/x-pack/auditbeat/module/system/process/process.go +++ b/x-pack/auditbeat/module/system/process/process.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/auditbeat/datastore" + "github.com/elastic/beats/auditbeat/helper/hasher" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" @@ -80,6 +81,7 @@ type MetricSet struct { log *logp.Logger bucket datastore.Bucket lastState time.Time + hasher *hasher.FileHasher suppressPermissionWarnings bool } @@ -90,6 +92,7 @@ type Process struct { UserInfo *types.UserInfo User *user.User Group *user.Group + Hashes map[hasher.HashType]hasher.Digest Error error } @@ -137,12 +140,18 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "failed to open persistent datastore") } + hasher, err := hasher.NewFileHasher(config.HasherConfig, nil) + if err != nil { + return nil, err + } + ms := &MetricSet{ SystemMetricSet: system.NewSystemMetricSet(base), config: config, log: logp.NewLogger(metricsetName), cache: cache.New(), bucket: bucket, + hasher: hasher, } // Load from disk: Time when state was last sent @@ -215,6 +224,8 @@ func (ms *MetricSet) reportState(report mb.ReporterV2) error { return errors.Wrap(err, "error generating state ID") } for _, p := range processes { + ms.enrichProcess(p) + if p.Error == nil { event := ms.processEvent(p, eventTypeState, eventActionExistingProcess) event.RootFields.Put("event.id", stateID.String()) @@ -255,6 +266,7 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { for _, cacheValue := range started { p := cacheValue.(*Process) + ms.enrichProcess(p) if p.Error == nil { report.Event(ms.processEvent(p, eventTypeEvent, eventActionProcessStarted)) @@ -275,6 +287,34 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { return nil } +// enrichProcess enriches a process with user lookup information +// and executable file hash. +func (ms *MetricSet) enrichProcess(process *Process) { + if process.UserInfo != nil { + goUser, err := user.LookupId(process.UserInfo.UID) + if err == nil { + process.User = goUser + } + + group, err := user.LookupGroupId(process.UserInfo.GID) + if err == nil { + process.Group = group + } + } + + if process.Info.Exe != "" { + hashes, err := ms.hasher.HashFile(process.Info.Exe) + if err != nil { + if process.Error == nil { + process.Error = errors.Wrapf(err, "failed to hash executable %v for PID %v", process.Info.Exe, + process.Info.PID) + } + } else { + process.Hashes = hashes + } + } +} + func (ms *MetricSet) processEvent(process *Process, eventType string, action eventAction) mb.Event { event := mb.Event{ RootFields: common.MapStr{ @@ -310,6 +350,13 @@ func (ms *MetricSet) processEvent(process *Process, eventType string, action eve event.RootFields.Put("user.group.name", process.Group.Name) } + if process.Hashes != nil { + for hashType, digest := range process.Hashes { + fieldName := "process.hash." + string(hashType) + event.RootFields.Put(fieldName, digest) + } + } + if process.Error != nil { event.RootFields.Put("error.message", process.Error.Error()) } @@ -411,16 +458,6 @@ func (ms *MetricSet) getProcesses() ([]*Process, error) { } } else { process.UserInfo = &userInfo - - goUser, err := user.LookupId(userInfo.UID) - if err == nil { - process.User = goUser - } - - group, err := user.LookupGroupId(userInfo.GID) - if err == nil { - process.Group = group - } } // Exclude Linux kernel processes, they are not very interesting. diff --git a/x-pack/auditbeat/module/system/process/process_test.go b/x-pack/auditbeat/module/system/process/process_test.go index a0999947a110..5fa6e16a7223 100644 --- a/x-pack/auditbeat/module/system/process/process_test.go +++ b/x-pack/auditbeat/module/system/process/process_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/auditbeat/core" + "github.com/elastic/beats/auditbeat/helper/hasher" abtest "github.com/elastic/beats/auditbeat/testing" "github.com/elastic/beats/libbeat/common" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -43,8 +44,12 @@ func TestData(t *testing.T) { func getConfig() map[string]interface{} { return map[string]interface{}{ - "module": "system", - "metricsets": []string{"process"}, + "module": "system", + "datasets": []string{"process"}, + + // To speed things up during testing, we effectively + // disable hashing. + "process.hash.max_file_size": 1, } } @@ -71,6 +76,7 @@ func TestProcessEvent(t *testing.T) { "process.executable": "/bin/zsh", "process.args": []string{"zsh"}, "process.start": "2019-01-01 00:00:01 +0000 UTC", + "process.hash.sha1": "3de6a0a1cf514d15a61d3c873e2a710977c1103d", "user.id": "1000", "user.name": "elastic", @@ -87,6 +93,8 @@ func TestProcessEvent(t *testing.T) { switch v := value.(type) { case time.Time: assert.Equalf(t, expFieldValue, v.String(), "Unexpected value for field %v.", expFieldName) + case hasher.Digest: + assert.Equalf(t, expFieldValue, string(v), "Unexpected value for field %v.", expFieldName) default: assert.Equalf(t, expFieldValue, value, "Unexpected value for field %v.", expFieldName) } @@ -121,6 +129,9 @@ func testProcess() *Process { Gid: "1000", Name: "elastic", }, + Hashes: map[hasher.HashType]hasher.Digest{ + hasher.SHA1: []byte("3de6a0a1cf514d15a61d3c873e2a710977c1103d"), + }, } } diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index a4956f77aa6a..f4eaec2bcd16 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -67,8 +67,11 @@ def test_metricset_process(self): fields.extend(["user.effective.id", "user.saved.id", "user.effective.group.id", "user.saved.group.id", "user.name", "user.group.name"]) - # Metricset is beta and that generates a warning, TODO: remove later - self.check_metricset("system", "process", COMMON_FIELDS + fields, warnings_allowed=True) + # process.hash.max_file_size: 1 - To speed things up during testing, we effectively disable hashing. + # errors_allowed|warnings_allowed=True - Disabling hashing causes the dataset to add an error to the event + # and log a warning. That should not fail the test. + self.check_metricset("system", "process", COMMON_FIELDS + fields, {"process.hash.max_file_size": 1}, + errors_allowed=True, warnings_allowed=True) @unittest.skipUnless(sys.platform == "linux2", "Only implemented for Linux") def test_metricset_socket(self):