From 79936dbde2206b6614b7032a29bb5434bab10071 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 29 Jan 2019 14:36:06 +0000 Subject: [PATCH 1/2] [Auditbeat] System module: Update and re-enable package dataset (#10225) Re-enables the disabled `package` dataset and brings it up to date with the other, soon-to-be released datasets. High-level changes: - Renamed to `package` (singular) - Scheduled state reporting based on `state.period` and `package.state.period` - Common fields: `event.kind`, `event.action`, `event.id`, `message` - Save/Restore package information to disk (cherry picked from commit 1e2c30af88559ddbad108769e26cc99a5dd77c48) --- CHANGELOG.next.asciidoc | 1 + auditbeat/docs/fields.asciidoc | 95 +++ x-pack/auditbeat/auditbeat.reference.yml | 2 + x-pack/auditbeat/auditbeat.yml | 1 + x-pack/auditbeat/docs/modules/system.asciidoc | 7 + .../docs/modules/system/package.asciidoc | 21 + x-pack/auditbeat/include/list.go | 1 + .../module/system/_meta/config.yml.tmpl | 9 +- .../module/system/_meta/docs.asciidoc | 2 + x-pack/auditbeat/module/system/fields.go | 2 +- .../module/system/package/_meta/data.json | 29 + .../module/system/package/_meta/docs.asciidoc | 8 + .../module/system/package/_meta/fields.yml | 41 ++ .../auditbeat/module/system/package/config.go | 30 + .../module/system/package/package.go | 539 ++++++++++++++++++ .../module/system/package/package_test.go | 36 ++ .../module/system/package/package_windows.go | 29 + .../auditbeat/tests/system/test_metricsets.py | 14 + 18 files changed, 863 insertions(+), 4 deletions(-) create mode 100644 x-pack/auditbeat/docs/modules/system/package.asciidoc create mode 100644 x-pack/auditbeat/module/system/package/_meta/data.json create mode 100644 x-pack/auditbeat/module/system/package/_meta/docs.asciidoc create mode 100644 x-pack/auditbeat/module/system/package/_meta/fields.yml create mode 100644 x-pack/auditbeat/module/system/package/config.go create mode 100644 x-pack/auditbeat/module/system/package/package.go create mode 100644 x-pack/auditbeat/module/system/package/package_test.go create mode 100644 x-pack/auditbeat/module/system/package/package_windows.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1f2d5e27624..442f61a4abf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -142,6 +142,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Add system module. {pull}9546[9546] - System module `process` dataset: Add user information to processes. {pull}9963[9963] +- Add system `package` dataset. {pull}10225[10225] *Filebeat* diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index 9c05d662277..5fde2c9eb58 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -3892,6 +3892,101 @@ type: keyword The operating system's kernel version. +-- + +[float] +== package fields + +`package` contains information about an installed or removed package. + + + +*`system.audit.package.name`*:: ++ +-- +type: keyword + +Package name. + + +-- + +*`system.audit.package.version`*:: ++ +-- +type: keyword + +Package version. + + +-- + +*`system.audit.package.release`*:: ++ +-- +type: keyword + +Package release. + + +-- + +*`system.audit.package.arch`*:: ++ +-- +type: keyword + +Package architecture. + + +-- + +*`system.audit.package.license`*:: ++ +-- +type: keyword + +Package license. + + +-- + +*`system.audit.package.installtime`*:: ++ +-- +type: date + +Package install time. + + +-- + +*`system.audit.package.size`*:: ++ +-- +type: long + +Package size. + + +-- + +*`system.audit.package.summary`*:: ++ +-- +Package summary. + + +-- + +*`system.audit.package.url`*:: ++ +-- +type: keyword + +Package URL. + + -- [float] diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 090cbe3c48d..b9d33bb7197 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -112,6 +112,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets - user # User information @@ -123,6 +124,7 @@ auditbeat.modules: # The state.period can be overridden for any dataset. # host.state.period: 12h + # package.state.period: 12h # process.state.period: 12h # socket.state.period: 12h # user.state.period: 12h diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index 421685744b8..939d037b424 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -50,6 +50,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets - user # User information diff --git a/x-pack/auditbeat/docs/modules/system.asciidoc b/x-pack/auditbeat/docs/modules/system.asciidoc index 64198c06d27..6e3c52b3985 100644 --- a/x-pack/auditbeat/docs/modules/system.asciidoc +++ b/x-pack/auditbeat/docs/modules/system.asciidoc @@ -51,6 +51,7 @@ sample suggested configuration. - module: system datasets: - host + - package - process - socket - user @@ -86,6 +87,7 @@ so a longer polling interval can be used. - module: system datasets: - host + - package - user period: 1m user.detect_password_changes: true @@ -111,6 +113,7 @@ auditbeat.modules: - module: system datasets: - host # General host information, e.g. uptime, IPs + - package # Installed, updated, and removed packages - process # Started and stopped processes - socket # Opened and closed sockets - user # User information @@ -133,6 +136,8 @@ The following datasets are available: * <<{beatname_lc}-dataset-system-host,host>> +* <<{beatname_lc}-dataset-system-package,package>> + * <<{beatname_lc}-dataset-system-process,process>> * <<{beatname_lc}-dataset-system-socket,socket>> @@ -141,6 +146,8 @@ The following datasets are available: include::system/host.asciidoc[] +include::system/package.asciidoc[] + include::system/process.asciidoc[] include::system/socket.asciidoc[] diff --git a/x-pack/auditbeat/docs/modules/system/package.asciidoc b/x-pack/auditbeat/docs/modules/system/package.asciidoc new file mode 100644 index 00000000000..ec87bd976da --- /dev/null +++ b/x-pack/auditbeat/docs/modules/system/package.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[id="{beatname_lc}-dataset-system-package"] +=== System package dataset + +include::../../../module/system/package/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the dataset, see the +<> section. + +Here is an example document generated by this dataset: + +[source,json] +---- +include::../../../module/system/package/_meta/data.json[] +---- diff --git a/x-pack/auditbeat/include/list.go b/x-pack/auditbeat/include/list.go index 6a86c7fcbac..4c2e3507ade 100644 --- a/x-pack/auditbeat/include/list.go +++ b/x-pack/auditbeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/auditbeat/module/system" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/host" + _ "github.com/elastic/beats/x-pack/auditbeat/module/system/package" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/process" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/socket" _ "github.com/elastic/beats/x-pack/auditbeat/module/system/user" diff --git a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl index f06f53cab3e..5ac79221791 100644 --- a/x-pack/auditbeat/module/system/_meta/config.yml.tmpl +++ b/x-pack/auditbeat/module/system/_meta/config.yml.tmpl @@ -7,9 +7,9 @@ - module: system datasets: - host # General host information, e.g. uptime, IPs - {{ if false -}} - - packages # Installed packages - {{- end -}} + {{ if ne .GOOS "windows" -}} + - package # Installed, updated, and removed packages + {{- end }} - process # Started and stopped processes {{ if eq .GOOS "linux" -}} - socket # Opened and closed sockets @@ -23,6 +23,9 @@ {{ if .Reference }} # The state.period can be overridden for any dataset. # host.state.period: 12h + {{ if ne .GOOS "windows" -}} + # package.state.period: 12h + {{- end }} # process.state.period: 12h {{ if eq .GOOS "linux" -}} # socket.state.period: 12h diff --git a/x-pack/auditbeat/module/system/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/_meta/docs.asciidoc index 4dc42e8c17d..e9da6bc8f55 100644 --- a/x-pack/auditbeat/module/system/_meta/docs.asciidoc +++ b/x-pack/auditbeat/module/system/_meta/docs.asciidoc @@ -46,6 +46,7 @@ sample suggested configuration. - module: system datasets: - host + - package - process - socket - user @@ -81,6 +82,7 @@ so a longer polling interval can be used. - module: system datasets: - host + - package - user period: 1m user.detect_password_changes: true diff --git a/x-pack/auditbeat/module/system/fields.go b/x-pack/auditbeat/module/system/fields.go index a62526effb6..5bb495e45ae 100644 --- a/x-pack/auditbeat/module/system/fields.go +++ b/x-pack/auditbeat/module/system/fields.go @@ -18,5 +18,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy0WE2P2zYQvftXDHLZXcBRLkVQ+FAgaYE2QNMssBugN3skjiR2JY7AoeJVfn1BfdmyKTvOenXz2H7vcd48kvZbeKJmBdKIo3IB4LQraAVvHtrCmwWAIkmsrpxms4LfFgAAjzkJAVoClxOkmgolkJEhi44UxE1b7zChZFUXFC0ALBWEQiug54qsLsk4LBbQA6xa6LdgsKRBUIS10q59A8A1Fa0gs1xXfSUgbYq2j5izuLEYQptF7J6NB9hAwsahNsN6ixYXtEnZlui/F+19a27Fw3Oodae2rpwuaSKgU1ywySblE5L987UFAm3AoGGhhI2SKMAYM7sZToWOLuH8yOzAY4V4+gaS1d9JBchi5oLQXML3QA502jshgDuOkAAv7DsbivzLgIAnarZs1SUC/sGSgNN27Af44bVXtQSKsgg+PjyeFMRpKuQioeQKxj/udHhUPwEn3Pcqr9ePv3q0EJMOmf6THPDpjxAF2iTXjhJX2ysuaAILt62jz7++X7//5S4kosSQiz/B/fnD74BKWRKhoHe6ChAdFM9wfLo/TcESoDjcPM+wbFj2ts+9HRMw5tq1YeHKnyHaZMMBMME43i53CqsCnUc8IJ3v+tmefHkYQXu3EzKOZQl1XBtXL2GrjeKt3EVBRUdxeqkaD9gr+YyJr/w7Q51iqYvmquQdZE9vSeXolqAo1miWkFqiWNS5jnwjK5rNVXX1mGHCJ7KGiuvxPQZG9EZ6mmMp42EuZF909fAAZ7PjPyTgS4HwXH4Tea1BBvgqZOHweNjrlj5EfDnb9JzYcWXX5frTezpLprS99sJuBHIuyUNT4tg2YWbJqbhiDADuLWcWS3AMtjaADgrO9EwK/WCu92Y2KMTRs7u43f0V3BNMruDwxcDf2tTPS3C5Fn8p9AnJKGHpRn1mHI5OtEEex/9RcqHATQt3Mrdomo5Uxgh7sRVa5++PtzE1bNT43o1AZXWJtv/WwT4bjjGcjjKcmYUfcgLG4T/ONZzM245eG0cZHUbkQvq57FUoEljc3EXmvLcD4Gl7R9f6T8Ot4W63HivaCRXpxU565a/l5Icj2R42gnsW0XFB8A2LmqT9B2AjOSrersd+zGDeThado+R+0LXp/ipoMSDVBd0td71dKy0YF6Q2yxnUjeEds+fowq7QZGS5FkDxGfM/gxI0fpMCbe6WgCbUnBYxsU3l9kG3OZmpZa03Xvs7csm7tqxAiEqZAXU8TAmgATItB6kR8cj9nc8FilsnuV/QfHSOfqB3zw+Z7e80CpvJHjMsdIvSCoBeQLT4PwAA///ff7jT" + return "eJy0WE1v20YQvetXDHKxDSjKpQgKHQokLdAGSBqjtoHerBF3RG693CF2lpbpX18sPyRRWkqRxfDmsfje2/levocnquYglXjKJwBee0NzeHdXG95NABRJ4nThNds5/DYBALjPSAjQEfiMYKXJKIGULDn0pGBZ1fYGE3JWpaHZBMCRIRSaA70U5HRO1qOZQAswr6Hfg8WcOkEzLJX29T8AfFXQHFLHZdFaItL6aLuIGYvfGGNog4jNswgAC0jYetS2O6+pcUHbFbscw3uznbeGTtw9+1q3asvC65x6AhrFhm3aMx+RHJ6HGgi0BYuWhRK2SmYRxiWzH+BU6Okczs/MHgJWjKd1IDn9SipCtmQ2hPYcvjvyoFdtJARwyxETEIS9sqVZ+DMi4ImqNTt1joC/MSfgVZ32HXz3d1A1BZqlM/h8d39UEK9WQn4mlIwQ+PutjoAaMuBI9IPK8fzxV4sWY9KxoL+RA778EaNAl2TaU+JLN+KBerBwXUf05dePjx9/uYmJyDEWxTdwf/v0O6BSjkQoGjtdRIj2jCc4vtwep2CJUOw3zxMsC5ad9rnTMQGXXPq6WLgIM0TbtBsAPYzDdrlVWBj0AXGPdNjrJ33y/W4D2kY7IetZplAuS+vLKay1VbyWm1lU0UE5XaomALZKvmESLP8OUK8w16YalbyBbOkdqQz9FBQtNdoprBzRUtQpjzyTE812VF0tZpzwiZwlMx7ffSRFr6SlOZSyyU1MnjCli7aPFuNoBaEFbcWjMaSAHTjK+ZlUxz/OZjLWlLhtRMHQnIgly4VUkVzp2Fp/jMnWQg4NqDGpdidTjM/ohOy4p2sho/OoycFRlsmOrsUc3CpFv16+LXdkASxKUuY5uuoNgM2LMczSmTHD8vDP18P+Uwq5i5pPADg5u8OPBIIpMrzP7zc/a5ACPAi5g7az4y29j3g5W39P3XKl43L9GWI6SKa0G/tgVwIZ5xSgKfHcT/GdwsnIjDiGAW4dpw5z8AyutIAeDKd6YAsIifm4k7NRIZ5e/Nnubj8BBILeJwD4buGrtuXLFHymJVxKQ4WklLA0qT6QDgcbdSePl/9RcqbARQ13YmOoGlLZlHAQW6Dz4f56vaSKrdr870qgcDq0suatvT0vXsZwvJThRC78UCRgk/yHdQ1H621Lr62nlPZL5Ez6odorUCRyuKGL1OnYdoDHw7uJWvtruLbcdOuNRXshszo7kkH5z4rkpwPZAXYGtyyil4bgGU1JUn+BXEiGitePG38MYF73Dp2hZCHRtW0+VdYYsNKGbqZb3z4qLbg0pBbTAdSF5S1z4GiKXaFNyXEpgBJqjC1BgjY0KdD2ZgpoY86pERNXFX4XdJ2R7Yesjk3Q/oF88qE2KxCiXAZAPXdZEu4IZGuO+mLQIB5Ef2d1RPGPSRYONFw6Bztd8/xQsMOdSmHV6zHdQdcotQBoBcwm/wcAAP//IWXfjA==" } diff --git a/x-pack/auditbeat/module/system/package/_meta/data.json b/x-pack/auditbeat/module/system/package/_meta/data.json new file mode 100644 index 00000000000..ca570150245 --- /dev/null +++ b/x-pack/auditbeat/module/system/package/_meta/data.json @@ -0,0 +1,29 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "action": "existing_package", + "dataset": "package", + "id": "9ac4ea4c-5a0c-475f-b4c9-ec9d981ff11b", + "kind": "state", + "module": "system" + }, + "message": "Package zstd (1.3.5) is already installed", + "service": { + "type": "system" + }, + "system": { + "audit": { + "package": { + "installtime": "2018-08-30T18:41:23.85657356+01:00", + "name": "zstd", + "summary": "Zstandard is a real-time compression algorithm", + "url": "http://zstd.net/", + "version": "1.3.5" + } + } + } +} \ No newline at end of file diff --git a/x-pack/auditbeat/module/system/package/_meta/docs.asciidoc b/x-pack/auditbeat/module/system/package/_meta/docs.asciidoc new file mode 100644 index 00000000000..13e2be806a7 --- /dev/null +++ b/x-pack/auditbeat/module/system/package/_meta/docs.asciidoc @@ -0,0 +1,8 @@ +[role="xpack"] + +experimental[] + +This is the `package` dataset of the system module. + +It is implemented for Linux distributions using dpkg as their package manager, +and for Homebrew on macOS (Darwin). diff --git a/x-pack/auditbeat/module/system/package/_meta/fields.yml b/x-pack/auditbeat/module/system/package/_meta/fields.yml new file mode 100644 index 00000000000..d87ba9aac06 --- /dev/null +++ b/x-pack/auditbeat/module/system/package/_meta/fields.yml @@ -0,0 +1,41 @@ +- name: package + type: group + description: > + `package` contains information about an installed or removed package. + release: experimental + fields: + - name: name + type: keyword + description: > + Package name. + - name: version + type: keyword + description: > + Package version. + - name: release + type: keyword + description: > + Package release. + - name: arch + type: keyword + description: > + Package architecture. + - name: license + type: keyword + description: > + Package license. + - name: installtime + type: date + description: > + Package install time. + - name: size + type: long + description: > + Package size. + - name: summary + description: > + Package summary. + - name: url + type: keyword + description: > + Package URL. diff --git a/x-pack/auditbeat/module/system/package/config.go b/x-pack/auditbeat/module/system/package/config.go new file mode 100644 index 00000000000..5af2d0a7d6a --- /dev/null +++ b/x-pack/auditbeat/module/system/package/config.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package pkg + +import ( + "time" +) + +// config defines the package metricset's configuration options. +type config struct { + StatePeriod time.Duration `config:"state.period"` + PackageStatePeriod time.Duration `config:"package.state.period"` +} + +func (c *config) effectiveStatePeriod() time.Duration { + if c.PackageStatePeriod != 0 { + return c.PackageStatePeriod + } + return c.StatePeriod +} + +func defaultConfig() config { + return config{ + StatePeriod: 12 * time.Hour, + } +} diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go new file mode 100644 index 00000000000..5030c521d1a --- /dev/null +++ b/x-pack/auditbeat/module/system/package/package.go @@ -0,0 +1,539 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package pkg + +import ( + "bufio" + "bytes" + "encoding/gob" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/OneOfOne/xxhash" + "github.com/gofrs/uuid" + "github.com/pkg/errors" + + "github.com/elastic/beats/auditbeat/datastore" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/auditbeat/cache" + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +const ( + moduleName = "system" + metricsetName = "package" + namespace = "system.audit.package" + + redhat = "redhat" + debian = "debian" + darwin = "darwin" + + dpkgStatusFile = "/var/lib/dpkg/status" + homebrewCellarPath = "/usr/local/Cellar" + + bucketName = "package.v1" + bucketKeyPackages = "packages" + bucketKeyStateTimestamp = "state_timestamp" + + eventTypeState = "state" + eventTypeEvent = "event" +) + +type eventAction uint8 + +const ( + eventActionExistingPackage eventAction = iota + eventActionPackageInstalled + eventActionPackageRemoved +) + +func (action eventAction) String() string { + switch action { + case eventActionExistingPackage: + return "existing_package" + case eventActionPackageInstalled: + return "package_installed" + case eventActionPackageRemoved: + return "package_removed" + default: + return "" + } +} + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + mb.WithNamespace(namespace), + ) +} + +// MetricSet collects data about the system's packages. +type MetricSet struct { + mb.BaseMetricSet + config config + log *logp.Logger + cache *cache.Cache + bucket datastore.Bucket + lastState time.Time + osFamily string +} + +// Package represents information for a package. +type Package struct { + Name string + Version string + Release string + Arch string + License string + InstallTime time.Time + Size uint64 + Summary string + URL string +} + +// Hash creates a hash for Package. +func (pkg Package) Hash() uint64 { + h := xxhash.New64() + h.WriteString(pkg.Name) + h.WriteString(pkg.InstallTime.String()) + return h.Sum64() +} + +func (pkg Package) toMapStr() common.MapStr { + mapstr := common.MapStr{ + "name": pkg.Name, + "version": pkg.Version, + "installtime": pkg.InstallTime, + } + + if pkg.Arch != "" { + mapstr.Put("arch", pkg.Arch) + } + + if pkg.License != "" { + mapstr.Put("license", pkg.License) + } + + if pkg.Release != "" { + mapstr.Put("release", pkg.Release) + } + + if pkg.Size != 0 { + mapstr.Put("size", pkg.Size) + } + + if pkg.Summary != "" { + mapstr.Put("summary", pkg.Summary) + } + + if pkg.URL != "" { + mapstr.Put("url", pkg.URL) + } + + return mapstr +} + +func getOS() (*types.OSInfo, error) { + host, err := sysinfo.Host() + if err != nil { + return nil, errors.Wrap(err, "error getting the OS") + } + + hostInfo := host.Info() + if hostInfo.OS == nil { + return nil, errors.New("no host info") + } + + return hostInfo.OS, nil +} + +// New constructs a new MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The %v/%v dataset is experimental", moduleName, metricsetName) + + config := defaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, errors.Wrapf(err, "failed to unpack the %v/%v config", moduleName, metricsetName) + } + + bucket, err := datastore.OpenBucket(bucketName) + if err != nil { + return nil, errors.Wrap(err, "failed to open persistent datastore") + } + + ms := &MetricSet{ + BaseMetricSet: base, + config: config, + log: logp.NewLogger(metricsetName), + cache: cache.New(), + bucket: bucket, + } + + osInfo, err := getOS() + if err != nil { + return nil, errors.Wrap(err, "error determining operating system") + } + ms.osFamily = osInfo.Family + switch osInfo.Family { + case redhat: + return nil, fmt.Errorf("RPM support is not yet implemented") + case debian: + if _, err := os.Stat(dpkgStatusFile); err != nil { + return nil, errors.Wrapf(err, "error looking up %s", dpkgStatusFile) + } + case darwin: + if _, err := os.Stat(homebrewCellarPath); err != nil { + return nil, errors.Wrapf(err, "error looking up %s - is Homebrew installed?", homebrewCellarPath) + } + default: + return nil, fmt.Errorf("this metricset does not support OS family %v", osInfo.Family) + } + + // Load from disk: Time when state was last sent + err = bucket.Load(bucketKeyStateTimestamp, func(blob []byte) error { + if len(blob) > 0 { + return ms.lastState.UnmarshalBinary(blob) + } + return nil + }) + if err != nil { + return nil, err + } + if !ms.lastState.IsZero() { + ms.log.Debugf("Last state was sent at %v. Next state update by %v.", ms.lastState, ms.lastState.Add(ms.config.effectiveStatePeriod())) + } else { + ms.log.Debug("No state timestamp found") + } + + // Load from disk: Packages + packages, err := ms.restorePackagesFromDisk() + if err != nil { + return nil, errors.Wrap(err, "failed to restore packages from disk") + } + ms.log.Debugf("Restored %d packages from disk", len(packages)) + + ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + + return ms, nil +} + +// Close cleans up the MetricSet when it finishes. +func (ms *MetricSet) Close() error { + if ms.bucket != nil { + return ms.bucket.Close() + } + return nil +} + +// Fetch collects data about the host. It is invoked periodically. +func (ms *MetricSet) Fetch(report mb.ReporterV2) { + needsStateUpdate := time.Since(ms.lastState) > ms.config.effectiveStatePeriod() + if needsStateUpdate || ms.cache.IsEmpty() { + ms.log.Debugf("State update needed (needsStateUpdate=%v, cache.IsEmpty()=%v)", needsStateUpdate, ms.cache.IsEmpty()) + err := ms.reportState(report) + if err != nil { + ms.log.Error(err) + report.Error(err) + } + ms.log.Debugf("Next state update by %v", ms.lastState.Add(ms.config.effectiveStatePeriod())) + } + + err := ms.reportChanges(report) + if err != nil { + ms.log.Error(err) + report.Error(err) + } +} + +// reportState reports all installed packages on the system. +func (ms *MetricSet) reportState(report mb.ReporterV2) error { + ms.lastState = time.Now() + + packages, err := getPackages(ms.osFamily) + if err != nil { + return errors.Wrap(err, "failed to get packages") + } + ms.log.Debugf("Found %v packages", len(packages)) + + stateID, err := uuid.NewV4() + if err != nil { + return errors.Wrap(err, "error generating state ID") + } + for _, pkg := range packages { + event := packageEvent(pkg, eventTypeState, eventActionExistingPackage) + event.RootFields.Put("event.id", stateID.String()) + report.Event(event) + } + + // This will initialize the cache with the current packages + ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + + // Save time so we know when to send the state again (config.StatePeriod) + timeBytes, err := ms.lastState.MarshalBinary() + if err != nil { + return err + } + err = ms.bucket.Store(bucketKeyStateTimestamp, timeBytes) + if err != nil { + return errors.Wrap(err, "error writing state timestamp to disk") + } + + return ms.savePackagesToDisk(packages) +} + +// reportChanges detects and reports any changes to installed packages on this system since the last call. +func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { + packages, err := getPackages(ms.osFamily) + if err != nil { + return errors.Wrap(err, "failed to get packages") + } + ms.log.Debugf("Found %v packages", len(packages)) + + installed, removed := ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + + for _, cacheValue := range installed { + report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageInstalled)) + } + + for _, cacheValue := range removed { + report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageRemoved)) + } + + if len(installed) > 0 || len(removed) > 0 { + return ms.savePackagesToDisk(packages) + } + + return nil +} + +func packageEvent(pkg *Package, eventType string, action eventAction) mb.Event { + return mb.Event{ + RootFields: common.MapStr{ + "event": common.MapStr{ + "kind": eventType, + "action": action.String(), + }, + "message": packageMessage(pkg, action), + }, + MetricSetFields: pkg.toMapStr(), + } +} + +func packageMessage(pkg *Package, action eventAction) string { + var actionString string + switch action { + case eventActionExistingPackage: + actionString = "is already installed" + case eventActionPackageInstalled: + actionString = "installed" + case eventActionPackageRemoved: + actionString = "removed" + } + + return fmt.Sprintf("Package %v (%v) %v", + pkg.Name, pkg.Version, actionString) +} + +func convertToCacheable(packages []*Package) []cache.Cacheable { + c := make([]cache.Cacheable, 0, len(packages)) + + for _, p := range packages { + c = append(c, p) + } + + return c +} + +// restorePackagesFromDisk loads the packages from disk. +func (ms *MetricSet) restorePackagesFromDisk() (packages []*Package, err error) { + var decoder *gob.Decoder + err = ms.bucket.Load(bucketKeyPackages, func(blob []byte) error { + if len(blob) > 0 { + buf := bytes.NewBuffer(blob) + decoder = gob.NewDecoder(buf) + } + return nil + }) + if err != nil { + return nil, err + } + + if decoder != nil { + for { + pkg := new(Package) + err = decoder.Decode(pkg) + if err == nil { + packages = append(packages, pkg) + } else if err == io.EOF { + // Read all packages + break + } else { + return nil, errors.Wrap(err, "error decoding packages") + } + } + } + + return packages, nil +} + +// Save packages to disk. +func (ms *MetricSet) savePackagesToDisk(packages []*Package) error { + var buf bytes.Buffer + encoder := gob.NewEncoder(&buf) + + for _, pkg := range packages { + err := encoder.Encode(*pkg) + if err != nil { + return errors.Wrap(err, "error encoding packages") + } + } + + err := ms.bucket.Store(bucketKeyPackages, buf.Bytes()) + if err != nil { + return errors.Wrap(err, "error writing packages to disk") + } + return nil +} + +func getPackages(osFamily string) (packages []*Package, err error) { + switch osFamily { + case redhat: + // TODO: Implement RPM + err = errors.New("RPM not yet supported") + case debian: + packages, err = listDebPackages() + if err != nil { + err = errors.Wrap(err, "error getting DEB packages") + } + case darwin: + packages, err = listBrewPackages() + if err != nil { + err = errors.Wrap(err, "error getting Homebrew packages") + } + default: + err = errors.Errorf("unknown OS %v - this should not have happened", osFamily) + } + + return +} + +func listDebPackages() ([]*Package, error) { + file, err := os.Open(dpkgStatusFile) + if err != nil { + return nil, errors.Wrapf(err, "error opening %s", dpkgStatusFile) + } + defer file.Close() + + var packages []*Package + pkg := &Package{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(strings.TrimSpace(line)) == 0 { + // empty line signals new package + packages = append(packages, pkg) + pkg = &Package{} + continue + } + if strings.HasPrefix(line, " ") { + // not interested in multi-lines for now + continue + } + words := strings.SplitN(line, ":", 2) + if len(words) != 2 { + return nil, fmt.Errorf("the following line was unexpected (no ':' found): '%s'", line) + } + value := strings.TrimSpace(words[1]) + switch strings.ToLower(words[0]) { + case "package": + pkg.Name = value + case "architecture": + pkg.Arch = value + case "version": + pkg.Version = value + case "description": + pkg.Summary = value + case "installed-size": + pkg.Size, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error converting %s to int", value) + } + default: + continue + } + } + if err = scanner.Err(); err != nil { + return nil, errors.Wrap(err, "error scanning file") + } + return packages, nil +} + +func listBrewPackages() ([]*Package, error) { + packageDirs, err := ioutil.ReadDir(homebrewCellarPath) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory %s", homebrewCellarPath) + } + + var packages []*Package + for _, packageDir := range packageDirs { + if !packageDir.IsDir() { + continue + } + pkgPath := path.Join(homebrewCellarPath, packageDir.Name()) + versions, err := ioutil.ReadDir(pkgPath) + if err != nil { + return nil, errors.Wrapf(err, "error reading directory: %s", pkgPath) + } + + for _, version := range versions { + if !version.IsDir() { + continue + } + pkg := &Package{ + Name: packageDir.Name(), + Version: version.Name(), + InstallTime: version.ModTime(), + } + + // read formula + formulaPath := path.Join(homebrewCellarPath, pkg.Name, pkg.Version, ".brew", pkg.Name+".rb") + file, err := os.Open(formulaPath) + if err != nil { + //fmt.Printf("WARNING: Can't get formula for package %s-%s\n", pkg.Name, pkg.Version) + // TODO: follow the path from INSTALL_RECEIPT.json to find the formula + continue + } + scanner := bufio.NewScanner(file) + count := 15 // only look into the first few lines of the formula + for scanner.Scan() { + count-- + if count == 0 { + break + } + line := scanner.Text() + if strings.HasPrefix(line, " desc ") { + pkg.Summary = strings.Trim(line[7:], " \"") + } else if strings.HasPrefix(line, " homepage ") { + pkg.URL = strings.Trim(line[11:], " \"") + } + } + + packages = append(packages, pkg) + } + } + return packages, nil +} diff --git a/x-pack/auditbeat/module/system/package/package_test.go b/x-pack/auditbeat/module/system/package/package_test.go new file mode 100644 index 00000000000..8e8cde472b8 --- /dev/null +++ b/x-pack/auditbeat/module/system/package/package_test.go @@ -0,0 +1,36 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package pkg + +import ( + "testing" + + "github.com/elastic/beats/auditbeat/core" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("received error: %+v", errs[0]) + } + + if len(events) == 0 { + t.Fatal("no events were generated") + } + + fullEvent := mbtest.StandardizeEvent(f, events[len(events)-1], core.AddDatasetToEvent) + mbtest.WriteEventToDataJSON(t, fullEvent, "") +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "datasets": []string{"package"}, + } +} diff --git a/x-pack/auditbeat/module/system/package/package_windows.go b/x-pack/auditbeat/module/system/package/package_windows.go new file mode 100644 index 00000000000..6811cb41c60 --- /dev/null +++ b/x-pack/auditbeat/module/system/package/package_windows.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package socket + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" +) + +const ( + moduleName = "system" + metricsetName = "package" +) + +func init() { + mb.Registry.MustAddMetricSet(moduleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// New returns an error. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return nil, fmt.Errorf("the %v/%v dataset is not supported on Windows", moduleName, metricsetName) +} diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index df4f642334f..e8007792c2a 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -1,5 +1,6 @@ import jinja2 import os +import platform import sys import time import unittest @@ -21,6 +22,19 @@ def test_metricset_host(self): # Metricset is experimental and that generates a warning, TODO: remove later self.check_metricset("system", "host", COMMON_FIELDS + fields, warnings_allowed=True) + @unittest.skipIf(sys.platform == "win32", "Not implemented for Windows") + @unittest.skipIf(sys.platform == "linux2" and platform.linux_distribution()[0] != "debian", + "Only implemented for Debian") + def test_metricset_package(self): + """ + package metricset collects information about installed packages on a system. + """ + + fields = ["system.audit.package.name", "system.audit.package.version", "system.audit.package.installtime"] + + # Metricset is experimental and that generates a warning, TODO: remove later + self.check_metricset("system", "package", COMMON_FIELDS + fields, warnings_allowed=True) + def test_metricset_process(self): """ process metricset collects information about processes running on a system. From 6675ddae5a0cbc83d5adbeb3c378103aed60907b Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Fri, 1 Feb 2019 00:37:52 +0000 Subject: [PATCH 2/2] Fix package name. --- x-pack/auditbeat/module/system/package/package_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/auditbeat/module/system/package/package_windows.go b/x-pack/auditbeat/module/system/package/package_windows.go index 6811cb41c60..498eea61111 100644 --- a/x-pack/auditbeat/module/system/package/package_windows.go +++ b/x-pack/auditbeat/module/system/package/package_windows.go @@ -4,7 +4,7 @@ // +build windows -package socket +package pkg import ( "fmt"