From aff92960eefea911818f81066bbaec22377386f0 Mon Sep 17 00:00:00 2001 From: Christoph Wurm Date: Tue, 5 Feb 2019 11:49:56 +0000 Subject: [PATCH] [Auditbeat] System module: Detect package updates (#10508) Detects package updates by checking if any of the "new" package objects have the same package name as one of the "old" package objects. The event will have `event.action: package_updated`. Also removes `InstallTime` from change detection. And for dpkg, reports packages as removed that have only been removed (`apt-get remove`) but not purged (`apt-get purge`). (cherry picked from commit 394d93d52754ea3777e3a7fcfe1729f5895d8ce8) --- .../module/system/package/package.go | 84 +++++++++++++++---- .../auditbeat/tests/system/test_metricsets.py | 2 +- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/x-pack/auditbeat/module/system/package/package.go b/x-pack/auditbeat/module/system/package/package.go index 44babb51f19..f25dbc2fbc4 100644 --- a/x-pack/auditbeat/module/system/package/package.go +++ b/x-pack/auditbeat/module/system/package/package.go @@ -9,6 +9,7 @@ package pkg import ( "bufio" "bytes" + "encoding/binary" "encoding/gob" "fmt" "io" @@ -57,6 +58,7 @@ const ( eventActionExistingPackage eventAction = iota eventActionPackageInstalled eventActionPackageRemoved + eventActionPackageUpdated ) func (action eventAction) String() string { @@ -67,6 +69,8 @@ func (action eventAction) String() string { return "package_installed" case eventActionPackageRemoved: return "package_removed" + case eventActionPackageUpdated: + return "package_updated" default: return "" } @@ -108,15 +112,20 @@ type Package struct { func (pkg Package) Hash() uint64 { h := xxhash.New64() h.WriteString(pkg.Name) - h.WriteString(pkg.InstallTime.String()) + h.WriteString(pkg.Version) + h.WriteString(pkg.Release) + binary.Write(h, binary.LittleEndian, pkg.Size) return h.Sum64() } func (pkg Package) toMapStr() common.MapStr { mapstr := common.MapStr{ - "name": pkg.Name, - "version": pkg.Version, - "installtime": pkg.InstallTime, + "name": pkg.Name, + "version": pkg.Version, + } + + if pkg.Release != "" { + mapstr.Put("release", pkg.Release) } if pkg.Arch != "" { @@ -127,8 +136,8 @@ func (pkg Package) toMapStr() common.MapStr { mapstr.Put("license", pkg.License) } - if pkg.Release != "" { - mapstr.Put("release", pkg.Release) + if !pkg.InstallTime.IsZero() { + mapstr.Put("installtime", pkg.InstallTime) } if pkg.Size != 0 { @@ -302,23 +311,55 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error { } ms.log.Debugf("Found %v packages", len(packages)) - installed, removed := ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + newInCache, missingFromCache := ms.cache.DiffAndUpdateCache(convertToCacheable(packages)) + newPackages := convertToPackage(newInCache) + missingPackages := convertToPackage(missingFromCache) + + // Package names of updated packages + updated := make(map[string]struct{}) + + for _, missingPkg := range missingPackages { + found := false + + // Using an inner loop is less efficient than using a map, but in this case + // we do not expect a lot of installed or removed packages all at once. + for _, newPkg := range newPackages { + if missingPkg.Name == newPkg.Name { + found = true + updated[newPkg.Name] = struct{}{} + report.Event(packageEvent(newPkg, eventTypeEvent, eventActionPackageUpdated)) + break + } + } - for _, cacheValue := range installed { - report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageInstalled)) + if !found { + report.Event(packageEvent(missingPkg, eventTypeEvent, eventActionPackageRemoved)) + } } - for _, cacheValue := range removed { - report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageRemoved)) + for _, newPkg := range newPackages { + if _, contains := updated[newPkg.Name]; !contains { + report.Event(packageEvent(newPkg, eventTypeEvent, eventActionPackageInstalled)) + } } - if len(installed) > 0 || len(removed) > 0 { + if len(newPackages) > 0 || len(missingPackages) > 0 { return ms.savePackagesToDisk(packages) } return nil } +func convertToPackage(cacheValues []interface{}) []*Package { + packages := make([]*Package, 0, len(cacheValues)) + + for _, c := range cacheValues { + packages = append(packages, c.(*Package)) + } + + return packages +} + func packageEvent(pkg *Package, eventType string, action eventAction) mb.Event { event := mb.Event{ RootFields: common.MapStr{ @@ -347,6 +388,8 @@ func packageMessage(pkg *Package, action eventAction) string { actionString = "installed" case eventActionPackageRemoved: actionString = "removed" + case eventActionPackageUpdated: + actionString = "updated" } return fmt.Sprintf("Package %v (%v) %v", @@ -444,16 +487,24 @@ func listDebPackages() ([]*Package, error) { defer file.Close() var packages []*Package + var skipPackage bool pkg := &Package{} scanner := bufio.NewScanner(file) for scanner.Scan() { line := scanner.Text() if len(strings.TrimSpace(line)) == 0 { // empty line signals new package - packages = append(packages, pkg) + if !skipPackage { + packages = append(packages, pkg) + } + skipPackage = false pkg = &Package{} continue + } else if skipPackage { + // Skipping this package - read on. + continue } + if strings.HasPrefix(line, " ") { // not interested in multi-lines for now continue @@ -466,6 +517,11 @@ func listDebPackages() ([]*Package, error) { switch strings.ToLower(words[0]) { case "package": pkg.Name = value + case "status": + if strings.HasPrefix(value, "deinstall ok") { + // Package was removed but not purged. We report both cases as removed. + skipPackage = true + } case "architecture": pkg.Arch = value case "version": @@ -482,7 +538,7 @@ func listDebPackages() ([]*Package, error) { } } if err = scanner.Err(); err != nil { - return nil, errors.Wrap(err, "error scanning file") + return nil, errors.Wrapf(err, "error scanning file %v", dpkgStatusFile) } return packages, nil } diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index ce45b0bbf9b..f44535ead3e 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -48,7 +48,7 @@ def test_metricset_package(self): package metricset collects information about installed packages on a system. """ - fields = ["system.audit.package.name", "system.audit.package.version", "system.audit.package.installtime"] + fields = ["system.audit.package.name", "system.audit.package.version"] # Metricset is experimental and that generates a warning, TODO: remove later self.check_metricset("system", "package", COMMON_FIELDS + fields, warnings_allowed=True)