[Auditbeat] Cherry-pick #10508 to 6.x: System module: Detect package updates

x-pack/auditbeat/module/system/package/package.go

@@ -9,6 +9,7 @@ package pkg
 import (
 	"bufio"
 	"bytes"
+	"encoding/binary"
 	"encoding/gob"
 	"fmt"
 	"io"
@@ -57,6 +58,7 @@ const (
 	eventActionExistingPackage eventAction = iota
 	eventActionPackageInstalled
 	eventActionPackageRemoved
+	eventActionPackageUpdated
 )
 
 func (action eventAction) String() string {
@@ -67,6 +69,8 @@ func (action eventAction) String() string {
 		return "package_installed"
 	case eventActionPackageRemoved:
 		return "package_removed"
+	case eventActionPackageUpdated:
+		return "package_updated"
 	default:
 		return ""
 	}
@@ -108,15 +112,20 @@ type Package struct {
 func (pkg Package) Hash() uint64 {
 	h := xxhash.New64()
 	h.WriteString(pkg.Name)
-	h.WriteString(pkg.InstallTime.String())
+	h.WriteString(pkg.Version)
+	h.WriteString(pkg.Release)
+	binary.Write(h, binary.LittleEndian, pkg.Size)
 	return h.Sum64()
 }
 
 func (pkg Package) toMapStr() common.MapStr {
 	mapstr := common.MapStr{
-		"name":        pkg.Name,
-		"version":     pkg.Version,
-		"installtime": pkg.InstallTime,
+		"name":    pkg.Name,
+		"version": pkg.Version,
+	}
+
+	if pkg.Release != "" {
+		mapstr.Put("release", pkg.Release)
 	}
 
 	if pkg.Arch != "" {
@@ -127,8 +136,8 @@ func (pkg Package) toMapStr() common.MapStr {
 		mapstr.Put("license", pkg.License)
 	}
 
-	if pkg.Release != "" {
-		mapstr.Put("release", pkg.Release)
+	if !pkg.InstallTime.IsZero() {
+		mapstr.Put("installtime", pkg.InstallTime)
 	}
 
 	if pkg.Size != 0 {
@@ -302,23 +311,55 @@ func (ms *MetricSet) reportChanges(report mb.ReporterV2) error {
 	}
 	ms.log.Debugf("Found %v packages", len(packages))
 
-	installed, removed := ms.cache.DiffAndUpdateCache(convertToCacheable(packages))
+	newInCache, missingFromCache := ms.cache.DiffAndUpdateCache(convertToCacheable(packages))
+	newPackages := convertToPackage(newInCache)
+	missingPackages := convertToPackage(missingFromCache)
+
+	// Package names of updated packages
+	updated := make(map[string]struct{})
+
+	for _, missingPkg := range missingPackages {
+		found := false
+
+		// Using an inner loop is less efficient than using a map, but in this case
+		// we do not expect a lot of installed or removed packages all at once.
+		for _, newPkg := range newPackages {
+			if missingPkg.Name == newPkg.Name {
+				found = true
+				updated[newPkg.Name] = struct{}{}
+				report.Event(packageEvent(newPkg, eventTypeEvent, eventActionPackageUpdated))
+				break
+			}
+		}
 
-	for _, cacheValue := range installed {
-		report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageInstalled))
+		if !found {
+			report.Event(packageEvent(missingPkg, eventTypeEvent, eventActionPackageRemoved))
+		}
 	}
 
-	for _, cacheValue := range removed {
-		report.Event(packageEvent(cacheValue.(*Package), eventTypeEvent, eventActionPackageRemoved))
+	for _, newPkg := range newPackages {
+		if _, contains := updated[newPkg.Name]; !contains {
+			report.Event(packageEvent(newPkg, eventTypeEvent, eventActionPackageInstalled))
+		}
 	}
 
-	if len(installed) > 0 || len(removed) > 0 {
+	if len(newPackages) > 0 || len(missingPackages) > 0 {
 		return ms.savePackagesToDisk(packages)
 	}
 
 	return nil
 }
 
+func convertToPackage(cacheValues []interface{}) []*Package {
+	packages := make([]*Package, 0, len(cacheValues))
+
+	for _, c := range cacheValues {
+		packages = append(packages, c.(*Package))
+	}
+
+	return packages
+}
+
 func packageEvent(pkg *Package, eventType string, action eventAction) mb.Event {
 	event := mb.Event{
 		RootFields: common.MapStr{
@@ -347,6 +388,8 @@ func packageMessage(pkg *Package, action eventAction) string {
 		actionString = "installed"
 	case eventActionPackageRemoved:
 		actionString = "removed"
+	case eventActionPackageUpdated:
+		actionString = "updated"
 	}
 
 	return fmt.Sprintf("Package %v (%v) %v",
@@ -444,16 +487,24 @@ func listDebPackages() ([]*Package, error) {
 	defer file.Close()
 
 	var packages []*Package
+	var skipPackage bool
 	pkg := &Package{}
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		line := scanner.Text()
 		if len(strings.TrimSpace(line)) == 0 {
 			// empty line signals new package
-			packages = append(packages, pkg)
+			if !skipPackage {
+				packages = append(packages, pkg)
+			}
+			skipPackage = false
 			pkg = &Package{}
 			continue
+		} else if skipPackage {
+			// Skipping this package - read on.
+			continue
 		}
+
 		if strings.HasPrefix(line, " ") {
 			// not interested in multi-lines for now
 			continue
@@ -466,6 +517,11 @@ func listDebPackages() ([]*Package, error) {
 		switch strings.ToLower(words[0]) {
 		case "package":
 			pkg.Name = value
+		case "status":
+			if strings.HasPrefix(value, "deinstall ok") {
+				// Package was removed but not purged. We report both cases as removed.
+				skipPackage = true
+			}
 		case "architecture":
 			pkg.Arch = value
 		case "version":
@@ -482,7 +538,7 @@ func listDebPackages() ([]*Package, error) {
 		}
 	}
 	if err = scanner.Err(); err != nil {
-		return nil, errors.Wrap(err, "error scanning file")
+		return nil, errors.Wrapf(err, "error scanning file %v", dpkgStatusFile)
 	}
 	return packages, nil
 }

x-pack/auditbeat/tests/system/test_metricsets.py

@@ -48,7 +48,7 @@ def test_metricset_package(self):
         package metricset collects information about installed packages on a system.
         """
 
-        fields = ["system.audit.package.name", "system.audit.package.version", "system.audit.package.installtime"]
+        fields = ["system.audit.package.name", "system.audit.package.version"]
 
         # Metricset is experimental and that generates a warning, TODO: remove later
         self.check_metricset("system", "package", COMMON_FIELDS + fields, warnings_allowed=True)