Skip to content

Commit

Permalink
Harden system/package parsing of deb size
Browse files Browse the repository at this point in the history 
Deb packages report their installed size in a field, Installed-Size,
which is an integer interpreted as KiB. Some unoficial packages are
adding a unit at the end of this field:

Installed-Size: 65M

Dpkg/Apt ignore everything after the number. Auditbeat is currently
failing to parse the list of packages once this mistake is reached.

This updates the dataset to:
- Do not fail when installed size can't be parsed.
- Understand prefixes k/K, m/M and G/b.

Fixes elastic#16661
  • Loading branch information
@adriansr
adriansr committed Mar 23, 2020
1 parent c65648a commit 18695e9
Show file tree
Hide file tree
Showing 3 changed files with 300 additions and 4 deletions.
48 changes: 44 additions & 4 deletions x-pack/auditbeat/module/system/package/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
if err == nil {
foundPackageManager = true

dpkgPackages, err := listDebPackages()
dpkgPackages, err := ms.listDebPackages()
if err != nil {
return nil, errors.Wrap(err, "error getting DEB packages")
}
Expand Down Expand Up @@ -499,7 +499,7 @@ func (ms *MetricSet) getPackages() (packages []*Package, err error) {
return packages, nil
}

func listDebPackages() ([]*Package, error) {
func (ms *MetricSet) listDebPackages() ([]*Package, error) {
dpkgStatusFile := filepath.Join(dpkgPath, "status")

file, err := os.Open(dpkgStatusFile)
Expand Down Expand Up @@ -556,9 +556,14 @@ func listDebPackages() ([]*Package, error) {
case "description":
pkg.Summary = value
case "installed-size":
pkg.Size, err = strconv.ParseUint(value, 10, 64)
pkg.Size, err = parseDpkgInstalledSize(value)
if err != nil {
return nil, errors.Wrapf(err, "error converting %s to int", value)
// If installed size is invalid, log a warning but still
// report the package with size=0.
ms.log.Warnw("Failed parsing installed size",
"package", pkg.Name,
"Installed-Size", value,
"Error", err)
}
case "homepage":
pkg.URL = value
Expand All @@ -578,3 +583,38 @@ func listDebPackages() ([]*Package, error) {

return packages, nil
}

func parseDpkgInstalledSize(value string) (size uint64, err error) {
// Installed-Size is an integer (KiB).
if size, err = strconv.ParseUint(value, 10, 64); err == nil {
return size, err
}

// Some rare third-party packages contain a unit at the end. This is ignored
// by dpkg tools. Try to parse to return as value as accurate as possible
// to what the package maintainer meant.
end := len(value)
for idx, chr := range value {
if chr < '0' || chr > '9' {
end = idx
break
}
}
var multiplier uint64
if end < len(value) {
switch value[end] {
case 'm', 'M':
multiplier = 1024
case 'g', 'G':
multiplier = 1024 * 1024
default:
multiplier = 1
}
}
if size, err = strconv.ParseUint(value[:end], 10, 64); err == nil {
return size * multiplier, nil
}

// Otherwise just return size=0 and an error.
return size, err
}
66 changes: 66 additions & 0 deletions x-pack/auditbeat/module/system/package/package_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,78 @@ func TestDpkg(t *testing.T) {
if assert.Len(t, events, 1) {
event := mbtest.StandardizeEvent(f, events[0], core.AddDatasetToEvent)
checkFieldValue(t, event, "system.audit.package.name", "test")
checkFieldValue(t, event, "system.audit.package.size", uint64(269))
checkFieldValue(t, event, "system.audit.package.summary", "Test Package")
checkFieldValue(t, event, "system.audit.package.url", "https://www.elastic.co/")
checkFieldValue(t, event, "system.audit.package.version", "8.2.0-1ubuntu2~18.04")
}
}

func TestDpkgInstalledSize(t *testing.T) {
expected := map[string]uint64{
"libquadmath0": 269,
"python-apt-common": 248,
"libnpth0": 32,
"bind9-host": 17,
"libpam-runtime": 300,
"libsepol1-dev": 1739 * 1024,
"libisl19": 17 * 1024 * 1024,
"netbase": 0,
"python2.7-minimal": 3685,
}

logp.TestingSetup()

defer abtest.SetupDataDir(t)()

// Disable all except dpkg
rpmPathOld := rpmPath
dpkgPathOld := dpkgPath
brewPathOld := homebrewCellarPath
defer func() {
rpmPath = rpmPathOld
dpkgPath = dpkgPathOld
homebrewCellarPath = brewPathOld
}()
rpmPath = "/does/not/exist"
homebrewCellarPath = "/does/not/exist"

var err error
dpkgPath, err = filepath.Abs("testdata/dpkg-size/")
if err != nil {
t.Fatal(err)
}

f := mbtest.NewReportingMetricSetV2(t, getConfig())
defer f.(*MetricSet).bucket.DeleteBucket()

events, errs := mbtest.ReportingFetchV2(f)
if len(errs) > 0 {
t.Fatalf("received error: %+v", errs[0])
}

got := make(map[string]uint64, len(events))
for _, ev := range events {
event := mbtest.StandardizeEvent(f, ev, core.AddDatasetToEvent)
name, err := event.GetValue("system.audit.package.name")
if err != nil {
t.Fatal(err)
}
size, err := event.GetValue("system.audit.package.size")
if err != nil {
size = uint64(0)
}
if !assert.IsType(t, "", name) {
t.Fatal("string expected")
}
if !assert.IsType(t, uint64(0), size) {
t.Fatal("uint64 expected")
}
got[name.(string)] = size.(uint64)
}
assert.Equal(t, expected, got)
}

func getConfig() map[string]interface{} {
return map[string]interface{}{
"module": "system",
Expand Down
190 changes: 190 additions & 0 deletions x-pack/auditbeat/module/system/package/testdata/dpkg-size/status
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
Package: libquadmath0
Status: install ok installed
Priority: optional
Section: libs
Installed-Size: 269B
Maintainer: Ubuntu Core developers <[email protected]>
Architecture: amd64
Multi-Arch: same
Source: gcc-8
Version: 8.3.0-26ubuntu1~18.04
Depends: gcc-8-base (= 8.3.0-26ubuntu1~18.04), libc6 (>= 2.23)
Description: GCC Quad-Precision Math Library
A library, which provides quad-precision mathematical functions on targets
supporting the __float128 datatype. The library is used to provide on such
targets the REAL(16) type in the GNU Fortran compiler.
Homepage: http://gcc.gnu.org/
Original-Maintainer: Debian GCC Maintainers <[email protected]>

Package: python-apt-common
Status: install ok installed
Priority: optional
Section: python
Installed-Size: 248KiB
Maintainer: Ubuntu Developers <[email protected]>
Architecture: all
Source: python-apt
Version: 1.6.5ubuntu0.2
Replaces: python-apt (<< 0.7.98+nmu1)
Breaks: python-apt (<< 0.7.98+nmu1)
Enhances: python-apt, python3-apt
Description: Python interface to libapt-pkg (locales)
The apt_pkg Python interface will provide full access to the internal
libapt-pkg structures allowing Python programs to easily perform a
variety of functions.
.
This package contains locales.
Original-Maintainer: APT Development Team <[email protected]>

Package: libnpth0
Status: install ok installed
Priority: optional
Section: libs
Installed-Size: 32trash
Maintainer: Ubuntu Developers <[email protected]>
Architecture: amd64
Multi-Arch: same
Source: npth
Version: 1.5-3
Depends: libc6 (>= 2.17)
Description: replacement for GNU Pth using system threads
nPth is a non-preemptive threads implementation using an API very
similar to the one known from GNU Pth. It has been designed as a
replacement of GNU Pth for non-ancient operating systems. In
contrast to GNU Pth it is based on the system's standard threads
implementation. Thus nPth allows the use of libraries which are not
compatible to GNU Pth.
Original-Maintainer: Eric Dorland <[email protected]>
Homepage: https://www.gnupg.org/

Package: bind9-host
Status: install ok installed
Priority: standard
Section: net
Installed-Size: 17.4
Maintainer: Ubuntu Developers <[email protected]>
Architecture: amd64
Source: bind9
Version: 1:9.11.3+dfsg-1ubuntu1.11
Provides: host
Depends: libbind9-160 (= 1:9.11.3+dfsg-1ubuntu1.11), libdns1100 (= 1:9.11.3+dfsg-1ubuntu1.11), libisc169 (= 1:9.11.3+dfsg-1ubuntu1.11), libisccfg160 (= 1:9.11.3+dfsg-1ubuntu1.11), liblwres160 (= 1:9.11.3+dfsg-1ubuntu1.11), libc6 (>= 2.4)
Description: DNS lookup utility (deprecated)
This package provides /usr/bin/host, a simple utility (bundled with the
BIND 9.X sources) which can be used for converting domain names to IP
addresses and the reverse.
.
This utility is deprecated, use dig or delv from the dnsutils package.
Homepage: https://www.isc.org/downloads/bind/
Original-Maintainer: Debian DNS Packaging <[email protected]>

Package: libpam-runtime
Status: install ok installed
Priority: required
Section: admin
Installed-Size: 300T
Maintainer: Ubuntu Developers <[email protected]>
Architecture: all
Multi-Arch: foreign
Source: pam
Version: 1.1.8-3.6ubuntu2
Replaces: libpam0g-dev, libpam0g-util
Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
Conflicts: libpam0g-util
Conffiles:
/etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
/etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
Description: Runtime support for the PAM library
Contains configuration files and directories required for
authentication to work on Debian systems. This package is required
on almost all installations.
Homepage: http://www.linux-pam.org/
Original-Maintainer: Steve Langasek <[email protected]>

Package: libsepol1-dev
Status: install ok installed
Priority: optional
Section: libdevel
Installed-Size: 1739m
Maintainer: Ubuntu Developers <[email protected]>
Architecture: amd64
Multi-Arch: same
Source: libsepol
Version: 2.7-1
Provides: libsepol-dev
Depends: libsepol1 (= 2.7-1)
Conflicts: libsepol-dev
Description: SELinux binary policy manipulation library and development files
libsepol allows programs to easily modify SELinux binary policies. This
means changing the default values for booleans, or reading the policy for
analysis.
.
This package contains the headers and archives used for linking it into your
programs.
Original-Maintainer: Debian SELinux maintainers <[email protected]>
Homepage: http://userspace.selinuxproject.org/

Package: libisl19
Status: install ok installed
Priority: optional
Section: libs
Installed-Size: 17G
Maintainer: Ubuntu Developers <[email protected]>
Architecture: amd64
Multi-Arch: same
Source: isl
Version: 0.19-1
Replaces: libisl-dbg (<< 0.19)
Depends: libc6 (>= 2.14), libgmp10
Breaks: libisl-dbg (<< 0.19)
Description: manipulating sets and relations of integer points bounded by linear constraints
isl is a library for manipulating sets and relations of integer points
bounded by linear constraints. Supported operations on sets include
intersection, union, set difference, emptiness check, convex hull,
(integer) affine hull, integer projection, and computing the lexicographic
minimum using parametric integer programming. It also includes an ILP solver
based on generalized basis reduction.
.
This package contains the runtime library.
Original-Maintainer: Debian GCC Maintainers <[email protected]>
Homepage: http://isl.gforge.inria.fr/

Package: netbase
Status: install ok installed
Priority: important
Section: admin
Maintainer: Ubuntu Developers <[email protected]>
Architecture: all
Multi-Arch: foreign
Version: 5.4
Conffiles:
/etc/protocols bb9c019d6524e913fd72441d58b68216
/etc/rpc f0b6f6352bf886623adc04183120f83b
/etc/services 567c100888518c1163b3462993de7d47
Description: Basic TCP/IP networking system
This package provides the necessary infrastructure for basic TCP/IP based
networking.
Original-Maintainer: Marco d'Itri <[email protected]>

Package: python2.7-minimal
Status: install ok installed
Priority: optional
Section: python
Installed-Size: 3685
Maintainer: Ubuntu Developers <[email protected]>
Architecture: amd64
Multi-Arch: allowed
Source: python2.7
Version: 2.7.17-1~18.04
Replaces: python2.7 (<< 2.7.8-7~)
Depends: libpython2.7-minimal (= 2.7.17-1~18.04)
Pre-Depends: libc6 (>= 2.15), zlib1g (>= 1:1.2.0)
Recommends: python2.7
Suggests: binfmt-support
Conflicts: binfmt-support (<< 1.1.2)
Description: Minimal subset of the Python language (version 2.7)
This package contains the interpreter and some essential modules. It can
be used in the boot process for some basic tasks.
See /usr/share/doc/python2.7-minimal/README.Debian for a list of the modules
contained in this package.
Original-Maintainer: Matthias Klose <[email protected]>

0 comments on commit 18695e9

Please sign in to comment.