From 09cb9bc27eddf445065582e3b71747ca2d15da7f Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Mon, 20 Nov 2023 17:51:28 +0100 Subject: [PATCH] CodeReview: rename, add a method to update intervals, add info to the payload, rephrase comments --- comp/README.md | 4 +- comp/metadata/bundle.go | 4 +- .../internal/util/inventory_payload.go | 16 +-- .../internal/util/inventory_payload_test.go | 1 - .../metadata/inventoryagent/inventoryagent.go | 2 +- .../inventoryhostimpl/inventoryhost.go | 11 +- .../inventoryhostimpl/inventoryhost_test.go | 74 ++++++------- comp/metadata/inventorysigning/README.md | 42 -------- .../inventorysigningimpl/commonsigning.go | 88 --------------- comp/metadata/packagesigning/README.md | 44 ++++++++ .../component.go | 6 +- .../component_mock.go | 4 +- .../packagesigningimpl/aptsigning.go} | 96 ++++++++++++----- .../packagesigningimpl/commonsigning.go | 77 +++++++++++++ .../packagesigningimpl/packagesigning.go} | 53 +++++---- .../packagesigningimpl/yumsigning.go} | 102 +++++++++--------- ...geSigning-component-f1487f8d92f4e749.yaml} | 2 +- 17 files changed, 334 insertions(+), 292 deletions(-) delete mode 100644 comp/metadata/inventorysigning/README.md delete mode 100644 comp/metadata/inventorysigning/inventorysigningimpl/commonsigning.go create mode 100644 comp/metadata/packagesigning/README.md rename comp/metadata/{inventorysigning => packagesigning}/component.go (63%) rename comp/metadata/{inventorysigning => packagesigning}/component_mock.go (74%) rename comp/metadata/{inventorysigning/inventorysigningimpl/debiansigning.go => packagesigning/packagesigningimpl/aptsigning.go} (58%) create mode 100644 comp/metadata/packagesigning/packagesigningimpl/commonsigning.go rename comp/metadata/{inventorysigning/inventorysigningimpl/inventorysigning.go => packagesigning/packagesigningimpl/packagesigning.go} (67%) rename comp/metadata/{inventorysigning/inventorysigningimpl/redhatsigning.go => packagesigning/packagesigningimpl/yumsigning.go} (62%) rename releasenotes/notes/{add-a-new-inventorySigning-component-to-collect-Linux-package-signature-information-f1487f8d92f4e749.yaml => add-new-packageSigning-component-f1487f8d92f4e749.yaml} (74%) diff --git a/comp/README.md b/comp/README.md index d0b7edb266e071..4c9242e5e72eb4 100644 --- a/comp/README.md +++ b/comp/README.md @@ -150,11 +150,11 @@ Package inventoryagent implements a component to generate the 'datadog_agent' me Package inventoryhost exposes the interface for the component to generate the 'host_metadata' metadata payload for inventory. -### [comp/metadata/inventorysigning](https://pkg.go.dev/github.com/DataDog/dd-agent-comp-experiments/comp/metadata/inventorysigning) +### [comp/metadata/packagesigning](https://pkg.go.dev/github.com/DataDog/dd-agent-comp-experiments/comp/metadata/packagesigning) *Datadog Team*: agent-platform -Package inventorysigning implements a component to generate the 'signing' metadata payload for DD inventory (REDAPL). +Package packagesigning implements a component to generate the 'signing' metadata payload for DD inventory (REDAPL). ### [comp/metadata/resources](https://pkg.go.dev/github.com/DataDog/dd-agent-comp-experiments/comp/metadata/resources) diff --git a/comp/metadata/bundle.go b/comp/metadata/bundle.go index eaae6e44407cbd..5f70ace35d9eb7 100644 --- a/comp/metadata/bundle.go +++ b/comp/metadata/bundle.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/host" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent" "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost/inventoryhostimpl" - "github.com/DataDog/datadog-agent/comp/metadata/inventorysigning/inventorysigningimpl" + "github.com/DataDog/datadog-agent/comp/metadata/packagesigning/packagesigningimpl" "github.com/DataDog/datadog-agent/comp/metadata/resources/resourcesimpl" "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -26,7 +26,7 @@ var Bundle = fxutil.Bundle( host.Module, inventoryagent.Module, inventoryhostimpl.Module, - inventorysigningimpl.Module, + packagesigningimpl.Module, ) // MockBundle defines the mock fx options for this bundle. diff --git a/comp/metadata/internal/util/inventory_payload.go b/comp/metadata/internal/util/inventory_payload.go index ce85c36fae666a..fca71c59d35525 100644 --- a/comp/metadata/internal/util/inventory_payload.go +++ b/comp/metadata/internal/util/inventory_payload.go @@ -102,18 +102,14 @@ type InventoryPayload struct { // CreateInventoryPayload returns an initialized InventoryPayload. 'getPayload' will be called each time a new payload // needs to be generated. -func CreateInventoryPayload(conf config.Component, l log.Component, s serializer.MetricSerializer, getPayload PayloadGetter, flareFileName string, customInterval time.Duration) InventoryPayload { +func CreateInventoryPayload(conf config.Component, l log.Component, s serializer.MetricSerializer, getPayload PayloadGetter, flareFileName string) InventoryPayload { minInterval := time.Duration(conf.GetInt("inventories_min_interval")) * time.Second - if customInterval > 0 { - minInterval = customInterval - } else if minInterval <= 0 { + if minInterval <= 0 { minInterval = defaultMinInterval } maxInterval := time.Duration(conf.GetInt("inventories_max_interval")) * time.Second - if customInterval > 0 { - maxInterval = customInterval - } else if maxInterval <= 0 { + if maxInterval <= 0 { maxInterval = defaultMaxInterval } @@ -204,3 +200,9 @@ func (i *InventoryPayload) fillFlare(fb flaretypes.FlareBuilder) error { fb.AddFileFromFunc(path, i.GetAsJSON) return nil } + +// SetIntervals update the default intervals between two payloads. +func (i *InventoryPayload) SetIntervals(minInterval, maxInterval time.Duration) { + defaultMinInterval = minInterval + defaultMaxInterval = maxInterval +} diff --git a/comp/metadata/internal/util/inventory_payload_test.go b/comp/metadata/internal/util/inventory_payload_test.go index 9545d01f88ac12..1972ae1bbe6d42 100644 --- a/comp/metadata/internal/util/inventory_payload_test.go +++ b/comp/metadata/internal/util/inventory_payload_test.go @@ -41,7 +41,6 @@ func getTestInventoryPayload(t *testing.T, confOverrides map[string]any) *Invent &serializer.MockSerializer{}, func() marshaler.JSONMarshaler { return &testPayload{} }, "test.json", - time.Duration(0), ) return &i } diff --git a/comp/metadata/inventoryagent/inventoryagent.go b/comp/metadata/inventoryagent/inventoryagent.go index 77c0038051e0e9..d3725242ba8110 100644 --- a/comp/metadata/inventoryagent/inventoryagent.go +++ b/comp/metadata/inventoryagent/inventoryagent.go @@ -91,7 +91,7 @@ func newInventoryAgentProvider(deps dependencies) provides { hostname: hname, data: make(agentMetadata), } - ia.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, ia.getPayload, "agent.json", time.Duration(0)) + ia.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, ia.getPayload, "agent.json") if ia.Enabled { ia.initData() diff --git a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go index 0ddffba5527786..97cfad7a9eaeda 100644 --- a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go +++ b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "fmt" - "runtime" "time" "github.com/DataDog/datadog-agent/comp/core/config" @@ -19,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/host/utils" "github.com/DataDog/datadog-agent/comp/metadata/internal/util" "github.com/DataDog/datadog-agent/comp/metadata/inventoryhost" + "github.com/DataDog/datadog-agent/comp/metadata/packagesigning/packagesigningimpl" "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" "github.com/DataDog/datadog-agent/pkg/gohai/cpu" "github.com/DataDog/datadog-agent/pkg/gohai/memory" @@ -92,7 +92,7 @@ type hostMetadata struct { DmiBoardVendor string `json:"dmi_board_vendor"` // from package repositories - GPGSigningEnabled bool `json:"gpg_signing_enabled"` + LinuxPackageSigningEnabled bool `json:"linux_package_signing_enabled"` } // Payload handles the JSON unmarshalling of the metadata payload @@ -148,7 +148,7 @@ func newInventoryHostProvider(deps dependencies) provides { hostname: hname, data: &hostMetadata{}, } - ih.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, ih.getPayload, "host.json", time.Duration(0)) + ih.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, ih.getPayload, "host.json") return provides{ Comp: ih, @@ -235,10 +235,7 @@ func (ih *invHost) fillData() { ih.data.CloudProviderHostID = cloudproviders.GetHostID(context.Background(), cloudProvider) ih.data.OsVersion = osVersionGet() - ih.data.GPGSigningEnabled = false - if runtime.GOOS == "linux" { - ih.data.GPGSigningEnabled = true // TODO implement gpgCheck parsing debian/redhat - } + ih.data.LinuxPackageSigningEnabled = packagesigningimpl.GetLinuxPackageSigningPolicy() } func (ih *invHost) getPayload() marshaler.JSONMarshaler { diff --git a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost_test.go b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost_test.go index 687730e5b4da9c..4f4ba8fc6a8ebf 100644 --- a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost_test.go +++ b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost_test.go @@ -127,36 +127,36 @@ func TestGetPayload(t *testing.T) { setupHostMetadataMock(t) expectedMetadata := &hostMetadata{ - CPUCores: 6, - CPULogicalProcessors: 6, - CPUVendor: "GenuineIntel", - CPUModel: "Intel_i7-8750H", - CPUModelID: "158", - CPUFamily: "6", - CPUStepping: "10", - CPUFrequency: 2208.006, - CPUCacheSize: 9437184, - KernelName: "Linux", - KernelRelease: "5.17.0-1-amd64", - KernelVersion: "Debian_5.17.3-1", - OS: "GNU/Linux", - CPUArchitecture: "unknown", - MemoryTotalKb: 1205632, - MemorySwapTotalKb: 1205632, - IPAddress: "192.168.24.138", - IPv6Address: "fe80::20c:29ff:feb6:d232", - MacAddress: "00:0c:29:b6:d2:32", - AgentVersion: version.AgentVersion, - CloudProvider: "some_cloud_provider", - CloudProviderAccountID: "some_host_id", - CloudProviderSource: "test_source", - CloudProviderHostID: "test_id_1234", - OsVersion: "testOS", - HypervisorGuestUUID: "hypervisorUUID", - DmiProductUUID: "dmiUUID", - DmiBoardAssetTag: "boardTag", - DmiBoardVendor: "boardVendor", - GPGSigningEnabled: true, + CPUCores: 6, + CPULogicalProcessors: 6, + CPUVendor: "GenuineIntel", + CPUModel: "Intel_i7-8750H", + CPUModelID: "158", + CPUFamily: "6", + CPUStepping: "10", + CPUFrequency: 2208.006, + CPUCacheSize: 9437184, + KernelName: "Linux", + KernelRelease: "5.17.0-1-amd64", + KernelVersion: "Debian_5.17.3-1", + OS: "GNU/Linux", + CPUArchitecture: "unknown", + MemoryTotalKb: 1205632, + MemorySwapTotalKb: 1205632, + IPAddress: "192.168.24.138", + IPv6Address: "fe80::20c:29ff:feb6:d232", + MacAddress: "00:0c:29:b6:d2:32", + AgentVersion: version.AgentVersion, + CloudProvider: "some_cloud_provider", + CloudProviderAccountID: "some_host_id", + CloudProviderSource: "test_source", + CloudProviderHostID: "test_id_1234", + OsVersion: "testOS", + HypervisorGuestUUID: "hypervisorUUID", + DmiProductUUID: "dmiUUID", + DmiBoardAssetTag: "boardTag", + DmiBoardVendor: "boardVendor", + LinuxPackageSigningEnabled: true, } ih := getTestInventoryHost(t) @@ -172,13 +172,13 @@ func TestGetPayloadError(t *testing.T) { p := ih.getPayload().(*Payload) expected := &hostMetadata{ - AgentVersion: version.AgentVersion, - CloudProvider: "some_cloud_provider", - CloudProviderAccountID: "some_host_id", - CloudProviderSource: "test_source", - CloudProviderHostID: "test_id_1234", - OsVersion: "testOS", - GPGSigningEnabled: true, + AgentVersion: version.AgentVersion, + CloudProvider: "some_cloud_provider", + CloudProviderAccountID: "some_host_id", + CloudProviderSource: "test_source", + CloudProviderHostID: "test_id_1234", + OsVersion: "testOS", + LinuxPackageSigningEnabled: true, } assert.Equal(t, expected, p.Metadata) } diff --git a/comp/metadata/inventorysigning/README.md b/comp/metadata/inventorysigning/README.md deleted file mode 100644 index d9cd8b1ec3f78a..00000000000000 --- a/comp/metadata/inventorysigning/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Inventory Signing Payload - -This component populates the Linux Packages signing information in the `inventories` product in DataDog. They fill the `host_signing` table. - -This is enabled by default but can be turned off using `inventories_enabled` config. - -The payload is sent every 12h (see `inventories_max_interval` in the config) or whenever it's updated with at most 1 -update every 5 minutes (see `inventories_min_interval`). - -# Content - -The `Set` method from the component allows the rest of the codebase to add any information to the payload. - -# Format - -The payload is a JSON dict with a list of keys, each having the following fields - -- `fingerprint` - **string**: the 8-char long key fingerprint. -- `expiration_date` - **string**: the expiration date of the key. -- `key_type` - **string**: the type of key. which represents how it is referenced in the host. Possible values are "signed-by", "trusted" or "debsig" for debianoids, "repo" or "rpm" for redhat-like (including SUSE) - - -## Example Payload - -Here an example of an signing inventory payload: - -``` -{ - "signing_keys": [ - { - "fingerprint": "12345ABC", - "expiration_date": "2023-02-12", - "key_type": "trusted", - }, - { - "fingerprint": "DEF90874", - "expiration_date": "9999-12-24", - "key_type": "debsig", - } - ] -} -``` diff --git a/comp/metadata/inventorysigning/inventorysigningimpl/commonsigning.go b/comp/metadata/inventorysigning/inventorysigningimpl/commonsigning.go deleted file mode 100644 index 91f5151e6cb79c..00000000000000 --- a/comp/metadata/inventorysigning/inventorysigningimpl/commonsigning.go +++ /dev/null @@ -1,88 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package inventorysigningimpl - -import ( - "bufio" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" -) - -// SigningKey represents relevant fields for a package signature key -type SigningKey struct { - Fingerprint string `json:"signing_key_fingerprint"` - ExpirationDate string `json:"signing_key_expiration_date"` - KeyType string `json:"signing_key_type"` -} - -// fileEntry implements the os.DirEntry interface to generate a list of files from string -type fileEntry struct { - name string -} - -func (fe fileEntry) Name() string { - return fe.name -} -func (fe fileEntry) IsDir() bool { - return false -} -func (fe fileEntry) Type() os.FileMode { - return 0 -} -func (fe fileEntry) Info() (os.FileInfo, error) { - return nil, nil -} - -func getKeysFromGPGFiles(files []os.DirEntry, parent string, keyType string) []SigningKey { - var keys []SigningKey - for _, file := range files { - keys = decryptGPGFile("gpg "+filepath.Join(parent, file.Name()), keyType) - - } - return keys -} - -func decryptGPGFile(gpgCommand string, keyType string) []SigningKey { - cmd := exec.Command("bash", "-c", gpgCommand) - output, err := cmd.Output() - if err != nil { - return nil - } - - // Create regular expressions for date and key ID extraction - dateRegex := regexp.MustCompile(`[0-9]{4}-[0-9]{2}-[0-9]{2}`) - keyIDRegex := regexp.MustCompile(`[A-Z0-9]+`) - - var keys []SigningKey - scanner := bufio.NewScanner(strings.NewReader(string(output))) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "pub") { - expDate := "9999-12-31" - dateList := dateRegex.FindAllString(line, -1) - if len(dateList) > 1 { - expDate = dateList[1] - } - // Read the next line to extract the key ID - if scanner.Scan() { - keyID := keyIDRegex.FindString(scanner.Text()) - keys = append(keys, SigningKey{ - Fingerprint: keyID, - ExpirationDate: expDate, - KeyType: keyType, - }) - } - } - } - - if err := scanner.Err(); err != nil { - return nil - } - return keys -} diff --git a/comp/metadata/packagesigning/README.md b/comp/metadata/packagesigning/README.md new file mode 100644 index 00000000000000..0db2162b1e0c96 --- /dev/null +++ b/comp/metadata/packagesigning/README.md @@ -0,0 +1,44 @@ +# Package Signing Payload + +This component populates the Linux Packages signing information in the `inventories` product in DataDog. They fill the `host_signing` table. + +This is enabled by default but can be turned off using `inventories_enabled` config. + +The payload is sent every 12h (see `inventories_max_interval` in the config). It has a different collection timeframe than the other inventory payloads. + +# Format + +The payload is a JSON dict with a list of keys, each having the following fields + +- `hostname` - **string**: the hostname of the agent as shown on the status page. +- `timestamp` - **int**: the timestamp when the payload was created. +- `agent_version` - **string**: the version of the Agent. +- `signing_keys` - **list of dict of string to JSON type** + - `fingerprint` - **string**: the 8-char long key fingerprint. + - `expiration_date` - **string**: the expiration date of the key. + - `key_type` - **string**: the type of key. which represents how it is referenced in the host. Possible values are "signed-by", "trusted" or "debsig" for debianoids, "repo" or "rpm" for redhat-like (including SUSE) + + +## Example Payload + +Here an example of an signing inventory payload: + +``` +{ + "hostname": "totoro", + "timestamp": 1631281754507358895, + "agent_version: "7.50.0", + "signing_keys": [ + { + "fingerprint": "12345ABC", + "expiration_date": "2023-02-12", + "key_type": "trusted", + }, + { + "fingerprint": "DEF90874", + "expiration_date": "9999-12-24", + "key_type": "debsig", + } + ] +} +``` diff --git a/comp/metadata/inventorysigning/component.go b/comp/metadata/packagesigning/component.go similarity index 63% rename from comp/metadata/inventorysigning/component.go rename to comp/metadata/packagesigning/component.go index 180adc4323c163..f11439e287d1fc 100644 --- a/comp/metadata/inventorysigning/component.go +++ b/comp/metadata/packagesigning/component.go @@ -3,8 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package inventorysigning implements a component to generate the 'signing' metadata payload for DD inventory (REDAPL). -package inventorysigning +// Package packagesigning implements a component to generate the 'signing' metadata payload for DD inventory (REDAPL). +package packagesigning // team: agent-platform @@ -12,6 +12,4 @@ package inventorysigning type Component interface { // GetAsJSON returns the payload as a JSON string. Useful to be displayed in the CLI or added to a flare. GetAsJSON() ([]byte, error) - // Refresh trigger a new payload to be send while still respecting the minimal interval between two updates. - Refresh() } diff --git a/comp/metadata/inventorysigning/component_mock.go b/comp/metadata/packagesigning/component_mock.go similarity index 74% rename from comp/metadata/inventorysigning/component_mock.go rename to comp/metadata/packagesigning/component_mock.go index bc0f2ee41b8a6e..1ef64644cd31fe 100644 --- a/comp/metadata/inventorysigning/component_mock.go +++ b/comp/metadata/packagesigning/component_mock.go @@ -5,9 +5,9 @@ //go:build test -package inventorysigning +package packagesigning -// Mock implements mock-specific methods for the inventorysigning component. +// Mock implements mock-specific methods for the packagesigning component. type Mock interface { Component } diff --git a/comp/metadata/inventorysigning/inventorysigningimpl/debiansigning.go b/comp/metadata/packagesigning/packagesigningimpl/aptsigning.go similarity index 58% rename from comp/metadata/inventorysigning/inventorysigningimpl/debiansigning.go rename to comp/metadata/packagesigning/packagesigningimpl/aptsigning.go index 4c3ee38c932338..f75d6713a5d5f2 100644 --- a/comp/metadata/inventorysigning/inventorysigningimpl/debiansigning.go +++ b/comp/metadata/packagesigning/packagesigningimpl/aptsigning.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package inventorysigningimpl +package packagesigningimpl import ( "bufio" @@ -13,32 +13,55 @@ import ( "regexp" ) -// GetDebianSignatureKeys returns the list of debian signature keys -func GetDebianSignatureKeys() []SigningKey { +// getNoDebsig returns the signature policy for the host. no-debsig means GPG check is enabled +func getNoDebsig() bool { + packageConfig := "/etc/dpkg/dpkg.cfg" + if _, err := os.Stat(packageConfig); os.IsExist(err) { + if file, err := os.Open(packageConfig); err == nil { + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if scanner.Text() == "no-debsig" { + return true + } + } + } + } + return false +} + +// getAPTSignatureKeys returns the list of debian signature keys +func getAPTSignatureKeys() []SigningKey { + allKeys := make(map[SigningKey]struct{}) // debian 11 and ubuntu 22.04 will be the last using legacy trusted.gpg.d folder and trusted.gpg file - allKeys := getTrustedKeys() + getTrustedKeys(allKeys) // Regular files are referenced in the sources.list file by signed-by=filename - allKeys = append(allKeys, getKeysFromGPGFiles(getSignedByFiles(), "", "signed-by")...) + for _, file := range getSignedByFiles() { + decryptGPGFile(allKeys, file.Name(), "signed-by") + } // In APT we can also sign packages with debsig - allKeys = append(allKeys, getDebsigKeys()...) - return allKeys + getDebsigKeys(allKeys) + var keyList []SigningKey + for keys := range allKeys { + keyList = append(keyList, keys) + } + return keyList } -func getTrustedKeys() []SigningKey { - var allKeys []SigningKey +func getTrustedKeys(allkeys map[SigningKey]struct{}) { // debian 11 and ubuntu 22.04 will be the last using legacy trusted.gpg.d folder and trusted.gpg file trustedFolder := "/etc/apt/trusted.gpg.d/" if _, err := os.Stat(trustedFolder); os.IsExist(err) { if files, err := os.ReadDir(trustedFolder); err == nil { - allKeys = append(allKeys, getKeysFromGPGFiles(files, trustedFolder, "trusted")...) + for _, file := range files { + decryptGPGFile(allkeys, filepath.Join(trustedFolder, file.Name()), "trusted") + } } } - var trustedFiles []os.DirEntry - trustedFiles = append(trustedFiles, fileEntry{name: "/etc/apt/trusted.gpg"}) - if _, err := os.Stat(trustedFiles[0].Name()); os.IsExist(err) { - allKeys = append(allKeys, getKeysFromGPGFiles(trustedFiles, "", "trusted")...) + trustedFile := "/etc/apt/trusted.gpg" + if _, err := os.Stat(trustedFile); os.IsExist(err) { + decryptGPGFile(allkeys, trustedFile, "trusted") } - return allKeys } func getSignedByFiles() []os.DirEntry { @@ -59,22 +82,21 @@ func getSignedByFiles() []os.DirEntry { return signedByFiles } -func getDebsigKeys() []SigningKey { - var allKeys []SigningKey +func getDebsigKeys(allKeys map[SigningKey]struct{}) { debsigPolicies := "/etc/debsig/policies/" debsigKeyring := "/usr/share/debsig/keyrings/" // Search in the policy files if _, err := os.Stat(debsigPolicies); os.IsExist(err) { - if files, err := os.ReadDir(debsigPolicies); err == nil { - for _, file := range files { - if file.IsDir() { - if policyFiles, err := os.ReadDir(debsigPolicies + file.Name()); err != nil { + if debsigDirs, err := os.ReadDir(debsigPolicies); err == nil { + for _, debsigDir := range debsigDirs { + if debsigDir.IsDir() { + if policyFiles, err := os.ReadDir(filepath.Join(debsigPolicies, debsigDir.Name())); err == nil { for _, policyFile := range policyFiles { // Get the gpg file name from policy files - if debsigFile, _ := getDebsigFileFromPolicy(filepath.Join(debsigPolicies, file.Name(), policyFile.Name())); debsigFile != "" { - debsigFilePath := filepath.Join(debsigKeyring, file.Name(), debsigFile) + if debsigFile := getDebsigFileFromPolicy(filepath.Join(debsigPolicies, debsigDir.Name(), policyFile.Name())); debsigFile != "" { + debsigFilePath := filepath.Join(debsigKeyring, debsigDir.Name(), debsigFile) if _, err := os.Stat(debsigFilePath); os.IsExist(err) { - allKeys = append(allKeys, getKeysFromGPGFiles([]os.DirEntry{fileEntry{name: debsigFilePath}}, "", "debsig")...) + decryptGPGFile(allKeys, debsigFilePath, "debsig") } } } @@ -83,7 +105,24 @@ func getDebsigKeys() []SigningKey { } } } - return allKeys +} + +// fileEntry implements the os.DirEntry interface to generate a list of files from string +type fileEntry struct { + name string +} + +func (fe fileEntry) Name() string { + return fe.name +} +func (fe fileEntry) IsDir() bool { + return false +} +func (fe fileEntry) Type() os.FileMode { + return 0 +} +func (fe fileEntry) Info() (os.FileInfo, error) { + return nil, nil } func extractPattern(filePath string, pattern *regexp.Regexp) []os.DirEntry { @@ -105,6 +144,7 @@ func extractPattern(filePath string, pattern *regexp.Regexp) []os.DirEntry { return patterns } +// policy structure to unmarshall the policy files type policy struct { XMLName xml.Name `xml:"Policy"` Origin origin `xml:"Origin"` @@ -133,12 +173,12 @@ type verification struct { Required required `xml:"Required"` } -func getDebsigFileFromPolicy(policyFile string) (string, error) { +func getDebsigFileFromPolicy(policyFile string) string { if xmlData, err := os.ReadFile(policyFile); err == nil { var policy policy if err = xml.Unmarshal(xmlData, &policy); err == nil { - return policy.Verification.Required.File, nil + return policy.Verification.Required.File } } - return "", nil + return "" } diff --git a/comp/metadata/packagesigning/packagesigningimpl/commonsigning.go b/comp/metadata/packagesigning/packagesigningimpl/commonsigning.go new file mode 100644 index 00000000000000..734301e893b1f6 --- /dev/null +++ b/comp/metadata/packagesigning/packagesigningimpl/commonsigning.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package packagesigningimpl + +import ( + "io" + "net/http" + "os" + "strings" + "time" + + pgp "github.com/ProtonMail/go-crypto/openpgp" +) + +// SigningKey represents relevant fields for a package signature key +type SigningKey struct { + Fingerprint string `json:"signing_key_fingerprint"` + ExpirationDate string `json:"signing_key_expiration_date"` + KeyType string `json:"signing_key_type"` +} + +// getPackageManager is a lazy implementation to detect if we use APT or YUM (RH or SUSE) +func getPackageManager() string { + if _, err := os.Stat("/etc/apt"); os.IsExist(err) { + return "apt" + } else if _, err := os.Stat("/etc/yum"); os.IsExist(err) { + return "yum" + } + return "zypper" +} + +// decryptGPGFile parse a gpg file (local or http) and extract signing keys information +func decryptGPGFile(allKeys map[SigningKey]struct{}, gpgFile string, keyType string) { + var reader io.Reader + if strings.HasPrefix(gpgFile, "http") { + response, err := http.Get(gpgFile) + if err != nil { + return + } + defer response.Body.Close() + reader = response.Body + } else { + file, err := os.Open(gpgFile) + if err != nil { + return + } + defer file.Close() + reader = file + } + decryptGPGReader(allKeys, reader, keyType) +} + +// decryptGPGReader extract keys from a reader, useful for rpm extraction +func decryptGPGReader(allKeys map[SigningKey]struct{}, reader io.Reader, keyType string) { + keyList, err := pgp.ReadArmoredKeyRing(reader) + if err != nil { + return + } + for _, key := range keyList { + fingerprint := key.PrimaryKey.KeyIdString() + expDate := "9999-12-31" + i := key.PrimaryIdentity() + keyLifetime := i.SelfSignature.KeyLifetimeSecs + if keyLifetime != nil { + expiry := key.PrimaryKey.CreationTime.Add(time.Duration(*i.SelfSignature.KeyLifetimeSecs) * time.Second) + expDate = expiry.Format("2006-01-02") + } + allKeys[SigningKey{ + Fingerprint: fingerprint, + ExpirationDate: expDate, + KeyType: keyType, + }] = struct{}{} + } +} diff --git a/comp/metadata/inventorysigning/inventorysigningimpl/inventorysigning.go b/comp/metadata/packagesigning/packagesigningimpl/packagesigning.go similarity index 67% rename from comp/metadata/inventorysigning/inventorysigningimpl/inventorysigning.go rename to comp/metadata/packagesigning/packagesigningimpl/packagesigning.go index 4ca4a59b123fb4..093db75f4ba3ef 100644 --- a/comp/metadata/inventorysigning/inventorysigningimpl/inventorysigning.go +++ b/comp/metadata/packagesigning/packagesigningimpl/packagesigning.go @@ -3,8 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -// Package inventorysigningimpl implements the inventory signing component, to collect package signing keys. -package inventorysigningimpl +// Package packagesigningimpl implements the inventory signing component, to collect package signing keys. +package packagesigningimpl import ( "context" @@ -17,22 +17,21 @@ import ( flaretypes "github.com/DataDog/datadog-agent/comp/core/flare/types" "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/metadata/internal/util" - "github.com/DataDog/datadog-agent/comp/metadata/inventorysigning" + "github.com/DataDog/datadog-agent/comp/metadata/packagesigning" "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/serializer/marshaler" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/hostname" - "github.com/DataDog/datadog-agent/pkg/util/kernel" "go.uber.org/fx" ) // Module defines the fx options for this component. var Module = fxutil.Component( - fx.Provide(newInventorySigningProvider), + fx.Provide(newpackagesigningProvider), ) -const defaultCollectInterval = 86400 * time.Second +const defaultCollectInterval = 12 * time.Hour // Payload handles the JSON unmarshalling of the metadata payload type Payload struct { @@ -57,7 +56,7 @@ type signingMetadata struct { SigningKeys []SigningKey `json:"signing_keys"` } -type invSigning struct { +type pkgSigning struct { util.InventoryPayload log log.Component @@ -77,20 +76,21 @@ type dependencies struct { type provides struct { fx.Out - Comp inventorysigning.Component + Comp packagesigning.Component Provider runnerimpl.Provider FlareProvider flaretypes.Provider } -func newInventorySigningProvider(deps dependencies) provides { +func newpackagesigningProvider(deps dependencies) provides { hname, _ := hostname.Get(context.Background()) - is := &invSigning{ + is := &pkgSigning{ conf: deps.Config, log: deps.Log, hostname: hname, data: &signingMetadata{}, } - is.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, is.getPayload, "signing.json", time.Duration(defaultCollectInterval)) + is.InventoryPayload.SetIntervals(defaultCollectInterval, defaultCollectInterval) + is.InventoryPayload = util.CreateInventoryPayload(deps.Config, deps.Log, deps.Serializer, is.getPayload, "signing.json") return provides{ Comp: is, @@ -99,20 +99,19 @@ func newInventorySigningProvider(deps dependencies) provides { } } -func (is *invSigning) fillData() { +func (is *pkgSigning) fillData() { if runtime.GOOS == "linux" { - if platform, err := kernel.Platform(); err == nil { - switch platform { - case "debian", "ubuntu": - is.data.SigningKeys = GetDebianSignatureKeys() - default: // We are in linux OS, all other distros are redhat based - is.data.SigningKeys = GetRedhatSignatureKeys() - } + pkgManager := getPackageManager() + switch pkgManager { + case "apt": + is.data.SigningKeys = getAPTSignatureKeys() + default: // yum and zypper are similar + is.data.SigningKeys = getYUMSignatureKeys(pkgManager) } } } -func (is *invSigning) getPayload() marshaler.JSONMarshaler { +func (is *pkgSigning) getPayload() marshaler.JSONMarshaler { is.fillData() return &Payload{ @@ -121,3 +120,17 @@ func (is *invSigning) getPayload() marshaler.JSONMarshaler { Metadata: is.data, } } + +// GetLinuxPackageSigningPolicy returns the global GPG signing policy of the host +func GetLinuxPackageSigningPolicy() bool { + if runtime.GOOS == "linux" { + pkgManager := getPackageManager() + switch pkgManager { + case "apt": + return getNoDebsig() + default: // yum and zypper are similar + return getMainGPGCheck(pkgManager) + } + } + return true +} diff --git a/comp/metadata/inventorysigning/inventorysigningimpl/redhatsigning.go b/comp/metadata/packagesigning/packagesigningimpl/yumsigning.go similarity index 62% rename from comp/metadata/inventorysigning/inventorysigningimpl/redhatsigning.go rename to comp/metadata/packagesigning/packagesigningimpl/yumsigning.go index 1259c41d7ab467..55aed1442a7b36 100644 --- a/comp/metadata/inventorysigning/inventorysigningimpl/redhatsigning.go +++ b/comp/metadata/packagesigning/packagesigningimpl/yumsigning.go @@ -3,89 +3,91 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package inventorysigningimpl +package packagesigningimpl import ( "bufio" - "fmt" "os" "os/exec" "regexp" "strings" ) -// GetRedhatSignatureKeys returns the list of keys used to sign RPM packages -func GetRedhatSignatureKeys() []SigningKey { - allKeys := getKeysFromRepoFiles() - allKeys = append(allKeys, getKeysFromRPMDB()...) - return allKeys +// GetMainGPGCheck returns gpgcheck setting for [main] table +func getMainGPGCheck(distribution string) bool { + repoConfig, _ := getRepoPathFromDistribution(distribution) + gpgCheck, _ := parseRepoFile(repoConfig, false) + return gpgCheck } -func getKeysFromRepoFiles() []SigningKey { - var allKeys []SigningKey +// GetYUMSignatureKeys returns the list of keys used to sign RPM packages +func getYUMSignatureKeys(distribution string) []SigningKey { + allKeys := make(map[SigningKey]struct{}) + getKeysFromRepoFiles(allKeys, distribution) + getKeysFromRPMDB(allKeys) + var keyList []SigningKey + for keys := range allKeys { + keyList = append(keyList, keys) + } + return keyList +} + +func getKeysFromRepoFiles(allKeys map[SigningKey]struct{}, platform string) { var gpgCheck bool - var gpgFiles []gpgReference - repoConfig := "/etc/yum.conf" + var gpgFiles []string + repoConfig, repoFilesDir := getRepoPathFromDistribution(platform) + + // First parsing of the main config file if _, err := os.Stat(repoConfig); os.IsExist(err) { gpgCheck, gpgFiles = parseRepoFile(repoConfig, false) - for _, file := range gpgFiles { - var gpgCmd string - if file.isHTTP { - gpgCmd = fmt.Sprintf("gpg <(curl -S %s)", file.path) - } else { - gpgCmd = fmt.Sprintf("gpg %s", file.path) - } - allKeys = append(allKeys, decryptGPGFile(gpgCmd, "repo")...) + for _, gpgFile := range gpgFiles { + decryptGPGFile(allKeys, gpgFile, "repo") } } - repoFiles := "/etc/yum.repos.d/" - if _, err := os.Stat(repoFiles); os.IsExist(err) { - if files, err := os.ReadDir(repoFiles); err == nil { + // Then parsing of the repo files + if _, err := os.Stat(repoFilesDir); os.IsExist(err) { + if files, err := os.ReadDir(repoFilesDir); err == nil { for _, file := range files { _, gpgFiles = parseRepoFile(file.Name(), gpgCheck) - for _, f := range gpgFiles { - var gpgCmd string - if f.isHTTP { - gpgCmd = fmt.Sprintf("gpg <(curl -S %s)", f.path) - } else { - gpgCmd = fmt.Sprintf("gpg %s", f.path) - } - allKeys = append(allKeys, decryptGPGFile(gpgCmd, "repo")...) + for _, gpgFile := range gpgFiles { + decryptGPGFile(allKeys, gpgFile, "repo") } } } } - return allKeys +} +// getRepoPathFromDistribution returns the path to the configuration file and the path to the repository files for RH or SUSE based OS +func getRepoPathFromDistribution(distribution string) (string, string) { + if distribution == "suse" { + return "/etc/zypp/zypp.conf", "/etc/zypp/repos.d/" + } + return "/etc/yum.conf", "/etc/yum.repos.d/" } -func getKeysFromRPMDB() []SigningKey { - // TODO handle SUSE - var allKeys []SigningKey +func getKeysFromRPMDB(allKeys map[SigningKey]struct{}) { // It seems not possible to get the expiration date from rpmdb, so we extract the list of keys and call gpg cmd := exec.Command("rpm", "-qa", "gpg-pubkey*") output, err := cmd.Output() if err != nil { - return nil + return } scanner := bufio.NewScanner(strings.NewReader(string(output))) for scanner.Scan() { - gpgCmd := fmt.Sprintf("gpg <(rpm -qi %s --qf '%%{PUBKEYS}\n')", scanner.Text()) - allKeys = append(allKeys, decryptGPGFile(gpgCmd, "rpm")...) + rpmCmd := exec.Command("rpm", "-qi", "%s", "--qf", "'%%{PUBKEYS}\n'") + rpmKey, err := rpmCmd.Output() + if err != nil { + return + } + decryptGPGReader(allKeys, strings.NewReader(string(rpmKey)), "rpm") } - return allKeys -} - -type gpgReference struct { - path string - isHTTP bool } // parseRepoFile extracts information from yum repo files // Save the global gpgcheck value when encountering a [main] table (should only occur on `/etc/yum.conf`) // Match several entries in gpgkey field, either file references (file://) or http(s)://. From observations, // these reference can be separated either by space or by new line. We assume it possible to mix file and http references -func parseRepoFile(inputFile string, gpgConf bool) (bool, []gpgReference) { +func parseRepoFile(inputFile string, gpgConf bool) (bool, []string) { file, err := os.Open(inputFile) if err != nil { @@ -99,8 +101,8 @@ func parseRepoFile(inputFile string, gpgConf bool) (bool, []gpgReference) { gpgCheck := gpgConf localGpgCheck := false inMain := false - var gpgFiles []gpgReference - var localGpgFiles []gpgReference + var gpgFiles []string + var localGpgFiles []string defer file.Close() if err == nil { scanner := bufio.NewScanner(file) @@ -131,12 +133,12 @@ func parseRepoFile(inputFile string, gpgConf bool) (bool, []gpgReference) { signedFile := signedBy.FindAllStringSubmatch(line, -1) for _, match := range signedFile { if len(match) > 1 { - localGpgFiles = append(localGpgFiles, gpgReference{match[1], false}) + localGpgFiles = append(localGpgFiles, match[1]) } } urls := urlmatch.FindAllString(strings.TrimPrefix(line, "gpgkey="), -1) for _, url := range urls { - localGpgFiles = append(localGpgFiles, gpgReference{url, true}) + localGpgFiles = append(localGpgFiles, url) } // Scan other lines in case of multiple gpgkey for scanner.Scan() { @@ -159,12 +161,12 @@ func parseRepoFile(inputFile string, gpgConf bool) (bool, []gpgReference) { signedFile := signedBy.FindAllStringSubmatch(cont, -1) for _, match := range signedFile { if len(match) > 1 { - localGpgFiles = append(localGpgFiles, gpgReference{match[1], false}) + localGpgFiles = append(localGpgFiles, match[1]) } } urls := urlmatch.FindAllString(strings.TrimSpace(cont), -1) for _, url := range urls { - localGpgFiles = append(localGpgFiles, gpgReference{url, true}) + localGpgFiles = append(localGpgFiles, url) } } diff --git a/releasenotes/notes/add-a-new-inventorySigning-component-to-collect-Linux-package-signature-information-f1487f8d92f4e749.yaml b/releasenotes/notes/add-new-packageSigning-component-f1487f8d92f4e749.yaml similarity index 74% rename from releasenotes/notes/add-a-new-inventorySigning-component-to-collect-Linux-package-signature-information-f1487f8d92f4e749.yaml rename to releasenotes/notes/add-new-packageSigning-component-f1487f8d92f4e749.yaml index 804160f556dd44..4a818d42bfbe18 100644 --- a/releasenotes/notes/add-a-new-inventorySigning-component-to-collect-Linux-package-signature-information-f1487f8d92f4e749.yaml +++ b/releasenotes/notes/add-new-packageSigning-component-f1487f8d92f4e749.yaml @@ -8,4 +8,4 @@ --- features: - | - Creation of a new inventorySigning component to collect Linux package signature information, to improve signature rotation process. + Creation of a new packageSigning component, to collect Linux package signature information, to improve signature rotation process.