Merge branch 'main' into refactor-configurable-host-tags

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v45979046-89a73a40
+      - image: gcr.io/datadoghq/agent-circleci-runner:v46542806-c7a4a6be
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.gitlab-ci.yml

@@ -173,15 +173,15 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_BUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_WINBUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_ARMBUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v46542806-c7a4a6be
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
   DEB_GPG_KEY_ID: c0962c7d

comp/core/flare/helpers/send_flare.go

@@ -34,8 +34,9 @@ var (
 
 // any modification to this struct should also be applied to datadog-agent/test/fakeintake/server/body.go
 type flareResponse struct {
-	CaseID int    `json:"case_id,omitempty"`
-	Error  string `json:"error,omitempty"`
+	CaseID      int    `json:"case_id,omitempty"`
+	Error       string `json:"error,omitempty"`
+	RequestUUID string `json:"request_uuid,omitempty"`
 }
 
 // FlareSource has metadata about why the flare was sent
@@ -184,7 +185,11 @@ func analyzeResponse(r *http.Response, apiKey string) (string, error) {
 	}
 
 	if res.Error != "" {
-		response := fmt.Sprintf("An error occurred while uploading the flare: %s. Please contact support by email.", res.Error)
+		var uuidReport string
+		if res.RequestUUID != "" {
+			uuidReport = fmt.Sprintf(" and facilitate the request uuid: `%s`", res.RequestUUID)
+		}
+		response := fmt.Sprintf("An error occurred while uploading the flare: %s. Please contact support by email%s.", res.Error, uuidReport)
 		return response, errors.New(res.Error)
 	}
 

comp/core/flare/helpers/send_flare_test.go

@@ -127,6 +127,19 @@ func TestAnalyzeResponse(t *testing.T) {
 	})
 
 	t.Run("error-from-server", func(t *testing.T) {
+		r := &http.Response{
+			StatusCode: 200,
+			Header:     http.Header{"Content-Type": []string{"application/json; charset=UTF-8"}},
+			Body:       io.NopCloser(bytes.NewBuffer([]byte("{\"case_id\": 1234, \"error\": \"uhoh\", \"request_uuid\": \"1dd9a912-843f-4987-9007-b915edb3d047\"}"))),
+		}
+		resstr, reserr := analyzeResponse(r, "abcdef")
+		require.Equal(t, errors.New("uhoh"), reserr)
+		require.Equal(t,
+			"An error occurred while uploading the flare: uhoh. Please contact support by email and facilitate the request uuid: `1dd9a912-843f-4987-9007-b915edb3d047`.",
+			resstr)
+	})
+
+	t.Run("error-from-server-with-no-request_uuid", func(t *testing.T) {
 		r := &http.Response{
 			StatusCode: 200,
 			Header:     http.Header{"Content-Type": []string{"application/json; charset=UTF-8"}},

pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go

@@ -20,6 +20,7 @@ import (
 	"unsafe"
 
 	manager "github.com/DataDog/ebpf-manager"
+	"github.com/cihub/seelog"
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/asm"
 	"github.com/cilium/ebpf/features"
@@ -305,9 +306,11 @@ func (k *Probe) getProgramStats(stats *model.EBPFStats) error {
 		stats.Programs = append(stats.Programs, ps)
 	}
 
-	log.Tracef("found %d programs", len(stats.Programs))
-	for _, ps := range stats.Programs {
-		log.Tracef("name=%s prog_id=%d type=%s", ps.Name, ps.ID, ps.Type.String())
+	if log.ShouldLog(seelog.TraceLvl) {
+		log.Tracef("found %d programs", len(stats.Programs))
+		for _, ps := range stats.Programs {
+			log.Tracef("name=%s prog_id=%d type=%s", ps.Name, ps.ID, ps.Type.String())
+		}
 	}
 
 	return nil

pkg/network/protocols/kafka/statkeeper.go

@@ -39,15 +39,24 @@ func NewStatkeeper(c *config.Config, telemetry *Telemetry) *StatKeeper {
 
 // Process processes the kafka transaction
 func (statKeeper *StatKeeper) Process(tx *EbpfTx) {
-	statKeeper.statsMutex.Lock()
-	defer statKeeper.statsMutex.Unlock()
+	latency := tx.RequestLatency()
+	// Produce requests with acks = 0 do not receive a response, and as a result, have no latency
+	if tx.APIKey() == FetchAPIKey && latency <= 0 {
+		statKeeper.telemetry.invalidLatency.Add(1)
+		return
+	}
 
+	// extractTopicName is an expensive operation but, it is also concurrent safe, so we can do it here
+	// without holding the lock.
 	key := Key{
 		RequestAPIKey:  tx.APIKey(),
 		RequestVersion: tx.APIVersion(),
 		TopicName:      statKeeper.extractTopicName(&tx.Transaction),
 		ConnectionKey:  tx.ConnTuple(),
 	}
+
+	statKeeper.statsMutex.Lock()
+	defer statKeeper.statsMutex.Unlock()
 	requestStats, ok := statKeeper.stats[key]
 	if !ok {
 		if len(statKeeper.stats) >= statKeeper.maxEntries {
@@ -58,13 +67,6 @@ func (statKeeper *StatKeeper) Process(tx *EbpfTx) {
 		statKeeper.stats[key] = requestStats
 	}
 
-	latency := tx.RequestLatency()
-	// Produce requests with acks = 0 do not receive a response, and as a result, have no latency
-	if key.RequestAPIKey == FetchAPIKey && latency <= 0 {
-		statKeeper.telemetry.invalidLatency.Add(1)
-		return
-	}
-
 	requestStats.AddRequest(int32(tx.ErrorCode()), int(tx.RecordsCount()), uint64(tx.Transaction.Tags), latency)
 }
 

pkg/network/protocols/postgres/model_linux.go

@@ -10,7 +10,6 @@ package postgres
 import (
 	"bytes"
 	"fmt"
-	"regexp"
 	"strings"
 
 	"github.com/DataDog/go-sqllexer"
@@ -26,6 +25,10 @@ const (
 	EmptyParameters = "EMPTY_PARAMETERS"
 )
 
+var (
+	postgresDBMS = sqllexer.WithDBMS(sqllexer.DBMSPostgres)
+)
+
 // EventWrapper wraps an ebpf event and provides additional methods to extract information from it.
 // We use this wrapper to avoid recomputing the same values (operation and table name) multiple times.
 type EventWrapper struct {
@@ -98,17 +101,10 @@ func (e *EventWrapper) extractParameters() string {
 	return string(b[idxParam:])
 }
 
-var re = regexp.MustCompile(`(?i)if\s+exists`)
-
 // extractTableName extracts the table name from the query.
 func (e *EventWrapper) extractTableName() string {
-	fragment := string(getFragment(&e.Tx))
-	// Temp solution for the fact that ObfuscateSQLString does not support "IF EXISTS" or "if exists", so we remove
-	// it from the fragment if found.
-	fragment = re.ReplaceAllString(fragment, "")
-
 	// Normalize the query without obfuscating it.
-	_, statementMetadata, err := e.normalizer.Normalize(fragment, sqllexer.WithDBMS(sqllexer.DBMSPostgres))
+	_, statementMetadata, err := e.normalizer.Normalize(string(getFragment(&e.Tx)), postgresDBMS)
 	if err != nil {
 		log.Debugf("unable to normalize due to: %s", err)
 		return "UNKNOWN"

pkg/security/probe/selftests/chmod.go

@@ -31,6 +31,7 @@ func (o *ChmodSelfTest) GetRuleDefinition() *rules.RuleDefinition {
 	return &rules.RuleDefinition{
 		ID:         o.ruleID,
 		Expression: fmt.Sprintf(`chmod.file.path == "%s"`, o.filename),
+		Silent:     true,
 	}
 }
 

pkg/security/probe/selftests/chown.go

@@ -32,6 +32,7 @@ func (o *ChownSelfTest) GetRuleDefinition() *rules.RuleDefinition {
 	return &rules.RuleDefinition{
 		ID:         o.ruleID,
 		Expression: fmt.Sprintf(`chown.file.path == "%s"`, o.filename),
+		Silent:     true,
 	}
 }
 

pkg/security/probe/selftests/create_file_windows.go

@@ -38,6 +38,7 @@ func (o *WindowsCreateFileSelfTest) GetRuleDefinition() *rules.RuleDefinition {
 	return &rules.RuleDefinition{
 		ID:         o.ruleID,
 		Expression: fmt.Sprintf(`create.file.name == "%s" && create.file.device_path =~ "%s" && process.pid == %d`, basename, filepath.ToSlash(devicePath), os.Getpid()),
+		Silent:     true,
 	}
 }
 

pkg/security/probe/selftests/ebpfless.go

@@ -31,6 +31,7 @@ func (o *EBPFLessSelfTest) GetRuleDefinition() *rules.RuleDefinition {
 		ID:         o.ruleID,
 		Expression: `exec.file.path != "" && process.parent.pid == 0 && process.ppid == 0`,
 		Every:      time.Duration(math.MaxInt64),
+		Silent:     true,
 	}
 }
 

pkg/security/probe/selftests/open.go

@@ -31,6 +31,7 @@ func (o *OpenSelfTest) GetRuleDefinition() *rules.RuleDefinition {
 	return &rules.RuleDefinition{
 		ID:         o.ruleID,
 		Expression: fmt.Sprintf(`open.file.path == "%s" && open.flags & O_CREAT > 0`, o.filename),
+		Silent:     true,
 	}
 }
 

pkg/security/probe/selftests/open_registry_key_windows.go

@@ -34,6 +34,7 @@ func (o *WindowsOpenRegistryKeyTest) GetRuleDefinition() *rules.RuleDefinition {
 	return &rules.RuleDefinition{
 		ID:         o.ruleID,
 		Expression: fmt.Sprintf(`open.registry.key_name == "%s" && process.pid == %d`, filepath.Base(o.keyPath), os.Getpid()),
+		Silent:     true,
 	}
 }
 

test/new-e2e/tests/agent-subcommands/check/check_common_test.go

@@ -7,19 +7,26 @@
 package check
 
 import (
+	_ "embed"
 	"fmt"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 type baseCheckSuite struct {
 	e2e.BaseSuite[environments.Host]
 }
 
+//go:embed fixtures/hello.yaml
+var customCheckYaml []byte
+
+//go:embed fixtures/hello.py
+var customCheckPython []byte
+
 func (v *baseCheckSuite) TestCheckDisk() {
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"disk"}))
 
@@ -43,8 +50,7 @@ func (v *baseCheckSuite) TestCustomCheck() {
 
 func (v *baseCheckSuite) TestCheckRate() {
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-rate", "--json"}))
-	data := parseCheckOutput([]byte(check))
-	require.NotNil(v.T(), data)
+	data := parseCheckOutput(v.T(), []byte(check))
 
 	metrics := data[0].Aggregator.Metrics
 
@@ -59,8 +65,7 @@ func (v *baseCheckSuite) TestCheckTimes() {
 	times := 10
 	check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-times", fmt.Sprint(times), "--json"}))
 
-	data := parseCheckOutput([]byte(check))
-	require.NotNil(v.T(), data)
+	data := parseCheckOutput(v.T(), []byte(check))
 
 	metrics := data[0].Aggregator.Metrics
 

test/new-e2e/tests/agent-subcommands/check/check_nix_test.go

@@ -7,7 +7,6 @@
 package check
 
 import (
-	_ "embed"
 	"testing"
 
 	"github.com/DataDog/test-infra-definitions/components/datadog/agentparams"
@@ -22,12 +21,6 @@ type linuxCheckSuite struct {
 	baseCheckSuite
 }
 
-//go:embed fixtures/hello.yaml
-var customCheckYaml []byte
-
-//go:embed fixtures/hello.py
-var customCheckPython []byte
-
 func TestLinuxCheckSuite(t *testing.T) {
 	t.Parallel()
 	e2e.Run(t, &linuxCheckSuite{}, e2e.WithProvisioner(awshost.ProvisionerNoFakeIntake(awshost.WithAgentOptions(

test/new-e2e/tests/agent-subcommands/check/check_win_test.go

@@ -22,13 +22,14 @@ type windowsCheckSuite struct {
 }
 
 func TestWindowsCheckSuite(t *testing.T) {
-	t.Skip("not working because of the following error: unable to import module 'hello': source code string cannot contain null bytes")
 	t.Parallel()
 	e2e.Run(t, &windowsCheckSuite{}, e2e.WithProvisioner(
 		awshost.ProvisionerNoFakeIntake(
 			awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault)),
 			awshost.WithAgentOptions(
 				agentparams.WithFile("C:/ProgramData/Datadog/conf.d/hello.d/conf.yaml", string(customCheckYaml), true),
 				agentparams.WithFile("C:/ProgramData/Datadog/checks.d/hello.py", string(customCheckPython), true),
-			))))
+			),
+		),
+	))
 }

test/new-e2e/tests/agent-subcommands/check/parse.go

@@ -6,7 +6,11 @@
 package check
 
 import (
+	"bytes"
 	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 type root struct {
@@ -27,11 +31,17 @@ type metric struct {
 	Type           string   `json:"type"`
 }
 
-func parseCheckOutput(check []byte) []root {
+func parseCheckOutput(t *testing.T, check []byte) []root {
+	// On Windows a warning is printed when running the check command with the wrong user
+	// This warning is not part of the JSON output and needs to be ignored when parsing
+	startIdx := bytes.IndexAny(check, "[{")
+	require.NotEqual(t, -1, startIdx, "Failed to find start of JSON output in check output: %v", string(check))
+
+	check = check[startIdx:]
+
 	var data []root
-	if err := json.Unmarshal([]byte(check), &data); err != nil {
-		return nil
-	}
+	err := json.Unmarshal([]byte(check), &data)
+	require.NoErrorf(t, err, "Failed to unmarshal check output: %v", string(check))
 
 	return data
 }

test/new-e2e/tests/containers/eks_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/eks"
 
+	"github.com/DataDog/datadog-agent/pkg/util/testutil/flake"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/components"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters"
@@ -33,6 +34,7 @@ func TestEKSSuite(t *testing.T) {
 	if err == nil {
 		initOnly = initOnlyParam
 	}
+	flake.Mark(t)
 	suite.Run(t, &eksSuite{initOnly: initOnly})
 }