Merge branch 'main' into guy.arbitman/USMON-1369

.gitlab/e2e/e2e.yml

@@ -277,6 +277,7 @@ new-e2e-aml:
     - deploy_deb_testing-a7_x64
     - deploy_windows_testing-a7
     - qa_agent
+    - qa_agent_jmx
     - qa_dca
   rules:
     - !reference [.on_aml_or_e2e_changes]

.run/README.md

@@ -0,0 +1,3 @@
+# IntelliJ Goland out of the box configuration
+
+This folder contains scripts and tasks for IntelliJ Goland to build and run the agent and its sub-processes

comp/core/tagger/impl-dual/dual.go

@@ -54,7 +54,8 @@ func NewComponent(req Requires) (Provides, error) {
 
 		return Provides{
 			local.Provides{
-				Comp: provide.Comp,
+				Comp:     provide.Comp,
+				Endpoint: provide.Endpoint,
 			},
 		}, nil
 	}

comp/core/tagger/impl-remote/remote.go

@@ -9,8 +9,10 @@ package remotetaggerimpl
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"net"
+	"net/http"
 	"time"
 
 	"github.com/cenkalti/backoff"
@@ -21,6 +23,7 @@ import (
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/metadata"
 
+	api "github.com/DataDog/datadog-agent/comp/api/api/def"
 	"github.com/DataDog/datadog-agent/comp/core/config"
 	log "github.com/DataDog/datadog-agent/comp/core/log/def"
 	taggercommon "github.com/DataDog/datadog-agent/comp/core/tagger/common"
@@ -35,6 +38,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/tagset"
 	"github.com/DataDog/datadog-agent/pkg/util/common"
 	grpcutil "github.com/DataDog/datadog-agent/pkg/util/grpc"
+	httputils "github.com/DataDog/datadog-agent/pkg/util/http"
 )
 
 const (
@@ -59,7 +63,8 @@ type Requires struct {
 type Provides struct {
 	compdef.Out
 
-	Comp tagger.Component
+	Comp     tagger.Component
+	Endpoint api.AgentEndpointProvider
 }
 
 type remoteTagger struct {
@@ -112,11 +117,12 @@ func NewComponent(req Requires) (Provides, error) {
 	}})
 
 	return Provides{
-		Comp: remoteTagger,
+		Comp:     remoteTagger,
+		Endpoint: api.NewAgentEndpointProvider(remoteTagger.writeList, "/tagger-list", "GET"),
 	}, nil
 }
 
-func newRemoteTagger(params tagger.RemoteParams, cfg config.Component, log log.Component, telemetryComp coretelemetry.Component) (tagger.Component, error) {
+func newRemoteTagger(params tagger.RemoteParams, cfg config.Component, log log.Component, telemetryComp coretelemetry.Component) (*remoteTagger, error) {
 	telemetryStore := telemetry.NewStore(telemetryComp)
 
 	target, err := params.RemoteTarget(cfg)
@@ -494,6 +500,17 @@ func (t *remoteTagger) startTaggerStream(maxElapsed time.Duration) error {
 	}, expBackoff)
 }
 
+func (t *remoteTagger) writeList(w http.ResponseWriter, _ *http.Request) {
+	response := t.List()
+
+	jsonTags, err := json.Marshal(response)
+	if err != nil {
+		httputils.SetJSONError(w, t.log.Errorf("Unable to marshal tagger list response: %s", err), 500)
+		return
+	}
+	w.Write(jsonTags)
+}
+
 func convertEventType(t pb.EventType) (types.EventType, error) {
 	switch t {
 	case pb.EventType_ADDED:

comp/core/tagger/impl-remote/remote_test.go

@@ -7,17 +7,22 @@ package remotetaggerimpl
 
 import (
 	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"runtime"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/DataDog/datadog-agent/comp/core/config"
 	logmock "github.com/DataDog/datadog-agent/comp/core/log/mock"
 	tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/types"
 	nooptelemetry "github.com/DataDog/datadog-agent/comp/core/telemetry/noopsimpl"
+	compdef "github.com/DataDog/datadog-agent/comp/def"
 	configmock "github.com/DataDog/datadog-agent/pkg/config/mock"
 	"github.com/DataDog/datadog-agent/pkg/util/grpc"
 )
@@ -72,3 +77,42 @@ func TestStartDoNotBlockIfServerIsNotAvailable(t *testing.T) {
 	require.NoError(t, err)
 	remoteTagger.Stop()
 }
+
+func TestNewComponentSetsTaggerListEndpoint(t *testing.T) {
+	req := Requires{
+		Lc:     compdef.NewTestLifecycle(t),
+		Config: configmock.New(t),
+		Log:    logmock.New(t),
+		Params: tagger.RemoteParams{
+			RemoteTarget: func(config.Component) (string, error) { return ":5001", nil },
+			RemoteTokenFetcher: func(config.Component) func() (string, error) {
+				return func() (string, error) {
+					return "something", nil
+				}
+			},
+		},
+		Telemetry: nooptelemetry.GetCompatComponent(),
+	}
+	provides, err := NewComponent(req)
+	require.NoError(t, err)
+
+	endpointProvider := provides.Endpoint.Provider
+
+	assert.Equal(t, []string{"GET"}, endpointProvider.Methods())
+	assert.Equal(t, "/tagger-list", endpointProvider.Route())
+
+	// Create a test server with the endpoint handler
+	server := httptest.NewServer(endpointProvider.HandlerFunc())
+	defer server.Close()
+
+	// Make a request to the endpoint
+	resp, err := http.Get(server.URL + "/tagger-list")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	var response types.TaggerListResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	require.NoError(t, err)
+	assert.NotNil(t, response.Entities)
+}

comp/otelcol/converter/impl/autoconfigure.go

@@ -120,6 +120,9 @@ func addComponentToPipeline(conf *confmap.Conf, comp component, pipelineName str
 // For example, if api key is not found in otel config, it can be retrieved from core
 // agent config instead.
 func addCoreAgentConfig(conf *confmap.Conf, coreCfg config.Component) {
+	if coreCfg == nil {
+		return
+	}
 	stringMapConf := conf.ToStringMap()
 	exporters, ok := stringMapConf["exporters"]
 	if !ok {
@@ -129,43 +132,61 @@ func addCoreAgentConfig(conf *confmap.Conf, coreCfg config.Component) {
 	if !ok {
 		return
 	}
-	datadog, ok := exporterMap["datadog"]
-	if !ok {
+	reg, err := regexp.Compile(secretRegex)
+	if err != nil {
 		return
 	}
-	datadogMap, ok := datadog.(map[string]any)
-	if !ok {
-		return
-	}
-	api, ok := datadogMap["api"]
-	if !ok {
-		return
-	}
-	apiMap, ok := api.(map[string]any)
-	if !ok {
-		return
-	}
-
-	apiKey, ok := apiMap["key"]
-	if ok {
-		key, ok := apiKey.(string)
-		if ok && key != "" {
-			match, _ := regexp.MatchString(secretRegex, apiKey.(string))
-			if !match {
+	for exporter := range exporterMap {
+		if componentName(exporter) == "datadog" {
+			datadog, ok := exporterMap[exporter]
+			if !ok {
 				return
 			}
+			datadogMap, ok := datadog.(map[string]any)
+			if !ok {
+				// datadog section is there, but there is nothing in it. We
+				// need to add it so we can add to it.
+				exporterMap[exporter] = make(map[string]any)
+				datadogMap = exporterMap[exporter].(map[string]any)
+			}
+			api, ok := datadogMap["api"]
+			// ok can be true if api section is there but contains nothing (api == nil).
+			// In which case, we need to add it so we can add to it.
+			if !ok || api == nil {
+				datadogMap["api"] = make(map[string]any, 2)
+				api = datadogMap["api"]
+			}
+			apiMap, ok := api.(map[string]any)
+			if !ok {
+				return
+			}
+
+			// api::site
+			apiSite := apiMap["site"]
+			if (apiSite == nil || apiSite == "") && coreCfg.Get("site") != nil {
+				apiMap["site"] = coreCfg.Get("site")
+			}
+
+			// api::key
+			var match bool
+			apiKey, ok := apiMap["key"]
+			if ok {
+				var key string
+				if keyString, okString := apiKey.(string); okString {
+					key = keyString
+				}
+				if ok && key != "" {
+					match = reg.Match([]byte(key))
+					if !match {
+						continue
+					}
+				}
+			}
+			// TODO: add logic to either fail or log message if api key not found
+			if (apiKey == nil || apiKey == "" || match) && coreCfg.Get("api_key") != nil {
+				apiMap["key"] = coreCfg.Get("api_key")
+			}
 		}
 	}
-	// this is the only reference to Requires.Conf
-	// TODO: add logic to either fail or log message if api key not found
-	if coreCfg != nil {
-		apiMap["key"] = coreCfg.Get("api_key")
-
-		apiSite, ok := apiMap["site"]
-		if ok && apiSite == "" {
-			apiMap["site"] = coreCfg.Get("site")
-		}
-	}
-
 	*conf = *confmap.NewFromStringMap(stringMapConf)
 }