Refactor sourcemap logic

Make use of newly integrated go-elasticsearch client for fetching sourcemaps from Elasticsearch. While on it, refactor logic in sourcemap package and tests to get rid of some technical dept. closes elastic#2868
simitt · Nov 13, 2019 · 2118846 · 2118846
1 parent 6d9e0a9
commit 2118846
Show file tree

Hide file tree

Showing 41 changed files with 1,909 additions and 1,969 deletions.
diff --git a/beater/api/mux.go b/beater/api/mux.go
@@ -24,6 +24,8 @@ import (
 
 	"github.com/elastic/beats/libbeat/monitoring"
 
+	"github.com/elastic/beats/libbeat/logp"
+
 	"github.com/elastic/apm-server/beater/api/asset/sourcemap"
 	"github.com/elastic/apm-server/beater/api/config/agent"
 	"github.com/elastic/apm-server/beater/api/intake"
@@ -39,7 +41,6 @@ import (
 	"github.com/elastic/apm-server/processor/stream"
 	"github.com/elastic/apm-server/publish"
 	"github.com/elastic/apm-server/transform"
-	"github.com/elastic/beats/libbeat/logp"
 )
 
 const (
@@ -213,12 +214,12 @@ func userMetaDataDecoder(beaterConfig *config.Config, d decoder.ReqDecoder) deco
 }
 
 func rumTransformConfig(beaterConfig *config.Config) (*transform.Config, error) {
-	mapper, err := beaterConfig.RumConfig.MemoizedSourcemapMapper()
+	store, err := beaterConfig.RumConfig.MemoizedSourcemapStore()
 	if err != nil {
 		return nil, err
 	}
 	return &transform.Config{
-		SourcemapMapper:     mapper,
+		SourcemapStore:      store,
 		LibraryPattern:      regexp.MustCompile(beaterConfig.RumConfig.LibraryPattern),
 		ExcludeFromGrouping: regexp.MustCompile(beaterConfig.RumConfig.ExcludeFromGrouping),
 	}, nil

diff --git a/beater/beater.go b/beater/beater.go
@@ -85,31 +85,20 @@ func checkConfig(logger *logp.Logger) error {
 	return nil
 }
 
-// Creates beater
+// New creates a beater instance using the provided configuration
 func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) {
 	logger := logp.NewLogger(logs.Beater)
 	if err := checkConfig(logger); err != nil {
 		return nil, err
 	}
-	beaterConfig, err := config.NewConfig(b.Info.Version, ucfg)
+	var esOutputCfg *common.Config
+	if isElasticsearchOutput(b) {
+		esOutputCfg = b.Config.Output.Config()
+	}
+	beaterConfig, err := config.NewConfig(b.Info.Version, ucfg, esOutputCfg)
 	if err != nil {
 		return nil, err
 	}
-	if beaterConfig.RumConfig.IsEnabled() {
-		if b.Config != nil && beaterConfig.RumConfig.SourceMapping.EsConfig == nil {
-			// fall back to elasticsearch output configuration for sourcemap storage if possible
-			if isElasticsearchOutput(b) {
-				logger.Info("Falling back to elasticsearch output for sourcemap storage")
-				beaterConfig.SetSourcemapElasticsearch(b.Config.Output.Config())
-			} else {
-				logger.Info("Unable to determine sourcemap storage, sourcemaps will not be applied")
-			}
-		}
-	}
-	if isElasticsearchOutput(b) &&
-		(b.Config.Output.Config().HasField("pipeline") || b.Config.Output.Config().HasField("pipelines")) {
-		beaterConfig.Pipeline = ""
-	}
 
 	bt := &beater{
 		config:  beaterConfig,

diff --git a/beater/beater_test.go b/beater/beater_test.go
@@ -28,9 +28,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 
-	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/transport/tlscommon"
@@ -41,6 +42,8 @@ import (
 	"github.com/elastic/beats/libbeat/publisher/processing"
 	"github.com/elastic/beats/libbeat/publisher/queue"
 	"github.com/elastic/beats/libbeat/publisher/queue/memqueue"
+
+	"github.com/elastic/apm-server/beater/config"
 )
 
 func TestBeatConfig(t *testing.T) {
@@ -392,24 +395,13 @@ func (bt *beater) wait() error {
 	}
 }
 
-func (bt *beater) sourcemapElasticsearchHosts() []string {
-	var content map[string]interface{}
-	if err := bt.config.RumConfig.SourceMapping.EsConfig.Unpack(&content); err != nil {
-		return nil
-	}
-	hostsContent := content["hosts"].([]interface{})
-	hosts := make([]string, len(hostsContent))
-	for i := range hostsContent {
-		hosts[i] = hostsContent[i].(string)
-	}
-	return hosts
-}
-
 func setupBeater(t *testing.T, apmBeat *beat.Beat, ucfg *common.Config, beatConfig *beat.BeatConfig) (*beater, func(), error) {
 	// create our beater
 	beatBeater, err := New(apmBeat, ucfg)
-	assert.NoError(t, err)
-	assert.NotNil(t, beatBeater)
+	if err != nil {
+		return nil, nil, err
+	}
+	require.NotNil(t, beatBeater)
 
 	c := make(chan error)
 	// start it