Read sourcemap using newly added go elasticsearch client (#2897)

Add the official go-elasticsearch client as dependency, and use it 
for fetching sourcemaps from Elasticsearch. Refactor current 
sourcemap logic to get rid of some technical dept. 

closes #2868

NOTICE.txt

@@ -511,6 +511,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
+--------------------------------------------------------------------
+Dependency: github.com/elastic/go-elasticsearch
+Revision: 413583419151bfec42078b5754120bb4d04bd537
+License type (autodetected): Apache-2.0
+./vendor/github.com/elastic/go-elasticsearch/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-lumber
 Revision: 616041e345fc33c97bc0eb0fa6b388aa07bca3e1

README.md

@@ -124,6 +124,16 @@ To update the dependency to a specific commit or branch run command as following
 ```
 BEATS_VERSION=f240148065af94d55c5149e444482b9635801f27 make update-beats
 ```
+### Go-elasticsearch client Update
+
+It is important to keep the used [go-elasticsearch client](https://github.com/elastic/go-elasticsearch) in sync with the according major version. 
+We also recommend to use the latest available client for minor versions.
+Since APM Server does not yet support go modules, you can update the dependency using govendor, e.g. by running: 
+```
+git clone --branch v7.4.1 https://github.com/elastic/go-elasticsearch.git $GOPATH/src/github.com/elastic/go-elasticsearch/v7
+govendor add github.com/elastic/go-elasticsearch/v7/^
+mv ./vendor/github.com/elastic/go-elasticsearch/v7/LICENSE ./vendor/github.com/elastic/go-elasticsearch/
+```
 
 ## Packaging
 

beater/api/mux.go

@@ -24,6 +24,8 @@ import (
 
 	"github.com/elastic/beats/libbeat/monitoring"
 
+	"github.com/elastic/beats/libbeat/logp"
+
 	"github.com/elastic/apm-server/beater/api/asset/sourcemap"
 	"github.com/elastic/apm-server/beater/api/config/agent"
 	"github.com/elastic/apm-server/beater/api/intake"
@@ -39,7 +41,6 @@ import (
 	"github.com/elastic/apm-server/processor/stream"
 	"github.com/elastic/apm-server/publish"
 	"github.com/elastic/apm-server/transform"
-	"github.com/elastic/beats/libbeat/logp"
 )
 
 const (
@@ -213,12 +214,12 @@ func userMetaDataDecoder(beaterConfig *config.Config, d decoder.ReqDecoder) deco
 }
 
 func rumTransformConfig(beaterConfig *config.Config) (*transform.Config, error) {
-	mapper, err := beaterConfig.RumConfig.MemoizedSourcemapMapper()
+	store, err := beaterConfig.RumConfig.MemoizedSourcemapStore()
 	if err != nil {
 		return nil, err
 	}
 	return &transform.Config{
-		SourcemapMapper:     mapper,
+		SourcemapStore:      store,
 		LibraryPattern:      regexp.MustCompile(beaterConfig.RumConfig.LibraryPattern),
 		ExcludeFromGrouping: regexp.MustCompile(beaterConfig.RumConfig.ExcludeFromGrouping),
 	}, nil

beater/beater.go

@@ -85,31 +85,20 @@ func checkConfig(logger *logp.Logger) error {
 	return nil
 }
 
-// Creates beater
+// New creates a beater instance using the provided configuration
 func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) {
 	logger := logp.NewLogger(logs.Beater)
 	if err := checkConfig(logger); err != nil {
 		return nil, err
 	}
-	beaterConfig, err := config.NewConfig(b.Info.Version, ucfg)
+	var esOutputCfg *common.Config
+	if isElasticsearchOutput(b) {
+		esOutputCfg = b.Config.Output.Config()
+	}
+	beaterConfig, err := config.NewConfig(b.Info.Version, ucfg, esOutputCfg)
 	if err != nil {
 		return nil, err
 	}
-	if beaterConfig.RumConfig.IsEnabled() {
-		if b.Config != nil && beaterConfig.RumConfig.SourceMapping.EsConfig == nil {
-			// fall back to elasticsearch output configuration for sourcemap storage if possible
-			if isElasticsearchOutput(b) {
-				logger.Info("Falling back to elasticsearch output for sourcemap storage")
-				beaterConfig.SetSourcemapElasticsearch(b.Config.Output.Config())
-			} else {
-				logger.Info("Unable to determine sourcemap storage, sourcemaps will not be applied")
-			}
-		}
-	}
-	if isElasticsearchOutput(b) &&
-		(b.Config.Output.Config().HasField("pipeline") || b.Config.Output.Config().HasField("pipelines")) {
-		beaterConfig.Pipeline = ""
-	}
 
 	bt := &beater{
 		config:  beaterConfig,