Send sourcemap docs to own index by default. (elastic#582)

Uploaded source map documents are now indexed
in their own `sourcemap` index by default.
Possible config options are explained in config files.

CHANGELOG.asciidoc

@@ -45,6 +45,7 @@ https://github.com/elastic/apm-server/compare/71df0d96445df35afe27f38bcf734a0828
 - Change `error.culprit` after successfully applying sourcemapping {pull}520[520]
 - Make `transaction.name` optional {pull}554[554]
 - Remove config files from beats. Manually add relevant config options {pull}578[578]
+- Use seperate index for uploaded `source maps` {pull}582[582].
 
 ==== Deprecated
 

_meta/beat.reference.yml

@@ -64,10 +64,10 @@ apm-server:
       #cache:
         #expiration: 5m
 
-      # Source maps are stored in the same index as transaction and error documents.
-      # If the default index pattern at 'outputs.elasticsearch.index' is changed,
-      # a matching index pattern needs to be specified here.
-      #index_pattern: "apm-*"
+      # Source maps are stored in a seperate index.
+      # If the default index pattern for source maps at 'outputs.elasticsearch.indices'
+      # is changed, a matching index pattern needs to be specified here.
+      #index_pattern: "apm-*-sourcemap*"
 
   # golang expvar support - https://golang.org/pkg/expvar/
   #expvar:
@@ -134,10 +134,30 @@ output.elasticsearch:
   #worker: 1
 
   # Optional index name. The default is "apm" plus version plus date
-  # and generates apm-version-YYYY.MM.DD keys.
-  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
+  # and generates apm-%{[beat.version]}-YYYY.MM.DD keys.
+  # In case you modify this pattern you must update following configuration accordingly:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #
   #index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
 
+  # By specifying additional indices documents matching the criteria are indexed in a seperate index.
+  # In case you are uploading source map documents (which is currently an experimental feature),
+  # source map documents are going to their own index with the configuration below.
+  #
+  # Be aware that there is only one Elasticsearch template and one Kibana Index Pattern,
+  # which needs to match against indices for ingested documents as well as for source maps.
+  # In case you modify the index patterns you must ensure to use the same prefix for all indices,
+  # and to accordingly set:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #indices:
+    #- index: "apm-%{[beat.version]}-sourcemap"
+      #when.contains:
+        #processor.event: "sourcemap"
+
   # Optional ingest node pipeline. By default no pipeline will be used.
   #pipeline: ""
 
@@ -256,6 +276,7 @@ output.elasticsearch:
 
 # The Elasticsearch index name. This overwrites the index name defined in the
 # dashboards and index pattern. Example: testbeat-*
+# The dashboards.index needs to be changed in case the elasticsearch index pattern is modified.
 #setup.dashboards.index:
 
 # Always use the Kibana API for loading the dashboards instead of autodetecting

_meta/beat.yml

@@ -21,11 +21,13 @@ apm-server:
   #ssl.key : "path/to/private_key"
 
 #==================== Elasticsearch template setting ==========================
+
 setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression
 
 #============================== Dashboards =====================================
+#
 # These settings control loading the sample dashboards to the Kibana index. Loading
 # the dashboards is disabled by default and can be enabled either by setting the
 # options here, or by using the `-setup` CLI flag or the `setup` command.
@@ -52,6 +54,31 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
+  # Optional index name. The default is "apm" plus version plus date
+  # and generates apm-%{[beat.version]}-YYYY.MM.DD keys.
+  # In case you modify this pattern you must update following configuration accordingly:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #
+  #index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
+
+  # By specifying additional indices documents matching the criteria are indexed in a seperate index.
+  # In case you are uploading source map documents (which is currently an experimental feature),
+  # source map documents are going to their own index with the configuration below.
+  #
+  # Be aware that there is only one Elasticsearch template and one Kibana Index Pattern,
+  # which needs to match against indices for ingested documents as well as for source maps.
+  # In case you modify the index patterns you must ensure to use the same prefix for all indices,
+  # and to accordingly set:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  indices:
+    - index: "apm-%{[beat.version]}-sourcemap"
+      when.contains:
+        processor.event: "sourcemap"
+
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

apm-server.reference.yml

@@ -64,10 +64,10 @@ apm-server:
       #cache:
         #expiration: 5m
 
-      # Source maps are stored in the same index as transaction and error documents.
-      # If the default index pattern at 'outputs.elasticsearch.index' is changed,
-      # a matching index pattern needs to be specified here.
-      #index_pattern: "apm-*"
+      # Source maps are stored in a seperate index.
+      # If the default index pattern for source maps at 'outputs.elasticsearch.indices'
+      # is changed, a matching index pattern needs to be specified here.
+      #index_pattern: "apm-*-sourcemap*"
 
   # golang expvar support - https://golang.org/pkg/expvar/
   #expvar:
@@ -134,10 +134,30 @@ output.elasticsearch:
   #worker: 1
 
   # Optional index name. The default is "apm" plus version plus date
-  # and generates apm-version-YYYY.MM.DD keys.
-  # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly.
+  # and generates apm-%{[beat.version]}-YYYY.MM.DD keys.
+  # In case you modify this pattern you must update following configuration accordingly:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #
   #index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
 
+  # By specifying additional indices documents matching the criteria are indexed in a seperate index.
+  # In case you are uploading source map documents (which is currently an experimental feature),
+  # source map documents are going to their own index with the configuration below.
+  #
+  # Be aware that there is only one Elasticsearch template and one Kibana Index Pattern,
+  # which needs to match against indices for ingested documents as well as for source maps.
+  # In case you modify the index patterns you must ensure to use the same prefix for all indices,
+  # and to accordingly set:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #indices:
+    #- index: "apm-%{[beat.version]}-sourcemap"
+      #when.contains:
+        #processor.event: "sourcemap"
+
   # Optional ingest node pipeline. By default no pipeline will be used.
   #pipeline: ""
 
@@ -256,6 +276,7 @@ output.elasticsearch:
 
 # The Elasticsearch index name. This overwrites the index name defined in the
 # dashboards and index pattern. Example: testbeat-*
+# The dashboards.index needs to be changed in case the elasticsearch index pattern is modified.
 #setup.dashboards.index:
 
 # Always use the Kibana API for loading the dashboards instead of autodetecting

apm-server.yml

@@ -21,11 +21,13 @@ apm-server:
   #ssl.key : "path/to/private_key"
 
 #==================== Elasticsearch template setting ==========================
+
 setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression
 
 #============================== Dashboards =====================================
+#
 # These settings control loading the sample dashboards to the Kibana index. Loading
 # the dashboards is disabled by default and can be enabled either by setting the
 # options here, or by using the `-setup` CLI flag or the `setup` command.
@@ -52,6 +54,31 @@ output.elasticsearch:
   # Array of hosts to connect to.
   hosts: ["localhost:9200"]
 
+  # Optional index name. The default is "apm" plus version plus date
+  # and generates apm-%{[beat.version]}-YYYY.MM.DD keys.
+  # In case you modify this pattern you must update following configuration accordingly:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  #
+  #index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
+
+  # By specifying additional indices documents matching the criteria are indexed in a seperate index.
+  # In case you are uploading source map documents (which is currently an experimental feature),
+  # source map documents are going to their own index with the configuration below.
+  #
+  # Be aware that there is only one Elasticsearch template and one Kibana Index Pattern,
+  # which needs to match against indices for ingested documents as well as for source maps.
+  # In case you modify the index patterns you must ensure to use the same prefix for all indices,
+  # and to accordingly set:
+  # * `setup.template.name`
+  # * `setup.template.pattern`
+  # * `setup.dashboards.index`
+  indices:
+    - index: "apm-%{[beat.version]}-sourcemap"
+      when.contains:
+        processor.event: "sourcemap"
+
   # Optional protocol and basic auth credentials.
   #protocol: "https"
   #username: "elastic"

beater/beater.go

@@ -22,7 +22,7 @@ type beater struct {
 
 // Creates beater
 func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) {
-	beaterConfig := defaultConfig()
+	beaterConfig := defaultConfig(b.Info.Version)
 	if err := ucfg.Unpack(beaterConfig); err != nil {
 		return nil, fmt.Errorf("Error reading config file: %v", err)
 	}

beater/beater_test.go

@@ -27,11 +27,12 @@ func TestBeatConfig(t *testing.T) {
 	tests := []struct {
 		conf       map[string]interface{}
 		beaterConf *Config
+		SmapIndex  string
 		msg        string
 	}{
 		{
 			conf:       map[string]interface{}{},
-			beaterConf: defaultConfig(),
+			beaterConf: defaultConfig("6.2.0"),
 			msg:        "Default config created for empty config.",
 		},
 		{
@@ -85,11 +86,12 @@ func TestBeatConfig(t *testing.T) {
 					RateLimit:    1000,
 					AllowOrigins: []string{"example*"},
 					SourceMapping: &SourceMapping{
-						Cache: &Cache{Expiration: 5 * time.Minute},
-						Index: "apm-test*",
+						Cache:        &Cache{Expiration: 5 * time.Minute},
+						IndexPattern: "apm-test*",
 					},
 					LibraryPattern:      "^custom",
 					ExcludeFromGrouping: "^grouping",
+					beatVersion:         "6.2.0",
 				},
 				ConcurrentRequests: 15,
 			},
@@ -137,10 +139,11 @@ func TestBeatConfig(t *testing.T) {
 						Cache: &Cache{
 							Expiration: 7 * time.Second,
 						},
-						Index: "apm-*",
+						IndexPattern: "apm-*-sourcemap*",
 					},
 					LibraryPattern:      "node_modules|bower_components|~",
 					ExcludeFromGrouping: "^/webpack",
+					beatVersion:         "6.2.0",
 				},
 				ConcurrentRequests: 40,
 			},
@@ -151,7 +154,7 @@ func TestBeatConfig(t *testing.T) {
 	for _, test := range tests {
 		ucfgConfig, err := common.NewConfigFrom(test.conf)
 		assert.NoError(t, err)
-		btr, err := New(&beat.Beat{}, ucfgConfig)
+		btr, err := New(&beat.Beat{Info: beat.Info{Version: "6.2.0"}}, ucfgConfig)
 		assert.NoError(t, err)
 		assert.NotNil(t, btr)
 		bt := btr.(*beater)
@@ -226,7 +229,7 @@ func SetupServer(b *testing.B) *http.ServeMux {
 	if err != nil {
 		b.Fatal(err)
 	}
-	return newMuxer(defaultConfig(), pub.Send)
+	return newMuxer(defaultConfig("7.0.0"), pub.Send)
 }
 
 func pluralize(entity string) string {

beater/config.go

@@ -1,6 +1,7 @@
 package beater
 
 import (
+	"regexp"
 	"time"
 
 	"github.com/elastic/apm-server/sourcemap"
@@ -33,11 +34,13 @@ type FrontendConfig struct {
 	LibraryPattern      string         `config:"library_pattern"`
 	ExcludeFromGrouping string         `config:"exclude_from_grouping"`
 	SourceMapping       *SourceMapping `config:"source_mapping"`
+
+	beatVersion string
 }
 
 type SourceMapping struct {
-	Cache *Cache `config:"cache"`
-	Index string `config:"index_pattern"`
+	Cache        *Cache `config:"cache"`
+	IndexPattern string `config:"index_pattern"`
 
 	esConfig *common.Config
 	mapper   sourcemap.Mapper
@@ -75,7 +78,7 @@ func (s *SourceMapping) isSetup() bool {
 	return s != nil && (s.esConfig != nil)
 }
 
-func (c *FrontendConfig) SmapMapper() (sourcemap.Mapper, error) {
+func (c *FrontendConfig) memoizedSmapMapper() (sourcemap.Mapper, error) {
 	smap := c.SourceMapping
 	if !c.isEnabled() || !smap.isSetup() {
 		return nil, nil
@@ -86,7 +89,7 @@ func (c *FrontendConfig) SmapMapper() (sourcemap.Mapper, error) {
 	smapConfig := sourcemap.Config{
 		CacheExpiration:     smap.Cache.Expiration,
 		ElasticsearchConfig: smap.esConfig,
-		Index:               smap.Index,
+		Index:               replaceVersion(c.SourceMapping.IndexPattern, c.beatVersion),
 	}
 	smapMapper, err := sourcemap.NewSmapMapper(smapConfig)
 	if err != nil {
@@ -96,7 +99,12 @@ func (c *FrontendConfig) SmapMapper() (sourcemap.Mapper, error) {
 	return c.SourceMapping.mapper, nil
 }
 
-func defaultConfig() *Config {
+func replaceVersion(pattern, version string) string {
+	re := regexp.MustCompile("%.*{.*beat.version.?}")
+	return re.ReplaceAllLiteralString(pattern, version)
+}
+
+func defaultConfig(beatVersion string) *Config {
 	return &Config{
 		Host:               "localhost:8200",
 		MaxUnzippedSize:    50 * 1024 * 1024, // 50mb
@@ -107,14 +115,15 @@ func defaultConfig() *Config {
 		ShutdownTimeout:    5 * time.Second,
 		SecretToken:        "",
 		Frontend: &FrontendConfig{
+			beatVersion:  beatVersion,
 			Enabled:      new(bool),
 			RateLimit:    10,
 			AllowOrigins: []string{"*"},
 			SourceMapping: &SourceMapping{
 				Cache: &Cache{
 					Expiration: 5 * time.Minute,
 				},
-				Index: "apm-*",
+				IndexPattern: "apm-*-sourcemap*",
 			},
 			LibraryPattern:      "node_modules|bower_components|~",
 			ExcludeFromGrouping: "^/webpack",

beater/config_test.go

@@ -59,8 +59,8 @@ func TestConfig(t *testing.T) {
 					RateLimit:    1000,
 					AllowOrigins: []string{"example*"},
 					SourceMapping: &SourceMapping{
-						Cache: &Cache{Expiration: 10 * time.Minute},
-						Index: "apm-test*",
+						Cache:        &Cache{Expiration: 10 * time.Minute},
+						IndexPattern: "apm-test*",
 					},
 					LibraryPattern:      "pattern",
 					ExcludeFromGrouping: "group_pattern",
@@ -99,7 +99,7 @@ func TestConfig(t *testing.T) {
 					RateLimit:    0,
 					AllowOrigins: nil,
 					SourceMapping: &SourceMapping{
-						Index: "",
+						IndexPattern: "",
 					},
 				},
 			},
@@ -157,3 +157,19 @@ func TestIsEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestReplaceBeatVersion(t *testing.T) {
+	cases := []struct {
+		version      string
+		indexPattern string
+		replaced     string
+	}{
+		{version: "", indexPattern: "", replaced: ""},
+		{version: "6.2.0", indexPattern: "apm-%{[beat.version]}", replaced: "apm-6.2.0"},
+		{version: "6.2.0", indexPattern: "apm-smap", replaced: "apm-smap"},
+	}
+	for _, test := range cases {
+		out := replaceVersion(test.indexPattern, test.version)
+		assert.Equal(t, test.replaced, out)
+	}
+}