Merge branch 'master' into fix_tar_step

NOTICE.txt

@@ -509,11 +509,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20210208210832-2363df416081
+Version: v7.0.0-alpha2.0.20210210141635-5475407dcdb0
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20210208210832-2363df416081/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20210210141635-5475407dcdb0/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -15926,11 +15926,11 @@ Contents of probable licence file $GOMODCACHE/github.com/pquerna/cachecontrol@v0
 
 --------------------------------------------------------------------------------
 Dependency : github.com/prometheus/procfs
-Version: v0.4.1
+Version: v0.5.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.4.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/prometheus/procfs@v0.5.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

apmpackage/apm/0.1.0/_dev/docs/README.template.md

@@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode,
 - `Host`: APM Server host and port to listen on.
 - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
 - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
-
+- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html)
+- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html).
 
 ## Traces
 

apmpackage/apm/0.1.0/agent/input/template.yml.hbs

@@ -1,4 +1,8 @@
 apm-server:
     host: {{host}}
     secret_token: {{secret_token}}
-    rum.enabled: {{enable_rum}}
+    rum:
+        enabled: {{enable_rum}}
+        source_mapping.elasticsearch.api_key: {{sourcemap_api_key}}
+    kibana:
+        api_key: {{kibana_api_key}}

apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json

@@ -0,0 +1,19 @@
+{
+    "policy": {
+        "phases": {
+            "hot": {
+                "actions": {
+                    "rollover": {
+                        "max_age": "1h"
+                    }
+                }
+            },
+            "delete": {
+                "min_age": "1h",
+                "actions": {
+                    "delete": {}
+                }
+            }
+        }
+    }
+}

apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ingest_pipeline/default.json

@@ -0,0 +1,11 @@
+{
+  "description": "Ingest pipeline for sampled trace documents",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{_ingest.timestamp}}"
+      }
+    }
+  ]
+}

apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.

apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/ecs.yml

@@ -0,0 +1,8 @@
+- name: event.ingested
+  type: date
+  description: |
+    Timestamp when an event arrived in the central data store.
+- name: trace.id
+  type: keyword
+  description: |
+    The ID of the sampled trace.

apmpackage/apm/0.1.0/data_stream/sampled_traces/fields/fields.yml

@@ -0,0 +1,6 @@
+# When changing fields or ILM policy, make sure to update
+# x-pack/apm-server/sampling/pubsub/datastream.go.
+- name: observer.id
+  type: keyword
+  description: |
+    The ID of the APM Server that indexed the sampled trace ID.

apmpackage/apm/0.1.0/data_stream/sampled_traces/manifest.yml

@@ -0,0 +1,4 @@
+title: APM tail-sampled traces
+type: traces
+dataset: sampled
+ilm_policy: traces-apm.sampled-default_policy

apmpackage/apm/0.1.0/docs/README.md

@@ -66,7 +66,8 @@ IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode,
 - `Host`: APM Server host and port to listen on.
 - `Secret token`: Authorization token for sending data to APM Server. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
 - `Enable RUM`: Enables support for RUM monitoring. See the [documentation](https://www.elastic.co/guide/en/apm/server/current/configuration-rum.html) for details.
-
+- `API Key for Central Configuration`: Gives privileges for APM Agent central configuration. See the [documentation](https://www.elastic.co/guide/en/kibana/master/agent-configuration.html)
+- `API Key for Sourcemaps`: Gives priveleges to read sourcemaps. See the [documentation](https://www.elastic.co/guide/en/apm/agent/rum-js/current/sourcemap.html).
 
 ## Traces
 

apmpackage/apm/0.1.0/manifest.yml

@@ -42,6 +42,18 @@ policy_templates:
         required: true
         show_user: true
         default: false
+      - name: kibana_api_key
+        type: string
+        title: API Key for Central Configuration
+        required: false
+        description: Enter as <Id>:<API Key>
+        show_user: true
+      - name: sourcemap_api_key
+        type: string
+        title: API Key for Sourcemaps
+        required: false
+        description: Enter as <Id>:<API Key>
+        show_user: true
     template_path: template.yml.hbs
 owner:
   github: elastic/apm-server

apmpackage/cmd/gen-package/genfields.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"io/ioutil"
+	"log"
 	"net/http"
 	"path/filepath"
 	"sort"
@@ -42,6 +43,7 @@ func generateFields(version string) map[string][]field {
 	inputFieldsFiles["app_metrics"] = filterInternalMetrics(inputFieldsFiles["internal_metrics"])
 
 	for streamType, inputFields := range inputFieldsFiles {
+		log.Printf("%s", streamType)
 		var ecsFields []field
 		var nonECSFields []field
 		for _, fields := range populateECSInfo(ecsFlatFields, inputFields) {

apmpackage/cmd/gen-package/main.go

@@ -33,6 +33,13 @@ var versionMapping = map[string]string{
 	"8.0":  "0.1.0",
 }
 
+// Some data streams may not have a counterpart template
+// in standalone apm-server, and so it does not make sense
+// to maintain a separate fields.yml.
+var handwritten = map[string]bool{
+	"sampled_traces": true,
+}
+
 func main() {
 	stackVersion := common.MustNewVersion(cmd.DefaultSettings().Version)
 	shortVersion := fmt.Sprintf("%d.%d", stackVersion.Major, stackVersion.Minor)
@@ -57,11 +64,28 @@ func clear(version string) {
 		panic(err)
 	}
 	for _, f := range fileInfo {
-		if f.IsDir() {
-			os.Remove(ecsFilePath(version, f.Name()))
-			os.Remove(fieldsFilePath(version, f.Name()))
-			os.RemoveAll(pipelinesPath(version, f.Name()))
+		if !f.IsDir() {
+			continue
+		}
+		name := f.Name()
+		if handwritten[name] {
+			continue
 		}
+		removeFile(ecsFilePath(version, name))
+		removeFile(fieldsFilePath(version, name))
+		removeDir(pipelinesPath(version, name))
 	}
 	ioutil.WriteFile(docsFilePath(version), nil, 0644)
 }
+
+func removeFile(path string) {
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		log.Fatal(err)
+	}
+}
+
+func removeDir(path string) {
+	if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) {
+		log.Fatal(err)
+	}
+}

beater/api/config/agent/handler_test.go

@@ -355,7 +355,7 @@ func TestIfNoneMatch(t *testing.T) {
 }
 
 func TestAgentConfigTraceContext(t *testing.T) {
-	kibanaCfg := libkibana.DefaultClientConfig()
+	kibanaCfg := config.KibanaConfig{Enabled: true, ClientConfig: libkibana.DefaultClientConfig()}
 	kibanaCfg.Host = "testKibana:12345"
 	client := kibana.NewConnectingClient(&kibanaCfg)
 	handler := Handler(client, &config.AgentConfig{Cache: &config.Cache{Expiration: 5 * time.Minute}})

beater/api/mux.go

@@ -20,9 +20,9 @@ package api
 import (
 	"net/http"
 
-	"github.com/elastic/beats/v7/libbeat/monitoring"
-
+	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/libbeat/monitoring"
 
 	"github.com/elastic/apm-server/beater/api/asset/sourcemap"
 	"github.com/elastic/apm-server/beater/api/config/agent"
@@ -64,13 +64,8 @@ const (
 	IntakeRUMV3Path = "/intake/v3/rum/events"
 )
 
-type route struct {
-	path      string
-	handlerFn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)
-}
-
 // NewMux registers apm handlers to paths building up the APM Server API.
-func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMux, error) {
+func NewMux(beatInfo beat.Info, beaterConfig *config.Config, report publish.Reporter) (*http.ServeMux, error) {
 	pool := request.NewContextPool()
 	mux := http.NewServeMux()
 	logger := logp.NewLogger(logs.Handler)
@@ -80,26 +75,36 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu
 		return nil, err
 	}
 
+	builder := routeBuilder{
+		info:        beatInfo,
+		cfg:         beaterConfig,
+		authBuilder: auth,
+		reporter:    report,
+	}
+
+	type route struct {
+		path      string
+		handlerFn func() (request.Handler, error)
+	}
 	routeMap := []route{
-		{RootPath, rootHandler},
-		{AssetSourcemapPath, sourcemapHandler},
-		{AgentConfigPath, backendAgentConfigHandler},
-		{AgentConfigRUMPath, rumAgentConfigHandler},
-		{IntakeRUMPath, rumIntakeHandler},
-		{IntakeRUMV3Path, rumV3IntakeHandler},
-		{IntakePath, backendIntakeHandler},
+		{RootPath, builder.rootHandler},
+		{AssetSourcemapPath, builder.sourcemapHandler},
+		{AgentConfigPath, builder.backendAgentConfigHandler},
+		{AgentConfigRUMPath, builder.rumAgentConfigHandler},
+		{IntakeRUMPath, builder.rumIntakeHandler},
+		{IntakeRUMV3Path, builder.rumV3IntakeHandler},
+		{IntakePath, builder.backendIntakeHandler},
 		// The profile endpoint is in Beta
-		{ProfilePath, profileHandler},
+		{ProfilePath, builder.profileHandler},
 	}
 
 	for _, route := range routeMap {
-		h, err := route.handlerFn(beaterConfig, auth, report)
+		h, err := route.handlerFn()
 		if err != nil {
 			return nil, err
 		}
 		logger.Infof("Path %s added to request handler", route.path)
 		mux.Handle(route.path, pool.HTTPHandler(h))
-
 	}
 	if beaterConfig.Expvar.IsEnabled() {
 		path := beaterConfig.Expvar.URL
@@ -109,49 +114,61 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu
 	return mux, nil
 }
 
-func profileHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) {
-	h := profile.Handler(reporter)
-	authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
-	return middleware.Wrap(h, backendMiddleware(cfg, authHandler, profile.MonitoringMap)...)
+type routeBuilder struct {
+	info        beat.Info
+	cfg         *config.Config
+	authBuilder *authorization.Builder
+	reporter    publish.Reporter
+}
+
+func (r *routeBuilder) profileHandler() (request.Handler, error) {
+	h := profile.Handler(r.reporter)
+	authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
+	return middleware.Wrap(h, backendMiddleware(r.cfg, authHandler, profile.MonitoringMap)...)
 }
 
-func backendIntakeHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) {
-	h := intake.Handler(stream.BackendProcessor(cfg), reporter)
-	authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
-	return middleware.Wrap(h, backendMiddleware(cfg, authHandler, intake.MonitoringMap)...)
+func (r *routeBuilder) backendIntakeHandler() (request.Handler, error) {
+	h := intake.Handler(stream.BackendProcessor(r.cfg), r.reporter)
+	authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
+	return middleware.Wrap(h, backendMiddleware(r.cfg, authHandler, intake.MonitoringMap)...)
 }
 
-func rumIntakeHandler(cfg *config.Config, _ *authorization.Builder, reporter publish.Reporter) (request.Handler, error) {
-	h := intake.Handler(stream.RUMV2Processor(cfg), reporter)
-	return middleware.Wrap(h, rumMiddleware(cfg, nil, intake.MonitoringMap)...)
+func (r *routeBuilder) rumIntakeHandler() (request.Handler, error) {
+	h := intake.Handler(stream.RUMV2Processor(r.cfg), r.reporter)
+	return middleware.Wrap(h, rumMiddleware(r.cfg, nil, intake.MonitoringMap)...)
 }
 
-func rumV3IntakeHandler(cfg *config.Config, _ *authorization.Builder, reporter publish.Reporter) (request.Handler, error) {
-	h := intake.Handler(stream.RUMV3Processor(cfg), reporter)
-	return middleware.Wrap(h, rumMiddleware(cfg, nil, intake.MonitoringMap)...)
+func (r *routeBuilder) rumV3IntakeHandler() (request.Handler, error) {
+	h := intake.Handler(stream.RUMV3Processor(r.cfg), r.reporter)
+	return middleware.Wrap(h, rumMiddleware(r.cfg, nil, intake.MonitoringMap)...)
 }
 
-func sourcemapHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) {
-	h := sourcemap.Handler(reporter)
-	authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action)
-	return middleware.Wrap(h, sourcemapMiddleware(cfg, authHandler)...)
+func (r *routeBuilder) sourcemapHandler() (request.Handler, error) {
+	h := sourcemap.Handler(r.reporter)
+	authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action)
+	return middleware.Wrap(h, sourcemapMiddleware(r.cfg, authHandler)...)
 }
 
-func backendAgentConfigHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) {
-	authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action)
-	return agentConfigHandler(cfg, authHandler, backendMiddleware)
+func (r *routeBuilder) rootHandler() (request.Handler, error) {
+	h := root.Handler(root.HandlerConfig{Version: r.info.Version})
+	return middleware.Wrap(h, rootMiddleware(r.cfg, r.authBuilder.ForAnyOfPrivileges(authorization.ActionAny))...)
 }
 
-func rumAgentConfigHandler(cfg *config.Config, _ *authorization.Builder, _ publish.Reporter) (request.Handler, error) {
-	return agentConfigHandler(cfg, nil, rumMiddleware)
+func (r *routeBuilder) backendAgentConfigHandler() (request.Handler, error) {
+	authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action)
+	return agentConfigHandler(r.cfg, authHandler, backendMiddleware)
+}
+
+func (r *routeBuilder) rumAgentConfigHandler() (request.Handler, error) {
+	return agentConfigHandler(r.cfg, nil, rumMiddleware)
 }
 
 type middlewareFunc func(*config.Config, *authorization.Handler, map[request.ResultID]*monitoring.Int) []middleware.Middleware
 
 func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, middlewareFunc middlewareFunc) (request.Handler, error) {
 	var client kibana.Client
 	if cfg.Kibana.Enabled {
-		client = kibana.NewConnectingClient(&cfg.Kibana.ClientConfig)
+		client = kibana.NewConnectingClient(&cfg.Kibana)
 	}
 	h := agent.Handler(client, cfg.AgentConfig)
 	msg := "Agent remote configuration is disabled. " +
@@ -162,11 +179,6 @@ func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler,
 	return middleware.Wrap(h, append(middlewareFunc(cfg, authHandler, agent.MonitoringMap), ks)...)
 }
 
-func rootHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) {
-	return middleware.Wrap(root.Handler(),
-		rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.ActionAny))...)
-}
-
 func apmMiddleware(m map[request.ResultID]*monitoring.Int) []middleware.Middleware {
 	return []middleware.Middleware{
 		middleware.LogMiddleware(),