diff --git a/beater/api/mux.go b/beater/api/mux.go index e10cf41de10..1515b58c588 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -24,6 +24,8 @@ import ( "github.com/elastic/beats/libbeat/monitoring" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/apm-server/beater/api/asset/sourcemap" "github.com/elastic/apm-server/beater/api/config/agent" "github.com/elastic/apm-server/beater/api/intake" @@ -39,7 +41,6 @@ import ( "github.com/elastic/apm-server/processor/stream" "github.com/elastic/apm-server/publish" "github.com/elastic/apm-server/transform" - "github.com/elastic/beats/libbeat/logp" ) const ( @@ -213,12 +214,12 @@ func userMetaDataDecoder(beaterConfig *config.Config, d decoder.ReqDecoder) deco } func rumTransformConfig(beaterConfig *config.Config) (*transform.Config, error) { - mapper, err := beaterConfig.RumConfig.MemoizedSourcemapMapper() + store, err := beaterConfig.RumConfig.MemoizedSourcemapStore() if err != nil { return nil, err } return &transform.Config{ - SourcemapMapper: mapper, + SourcemapStore: store, LibraryPattern: regexp.MustCompile(beaterConfig.RumConfig.LibraryPattern), ExcludeFromGrouping: regexp.MustCompile(beaterConfig.RumConfig.ExcludeFromGrouping), }, nil diff --git a/beater/beater.go b/beater/beater.go index 716235455ae..6f73600b331 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -85,31 +85,20 @@ func checkConfig(logger *logp.Logger) error { return nil } -// Creates beater +// New creates a beater instance using the provided configuration func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) { logger := logp.NewLogger(logs.Beater) if err := checkConfig(logger); err != nil { return nil, err } - beaterConfig, err := config.NewConfig(b.Info.Version, ucfg) + var esOutputCfg *common.Config + if isElasticsearchOutput(b) { + esOutputCfg = b.Config.Output.Config() + } + beaterConfig, err := config.NewConfig(b.Info.Version, ucfg, esOutputCfg) if err != nil { return nil, err } - if beaterConfig.RumConfig.IsEnabled() { - if b.Config != nil && beaterConfig.RumConfig.SourceMapping.EsConfig == nil { - // fall back to elasticsearch output configuration for sourcemap storage if possible - if isElasticsearchOutput(b) { - logger.Info("Falling back to elasticsearch output for sourcemap storage") - beaterConfig.SetSourcemapElasticsearch(b.Config.Output.Config()) - } else { - logger.Info("Unable to determine sourcemap storage, sourcemaps will not be applied") - } - } - } - if isElasticsearchOutput(b) && - (b.Config.Output.Config().HasField("pipeline") || b.Config.Output.Config().HasField("pipelines")) { - beaterConfig.Pipeline = "" - } bt := &beater{ config: beaterConfig, diff --git a/beater/beater_test.go b/beater/beater_test.go index b8abdfc4dca..bf04b505bb6 100644 --- a/beater/beater_test.go +++ b/beater/beater_test.go @@ -28,9 +28,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" - "github.com/elastic/apm-server/beater/config" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/transport/tlscommon" @@ -41,6 +42,8 @@ import ( "github.com/elastic/beats/libbeat/publisher/processing" "github.com/elastic/beats/libbeat/publisher/queue" "github.com/elastic/beats/libbeat/publisher/queue/memqueue" + + "github.com/elastic/apm-server/beater/config" ) func TestBeatConfig(t *testing.T) { @@ -392,24 +395,13 @@ func (bt *beater) wait() error { } } -func (bt *beater) sourcemapElasticsearchHosts() []string { - var content map[string]interface{} - if err := bt.config.RumConfig.SourceMapping.EsConfig.Unpack(&content); err != nil { - return nil - } - hostsContent := content["hosts"].([]interface{}) - hosts := make([]string, len(hostsContent)) - for i := range hostsContent { - hosts[i] = hostsContent[i].(string) - } - return hosts -} - func setupBeater(t *testing.T, apmBeat *beat.Beat, ucfg *common.Config, beatConfig *beat.BeatConfig) (*beater, func(), error) { // create our beater beatBeater, err := New(apmBeat, ucfg) - assert.NoError(t, err) - assert.NotNil(t, beatBeater) + if err != nil { + return nil, nil, err + } + require.NotNil(t, beatBeater) c := make(chan error) // start it diff --git a/beater/config/config.go b/beater/config/config.go index ce45edfe3ad..aa5d20337fa 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -18,20 +18,19 @@ package config import ( - "fmt" "net" "path/filepath" "regexp" - "strings" "time" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/transport/tlscommon" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/paths" - "github.com/elastic/apm-server/sourcemap" + logs "github.com/elastic/apm-server/log" ) const ( @@ -43,6 +42,10 @@ const ( msgInvalidConfigAgentCfg = "invalid value for `apm-server.agent.config.cache.expiration`, only accepting full seconds" ) +var ( + regexObserverVersion = regexp.MustCompile("%.*{.*observer.version.?}") +) + // Config holds configuration information nested under the key `apm-server` type Config struct { Host string `config:"host"` @@ -52,7 +55,6 @@ type Config struct { WriteTimeout time.Duration `config:"write_timeout"` MaxEventSize int `config:"max_event_size"` ShutdownTimeout time.Duration `config:"shutdown_timeout"` - SecretToken string `config:"secret_token"` TLS *tlscommon.ServerConfig `config:"ssl"` MaxConnections int `config:"max_connections"` Expvar *ExpvarConfig `config:"expvar"` @@ -63,6 +65,7 @@ type Config struct { Mode Mode `config:"mode"` Kibana *common.Config `config:"kibana"` AgentConfig *AgentConfig `config:"agent.config"` + SecretToken string `config:"secret_token"` Pipeline string } @@ -73,61 +76,22 @@ type ExpvarConfig struct { URL string `config:"url"` } -// RumConfig holds config information related to the RUM endpoint -type RumConfig struct { - Enabled *bool `config:"enabled"` - EventRate *EventRate `config:"event_rate"` - AllowOrigins []string `config:"allow_origins"` - LibraryPattern string `config:"library_pattern"` - ExcludeFromGrouping string `config:"exclude_from_grouping"` - SourceMapping *SourceMapping `config:"source_mapping"` - - BeatVersion string -} - -// EventRate holds config information about event rate limiting -type EventRate struct { - Limit int `config:"limit"` - LruSize int `config:"lru_size"` -} - -// RegisterConfig holds ingest config information -type RegisterConfig struct { - Ingest *IngestConfig `config:"ingest"` -} - -// IngestConfig holds config pipeline ingest information -type IngestConfig struct { - Pipeline *PipelineConfig `config:"pipeline"` -} - -// PipelineConfig holds config information about registering ingest pipelines -type PipelineConfig struct { - Enabled *bool `config:"enabled"` - Overwrite *bool `config:"overwrite"` - Path string -} - // AgentConfig holds remote agent config information type AgentConfig struct { Cache *Cache `config:"cache"` } -// SourceMapping holds sourecemap config information -type SourceMapping struct { - Cache *Cache `config:"cache"` - Enabled *bool `config:"enabled"` - IndexPattern string `config:"index_pattern"` - - EsConfig *common.Config `config:"elasticsearch"` - mapper sourcemap.Mapper -} - // Cache holds config information about cache expiration type Cache struct { Expiration time.Duration `config:"expiration"` } +// LimitedCache holds config information about cache expiration +type LimitedCache struct { + Expiration time.Duration `config:"expiration"` + Size int `config:"size"` +} + // InstrumentationConfig holds config information about self instrumenting the APM Server type InstrumentationConfig struct { Enabled *bool `config:"enabled"` @@ -136,30 +100,9 @@ type InstrumentationConfig struct { SecretToken string `config:"secret_token"` } -//Mode enumerates the APM Server env -type Mode uint8 - -const ( - // ModeProduction is the default mode - ModeProduction Mode = iota - - // ModeExperimental should only be used in development environments. It allows to circumvent some restrictions - // on the Intake API for faster development. - ModeExperimental -) - -// Unpack parses the given string into a Mode value -func (m *Mode) Unpack(s string) error { - if strings.ToLower(s) == "experimental" { - *m = ModeExperimental - return nil - } - *m = ModeProduction - return nil -} - // NewConfig creates a Config struct based on the default config and the given input params -func NewConfig(version string, ucfg *common.Config) (*Config, error) { +func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { + logger := logp.NewLogger(logs.Config) c := DefaultConfig(version) if ucfg.HasField("ssl") { @@ -181,23 +124,16 @@ func NewConfig(version string, ucfg *common.Config) (*Config, error) { if float64(int(c.AgentConfig.Cache.Expiration.Seconds())) != c.AgentConfig.Cache.Expiration.Seconds() { return nil, errors.New(msgInvalidConfigAgentCfg) } - if c.RumConfig.IsEnabled() { - if _, err := regexp.Compile(c.RumConfig.LibraryPattern); err != nil { - return nil, errors.New(fmt.Sprintf("Invalid regex for `library_pattern`: %v", err.Error())) - } - if _, err := regexp.Compile(c.RumConfig.ExcludeFromGrouping); err != nil { - return nil, errors.New(fmt.Sprintf("Invalid regex for `exclude_from_grouping`: %v", err.Error())) - } - } - return c, nil -} + if outputESCfg != nil && (outputESCfg.HasField("pipeline") || outputESCfg.HasField("pipelines")) { + c.Pipeline = "" + } -// SetSourcemapElasticsearch sets the Elasticsearch configuration for uploading sourcemaps -func (c *Config) SetSourcemapElasticsearch(esConfig *common.Config) { - if c != nil && c.RumConfig.IsEnabled() && c.RumConfig.SourceMapping != nil { - c.RumConfig.SourceMapping.EsConfig = esConfig + if err := c.RumConfig.setup(logger, outputESCfg); err != nil { + return nil, err } + + return c, nil } // IsEnabled indicates whether expvar is enabled or not @@ -205,21 +141,6 @@ func (c *ExpvarConfig) IsEnabled() bool { return c != nil && (c.Enabled == nil || *c.Enabled) } -// IsEnabled indicates whether RUM endpoint is enabled or not -func (c *RumConfig) IsEnabled() bool { - return c != nil && (c.Enabled != nil && *c.Enabled) -} - -// IsEnabled indicates whether sourcemap handling is enabled or not -func (s *SourceMapping) IsEnabled() bool { - return s == nil || s.Enabled == nil || *s.Enabled -} - -// IsSetup indicates whether there is an Elasticsearch configured for sourcemap handling -func (s *SourceMapping) IsSetup() bool { - return s != nil && (s.EsConfig != nil) -} - // IsEnabled indicates whether pipeline registration is enabled or not func (c *PipelineConfig) IsEnabled() bool { return c != nil && (c.Enabled == nil || *c.Enabled) @@ -230,28 +151,6 @@ func (c *PipelineConfig) ShouldOverwrite() bool { return c != nil && (c.Overwrite != nil && *c.Overwrite) } -// MemoizedSourcemapMapper creates the sourcemap mapper once and then caches it -func (c *RumConfig) MemoizedSourcemapMapper() (sourcemap.Mapper, error) { - if !c.IsEnabled() || !c.SourceMapping.IsEnabled() || !c.SourceMapping.IsSetup() { - return nil, nil - } - if c.SourceMapping.mapper != nil { - return c.SourceMapping.mapper, nil - } - - sourcemapConfig := sourcemap.Config{ - CacheExpiration: c.SourceMapping.Cache.Expiration, - ElasticsearchConfig: c.SourceMapping.EsConfig, - Index: replaceVersion(c.SourceMapping.IndexPattern, c.BeatVersion), - } - mapper, err := sourcemap.NewSmapMapper(sourcemapConfig) - if err != nil { - return nil, err - } - c.SourceMapping.mapper = mapper - return c.SourceMapping.mapper, nil -} - // IsEnabled indicates whether self instrumentation is enabled func (c *InstrumentationConfig) IsEnabled() bool { // self instrumentation is disabled by default. @@ -259,27 +158,7 @@ func (c *InstrumentationConfig) IsEnabled() bool { } func replaceVersion(pattern, version string) string { - re := regexp.MustCompile("%.*{.*observer.version.?}") - return re.ReplaceAllLiteralString(pattern, version) -} - -func defaultRum(beatVersion string) *RumConfig { - return &RumConfig{ - EventRate: &EventRate{ - Limit: 300, - LruSize: 1000, - }, - AllowOrigins: []string{"*"}, - SourceMapping: &SourceMapping{ - Cache: &Cache{ - Expiration: 5 * time.Minute, - }, - IndexPattern: "apm-*-sourcemap*", - }, - LibraryPattern: "node_modules|bower_components|~", - ExcludeFromGrouping: "^/webpack", - BeatVersion: beatVersion, - } + return regexObserverVersion.ReplaceAllLiteralString(pattern, version) } // DefaultConfig returns a config with default settings for `apm-server` config options. @@ -294,28 +173,12 @@ func DefaultConfig(beatVersion string) *Config { WriteTimeout: 30 * time.Second, MaxEventSize: 300 * 1024, // 300 kb ShutdownTimeout: 5 * time.Second, - SecretToken: "", AugmentEnabled: true, Expvar: &ExpvarConfig{ Enabled: new(bool), URL: "/debug/vars", }, - RumConfig: &RumConfig{ - EventRate: &EventRate{ - Limit: 300, - LruSize: 1000, - }, - AllowOrigins: []string{"*"}, - SourceMapping: &SourceMapping{ - Cache: &Cache{ - Expiration: 5 * time.Minute, - }, - IndexPattern: "apm-*-sourcemap*", - }, - LibraryPattern: "node_modules|bower_components|~", - ExcludeFromGrouping: "^/webpack", - BeatVersion: beatVersion, - }, + RumConfig: defaultRum(beatVersion), Register: &RegisterConfig{ Ingest: &IngestConfig{ Pipeline: &PipelineConfig{ diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 23aef6ad80e..5469ec98bd3 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -19,6 +19,7 @@ package config import ( "fmt" + "path/filepath" "testing" "time" @@ -29,151 +30,208 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/outputs" - "github.com/elastic/go-ucfg/yaml" ) -func TestConfig(t *testing.T) { - truthy := true - cases := []struct { - config []byte - expectedConfig Config +func Test_UnpackConfig(t *testing.T) { + falsy, truthy := false, true + version := "8.0.0" + + tests := map[string]struct { + inpCfg map[string]interface{} + outCfg *Config + outCfgWithES *Config }{ - { - config: []byte(`{ - "host": "localhost:3000", - "max_header_size": 8, - "idle_timeout": 4s, - "read_timeout": 3s, - "write_timeout": 4s, - "shutdown_timeout": 9s, - "secret_token": "1234random", - "ssl": { - "enabled": true, - "key": "1234key", - "certificate": "1234cert", - "certificate_authorities": ["./ca.cert.pem"] + "default config": { + inpCfg: map[string]interface{}{}, + outCfg: DefaultConfig(version), + outCfgWithES: DefaultConfig(version), }, - "rum": { - "enabled": true, - "event_rate": { - "limit": 8000, - "lru_size": 2000, - }, - "allow_origins": ["rum*"], - "source_mapping": { - "cache": { - "expiration": 1m, + "overwrite default": { + inpCfg: map[string]interface{}{ + "host": "localhost:3000", + "max_header_size": 8, + "max_event_size": 100, + "idle_timeout": 5 * time.Second, + "read_timeout": 3 * time.Second, + "write_timeout": 4 * time.Second, + "shutdown_timeout": 9 * time.Second, + "capture_personal_data": true, + "secret_token": "1234random", + "ssl": map[string]interface{}{ + "enabled": true, + "key": "1234key", + "certificate": "1234cert", + "certificate_authorities": []string{"./ca.cert.pem"}, + "client_authentication": "none", + }, + "expvar": map[string]interface{}{ + "enabled": true, + "url": "/debug/vars", + }, + "rum": map[string]interface{}{ + "enabled": true, + "event_rate": map[string]interface{}{ + "limit": 7200, + "lru_size": 2000, + }, + "allow_origins": []string{"example*"}, + "source_mapping": map[string]interface{}{ + "cache": map[string]interface{}{ + "expiration": 8 * time.Minute, + }, + "index_pattern": "apm-test*", + }, + "library_pattern": "^custom", + "exclude_from_grouping": "^grouping", }, - "index_pattern": "apm-rum-test*" + "register": map[string]interface{}{ + "ingest": map[string]interface{}{ + "pipeline": map[string]interface{}{ + "overwrite": false, + "path": filepath.Join("tmp", "definition.json"), + }, + }, + }, + "kibana": map[string]interface{}{"enabled": "true"}, + "agent.config.cache.expiration": "2m", }, - "library_pattern": "pattern-rum", - "exclude_from_grouping": "group_pattern-rum", - }, - "register": { - "ingest": { - "pipeline": { - enabled: true, - overwrite: true, - path: "tmp", - } - } - } - }`), - expectedConfig: Config{ + outCfg: &Config{ Host: "localhost:3000", MaxHeaderSize: 8, - IdleTimeout: 4000000000, + MaxEventSize: 100, + IdleTimeout: 5000000000, ReadTimeout: 3000000000, WriteTimeout: 4000000000, ShutdownTimeout: 9000000000, SecretToken: "1234random", TLS: &tlscommon.ServerConfig{ Enabled: &truthy, - CAs: []string{"./ca.cert.pem"}, Certificate: outputs.CertificateConfig{Certificate: "1234cert", Key: "1234key"}, - ClientAuth: 4}, //4=RequireAndVerifyClientCert + ClientAuth: 0, + CAs: []string{"./ca.cert.pem"}, + }, + AugmentEnabled: true, + Expvar: &ExpvarConfig{ + Enabled: &truthy, + URL: "/debug/vars", + }, RumConfig: &RumConfig{ Enabled: &truthy, EventRate: &EventRate{ - Limit: 8000, + Limit: 7200, LruSize: 2000, }, - AllowOrigins: []string{"rum*"}, + AllowOrigins: []string{"example*"}, SourceMapping: &SourceMapping{ - Cache: &Cache{Expiration: 1 * time.Minute}, - IndexPattern: "apm-rum-test*", + Cache: &Cache{Expiration: 8 * time.Minute}, + IndexPattern: "apm-test*", }, - LibraryPattern: "pattern-rum", - ExcludeFromGrouping: "group_pattern-rum", + LibraryPattern: "^custom", + ExcludeFromGrouping: "^grouping", + BeatVersion: version, }, Register: &RegisterConfig{ Ingest: &IngestConfig{ Pipeline: &PipelineConfig{ Enabled: &truthy, - Overwrite: &truthy, - Path: "tmp", + Overwrite: &falsy, + Path: filepath.Join("tmp", "definition.json"), }, }, }, + Kibana: common.MustNewConfigFrom(map[string]interface{}{"enabled": "true"}), + AgentConfig: &AgentConfig{Cache: &Cache{Expiration: 2 * time.Minute}}, + Pipeline: DefaultAPMPipeline, }, }, - { - config: []byte(`{ - "host": "localhost:8200", - "max_header_size": 8, - "read_timeout": 3s, - "write_timeout": 2s, - "shutdown_timeout": 5s, - "secret_token": "1234random", - "rum": { - "source_mapping": {} - }, - "register": {} - }`), - expectedConfig: Config{ - Host: "localhost:8200", - MaxHeaderSize: 8, - ReadTimeout: 3000000000, - WriteTimeout: 2000000000, + "merge config with default": { + inpCfg: map[string]interface{}{ + "host": "localhost:3000", + "secret_token": "1234random", + "ssl": map[string]interface{}{ + "enabled": true, + }, + "expvar": map[string]interface{}{ + "enabled": true, + "url": "/debug/vars", + }, + "rum": map[string]interface{}{ + "enabled": true, + "source_mapping": map[string]interface{}{ + "cache": map[string]interface{}{ + "expiration": 7, + }, + }, + "library_pattern": "rum", + }, + "register": map[string]interface{}{ + "ingest": map[string]interface{}{ + "pipeline": map[string]interface{}{ + "enabled": false, + }, + }, + }, + }, + outCfg: &Config{ + Host: "localhost:3000", + MaxHeaderSize: 1048576, + MaxEventSize: 307200, + IdleTimeout: 45000000000, + ReadTimeout: 30000000000, + WriteTimeout: 30000000000, ShutdownTimeout: 5000000000, SecretToken: "1234random", + TLS: &tlscommon.ServerConfig{ + Enabled: &truthy, + Certificate: outputs.CertificateConfig{Certificate: "", Key: ""}, + ClientAuth: 3}, + AugmentEnabled: true, + Expvar: &ExpvarConfig{ + Enabled: &truthy, + URL: "/debug/vars", + }, RumConfig: &RumConfig{ - Enabled: nil, - EventRate: nil, - AllowOrigins: nil, + Enabled: &truthy, + EventRate: &EventRate{ + Limit: 300, + LruSize: 1000, + }, + AllowOrigins: []string{"*"}, SourceMapping: &SourceMapping{ - IndexPattern: "", + Cache: &Cache{ + Expiration: 7 * time.Second, + }, + IndexPattern: "apm-*-sourcemap*", }, + LibraryPattern: "rum", + ExcludeFromGrouping: "^/webpack", + BeatVersion: "8.0.0", }, Register: &RegisterConfig{ - Ingest: nil, + Ingest: &IngestConfig{ + Pipeline: &PipelineConfig{ + Enabled: &falsy, + Path: filepath.Join("ingest", "pipeline", "definition.json"), + }, + }, }, - }, - }, - { - config: []byte(`{ }`), - expectedConfig: Config{ - Host: "", - MaxHeaderSize: 0, - IdleTimeout: 0, - ReadTimeout: 0, - WriteTimeout: 0, - ShutdownTimeout: 0, - SecretToken: "", - TLS: nil, - RumConfig: nil, + Kibana: common.MustNewConfigFrom(map[string]interface{}{"enabled": "false"}), + AgentConfig: &AgentConfig{Cache: &Cache{Expiration: 30 * time.Second}}, + Pipeline: DefaultAPMPipeline, }, }, } - for idx, test := range cases { - cfg, err := yaml.NewConfig(test.config) - require.NoError(t, err) - var beaterConfig Config - err = cfg.Unpack(&beaterConfig) - require.NoError(t, err) - msg := fmt.Sprintf("Test number %v failed. Config: %v, ExpectedConfig: %v", idx, beaterConfig, test.expectedConfig) - assert.Equal(t, test.expectedConfig, beaterConfig, msg) + for name, test := range tests { + t.Run(name+"no outputESCfg", func(t *testing.T) { + inpCfg, err := common.NewConfigFrom(test.inpCfg) + assert.NoError(t, err) + + cfg, err := NewConfig(version, inpCfg, nil) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.Equal(t, test.outCfg, cfg) + }) } } @@ -193,70 +251,6 @@ func TestReplaceBeatVersion(t *testing.T) { } } -func TestIsRumEnabled(t *testing.T) { - truthy := true - for _, td := range []struct { - c *Config - enabled bool - }{ - {c: &Config{RumConfig: &RumConfig{Enabled: new(bool)}}, enabled: false}, - {c: &Config{RumConfig: &RumConfig{Enabled: &truthy}}, enabled: true}, - } { - assert.Equal(t, td.enabled, td.c.RumConfig.IsEnabled()) - - } -} - -func TestDefaultRum(t *testing.T) { - c := DefaultConfig("7.0.0") - assert.Equal(t, c.RumConfig, defaultRum("7.0.0")) -} - -func TestMemoizedSourcemapMapper(t *testing.T) { - truthy := true - esConfig, err := common.NewConfigFrom(map[string]interface{}{ - "hosts": []string{"localhost:0"}, - }) - require.NoError(t, err) - mapping := SourceMapping{ - Cache: &Cache{Expiration: 1 * time.Minute}, - IndexPattern: "apm-rum-test*", - EsConfig: esConfig, - } - - for idx, td := range []struct { - c *Config - mapper bool - e error - }{ - {c: &Config{RumConfig: &RumConfig{}}, mapper: false, e: nil}, - {c: &Config{RumConfig: &RumConfig{Enabled: new(bool)}}, mapper: false, e: nil}, - {c: &Config{RumConfig: &RumConfig{Enabled: &truthy}}, mapper: false, e: nil}, - {c: &Config{RumConfig: &RumConfig{SourceMapping: &mapping}}, mapper: false, e: nil}, - {c: &Config{ - RumConfig: &RumConfig{ - Enabled: &truthy, - SourceMapping: &SourceMapping{ - Cache: &Cache{Expiration: 1 * time.Minute}, - IndexPattern: "apm-rum-test*", - }, - }}, - mapper: false, - e: nil}, - {c: &Config{RumConfig: &RumConfig{Enabled: &truthy, SourceMapping: &mapping}}, - mapper: true, - e: nil}, - } { - mapper, e := td.c.RumConfig.MemoizedSourcemapMapper() - if td.mapper { - assert.NotNil(t, mapper, fmt.Sprintf("Test number <%v> failed", idx)) - } else { - assert.Nil(t, mapper, fmt.Sprintf("Test number <%v> failed", idx)) - } - assert.Equal(t, td.e, e) - } -} - func TestPipeline(t *testing.T) { truthy, falsy := true, false cases := []struct { @@ -314,7 +308,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.ClientAuth, cfg.TLS.ClientAuth) }) @@ -340,7 +334,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.VerificationMode, cfg.TLS.VerificationMode) }) @@ -373,22 +367,39 @@ func TestTLSSettings(t *testing.T) { func TestAgentConfig(t *testing.T) { t.Run("InvalidValueTooSmall", func(t *testing.T) { cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"})) + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("InvalidUnit", func(t *testing.T) { cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"})) + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("Valid", func(t *testing.T) { cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"})) + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"}), nil) require.NoError(t, err) assert.Equal(t, time.Second*123, cfg.AgentConfig.Cache.Expiration) }) } + +func TestSourcemapESConfig(t *testing.T) { + version := "8.0.0" + ucfg, err := common.NewConfigFrom(`{"rum":{"enabled":true}}`) + require.NoError(t, err) + + // no es config given + cfg, err := NewConfig(version, ucfg, nil) + require.NoError(t, err) + assert.Nil(t, cfg.RumConfig.SourceMapping.ESConfig) + + // with es config + outputESCfg := common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`) + cfg, err = NewConfig(version, ucfg, outputESCfg) + require.NoError(t, err) + assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) +} diff --git a/beater/config/config_helper.go b/beater/config/helper.go similarity index 100% rename from beater/config/config_helper.go rename to beater/config/helper.go diff --git a/beater/config/config_helper_test.go b/beater/config/helper_test.go similarity index 100% rename from beater/config/config_helper_test.go rename to beater/config/helper_test.go diff --git a/sourcemap/id.go b/beater/config/mode.go similarity index 55% rename from sourcemap/id.go rename to beater/config/mode.go index b5730fb2505..ff2a2f58a90 100644 --- a/sourcemap/id.go +++ b/beater/config/mode.go @@ -15,41 +15,28 @@ // specific language governing permissions and limitations // under the License. -package sourcemap +package config -import ( - "fmt" - "strings" -) +import "strings" -type Id struct { - ServiceName string - ServiceVersion string - Path string -} +//Mode enumerates the APM Server env +type Mode uint8 -func (i *Id) Key() string { - info := []string{} - info = add(info, i.ServiceName) - info = add(info, i.ServiceVersion) - info = add(info, i.Path) - return strings.Join(info, "_") -} +const ( + // ModeProduction is the default mode + ModeProduction Mode = iota -func (i *Id) String() string { - return fmt.Sprintf("Service Name: '%s', Service Version: '%s' and Path: '%s'", - i.ServiceName, - i.ServiceVersion, - i.Path) -} - -func (i *Id) Valid() bool { - return i.ServiceName != "" && i.ServiceVersion != "" && i.Path != "" -} + // ModeExperimental should only be used in development environments. It allows to circumvent some restrictions + // on the Intake API for faster development. + ModeExperimental +) -func add(a []string, s string) []string { - if s != "" { - a = append(a, s) +// Unpack parses the given string into a Mode value +func (m *Mode) Unpack(s string) error { + if strings.ToLower(s) == "experimental" { + *m = ModeExperimental + return nil } - return a + *m = ModeProduction + return nil } diff --git a/beater/config/register.go b/beater/config/register.go new file mode 100644 index 00000000000..cab44410985 --- /dev/null +++ b/beater/config/register.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +// RegisterConfig holds ingest config information +type RegisterConfig struct { + Ingest *IngestConfig `config:"ingest"` +} + +// IngestConfig holds config pipeline ingest information +type IngestConfig struct { + Pipeline *PipelineConfig `config:"pipeline"` +} + +// PipelineConfig holds config information about registering ingest pipelines +type PipelineConfig struct { + Enabled *bool `config:"enabled"` + Overwrite *bool `config:"overwrite"` + Path string +} diff --git a/beater/config/rum.go b/beater/config/rum.go new file mode 100644 index 00000000000..b83ecb08e68 --- /dev/null +++ b/beater/config/rum.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "regexp" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/apm-server/elasticsearch" + "github.com/elastic/apm-server/sourcemap" +) + +const ( + allowAllOrigins = "*" + defaultEventRateLimit = 300 + defaultEventRateLRUSize = 1000 + defaultExcludeFromGrouping = "^/webpack" + defaultLibraryPattern = "node_modules|bower_components|~" + defaultSourcemapCacheExpiration = 5 * time.Minute + defaultSourcemapIndexPattern = "apm-*-sourcemap*" + + esConnectionTimeout = 5 * time.Second +) + +// RumConfig holds config information related to the RUM endpoint +type RumConfig struct { + Enabled *bool `config:"enabled"` + EventRate *EventRate `config:"event_rate"` + AllowOrigins []string `config:"allow_origins"` + LibraryPattern string `config:"library_pattern"` + ExcludeFromGrouping string `config:"exclude_from_grouping"` + SourceMapping *SourceMapping `config:"source_mapping"` + + BeatVersion string +} + +// EventRate holds config information about event rate limiting +type EventRate struct { + Limit int `config:"limit"` + LruSize int `config:"lru_size"` +} + +// SourceMapping holds sourecemap config information +type SourceMapping struct { + Cache *Cache `config:"cache"` + Enabled *bool `config:"enabled"` + IndexPattern string `config:"index_pattern"` + ESConfig *elasticsearch.Config `config:"elasticsearch"` + store *sourcemap.Store +} + +// IsEnabled indicates whether RUM endpoint is enabled or not +func (c *RumConfig) IsEnabled() bool { + return c != nil && (c.Enabled != nil && *c.Enabled) +} + +// IsEnabled indicates whether sourcemap handling is enabled or not +func (s *SourceMapping) IsEnabled() bool { + return s == nil || s.Enabled == nil || *s.Enabled +} + +// MemoizedSourcemapStore creates the sourcemap store once and then caches it +func (c *RumConfig) MemoizedSourcemapStore() (*sourcemap.Store, error) { + if !c.IsEnabled() || !c.SourceMapping.IsEnabled() || !c.SourceMapping.isSetup() { + return nil, nil + } + if c.SourceMapping.store != nil { + return c.SourceMapping.store, nil + } + + esClient, err := elasticsearch.NewClient(c.SourceMapping.ESConfig) + if err != nil { + return nil, err + } + index := replaceVersion(c.SourceMapping.IndexPattern, c.BeatVersion) + store, err := sourcemap.NewStore(esClient, index, c.SourceMapping.Cache.Expiration) + if err != nil { + return nil, err + } + c.SourceMapping.store = store + return store, nil +} + +func (c *RumConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { + if !c.IsEnabled() { + return nil + } + + if _, err := regexp.Compile(c.LibraryPattern); err != nil { + return errors.Wrapf(err, "Invalid regex for `library_pattern`: ") + } + if _, err := regexp.Compile(c.ExcludeFromGrouping); err != nil { + return errors.Wrapf(err, "Invalid regex for `exclude_from_grouping`: ") + } + + if c.SourceMapping == nil || c.SourceMapping.ESConfig != nil { + return nil + } + + // fall back to elasticsearch output configuration for sourcemap storage if possible + if outputESCfg == nil { + log.Info("Unable to determine sourcemap storage, sourcemaps will not be applied") + return nil + } + log.Info("Falling back to elasticsearch output for sourcemap storage") + esCfg := &elasticsearch.Config{Hosts: []string{"localhost:9200"}, Protocol: "http", Timeout: esConnectionTimeout} + if err := outputESCfg.Unpack(esCfg); err != nil { + return errors.Wrap(err, "unpacking Elasticsearch config into Sourcemap config") + } + c.SourceMapping.ESConfig = esCfg + return nil +} + +func (s *SourceMapping) isSetup() bool { + return s != nil && (s.ESConfig != nil) +} + +func defaultRum(beatVersion string) *RumConfig { + return &RumConfig{ + EventRate: &EventRate{ + Limit: defaultEventRateLimit, + LruSize: defaultEventRateLRUSize, + }, + AllowOrigins: []string{allowAllOrigins}, + SourceMapping: &SourceMapping{ + Cache: &Cache{ + Expiration: defaultSourcemapCacheExpiration, + }, + IndexPattern: defaultSourcemapIndexPattern, + }, + LibraryPattern: defaultLibraryPattern, + ExcludeFromGrouping: defaultExcludeFromGrouping, + BeatVersion: beatVersion, + } +} diff --git a/beater/config/rum_test.go b/beater/config/rum_test.go new file mode 100644 index 00000000000..4e5d7b3304c --- /dev/null +++ b/beater/config/rum_test.go @@ -0,0 +1,89 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package config + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/apm-server/elasticsearch" +) + +func TestIsRumEnabled(t *testing.T) { + truthy := true + for _, td := range []struct { + c *Config + enabled bool + }{ + {c: &Config{RumConfig: &RumConfig{Enabled: new(bool)}}, enabled: false}, + {c: &Config{RumConfig: &RumConfig{Enabled: &truthy}}, enabled: true}, + } { + assert.Equal(t, td.enabled, td.c.RumConfig.IsEnabled()) + + } +} + +func TestDefaultRum(t *testing.T) { + c := DefaultConfig("7.0.0") + assert.Equal(t, defaultRum("7.0.0"), c.RumConfig) +} + +func TestMemoizedSourcemapMapper(t *testing.T) { + truthy := true + esConfig := elasticsearch.Config{Hosts: []string{"localhost:0"}} + mapping := SourceMapping{ + Cache: &Cache{Expiration: 1 * time.Minute}, + IndexPattern: "apm-rum-test*", + ESConfig: &esConfig, + } + + for idx, td := range []struct { + c *Config + mapper bool + e error + }{ + {c: &Config{RumConfig: &RumConfig{}}, mapper: false, e: nil}, + {c: &Config{RumConfig: &RumConfig{Enabled: new(bool)}}, mapper: false, e: nil}, + {c: &Config{RumConfig: &RumConfig{Enabled: &truthy}}, mapper: false, e: nil}, + {c: &Config{RumConfig: &RumConfig{SourceMapping: &mapping}}, mapper: false, e: nil}, + {c: &Config{ + RumConfig: &RumConfig{ + Enabled: &truthy, + SourceMapping: &SourceMapping{ + Cache: &Cache{Expiration: 1 * time.Minute}, + IndexPattern: "apm-rum-test*", + }, + }}, + mapper: false, + e: nil}, + {c: &Config{RumConfig: &RumConfig{Enabled: &truthy, SourceMapping: &mapping}}, + mapper: true, + e: nil}, + } { + mapper, e := td.c.RumConfig.MemoizedSourcemapStore() + if td.mapper { + assert.NotNil(t, mapper, fmt.Sprintf("Test number <%v> failed", idx)) + } else { + assert.Nil(t, mapper, fmt.Sprintf("Test number <%v> failed", idx)) + } + assert.Equal(t, td.e, e) + } +} diff --git a/beater/server_test.go b/beater/server_test.go index bece4753b41..328ef6b022d 100644 --- a/beater/server_test.go +++ b/beater/server_test.go @@ -40,6 +40,7 @@ import ( "github.com/elastic/apm-server/beater/api" "github.com/elastic/apm-server/beater/config" + "github.com/elastic/apm-server/elasticsearch" "github.com/elastic/apm-server/tests/loader" ) @@ -234,7 +235,8 @@ func TestServerRumSwitch(t *testing.T) { func TestServerSourcemapBadConfig(t *testing.T) { ucfg, err := common.NewConfigFrom(m{"rum": m{"enabled": true, "source_mapping": m{"elasticsearch": m{"hosts": []string{}}}}}) require.NoError(t, err) - _, teardown, err := setupServer(t, ucfg, nil, nil) + s, teardown, err := setupServer(t, ucfg, nil, nil) + require.Nil(t, s) if err == nil { defer teardown() } @@ -318,18 +320,17 @@ func TestServerNoContentType(t *testing.T) { } func TestServerSourcemapElasticsearch(t *testing.T) { - cases := []struct { - expected []string + for name, tc := range map[string]struct { + expected elasticsearch.Hosts config m outputConfig m }{ - { + "nil": { expected: nil, config: m{}, }, - { - // source_mapping.elasticsearch.hosts set - expected: []string{"localhost:5200"}, + "esConfigured": { + expected: elasticsearch.Hosts{"localhost:5200"}, config: m{ "rum": m{ "enabled": "true", @@ -337,9 +338,8 @@ func TestServerSourcemapElasticsearch(t *testing.T) { }, }, }, - { - // source_mapping.elasticsearch.hosts not set, elasticsearch.enabled = true - expected: []string{"localhost:5201"}, + "esFromOutput": { + expected: elasticsearch.Hosts{"localhost:5201"}, config: m{ "rum": m{ "enabled": "true", @@ -352,8 +352,7 @@ func TestServerSourcemapElasticsearch(t *testing.T) { }, }, }, - { - // source_mapping.elasticsearch.hosts not set, elasticsearch.enabled = false + "esOutputDisabled": { expected: nil, config: m{ "rum": m{ @@ -367,24 +366,22 @@ func TestServerSourcemapElasticsearch(t *testing.T) { }, }, }, - } - for _, testCase := range cases { - ucfg, err := common.NewConfigFrom(testCase.config) - if !assert.NoError(t, err) { - continue - } + } { + t.Run(name, func(t *testing.T) { + ucfg, err := common.NewConfigFrom(tc.config) + require.NoError(t, err) - var beatConfig beat.BeatConfig - ocfg, err := common.NewConfigFrom(testCase.outputConfig) - if !assert.NoError(t, err) { - continue - } - beatConfig.Output.Unpack(ocfg) - apm, teardown, err := setupServer(t, ucfg, &beatConfig, nil) - if assert.NoError(t, err) { - assert.Equal(t, testCase.expected, apm.sourcemapElasticsearchHosts()) - } - teardown() + var beatConfig beat.BeatConfig + ocfg, err := common.NewConfigFrom(tc.outputConfig) + require.NoError(t, err) + require.NoError(t, beatConfig.Output.Unpack(ocfg)) + apm, teardown, err := setupServer(t, ucfg, &beatConfig, nil) + require.NoError(t, err) + if tc.expected != nil { + assert.Equal(t, tc.expected, apm.config.RumConfig.SourceMapping.ESConfig.Hosts) + } + teardown() + }) } } diff --git a/elasticsearch/client.go b/elasticsearch/client.go index ed6a381acf0..f99b669ddb6 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -35,7 +35,7 @@ import ( const ( prefixHTTP = "http" prefixHTTPSchema = prefixHTTP + "://" - esDefaultPort = 9200 + defaultESPort = 9200 ) var ( @@ -122,7 +122,7 @@ func httpProxyURL(cfg *Config) (func(*http.Request) (*url.URL, error), error) { func addresses(cfg *Config) ([]string, error) { var addresses []string for _, host := range cfg.Hosts { - address, err := common.MakeURL(cfg.Protocol, cfg.Path, host, esDefaultPort) + address, err := common.MakeURL(cfg.Protocol, cfg.Path, host, defaultESPort) if err != nil { return nil, err } diff --git a/elasticsearch/test/client.go b/elasticsearch/test/client.go new file mode 100644 index 00000000000..c2443a48d03 --- /dev/null +++ b/elasticsearch/test/client.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package test + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pkg/errors" + + "github.com/elastic/go-elasticsearch/v8" +) + +// Transport can be used to pass to test Elasticsearch Client for more control over client behavior +type Transport struct { + roundTripFn func(req *http.Request) (*http.Response, error) + executed int +} + +// RoundTrip implements http.RoundTripper interface +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + t.executed++ + return t.roundTripFn(req) +} + +// NewTransport creates test transport instance returning status code and body according to input parameters when called. +func NewTransport(t *testing.T, statusCode int, esBody map[string]interface{}) *Transport { + var body io.ReadCloser + if esBody == nil { + body = ioutil.NopCloser(bytes.NewReader([]byte{})) + } else { + resp, err := json.Marshal(esBody) + require.NoError(t, err) + body = ioutil.NopCloser(bytes.NewReader(resp)) + } + return &Transport{ + roundTripFn: func(_ *http.Request) (*http.Response, error) { + if statusCode == http.StatusInternalServerError { + return &http.Response{}, errors.New("Internal server error") + } + return &http.Response{StatusCode: statusCode, Body: body}, nil + }, + } +} + +// NewElasticsearchClient creates ES client using the given transport instance +func NewElasticsearchClient(transport *Transport) (*elasticsearch.Client, error) { + return elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{}, Transport: transport}) +} diff --git a/log/selectors.go b/log/selectors.go index 538d542ab91..d7d3097cbbb 100644 --- a/log/selectors.go +++ b/log/selectors.go @@ -20,6 +20,7 @@ package logs // logging selectors const ( Beater = "beater" + Config = "config" Handler = "handler" Ilm = "ilm" IndexManagement = "index-management" diff --git a/model/error/event.go b/model/error/event.go index 0391f505b33..5db6debc35e 100644 --- a/model/error/event.go +++ b/model/error/event.go @@ -254,7 +254,7 @@ func (e *Event) fields(tctx *transform.Context) common.MapStr { } func (e *Event) updateCulprit(tctx *transform.Context) { - if tctx.Config.SourcemapMapper == nil { + if tctx.Config.SourcemapStore == nil { return } var fr *m.StacktraceFrame diff --git a/model/error/event_test.go b/model/error/event_test.go index fd754ab93c7..e7381638bd6 100644 --- a/model/error/event_test.go +++ b/model/error/event_test.go @@ -24,20 +24,17 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" - "os" - "path/filepath" "testing" "time" - s "github.com/go-sourcemap/sourcemap" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" m "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/metadata" "github.com/elastic/apm-server/sourcemap" + "github.com/elastic/apm-server/sourcemap/test" "github.com/elastic/apm-server/transform" "github.com/elastic/apm-server/utility" @@ -136,7 +133,7 @@ func TestErrorEventDecode(t *testing.T) { "stacktrace": "123", }, }, - err: m.ErrInvalidStacktraceType, + err: errors.New("invalid type for stacktrace"), }, "invalid type for log stacktrace": { input: map[string]interface{}{ @@ -146,7 +143,7 @@ func TestErrorEventDecode(t *testing.T) { "stacktrace": "123", }, }, - err: m.ErrInvalidStacktraceType, + err: errors.New("invalid type for stacktrace"), }, "minimal valid error": { input: map[string]interface{}{ @@ -440,68 +437,60 @@ func TestEventFields(t *testing.T) { baseLogGroupingKey := hex.EncodeToString(baseLogHash.Sum(nil)) trId := "945254c5-67a5-417e-8a4e-aa29efcbfb79" - tests := []struct { + tests := map[string]struct { Event Event Output common.MapStr - Msg string }{ - { + "minimal": { Event: Event{}, Output: common.MapStr{ "grouping_key": hex.EncodeToString(md5.New().Sum(nil)), }, - Msg: "Minimal Event", }, - { + "withLog": { Event: Event{Log: baseLog()}, Output: common.MapStr{ "log": common.MapStr{"message": "error log message"}, "grouping_key": baseLogGroupingKey, }, - Msg: "Minimal Event wth log", }, - { + "withLogAndException": { Event: Event{Exception: baseException(), Log: baseLog()}, Output: common.MapStr{ "exception": []common.MapStr{{"message": "exception message"}}, "log": common.MapStr{"message": "error log message"}, "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth log and exception", }, - { + "withException": { Event: Event{Exception: baseException()}, Output: common.MapStr{ "exception": []common.MapStr{{"message": "exception message"}}, "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event with exception", }, - { + "stringCode": { Event: Event{Exception: baseException().withCode("13")}, Output: common.MapStr{ "exception": []common.MapStr{{"message": "exception message", "code": "13"}}, "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event with exception and string code", }, - { + "intCode": { Event: Event{Exception: baseException().withCode(13)}, Output: common.MapStr{ "exception": []common.MapStr{{"message": "exception message", "code": "13"}}, "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth exception and int code", }, - { + "floatCode": { Event: Event{Exception: baseException().withCode(13.0)}, Output: common.MapStr{ "exception": []common.MapStr{{"message": "exception message", "code": "13"}}, "grouping_key": baseExceptionGroupingKey, }, - Msg: "Minimal Event wth exception and float code", }, - { + "withFrames": { Event: Event{ Id: &id, Timestamp: time.Now(), @@ -537,23 +526,24 @@ func TestEventFields(t *testing.T) { }, "grouping_key": "2725d2590215a6e975f393bf438f90ef", }, - Msg: "Event with frames", }, } - s := "myservice" - tctx := &transform.Context{ - Config: transform.Config{SourcemapMapper: &sourcemap.SmapMapper{}}, - Metadata: metadata.Metadata{ - Service: &metadata.Service{Name: &s}, - }, - } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + s := "myservice" + tctx := &transform.Context{ + Config: transform.Config{SourcemapStore: &sourcemap.Store{}}, + Metadata: metadata.Metadata{ + Service: &metadata.Service{Name: &s, Version: &s}, + }, + } - for idx, test := range tests { - output := test.Event.Transform(tctx) - require.Len(t, output, 1) - fields := output[0].Fields["error"] - assert.Equal(t, test.Output, fields, fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) + output := tc.Event.Transform(tctx) + require.Len(t, output, 1) + fields := output[0].Fields["error"] + assert.Equal(t, tc.Output, fields) + }) } } @@ -561,9 +551,9 @@ func TestEvents(t *testing.T) { timestamp := time.Date(2019, 1, 3, 15, 17, 4, 908.596*1e6, time.FixedZone("+0100", 3600)) timestampUs := timestamp.UnixNano() / 1000 - serviceName, agentName, agentVersion := "myservice", "go", "1.0" + serviceName, agentName, version := "myservice", "go", "1.0" service := metadata.Service{ - Name: &serviceName, Agent: metadata.Agent{Name: &agentName, Version: &agentVersion}, + Name: &serviceName, Version: &version, Agent: metadata.Agent{Name: &agentName, Version: &version}, } serviceVersion := "1.2.3" exMsg := "exception message" @@ -577,16 +567,16 @@ func TestEvents(t *testing.T) { labels := m.Labels(common.MapStr{"key": true}) custom := m.Custom(common.MapStr{"foo": "bar"}) metadataLabels := common.MapStr{"label": 101} - tests := []struct { + for name, tc := range map[string]struct { Transformable transform.Transformable Output common.MapStr Msg string }{ - { + "valid": { Transformable: &Event{Timestamp: timestamp}, Output: common.MapStr{ "agent": common.MapStr{"name": "go", "version": "1.0"}, - "service": common.MapStr{"name": "myservice"}, + "service": common.MapStr{"name": "myservice", "version": "1.0"}, "error": common.MapStr{ "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", }, @@ -595,14 +585,13 @@ func TestEvents(t *testing.T) { "timestamp": common.MapStr{"us": timestampUs}, "labels": common.MapStr{"label": 101}, }, - Msg: "Payload with valid Event.", }, - { + "notSampled": { Transformable: &Event{Timestamp: timestamp, TransactionSampled: &sampledFalse}, Output: common.MapStr{ "transaction": common.MapStr{"sampled": false}, "agent": common.MapStr{"name": "go", "version": "1.0"}, - "service": common.MapStr{"name": "myservice"}, + "service": common.MapStr{"name": "myservice", "version": "1.0"}, "error": common.MapStr{ "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", }, @@ -611,9 +600,8 @@ func TestEvents(t *testing.T) { "timestamp": common.MapStr{"us": timestampUs}, "labels": common.MapStr{"label": 101}, }, - Msg: "Payload with valid Event.", }, - { + "withMeta": { Transformable: &Event{Timestamp: timestamp, TransactionType: &transactionType}, Output: common.MapStr{ "transaction": common.MapStr{"type": "request"}, @@ -621,15 +609,14 @@ func TestEvents(t *testing.T) { "grouping_key": "d41d8cd98f00b204e9800998ecf8427e", }, "processor": common.MapStr{"event": "error", "name": "error"}, - "service": common.MapStr{"name": "myservice"}, + "service": common.MapStr{"name": "myservice", "version": "1.0"}, "user": common.MapStr{"id": uid}, "timestamp": common.MapStr{"us": timestampUs}, "agent": common.MapStr{"name": "go", "version": "1.0"}, "labels": common.MapStr{"label": 101}, }, - Msg: "Payload with valid Event.", }, - { + "withContext": { Transformable: &Event{ Timestamp: timestamp, Log: baseLog(), @@ -647,7 +634,7 @@ func TestEvents(t *testing.T) { Output: common.MapStr{ "labels": common.MapStr{"key": true, "label": 101}, - "service": common.MapStr{"name": "myservice"}, + "service": common.MapStr{"name": "myservice", "version": "1.0"}, "agent": common.MapStr{"name": "go", "version": "1.0"}, "user": common.MapStr{"email": email}, "client": common.MapStr{"ip": userIp}, @@ -676,9 +663,8 @@ func TestEvents(t *testing.T) { "transaction": common.MapStr{"id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "sampled": true}, "timestamp": common.MapStr{"us": timestampUs}, }, - Msg: "Payload with Event with Context.", }, - { + "deepUpdateService": { Transformable: &Event{Timestamp: timestamp, Service: &metadata.Service{Version: &serviceVersion}}, Output: common.MapStr{ "service": common.MapStr{"name": serviceName, "version": serviceVersion}, @@ -689,23 +675,23 @@ func TestEvents(t *testing.T) { "processor": common.MapStr{"event": "error", "name": "error"}, "timestamp": common.MapStr{"us": timestampUs}, }, - Msg: "Deep update service fields", }, - } + } { + t.Run(name, func(t *testing.T) { + me := metadata.NewMetadata(&service, nil, nil, &metadata.User{Id: &uid}, metadataLabels) + tctx := &transform.Context{ + Metadata: *me, + Config: transform.Config{SourcemapStore: &sourcemap.Store{}}, + RequestTime: timestamp, + } - me := metadata.NewMetadata(&service, nil, nil, &metadata.User{Id: &uid}, metadataLabels) - tctx := &transform.Context{ - Metadata: *me, - Config: transform.Config{SourcemapMapper: &sourcemap.SmapMapper{}}, - RequestTime: timestamp, - } + outputEvents := tc.Transformable.Transform(tctx) + require.Len(t, outputEvents, 1) + outputEvent := outputEvents[0] + assert.Equal(t, tc.Output, outputEvent.Fields) + assert.Equal(t, timestamp, outputEvent.Timestamp) - for idx, test := range tests { - outputEvents := test.Transformable.Transform(tctx) - require.Len(t, outputEvents, 1) - outputEvent := outputEvents[0] - assert.Equal(t, test.Output, outputEvent.Fields, fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) - assert.Equal(t, timestamp, outputEvent.Timestamp, fmt.Sprintf("Bad timestamp at idx %v; %s", idx, test.Msg)) + }) } } @@ -722,7 +708,7 @@ func TestCulprit(t *testing.T) { &m.StacktraceFrame{Filename: "f", Function: &fct, Sourcemap: m.Sourcemap{Updated: &truthy}}, &m.StacktraceFrame{Filename: "bar", Function: &fct, Sourcemap: m.Sourcemap{Updated: &truthy}}, } - mapper := sourcemap.SmapMapper{} + store := &sourcemap.Store{} tests := []struct { event Event config transform.Config @@ -737,13 +723,13 @@ func TestCulprit(t *testing.T) { }, { event: Event{Culprit: &c}, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "foo", msg: "No Stacktrace Frame given.", }, { event: Event{Culprit: &c, Log: &Log{Stacktrace: st}}, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "foo", msg: "Log.StacktraceFrame has no updated frame", }, @@ -759,7 +745,7 @@ func TestCulprit(t *testing.T) { }, }, }, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "f", msg: "Adapt culprit to first valid Log.StacktraceFrame information.", }, @@ -768,7 +754,7 @@ func TestCulprit(t *testing.T) { Culprit: &c, Exception: &Exception{Stacktrace: stUpdate}, }, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "f in fct", msg: "Adapt culprit to first valid Exception.StacktraceFrame information.", }, @@ -778,7 +764,7 @@ func TestCulprit(t *testing.T) { Log: &Log{Stacktrace: st}, Exception: &Exception{Stacktrace: stUpdate}, }, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "f in fct", msg: "Log and Exception StacktraceFrame given, only one changes culprit.", }, @@ -796,7 +782,7 @@ func TestCulprit(t *testing.T) { }, Exception: &Exception{Stacktrace: stUpdate}, }, - config: transform.Config{SourcemapMapper: &mapper}, + config: transform.Config{SourcemapStore: store}, culprit: "a in fct", msg: "Log Stacktrace is prioritized over Exception StacktraceFrame", }, @@ -1011,51 +997,31 @@ func md5With(args ...string) []byte { } func TestSourcemapping(t *testing.T) { - c1 := 18 - lineno := 1 - empty := "" + col, line, path := 23, 1, "../a/b" exMsg := "exception message" - event := Event{Exception: &Exception{ - Message: &exMsg, - Stacktrace: m.Stacktrace{ - &m.StacktraceFrame{Filename: "/a/b/c", Lineno: &lineno, Colno: &c1}, - }, + event1 := Event{Exception: &Exception{Message: &exMsg, + Stacktrace: m.Stacktrace{&m.StacktraceFrame{Filename: "/a/b/c", Lineno: &line, Colno: &col, AbsPath: &path}}, + }} + event2 := Event{Exception: &Exception{Message: &exMsg, + Stacktrace: m.Stacktrace{&m.StacktraceFrame{Filename: "/a/b/c", Lineno: &line, Colno: &col, AbsPath: &path}}, }} + + // transform without sourcemap store + str := "foo" tctx := &transform.Context{ - Config: transform.Config{SourcemapMapper: nil}, - Metadata: metadata.Metadata{ - Service: &metadata.Service{Name: &empty}, - }, + Config: transform.Config{SourcemapStore: nil}, + Metadata: metadata.Metadata{Service: &metadata.Service{Name: &str, Version: &str}}, } - trNoSmap := event.fields(tctx) - - event2 := Event{Exception: &Exception{ - Message: &exMsg, - Stacktrace: m.Stacktrace{ - &m.StacktraceFrame{Filename: "/a/b/c", Lineno: &lineno, Colno: &c1}, - }, - }} - mapper := sourcemap.SmapMapper{Accessor: &fakeAcc{}} + transformedNoSourcemap := event1.fields(tctx) - tctx.Config = transform.Config{SourcemapMapper: &mapper} - trWithSmap := event2.fields(tctx) + // transform with sourcemap store + store, err := sourcemap.NewStore(test.ESClientWithValidSourcemap(t), "apm-*sourcemap*", time.Minute) + require.NoError(t, err) + tctx.Config = transform.Config{SourcemapStore: store} + transformedWithSourcemap := event2.fields(tctx) - assert.Equal(t, 1, *event.Exception.Stacktrace[0].Lineno) + // ensure events have different line number and grouping keys + assert.Equal(t, 1, *event1.Exception.Stacktrace[0].Lineno) assert.Equal(t, 5, *event2.Exception.Stacktrace[0].Lineno) - - assert.NotEqual(t, trNoSmap["grouping_key"], trWithSmap["grouping_key"]) -} - -type fakeAcc struct{} - -func (ac *fakeAcc) Fetch(smapId sourcemap.Id) (*s.Consumer, error) { - file := "bundle.js.map" - current, _ := os.Getwd() - path := filepath.Join(current, "../../testdata/sourcemap/", file) - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return s.Parse("", fileBytes) + assert.NotEqual(t, transformedNoSourcemap["grouping_key"], transformedWithSourcemap["grouping_key"]) } -func (a *fakeAcc) Remove(smapId sourcemap.Id) {} diff --git a/model/sourcemap/payload.go b/model/sourcemap/payload.go index 02673a8155b..b76d02d73af 100644 --- a/model/sourcemap/payload.go +++ b/model/sourcemap/payload.go @@ -29,7 +29,6 @@ import ( logs "github.com/elastic/apm-server/log" "github.com/elastic/apm-server/model/sourcemap/generated/schema" - smap "github.com/elastic/apm-server/sourcemap" "github.com/elastic/apm-server/transform" "github.com/elastic/apm-server/utility" "github.com/elastic/apm-server/validation" @@ -66,14 +65,10 @@ func (pa *Sourcemap) Transform(tctx *transform.Context) []beat.Event { return nil } - if tctx.Config.SourcemapMapper == nil { + if tctx.Config.SourcemapStore == nil { logp.NewLogger(logs.Sourcemap).Error("Sourcemap Accessor is nil, cache cannot be invalidated.") } else { - tctx.Config.SourcemapMapper.NewSourcemapAdded(smap.Id{ - ServiceName: pa.ServiceName, - ServiceVersion: pa.ServiceVersion, - Path: pa.BundleFilepath, - }) + tctx.Config.SourcemapStore.Added(pa.ServiceName, pa.ServiceVersion, pa.BundleFilepath) } ev := beat.Event{ diff --git a/model/sourcemap/payload_test.go b/model/sourcemap/payload_test.go index aaeef1d5972..8904eda70c0 100644 --- a/model/sourcemap/payload_test.go +++ b/model/sourcemap/payload_test.go @@ -18,13 +18,20 @@ package sourcemap import ( + "net/http" "testing" "time" + "go.uber.org/zap/zapcore" + + "github.com/elastic/beats/libbeat/logp" + s "github.com/go-sourcemap/sourcemap" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + estest "github.com/elastic/apm-server/elasticsearch/test" + logs "github.com/elastic/apm-server/log" "github.com/elastic/apm-server/sourcemap" "github.com/elastic/apm-server/tests/loader" "github.com/elastic/apm-server/transform" @@ -95,43 +102,56 @@ func TestParseSourcemaps(t *testing.T) { } func TestInvalidateCache(t *testing.T) { + // load sourcemap from file and decode data, err := loader.LoadData("../testdata/sourcemap/payload.json") assert.NoError(t, err) - - smapId := sourcemap.Id{Path: "/tmp"} - smapMapper := smapMapperFake{ - c: map[string]*sourcemap.Mapping{ - "/tmp": &(sourcemap.Mapping{}), - }, - } - mapping, err := smapMapper.Apply(smapId, 0, 0) - require.NoError(t, err) - assert.NotNil(t, mapping) - - conf := transform.Config{SourcemapMapper: &smapMapper} - tctx := &transform.Context{Config: conf} - - sourcemap, err := DecodeSourcemap(data) - require.NoError(t, err) - sourcemap.Transform(tctx) - - sourcemap, err = DecodeSourcemap(data) - require.NoError(t, err) - sourcemap.Transform(tctx) - - mapping, err = smapMapper.Apply(smapId, 0, 0) + decoded, err := DecodeSourcemap(data) require.NoError(t, err) - assert.Nil(t, mapping) -} - -type smapMapperFake struct { - c map[string]*sourcemap.Mapping -} - -func (a *smapMapperFake) Apply(id sourcemap.Id, lineno, colno int) (*sourcemap.Mapping, error) { - return a.c[id.Path], nil -} - -func (sm *smapMapperFake) NewSourcemapAdded(id sourcemap.Id) { - sm.c = map[string]*sourcemap.Mapping{} + event := decoded.(*Sourcemap) + + t.Run("withSourcemapStore", func(t *testing.T) { + // collect logs + require.NoError(t, logp.DevelopmentSetup(logp.ToObserverOutput())) + + // create sourcemap store + client, err := estest.NewElasticsearchClient(estest.NewTransport(t, http.StatusOK, nil)) + require.NoError(t, err) + store, err := sourcemap.NewStore(client, "foo", time.Minute) + require.NoError(t, err) + + // transform with sourcemap store + event.Transform(&transform.Context{Config: transform.Config{SourcemapStore: store}}) + + logCollection := logp.ObserverLogs().TakeAll() + assert.Equal(t, 2, len(logCollection)) + + // first sourcemap was added + for i, entry := range logCollection { + assert.Equal(t, logs.Sourcemap, entry.LoggerName) + assert.Equal(t, zapcore.DebugLevel, entry.Level) + if i == 0 { + assert.Contains(t, entry.Message, "Added id service_1_js/bundle.js. Cache now has 1 entries.") + } else { + assert.Contains(t, entry.Message, "Removed id service_1_js/bundle.js. Cache now has 0 entries.") + } + } + + }) + + t.Run("noSourcemapStore", func(t *testing.T) { + // collect logs + require.NoError(t, logp.DevelopmentSetup(logp.ToObserverOutput())) + + // transform with sourcemap store + event.Transform(&transform.Context{Config: transform.Config{SourcemapStore: nil}}) + + logCollection := logp.ObserverLogs().TakeAll() + assert.Equal(t, 1, len(logCollection)) + for _, entry := range logCollection { + assert.Equal(t, logs.Sourcemap, entry.LoggerName) + assert.Equal(t, zapcore.ErrorLevel, entry.Level) + assert.Contains(t, entry.Message, "cache cannot be invalidated") + } + + }) } diff --git a/model/span/event_test.go b/model/span/event_test.go index 7af3a98cf99..9d1120005c0 100644 --- a/model/span/event_test.go +++ b/model/span/event_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/elastic/apm-server/sourcemap" "github.com/elastic/apm-server/utility" "github.com/stretchr/testify/assert" @@ -33,7 +34,6 @@ import ( m "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/metadata" - "github.com/elastic/apm-server/sourcemap" "github.com/elastic/apm-server/transform" ) @@ -104,7 +104,7 @@ func TestDecodeSpan(t *testing.T) { "name": name, "type": "db.postgresql.query.custom", "start": start, "duration": duration, "parent_id": parentId, "timestamp": timestampEpoch, "id": id, "trace_id": traceId, "stacktrace": []interface{}{"foo"}, }, - err: m.ErrInvalidStacktraceFrameType.Error(), + err: "invalid type for stacktrace frame", }, "minimal payload": { input: map[string]interface{}{ @@ -230,8 +230,8 @@ func TestDecodeSpan(t *testing.T) { func TestSpanTransform(t *testing.T) { path := "test/path" start := 0.65 - serviceName, env := "myService", "staging" - service := metadata.Service{Name: &serviceName, Environment: &env} + serviceName, serviceVersion, env := "myService", "1.2", "staging" + service := metadata.Service{Name: &serviceName, Version: &serviceVersion, Environment: &env} hexId, parentId, traceId := "0147258369012345", "abcdef0123456789", "01234567890123456789abcdefa" subtype := "myspansubtype" action := "myspanquery" @@ -319,7 +319,7 @@ func TestSpanTransform(t *testing.T) { } tctx := &transform.Context{ - Config: transform.Config{SourcemapMapper: &sourcemap.SmapMapper{}}, + Config: transform.Config{SourcemapStore: &sourcemap.Store{}}, Metadata: metadata.Metadata{Service: &service, Labels: metadataLabels}, RequestTime: timestamp, } diff --git a/model/stacktrace.go b/model/stacktrace.go index 763cf5ff306..d2f335b2c3f 100644 --- a/model/stacktrace.go +++ b/model/stacktrace.go @@ -20,15 +20,18 @@ package model import ( "errors" - logs "github.com/elastic/apm-server/log" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + logs "github.com/elastic/apm-server/log" + "github.com/elastic/apm-server/transform" ) -var ErrInvalidStacktraceType = errors.New("invalid type for stacktrace") +var ( + errInvalidStacktraceType = errors.New("invalid type for stacktrace") + msgServiceInvalidForSourcemapping = "Cannot apply sourcemap without a service name or service version" +) type Stacktrace []*StacktraceFrame @@ -38,7 +41,7 @@ func DecodeStacktrace(input interface{}, err error) (*Stacktrace, error) { } raw, ok := input.([]interface{}) if !ok { - return nil, ErrInvalidStacktraceType + return nil, errInvalidStacktraceType } st := make(Stacktrace, len(raw)) for idx, fr := range raw { @@ -67,33 +70,44 @@ func (st *Stacktrace) Transform(tctx *transform.Context) []common.MapStr { // - lineno // - abs_path is set to the cleaned abs_path // - sourcmeap.updated is set to true + + if tctx.Config.SourcemapStore == nil { + return st.transform(tctx, noSourcemapping) + } + if tctx.Metadata.Service == nil || tctx.Metadata.Service.Name == nil || tctx.Metadata.Service.Version == nil { + logp.NewLogger(logs.Stacktrace).Warn(msgServiceInvalidForSourcemapping) + return st.transform(tctx, noSourcemapping) + } + + var errMsg string + var sourcemapErrorSet = map[string]interface{}{} + logger := logp.NewLogger(logs.Stacktrace) + fct := "" + return st.transform(tctx, func(frame *StacktraceFrame) { + fct, errMsg = frame.applySourcemap(tctx.Config.SourcemapStore, tctx.Metadata.Service, fct) + if errMsg != "" { + if _, ok := sourcemapErrorSet[errMsg]; !ok { + logger.Warn(errMsg) + sourcemapErrorSet[errMsg] = nil + } + } + }) +} + +func (st *Stacktrace) transform(ctx *transform.Context, apply func(*StacktraceFrame)) []common.MapStr { frameCount := len(*st) if frameCount == 0 { return nil } + var fr *StacktraceFrame frames := make([]common.MapStr, frameCount) - - fct := "" - var sourcemapErrorSet = make(map[string]struct{}) - for idx := frameCount - 1; idx >= 0; idx-- { fr = (*st)[idx] - if tctx.Config.SourcemapMapper != nil && tctx.Metadata.Service != nil { - var errMsg string - fct, errMsg = fr.applySourcemap(tctx.Config.SourcemapMapper, tctx.Metadata.Service, fct) - sourcemapErrorSet[errMsg] = struct{}{} - } - frames[idx] = fr.Transform(tctx) + apply(fr) + frames[idx] = fr.Transform(ctx) } - if len(sourcemapErrorSet) > 0 { - logger := logp.NewLogger(logs.Stacktrace) - for errMsg := range sourcemapErrorSet { - if errMsg != "" { - logger.Warn(errMsg) - } - } - } - return frames } + +func noSourcemapping(_ *StacktraceFrame) {} diff --git a/model/stacktrace_frame.go b/model/stacktrace_frame.go index 637abc34b87..dbdc76e5f05 100644 --- a/model/stacktrace_frame.go +++ b/model/stacktrace_frame.go @@ -19,13 +19,15 @@ package model import ( "errors" + "fmt" "regexp" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/apm-server/model/metadata" "github.com/elastic/apm-server/sourcemap" "github.com/elastic/apm-server/transform" "github.com/elastic/apm-server/utility" - "github.com/elastic/beats/libbeat/common" ) type StacktraceFrame struct { @@ -63,7 +65,11 @@ type Original struct { sourcemapCopied bool } -var ErrInvalidStacktraceFrameType = errors.New("invalid type for stacktrace frame") +type sourcemapError interface { + NoSourcemapFound() bool +} + +var errInvalidStacktraceFrameType = errors.New("invalid type for stacktrace frame") func DecodeStacktraceFrame(input interface{}, err error) (*StacktraceFrame, error) { if input == nil || err != nil { @@ -71,7 +77,7 @@ func DecodeStacktraceFrame(input interface{}, err error) (*StacktraceFrame, erro } raw, ok := input.(map[string]interface{}) if !ok { - return nil, ErrInvalidStacktraceFrameType + return nil, errInvalidStacktraceFrameType } decoder := utility.ManualDecoder{} frame := StacktraceFrame{ @@ -156,50 +162,66 @@ func (s *StacktraceFrame) setLibraryFrame(pattern *regexp.Regexp) { s.LibraryFrame = &libraryFrame } -func (s *StacktraceFrame) applySourcemap(mapper sourcemap.Mapper, service *metadata.Service, prevFunction string) (string, string) { - s.setOriginalSourcemapData() +func (s *StacktraceFrame) applySourcemap(store *sourcemap.Store, service *metadata.Service, prevFunction string) (function string, errMsg string) { + function = prevFunction - if s.Original.Colno == nil { - errMsg := "Colno mandatory for sourcemapping." - s.updateError(errMsg) - return prevFunction, errMsg - } - if s.Original.Lineno == nil { - errMsg := "Lineno mandatory for sourcemapping." + var valid bool + if valid, errMsg = validForSourcemapping(s, service); !valid { s.updateError(errMsg) - return prevFunction, errMsg + return } - sourcemapId, errMsg := s.buildSourcemapId(service) - if errMsg != "" { - return prevFunction, errMsg - } - mapping, err := mapper.Apply(sourcemapId, *s.Original.Lineno, *s.Original.Colno) + s.setOriginalSourcemapData() + + path := utility.CleanUrlPath(*s.Original.AbsPath) + mapper, err := store.Fetch(*service.Name, *service.Version, path) if err != nil { - e, isSourcemapError := err.(sourcemap.Error) - if !isSourcemapError || e.Kind == sourcemap.MapError || e.Kind == sourcemap.KeyError { + if e, ok := err.(sourcemapError); !ok || e.NoSourcemapFound() { s.updateError(err.Error()) } - return prevFunction, err.Error() + errMsg = err.Error() + return } - if mapping.Filename != "" { - s.Filename = mapping.Filename + file, fct, line, col, ctxLine, preCtx, postCtx, ok := mapper.Apply(*s.Original.Lineno, *s.Original.Colno) + if !ok { + errMsg = fmt.Sprintf("No Sourcemap found for Lineno %v, Colno %v", *s.Original.Lineno, *s.Original.Colno) + s.updateError(errMsg) + return + } + + if file != "" { + s.Filename = file } - s.Colno = &mapping.Colno - s.Lineno = &mapping.Lineno - s.AbsPath = &mapping.Path + s.Colno = &col + s.Lineno = &line + s.AbsPath = &path s.updateSmap(true) s.Function = &prevFunction - s.ContextLine = &mapping.ContextLine - s.PreContext = mapping.PreContext - s.PostContext = mapping.PostContext + s.ContextLine = &ctxLine + s.PreContext = preCtx + s.PostContext = postCtx + + if fct != "" { + function = fct + return + } + function = "" + return +} - if mapping.Function != "" { - return mapping.Function, "" +func validForSourcemapping(s *StacktraceFrame, service *metadata.Service) (bool, string) { + if s.Colno == nil { + return false, "Colno mandatory for sourcemapping." } - return "", "" + if s.Lineno == nil { + return false, "Lineno mandatory for sourcemapping." + } + if s.AbsPath == nil { + return false, "AbsPath mandatory for sourcemapping." + } + return true, "" } func (s *StacktraceFrame) setOriginalSourcemapData() { @@ -215,20 +237,6 @@ func (s *StacktraceFrame) setOriginalSourcemapData() { s.Original.sourcemapCopied = true } -func (s *StacktraceFrame) buildSourcemapId(service *metadata.Service) (sourcemap.Id, string) { - if service.Name == nil { - return sourcemap.Id{}, "Cannot apply sourcemap without a service name." - } - id := sourcemap.Id{ServiceName: *service.Name} - if service.Version != nil { - id.ServiceVersion = *service.Version - } - if s.AbsPath != nil { - id.Path = utility.CleanUrlPath(*s.AbsPath) - } - return id, "" -} - func (s *StacktraceFrame) updateError(errMsg string) { s.Sourcemap.Error = &errMsg s.updateSmap(false) diff --git a/model/stacktrace_frame_test.go b/model/stacktrace_frame_test.go index 8c2b0591388..859fdc28fc4 100644 --- a/model/stacktrace_frame_test.go +++ b/model/stacktrace_frame_test.go @@ -22,13 +22,16 @@ import ( "fmt" "regexp" "testing" + "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" + "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/apm-server/model/metadata" "github.com/elastic/apm-server/sourcemap" + "github.com/elastic/apm-server/sourcemap/test" "github.com/elastic/apm-server/transform" "github.com/elastic/apm-server/utility" @@ -48,7 +51,7 @@ func TestStacktraceFrameDecode(t *testing.T) { }{ {input: nil, err: nil, s: nil}, {input: nil, inpErr: errors.New("a"), err: errors.New("a"), s: nil}, - {input: "", err: ErrInvalidStacktraceFrameType, s: nil}, + {input: "", err: errInvalidStacktraceFrameType, s: nil}, { input: map[string]interface{}{}, err: utility.ErrFetch, @@ -159,177 +162,139 @@ func TestStacktraceFrameTransform(t *testing.T) { } } -func TestApplySourcemap(t *testing.T) { - colno := 1 - l0, l5, l6, l7, l8, l9 := 0, 5, 6, 7, 8, 9 - fct := "original function" - absPath := "original path" - tests := []struct { - fr StacktraceFrame - lineno, colno int - filename, function, absPath string - smapUpdated bool - smapError string - fct string - outFct string - msg string - }{ - { - fr: StacktraceFrame{Lineno: &l0, Function: &fct, AbsPath: &absPath}, - lineno: l0, - filename: "", - function: "original function", - absPath: "original path", - smapUpdated: false, - smapError: "Colno mandatory for sourcemapping.", - fct: "", - outFct: "", - msg: "No colno", - }, - { - fr: StacktraceFrame{ - Colno: &colno, - Lineno: &l9, - Filename: "filename", - Function: &fct, - AbsPath: &absPath, - }, - colno: 1, - lineno: l9, - filename: "filename", - function: "original function", - absPath: "original path", - smapUpdated: false, - smapError: "Some untyped error", - fct: "", - outFct: "", - msg: "Some error occured in mapper.", - }, - { - fr: StacktraceFrame{Colno: &colno, Lineno: &l8, Function: &fct, AbsPath: &absPath}, - colno: 1, - lineno: l8, - filename: "", - function: "original function", - absPath: "original path", - fct: "", - outFct: "", - msg: "Some access error occured in mapper.", - }, - { - fr: StacktraceFrame{Colno: &colno, Lineno: &l7, Function: &fct, AbsPath: &absPath}, - colno: 1, - lineno: l7, - filename: "", - function: "original function", - absPath: "original path", - smapUpdated: false, - smapError: "Some mapping error", - fct: "", - outFct: "", - msg: "Some mapping error occured in mapper.", - }, - { - fr: StacktraceFrame{Colno: &colno, Lineno: &l6, Function: &fct, AbsPath: &absPath}, - colno: 1, - lineno: l6, - filename: "", - function: "original function", - absPath: "original path", - smapUpdated: false, - smapError: "Some key error", - fct: "", - outFct: "", - msg: "Some key error occured in mapper.", - }, - { - fr: StacktraceFrame{ - Colno: &colno, - Lineno: &l5, - Filename: "original filename", - Function: &fct, - AbsPath: &absPath, - }, - colno: 100, - lineno: 500, - filename: "original filename", - function: "other function", - absPath: "changed path", - smapUpdated: true, - - fct: "other function", - outFct: "", - msg: "Sourcemap with empty filename and function mapping applied.", - }, - { - fr: StacktraceFrame{ - Colno: &colno, - Lineno: &l0, - Filename: "original filename", - Function: &fct, - AbsPath: &absPath, - }, - colno: 100, - lineno: 400, - filename: "changed filename", - function: "prev function", - absPath: "changed path", - smapUpdated: true, +func TestSourcemap_Apply(t *testing.T) { - fct: "prev function", - outFct: "changed function", - msg: "Full sourcemap mapping applied.", - }, + name, version, col, line, path := "myservice", "2.1.4", 10, 15, "/../a/path" + validService := func() *metadata.Service { + return &metadata.Service{Name: &name, Version: &version} + } + validFrame := func() *StacktraceFrame { + return &StacktraceFrame{Colno: &col, Lineno: &line, AbsPath: &path} } - ver, name := "1", "foo" - service := &metadata.Service{Name: &name, Version: &ver} - for idx, test := range tests { - // check that original data are preserved, - // even when Transform function is applied twice. - if test.smapUpdated { - origAbsPath := *test.fr.AbsPath - origFilename := test.fr.Filename - origLineno := test.fr.Lineno - origColno := test.fr.Colno - origFunction := test.fr.Function - - (&test.fr).applySourcemap(&FakeMapper{}, service, test.fct) - (&test.fr).applySourcemap(&FakeMapper{}, service, test.fct) + t.Run("frame", func(t *testing.T) { + for name, tc := range map[string]struct { + frame *StacktraceFrame - assert.Equal(t, origAbsPath, *test.fr.Original.AbsPath) - assert.Equal(t, origFilename, test.fr.Original.Filename) - assert.Equal(t, origLineno, test.fr.Original.Lineno) - if origColno == nil { - assert.Nil(t, test.fr.Original.Colno) - } else { - assert.Equal(t, *origColno, *test.fr.Original.Colno) - } - if origFunction == nil { - assert.Nil(t, test.fr.Original.Function) - } else { - assert.Equal(t, *origFunction, *test.fr.Original.Function) - } + expectedErrorMsg string + }{ + "noColumn": { + frame: &StacktraceFrame{}, + expectedErrorMsg: "Colno mandatory"}, + "noLine": { + frame: &StacktraceFrame{Colno: &col}, + expectedErrorMsg: "Lineno mandatory"}, + "noPath": { + frame: &StacktraceFrame{Colno: &col, Lineno: &line}, + expectedErrorMsg: "AbsPath mandatory", + }, + } { + t.Run(name, func(t *testing.T) { + function, msg := tc.frame.applySourcemap(&sourcemap.Store{}, validService(), "foo") + assert.Equal(t, "foo", function) + assert.Contains(t, msg, tc.expectedErrorMsg) + assert.Equal(t, new(bool), tc.frame.Sourcemap.Updated) + require.NotNil(t, tc.frame.Sourcemap.Error) + assert.Contains(t, *tc.frame.Sourcemap.Error, msg) + assert.Zero(t, tc.frame.Original) + }) } + }) - // check that source mapping is applied as expected - output, errMsg := (&test.fr).applySourcemap(&FakeMapper{}, service, test.fct) - assert.Equal(t, test.smapError, errMsg) - assert.Equal(t, test.outFct, output) - assert.Equal(t, test.lineno, *test.fr.Lineno, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) - assert.Equal(t, test.filename, test.fr.Filename, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) - assert.Equal(t, test.function, *test.fr.Function, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) - assert.Equal(t, test.absPath, *test.fr.AbsPath, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) - if test.colno != 0 { - assert.Equal(t, test.colno, *test.fr.Colno, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) + t.Run("errorPerFrame", func(t *testing.T) { + for name, tc := range map[string]struct { + store *sourcemap.Store + expectedErrorMsg string + }{ + "noSourcemap": {store: testSourcemapStore(t, test.ESClientWithSourcemapNotFound(t)), + expectedErrorMsg: "No Sourcemap available"}, + "noMapping": {store: testSourcemapStore(t, test.ESClientWithValidSourcemap(t)), + expectedErrorMsg: "No Sourcemap found for Lineno", + }, + } { + t.Run(name, func(t *testing.T) { + frame := validFrame() + function, msg := frame.applySourcemap(tc.store, validService(), "xyz") + assert.Equal(t, "xyz", function) + require.Contains(t, msg, tc.expectedErrorMsg) + assert.NotZero(t, frame.Sourcemap.Error) + assert.Equal(t, new(bool), frame.Sourcemap.Updated) + }) } - if test.smapError != "" || test.smapUpdated { - assert.Equal(t, test.smapUpdated, *test.fr.Sourcemap.Updated, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) + }) + + t.Run("mappingError", func(t *testing.T) { + for name, tc := range map[string]struct { + store *sourcemap.Store + expectedErrorMsg string + }{ + "ESUnavailable": {store: testSourcemapStore(t, test.ESClientUnavailable(t)), + expectedErrorMsg: "Internal server error"}, + "invalidSourcemap": {store: testSourcemapStore(t, test.ESClientWithInvalidSourcemap(t)), + expectedErrorMsg: "Could not parse Sourcemap."}, + "unsupportedSourcemap": {store: testSourcemapStore(t, test.ESClientWithUnsupportedSourcemap(t)), + expectedErrorMsg: "only 3rd version is supported"}, + } { + t.Run(name, func(t *testing.T) { + frame := validFrame() + function, msg := frame.applySourcemap(tc.store, validService(), "xyz") + assert.Equal(t, "xyz", function) + require.Contains(t, msg, tc.expectedErrorMsg) + assert.NotZero(t, msg) + assert.Zero(t, frame.Sourcemap) + }) } - if test.smapError != "" { - assert.Equal(t, test.smapError, *test.fr.Sourcemap.Error, fmt.Sprintf("Failed at idx %v; %s", idx, test.msg)) + }) + + t.Run("mapping", func(t *testing.T) { + + for name, tc := range map[string]struct { + origCol, origLine int + origPath string + + function, file, path, ctxLine string + preCtx, postCtx []string + col, line int + }{ + "withFunction": {origCol: 67, origLine: 1, origPath: "/../a/path", + function: "exports", file: "", path: "/a/path", ctxLine: " \t\t\texports: {},", col: 0, line: 13, + preCtx: []string{" \t\tif(installedModules[moduleId])", " \t\t\treturn installedModules[moduleId].exports;", "", " \t\t// Create a new module (and put it into the cache)", " \t\tvar module = installedModules[moduleId] = {"}, + postCtx: []string{" \t\t\tid: moduleId,", " \t\t\tloaded: false", " \t\t};", "", " \t\t// Execute the module function"}}, + "withFilename": {origCol: 7, origLine: 1, origPath: "/../a/path", + function: "", file: "webpack:///bundle.js", path: "/a/path", + ctxLine: "/******/ (function(modules) { // webpackBootstrap", preCtx: []string{}, + postCtx: []string{"/******/ \t// The module cache", "/******/ \tvar installedModules = {};", "/******/", "/******/ \t// The require function", "/******/ \tfunction __webpack_require__(moduleId) {"}, + col: 9, line: 1}, + "withoutFilename": {origCol: 23, origLine: 1, origPath: "/../a/path", + function: "__webpack_require__", file: "", path: "/a/path", ctxLine: " \tfunction __webpack_require__(moduleId) {", + preCtx: []string{" \t// The module cache", " \tvar installedModules = {};", "", " \t// The require function"}, + postCtx: []string{"", " \t\t// Check if module is in cache", " \t\tif(installedModules[moduleId])", " \t\t\treturn installedModules[moduleId].exports;", ""}, + col: 0, line: 5}, + } { + t.Run(name, func(t *testing.T) { + frame := &StacktraceFrame{Colno: &tc.origCol, Lineno: &tc.origLine, AbsPath: &tc.origPath} + + prevFunction := "xyz" + function, msg := frame.applySourcemap(testSourcemapStore(t, test.ESClientWithValidSourcemap(t)), validService(), prevFunction) + require.Empty(t, msg) + assert.Zero(t, frame.Sourcemap.Error) + updated := true + assert.Equal(t, &updated, frame.Sourcemap.Updated) + + assert.Equal(t, tc.function, function) + assert.Equal(t, tc.file, frame.Filename) + assert.Equal(t, prevFunction, *frame.Function) + assert.Equal(t, tc.col, *frame.Colno) + assert.Equal(t, tc.line, *frame.Lineno) + assert.Equal(t, tc.path, *frame.AbsPath) + assert.Equal(t, tc.ctxLine, *frame.ContextLine) + assert.Equal(t, tc.preCtx, frame.PreContext) + assert.Equal(t, tc.postCtx, frame.PostContext) + + assert.NotZero(t, frame.Original) + }) } - } + }) } func TestIsLibraryFrame(t *testing.T) { @@ -514,63 +479,8 @@ func TestLibraryFrame(t *testing.T) { } } -func TestBuildSourcemap(t *testing.T) { - version := "1.0" - path := "././a/b/../c" - serviceName, empty := "foo", "" - tests := []struct { - service metadata.Service - fr StacktraceFrame - out string - err string - }{ - {service: metadata.Service{}, fr: StacktraceFrame{}, out: "", err: "Cannot apply sourcemap without a service name."}, - {service: metadata.Service{Version: &version, Name: &empty}, fr: StacktraceFrame{}, out: "1.0"}, - {service: metadata.Service{Name: &serviceName}, fr: StacktraceFrame{}, out: "foo"}, - {service: metadata.Service{Name: &empty}, fr: StacktraceFrame{AbsPath: &path}, out: "a/c"}, - { - service: metadata.Service{Name: &serviceName, Version: &version}, - fr: StacktraceFrame{AbsPath: &path}, - out: "foo_1.0_a/c", - }, - } - for _, test := range tests { - id, errStr := test.fr.buildSourcemapId(&test.service) - require.Equal(t, test.err, errStr) - assert.Equal(t, test.out, (&id).Key()) - } -} - -// Fake implemenations for Mapper - -type FakeMapper struct{} - -func (m *FakeMapper) Apply(smapId sourcemap.Id, lineno, colno int) (*sourcemap.Mapping, error) { - switch lineno { - case 9: - return nil, errors.New("Some untyped error") - case 8: - return nil, sourcemap.Error{Kind: sourcemap.AccessError} - case 7: - return nil, sourcemap.Error{Kind: sourcemap.MapError, Msg: "Some mapping error"} - case 6: - return nil, sourcemap.Error{Kind: sourcemap.KeyError, Msg: "Some key error"} - case 5: - return &sourcemap.Mapping{ - Filename: "", - Function: "", - Colno: 100, - Lineno: 500, - Path: "changed path", - }, nil - default: - return &sourcemap.Mapping{ - Filename: "changed filename", - Function: "changed function", - Colno: 100, - Lineno: 400, - Path: "changed path", - }, nil - } +func testSourcemapStore(t *testing.T, client *elasticsearch.Client) *sourcemap.Store { + store, err := sourcemap.NewStore(client, "apm-*sourcemap*", time.Minute) + require.NoError(t, err) + return store } -func (m *FakeMapper) NewSourcemapAdded(smapId sourcemap.Id) {} diff --git a/model/stacktrace_test.go b/model/stacktrace_test.go index 9695c274a7d..8f7aae38c57 100644 --- a/model/stacktrace_test.go +++ b/model/stacktrace_test.go @@ -24,9 +24,11 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/apm-server/model/metadata" + "github.com/elastic/apm-server/sourcemap/test" "github.com/elastic/apm-server/transform" - "github.com/elastic/beats/libbeat/common" ) func TestStacktraceDecode(t *testing.T) { @@ -41,7 +43,7 @@ func TestStacktraceDecode(t *testing.T) { {input: "", err: errors.New("invalid type for stacktrace"), s: nil}, { input: []interface{}{"foo"}, - err: ErrInvalidStacktraceFrameType, + err: errInvalidStacktraceFrameType, s: &Stacktrace{nil}, }, { @@ -155,23 +157,21 @@ func TestStacktraceTransform(t *testing.T) { } func TestStacktraceTransformWithSourcemapping(t *testing.T) { - colno := 1 - l4, l5, l6, l8 := 4, 5, 6, 8 - fct := "original function" - absPath, serviceName := "original path", "service1" - service := metadata.Service{Name: &serviceName} + int1, int6, int7, int67 := 1, 6, 7, 67 + fct1, fct2 := "function foo", "function bar" + absPath, serviceName, serviceVersion := "/../a/c", "service1", "2.4.1" + service := metadata.Service{Name: &serviceName, Version: &serviceVersion} - tests := []struct { + for name, tc := range map[string]struct { Stacktrace Stacktrace Output []common.MapStr Msg string }{ - { + "emptyStacktrace": { Stacktrace: Stacktrace{}, Output: nil, - Msg: "Empty Stacktrace", }, - { + "emptyFrame": { Stacktrace: Stacktrace{&StacktraceFrame{}}, Output: []common.MapStr{ {"filename": "", @@ -182,10 +182,9 @@ func TestStacktraceTransformWithSourcemapping(t *testing.T) { }, }, }, - Msg: "Stacktrace with empty Frame", }, - { - Stacktrace: Stacktrace{&StacktraceFrame{Colno: &colno}}, + "noLineno": { + Stacktrace: Stacktrace{&StacktraceFrame{Colno: &int1}}, Output: []common.MapStr{ {"filename": "", "line": common.MapStr{"column": 1}, @@ -196,97 +195,115 @@ func TestStacktraceTransformWithSourcemapping(t *testing.T) { }, }, }, - Msg: "Stacktrace with no lineno", }, - { + "sourcemapApplied": { Stacktrace: Stacktrace{ &StacktraceFrame{ - Colno: &colno, - Lineno: &l4, + Colno: &int7, + Lineno: &int1, Filename: "original filename", - Function: &fct, + Function: &fct1, AbsPath: &absPath, }, - &StacktraceFrame{Colno: &colno, Lineno: &l6, Function: &fct, AbsPath: &absPath}, - &StacktraceFrame{Colno: &colno, Lineno: &l8, Function: &fct, AbsPath: &absPath}, &StacktraceFrame{ - Colno: &colno, - Lineno: &l5, - Filename: "original filename", - Function: &fct, + Colno: &int67, + Lineno: &int1, + Filename: "myfilename", + Function: &fct2, AbsPath: &absPath, }, &StacktraceFrame{ - Colno: &colno, - Lineno: &l4, - Filename: "/webpack", + Colno: &int7, + Lineno: &int1, + Filename: "myfilename", + Function: &fct2, AbsPath: &absPath, }, + &StacktraceFrame{Colno: &int1, Lineno: &int6, Function: &fct2, AbsPath: &absPath}, }, Output: []common.MapStr{ { - "abs_path": "changed path", "filename": "changed filename", "function": "", - "line": common.MapStr{"column": 100, "number": 400, "context": ""}, + "abs_path": "/a/c", + "filename": "webpack:///bundle.js", + "function": "exports", + "context": common.MapStr{ + "post": []string{"/******/ \t// The module cache", "/******/ \tvar installedModules = {};", "/******/", "/******/ \t// The require function", "/******/ \tfunction __webpack_require__(moduleId) {"}}, + "line": common.MapStr{ + "column": 9, + "number": 1, + "context": "/******/ (function(modules) { // webpackBootstrap"}, "exclude_from_grouping": false, "sourcemap": common.MapStr{"updated": true}, "original": common.MapStr{ - "abs_path": "original path", - "colno": 1, + "abs_path": "/../a/c", + "colno": 7, "filename": "original filename", - "function": "original function", - "lineno": 4, + "function": "function foo", + "lineno": 1, }, }, { - "abs_path": "original path", "filename": "", "function": "original function", - "line": common.MapStr{"column": 1, "number": 6}, - "exclude_from_grouping": false, - "sourcemap": common.MapStr{"updated": false, "error": "Some key error"}, - }, - { - "abs_path": "original path", "filename": "", "function": "original function", - "line": common.MapStr{"column": 1, "number": 8}, - "exclude_from_grouping": false, - }, - { - "abs_path": "changed path", "filename": "original filename", "function": "changed function", - "line": common.MapStr{"column": 100, "number": 500, "context": ""}, + "abs_path": "/a/c", + "filename": "myfilename", + "function": "", //prev function + "context": common.MapStr{ + "post": []string{" \t\t\tid: moduleId,", " \t\t\tloaded: false", " \t\t};", "", " \t\t// Execute the module function"}, + "pre": []string{" \t\tif(installedModules[moduleId])", " \t\t\treturn installedModules[moduleId].exports;", "", " \t\t// Create a new module (and put it into the cache)", " \t\tvar module = installedModules[moduleId] = {"}}, + "line": common.MapStr{ + "column": 0, + "number": 13, + "context": " \t\t\texports: {},"}, "exclude_from_grouping": false, "sourcemap": common.MapStr{"updated": true}, "original": common.MapStr{ - "abs_path": "original path", - "colno": 1, - "filename": "original filename", - "function": "original function", - "lineno": 5, + "abs_path": "/../a/c", + "colno": 67, + "filename": "myfilename", + "function": "function bar", + "lineno": 1, }, }, { - "abs_path": "changed path", "filename": "changed filename", "function": "", - "line": common.MapStr{"column": 100, "number": 400, "context": ""}, + "abs_path": "/a/c", + "filename": "webpack:///bundle.js", + "function": "", //prev function + "context": common.MapStr{ + "post": []string{"/******/ \t// The module cache", "/******/ \tvar installedModules = {};", "/******/", "/******/ \t// The require function", "/******/ \tfunction __webpack_require__(moduleId) {"}}, + "line": common.MapStr{ + "column": 9, + "number": 1, + "context": "/******/ (function(modules) { // webpackBootstrap"}, "exclude_from_grouping": false, "sourcemap": common.MapStr{"updated": true}, "original": common.MapStr{ - "abs_path": "original path", - "colno": 1, - "filename": "/webpack", - "lineno": 4, + "abs_path": "/../a/c", + "colno": 7, + "filename": "myfilename", + "function": "function bar", + "lineno": 1, }, }, + { + "abs_path": "/../a/c", + "filename": "", + "function": fct2, + "line": common.MapStr{"column": 1, "number": 6}, + "exclude_from_grouping": false, + "sourcemap": common.MapStr{"updated": false, "error": "No Sourcemap found for Lineno 6, Colno 1"}, + }, }, - Msg: "Stacktrace with sourcemapping", }, - } - - for idx, test := range tests { - tctx := &transform.Context{ - Config: transform.Config{SourcemapMapper: &FakeMapper{}}, - Metadata: metadata.Metadata{Service: &service}, - } + } { + t.Run(name, func(t *testing.T) { + tctx := &transform.Context{ + Config: transform.Config{SourcemapStore: testSourcemapStore(t, test.ESClientWithValidSourcemap(t))}, + Metadata: metadata.Metadata{Service: &service}, + } - // run `Stacktrace.Transform` twice to ensure method is idempotent - test.Stacktrace.Transform(tctx) - output := test.Stacktrace.Transform(tctx) - assert.Equal(t, test.Output, output, fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) + // run `Stacktrace.Transform` twice to ensure method is idempotent + tc.Stacktrace.Transform(tctx) + output := tc.Stacktrace.Transform(tctx) + assert.Equal(t, tc.Output, output) + }) } } diff --git a/sourcemap/accessor.go b/sourcemap/accessor.go deleted file mode 100644 index b956e29b80f..00000000000 --- a/sourcemap/accessor.go +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "fmt" - - "github.com/go-sourcemap/sourcemap" -) - -type Accessor interface { - Fetch(id Id) (*sourcemap.Consumer, error) - Remove(id Id) -} - -type SmapAccessor struct { - es elasticsearch - cache *cache -} - -func NewSmapAccessor(config Config) (*SmapAccessor, error) { - es, err := NewElasticsearch(config.ElasticsearchConfig, config.Index) - if err != nil { - return nil, err - } - cache, err := newCache(config.CacheExpiration) - if err != nil { - return nil, err - } - return &SmapAccessor{ - es: es, - cache: cache, - }, nil -} - -func (s *SmapAccessor) Fetch(id Id) (*sourcemap.Consumer, error) { - if !id.Valid() { - return nil, Error{ - Msg: fmt.Sprintf("Sourcemap Key Error for %v", id.String()), - Kind: KeyError, - } - } - consumer, found := s.cache.fetch(id) - if consumer != nil { - // avoid fetching ES again when key was already queried - // but no Sourcemap was found for it. - return consumer, nil - } - if found { - return nil, errSmapNotAvailable(id) - } - consumer, err := s.es.fetch(id) - if err != nil { - return nil, err - } - s.cache.add(id, consumer) - if consumer == nil { - return nil, errSmapNotAvailable(id) - } - return consumer, nil -} - -func (s *SmapAccessor) Remove(id Id) { - s.cache.remove(id) -} - -func errSmapNotAvailable(id Id) Error { - return Error{ - Msg: fmt.Sprintf("No Sourcemap available for %v.", id.String()), - Kind: MapError, - } -} diff --git a/sourcemap/accessor_test.go b/sourcemap/accessor_test.go deleted file mode 100644 index ad3c69abb83..00000000000 --- a/sourcemap/accessor_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" - - "github.com/go-sourcemap/sourcemap" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewSmapAccessor(t *testing.T) { - // Init problem with Elasticsearch - smapAcc, err := NewSmapAccessor(Config{}) - assert.Nil(t, smapAcc) - require.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, InitError) - assert.Contains(t, err.Error(), "ES Client cannot be initialized") - - // Init problem with cache - config := Config{ElasticsearchConfig: getFakeESConfig(nil), CacheExpiration: -1} - smapAcc, err = NewSmapAccessor(config) - assert.Nil(t, smapAcc) - require.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, InitError) - assert.Contains(t, err.Error(), "Cache cannot be initialized") - - // Init ok - minimalConfig := Config{ElasticsearchConfig: getFakeESConfig(nil)} - smapAcc, err = NewSmapAccessor(minimalConfig) - require.NoError(t, err) - assert.NotNil(t, smapAcc.es) - assert.NotNil(t, smapAcc.cache) - - smapAcc, err = NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - assert.NotNil(t, smapAcc.es) - assert.NotNil(t, smapAcc.cache) -} - -func TestFetchKeyError(t *testing.T) { - id := Id{Path: "/tmp"} - smapAcc, err := NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - c, err := smapAcc.Fetch(id) - require.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, KeyError) - assert.Nil(t, c) -} - -func TestFetchAccessError(t *testing.T) { - smapAcc, err := NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - c, err := smapAcc.Fetch(getFakeId()) - require.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, AccessError) - assert.Nil(t, c) -} - -func TestFetchAndCaching(t *testing.T) { - id := getFakeId() - smapAcc, err := NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - smapAcc.es = &FakeESAccessor{} - - // at the beginning cache is empty - cached, found := smapAcc.cache.fetch(id) - assert.Nil(t, cached) - assert.False(t, found) - - // smap fetched from ES - c, err := smapAcc.Fetch(id) - require.NoError(t, err) - assert.NotNil(t, c) - - // ensure that smap is cached - cached, found = smapAcc.cache.fetch(id) - assert.NotNil(t, cached) - assert.True(t, found) - assert.Equal(t, c, cached) -} - -func TestFetchCacheEmptyValueWhenSmapNotFound(t *testing.T) { - smapAcc, err := NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - smapAcc.es = &FakeESAccessor{} - id := Id{Path: "/tmp/123", ServiceName: "foo", ServiceVersion: "bar"} - - // at the beginning cache is empty - cached, found := smapAcc.cache.fetch(id) - assert.Nil(t, cached) - assert.False(t, found) - - // no smap found for given id when fetching from ES - c, err := smapAcc.Fetch(id) - assert.Nil(t, c) - require.Error(t, err) - assert.Contains(t, err.Error(), "No Sourcemap available") - assert.Equal(t, (err.(Error)).Kind, MapError) - - // check that cache value is now set with null value - cached, found = smapAcc.cache.fetch(id) - assert.Nil(t, cached) - assert.True(t, found) - - // check that error is returned also when empty value is fetched from cache - c, err = smapAcc.Fetch(id) - require.Error(t, err) - assert.Nil(t, c) - assert.Contains(t, err.Error(), "No Sourcemap available") - assert.Equal(t, (err.(Error)).Kind, MapError) -} - -func TestSourcemapRemovedFromCache(t *testing.T) { - id := getFakeId() - smap := getFakeSmap() - - smapAcc, err := NewSmapAccessor(getFakeConfig()) - require.NoError(t, err) - smapAcc.cache.add(id, smap) - cached, found := smapAcc.cache.fetch(id) - assert.True(t, found) - assert.Equal(t, smap, cached) - - smapAcc.Remove(id) - cached, found = smapAcc.cache.fetch(id) - assert.Nil(t, cached) - assert.False(t, found) -} - -type FakeESAccessor struct{} - -func (es *FakeESAccessor) fetch(id Id) (*sourcemap.Consumer, error) { - if id.Path == "/tmp" { - return getFakeSmap(), nil - } else { - return nil, nil - } -} - -func getFakeId() Id { - return Id{Path: "/tmp", ServiceName: "foo", ServiceVersion: "1.0"} -} - -func getFakeSmap() *sourcemap.Consumer { - cwd, _ := os.Getwd() - data, err := ioutil.ReadFile(filepath.Join(cwd, "..", "testdata/sourcemap/bundle.js.map")) - if err != nil { - panic(err) - } - smap, err := sourcemap.Parse("", data) - if err != nil { - panic(err) - } - return smap -} - -func getFakeConfig() Config { - return Config{ - ElasticsearchConfig: getFakeESConfig(nil), - CacheExpiration: 1 * time.Second, - Index: "test-index", - } -} diff --git a/sourcemap/cache.go b/sourcemap/cache.go deleted file mode 100644 index 7fec0c845b7..00000000000 --- a/sourcemap/cache.go +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "math" - "time" - - "github.com/go-sourcemap/sourcemap" - gocache "github.com/patrickmn/go-cache" - - "github.com/elastic/beats/libbeat/logp" - - logs "github.com/elastic/apm-server/log" -) - -const ( - minCleanupIntervalSeconds float64 = 60 -) - -type cache struct { - goca *gocache.Cache - logger *logp.Logger -} - -func newCache(expiration time.Duration) (*cache, error) { - if expiration < 0 { - return nil, Error{ - Msg: "Cache cannot be initialized. Expiration and CleanupInterval need to be >= 0", - Kind: InitError, - } - } - return &cache{ - goca: gocache.New(expiration, cleanupInterval(expiration)), - logger: logp.NewLogger(logs.Sourcemap), - }, nil -} - -func (c *cache) add(id Id, consumer *sourcemap.Consumer) { - c.goca.Set(id.Key(), consumer, gocache.DefaultExpiration) - if !c.logger.IsDebug() { - return - } - c.logger.Debugf("Added id %v. Cache now has %v entries.", id.Key(), c.goca.ItemCount()) -} - -func (c *cache) remove(id Id) { - c.goca.Delete(id.Key()) - if !c.logger.IsDebug() { - return - } - c.logger.Debugf("Removed id %v. Cache now has %v entries.", id.Key(), c.goca.ItemCount()) -} - -func (c *cache) fetch(id Id) (*sourcemap.Consumer, bool) { - if cached, found := c.goca.Get(id.Key()); found { - if cached == nil { - // in case empty value was cached - // return found=true - return nil, true - } - return cached.(*sourcemap.Consumer), true - } - return nil, false -} - -func cleanupInterval(ttl time.Duration) time.Duration { - return time.Duration(math.Max(ttl.Seconds(), minCleanupIntervalSeconds)) * time.Second -} diff --git a/sourcemap/cache_test.go b/sourcemap/cache_test.go deleted file mode 100644 index 58c992a1237..00000000000 --- a/sourcemap/cache_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestCache(t *testing.T) { - _, err := newCache(-1 * time.Second) - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, InitError) - assert.Contains(t, err.Error(), "Cache cannot be initialized") - - _, err = newCache(1 * time.Second) - assert.NoError(t, err) -} - -func TestAddAndFetch(t *testing.T) { - c, err := newCache(60 * time.Second) - assert.NoError(t, err) - testSmap := getFakeSmap() - id := fakeSmapId() - - //check that cache is nil - smap, found := c.fetch(id) - assert.Nil(t, smap) - assert.False(t, found) - - //add to cache and check that value is cached - c.add(id, testSmap) - smap, found = c.fetch(id) - assert.Equal(t, smap, testSmap) - assert.True(t, found) - - //add nil value to cache and check that value is cached - c.add(id, nil) - smap, found = c.fetch(id) - assert.Nil(t, smap) - assert.True(t, found) -} - -func TestRemove(t *testing.T) { - c, err := newCache(60 * time.Second) - assert.NoError(t, err) - id := fakeSmapId() - testSmap := getFakeSmap() - - c.add(id, testSmap) - smap, _ := c.fetch(id) - assert.Equal(t, smap, testSmap) - - c.remove(id) - smap, found := c.fetch(id) - assert.Nil(t, smap) - assert.False(t, found) -} - -func TestExpiration(t *testing.T) { - expiration := 25 * time.Millisecond - c, err := newCache(expiration) - assert.NoError(t, err) - id := fakeSmapId() - testSmap := getFakeSmap() - - c.add(id, testSmap) - smap, found := c.fetch(id) - assert.Equal(t, smap, testSmap) - assert.True(t, found) - - //let the cache expire - time.Sleep(expiration + 1*time.Millisecond) - smap, found = c.fetch(id) - assert.Nil(t, smap) - assert.False(t, found) -} - -func TestCleanupInterval(t *testing.T) { - tests := []struct { - ttl time.Duration - expected float64 - }{ - {expected: 1}, - {ttl: 30 * time.Second, expected: 1}, - {ttl: 30 * time.Second, expected: 1}, - {ttl: 60 * time.Second, expected: 1}, - {ttl: 61 * time.Second, expected: 61.0 / 60}, - {ttl: 5 * time.Minute, expected: 5}, - } - for idx, test := range tests { - out := cleanupInterval(test.ttl) - assert.Equal(t, test.expected, out.Minutes(), - fmt.Sprintf("(%v) expected %v minutes, received %v minutes", idx, test.expected, out.Minutes())) - } -} - -func fakeSmapId() Id { - serviceName := "foo" - serviceVersion := "bar" - path := "bundle.js.map" - return Id{serviceName, serviceVersion, path} -} diff --git a/sourcemap/elasticsearch.go b/sourcemap/elasticsearch.go deleted file mode 100644 index 2de607c5013..00000000000 --- a/sourcemap/elasticsearch.go +++ /dev/null @@ -1,168 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "encoding/json" - "fmt" - "sync" - - logs "github.com/elastic/apm-server/log" - - "github.com/elastic/apm-server/utility" - - "github.com/go-sourcemap/sourcemap" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" - es "github.com/elastic/beats/libbeat/outputs/elasticsearch" -) - -type elasticsearch interface { - fetch(id Id) (*sourcemap.Consumer, error) -} - -type smapElasticsearch struct { - mu sync.Mutex // guards clients - clients []es.Client - - index string - logger *logp.Logger -} - -func NewElasticsearch(config *common.Config, index string) (*smapElasticsearch, error) { - esClients, err := es.NewElasticsearchClients(config) - if err != nil || esClients == nil || len(esClients) == 0 { - return nil, Error{ - Msg: fmt.Sprintf("Sourcemap ES Client cannot be initialized. %v", err.Error()), - Kind: InitError, - } - } - if index == "" { - index = "*" - } - return &smapElasticsearch{ - clients: esClients, - index: index, - logger: logp.NewLogger(logs.Sourcemap), - }, nil -} - -func (e *smapElasticsearch) fetch(id Id) (*sourcemap.Consumer, error) { - result, err := e.runESQuery(query(id)) - if err != nil { - return nil, err - } - return e.parseResult(result, id) -} - -func (e *smapElasticsearch) runESQuery(body map[string]interface{}) (*es.SearchResults, error) { - var err error - var result *es.SearchResults - e.mu.Lock() - defer e.mu.Unlock() - for _, client := range e.clients { - _, result, err = client.Connection.SearchURIWithBody(e.index, "", nil, body) - if err == nil { - return result, nil - } - } - if err != nil { - return nil, Error{Msg: err.Error(), Kind: AccessError} - } - return result, nil -} - -func (e *smapElasticsearch) parseResult(result *es.SearchResults, id Id) (*sourcemap.Consumer, error) { - if result.Hits.Total.Value == 0 { - return nil, nil - } - if result.Hits.Total.Value > 1 { - e.logger.Warnf("%d sourcemaps found for service %s version %s and file %s, using the most recent one", - result.Hits.Total.Value, id.ServiceName, id.ServiceVersion, id.Path) - } - smap, err := parseSmap(result.Hits.Hits[0]) - if err != nil { - return nil, err - } - cons, err := sourcemap.Parse("", []byte(smap)) - if err != nil { - return nil, Error{ - Msg: fmt.Sprintf("Could not parse Sourcemap. %v", err.Error()), - Kind: ParseError, - } - } - return cons, nil -} - -func query(id Id) map[string]interface{} { - return map[string]interface{}{ - "query": map[string]interface{}{ - "bool": map[string]interface{}{ - "must": []map[string]interface{}{ - {"term": map[string]interface{}{"processor.name": "sourcemap"}}, - {"term": map[string]interface{}{"sourcemap.service.name": id.ServiceName}}, - {"term": map[string]interface{}{"sourcemap.service.version": id.ServiceVersion}}, - {"bool": map[string]interface{}{ - "should": []map[string]interface{}{ - {"term": map[string]interface{}{"sourcemap.bundle_filepath": map[string]interface{}{ - "value": id.Path, - // prefer full url match - "boost": 2.0, - }}}, - {"term": map[string]interface{}{"sourcemap.bundle_filepath": utility.UrlPath(id.Path)}}, - }, - }}, - }, - }, - }, - "size": 1, - "sort": []map[string]interface{}{ - { - "_score": map[string]interface{}{ - "order": "desc", - }, - }, - { - "@timestamp": map[string]interface{}{ - "order": "desc", - }, - }, - }, - "_source": "sourcemap.sourcemap", - } -} - -func parseSmap(result []byte) (string, error) { - var smap struct { - Source struct { - Sourcemap struct { - Sourcemap string - } - } `json:"_source"` - } - err := json.Unmarshal(result, &smap) - if err != nil { - return "", Error{Msg: err.Error(), Kind: ParseError} - } - // until https://github.com/golang/go/issues/19858 is resolved - if smap.Source.Sourcemap.Sourcemap == "" { - return "", Error{Msg: "Sourcemapping ES Result not in expected format", Kind: ParseError} - } - return smap.Source.Sourcemap.Sourcemap, nil -} diff --git a/sourcemap/elasticsearch_test.go b/sourcemap/elasticsearch_test.go deleted file mode 100644 index bc993f09535..00000000000 --- a/sourcemap/elasticsearch_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/common" - es "github.com/elastic/beats/libbeat/outputs/elasticsearch" -) - -func TestNewElasticsearch(t *testing.T) { - _, err := NewElasticsearch(getFakeESConfig(map[string]interface{}{}), "apm") - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, InitError) - assert.Contains(t, err.Error(), "ES Client cannot be initialized") - - es, err := NewElasticsearch(getFakeESConfig(nil), "") - assert.NoError(t, err) - assert.Equal(t, "*", es.index) -} - -func TestNoElasticsearchConnection(t *testing.T) { - es, err := NewElasticsearch(getFakeESConfig(nil), "") - assert.NoError(t, err) - c, err := es.fetch(Id{ServiceName: "testService"}) - assert.Nil(t, c) - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, AccessError) - assert.Contains(t, err.Error(), "connection") -} - -func TestParseResultNoSourcemap(t *testing.T) { - id := Id{Path: "/tmp"} - result := &es.SearchResults{} - c, err := (&smapElasticsearch{}).parseResult(result, id) - assert.Nil(t, c) - assert.NoError(t, err) -} - -func TestParseResultParseError(t *testing.T) { - id := Id{Path: "/tmp"} - result := &es.SearchResults{ - Hits: es.Hits{ - Hits: []json.RawMessage{ - {}, - }, - Total: es.Total{Value: 1}, - }, - } - c, err := (&smapElasticsearch{}).parseResult(result, id) - assert.Nil(t, c) - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, ParseError) - - result = &es.SearchResults{ - Hits: es.Hits{ - Hits: []json.RawMessage{ - []byte(`{"_id": "1","_source": {"sourcemap": {"sourcemap": "map"}}}`), - }, - Total: es.Total{Value: 1}, - }, - } - c, err = (&smapElasticsearch{}).parseResult(result, id) - assert.Nil(t, c) - assert.Error(t, err) - assert.Contains(t, err.Error(), "Could not parse Sourcemap") - assert.Equal(t, (err.(Error)).Kind, ParseError) -} - -func TestParseSourcemapResult(t *testing.T) { - smap, err := parseSmap([]byte(`{ -"_id": "1", -"_source": { -"sourcemap": { -"sourcemap": "map" -} -} -} -`)) - assert.NoError(t, err) - assert.Equal(t, "map", smap) -} - -func TestParseSourcemapResultError(t *testing.T) { - // valid json, missing sourcemap - _, err := parseSmap([]byte(`{ -"_id": "1", -"_source": { -"foo": "bar" -} -} -`)) - assert.Error(t, err) - - // invalid json - _, err = parseSmap([]byte(`{`)) - assert.Error(t, err) -} - -func getFakeESConfig(cfg map[string]interface{}) *common.Config { - if cfg == nil { - cfg = map[string]interface{}{ - "hosts": []string{ - "http://localhost:9288", - "http://localhost:9898", - }, - } - } - c, _ := common.NewConfigFrom(cfg) - return c -} diff --git a/sourcemap/error.go b/sourcemap/error.go index ff4eb803249..fd23b6a0910 100644 --- a/sourcemap/error.go +++ b/sourcemap/error.go @@ -17,21 +17,16 @@ package sourcemap -type Enum string - -const ( - InitError Enum = "InitError" - AccessError Enum = "AccessError" - ParseError Enum = "ParseError" - MapError Enum = "MapError" - KeyError Enum = "KeyError" -) +type internalErr struct { + err error + noSourcemap bool + temporary bool +} -type Error struct { - Msg string - Kind Enum +func (e *internalErr) Error() string { + return e.err.Error() } -func (e Error) Error() string { - return e.Msg +func (e *internalErr) NoSourcemapFound() bool { + return e.noSourcemap } diff --git a/sourcemap/error_test.go b/sourcemap/error_test.go deleted file mode 100644 index 43969603f87..00000000000 --- a/sourcemap/error_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestError(t *testing.T) { - tests := []struct { - Msg string - Kind Enum - OutMsg string - OutKind string - }{ - {OutMsg: "", OutKind: ""}, - {Msg: "Init failed", Kind: InitError, OutMsg: "Init failed", OutKind: "InitError"}, - {Msg: "Access failed", Kind: AccessError, OutMsg: "Access failed", OutKind: "AccessError"}, - {Msg: "Map failed", Kind: MapError, OutMsg: "Map failed", OutKind: "MapError"}, - {Msg: "Parse failed", Kind: ParseError, OutMsg: "Parse failed", OutKind: "ParseError"}, - {Msg: "Key error", Kind: KeyError, OutMsg: "Key error", OutKind: "KeyError"}, - } - for idx, test := range tests { - err := Error{Msg: test.Msg, Kind: test.Kind} - assert.Equal(t, test.OutMsg, err.Msg, - fmt.Sprintf("(%v): Expected <%v>, Received <%v> ", idx, test.OutMsg, err.Msg)) - assert.Equal(t, test.OutKind, string(err.Kind), - fmt.Sprintf("(%v): Expected <%v>, Received <%v> ", idx, test.OutKind, err.Kind)) - } -} diff --git a/sourcemap/es_store.go b/sourcemap/es_store.go new file mode 100644 index 00000000000..63ca7e91d1e --- /dev/null +++ b/sourcemap/es_store.go @@ -0,0 +1,187 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + + "github.com/elastic/apm-server/utility" +) + +const ( + boolStr = "bool" + boostStr = "boost" + descStr = "desc" + mustStr = "must" + orderStr = "order" + processorNameStr = "processor.name" + queryStr = "query" + scoreStr = "_score" + shouldStr = "should" + sizeStr = "size" + sortStr = "sort" + sourcemapStr = "sourcemap" + sourcemapBundleFilepathStr = "sourcemap.bundle_filepath" + sourcemapServiceNameStr = "sourcemap.service.name" + sourcemapServiceVersionStr = "sourcemap.service.version" + sourcemapSourcemapStr = "sourcemap.sourcemap" + sourceStr = "_source" + termStr = "term" + timestampStr = "@timestamp" + valueStr = "value" + + emptyResult = "" + + errMsgParseSourcemap = "Could not parse Sourcemap." +) + +var ( + errSourcemapWrongFormat = errors.New("Sourcemapping ES Result not in expected format") +) + +type esStore struct { + client *elasticsearch.Client + index string + logger *logp.Logger +} + +type esSourcemapResponse struct { + Hits struct { + Total struct { + Value int + } + Hits []struct { + Source struct { + Sourcemap struct { + Sourcemap string + } + } `json:"_source"` + } + } `json:"hits"` +} + +func (s *esStore) fetch(name, version, path string) (string, error) { + response, err := s.runSearchQuery(name, version, path) + if err != nil { + return "", &internalErr{err: err, temporary: true} + } + defer response.Body.Close() + // handle error response + if response.IsError() { + if response.StatusCode == http.StatusNotFound { + return "", nil + } + b, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", &internalErr{err: errors.Wrap(err, errMsgParseSourcemap)} + } + return "", &internalErr{err: errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b))} + } + + // parse response + return parse(response, name, version, path, s.logger) +} + +func (s *esStore) runSearchQuery(name, version, path string) (*esapi.Response, error) { + // build and encode the query + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(query(name, version, path)); err != nil { + return nil, &internalErr{err: err} + } + + // Perform the runSearchQuery request. + return s.client.Search( + s.client.Search.WithContext(context.Background()), //TODO: should we cancel with timeout? + s.client.Search.WithIndex(s.index), + s.client.Search.WithBody(&buf), + s.client.Search.WithTrackTotalHits(true), + s.client.Search.WithPretty(), + ) +} + +func parse(response *esapi.Response, name, version, path string, logger *logp.Logger) (string, error) { + var esSourcemapResponse esSourcemapResponse + if err := json.NewDecoder(response.Body).Decode(&esSourcemapResponse); err != nil { + return "", &internalErr{err: err} + } + hits := esSourcemapResponse.Hits.Total.Value + if hits == 0 { + return emptyResult, nil + } + + var esSourcemap string + if hits > 1 { + logger.Warnf("%d sourcemaps found for service %s version %s and file %s, using the most recent one", + hits, name, version, path) + } + esSourcemap = esSourcemapResponse.Hits.Hits[0].Source.Sourcemap.Sourcemap + // until https://github.com/golang/go/issues/19858 is resolved + if esSourcemap == emptyResult { + return emptyResult, &internalErr{err: errSourcemapWrongFormat} + } + return esSourcemap, nil +} + +func query(name, version, path string) map[string]interface{} { + return map[string]interface{}{ + queryStr: map[string]interface{}{ + boolStr: map[string]interface{}{ + mustStr: []map[string]interface{}{ + {termStr: map[string]interface{}{processorNameStr: sourcemapStr}}, + {termStr: map[string]interface{}{sourcemapServiceNameStr: name}}, + {termStr: map[string]interface{}{sourcemapServiceVersionStr: version}}, + {boolStr: map[string]interface{}{ + shouldStr: []map[string]interface{}{ + {termStr: map[string]interface{}{sourcemapBundleFilepathStr: map[string]interface{}{ + valueStr: path, + // prefer full url match + boostStr: 2.0, + }}}, + {termStr: map[string]interface{}{sourcemapBundleFilepathStr: utility.UrlPath(path)}}, + }, + }}, + }, + }, + }, + sizeStr: 1, + sortStr: []map[string]interface{}{ + { + scoreStr: map[string]interface{}{ + orderStr: descStr, + }, + }, + { + timestampStr: map[string]interface{}{ + orderStr: descStr, + }, + }, + }, + sourceStr: sourcemapSourcemapStr, + } +} diff --git a/sourcemap/es_store_test.go b/sourcemap/es_store_test.go new file mode 100644 index 00000000000..c2941dbc1f5 --- /dev/null +++ b/sourcemap/es_store_test.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "net/http" + "testing" + + "github.com/go-sourcemap/sourcemap" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/go-elasticsearch/v8" + + estest "github.com/elastic/apm-server/elasticsearch/test" + logs "github.com/elastic/apm-server/log" + "github.com/elastic/apm-server/sourcemap/test" +) + +//func TestABC(t *testing.T) { +// client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{"http://localhost:9200"}}) +// require.NoError(t, err) +// f := esFetcher{client: client, index: "apm-*-sourcemap", logger: logp.NewLogger(logs.Sourcemap)} +// c, err := f.fetch(Id{Path: "bundle_no_mapping.js.map", ServiceName: "apm-agent-js", ServiceVersion: "1.0.1"}) +// assert.NoError(t, err) +// assert.Nil(t, c) +// c.File() +//} +func Test_esFetcher_fetchError(t *testing.T) { + for name, tc := range map[string]struct { + statusCode int + esBody map[string]interface{} + temporary bool + }{ + "es not reachable": { + statusCode: http.StatusInternalServerError, temporary: true, + }, + "es bad request": { + statusCode: http.StatusBadRequest, + }, + "empty sourcemap string": { + esBody: map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{"value": 1}, + "hits": []map[string]interface{}{ + {"_source": map[string]interface{}{ + "sourcemap": map[string]interface{}{ + "sourcemap": ""}}}}}}, + }, + } { + t.Run(name, func(t *testing.T) { + statusCode := tc.statusCode + if statusCode == 0 { + statusCode = http.StatusOK + } + client, err := estest.NewElasticsearchClient(estest.NewTransport(t, statusCode, tc.esBody)) + require.NoError(t, err) + consumer, err := testESStore(client).fetch("abc", "1.0", "/tmp") + require.Error(t, err) + assert.Equal(t, tc.temporary, err.(*internalErr).temporary) + assert.Empty(t, consumer) + }) + } +} + +func Test_esFetcher_fetch(t *testing.T) { + for name, tc := range map[string]struct { + client *elasticsearch.Client + filePath string + }{ + "no sourcemap found": {client: test.ESClientWithSourcemapNotFound(t)}, + "valid sourcemap found": {client: test.ESClientWithValidSourcemap(t), filePath: "bundle.js"}, + } { + t.Run(name, func(t *testing.T) { + sourcemapStr, err := testESStore(tc.client).fetch("abc", "1.0", "/tmp") + require.NoError(t, err) + + if tc.filePath == "" { + assert.Empty(t, sourcemapStr) + } else { + sourcemapConsumer, err := sourcemap.Parse("", []byte(sourcemapStr)) + require.NoError(t, err) + assert.Equal(t, tc.filePath, sourcemapConsumer.File()) + } + }) + } +} + +func testESStore(client *elasticsearch.Client) *esStore { + return &esStore{client: client, index: "apm-sourcemap", logger: logp.NewLogger(logs.Sourcemap)} +} diff --git a/sourcemap/id_test.go b/sourcemap/id_test.go deleted file mode 100644 index 18f71b2aab2..00000000000 --- a/sourcemap/id_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSmapIdKey(t *testing.T) { - tests := []struct { - id Id - key string - valid bool - }{ - { - id: Id{ServiceName: "foo", ServiceVersion: "1.0", Path: "a/b"}, - key: "foo_1.0_a/b", - valid: true, - }, - { - id: Id{ServiceName: "foo", ServiceVersion: "bar"}, - key: "foo_bar", - valid: false, - }, - {id: Id{ServiceName: "foo"}, key: "foo", valid: false}, - {id: Id{ServiceVersion: "1"}, key: "1", valid: false}, - {id: Id{Path: "/tmp/a"}, key: "/tmp/a", valid: false}, - } - for idx, test := range tests { - assert.Equal(t, test.key, test.id.Key(), - fmt.Sprintf("(%v) Expected Key() to return <%v> but received<%v>", idx, test.key, test.id.Key())) - assert.Equal(t, test.valid, test.id.Valid(), - fmt.Sprintf("(%v) Expected Valid() to be <%v>", idx, test.valid)) - } -} diff --git a/sourcemap/mapper.go b/sourcemap/mapper.go index 1eca3f058a5..e4216b35da9 100644 --- a/sourcemap/mapper.go +++ b/sourcemap/mapper.go @@ -18,91 +18,33 @@ package sourcemap import ( - "fmt" "strings" - "time" - logs "github.com/elastic/apm-server/log" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" + "github.com/go-sourcemap/sourcemap" ) const sourcemapContentSnippetSize = 5 -type Mapper interface { - Apply(Id, int, int) (*Mapping, error) - NewSourcemapAdded(id Id) -} - -type SmapMapper struct { - Accessor Accessor - logger *logp.Logger +// Mapper holds a sourcemapper instance to be able to apply sourcemapping +type Mapper struct { + sourcemapConsumer *sourcemap.Consumer } -type Config struct { - CacheExpiration time.Duration - ElasticsearchConfig *common.Config - Index string -} +// Apply sourcemapping for given line and column and return values after sourcemapping +func (m *Mapper) Apply(lineno, colno int) ( + file string, function string, line int, col int, + contextLine string, preContext []string, postContext []string, ok bool) { -type Mapping struct { - Filename string - Function string - Colno int - Lineno int - Path string - ContextLine string - PreContext []string - PostContext []string -} - -func NewSmapMapper(config Config) (*SmapMapper, error) { - accessor, err := NewSmapAccessor(config) - if err != nil { - return nil, err + if m.sourcemapConsumer == nil { + return } - return &SmapMapper{ - Accessor: accessor, - logger: logp.NewLogger(logs.Sourcemap), - }, nil -} -func (m *SmapMapper) Apply(id Id, lineno, colno int) (*Mapping, error) { - smapCons, err := m.Accessor.Fetch(id) - if err != nil { - return nil, err - } - - file, funct, line, col, ok := smapCons.Source(lineno, colno) - if !ok { - return nil, Error{ - Msg: fmt.Sprintf("No Sourcemap found for id %v, Lineno %v, Colno %v", - id.String(), lineno, colno), - Kind: KeyError, - } - } - src := strings.Split(smapCons.SourceContent(file), "\n") - return &Mapping{ - Filename: file, - Function: funct, - Lineno: line, - Colno: col, - Path: id.Path, - // line is 1-based - ContextLine: strings.Join(subSlice(line-1, line, src), ""), - PreContext: subSlice(line-1-sourcemapContentSnippetSize, line-1, src), - PostContext: subSlice(line, line+sourcemapContentSnippetSize, src), - }, nil -} - -func (m *SmapMapper) NewSourcemapAdded(id Id) { - _, err := m.Accessor.Fetch(id) - if err == nil { - m.logger.Warnf("Overriding sourcemap for service %s version %s and file %s", - id.ServiceName, id.ServiceVersion, id.Path) - } - m.Accessor.Remove(id) + file, function, line, col, ok = m.sourcemapConsumer.Source(lineno, colno) + src := strings.Split(m.sourcemapConsumer.SourceContent(file), "\n") + contextLine = strings.Join(subSlice(line-1, line, src), "") + preContext = subSlice(line-1-sourcemapContentSnippetSize, line-1, src) + postContext = subSlice(line, line+sourcemapContentSnippetSize, src) + return } func subSlice(from, to int, content []string) []string { diff --git a/sourcemap/mapper_test.go b/sourcemap/mapper_test.go index d69a2f5e367..8886f53b98d 100644 --- a/sourcemap/mapper_test.go +++ b/sourcemap/mapper_test.go @@ -18,54 +18,48 @@ package sourcemap import ( - "io/ioutil" - "os" - "path/filepath" "testing" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/sourcemap/test" + "github.com/go-sourcemap/sourcemap" "github.com/stretchr/testify/assert" ) -func TestNewSmapMapper(t *testing.T) { - mapper, err := NewSmapMapper(Config{}) - assert.Nil(t, mapper) - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, InitError) - - mapper, err = NewSmapMapper(getFakeConfig()) - assert.NoError(t, err) - assert.NotNil(t, mapper) -} - func TestApply(t *testing.T) { - mapper, err := NewSmapMapper(getFakeConfig()) - assert.NoError(t, err) - assert.NotNil(t, mapper) + // no sourcemapConsumer + _, _, _, _, _, _, _, ok := (&Mapper{}).Apply(0, 0) + assert.False(t, ok) - // error occurs - mapping, err := mapper.Apply(Id{}, 0, 0) - assert.Nil(t, mapping) - assert.Error(t, err) - assert.Equal(t, (err.(Error)).Kind, KeyError) + sourcemapConsumer, err := sourcemap.Parse("", []byte(test.ValidSourcemap)) + require.NoError(t, err) + m := &Mapper{sourcemapConsumer: sourcemapConsumer} - // no mapping found in sourcemap - mapper.Accessor = &fakeAccessor{} - mapping, err = mapper.Apply(Id{Path: "bundle.js.map"}, 0, 0) - assert.Nil(t, mapping) - assert.Error(t, err) - assert.Contains(t, err.Error(), "No Sourcemap found") - assert.Equal(t, (err.(Error)).Kind, KeyError) + t.Run("notOK", func(t *testing.T) { + // nothing found for lineno and colno + file, fc, line, col, ctxLine, _, _, ok := m.Apply(0, 0) + require.False(t, ok) + assert.Zero(t, file) + assert.Zero(t, fc) + assert.Zero(t, line) + assert.Zero(t, col) + assert.Zero(t, ctxLine) + }) - // mapping found in minified sourcemap - mapping, err = mapper.Apply(Id{Path: "bundle.js.map"}, 1, 7) - assert.NoError(t, err) - assert.NotNil(t, mapping) - assert.Equal(t, "webpack:///bundle.js", mapping.Filename) - assert.Equal(t, "", mapping.Function) - assert.Equal(t, 1, mapping.Lineno) - assert.Equal(t, 9, mapping.Colno) - assert.Equal(t, "bundle.js.map", mapping.Path) + t.Run("OK", func(t *testing.T) { + // mapping found in minified sourcemap + file, fc, line, col, ctxLine, preCtx, postCtx, ok := m.Apply(1, 7) + require.True(t, ok) + assert.Equal(t, "webpack:///bundle.js", file) + assert.Equal(t, "", fc) + assert.Equal(t, 1, line) + assert.Equal(t, 9, col) + assert.Equal(t, "/******/ (function(modules) { // webpackBootstrap", ctxLine) + assert.Equal(t, []string{}, preCtx) + assert.NotZero(t, postCtx) + }) } func TestSubSlice(t *testing.T) { @@ -94,16 +88,3 @@ func TestSubSlice(t *testing.T) { assert.Equal(t, []string{}, subSlice(test.start, test.end, []string{})) } } - -type fakeAccessor struct{} - -func (ac *fakeAccessor) Fetch(smapId Id) (*sourcemap.Consumer, error) { - current, _ := os.Getwd() - path := filepath.Join(current, "../testdata/sourcemap/", smapId.Path) - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return sourcemap.Parse("", fileBytes) -} -func (ac *fakeAccessor) Remove(smapId Id) {} diff --git a/sourcemap/store.go b/sourcemap/store.go new file mode 100644 index 00000000000..e6e008ee922 --- /dev/null +++ b/sourcemap/store.go @@ -0,0 +1,137 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/go-sourcemap/sourcemap" + gocache "github.com/patrickmn/go-cache" + "github.com/pkg/errors" + + "github.com/elastic/go-elasticsearch/v8" + + "github.com/elastic/beats/libbeat/logp" + + logs "github.com/elastic/apm-server/log" +) + +const ( + minCleanupIntervalSeconds float64 = 60 +) + +var ( + errInit = errors.New("Cache cannot be initialized. Expiration and CleanupInterval need to be >= 0") +) + +// Store holds information necessary to fetch a sourcemap, either from an Elasticsearch instance or an internal cache. +type Store struct { + cache *gocache.Cache + esStore *esStore + logger *logp.Logger +} + +// NewStore creates a new instance for fetching sourcemaps. The client and index parameters are needed to be able to +// fetch sourcemaps from Elasticsearch. The expiration time is used for the internal cache. +func NewStore(client *elasticsearch.Client, index string, expiration time.Duration) (*Store, error) { + if expiration < 0 { + return nil, &internalErr{err: errInit} + } + logger := logp.NewLogger(logs.Sourcemap) + return &Store{ + cache: gocache.New(expiration, cleanupInterval(expiration)), + esStore: &esStore{client: client, index: index, logger: logger}, + logger: logger, + }, nil +} + +// Fetch a sourcemap from the store. +func (s *Store) Fetch(name string, version string, path string) (*Mapper, error) { + key := key([]string{name, version, path}) + + // fetch from cache + if val, found := s.cache.Get(key); found { + sourcemapConsumer, ok := val.(*sourcemap.Consumer) + if ok && sourcemapConsumer != nil { + return &Mapper{sourcemapConsumer: sourcemapConsumer}, nil + } + return nil, errNoSourcemap(name, version, path) + } + + // fetch from Elasticsearch and ensure caching for all non-temporary results + sourcemapStr, err := s.esStore.fetch(name, version, path) + if err != nil { + if e, ok := err.(*internalErr); ok && !e.temporary { + s.add(key, nil) + } + return nil, err + } + + if sourcemapStr == emptyResult { + s.add(key, nil) + return nil, errNoSourcemap(name, version, path) + } + + consumer, err := sourcemap.Parse("", []byte(sourcemapStr)) + if err != nil { + s.add(key, nil) + return nil, &internalErr{err: errors.Wrap(err, errMsgParseSourcemap)} + } + s.add(key, consumer) + return &Mapper{sourcemapConsumer: consumer}, nil +} + +// Added ensures the internal cache is cleared for the given parameters. This should be called when a sourcemap is uploaded. +func (s *Store) Added(name string, version string, path string) { + if _, err := s.Fetch(name, version, path); err == nil { + s.logger.Warnf("Overriding sourcemap for service %s version %s and file %s", + name, version, path) + } + key := key([]string{name, version, path}) + s.cache.Delete(key) + if !s.logger.IsDebug() { + return + } + s.logger.Debugf("Removed id %v. Cache now has %v entries.", key, s.cache.ItemCount()) +} + +func (s *Store) add(key string, consumer *sourcemap.Consumer) { + s.cache.SetDefault(key, consumer) + if !s.logger.IsDebug() { + return + } + s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.ItemCount()) +} + +func key(s []string) string { + return strings.Join(s, "_") +} + +func cleanupInterval(ttl time.Duration) time.Duration { + return time.Duration(math.Max(ttl.Seconds(), minCleanupIntervalSeconds)) * time.Second +} + +func errNoSourcemap(name, version, path string) *internalErr { + return &internalErr{ + err: errors.New(fmt.Sprintf( + "No Sourcemap available for ServiceName %s, ServiceVersion %s, Path %s.", name, version, path)), + noSourcemap: true} +} diff --git a/sourcemap/store_test.go b/sourcemap/store_test.go new file mode 100644 index 00000000000..ec8902c06b6 --- /dev/null +++ b/sourcemap/store_test.go @@ -0,0 +1,194 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "fmt" + "testing" + "time" + + "github.com/go-sourcemap/sourcemap" + gocache "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/go-elasticsearch/v8" + + "github.com/elastic/apm-server/sourcemap/test" +) + +func Test_NewStore(t *testing.T) { + _, err := NewStore(nil, "", -1) + require.Error(t, err) + + f, err := NewStore(nil, "", 100) + require.NoError(t, err) + assert.NotNil(t, f.cache) +} + +func TestStore_Fetch(t *testing.T) { + serviceName, serviceVersion, path := "foo", "1.0.1", "/tmp" + key := "foo_1.0.1_/tmp" + + t.Run("cache", func(t *testing.T) { + t.Run("nil", func(t *testing.T) { + var nilConsumer *sourcemap.Consumer + store := testStore(t, test.ESClientWithValidSourcemap(t)) //if ES was queried, it would return a valid sourcemap + store.add(key, nilConsumer) + + mapper, err := store.Fetch(serviceName, serviceVersion, path) + require.Error(t, err) + assert.Contains(t, err.Error(), "No Sourcemap available for ServiceName") + require.Nil(t, mapper) + }) + + t.Run("sourcemapConsumer", func(t *testing.T) { + consumer := &sourcemap.Consumer{} + store := testStore(t, test.ESClientUnavailable(t)) //if ES was queried, it would return a server error + store.add(key, consumer) + + mapper, err := store.Fetch(serviceName, serviceVersion, path) + require.NoError(t, err) + require.NotNil(t, mapper) + assert.Equal(t, consumer, mapper.sourcemapConsumer) + + }) + }) + + t.Run("validFromES", func(t *testing.T) { + store := testStore(t, test.ESClientWithValidSourcemap(t)) + mapper, err := store.Fetch(serviceName, serviceVersion, path) + require.NoError(t, err) + require.NotNil(t, mapper) + assert.NotNil(t, mapper.sourcemapConsumer) + + // ensure sourcemap is added to cache + cached, found := store.cache.Get(key) + require.True(t, found) + assert.Equal(t, mapper.sourcemapConsumer, cached) + }) + + t.Run("invalidFromES", func(t *testing.T) { + for name, client := range map[string]*elasticsearch.Client{ + "notFound": test.ESClientWithSourcemapNotFound(t), + "invalid": test.ESClientWithInvalidSourcemap(t), + "unsupportedVersion": test.ESClientWithUnsupportedSourcemap(t), + } { + t.Run(name, func(t *testing.T) { + store := testStore(t, client) + //not cached + cached, found := store.cache.Get(key) + require.False(t, found) + assert.Nil(t, cached) + + //fetch nil value, leading to error + mapper, err := store.Fetch(serviceName, serviceVersion, path) + require.Error(t, err) + require.Nil(t, mapper) + + // ensure nil value is added to cache + cached, found = store.cache.Get(key) + assert.True(t, found) + assert.Nil(t, cached) + }) + } + }) + + t.Run("noConnectionToES", func(t *testing.T) { + store := testStore(t, test.ESClientUnavailable(t)) + //not cached + _, found := store.cache.Get(key) + require.False(t, found) + + //fetch nil value, leading to error + mapper, err := store.Fetch(serviceName, serviceVersion, path) + require.Error(t, err) + require.Nil(t, mapper) + + // ensure not cached + _, found = store.cache.Get(key) + assert.False(t, found) + }) +} + +func TestStore_Added(t *testing.T) { + name, version, path := "foo", "1.0.1", "/tmp" + key := "foo_1.0.1_/tmp" + + // setup + // remove empty sourcemap from cache, and valid one with File() == "bundle.js" from Elasticsearch + store := testStore(t, test.ESClientWithValidSourcemap(t)) + store.add(key, &sourcemap.Consumer{}) + + mapper, err := store.Fetch(name, version, path) + require.NoError(t, err) + require.NotNil(t, mapper) + assert.Equal(t, &sourcemap.Consumer{}, mapper.sourcemapConsumer) + assert.Equal(t, "", mapper.sourcemapConsumer.File()) + + // remove from cache, afterwards sourcemap should be fetched from ES + store.Added(name, version, path) + mapper, err = store.Fetch(name, version, path) + require.NoError(t, err) + require.NotNil(t, mapper) + assert.NotNil(t, &sourcemap.Consumer{}, mapper.sourcemapConsumer) + assert.Equal(t, "bundle.js", mapper.sourcemapConsumer.File()) +} + +func TestExpiration(t *testing.T) { + store := testStore(t, test.ESClientUnavailable(t)) //if ES was queried it would return an error + store.cache = gocache.New(25*time.Millisecond, 100) + store.add("foo_1.0.1_/tmp", &sourcemap.Consumer{}) + name, version, path := "foo", "1.0.1", "/tmp" + + // sourcemap is cached + mapper, err := store.Fetch(name, version, path) + require.NoError(t, err) + assert.Equal(t, &sourcemap.Consumer{}, mapper.sourcemapConsumer) + + time.Sleep(25 * time.Millisecond) + // cache is cleared, sourcemap is fetched from ES leading to an error + mapper, err = store.Fetch(name, version, path) + require.Error(t, err) + assert.Nil(t, mapper) +} + +func TestCleanupInterval(t *testing.T) { + tests := []struct { + ttl time.Duration + expected float64 + }{ + {expected: 1}, + {ttl: 30 * time.Second, expected: 1}, + {ttl: 30 * time.Second, expected: 1}, + {ttl: 60 * time.Second, expected: 1}, + {ttl: 61 * time.Second, expected: 61.0 / 60}, + {ttl: 5 * time.Minute, expected: 5}, + } + for idx, test := range tests { + out := cleanupInterval(test.ttl) + assert.Equal(t, test.expected, out.Minutes(), + fmt.Sprintf("(%v) expected %v minutes, received %v minutes", idx, test.expected, out.Minutes())) + } +} + +func testStore(t *testing.T, client *elasticsearch.Client) *Store { + store, err := NewStore(client, "apm-*sourcemap*", time.Minute) + require.NoError(t, err) + return store +} diff --git a/sourcemap/test/es_client.go b/sourcemap/test/es_client.go new file mode 100644 index 00000000000..40d6f9fd167 --- /dev/null +++ b/sourcemap/test/es_client.go @@ -0,0 +1,150 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package test + +import ( + "net/http" + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/elasticsearch/test" +) + +//ValidSourcemap represents an example for a valid sourcemap string +var ValidSourcemap = `{ + "version": 3, + "sources": [ + "webpack:///bundle.js", + "", + "webpack:///./scripts/index.js", + "webpack:///./index.html", + "webpack:///./scripts/app.js" + ], + "names": [ + "modules", + "__webpack_require__", + "moduleId", + "installedModules", + "exports", + "module", + "id", + "loaded", + "call", + "m", + "c", + "p", + "foo", + "console", + "log", + "foobar" + ], + "mappings": "CAAS,SAAUA,GCInB,QAAAC,GAAAC,GAGA,GAAAC,EAAAD,GACA,MAAAC,GAAAD,GAAAE,OAGA,IAAAC,GAAAF,EAAAD,IACAE,WACAE,GAAAJ,EACAK,QAAA,EAUA,OANAP,GAAAE,GAAAM,KAAAH,EAAAD,QAAAC,IAAAD,QAAAH,GAGAI,EAAAE,QAAA,EAGAF,EAAAD,QAvBA,GAAAD,KAqCA,OATAF,GAAAQ,EAAAT,EAGAC,EAAAS,EAAAP,EAGAF,EAAAU,EAAA,GAGAV,EAAA,KDMM,SAASI,EAAQD,EAASH,GE3ChCA,EAAA,GAEAA,EAAA,GAEAW,OFmDM,SAASP,EAAQD,EAASH,GGxDhCI,EAAAD,QAAAH,EAAAU,EAAA,cH8DM,SAASN,EAAQD,GI9DvB,QAAAQ,KACAC,QAAAC,IAAAC,QAGAH", + "file": "bundle.js", + "sourcesContent": [ + "/******/ (function(modules) { // webpackBootstrap\n/******/ \t// The module cache\n/******/ \tvar installedModules = {};\n/******/\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(installedModules[moduleId])\n/******/ \t\t\treturn installedModules[moduleId].exports;\n/******/\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = installedModules[moduleId] = {\n/******/ \t\t\texports: {},\n/******/ \t\t\tid: moduleId,\n/******/ \t\t\tloaded: false\n/******/ \t\t};\n/******/\n/******/ \t\t// Execute the module function\n/******/ \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n/******/\n/******/ \t\t// Flag the module as loaded\n/******/ \t\tmodule.loaded = true;\n/******/\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/\n/******/\n/******/ \t// expose the modules object (__webpack_modules__)\n/******/ \t__webpack_require__.m = modules;\n/******/\n/******/ \t// expose the module cache\n/******/ \t__webpack_require__.c = installedModules;\n/******/\n/******/ \t// __webpack_public_path__\n/******/ \t__webpack_require__.p = \"\";\n/******/\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(0);\n/******/ })\n/************************************************************************/\n/******/ ([\n/* 0 */\n/***/ function(module, exports, __webpack_require__) {\n\n\t// Webpack\n\t__webpack_require__(1)\n\t\n\t__webpack_require__(2)\n\t\n\tfoo()\n\n\n/***/ },\n/* 1 */\n/***/ function(module, exports, __webpack_require__) {\n\n\tmodule.exports = __webpack_require__.p + \"index.html\"\n\n/***/ },\n/* 2 */\n/***/ function(module, exports) {\n\n\tfunction foo() {\n\t console.log(foobar)\n\t}\n\t\n\tfoo()\n\n\n/***/ }\n/******/ ]);\n\n\n/** WEBPACK FOOTER **\n ** bundle.js\n **/", + " \t// The module cache\n \tvar installedModules = {};\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId])\n \t\t\treturn installedModules[moduleId].exports;\n\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\texports: {},\n \t\t\tid: moduleId,\n \t\t\tloaded: false\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.loaded = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"\";\n\n \t// Load entry module and return exports\n \treturn __webpack_require__(0);\n\n\n\n/** WEBPACK FOOTER **\n ** webpack/bootstrap 6002740481c9666b0d38\n **/", + "// Webpack\nrequire('../index.html')\n\nrequire('./app')\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/index.js\n ** module id = 0\n ** module chunks = 0\n **/", + "module.exports = __webpack_public_path__ + \"index.html\"\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./index.html\n ** module id = 1\n ** module chunks = 0\n **/", + "function foo() {\n console.log(foobar)\n}\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/app.js\n ** module id = 2\n ** module chunks = 0\n **/" + ], + "sourceRoot": "" +}` + +// ESClientWithValidSourcemap returns an elasticsearch client that will always return a document containing +// a valid sourcemap. +func ESClientWithValidSourcemap(t *testing.T) *elasticsearch.Client { + client, err := test.NewElasticsearchClient(test.NewTransport(t, http.StatusOK, validSourcemapFromES())) + require.NoError(t, err) + return client +} + +// ESClientUnavailable returns an elasticsearch client that will always return an internal server error +func ESClientUnavailable(t *testing.T) *elasticsearch.Client { + client, err := test.NewElasticsearchClient(test.NewTransport(t, http.StatusInternalServerError, nil)) + require.NoError(t, err) + return client +} + +// ESClientWithInvalidSourcemap returns an elasticsearch client that will always return a document containing +// an invalid sourcemap. +func ESClientWithInvalidSourcemap(t *testing.T) *elasticsearch.Client { + client, err := test.NewElasticsearchClient(test.NewTransport(t, http.StatusOK, invalidSourcemapFromES())) + require.NoError(t, err) + return client +} + +// ESClientWithUnsupportedSourcemap returns an elasticsearch client that will always return a document containing +// a sourcemap with an unsupported version. +func ESClientWithUnsupportedSourcemap(t *testing.T) *elasticsearch.Client { + client, err := test.NewElasticsearchClient(test.NewTransport(t, http.StatusOK, sourcemapUnsupportedVersionFromES())) + require.NoError(t, err) + return client +} + +// ESClientWithSourcemapNotFound returns an elasticsearch client that will always return a not found error +func ESClientWithSourcemapNotFound(t *testing.T) *elasticsearch.Client { + client, err := test.NewElasticsearchClient(test.NewTransport(t, http.StatusNotFound, sourcemapNotFoundFromES())) + require.NoError(t, err) + return client +} + +func validSourcemapFromES() map[string]interface{} { + return map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{"value": 1}, + "hits": []map[string]interface{}{ + {"_source": map[string]interface{}{ + "sourcemap": map[string]interface{}{ + "sourcemap": ValidSourcemap}}}}}} +} + +func sourcemapNotFoundFromES() map[string]interface{} { + return map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{"value": 0}}} +} + +func invalidSourcemapFromES() map[string]interface{} { + return map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{"value": 1}, + "hits": []map[string]interface{}{ + {"_source": map[string]interface{}{ + "sourcemap": map[string]interface{}{ + "sourcemap": "foo"}}}}}} +} + +func sourcemapUnsupportedVersionFromES() map[string]interface{} { + return map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{"value": 1}, + "hits": []map[string]interface{}{ + {"_source": map[string]interface{}{ + "sourcemap": map[string]interface{}{ + "sourcemap": `{ + "version": 1, + "sources": ["webpack:///bundle.js"], + "names": [], + "mappings": "CAAS", + "file": "bundle.js", + "sourcesContent": [], + "sourceRoot": "" + }`}}}}}} +} diff --git a/transform/transform.go b/transform/transform.go index 59e6982653e..a571cca9732 100644 --- a/transform/transform.go +++ b/transform/transform.go @@ -21,9 +21,10 @@ import ( "regexp" "time" + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/apm-server/model/metadata" "github.com/elastic/apm-server/sourcemap" - "github.com/elastic/beats/libbeat/beat" ) type Transformable interface { @@ -39,5 +40,5 @@ type Context struct { type Config struct { LibraryPattern *regexp.Regexp ExcludeFromGrouping *regexp.Regexp - SourcemapMapper sourcemap.Mapper + SourcemapStore *sourcemap.Store }