diff --git a/cmd/es-rollover/app/init/flags.go b/cmd/es-rollover/app/init/flags.go index 835dc627a58..f2af75cddc4 100644 --- a/cmd/es-rollover/app/init/flags.go +++ b/cmd/es-rollover/app/init/flags.go @@ -17,25 +17,10 @@ package init import ( "flag" - cfg "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/spf13/viper" "github.com/jaegertracing/jaeger/cmd/es-rollover/app" -) - -const ( - spanShards = "shards-span" - serviceShards = "shards-service" - dependenciesShards = "shards-dependencies" - samplingShards = "shards-sampling" - spanReplicas = "replicas-span" - serviceReplicas = "replicas-service" - dependenciesReplicas = "replicas-dependencies" - samplingReplicas = "replicas-sampling" - prioritySpanTemplate = "priority-span-template" - priorityServiceTemplate = "priority-service-template" - priorityDependenciesTemplate = "priority-dependencies-template" - prioritySamplingTemplate = "priority-sampling-template" + cfg "github.com/jaegertracing/jaeger/pkg/es/config" ) // Config holds configuration for index cleaner binary. @@ -46,34 +31,34 @@ type Config struct { // AddFlags adds flags for TLS to the FlagSet. func (*Config) AddFlags(flags *flag.FlagSet) { - flags.Int64(spanShards, 5, "Number of span index shards") - flags.Int64(serviceShards, 5, "Number of service index shards") - flags.Int64(dependenciesShards, 5, "Number of dependencies index shards") - flags.Int64(samplingShards, 5, "Number of sampling index shards") + flags.Int64(cfg.GetNumShardSpanFlag(), 5, "Number of span index shards") + flags.Int64(cfg.GetNumShardServiceFlag(), 5, "Number of service index shards") + flags.Int64(cfg.GetNumShardDependenciesFlag(), 5, "Number of dependencies index shards") + flags.Int64(cfg.GetNumShardSamplingFlag(), 5, "Number of sampling index shards") - flags.Int64(spanReplicas, 1, "Number of span index replicas") - flags.Int64(serviceReplicas, 1, "Number of services index replicas") - flags.Int64(dependenciesReplicas, 1, "Number of dependencies index replicas") - flags.Int64(samplingReplicas, 1, "Number of sampling index replicas") + flags.Int64(cfg.GetNumReplicasSpanFlag(), 1, "Number of span index replicas") + flags.Int64(cfg.GetNumReplicasServiceFlag(), 1, "Number of services index replicas") + flags.Int64(cfg.GetNumReplicasDependenciesFlag(), 1, "Number of dependencies index replicas") + flags.Int64(cfg.GetNumReplicasSamplingFlag(), 1, "Number of sampling index replicas") - flags.Int64(prioritySpanTemplate, 0, "Priority of jaeger-span index template (ESv8 only)") - flags.Int64(priorityServiceTemplate, 0, "Priority of jaeger-service index template (ESv8 only)") - flags.Int64(priorityDependenciesTemplate, 0, "Priority of jaeger-dependencies index template (ESv8 only)") - flags.Int64(prioritySamplingTemplate, 0, "Priority of jaeger-sampling index template (ESv8 only)") + flags.Int64(cfg.GetPrioritySpanTemplate(), 0, "Priority of jaeger-span index template (ESv8 only)") + flags.Int64(cfg.GetPriorityServiceTemplate(), 0, "Priority of jaeger-service index template (ESv8 only)") + flags.Int64(cfg.GetPriorityDependenciesTemplate(), 0, "Priority of jaeger-dependencies index template (ESv8 only)") + flags.Int64(cfg.GetPrioritySamplingTemplate(), 0, "Priority of jaeger-sampling index template (ESv8 only)") } // InitFromViper initializes config from viper.Viper. func (c *Config) InitFromViper(v *viper.Viper) { - c.Indices.Spans.TemplateOptions.NumShards = v.GetInt(spanShards) - c.Indices.Spans.TemplateOptions.NumReplicas = v.GetInt(spanReplicas) - c.Indices.Services.TemplateOptions.NumShards = v.GetInt(serviceShards) - c.Indices.Services.TemplateOptions.NumReplicas = v.GetInt(serviceReplicas) - c.Indices.Dependencies.TemplateOptions.NumShards = v.GetInt(dependenciesShards) - c.Indices.Dependencies.TemplateOptions.NumReplicas = v.GetInt(dependenciesReplicas) - c.Indices.Sampling.TemplateOptions.NumShards = v.GetInt(samplingShards) - c.Indices.Sampling.TemplateOptions.NumReplicas = v.GetInt(samplingReplicas) - c.Indices.Spans.TemplateOptions.Priority = v.GetInt(prioritySpanTemplate) - c.Indices.Services.TemplateOptions.Priority = v.GetInt(priorityServiceTemplate) - c.Indices.Dependencies.TemplateOptions.Priority = v.GetInt(priorityDependenciesTemplate) - c.Indices.Sampling.TemplateOptions.Priority = v.GetInt(prioritySamplingTemplate) + c.Indices.Spans.TemplateNumShards = v.GetInt(cfg.GetNumShardSpanFlag()) + c.Indices.Spans.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasSpanFlag()) + c.Indices.Services.TemplateNumShards = v.GetInt(cfg.GetNumShardServiceFlag()) + c.Indices.Services.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasServiceFlag()) + c.Indices.Dependencies.TemplateNumShards = v.GetInt(cfg.GetNumShardDependenciesFlag()) + c.Indices.Dependencies.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasDependenciesFlag()) + c.Indices.Sampling.TemplateNumShards = v.GetInt(cfg.GetNumShardSamplingFlag()) + c.Indices.Sampling.TemplateNumReplicas = v.GetInt(cfg.GetNumShardSamplingFlag()) + c.Indices.Spans.TemplatePriority = v.GetInt(cfg.GetPrioritySpanTemplate()) + c.Indices.Services.TemplatePriority = v.GetInt(cfg.GetPriorityServiceTemplate()) + c.Indices.Dependencies.TemplatePriority = v.GetInt(cfg.GetPriorityDependenciesTemplate()) + c.Indices.Sampling.TemplatePriority = v.GetInt(cfg.GetPrioritySamplingTemplate()) } diff --git a/cmd/esmapping-generator/app/flags.go b/cmd/esmapping-generator/app/flags.go index 1201a5345d0..e61c341945b 100644 --- a/cmd/esmapping-generator/app/flags.go +++ b/cmd/esmapping-generator/app/flags.go @@ -15,8 +15,9 @@ package app import ( - "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/spf13/cobra" + + "github.com/jaegertracing/jaeger/pkg/es/config" ) // Options represent configurable parameters for jaeger-esmapping-generator @@ -30,19 +31,11 @@ type Options struct { } const ( - mappingFlag = "mapping" - esVersionFlag = "es-version" - spanShardsFlag = "shards-span" - serviceShardsFlag = "shards-service" - dependenciesShardsFlag = "shards-dependencies" - samplingShardsFlag = "shards-sampling" - spanReplicasFlag = "replicas-span" - serviceReplicasFlag = "replicas-service" - dependenciesReplicasFlag = "replicas-dependencies" - samplingReplicasFlag = "replicas-sampling" - indexPrefixFlag = "index-prefix" - useILMFlag = "use-ilm" - ilmPolicyNameFlag = "ilm-policy-name" + mappingFlag = "mapping" + esVersionFlag = "es-version" + indexPrefixFlag = "index-prefix" + useILMFlag = "use-ilm" + ilmPolicyNameFlag = "ilm-policy-name" ) // AddFlags adds flags for esmapping-generator main program @@ -58,43 +51,43 @@ func (o *Options) AddFlags(command *cobra.Command) { 7, "The major Elasticsearch version") command.Flags().IntVar( - &o.Indices.Spans.TemplateOptions.NumShards, - spanShardsFlag, + &o.Indices.Spans.TemplateNumShards, + config.GetNumShardSpanFlag(), 5, "The number of shards per span index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Spans.TemplateOptions.NumReplicas, - spanReplicasFlag, + &o.Indices.Spans.TemplateNumReplicas, + config.GetNumReplicasSpanFlag(), 1, "The number of replicas per index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Services.TemplateOptions.NumShards, - serviceShardsFlag, + &o.Indices.Services.TemplateNumShards, + config.GetNumShardServiceFlag(), 5, "The number of shards per service index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Services.TemplateOptions.NumReplicas, - serviceReplicasFlag, + &o.Indices.Services.TemplateNumReplicas, + config.GetNumReplicasServiceFlag(), 1, "The number of replicas per service index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Dependencies.TemplateOptions.NumShards, - dependenciesShardsFlag, + &o.Indices.Dependencies.TemplateNumShards, + config.GetNumShardDependenciesFlag(), 5, "The number of shards per dependencies index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Dependencies.TemplateOptions.NumReplicas, - dependenciesReplicasFlag, + &o.Indices.Dependencies.TemplateNumReplicas, + config.GetNumReplicasDependenciesFlag(), 1, "The number of replicas per dependencies index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Sampling.TemplateOptions.NumShards, - samplingShardsFlag, + &o.Indices.Sampling.TemplateNumShards, + config.GetNumShardSamplingFlag(), 5, "The number of shards per sampling index in Elasticsearch") command.Flags().IntVar( - &o.Indices.Sampling.TemplateOptions.NumReplicas, - samplingReplicasFlag, + &o.Indices.Sampling.TemplateNumReplicas, + config.GetNumReplicasSamplingFlag(), 1, "The number of replicas per sampling index in Elasticsearch") command.Flags().StringVar( diff --git a/cmd/esmapping-generator/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go index 75b2679e02a..fe5282e5c86 100644 --- a/cmd/esmapping-generator/app/renderer/render_test.go +++ b/cmd/esmapping-generator/app/renderer/render_test.go @@ -16,7 +16,6 @@ package renderer import ( "errors" - "github.com/jaegertracing/jaeger/pkg/es/config" "io" "testing" @@ -25,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" + "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/es/mocks" "github.com/jaegertracing/jaeger/pkg/testutils" ) diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml index a77f1f7797d..bcff0987ca9 100644 --- a/cmd/jaeger/config-elasticsearch.yaml +++ b/cmd/jaeger/config-elasticsearch.yaml @@ -21,15 +21,23 @@ extensions: spans: date_layout: "2006-01-02" rollover_frequency: "day" + template_num_shards: 5 + template_num_replicas: 1 services: date_layout: "2006-01-02" rollover_frequency: "day" + template_num_shards: 5 + template_num_replicas: 1 dependencies: date_layout: "2006-01-02" rollover_frequency: "day" + template_num_shards: 5 + template_num_replicas: 1 sampling: date_layout: "2006-01-02" rollover_frequency: "day" + template_num_shards: 5 + template_num_replicas: 1 another_storage: elasticsearch: index_prefix: "jaeger-archive" diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 148ffa4ac11..194a537c30e 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -52,9 +52,11 @@ type TemplateOptions struct { // IndexOptions describes the index format and rollover frequency type IndexOptions struct { - DateLayout string `mapstructure:"date_layout"` - RolloverFrequency string `mapstructure:"rollover_frequency"` // "hour" or "day" - TemplateOptions TemplateOptions `mapstructure:"template"` + DateLayout string `mapstructure:"date_layout"` + RolloverFrequency string `mapstructure:"rollover_frequency"` // "hour" or "day" + TemplatePriority int `mapstructure:"template_priority"` + TemplateNumShards int `mapstructure:"template_num_shards"` + TemplateNumReplicas int `mapstructure:"template_num_replicas"` } // Indices describes different configuration options for each index type @@ -230,17 +232,25 @@ func newElasticsearchV8(c *Configuration, logger *zap.Logger) (*esV8.Client, err return esV8.NewClient(options) } -func setFieldIfZero(cfg, source *IndexOptions) { - if cfg.TemplateOptions.NumShards == 0 { - cfg.TemplateOptions.NumShards = source.TemplateOptions.NumShards +func setDefaultIndexOptions(cfg, source *IndexOptions) { + if cfg.TemplateNumShards == 0 { + cfg.TemplateNumShards = source.TemplateNumShards } - if cfg.TemplateOptions.NumReplicas == 0 { - cfg.TemplateOptions.NumReplicas = source.TemplateOptions.NumReplicas + if cfg.TemplateNumReplicas == 0 { + cfg.TemplateNumReplicas = source.TemplateNumReplicas } - if cfg.TemplateOptions.Priority == 0 { - cfg.TemplateOptions.Priority = source.TemplateOptions.Priority + if cfg.TemplatePriority == 0 { + cfg.TemplatePriority = source.TemplatePriority + } + + if cfg.DateLayout == "" { + cfg.DateLayout = source.DateLayout + } + + if cfg.RolloverFrequency == "" { + cfg.RolloverFrequency = source.RolloverFrequency } } @@ -265,9 +275,9 @@ func (c *Configuration) ApplyDefaults(source *Configuration) { c.AdaptiveSamplingLookback = source.AdaptiveSamplingLookback } - setFieldIfZero(&c.Indices.Spans, &source.Indices.Spans) - setFieldIfZero(&c.Indices.Services, &source.Indices.Services) - setFieldIfZero(&c.Indices.Dependencies, &source.Indices.Dependencies) + setDefaultIndexOptions(&c.Indices.Spans, &source.Indices.Spans) + setDefaultIndexOptions(&c.Indices.Services, &source.Indices.Services) + setDefaultIndexOptions(&c.Indices.Dependencies, &source.Indices.Dependencies) if c.BulkSize == 0 { c.BulkSize = source.BulkSize diff --git a/pkg/es/config/flagutils.go b/pkg/es/config/flagutils.go new file mode 100644 index 00000000000..3b02ed3bc8e --- /dev/null +++ b/pkg/es/config/flagutils.go @@ -0,0 +1,90 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "fmt" + +type IndexType string + +const ( + SPAN IndexType = "span" + SERVICE IndexType = "service" + DEPENDENCIES IndexType = "dependencies" + SAMPLING IndexType = "sampling" + + numShards = "num-shards-" + numReplicas = "num-replicas-" + priority = "priority-%s-template" +) + +func GetNumShardSpanFlag() string { + return getNumShardPrefix(SPAN) +} + +func GetNumShardServiceFlag() string { + return getNumShardPrefix(SERVICE) +} + +func GetNumShardSamplingFlag() string { + return getNumShardPrefix(SAMPLING) +} + +func GetNumShardDependenciesFlag() string { + return getNumShardPrefix(DEPENDENCIES) +} + +func GetNumReplicasSpanFlag() string { + return getNumReplicasPrefix(SPAN) +} + +func GetNumReplicasServiceFlag() string { + return getNumReplicasPrefix(SERVICE) +} + +func GetNumReplicasDependenciesFlag() string { + return getNumReplicasPrefix(DEPENDENCIES) +} + +func GetNumReplicasSamplingFlag() string { + return getNumReplicasPrefix(SAMPLING) +} + +func getNumShardPrefix(indexType IndexType) string { + return string(numShards + indexType) +} + +func getNumReplicasPrefix(indexType IndexType) string { + return string(numReplicas + indexType) +} + +func GetPrioritySpanTemplate() string { + return getPriorityTemplate(SPAN) +} + +func GetPriorityServiceTemplate() string { + return getPriorityTemplate(SERVICE) +} + +func GetPriorityDependenciesTemplate() string { + return getPriorityTemplate(DEPENDENCIES) +} + +func GetPrioritySamplingTemplate() string { + return getPriorityTemplate(SAMPLING) +} + +func getPriorityTemplate(indexType IndexType) string { + return fmt.Sprintf(priority, indexType) +} diff --git a/pkg/es/config/flagutils_test.go b/pkg/es/config/flagutils_test.go new file mode 100644 index 00000000000..eb52a2c0e53 --- /dev/null +++ b/pkg/es/config/flagutils_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2019 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetFlags(t *testing.T) { + testCase := []struct { + name string + indexType IndexType + expectedNumShardsFlag string + expectedNumReplicasFlag string + expectedPriorityFlag string + }{ + { + name: "span-options", + indexType: SPAN, + expectedNumShardsFlag: "num-shards-span", + expectedNumReplicasFlag: "num-replicas-span", + expectedPriorityFlag: "priority-span-template", + }, + { + name: "service-options", + indexType: SERVICE, + expectedNumShardsFlag: "num-shards-service", + expectedNumReplicasFlag: "num-replicas-service", + expectedPriorityFlag: "priority-service-template", + }, + } + + for _, tt := range testCase { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedNumShardsFlag, getNumShardPrefix(tt.indexType)) + assert.Equal(t, tt.expectedNumReplicasFlag, getNumReplicasPrefix(tt.indexType)) + assert.Equal(t, tt.expectedPriorityFlag, getPriorityTemplate(tt.indexType)) + }) + } +} diff --git a/plugin/storage/es/mappings/jaeger-dependencies-6.json b/plugin/storage/es/mappings/jaeger-dependencies-6.json index 325a4b4e42b..ce47b95e257 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-6.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": {{ .Indices.Dependencies.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Dependencies.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Dependencies.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Dependencies.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-dependencies-7.json b/plugin/storage/es/mappings/jaeger-dependencies-7.json index 718bd91fe66..032e5f1432e 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-7.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Indices.Dependencies.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Dependencies.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Dependencies.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Dependencies.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-dependencies-8.json b/plugin/storage/es/mappings/jaeger-dependencies-8.json index 0bf92280cd5..de6109a79de 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-8.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-8.json @@ -1,5 +1,5 @@ { - "priority": {{ .Indices.Dependencies.TemplateOptions.Priority }}, + "priority": {{ .Indices.Dependencies.TemplatePriority }}, "index_patterns": "{{ .IndexPrefix }}jaeger-dependencies-*", "template": { {{- if .UseILM }} @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Indices.Dependencies.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Dependencies.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Dependencies.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Dependencies.TemplateNumReplicas }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-sampling-6.json b/plugin/storage/es/mappings/jaeger-sampling-6.json index 2f56428b726..4da6696801f 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-6.json +++ b/plugin/storage/es/mappings/jaeger-sampling-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-sampling-*", "settings":{ - "index.number_of_shards": {{ .Indices.Sampling.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Sampling.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Sampling.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Sampling.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":false }, diff --git a/plugin/storage/es/mappings/jaeger-sampling-7.json b/plugin/storage/es/mappings/jaeger-sampling-7.json index 1cad81f3dd9..79ca60fe237 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-7.json +++ b/plugin/storage/es/mappings/jaeger-sampling-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Indices.Sampling.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Sampling.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Sampling.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Sampling.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":false {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-sampling-8.json b/plugin/storage/es/mappings/jaeger-sampling-8.json index de5ceee1f08..c18b348a8d3 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-8.json +++ b/plugin/storage/es/mappings/jaeger-sampling-8.json @@ -1,5 +1,5 @@ { - "priority": {{ .Indices.Sampling.TemplateOptions.Priority }}, + "priority": {{ .Indices.Sampling.TemplatePriority }}, "index_patterns": "{{ .IndexPrefix }}jaeger-sampling-*", "template": { {{- if .UseILM }} @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Indices.Sampling.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Sampling.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Sampling.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Sampling.TemplateNumReplicas }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": false {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-service-6.json b/plugin/storage/es/mappings/jaeger-service-6.json index 06fab4e81ef..5c29a51f286 100644 --- a/plugin/storage/es/mappings/jaeger-service-6.json +++ b/plugin/storage/es/mappings/jaeger-service-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-service-*", "settings":{ - "index.number_of_shards": {{ .Indices.Services.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Services.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Services.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Services.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json index 6b47145ec58..2910e1d9d67 100644 --- a/plugin/storage/es/mappings/jaeger-service-7.json +++ b/plugin/storage/es/mappings/jaeger-service-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Indices.Services.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Services.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Services.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Services.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-service-8.json b/plugin/storage/es/mappings/jaeger-service-8.json index 4816bbf393d..0b1e78c500f 100644 --- a/plugin/storage/es/mappings/jaeger-service-8.json +++ b/plugin/storage/es/mappings/jaeger-service-8.json @@ -1,5 +1,5 @@ { - "priority": {{ .Indices.Services.TemplateOptions.Priority }}, + "priority": {{ .Indices.Services.TemplatePriority }}, "index_patterns": "{{ .IndexPrefix }}jaeger-service-*", "template": { {{- if .UseILM }} @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Indices.Services.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Services.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Services.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Services.TemplateNumReplicas }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-span-6.json b/plugin/storage/es/mappings/jaeger-span-6.json index c01702807fd..f544bdb0c17 100644 --- a/plugin/storage/es/mappings/jaeger-span-6.json +++ b/plugin/storage/es/mappings/jaeger-span-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-span-*", "settings":{ - "index.number_of_shards": {{ .Indices.Spans.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Spans.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Spans.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Spans.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index 0e4ad5cadf4..0dbd9942ad6 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Indices.Spans.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Spans.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Spans.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Spans.TemplateNumReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-span-8.json b/plugin/storage/es/mappings/jaeger-span-8.json index c92949c3c52..63156450a7a 100644 --- a/plugin/storage/es/mappings/jaeger-span-8.json +++ b/plugin/storage/es/mappings/jaeger-span-8.json @@ -1,5 +1,5 @@ { - "priority": {{ .Indices.Spans.TemplateOptions.Priority }}, + "priority": {{ .Indices.Spans.TemplatePriority }}, "index_patterns": "{{ .IndexPrefix }}jaeger-span-*", "template": { @@ -9,8 +9,8 @@ }, {{- end}} "settings": { - "index.number_of_shards": {{ .Indices.Spans.TemplateOptions.NumShards }}, - "index.number_of_replicas": {{ .Indices.Spans.TemplateOptions.NumReplicas }}, + "index.number_of_shards": {{ .Indices.Spans.TemplateNumShards }}, + "index.number_of_replicas": {{ .Indices.Spans.TemplateNumReplicas }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go index a968a47e533..d0b1909f17e 100644 --- a/plugin/storage/es/mappings/mapping.go +++ b/plugin/storage/es/mappings/mapping.go @@ -17,10 +17,10 @@ package mappings import ( "bytes" "embed" - "github.com/jaegertracing/jaeger/pkg/es/config" "strings" "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/pkg/es/config" ) // MAPPINGS contains embedded index templates. diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go index 0d829364f1f..93ef0099400 100644 --- a/plugin/storage/es/mappings/mapping_test.go +++ b/plugin/storage/es/mappings/mapping_test.go @@ -18,7 +18,6 @@ import ( "embed" "errors" "fmt" - "github.com/jaegertracing/jaeger/pkg/es/config" "io" "os" "testing" @@ -29,6 +28,7 @@ import ( "github.com/stretchr/testify/require" "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/es/mocks" "github.com/jaegertracing/jaeger/pkg/testutils" ) @@ -383,7 +383,6 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - mappingBuilder := MappingBuilder{ TemplateBuilder: test.mockNewTextTemplateBuilder(), Indices: config.Indices{ diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 11e3fc9e2a6..405b9bae5a5 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -29,54 +29,47 @@ import ( ) const ( - suffixUsername = ".username" - suffixPassword = ".password" - suffixSniffer = ".sniffer" - suffixSnifferTLSEnabled = ".sniffer-tls-enabled" - suffixTokenPath = ".token-file" - suffixPasswordPath = ".password-file" - suffixServerURLs = ".server-urls" - suffixRemoteReadClusters = ".remote-read-clusters" - suffixMaxSpanAge = ".max-span-age" - suffixAdaptiveSamplingLookback = ".adaptive-sampling.lookback" - suffixNumShards = ".num-shards" - suffixNumReplicas = ".num-replicas" - suffixNumShardsSpan = ".num-shards-spans" - suffixNumShardsServices = ".num-shards-services" - suffixNumShardsSampling = ".num-shards-sampling" - suffixNumShardsDependencies = ".num-shards-dependencies" - suffixNumReplicasSpan = ".num-replicas-spans" - suffixNumReplicasServices = ".num-replicas-services" - suffixNumReplicasSampling = ".num-replicas-sampling" - suffixNumReplicasDependencies = ".num-replicas-dependencies" - suffixPrioritySpanTemplate = ".prioirity-span-template" - suffixPriorityServiceTemplate = ".prioirity-service-template" - suffixPriorityDependenciesTemplate = ".prioirity-dependencies-template" - suffixBulkSize = ".bulk.size" - suffixBulkWorkers = ".bulk.workers" - suffixBulkActions = ".bulk.actions" - suffixBulkFlushInterval = ".bulk.flush-interval" - suffixTimeout = ".timeout" - suffixIndexPrefix = ".index-prefix" - suffixIndexDateSeparator = ".index-date-separator" - suffixIndexRolloverFrequencySpans = ".index-rollover-frequency-spans" - suffixIndexRolloverFrequencyServices = ".index-rollover-frequency-services" - suffixIndexRolloverFrequencySampling = ".index-rollover-frequency-adaptive-sampling" - suffixIndexRolloverFrequencyDependencies = ".index-rollover-frequency-adaptive-dependencies" - suffixServiceCacheTTL = ".service-cache-ttl" - suffixTagsAsFields = ".tags-as-fields" - suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" - suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" - suffixTagsFile = suffixTagsAsFields + ".config-file" - suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" - suffixReadAlias = ".use-aliases" - suffixUseILM = ".use-ilm" - suffixCreateIndexTemplate = ".create-index-templates" - suffixEnabled = ".enabled" - suffixVersion = ".version" - suffixMaxDocCount = ".max-doc-count" - suffixLogLevel = ".log-level" - suffixSendGetBodyAs = ".send-get-body-as" + suffix = "." + username = "username" + password = "password" + sniffer = "sniffer" + snifferTLSEnabled = "sniffer-tls-enabled" + tokenPath = "token-file" + passwordPath = "password-file" + serverURLs = "server-urls" + remoteReadClusters = "remote-read-clusters" + maxSpanAge = "max-span-age" + adaptiveSamplingLookback = "adaptive-sampling.lookback" + numShards = "num-shards" + numReplicas = "num-replicas" + prioritySpanTemplate = "prioirity-span-template" + priorityServiceTemplate = "prioirity-service-template" + priorityDependenciesTemplate = "prioirity-dependencies-template" + bulkSize = "bulk.size" + bulkWorkers = "bulk.workers" + bulkActions = "bulk.actions" + bulkFlushInterval = "bulk.flush-interval" + timeout = "timeout" + indexPrefix = "index-prefix" + indexDateSeparator = "index-date-separator" + indexRolloverFrequencySpans = "index-rollover-frequency-spans" + indexRolloverFrequencyServices = "index-rollover-frequency-services" + indexRolloverFrequencySampling = "index-rollover-frequency-adaptive-sampling" + indexRolloverFrequencyDependencies = "index-rollover-frequency-adaptive-dependencies" + serviceCacheTTL = "service-cache-ttl" + tagsAsFields = "tags-as-fields" + tagsAsFieldsAll = tagsAsFields + ".all" + tagsAsFieldsInclude = tagsAsFields + ".include" + tagsFile = tagsAsFields + ".config-file" + tagDeDotChar = tagsAsFields + ".dot-replacement" + readAlias = "use-aliases" + useILM = "use-ilm" + createIndexTemplate = "create-index-templates" + enabled = "enabled" + version = "version" + maxDocCount = "max-doc-count" + logLevel = "log-level" + sendGetBodyAs = "send-get-body-as" // default number of documents to return from a query (elasticsearch allowed limit) // see search.max_buckets and index.max_result_window defaultMaxDocCount = 10_000 @@ -89,17 +82,13 @@ const ( defaultSendGetBodyAs = "" ) -var ( - defaultIndexOptions = config.IndexOptions{ - DateLayout: initDateLayout(defaultIndexRolloverFrequency, defaultIndexDateSeparator), - RolloverFrequency: defaultIndexRolloverFrequency, - TemplateOptions: config.TemplateOptions{ - Priority: 0, - NumShards: 5, - NumReplicas: 1, - }, - } -) +var defaultIndexOptions = config.IndexOptions{ + DateLayout: initDateLayout(defaultIndexRolloverFrequency, defaultIndexDateSeparator), + RolloverFrequency: defaultIndexRolloverFrequency, + TemplateNumShards: 5, + TemplateNumReplicas: 1, + TemplatePriority: 0, +} // TODO this should be moved next to config.Configuration struct (maybe ./flags package) @@ -155,208 +144,216 @@ func (opt *Options) AddFlags(flagSet *flag.FlagSet) { } } +func flagWithSuffix(namespace, flag string) string { + if namespace != "" { + return namespace + suffix + flag + } else { + return suffix + flag + } +} + func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { flagSet.String( - nsConfig.namespace+suffixUsername, + flagWithSuffix(nsConfig.namespace, username), nsConfig.Username, "The username required by Elasticsearch. The basic authentication also loads CA if it is specified.") flagSet.String( - nsConfig.namespace+suffixPassword, + flagWithSuffix(nsConfig.namespace, password), nsConfig.Password, "The password required by Elasticsearch") flagSet.String( - nsConfig.namespace+suffixTokenPath, + flagWithSuffix(nsConfig.namespace, tokenPath), nsConfig.TokenFilePath, "Path to a file containing bearer token. This flag also loads CA if it is specified.") flagSet.String( - nsConfig.namespace+suffixPasswordPath, + flagWithSuffix(nsConfig.namespace, passwordPath), nsConfig.PasswordFilePath, "Path to a file containing password. This file is watched for changes.") flagSet.Bool( - nsConfig.namespace+suffixSniffer, + flagWithSuffix(nsConfig.namespace, sniffer), nsConfig.Sniffer, "The sniffer config for Elasticsearch; client uses sniffing process to find all nodes automatically, disable if not required") flagSet.String( - nsConfig.namespace+suffixServerURLs, + flagWithSuffix(nsConfig.namespace, serverURLs), defaultServerURL, "The comma-separated list of Elasticsearch servers, must be full url i.e. http://localhost:9200") flagSet.String( - nsConfig.namespace+suffixRemoteReadClusters, + flagWithSuffix(nsConfig.namespace, remoteReadClusters), defaultRemoteReadClusters, "Comma-separated list of Elasticsearch remote cluster names for cross-cluster querying."+ "See Elasticsearch remote clusters and cross-cluster query api.") flagSet.Duration( - nsConfig.namespace+suffixTimeout, + flagWithSuffix(nsConfig.namespace, timeout), nsConfig.Timeout, "Timeout used for queries. A Timeout of zero means no timeout") // TODO deprecated flag to be removed flagSet.Int( - nsConfig.namespace+suffixNumShards, - defaultIndexOptions.TemplateOptions.NumShards, + flagWithSuffix(nsConfig.namespace, numShards), + defaultIndexOptions.TemplateNumShards, "(deprecated, will be removed in the future, use .num-shards-spans or .num-shards-services or .num-shards-sampling or .num-shards-dependencies instead) The number of shards per index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumReplicas, - defaultIndexOptions.TemplateOptions.NumReplicas, + flagWithSuffix(nsConfig.namespace, numReplicas), + defaultIndexOptions.TemplateNumReplicas, "(deprecated, will be removed in the future, use .num-replicas-spans or .num-replicas-services or .num-replicas-sampling or .num-replicas-dependencies instead) The number of replicas per index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumShardsSpan, - nsConfig.Indices.Spans.TemplateOptions.NumShards, + flagWithSuffix(nsConfig.namespace, config.GetNumShardSpanFlag()), + nsConfig.Indices.Spans.TemplateNumShards, "The number of shards per span index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumShardsServices, - nsConfig.Indices.Services.TemplateOptions.NumShards, + flagWithSuffix(nsConfig.namespace, config.GetNumShardServiceFlag()), + nsConfig.Indices.Services.TemplateNumShards, "The number of shards per service index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumShardsSampling, - nsConfig.Indices.Sampling.TemplateOptions.NumShards, + flagWithSuffix(nsConfig.namespace, config.GetNumShardSamplingFlag()), + nsConfig.Indices.Sampling.TemplateNumShards, "The number of shards per sampling index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumShardsDependencies, - nsConfig.Indices.Dependencies.TemplateOptions.NumShards, + flagWithSuffix(nsConfig.namespace, config.GetNumShardDependenciesFlag()), + nsConfig.Indices.Dependencies.TemplateNumShards, "The number of shards per dependencies index in Elasticsearch") flagSet.Duration( - nsConfig.namespace+suffixServiceCacheTTL, + flagWithSuffix(nsConfig.namespace, serviceCacheTTL), nsConfig.ServiceCacheTTL, "The TTL for the cache of known service names", ) flagSet.Int( - nsConfig.namespace+suffixNumReplicasSpan, - nsConfig.Indices.Spans.TemplateOptions.NumReplicas, + flagWithSuffix(nsConfig.namespace, config.GetNumReplicasSpanFlag()), + nsConfig.Indices.Spans.TemplateNumReplicas, "The number of replicas per span index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumReplicasServices, - nsConfig.Indices.Services.TemplateOptions.NumReplicas, + flagWithSuffix(nsConfig.namespace, config.GetNumReplicasServiceFlag()), + nsConfig.Indices.Services.TemplateNumReplicas, "The number of replicas per service index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumReplicasSampling, - nsConfig.Indices.Sampling.TemplateOptions.NumReplicas, + flagWithSuffix(nsConfig.namespace, config.GetNumReplicasSamplingFlag()), + nsConfig.Indices.Sampling.TemplateNumReplicas, "The number of replicas per sampling index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixNumReplicasDependencies, - nsConfig.Indices.Dependencies.TemplateOptions.NumReplicas, + flagWithSuffix(nsConfig.namespace, config.GetNumReplicasDependenciesFlag()), + nsConfig.Indices.Dependencies.TemplateNumReplicas, "The number of replicas per dependencies index in Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixPrioritySpanTemplate, - nsConfig.Indices.Spans.TemplateOptions.Priority, + flagWithSuffix(nsConfig.namespace, prioritySpanTemplate), + nsConfig.Indices.Spans.TemplatePriority, "Priority of jaeger-span index template (ESv8 only)") flagSet.Int( - nsConfig.namespace+suffixPriorityServiceTemplate, - nsConfig.Indices.Services.TemplateOptions.Priority, + flagWithSuffix(nsConfig.namespace, priorityServiceTemplate), + nsConfig.Indices.Services.TemplatePriority, "Priority of jaeger-service index template (ESv8 only)") flagSet.Int( - nsConfig.namespace+suffixPriorityDependenciesTemplate, - nsConfig.Indices.Dependencies.TemplateOptions.Priority, + flagWithSuffix(nsConfig.namespace, priorityDependenciesTemplate), + nsConfig.Indices.Dependencies.TemplatePriority, "Priority of jaeger-dependecies index template (ESv8 only)") flagSet.Int( - nsConfig.namespace+suffixBulkSize, + flagWithSuffix(nsConfig.namespace, bulkSize), nsConfig.BulkSize, "The number of bytes that the bulk requests can take up before the bulk processor decides to commit") flagSet.Int( - nsConfig.namespace+suffixBulkWorkers, + flagWithSuffix(nsConfig.namespace, bulkWorkers), nsConfig.BulkWorkers, "The number of workers that are able to receive bulk requests and eventually commit them to Elasticsearch") flagSet.Int( - nsConfig.namespace+suffixBulkActions, + flagWithSuffix(nsConfig.namespace, bulkActions), nsConfig.BulkActions, "The number of requests that can be enqueued before the bulk processor decides to commit") flagSet.Duration( - nsConfig.namespace+suffixBulkFlushInterval, + flagWithSuffix(nsConfig.namespace, bulkFlushInterval), nsConfig.BulkFlushInterval, "A time.Duration after which bulk requests are committed, regardless of other thresholds. Set to zero to disable. By default, this is disabled.") flagSet.String( - nsConfig.namespace+suffixIndexPrefix, + flagWithSuffix(nsConfig.namespace, indexPrefix), nsConfig.IndexPrefix, "Optional prefix of Jaeger indices. For example \"production\" creates \"production-jaeger-*\".") flagSet.String( - nsConfig.namespace+suffixIndexDateSeparator, + flagWithSuffix(nsConfig.namespace, indexDateSeparator), defaultIndexDateSeparator, "Optional date separator of Jaeger indices. For example \".\" creates \"jaeger-span-2020.11.20\".") flagSet.String( - nsConfig.namespace+suffixIndexRolloverFrequencySpans, + flagWithSuffix(nsConfig.namespace, indexRolloverFrequencySpans), defaultIndexRolloverFrequency, "Rotates jaeger-span indices over the given period. For example \"day\" creates \"jaeger-span-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") flagSet.String( - nsConfig.namespace+suffixIndexRolloverFrequencyServices, + flagWithSuffix(nsConfig.namespace, indexRolloverFrequencyServices), defaultIndexRolloverFrequency, "Rotates jaeger-service indices over the given period. For example \"day\" creates \"jaeger-service-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") flagSet.String( - nsConfig.namespace+suffixIndexRolloverFrequencySampling, + flagWithSuffix(nsConfig.namespace, indexRolloverFrequencySampling), defaultIndexRolloverFrequency, "Rotates jaeger-sampling indices over the given period. For example \"day\" creates \"jaeger-sampling-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") flagSet.String( - nsConfig.namespace+suffixIndexRolloverFrequencyDependencies, + flagWithSuffix(nsConfig.namespace, indexRolloverFrequencyDependencies), defaultIndexRolloverFrequency, "Rotates jaeger-dependencies indices over the given period. For example \"day\" creates \"jaeger-dependencies-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") flagSet.Bool( - nsConfig.namespace+suffixTagsAsFieldsAll, + flagWithSuffix(nsConfig.namespace, tagsAsFieldsAll), nsConfig.Tags.AllAsFields, - "(experimental) Store all span and process tags as object fields. If true "+suffixTagsFile+" and "+suffixTagsAsFieldsInclude+" is ignored. Binary tags are always stored as nested objects.") + "(experimental) Store all span and process tags as object fields. If true "+tagsFile+" and "+tagsAsFieldsInclude+" is ignored. Binary tags are always stored as nested objects.") flagSet.String( - nsConfig.namespace+suffixTagsAsFieldsInclude, + flagWithSuffix(nsConfig.namespace, tagsAsFieldsInclude), nsConfig.Tags.Include, - "(experimental) Comma delimited list of tag keys which will be stored as object fields. Merged with the contents of "+suffixTagsFile) + "(experimental) Comma delimited list of tag keys which will be stored as object fields. Merged with the contents of "+tagsFile) flagSet.String( - nsConfig.namespace+suffixTagsFile, + flagWithSuffix(nsConfig.namespace, tagsFile), nsConfig.Tags.File, - "(experimental) Optional path to a file containing tag keys which will be stored as object fields. Each key should be on a separate line. Merged with "+suffixTagsAsFieldsInclude) + "(experimental) Optional path to a file containing tag keys which will be stored as object fields. Each key should be on a separate line. Merged with "+tagsAsFieldsInclude) flagSet.String( - nsConfig.namespace+suffixTagDeDotChar, + flagWithSuffix(nsConfig.namespace, tagDeDotChar), nsConfig.Tags.DotReplacement, "(experimental) The character used to replace dots (\".\") in tag keys stored as object fields.") flagSet.Bool( - nsConfig.namespace+suffixReadAlias, + flagWithSuffix(nsConfig.namespace, readAlias), nsConfig.UseReadWriteAliases, "Use read and write aliases for indices. Use this option with Elasticsearch rollover "+ "API. It requires an external component to create aliases before startup and then performing its management. "+ - "Note that es"+suffixMaxSpanAge+" will influence trace search window start times.") + "Note that es"+maxSpanAge+" will influence trace search window start times.") flagSet.Bool( - nsConfig.namespace+suffixUseILM, + flagWithSuffix(nsConfig.namespace, useILM), nsConfig.UseILM, - "(experimental) Option to enable ILM for jaeger span & service indices. Use this option with "+nsConfig.namespace+suffixReadAlias+". "+ + "(experimental) Option to enable ILM for jaeger span & service indices. Use this option with "+nsConfig.namespace+readAlias+". "+ "It requires an external component to create aliases before startup and then performing its management. "+ "ILM policy must be manually created in ES before startup. Supported only for elasticsearch version 7+.") flagSet.Bool( - nsConfig.namespace+suffixCreateIndexTemplate, + flagWithSuffix(nsConfig.namespace, createIndexTemplate), nsConfig.CreateIndexTemplates, "Create index templates at application startup. Set to false when templates are installed manually.") flagSet.Uint( - nsConfig.namespace+suffixVersion, + flagWithSuffix(nsConfig.namespace, version), 0, "The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.") flagSet.Bool( - nsConfig.namespace+suffixSnifferTLSEnabled, + flagWithSuffix(nsConfig.namespace, snifferTLSEnabled), nsConfig.SnifferTLSEnabled, "Option to enable TLS when sniffing an Elasticsearch Cluster ; client uses sniffing process to find all nodes automatically, disabled by default") flagSet.Int( - nsConfig.namespace+suffixMaxDocCount, + flagWithSuffix(nsConfig.namespace, maxDocCount), nsConfig.MaxDocCount, "The maximum document count to return from an Elasticsearch query. This will also apply to aggregations.") flagSet.String( - nsConfig.namespace+suffixLogLevel, + flagWithSuffix(nsConfig.namespace, logLevel), nsConfig.LogLevel, "The Elasticsearch client log-level. Valid levels: [debug, info, error]") flagSet.String( - nsConfig.namespace+suffixSendGetBodyAs, + flagWithSuffix(nsConfig.namespace, sendGetBodyAs), nsConfig.SendGetBodyAs, "HTTP verb for requests that contain a body [GET, POST].") flagSet.Duration( - nsConfig.namespace+suffixAdaptiveSamplingLookback, + flagWithSuffix(nsConfig.namespace, adaptiveSamplingLookback), nsConfig.AdaptiveSamplingLookback, "How far back to look for the latest adaptive sampling probabilities") if nsConfig.namespace == archiveNamespace { flagSet.Bool( - nsConfig.namespace+suffixEnabled, + flagWithSuffix(nsConfig.namespace, enabled), nsConfig.Enabled, "Enable extra storage") } else { // MaxSpanAge is only relevant when searching for unarchived traces. // Archived traces are searched with no look-back limit. flagSet.Duration( - nsConfig.namespace+suffixMaxSpanAge, + flagWithSuffix(nsConfig.namespace, maxSpanAge), nsConfig.MaxSpanAge, "The maximum lookback for spans in Elasticsearch") } @@ -379,66 +376,67 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { return newNum } } - cfg.Username = v.GetString(cfg.namespace + suffixUsername) - cfg.Password = v.GetString(cfg.namespace + suffixPassword) - cfg.TokenFilePath = v.GetString(cfg.namespace + suffixTokenPath) - cfg.PasswordFilePath = v.GetString(cfg.namespace + suffixPasswordPath) - cfg.Sniffer = v.GetBool(cfg.namespace + suffixSniffer) - cfg.SnifferTLSEnabled = v.GetBool(cfg.namespace + suffixSnifferTLSEnabled) - cfg.Servers = strings.Split(stripWhiteSpace(v.GetString(cfg.namespace+suffixServerURLs)), ",") - cfg.MaxSpanAge = v.GetDuration(cfg.namespace + suffixMaxSpanAge) - cfg.AdaptiveSamplingLookback = v.GetDuration(cfg.namespace + suffixAdaptiveSamplingLookback) - - deprecatedNumShards := v.GetInt(cfg.namespace + suffixNumShards) - deprecatedReplicaShards := v.GetInt(cfg.namespace + suffixNumReplicas) - - cfg.Indices.Spans.TemplateOptions.NumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(cfg.namespace+suffixNumShardsSpan)) - cfg.Indices.Services.TemplateOptions.NumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(cfg.namespace+suffixNumShardsServices)) - cfg.Indices.Sampling.TemplateOptions.NumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(cfg.namespace+suffixNumShardsSampling)) - cfg.Indices.Dependencies.TemplateOptions.NumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(cfg.namespace+suffixNumShardsDependencies)) - cfg.Indices.Spans.TemplateOptions.NumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(cfg.namespace+suffixNumReplicasSpan)) - cfg.Indices.Services.TemplateOptions.NumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(cfg.namespace+suffixNumReplicasServices)) - cfg.Indices.Sampling.TemplateOptions.NumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(cfg.namespace+suffixNumReplicasSampling)) - cfg.Indices.Dependencies.TemplateOptions.NumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(cfg.namespace+suffixNumReplicasDependencies)) - - cfg.Indices.Spans.TemplateOptions.Priority = v.GetInt(cfg.namespace + suffixPrioritySpanTemplate) - cfg.Indices.Services.TemplateOptions.Priority = v.GetInt(cfg.namespace + suffixPriorityServiceTemplate) - cfg.Indices.Dependencies.TemplateOptions.Priority = v.GetInt(cfg.namespace + suffixPriorityDependenciesTemplate) - - cfg.BulkSize = v.GetInt(cfg.namespace + suffixBulkSize) - cfg.BulkWorkers = v.GetInt(cfg.namespace + suffixBulkWorkers) - cfg.BulkActions = v.GetInt(cfg.namespace + suffixBulkActions) - cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval) - cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout) - cfg.ServiceCacheTTL = v.GetDuration(cfg.namespace + suffixServiceCacheTTL) - cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix) - cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll) - cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude) - cfg.Tags.File = v.GetString(cfg.namespace + suffixTagsFile) - cfg.Tags.DotReplacement = v.GetString(cfg.namespace + suffixTagDeDotChar) - cfg.UseReadWriteAliases = v.GetBool(cfg.namespace + suffixReadAlias) - cfg.Enabled = v.GetBool(cfg.namespace + suffixEnabled) - cfg.CreateIndexTemplates = v.GetBool(cfg.namespace + suffixCreateIndexTemplate) - cfg.Version = uint(v.GetInt(cfg.namespace + suffixVersion)) - cfg.LogLevel = v.GetString(cfg.namespace + suffixLogLevel) - cfg.SendGetBodyAs = v.GetString(cfg.namespace + suffixSendGetBodyAs) - - cfg.MaxDocCount = v.GetInt(cfg.namespace + suffixMaxDocCount) - cfg.UseILM = v.GetBool(cfg.namespace + suffixUseILM) + cfg.Username = v.GetString(flagWithSuffix(cfg.namespace, username)) + cfg.Password = v.GetString(flagWithSuffix(cfg.namespace, password)) + cfg.TokenFilePath = v.GetString(flagWithSuffix(cfg.namespace, tokenPath)) + cfg.PasswordFilePath = v.GetString(flagWithSuffix(cfg.namespace, passwordPath)) + cfg.Sniffer = v.GetBool(flagWithSuffix(cfg.namespace, sniffer)) + cfg.SnifferTLSEnabled = v.GetBool(flagWithSuffix(cfg.namespace, snifferTLSEnabled)) + cfg.Servers = strings.Split(stripWhiteSpace(v.GetString(flagWithSuffix(cfg.namespace, serverURLs))), ",") + cfg.MaxSpanAge = v.GetDuration(flagWithSuffix(cfg.namespace, maxSpanAge)) + cfg.AdaptiveSamplingLookback = v.GetDuration(flagWithSuffix(cfg.namespace, adaptiveSamplingLookback)) + + deprecatedNumShards := v.GetInt(flagWithSuffix(cfg.namespace, numShards)) + deprecatedReplicaShards := v.GetInt(flagWithSuffix(cfg.namespace, numReplicas)) + + cfg.Indices.Spans.TemplateNumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumShardSpanFlag()))) + cfg.Indices.Services.TemplateNumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumShardServiceFlag()))) + cfg.Indices.Sampling.TemplateNumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumShardSamplingFlag()))) + cfg.Indices.Dependencies.TemplateNumShards = overrideIndexShardsNums(deprecatedNumShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumShardDependenciesFlag()))) + + cfg.Indices.Spans.TemplateNumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumReplicasSpanFlag()))) + cfg.Indices.Services.TemplateNumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumReplicasServiceFlag()))) + cfg.Indices.Sampling.TemplateNumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumReplicasSamplingFlag()))) + cfg.Indices.Dependencies.TemplateNumReplicas = overrideIndexShardsNums(deprecatedReplicaShards, v.GetInt(flagWithSuffix(cfg.namespace, config.GetNumReplicasDependenciesFlag()))) + + cfg.Indices.Spans.TemplatePriority = v.GetInt(flagWithSuffix(cfg.namespace, prioritySpanTemplate)) + cfg.Indices.Services.TemplatePriority = v.GetInt(flagWithSuffix(cfg.namespace, priorityServiceTemplate)) + cfg.Indices.Dependencies.TemplatePriority = v.GetInt(flagWithSuffix(cfg.namespace, priorityDependenciesTemplate)) + + cfg.BulkSize = v.GetInt(flagWithSuffix(cfg.namespace, bulkSize)) + cfg.BulkWorkers = v.GetInt(flagWithSuffix(cfg.namespace, bulkWorkers)) + cfg.BulkActions = v.GetInt(flagWithSuffix(cfg.namespace, bulkActions)) + cfg.BulkFlushInterval = v.GetDuration(flagWithSuffix(cfg.namespace, bulkFlushInterval)) + cfg.Timeout = v.GetDuration(flagWithSuffix(cfg.namespace, timeout)) + cfg.ServiceCacheTTL = v.GetDuration(flagWithSuffix(cfg.namespace, serviceCacheTTL)) + cfg.IndexPrefix = v.GetString(flagWithSuffix(cfg.namespace, indexPrefix)) + cfg.Tags.AllAsFields = v.GetBool(flagWithSuffix(cfg.namespace, tagsAsFieldsAll)) + cfg.Tags.Include = v.GetString(flagWithSuffix(cfg.namespace, tagsAsFieldsInclude)) + cfg.Tags.File = v.GetString(flagWithSuffix(cfg.namespace, tagsFile)) + cfg.Tags.DotReplacement = v.GetString(flagWithSuffix(cfg.namespace, tagDeDotChar)) + cfg.UseReadWriteAliases = v.GetBool(flagWithSuffix(cfg.namespace, readAlias)) + cfg.Enabled = v.GetBool(flagWithSuffix(cfg.namespace, enabled)) + cfg.CreateIndexTemplates = v.GetBool(flagWithSuffix(cfg.namespace, createIndexTemplate)) + cfg.Version = uint(v.GetInt(flagWithSuffix(cfg.namespace, version))) + cfg.LogLevel = v.GetString(flagWithSuffix(cfg.namespace, logLevel)) + cfg.SendGetBodyAs = v.GetString(flagWithSuffix(cfg.namespace, sendGetBodyAs)) + + cfg.MaxDocCount = v.GetInt(flagWithSuffix(cfg.namespace, maxDocCount)) + cfg.UseILM = v.GetBool(flagWithSuffix(cfg.namespace, useILM)) // TODO: Need to figure out a better way for do this. cfg.AllowTokenFromContext = v.GetBool(bearertoken.StoragePropagationKey) - remoteReadClusters := stripWhiteSpace(v.GetString(cfg.namespace + suffixRemoteReadClusters)) - if len(remoteReadClusters) > 0 { - cfg.RemoteReadClusters = strings.Split(remoteReadClusters, ",") + rReadClusters := stripWhiteSpace(v.GetString(flagWithSuffix(cfg.namespace, remoteReadClusters))) + if len(rReadClusters) > 0 { + cfg.RemoteReadClusters = strings.Split(rReadClusters, ",") } - cfg.Indices.Spans.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySpans)) - cfg.Indices.Services.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencyServices)) - cfg.Indices.Sampling.RolloverFrequency = strings.ToLower(v.GetString(cfg.namespace + suffixIndexRolloverFrequencySampling)) + cfg.Indices.Spans.RolloverFrequency = strings.ToLower(v.GetString(flagWithSuffix(cfg.namespace, indexRolloverFrequencySpans))) + cfg.Indices.Services.RolloverFrequency = strings.ToLower(v.GetString(flagWithSuffix(cfg.namespace, indexRolloverFrequencyServices))) + cfg.Indices.Sampling.RolloverFrequency = strings.ToLower(v.GetString(flagWithSuffix(cfg.namespace, indexRolloverFrequencySampling))) - separator := v.GetString(cfg.namespace + suffixIndexDateSeparator) + separator := v.GetString(flagWithSuffix(cfg.namespace, indexDateSeparator)) cfg.Indices.Spans.DateLayout = initDateLayout(cfg.Indices.Spans.RolloverFrequency, separator) cfg.Indices.Services.DateLayout = initDateLayout(cfg.Indices.Services.RolloverFrequency, separator) cfg.Indices.Sampling.DateLayout = initDateLayout(cfg.Indices.Sampling.RolloverFrequency, separator) diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index a64584a5730..501a3909bf7 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -33,14 +33,14 @@ func TestOptions(t *testing.T) { assert.Empty(t, primary.PasswordFilePath) assert.NotEmpty(t, primary.Servers) assert.Empty(t, primary.RemoteReadClusters) - assert.Equal(t, 5, primary.Indices.Spans.TemplateOptions.NumShards) - assert.Equal(t, 5, primary.Indices.Services.TemplateOptions.NumShards) - assert.Equal(t, 5, primary.Indices.Sampling.TemplateOptions.NumShards) - assert.Equal(t, 5, primary.Indices.Dependencies.TemplateOptions.NumShards) - assert.Equal(t, 1, primary.Indices.Spans.TemplateOptions.NumReplicas) - assert.Equal(t, 1, primary.Indices.Services.TemplateOptions.NumReplicas) - assert.Equal(t, 1, primary.Indices.Sampling.TemplateOptions.NumReplicas) - assert.Equal(t, 1, primary.Indices.Dependencies.TemplateOptions.NumReplicas) + assert.Equal(t, 5, primary.Indices.Spans.TemplateNumShards) + assert.Equal(t, 5, primary.Indices.Services.TemplateNumShards) + assert.Equal(t, 5, primary.Indices.Sampling.TemplateNumShards) + assert.Equal(t, 5, primary.Indices.Dependencies.TemplateNumShards) + assert.Equal(t, 1, primary.Indices.Spans.TemplateNumReplicas) + assert.Equal(t, 1, primary.Indices.Services.TemplateNumReplicas) + assert.Equal(t, 1, primary.Indices.Sampling.TemplateNumReplicas) + assert.Equal(t, 1, primary.Indices.Dependencies.TemplateNumReplicas) assert.Equal(t, 72*time.Hour, primary.MaxSpanAge) assert.False(t, primary.Sniffer) assert.False(t, primary.SnifferTLSEnabled) @@ -110,14 +110,14 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, []string{"3.3.3.3", "4.4.4.4"}, aux.Servers) assert.Equal(t, "hello", aux.Username) assert.Equal(t, "world", aux.Password) - assert.Equal(t, 5, aux.Indices.Spans.TemplateOptions.NumShards) - assert.Equal(t, 5, aux.Indices.Services.TemplateOptions.NumShards) - assert.Equal(t, 5, aux.Indices.Sampling.TemplateOptions.NumShards) - assert.Equal(t, 5, aux.Indices.Dependencies.TemplateOptions.NumShards) - assert.Equal(t, 10, aux.Indices.Spans.TemplateOptions.NumReplicas) - assert.Equal(t, 10, aux.Indices.Services.TemplateOptions.NumReplicas) - assert.Equal(t, 10, aux.Indices.Sampling.TemplateOptions.NumReplicas) - assert.Equal(t, 10, aux.Indices.Dependencies.TemplateOptions.NumReplicas) + assert.Equal(t, 5, aux.Indices.Spans.TemplateNumShards) + assert.Equal(t, 5, aux.Indices.Services.TemplateNumShards) + assert.Equal(t, 5, aux.Indices.Sampling.TemplateNumShards) + assert.Equal(t, 5, aux.Indices.Dependencies.TemplateNumShards) + assert.Equal(t, 10, aux.Indices.Spans.TemplateNumReplicas) + assert.Equal(t, 10, aux.Indices.Services.TemplateNumReplicas) + assert.Equal(t, 10, aux.Indices.Sampling.TemplateNumReplicas) + assert.Equal(t, 10, aux.Indices.Dependencies.TemplateNumReplicas) assert.Equal(t, 24*time.Hour, aux.MaxSpanAge) assert.True(t, aux.Sniffer) assert.True(t, aux.Tags.AllAsFields) diff --git a/plugin/storage/es/spanstore/reader_test.go b/plugin/storage/es/spanstore/reader_test.go index ca4db9fa203..6a86d45d90f 100644 --- a/plugin/storage/es/spanstore/reader_test.go +++ b/plugin/storage/es/spanstore/reader_test.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/jaegertracing/jaeger/pkg/es/config" "os" "reflect" "testing" @@ -38,6 +37,7 @@ import ( "github.com/jaegertracing/jaeger/internal/metricstest" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/es/mocks" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel"