diff --git a/Makefile b/Makefile index 5ceb601fa0fc..bcf46a4eee9c 100644 --- a/Makefile +++ b/Makefile @@ -225,13 +225,13 @@ build-tracegen: build-anonymizer: $(GOBUILD) -o ./cmd/anonymizer/anonymizer-$(GOOS)-$(GOARCH) ./cmd/anonymizer/main.go -.PHONY: build-templatizer -build-templatizer: - $(GOBUILD) -o ./plugin/storage/es/templatizer-$(GOOS)-$(GOARCH) ./cmd/templatizer/main.go +.PHONY: build-esmapping-generator +build-esmapping-generator: + $(GOBUILD) -o ./plugin/storage/es/esmapping-generator-$(GOOS)-$(GOARCH) ./cmd/esmapping-generator/main.go -.PHONY: build-templatizer-linux -build-templatizer-linux: - GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/templatizer ./cmd/templatizer/main.go +.PHONY: build-esmapping-generator-linux +build-esmapping-generator-linux: + GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go .PHONY: docker-hotrod docker-hotrod: @@ -352,7 +352,7 @@ docker-images-cassandra: @echo "Finished building jaeger-cassandra-schema ==============" .PHONY: docker-images-elastic -docker-images-elastic: build-templatizer-linux +docker-images-elastic: build-esmapping-generator-linux docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover @echo "Finished building jaeger-es-indices-clean ==============" diff --git a/cmd/templatizer/app/flags.go b/cmd/esmapping-generator/app/flags.go similarity index 79% rename from cmd/templatizer/app/flags.go rename to cmd/esmapping-generator/app/flags.go index 3b20a326ec3e..b5d4c18d43bd 100644 --- a/cmd/templatizer/app/flags.go +++ b/cmd/esmapping-generator/app/flags.go @@ -18,7 +18,7 @@ import ( "github.com/spf13/cobra" ) -// Options represent configurable parameters for jaeger-templatizer +// Options represent configurable parameters for jaeger-esmapping-generator type Options struct { Mapping string EsVersion int64 @@ -30,49 +30,43 @@ type Options struct { const ( mappingFlag = "mapping" - esVersionFlag = "esVersion" + esVersionFlag = "es-version" shardsFlag = "shards" replicasFlag = "replicas" - esPrefixFlag = "esPrefix" - useILMFlag = "useILM" + esPrefixFlag = "es-prefix" + useILMFlag = "use-ilm" ) -// AddFlags adds flags for templatizer main program +// AddFlags adds flags for esmapping-generator main program func (o *Options) AddFlags(command *cobra.Command) { - command.Flags().StringVarP( + command.Flags().StringVar( &o.Mapping, mappingFlag, - "m", "", "The index mapping the template will be applied to. Pass either jaeger-span or jaeger-service") - command.Flags().Int64VarP( + command.Flags().Int64Var( &o.EsVersion, esVersionFlag, - "v", 7, "The major Elasticsearch version") - command.Flags().Int64VarP( + command.Flags().Int64Var( &o.Shards, shardsFlag, - "s", 5, "The number of shards per index in Elasticsearch") - command.Flags().Int64VarP( + command.Flags().Int64Var( &o.Replicas, replicasFlag, - "r", 1, "The number of replicas per index in Elasticsearch") - command.Flags().StringVarP( + command.Flags().StringVar( &o.EsPrefix, esPrefixFlag, - "p", "", "Specifies index prefix") - command.Flags().StringVarP( + command.Flags().StringVar( &o.UseILM, useILMFlag, - "i", "false", "Set to true to use ILM for managing lifecycle of jaeger indices") diff --git a/cmd/templatizer/app/flags_test.go b/cmd/esmapping-generator/app/flags_test.go similarity index 90% rename from cmd/templatizer/app/flags_test.go rename to cmd/esmapping-generator/app/flags_test.go index 2d3149a3c6bc..72866fc4e5bc 100644 --- a/cmd/templatizer/app/flags_test.go +++ b/cmd/esmapping-generator/app/flags_test.go @@ -15,6 +15,7 @@ package app import ( + "github.com/stretchr/testify/require" "testing" "github.com/spf13/cobra" @@ -39,15 +40,15 @@ func TestOptionsWithFlags(t *testing.T) { c := cobra.Command{} o.AddFlags(&c) - c.ParseFlags([]string{ + err := c.ParseFlags([]string{ "--mapping=jaeger-span", - "--esVersion=6", + "--es-version=6", "--shards=5", "--replicas=1", - "--esPrefix=test", - "--useILM=true", + "--es-prefix=test", + "--use-ilm=true", }) - + require.NoError(t, err) assert.Equal(t, "jaeger-span", o.Mapping) assert.Equal(t, int64(6), o.EsVersion) assert.Equal(t, int64(5), o.Shards) diff --git a/cmd/templatizer/app/renderer/render.go b/cmd/esmapping-generator/app/renderer/render.go similarity index 91% rename from cmd/templatizer/app/renderer/render.go rename to cmd/esmapping-generator/app/renderer/render.go index aa54414ec186..64a7e90f9751 100644 --- a/cmd/templatizer/app/renderer/render.go +++ b/cmd/esmapping-generator/app/renderer/render.go @@ -17,12 +17,12 @@ package renderer import ( "strconv" - "github.com/jaegertracing/jaeger/cmd/templatizer/app" + "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" esTemplate "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/plugin/storage/es" ) -var allowedValues = map[string]struct{}{ +var supportedMappings = map[string]struct{}{ "jaeger-span": {}, "jaeger-service": {}, } @@ -41,6 +41,6 @@ func GetMappingAsString(builder esTemplate.TemplateBuilder, opt *app.Options) (s // IsValidOption checks if passed option is a valid index template. func IsValidOption(val string) bool { - _, ok := allowedValues[val] + _, ok := supportedMappings[val] return ok } diff --git a/cmd/templatizer/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go similarity index 98% rename from cmd/templatizer/app/renderer/render_test.go rename to cmd/esmapping-generator/app/renderer/render_test.go index 2d74c9f4b971..f1b710f6252c 100644 --- a/cmd/templatizer/app/renderer/render_test.go +++ b/cmd/esmapping-generator/app/renderer/render_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/jaegertracing/jaeger/cmd/templatizer/app" + "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" "github.com/jaegertracing/jaeger/pkg/es/mocks" ) diff --git a/cmd/templatizer/main.go b/cmd/esmapping-generator/main.go similarity index 79% rename from cmd/templatizer/main.go rename to cmd/esmapping-generator/main.go index 524022cc53dd..bb46e64cf623 100644 --- a/cmd/templatizer/main.go +++ b/cmd/esmapping-generator/main.go @@ -21,8 +21,8 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" - "github.com/jaegertracing/jaeger/cmd/templatizer/app" - "github.com/jaegertracing/jaeger/cmd/templatizer/app/renderer" + "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" + "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app/renderer" "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/pkg/version" ) @@ -31,9 +31,9 @@ func main() { logger, _ := zap.NewDevelopment() var options = app.Options{} var command = &cobra.Command{ - Use: "jaeger-templatizer", - Short: "Jaeger templatizer prints rendered mappings as string", - Long: `Jaeger templatizer renders passed templates with provided values and prints rendered output to stdout`, + Use: "jaeger-esmapping-generator", + Short: "Jaeger esmapping-generator prints rendered mappings as string", + Long: `Jaeger esmapping-generator renders passed templates with provided values and prints rendered output to stdout`, Run: func(cmd *cobra.Command, args []string) { if !renderer.IsValidOption(options.Mapping) { diff --git a/examples/hotrod/services/frontend/gen_assets.go b/examples/hotrod/services/frontend/gen_assets.go index 46b66108f8a5..b924f0754ea9 100644 --- a/examples/hotrod/services/frontend/gen_assets.go +++ b/examples/hotrod/services/frontend/gen_assets.go @@ -213,7 +213,7 @@ var _escData = map[string]*_escFile{ name: "index.html", local: "examples/hotrod/services/frontend/web_assets/index.html", size: 4058, - modtime: 1608397661, + modtime: 1601939528, compressed: ` H4sIAAAAAAAC/9RXbVPjOPJ/n0/Rf23AyT/EzgMMTIgzxZJZhtmdZS7AbM1NUYcsd2yBLWUkOYFN5btf yXaCA0zd3au7zYtE6m5196+fFA1jkyajGsAwRUOBxVRpND45v7xoHx0dvG13yRNX0BR9Mue4mEllCDAp @@ -250,7 +250,7 @@ KtU3UHprcaxum88KvCzyyrNp6BVP+n8GAAD//wz8Z3XaDwAA name: "jquery-3.1.1.min.js", local: "examples/hotrod/services/frontend/web_assets/jquery-3.1.1.min.js", size: 86709, - modtime: 1608397661, + modtime: 1598995573, compressed: ` H4sIAAAAAAAC/8y9fZebONIo/v/9FG02D4PaMm1nZvbewa1wMslkN7vztpPMzO5iskeAwLgxuAGnO2PY z/47KkkgME5mn+fec34nJ21A71KpVFWql5vr2dXub0dWfrh6/7m9sldXzZUVIvXtVXHMI1qnRX7VXO3u diff --git a/go.mod b/go.mod index 9f7904e1dd73..1f091a21d491 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/gorilla/mux v1.7.4 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 github.com/grpc-ecosystem/grpc-gateway v1.14.5 - github.com/hashicorp/go-hclog v0.14.1 + github.com/hashicorp/go-hclog v0.15.0 github.com/hashicorp/go-plugin v1.4.0 github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d // indirect github.com/kr/pretty v0.2.1 @@ -68,7 +68,7 @@ require ( github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c go.mongodb.org/mongo-driver v1.3.2 // indirect - go.uber.org/atomic v1.6.0 + go.uber.org/atomic v1.7.0 go.uber.org/automaxprocs v1.4.0 go.uber.org/zap v1.16.0 golang.org/x/lint v0.0.0-20200302205851-738671d3881b diff --git a/go.sum b/go.sum index 7cd41a8cbb50..af6ae9b959a9 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.15.0 h1:qMuK0wxsoW4D0ddCCYwPSTm4KQv1X1ke3WmPWZ0Mvsk= +github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= @@ -627,6 +629,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index dd7ebf0a1cab..d5efe4c94118 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -65,7 +65,7 @@ type Configuration struct { TLS tlscfg.Options `mapstructure:"tls"` UseReadWriteAliases bool `mapstructure:"use_aliases"` CreateIndexTemplates bool `mapstructure:"create_mappings"` - UseILM bool `mapstructure:"-"` + UseILM bool `mapstructure:"use_ilm"` Version uint `mapstructure:"version"` } diff --git a/pkg/es/textTemplate.go b/pkg/es/textTemplate.go index 304b8b63476d..9d89d05712c9 100644 --- a/pkg/es/textTemplate.go +++ b/pkg/es/textTemplate.go @@ -34,6 +34,6 @@ type TemplateBuilder interface { type TextTemplateBuilder struct{} // Parse is a wrapper for template.Parse -func (t TextTemplateBuilder) Parse(mapping string) (TemplateApplier, error) { - return template.New("mapping").Parse(mapping) +func (t TextTemplateBuilder) Parse(tmpl string) (TemplateApplier, error) { + return template.New("mapping").Parse(tmpl) } diff --git a/plugin/storage/es/Dockerfile.rollover b/plugin/storage/es/Dockerfile.rollover index eac9401a5317..f58d054b6e5b 100644 --- a/plugin/storage/es/Dockerfile.rollover +++ b/plugin/storage/es/Dockerfile.rollover @@ -6,6 +6,6 @@ RUN pip install urllib3==1.24.3 RUN pip install elasticsearch elasticsearch-curator COPY ./mappings/* /mappings/ COPY esRollover.py /es-rollover/ -COPY templatizer /usr/bin/ +COPY esmapping-generator /usr/bin/ ENTRYPOINT ["python3", "/es-rollover/esRollover.py"] diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index cb63110895d3..bae5a893313b 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -195,9 +195,9 @@ def str2bool(v): def fix_mapping(template_name, esVersion, shards, replicas, esprefix, use_ilm): - output = subprocess.Popen(['templatizer', '--mapping', template_name, '--esVersion', str(esVersion), + output = subprocess.Popen(['esmapping-generator', '--mapping', template_name, '--es-version', str(esVersion), '--shards', str(shards), '--replicas', - str(replicas), '--esPrefix', esprefix, '--useILM', str(use_ilm)], + str(replicas), '--es-prefix', esprefix, '--use-ilm', str(use_ilm)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) mapping, stderr = output.communicate() diff --git a/plugin/storage/es/spanstore/writer.go b/plugin/storage/es/spanstore/writer.go index 8bcdc67ca484..87b6bbff3b2c 100644 --- a/plugin/storage/es/spanstore/writer.go +++ b/plugin/storage/es/spanstore/writer.go @@ -17,6 +17,7 @@ package spanstore import ( "context" + "strings" "time" "github.com/uber/jaeger-lib/metrics" @@ -102,7 +103,7 @@ func NewSpanWriter(p SpanWriterParams) *SpanWriter { // CreateTemplates creates index templates. func (s *SpanWriter) CreateTemplates(spanTemplate, serviceTemplate, indexPrefix string) error { - if indexPrefix != "" { + if indexPrefix != "" && !strings.HasSuffix(indexPrefix, "-") { indexPrefix += "-" } _, err := s.client.CreateTemplate(indexPrefix + "jaeger-span").Body(spanTemplate).Do(context.Background()) diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go index ac93d2659dd1..7eb372f98ce5 100644 --- a/plugin/storage/integration/elasticsearch_test.go +++ b/plugin/storage/integration/elasticsearch_test.go @@ -31,7 +31,7 @@ import ( "go.uber.org/zap" "github.com/jaegertracing/jaeger/model" - esTemplate "github.com/jaegertracing/jaeger/pkg/es" + estemplate "github.com/jaegertracing/jaeger/pkg/es" eswrapper "github.com/jaegertracing/jaeger/pkg/es/wrapper" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es" @@ -108,7 +108,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro return err } client := eswrapper.WrapESClient(s.client, bp, esVersion) - spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion(), indexPrefix, false) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(estemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion(), indexPrefix, false) if err != nil { return err } @@ -138,7 +138,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro MaxDocCount: defaultMaxDocCount, }) dependencyStore := dependencystore.NewDependencyStore(client, s.logger, indexPrefix, indexDateLayout, defaultMaxDocCount) - depMapping, err := es.GetDependenciesMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion()) + depMapping, err := es.GetDependenciesMappings(estemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion()) if err != nil { return err }