#2048 - Support Elasticsearch ILM for managing jaeger indices

Makefile

@@ -129,11 +129,19 @@ test-compile-es-scripts:
 
 .PHONY: index-cleaner-integration-test
 index-cleaner-integration-test: docker-images-elastic
-	# Expire tests results for storage integration tests since the environment might change
+	# Expire test results for storage integration tests since the environment might change
 	# even though the code remains the same.
 	go clean -testcache
 	bash -c "set -e; set -o pipefail; $(GOTEST) -tags index_cleaner $(STORAGE_PKGS) | $(COLORIZE)"
 
+.PHONY: index-rollover-integration-test
+index-rollover-integration-test: docker-images-elastic
+	# Expire test results for storage integration tests since the environment might change
+	# even though the code remains the same.
+	go clean -testcache
+	bash -c "set -e; set -o pipefail; $(GOTEST) -tags index_rollover $(STORAGE_PKGS) | $(COLORIZE)"
+
+
 .PHONY: token-propagation-integration-test
 token-propagation-integration-test:
 	go clean -testcache
@@ -217,6 +225,14 @@ build-tracegen:
 build-anonymizer:
 	$(GOBUILD) -o ./cmd/anonymizer/anonymizer-$(GOOS)-$(GOARCH) ./cmd/anonymizer/main.go
 
+.PHONY: build-esmapping-generator
+build-esmapping-generator:
+	$(GOBUILD) -o ./plugin/storage/es/esmapping-generator-$(GOOS)-$(GOARCH) ./cmd/esmapping-generator/main.go
+
+.PHONY: build-esmapping-generator-linux
+build-esmapping-generator-linux:
+	 GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go
+
 .PHONY: docker-hotrod
 docker-hotrod:
 	GOOS=linux $(MAKE) build-examples
@@ -336,7 +352,7 @@ docker-images-cassandra:
 	@echo "Finished building jaeger-cassandra-schema =============="
 
 .PHONY: docker-images-elastic
-docker-images-elastic:
+docker-images-elastic: build-esmapping-generator-linux
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover
 	@echo "Finished building jaeger-es-indices-clean =============="

cmd/esmapping-generator/app/flags.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// Options represent configurable parameters for jaeger-esmapping-generator
+type Options struct {
+	Mapping   string
+	EsVersion int64
+	Shards    int64
+	Replicas  int64
+	EsPrefix  string
+	UseILM    string // using string as util is being used in python and using bool leads to type issues.
+}
+
+const (
+	mappingFlag   = "mapping"
+	esVersionFlag = "es-version"
+	shardsFlag    = "shards"
+	replicasFlag  = "replicas"
+	esPrefixFlag  = "es-prefix"
+	useILMFlag    = "use-ilm"
+)
+
+// AddFlags adds flags for esmapping-generator main program
+func (o *Options) AddFlags(command *cobra.Command) {
+	command.Flags().StringVar(
+		&o.Mapping,
+		mappingFlag,
+		"",
+		"The index mapping the template will be applied to. Pass either jaeger-span or jaeger-service")
+	command.Flags().Int64Var(
+		&o.EsVersion,
+		esVersionFlag,
+		7,
+		"The major Elasticsearch version")
+	command.Flags().Int64Var(
+		&o.Shards,
+		shardsFlag,
+		5,
+		"The number of shards per index in Elasticsearch")
+	command.Flags().Int64Var(
+		&o.Replicas,
+		replicasFlag,
+		1,
+		"The number of replicas per index in Elasticsearch")
+	command.Flags().StringVar(
+		&o.EsPrefix,
+		esPrefixFlag,
+		"",
+		"Specifies index prefix")
+	command.Flags().StringVar(
+		&o.UseILM,
+		useILMFlag,
+		"false",
+		"Set to true to use ILM for managing lifecycle of jaeger indices")
+
+	// mark mapping flag as mandatory
+	command.MarkFlagRequired(mappingFlag)
+}

cmd/esmapping-generator/app/flags_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptionsWithDefaultFlags(t *testing.T) {
+	o := Options{}
+	c := cobra.Command{}
+	o.AddFlags(&c)
+
+	assert.Equal(t, "", o.Mapping)
+	assert.Equal(t, int64(7), o.EsVersion)
+	assert.Equal(t, int64(5), o.Shards)
+	assert.Equal(t, int64(1), o.Replicas)
+	assert.Equal(t, "", o.EsPrefix)
+	assert.Equal(t, "false", o.UseILM)
+}
+
+func TestOptionsWithFlags(t *testing.T) {
+	o := Options{}
+	c := cobra.Command{}
+
+	o.AddFlags(&c)
+	err := c.ParseFlags([]string{
+		"--mapping=jaeger-span",
+		"--es-version=6",
+		"--shards=5",
+		"--replicas=1",
+		"--es-prefix=test",
+		"--use-ilm=true",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "jaeger-span", o.Mapping)
+	assert.Equal(t, int64(6), o.EsVersion)
+	assert.Equal(t, int64(5), o.Shards)
+	assert.Equal(t, int64(1), o.Replicas)
+	assert.Equal(t, "test", o.EsPrefix)
+	assert.Equal(t, "true", o.UseILM)
+}

cmd/esmapping-generator/app/renderer/render.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2021 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package renderer
+
+import (
+	"strconv"
+
+	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
+	estemplate "github.com/jaegertracing/jaeger/pkg/es"
+	"github.com/jaegertracing/jaeger/plugin/storage/es"
+)
+
+var supportedMappings = map[string]struct{}{
+	"jaeger-span":    {},
+	"jaeger-service": {},
+}
+
+// GetMappingAsString returns rendered index templates as string
+func GetMappingAsString(builder estemplate.TemplateBuilder, opt *app.Options) (string, error) {
+	if opt.EsVersion == 7 {
+		enableILM, err := strconv.ParseBool(opt.UseILM)
+		if err != nil {
+			return "", err
+		}
+		return es.FixMapping(builder, es.LoadMapping("/"+opt.Mapping+"-7.json"), opt.Shards, opt.Replicas, opt.EsPrefix, enableILM)
+	}
+	return es.FixMapping(builder, es.LoadMapping("/"+opt.Mapping+".json"), opt.Shards, opt.Replicas, "", false)
+}
+
+// IsValidOption checks if passed option is a valid index template.
+func IsValidOption(val string) bool {
+	_, ok := supportedMappings[val]
+	return ok
+}

cmd/esmapping-generator/app/renderer/render_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2021 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package renderer
+
+import (
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
+	"github.com/jaegertracing/jaeger/pkg/es/mocks"
+)
+
+func TestIsValidOption(t *testing.T) {
+	tests := []struct {
+		name          string
+		arg           string
+		expectedValue bool
+	}{{name: "span mapping", arg: "jaeger-span", expectedValue: true},
+		{name: "service mapping", arg: "jaeger-service", expectedValue: true},
+		{name: "Invalid mapping", arg: "dependency-service", expectedValue: false},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expectedValue, IsValidOption(test.arg))
+		})
+	}
+}
+
+func Test_getMappingAsString(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    app.Options
+		want    string
+		wantErr error
+	}{
+		{
+			name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"},
+			want: "ES version 7",
+		},
+		{
+			name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"},
+			want: "ES version 6",
+		},
+		{
+			name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"},
+			wantErr: errors.New("parse error"),
+		},
+		{
+			name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"},
+			wantErr: errors.New("parse error"),
+		},
+		{
+			name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "foo"},
+			wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Prepare
+			mockTemplateApplier := &mocks.TemplateApplier{}
+			mockTemplateApplier.On("Execute", mock.Anything, mock.Anything).Return(
+				func(wr io.Writer, data interface{}) error {
+					wr.Write([]byte(tt.want))
+					return nil
+				},
+			)
+			mockTemplateBuilder := &mocks.TemplateBuilder{}
+			mockTemplateBuilder.On("Parse", mock.Anything).Return(mockTemplateApplier, tt.wantErr)
+
+			// Test
+			got, err := GetMappingAsString(mockTemplateBuilder, &tt.args)
+
+			// Validate
+			if tt.wantErr != nil {
+				assert.EqualError(t, err, tt.wantErr.Error())
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

cmd/esmapping-generator/main.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2020 The Jaeger Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+
+	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
+	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app/renderer"
+	"github.com/jaegertracing/jaeger/pkg/es"
+	"github.com/jaegertracing/jaeger/pkg/version"
+)
+
+func main() {
+	logger, _ := zap.NewDevelopment()
+	var options = app.Options{}
+	var command = &cobra.Command{
+		Use:   "jaeger-esmapping-generator",
+		Short: "Jaeger esmapping-generator prints rendered mappings as string",
+		Long:  `Jaeger esmapping-generator renders passed templates with provided values and prints rendered output to stdout`,
+		Run: func(cmd *cobra.Command, args []string) {
+
+			if !renderer.IsValidOption(options.Mapping) {
+				logger.Fatal("please pass either 'jaeger-service' or 'jaeger-span' as argument")
+			}
+
+			parsedMapping, err := renderer.GetMappingAsString(es.TextTemplateBuilder{}, &options)
+			if err != nil {
+				logger.Fatal(err.Error())
+			}
+			print(parsedMapping)
+		},
+	}
+
+	options.AddFlags(command)
+
+	command.AddCommand(version.Command())
+
+	if err := command.Execute(); err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+}