jaegertracing#2048[WIP] - Add support for enabling ILM using command …

…line param

Signed-off-by: santosh <[email protected]>

cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go

@@ -31,7 +31,7 @@ func newExporter(ctx context.Context, config *Config, params component.ExporterC
 		return nil, err
 	}
 	if config.Primary.IsCreateIndexTemplates() {
-		spanMapping, serviceMapping := es.GetSpanServiceMappings(esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion()))
+		spanMapping, serviceMapping := es.GetSpanServiceMappings(esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion()),esCfg.GetIndexPrefix(),esCfg.GetUseILM())
 		if err = w.CreateTemplates(ctx, spanMapping, serviceMapping); err != nil {
 			return nil, err
 		}

cmd/opentelemetry/go.sum

@@ -282,6 +282,8 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/flosch/pongo2/v4 v4.0.1 h1:l/49mdkeTYSHzH9N4Iwjs0yMqELMKCsYv5PzTm37Gio=
+github.com/flosch/pongo2/v4 v4.0.1/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=

go.mod

@@ -3,18 +3,17 @@ module github.com/jaegertracing/jaeger
 go 1.14
 
 require (
-	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/DataDog/zstd v1.4.4 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v0.9.0 // indirect
 	github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f
 	github.com/apache/thrift v0.13.0
 	github.com/bsm/sarama-cluster v2.1.13+incompatible
-	github.com/cpuguy83/go-md2man v1.0.10 // indirect
 	github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b
 	github.com/dgraph-io/badger v1.6.2
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/fatih/color v1.9.0 // indirect
+	github.com/flosch/pongo2/v4 v4.0.1
 	github.com/fortytw2/leaktest v1.3.0 // indirect
 	github.com/frankban/quicktest v1.7.3 // indirect
 	github.com/fsnotify/fsnotify v1.4.7

go.sum

@@ -89,8 +89,6 @@ github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger v1.5.3 h1:5oWIuRvwn93cie+OSt1zSnkaIQ1JFQM8bGlIv6O6Sts=
-github.com/dgraph-io/badger v1.5.3/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
 github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
 github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po=
@@ -121,6 +119,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/flosch/pongo2/v4 v4.0.1 h1:l/49mdkeTYSHzH9N4Iwjs0yMqELMKCsYv5PzTm37Gio=
+github.com/flosch/pongo2/v4 v4.0.1/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -421,6 +421,8 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
 github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
@@ -792,6 +794,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=

pkg/es/config/config.go

@@ -64,6 +64,7 @@ type Configuration struct {
 	TLS                   tlscfg.Options `mapstructure:"tls"`
 	UseReadWriteAliases   bool           `mapstructure:"use_aliases"`
 	CreateIndexTemplates  bool           `mapstructure:"create_mappings"`
+	UseILM			      bool			 `mapstructure:"-"`
 	Version               uint           `mapstructure:"version"`
 }
 
@@ -98,6 +99,7 @@ type ClientBuilder interface {
 	IsCreateIndexTemplates() bool
 	GetVersion() uint
 	TagKeysAsFields() ([]string, error)
+	GetUseILM() bool
 }
 
 // NewClient creates a new ElasticSearch client
@@ -286,6 +288,11 @@ func (c *Configuration) GetUseReadWriteAliases() bool {
 	return c.UseReadWriteAliases
 }
 
+// GetUseILM indicates whether ILM should be used
+func (c *Configuration) GetUseILM() bool {
+	return c.UseILM
+}
+
 // GetTokenFilePath returns file path containing the bearer token
 func (c *Configuration) GetTokenFilePath() string {
 	return c.TokenFilePath

plugin/storage/es/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3-alpine
+FROM python:3.7-alpine
 
 # Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494
 RUN pip install urllib3==1.24.3

plugin/storage/es/Dockerfile.rollover

@@ -1,9 +1,9 @@
-FROM python:3-alpine
+FROM python:3.7-alpine3.10
 
 # Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494
 RUN pip install urllib3==1.24.3
 
-RUN pip install elasticsearch elasticsearch-curator pathlib2
+RUN pip install elasticsearch elasticsearch-curator pathlib2 Jinja2
 COPY ./mappings/* /mappings/
 COPY esRollover.py /es-rollover/
 

plugin/storage/es/esRollover.py

@@ -11,6 +11,7 @@
 import re
 from pathlib2 import Path
 from requests.auth import HTTPBasicAuth
+from jinja2 import Template
 
 
 ARCHIVE_INDEX = 'jaeger-span-archive'
@@ -36,6 +37,7 @@ def main():
         print('ES_TLS_CA ... Path to TLS CA file.')
         print('ES_TLS_CERT ... Path to TLS certificate file.')
         print('ES_TLS_KEY ... Path to TLS key file.')
+        print('ES_USE_ILM .. Use ILM to manage jaeger indices.')
         print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
         print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.')
         print('init configuration:')
@@ -73,11 +75,13 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t
         shards = os.getenv('SHARDS', SHARDS)
         replicas = os.getenv('REPLICAS', REPLICAS)
         esVersion = get_version(client)
+        use_ilm = str2bool(os.getenv("ES_USE_ILM", 'false'))
         if esVersion == 7:
             mapping = Path('./mappings/'+template_name+'-7.json').read_text()
         else:
             mapping = Path('./mappings/'+template_name+'.json').read_text()
-        create_index_template(fix_mapping(mapping, shards, replicas, prefix), prefix+template_name+"-with-ilm")
+            use_ilm = False
+        create_index_template(fix_mapping(mapping, shards, replicas, prefix, use_ilm), prefix+template_name+"-override-template")
 
         index = index_to_rollover + '-000001'
         create_index(client, index)
@@ -178,10 +182,9 @@ def str2bool(v):
     return v.lower() in ('true', '1')
 
 
-def fix_mapping(mapping, shards, replicas, prefix):
-    mapping = mapping.replace("${__NUMBER_OF_SHARDS__}", str(shards))
-    mapping = mapping.replace("${__NUMBER_OF_REPLICAS__}", str(replicas))
-    mapping = mapping.replace("${__PREFIX__}", str(prefix))
+def fix_mapping(mapping, shards, replicas, esprefix, use_ilm):
+    template = Template(mapping)
+    mapping = template.render(NumberOfShards=shards, NumberOfReplicas=replicas, ESPrefix=esprefix, UseILM=use_ilm, Order=2)
     return mapping
 
 

plugin/storage/es/factory.go

@@ -19,9 +19,8 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"strconv"
-	"strings"
 
+	"github.com/flosch/pongo2/v4"
 	"github.com/spf13/viper"
 	"github.com/uber/jaeger-lib/metrics"
 	"go.uber.org/zap"
@@ -165,7 +164,7 @@ func createSpanWriter(
 		return nil, err
 	}
 
-	spanMapping, serviceMapping := GetSpanServiceMappings(cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion())
+	spanMapping, serviceMapping := GetSpanServiceMappings(cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion(), cfg.GetIndexPrefix(), cfg.GetUseILM())
 	writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{
 		Client:              client,
 		Logger:              logger,
@@ -187,32 +186,36 @@ func createSpanWriter(
 }
 
 // GetSpanServiceMappings returns span and service mappings
-func GetSpanServiceMappings(shards, replicas int64, esVersion uint) (string, string) {
+func GetSpanServiceMappings(shards, replicas int64, esVersion uint, esPrefix string, useILM bool) (string, string) {
 	if esVersion == 7 {
-		return fixMapping(loadMapping("/jaeger-span-7.json"), shards, replicas),
-			fixMapping(loadMapping("/jaeger-service-7.json"), shards, replicas)
+		return fixMapping(loadMapping("/jaeger-span-7.json"), shards, replicas, esPrefix, useILM),
+			fixMapping(loadMapping("/jaeger-service-7.json"), shards, replicas, esPrefix, useILM)
 	}
-	return fixMapping(loadMapping("/jaeger-span.json"), shards, replicas),
-		fixMapping(loadMapping("/jaeger-service.json"), shards, replicas)
+	return fixMapping(loadMapping("/jaeger-span.json"), shards, replicas, "", false),
+		fixMapping(loadMapping("/jaeger-service.json"), shards, replicas, "", false)
 }
 
 // GetDependenciesMappings returns dependencies mappings
 func GetDependenciesMappings(shards, replicas int64, esVersion uint) string {
 	if esVersion == 7 {
-		return fixMapping(loadMapping("/jaeger-dependencies-7.json"), shards, replicas)
+		return fixMapping(loadMapping("/jaeger-dependencies-7.json"), shards, replicas, "", false)
 	}
-	return fixMapping(loadMapping("/jaeger-dependencies.json"), shards, replicas)
+	return fixMapping(loadMapping("/jaeger-dependencies.json"), shards, replicas, "", false)
 }
 
 func loadMapping(name string) string {
 	s, _ := mappings.FSString(false, name)
 	return s
 }
 
-func fixMapping(mapping string, shards, replicas int64) string {
-	mapping = strings.Replace(mapping, "${__NUMBER_OF_SHARDS__}", strconv.FormatInt(shards, 10), 1)
-	mapping = strings.Replace(mapping, "${__NUMBER_OF_REPLICAS__}", strconv.FormatInt(replicas, 10), 1)
-	return mapping
+func fixMapping(mapping string, shards, replicas int64, esPrefix string, useILM bool) string {
+	t, _:= pongo2.FromString(mapping)
+	if esPrefix != "" {
+		esPrefix += "-"
+	}
+	fixedMapping, _ := t.Execute(pongo2.Context{"NumberOfShards": shards, "NumberOfReplicas": replicas, "ESPrefix": esPrefix, "UseILM": useILM})
+
+	return fixedMapping
 }
 
 var _ io.Closer = (*Factory)(nil)

plugin/storage/es/factory_test.go

@@ -18,10 +18,9 @@ package es
 import (
 	"context"
 	"errors"
+	"github.com/flosch/pongo2/v4"
 	"io/ioutil"
 	"os"
-	"strconv"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -164,9 +163,9 @@ func TestTagKeysAsFields(t *testing.T) {
 }
 
 func TestFactory_LoadMapping(t *testing.T) {
-	spanMapping5, serviceMapping5 := GetSpanServiceMappings(10, 0, 5)
-	spanMapping6, serviceMapping6 := GetSpanServiceMappings(10, 0, 6)
-	spanMapping7, serviceMapping7 := GetSpanServiceMappings(10, 0, 7)
+	spanMapping5, serviceMapping5 := GetSpanServiceMappings(10, 0, 5,"",false)
+	spanMapping6, serviceMapping6 := GetSpanServiceMappings(10, 0, 6,"",false)
+	spanMapping7, serviceMapping7 := GetSpanServiceMappings(10, 0, 7,"test",true)
 	dependenciesMapping6 := GetDependenciesMappings(10, 0, 6)
 	dependenciesMapping7 := GetDependenciesMappings(10, 0, 7)
 	tests := []struct {
@@ -189,11 +188,17 @@ func TestFactory_LoadMapping(t *testing.T) {
 		b, err := ioutil.ReadAll(f)
 		require.NoError(t, err)
 		assert.Equal(t, string(b), mapping)
-
-		expectedMapping := string(b)
-		expectedMapping = strings.Replace(expectedMapping, "${__NUMBER_OF_SHARDS__}", strconv.FormatInt(10, 10), 1)
-		expectedMapping = strings.Replace(expectedMapping, "${__NUMBER_OF_REPLICAS__}", strconv.FormatInt(0, 10), 1)
-		assert.Equal(t, expectedMapping, fixMapping(mapping, 10, 0))
+		tempMapping, err1 := pongo2.FromString(mapping)
+		assert.NoError(t,err1)
+		if test.name != "/jaeger-service-7.json" && test.name != "/jaeger-span-7.json" {
+			expectedMapping, err := tempMapping.Execute(pongo2.Context{"NumberOfShards": 10, "NumberOfReplicas": 0, "ESPrefix": "", "UseILM": false})
+			assert.NoError(t,err)
+			assert.Equal(t,expectedMapping,fixMapping(mapping, 10, 0, "", false))
+		}else {
+			expectedMapping, err := tempMapping.Execute(pongo2.Context{"NumberOfShards": 10, "NumberOfReplicas": 0, "ESPrefix": "test-", "UseILM": true})
+			assert.NoError(t,err)
+			assert.Equal(t,expectedMapping,fixMapping(mapping, 10, 0, "test", true))
+		}
 	}
 }