[jaeger-v2] Refactor ElasticSearch/OpenSearch Storage Configurations (j…

…aegertracing#6060)

## Which problem is this PR solving?
- Part of jaegertracing#6059

## Description of the changes
- Continuation of jaegertracing#5790
- Refactors the configurations for ElasticSearch and OpenSearch to
simplify the configuration by having more logical groupings

## How was this change tested?
- Unit Tests and E2E Integration Tests

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `yarn lint` and `yarn test`

---------

Signed-off-by: Jared Tan <[email protected]>
Signed-off-by: Mahad Zaryab <[email protected]>
Signed-off-by: Yuri Shkuro <[email protected]>
Co-authored-by: Jared Tan <[email protected]>
Co-authored-by: Yuri Shkuro <[email protected]>

cmd/es-rollover/app/init/action.go

@@ -13,6 +13,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
 	"github.com/jaegertracing/jaeger/pkg/es"
 	"github.com/jaegertracing/jaeger/pkg/es/client"
+	"github.com/jaegertracing/jaeger/pkg/es/config"
 	"github.com/jaegertracing/jaeger/pkg/es/filter"
 	"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
 )
@@ -28,19 +29,15 @@ type Action struct {
 }
 
 func (c Action) getMapping(version uint, templateName string) (string, error) {
+	c.Config.Indices.IndexPrefix = config.IndexPrefix(c.Config.Config.IndexPrefix)
 	mappingBuilder := mappings.MappingBuilder{
-		TemplateBuilder:              es.TextTemplateBuilder{},
-		PrioritySpanTemplate:         int64(c.Config.PrioritySpanTemplate),
-		PriorityServiceTemplate:      int64(c.Config.PriorityServiceTemplate),
-		PriorityDependenciesTemplate: int64(c.Config.PriorityDependenciesTemplate),
-		PrioritySamplingTemplate:     int64(c.Config.PrioritySamplingTemplate),
-		Shards:                       int64(c.Config.Shards),
-		Replicas:                     int64(c.Config.Replicas),
-		IndexPrefix:                  c.Config.IndexPrefix,
-		UseILM:                       c.Config.UseILM,
-		ILMPolicyName:                c.Config.ILMPolicyName,
-		EsVersion:                    version,
+		TemplateBuilder: es.TextTemplateBuilder{},
+		Indices:         c.Config.Indices,
+		UseILM:          c.Config.UseILM,
+		ILMPolicyName:   c.Config.ILMPolicyName,
+		EsVersion:       version,
 	}
+
 	return mappingBuilder.GetMapping(templateName)
 }
 
@@ -62,7 +59,7 @@ func (c Action) Do() error {
 			return fmt.Errorf("ILM policy %s doesn't exist in Elasticsearch. Please create it and re-run init", c.Config.ILMPolicyName)
 		}
 	}
-	rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.IndexPrefix)
+	rolloverIndices := app.RolloverIndices(c.Config.Archive, c.Config.SkipDependencies, c.Config.AdaptiveSampling, c.Config.Config.IndexPrefix)
 	for _, indexName := range rolloverIndices {
 		if err := c.init(version, indexName); err != nil {
 			return err
@@ -114,7 +111,7 @@ func (c Action) init(version uint, indexopt app.IndexOption) error {
 		return err
 	}
 
-	jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.IndexPrefix)
+	jaegerIndices, err := c.IndicesClient.GetJaegerIndices(c.Config.Config.IndexPrefix)
 	if err != nil {
 		return err
 	}

cmd/es-rollover/app/init/flags.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
+	cfg "github.com/jaegertracing/jaeger/pkg/es/config"
 )
 
 const (
@@ -21,14 +22,10 @@ const (
 )
 
 // Config holds configuration for index cleaner binary.
+// Config.IndexPrefix supersedes Indices.IndexPrefix
 type Config struct {
 	app.Config
-	Shards                       int
-	Replicas                     int
-	PrioritySpanTemplate         int
-	PriorityServiceTemplate      int
-	PriorityDependenciesTemplate int
-	PrioritySamplingTemplate     int
+	cfg.Indices
 }
 
 // AddFlags adds flags for TLS to the FlagSet.
@@ -43,10 +40,18 @@ func (*Config) AddFlags(flags *flag.FlagSet) {
 
 // InitFromViper initializes config from viper.Viper.
 func (c *Config) InitFromViper(v *viper.Viper) {
-	c.Shards = v.GetInt(shards)
-	c.Replicas = v.GetInt(replicas)
-	c.PrioritySpanTemplate = v.GetInt(prioritySpanTemplate)
-	c.PriorityServiceTemplate = v.GetInt(priorityServiceTemplate)
-	c.PriorityDependenciesTemplate = v.GetInt(priorityDependenciesTemplate)
-	c.PrioritySamplingTemplate = v.GetInt(prioritySamplingTemplate)
+	c.Indices.Spans.Shards = v.GetInt64(shards)
+	c.Indices.Services.Shards = v.GetInt64(shards)
+	c.Indices.Dependencies.Shards = v.GetInt64(shards)
+	c.Indices.Sampling.Shards = v.GetInt64(shards)
+
+	c.Indices.Spans.Replicas = v.GetInt64(replicas)
+	c.Indices.Services.Replicas = v.GetInt64(replicas)
+	c.Indices.Dependencies.Replicas = v.GetInt64(replicas)
+	c.Indices.Sampling.Replicas = v.GetInt64(replicas)
+
+	c.Indices.Spans.Priority = v.GetInt64(prioritySpanTemplate)
+	c.Indices.Services.Priority = v.GetInt64(priorityServiceTemplate)
+	c.Indices.Dependencies.Priority = v.GetInt64(priorityDependenciesTemplate)
+	c.Indices.Sampling.Priority = v.GetInt64(prioritySamplingTemplate)
 }

cmd/es-rollover/app/init/flags_test.go

@@ -33,10 +33,10 @@ func TestBindFlags(t *testing.T) {
 	require.NoError(t, err)
 
 	c.InitFromViper(v)
-	assert.Equal(t, 8, c.Shards)
-	assert.Equal(t, 16, c.Replicas)
-	assert.Equal(t, 300, c.PrioritySpanTemplate)
-	assert.Equal(t, 301, c.PriorityServiceTemplate)
-	assert.Equal(t, 302, c.PriorityDependenciesTemplate)
-	assert.Equal(t, 303, c.PrioritySamplingTemplate)
+	assert.EqualValues(t, 8, c.Indices.Spans.Shards)
+	assert.EqualValues(t, 16, c.Indices.Spans.Replicas)
+	assert.EqualValues(t, 300, c.Indices.Spans.Priority)
+	assert.EqualValues(t, 301, c.Indices.Services.Priority)
+	assert.EqualValues(t, 302, c.Indices.Dependencies.Priority)
+	assert.EqualValues(t, 303, c.Indices.Sampling.Priority)
 }

cmd/esmapping-generator/app/flags_test.go

@@ -20,8 +20,9 @@ func TestOptionsWithDefaultFlags(t *testing.T) {
 
 	assert.Equal(t, "", o.Mapping)
 	assert.Equal(t, uint(7), o.EsVersion)
-	assert.Equal(t, int64(5), o.Shards)
-	assert.Equal(t, int64(1), o.Replicas)
+	assert.EqualValues(t, 5, o.Shards)
+	assert.EqualValues(t, 1, o.Replicas)
+
 	assert.Equal(t, "", o.IndexPrefix)
 	assert.Equal(t, "false", o.UseILM)
 	assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)

cmd/esmapping-generator/app/renderer/render.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
 	"github.com/jaegertracing/jaeger/pkg/es"
+	cfg "github.com/jaegertracing/jaeger/pkg/es/config"
 	"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
 )
 
@@ -25,14 +26,22 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
 		return "", err
 	}
 
+	indexOpts := cfg.IndexOptions{
+		Shards:   opt.Shards,
+		Replicas: opt.Replicas,
+	}
 	mappingBuilder := mappings.MappingBuilder{
 		TemplateBuilder: builder,
-		Shards:          opt.Shards,
-		Replicas:        opt.Replicas,
-		EsVersion:       opt.EsVersion,
-		IndexPrefix:     opt.IndexPrefix,
-		UseILM:          enableILM,
-		ILMPolicyName:   opt.ILMPolicyName,
+		Indices: cfg.Indices{
+			IndexPrefix:  cfg.IndexPrefix(opt.IndexPrefix),
+			Spans:        indexOpts,
+			Services:     indexOpts,
+			Dependencies: indexOpts,
+			Sampling:     indexOpts,
+		},
+		EsVersion:     opt.EsVersion,
+		UseILM:        enableILM,
+		ILMPolicyName: opt.ILMPolicyName,
 	}
 	return mappingBuilder.GetMapping(opt.Mapping)
 }

cmd/jaeger/config-elasticsearch.yaml

@@ -22,10 +22,32 @@ extensions:
     backends:
       some_storage:
         elasticsearch:
-          index_prefix: "jaeger-main"
+          indices:
+            index_prefix: "jaeger-main"
+            spans:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            services:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            dependencies:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            sampling:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
       another_storage:
         elasticsearch:
-          index_prefix: "jaeger-archive"
+          indices:
+            index_prefix: "jaeger-archive"
 
 receivers:
   otlp:

cmd/jaeger/config-opensearch.yaml

@@ -22,11 +22,32 @@ extensions:
     backends:
       some_storage:
         opensearch:
-          index_prefix: "jaeger-main"
-
+          indices:
+            index_prefix: "jaeger-main"
+            spans:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            services:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            dependencies:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
+            sampling:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+              shards: 5
+              replicas: 1
       another_storage:
         opensearch:
-          index_prefix: "jaeger-main"
+          indices:
+            index_prefix: "jaeger-archive"
 
 receivers:
   otlp: