Skip to content

Commit

Permalink
polish
Browse files Browse the repository at this point in the history
  • Loading branch information
JaredTan95 committed Aug 7, 2024
1 parent d999e1a commit 20bdffb
Show file tree
Hide file tree
Showing 24 changed files with 446 additions and 309 deletions.
65 changes: 25 additions & 40 deletions cmd/es-rollover/app/init/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,10 @@ package init
import (
"flag"

cfg "github.com/jaegertracing/jaeger/pkg/es/config"
"github.com/spf13/viper"

"github.com/jaegertracing/jaeger/cmd/es-rollover/app"
)

const (
spanShards = "shards-span"
serviceShards = "shards-service"
dependenciesShards = "shards-dependencies"
samplingShards = "shards-sampling"
spanReplicas = "replicas-span"
serviceReplicas = "replicas-service"
dependenciesReplicas = "replicas-dependencies"
samplingReplicas = "replicas-sampling"
prioritySpanTemplate = "priority-span-template"
priorityServiceTemplate = "priority-service-template"
priorityDependenciesTemplate = "priority-dependencies-template"
prioritySamplingTemplate = "priority-sampling-template"
cfg "github.com/jaegertracing/jaeger/pkg/es/config"
)

// Config holds configuration for index cleaner binary.
Expand All @@ -46,34 +31,34 @@ type Config struct {

// AddFlags adds flags for TLS to the FlagSet.
func (*Config) AddFlags(flags *flag.FlagSet) {
flags.Int64(spanShards, 5, "Number of span index shards")
flags.Int64(serviceShards, 5, "Number of service index shards")
flags.Int64(dependenciesShards, 5, "Number of dependencies index shards")
flags.Int64(samplingShards, 5, "Number of sampling index shards")
flags.Int64(cfg.GetNumShardSpanFlag(), 5, "Number of span index shards")
flags.Int64(cfg.GetNumShardServiceFlag(), 5, "Number of service index shards")
flags.Int64(cfg.GetNumShardDependenciesFlag(), 5, "Number of dependencies index shards")
flags.Int64(cfg.GetNumShardSamplingFlag(), 5, "Number of sampling index shards")

flags.Int64(spanReplicas, 1, "Number of span index replicas")
flags.Int64(serviceReplicas, 1, "Number of services index replicas")
flags.Int64(dependenciesReplicas, 1, "Number of dependencies index replicas")
flags.Int64(samplingReplicas, 1, "Number of sampling index replicas")
flags.Int64(cfg.GetNumReplicasSpanFlag(), 1, "Number of span index replicas")
flags.Int64(cfg.GetNumReplicasServiceFlag(), 1, "Number of services index replicas")
flags.Int64(cfg.GetNumReplicasDependenciesFlag(), 1, "Number of dependencies index replicas")
flags.Int64(cfg.GetNumReplicasSamplingFlag(), 1, "Number of sampling index replicas")

flags.Int64(prioritySpanTemplate, 0, "Priority of jaeger-span index template (ESv8 only)")
flags.Int64(priorityServiceTemplate, 0, "Priority of jaeger-service index template (ESv8 only)")
flags.Int64(priorityDependenciesTemplate, 0, "Priority of jaeger-dependencies index template (ESv8 only)")
flags.Int64(prioritySamplingTemplate, 0, "Priority of jaeger-sampling index template (ESv8 only)")
flags.Int64(cfg.GetPrioritySpanTemplate(), 0, "Priority of jaeger-span index template (ESv8 only)")
flags.Int64(cfg.GetPriorityServiceTemplate(), 0, "Priority of jaeger-service index template (ESv8 only)")
flags.Int64(cfg.GetPriorityDependenciesTemplate(), 0, "Priority of jaeger-dependencies index template (ESv8 only)")
flags.Int64(cfg.GetPrioritySamplingTemplate(), 0, "Priority of jaeger-sampling index template (ESv8 only)")
}

// InitFromViper initializes config from viper.Viper.
func (c *Config) InitFromViper(v *viper.Viper) {
c.Indices.Spans.TemplateOptions.NumShards = v.GetInt(spanShards)
c.Indices.Spans.TemplateOptions.NumReplicas = v.GetInt(spanReplicas)
c.Indices.Services.TemplateOptions.NumShards = v.GetInt(serviceShards)
c.Indices.Services.TemplateOptions.NumReplicas = v.GetInt(serviceReplicas)
c.Indices.Dependencies.TemplateOptions.NumShards = v.GetInt(dependenciesShards)
c.Indices.Dependencies.TemplateOptions.NumReplicas = v.GetInt(dependenciesReplicas)
c.Indices.Sampling.TemplateOptions.NumShards = v.GetInt(samplingShards)
c.Indices.Sampling.TemplateOptions.NumReplicas = v.GetInt(samplingReplicas)
c.Indices.Spans.TemplateOptions.Priority = v.GetInt(prioritySpanTemplate)
c.Indices.Services.TemplateOptions.Priority = v.GetInt(priorityServiceTemplate)
c.Indices.Dependencies.TemplateOptions.Priority = v.GetInt(priorityDependenciesTemplate)
c.Indices.Sampling.TemplateOptions.Priority = v.GetInt(prioritySamplingTemplate)
c.Indices.Spans.TemplateNumShards = v.GetInt(cfg.GetNumShardSpanFlag())
c.Indices.Spans.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasSpanFlag())
c.Indices.Services.TemplateNumShards = v.GetInt(cfg.GetNumShardServiceFlag())
c.Indices.Services.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasServiceFlag())
c.Indices.Dependencies.TemplateNumShards = v.GetInt(cfg.GetNumShardDependenciesFlag())
c.Indices.Dependencies.TemplateNumReplicas = v.GetInt(cfg.GetNumReplicasDependenciesFlag())
c.Indices.Sampling.TemplateNumShards = v.GetInt(cfg.GetNumShardSamplingFlag())
c.Indices.Sampling.TemplateNumReplicas = v.GetInt(cfg.GetNumShardSamplingFlag())
c.Indices.Spans.TemplatePriority = v.GetInt(cfg.GetPrioritySpanTemplate())
c.Indices.Services.TemplatePriority = v.GetInt(cfg.GetPriorityServiceTemplate())
c.Indices.Dependencies.TemplatePriority = v.GetInt(cfg.GetPriorityDependenciesTemplate())
c.Indices.Sampling.TemplatePriority = v.GetInt(cfg.GetPrioritySamplingTemplate())
}
53 changes: 23 additions & 30 deletions cmd/esmapping-generator/app/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
package app

import (
"github.com/jaegertracing/jaeger/pkg/es/config"
"github.com/spf13/cobra"

"github.com/jaegertracing/jaeger/pkg/es/config"
)

// Options represent configurable parameters for jaeger-esmapping-generator
Expand All @@ -30,19 +31,11 @@ type Options struct {
}

const (
mappingFlag = "mapping"
esVersionFlag = "es-version"
spanShardsFlag = "shards-span"
serviceShardsFlag = "shards-service"
dependenciesShardsFlag = "shards-dependencies"
samplingShardsFlag = "shards-sampling"
spanReplicasFlag = "replicas-span"
serviceReplicasFlag = "replicas-service"
dependenciesReplicasFlag = "replicas-dependencies"
samplingReplicasFlag = "replicas-sampling"
indexPrefixFlag = "index-prefix"
useILMFlag = "use-ilm"
ilmPolicyNameFlag = "ilm-policy-name"
mappingFlag = "mapping"
esVersionFlag = "es-version"
indexPrefixFlag = "index-prefix"
useILMFlag = "use-ilm"
ilmPolicyNameFlag = "ilm-policy-name"
)

// AddFlags adds flags for esmapping-generator main program
Expand All @@ -58,43 +51,43 @@ func (o *Options) AddFlags(command *cobra.Command) {
7,
"The major Elasticsearch version")
command.Flags().IntVar(
&o.Indices.Spans.TemplateOptions.NumShards,
spanShardsFlag,
&o.Indices.Spans.TemplateNumShards,
config.GetNumShardSpanFlag(),
5,
"The number of shards per span index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Spans.TemplateOptions.NumReplicas,
spanReplicasFlag,
&o.Indices.Spans.TemplateNumReplicas,
config.GetNumReplicasSpanFlag(),
1,
"The number of replicas per index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Services.TemplateOptions.NumShards,
serviceShardsFlag,
&o.Indices.Services.TemplateNumShards,
config.GetNumShardServiceFlag(),
5,
"The number of shards per service index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Services.TemplateOptions.NumReplicas,
serviceReplicasFlag,
&o.Indices.Services.TemplateNumReplicas,
config.GetNumReplicasServiceFlag(),
1,
"The number of replicas per service index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Dependencies.TemplateOptions.NumShards,
dependenciesShardsFlag,
&o.Indices.Dependencies.TemplateNumShards,
config.GetNumShardDependenciesFlag(),
5,
"The number of shards per dependencies index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Dependencies.TemplateOptions.NumReplicas,
dependenciesReplicasFlag,
&o.Indices.Dependencies.TemplateNumReplicas,
config.GetNumReplicasDependenciesFlag(),
1,
"The number of replicas per dependencies index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Sampling.TemplateOptions.NumShards,
samplingShardsFlag,
&o.Indices.Sampling.TemplateNumShards,
config.GetNumShardSamplingFlag(),
5,
"The number of shards per sampling index in Elasticsearch")
command.Flags().IntVar(
&o.Indices.Sampling.TemplateOptions.NumReplicas,
samplingReplicasFlag,
&o.Indices.Sampling.TemplateNumReplicas,
config.GetNumReplicasSamplingFlag(),
1,
"The number of replicas per sampling index in Elasticsearch")
command.Flags().StringVar(
Expand Down
2 changes: 1 addition & 1 deletion cmd/esmapping-generator/app/renderer/render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package renderer

import (
"errors"
"github.com/jaegertracing/jaeger/pkg/es/config"
"io"
"testing"

Expand All @@ -25,6 +24,7 @@ import (
"github.com/stretchr/testify/require"

"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
"github.com/jaegertracing/jaeger/pkg/es/config"
"github.com/jaegertracing/jaeger/pkg/es/mocks"
"github.com/jaegertracing/jaeger/pkg/testutils"
)
Expand Down
8 changes: 8 additions & 0 deletions cmd/jaeger/config-elasticsearch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,23 @@ extensions:
spans:
date_layout: "2006-01-02"
rollover_frequency: "day"
template_num_shards: 5
template_num_replicas: 1
services:
date_layout: "2006-01-02"
rollover_frequency: "day"
template_num_shards: 5
template_num_replicas: 1
dependencies:
date_layout: "2006-01-02"
rollover_frequency: "day"
template_num_shards: 5
template_num_replicas: 1
sampling:
date_layout: "2006-01-02"
rollover_frequency: "day"
template_num_shards: 5
template_num_replicas: 1
another_storage:
elasticsearch:
index_prefix: "jaeger-archive"
Expand Down
36 changes: 23 additions & 13 deletions pkg/es/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,11 @@ type TemplateOptions struct {

// IndexOptions describes the index format and rollover frequency
type IndexOptions struct {
DateLayout string `mapstructure:"date_layout"`
RolloverFrequency string `mapstructure:"rollover_frequency"` // "hour" or "day"
TemplateOptions TemplateOptions `mapstructure:"template"`
DateLayout string `mapstructure:"date_layout"`
RolloverFrequency string `mapstructure:"rollover_frequency"` // "hour" or "day"
TemplatePriority int `mapstructure:"template_priority"`
TemplateNumShards int `mapstructure:"template_num_shards"`
TemplateNumReplicas int `mapstructure:"template_num_replicas"`
}

// Indices describes different configuration options for each index type
Expand Down Expand Up @@ -230,17 +232,25 @@ func newElasticsearchV8(c *Configuration, logger *zap.Logger) (*esV8.Client, err
return esV8.NewClient(options)
}

func setFieldIfZero(cfg, source *IndexOptions) {
if cfg.TemplateOptions.NumShards == 0 {
cfg.TemplateOptions.NumShards = source.TemplateOptions.NumShards
func setDefaultIndexOptions(cfg, source *IndexOptions) {
if cfg.TemplateNumShards == 0 {
cfg.TemplateNumShards = source.TemplateNumShards
}

if cfg.TemplateOptions.NumReplicas == 0 {
cfg.TemplateOptions.NumReplicas = source.TemplateOptions.NumReplicas
if cfg.TemplateNumReplicas == 0 {
cfg.TemplateNumReplicas = source.TemplateNumReplicas
}

if cfg.TemplateOptions.Priority == 0 {
cfg.TemplateOptions.Priority = source.TemplateOptions.Priority
if cfg.TemplatePriority == 0 {
cfg.TemplatePriority = source.TemplatePriority
}

if cfg.DateLayout == "" {
cfg.DateLayout = source.DateLayout
}

if cfg.RolloverFrequency == "" {
cfg.RolloverFrequency = source.RolloverFrequency
}
}

Expand All @@ -265,9 +275,9 @@ func (c *Configuration) ApplyDefaults(source *Configuration) {
c.AdaptiveSamplingLookback = source.AdaptiveSamplingLookback
}

setFieldIfZero(&c.Indices.Spans, &source.Indices.Spans)
setFieldIfZero(&c.Indices.Services, &source.Indices.Services)
setFieldIfZero(&c.Indices.Dependencies, &source.Indices.Dependencies)
setDefaultIndexOptions(&c.Indices.Spans, &source.Indices.Spans)
setDefaultIndexOptions(&c.Indices.Services, &source.Indices.Services)
setDefaultIndexOptions(&c.Indices.Dependencies, &source.Indices.Dependencies)

if c.BulkSize == 0 {
c.BulkSize = source.BulkSize
Expand Down
90 changes: 90 additions & 0 deletions pkg/es/config/flagutils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import "fmt"

type IndexType string

const (
SPAN IndexType = "span"
SERVICE IndexType = "service"
DEPENDENCIES IndexType = "dependencies"
SAMPLING IndexType = "sampling"

numShards = "num-shards-"
numReplicas = "num-replicas-"
priority = "priority-%s-template"
)

func GetNumShardSpanFlag() string {
return getNumShardPrefix(SPAN)
}

func GetNumShardServiceFlag() string {
return getNumShardPrefix(SERVICE)
}

func GetNumShardSamplingFlag() string {
return getNumShardPrefix(SAMPLING)
}

func GetNumShardDependenciesFlag() string {
return getNumShardPrefix(DEPENDENCIES)
}

func GetNumReplicasSpanFlag() string {
return getNumReplicasPrefix(SPAN)
}

func GetNumReplicasServiceFlag() string {
return getNumReplicasPrefix(SERVICE)
}

func GetNumReplicasDependenciesFlag() string {
return getNumReplicasPrefix(DEPENDENCIES)
}

func GetNumReplicasSamplingFlag() string {
return getNumReplicasPrefix(SAMPLING)
}

func getNumShardPrefix(indexType IndexType) string {
return string(numShards + indexType)
}

func getNumReplicasPrefix(indexType IndexType) string {
return string(numReplicas + indexType)
}

func GetPrioritySpanTemplate() string {
return getPriorityTemplate(SPAN)
}

func GetPriorityServiceTemplate() string {
return getPriorityTemplate(SERVICE)
}

func GetPriorityDependenciesTemplate() string {
return getPriorityTemplate(DEPENDENCIES)
}

func GetPrioritySamplingTemplate() string {
return getPriorityTemplate(SAMPLING)
}

func getPriorityTemplate(indexType IndexType) string {
return fmt.Sprintf(priority, indexType)
}
Loading

0 comments on commit 20bdffb

Please sign in to comment.