From 530be87cc850571eb543b353b78c1bc50a376e18 Mon Sep 17 00:00:00 2001 From: odubajDT <93584209+odubajDT@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:49:09 +0200 Subject: [PATCH] [exporter/clickhouse]: Add the ability to override default table names for all metric types (#34251) **Description:** - add the ability to override default table names for all metric types **Link to tracking Issue:** #34225 **Testing:** - unit tests **Documentation:** --------- Signed-off-by: odubajDT --- .chloggen/metrics-tables-clickhouse.yaml | 28 ++++ exporter/clickhouseexporter/README.md | 24 ++- exporter/clickhouseexporter/config.go | 61 ++++++++ exporter/clickhouseexporter/config_test.go | 142 ++++++++++++++++-- .../clickhouseexporter/exporter_metrics.go | 28 +++- .../exporter_metrics_test.go | 22 +++ exporter/clickhouseexporter/factory.go | 9 +- .../internal/exponential_histogram_metrics.go | 4 +- .../internal/gauge_metrics.go | 4 +- .../internal/histogram_metrics.go | 4 +- .../internal/metrics_model.go | 36 +++-- .../internal/sum_metrics.go | 4 +- .../internal/summary_metrics.go | 4 +- .../clickhouseexporter/testdata/config.yaml | 11 ++ 14 files changed, 337 insertions(+), 44 deletions(-) create mode 100644 .chloggen/metrics-tables-clickhouse.yaml diff --git a/.chloggen/metrics-tables-clickhouse.yaml b/.chloggen/metrics-tables-clickhouse.yaml new file mode 100644 index 000000000000..fb63b0eeb94f --- /dev/null +++ b/.chloggen/metrics-tables-clickhouse.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/clickhouse + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add the ability to override default table names for all metric types. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34225] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + 'metrics_table_name' of the clickhouse exporter config is deprecated and newly introduced parameter 'metrics_tables' should be used instead. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index d4d3e52c90ff..ca27d4cf04c9 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -294,7 +294,17 @@ ClickHouse tables: - `logs_table_name` (default = otel_logs): The table name for logs. - `traces_table_name` (default = otel_traces): The table name for traces. -- `metrics_table_name` (default = otel_metrics): The table name for metrics. +- `metrics_tables` + - `gauge` + - `name` (default = "otel_metrics_gauge") + - `sum` + - `name` (default = "otel_metrics_sum") + - `summary` + - `name` (default = "otel_metrics_summary") + - `histogram` + - `name` (default = "otel_metrics_histogram") + - `exponential_histogram` + - `name` (default = "otel_metrics_exp_histogram") Cluster definition: @@ -365,8 +375,18 @@ exporters: create_schema: true logs_table_name: otel_logs traces_table_name: otel_traces - metrics_table_name: otel_metrics timeout: 5s + metrics_tables: + gauge: + name: "otel_metrics_gauge" + sum: + name: "otel_metrics_sum" + summary: + name: "otel_metrics_summary" + histogram: + name: "otel_metrics_histogram" + exponential_histogram: + name: "otel_metrics_exp_histogram" retry_on_failure: enabled: true initial_interval: 5s diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index 15c6cf90c9f5..f21d96a77830 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -14,6 +14,8 @@ import ( "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" ) // Config defines configuration for Elastic exporter. @@ -37,6 +39,10 @@ type Config struct { // TracesTableName is the table name for traces. default is `otel_traces`. TracesTableName string `mapstructure:"traces_table_name"` // MetricsTableName is the table name for metrics. default is `otel_metrics`. + // + // Deprecated: MetricsTableName exists for historical compatibility + // and should not be used. To set the metrics tables name, + // use the MetricsTables parameter instead. MetricsTableName string `mapstructure:"metrics_table_name"` // TTL is The data time-to-live example 30m, 48h. 0 means no ttl. TTL time.Duration `mapstructure:"ttl"` @@ -52,6 +58,21 @@ type Config struct { // Ignored if async inserts are configured in the `endpoint` or `connection_params`. // Async inserts may still be overridden server-side. AsyncInsert bool `mapstructure:"async_insert"` + // MetricsTables defines the table names for metric types. + MetricsTables MetricTablesConfig `mapstructure:"metrics_tables"` +} + +type MetricTablesConfig struct { + // Gauge is the table name for gauge metric type. default is `otel_metrics_gauge`. + Gauge internal.MetricTypeConfig `mapstructure:"gauge"` + // Sum is the table name for sum metric type. default is `otel_metrics_sum`. + Sum internal.MetricTypeConfig `mapstructure:"sum"` + // Summary is the table name for summary metric type. default is `otel_metrics_summary`. + Summary internal.MetricTypeConfig `mapstructure:"summary"` + // Histogram is the table name for histogram metric type. default is `otel_metrics_histogram`. + Histogram internal.MetricTypeConfig `mapstructure:"histogram"` + // ExponentialHistogram is the table name for exponential histogram metric type. default is `otel_metrics_exponential_histogram`. + ExponentialHistogram internal.MetricTypeConfig `mapstructure:"exponential_histogram"` } // TableEngine defines the ENGINE string value when creating the table. @@ -62,6 +83,12 @@ type TableEngine struct { const defaultDatabase = "default" const defaultTableEngineName = "MergeTree" +const defaultMetricTableName = "otel_metrics" +const defaultGaugeSuffix = "_gauge" +const defaultSumSuffix = "_sum" +const defaultSummarySuffix = "_summary" +const defaultHistogramSuffix = "_histogram" +const defaultExpHistogramSuffix = "_exponential_histogram" var ( errConfigNoEndpoint = errors.New("endpoint must be specified") @@ -78,6 +105,8 @@ func (cfg *Config) Validate() (err error) { err = errors.Join(err, e) } + cfg.buildMetricTableNames() + // Validate DSN with clickhouse driver. // Last chance to catch invalid config. if _, e := clickhouse.ParseDSN(dsn); e != nil { @@ -153,6 +182,38 @@ func (cfg *Config) shouldCreateSchema() bool { return cfg.CreateSchema } +func (cfg *Config) buildMetricTableNames() { + tableName := defaultMetricTableName + + if len(cfg.MetricsTableName) != 0 && !cfg.areMetricTableNamesSet() { + tableName = cfg.MetricsTableName + } + + if len(cfg.MetricsTables.Gauge.Name) == 0 { + cfg.MetricsTables.Gauge.Name = tableName + defaultGaugeSuffix + } + if len(cfg.MetricsTables.Sum.Name) == 0 { + cfg.MetricsTables.Sum.Name = tableName + defaultSumSuffix + } + if len(cfg.MetricsTables.Summary.Name) == 0 { + cfg.MetricsTables.Summary.Name = tableName + defaultSummarySuffix + } + if len(cfg.MetricsTables.Histogram.Name) == 0 { + cfg.MetricsTables.Histogram.Name = tableName + defaultHistogramSuffix + } + if len(cfg.MetricsTables.ExponentialHistogram.Name) == 0 { + cfg.MetricsTables.ExponentialHistogram.Name = tableName + defaultExpHistogramSuffix + } +} + +func (cfg *Config) areMetricTableNamesSet() bool { + return len(cfg.MetricsTables.Gauge.Name) != 0 || + len(cfg.MetricsTables.Sum.Name) != 0 || + len(cfg.MetricsTables.Summary.Name) != 0 || + len(cfg.MetricsTables.Histogram.Name) != 0 || + len(cfg.MetricsTables.ExponentialHistogram.Name) != 0 +} + // tableEngineString generates the ENGINE string. func (cfg *Config) tableEngineString() string { engine := cfg.TableEngine.Name diff --git a/exporter/clickhouseexporter/config_test.go b/exporter/clickhouseexporter/config_test.go index b97ba553e41a..b4b3ff73c633 100644 --- a/exporter/clickhouseexporter/config_test.go +++ b/exporter/clickhouseexporter/config_test.go @@ -19,6 +19,7 @@ import ( "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/exporter/exporterhelper" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal/metadata" ) @@ -47,15 +48,14 @@ func TestLoadConfig(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "full"), expected: &Config{ - Endpoint: defaultEndpoint, - Database: "otel", - Username: "foo", - Password: "bar", - TTL: 72 * time.Hour, - LogsTableName: "otel_logs", - TracesTableName: "otel_traces", - MetricsTableName: "otel_metrics", - CreateSchema: true, + Endpoint: defaultEndpoint, + Database: "otel", + Username: "foo", + Password: "bar", + TTL: 72 * time.Hour, + LogsTableName: "otel_logs", + TracesTableName: "otel_traces", + CreateSchema: true, TimeoutSettings: exporterhelper.TimeoutConfig{ Timeout: 5 * time.Second, }, @@ -67,6 +67,13 @@ func TestLoadConfig(t *testing.T) { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "otel_metrics_custom_gauge"}, + Sum: internal.MetricTypeConfig{Name: "otel_metrics_custom_sum"}, + Summary: internal.MetricTypeConfig{Name: "otel_metrics_custom_summary"}, + Histogram: internal.MetricTypeConfig{Name: "otel_metrics_custom_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "otel_metrics_custom_exp_histogram"}, + }, ConnectionParams: map[string]string{}, QueueSettings: exporterhelper.QueueConfig{ Enabled: true, @@ -102,6 +109,123 @@ func withDefaultConfig(fns ...func(*Config)) *Config { return cfg } +func TestBuildMetricMetricTableNames(t *testing.T) { + tests := []struct { + name string + cfg Config + want Config + }{ + { + name: "nothing set", + cfg: Config{}, + want: Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "otel_metrics_gauge"}, + Sum: internal.MetricTypeConfig{Name: "otel_metrics_sum"}, + Summary: internal.MetricTypeConfig{Name: "otel_metrics_summary"}, + Histogram: internal.MetricTypeConfig{Name: "otel_metrics_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "otel_metrics_exponential_histogram"}, + }, + }, + }, + { + name: "only metric_table_name set", + cfg: Config{ + MetricsTableName: "table_name", + }, + want: Config{ + MetricsTableName: "table_name", + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "table_name_gauge"}, + Sum: internal.MetricTypeConfig{Name: "table_name_sum"}, + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exponential_histogram"}, + }, + }, + }, + { + name: "only metric_tables set fully", + cfg: Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "table_name_gauge"}, + Sum: internal.MetricTypeConfig{Name: "table_name_sum"}, + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exponential_histogram"}, + }, + }, + want: Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "table_name_gauge"}, + Sum: internal.MetricTypeConfig{Name: "table_name_sum"}, + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exponential_histogram"}, + }, + }, + }, + { + name: "only metric_tables set partially", + cfg: Config{ + MetricsTables: MetricTablesConfig{ + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exp_histogram"}, + }, + }, + want: Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "otel_metrics_gauge"}, + Sum: internal.MetricTypeConfig{Name: "otel_metrics_sum"}, + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exp_histogram"}, + }, + }, + }, + { + name: "only metric_tables set partially with metric_table_name", + cfg: Config{ + MetricsTableName: "custom_name", + MetricsTables: MetricTablesConfig{ + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exp_histogram"}, + }, + }, + want: Config{ + MetricsTableName: "custom_name", + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "otel_metrics_gauge"}, + Sum: internal.MetricTypeConfig{Name: "otel_metrics_sum"}, + Summary: internal.MetricTypeConfig{Name: "table_name_summary"}, + Histogram: internal.MetricTypeConfig{Name: "table_name_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "table_name_exp_histogram"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.cfg.buildMetricTableNames() + require.Equal(t, tt.want, tt.cfg) + }) + } +} + +func TestAreMetricTableNamesSet(t *testing.T) { + cfg := Config{} + require.False(t, cfg.areMetricTableNamesSet()) + + cfg = Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "gauge"}, + }, + } + require.True(t, cfg.areMetricTableNamesSet()) +} + func TestConfig_buildDSN(t *testing.T) { type fields struct { Endpoint string diff --git a/exporter/clickhouseexporter/exporter_metrics.go b/exporter/clickhouseexporter/exporter_metrics.go index 6f11ba940d57..66f67b5cc065 100644 --- a/exporter/clickhouseexporter/exporter_metrics.go +++ b/exporter/clickhouseexporter/exporter_metrics.go @@ -19,8 +19,9 @@ import ( type metricsExporter struct { client *sql.DB - logger *zap.Logger - cfg *Config + logger *zap.Logger + cfg *Config + tablesConfig internal.MetricTablesConfigMapper } func newMetricsExporter(logger *zap.Logger, cfg *Config) (*metricsExporter, error) { @@ -29,10 +30,13 @@ func newMetricsExporter(logger *zap.Logger, cfg *Config) (*metricsExporter, erro return nil, err } + tablesConfig := generateMetricTablesConfigMapper(cfg) + return &metricsExporter{ - client: client, - logger: logger, - cfg: cfg, + client: client, + logger: logger, + cfg: cfg, + tablesConfig: tablesConfig, }, nil } @@ -48,7 +52,17 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error { } ttlExpr := generateTTLExpr(e.cfg.TTL, "toDateTime(TimeUnix)") - return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.clusterString(), e.cfg.tableEngineString(), ttlExpr, e.client) + return internal.NewMetricsTable(ctx, e.tablesConfig, e.cfg.clusterString(), e.cfg.tableEngineString(), ttlExpr, e.client) +} + +func generateMetricTablesConfigMapper(cfg *Config) internal.MetricTablesConfigMapper { + return internal.MetricTablesConfigMapper{ + pmetric.MetricTypeGauge: cfg.MetricsTables.Gauge, + pmetric.MetricTypeSum: cfg.MetricsTables.Sum, + pmetric.MetricTypeSummary: cfg.MetricsTables.Summary, + pmetric.MetricTypeHistogram: cfg.MetricsTables.Histogram, + pmetric.MetricTypeExponentialHistogram: cfg.MetricsTables.ExponentialHistogram, + } } // shutdown will shut down the exporter. @@ -60,7 +74,7 @@ func (e *metricsExporter) shutdown(_ context.Context) error { } func (e *metricsExporter) pushMetricsData(ctx context.Context, md pmetric.Metrics) error { - metricsMap := internal.NewMetricsModel(e.cfg.MetricsTableName) + metricsMap := internal.NewMetricsModel(e.tablesConfig) for i := 0; i < md.ResourceMetrics().Len(); i++ { metrics := md.ResourceMetrics().At(i) resAttr := attributesToMap(metrics.Resource().Attributes()) diff --git a/exporter/clickhouseexporter/exporter_metrics_test.go b/exporter/clickhouseexporter/exporter_metrics_test.go index 5a705c3b6862..2000303cab67 100644 --- a/exporter/clickhouseexporter/exporter_metrics_test.go +++ b/exporter/clickhouseexporter/exporter_metrics_test.go @@ -17,6 +17,8 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap/zaptest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" ) func TestMetricsClusterConfig(t *testing.T) { @@ -33,6 +35,26 @@ func TestMetricsTableEngineConfig(t *testing.T) { }) } +func Test_generateMetricMetricTableNames(t *testing.T) { + cfg := Config{ + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: "otel_metrics_custom_gauge"}, + Sum: internal.MetricTypeConfig{Name: "otel_metrics_custom_sum"}, + Summary: internal.MetricTypeConfig{Name: "otel_metrics_custom_summary"}, + Histogram: internal.MetricTypeConfig{Name: "otel_metrics_custom_histogram"}, + ExponentialHistogram: internal.MetricTypeConfig{Name: "otel_metrics_custom_exp_histogram"}, + }, + } + + require.Equal(t, internal.MetricTablesConfigMapper{ + pmetric.MetricTypeGauge: cfg.MetricsTables.Gauge, + pmetric.MetricTypeSum: cfg.MetricsTables.Sum, + pmetric.MetricTypeSummary: cfg.MetricsTables.Summary, + pmetric.MetricTypeHistogram: cfg.MetricsTables.Histogram, + pmetric.MetricTypeExponentialHistogram: cfg.MetricsTables.ExponentialHistogram, + }, generateMetricTablesConfigMapper(&cfg)) +} + func TestExporter_pushMetricsData(t *testing.T) { t.Parallel() t.Run("push success", func(t *testing.T) { diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index c4f632cab2f8..6faf3e3acfdb 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -15,6 +15,7 @@ import ( "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal/metadata" ) @@ -38,10 +39,16 @@ func createDefaultConfig() component.Config { Database: defaultDatabase, LogsTableName: "otel_logs", TracesTableName: "otel_traces", - MetricsTableName: "otel_metrics", TTL: 0, CreateSchema: true, AsyncInsert: true, + MetricsTables: MetricTablesConfig{ + Gauge: internal.MetricTypeConfig{Name: defaultMetricTableName + defaultGaugeSuffix}, + Sum: internal.MetricTypeConfig{Name: defaultMetricTableName + defaultSumSuffix}, + Summary: internal.MetricTypeConfig{Name: defaultMetricTableName + defaultSummarySuffix}, + Histogram: internal.MetricTypeConfig{Name: defaultMetricTableName + defaultHistogramSuffix}, + ExponentialHistogram: internal.MetricTypeConfig{Name: defaultMetricTableName + defaultExpHistogramSuffix}, + }, } } diff --git a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go index 380ef9a581eb..82d93dcf366f 100644 --- a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go @@ -18,7 +18,7 @@ import ( const ( // language=ClickHouse SQL createExpHistogramTableSQL = ` -CREATE TABLE IF NOT EXISTS %s_exponential_histogram %s ( +CREATE TABLE IF NOT EXISTS %s %s ( ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), ResourceSchemaUrl String CODEC(ZSTD(1)), ScopeName String CODEC(ZSTD(1)), @@ -65,7 +65,7 @@ ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL - insertExpHistogramTableSQL = `INSERT INTO %s_exponential_histogram ( + insertExpHistogramTableSQL = `INSERT INTO %s ( ResourceAttributes, ResourceSchemaUrl, ScopeName, diff --git a/exporter/clickhouseexporter/internal/gauge_metrics.go b/exporter/clickhouseexporter/internal/gauge_metrics.go index 62db51fea5a7..596f3fa654ed 100644 --- a/exporter/clickhouseexporter/internal/gauge_metrics.go +++ b/exporter/clickhouseexporter/internal/gauge_metrics.go @@ -18,7 +18,7 @@ import ( const ( // language=ClickHouse SQL createGaugeTableSQL = ` -CREATE TABLE IF NOT EXISTS %s_gauge %s ( +CREATE TABLE IF NOT EXISTS %s %s ( ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), ResourceSchemaUrl String CODEC(ZSTD(1)), ScopeName String CODEC(ZSTD(1)), @@ -55,7 +55,7 @@ ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL - insertGaugeTableSQL = `INSERT INTO %s_gauge ( + insertGaugeTableSQL = `INSERT INTO %s ( ResourceAttributes, ResourceSchemaUrl, ScopeName, diff --git a/exporter/clickhouseexporter/internal/histogram_metrics.go b/exporter/clickhouseexporter/internal/histogram_metrics.go index eed55f8b0d4e..544e019cf8a3 100644 --- a/exporter/clickhouseexporter/internal/histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/histogram_metrics.go @@ -18,7 +18,7 @@ import ( const ( // language=ClickHouse SQL createHistogramTableSQL = ` -CREATE TABLE IF NOT EXISTS %s_histogram %s ( +CREATE TABLE IF NOT EXISTS %s %s ( ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), ResourceSchemaUrl String CODEC(ZSTD(1)), ScopeName String CODEC(ZSTD(1)), @@ -61,7 +61,7 @@ ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL - insertHistogramTableSQL = `INSERT INTO %s_histogram ( + insertHistogramTableSQL = `INSERT INTO %s ( ResourceAttributes, ResourceSchemaUrl, ScopeName, diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index b4a1e6a3ab71..92c51b550643 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -18,16 +18,22 @@ import ( "go.uber.org/zap" ) -var supportedMetricTypes = map[string]struct{}{ - createGaugeTableSQL: {}, - createSumTableSQL: {}, - createHistogramTableSQL: {}, - createExpHistogramTableSQL: {}, - createSummaryTableSQL: {}, +var supportedMetricTypes = map[pmetric.MetricType]string{ + pmetric.MetricTypeGauge: createGaugeTableSQL, + pmetric.MetricTypeSum: createSumTableSQL, + pmetric.MetricTypeHistogram: createHistogramTableSQL, + pmetric.MetricTypeExponentialHistogram: createExpHistogramTableSQL, + pmetric.MetricTypeSummary: createSummaryTableSQL, } var logger *zap.Logger +type MetricTablesConfigMapper map[pmetric.MetricType]MetricTypeConfig + +type MetricTypeConfig struct { + Name string `mapstructure:"name"` +} + // MetricsModel is used to group metric data and insert into clickhouse // any type of metrics need implement it. type MetricsModel interface { @@ -51,9 +57,9 @@ func SetLogger(l *zap.Logger) { } // NewMetricsTable create metric tables with an expiry time to storage metric telemetry data -func NewMetricsTable(ctx context.Context, tableName, cluster, engine, ttlExpr string, db *sql.DB) error { - for table := range supportedMetricTypes { - query := fmt.Sprintf(table, tableName, cluster, engine, ttlExpr) +func NewMetricsTable(ctx context.Context, tablesConfig MetricTablesConfigMapper, cluster, engine, ttlExpr string, db *sql.DB) error { + for key, queryTemplate := range supportedMetricTypes { + query := fmt.Sprintf(queryTemplate, tablesConfig[key].Name, cluster, engine, ttlExpr) if _, err := db.ExecContext(ctx, query); err != nil { return fmt.Errorf("exec create metrics table sql: %w", err) } @@ -62,22 +68,22 @@ func NewMetricsTable(ctx context.Context, tableName, cluster, engine, ttlExpr st } // NewMetricsModel create a model for contain different metric data -func NewMetricsModel(tableName string) map[pmetric.MetricType]MetricsModel { +func NewMetricsModel(tablesConfig MetricTablesConfigMapper) map[pmetric.MetricType]MetricsModel { return map[pmetric.MetricType]MetricsModel{ pmetric.MetricTypeGauge: &gaugeMetrics{ - insertSQL: fmt.Sprintf(insertGaugeTableSQL, tableName), + insertSQL: fmt.Sprintf(insertGaugeTableSQL, tablesConfig[pmetric.MetricTypeGauge].Name), }, pmetric.MetricTypeSum: &sumMetrics{ - insertSQL: fmt.Sprintf(insertSumTableSQL, tableName), + insertSQL: fmt.Sprintf(insertSumTableSQL, tablesConfig[pmetric.MetricTypeSum].Name), }, pmetric.MetricTypeHistogram: &histogramMetrics{ - insertSQL: fmt.Sprintf(insertHistogramTableSQL, tableName), + insertSQL: fmt.Sprintf(insertHistogramTableSQL, tablesConfig[pmetric.MetricTypeHistogram].Name), }, pmetric.MetricTypeExponentialHistogram: &expHistogramMetrics{ - insertSQL: fmt.Sprintf(insertExpHistogramTableSQL, tableName), + insertSQL: fmt.Sprintf(insertExpHistogramTableSQL, tablesConfig[pmetric.MetricTypeExponentialHistogram].Name), }, pmetric.MetricTypeSummary: &summaryMetrics{ - insertSQL: fmt.Sprintf(insertSummaryTableSQL, tableName), + insertSQL: fmt.Sprintf(insertSummaryTableSQL, tablesConfig[pmetric.MetricTypeSummary].Name), }, } } diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index eab79064140a..4da0faa5cb3a 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -18,7 +18,7 @@ import ( const ( // language=ClickHouse SQL createSumTableSQL = ` -CREATE TABLE IF NOT EXISTS %s_sum %s ( +CREATE TABLE IF NOT EXISTS %s %s ( ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), ResourceSchemaUrl String CODEC(ZSTD(1)), ScopeName String CODEC(ZSTD(1)), @@ -57,7 +57,7 @@ ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL - insertSumTableSQL = `INSERT INTO %s_sum ( + insertSumTableSQL = `INSERT INTO %s ( ResourceAttributes, ResourceSchemaUrl, ScopeName, diff --git a/exporter/clickhouseexporter/internal/summary_metrics.go b/exporter/clickhouseexporter/internal/summary_metrics.go index 994616dcd6dc..3182ffee452c 100644 --- a/exporter/clickhouseexporter/internal/summary_metrics.go +++ b/exporter/clickhouseexporter/internal/summary_metrics.go @@ -18,7 +18,7 @@ import ( const ( // language=ClickHouse SQL createSummaryTableSQL = ` -CREATE TABLE IF NOT EXISTS %s_summary %s ( +CREATE TABLE IF NOT EXISTS %s %s ( ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), ResourceSchemaUrl String CODEC(ZSTD(1)), ScopeName String CODEC(ZSTD(1)), @@ -53,7 +53,7 @@ ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL - insertSummaryTableSQL = `INSERT INTO %s_summary ( + insertSummaryTableSQL = `INSERT INTO %s ( ResourceAttributes, ResourceSchemaUrl, ScopeName, diff --git a/exporter/clickhouseexporter/testdata/config.yaml b/exporter/clickhouseexporter/testdata/config.yaml index 1531e4578782..9eb443cde3da 100644 --- a/exporter/clickhouseexporter/testdata/config.yaml +++ b/exporter/clickhouseexporter/testdata/config.yaml @@ -17,6 +17,17 @@ clickhouse/full: sending_queue: queue_size: 100 storage: file_storage/clickhouse + metrics_tables: + gauge: + name: "otel_metrics_custom_gauge" + sum: + name: "otel_metrics_custom_sum" + summary: + name: "otel_metrics_custom_summary" + histogram: + name: "otel_metrics_custom_histogram" + exponential_histogram: + name: "otel_metrics_custom_exp_histogram" clickhouse/invalid-endpoint: endpoint: 127.0.0.1:9000