prometheusexporter: Automatic normalization of metrics from Otel to P…

…rometheus naming convention (#10028)

* Issue #8950 `prometheusexporter`: Automatic rename of metrics to follow Prometheus naming convention

* Updated **prometheusremotewriteexporter** to normalize metric names
* Moved common code to new module /pkg/translator/prometheus
* Added documentation

Co-authored-by: Dan Jaglowski <[email protected]>

.github/ALLOWLIST

@@ -28,6 +28,7 @@ internal/sharedcomponent
 internal/tools
 pkg/batchperresourceattr
 pkg/experimentalmetricmetadata
+pkg/translator/prometheus
 pkg/translator/prometheusremotewrite
 pkg/translator/signalfx
 pkg/winperfcounters

.github/CODEOWNERS

@@ -14,7 +14,7 @@
 # NOTE: Lines should be entered in the following format:
 # <component_path_relative_from_project_root>/<min_1_space><owner_1><space><owner_2><space>..<owner_n>
 # extension/oauth2clientauthextension/                 @open-telemetry/collector-contrib-approvers @pavankrish123 @jpkrohling
-# Path separator and minimum of 1 space between component path and owners is 
+# Path separator and minimum of 1 space between component path and owners is
 # important for validation steps
 #
 

.github/dependabot.yml

@@ -374,6 +374,10 @@ updates:
     directory: "/pkg/translator/opencensus"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/pkg/translator/prometheus"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod"
     directory: "/pkg/translator/prometheusremotewrite"
     schedule:

CHANGELOG.md

@@ -76,6 +76,10 @@
 - `pkg/stanza`: Removed reference to deprecated `ClusterName` (#10426)
 - `couchbasereceiver`: Fully removed unimplemented Couchbase receiver (#10482)
 - `hostmetricsreciever`: Fix Load Scraper to normalize 1m, 5m, and 15m averages independently (#8267)
+- `prometheusexporter`: Automatically rename metrics with units to follow Prometheus naming convention (#8950)
+
+### 🚩 Deprecations 🚩
+
 
 ### 🚀 New components 🚀
 

cmd/configschema/go.mod

@@ -324,6 +324,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.54.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.54.0 // indirect
@@ -714,6 +715,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => ../../pkg/translator/prometheus
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => ../../pkg/translator/prometheusremotewrite
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx => ../../pkg/translator/signalfx

exporter/awsprometheusremotewriteexporter/go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.54.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.54.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.35.0 // indirect
@@ -66,4 +67,6 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/corei
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => ../../pkg/resourcetotelemetry
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => ../../pkg/translator/prometheus
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => ../../pkg/translator/prometheusremotewrite

exporter/prometheusexporter/README.md

@@ -40,6 +40,10 @@ exporters:
       enabled: true
 ```
 
+## Metric names and labels normalization
+
+OpenTelemetry metric names and attributes are normalized to be compliant with Prometheus naming rules. [Details on this normalization process are described in the Prometheus translator module](../../pkg/translator/prometheus/).
+
 [beta]:https://github.com/open-telemetry/opentelemetry-collector#beta
 [contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
-[core]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol
+[core]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol

exporter/prometheusexporter/collector.go

@@ -24,26 +24,26 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
 	"go.uber.org/zap"
+
+	prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
 )
 
 type collector struct {
 	accumulator accumulator
 	logger      *zap.Logger
 
-	sendTimestamps    bool
-	namespace         string
-	constLabels       prometheus.Labels
-	skipSanitizeLabel bool
+	sendTimestamps bool
+	namespace      string
+	constLabels    prometheus.Labels
 }
 
 func newCollector(config *Config, logger *zap.Logger) *collector {
 	return &collector{
-		accumulator:       newAccumulator(logger, config.MetricExpiration),
-		logger:            logger,
-		namespace:         sanitize(config.Namespace, config.skipSanitizeLabel),
-		sendTimestamps:    config.SendTimestamps,
-		constLabels:       config.ConstLabels,
-		skipSanitizeLabel: config.skipSanitizeLabel,
+		accumulator:    newAccumulator(logger, config.MetricExpiration),
+		logger:         logger,
+		namespace:      prometheustranslator.CleanUpString(config.Namespace),
+		sendTimestamps: config.SendTimestamps,
+		constLabels:    config.ConstLabels,
 	}
 }
 
@@ -75,19 +75,12 @@ func (c *collector) convertMetric(metric pmetric.Metric, resourceAttrs pcommon.M
 	return nil, errUnknownMetricType
 }
 
-func (c *collector) metricName(namespace string, metric pmetric.Metric) string {
-	if namespace != "" {
-		return namespace + "_" + sanitize(metric.Name(), c.skipSanitizeLabel)
-	}
-	return sanitize(metric.Name(), c.skipSanitizeLabel)
-}
-
 func (c *collector) getMetricMetadata(metric pmetric.Metric, attributes pcommon.Map, resourceAttrs pcommon.Map) (*prometheus.Desc, []string) {
 	keys := make([]string, 0, attributes.Len()+2) // +2 for job and instance labels.
 	values := make([]string, 0, attributes.Len()+2)
 
 	attributes.Range(func(k string, v pcommon.Value) bool {
-		keys = append(keys, sanitize(k, c.skipSanitizeLabel))
+		keys = append(keys, prometheustranslator.NormalizeLabel(k))
 		values = append(values, v.AsString())
 		return true
 	})
@@ -108,7 +101,7 @@ func (c *collector) getMetricMetadata(metric pmetric.Metric, attributes pcommon.
 	}
 
 	return prometheus.NewDesc(
-		c.metricName(c.namespace, metric),
+		prometheustranslator.BuildPromCompliantName(metric, c.namespace),
 		metric.Description(),
 		keys,
 		c.constLabels,

exporter/prometheusexporter/config.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/collector/config"
-	"go.opentelemetry.io/collector/service/featuregate"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry"
 )
@@ -45,19 +44,10 @@ type Config struct {
 
 	// ResourceToTelemetrySettings defines configuration for converting resource attributes to metric labels.
 	ResourceToTelemetrySettings resourcetotelemetry.Settings `mapstructure:"resource_to_telemetry_conversion"`
-
-	// skipSanitizeLabel if enabled, labels that start with _ are not sanitized
-	skipSanitizeLabel bool
 }
 
 var _ config.Exporter = (*Config)(nil)
 
-var dropSanitizationGate = featuregate.Gate{
-	ID:          "exporter.prometheus.PermissiveLabelSanitization",
-	Enabled:     false,
-	Description: "Controls whether to change labels starting with '_' to 'key_'",
-}
-
 // Validate checks if the exporter configuration is valid
 func (cfg *Config) Validate() error {
 	return nil

exporter/prometheusexporter/config_test.go

@@ -27,6 +27,7 @@ import (
 )
 
 func TestLoadConfig(t *testing.T) {
+
 	factories, err := componenttest.NopFactories()
 	assert.NoError(t, err)
 
@@ -50,8 +51,8 @@ func TestLoadConfig(t *testing.T) {
 				"label1":        "value1",
 				"another label": "spaced value",
 			},
-			SendTimestamps:    true,
-			MetricExpiration:  60 * time.Minute,
-			skipSanitizeLabel: false,
+			SendTimestamps:   true,
+			MetricExpiration: 60 * time.Minute,
 		})
+
 }

exporter/prometheusexporter/factory.go

@@ -22,7 +22,6 @@ import (
 	"go.opentelemetry.io/collector/config"
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
-	"go.opentelemetry.io/collector/service/featuregate"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry"
 )
@@ -42,11 +41,10 @@ func NewFactory() component.ExporterFactory {
 
 func createDefaultConfig() config.Exporter {
 	return &Config{
-		ExporterSettings:  config.NewExporterSettings(config.NewComponentID(typeStr)),
-		ConstLabels:       map[string]string{},
-		SendTimestamps:    false,
-		MetricExpiration:  time.Minute * 5,
-		skipSanitizeLabel: featuregate.GetRegistry().IsEnabled(dropSanitizationGate.ID),
+		ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+		ConstLabels:      map[string]string{},
+		SendTimestamps:   false,
+		MetricExpiration: time.Minute * 5,
 	}
 }
 

exporter/prometheusexporter/go.mod

@@ -5,6 +5,7 @@ go 1.17
 require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.54.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.54.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.54.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.54.0
 	github.com/prometheus/client_golang v1.12.2
 	github.com/prometheus/client_model v0.2.0
@@ -153,8 +154,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/corei
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus
 
-replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => ../../pkg/translator/prometheusremotewrite
-
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => ../../pkg/resourcetotelemetry
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => ../../receiver/prometheusreceiver
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => ../../pkg/translator/prometheus

exporter/prometheusexporter/prometheus_test.go

@@ -141,14 +141,14 @@ func TestPrometheusExporter_endToEnd(t *testing.T) {
 		blob, _ := ioutil.ReadAll(res.Body)
 		_ = res.Body.Close()
 		want := []string{
-			`# HELP test_metric_1_this_one_there_where_ Extra ones`,
-			`# TYPE test_metric_1_this_one_there_where_ counter`,
-			fmt.Sprintf(`test_metric_1_this_one_there_where_{arch="x86",code1="one1",foo1="bar1",os="windows"} %v`, 99+128),
-			fmt.Sprintf(`test_metric_1_this_one_there_where_{arch="x86",code1="one1",foo1="bar1",os="linux"} %v`, 100+128),
-			`# HELP test_metric_2_this_one_there_where_ Extra ones`,
-			`# TYPE test_metric_2_this_one_there_where_ counter`,
-			fmt.Sprintf(`test_metric_2_this_one_there_where_{arch="x86",code1="one1",foo1="bar1",os="windows"} %v`, 99+delta),
-			fmt.Sprintf(`test_metric_2_this_one_there_where_{arch="x86",code1="one1",foo1="bar1",os="linux"} %v`, 100+delta),
+			`# HELP test_metric_1_this_one_there_where Extra ones`,
+			`# TYPE test_metric_1_this_one_there_where counter`,
+			fmt.Sprintf(`test_metric_1_this_one_there_where{arch="x86",code1="one1",foo1="bar1",os="windows"} %v`, 99+128),
+			fmt.Sprintf(`test_metric_1_this_one_there_where{arch="x86",code1="one1",foo1="bar1",os="linux"} %v`, 100+128),
+			`# HELP test_metric_2_this_one_there_where Extra ones`,
+			`# TYPE test_metric_2_this_one_there_where counter`,
+			fmt.Sprintf(`test_metric_2_this_one_there_where{arch="x86",code1="one1",foo1="bar1",os="windows"} %v`, 99+delta),
+			fmt.Sprintf(`test_metric_2_this_one_there_where{arch="x86",code1="one1",foo1="bar1",os="linux"} %v`, 100+delta),
 		}
 
 		for _, w := range want {
@@ -217,14 +217,14 @@ func TestPrometheusExporter_endToEndWithTimestamps(t *testing.T) {
 		blob, _ := ioutil.ReadAll(res.Body)
 		_ = res.Body.Close()
 		want := []string{
-			`# HELP test_metric_1_this_one_there_where_ Extra ones`,
-			`# TYPE test_metric_1_this_one_there_where_ counter`,
-			fmt.Sprintf(`test_metric_1_this_one_there_where_{arch="x86",code2="one2",foo2="bar2",os="windows"} %v %v`, 99+128, 1543160298100+128000),
-			fmt.Sprintf(`test_metric_1_this_one_there_where_{arch="x86",code2="one2",foo2="bar2",os="linux"} %v %v`, 100+128, 1543160298100),
-			`# HELP test_metric_2_this_one_there_where_ Extra ones`,
-			`# TYPE test_metric_2_this_one_there_where_ counter`,
-			fmt.Sprintf(`test_metric_2_this_one_there_where_{arch="x86",code2="one2",foo2="bar2",os="windows"} %v %v`, 99+delta, 1543160298100+delta*1000),
-			fmt.Sprintf(`test_metric_2_this_one_there_where_{arch="x86",code2="one2",foo2="bar2",os="linux"} %v %v`, 100+delta, 1543160298100),
+			`# HELP test_metric_1_this_one_there_where Extra ones`,
+			`# TYPE test_metric_1_this_one_there_where counter`,
+			fmt.Sprintf(`test_metric_1_this_one_there_where{arch="x86",code2="one2",foo2="bar2",os="windows"} %v %v`, 99+128, 1543160298100+128000),
+			fmt.Sprintf(`test_metric_1_this_one_there_where{arch="x86",code2="one2",foo2="bar2",os="linux"} %v %v`, 100+128, 1543160298100),
+			`# HELP test_metric_2_this_one_there_where Extra ones`,
+			`# TYPE test_metric_2_this_one_there_where counter`,
+			fmt.Sprintf(`test_metric_2_this_one_there_where{arch="x86",code2="one2",foo2="bar2",os="windows"} %v %v`, 99+delta, 1543160298100+delta*1000),
+			fmt.Sprintf(`test_metric_2_this_one_there_where{arch="x86",code2="one2",foo2="bar2",os="linux"} %v %v`, 100+delta, 1543160298100),
 		}
 
 		for _, w := range want {