Prometheus metric exporter (#378)

prometheus-exporter: initial commit

examples/metrics/prometheus.py

@@ -0,0 +1,55 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module serves as an example for a simple application using metrics
+Examples show how to recording affects the collection of metrics to be exported
+"""
+
+from prometheus_client import start_http_server
+
+from opentelemetry import metrics
+from opentelemetry.ext.prometheus import PrometheusMetricsExporter
+from opentelemetry.sdk.metrics import Counter, Meter
+from opentelemetry.sdk.metrics.export.controller import PushController
+
+# Start Prometheus client
+start_http_server(port=8000, addr="localhost")
+
+# Meter is responsible for creating and recording metrics
+metrics.set_preferred_meter_implementation(lambda _: Meter())
+meter = metrics.meter()
+# exporter to export metrics to Prometheus
+prefix = "MyAppPrefix"
+exporter = PrometheusMetricsExporter(prefix)
+# controller collects metrics created from meter and exports it via the
+# exporter every interval
+controller = PushController(meter, exporter, 5)
+
+counter = meter.create_metric(
+    "requests",
+    "number of requests",
+    "requests",
+    int,
+    Counter,
+    ("environment",),
+)
+
+# Labelsets are used to identify key-values that are associated with a specific
+# metric that you want to record. These are useful for pre-aggregation and can
+# be used to store custom dimensions pertaining to a metric
+label_set = meter.get_label_set({"environment": "staging"})
+
+counter.add(25, label_set)
+input("Press any key to exit...")

ext/opentelemetry-ext-prometheus/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## Unreleased
+

ext/opentelemetry-ext-prometheus/README.rst

@@ -0,0 +1,72 @@
+OpenTelemetry Prometheus Exporter
+=============================
+
+|pypi|
+
+.. |pypi| image:: https://badge.fury.io/py/opentelemetry-ext-prometheus.svg
+   :target: https://pypi.org/project/opentelemetry-ext-prometheus/
+
+This library allows to export metrics data to `Prometheus <https://prometheus.io/>`_.
+
+Installation
+------------
+
+::
+
+     pip install opentelemetry-ext-prometheus
+
+
+Usage
+-----
+
+The **OpenTelemetry Prometheus Exporter** allows to export `OpenTelemetry`_ metrics to `Prometheus`_.
+
+
+.. _Prometheus: https://prometheus.io/
+.. _OpenTelemetry: https://github.com/open-telemetry/opentelemetry-python/
+
+.. code:: python
+
+    from opentelemetry import metrics
+    from opentelemetry.ext.prometheus import PrometheusMetricsExporter
+    from opentelemetry.sdk.metrics import Counter, Meter
+    from opentelemetry.sdk.metrics.export.controller import PushController
+    from prometheus_client import start_http_server
+
+    # Start Prometheus client
+    start_http_server(port=8000, addr="localhost")
+
+    # Meter is responsible for creating and recording metrics
+    metrics.set_preferred_meter_implementation(lambda _: Meter())
+    meter = metrics.meter()
+    # exporter to export metrics to Prometheus
+    prefix = "MyAppPrefix"
+    exporter = PrometheusMetricsExporter(prefix)
+    # controller collects metrics created from meter and exports it via the
+    # exporter every interval
+    controller = PushController(meter, exporter, 5)
+
+    counter = meter.create_metric(
+        "requests",
+        "number of requests",
+        "requests",
+        int,
+        Counter,
+        ("environment",),
+    )
+
+    # Labelsets are used to identify key-values that are associated with a specific
+    # metric that you want to record. These are useful for pre-aggregation and can
+    # be used to store custom dimensions pertaining to a metric
+    label_set = meter.get_label_set({"environment": "staging"})
+
+    counter.add(25, label_set)
+    input("Press any key to exit...")
+
+
+
+References
+----------
+
+* `Prometheus <https://prometheus.io/>`_
+* `OpenTelemetry Project <https://opentelemetry.io/>`_

ext/opentelemetry-ext-prometheus/setup.cfg

@@ -0,0 +1,47 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[metadata]
+name = opentelemetry-ext-prometheus
+description = Prometheus Metric Exporter for OpenTelemetry
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+author = OpenTelemetry Authors
+author_email = [email protected]
+url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-ext-prometheus
+platforms = any
+license = Apache-2.0
+classifiers =
+    Development Status :: 3 - Alpha
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+
+[options]
+python_requires = >=3.4
+package_dir=
+    =src
+packages=find_namespace:
+install_requires =
+    prometheus_client >= 0.5.0, < 1.0.0
+    opentelemetry-api
+    opentelemetry-sdk
+
+[options.packages.find]
+where = src

ext/opentelemetry-ext-prometheus/setup.py

@@ -0,0 +1,26 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+import setuptools
+
+BASE_DIR = os.path.dirname(__file__)
+VERSION_FILENAME = os.path.join(
+    BASE_DIR, "src", "opentelemetry", "ext", "prometheus", "version.py"
+)
+PACKAGE_INFO = {}
+with open(VERSION_FILENAME) as f:
+    exec(f.read(), PACKAGE_INFO)
+
+setuptools.setup(version=PACKAGE_INFO["__version__"])

ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/__init__.py

@@ -0,0 +1,147 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Prometheus Metrics Exporter for OpenTelemetry."""
+
+import collections
+import logging
+import re
+from typing import Sequence
+
+from prometheus_client import start_http_server
+from prometheus_client.core import (
+    REGISTRY,
+    CollectorRegistry,
+    CounterMetricFamily,
+    GaugeMetricFamily,
+    UnknownMetricFamily,
+)
+
+from opentelemetry.metrics import Counter, Gauge, Measure, Metric
+from opentelemetry.sdk.metrics.export import (
+    MetricRecord,
+    MetricsExporter,
+    MetricsExportResult,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class PrometheusMetricsExporter(MetricsExporter):
+    """Prometheus metric exporter for OpenTelemetry.
+
+    Args:
+        prefix: single-word application prefix relevant to the domain
+        the metric belongs to.
+    """
+
+    def __init__(self, prefix: str = ""):
+        self._collector = CustomCollector(prefix)
+        REGISTRY.register(self._collector)
+
+    def export(
+        self, metric_records: Sequence[MetricRecord]
+    ) -> MetricsExportResult:
+        self._collector.add_metrics_data(metric_records)
+        return MetricsExportResult.SUCCESS
+
+    def shutdown(self) -> None:
+        REGISTRY.unregister(self._collector)
+
+
+class CustomCollector:
+    """ CustomCollector represents the Prometheus Collector object
+        https://github.com/prometheus/client_python#custom-collectors
+    """
+
+    def __init__(self, prefix: str = ""):
+        self._prefix = prefix
+        self._metrics_to_export = collections.deque()
+        self._non_letters_nor_digits_re = re.compile(
+            r"[^\w]", re.UNICODE | re.IGNORECASE
+        )
+
+    def add_metrics_data(self, metric_records: Sequence[MetricRecord]):
+        self._metrics_to_export.append(metric_records)
+
+    def collect(self):
+        """Collect fetches the metrics from OpenTelemetry
+        and delivers them as Prometheus Metrics.
+        Collect is invoked every time a prometheus.Gatherer is run
+        for example when the HTTP endpoint is invoked by Prometheus.
+        """
+
+        while self._metrics_to_export:
+            for metric_record in self._metrics_to_export.popleft():
+                prometheus_metric = self._translate_to_prometheus(
+                    metric_record
+                )
+                if prometheus_metric is not None:
+                    yield prometheus_metric
+
+    def _translate_to_prometheus(self, metric_record: MetricRecord):
+        prometheus_metric = None
+        label_values = []
+        label_keys = []
+        for label_tuple in metric_record.label_set.labels:
+            label_keys.append(self._sanitize(label_tuple[0]))
+            label_values.append(label_tuple[1])
+
+        metric_name = ""
+        if self._prefix != "":
+            metric_name = self._prefix + "_"
+        metric_name += self._sanitize(metric_record.metric.name)
+
+        if isinstance(metric_record.metric, Counter):
+            prometheus_metric = CounterMetricFamily(
+                name=metric_name,
+                documentation=metric_record.metric.description,
+                labels=label_keys,
+            )
+            prometheus_metric.add_metric(
+                labels=label_values, value=metric_record.aggregator.checkpoint
+            )
+
+        elif isinstance(metric_record.metric, Gauge):
+            prometheus_metric = GaugeMetricFamily(
+                name=metric_name,
+                documentation=metric_record.metric.description,
+                labels=label_keys,
+            )
+            prometheus_metric.add_metric(
+                labels=label_values, value=metric_record.aggregator.checkpoint
+            )
+
+        # TODO: Add support for histograms when supported in OT
+        elif isinstance(metric_record.metric, Measure):
+            prometheus_metric = UnknownMetricFamily(
+                name=metric_name,
+                documentation=metric_record.metric.description,
+                labels=label_keys,
+            )
+            prometheus_metric.add_metric(
+                labels=label_values, value=metric_record.aggregator.checkpoint
+            )
+
+        else:
+            logger.warning(
+                "Unsupported metric type. %s", type(metric_record.metric)
+            )
+        return prometheus_metric
+
+    def _sanitize(self, key):
+        """ sanitize the given metric name or label according to Prometheus rule.
+        Replace all characters other than [A-Za-z0-9_] with '_'.
+        """
+        return self._non_letters_nor_digits_re.sub("_", key)

ext/opentelemetry-ext-prometheus/src/opentelemetry/ext/prometheus/version.py

@@ -0,0 +1,15 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.4.dev0"

ext/opentelemetry-ext-prometheus/tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.